From 39a028f93f8ce778ee0e8fc552ab7932d8d12110 Mon Sep 17 00:00:00 2001 From: Luhaozhu Date: Thu, 22 Oct 2026 17:49:04 -0700 Subject: [PATCH 1/4] Sync CE native MCP runtime, durable automation, and modular backend --- .../references/architecture.md | 12 +- Makefile | 6 +- ...uirements-desktop-linux-aarch64-py311.lock | 50 +- ...quirements-desktop-linux-x86_64-py311.lock | 58 +- ...uirements-desktop-macos-aarch64-py311.lock | 58 +- ...quirements-desktop-macos-x86_64-py311.lock | 58 +- .../requirements-desktop-windows-py311.lock | 60 +- desktop/requirements-desktop.txt | 11 +- desktop/scripts/build-runtime.mjs | 3 + desktop/scripts/desktop-dependencies.test.mjs | 33 +- desktop/scripts/test-native-tools.py | 2 +- docker-compose.application-hosting.yml | 2 +- docker-compose.yml | 3 +- docker/Dockerfile.opensandbox-server | 1 + ...Dockerfile.opensandbox-server.dockerignore | 1 + docker/Dockerfile.personal-agent | 2 +- docker/opensandbox_capacity.py | 31 + docker/opensandbox_mount_recovery.py | 22 +- docker/patch_opensandbox_server.py | 11 + document/en/api/overview.md | 2 +- document/en/architecture/backend.md | 12 +- document/en/architecture/data-model.md | 2 +- document/en/architecture/overview.md | 8 +- .../en/deployment/environment-variables.md | 10 +- document/en/deployment/offline-production.md | 14 +- document/en/deployment/quick-install.md | 2 +- document/en/deployment/space-files.md | 12 +- document/en/development/backend.md | 6 +- .../en/development/workbench-migration.md | 29 +- document/en/editions/build-ce.md | 6 +- document/en/modules/agent-skills.md | 20 +- document/en/modules/auth.md | 4 +- document/en/modules/automation.md | 38 +- document/en/modules/canvas-artifacts.md | 10 +- document/en/modules/catalog.md | 15 +- document/en/modules/chat.md | 24 +- document/en/modules/knowledge-base.md | 14 +- document/en/modules/mcp-tools.md | 31 +- document/en/modules/model-providers.md | 10 +- document/en/modules/projects-myspace.md | 21 +- document/en/modules/prompts.md | 27 +- document/en/modules/sandbox.md | 36 +- document/en/modules/storage.md | 2 +- document/zh-CN/api/overview.md | 2 +- document/zh-CN/architecture/backend.md | 12 +- document/zh-CN/architecture/data-model.md | 2 +- document/zh-CN/architecture/overview.md | 6 +- .../zh-CN/deployment/environment-variables.md | 10 +- .../zh-CN/deployment/offline-production.md | 13 +- document/zh-CN/deployment/quick-install.md | 2 +- document/zh-CN/deployment/space-files.md | 16 +- document/zh-CN/development/backend.md | 6 +- .../zh-CN/development/workbench-migration.md | 29 +- document/zh-CN/editions/build-ce.md | 6 +- document/zh-CN/modules/agent-skills.md | 20 +- document/zh-CN/modules/auth.md | 4 +- document/zh-CN/modules/automation.md | 34 +- document/zh-CN/modules/canvas-artifacts.md | 10 +- document/zh-CN/modules/catalog.md | 15 +- document/zh-CN/modules/chat.md | 24 +- document/zh-CN/modules/knowledge-base.md | 14 +- document/zh-CN/modules/mcp-tools.md | 31 +- document/zh-CN/modules/model-providers.md | 10 +- document/zh-CN/modules/projects-myspace.md | 19 +- document/zh-CN/modules/prompts.md | 27 +- document/zh-CN/modules/sandbox.md | 27 +- document/zh-CN/modules/storage.md | 2 +- pyproject.toml | 14 +- requirements.txt | 17 +- scripts/performance/benchmark_preview.py | 2 +- scripts/performance/serve_fixture.py | 6 +- scripts/refactor/__init__.py | 1 + scripts/refactor/check_ce_paths.py | 79 + scripts/refactor/check_line_budget.py | 37 + scripts/refactor/check_test_baseline.py | 75 + scripts/refactor/import_contract.py | 209 + scripts/refactor/rewrite_imports.py | 357 ++ scripts/refactor/snapshot_routes.py | 35 + scripts/refactor/snapshot_schema.py | 81 + scripts/refactor/tests/test_edition_seams.py | 33 + scripts/refactor/tests/test_guards.py | 127 + .../refactor/tests/test_package_markers.py | 20 + .../refactor/tests/test_private_imports.py | 41 + .../refactor/tests/test_rewrite_imports.py | 142 + .../refactor/tests/test_snapshot_schema.py | 33 + .../refactor/tests/test_supplemental_moves.py | 47 + scripts/refactor/tests/test_test_baseline.py | 22 + scripts/refactor/tests/test_test_moves.py | 45 + scripts/repair_artifact_storage_keys.py | 4 +- scripts/sync-workbench-schema.mjs | 4 +- scripts/test-workbench-build.mjs | 2 +- src/backend/alembic/env.py | 4 + .../alembic/versions/ce_0001_initial.py | 2 +- .../versions/ce_0002_reconcile_schema.py | 2 +- .../versions/ce_0003_mcp_marketplace.py | 2 +- .../versions/ce_0009_desktop_delivery.py | 2 +- .../versions/ce_0010_desktop_channel_relay.py | 2 +- .../alembic/versions/ce_0012_model_steps.py | 3 +- .../alembic/versions/ce_0013_tool_media.py | 2 +- .../versions/ce_0014_site_access_password.py | 2 +- .../versions/ce_0019_history_display.py | 2 +- .../versions/ce_0020_plugin_resources.py | 2 +- .../versions/ce_0021_retire_site_storage.py | 2 +- .../versions/ce_0028_personal_agent.py | 4 +- .../ce_0031_automation_lease_token.py | 19 + .../versions/ce_0032_automation_queue.py | 25 + .../versions/ce_0033_automation_targets.py | 17 + src/backend/api/__init__.py | 19 +- src/backend/api/app.py | 28 +- src/backend/api/deps.py | 2 +- src/backend/api/edition.py | 13 + src/backend/api/health.py | 2 +- src/backend/api/lifecycle.py | 16 +- src/backend/api/middleware/agent_api_scope.py | 11 +- src/backend/api/middleware/logging.py | 2 +- src/backend/api/openapi.py | 2 +- .../v1/agents.py => personal_agents.py} | 10 +- src/backend/api/routes/__init__.py | 15 +- src/backend/api/routes/file_archive.py | 11 +- src/backend/api/routes/file_location.py | 2 +- src/backend/api/routes/files.py | 14 +- src/backend/api/routes/v1/__init__.py | 53 +- src/backend/api/routes/v1/agent_api_keys.py | 8 +- src/backend/api/routes/v1/agent_responses.py | 17 +- src/backend/api/routes/v1/api_keys.py | 2 +- src/backend/api/routes/v1/artifacts.py | 24 +- src/backend/api/routes/v1/audio.py | 2 +- src/backend/api/routes/v1/auth.py | 6 +- src/backend/api/routes/v1/catalog.py | 31 +- .../api/routes/v1/catalog_mutations.py | 12 +- src/backend/api/routes/v1/chat_forks.py | 2 +- src/backend/api/routes/v1/chat_modes.py | 2 +- src/backend/api/routes/v1/chat_runs.py | 17 +- src/backend/api/routes/v1/chats/confirms.py | 8 +- src/backend/api/routes/v1/chats/feedback.py | 14 +- src/backend/api/routes/v1/chats/history.py | 16 +- .../api/routes/v1/chats/pending_access.py | 8 +- src/backend/api/routes/v1/chats/questions.py | 2 +- src/backend/api/routes/v1/chats/reruns.py | 81 +- src/backend/api/routes/v1/chats/run_views.py | 28 +- src/backend/api/routes/v1/chats/sessions.py | 20 +- src/backend/api/routes/v1/chats/sidebar.py | 12 +- src/backend/api/routes/v1/classify.py | 2 +- src/backend/api/routes/v1/commands.py | 11 +- src/backend/api/routes/v1/content.py | 7 +- .../api/routes/v1/desktop_capability.py | 824 --- src/backend/api/routes/v1/evolution.py | 29 +- src/backend/api/routes/v1/file_parse.py | 3 +- src/backend/api/routes/v1/file_text.py | 2 +- src/backend/api/routes/v1/file_upload.py | 18 +- src/backend/api/routes/v1/internal_jobs.py | 10 +- .../api/routes/v1/job_callback_session.py | 2 +- src/backend/api/routes/v1/jobs.py | 4 +- src/backend/api/routes/v1/kb.py | 873 ---- src/backend/api/routes/v1/loops.py | 28 +- src/backend/api/routes/v1/me_capabilities.py | 46 +- src/backend/api/routes/v1/me_skill_files.py | 38 +- src/backend/api/routes/v1/me_system.py | 12 +- src/backend/api/routes/v1/memories.py | 18 +- src/backend/api/routes/v1/mock_sso.py | 6 +- src/backend/api/routes/v1/model_reasoning.py | 4 +- src/backend/api/routes/v1/models.py | 57 +- src/backend/api/routes/v1/myspace_folders.py | 4 +- src/backend/api/routes/v1/ontologies.py | 17 +- src/backend/api/routes/v1/plans.py | 47 +- src/backend/api/routes/v1/plugin_resources.py | 32 +- src/backend/api/routes/v1/plugins.py | 35 +- src/backend/api/routes/v1/project_api_keys.py | 8 +- .../api/routes/v1/project_environments.py | 8 +- src/backend/api/routes/v1/projects.py | 10 +- .../api/routes/v1/prompt_management.py | 68 +- src/backend/api/routes/v1/public_chats.py | 6 +- src/backend/api/routes/v1/space_files.py | 14 +- src/backend/api/routes/v1/space_previews.py | 4 +- src/backend/api/routes/v1/summary.py | 2 +- src/backend/api/routes/v1/tool_approval.py | 10 +- src/backend/api/routes/v1/users.py | 12 +- src/backend/api/startup/bootstrap.py | 13 +- src/backend/api/startup/commands.py | 8 +- src/backend/api/startup/memory.py | 10 +- src/backend/api/startup/plugins.py | 31 +- src/backend/api/startup/runtime.py | 80 +- src/backend/api/startup/workers.py | 63 +- .../content => api/transports}/__init__.py | 0 src/backend/api/transports/run_sse.py | 85 + src/backend/cli.py | 117 +- src/backend/core/agent_runtime/README.md | 18 + .../core/agent_runtime/README.zh-CN.md | 18 + .../resources => agent_runtime}/__init__.py | 0 src/backend/core/agent_runtime/budget.py | 74 + src/backend/core/agent_runtime/direct.py | 458 ++ .../core/agent_runtime/direct_state.py | 46 + src/backend/core/agent_runtime/events.py | 71 + src/backend/core/agent_runtime/host.py | 11 + .../agent_runtime/integrations}/__init__.py | 0 .../integrations}/compaction_stream.py | 8 +- .../agent_runtime/integrations/evolution.py | 160 + .../core/agent_runtime/integrations/memory.py | 304 ++ .../integrations/memory_budget.py | 34 + .../integrations/memory_identity.py | 74 + .../integrations/memory_session.py | 178 + .../integrations/memory_state.py | 23 + .../integrations/ontology_evidence.py | 229 + .../integrations/ontology_repair.py | 366 ++ src/backend/core/agent_runtime/main_state.py | 45 + .../agent_runtime}/plugin_visibility.py | 14 +- .../core/agent_runtime/prepare_direct.py | 126 + .../core/agent_runtime/prepare_main.py | 288 + src/backend/core/agent_runtime/runner.py | 444 ++ src/backend/core/agent_runtime/skills.py | 176 + .../agent_runtime}/strategy.py | 0 .../core/agent_runtime/stream_direct.py | 298 ++ src/backend/core/agent_runtime/stream_main.py | 354 ++ src/backend/core/agent_runtime/targets.py | 208 + .../core/agent_skills/backends/__init__.py | 14 - .../core/agent_skills/cache_refresh.py | 51 - .../agent_api.py} | 2 +- src/backend/core/auth/agent_api_scope.py | 2 +- .../core/{services => auth}/api_key_runs.py | 12 +- .../api_key_service.py => auth/api_keys.py} | 12 +- src/backend/core/auth/backend.py | 24 +- src/backend/core/auth/capabilities.py | 4 +- src/backend/core/auth/edition.py | 5 + src/backend/core/auth/local_user_bootstrap.py | 83 + src/backend/core/auth/local_user_policy.py | 16 + .../local_users.py} | 155 +- .../core/auth/local_users_constants.py | 40 + src/backend/core/auth/mock_ticket_store.py | 26 +- src/backend/core/auth/model_selection.py | 52 + src/backend/core/auth/oa_ticket_store.py | 23 +- ...t_policy.py => personal_account_policy.py} | 0 ...count_view.py => personal_account_view.py} | 0 ...es.py => personal_edition_capabilities.py} | 0 ...iface.py => personal_permissions_iface.py} | 0 src/backend/core/auth/project_api_scope.py | 2 +- src/backend/core/auth/public_chat.py | 6 +- src/backend/core/auth/public_identity.py | 36 + src/backend/core/auth/session.py | 33 +- .../user_service.py => auth/users.py} | 11 +- src/backend/core/capabilities/__init__.py | 17 +- .../core/capabilities/agents/__init__.py | 1 + .../agents/available_resources.py | 273 + src/backend/core/capabilities/agents/base.py | 435 ++ .../capabilities/agents/base_constants.py | 33 + .../agents/builtin.py} | 4 +- .../agents/edition.py} | 2 +- .../agents/markdown.py} | 0 .../capabilities/agents/version_history.py | 180 + src/backend/core/capabilities/archive.py | 18 +- .../core/capabilities/catalog/__init__.py | 1 + .../catalog/api.py} | 14 +- .../catalog/common.py} | 4 +- .../catalog/details.py} | 20 +- .../catalog/loader.py} | 57 +- .../catalog/migration.py} | 4 +- .../catalog/overrides.py} | 0 .../catalog/resolver.py} | 78 +- .../catalog/runtime.py} | 53 +- .../core/capabilities/identity_rules.py | 36 + .../manifest_protocol.py} | 3 +- src/backend/core/capabilities/mcp/__init__.py | 1 + .../mcp/builtin_specs.py} | 4 - .../mcp/edition.py} | 0 .../mcp/management.py} | 50 +- .../mcp/oauth.py} | 289 +- .../core/capabilities/mcp/oauth_storage.py | 124 + .../mcp/service.py} | 60 +- .../core/capabilities/mcp/tool_metadata.py | 13 + .../core/capabilities/mcp/tool_schema.py | 20 + .../mcp/tool_schema_backfill.py} | 16 +- .../core/capabilities/plugins/README.md | 23 + .../{ => capabilities}/plugins/__init__.py | 0 .../{ => capabilities}/plugins/connections.py | 2 +- .../plugins/management/__init__.py | 43 + .../plugins/management/admin_config.py | 18 +- .../plugins/management/bundle_upgrade.py} | 8 +- .../plugins/management/cloud.py} | 52 +- .../plugins/management/components.py | 32 +- .../plugins/management/contract.py} | 0 .../plugins/management/defaults.py | 2 +- .../plugins/management/details.py | 76 +- .../plugins/management/errors.py} | 0 .../plugins/management/installation.py | 72 +- .../plugins/management/lifecycle.py | 42 +- .../plugins/management/package.py} | 10 +- .../plugins/management/packages.py | 33 +- .../plugins/management/projection.py | 28 +- .../plugins/management/queries.py | 50 +- .../plugins/packaging/__init__.py | 0 .../plugins/packaging/archive.py | 4 +- .../plugins/packaging/definitions.py | 4 +- .../plugins/packaging/importer.py | 24 +- .../plugins/packaging/models.py | 0 .../plugins/packaging/presentation.py | 31 + .../plugins/packaging/runtime_assets.py | 0 .../plugins/packaging/skill_transform.py | 2 +- .../plugins/packaging/sources.py | 31 +- .../plugins/resources/__init__.py | 1 + .../plugins/resources/artifacts.py | 4 +- .../plugins/resources/authentication.py | 0 .../plugins/resources/configuration.py | 0 .../plugins/resources/confinement.py | 7 +- .../plugins/resources/crypto.py | 0 .../plugins/resources/deadlines.py | 48 + .../plugins/resources/installation.py | 41 +- .../plugins/resources/lifecycle.py | 21 +- .../capabilities/plugins/resources/owners.py | 66 + .../plugins/resources/prewarm.py | 70 +- .../plugins/resources/reservation.py | 4 +- .../plugins/resources/service.py | 52 +- .../plugins/resources/store.py | 2 +- .../plugins/resources/stream.py | 6 +- .../plugins/resources/transport.py | 0 .../plugins/runtime/__init__.py | 28 +- .../plugins/runtime/activation_history.py | 2 +- .../plugins/runtime/models.py | 0 .../plugins/runtime/resolution.py | 27 +- .../plugins/runtime/sticky_selection.py | 35 +- .../capabilities/{ref.py => reference.py} | 2 +- .../core/capabilities/runtime/__init__.py | 37 - src/backend/core/capabilities/skills.py | 656 --- .../skills}/__init__.py | 1 - .../capabilities/skills/backends/__init__.py | 14 + .../skills}/backends/composite.py | 6 +- .../skills}/backends/database.py | 2 +- .../skills}/backends/filesystem.py | 2 +- .../skills}/backends/protocol.py | 0 .../skills}/binary_files.py | 0 .../core/capabilities/skills/cache_refresh.py | 50 + .../skills}/config.py | 56 +- .../core/capabilities/skills/deps/__init__.py | 1 + .../skills/deps/aggregator.py} | 4 +- .../skills/deps/detector.py} | 0 .../skills/deps/prober.py} | 0 .../skills/deps/requests.py} | 2 +- .../skills/icons.py} | 0 .../core/capabilities/skills/installation.py | 117 + .../skills}/loader.py | 165 +- .../capabilities/skills/loader_factory.py | 21 + .../skills/management.py} | 18 +- .../skills}/publication.py | 41 +- .../skills}/registry.py | 12 +- .../capabilities/skills/runtime_loader.py | 115 + .../capabilities/skills/script_discovery.py | 65 + .../skills}/selection_record.py | 4 +- .../skills}/skill_archive.py | 8 +- .../core/capabilities/skills/zip_reader.py | 124 + .../core/channels/adapters/dingtalk.py | 854 --- src/backend/core/channels/adapters/lark.py | 683 --- src/backend/core/channels/inbound.py | 1005 ---- .../{services => chat}/audio_transcription.py | 2 +- src/backend/core/chat/compaction/__init__.py | 1 + .../compaction/budget_checkpoint.py} | 4 +- .../compaction/coordinator.py} | 24 +- .../compaction/history.py} | 8 +- .../compaction/progress.py} | 0 .../compaction/service.py} | 20 +- .../compaction/summary.py} | 19 +- .../compaction/usage_projection.py} | 4 +- .../core/chat/compaction_checkpoints.py | 259 + src/backend/core/chat/compaction_snapshots.py | 265 + src/backend/core/chat/compaction_types.py | 36 + src/backend/core/chat/context.py | 25 +- .../chat_edition.py => chat/edition.py} | 0 .../{orchestration => core/chat}/followups.py | 57 +- .../chat_fork_service.py => chat/fork.py} | 2 +- src/backend/core/chat/history_operations.py | 49 + src/backend/core/chat/limits.py | 88 + .../chat}/message_parser.py | 5 +- src/backend/core/chat/message_queries.py | 91 + src/backend/core/chat/message_writes.py | 299 ++ src/backend/core/chat/mode_seed.py | 51 + src/backend/core/chat/mode_types.py | 48 + src/backend/core/chat/mode_validation.py | 107 + .../chat_mode_service.py => chat/modes.py} | 295 +- .../operations}/__init__.py | 0 .../operations}/admission.py | 66 +- .../operations}/admission_errors.py | 2 +- .../operations}/agent_targets.py | 16 +- .../operations}/commands.py | 18 +- .../operations}/history.py | 10 +- .../operations}/invocation.py | 31 +- .../operations}/projects.py | 6 +- .../operations}/request_context.py | 25 +- .../operations}/runs.py | 14 +- .../operations}/schemas.py | 0 .../operations}/session_context.py | 33 +- .../operations}/sessions.py | 40 +- .../plan_service.py => chat/plans.py} | 0 .../core/{services => chat}/project_init.py | 8 +- src/backend/core/chat/projection/__init__.py | 0 src/backend/core/chat/projection/codec.py | 8 + .../chat/projection/reducer.py} | 134 +- .../chat/projection/storage.py} | 8 +- .../chat/projection/subscription.py} | 32 +- .../public_chat_service.py => chat/public.py} | 43 +- .../references.py} | 6 +- src/backend/core/chat/runtime/__init__.py | 0 src/backend/core/chat/runtime/admission.py | 98 + src/backend/core/chat/runtime/chunks.py | 183 + .../chat/runtime/compaction.py} | 23 +- src/backend/core/chat/runtime/completion.py | 231 + src/backend/core/chat/runtime/followups.py | 63 + src/backend/core/chat/runtime/handoff.py | 272 + .../chat/runtime}/job_wakeup.py | 30 +- src/backend/core/chat/runtime/pauses.py | 58 + src/backend/core/chat/runtime/recovery.py | 176 + .../core/chat/runtime/recovery_snapshot.py | 201 + src/backend/core/chat/runtime/snapshots.py | 201 + src/backend/core/chat/runtime/steering.py | 199 + src/backend/core/chat/runtime/turn_state.py | 44 + src/backend/core/chat/runtime/worker.py | 404 ++ src/backend/core/chat/schemas.py | 2 +- .../chat_sequencer.py => chat/sequencer.py} | 12 +- src/backend/core/chat/service.py | 256 + src/backend/core/chat/session_operations.py | 284 + .../core/{services => chat}/sidebar_status.py | 0 .../chat_steer_service.py => chat/steer.py} | 7 +- src/backend/core/chat/steer_hook.py | 70 + .../core/{services => chat}/steer_queue.py | 0 .../subagent_routing.py} | 0 .../chat_titles.py => chat/titles.py} | 21 +- src/backend/core/chat/tool_log.py | 2 +- .../chat}/tool_payloads.py | 45 +- src/backend/core/config/__init__.py | 11 +- src/backend/core/config/display_names.py | 2 +- .../{edition_display_names.py => edition.py} | 8 + src/backend/core/config/environment.py | 3 - src/backend/core/config/identity_settings.py | 3 +- src/backend/core/config/mcp_config.py | 8 +- src/backend/core/config/product_settings.py | 5 +- src/backend/core/config/runtime_env.py | 49 +- src/backend/core/config/runtime_settings.py | 33 +- src/backend/core/config/sandbox_settings.py | 9 +- src/backend/core/config/settings.py | 7 +- src/backend/core/config/store/__init__.py | 1 + src/backend/core/config/store/config_keys.py | 12 + .../core/config/store/execution_options.py | 93 + src/backend/core/config/store/runtime_env.py | 52 + .../core/config/store/system_config.py | 120 + .../config/store/system_config_constants.py | 302 ++ src/backend/core/config/user_intros.py | 2 +- src/backend/core/content/content_blocks.py | 701 --- src/backend/core/content/file_parser.py | 752 --- src/backend/core/db/__init__.py | 28 +- src/backend/core/db/artifact_scope.py | 13 + src/backend/core/db/chat_search.py | 2 +- src/backend/core/db/edition.py | 27 + src/backend/core/db/engine.py | 6 +- src/backend/core/db/history_projection.py | 2 +- src/backend/core/db/models/__init__.py | 212 +- src/backend/core/db/models/admin.py | 338 +- src/backend/core/db/models/agent.py | 73 +- src/backend/core/db/models/artifact.py | 17 +- src/backend/core/db/models/channel_relay.py | 40 - src/backend/core/db/models/chat.py | 350 +- src/backend/core/db/models/chat_mode.py | 5 +- src/backend/core/db/models/chat_resources.py | 116 + src/backend/core/db/models/chat_runs.py | 199 + src/backend/core/db/models/column_types.py | 9 + src/backend/core/db/models/config.py | 12 +- src/backend/core/db/models/evolution.py | 10 +- src/backend/core/db/models/identity.py | 285 +- src/backend/core/db/models/job.py | 14 +- src/backend/core/db/models/kb_wiki.py | 224 - src/backend/core/db/models/knowledge.py | 182 +- src/backend/core/db/models/logs.py | 13 +- src/backend/core/db/models/memory.py | 15 +- src/backend/core/db/models/observability.py | 61 - src/backend/core/db/models/ontology.py | 6 +- src/backend/core/db/models/project.py | 14 +- src/backend/core/db/personal_file_names.py | 13 +- ...extensions.py => personal_model_fields.py} | 0 .../{edition_tables.py => personal_schema.py} | 6 +- src/backend/core/db/repository/__init__.py | 39 +- src/backend/core/db/repository/artifact.py | 8 +- src/backend/core/db/repository/catalog.py | 5 +- src/backend/core/db/repository/chat.py | 15 +- src/backend/core/db/repository/folder.py | 40 + .../{agent.py => personal_agent.py} | 0 .../{audit.py => personal_audit.py} | 2 +- src/backend/core/db/repository/user.py | 115 +- .../core/db/site_storage_retirement.py | 8 +- src/backend/core/db/system_configuration.py | 146 + src/backend/core/db/user_cleanup.py | 13 + src/backend/core/evolution/__init__.py | 6 +- src/backend/core/evolution/activation.py | 1502 ------ src/backend/core/evolution/agent_profile.py | 505 -- .../core/evolution/assessment/__init__.py | 1 + .../{ => assessment}/applicability.py | 2 +- .../evolution/{ => assessment}/benchmark.py | 6 +- .../evolution/{ => assessment}/evaluator.py | 0 .../evolution/{ => assessment}/experiments.py | 2 +- .../{ => assessment}/fault_injection.py | 15 +- .../evolution/{ => assessment}/metrics.py | 0 .../evolution/{ => assessment}/similarity.py | 9 +- .../evolution/{ => assessment}/sop_judge.py | 2 +- src/backend/core/evolution/backfill.py | 9 +- src/backend/core/evolution/contract.py | 4 +- src/backend/core/evolution/cycle/__init__.py | 1 + .../core/evolution/cycle/activation.py | 353 ++ .../evolution/cycle/activation_constants.py | 11 + .../core/evolution/cycle/activation_policy.py | 196 + .../core/evolution/cycle/activation_types.py | 9 + .../core/evolution/cycle/candidate_records.py | 197 + .../evolution/cycle/composition_candidates.py | 221 + .../core/evolution/cycle/cycle_extras.py | 32 + .../evolution/cycle/cycle_extras_constants.py | 20 + .../core/evolution/cycle/episode_views.py | 178 + .../core/evolution/cycle/evaluation.py | 153 + .../core/evolution/cycle/extra_findings.py | 166 + .../core/evolution/cycle/graph_candidates.py | 156 + src/backend/core/evolution/cycle/loop.py | 292 ++ .../core/evolution/cycle/memory_candidates.py | 286 + .../core/evolution/cycle/memory_findings.py | 109 + src/backend/core/evolution/cycle/ordering.py | 103 + src/backend/core/evolution/cycle/patterns.py | 113 + .../core/evolution/cycle/profile_release.py | 281 + src/backend/core/evolution/cycle/promotion.py | 317 ++ .../evolution/cycle/promotion_constants.py | 80 + .../core/evolution/cycle/promotion_types.py | 94 + .../core/evolution/cycle/prompt_candidates.py | 95 + .../core/evolution/{ => cycle}/readiness.py | 25 +- .../core/evolution/{ => cycle}/release.py | 0 .../core/evolution/cycle/retirement.py | 158 + .../evolution/{ => cycle}/runtime_binding.py | 85 +- .../core/evolution/cycle/skill_document.py | 298 ++ .../core/evolution/cycle/skill_release.py | 355 ++ .../core/evolution/{ => cycle}/timescales.py | 2 +- .../core/evolution/{ => cycle}/trigger.py | 77 +- .../core/evolution/cycle/usage_evidence.py | 283 + src/backend/core/evolution/cycle_extras.py | 653 --- .../core/evolution/evidence/__init__.py | 1 + .../core/evolution/{ => evidence}/cassette.py | 19 +- .../{ => evidence}/counterfactual.py | 2 +- .../core/evolution/{ => evidence}/credit.py | 2 +- .../core/evolution/{ => evidence}/events.py | 2 +- .../{ => evidence}/evidence_contract.py | 0 .../{ => evidence}/evidence_resolver.py | 8 +- .../core/evolution/{ => evidence}/exposure.py | 10 +- .../core/evolution/{ => evidence}/lineage.py | 2 +- .../core/evolution/{ => evidence}/outcomes.py | 0 .../core/evolution/{ => evidence}/replay.py | 0 .../{ => evidence}/trace_assembler.py | 6 +- .../evolution/{ => evidence}/trace_store.py | 4 +- .../core/evolution/generators/__init__.py | 1 + .../evolution/{ => generators}/graph_scan.py | 0 .../{ => generators}/memory_apply.py | 8 +- .../evolution/{ => generators}/memory_ops.py | 2 +- .../evolution/{ => generators}/memory_scan.py | 2 +- .../{ => generators}/ontology_bridge.py | 0 .../{ => generators}/orchestration_gen.py | 2 +- .../evolution/{ => generators}/prompt_gen.py | 0 .../skill_candidate_engine.py | 14 +- .../evolution/generators/skill_drafting.py | 185 + .../evolution/generators/skill_evidence.py | 124 + .../core/evolution/generators/skill_gen.py | 95 + .../generators/skill_gen_constants.py | 109 + src/backend/core/evolution/loop.py | 1576 ------ .../core/evolution/profiles/__init__.py | 1 + .../core/evolution/profiles/agent_profile.py | 259 + .../profiles/agent_profile_constants.py | 102 + .../core/evolution/profiles/change_preview.py | 85 + .../evolution/{ => profiles}/console_view.py | 2 - .../evolution/{ => profiles}/federation.py | 2 +- .../core/evolution/{ => profiles}/personal.py | 52 +- .../personal_review.py} | 219 +- .../core/evolution/{ => profiles}/policies.py | 22 +- .../core/evolution/{ => profiles}/prefs.py | 0 .../evolution/profiles/profile_selection.py | 147 + .../core/evolution/profiles/stall_hook.py | 103 + .../core/evolution/profiles/user_settings.py | 70 + .../profiles/user_settings_constants.py | 14 + src/backend/core/evolution/promotion.py | 890 ---- .../core/evolution/settlement/__init__.py | 1 + .../evolution/settlement/memory_service.py | 39 + .../evolution/{ => settlement}/settlement.py | 0 .../{ => settlement}/settlement_runner.py | 43 +- .../{ => settlement}/settlement_store.py | 2 +- src/backend/core/evolution/skill_gen.py | 521 -- src/backend/core/extensions/__init__.py | 69 - src/backend/core/extensions/registry.py | 107 - src/backend/core/files/__init__.py | 1 + src/backend/core/files/artifact_bytes.py | 75 + .../core/{ => files}/artifacts/__init__.py | 1 - .../artifacts/backfill.py} | 10 +- src/backend/core/files/artifacts/generated.py | 210 + .../artifacts/key_repair.py} | 0 .../{ => files}/artifacts/local_reference.py | 31 +- .../artifacts/location.py} | 24 +- .../artifacts/reader.py} | 19 +- .../artifacts/refs.py} | 10 +- .../artifacts/service.py} | 15 +- .../core/{ => files}/artifacts/store.py | 26 +- .../artifacts/summary.py} | 8 +- .../artifacts/text.py} | 15 +- src/backend/core/files/content/__init__.py | 1 + .../{ => files}/content/archive_browser.py | 2 +- .../{ => files}/content/archive_limits.py | 0 src/backend/core/files/content/branding.py | 299 ++ .../core/files/content/content_blocks.py | 183 + .../files/content/content_blocks_constants.py | 181 + .../core/files/content/file_formats.py | 33 + src/backend/core/files/content/file_parser.py | 41 + .../{ => files}/content/file_validation.py | 0 .../core/{ => files}/content/office.py | 0 .../content/office_preview_cache.py | 2 +- .../core/files/content/parser_config.py | 47 + src/backend/core/files/content/pdf_parser.py | 125 + .../core/files/content/presentation_parser.py | 63 + .../core/files/content/spreadsheet_parser.py | 332 ++ .../core/{ => files}/content/svg_fit.py | 5 +- src/backend/core/files/content/text_parser.py | 21 + src/backend/core/files/content/word_parser.py | 77 + src/backend/core/files/edition.py | 98 + .../core/{services => files}/folder_batch.py | 47 +- .../personal_operations.py} | 10 +- .../core/{ => files}/spaces/__init__.py | 0 src/backend/core/{ => files}/spaces/access.py | 4 +- src/backend/core/{ => files}/spaces/files.py | 8 +- src/backend/core/{ => files}/spaces/leases.py | 24 +- src/backend/core/{ => files}/spaces/mounts.py | 32 +- .../core/{ => files}/spaces/projects.py | 6 +- .../core/{ => files}/spaces/recovery.py | 4 +- .../core/{ => files}/spaces/sandbox.py | 26 +- src/backend/core/{ => files}/spaces/search.py | 2 +- src/backend/core/{ => files}/spaces/text.py | 4 +- src/backend/core/{ => files}/spaces/tree.py | 0 .../core/{ => files}/storage/__init__.py | 17 +- .../core/{ => files}/storage/entries.py | 0 .../core/{ => files}/storage/factory.py | 32 +- src/backend/core/{ => files}/storage/local.py | 8 +- .../{ => files}/storage/local_directory.py | 4 +- .../core/{ => files}/storage/local_paths.py | 2 +- .../core/{ => files}/storage/protocol.py | 0 src/backend/core/{ => files}/storage/s3.py | 4 +- .../core/{ => files}/storage/s3_directory.py | 2 +- .../core/{services => files}/upload_retry.py | 4 +- .../capability_errors.py} | 0 src/backend/core/harness/context_items.py | 6 + src/backend/core/harness/events.py | 67 +- .../{llm => harness}/execution_manifest.py | 2 +- src/backend/core/harness/hooks.py | 151 +- .../harness_ledger.py => harness/ledger.py} | 163 +- src/backend/core/harness/log_audit.py | 144 + src/backend/core/harness/log_context.py | 161 + src/backend/core/harness/log_records.py | 212 + src/backend/core/harness/log_writer.py | 121 + .../core/harness/log_writer_constants.py | 32 + .../core/harness/observation_writer.py | 134 + src/backend/core/harness/registry/__init__.py | 79 + .../core/harness/registry/authentication.py | 26 + .../core/harness/registry/build_governance.py | 51 + .../harness/registry/cache_invalidation.py | 27 + .../harness/registry/capability_management.py | 27 + .../harness/registry/capability_revisions.py | 34 + .../harness/registry/capability_runtime.py | 74 + .../core/harness/registry/catalog_options.py | 26 + .../core/harness/registry/chat_execution.py | 27 + .../core/harness/registry/chat_history.py | 21 + .../core/harness/registry/compaction.py | 62 + .../core/harness/registry/device_catalog.py | 79 + .../registry}/edition.py | 0 .../core/harness/registry/evaluation.py | 72 + .../harness/registry/execution_account.py | 35 + .../harness/registry/execution_catalog.py | 51 + .../harness/registry/execution_identity.py | 78 + .../core/harness/registry/hook_factories.py | 69 + .../core/harness/registry/host_runtime.py | 13 + .../core/harness/registry/job_operations.py | 21 + .../core/harness/registry/local_files.py | 65 + .../harness/registry/local_publication.py | 69 + .../harness/registry/marketplace_catalog.py | 44 + .../core/harness/registry/mcp_clients.py | 21 + .../harness/registry/model_credentials.py | 70 + .../core/harness/registry/model_services.py | 59 + .../core/harness/registry/package_sources.py | 48 + .../core/harness/registry/package_views.py | 66 + src/backend/core/harness/registry/registry.py | 169 + .../core/harness/registry/resource_owners.py | 48 + .../harness/registry/retrieval_context.py | 37 + .../core/harness/registry/run_cancellation.py | 37 + .../core/harness/registry/run_capabilities.py | 114 + .../core/harness/registry/schema_lifecycle.py | 36 + .../core/harness/registry/skill_content.py | 24 + .../harness/registry/system_configuration.py | 31 + .../core/harness/registry/tool_operations.py | 59 + .../registry/transfer_channels.py} | 64 +- .../core/harness/registry/user_cleanup.py | 27 + src/backend/core/harness/tool_call_context.py | 5 + .../core/harness/tool_effect_context.py | 18 + src/backend/core/harness/usage.py | 41 +- src/backend/core/infra/background.py | 18 +- src/backend/core/infra/crypto.py | 28 +- .../junction.py => infra/directory_links.py} | 32 +- src/backend/core/infra/dns_cache.py | 58 +- src/backend/core/infra/failures.py | 13 + .../lockfile.py => infra/file_lock.py} | 0 src/backend/core/{ => infra}/immutable.py | 0 src/backend/core/infra/json_output.py | 49 + src/backend/core/infra/leader.py | 225 +- .../core/infra/local_runtime/__init__.py | 1 + .../infra/local_runtime/dynamic_servers.py | 64 + .../infra/local_runtime/lifecycle.py} | 33 +- .../local_runtime/runner_auth.py} | 0 .../infra/local_runtime/startup.py} | 68 +- src/backend/core/infra/model_output.py | 95 + src/backend/core/infra/probes/__init__.py | 1 + .../probes/edition.py} | 0 .../probes}/service_probes.py | 25 +- src/backend/core/infra/rate_limit.py | 21 - src/backend/core/infra/redis.py | 27 +- src/backend/core/infra/runtime_state.py | 24 +- .../text_bounds.py} | 0 src/backend/core/infra/time.py | 14 +- src/backend/core/infra/worker_count.py | 2 +- .../{orchestration => core}/jobs/__init__.py | 0 .../{orchestration => core}/jobs/agent.py | 14 +- .../{orchestration => core}/jobs/callback.py | 0 .../{orchestration => core}/jobs/files.py | 6 +- .../jobs/notifications.py | 22 +- .../{orchestration => core}/jobs/owner.py | 43 +- .../{orchestration => core}/jobs/process.py | 13 +- src/backend/core/jobs/reminder_hook.py | 80 + .../{orchestration => core}/jobs/runner.py | 0 .../job_runtime.py => core/jobs/runtime.py} | 32 +- .../{orchestration => core}/jobs/sdk.py | 0 .../{orchestration => core}/jobs/state.py | 2 +- .../job_service.py => jobs/store.py} | 0 .../jobs/supervisor.py | 38 +- src/backend/core/jobs/tool_operations.py | 387 ++ src/backend/core/kb/kb_assets.py | 567 -- src/backend/core/kb/kb_parser.py | 933 ---- src/backend/core/kb/kb_vector.py | 755 --- src/backend/core/kb/wiki/local_provider.py | 523 -- src/backend/core/llm/__init__.py | 14 +- src/backend/core/llm/_distill_shared.py | 156 - src/backend/core/llm/api_runtime/__init__.py | 1 + .../agent.py} | 6 +- .../agent_skills.py} | 2 +- .../agent_tools.py} | 27 +- .../project.py} | 2 +- src/backend/core/llm/capability_tools.py | 106 - src/backend/core/llm/citations/__init__.py | 1 + src/backend/core/llm/citations/allocator.py | 115 + src/backend/core/llm/citations/anchor.py | 63 + .../core/llm/citations/anchor_state.py | 101 + src/backend/core/llm/citations/extraction.py | 208 + src/backend/core/llm/citations/hook.py | 38 + src/backend/core/llm/citations/types.py | 63 + src/backend/core/llm/command_middleware.py | 39 - src/backend/core/llm/context/__init__.py | 1 + src/backend/core/llm/context/adapter.py | 362 ++ .../core/llm/context/attachment_hook.py | 81 + .../context/budget.py} | 2 +- .../llm/{ => context}/compacting_agent.py | 20 +- .../core/llm/{ => context}/compaction.py | 11 +- src/backend/core/llm/context/content.py | 212 + src/backend/core/llm/context/ir.py | 327 ++ src/backend/core/llm/context/ir_constants.py | 86 + src/backend/core/llm/context/items.py | 215 + .../core/llm/context/message_defaults.py | 78 + .../core/llm/{ => context}/offloader.py | 4 +- .../core/llm/context/provider_conversion.py | 239 + .../core/llm/context/reasoning_hooks.py | 49 + .../core/llm/context/reference_conversion.py | 79 + src/backend/core/llm/context/rendering.py | 114 + .../{context_safety.py => context/safety.py} | 16 +- .../core/llm/{ => context}/system_reminder.py | 4 +- .../{context_usage.py => context/usage.py} | 2 +- .../core/llm/context/workspace_hooks.py | 125 + src/backend/core/llm/context_adapter.py | 867 --- src/backend/core/llm/context_ir.py | 855 --- src/backend/core/llm/factory/README.md | 6 +- src/backend/core/llm/factory/__init__.py | 6 +- src/backend/core/llm/factory/assembly.py | 18 +- src/backend/core/llm/factory/build.py | 6 +- src/backend/core/llm/factory/defaults.py | 16 +- .../core/llm/factory/prompts/policy.py | 3 +- .../core/llm/factory/prompts/system.py | 39 +- src/backend/core/llm/factory/request.py | 4 +- .../llm/factory/runtime/agent_construction.py | 21 +- .../capability_bindings.py} | 11 +- .../llm/factory/runtime/execution_budget.py | 10 +- .../core/llm/factory/runtime/listeners.py | 33 +- .../llm/factory/runtime/manifest_binding.py | 4 +- .../llm/factory/runtime/middleware_stack.py | 130 +- .../llm/factory/runtime/model_selection.py | 20 +- .../llm/factory/selection/capabilities.py | 58 +- .../llm/factory/selection/entry_policy.py | 20 +- .../llm/factory/selection/mode_bindings.py | 7 +- .../factory/selection/profile_selection.py | 12 +- .../factory/selection/progressive_bindings.py | 24 +- .../llm/factory/selection/server_selection.py | 10 +- .../factory/selection/sticky_capabilities.py | 26 +- .../factory/selection/subagent_bindings.py | 8 +- .../registry.py => core/llm/factory/specs.py} | 2 - .../core/llm/factory/tools/definitions.py | 35 + .../core/llm/factory/tools/interactive.py | 2 +- .../core/llm/factory/tools/mcp_config.py | 55 +- .../core/llm/factory/tools/mcp_connections.py | 226 +- src/backend/core/llm/factory/tools/native.py | 82 +- .../llm/factory/tools/skill_registration.py | 26 +- .../core/llm/factory/tools/tool_surface.py | 46 +- src/backend/core/llm/finish_guard.py | 85 - src/backend/core/llm/hooks.py | 588 --- src/backend/core/llm/host_services.py | 31 + src/backend/core/llm/management_tool.py | 84 - src/backend/core/llm/mcp/__init__.py | 1 + src/backend/core/llm/mcp/callbacks.py | 168 + src/backend/core/llm/mcp/chat_sessions.py | 181 + src/backend/core/llm/mcp/client.py | 149 + src/backend/core/llm/mcp/context_tools.py | 69 + src/backend/core/llm/mcp/deadline.py | 40 + .../{mcp_invocation.py => mcp/invocation.py} | 10 +- src/backend/core/llm/mcp/lifetime.py | 68 + src/backend/core/llm/mcp/manager.py | 62 + src/backend/core/llm/mcp/results.py | 67 + src/backend/core/llm/mcp/session.py | 147 + src/backend/core/llm/mcp/tools.py | 113 + src/backend/core/llm/mcp_manager.py | 336 -- src/backend/core/llm/mcp_pool.py | 387 -- src/backend/core/llm/middleware/__init__.py | 1 + .../migration_map.py} | 67 +- src/backend/core/llm/middlewares.py | 1450 ----- src/backend/core/llm/persona_distiller.py | 226 - src/backend/core/llm/ports.py | 79 + .../llm/project_environment_middleware.py | 27 - src/backend/core/llm/prompt_files.py | 138 + src/backend/core/llm/prompt_kinds.py | 193 + src/backend/core/llm/prompt_rendering.py | 137 + src/backend/core/llm/prompt_storage.py | 99 + src/backend/core/llm/prompt_versions.py | 296 ++ src/backend/core/llm/prompt_versions_state.py | 30 + src/backend/core/llm/providers/__init__.py | 13 - src/backend/core/llm/runtime/__init__.py | 1 + .../{ => runtime}/agentscope_hook_adapter.py | 223 +- .../core/llm/runtime/context_hook_adapter.py | 80 + src/backend/core/llm/runtime/file_context.py | 277 + src/backend/core/llm/runtime/finish_guard.py | 54 + .../core/llm/runtime/hooks_constants.py | 17 + .../llm/{ => runtime}/human_interaction.py | 4 +- .../llm/{ => runtime}/interaction_store.py | 0 .../core/llm/{ => runtime}/manifest_agent.py | 17 +- .../core/llm/{ => runtime}/message_compat.py | 16 +- src/backend/core/llm/runtime/model_cache.py | 133 + .../llm/runtime/model_selection_adapter.py | 28 + .../core/llm/{ => runtime}/model_steps.py | 5 +- .../llm/{ => runtime}/project_environment.py | 33 +- .../llm/runtime/project_environment_hook.py | 19 + .../core/llm/runtime/review_text_agent.py | 51 + .../llm/{ => runtime}/session_capabilities.py | 2 +- src/backend/core/llm/runtime/state.py | 75 + src/backend/core/llm/runtime/stream_events.py | 285 + src/backend/core/llm/runtime/stream_loop.py | 309 ++ src/backend/core/llm/runtime/stream_text.py | 63 + src/backend/core/llm/runtime/streaming.py | 280 + .../core/llm/runtime/streaming_constants.py | 17 + .../llm/runtime}/tool_callbacks.py | 24 +- .../llm/runtime/tool_execution_adapter.py | 210 + .../core/llm/runtime/tool_hook_adapter.py | 81 + .../llm/{ => runtime}/tool_media_store.py | 24 +- .../core/llm/runtime/tool_result_adapter.py | 53 + src/backend/core/llm/runtime/turn_state.py | 41 + .../core/llm/{ => runtime}/workspace.py | 0 .../selector.py => llm/skill_selection.py} | 23 +- src/backend/core/llm/steer_middleware.py | 96 - src/backend/core/llm/subagents/__init__.py | 1 + .../execution.py} | 44 +- .../prompts.py} | 0 .../sessions.py} | 2 +- .../stream.py} | 39 +- .../{subagent_tool.py => subagents/tool.py} | 72 +- src/backend/core/llm/tool_collector.py | 270 - .../core/llm/tool_permissions/__init__.py | 95 +- src/backend/core/llm/tool_permissions/hook.py | 71 + .../core/llm/tool_permissions/middleware.py | 86 - .../core/llm/tool_permissions/pending.py | 4 +- .../core/llm/tool_permissions/service.py | 35 +- src/backend/core/llm/tools/__init__.py | 9 +- src/backend/core/llm/tools/_common.py | 23 +- src/backend/core/llm/tools/_tool_helpers.py | 208 +- .../core/llm/tools/ask_user_question_tool.py | 11 +- .../core/llm/tools/chat_history_tool.py | 51 +- .../cloud_gateway.py} | 119 +- src/backend/core/llm/tools/cloud_manifest.py | 55 + .../__init__.py | 0 .../gate.py | 3 +- .../pending.py | 13 +- .../shared.py | 2 +- .../state.py | 104 +- .../core/llm/tools/data_context_tool.py | 96 - src/backend/core/llm/tools/edit_tool.py | 25 +- src/backend/core/llm/tools/glob_tool.py | 7 +- src/backend/core/llm/tools/grep_tool.py | 12 +- src/backend/core/llm/tools/job_tool.py | 383 +- src/backend/core/llm/tools/management.py | 45 + .../tools/ontology_toolkit.py} | 30 +- src/backend/core/llm/tools/pin_tool.py | 36 +- .../plan_update.py} | 19 +- .../tools/plugin_activation.py} | 122 +- .../llm/tools/project_instructions_tool.py | 30 +- .../core/llm/tools/project_source_access.py | 6 +- .../core/llm/tools/read_artifact_tool.py | 42 +- src/backend/core/llm/tools/read_image_tool.py | 34 +- src/backend/core/llm/tools/read_tool.py | 27 +- .../core/llm/tools/sandbox_artifact_tools.py | 34 +- src/backend/core/llm/tools/sandbox_tool.py | 75 +- src/backend/core/llm/tools/skill_tool.py | 51 +- src/backend/core/llm/tools/user_questions.py | 67 +- src/backend/core/llm/tools/write_tool.py | 30 +- src/backend/core/memory/access.py | 4 +- src/backend/core/memory/audit.py | 166 +- src/backend/core/memory/backend.py | 96 +- src/backend/core/memory/context.py | 2 +- src/backend/core/memory/edition.py | 4 + src/backend/core/memory/effect_lane.py | 21 +- src/backend/core/memory/executor.py | 5 +- src/backend/core/memory/extractors/_base.py | 49 +- src/backend/core/memory/extractors/graph.py | 2 +- src/backend/core/memory/extractors/writers.py | 10 +- src/backend/core/memory/graph.py | 345 +- src/backend/core/memory/graph_contract.py | 122 + src/backend/core/memory/graph_state.py | 141 + src/backend/core/memory/outbox/enqueue.py | 3 +- src/backend/core/memory/outbox/jobs.py | 35 +- src/backend/core/memory/outbox/leases.py | 3 +- src/backend/core/memory/outbox/settlement.py | 18 +- src/backend/core/memory/outbox/worker.py | 36 +- src/backend/core/memory/pipeline.py | 31 +- src/backend/core/memory/preview.py | 5 +- src/backend/core/memory/procedures.py | 3 +- src/backend/core/memory/queries.py | 3 +- src/backend/core/memory/sanitizer.py | 39 +- src/backend/core/memory/service.py | 9 +- src/backend/core/memory/session_snapshot.py | 2 +- .../settings.py} | 2 +- src/backend/core/memory/settlement_port.py | 33 + src/backend/core/memory/ttl_sweeper.py | 2 +- src/backend/core/memory/weights.py | 23 +- src/backend/core/models/__init__.py | 1 + src/backend/core/models/admission.py | 21 + .../api_protocol_backfill.py | 8 +- .../core/{llm => models}/chat_models.py | 47 +- .../core/models/completions/__init__.py | 1 + .../{llm => models/completions}/classifier.py | 25 +- .../completions}/single_turn.py | 10 +- .../{llm => models/completions}/summarizer.py | 25 +- .../model_config.py => models/config.py} | 2 +- src/backend/core/models/constants.py | 5 + .../context_window.py} | 2 +- src/backend/core/models/embedding.py | 123 + src/backend/core/models/embedding_state.py | 20 + src/backend/core/{llm => models}/failover.py | 36 +- .../import_validation.py} | 4 +- src/backend/core/models/providers/__init__.py | 3 + .../providers/_image_tokens.py | 2 +- .../providers/context_probe.py | 2 +- .../providers/openai_compat.py | 20 +- .../providers/protocol_probe.py | 0 .../providers/reasoning_probe.py | 6 +- .../{llm => models}/providers/registry.py | 2 +- .../{llm => models}/providers/stream_retry.py | 4 +- .../providers/vendor_models.py | 12 +- .../core/{llm => models}/reasoning_effort.py | 0 .../core/{llm => models}/reasoning_replay.py | 0 .../repository.py} | 4 +- src/backend/core/models/reranking.py | 63 + .../responses.py} | 97 +- .../selection.py} | 46 +- src/backend/core/models/selection_hook.py | 20 + .../{llm => models}/tool_call_identity.py | 0 .../core/{llm => models}/tool_result_media.py | 63 +- .../{llm/model_usage.py => models/usage.py} | 4 +- src/backend/core/models/vision/__init__.py | 34 + .../core/{ => models}/vision/attachments.py | 18 +- .../core/{ => models}/vision/prompt.py | 2 +- .../core/{ => models}/vision/provider.py | 12 +- .../core/{ => models}/vision/render.py | 4 +- .../core/{ => models}/vision/schema.py | 0 .../core/{ => models}/vision/service.py | 14 +- src/backend/core/myspace/duplicate_cleanup.py | 63 - src/backend/core/ontology/__init__.py | 3 +- src/backend/core/ontology/audit_events.py | 71 + src/backend/core/ontology/build_validator.py | 6 +- .../evolution.py} | 36 +- src/backend/core/ontology/host.py | 13 + src/backend/core/ontology/manual_review.py | 85 + .../core/ontology/output_review_state.py | 60 + .../ontology_policy.py => ontology/policy.py} | 4 +- .../core/ontology/review_operations.py | 102 + src/backend/core/ontology/review_prompt.py | 99 + src/backend/core/ontology/reviewer.py | 319 ++ src/backend/core/ontology/reviewer_state.py | 19 + src/backend/core/ontology/runtime_gates.py | 157 + src/backend/core/ontology/runtime_pack.py | 357 ++ src/backend/core/ontology/runtime_policy.py | 158 + .../service.py} | 262 +- src/backend/core/ontology/tool_hook.py | 183 + .../core/ontology/validation_models.py | 33 + src/backend/core/ontology/validator.py | 634 +-- src/backend/core/plugins/README.md | 28 - .../core/plugins/management/__init__.py | 80 - src/backend/core/plugins/resources/owners.py | 112 - src/backend/core/projects/__init__.py | 1 + .../project_access.py => projects/access.py} | 2 +- .../api.py} | 2 +- .../project_service.py => projects/base.py} | 87 +- src/backend/core/projects/edition.py | 8 + .../environment}/__init__.py | 0 .../environment}/config.py | 0 .../environment}/dependencies.py | 2 +- .../environment/service.py} | 12 +- .../environment_resources.py} | 22 +- .../files.py} | 14 +- .../instructions.py} | 20 +- .../local_workspace.py} | 2 +- .../personal_scope.py} | 8 +- src/backend/core/projects/personal_summary.py | 44 + .../personal_transfer.py} | 0 src/backend/core/projects/project_names.py | 27 + .../project_source.py => projects/source.py} | 20 +- .../user_folders.py} | 23 +- src/backend/core/runs/__init__.py | 0 .../runs/api_recovery.py} | 6 +- .../run_binding.py => runs/binding.py} | 15 +- src/backend/core/runs/clock.py | 55 + src/backend/core/runs/events/__init__.py | 0 src/backend/core/runs/events/projection.py | 13 + .../runs/events/store.py} | 88 +- src/backend/core/runs/events/subscription.py | 81 + src/backend/core/runs/events/writer.py | 197 + src/backend/core/runs/host.py | 41 + .../run_journal.py => runs/journal.py} | 32 +- .../journal_domain.py} | 0 .../journal_lifecycle.py} | 20 +- .../journal_recovery.py} | 9 +- src/backend/core/runs/ownership.py | 276 + src/backend/core/runs/runtime.py | 275 + src/backend/core/runs/settings.py | 98 + src/backend/core/runs/tasks.py | 23 + .../runs}/tool_effect_recovery.py | 47 +- .../core/runs/tool_effects/__init__.py | 1 + .../core/runs/tool_effects/arguments.py | 65 + src/backend/core/runs/tool_effects/errors.py | 43 + src/backend/core/runs/tool_effects/gateway.py | 256 + src/backend/core/runs/tool_effects/hooks.py | 14 + src/backend/core/runs/tool_effects/host.py | 28 + .../runs/tool_effects/intent_transactions.py | 342 ++ .../runs/tool_effects/invocation_claims.py | 123 + src/backend/core/runs/tool_effects/journal.py | 223 + .../runs/tool_effects/journal_constants.py | 32 + .../core/runs/tool_effects/recovery.py | 176 + .../core/runs/tool_effects/recovery_claims.py | 166 + .../core/runs/tool_effects/registry.py | 95 + .../runs/tool_effects/result_transactions.py | 285 + .../core/runs/tool_effects/retention.py | 154 + src/backend/core/runs/tool_effects/types.py | 74 + src/backend/core/runs/watchdog.py | 319 ++ src/backend/core/sandbox/commands/__init__.py | 1 + .../core/sandbox/commands/completion_hook.py | 44 + .../{services => sandbox}/commands/context.py | 10 +- .../commands/delivery.py | 13 +- .../sandbox}/commands/events.py | 15 +- .../sandbox}/commands/notifications.py | 27 +- src/backend/core/sandbox/commands/runtime.py | 449 ++ .../{services => sandbox}/commands/store.py | 13 +- .../sandbox}/commands/subscription.py | 40 +- .../core/sandbox/credential_dispatch.py | 30 + src/backend/core/sandbox/factory.py | 33 +- .../core/sandbox/oslayer/windows_runtime.py | 36 +- .../permissions}/account.py | 4 +- .../permissions/approval.py} | 6 +- .../permissions/dispatch_models.py} | 4 +- .../core/sandbox/permissions/os_gate.py | 2 +- .../permissions/tool_specs.py} | 20 +- .../core/sandbox/permissions/vocabulary.py | 26 + src/backend/core/sandbox/processes/cloud.py | 16 +- .../core/sandbox/processes/completion.py | 10 +- .../core/sandbox/processes/dispatch.py | 2 +- .../core/sandbox/processes/interactive.py | 3 +- .../core/sandbox/providers/account_env.py | 27 +- src/backend/core/sandbox/providers/edition.py | 2 +- .../providers/opensandbox/account_volumes.py | 292 -- .../sandbox/providers/opensandbox/admin.py | 5 +- .../sandbox/providers/opensandbox/adoption.py | 242 +- .../sandbox/providers/opensandbox/binding.py | 2 +- .../sandbox/providers/opensandbox/creation.py | 72 +- .../providers/opensandbox/execution.py | 7 +- .../providers/opensandbox/lifecycle.py | 4 +- .../sandbox/providers/opensandbox/models.py | 9 +- .../sandbox/providers/opensandbox/pool.py | 3 +- .../sandbox/providers/opensandbox/provider.py | 59 +- .../providers/opensandbox/reclamation.py | 4 +- .../sandbox/providers/opensandbox/runtime.py | 15 +- .../providers/opensandbox/snapshots.py | 44 +- .../sandbox/providers/opensandbox/tracking.py | 31 + .../providers/opensandbox/user_pool.py | 6 +- .../sandbox/providers/opensandbox/volumes.py | 85 +- .../providers/script_runner/provider.py | 44 +- .../core/sandbox/runner/native_group.py | 28 +- src/backend/core/sandbox/runner/server.py | 1 - src/backend/core/sandbox/sessions/registry.py | 21 +- src/backend/core/sandbox/sessions/router.py | 15 +- .../core/sandbox/workspace/container.py | 15 +- .../core/sandbox/workspace/credentials.py | 141 - src/backend/core/sandbox/workspace/desktop.py | 2 +- .../workspace/resolution.py} | 14 +- src/backend/core/services/__init__.py | 19 - .../core/services/agent_market_categories.py | 77 - .../core/services/agent_market_service.py | 940 ---- src/backend/core/services/artifact_edition.py | 66 - src/backend/core/services/chat_service.py | 1301 ----- .../core/services/commands/__init__.py | 1 - .../core/services/desktop_cloud_bridge.py | 960 ---- .../core/services/edition_system_config.py | 6 - .../core/services/edition_workspace.py | 52 - src/backend/core/services/kb_edition.py | 12 - src/backend/core/services/lark_service.py | 752 --- src/backend/core/services/log_service.py | 649 --- .../core/services/marketplace_service.py | 1400 ----- .../core/services/mcp_marketplace_service.py | 1772 ------- .../core/services/mcp_tool_metadata.py | 19 - .../core/services/prompt_version_service.py | 889 ---- .../core/services/skill_draft_service.py | 192 - src/backend/core/services/system_config.py | 655 --- .../core/services/tool_effect_ledger.py | 1808 ------- src/backend/core/services/user_agent_base.py | 854 --- src/backend/core/spaces/edition_access.py | 17 - src/backend/core/spaces/edition_revocation.py | 3 - src/backend/core/vision/__init__.py | 45 - src/backend/extensions/README.md | 2 +- .../extensions/mcp_servers/_packaging.py | 6 +- src/backend/extensions/mcp_servers/_serve.py | 28 +- .../mcp_servers/agent_manager_mcp/impl.py | 48 +- .../mcp_servers/agent_manager_mcp/server.py | 4 +- .../mcp_servers/automation_task_mcp/impl.py | 18 +- .../mcp_servers/automation_task_mcp/server.py | 16 +- .../mcp_servers/browser_runtime_mcp/server.py | 8 +- .../internet_search_mcp/_selftest.py | 2 +- .../mcp_servers/internet_search_mcp/impl.py | 4 +- .../mcp_servers/internet_search_mcp/server.py | 35 +- .../mcp_servers/management_registration.py | 8 +- .../mcp_servers/plugin_manager_mcp/impl.py | 11 +- .../mcp_servers/plugin_manager_mcp/server.py | 4 +- .../retrieve_dataset_content_mcp/impl.py | 18 +- .../local_impl.py | 25 +- .../retrieve_dataset_content_mcp/server.py | 47 +- .../retrieve_dataset_content_mcp/wiki_impl.py | 12 +- .../mcp_servers/site_publish_mcp/server.py | 6 +- .../mcp_servers/skill_manager_mcp/impl.py | 21 +- .../mcp_servers/skill_manager_mcp/server.py | 4 +- .../space_files_mcp/permissions.py | 8 +- .../mcp_servers/space_files_mcp/server.py | 10 +- .../mcp_servers/web_fetch_mcp/server.py | 4 +- .../marketplace/agent-manager/mcp.json | 2 +- .../plugins/marketplace/automation/mcp.json | 2 +- .../marketplace/plugin-manager/mcp.json | 2 +- .../plugin-creator/scripts/validate_plugin.py | 2 +- .../plugins/marketplace/sites/mcp.json | 2 +- .../marketplace/skill-manager/mcp.json | 2 +- .../skill-creator/references/skill-anatomy.md | 2 +- .../skill-creator/scripts/quick_validate.py | 2 +- .../skills/marketplace/data-mask/config.yaml | 166 +- src/backend/features/__init__.py | 157 + src/backend/features/automation/README.md | 15 + src/backend/features/automation/__init__.py | 1 + src/backend/features/automation/admission.py | 80 + .../features/automation/api/__init__.py | 1 + .../automation/api}/automations.py | 27 +- .../automation}/delivery_targets.py | 19 +- .../automation/execution.py} | 4 +- src/backend/features/automation/extension.py | 28 + .../automation/loops.py} | 5 +- .../automation/models.py} | 59 +- .../automation/notifications.py} | 0 .../automation/remote_effect.py} | 3 +- src/backend/features/automation/runs.py | 66 + src/backend/features/automation/scheduling.py | 23 + src/backend/features/automation/schema.py | 59 + .../automation/service.py} | 155 +- .../automation/tool_routing.py} | 24 +- src/backend/features/desktop/README.md | 11 + src/backend/features/desktop/__init__.py | 1 + src/backend/features/desktop/api/__init__.py | 1 + .../features/desktop/api/capability_auth.py | 48 + .../features/desktop/api/capability_bridge.py | 92 + .../desktop/api/capability_dependencies.py | 107 + .../desktop/api/capability_manifests.py | 145 + .../desktop/api/capability_mcp_gateway.py | 79 + .../desktop/api/capability_model_gateway.py | 89 + .../features/desktop/api/capability_tools.py | 192 + .../desktop/api/capability_workcopies.py | 38 + .../v1 => features/desktop/api}/desktop.py | 2 +- .../desktop/api}/desktop_capabilities.py | 104 +- .../desktop/api/desktop_capability.py | 12 + .../desktop/api/desktop_capability_state.py | 52 + .../desktop/api}/desktop_login.py | 2 +- .../desktop/api}/desktop_observability.py | 8 +- .../desktop/api}/device_link.py | 6 +- .../v1 => features/desktop/api}/local.py | 20 +- .../desktop/api}/local_file_actions.py | 4 +- src/backend/features/desktop/auth/__init__.py | 1 + .../desktop/auth/bridge.py} | 8 +- .../desktop/auth/login_store.py} | 0 .../desktop/auth/ticket_store.py} | 0 .../features/desktop/bridge/__init__.py | 1 + .../desktop/bridge/bundles.py} | 42 +- src/backend/features/desktop/bridge/client.py | 214 + .../features/desktop/bridge/client_state.py | 48 + .../features/desktop/bridge/gateway.py | 225 + .../features/desktop/bridge/identity.py | 254 + .../desktop/bridge/managed_connectors.py | 144 + .../desktop/bridge/model_credentials.py} | 25 +- .../desktop/bridge/release_events.py} | 0 .../desktop/bridge/resource_plugins.py} | 19 +- .../desktop/bridge/skills.py} | 47 +- src/backend/features/desktop/bridge/status.py | 113 + .../features/desktop/capabilities/__init__.py | 16 + .../desktop}/capabilities/agents.py | 17 +- .../desktop}/capabilities/change_merge.py | 10 +- .../desktop}/capabilities/change_signal.py | 0 .../desktop}/capabilities/change_sync.py | 60 +- .../desktop}/capabilities/connectors.py | 12 +- .../desktop}/capabilities/credentials.py | 0 .../desktop}/capabilities/dependency.py | 28 +- .../capabilities/dependency_declarations.py | 0 .../desktop}/capabilities/device_catalog.py | 90 +- .../capabilities/device_plugin_catalog.py | 69 +- .../desktop}/capabilities/discovery.py | 6 +- .../desktop}/capabilities/invocation.py | 34 +- .../capabilities/local_plugin_runtime.py | 21 +- .../desktop}/capabilities/local_projection.py | 27 +- .../capabilities/management_context.py | 17 +- .../desktop}/capabilities/manifest_order.py | 2 +- .../desktop}/capabilities/mcp_json.py | 8 +- .../desktop}/capabilities/migration.py | 25 +- .../desktop}/capabilities/paths.py | 31 +- .../desktop}/capabilities/plugins.py | 47 +- .../desktop}/capabilities/preparation.py | 28 +- .../desktop}/capabilities/readiness.py | 24 +- .../desktop}/capabilities/registry.py | 115 +- .../desktop/capabilities/registry_types.py | 79 + .../desktop}/capabilities/resolver.py | 4 +- .../desktop/capabilities/runtime/__init__.py | 37 + .../capabilities/runtime/connectors.py | 23 +- .../capabilities/runtime/dependencies.py | 18 +- .../capabilities/runtime/preparation.py | 25 +- .../desktop}/capabilities/runtime/state.py | 52 +- .../capabilities/session_availability.py | 0 .../desktop/capabilities/skill_account.py | 54 + .../desktop/capabilities/skill_candidates.py | 140 + .../desktop/capabilities/skill_hashes.py | 101 + .../desktop/capabilities/skill_paths.py | 29 + .../desktop/capabilities/skill_publication.py | 88 + .../desktop/capabilities/skill_views.py | 62 + .../features/desktop/capabilities/skills.py | 161 + .../desktop/capabilities/skills_state.py | 35 + .../desktop}/capabilities/store.py | 17 +- .../desktop}/capabilities/view.py | 4 +- .../desktop}/capabilities/warmup.py | 17 +- .../features/desktop/capability/__init__.py | 1 + .../desktop/capability/configs.py} | 6 +- .../desktop/capability/credentials.py} | 2 +- .../desktop/capability/entities.py} | 24 +- .../desktop/capability/mcp.py} | 40 +- .../desktop/capability/models.py} | 16 +- .../desktop/capability/revision_provider.py | 22 + .../desktop/capability/security.py} | 9 +- .../desktop/capability/service.py} | 43 +- .../desktop/capability/skills.py} | 18 +- .../desktop/capability/sync_check.py} | 29 +- .../desktop/capability/workcopies.py} | 45 +- src/backend/features/desktop/edition.py | 23 + src/backend/features/desktop/extension.py | 4 + .../features/desktop/gateway/__init__.py | 1 + .../desktop/gateway/observability.py} | 4 +- .../features/desktop/local/__init__.py | 1 + .../desktop/local/file_actions.py} | 2 +- .../desktop/local/grants.py} | 0 .../desktop/local/management_migration.py} | 16 +- .../desktop/local/manager_execution.py | 51 + .../desktop/local/market_install.py} | 14 +- .../desktop/local}/native_app_icons.py | 0 .../desktop/local/plugins}/__init__.py | 0 .../desktop/local/plugins}/assets.py | 6 +- .../local/plugins}/legacy_migration.py | 27 +- .../desktop/local/plugins}/service.py | 66 +- .../desktop/local/plugins/sidecars.py | 33 + .../desktop/local/publication_provider.py | 101 + .../features/desktop/local/skills/__init__.py | 1 + .../desktop/local/skills/editor.py} | 28 +- .../desktop/local/skills/service.py} | 36 +- .../desktop/local/skills/upload.py} | 34 +- .../desktop/local/snapshots.py} | 0 .../desktop/models.py} | 69 +- .../desktop/observability/__init__.py | 1 + .../desktop/observability/context.py} | 8 +- .../desktop/observability/delivery.py} | 11 +- .../desktop/observability/fields.py} | 2 +- .../desktop/observability/projection.py} | 8 +- .../desktop/observability/sync.py} | 13 +- .../desktop/plugin_resolution.py} | 39 +- .../desktop/run_capabilities/__init__.py | 1 + .../desktop/run_capabilities/preparation.py} | 22 +- .../desktop/run_capabilities/provider.py | 118 + .../desktop/skill_backend.py} | 20 +- src/backend/features/evaluation/README.md | 7 + src/backend/features/evaluation/__init__.py | 1 + .../evaluation/admission.py} | 2 +- .../features/evaluation/api/__init__.py | 1 + .../evaluation/api}/evaluation_sandboxes.py | 2 +- src/backend/features/evaluation/extension.py | 23 + src/backend/features/evaluation/models.py | 1 + src/backend/features/evaluation/policy.py | 27 + .../features/evaluation/runtime/__init__.py | 1 + .../evaluation/runtime/paths.py} | 0 .../evaluation/runtime/process_tools.py} | 7 +- .../evaluation/runtime/runtime.py} | 0 .../evaluation/runtime/tools.py} | 59 +- .../evaluation/sandbox_provider}/__init__.py | 0 .../evaluation/sandbox_provider}/binding.py | 0 .../sandbox_provider}/process_guard.py | 0 .../evaluation/sandbox_provider}/provider.py | 4 +- .../evaluation/sandboxes.py} | 16 +- src/backend/features/hosted_apps/README.md | 9 + src/backend/features/hosted_apps/__init__.py | 1 + .../features/hosted_apps/api/__init__.py | 1 + .../api}/application_transports.py | 48 +- .../hosted_apps/api}/applications.py | 48 +- .../hosted_apps/collections.py} | 8 +- .../hosted_apps/config.py} | 2 +- .../hosted_apps/data.py} | 20 +- .../hosted_apps/deployments.py} | 13 +- .../hosted_apps/export.py} | 6 +- src/backend/features/hosted_apps/extension.py | 17 + .../hosted_apps/history.py} | 8 +- .../hosted_apps/hosting_setup.py} | 4 +- .../hosted_apps/industry.py} | 9 +- .../hosted_apps/lifecycle.py} | 14 +- .../hosted_apps/mcp.py} | 34 +- src/backend/features/hosted_apps/models.py | 5 + .../hosted_apps/operations.py} | 21 +- .../hosted_apps/projects.py} | 10 +- .../hosted_apps/publication.py} | 17 +- .../hosted_apps/relational.py} | 2 +- .../hosted_apps/resources/__init__.py | 1 + .../hosted_apps/resources/admission.py} | 18 +- .../hosted_apps/resources/cleanup.py} | 16 +- .../hosted_apps/schema.py} | 0 .../hosted_apps/sql_policy.py} | 2 +- .../hosted_apps/store.py} | 10 +- src/backend/features/installed.py | 3 + src/backend/features/knowledge/README.md | 11 + .../kb => features/knowledge}/__init__.py | 2 +- .../features/knowledge/api/__init__.py | 1 + src/backend/features/knowledge/api/kb.py | 7 + .../features/knowledge/api/kb_assets.py | 147 + .../features/knowledge/api/kb_chunks.py | 175 + .../features/knowledge/api/kb_documents.py | 255 + .../features/knowledge/api/kb_helpers.py | 105 + .../features/knowledge/api/kb_mutations.py | 117 + .../features/knowledge/api/kb_spaces.py | 116 + .../features/knowledge/api/kb_state.py | 8 + .../v1 => features/knowledge/api}/kb_wiki.py | 24 +- .../features/knowledge/asset_captions.py | 181 + .../features/knowledge/asset_indexing.py | 88 + .../features/knowledge/catalog_options.py | 7 + .../knowledge/disabled_provider.py} | 4 +- .../knowledge/disabled_retrieval.py} | 0 .../knowledge}/document_status.py | 0 src/backend/features/knowledge/edition.py | 33 + src/backend/features/knowledge/enrichment.py | 126 + src/backend/features/knowledge/extension.py | 19 + .../kb => features/knowledge}/index_queue.py | 4 +- .../kb => features/knowledge}/index_worker.py | 8 +- src/backend/features/knowledge/kb_assets.py | 231 + .../features/knowledge/kb_assets_state.py | 31 + src/backend/features/knowledge/kb_parser.py | 116 + .../features/knowledge/kb_parser_state.py | 34 + src/backend/features/knowledge/kb_vector.py | 149 + .../features/knowledge/kb_vector_state.py | 27 + src/backend/features/knowledge/models.py | 377 ++ .../features/knowledge/parser_chunks.py | 278 + .../features/knowledge/parser_extract.py | 302 ++ .../features/knowledge/parser_semantic.py | 130 + .../features/knowledge/parser_tokens.py | 26 + .../features/knowledge/parser_types.py | 43 + .../knowledge/personal_permissions.py} | 2 +- .../knowledge/personal_request_models.py} | 0 .../knowledge/processing.py} | 193 +- .../knowledge/repository.py} | 4 +- .../knowledge/service.py} | 34 +- .../features/knowledge/vector_client.py | 166 + .../features/knowledge/vector_fields.py | 16 + .../features/knowledge/vector_indexing.py | 204 + .../knowledge}/wiki/__init__.py | 2 +- .../kb => features/knowledge}/wiki/cite.py | 10 +- .../kb => features/knowledge}/wiki/config.py | 2 +- .../kb => features/knowledge}/wiki/dedup.py | 16 +- .../kb => features/knowledge}/wiki/extract.py | 12 +- .../knowledge}/wiki/finalize.py | 17 +- .../kb => features/knowledge}/wiki/jobs.py | 4 +- .../kb => features/knowledge}/wiki/linkify.py | 0 .../kb => features/knowledge}/wiki/llm.py | 10 +- .../features/knowledge/wiki/local_graph.py | 156 + .../knowledge/wiki/local_projection.py | 51 + .../features/knowledge/wiki/local_provider.py | 271 + .../knowledge/wiki/local_provider_state.py | 15 + .../knowledge}/wiki/pipeline.py | 26 +- .../kb => features/knowledge}/wiki/prompts.py | 0 .../kb => features/knowledge}/wiki/reduce.py | 17 +- .../kb => features/knowledge}/wiki/slug.py | 0 .../kb => features/knowledge}/wiki/store.py | 8 +- .../knowledge}/wiki/taxonomy.py | 14 +- .../kb => features/knowledge}/wiki/worker.py | 12 +- .../kb => features/knowledge}/wiki_router.py | 16 +- src/backend/features/marketplace/README.md | 15 + src/backend/features/marketplace/__init__.py | 1 + .../features/marketplace/agent_catalog.py | 119 + .../features/marketplace/agent_categories.py | 38 + .../marketplace/agent_installation.py | 240 + .../features/marketplace/agent_metadata.py | 101 + .../features/marketplace/agent_publishing.py | 165 + .../marketplace/agent_submission_data.py | 155 + .../features/marketplace/agent_submissions.py | 160 + .../marketplace/agent_versions.py} | 31 +- .../features/marketplace/agents_state.py | 14 + .../features/marketplace/api/__init__.py | 1 + .../marketplace/api}/agent_marketplace.py | 28 +- .../marketplace/api}/catalog_ownership.py | 9 +- .../marketplace/api}/marketplace.py | 27 +- .../marketplace/api}/mcp_marketplace.py | 37 +- .../features/marketplace/core_catalog.py | 108 + src/backend/features/marketplace/edition.py | 10 + src/backend/features/marketplace/extension.py | 4 + .../marketplace/mcp_administration.py | 291 ++ .../marketplace/mcp_catalog.py} | 0 .../features/marketplace/mcp_credentials.py | 130 + .../features/marketplace/mcp_curated.py | 127 + .../features/marketplace/mcp_installation.py | 212 + .../features/marketplace/mcp_listing.py | 142 + .../marketplace/mcp_monitor.py} | 4 +- .../features/marketplace/mcp_publishing.py | 151 + .../features/marketplace/mcp_revalidation.py | 118 + src/backend/features/marketplace/mcp_state.py | 17 + .../features/marketplace/mcp_submissions.py | 272 + .../features/marketplace/mcp_validation.py | 166 + .../features/marketplace/mcp_versions.py | 221 + src/backend/features/marketplace/models.py | 388 ++ .../features/marketplace/package_source.py | 35 + .../marketplace/personal_listing.py} | 2 +- .../marketplace/personal_visibility.py} | 0 .../marketplace/plugin_market.py} | 63 +- .../features/marketplace/skill_archives.py | 49 + .../features/marketplace/skill_catalog.py | 184 + .../marketplace/skill_installation.py | 199 + .../features/marketplace/skill_metadata.py | 174 + .../features/marketplace/skill_publishing.py | 231 + .../marketplace/skill_submission_data.py | 80 + .../features/marketplace/skill_submissions.py | 190 + .../marketplace/skill_versions.py} | 70 +- .../features/marketplace/skills_state.py | 56 + .../features/marketplace/versions/__init__.py | 1 + .../marketplace/versions/archive.py} | 0 .../marketplace/versions/config.py} | 4 +- .../marketplace/versions/sources.py} | 50 +- .../marketplace/versions/store.py} | 37 +- src/backend/features/sites/README.md | 11 + src/backend/features/sites/__init__.py | 1 + src/backend/features/sites/api/__init__.py | 1 + .../sites/api}/internal_site_auth.py | 0 .../sites/api}/internal_sites.py | 39 +- .../sites/api}/local_site_sources.py | 3 +- .../sites/api}/site_gate.py | 0 .../routes/v1 => features/sites/api}/sites.py | 11 +- .../sites/api}/sites_serve.py | 17 +- .../sites/builtin_upgrade.py} | 29 +- .../sites/desktop_publish.py} | 64 +- src/backend/features/sites/edition.py | 10 + src/backend/features/sites/extension.py | 34 + .../sites/listing.py} | 8 +- .../sites/local_sources.py} | 28 +- .../sites/management.py} | 14 +- .../site.py => features/sites/models.py} | 2 +- .../sites/packaging.py} | 17 +- .../sites/password.py} | 0 .../sites/personal_access.py} | 2 +- .../sites/personal_scope.py} | 0 .../sites/rate_limit.py} | 0 .../site.py => features/sites/repository.py} | 2 +- .../sites/service.py} | 17 +- src/backend/features/sites/tools/__init__.py | 1 + .../sites/tools/design_picker.py} | 46 +- src/backend/features/workbench/README.md | 9 + .../workbench}/__init__.py | 0 .../workbench}/activity.py | 6 +- .../features/workbench/api/__init__.py | 1 + .../workbench/api}/plugin_ui.py | 12 +- .../workbench/api}/workbench.py | 44 +- .../workbench/api}/workbench_activity.py | 4 +- .../api}/workbench_application_config.py | 8 +- .../workbench/api}/workbench_auth.py | 4 +- .../workbench/api}/workbench_capabilities.py | 14 +- .../workbench/api}/workbench_diagnostics.py | 6 +- .../workbench/api}/workbench_events.py | 2 +- .../workbench/api}/workbench_preferences.py | 2 +- .../workbench/api}/workbench_resources.py | 15 +- .../api}/workbench_revision_leases.py | 2 +- .../workbench/api}/workbench_state.py | 2 +- .../workbench/api}/workbench_upgrades.py | 2 +- .../workbench/api}/workbench_workspace.py | 12 +- .../workbench}/application_config.py | 14 +- .../workbench}/application_config_bindings.py | 26 +- .../workbench}/background.py | 10 +- .../workbench}/capabilities.py | 14 +- .../workbench}/catalog.py | 6 +- .../workbench}/coordination.py | 0 .../workbench}/diagnostic_probe.py | 2 +- .../workbench}/diagnostic_resources.py | 0 .../workbench}/diagnostics.py | 2 +- .../workbench}/directory_revision.py | 2 +- src/backend/features/workbench/edition.py | 10 + src/backend/features/workbench/extension.py | 4 + .../workbench}/external.py | 12 +- .../workbench}/external_auth.py | 4 +- .../workbench}/external_boundary.py | 2 +- .../workbench}/external_embed.py | 2 +- .../workbench}/installation_binding.py | 24 +- .../workbench}/local_plugin_upgrades.py | 32 +- .../workbench}/migration.py | 14 +- .../workbench}/migration_compat.py | 25 +- .../workbench}/migration_plan.py | 12 +- .../workbench}/migration_snapshot.py | 10 +- src/backend/features/workbench/models.py | 1 + .../workbench}/origin.py | 0 .../features/workbench/package_source.py | 19 + .../workbench}/plugin_candidate_storage.py | 33 +- .../workbench}/plugin_catalog.py | 10 +- .../workbench}/plugin_revisions.py | 36 +- .../workbench}/plugin_upgrades.py | 26 +- .../workbench}/policy.py | 6 +- .../workbench}/preferences.py | 4 +- .../workbench}/protocol.py | 0 .../workbench/resource_deadlines.py} | 48 +- .../features/workbench/resource_owner.py | 65 + .../workbench}/revocation.py | 18 +- .../workbench}/revocation_sweep.py | 2 +- .../schemas/application.schema.json | 0 .../workbench}/schemas/host.json | 0 .../workbench}/schemas/module.schema.json | 0 .../workbench}/state.py | 8 +- .../workbench}/state_recovery.py | 0 .../workbench}/ui/__init__.py | 0 .../workbench}/ui/contract.py | 2 +- .../workbench}/ui/contributions.py | 20 +- .../workbench}/ui/data_proxy.py | 12 +- .../workbench}/ui/declarations.py | 2 +- .../workbench}/ui/primitives.py | 2 +- .../workbench}/ui/workspace_enabled.py | 0 .../workbench}/workspace_page.py | 4 +- src/backend/hugagent_cli/schema.py | 20 + src/backend/integrations/__init__.py | 8 + src/backend/integrations/api/__init__.py | 1 + .../v1 => integrations/api}/integrations.py | 8 +- src/backend/integrations/channels/README.md | 9 + .../channels/__init__.py | 9 +- .../channels/adapters/__init__.py | 0 .../channels/adapters/dingtalk.py | 325 ++ .../channels/adapters/dingtalk_delivery.py | 213 + .../channels/adapters/dingtalk_history.py | 361 ++ .../channels/adapters/dingtalk_settings.py | 3 + .../integrations/channels/adapters/lark.py | 261 + .../channels/adapters/lark_delivery.py | 152 + .../channels/adapters/lark_messages.py | 317 ++ .../channels/adapters/lark_settings.py | 3 + .../channels/adapters/wecom.py | 2 +- .../channels/adapters/weixin.py | 2 +- .../integrations/channels/api/__init__.py | 1 + .../channels/api}/channel_desktop.py | 14 +- .../channels/api}/channels.py | 10 +- .../channels/desktop_worker.py} | 42 +- .../integrations/channels/extension.py | 7 + src/backend/integrations/channels/inbound.py | 347 ++ .../channels/inbound_attachments.py | 73 + .../integrations/channels/inbound_delivery.py | 118 + .../integrations/channels/inbound_history.py | 147 + .../channels/inbound_observation.py | 168 + .../integrations/channels/inbound_sessions.py | 131 + .../integrations/channels/inbound_state.py | 46 + .../channels/manager.py | 10 +- .../channels/markdown.py | 0 src/backend/integrations/channels/models.py | 133 + .../channels/outbound.py | 8 +- .../channels/protocol.py | 0 .../channels/registry.py | 10 +- .../channels/relay.py} | 4 +- .../channels/relay_dispatch.py} | 14 +- .../channels/relay_rpc.py} | 8 +- .../channels/repository.py} | 2 +- .../channels/service.py} | 20 +- .../integrations/channels/tools/__init__.py | 1 + .../channels/tools/attachment.py} | 36 +- src/backend/integrations/dingtalk/README.md | 5 + src/backend/integrations/dingtalk/__init__.py | 1 + .../integrations/dingtalk/cube_credentials.py | 127 + .../integrations/dingtalk/environment.py | 29 + .../integrations/dingtalk/extension.py | 10 + src/backend/integrations/dingtalk/models.py | 61 + src/backend/integrations/dingtalk/mounts.py | 77 + src/backend/integrations/dingtalk/parsing.py | 105 + .../integrations/dingtalk/repository.py | 39 + src/backend/integrations/dingtalk/sandbox.py | 15 + .../dingtalk/service.py} | 120 +- src/backend/integrations/dingtalk/storage.py | 34 + src/backend/integrations/email/README.md | 5 + src/backend/integrations/email/__init__.py | 1 + .../integrations/email/cube_credentials.py | 53 + src/backend/integrations/email/extension.py | 10 + src/backend/integrations/email/models.py | 65 + src/backend/integrations/email/mounts.py | 72 + src/backend/integrations/email/repository.py | 37 + src/backend/integrations/email/sandbox.py | 14 + .../email/service.py} | 12 +- src/backend/integrations/email/storage.py | 36 + src/backend/integrations/lark/README.md | 5 + src/backend/integrations/lark/__init__.py | 1 + .../integrations/lark/configuration.py | 200 + .../integrations/lark/cube_credentials.py | 90 + src/backend/integrations/lark/extension.py | 10 + src/backend/integrations/lark/login.py | 114 + src/backend/integrations/lark/models.py | 67 + src/backend/integrations/lark/mounts.py | 75 + src/backend/integrations/lark/parsing.py | 141 + src/backend/integrations/lark/process.py | 59 + src/backend/integrations/lark/repository.py | 37 + src/backend/integrations/lark/sandbox.py | 14 + src/backend/integrations/lark/service.py | 189 + src/backend/integrations/lark/state.py | 49 + src/backend/integrations/lark/storage.py | 51 + src/backend/integrations/yida/README.md | 5 + src/backend/integrations/yida/__init__.py | 1 + .../integrations/yida/cube_credentials.py | 152 + .../yida/execution.py} | 4 +- src/backend/integrations/yida/extension.py | 13 + src/backend/integrations/yida/models.py | 1 + src/backend/integrations/yida/mounts.py | 68 + src/backend/integrations/yida/sandbox.py | 18 + .../yida/service.py} | 6 +- .../integrations/yida/skill_content.py | 12 + src/backend/integrations/yida/storage.py | 52 + src/backend/orchestration/__init__.py | 5 +- .../orchestration/automations/__init__.py | 0 .../automation_artifacts.py | 3 +- .../automation_conversation.py | 176 +- .../automations/automation_delivery.py | 62 + .../automation_dispatch.py | 153 +- .../automations/automation_lease.py | 113 + .../automations/automation_recovery.py | 39 + .../automations/automation_scheduler.py | 136 + .../orchestration/chat_run_executor.py | 4651 ----------------- src/backend/orchestration/citation_anchor.py | 551 -- .../orchestration/commands/__init__.py | 1 - src/backend/orchestration/commands/runtime.py | 262 - src/backend/orchestration/configuration.py | 6 + .../orchestration/local_plugin_sidecars.py | 75 - src/backend/orchestration/loops/__init__.py | 0 src/backend/orchestration/loops/admission.py | 121 + .../{autonomous_loop.py => loops/driver.py} | 26 +- .../{loop_planner.py => loops/planner.py} | 14 +- .../orchestration/loops/presentation.py | 71 + src/backend/orchestration/loops/recovery.py | 88 + .../requirements.py} | 59 +- .../loop_reviewer.py => loops/reviewer.py} | 161 +- src/backend/orchestration/loops/run.py | 499 ++ .../orchestration/memory_integration.py | 576 -- src/backend/orchestration/pending.py | 36 + src/backend/orchestration/plans/__init__.py | 0 src/backend/orchestration/plans/admission.py | 171 + .../orchestration/plans/execute_run.py | 328 ++ .../orchestration/plans/generate_run.py | 275 + .../{subagents/plan_mode.py => plans/mode.py} | 80 +- .../schedulers/automation_lease.py | 58 - .../schedulers/automation_recovery.py | 45 - .../schedulers/automation_scheduler.py | 330 -- src/backend/orchestration/streaming.py | 999 ---- .../orchestration/subagents/__init__.py | 2 - .../subagents/ontology_reviewer.py | 656 --- .../{schedulers => workers}/__init__.py | 0 .../{schedulers => workers}/_worker_base.py | 7 +- .../workers}/edition_startup.py | 0 .../evolution_scheduler.py | 4 +- .../memory_ttl_scheduler.py | 6 +- .../persona_distill_worker.py | 6 +- .../orchestration/workers/run_recovery.py | 37 + src/backend/orchestration/workflow.py | 3463 ------------ src/backend/personal_agent/README.md | 2 +- src/backend/personal_agent/access.py | 2 +- .../personal_agent/api/conversation.py | 2 +- .../personal_agent/channels/channel_bridge.py | 21 +- .../personal_agent/computer/computer.py | 5 +- .../personal_agent/computer/devices.py | 2 +- .../personal_agent/computer/driver_install.py | 2 +- .../personal_agent/computer/sandbox.py | 14 +- .../computer/screenshot_pruning.py | 72 +- .../conversation/conversations.py | 3 +- .../personal_agent/conversation/front.py | 36 +- .../personal_agent/delivery/delivery.py | 4 +- .../personal_agent/device_link/README.md | 3 + .../device_link/__init__.py | 2 +- .../device_link/driver.py | 0 .../device_link/local_api.py | 0 .../device_link/runner.py | 6 +- .../device_link/store.py | 0 src/backend/personal_agent/extension.py | 26 +- .../personal_agent/identity/capabilities.py | 8 +- src/backend/personal_agent/identity/config.py | 2 +- .../personal_agent/identity/service.py | 2 +- .../personal_agent/scheduling/schedules.py | 2 +- .../personal_agent/sessions/session_cloud.py | 25 +- .../personal_agent/sessions/session_watch.py | 2 +- .../personal_agent/tasks/attachments.py | 4 +- src/backend/personal_agent/tasks/runtime.py | 10 +- src/backend/personal_agent/tasks/service.py | 2 +- src/backend/personal_agent/tasks/steering.py | 6 +- .../personal_agent/tasks/task_threads.py | 6 +- src/backend/personal_agent/tasks/worker.py | 14 +- src/backend/personal_agent/tools/email.py | 5 +- .../personal_agent/tools/front_tools.py | 35 +- src/backend/personal_agent/tools/responses.py | 4 +- src/backend/personal_agent/tools/tools.py | 31 +- .../personal_agent/workspace/dream_plan.py | 4 +- .../personal_agent/workspace/prompting.py | 7 +- src/backend/prompts/desktop_templates.py | 5 +- src/backend/prompts/desktop_workspace.py | 6 +- src/backend/prompts/kb_lite_section.py | 4 +- src/backend/prompts/project_section.py | 7 +- src/backend/prompts/prompt_runtime.py | 20 +- src/backend/prompts/system_builder.py | 13 +- src/backend/scripts/_loop_convergence_unit.py | 147 - src/backend/scripts/_loop_db_resume_unit.py | 92 - .../scripts/_loop_evaluator_route_unit.py | 88 - src/backend/scripts/_loop_integration_test.py | 88 - src/backend/scripts/_loop_llmeval_smoke.py | 48 - src/backend/scripts/_loop_preflight.py | 73 - src/backend/scripts/_loop_selfverify_smoke.py | 66 - src/backend/scripts/_loop_smoke.py | 90 - src/backend/scripts/_loop_test_edgebench.py | 99 - src/backend/scripts/backend_entrypoint.sh | 4 +- src/backend/scripts/desktop_cloud_wrapper.py | 76 - src/backend/scripts/desktop_fixture_mcp.py | 50 - src/backend/scripts/dual_mode_e2e.py | 947 ---- src/backend/scripts/edgebench/ann_judge.py | 136 - .../scripts/edgebench/perm_opt_judge.py | 88 - src/backend/scripts/edit_chat_message.py | 198 - src/backend/scripts/evaluation_sandbox_e2e.py | 89 - src/backend/scripts/export_content.py | 317 -- .../scripts/harness_fault_injection.py | 637 --- src/backend/scripts/import_cherry_agents.py | 372 -- src/backend/scripts/import_content.py | 379 -- .../scripts/run_evolution_benchmark.py | 308 -- src/backend/scripts/run_evolution_llm_ab.py | 621 --- src/backend/scripts/workbench_migration.py | 66 - src/backend/tests/_capability_index.py | 11 +- .../tests/api/api_routes_v1_selftest.py | 2 +- src/backend/tests/api/catalog_api_selftest.py | 9 +- src/backend/tests/api/test_agent_api_keys.py | 7 +- .../api/test_agent_response_admission.py | 71 +- .../tests/api/test_agent_response_scope.py | 23 +- src/backend/tests/api/test_agent_responses.py | 12 +- .../test_api_key_run_lifecycle_postgres.py | 6 +- src/backend/tests/api/test_archive_files.py | 4 +- src/backend/tests/api/test_archive_safety.py | 4 +- .../tests/api/test_cancel_continue_e2e.py | 34 +- .../api/test_chat_connector_selection.py | 4 +- .../api/test_chat_event_loop_performance.py | 8 +- .../tests/api/test_chat_fork_derived_data.py | 5 +- src/backend/tests/api/test_chat_forks.py | 13 +- .../tests/api/test_chat_reference_api.py | 4 +- .../tests/api/test_chat_rerun_options.py | 18 +- .../api/test_chat_rerun_start_failure.py | 10 +- .../tests/api/test_chat_run_steer_api.py | 9 +- .../tests/api/test_chat_stream_sequencer.py | 56 +- .../tests/api/test_chat_thinking_mode.py | 4 +- .../tests/api/test_cross_turn_file_access.py | 10 +- .../tests/api/test_deep_folder_repair.py | 2 +- .../tests/api/test_evaluation_sandboxes.py | 2 +- src/backend/tests/api/test_file_location.py | 2 +- .../tests/api/test_file_open_browser_e2e.py | 2 +- .../tests/api/test_loop_cancel_scope.py | 39 + src/backend/tests/api/test_me_system.py | 2 +- .../tests/api/test_page_config_backfill.py | 6 +- .../tests/api/test_plugin_delete_isolation.py | 2 +- .../tests/api/test_project_environments.py | 5 +- src/backend/tests/api/test_public_chats.py | 34 +- .../tests/browser_automation_server.py | 81 +- .../test_credential_continuity.py | 189 - .../ce_release/test_ce_api_key_lifecycle.py | 8 +- .../ce_release/test_ce_history_display.py | 2 +- .../ce_release/test_ce_release_regressions.py | 13 +- .../tests/ce_release/test_ce_tool_media.py | 8 +- .../test_desktop_capability_wiring.py | 6 +- .../ce_release/test_personal_agent_ce.py | 12 +- .../test_workbench_migration_snapshot.py | 6 +- src/backend/tests/chat/test_display_bounds.py | 2 +- src/backend/tests/chat/test_sidebar_order.py | 14 +- .../chat/test_snapshot_transaction_cleanup.py | 23 + .../tests/chat/test_user_questions_api.py | 37 +- src/backend/tests/conftest.py | 48 +- .../tests/content/artifacts_selftest.py | 4 +- src/backend/tests/content/catalog_selftest.py | 5 +- .../tests/content/content_blocks_selftest.py | 2 +- .../tests/content/security_selftest.py | 4 +- .../content/test_artifact_default_path.py | 24 + .../content/test_artifact_record_store.py | 4 +- .../content/test_artifact_storage_keys.py | 10 +- src/backend/tests/content/test_extra_files.py | 42 +- .../tests/content/test_file_preview.py | 2 +- .../tests/content/test_local_file_location.py | 4 +- .../content/test_office_preview_cache.py | 6 +- .../test_storage_existence_is_strict.py | 6 +- .../runs/test_automation_journal_guard.py | 89 + .../tests/e2e/_e2e_regression_smoke.py | 2 +- src/backend/tests/e2e/agentscope_poc.py | 2 +- src/backend/tests/e2e/core_external_server.py | 4 +- src/backend/tests/e2e/core_refactor_server.py | 2 +- src/backend/tests/e2e/model_tests.py | 4 +- src/backend/tests/evolution/conftest.py | 2 +- .../test_benchmark_and_activation.py | 24 +- .../evolution/test_candidate_reasoning.py | 32 +- .../evolution/test_cassette_and_replay.py | 4 +- .../tests/evolution/test_compiler_v2.py | 73 +- .../evolution/test_console_and_metrics.py | 8 +- .../tests/evolution/test_credit_assigner.py | 2 +- .../evolution/test_engines_end_to_end.py | 71 +- .../tests/evolution/test_episode_assembler.py | 32 +- .../test_experiments_and_federation.py | 4 +- .../tests/evolution/test_exposure_and_ramp.py | 9 +- .../evolution/test_fault_injection_gate.py | 4 +- .../tests/evolution/test_mechanisms.py | 6 +- .../tests/evolution/test_ontology_bridge.py | 2 +- .../evolution/test_outcomes_and_backfill.py | 6 +- .../tests/evolution/test_personal_approval.py | 61 +- .../test_personal_cycle_and_prefs.py | 20 +- .../evolution/test_policies_and_memory_ops.py | 67 +- .../tests/evolution/test_promotion_chain.py | 98 +- .../evolution/test_react_intervention.py | 29 +- .../tests/evolution/test_readiness_gate.py | 18 +- .../tests/evolution/test_realistic_corpus.py | 46 +- .../evolution/test_release_state_machine.py | 2 +- .../tests/evolution/test_runtime_binding.py | 8 +- .../tests/evolution/test_settlement.py | 2 +- .../tests/evolution/test_settlement_runner.py | 4 +- .../evolution/test_skill_selection_record.py | 6 +- src/backend/tests/evolution/test_sop_judge.py | 2 +- .../tests/evolution/test_three_engines.py | 59 +- src/backend/tests/evolution/test_trigger.py | 9 +- .../tests/evolution/test_turn_card_edits.py | 4 +- .../tests/evolution/test_user_contribution.py | 72 +- src/backend/tests/features/__init__.py | 1 + .../tests/features/automation/__init__.py | 1 + .../tests/features/automation/run_fixtures.py | 15 + .../test_automation_delivery_deadline.py | 179 + .../test_automation_execution_location.py | 94 +- .../automation/test_automation_lease_close.py | 199 + .../test_automation_manual_trigger.py | 81 + .../test_automation_notifications.py | 2 +- .../test_automation_oneshot_finalize.py | 49 + .../test_automation_ownership_postgres.py | 173 + .../test_automation_remote_effect.py | 64 +- .../test_automation_run_ownership.py | 256 + .../test_delivery_target_migration.py | 31 + .../automation/test_durable_admission.py | 96 + .../automation}/test_loop_service_harness.py | 2 +- .../automation/test_queue_postgres.py | 94 + .../tests/features/desktop/__init__.py | 1 + .../features/desktop/capabilities/__init__.py | 1 + .../desktop}/capabilities/_cloud_identity.py | 0 .../desktop}/capabilities/conftest.py | 34 +- .../capabilities/management_browser_server.py | 17 +- .../capabilities/runtime_recovery_support.py | 19 +- .../test_account_authorization.py | 30 +- .../capabilities/test_agent_rerun_identity.py | 60 +- .../capabilities/test_agent_runtime_grants.py | 22 +- .../capabilities/test_agents_plugins.py | 42 +- .../test_autonomous_loop_scopes.py | 95 +- .../capabilities/test_available_runtime.py | 65 +- .../test_capability_change_sync.py | 56 +- .../test_catalog_cloud_only_in_hybrid.py | 41 +- .../test_catalog_plugin_components_hidden.py | 8 +- .../capabilities/test_change_merge.py | 4 +- .../capabilities/test_change_signal.py | 2 +- .../capabilities/test_cloud_identity.py | 2 +- .../capabilities/test_cloud_management.py | 17 +- .../test_cloud_parallel_preparation.py | 33 +- .../test_cloud_plugin_binding_runtime.py | 32 +- .../capabilities/test_cloud_sticky_plugins.py | 52 +- .../capabilities/test_cloud_workcopies.py | 25 +- .../desktop}/capabilities/test_core_layers.py | 13 +- .../test_credential_continuity.py | 193 + .../capabilities/test_dependency_closure.py | 53 +- .../test_desktop_capabilities_api.py | 52 +- .../test_desktop_content_safety.py | 67 +- .../test_desktop_declaration_export.py | 21 +- .../test_desktop_model_reasoning_safety.py | 21 +- .../test_desktop_resource_plugins.py | 28 +- .../test_desktop_token_security.py | 36 +- .../capabilities/test_desktop_warmup.py | 28 +- .../capabilities/test_device_api_contract.py | 58 +- .../test_device_catalog_projection.py | 34 +- .../test_dynamic_model_epoch_cache.py | 28 +- .../test_explicit_desktop_invocation.py | 27 +- .../test_gateway_invocation_proof.py | 16 +- .../test_gateway_unknown_outcome.py | 52 +- .../capabilities/test_hybrid_owned_plugins.py | 22 +- .../test_initial_sync_readiness.py | 30 +- .../test_junction_windows_paths.py | 10 +- .../capabilities/test_loader_store_mode.py | 34 +- .../test_local_folder_discovery.py | 4 +- .../test_local_management_migration.py | 22 +- .../test_local_skill_projection.py | 17 +- .../capabilities/test_management_readiness.py | 26 +- .../capabilities/test_management_tools.py | 61 +- .../capabilities/test_manager_lifecycle.py | 8 +- .../capabilities/test_manager_regressions.py | 29 +- .../capabilities/test_manifest_order.py | 32 +- .../capabilities/test_manifest_tool_schema.py | 27 +- .../desktop}/capabilities/test_mcp_json.py | 2 +- .../capabilities/test_mcp_json_consistency.py | 2 +- .../capabilities/test_memory_epoch_cache.py | 31 +- .../desktop}/capabilities/test_migration.py | 9 +- .../test_migration_preservation.py | 5 +- .../test_model_credential_reference.py | 13 +- .../test_model_error_diagnostics.py | 6 +- .../test_owned_connector_assembly.py | 38 +- .../test_partial_session_availability.py | 33 +- .../capabilities/test_plugin_connections.py | 10 +- .../capabilities/test_plugin_device_assets.py | 10 +- .../test_prepare_lock_not_held_over_io.py | 2 +- .../capabilities/test_progressive_plugins.py | 166 +- .../test_runtime_frozen_bindings.py | 142 +- .../test_runtime_publication_recovery.py | 120 +- .../capabilities/test_runtime_recovery.py | 72 +- .../test_runtime_scope_consumers.py | 40 +- .../capabilities/test_runtime_scopes.py | 98 +- .../test_runtime_selection_recovery.py | 99 +- .../test_runtime_session_recovery.py | 103 +- .../test_selection_dependency_sources.py | 36 +- .../capabilities/test_selection_readiness.py | 14 +- .../capabilities/test_store_file_integrity.py | 19 +- .../test_toolkit_runtime_alias.py | 40 +- .../capabilities/test_user_facing_errors.py | 11 +- .../capabilities/test_view_os_errors.py | 7 +- .../test_workbench_local_upgrades.py | 16 +- .../desktop}/desktop_login_browser_server.py | 6 +- .../tests/features/desktop/index_fixtures.py | 13 + .../features/desktop/observability_cases_1.py | 310 ++ .../features/desktop/observability_cases_2.py | 270 + .../features/desktop/observability_support.py | 32 + .../desktop}/run_desktop_browser_e2e.py | 2 +- .../desktop}/test_desktop_auth.py | 4 +- .../desktop}/test_desktop_bridge.py | 26 +- .../desktop}/test_desktop_capability.py | 101 +- .../test_desktop_capability_models.py | 27 +- .../desktop}/test_desktop_cloud_skills.py | 70 +- .../desktop}/test_desktop_device_login.py | 9 +- .../test_desktop_memory_persistence.py | 8 +- .../desktop}/test_desktop_model_gateway.py | 45 +- .../desktop/test_desktop_observability.py | 20 + ...op_observability_authenticated_delivery.py | 54 +- .../test_desktop_observability_delivery.py | 64 +- .../desktop}/test_desktop_release_events.py | 4 +- .../desktop}/test_desktop_sync_locking.py | 31 +- .../desktop}/test_desktop_updates.py | 4 +- .../desktop}/test_local_file_actions.py | 20 +- .../desktop}/test_local_grant_service.py | 2 +- .../desktop}/test_local_snapshot_service.py | 2 +- .../tests/features/evaluation/__init__.py | 1 + .../tests/features/evaluation/conftest.py | 9 + .../evaluation}/test_evaluation_admission.py | 4 +- .../evaluation}/test_evaluation_binding.py | 2 +- .../evaluation}/test_evaluation_factory.py | 15 +- .../test_evaluation_process_guard.py | 2 +- .../evaluation}/test_evaluation_provider.py | 10 +- .../evaluation}/test_evaluation_router.py | 38 +- .../evaluation}/test_evaluation_runtime.py | 36 +- .../evaluation}/test_evaluation_service.py | 6 +- .../tests/features/hosted_apps/__init__.py | 1 + .../application_hosting_support.py | 19 +- .../test_application_agent_contract.py | 12 +- .../test_application_collections.py | 18 +- .../hosted_apps}/test_application_desktop.py | 65 +- .../hosted_apps}/test_application_export.py | 2 +- .../hosted_apps}/test_application_hosting.py | 40 +- .../test_application_personal_mcp.py | 30 +- .../test_application_project_tools.py | 16 +- .../hosted_apps}/test_application_projects.py | 9 +- .../test_application_retirement.py | 10 +- .../hosted_apps}/test_hosting_remediation.py | 28 +- .../hosted_apps/test_schema_lifecycle.py | 60 + .../tests/features/knowledge/__init__.py | 1 + .../knowledge/kb_multimodal_assets_cases_1.py | 353 ++ .../knowledge/kb_multimodal_assets_cases_2.py | 120 + .../knowledge/kb_multimodal_assets_support.py | 235 + .../knowledge/kb_wiki_end_to_end_cases_1.py | 349 ++ .../knowledge/kb_wiki_end_to_end_cases_2.py | 175 + .../knowledge/kb_wiki_end_to_end_support.py | 197 + .../knowledge/test_host_contributions.py | 68 + .../knowledge}/test_kb_backend_routing.py | 59 +- .../knowledge}/test_kb_document_status.py | 7 +- .../knowledge}/test_kb_index_queue.py | 5 +- .../knowledge}/test_kb_index_worker.py | 12 +- .../knowledge/test_kb_multimodal_assets.py | 37 + .../knowledge}/test_kb_vector_safety.py | 51 +- .../knowledge}/test_kb_vector_timeouts.py | 27 +- .../knowledge/test_kb_wiki_end_to_end.py | 31 + .../knowledge}/test_kb_wiki_pipeline.py | 24 +- .../tests/features/marketplace/__init__.py | 1 + .../marketplace/mcp_marketplace_cases_1.py | 434 ++ .../marketplace/mcp_marketplace_cases_2.py | 466 ++ .../marketplace/mcp_marketplace_cases_3.py | 458 ++ .../marketplace/mcp_marketplace_cases_4.py | 435 ++ .../marketplace/mcp_marketplace_support.py | 269 + .../marketplace}/test_agent_marketplace.py | 49 +- .../marketplace}/test_marketplace_service.py | 83 +- .../test_marketplace_submission.py | 90 +- .../marketplace/test_mcp_marketplace.py | 69 + .../marketplace/test_oauth_sdk2_contract.py | 113 + src/backend/tests/features/sites/__init__.py | 1 + src/backend/tests/features/sites/conftest.py | 2 + .../sites}/test_cloud_site_publish.py | 38 +- .../sites}/test_design_pick.py | 2 +- .../sites}/test_desktop_site_paths.py | 20 +- .../sites}/test_local_project_site_publish.py | 18 +- .../sites}/test_local_site_sources.py | 56 +- .../sites}/test_site_access_password.py | 16 +- .../sites}/test_site_contention_postgres.py | 17 +- .../sites}/test_site_listing.py | 19 +- .../sites}/test_site_plugin_upgrade.py | 40 +- .../sites}/test_site_rate_limit.py | 2 +- .../test_execution_identity_contract.py | 44 + .../tests/features/test_host_runtime.py | 18 + .../tests/features/test_installed_modules.py | 63 + .../features/test_run_capability_host.py | 15 + .../tests/features/workbench/__init__.py | 1 + .../workbench}/test_plugin_ui_contract.py | 10 +- .../workbench}/test_plugin_ui_routes.py | 4 +- .../test_workbench_activity_routes.py | 6 +- .../test_workbench_actual_process_observer.py | 4 +- .../test_workbench_application_config_http.py | 12 +- .../test_workbench_approval_lease_http.py | 16 +- .../workbench}/test_workbench_auth_routes.py | 8 +- .../test_workbench_background_subpage_http.py | 6 +- .../test_workbench_candidate_storage.py | 12 +- .../test_workbench_capability_routes.py | 6 +- .../test_workbench_closing_isolation_http.py | 14 +- .../test_workbench_cookie_boundaries.py | 2 +- .../workbench}/test_workbench_coordination.py | 2 +- .../test_workbench_diagnostic_failure_http.py | 4 +- .../workbench}/test_workbench_diagnostics.py | 4 +- .../workbench}/test_workbench_embed_routes.py | 2 +- .../test_workbench_external_routes.py | 0 .../workbench}/test_workbench_invalidation.py | 14 +- .../workbench}/test_workbench_migration.py | 8 +- .../test_workbench_migration_auth_gate.py | 6 +- .../test_workbench_observer_paths.py | 2 +- ...st_workbench_official_capabilities_http.py | 4 +- .../workbench}/test_workbench_origins.py | 10 +- .../test_workbench_plugin_routes.py | 4 +- .../test_workbench_preferences_routes.py | 2 +- .../workbench}/test_workbench_protocol.py | 8 +- .../test_workbench_provider_scope_http.py | 4 +- ...est_workbench_resource_owner_migrations.py | 2 +- .../workbench}/test_workbench_resources.py | 31 +- .../test_workbench_revocation_failure_http.py | 28 +- .../test_workbench_routing_routes.py | 4 +- .../test_workbench_state_binding.py | 2 +- .../test_workbench_state_concurrency.py | 2 +- .../workbench}/test_workbench_state_routes.py | 2 +- .../test_workbench_upgrade_routes.py | 4 +- .../workbench_capabilities_http_fixture.py | 6 +- .../workbench_diagnostic_http_fixture.py | 2 +- .../workbench_embed_http_fixture.py | 2 +- .../workbench}/workbench_http_fixture.py | 2 +- .../workbench_observed_runner_fixture.py | 4 +- .../workbench}/workbench_observer_paths.py | 2 +- .../workbench_overflow_http_fixture.py | 2 +- .../workbench_plugin_http_fixture.py | 2 +- .../workbench}/workbench_plugin_support.py | 0 .../workbench}/workbench_resource_observer.py | 6 +- .../workbench_routing_http_fixture.py | 2 +- src/backend/tests/infra/test_core_cleanup.py | 23 +- .../integration/background_command_server.py | 13 +- .../integration/job_agent_local_smoke.py | 15 +- .../tests/integration/job_local_smoke.py | 18 +- .../tests/integration/job_managed_smoke.py | 12 +- src/backend/tests/integrations/__init__.py | 1 + .../tests/integrations/channels/__init__.py | 1 + .../integrations/channels/bot_cases_1.py | 381 ++ .../integrations/channels/bot_cases_2.py | 362 ++ .../integrations/channels/bot_cases_3.py | 374 ++ .../integrations/channels/bot_cases_4.py | 382 ++ .../integrations/channels/bot_cases_5.py | 381 ++ .../integrations/channels/bot_cases_6.py | 145 + .../integrations/channels/bot_support.py | 179 + .../channels/test_adapter_method_binding.py | 16 + .../channels/test_channel_bots.py | 115 + .../channels}/test_channel_bots_api.py | 6 +- .../channels}/test_channel_desktop_e2e.py | 42 +- .../channels}/test_channel_desktop_http.py | 31 +- .../test_channel_desktop_migration.py | 2 +- .../channels}/test_channel_desktop_relay.py | 13 +- .../test_channel_desktop_relay_api.py | 19 +- .../channels}/test_channel_desktop_worker.py | 52 +- .../test_channel_receiver_lifecycle.py | 10 +- .../test_weixin_connection_health.py | 10 +- .../channels}/test_weixin_qr_protocol.py | 6 +- .../dingtalk}/test_dingtalk_integration.py | 34 +- .../email}/test_email_integration.py | 42 +- .../integrations/lark/test_app_lifecycle.py | 48 + .../lark}/test_lark_auth_status.py | 39 +- .../yida}/test_yida_integration.py | 29 +- src/backend/tests/llm/command_test_support.py | 15 +- .../tests/llm/e2e_multiturn_toolcall.py | 4 +- ...le_context_hook_user_id_gating_selftest.py | 8 +- .../tests/llm/local_project_test_support.py | 29 +- src/backend/tests/llm/mcp_env_debug.py | 6 +- .../tests/llm/test_agent_api_execution.py | 15 +- .../tests/llm/test_agent_api_runtime.py | 51 +- .../tests/llm/test_api_protocol_backfill.py | 8 +- src/backend/tests/llm/test_approval_modes.py | 24 +- src/backend/tests/llm/test_artifact_refs.py | 2 +- .../tests/llm/test_ask_user_question_tool.py | 14 +- .../tests/llm/test_bash_process_tools.py | 23 +- .../tests/llm/test_builtin_subagents.py | 19 +- .../tests/llm/test_cancelled_replay.py | 2 +- .../tests/llm/test_chat_history_tool.py | 18 +- .../tests/llm/test_chat_model_stream_retry.py | 8 +- src/backend/tests/llm/test_compaction.py | 2 +- src/backend/tests/llm/test_compaction_e2e.py | 14 +- .../tests/llm/test_compaction_integration.py | 22 +- .../tests/llm/test_compaction_mid_turn.py | 35 +- .../tests/llm/test_compaction_observation.py | 16 +- .../tests/llm/test_compaction_progress.py | 56 +- .../llm/test_compaction_summary_requests.py | 6 +- .../tests/llm/test_compaction_watermark.py | 38 +- .../llm/test_configured_reasoning_effort.py | 10 +- src/backend/tests/llm/test_context_adapter.py | 26 +- src/backend/tests/llm/test_context_ir.py | 8 +- .../llm/test_context_order_user_input.py | 16 +- src/backend/tests/llm/test_context_probe.py | 4 +- src/backend/tests/llm/test_context_safety.py | 28 +- .../tests/llm/test_context_step_units.py | 25 +- .../llm/test_context_usage_adversarial.py | 10 +- .../tests/llm/test_context_usage_snapshot.py | 13 +- .../tests/llm/test_execution_manifest.py | 10 +- .../llm/test_execution_surface_limits.py | 37 +- .../llm/test_execution_surface_snapshot.py | 24 +- .../tests/llm/test_failover_context_safety.py | 8 +- ...st_file_context_offloads_blocking_parse.py | 25 +- .../tests/llm/test_file_context_preview.py | 18 +- .../llm/test_live_approval_integration.py | 8 +- .../tests/llm/test_live_approval_repair.py | 30 +- .../tests/llm/test_local_bash_permission.py | 52 +- .../llm/test_local_confinement_contract.py | 11 +- .../tests/llm/test_local_file_permission.py | 46 +- .../tests/llm/test_local_myspace_block.py | 2 +- .../tests/llm/test_local_project_cwd.py | 57 +- .../tests/llm/test_local_project_delivery.py | 73 +- .../tests/llm/test_local_project_pin.py | 74 +- .../tests/llm/test_local_session_reference.py | 24 +- .../tests/llm/test_manifest_mcp_client.py | 27 +- src/backend/tests/llm/test_mcp_factory_e2e.py | 88 + .../tests/llm/test_mcp_human_wait_timeout.py | 271 + .../tests/llm/test_mcp_native_adversarial.py | 219 + .../tests/llm/test_mcp_native_capabilities.py | 169 + .../tests/llm/test_mcp_parallel_execution.py | 36 +- src/backend/tests/llm/test_mcp_pool_cache.py | 203 - .../tests/llm/test_mcp_runtime_pool.py | 272 + .../tests/llm/test_mcp_session_ownership.py | 128 + src/backend/tests/llm/test_mcp_timeouts.py | 32 +- .../llm/test_migration_vendor_datasource.py | 114 + src/backend/tests/llm/test_model_failover.py | 27 +- src/backend/tests/llm/test_model_steps.py | 26 +- .../tests/llm/test_model_usage_extraction.py | 2 +- .../llm/test_model_usage_scope_teardown.py | 2 +- src/backend/tests/llm/test_offloader.py | 40 +- src/backend/tests/llm/test_openai_provider.py | 14 +- .../llm/test_permission_ticket_isolation.py | 21 +- .../llm/test_plan_update_tool_description.py | 10 +- .../llm/test_post_turn_measured_compaction.py | 14 +- .../llm/test_project_environment_factory.py | 32 +- .../llm/test_project_environment_policy.py | 14 +- src/backend/tests/llm/test_protocol_probe.py | 8 +- .../tests/llm/test_read_image_native.py | 26 +- .../tests/llm/test_reasoning_budget.py | 6 +- .../llm/test_reasoning_echo_formatter.py | 4 +- src/backend/tests/llm/test_reasoning_probe.py | 2 +- .../tests/llm/test_reasoning_replay_policy.py | 10 +- .../llm/test_reasoning_retained_in_history.py | 8 +- .../llm/test_responses_reasoning_replay.py | 7 +- .../llm/test_responses_thinking_stream.py | 94 +- .../tests/llm/test_responses_tool_call_id.py | 20 +- .../llm/test_sandbox_session_resolution.py | 8 +- .../llm/test_selected_model_context_window.py | 27 +- .../llm/test_shared_human_interactions.py | 8 +- .../tests/llm/test_single_turn_protocol.py | 2 +- .../tests/llm/test_stream_wrapper_cleanup.py | 30 + .../llm/test_structured_reasoning_flag.py | 4 +- .../tests/llm/test_subagent_cancellation.py | 25 +- .../tests/llm/test_subagent_model_config.py | 15 +- src/backend/tests/llm/test_subagent_resume.py | 10 +- .../tests/llm/test_subagent_tool_streaming.py | 12 +- src/backend/tests/llm/test_system_reminder.py | 7 +- .../test_thinking_modes_across_protocols.py | 10 +- .../tests/llm/test_tool_call_identity.py | 2 +- .../tests/llm/test_tool_media_no_disk.py | 10 +- .../llm/test_tool_permission_hardening.py | 17 +- .../tests/llm/test_tool_permissions.py | 43 +- .../tests/llm/test_tool_result_images.py | 46 +- src/backend/tests/llm/test_vendor_timeouts.py | 4 +- src/backend/tests/llm/test_vision_bridge.py | 20 +- .../tests/llm/test_vision_bridge_e2e.py | 6 +- .../tests/llm/test_vision_injection.py | 76 +- .../tests/llm/test_vision_progress_events.py | 44 +- .../tests/llm/test_vision_role_capability.py | 7 +- src/backend/tests/llm/test_workspace.py | 13 +- .../tests/llm/test_workspace_profiles.py | 99 +- src/backend/tests/llm/tool_call_debug.py | 4 + .../tests/llm/vision_fullstack_check.py | 4 +- src/backend/tests/llm/vision_live_check.py | 12 +- .../tests/llm/vision_role_live_check.py | 10 +- src/backend/tests/mcp/test_internet_search.py | 12 +- .../tests/mcp/test_wiki_tool_gating.py | 13 +- .../tests/mcp_servers/test_search_batch.py | 28 +- .../tests/mcp_servers/test_web_fetch.py | 3 +- .../tests/memory/outbox_test_support.py | 2 +- .../test_ce_overlay_memory_contracts.py | 41 +- src/backend/tests/memory/test_graph_memory.py | 22 +- .../memory/test_identity_frozen_block.py | 6 +- .../tests/memory/test_memory_outbox.py | 4 +- .../memory/test_memory_outbox_receipts.py | 8 +- .../memory/test_memory_outbox_recovery.py | 16 +- .../memory/test_memory_retrieval_lifecycle.py | 36 +- .../memory/test_session_frozen_snapshot.py | 34 +- .../misc/chat_metadata_merge_selftest.py | 2 +- src/backend/tests/misc/conftest.py | 15 +- src/backend/tests/misc/test_identity_dedup.py | 2 +- .../tests/ontology/test_ontology_reviewer.py | 40 +- .../ontology/test_ontology_revision_accept.py | 10 +- .../test_ontology_workflow_evidence.py | 17 +- .../agent_mention_parse_selftest.py | 2 +- .../orchestration/automation_test_support.py | 17 + .../orchestration/command_test_support.py | 2 +- .../tests/orchestration/hook_support.py | 53 + .../orchestration/recovery_test_support.py | 17 +- .../orchestration/router_strategy_selftest.py | 2 +- .../orchestration/routing_import_selftest.py | 2 +- .../orchestration/stream_protocol_selftest.py | 7 +- .../streaming_tool_call_dedupe_selftest.py | 11 +- .../orchestration/test_agent_api_recovery.py | 40 +- .../test_agent_runtime_resources.py | 20 + .../test_agentscope_hook_adapter.py | 31 +- .../test_automation_admission.py | 50 +- .../orchestration/test_automation_handoff.py | 50 +- .../test_automation_live_conversations.py | 125 +- .../orchestration/test_automation_reaper.py | 4 +- .../test_automation_unbounded.py | 65 +- .../test_autonomous_loop_driver.py | 12 +- .../orchestration/test_background_commands.py | 87 +- .../test_chat_resume_checkpoint.py | 16 +- .../test_chat_run_cancel_persistence.py | 57 +- .../test_chat_run_follow_recovery.py | 104 +- .../orchestration/test_chat_run_human_wait.py | 24 +- .../orchestration/test_chat_run_recovery.py | 37 +- .../test_chat_run_restart_recovery.py | 89 +- .../test_chat_run_worker_recovery.py | 56 +- .../tests/orchestration/test_chat_steer.py | 104 +- .../orchestration/test_command_events.py | 57 +- .../orchestration/test_command_recovery.py | 158 +- .../test_command_worker_lifecycle.py | 18 +- .../orchestration/test_context_budget_stop.py | 41 +- .../orchestration/test_context_provenance.py | 11 +- .../test_context_stats_finalization_race.py | 6 +- .../test_direct_subagent_routing.py | 79 +- .../test_event_offset_fencing.py | 11 +- .../test_execution_manifest_binding.py | 128 +- .../tests/orchestration/test_finish_guard.py | 79 +- .../test_harness_e2e_fault_matrix.py | 18 +- .../test_harness_ledger_postgres.py | 4 +- .../test_harness_process_restart.py | 52 - .../test_harness_usage_ledger.py | 57 +- .../tests/orchestration/test_hook_bus.py | 141 + .../orchestration/test_inactivity_watchdog.py | 2 +- .../orchestration/test_inflight_message.py | 40 +- .../test_job_callback_transport.py | 7 +- .../orchestration/test_job_conflict_policy.py | 13 +- .../orchestration/test_job_derived_agent.py | 56 +- .../tests/orchestration/test_job_failfast.py | 35 +- .../test_job_inherited_grants.py | 18 +- .../orchestration/test_job_local_callback.py | 5 +- .../orchestration/test_job_managed_runtime.py | 46 +- .../orchestration/test_job_orphan_reaper.py | 23 +- .../orchestration/test_job_resilience.py | 4 +- .../orchestration/test_job_sdk_ledger.py | 2 +- .../orchestration/test_job_ui_progress.py | 12 +- .../test_job_wake_delivery_format.py | 2 +- .../test_job_wake_message_visibility.py | 32 +- .../tests/orchestration/test_job_wake_mode.py | 34 +- .../test_job_windows_attempt_paths.py | 5 +- .../test_loop_reviewer_registry_tool.py | 4 +- .../test_missing_tool_call_synthesis.py | 4 +- .../test_model_progress_signal.py | 22 +- .../test_model_steps_pipeline.py | 25 +- .../test_model_steps_plan_projection.py | 66 +- .../orchestration/test_model_steps_worker.py | 47 +- .../orchestration/test_observation_writer.py | 146 + .../orchestration/test_pending_workflows.py | 68 + .../orchestration/test_plan_early_cancel.py | 84 + .../test_plan_progress_closeout.py | 10 +- .../orchestration/test_plan_stale_reminder.py | 57 +- .../test_project_api_recovery.py | 25 +- .../test_project_init_stream_input.py | 5 +- .../orchestration/test_run_event_stream.py | 16 +- .../tests/orchestration/test_run_journal.py | 9 +- .../test_run_journal_postgres.py | 15 +- .../test_runtime_phase_lifecycle.py | 203 + .../orchestration/test_site_mode_prompt.py | 11 +- .../tests/orchestration/test_stale_reaper.py | 123 +- .../tests/orchestration/test_steer_queue.py | 13 +- .../test_structured_reasoning_protocol.py | 2 +- .../test_thinking_block_persistence.py | 22 +- .../test_thinking_suppression.py | 2 +- .../orchestration/test_tool_effect_ledger.py | 1405 +---- .../test_tool_effect_ledger_2.py | 429 ++ .../test_tool_effect_ledger_3.py | 477 ++ .../test_tool_effect_ledger_4.py | 459 ++ .../test_tool_effect_ledger_5.py | 133 + .../test_user_question_passthrough.py | 42 +- .../tests/orchestration/test_worker_base.py | 3 +- .../test_workflow_hint_consistency.py | 2 +- .../test_workflow_model_runtime.py | 10 +- .../orchestration/tool_effect_support.py | 81 + .../{ => personal_agent}/test_device_link.py | 8 +- .../prompts/test_desktop_delivery_tools.py | 2 +- .../tests/prompts/test_desktop_environment.py | 13 +- .../prompts/test_desktop_prompt_versions.py | 40 +- ...est_local_mode_drops_container_sections.py | 16 +- .../prompts/test_local_project_section.py | 16 +- .../tests/prompts/test_project_isolation.py | 2 +- .../prompts/test_prompt_cache_full_hash.py | 4 +- .../test_prompt_parts_from_filesystem.py | 56 +- .../prompts/test_prompt_version_freshness.py | 30 +- .../tests/run_browser_opensandbox_e2e.py | 8 +- .../tests/run_command_opensandbox_e2e.py | 12 +- .../tests/run_personal_agent_cli_grant_e2e.py | 3 +- src/backend/tests/run_personal_agent_e2e.py | 6 +- .../tests/run_personal_agent_key_e2e.py | 2 +- ...run_personal_agent_project_sessions_e2e.py | 5 +- .../tests/sandbox/e2e_myspace_bulk_live.py | 6 +- .../tests/sandbox/fixtures/egress_gateway.py | 14 +- .../tests/sandbox/provider_test_support.py | 5 +- .../test_adopt_respects_session_binding.py | 3 - .../tests/sandbox/test_artifact_streaming.py | 8 +- .../sandbox/test_cloud_process_sessions.py | 6 +- .../tests/sandbox/test_controller_capacity.py | 37 + .../sandbox/test_creation_adoption_race.py | 9 +- .../test_credential_purge_preserves_mounts.py | 21 +- .../sandbox/test_cube_skill_push_ownership.py | 7 +- .../sandbox/test_idle_reclaim_snapshot.py | 3 +- .../sandbox/test_multiworker_lifecycle.py | 2 +- .../sandbox/test_multiworker_transactions.py | 2 +- .../sandbox/test_opensandbox_expirations.py | 2 + .../sandbox/test_opensandbox_provider.py | 30 + src/backend/tests/sandbox/test_os_sandbox.py | 4 +- .../sandbox/test_pool_retirement_guards.py | 18 +- .../sandbox/test_product_session_scope.py | 2 +- .../tests/sandbox/test_runtime_ownership.py | 61 + .../tests/sandbox/test_sandbox_accounting.py | 14 +- .../tests/sandbox/test_sandbox_packaging.py | 16 +- .../sandbox/test_service_loop_affinity.py | 4 +- .../tests/sandbox/test_session_affinity.py | 2 +- .../sandbox/test_session_lock_cross_loop.py | 2 +- .../tests/sandbox/test_user_bound_sandbox.py | 21 - .../services/project_instructions_support.py | 5 +- .../tests/services/test_artifact_reader.py | 41 +- .../tests/services/test_artifact_summary.py | 20 +- src/backend/tests/services/test_audio_live.py | 2 +- .../tests/services/test_audio_models.py | 8 +- .../tests/services/test_audio_routes.py | 4 +- .../services/test_audio_transcription.py | 4 +- .../tests/services/test_chat_mode_service.py | 3 +- .../services/test_chat_reference_service.py | 11 +- .../tests/services/test_chat_sequencer.py | 74 +- .../tests/services/test_chat_titles.py | 33 +- .../services/test_kb_multimodal_assets.py | 657 --- .../tests/services/test_kb_wiki_end_to_end.py | 663 --- .../services/test_memory_settings_service.py | 16 +- .../services/test_model_config_consistency.py | 8 +- .../services/test_model_config_export.py | 8 +- .../services/test_plan_execution_snapshot.py | 2 +- .../test_project_context_no_inventory.py | 8 +- .../services/test_project_environment.py | 17 +- .../services/test_project_instructions.py | 35 +- .../test_project_instructions_dispatch.py | 26 +- .../tests/services/test_retire_chart_mcp.py | 2 +- .../services/test_turbo_capability_config.py | 14 +- .../services/test_user_model_selection.py | 13 +- .../services/user_folder_service_selftest.py | 6 +- .../tests/skills/agent_skills_selftest.py | 4 +- src/backend/tests/skills/backends_selftest.py | 6 +- src/backend/tests/skills/loader_selftest.py | 7 +- .../tests/skills/skill_loading_selftest.py | 2 +- .../tests/skills/skill_selection_selftest.py | 8 +- .../skills/test_database_backend_lazy.py | 10 +- .../tests/skills/test_deps_detector.py | 2 +- .../skills/test_read_artifact_pptx_xlsx.py | 54 +- src/backend/tests/skills/test_registry.py | 6 +- .../tests/skills/test_sandbox_skill_purge.py | 17 +- .../skills/test_skill_deps_aggregator.py | 10 +- .../tests/skills/test_skill_publication.py | 67 +- .../skills/test_skill_publication_docker.py | 2 +- .../skills/test_skill_selector_context_ir.py | 6 +- .../tests/skills/test_skill_zip_export.py | 20 +- .../tests/test_add_agent_permission.py | 4 +- .../tests/test_agent_manager_plugin.py | 13 +- src/backend/tests/test_agent_markdown.py | 7 +- .../tests/test_agent_version_history.py | 15 + .../tests/test_automation_manual_trigger.py | 77 - .../tests/test_automation_oneshot_finalize.py | 109 - src/backend/tests/test_browser_prewarm.py | 22 +- src/backend/tests/test_browser_remediation.py | 4 +- .../tests/test_browser_resource_assets.py | 4 +- .../tests/test_browser_resource_security.py | 20 +- .../tests/test_browser_resource_transport.py | 6 +- src/backend/tests/test_ce_agent_visibility.py | 6 +- .../tests/test_ce_runtime_contracts.py | 24 +- src/backend/tests/test_channel_bots.py | 2109 -------- src/backend/tests/test_citation_anchor.py | 16 +- src/backend/tests/test_cli_cold_bootstrap.py | 48 + .../tests/test_content_block_id_migration.py | 2 +- src/backend/tests/test_credential_registry.py | 44 + src/backend/tests/test_db_import_order.py | 103 + .../tests/test_default_plugin_bootstrap.py | 10 +- .../tests/test_desktop_observability.py | 633 --- .../test_explicit_capability_invocation.py | 21 +- src/backend/tests/test_extensions_layout.py | 28 +- src/backend/tests/test_hook_contributions.py | 47 + .../tests/test_iter_budget_middleware.py | 69 +- src/backend/tests/test_leader_election.py | 48 + .../tests/test_local_memory_defaults.py | 3 +- src/backend/tests/test_local_onboarding.py | 95 + .../tests/test_local_plugin_sidecars.py | 75 +- src/backend/tests/test_local_profile.py | 158 +- src/backend/tests/test_local_subprocess.py | 95 +- src/backend/tests/test_management_failures.py | 8 +- src/backend/tests/test_mcp_canonical_url.py | 30 + src/backend/tests/test_mcp_marketplace.py | 1758 ------- src/backend/tests/test_package_boundaries.py | 16 +- src/backend/tests/test_package_sources.py | 23 + src/backend/tests/test_personal_agent.py | 8 +- .../tests/test_personal_agent_access.py | 10 +- .../tests/test_personal_agent_capabilities.py | 14 +- .../test_personal_agent_channel_files.py | 8 +- .../test_personal_agent_channel_sessions.py | 7 +- .../tests/test_personal_agent_cold_start.py | 10 +- .../test_personal_agent_computer_wait.py | 6 +- .../tests/test_personal_agent_delivery.py | 11 +- .../tests/test_personal_agent_fixes.py | 35 +- .../tests/test_personal_agent_persona.py | 15 +- .../tests/test_personal_agent_prompt_cache.py | 22 + .../tests/test_personal_agent_routing.py | 27 +- .../tests/test_personal_agent_sessions.py | 2 +- .../tests/test_personal_agent_steer.py | 17 +- .../test_personal_agent_tool_regressions.py | 30 +- .../tests/test_personal_canonical_history.py | 4 +- .../test_personal_task_transcript_order.py | 18 +- src/backend/tests/test_plugin_components.py | 66 +- .../tests/test_plugin_manager_plugin.py | 17 +- .../tests/test_plugin_progressive_loading.py | 89 +- .../tests/test_plugin_resource_contract.py | 2 +- src/backend/tests/test_plugin_system.py | 17 +- src/backend/tests/test_redis_stream_pool.py | 6 +- .../test_retired_batch_chat_admission_http.py | 32 +- src/backend/tests/test_route_import_order.py | 74 + .../tests/test_session_capabilities.py | 4 +- .../tests/test_session_database_isolation.py | 23 + src/backend/tests/test_sidebar_task_status.py | 13 +- .../tests/test_skill_manager_plugin.py | 23 +- src/backend/tests/test_tool_contributions.py | 90 + .../tests/test_user_agent_resources.py | 16 +- src/backend/tests/test_user_cleanup.py | 34 + src/backend/tests/test_visualize_plugin.py | 18 +- src/backend/tests/test_yida_local_login.py | 12 +- src/frontend/Dockerfile | 2 +- src/frontend/Dockerfile.dockerignore | 8 +- src/frontend/scripts/archive-e2e-fixtures.py | 4 +- .../scripts/chat-customization-environment.py | 2 +- .../scripts/component-e2e-environment.py | 2 +- .../scripts/customer-devkit-environment.py | 4 +- .../scripts/grapheme-e2e-environment.py | 2 +- .../scripts/test-desktop-login-browser.mjs | 2 +- .../test-sidebar-task-status-browser.mjs | 1 - .../test-workbench-capabilities-browser.mjs | 2 +- .../test-workbench-diagnostics-browser.mjs | 2 +- .../scripts/test-workbench-embed-browser.mjs | 2 +- .../scripts/test-workbench-plugin-browser.mjs | 4 +- .../scripts/test-workspace-http-browser.mjs | 4 +- .../automation/RunTimelinePanel.tsx | 6 +- .../components/automation/automationUtils.ts | 2 + .../src/hooks/chatStream/lifecycleAndTools.ts | 14 + src/frontend/src/plugin-ui/README.md | 4 +- src/frontend/src/types/automations.ts | 2 +- 2513 files changed, 85988 insertions(+), 79469 deletions(-) create mode 100644 docker/opensandbox_capacity.py create mode 100644 scripts/refactor/__init__.py create mode 100644 scripts/refactor/check_ce_paths.py create mode 100644 scripts/refactor/check_line_budget.py create mode 100644 scripts/refactor/check_test_baseline.py create mode 100644 scripts/refactor/import_contract.py create mode 100644 scripts/refactor/rewrite_imports.py create mode 100644 scripts/refactor/snapshot_routes.py create mode 100644 scripts/refactor/snapshot_schema.py create mode 100644 scripts/refactor/tests/test_edition_seams.py create mode 100644 scripts/refactor/tests/test_guards.py create mode 100644 scripts/refactor/tests/test_package_markers.py create mode 100644 scripts/refactor/tests/test_private_imports.py create mode 100644 scripts/refactor/tests/test_rewrite_imports.py create mode 100644 scripts/refactor/tests/test_snapshot_schema.py create mode 100644 scripts/refactor/tests/test_supplemental_moves.py create mode 100644 scripts/refactor/tests/test_test_baseline.py create mode 100644 scripts/refactor/tests/test_test_moves.py create mode 100644 src/backend/alembic/versions/ce_0031_automation_lease_token.py create mode 100644 src/backend/alembic/versions/ce_0032_automation_queue.py create mode 100644 src/backend/alembic/versions/ce_0033_automation_targets.py create mode 100644 src/backend/api/edition.py rename src/backend/api/{routes/v1/agents.py => personal_agents.py} (96%) delete mode 100644 src/backend/api/routes/v1/desktop_capability.py delete mode 100644 src/backend/api/routes/v1/kb.py rename src/backend/{core/content => api/transports}/__init__.py (100%) create mode 100644 src/backend/api/transports/run_sse.py create mode 100644 src/backend/core/agent_runtime/README.md create mode 100644 src/backend/core/agent_runtime/README.zh-CN.md rename src/backend/core/{plugins/resources => agent_runtime}/__init__.py (100%) create mode 100644 src/backend/core/agent_runtime/budget.py create mode 100644 src/backend/core/agent_runtime/direct.py create mode 100644 src/backend/core/agent_runtime/direct_state.py create mode 100644 src/backend/core/agent_runtime/events.py create mode 100644 src/backend/core/agent_runtime/host.py rename src/backend/{tests/capabilities => core/agent_runtime/integrations}/__init__.py (100%) rename src/backend/{orchestration => core/agent_runtime/integrations}/compaction_stream.py (95%) create mode 100644 src/backend/core/agent_runtime/integrations/evolution.py create mode 100644 src/backend/core/agent_runtime/integrations/memory.py create mode 100644 src/backend/core/agent_runtime/integrations/memory_budget.py create mode 100644 src/backend/core/agent_runtime/integrations/memory_identity.py create mode 100644 src/backend/core/agent_runtime/integrations/memory_session.py create mode 100644 src/backend/core/agent_runtime/integrations/memory_state.py create mode 100644 src/backend/core/agent_runtime/integrations/ontology_evidence.py create mode 100644 src/backend/core/agent_runtime/integrations/ontology_repair.py create mode 100644 src/backend/core/agent_runtime/main_state.py rename src/backend/{orchestration/subagents => core/agent_runtime}/plugin_visibility.py (91%) create mode 100644 src/backend/core/agent_runtime/prepare_direct.py create mode 100644 src/backend/core/agent_runtime/prepare_main.py create mode 100644 src/backend/core/agent_runtime/runner.py create mode 100644 src/backend/core/agent_runtime/skills.py rename src/backend/{orchestration => core/agent_runtime}/strategy.py (100%) create mode 100644 src/backend/core/agent_runtime/stream_direct.py create mode 100644 src/backend/core/agent_runtime/stream_main.py create mode 100644 src/backend/core/agent_runtime/targets.py delete mode 100644 src/backend/core/agent_skills/backends/__init__.py delete mode 100644 src/backend/core/agent_skills/cache_refresh.py rename src/backend/core/{services/agent_api_service.py => auth/agent_api.py} (99%) rename src/backend/core/{services => auth}/api_key_runs.py (91%) rename src/backend/core/{services/api_key_service.py => auth/api_keys.py} (95%) create mode 100644 src/backend/core/auth/edition.py create mode 100644 src/backend/core/auth/local_user_bootstrap.py create mode 100644 src/backend/core/auth/local_user_policy.py rename src/backend/core/{services/local_user_service.py => auth/local_users.py} (79%) create mode 100644 src/backend/core/auth/local_users_constants.py create mode 100644 src/backend/core/auth/model_selection.py rename src/backend/core/auth/{account_policy.py => personal_account_policy.py} (100%) rename src/backend/core/auth/{account_view.py => personal_account_view.py} (100%) rename src/backend/core/auth/{edition_capabilities.py => personal_edition_capabilities.py} (100%) rename src/backend/core/auth/{permissions_iface.py => personal_permissions_iface.py} (100%) create mode 100644 src/backend/core/auth/public_identity.py rename src/backend/core/{services/user_service.py => auth/users.py} (92%) create mode 100644 src/backend/core/capabilities/agents/__init__.py create mode 100644 src/backend/core/capabilities/agents/available_resources.py create mode 100644 src/backend/core/capabilities/agents/base.py create mode 100644 src/backend/core/capabilities/agents/base_constants.py rename src/backend/core/{llm/builtin_subagents.py => capabilities/agents/builtin.py} (98%) rename src/backend/core/{services/user_agent_service.py => capabilities/agents/edition.py} (64%) rename src/backend/core/{services/agent_markdown.py => capabilities/agents/markdown.py} (100%) create mode 100644 src/backend/core/capabilities/agents/version_history.py create mode 100644 src/backend/core/capabilities/catalog/__init__.py rename src/backend/core/{config/catalog.py => capabilities/catalog/api.py} (92%) rename src/backend/core/{config/catalog_common.py => capabilities/catalog/common.py} (90%) rename src/backend/core/{config/catalog_details.py => capabilities/catalog/details.py} (96%) rename src/backend/core/{config/catalog_loader.py => capabilities/catalog/loader.py} (85%) rename src/backend/core/{config/catalog_migration.py => capabilities/catalog/migration.py} (97%) rename src/backend/core/{services/catalog_service.py => capabilities/catalog/overrides.py} (100%) rename src/backend/core/{config/catalog_resolver.py => capabilities/catalog/resolver.py} (87%) rename src/backend/core/{config/catalog_runtime.py => capabilities/catalog/runtime.py} (87%) create mode 100644 src/backend/core/capabilities/identity_rules.py rename src/backend/core/{services/desktop_capability_protocol.py => capabilities/manifest_protocol.py} (99%) create mode 100644 src/backend/core/capabilities/mcp/__init__.py rename src/backend/core/{services/mcp_builtin_specs.py => capabilities/mcp/builtin_specs.py} (94%) rename src/backend/core/{services/edition_mcp.py => capabilities/mcp/edition.py} (100%) rename src/backend/core/{services/mcp_management_service.py => capabilities/mcp/management.py} (90%) rename src/backend/core/{services/mcp_oauth_service.py => capabilities/mcp/oauth.py} (54%) create mode 100644 src/backend/core/capabilities/mcp/oauth_storage.py rename src/backend/core/{services/mcp_service.py => capabilities/mcp/service.py} (91%) create mode 100644 src/backend/core/capabilities/mcp/tool_metadata.py create mode 100644 src/backend/core/capabilities/mcp/tool_schema.py rename src/backend/core/{services/mcp_tool_schema_backfill.py => capabilities/mcp/tool_schema_backfill.py} (87%) create mode 100644 src/backend/core/capabilities/plugins/README.md rename src/backend/core/{ => capabilities}/plugins/__init__.py (100%) rename src/backend/core/{ => capabilities}/plugins/connections.py (83%) create mode 100644 src/backend/core/capabilities/plugins/management/__init__.py rename src/backend/core/{ => capabilities}/plugins/management/admin_config.py (88%) rename src/backend/core/{services/manager_bundle_upgrade.py => capabilities/plugins/management/bundle_upgrade.py} (92%) rename src/backend/core/{services/cloud_management.py => capabilities/plugins/management/cloud.py} (78%) rename src/backend/core/{ => capabilities}/plugins/management/components.py (88%) rename src/backend/core/{services/management_contract.py => capabilities/plugins/management/contract.py} (100%) rename src/backend/core/{ => capabilities}/plugins/management/defaults.py (93%) rename src/backend/core/{ => capabilities}/plugins/management/details.py (58%) rename src/backend/core/{services/management_errors.py => capabilities/plugins/management/errors.py} (100%) rename src/backend/core/{ => capabilities}/plugins/management/installation.py (81%) rename src/backend/core/{ => capabilities}/plugins/management/lifecycle.py (87%) rename src/backend/core/{services/management_package.py => capabilities/plugins/management/package.py} (91%) rename src/backend/core/{ => capabilities}/plugins/management/packages.py (70%) rename src/backend/core/{ => capabilities}/plugins/management/projection.py (63%) rename src/backend/core/{ => capabilities}/plugins/management/queries.py (86%) rename src/backend/core/{ => capabilities}/plugins/packaging/__init__.py (100%) rename src/backend/core/{ => capabilities}/plugins/packaging/archive.py (96%) rename src/backend/core/{ => capabilities}/plugins/packaging/definitions.py (94%) rename src/backend/core/{ => capabilities}/plugins/packaging/importer.py (96%) rename src/backend/core/{ => capabilities}/plugins/packaging/models.py (100%) create mode 100644 src/backend/core/capabilities/plugins/packaging/presentation.py rename src/backend/core/{ => capabilities}/plugins/packaging/runtime_assets.py (100%) rename src/backend/core/{ => capabilities}/plugins/packaging/skill_transform.py (96%) rename src/backend/core/{ => capabilities}/plugins/packaging/sources.py (83%) create mode 100644 src/backend/core/capabilities/plugins/resources/__init__.py rename src/backend/core/{ => capabilities}/plugins/resources/artifacts.py (93%) rename src/backend/core/{ => capabilities}/plugins/resources/authentication.py (100%) rename src/backend/core/{ => capabilities}/plugins/resources/configuration.py (100%) rename src/backend/core/{ => capabilities}/plugins/resources/confinement.py (83%) rename src/backend/core/{ => capabilities}/plugins/resources/crypto.py (100%) create mode 100644 src/backend/core/capabilities/plugins/resources/deadlines.py rename src/backend/core/{ => capabilities}/plugins/resources/installation.py (56%) rename src/backend/core/{ => capabilities}/plugins/resources/lifecycle.py (84%) create mode 100644 src/backend/core/capabilities/plugins/resources/owners.py rename src/backend/core/{ => capabilities}/plugins/resources/prewarm.py (72%) rename src/backend/core/{ => capabilities}/plugins/resources/reservation.py (96%) rename src/backend/core/{ => capabilities}/plugins/resources/service.py (84%) rename src/backend/core/{ => capabilities}/plugins/resources/store.py (97%) rename src/backend/core/{ => capabilities}/plugins/resources/stream.py (95%) rename src/backend/core/{ => capabilities}/plugins/resources/transport.py (100%) rename src/backend/core/{ => capabilities}/plugins/runtime/__init__.py (76%) rename src/backend/core/{ => capabilities}/plugins/runtime/activation_history.py (97%) rename src/backend/core/{ => capabilities}/plugins/runtime/models.py (100%) rename src/backend/core/{ => capabilities}/plugins/runtime/resolution.py (91%) rename src/backend/core/{ => capabilities}/plugins/runtime/sticky_selection.py (85%) rename src/backend/core/capabilities/{ref.py => reference.py} (98%) delete mode 100644 src/backend/core/capabilities/runtime/__init__.py delete mode 100644 src/backend/core/capabilities/skills.py rename src/backend/core/{agent_skills => capabilities/skills}/__init__.py (97%) create mode 100644 src/backend/core/capabilities/skills/backends/__init__.py rename src/backend/core/{agent_skills => capabilities/skills}/backends/composite.py (97%) rename src/backend/core/{agent_skills => capabilities/skills}/backends/database.py (99%) rename src/backend/core/{agent_skills => capabilities/skills}/backends/filesystem.py (98%) rename src/backend/core/{agent_skills => capabilities/skills}/backends/protocol.py (100%) rename src/backend/core/{agent_skills => capabilities/skills}/binary_files.py (100%) create mode 100644 src/backend/core/capabilities/skills/cache_refresh.py rename src/backend/core/{agent_skills => capabilities/skills}/config.py (89%) create mode 100644 src/backend/core/capabilities/skills/deps/__init__.py rename src/backend/core/{services/skill_deps_aggregator.py => capabilities/skills/deps/aggregator.py} (98%) rename src/backend/core/{agent_skills/deps_detector.py => capabilities/skills/deps/detector.py} (100%) rename src/backend/core/{services/skill_deps_prober.py => capabilities/skills/deps/prober.py} (100%) rename src/backend/core/{services/skill_deps_request_service.py => capabilities/skills/deps/requests.py} (99%) rename src/backend/core/{services/skill_icon_service.py => capabilities/skills/icons.py} (100%) create mode 100644 src/backend/core/capabilities/skills/installation.py rename src/backend/core/{agent_skills => capabilities/skills}/loader.py (77%) create mode 100644 src/backend/core/capabilities/skills/loader_factory.py rename src/backend/core/{services/skill_management_service.py => capabilities/skills/management.py} (93%) rename src/backend/core/{agent_skills => capabilities/skills}/publication.py (89%) rename src/backend/core/{agent_skills => capabilities/skills}/registry.py (97%) create mode 100644 src/backend/core/capabilities/skills/runtime_loader.py create mode 100644 src/backend/core/capabilities/skills/script_discovery.py rename src/backend/core/{agent_skills => capabilities/skills}/selection_record.py (98%) rename src/backend/core/{agent_skills => capabilities/skills}/skill_archive.py (88%) create mode 100644 src/backend/core/capabilities/skills/zip_reader.py delete mode 100644 src/backend/core/channels/adapters/dingtalk.py delete mode 100644 src/backend/core/channels/adapters/lark.py delete mode 100644 src/backend/core/channels/inbound.py rename src/backend/core/{services => chat}/audio_transcription.py (98%) create mode 100644 src/backend/core/chat/compaction/__init__.py rename src/backend/core/{services/context_budget_checkpoint.py => chat/compaction/budget_checkpoint.py} (97%) rename src/backend/core/{services/compaction_coordinator.py => chat/compaction/coordinator.py} (94%) rename src/backend/core/{services/compaction_history.py => chat/compaction/history.py} (93%) rename src/backend/core/{services/compaction_progress.py => chat/compaction/progress.py} (100%) rename src/backend/core/{services/compaction_service.py => chat/compaction/service.py} (96%) rename src/backend/core/{services/compaction_summary.py => chat/compaction/summary.py} (93%) rename src/backend/core/{services/context_usage_projection.py => chat/compaction/usage_projection.py} (97%) create mode 100644 src/backend/core/chat/compaction_checkpoints.py create mode 100644 src/backend/core/chat/compaction_snapshots.py create mode 100644 src/backend/core/chat/compaction_types.py rename src/backend/core/{services/chat_edition.py => chat/edition.py} (100%) rename src/backend/{orchestration => core/chat}/followups.py (83%) rename src/backend/core/{services/chat_fork_service.py => chat/fork.py} (99%) create mode 100644 src/backend/core/chat/history_operations.py create mode 100644 src/backend/core/chat/limits.py rename src/backend/{orchestration => core/chat}/message_parser.py (97%) create mode 100644 src/backend/core/chat/message_queries.py create mode 100644 src/backend/core/chat/message_writes.py create mode 100644 src/backend/core/chat/mode_seed.py create mode 100644 src/backend/core/chat/mode_types.py create mode 100644 src/backend/core/chat/mode_validation.py rename src/backend/core/{services/chat_mode_service.py => chat/modes.py} (62%) rename src/backend/core/{services/session_operations => chat/operations}/__init__.py (100%) rename src/backend/core/{services/session_operations => chat/operations}/admission.py (81%) rename src/backend/core/{services/session_operations => chat/operations}/admission_errors.py (92%) rename src/backend/core/{services/session_operations => chat/operations}/agent_targets.py (94%) rename src/backend/core/{services/session_operations => chat/operations}/commands.py (82%) rename src/backend/core/{services/session_operations => chat/operations}/history.py (92%) rename src/backend/core/{services/session_operations => chat/operations}/invocation.py (85%) rename src/backend/core/{services/session_operations => chat/operations}/projects.py (90%) rename src/backend/core/{services/session_operations => chat/operations}/request_context.py (91%) rename src/backend/core/{services/session_operations => chat/operations}/runs.py (90%) rename src/backend/core/{services/session_operations => chat/operations}/schemas.py (100%) rename src/backend/core/{services/session_operations => chat/operations}/session_context.py (86%) rename src/backend/core/{services/session_operations => chat/operations}/sessions.py (86%) rename src/backend/core/{services/plan_service.py => chat/plans.py} (100%) rename src/backend/core/{services => chat}/project_init.py (91%) create mode 100644 src/backend/core/chat/projection/__init__.py create mode 100644 src/backend/core/chat/projection/codec.py rename src/backend/{orchestration/run_projection.py => core/chat/projection/reducer.py} (62%) rename src/backend/{orchestration/run_projection_storage.py => core/chat/projection/storage.py} (89%) rename src/backend/{orchestration/run_subscription.py => core/chat/projection/subscription.py} (80%) rename src/backend/core/{services/public_chat_service.py => chat/public.py} (78%) rename src/backend/core/{services/chat_reference_service.py => chat/references.py} (98%) create mode 100644 src/backend/core/chat/runtime/__init__.py create mode 100644 src/backend/core/chat/runtime/admission.py create mode 100644 src/backend/core/chat/runtime/chunks.py rename src/backend/{orchestration/post_turn_compaction.py => core/chat/runtime/compaction.py} (84%) create mode 100644 src/backend/core/chat/runtime/completion.py create mode 100644 src/backend/core/chat/runtime/followups.py create mode 100644 src/backend/core/chat/runtime/handoff.py rename src/backend/{orchestration => core/chat/runtime}/job_wakeup.py (94%) create mode 100644 src/backend/core/chat/runtime/pauses.py create mode 100644 src/backend/core/chat/runtime/recovery.py create mode 100644 src/backend/core/chat/runtime/recovery_snapshot.py create mode 100644 src/backend/core/chat/runtime/snapshots.py create mode 100644 src/backend/core/chat/runtime/steering.py create mode 100644 src/backend/core/chat/runtime/turn_state.py create mode 100644 src/backend/core/chat/runtime/worker.py rename src/backend/core/{services/chat_sequencer.py => chat/sequencer.py} (97%) create mode 100644 src/backend/core/chat/service.py create mode 100644 src/backend/core/chat/session_operations.py rename src/backend/core/{services => chat}/sidebar_status.py (100%) rename src/backend/core/{services/chat_steer_service.py => chat/steer.py} (95%) create mode 100644 src/backend/core/chat/steer_hook.py rename src/backend/core/{services => chat}/steer_queue.py (100%) rename src/backend/core/{services/subagent_routing_service.py => chat/subagent_routing.py} (100%) rename src/backend/core/{services/chat_titles.py => chat/titles.py} (92%) rename src/backend/{orchestration => core/chat}/tool_payloads.py (92%) rename src/backend/core/config/{edition_display_names.py => edition.py} (78%) create mode 100644 src/backend/core/config/store/__init__.py create mode 100644 src/backend/core/config/store/config_keys.py create mode 100644 src/backend/core/config/store/execution_options.py create mode 100644 src/backend/core/config/store/runtime_env.py create mode 100644 src/backend/core/config/store/system_config.py create mode 100644 src/backend/core/config/store/system_config_constants.py delete mode 100644 src/backend/core/content/content_blocks.py delete mode 100644 src/backend/core/content/file_parser.py create mode 100644 src/backend/core/db/artifact_scope.py create mode 100644 src/backend/core/db/edition.py delete mode 100644 src/backend/core/db/models/channel_relay.py create mode 100644 src/backend/core/db/models/chat_resources.py create mode 100644 src/backend/core/db/models/chat_runs.py create mode 100644 src/backend/core/db/models/column_types.py delete mode 100644 src/backend/core/db/models/kb_wiki.py delete mode 100644 src/backend/core/db/models/observability.py rename src/backend/core/db/{model_extensions.py => personal_model_fields.py} (100%) rename src/backend/core/db/{edition_tables.py => personal_schema.py} (97%) create mode 100644 src/backend/core/db/repository/folder.py rename src/backend/core/db/repository/{agent.py => personal_agent.py} (100%) rename src/backend/core/db/repository/{audit.py => personal_audit.py} (96%) create mode 100644 src/backend/core/db/system_configuration.py create mode 100644 src/backend/core/db/user_cleanup.py delete mode 100644 src/backend/core/evolution/activation.py delete mode 100644 src/backend/core/evolution/agent_profile.py create mode 100644 src/backend/core/evolution/assessment/__init__.py rename src/backend/core/evolution/{ => assessment}/applicability.py (98%) rename src/backend/core/evolution/{ => assessment}/benchmark.py (98%) rename src/backend/core/evolution/{ => assessment}/evaluator.py (100%) rename src/backend/core/evolution/{ => assessment}/experiments.py (99%) rename src/backend/core/evolution/{ => assessment}/fault_injection.py (98%) rename src/backend/core/evolution/{ => assessment}/metrics.py (100%) rename src/backend/core/evolution/{ => assessment}/similarity.py (98%) rename src/backend/core/evolution/{ => assessment}/sop_judge.py (98%) create mode 100644 src/backend/core/evolution/cycle/__init__.py create mode 100644 src/backend/core/evolution/cycle/activation.py create mode 100644 src/backend/core/evolution/cycle/activation_constants.py create mode 100644 src/backend/core/evolution/cycle/activation_policy.py create mode 100644 src/backend/core/evolution/cycle/activation_types.py create mode 100644 src/backend/core/evolution/cycle/candidate_records.py create mode 100644 src/backend/core/evolution/cycle/composition_candidates.py create mode 100644 src/backend/core/evolution/cycle/cycle_extras.py create mode 100644 src/backend/core/evolution/cycle/cycle_extras_constants.py create mode 100644 src/backend/core/evolution/cycle/episode_views.py create mode 100644 src/backend/core/evolution/cycle/evaluation.py create mode 100644 src/backend/core/evolution/cycle/extra_findings.py create mode 100644 src/backend/core/evolution/cycle/graph_candidates.py create mode 100644 src/backend/core/evolution/cycle/loop.py create mode 100644 src/backend/core/evolution/cycle/memory_candidates.py create mode 100644 src/backend/core/evolution/cycle/memory_findings.py create mode 100644 src/backend/core/evolution/cycle/ordering.py create mode 100644 src/backend/core/evolution/cycle/patterns.py create mode 100644 src/backend/core/evolution/cycle/profile_release.py create mode 100644 src/backend/core/evolution/cycle/promotion.py create mode 100644 src/backend/core/evolution/cycle/promotion_constants.py create mode 100644 src/backend/core/evolution/cycle/promotion_types.py create mode 100644 src/backend/core/evolution/cycle/prompt_candidates.py rename src/backend/core/evolution/{ => cycle}/readiness.py (93%) rename src/backend/core/evolution/{ => cycle}/release.py (100%) create mode 100644 src/backend/core/evolution/cycle/retirement.py rename src/backend/core/evolution/{ => cycle}/runtime_binding.py (87%) create mode 100644 src/backend/core/evolution/cycle/skill_document.py create mode 100644 src/backend/core/evolution/cycle/skill_release.py rename src/backend/core/evolution/{ => cycle}/timescales.py (99%) rename src/backend/core/evolution/{ => cycle}/trigger.py (81%) create mode 100644 src/backend/core/evolution/cycle/usage_evidence.py delete mode 100644 src/backend/core/evolution/cycle_extras.py create mode 100644 src/backend/core/evolution/evidence/__init__.py rename src/backend/core/evolution/{ => evidence}/cassette.py (94%) rename src/backend/core/evolution/{ => evidence}/counterfactual.py (99%) rename src/backend/core/evolution/{ => evidence}/credit.py (99%) rename src/backend/core/evolution/{ => evidence}/events.py (99%) rename src/backend/core/evolution/{ => evidence}/evidence_contract.py (100%) rename src/backend/core/evolution/{ => evidence}/evidence_resolver.py (98%) rename src/backend/core/evolution/{ => evidence}/exposure.py (95%) rename src/backend/core/evolution/{ => evidence}/lineage.py (99%) rename src/backend/core/evolution/{ => evidence}/outcomes.py (100%) rename src/backend/core/evolution/{ => evidence}/replay.py (100%) rename src/backend/core/evolution/{ => evidence}/trace_assembler.py (98%) rename src/backend/core/evolution/{ => evidence}/trace_store.py (97%) create mode 100644 src/backend/core/evolution/generators/__init__.py rename src/backend/core/evolution/{ => generators}/graph_scan.py (100%) rename src/backend/core/evolution/{ => generators}/memory_apply.py (98%) rename src/backend/core/evolution/{ => generators}/memory_ops.py (99%) rename src/backend/core/evolution/{ => generators}/memory_scan.py (99%) rename src/backend/core/evolution/{ => generators}/ontology_bridge.py (100%) rename src/backend/core/evolution/{ => generators}/orchestration_gen.py (99%) rename src/backend/core/evolution/{ => generators}/prompt_gen.py (100%) rename src/backend/core/evolution/{ => generators}/skill_candidate_engine.py (96%) create mode 100644 src/backend/core/evolution/generators/skill_drafting.py create mode 100644 src/backend/core/evolution/generators/skill_evidence.py create mode 100644 src/backend/core/evolution/generators/skill_gen.py create mode 100644 src/backend/core/evolution/generators/skill_gen_constants.py delete mode 100644 src/backend/core/evolution/loop.py create mode 100644 src/backend/core/evolution/profiles/__init__.py create mode 100644 src/backend/core/evolution/profiles/agent_profile.py create mode 100644 src/backend/core/evolution/profiles/agent_profile_constants.py create mode 100644 src/backend/core/evolution/profiles/change_preview.py rename src/backend/core/evolution/{ => profiles}/console_view.py (99%) rename src/backend/core/evolution/{ => profiles}/federation.py (99%) rename src/backend/core/evolution/{ => profiles}/personal.py (88%) rename src/backend/core/evolution/{user_settings.py => profiles/personal_review.py} (60%) rename src/backend/core/evolution/{ => profiles}/policies.py (88%) rename src/backend/core/evolution/{ => profiles}/prefs.py (100%) create mode 100644 src/backend/core/evolution/profiles/profile_selection.py create mode 100644 src/backend/core/evolution/profiles/stall_hook.py create mode 100644 src/backend/core/evolution/profiles/user_settings.py create mode 100644 src/backend/core/evolution/profiles/user_settings_constants.py delete mode 100644 src/backend/core/evolution/promotion.py create mode 100644 src/backend/core/evolution/settlement/__init__.py create mode 100644 src/backend/core/evolution/settlement/memory_service.py rename src/backend/core/evolution/{ => settlement}/settlement.py (100%) rename src/backend/core/evolution/{ => settlement}/settlement_runner.py (88%) rename src/backend/core/evolution/{ => settlement}/settlement_store.py (98%) delete mode 100644 src/backend/core/evolution/skill_gen.py delete mode 100644 src/backend/core/extensions/__init__.py delete mode 100644 src/backend/core/extensions/registry.py create mode 100644 src/backend/core/files/__init__.py create mode 100644 src/backend/core/files/artifact_bytes.py rename src/backend/core/{ => files}/artifacts/__init__.py (96%) rename src/backend/core/{services/artifact_backfill.py => files/artifacts/backfill.py} (95%) create mode 100644 src/backend/core/files/artifacts/generated.py rename src/backend/core/{content/artifact_key_repair.py => files/artifacts/key_repair.py} (100%) rename src/backend/core/{ => files}/artifacts/local_reference.py (89%) rename src/backend/core/{services/artifact_location.py => files/artifacts/location.py} (52%) rename src/backend/core/{content/artifact_reader.py => files/artifacts/reader.py} (91%) rename src/backend/core/{content/artifact_refs.py => files/artifacts/refs.py} (93%) rename src/backend/core/{services/artifact_service.py => files/artifacts/service.py} (93%) rename src/backend/core/{ => files}/artifacts/store.py (96%) rename src/backend/core/{content/artifact_summary.py => files/artifacts/summary.py} (97%) rename src/backend/core/{services/artifact_text_service.py => files/artifacts/text.py} (92%) create mode 100644 src/backend/core/files/content/__init__.py rename src/backend/core/{ => files}/content/archive_browser.py (99%) rename src/backend/core/{ => files}/content/archive_limits.py (100%) create mode 100644 src/backend/core/files/content/branding.py create mode 100644 src/backend/core/files/content/content_blocks.py create mode 100644 src/backend/core/files/content/content_blocks_constants.py create mode 100644 src/backend/core/files/content/file_formats.py create mode 100644 src/backend/core/files/content/file_parser.py rename src/backend/core/{ => files}/content/file_validation.py (100%) rename src/backend/core/{ => files}/content/office.py (100%) rename src/backend/core/{ => files}/content/office_preview_cache.py (98%) create mode 100644 src/backend/core/files/content/parser_config.py create mode 100644 src/backend/core/files/content/pdf_parser.py create mode 100644 src/backend/core/files/content/presentation_parser.py create mode 100644 src/backend/core/files/content/spreadsheet_parser.py rename src/backend/core/{ => files}/content/svg_fit.py (97%) create mode 100644 src/backend/core/files/content/text_parser.py create mode 100644 src/backend/core/files/content/word_parser.py create mode 100644 src/backend/core/files/edition.py rename src/backend/core/{services => files}/folder_batch.py (69%) rename src/backend/core/{services/personal_file_operations.py => files/personal_operations.py} (86%) rename src/backend/core/{ => files}/spaces/__init__.py (100%) rename src/backend/core/{ => files}/spaces/access.py (95%) rename src/backend/core/{ => files}/spaces/files.py (96%) rename src/backend/core/{ => files}/spaces/leases.py (86%) rename src/backend/core/{ => files}/spaces/mounts.py (78%) rename src/backend/core/{ => files}/spaces/projects.py (94%) rename src/backend/core/{ => files}/spaces/recovery.py (94%) rename src/backend/core/{ => files}/spaces/sandbox.py (73%) rename src/backend/core/{ => files}/spaces/search.py (97%) rename src/backend/core/{ => files}/spaces/text.py (92%) rename src/backend/core/{ => files}/spaces/tree.py (100%) rename src/backend/core/{ => files}/storage/__init__.py (56%) rename src/backend/core/{ => files}/storage/entries.py (100%) rename src/backend/core/{ => files}/storage/factory.py (81%) rename src/backend/core/{ => files}/storage/local.py (96%) rename src/backend/core/{ => files}/storage/local_directory.py (94%) rename src/backend/core/{ => files}/storage/local_paths.py (98%) rename src/backend/core/{ => files}/storage/protocol.py (100%) rename src/backend/core/{ => files}/storage/s3.py (98%) rename src/backend/core/{ => files}/storage/s3_directory.py (96%) rename src/backend/core/{services => files}/upload_retry.py (92%) rename src/backend/core/{capabilities/errors.py => harness/capability_errors.py} (100%) create mode 100644 src/backend/core/harness/context_items.py rename src/backend/core/{llm => harness}/execution_manifest.py (99%) rename src/backend/core/{services/harness_ledger.py => harness/ledger.py} (53%) create mode 100644 src/backend/core/harness/log_audit.py create mode 100644 src/backend/core/harness/log_context.py create mode 100644 src/backend/core/harness/log_records.py create mode 100644 src/backend/core/harness/log_writer.py create mode 100644 src/backend/core/harness/log_writer_constants.py create mode 100644 src/backend/core/harness/observation_writer.py create mode 100644 src/backend/core/harness/registry/__init__.py create mode 100644 src/backend/core/harness/registry/authentication.py create mode 100644 src/backend/core/harness/registry/build_governance.py create mode 100644 src/backend/core/harness/registry/cache_invalidation.py create mode 100644 src/backend/core/harness/registry/capability_management.py create mode 100644 src/backend/core/harness/registry/capability_revisions.py create mode 100644 src/backend/core/harness/registry/capability_runtime.py create mode 100644 src/backend/core/harness/registry/catalog_options.py create mode 100644 src/backend/core/harness/registry/chat_execution.py create mode 100644 src/backend/core/harness/registry/chat_history.py create mode 100644 src/backend/core/harness/registry/compaction.py create mode 100644 src/backend/core/harness/registry/device_catalog.py rename src/backend/core/{extensions => harness/registry}/edition.py (100%) create mode 100644 src/backend/core/harness/registry/evaluation.py create mode 100644 src/backend/core/harness/registry/execution_account.py create mode 100644 src/backend/core/harness/registry/execution_catalog.py create mode 100644 src/backend/core/harness/registry/execution_identity.py create mode 100644 src/backend/core/harness/registry/hook_factories.py create mode 100644 src/backend/core/harness/registry/host_runtime.py create mode 100644 src/backend/core/harness/registry/job_operations.py create mode 100644 src/backend/core/harness/registry/local_files.py create mode 100644 src/backend/core/harness/registry/local_publication.py create mode 100644 src/backend/core/harness/registry/marketplace_catalog.py create mode 100644 src/backend/core/harness/registry/mcp_clients.py create mode 100644 src/backend/core/harness/registry/model_credentials.py create mode 100644 src/backend/core/harness/registry/model_services.py create mode 100644 src/backend/core/harness/registry/package_sources.py create mode 100644 src/backend/core/harness/registry/package_views.py create mode 100644 src/backend/core/harness/registry/registry.py create mode 100644 src/backend/core/harness/registry/resource_owners.py create mode 100644 src/backend/core/harness/registry/retrieval_context.py create mode 100644 src/backend/core/harness/registry/run_cancellation.py create mode 100644 src/backend/core/harness/registry/run_capabilities.py create mode 100644 src/backend/core/harness/registry/schema_lifecycle.py create mode 100644 src/backend/core/harness/registry/skill_content.py create mode 100644 src/backend/core/harness/registry/system_configuration.py create mode 100644 src/backend/core/harness/registry/tool_operations.py rename src/backend/core/{services/desktop_gateway_uploads.py => harness/registry/transfer_channels.py} (53%) create mode 100644 src/backend/core/harness/registry/user_cleanup.py create mode 100644 src/backend/core/harness/tool_call_context.py create mode 100644 src/backend/core/harness/tool_effect_context.py rename src/backend/core/{capabilities/junction.py => infra/directory_links.py} (92%) create mode 100644 src/backend/core/infra/failures.py rename src/backend/core/{capabilities/lockfile.py => infra/file_lock.py} (100%) rename src/backend/core/{ => infra}/immutable.py (100%) create mode 100644 src/backend/core/infra/json_output.py create mode 100644 src/backend/core/infra/local_runtime/__init__.py create mode 100644 src/backend/core/infra/local_runtime/dynamic_servers.py rename src/backend/{orchestration/local_sidecar_lifecycle.py => core/infra/local_runtime/lifecycle.py} (89%) rename src/backend/core/{sandbox/providers/script_runner/auth.py => infra/local_runtime/runner_auth.py} (100%) rename src/backend/{orchestration/local_subprocess.py => core/infra/local_runtime/startup.py} (93%) create mode 100644 src/backend/core/infra/model_output.py create mode 100644 src/backend/core/infra/probes/__init__.py rename src/backend/core/{services/edition_service_probe.py => infra/probes/edition.py} (100%) rename src/backend/core/{services => infra/probes}/service_probes.py (89%) delete mode 100644 src/backend/core/infra/rate_limit.py rename src/backend/core/{chat/display_bounds.py => infra/text_bounds.py} (100%) rename src/backend/{orchestration => core}/jobs/__init__.py (100%) rename src/backend/{orchestration => core}/jobs/agent.py (94%) rename src/backend/{orchestration => core}/jobs/callback.py (100%) rename src/backend/{orchestration => core}/jobs/files.py (95%) rename src/backend/{orchestration => core}/jobs/notifications.py (81%) rename src/backend/{orchestration => core}/jobs/owner.py (62%) rename src/backend/{orchestration => core}/jobs/process.py (95%) create mode 100644 src/backend/core/jobs/reminder_hook.py rename src/backend/{orchestration => core}/jobs/runner.py (100%) rename src/backend/{orchestration/job_runtime.py => core/jobs/runtime.py} (88%) rename src/backend/{orchestration => core}/jobs/sdk.py (100%) rename src/backend/{orchestration => core}/jobs/state.py (98%) rename src/backend/core/{services/job_service.py => jobs/store.py} (100%) rename src/backend/{orchestration => core}/jobs/supervisor.py (88%) create mode 100644 src/backend/core/jobs/tool_operations.py delete mode 100644 src/backend/core/kb/kb_assets.py delete mode 100644 src/backend/core/kb/kb_parser.py delete mode 100644 src/backend/core/kb/kb_vector.py delete mode 100644 src/backend/core/kb/wiki/local_provider.py delete mode 100644 src/backend/core/llm/_distill_shared.py create mode 100644 src/backend/core/llm/api_runtime/__init__.py rename src/backend/core/llm/{agent_api_runtime.py => api_runtime/agent.py} (97%) rename src/backend/core/llm/{agent_api_skills.py => api_runtime/agent_skills.py} (96%) rename src/backend/core/llm/{agent_api_tools.py => api_runtime/agent_tools.py} (90%) rename src/backend/core/llm/{project_api_runtime.py => api_runtime/project.py} (94%) delete mode 100644 src/backend/core/llm/capability_tools.py create mode 100644 src/backend/core/llm/citations/__init__.py create mode 100644 src/backend/core/llm/citations/allocator.py create mode 100644 src/backend/core/llm/citations/anchor.py create mode 100644 src/backend/core/llm/citations/anchor_state.py create mode 100644 src/backend/core/llm/citations/extraction.py create mode 100644 src/backend/core/llm/citations/hook.py create mode 100644 src/backend/core/llm/citations/types.py delete mode 100644 src/backend/core/llm/command_middleware.py create mode 100644 src/backend/core/llm/context/__init__.py create mode 100644 src/backend/core/llm/context/adapter.py create mode 100644 src/backend/core/llm/context/attachment_hook.py rename src/backend/core/{services/compaction_budget.py => llm/context/budget.py} (98%) rename src/backend/core/llm/{ => context}/compacting_agent.py (96%) rename src/backend/core/llm/{ => context}/compaction.py (98%) create mode 100644 src/backend/core/llm/context/content.py create mode 100644 src/backend/core/llm/context/ir.py create mode 100644 src/backend/core/llm/context/ir_constants.py create mode 100644 src/backend/core/llm/context/items.py create mode 100644 src/backend/core/llm/context/message_defaults.py rename src/backend/core/llm/{ => context}/offloader.py (96%) create mode 100644 src/backend/core/llm/context/provider_conversion.py create mode 100644 src/backend/core/llm/context/reasoning_hooks.py create mode 100644 src/backend/core/llm/context/reference_conversion.py create mode 100644 src/backend/core/llm/context/rendering.py rename src/backend/core/llm/{context_safety.py => context/safety.py} (87%) rename src/backend/core/llm/{ => context}/system_reminder.py (94%) rename src/backend/core/llm/{context_usage.py => context/usage.py} (99%) create mode 100644 src/backend/core/llm/context/workspace_hooks.py delete mode 100644 src/backend/core/llm/context_adapter.py delete mode 100644 src/backend/core/llm/context_ir.py rename src/backend/core/llm/factory/{selection/device_bindings.py => runtime/capability_bindings.py} (93%) rename src/backend/{orchestration/registry.py => core/llm/factory/specs.py} (99%) create mode 100644 src/backend/core/llm/factory/tools/definitions.py delete mode 100644 src/backend/core/llm/finish_guard.py delete mode 100644 src/backend/core/llm/hooks.py create mode 100644 src/backend/core/llm/host_services.py delete mode 100644 src/backend/core/llm/management_tool.py create mode 100644 src/backend/core/llm/mcp/__init__.py create mode 100644 src/backend/core/llm/mcp/callbacks.py create mode 100644 src/backend/core/llm/mcp/chat_sessions.py create mode 100644 src/backend/core/llm/mcp/client.py create mode 100644 src/backend/core/llm/mcp/context_tools.py create mode 100644 src/backend/core/llm/mcp/deadline.py rename src/backend/core/llm/{mcp_invocation.py => mcp/invocation.py} (87%) create mode 100644 src/backend/core/llm/mcp/lifetime.py create mode 100644 src/backend/core/llm/mcp/manager.py create mode 100644 src/backend/core/llm/mcp/results.py create mode 100644 src/backend/core/llm/mcp/session.py create mode 100644 src/backend/core/llm/mcp/tools.py delete mode 100644 src/backend/core/llm/mcp_manager.py delete mode 100644 src/backend/core/llm/mcp_pool.py create mode 100644 src/backend/core/llm/middleware/__init__.py rename src/backend/core/llm/{middleware_migration.py => middleware/migration_map.py} (51%) delete mode 100644 src/backend/core/llm/middlewares.py delete mode 100644 src/backend/core/llm/persona_distiller.py create mode 100644 src/backend/core/llm/ports.py delete mode 100644 src/backend/core/llm/project_environment_middleware.py create mode 100644 src/backend/core/llm/prompt_files.py create mode 100644 src/backend/core/llm/prompt_kinds.py create mode 100644 src/backend/core/llm/prompt_rendering.py create mode 100644 src/backend/core/llm/prompt_storage.py create mode 100644 src/backend/core/llm/prompt_versions.py create mode 100644 src/backend/core/llm/prompt_versions_state.py delete mode 100644 src/backend/core/llm/providers/__init__.py create mode 100644 src/backend/core/llm/runtime/__init__.py rename src/backend/core/llm/{ => runtime}/agentscope_hook_adapter.py (63%) create mode 100644 src/backend/core/llm/runtime/context_hook_adapter.py create mode 100644 src/backend/core/llm/runtime/file_context.py create mode 100644 src/backend/core/llm/runtime/finish_guard.py create mode 100644 src/backend/core/llm/runtime/hooks_constants.py rename src/backend/core/llm/{ => runtime}/human_interaction.py (86%) rename src/backend/core/llm/{ => runtime}/interaction_store.py (100%) rename src/backend/core/llm/{ => runtime}/manifest_agent.py (93%) rename src/backend/core/llm/{ => runtime}/message_compat.py (93%) create mode 100644 src/backend/core/llm/runtime/model_cache.py create mode 100644 src/backend/core/llm/runtime/model_selection_adapter.py rename src/backend/core/llm/{ => runtime}/model_steps.py (98%) rename src/backend/core/llm/{ => runtime}/project_environment.py (88%) create mode 100644 src/backend/core/llm/runtime/project_environment_hook.py create mode 100644 src/backend/core/llm/runtime/review_text_agent.py rename src/backend/core/llm/{ => runtime}/session_capabilities.py (97%) create mode 100644 src/backend/core/llm/runtime/state.py create mode 100644 src/backend/core/llm/runtime/stream_events.py create mode 100644 src/backend/core/llm/runtime/stream_loop.py create mode 100644 src/backend/core/llm/runtime/stream_text.py create mode 100644 src/backend/core/llm/runtime/streaming.py create mode 100644 src/backend/core/llm/runtime/streaming_constants.py rename src/backend/{orchestration => core/llm/runtime}/tool_callbacks.py (70%) create mode 100644 src/backend/core/llm/runtime/tool_execution_adapter.py create mode 100644 src/backend/core/llm/runtime/tool_hook_adapter.py rename src/backend/core/llm/{ => runtime}/tool_media_store.py (95%) create mode 100644 src/backend/core/llm/runtime/tool_result_adapter.py create mode 100644 src/backend/core/llm/runtime/turn_state.py rename src/backend/core/llm/{ => runtime}/workspace.py (100%) rename src/backend/core/{agent_skills/selector.py => llm/skill_selection.py} (91%) delete mode 100644 src/backend/core/llm/steer_middleware.py create mode 100644 src/backend/core/llm/subagents/__init__.py rename src/backend/core/llm/{_subagent_execution.py => subagents/execution.py} (89%) rename src/backend/core/llm/{_subagent_prompts.py => subagents/prompts.py} (100%) rename src/backend/core/llm/{subagent_sessions.py => subagents/sessions.py} (98%) rename src/backend/core/llm/{_subagent_stream.py => subagents/stream.py} (80%) rename src/backend/core/llm/{subagent_tool.py => subagents/tool.py} (89%) delete mode 100644 src/backend/core/llm/tool_collector.py create mode 100644 src/backend/core/llm/tool_permissions/hook.py delete mode 100644 src/backend/core/llm/tool_permissions/middleware.py rename src/backend/core/llm/{gateway_mcp_tool.py => tools/cloud_gateway.py} (71%) create mode 100644 src/backend/core/llm/tools/cloud_manifest.py rename src/backend/core/llm/tools/{_tool_confirmation => confirmation}/__init__.py (100%) rename src/backend/core/llm/tools/{_tool_confirmation => confirmation}/gate.py (99%) rename src/backend/core/llm/tools/{_tool_confirmation => confirmation}/pending.py (96%) rename src/backend/core/llm/tools/{_tool_confirmation => confirmation}/shared.py (98%) rename src/backend/core/llm/tools/{_tool_confirmation => confirmation}/state.py (89%) delete mode 100644 src/backend/core/llm/tools/data_context_tool.py create mode 100644 src/backend/core/llm/tools/management.py rename src/backend/core/{ontology/toolkit.py => llm/tools/ontology_toolkit.py} (85%) rename src/backend/core/llm/{plan_update_tool.py => tools/plan_update.py} (92%) rename src/backend/core/{plugins/runtime/activation.py => llm/tools/plugin_activation.py} (78%) create mode 100644 src/backend/core/memory/edition.py create mode 100644 src/backend/core/memory/graph_contract.py create mode 100644 src/backend/core/memory/graph_state.py rename src/backend/core/{services/memory_settings_service.py => memory/settings.py} (98%) create mode 100644 src/backend/core/memory/settlement_port.py create mode 100644 src/backend/core/models/__init__.py create mode 100644 src/backend/core/models/admission.py rename src/backend/core/{services => models}/api_protocol_backfill.py (92%) rename src/backend/core/{llm => models}/chat_models.py (92%) create mode 100644 src/backend/core/models/completions/__init__.py rename src/backend/core/{llm => models/completions}/classifier.py (84%) rename src/backend/core/{llm => models/completions}/single_turn.py (96%) rename src/backend/core/{llm => models/completions}/summarizer.py (86%) rename src/backend/core/{services/model_config.py => models/config.py} (99%) create mode 100644 src/backend/core/models/constants.py rename src/backend/core/{llm/context_manager.py => models/context_window.py} (97%) create mode 100644 src/backend/core/models/embedding.py create mode 100644 src/backend/core/models/embedding_state.py rename src/backend/core/{llm => models}/failover.py (94%) rename src/backend/core/{db/model_import_validation.py => models/import_validation.py} (93%) create mode 100644 src/backend/core/models/providers/__init__.py rename src/backend/core/{llm => models}/providers/_image_tokens.py (96%) rename src/backend/core/{llm => models}/providers/context_probe.py (99%) rename src/backend/core/{llm => models}/providers/openai_compat.py (96%) rename src/backend/core/{llm => models}/providers/protocol_probe.py (100%) rename src/backend/core/{llm => models}/providers/reasoning_probe.py (97%) rename src/backend/core/{llm => models}/providers/registry.py (99%) rename src/backend/core/{llm => models}/providers/stream_retry.py (96%) rename src/backend/core/{llm => models}/providers/vendor_models.py (97%) rename src/backend/core/{llm => models}/reasoning_effort.py (100%) rename src/backend/core/{llm => models}/reasoning_replay.py (100%) rename src/backend/core/{db/model_repository.py => models/repository.py} (99%) create mode 100644 src/backend/core/models/reranking.py rename src/backend/core/{llm/responses_models.py => models/responses.py} (80%) rename src/backend/core/{services/user_model_selection.py => models/selection.py} (63%) create mode 100644 src/backend/core/models/selection_hook.py rename src/backend/core/{llm => models}/tool_call_identity.py (100%) rename src/backend/core/{llm => models}/tool_result_media.py (69%) rename src/backend/core/{llm/model_usage.py => models/usage.py} (98%) create mode 100644 src/backend/core/models/vision/__init__.py rename src/backend/core/{ => models}/vision/attachments.py (83%) rename src/backend/core/{ => models}/vision/prompt.py (98%) rename src/backend/core/{ => models}/vision/provider.py (97%) rename src/backend/core/{ => models}/vision/render.py (98%) rename src/backend/core/{ => models}/vision/schema.py (100%) rename src/backend/core/{ => models}/vision/service.py (96%) delete mode 100644 src/backend/core/myspace/duplicate_cleanup.py create mode 100644 src/backend/core/ontology/audit_events.py rename src/backend/core/{services/ontology_evolution_service.py => ontology/evolution.py} (96%) create mode 100644 src/backend/core/ontology/host.py create mode 100644 src/backend/core/ontology/manual_review.py create mode 100644 src/backend/core/ontology/output_review_state.py rename src/backend/core/{services/ontology_policy.py => ontology/policy.py} (97%) create mode 100644 src/backend/core/ontology/review_operations.py create mode 100644 src/backend/core/ontology/review_prompt.py create mode 100644 src/backend/core/ontology/reviewer.py create mode 100644 src/backend/core/ontology/reviewer_state.py create mode 100644 src/backend/core/ontology/runtime_gates.py create mode 100644 src/backend/core/ontology/runtime_pack.py create mode 100644 src/backend/core/ontology/runtime_policy.py rename src/backend/core/{services/ontology_service.py => ontology/service.py} (68%) create mode 100644 src/backend/core/ontology/tool_hook.py create mode 100644 src/backend/core/ontology/validation_models.py delete mode 100644 src/backend/core/plugins/README.md delete mode 100644 src/backend/core/plugins/management/__init__.py delete mode 100644 src/backend/core/plugins/resources/owners.py create mode 100644 src/backend/core/projects/__init__.py rename src/backend/core/{services/project_access.py => projects/access.py} (90%) rename src/backend/core/{services/project_api_service.py => projects/api.py} (97%) rename src/backend/core/{services/project_service.py => projects/base.py} (85%) create mode 100644 src/backend/core/projects/edition.py rename src/backend/core/{project_environment => projects/environment}/__init__.py (100%) rename src/backend/core/{project_environment => projects/environment}/config.py (100%) rename src/backend/core/{project_environment => projects/environment}/dependencies.py (96%) rename src/backend/core/{services/project_environment.py => projects/environment/service.py} (86%) rename src/backend/core/{services/project_environment_resources.py => projects/environment_resources.py} (90%) rename src/backend/core/{services/project_file_service.py => projects/files.py} (78%) rename src/backend/core/{services/project_instructions.py => projects/instructions.py} (95%) rename src/backend/core/{services/local_project_workspace.py => projects/local_workspace.py} (97%) rename src/backend/core/{services/project_scope.py => projects/personal_scope.py} (95%) create mode 100644 src/backend/core/projects/personal_summary.py rename src/backend/core/{services/project_transfer.py => projects/personal_transfer.py} (100%) create mode 100644 src/backend/core/projects/project_names.py rename src/backend/core/{services/project_source.py => projects/source.py} (91%) rename src/backend/core/{services/user_folder_service.py => projects/user_folders.py} (95%) create mode 100644 src/backend/core/runs/__init__.py rename src/backend/{orchestration/agent_api_recovery.py => core/runs/api_recovery.py} (94%) rename src/backend/core/{services/run_binding.py => runs/binding.py} (92%) create mode 100644 src/backend/core/runs/clock.py create mode 100644 src/backend/core/runs/events/__init__.py create mode 100644 src/backend/core/runs/events/projection.py rename src/backend/{orchestration/run_event_stream.py => core/runs/events/store.py} (84%) create mode 100644 src/backend/core/runs/events/subscription.py create mode 100644 src/backend/core/runs/events/writer.py create mode 100644 src/backend/core/runs/host.py rename src/backend/core/{services/run_journal.py => runs/journal.py} (94%) rename src/backend/core/{services/run_journal_domain.py => runs/journal_domain.py} (100%) rename src/backend/core/{services/run_journal_lifecycle.py => runs/journal_lifecycle.py} (93%) rename src/backend/core/{services/run_journal_recovery.py => runs/journal_recovery.py} (97%) create mode 100644 src/backend/core/runs/ownership.py create mode 100644 src/backend/core/runs/runtime.py create mode 100644 src/backend/core/runs/settings.py create mode 100644 src/backend/core/runs/tasks.py rename src/backend/{orchestration => core/runs}/tool_effect_recovery.py (82%) create mode 100644 src/backend/core/runs/tool_effects/__init__.py create mode 100644 src/backend/core/runs/tool_effects/arguments.py create mode 100644 src/backend/core/runs/tool_effects/errors.py create mode 100644 src/backend/core/runs/tool_effects/gateway.py create mode 100644 src/backend/core/runs/tool_effects/hooks.py create mode 100644 src/backend/core/runs/tool_effects/host.py create mode 100644 src/backend/core/runs/tool_effects/intent_transactions.py create mode 100644 src/backend/core/runs/tool_effects/invocation_claims.py create mode 100644 src/backend/core/runs/tool_effects/journal.py create mode 100644 src/backend/core/runs/tool_effects/journal_constants.py create mode 100644 src/backend/core/runs/tool_effects/recovery.py create mode 100644 src/backend/core/runs/tool_effects/recovery_claims.py create mode 100644 src/backend/core/runs/tool_effects/registry.py create mode 100644 src/backend/core/runs/tool_effects/result_transactions.py create mode 100644 src/backend/core/runs/tool_effects/retention.py create mode 100644 src/backend/core/runs/tool_effects/types.py create mode 100644 src/backend/core/runs/watchdog.py create mode 100644 src/backend/core/sandbox/commands/__init__.py create mode 100644 src/backend/core/sandbox/commands/completion_hook.py rename src/backend/core/{services => sandbox}/commands/context.py (85%) rename src/backend/core/{services => sandbox}/commands/delivery.py (92%) rename src/backend/{orchestration => core/sandbox}/commands/events.py (93%) rename src/backend/{orchestration => core/sandbox}/commands/notifications.py (77%) create mode 100644 src/backend/core/sandbox/commands/runtime.py rename src/backend/core/{services => sandbox}/commands/store.py (95%) rename src/backend/{orchestration => core/sandbox}/commands/subscription.py (69%) create mode 100644 src/backend/core/sandbox/credential_dispatch.py rename src/backend/core/{llm/tool_permissions => sandbox/permissions}/account.py (90%) rename src/backend/core/{llm/tool_permissions/preferences.py => sandbox/permissions/approval.py} (94%) rename src/backend/core/{llm/tool_permissions/models.py => sandbox/permissions/dispatch_models.py} (98%) rename src/backend/core/{llm/tool_permissions/specs.py => sandbox/permissions/tool_specs.py} (95%) create mode 100644 src/backend/core/sandbox/permissions/vocabulary.py delete mode 100644 src/backend/core/sandbox/providers/opensandbox/account_volumes.py rename src/backend/core/{llm/tools/_paths.py => sandbox/workspace/resolution.py} (88%) delete mode 100644 src/backend/core/services/__init__.py delete mode 100644 src/backend/core/services/agent_market_categories.py delete mode 100644 src/backend/core/services/agent_market_service.py delete mode 100644 src/backend/core/services/artifact_edition.py delete mode 100644 src/backend/core/services/chat_service.py delete mode 100644 src/backend/core/services/commands/__init__.py delete mode 100644 src/backend/core/services/desktop_cloud_bridge.py delete mode 100644 src/backend/core/services/edition_system_config.py delete mode 100644 src/backend/core/services/edition_workspace.py delete mode 100644 src/backend/core/services/kb_edition.py delete mode 100644 src/backend/core/services/lark_service.py delete mode 100644 src/backend/core/services/log_service.py delete mode 100644 src/backend/core/services/marketplace_service.py delete mode 100644 src/backend/core/services/mcp_marketplace_service.py delete mode 100644 src/backend/core/services/mcp_tool_metadata.py delete mode 100644 src/backend/core/services/prompt_version_service.py delete mode 100644 src/backend/core/services/skill_draft_service.py delete mode 100644 src/backend/core/services/system_config.py delete mode 100644 src/backend/core/services/tool_effect_ledger.py delete mode 100644 src/backend/core/services/user_agent_base.py delete mode 100644 src/backend/core/spaces/edition_access.py delete mode 100644 src/backend/core/spaces/edition_revocation.py delete mode 100644 src/backend/core/vision/__init__.py create mode 100644 src/backend/features/__init__.py create mode 100644 src/backend/features/automation/README.md create mode 100644 src/backend/features/automation/__init__.py create mode 100644 src/backend/features/automation/admission.py create mode 100644 src/backend/features/automation/api/__init__.py rename src/backend/{api/routes/v1 => features/automation/api}/automations.py (94%) rename src/backend/{core/services => features/automation}/delivery_targets.py (75%) rename src/backend/{core/services/automation_execution.py => features/automation/execution.py} (96%) create mode 100644 src/backend/features/automation/extension.py rename src/backend/{core/services/loop_service.py => features/automation/loops.py} (99%) rename src/backend/{core/db/models/automation.py => features/automation/models.py} (64%) rename src/backend/{core/services/automation_notifications.py => features/automation/notifications.py} (100%) rename src/backend/{core/services/automation_remote_effect.py => features/automation/remote_effect.py} (97%) create mode 100644 src/backend/features/automation/runs.py create mode 100644 src/backend/features/automation/scheduling.py create mode 100644 src/backend/features/automation/schema.py rename src/backend/{core/services/automation_service.py => features/automation/service.py} (73%) rename src/backend/{core/services/automation_tool_routing.py => features/automation/tool_routing.py} (85%) create mode 100644 src/backend/features/desktop/README.md create mode 100644 src/backend/features/desktop/__init__.py create mode 100644 src/backend/features/desktop/api/__init__.py create mode 100644 src/backend/features/desktop/api/capability_auth.py create mode 100644 src/backend/features/desktop/api/capability_bridge.py create mode 100644 src/backend/features/desktop/api/capability_dependencies.py create mode 100644 src/backend/features/desktop/api/capability_manifests.py create mode 100644 src/backend/features/desktop/api/capability_mcp_gateway.py create mode 100644 src/backend/features/desktop/api/capability_model_gateway.py create mode 100644 src/backend/features/desktop/api/capability_tools.py create mode 100644 src/backend/features/desktop/api/capability_workcopies.py rename src/backend/{api/routes/v1 => features/desktop/api}/desktop.py (99%) rename src/backend/{api/routes/v1 => features/desktop/api}/desktop_capabilities.py (80%) create mode 100644 src/backend/features/desktop/api/desktop_capability.py create mode 100644 src/backend/features/desktop/api/desktop_capability_state.py rename src/backend/{api/routes/v1 => features/desktop/api}/desktop_login.py (99%) rename src/backend/{api/routes/v1 => features/desktop/api}/desktop_observability.py (90%) rename src/backend/{api/routes/v1 => features/desktop/api}/device_link.py (95%) rename src/backend/{api/routes/v1 => features/desktop/api}/local.py (82%) rename src/backend/{api/routes => features/desktop/api}/local_file_actions.py (93%) create mode 100644 src/backend/features/desktop/auth/__init__.py rename src/backend/{core/auth/desktop_bridge.py => features/desktop/auth/bridge.py} (95%) rename src/backend/{core/auth/desktop_login_store.py => features/desktop/auth/login_store.py} (100%) rename src/backend/{core/auth/desktop_ticket_store.py => features/desktop/auth/ticket_store.py} (100%) create mode 100644 src/backend/features/desktop/bridge/__init__.py rename src/backend/{core/services/desktop_cloud_bundles.py => features/desktop/bridge/bundles.py} (88%) create mode 100644 src/backend/features/desktop/bridge/client.py create mode 100644 src/backend/features/desktop/bridge/client_state.py create mode 100644 src/backend/features/desktop/bridge/gateway.py create mode 100644 src/backend/features/desktop/bridge/identity.py create mode 100644 src/backend/features/desktop/bridge/managed_connectors.py rename src/backend/{core/services/desktop_model_credentials.py => features/desktop/bridge/model_credentials.py} (86%) rename src/backend/{core/services/desktop_release_events.py => features/desktop/bridge/release_events.py} (100%) rename src/backend/{core/services/desktop_resource_plugins.py => features/desktop/bridge/resource_plugins.py} (86%) rename src/backend/{core/services/desktop_cloud_skills.py => features/desktop/bridge/skills.py} (86%) create mode 100644 src/backend/features/desktop/bridge/status.py create mode 100644 src/backend/features/desktop/capabilities/__init__.py rename src/backend/{core => features/desktop}/capabilities/agents.py (93%) rename src/backend/{core => features/desktop}/capabilities/change_merge.py (94%) rename src/backend/{core => features/desktop}/capabilities/change_signal.py (100%) rename src/backend/{core => features/desktop}/capabilities/change_sync.py (87%) rename src/backend/{core => features/desktop}/capabilities/connectors.py (90%) rename src/backend/{core => features/desktop}/capabilities/credentials.py (100%) rename src/backend/{core => features/desktop}/capabilities/dependency.py (94%) rename src/backend/{core => features/desktop}/capabilities/dependency_declarations.py (100%) rename src/backend/{core => features/desktop}/capabilities/device_catalog.py (81%) rename src/backend/{core => features/desktop}/capabilities/device_plugin_catalog.py (69%) rename src/backend/{core => features/desktop}/capabilities/discovery.py (85%) rename src/backend/{core => features/desktop}/capabilities/invocation.py (74%) rename src/backend/{core => features/desktop}/capabilities/local_plugin_runtime.py (72%) rename src/backend/{core => features/desktop}/capabilities/local_projection.py (79%) rename src/backend/{core => features/desktop}/capabilities/management_context.py (82%) rename src/backend/{core => features/desktop}/capabilities/manifest_order.py (96%) rename src/backend/{core => features/desktop}/capabilities/mcp_json.py (98%) rename src/backend/{core => features/desktop}/capabilities/migration.py (92%) rename src/backend/{core => features/desktop}/capabilities/paths.py (80%) rename src/backend/{core => features/desktop}/capabilities/plugins.py (87%) rename src/backend/{core => features/desktop}/capabilities/preparation.py (89%) rename src/backend/{core => features/desktop}/capabilities/readiness.py (92%) rename src/backend/{core => features/desktop}/capabilities/registry.py (84%) create mode 100644 src/backend/features/desktop/capabilities/registry_types.py rename src/backend/{core => features/desktop}/capabilities/resolver.py (98%) create mode 100644 src/backend/features/desktop/capabilities/runtime/__init__.py rename src/backend/{core => features/desktop}/capabilities/runtime/connectors.py (92%) rename src/backend/{core => features/desktop}/capabilities/runtime/dependencies.py (93%) rename src/backend/{core => features/desktop}/capabilities/runtime/preparation.py (86%) rename src/backend/{core => features/desktop}/capabilities/runtime/state.py (89%) rename src/backend/{core => features/desktop}/capabilities/session_availability.py (100%) create mode 100644 src/backend/features/desktop/capabilities/skill_account.py create mode 100644 src/backend/features/desktop/capabilities/skill_candidates.py create mode 100644 src/backend/features/desktop/capabilities/skill_hashes.py create mode 100644 src/backend/features/desktop/capabilities/skill_paths.py create mode 100644 src/backend/features/desktop/capabilities/skill_publication.py create mode 100644 src/backend/features/desktop/capabilities/skill_views.py create mode 100644 src/backend/features/desktop/capabilities/skills.py create mode 100644 src/backend/features/desktop/capabilities/skills_state.py rename src/backend/{core => features/desktop}/capabilities/store.py (94%) rename src/backend/{core => features/desktop}/capabilities/view.py (97%) rename src/backend/{core => features/desktop}/capabilities/warmup.py (72%) create mode 100644 src/backend/features/desktop/capability/__init__.py rename src/backend/{core/services/desktop_capability_configs.py => features/desktop/capability/configs.py} (91%) rename src/backend/{core/services/desktop_capability_credentials.py => features/desktop/capability/credentials.py} (98%) rename src/backend/{core/services/desktop_capability_entities.py => features/desktop/capability/entities.py} (91%) rename src/backend/{core/services/desktop_capability_mcp.py => features/desktop/capability/mcp.py} (85%) rename src/backend/{core/services/desktop_capability_models.py => features/desktop/capability/models.py} (93%) create mode 100644 src/backend/features/desktop/capability/revision_provider.py rename src/backend/{core/services/desktop_capability_security.py => features/desktop/capability/security.py} (96%) rename src/backend/{core/services/desktop_capability.py => features/desktop/capability/service.py} (90%) rename src/backend/{core/services/desktop_capability_skills.py => features/desktop/capability/skills.py} (86%) rename src/backend/{core/services/desktop_capability_sync_check.py => features/desktop/capability/sync_check.py} (79%) rename src/backend/{core/services/capability_workcopies.py => features/desktop/capability/workcopies.py} (89%) create mode 100644 src/backend/features/desktop/edition.py create mode 100644 src/backend/features/desktop/extension.py create mode 100644 src/backend/features/desktop/gateway/__init__.py rename src/backend/{core/services/desktop_gateway_observability.py => features/desktop/gateway/observability.py} (98%) create mode 100644 src/backend/features/desktop/local/__init__.py rename src/backend/{core/services/local_file_actions.py => features/desktop/local/file_actions.py} (99%) rename src/backend/{core/services/local_grant_service.py => features/desktop/local/grants.py} (100%) rename src/backend/{core/services/local_management_migration.py => features/desktop/local/management_migration.py} (85%) create mode 100644 src/backend/features/desktop/local/manager_execution.py rename src/backend/{core/services/local_market_install.py => features/desktop/local/market_install.py} (61%) rename src/backend/{core/services => features/desktop/local}/native_app_icons.py (100%) rename src/backend/{core/plugins/local => features/desktop/local/plugins}/__init__.py (100%) rename src/backend/{core/plugins/local => features/desktop/local/plugins}/assets.py (95%) rename src/backend/{core/plugins/local => features/desktop/local/plugins}/legacy_migration.py (87%) rename src/backend/{core/plugins/local => features/desktop/local/plugins}/service.py (85%) create mode 100644 src/backend/features/desktop/local/plugins/sidecars.py create mode 100644 src/backend/features/desktop/local/publication_provider.py create mode 100644 src/backend/features/desktop/local/skills/__init__.py rename src/backend/{core/services/local_skill_editor.py => features/desktop/local/skills/editor.py} (86%) rename src/backend/{core/services/local_skill_service.py => features/desktop/local/skills/service.py} (81%) rename src/backend/{core/services/local_skill_upload.py => features/desktop/local/skills/upload.py} (74%) rename src/backend/{core/services/local_snapshot_service.py => features/desktop/local/snapshots.py} (100%) rename src/backend/{core/db/models/capability.py => features/desktop/models.py} (60%) create mode 100644 src/backend/features/desktop/observability/__init__.py rename src/backend/{core/services/desktop_observation_context.py => features/desktop/observability/context.py} (78%) rename src/backend/{core/services/desktop_observability_delivery.py => features/desktop/observability/delivery.py} (86%) rename src/backend/{core/services/observability_projection.py => features/desktop/observability/fields.py} (96%) rename src/backend/{core/services/desktop_observability.py => features/desktop/observability/projection.py} (98%) rename src/backend/{core/services/desktop_observability_sync.py => features/desktop/observability/sync.py} (96%) rename src/backend/{core/plugins/runtime/desktop_resolution.py => features/desktop/plugin_resolution.py} (85%) create mode 100644 src/backend/features/desktop/run_capabilities/__init__.py rename src/backend/{core/llm/factory/selection/device_preparation.py => features/desktop/run_capabilities/preparation.py} (83%) create mode 100644 src/backend/features/desktop/run_capabilities/provider.py rename src/backend/{core/agent_skills/backends/capability_store.py => features/desktop/skill_backend.py} (86%) create mode 100644 src/backend/features/evaluation/README.md create mode 100644 src/backend/features/evaluation/__init__.py rename src/backend/{core/services/evaluation_admission.py => features/evaluation/admission.py} (93%) create mode 100644 src/backend/features/evaluation/api/__init__.py rename src/backend/{api/routes/v1 => features/evaluation/api}/evaluation_sandboxes.py (98%) create mode 100644 src/backend/features/evaluation/extension.py create mode 100644 src/backend/features/evaluation/models.py create mode 100644 src/backend/features/evaluation/policy.py create mode 100644 src/backend/features/evaluation/runtime/__init__.py rename src/backend/{core/llm/evaluation_paths.py => features/evaluation/runtime/paths.py} (100%) rename src/backend/{core/llm/evaluation_process_tools.py => features/evaluation/runtime/process_tools.py} (87%) rename src/backend/{core/llm/evaluation_runtime.py => features/evaluation/runtime/runtime.py} (100%) rename src/backend/{core/llm/evaluation_tools.py => features/evaluation/runtime/tools.py} (57%) rename src/backend/{core/sandbox/providers/evaluation => features/evaluation/sandbox_provider}/__init__.py (100%) rename src/backend/{core/sandbox/providers/evaluation => features/evaluation/sandbox_provider}/binding.py (100%) rename src/backend/{core/sandbox/providers/evaluation => features/evaluation/sandbox_provider}/process_guard.py (100%) rename src/backend/{core/sandbox/providers/evaluation => features/evaluation/sandbox_provider}/provider.py (98%) rename src/backend/{core/services/evaluation_sandbox_service.py => features/evaluation/sandboxes.py} (93%) create mode 100644 src/backend/features/hosted_apps/README.md create mode 100644 src/backend/features/hosted_apps/__init__.py create mode 100644 src/backend/features/hosted_apps/api/__init__.py rename src/backend/{api/routes/v1 => features/hosted_apps/api}/application_transports.py (79%) rename src/backend/{api/routes/v1 => features/hosted_apps/api}/applications.py (83%) rename src/backend/{core/services/application_collections.py => features/hosted_apps/collections.py} (92%) rename src/backend/{core/config/application_hosting.py => features/hosted_apps/config.py} (87%) rename src/backend/{core/services/application_data.py => features/hosted_apps/data.py} (95%) rename src/backend/{core/services/application_deployments.py => features/hosted_apps/deployments.py} (92%) rename src/backend/{core/services/application_export.py => features/hosted_apps/export.py} (89%) create mode 100644 src/backend/features/hosted_apps/extension.py rename src/backend/{core/services/application_history.py => features/hosted_apps/history.py} (91%) rename src/backend/{core/services/application_hosting_setup.py => features/hosted_apps/hosting_setup.py} (96%) rename src/backend/{core/services/application_industry.py => features/hosted_apps/industry.py} (95%) rename src/backend/{core/services/application_lifecycle.py => features/hosted_apps/lifecycle.py} (88%) rename src/backend/{core/services/application_mcp.py => features/hosted_apps/mcp.py} (82%) create mode 100644 src/backend/features/hosted_apps/models.py rename src/backend/{core/services/application_operations.py => features/hosted_apps/operations.py} (84%) rename src/backend/{core/services/application_projects.py => features/hosted_apps/projects.py} (95%) rename src/backend/{core/services/application_publication.py => features/hosted_apps/publication.py} (93%) rename src/backend/{core/services/application_relational.py => features/hosted_apps/relational.py} (98%) create mode 100644 src/backend/features/hosted_apps/resources/__init__.py rename src/backend/{core/plugins/resources/application.py => features/hosted_apps/resources/admission.py} (88%) rename src/backend/{core/plugins/resources/application_cleanup.py => features/hosted_apps/resources/cleanup.py} (72%) rename src/backend/{core/services/application_schema.py => features/hosted_apps/schema.py} (100%) rename src/backend/{core/services/application_sql_policy.py => features/hosted_apps/sql_policy.py} (96%) rename src/backend/{core/services/application_store.py => features/hosted_apps/store.py} (94%) create mode 100644 src/backend/features/installed.py create mode 100644 src/backend/features/knowledge/README.md rename src/backend/{core/kb => features/knowledge}/__init__.py (64%) create mode 100644 src/backend/features/knowledge/api/__init__.py create mode 100644 src/backend/features/knowledge/api/kb.py create mode 100644 src/backend/features/knowledge/api/kb_assets.py create mode 100644 src/backend/features/knowledge/api/kb_chunks.py create mode 100644 src/backend/features/knowledge/api/kb_documents.py create mode 100644 src/backend/features/knowledge/api/kb_helpers.py create mode 100644 src/backend/features/knowledge/api/kb_mutations.py create mode 100644 src/backend/features/knowledge/api/kb_spaces.py create mode 100644 src/backend/features/knowledge/api/kb_state.py rename src/backend/{api/routes/v1 => features/knowledge/api}/kb_wiki.py (93%) create mode 100644 src/backend/features/knowledge/asset_captions.py create mode 100644 src/backend/features/knowledge/asset_indexing.py create mode 100644 src/backend/features/knowledge/catalog_options.py rename src/backend/{core/kb/external_provider.py => features/knowledge/disabled_provider.py} (85%) rename src/backend/{core/kb/external_retrieval.py => features/knowledge/disabled_retrieval.py} (100%) rename src/backend/{core/kb => features/knowledge}/document_status.py (100%) create mode 100644 src/backend/features/knowledge/edition.py create mode 100644 src/backend/features/knowledge/enrichment.py create mode 100644 src/backend/features/knowledge/extension.py rename src/backend/{core/kb => features/knowledge}/index_queue.py (99%) rename src/backend/{core/kb => features/knowledge}/index_worker.py (97%) create mode 100644 src/backend/features/knowledge/kb_assets.py create mode 100644 src/backend/features/knowledge/kb_assets_state.py create mode 100644 src/backend/features/knowledge/kb_parser.py create mode 100644 src/backend/features/knowledge/kb_parser_state.py create mode 100644 src/backend/features/knowledge/kb_vector.py create mode 100644 src/backend/features/knowledge/kb_vector_state.py create mode 100644 src/backend/features/knowledge/models.py create mode 100644 src/backend/features/knowledge/parser_chunks.py create mode 100644 src/backend/features/knowledge/parser_extract.py create mode 100644 src/backend/features/knowledge/parser_semantic.py create mode 100644 src/backend/features/knowledge/parser_tokens.py create mode 100644 src/backend/features/knowledge/parser_types.py rename src/backend/{core/auth/kb_permissions.py => features/knowledge/personal_permissions.py} (97%) rename src/backend/{api/routes/v1/kb_models.py => features/knowledge/personal_request_models.py} (100%) rename src/backend/{core/content/kb_processing.py => features/knowledge/processing.py} (67%) rename src/backend/{core/db/repository/kb.py => features/knowledge/repository.py} (97%) rename src/backend/{core/services/kb_service.py => features/knowledge/service.py} (95%) create mode 100644 src/backend/features/knowledge/vector_client.py create mode 100644 src/backend/features/knowledge/vector_fields.py create mode 100644 src/backend/features/knowledge/vector_indexing.py rename src/backend/{core/kb => features/knowledge}/wiki/__init__.py (92%) rename src/backend/{core/kb => features/knowledge}/wiki/cite.py (95%) rename src/backend/{core/kb => features/knowledge}/wiki/config.py (99%) rename src/backend/{core/kb => features/knowledge}/wiki/dedup.py (94%) rename src/backend/{core/kb => features/knowledge}/wiki/extract.py (93%) rename src/backend/{core/kb => features/knowledge}/wiki/finalize.py (92%) rename src/backend/{core/kb => features/knowledge}/wiki/jobs.py (99%) rename src/backend/{core/kb => features/knowledge}/wiki/linkify.py (100%) rename src/backend/{core/kb => features/knowledge}/wiki/llm.py (96%) create mode 100644 src/backend/features/knowledge/wiki/local_graph.py create mode 100644 src/backend/features/knowledge/wiki/local_projection.py create mode 100644 src/backend/features/knowledge/wiki/local_provider.py create mode 100644 src/backend/features/knowledge/wiki/local_provider_state.py rename src/backend/{core/kb => features/knowledge}/wiki/pipeline.py (95%) rename src/backend/{core/kb => features/knowledge}/wiki/prompts.py (100%) rename src/backend/{core/kb => features/knowledge}/wiki/reduce.py (93%) rename src/backend/{core/kb => features/knowledge}/wiki/slug.py (100%) rename src/backend/{core/kb => features/knowledge}/wiki/store.py (98%) rename src/backend/{core/kb => features/knowledge}/wiki/taxonomy.py (89%) rename src/backend/{core/kb => features/knowledge}/wiki/worker.py (94%) rename src/backend/{core/kb => features/knowledge}/wiki_router.py (88%) create mode 100644 src/backend/features/marketplace/README.md create mode 100644 src/backend/features/marketplace/__init__.py create mode 100644 src/backend/features/marketplace/agent_catalog.py create mode 100644 src/backend/features/marketplace/agent_categories.py create mode 100644 src/backend/features/marketplace/agent_installation.py create mode 100644 src/backend/features/marketplace/agent_metadata.py create mode 100644 src/backend/features/marketplace/agent_publishing.py create mode 100644 src/backend/features/marketplace/agent_submission_data.py create mode 100644 src/backend/features/marketplace/agent_submissions.py rename src/backend/{core/services/marketplace_agent_versions.py => features/marketplace/agent_versions.py} (88%) create mode 100644 src/backend/features/marketplace/agents_state.py create mode 100644 src/backend/features/marketplace/api/__init__.py rename src/backend/{api/routes/v1 => features/marketplace/api}/agent_marketplace.py (82%) rename src/backend/{api/routes/v1 => features/marketplace/api}/catalog_ownership.py (94%) rename src/backend/{api/routes/v1 => features/marketplace/api}/marketplace.py (84%) rename src/backend/{api/routes/v1 => features/marketplace/api}/mcp_marketplace.py (84%) create mode 100644 src/backend/features/marketplace/core_catalog.py create mode 100644 src/backend/features/marketplace/edition.py create mode 100644 src/backend/features/marketplace/extension.py create mode 100644 src/backend/features/marketplace/mcp_administration.py rename src/backend/{core/config/mcp_marketplace_catalog.py => features/marketplace/mcp_catalog.py} (100%) create mode 100644 src/backend/features/marketplace/mcp_credentials.py create mode 100644 src/backend/features/marketplace/mcp_curated.py create mode 100644 src/backend/features/marketplace/mcp_installation.py create mode 100644 src/backend/features/marketplace/mcp_listing.py rename src/backend/{core/services/mcp_marketplace_monitor.py => features/marketplace/mcp_monitor.py} (94%) create mode 100644 src/backend/features/marketplace/mcp_publishing.py create mode 100644 src/backend/features/marketplace/mcp_revalidation.py create mode 100644 src/backend/features/marketplace/mcp_state.py create mode 100644 src/backend/features/marketplace/mcp_submissions.py create mode 100644 src/backend/features/marketplace/mcp_validation.py create mode 100644 src/backend/features/marketplace/mcp_versions.py create mode 100644 src/backend/features/marketplace/models.py create mode 100644 src/backend/features/marketplace/package_source.py rename src/backend/{core/services/marketplace_listing.py => features/marketplace/personal_listing.py} (97%) rename src/backend/{core/auth/marketplace_visibility.py => features/marketplace/personal_visibility.py} (100%) rename src/backend/{core/plugins/management/market.py => features/marketplace/plugin_market.py} (73%) create mode 100644 src/backend/features/marketplace/skill_archives.py create mode 100644 src/backend/features/marketplace/skill_catalog.py create mode 100644 src/backend/features/marketplace/skill_installation.py create mode 100644 src/backend/features/marketplace/skill_metadata.py create mode 100644 src/backend/features/marketplace/skill_publishing.py create mode 100644 src/backend/features/marketplace/skill_submission_data.py create mode 100644 src/backend/features/marketplace/skill_submissions.py rename src/backend/{core/services/marketplace_skill_versions.py => features/marketplace/skill_versions.py} (72%) create mode 100644 src/backend/features/marketplace/skills_state.py create mode 100644 src/backend/features/marketplace/versions/__init__.py rename src/backend/{core/services/marketplace_version_archive.py => features/marketplace/versions/archive.py} (100%) rename src/backend/{core/services/marketplace_version_config.py => features/marketplace/versions/config.py} (95%) rename src/backend/{core/services/marketplace_version_sources.py => features/marketplace/versions/sources.py} (84%) rename src/backend/{core/services/marketplace_version_store.py => features/marketplace/versions/store.py} (90%) create mode 100644 src/backend/features/sites/README.md create mode 100644 src/backend/features/sites/__init__.py create mode 100644 src/backend/features/sites/api/__init__.py rename src/backend/{api/routes/v1 => features/sites/api}/internal_site_auth.py (100%) rename src/backend/{api/routes/v1 => features/sites/api}/internal_sites.py (94%) rename src/backend/{api/routes/v1 => features/sites/api}/local_site_sources.py (94%) rename src/backend/{api/routes => features/sites/api}/site_gate.py (100%) rename src/backend/{api/routes/v1 => features/sites/api}/sites.py (95%) rename src/backend/{api/routes => features/sites/api}/sites_serve.py (96%) rename src/backend/{core/plugins/local/site_upgrade.py => features/sites/builtin_upgrade.py} (83%) rename src/backend/{core/services/desktop_site_publish.py => features/sites/desktop_publish.py} (83%) create mode 100644 src/backend/features/sites/edition.py create mode 100644 src/backend/features/sites/extension.py rename src/backend/{core/services/site_listing.py => features/sites/listing.py} (94%) rename src/backend/{core/services/local_site_sources.py => features/sites/local_sources.py} (93%) rename src/backend/{core/services/site_management.py => features/sites/management.py} (95%) rename src/backend/{core/db/models/site.py => features/sites/models.py} (97%) rename src/backend/{core/services/site_packaging.py => features/sites/packaging.py} (92%) rename src/backend/{core/services/site_password.py => features/sites/password.py} (100%) rename src/backend/{core/services/site_access_policy.py => features/sites/personal_access.py} (97%) rename src/backend/{core/db/models/site_scope.py => features/sites/personal_scope.py} (100%) rename src/backend/{core/services/site_rate_limit.py => features/sites/rate_limit.py} (100%) rename src/backend/{core/db/repository/site.py => features/sites/repository.py} (98%) rename src/backend/{core/services/site_service.py => features/sites/service.py} (97%) create mode 100644 src/backend/features/sites/tools/__init__.py rename src/backend/{core/llm/tools/design_picker_tool.py => features/sites/tools/design_picker.py} (86%) create mode 100644 src/backend/features/workbench/README.md rename src/backend/{core/applications => features/workbench}/__init__.py (100%) rename src/backend/{core/applications => features/workbench}/activity.py (98%) create mode 100644 src/backend/features/workbench/api/__init__.py rename src/backend/{api/routes/v1 => features/workbench/api}/plugin_ui.py (93%) rename src/backend/{api/routes/v1 => features/workbench/api}/workbench.py (86%) rename src/backend/{api/routes/v1 => features/workbench/api}/workbench_activity.py (98%) rename src/backend/{api/routes/v1 => features/workbench/api}/workbench_application_config.py (92%) rename src/backend/{api/routes/v1 => features/workbench/api}/workbench_auth.py (98%) rename src/backend/{api/routes/v1 => features/workbench/api}/workbench_capabilities.py (96%) rename src/backend/{api/routes/v1 => features/workbench/api}/workbench_diagnostics.py (97%) rename src/backend/{api/routes/v1 => features/workbench/api}/workbench_events.py (97%) rename src/backend/{api/routes/v1 => features/workbench/api}/workbench_preferences.py (97%) rename src/backend/{api/routes/v1 => features/workbench/api}/workbench_resources.py (87%) rename src/backend/{api/routes/v1 => features/workbench/api}/workbench_revision_leases.py (97%) rename src/backend/{api/routes/v1 => features/workbench/api}/workbench_state.py (98%) rename src/backend/{api/routes/v1 => features/workbench/api}/workbench_upgrades.py (98%) rename src/backend/{api/routes/v1 => features/workbench/api}/workbench_workspace.py (83%) rename src/backend/{core/applications => features/workbench}/application_config.py (93%) rename src/backend/{core/applications => features/workbench}/application_config_bindings.py (82%) rename src/backend/{core/applications => features/workbench}/background.py (95%) rename src/backend/{core/applications => features/workbench}/capabilities.py (95%) rename src/backend/{core/applications => features/workbench}/catalog.py (96%) rename src/backend/{core/applications => features/workbench}/coordination.py (100%) rename src/backend/{core/applications => features/workbench}/diagnostic_probe.py (98%) rename src/backend/{core/applications => features/workbench}/diagnostic_resources.py (100%) rename src/backend/{core/applications => features/workbench}/diagnostics.py (99%) rename src/backend/{core/applications => features/workbench}/directory_revision.py (96%) create mode 100644 src/backend/features/workbench/edition.py create mode 100644 src/backend/features/workbench/extension.py rename src/backend/{core/applications => features/workbench}/external.py (94%) rename src/backend/{core/applications => features/workbench}/external_auth.py (99%) rename src/backend/{core/applications => features/workbench}/external_boundary.py (97%) rename src/backend/{core/applications => features/workbench}/external_embed.py (97%) rename src/backend/{core/applications => features/workbench}/installation_binding.py (77%) rename src/backend/{core/applications => features/workbench}/local_plugin_upgrades.py (86%) rename src/backend/{core/applications => features/workbench}/migration.py (97%) rename src/backend/{core/applications => features/workbench}/migration_compat.py (88%) rename src/backend/{core/applications => features/workbench}/migration_plan.py (96%) rename src/backend/{core/applications => features/workbench}/migration_snapshot.py (91%) create mode 100644 src/backend/features/workbench/models.py rename src/backend/{core/applications => features/workbench}/origin.py (100%) create mode 100644 src/backend/features/workbench/package_source.py rename src/backend/{core/applications => features/workbench}/plugin_candidate_storage.py (88%) rename src/backend/{core/applications => features/workbench}/plugin_catalog.py (95%) rename src/backend/{core/applications => features/workbench}/plugin_revisions.py (89%) rename src/backend/{core/applications => features/workbench}/plugin_upgrades.py (90%) rename src/backend/{core/applications => features/workbench}/policy.py (94%) rename src/backend/{core/applications => features/workbench}/preferences.py (96%) rename src/backend/{core/applications => features/workbench}/protocol.py (100%) rename src/backend/{core/plugins/resources/deadlines.py => features/workbench/resource_deadlines.py} (67%) create mode 100644 src/backend/features/workbench/resource_owner.py rename src/backend/{core/applications => features/workbench}/revocation.py (94%) rename src/backend/{core/applications => features/workbench}/revocation_sweep.py (96%) rename src/backend/{core/applications => features/workbench}/schemas/application.schema.json (100%) rename src/backend/{core/applications => features/workbench}/schemas/host.json (100%) rename src/backend/{core/applications => features/workbench}/schemas/module.schema.json (100%) rename src/backend/{core/applications => features/workbench}/state.py (96%) rename src/backend/{core/applications => features/workbench}/state_recovery.py (100%) rename src/backend/{core/plugins => features/workbench}/ui/__init__.py (100%) rename src/backend/{core/plugins => features/workbench}/ui/contract.py (99%) rename src/backend/{core/plugins => features/workbench}/ui/contributions.py (87%) rename src/backend/{core/plugins => features/workbench}/ui/data_proxy.py (97%) rename src/backend/{core/plugins => features/workbench}/ui/declarations.py (99%) rename src/backend/{core/plugins => features/workbench}/ui/primitives.py (99%) rename src/backend/{core/plugins => features/workbench}/ui/workspace_enabled.py (100%) rename src/backend/{core/applications => features/workbench}/workspace_page.py (98%) create mode 100644 src/backend/hugagent_cli/schema.py create mode 100644 src/backend/integrations/__init__.py create mode 100644 src/backend/integrations/api/__init__.py rename src/backend/{api/routes/v1 => integrations/api}/integrations.py (97%) create mode 100644 src/backend/integrations/channels/README.md rename src/backend/{core => integrations}/channels/__init__.py (75%) rename src/backend/{core => integrations}/channels/adapters/__init__.py (100%) create mode 100644 src/backend/integrations/channels/adapters/dingtalk.py create mode 100644 src/backend/integrations/channels/adapters/dingtalk_delivery.py create mode 100644 src/backend/integrations/channels/adapters/dingtalk_history.py create mode 100644 src/backend/integrations/channels/adapters/dingtalk_settings.py create mode 100644 src/backend/integrations/channels/adapters/lark.py create mode 100644 src/backend/integrations/channels/adapters/lark_delivery.py create mode 100644 src/backend/integrations/channels/adapters/lark_messages.py create mode 100644 src/backend/integrations/channels/adapters/lark_settings.py rename src/backend/{core => integrations}/channels/adapters/wecom.py (99%) rename src/backend/{core => integrations}/channels/adapters/weixin.py (99%) create mode 100644 src/backend/integrations/channels/api/__init__.py rename src/backend/{api/routes/v1 => integrations/channels/api}/channel_desktop.py (89%) rename src/backend/{api/routes/v1 => integrations/channels/api}/channels.py (97%) rename src/backend/{core/services/channel_desktop_worker.py => integrations/channels/desktop_worker.py} (89%) create mode 100644 src/backend/integrations/channels/extension.py create mode 100644 src/backend/integrations/channels/inbound.py create mode 100644 src/backend/integrations/channels/inbound_attachments.py create mode 100644 src/backend/integrations/channels/inbound_delivery.py create mode 100644 src/backend/integrations/channels/inbound_history.py create mode 100644 src/backend/integrations/channels/inbound_observation.py create mode 100644 src/backend/integrations/channels/inbound_sessions.py create mode 100644 src/backend/integrations/channels/inbound_state.py rename src/backend/{core => integrations}/channels/manager.py (97%) rename src/backend/{core => integrations}/channels/markdown.py (100%) create mode 100644 src/backend/integrations/channels/models.py rename src/backend/{core => integrations}/channels/outbound.py (94%) rename src/backend/{core => integrations}/channels/protocol.py (100%) rename src/backend/{core => integrations}/channels/registry.py (88%) rename src/backend/{core/services/channel_relay.py => integrations/channels/relay.py} (98%) rename src/backend/{core/services/channel_relay_dispatch.py => integrations/channels/relay_dispatch.py} (89%) rename src/backend/{core/services/channel_relay_rpc.py => integrations/channels/relay_rpc.py} (95%) rename src/backend/{core/db/repository/channel.py => integrations/channels/repository.py} (98%) rename src/backend/{core/services/channel_service.py => integrations/channels/service.py} (97%) create mode 100644 src/backend/integrations/channels/tools/__init__.py rename src/backend/{core/llm/tools/channel_attachment_tool.py => integrations/channels/tools/attachment.py} (87%) create mode 100644 src/backend/integrations/dingtalk/README.md create mode 100644 src/backend/integrations/dingtalk/__init__.py create mode 100644 src/backend/integrations/dingtalk/cube_credentials.py create mode 100644 src/backend/integrations/dingtalk/environment.py create mode 100644 src/backend/integrations/dingtalk/extension.py create mode 100644 src/backend/integrations/dingtalk/models.py create mode 100644 src/backend/integrations/dingtalk/mounts.py create mode 100644 src/backend/integrations/dingtalk/parsing.py create mode 100644 src/backend/integrations/dingtalk/repository.py create mode 100644 src/backend/integrations/dingtalk/sandbox.py rename src/backend/{core/services/dingtalk_service.py => integrations/dingtalk/service.py} (80%) create mode 100644 src/backend/integrations/dingtalk/storage.py create mode 100644 src/backend/integrations/email/README.md create mode 100644 src/backend/integrations/email/__init__.py create mode 100644 src/backend/integrations/email/cube_credentials.py create mode 100644 src/backend/integrations/email/extension.py create mode 100644 src/backend/integrations/email/models.py create mode 100644 src/backend/integrations/email/mounts.py create mode 100644 src/backend/integrations/email/repository.py create mode 100644 src/backend/integrations/email/sandbox.py rename src/backend/{core/services/email_service.py => integrations/email/service.py} (98%) create mode 100644 src/backend/integrations/email/storage.py create mode 100644 src/backend/integrations/lark/README.md create mode 100644 src/backend/integrations/lark/__init__.py create mode 100644 src/backend/integrations/lark/configuration.py create mode 100644 src/backend/integrations/lark/cube_credentials.py create mode 100644 src/backend/integrations/lark/extension.py create mode 100644 src/backend/integrations/lark/login.py create mode 100644 src/backend/integrations/lark/models.py create mode 100644 src/backend/integrations/lark/mounts.py create mode 100644 src/backend/integrations/lark/parsing.py create mode 100644 src/backend/integrations/lark/process.py create mode 100644 src/backend/integrations/lark/repository.py create mode 100644 src/backend/integrations/lark/sandbox.py create mode 100644 src/backend/integrations/lark/service.py create mode 100644 src/backend/integrations/lark/state.py create mode 100644 src/backend/integrations/lark/storage.py create mode 100644 src/backend/integrations/yida/README.md create mode 100644 src/backend/integrations/yida/__init__.py create mode 100644 src/backend/integrations/yida/cube_credentials.py rename src/backend/{core/services/yida_execution.py => integrations/yida/execution.py} (97%) create mode 100644 src/backend/integrations/yida/extension.py create mode 100644 src/backend/integrations/yida/models.py create mode 100644 src/backend/integrations/yida/mounts.py create mode 100644 src/backend/integrations/yida/sandbox.py rename src/backend/{core/services/yida_service.py => integrations/yida/service.py} (99%) create mode 100644 src/backend/integrations/yida/skill_content.py create mode 100644 src/backend/integrations/yida/storage.py create mode 100644 src/backend/orchestration/automations/__init__.py rename src/backend/orchestration/{schedulers => automations}/automation_artifacts.py (97%) rename src/backend/orchestration/{schedulers => automations}/automation_conversation.py (61%) create mode 100644 src/backend/orchestration/automations/automation_delivery.py rename src/backend/orchestration/{schedulers => automations}/automation_dispatch.py (62%) create mode 100644 src/backend/orchestration/automations/automation_lease.py create mode 100644 src/backend/orchestration/automations/automation_recovery.py create mode 100644 src/backend/orchestration/automations/automation_scheduler.py delete mode 100644 src/backend/orchestration/chat_run_executor.py delete mode 100644 src/backend/orchestration/citation_anchor.py delete mode 100644 src/backend/orchestration/commands/__init__.py delete mode 100644 src/backend/orchestration/commands/runtime.py create mode 100644 src/backend/orchestration/configuration.py delete mode 100644 src/backend/orchestration/local_plugin_sidecars.py create mode 100644 src/backend/orchestration/loops/__init__.py create mode 100644 src/backend/orchestration/loops/admission.py rename src/backend/orchestration/{autonomous_loop.py => loops/driver.py} (98%) rename src/backend/orchestration/{loop_planner.py => loops/planner.py} (97%) create mode 100644 src/backend/orchestration/loops/presentation.py create mode 100644 src/backend/orchestration/loops/recovery.py rename src/backend/orchestration/{loop_evaluator.py => loops/requirements.py} (83%) rename src/backend/orchestration/{subagents/loop_reviewer.py => loops/reviewer.py} (78%) create mode 100644 src/backend/orchestration/loops/run.py delete mode 100644 src/backend/orchestration/memory_integration.py create mode 100644 src/backend/orchestration/pending.py create mode 100644 src/backend/orchestration/plans/__init__.py create mode 100644 src/backend/orchestration/plans/admission.py create mode 100644 src/backend/orchestration/plans/execute_run.py create mode 100644 src/backend/orchestration/plans/generate_run.py rename src/backend/orchestration/{subagents/plan_mode.py => plans/mode.py} (96%) delete mode 100644 src/backend/orchestration/schedulers/automation_lease.py delete mode 100644 src/backend/orchestration/schedulers/automation_recovery.py delete mode 100644 src/backend/orchestration/schedulers/automation_scheduler.py delete mode 100644 src/backend/orchestration/streaming.py delete mode 100644 src/backend/orchestration/subagents/__init__.py delete mode 100644 src/backend/orchestration/subagents/ontology_reviewer.py rename src/backend/orchestration/{schedulers => workers}/__init__.py (100%) rename src/backend/orchestration/{schedulers => workers}/_worker_base.py (98%) rename src/backend/{core/services => orchestration/workers}/edition_startup.py (100%) rename src/backend/orchestration/{schedulers => workers}/evolution_scheduler.py (98%) rename src/backend/orchestration/{schedulers => workers}/memory_ttl_scheduler.py (95%) rename src/backend/orchestration/{schedulers => workers}/persona_distill_worker.py (94%) create mode 100644 src/backend/orchestration/workers/run_recovery.py delete mode 100644 src/backend/orchestration/workflow.py create mode 100644 src/backend/personal_agent/device_link/README.md rename src/backend/{core => personal_agent}/device_link/__init__.py (84%) rename src/backend/{core => personal_agent}/device_link/driver.py (100%) rename src/backend/{core => personal_agent}/device_link/local_api.py (100%) rename src/backend/{core => personal_agent}/device_link/runner.py (98%) rename src/backend/{core => personal_agent}/device_link/store.py (100%) delete mode 100644 src/backend/scripts/_loop_convergence_unit.py delete mode 100644 src/backend/scripts/_loop_db_resume_unit.py delete mode 100644 src/backend/scripts/_loop_evaluator_route_unit.py delete mode 100644 src/backend/scripts/_loop_integration_test.py delete mode 100644 src/backend/scripts/_loop_llmeval_smoke.py delete mode 100644 src/backend/scripts/_loop_preflight.py delete mode 100644 src/backend/scripts/_loop_selfverify_smoke.py delete mode 100644 src/backend/scripts/_loop_smoke.py delete mode 100644 src/backend/scripts/_loop_test_edgebench.py delete mode 100644 src/backend/scripts/desktop_cloud_wrapper.py delete mode 100644 src/backend/scripts/desktop_fixture_mcp.py delete mode 100644 src/backend/scripts/dual_mode_e2e.py delete mode 100644 src/backend/scripts/edgebench/ann_judge.py delete mode 100644 src/backend/scripts/edgebench/perm_opt_judge.py delete mode 100644 src/backend/scripts/edit_chat_message.py delete mode 100644 src/backend/scripts/evaluation_sandbox_e2e.py delete mode 100644 src/backend/scripts/export_content.py delete mode 100755 src/backend/scripts/harness_fault_injection.py delete mode 100644 src/backend/scripts/import_cherry_agents.py delete mode 100644 src/backend/scripts/import_content.py delete mode 100644 src/backend/scripts/run_evolution_benchmark.py delete mode 100644 src/backend/scripts/run_evolution_llm_ab.py delete mode 100644 src/backend/scripts/workbench_migration.py create mode 100644 src/backend/tests/api/test_loop_cancel_scope.py delete mode 100644 src/backend/tests/capabilities/test_credential_continuity.py create mode 100644 src/backend/tests/chat/test_snapshot_transaction_cleanup.py create mode 100644 src/backend/tests/content/test_artifact_default_path.py create mode 100644 src/backend/tests/core/runs/test_automation_journal_guard.py create mode 100644 src/backend/tests/features/__init__.py create mode 100644 src/backend/tests/features/automation/__init__.py create mode 100644 src/backend/tests/features/automation/run_fixtures.py create mode 100644 src/backend/tests/features/automation/test_automation_delivery_deadline.py rename src/backend/tests/{ => features/automation}/test_automation_execution_location.py (80%) create mode 100644 src/backend/tests/features/automation/test_automation_lease_close.py create mode 100644 src/backend/tests/features/automation/test_automation_manual_trigger.py rename src/backend/tests/{services => features/automation}/test_automation_notifications.py (98%) create mode 100644 src/backend/tests/features/automation/test_automation_oneshot_finalize.py create mode 100644 src/backend/tests/features/automation/test_automation_ownership_postgres.py rename src/backend/tests/{ => features/automation}/test_automation_remote_effect.py (74%) create mode 100644 src/backend/tests/features/automation/test_automation_run_ownership.py create mode 100644 src/backend/tests/features/automation/test_delivery_target_migration.py create mode 100644 src/backend/tests/features/automation/test_durable_admission.py rename src/backend/tests/{services => features/automation}/test_loop_service_harness.py (98%) create mode 100644 src/backend/tests/features/automation/test_queue_postgres.py create mode 100644 src/backend/tests/features/desktop/__init__.py create mode 100644 src/backend/tests/features/desktop/capabilities/__init__.py rename src/backend/tests/{ => features/desktop}/capabilities/_cloud_identity.py (100%) rename src/backend/tests/{ => features/desktop}/capabilities/conftest.py (67%) rename src/backend/tests/{ => features/desktop}/capabilities/management_browser_server.py (88%) rename src/backend/tests/{ => features/desktop}/capabilities/runtime_recovery_support.py (75%) rename src/backend/tests/{ => features/desktop}/capabilities/test_account_authorization.py (52%) rename src/backend/tests/{ => features/desktop}/capabilities/test_agent_rerun_identity.py (79%) rename src/backend/tests/{ => features/desktop}/capabilities/test_agent_runtime_grants.py (62%) rename src/backend/tests/{ => features/desktop}/capabilities/test_agents_plugins.py (90%) rename src/backend/tests/{ => features/desktop}/capabilities/test_autonomous_loop_scopes.py (73%) rename src/backend/tests/{ => features/desktop}/capabilities/test_available_runtime.py (82%) rename src/backend/tests/{ => features/desktop}/capabilities/test_capability_change_sync.py (81%) rename src/backend/tests/{ => features/desktop}/capabilities/test_catalog_cloud_only_in_hybrid.py (72%) rename src/backend/tests/{ => features/desktop}/capabilities/test_catalog_plugin_components_hidden.py (87%) rename src/backend/tests/{ => features/desktop}/capabilities/test_change_merge.py (90%) rename src/backend/tests/{ => features/desktop}/capabilities/test_change_signal.py (97%) rename src/backend/tests/{ => features/desktop}/capabilities/test_cloud_identity.py (96%) rename src/backend/tests/{ => features/desktop}/capabilities/test_cloud_management.py (88%) rename src/backend/tests/{ => features/desktop}/capabilities/test_cloud_parallel_preparation.py (68%) rename src/backend/tests/{ => features/desktop}/capabilities/test_cloud_plugin_binding_runtime.py (76%) rename src/backend/tests/{ => features/desktop}/capabilities/test_cloud_sticky_plugins.py (83%) rename src/backend/tests/{ => features/desktop}/capabilities/test_cloud_workcopies.py (91%) rename src/backend/tests/{ => features/desktop}/capabilities/test_core_layers.py (96%) create mode 100644 src/backend/tests/features/desktop/capabilities/test_credential_continuity.py rename src/backend/tests/{ => features/desktop}/capabilities/test_dependency_closure.py (87%) rename src/backend/tests/{ => features/desktop}/capabilities/test_desktop_capabilities_api.py (85%) rename src/backend/tests/{ => features/desktop}/capabilities/test_desktop_content_safety.py (87%) rename src/backend/tests/{ => features/desktop}/capabilities/test_desktop_declaration_export.py (89%) rename src/backend/tests/{ => features/desktop}/capabilities/test_desktop_model_reasoning_safety.py (88%) rename src/backend/tests/{ => features/desktop}/capabilities/test_desktop_resource_plugins.py (87%) rename src/backend/tests/{ => features/desktop}/capabilities/test_desktop_token_security.py (90%) rename src/backend/tests/{ => features/desktop}/capabilities/test_desktop_warmup.py (61%) rename src/backend/tests/{ => features/desktop}/capabilities/test_device_api_contract.py (74%) rename src/backend/tests/{ => features/desktop}/capabilities/test_device_catalog_projection.py (91%) rename src/backend/tests/{ => features/desktop}/capabilities/test_dynamic_model_epoch_cache.py (80%) rename src/backend/tests/{ => features/desktop}/capabilities/test_explicit_desktop_invocation.py (79%) rename src/backend/tests/{ => features/desktop}/capabilities/test_gateway_invocation_proof.py (78%) rename src/backend/tests/{ => features/desktop}/capabilities/test_gateway_unknown_outcome.py (61%) rename src/backend/tests/{ => features/desktop}/capabilities/test_hybrid_owned_plugins.py (50%) rename src/backend/tests/{ => features/desktop}/capabilities/test_initial_sync_readiness.py (51%) rename src/backend/tests/{ => features/desktop}/capabilities/test_junction_windows_paths.py (86%) rename src/backend/tests/{ => features/desktop}/capabilities/test_loader_store_mode.py (74%) rename src/backend/tests/{ => features/desktop}/capabilities/test_local_folder_discovery.py (90%) rename src/backend/tests/{ => features/desktop}/capabilities/test_local_management_migration.py (83%) rename src/backend/tests/{ => features/desktop}/capabilities/test_local_skill_projection.py (83%) rename src/backend/tests/{ => features/desktop}/capabilities/test_management_readiness.py (87%) rename src/backend/tests/{ => features/desktop}/capabilities/test_management_tools.py (56%) rename src/backend/tests/{ => features/desktop}/capabilities/test_manager_lifecycle.py (91%) rename src/backend/tests/{ => features/desktop}/capabilities/test_manager_regressions.py (85%) rename src/backend/tests/{ => features/desktop}/capabilities/test_manifest_order.py (85%) rename src/backend/tests/{ => features/desktop}/capabilities/test_manifest_tool_schema.py (76%) rename src/backend/tests/{ => features/desktop}/capabilities/test_mcp_json.py (98%) rename src/backend/tests/{ => features/desktop}/capabilities/test_mcp_json_consistency.py (99%) rename src/backend/tests/{ => features/desktop}/capabilities/test_memory_epoch_cache.py (79%) rename src/backend/tests/{ => features/desktop}/capabilities/test_migration.py (89%) rename src/backend/tests/{ => features/desktop}/capabilities/test_migration_preservation.py (96%) rename src/backend/tests/{ => features/desktop}/capabilities/test_model_credential_reference.py (86%) rename src/backend/tests/{ => features/desktop}/capabilities/test_model_error_diagnostics.py (83%) rename src/backend/tests/{ => features/desktop}/capabilities/test_owned_connector_assembly.py (55%) rename src/backend/tests/{ => features/desktop}/capabilities/test_partial_session_availability.py (73%) rename src/backend/tests/{ => features/desktop}/capabilities/test_plugin_connections.py (89%) rename src/backend/tests/{ => features/desktop}/capabilities/test_plugin_device_assets.py (89%) rename src/backend/tests/{ => features/desktop}/capabilities/test_prepare_lock_not_held_over_io.py (98%) rename src/backend/tests/{ => features/desktop}/capabilities/test_progressive_plugins.py (72%) rename src/backend/tests/{ => features/desktop}/capabilities/test_runtime_frozen_bindings.py (63%) rename src/backend/tests/{ => features/desktop}/capabilities/test_runtime_publication_recovery.py (70%) rename src/backend/tests/{ => features/desktop}/capabilities/test_runtime_recovery.py (58%) rename src/backend/tests/{ => features/desktop}/capabilities/test_runtime_scope_consumers.py (87%) rename src/backend/tests/{ => features/desktop}/capabilities/test_runtime_scopes.py (82%) rename src/backend/tests/{ => features/desktop}/capabilities/test_runtime_selection_recovery.py (66%) rename src/backend/tests/{ => features/desktop}/capabilities/test_runtime_session_recovery.py (68%) rename src/backend/tests/{ => features/desktop}/capabilities/test_selection_dependency_sources.py (79%) rename src/backend/tests/{ => features/desktop}/capabilities/test_selection_readiness.py (80%) rename src/backend/tests/{ => features/desktop}/capabilities/test_store_file_integrity.py (88%) rename src/backend/tests/{ => features/desktop}/capabilities/test_toolkit_runtime_alias.py (83%) rename src/backend/tests/{ => features/desktop}/capabilities/test_user_facing_errors.py (81%) rename src/backend/tests/{ => features/desktop}/capabilities/test_view_os_errors.py (95%) rename src/backend/tests/{ => features/desktop}/capabilities/test_workbench_local_upgrades.py (84%) rename src/backend/tests/{ => features/desktop}/desktop_login_browser_server.py (91%) create mode 100644 src/backend/tests/features/desktop/index_fixtures.py create mode 100644 src/backend/tests/features/desktop/observability_cases_1.py create mode 100644 src/backend/tests/features/desktop/observability_cases_2.py create mode 100644 src/backend/tests/features/desktop/observability_support.py rename src/backend/tests/{ => features/desktop}/run_desktop_browser_e2e.py (99%) rename src/backend/tests/{ => features/desktop}/test_desktop_auth.py (96%) rename src/backend/tests/{misc => features/desktop}/test_desktop_bridge.py (90%) rename src/backend/tests/{misc => features/desktop}/test_desktop_capability.py (81%) rename src/backend/tests/{misc => features/desktop}/test_desktop_capability_models.py (91%) rename src/backend/tests/{misc => features/desktop}/test_desktop_cloud_skills.py (84%) rename src/backend/tests/{ => features/desktop}/test_desktop_device_login.py (97%) rename src/backend/tests/{ => features/desktop}/test_desktop_memory_persistence.py (96%) rename src/backend/tests/{api => features/desktop}/test_desktop_model_gateway.py (88%) create mode 100644 src/backend/tests/features/desktop/test_desktop_observability.py rename src/backend/tests/{ => features/desktop}/test_desktop_observability_authenticated_delivery.py (77%) rename src/backend/tests/{ => features/desktop}/test_desktop_observability_delivery.py (75%) rename src/backend/tests/{api => features/desktop}/test_desktop_release_events.py (96%) rename src/backend/tests/{ => features/desktop}/test_desktop_sync_locking.py (77%) rename src/backend/tests/{api => features/desktop}/test_desktop_updates.py (98%) rename src/backend/tests/{api => features/desktop}/test_local_file_actions.py (89%) rename src/backend/tests/{services => features/desktop}/test_local_grant_service.py (98%) rename src/backend/tests/{services => features/desktop}/test_local_snapshot_service.py (96%) create mode 100644 src/backend/tests/features/evaluation/__init__.py create mode 100644 src/backend/tests/features/evaluation/conftest.py rename src/backend/tests/{sandbox => features/evaluation}/test_evaluation_admission.py (97%) rename src/backend/tests/{sandbox => features/evaluation}/test_evaluation_binding.py (98%) rename src/backend/tests/{llm => features/evaluation}/test_evaluation_factory.py (87%) rename src/backend/tests/{sandbox => features/evaluation}/test_evaluation_process_guard.py (97%) rename src/backend/tests/{sandbox => features/evaluation}/test_evaluation_provider.py (96%) rename src/backend/tests/{sandbox => features/evaluation}/test_evaluation_router.py (78%) rename src/backend/tests/{llm => features/evaluation}/test_evaluation_runtime.py (88%) rename src/backend/tests/{sandbox => features/evaluation}/test_evaluation_service.py (94%) create mode 100644 src/backend/tests/features/hosted_apps/__init__.py rename src/backend/tests/{api => features/hosted_apps}/application_hosting_support.py (77%) rename src/backend/tests/{api => features/hosted_apps}/test_application_agent_contract.py (79%) rename src/backend/tests/{api => features/hosted_apps}/test_application_collections.py (87%) rename src/backend/tests/{services => features/hosted_apps}/test_application_desktop.py (80%) rename src/backend/tests/{api => features/hosted_apps}/test_application_export.py (91%) rename src/backend/tests/{api => features/hosted_apps}/test_application_hosting.py (92%) rename src/backend/tests/{api => features/hosted_apps}/test_application_personal_mcp.py (91%) rename src/backend/tests/{api => features/hosted_apps}/test_application_project_tools.py (87%) rename src/backend/tests/{api => features/hosted_apps}/test_application_projects.py (94%) rename src/backend/tests/{api => features/hosted_apps}/test_application_retirement.py (94%) rename src/backend/tests/{api => features/hosted_apps}/test_hosting_remediation.py (82%) create mode 100644 src/backend/tests/features/hosted_apps/test_schema_lifecycle.py create mode 100644 src/backend/tests/features/knowledge/__init__.py create mode 100644 src/backend/tests/features/knowledge/kb_multimodal_assets_cases_1.py create mode 100644 src/backend/tests/features/knowledge/kb_multimodal_assets_cases_2.py create mode 100644 src/backend/tests/features/knowledge/kb_multimodal_assets_support.py create mode 100644 src/backend/tests/features/knowledge/kb_wiki_end_to_end_cases_1.py create mode 100644 src/backend/tests/features/knowledge/kb_wiki_end_to_end_cases_2.py create mode 100644 src/backend/tests/features/knowledge/kb_wiki_end_to_end_support.py create mode 100644 src/backend/tests/features/knowledge/test_host_contributions.py rename src/backend/tests/{mcp => features/knowledge}/test_kb_backend_routing.py (74%) rename src/backend/tests/{ => features/knowledge}/test_kb_document_status.py (89%) rename src/backend/tests/{services => features/knowledge}/test_kb_index_queue.py (98%) rename src/backend/tests/{services => features/knowledge}/test_kb_index_worker.py (88%) create mode 100644 src/backend/tests/features/knowledge/test_kb_multimodal_assets.py rename src/backend/tests/{mcp => features/knowledge}/test_kb_vector_safety.py (57%) rename src/backend/tests/{mcp => features/knowledge}/test_kb_vector_timeouts.py (68%) create mode 100644 src/backend/tests/features/knowledge/test_kb_wiki_end_to_end.py rename src/backend/tests/{services => features/knowledge}/test_kb_wiki_pipeline.py (96%) create mode 100644 src/backend/tests/features/marketplace/__init__.py create mode 100644 src/backend/tests/features/marketplace/mcp_marketplace_cases_1.py create mode 100644 src/backend/tests/features/marketplace/mcp_marketplace_cases_2.py create mode 100644 src/backend/tests/features/marketplace/mcp_marketplace_cases_3.py create mode 100644 src/backend/tests/features/marketplace/mcp_marketplace_cases_4.py create mode 100644 src/backend/tests/features/marketplace/mcp_marketplace_support.py rename src/backend/tests/{ => features/marketplace}/test_agent_marketplace.py (69%) rename src/backend/tests/{skills => features/marketplace}/test_marketplace_service.py (73%) rename src/backend/tests/{skills => features/marketplace}/test_marketplace_submission.py (54%) create mode 100644 src/backend/tests/features/marketplace/test_mcp_marketplace.py create mode 100644 src/backend/tests/features/marketplace/test_oauth_sdk2_contract.py create mode 100644 src/backend/tests/features/sites/__init__.py create mode 100644 src/backend/tests/features/sites/conftest.py rename src/backend/tests/{capabilities => features/sites}/test_cloud_site_publish.py (82%) rename src/backend/tests/{llm => features/sites}/test_design_pick.py (99%) rename src/backend/tests/{llm => features/sites}/test_desktop_site_paths.py (88%) rename src/backend/tests/{capabilities => features/sites}/test_local_project_site_publish.py (93%) rename src/backend/tests/{capabilities => features/sites}/test_local_site_sources.py (88%) rename src/backend/tests/{api => features/sites}/test_site_access_password.py (95%) rename src/backend/tests/{api => features/sites}/test_site_contention_postgres.py (92%) rename src/backend/tests/{capabilities => features/sites}/test_site_listing.py (90%) rename src/backend/tests/{capabilities => features/sites}/test_site_plugin_upgrade.py (85%) rename src/backend/tests/{services => features/sites}/test_site_rate_limit.py (97%) create mode 100644 src/backend/tests/features/test_execution_identity_contract.py create mode 100644 src/backend/tests/features/test_host_runtime.py create mode 100644 src/backend/tests/features/test_installed_modules.py create mode 100644 src/backend/tests/features/test_run_capability_host.py create mode 100644 src/backend/tests/features/workbench/__init__.py rename src/backend/tests/{ => features/workbench}/test_plugin_ui_contract.py (97%) rename src/backend/tests/{ => features/workbench}/test_plugin_ui_routes.py (97%) rename src/backend/tests/{ => features/workbench}/test_workbench_activity_routes.py (98%) rename src/backend/tests/{ => features/workbench}/test_workbench_actual_process_observer.py (93%) rename src/backend/tests/{ => features/workbench}/test_workbench_application_config_http.py (97%) rename src/backend/tests/{ => features/workbench}/test_workbench_approval_lease_http.py (95%) rename src/backend/tests/{ => features/workbench}/test_workbench_auth_routes.py (98%) rename src/backend/tests/{ => features/workbench}/test_workbench_background_subpage_http.py (97%) rename src/backend/tests/{ => features/workbench}/test_workbench_candidate_storage.py (91%) rename src/backend/tests/{ => features/workbench}/test_workbench_capability_routes.py (98%) rename src/backend/tests/{ => features/workbench}/test_workbench_closing_isolation_http.py (94%) rename src/backend/tests/{ => features/workbench}/test_workbench_cookie_boundaries.py (94%) rename src/backend/tests/{ => features/workbench}/test_workbench_coordination.py (97%) rename src/backend/tests/{ => features/workbench}/test_workbench_diagnostic_failure_http.py (97%) rename src/backend/tests/{ => features/workbench}/test_workbench_diagnostics.py (98%) rename src/backend/tests/{ => features/workbench}/test_workbench_embed_routes.py (96%) rename src/backend/tests/{ => features/workbench}/test_workbench_external_routes.py (100%) rename src/backend/tests/{ => features/workbench}/test_workbench_invalidation.py (94%) rename src/backend/tests/{ => features/workbench}/test_workbench_migration.py (98%) rename src/backend/tests/{ => features/workbench}/test_workbench_migration_auth_gate.py (95%) rename src/backend/tests/{ => features/workbench}/test_workbench_observer_paths.py (93%) rename src/backend/tests/{ => features/workbench}/test_workbench_official_capabilities_http.py (93%) rename src/backend/tests/{ => features/workbench}/test_workbench_origins.py (95%) rename src/backend/tests/{ => features/workbench}/test_workbench_plugin_routes.py (98%) rename src/backend/tests/{ => features/workbench}/test_workbench_preferences_routes.py (99%) rename src/backend/tests/{ => features/workbench}/test_workbench_protocol.py (95%) rename src/backend/tests/{ => features/workbench}/test_workbench_provider_scope_http.py (93%) rename src/backend/tests/{ => features/workbench}/test_workbench_resource_owner_migrations.py (99%) rename src/backend/tests/{ => features/workbench}/test_workbench_resources.py (89%) rename src/backend/tests/{ => features/workbench}/test_workbench_revocation_failure_http.py (91%) rename src/backend/tests/{ => features/workbench}/test_workbench_routing_routes.py (94%) rename src/backend/tests/{ => features/workbench}/test_workbench_state_binding.py (94%) rename src/backend/tests/{ => features/workbench}/test_workbench_state_concurrency.py (93%) rename src/backend/tests/{ => features/workbench}/test_workbench_state_routes.py (98%) rename src/backend/tests/{ => features/workbench}/test_workbench_upgrade_routes.py (97%) rename src/backend/tests/{ => features/workbench}/workbench_capabilities_http_fixture.py (96%) rename src/backend/tests/{ => features/workbench}/workbench_diagnostic_http_fixture.py (95%) rename src/backend/tests/{ => features/workbench}/workbench_embed_http_fixture.py (95%) rename src/backend/tests/{ => features/workbench}/workbench_http_fixture.py (95%) rename src/backend/tests/{ => features/workbench}/workbench_observed_runner_fixture.py (90%) rename src/backend/tests/{ => features/workbench}/workbench_observer_paths.py (98%) rename src/backend/tests/{ => features/workbench}/workbench_overflow_http_fixture.py (95%) rename src/backend/tests/{ => features/workbench}/workbench_plugin_http_fixture.py (96%) rename src/backend/tests/{ => features/workbench}/workbench_plugin_support.py (100%) rename src/backend/tests/{ => features/workbench}/workbench_resource_observer.py (92%) rename src/backend/tests/{ => features/workbench}/workbench_routing_http_fixture.py (88%) create mode 100644 src/backend/tests/integrations/__init__.py create mode 100644 src/backend/tests/integrations/channels/__init__.py create mode 100644 src/backend/tests/integrations/channels/bot_cases_1.py create mode 100644 src/backend/tests/integrations/channels/bot_cases_2.py create mode 100644 src/backend/tests/integrations/channels/bot_cases_3.py create mode 100644 src/backend/tests/integrations/channels/bot_cases_4.py create mode 100644 src/backend/tests/integrations/channels/bot_cases_5.py create mode 100644 src/backend/tests/integrations/channels/bot_cases_6.py create mode 100644 src/backend/tests/integrations/channels/bot_support.py create mode 100644 src/backend/tests/integrations/channels/test_adapter_method_binding.py create mode 100644 src/backend/tests/integrations/channels/test_channel_bots.py rename src/backend/tests/{ => integrations/channels}/test_channel_bots_api.py (94%) rename src/backend/tests/{ => integrations/channels}/test_channel_desktop_e2e.py (88%) rename src/backend/tests/{ => integrations/channels}/test_channel_desktop_http.py (89%) rename src/backend/tests/{ => integrations/channels}/test_channel_desktop_migration.py (97%) rename src/backend/tests/{ => integrations/channels}/test_channel_desktop_relay.py (88%) rename src/backend/tests/{ => integrations/channels}/test_channel_desktop_relay_api.py (89%) rename src/backend/tests/{ => integrations/channels}/test_channel_desktop_worker.py (74%) rename src/backend/tests/{ => integrations/channels}/test_channel_receiver_lifecycle.py (94%) rename src/backend/tests/{ => integrations/channels}/test_weixin_connection_health.py (95%) rename src/backend/tests/{ => integrations/channels}/test_weixin_qr_protocol.py (96%) rename src/backend/tests/{ => integrations/dingtalk}/test_dingtalk_integration.py (93%) rename src/backend/tests/{ => integrations/email}/test_email_integration.py (91%) create mode 100644 src/backend/tests/integrations/lark/test_app_lifecycle.py rename src/backend/tests/{ => integrations/lark}/test_lark_auth_status.py (79%) rename src/backend/tests/{ => integrations/yida}/test_yida_integration.py (94%) create mode 100644 src/backend/tests/llm/test_mcp_factory_e2e.py create mode 100644 src/backend/tests/llm/test_mcp_human_wait_timeout.py create mode 100644 src/backend/tests/llm/test_mcp_native_adversarial.py create mode 100644 src/backend/tests/llm/test_mcp_native_capabilities.py delete mode 100644 src/backend/tests/llm/test_mcp_pool_cache.py create mode 100644 src/backend/tests/llm/test_mcp_runtime_pool.py create mode 100644 src/backend/tests/llm/test_mcp_session_ownership.py create mode 100644 src/backend/tests/llm/test_migration_vendor_datasource.py create mode 100644 src/backend/tests/llm/test_stream_wrapper_cleanup.py create mode 100644 src/backend/tests/orchestration/automation_test_support.py create mode 100644 src/backend/tests/orchestration/hook_support.py create mode 100644 src/backend/tests/orchestration/test_agent_runtime_resources.py delete mode 100644 src/backend/tests/orchestration/test_harness_process_restart.py create mode 100644 src/backend/tests/orchestration/test_observation_writer.py create mode 100644 src/backend/tests/orchestration/test_pending_workflows.py create mode 100644 src/backend/tests/orchestration/test_plan_early_cancel.py create mode 100644 src/backend/tests/orchestration/test_runtime_phase_lifecycle.py create mode 100644 src/backend/tests/orchestration/test_tool_effect_ledger_2.py create mode 100644 src/backend/tests/orchestration/test_tool_effect_ledger_3.py create mode 100644 src/backend/tests/orchestration/test_tool_effect_ledger_4.py create mode 100644 src/backend/tests/orchestration/test_tool_effect_ledger_5.py create mode 100644 src/backend/tests/orchestration/tool_effect_support.py rename src/backend/tests/{ => personal_agent}/test_device_link.py (97%) create mode 100644 src/backend/tests/sandbox/test_controller_capacity.py create mode 100644 src/backend/tests/sandbox/test_runtime_ownership.py delete mode 100644 src/backend/tests/services/test_kb_multimodal_assets.py delete mode 100644 src/backend/tests/services/test_kb_wiki_end_to_end.py create mode 100644 src/backend/tests/test_agent_version_history.py delete mode 100644 src/backend/tests/test_automation_manual_trigger.py delete mode 100644 src/backend/tests/test_automation_oneshot_finalize.py delete mode 100644 src/backend/tests/test_channel_bots.py create mode 100644 src/backend/tests/test_cli_cold_bootstrap.py create mode 100644 src/backend/tests/test_credential_registry.py create mode 100644 src/backend/tests/test_db_import_order.py delete mode 100644 src/backend/tests/test_desktop_observability.py create mode 100644 src/backend/tests/test_hook_contributions.py create mode 100644 src/backend/tests/test_local_onboarding.py create mode 100644 src/backend/tests/test_mcp_canonical_url.py delete mode 100644 src/backend/tests/test_mcp_marketplace.py create mode 100644 src/backend/tests/test_package_sources.py create mode 100644 src/backend/tests/test_personal_agent_prompt_cache.py create mode 100644 src/backend/tests/test_route_import_order.py create mode 100644 src/backend/tests/test_session_database_isolation.py create mode 100644 src/backend/tests/test_tool_contributions.py create mode 100644 src/backend/tests/test_user_cleanup.py diff --git a/.claude/skills/hugagent-backend-dev/references/architecture.md b/.claude/skills/hugagent-backend-dev/references/architecture.md index 34c6a766..5fda6ac0 100644 --- a/.claude/skills/hugagent-backend-dev/references/architecture.md +++ b/.claude/skills/hugagent-backend-dev/references/architecture.md @@ -48,10 +48,10 @@ POST /v1/agents/responses → api/routes/v1/chats/__init__.py → orchestration/chat_run_executor.py # ChatRun + Redis Stream（后台 run，SSE 跟随，断线续播） → orchestration/workflow.py # 流式编排主入口 - → core/services/chat_mode_service.py # 解析对话模式 ChatModeSpec（standard/turbo/市场模式；装配契约） + → core/chat/modes.py # 解析对话模式 ChatModeSpec（standard/turbo/市场模式；装配契约） → orchestration/memory_integration.py # 检索分层记忆并注入（user-role 冻结块，600ms 预算） → core/llm/factory/build.py # 构建 AgentScope 2.0 ReActAgent（按 ChatModeSpec 收窄工具面） - → core/llm/mcp_manager.py # MCP 客户端池（streamable-http → mcp 容器） + → core/llm/mcp/manager.py # MCP 客户端池（streamable-http → mcp 容器） → core/config/mcp_config.py # MCP server 定义（端口真源 mcp_servers/_ports.py） → core/llm/middlewares.py # AS2 中间件（动态模型、文件上下文、CitationAnchorMiddleware、OntologyGateMiddleware） → prompts/prompt_runtime.py # 装配系统提示词（DB 版本池优先，prompt_text/ 兜底） @@ -96,13 +96,13 @@ SSE 事件（前端解析真源 `src/frontend/src/hooks/chatStream.ts`）： | 健康检查 | `api/health.py` | /health, /ready, /live | | Schema | `api/schemas.py` | 请求/响应 Pydantic 模型 | | 路由 | `api/routes/v1/` | 66 个路由文件 | -| 技能引擎 | `core/agent_skills/` | SKILL.md 解析、多源加载、{dir} 沙箱路径注入 | -| 生成物 | `core/artifacts/` | 注册与下载（store.py，local/oss 双模） | +| 技能引擎 | `core/capabilities/skills/` | SKILL.md 解析、多源加载、{dir} 沙箱路径注入 | +| 生成物 | `core/files/artifacts/` | 注册与下载（store.py，local/oss 双模） | | 认证 | `core/auth/` | backend.py, session.py, sso.py, permissions_iface.py（CE/EE 接缝） | | 渠道 | `core/channels/` | 入站机器人框架：钉钉/飞书/企微/微信 adapter；加渠道 = 写 adapter + registry.py 注册 | | 聊天 | `core/chat/` | context.py, tool_log.py | | 配置 | `core/config/` | settings.py + catalog 五件套（catalog.json/catalog.py/loader/resolver/migration）+ mcp_config.py | -| 内容 | `core/content/` | 内容块、file_parser.py | +| 内容 | `core/files/content/` | 内容块、file_parser.py | | 数据库 | `core/db/` | engine.py, models/（17 领域文件）, repository/（10 领域文件）, model_extensions.py, edition_tables.py（垫片） | | 进化 | `core/evolution/` | GCE 共享契约（Episode/候选/结算）；控制面在 edition_ee/evolution | | 基础设施 | `core/infra/` | exceptions, responses, logging, metrics, 限流, Redis, crypto | @@ -113,7 +113,7 @@ SSE 事件（前端解析真源 `src/frontend/src/hooks/chatStream.ts`）： | 本体 | `core/ontology/` | Domain Pack 校验、运行时选取、工具闸（schemas/validator/revision/toolkit） | | 沙箱 | `core/sandbox/` | protocol.py + script_runner / opensandbox / cube 三实现 | | 服务 | `core/services/` | 70+ 业务服务（user/chat/chat_mode/channel/loop/kb/plugin/prompt_version/marketplace…） | -| 存储 | `core/storage/` | protocol, local, s3, oss | +| 存储 | `core/files/storage/` | protocol, local, s3, oss | | 编排 | `orchestration/` | workflow, chat_run_executor, streaming, citation_anchor, autonomous_loop, followups, batch_orchestrator, schedulers/, subagents/ | | 提示词 | `prompts/` | prompt_runtime + provider + prompt_text/{default,code_exec,distillation,plan_mode,subagents,turbo}/ | | MCP 服务器 | `mcp_servers/` | 12 个 server 目录、11 个在役（端口 9100–9114，真源 _ports.py；9105/9109-9111 为退役保留位） | diff --git a/Makefile b/Makefile index 365c5fa5..f3ae4788 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ .PHONY: help install install-dev dev test selftest lint format type-check security-scan \ - migrate migrate-new migrate-down migrate-history db-reset db-seed \ + migrate migrate-new migrate-down migrate-history db-reset \ build build-no-cache up up-recreate down down-volumes restart logs logs-backend logs-frontend ps \ health-check clean clean-docker clean-all @@ -32,7 +32,6 @@ help: @echo " make migrate-down - Rollback one migration" @echo " make migrate-history - Show migration history" @echo " make db-reset - Reset DB to latest (destructive)" - @echo " make db-seed - Seed sample data" @echo "" @echo "Docker:" @echo " make build - Build images" @@ -105,9 +104,6 @@ db-reset: alembic downgrade base alembic upgrade head -db-seed: - PYTHONPATH=$(PYTHONPATH_BACKEND) $(PYTHON) src/backend/scripts/seed_data.py - build: $(COMPOSE) build diff --git a/desktop/requirements-desktop-linux-aarch64-py311.lock b/desktop/requirements-desktop-linux-aarch64-py311.lock index 27cb3904..1420921a 100644 --- a/desktop/requirements-desktop-linux-aarch64-py311.lock +++ b/desktop/requirements-desktop-linux-aarch64-py311.lock @@ -1,8 +1,8 @@ # This file is generated. Do not edit it by hand. # Regenerate with: npm --prefix desktop run lock:desktop -# input-sha256: d18da0b813b601a5c727eb280b730ddb3b307fb487ab2a2bb28d4c568a336990 +# input-sha256: b9680ef14192ea099c4a7159c08c064cbd555bd681c00c49c5c74b1f7f3d4046 # Target: CPython 3.11 on Linux aarch64 (UOS 1070 / manylinux 2.28) -agentscope==2.0.0 +agentscope @ https://github.com/agentscope-ai/agentscope/archive/ed357feb78a45ff02df155bb5610eab9bf543832.tar.gz # via -r desktop/requirements-desktop.txt aiofiles==25.1.0 # via agentscope @@ -105,8 +105,6 @@ defusedxml==0.7.1 # via # -r desktop/requirements-desktop.txt # markitdown -deprecated==1.3.1 - # via limits dingtalk-stream==0.24.3 # via -r desktop/requirements-desktop.txt distro==1.9.0 @@ -183,18 +181,18 @@ httpx==0.28.1 # huggingface-hub # lark-oapi # litellm - # mcp # mem0ai # ollama # openai # qdrant-client # tavily-python httpx-sse==0.4.3 + # via dashscope +httpx2==2.12.0 # via - # dashscope + # agentscope + # anthropic # mcp -httpx2==2.12.0 - # via anthropic huggingface-hub==1.28.0 # via tokenizers hyperframe==6.1.0 @@ -220,7 +218,7 @@ jmespath==1.1.0 # via # boto3 # botocore -json-repair==0.63.3 +json-repair==0.64.1 # via agentscope json5==0.15.0 # via agentscope @@ -228,6 +226,7 @@ jsonschema==4.26.0 # via # -r desktop/requirements-desktop.txt # agentscope + # json-repair # litellm # mcp jsonschema-specifications==2025.9.1 @@ -236,8 +235,6 @@ kiwisolver==1.5.0 # via matplotlib lark-oapi==1.7.3 # via -r desktop/requirements-desktop.txt -limits==5.8.0 - # via slowapi linkify-it-py==2.1.0 # via markdown-it-py litellm==1.98.0 @@ -270,10 +267,12 @@ matplotlib==3.11.1 # via # -r desktop/requirements-desktop.txt # seaborn -mcp==1.29.0 +mcp==2.3.0 # via # -r desktop/requirements-desktop.txt # agentscope +mcp-types==2.3.0 + # via mcp mdit-py-plugins==0.6.1 # via textual mdurl==0.1.2 @@ -316,6 +315,7 @@ openpyxl==3.1.5 opentelemetry-api==1.44.0 # via # agentscope + # mcp # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http # opentelemetry-sdk @@ -348,7 +348,6 @@ packaging==26.3 # via # faiss-cpu # huggingface-hub - # limits # matplotlib # onnxruntime # pikepdf @@ -415,8 +414,10 @@ pydantic==2.13.4 # dashscope # fastapi # google-genai + # json-repair # litellm # mcp + # mcp-types # mem0ai # ollama # openai @@ -425,9 +426,7 @@ pydantic==2.13.4 pydantic-core==2.46.4 # via pydantic pydantic-settings==2.15.0 - # via - # litellm - # mcp + # via litellm pyee==13.0.1 # via playwright pygments==2.21.0 @@ -441,7 +440,9 @@ pymilvus==2.5.18 pyparsing==3.3.2 # via matplotlib pypdf==6.16.2 - # via -r desktop/requirements-desktop.txt + # via + # -r desktop/requirements-desktop.txt + # agentscope pypdf2==3.0.1 # via -r desktop/requirements-desktop.txt pypdfium2==5.13.0 @@ -518,11 +519,10 @@ requests-toolbelt==1.0.0 # via lark-oapi rich==15.0.0 # via + # agentscope # dashscope # textual # typer -ripgrep==14.1.0 - # via agentscope rpds-py==2026.6.3 # via # jsonschema @@ -549,8 +549,6 @@ six==1.17.0 # via # markdownify # python-dateutil -slowapi==0.1.10 - # via -r desktop/requirements-desktop.txt sniffio==1.3.1 # via # anthropic @@ -564,7 +562,9 @@ sqlalchemy==2.0.52 # alembic # mem0ai sse-starlette==3.0.3 - # via mcp + # via + # -r desktop/requirements-desktop.txt + # mcp starlette==0.46.2 # via # fastapi @@ -615,8 +615,8 @@ typing-extensions==4.16.0 # grpcio # httpx2 # huggingface-hub - # limits # mcp + # mcp-types # openai # opentelemetry-api # opentelemetry-exporter-otlp-proto-grpc @@ -639,6 +639,8 @@ typing-inspection==0.4.4 # mcp # pydantic # pydantic-settings +tzdata==2026.5 + # via agentscope uc-micro-py==2.0.0 # via linkify-it-py ujson==5.13.0 @@ -667,8 +669,6 @@ werkzeug==3.1.8 # via -r desktop/requirements-desktop.txt wheel==0.48.0 # via -r desktop/requirements-desktop.txt -wrapt==2.3.0 - # via deprecated wsproto==1.3.2 # via simple-websocket xlsxwriter==3.2.9 diff --git a/desktop/requirements-desktop-linux-x86_64-py311.lock b/desktop/requirements-desktop-linux-x86_64-py311.lock index 1ae48d54..a71239c2 100644 --- a/desktop/requirements-desktop-linux-x86_64-py311.lock +++ b/desktop/requirements-desktop-linux-x86_64-py311.lock @@ -1,8 +1,8 @@ # This file is generated. Do not edit it by hand. # Regenerate with: npm --prefix desktop run lock:desktop -# input-sha256: d18da0b813b601a5c727eb280b730ddb3b307fb487ab2a2bb28d4c568a336990 +# input-sha256: b9680ef14192ea099c4a7159c08c064cbd555bd681c00c49c5c74b1f7f3d4046 # Target: CPython 3.11 on Linux x86_64 -agentscope==2.0.0 +agentscope @ https://github.com/agentscope-ai/agentscope/archive/ed357feb78a45ff02df155bb5610eab9bf543832.tar.gz # via -r desktop/requirements-desktop.txt aiofiles==25.1.0 # via agentscope @@ -31,6 +31,7 @@ anyio==4.14.2 # anthropic # google-genai # httpx + # httpx2 # mcp # openai # sse-starlette @@ -98,8 +99,6 @@ defusedxml==0.7.1 # via # -r desktop/requirements-desktop.txt # markitdown -deprecated==1.3.1 - # via limits dingtalk-stream==0.24.3 # via -r desktop/requirements-desktop.txt distro==1.9.0 @@ -155,6 +154,7 @@ grpcio==1.83.0 h11==0.16.0 # via # httpcore + # httpcore2 # uvicorn # wsproto h2==4.4.0 @@ -165,6 +165,8 @@ hpack==4.2.0 # via h2 httpcore==1.0.9 # via httpx +httpcore2==2.16.0 + # via httpx2 httpx==0.28.1 # via # -r desktop/requirements-desktop.txt @@ -175,15 +177,16 @@ httpx==0.28.1 # huggingface-hub # lark-oapi # litellm - # mcp # mem0ai # ollama # openai # qdrant-client # tavily-python httpx-sse==0.4.3 + # via dashscope +httpx2==2.16.0 # via - # dashscope + # agentscope # mcp huggingface-hub==1.26.0 # via tokenizers @@ -193,6 +196,7 @@ idna==3.18 # via # anyio # httpx + # httpx2 # requests # yarl importlib-metadata==8.9.0 @@ -205,7 +209,7 @@ jiter==0.16.0 # via # anthropic # openai -json-repair==0.61.7 +json-repair==0.64.1 # via agentscope json5==0.15.0 # via agentscope @@ -213,6 +217,7 @@ jsonschema==4.26.0 # via # -r desktop/requirements-desktop.txt # agentscope + # json-repair # litellm # mcp jsonschema-specifications==2025.9.1 @@ -221,8 +226,6 @@ kiwisolver==1.5.0 # via matplotlib lark-oapi==1.7.1 # via -r desktop/requirements-desktop.txt -limits==5.8.0 - # via slowapi litellm==1.94.1 # via -r desktop/requirements-desktop.txt lxml==6.1.1 @@ -250,10 +253,12 @@ matplotlib==3.11.1 # via # -r desktop/requirements-desktop.txt # seaborn -mcp==1.29.0 +mcp==2.3.0 # via # -r desktop/requirements-desktop.txt # agentscope +mcp-types==2.3.0 + # via mcp mdurl==0.1.2 # via markdown-it-py mem0ai==2.0.14 @@ -294,6 +299,7 @@ openpyxl==3.1.5 opentelemetry-api==1.44.0 # via # agentscope + # mcp # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http # opentelemetry-sdk @@ -326,7 +332,6 @@ packaging==26.2 # via # faiss-cpu # huggingface-hub - # limits # matplotlib # onnxruntime # pikepdf @@ -388,17 +393,16 @@ pydantic==2.13.4 # anthropic # fastapi # google-genai + # json-repair # litellm # mcp + # mcp-types # mem0ai # ollama # openai - # pydantic-settings # qdrant-client pydantic-core==2.46.4 # via pydantic -pydantic-settings==2.14.2 - # via mcp pyee==13.0.1 # via playwright pygments==2.20.0 @@ -410,7 +414,9 @@ pymilvus==2.5.18 pyparsing==3.3.2 # via matplotlib pypdf==6.14.2 - # via -r desktop/requirements-desktop.txt + # via + # -r desktop/requirements-desktop.txt + # agentscope pypdf2==3.0.1 # via -r desktop/requirements-desktop.txt pypdfium2==5.12.1 @@ -429,7 +435,6 @@ python-dotenv==1.2.2 # -r desktop/requirements-desktop.txt # litellm # magika - # pydantic-settings # pymilvus python-engineio==4.13.3 # via python-socketio @@ -485,10 +490,9 @@ requests-toolbelt==1.0.0 # via lark-oapi rich==15.0.0 # via + # agentscope # dashscope # typer -ripgrep==14.1.0 - # via agentscope rpds-py==2026.6.3 # via # jsonschema @@ -513,8 +517,6 @@ six==1.17.0 # via # markdownify # python-dateutil -slowapi==0.1.10 - # via -r desktop/requirements-desktop.txt sniffio==1.3.1 # via # anthropic @@ -528,7 +530,9 @@ sqlalchemy==2.0.51 # alembic # mem0ai sse-starlette==3.0.3 - # via mcp + # via + # -r desktop/requirements-desktop.txt + # mcp starlette==0.46.2 # via # fastapi @@ -556,6 +560,10 @@ tree-sitter==0.26.0 # via agentscope tree-sitter-bash==0.25.1 # via agentscope +truststore==0.10.5 + # via + # httpcore2 + # httpx2 typer==0.27.0 # via dashscope typing-extensions==4.16.0 @@ -569,9 +577,10 @@ typing-extensions==4.16.0 # fastapi # google-genai # grpcio + # httpx2 # huggingface-hub - # limits # mcp + # mcp-types # openai # opentelemetry-api # opentelemetry-exporter-otlp-proto-grpc @@ -592,7 +601,8 @@ typing-inspection==0.4.2 # via # mcp # pydantic - # pydantic-settings +tzdata==2026.5 + # via agentscope ujson==5.13.0 # via pymilvus urllib3==2.7.0 @@ -616,8 +626,6 @@ werkzeug==3.1.8 # via -r desktop/requirements-desktop.txt wheel==0.47.0 # via -r desktop/requirements-desktop.txt -wrapt==2.3.0 - # via deprecated wsproto==1.3.2 # via simple-websocket xlsxwriter==3.2.9 diff --git a/desktop/requirements-desktop-macos-aarch64-py311.lock b/desktop/requirements-desktop-macos-aarch64-py311.lock index 887a7ba2..9df1e05d 100644 --- a/desktop/requirements-desktop-macos-aarch64-py311.lock +++ b/desktop/requirements-desktop-macos-aarch64-py311.lock @@ -1,8 +1,8 @@ # This file is generated. Do not edit it by hand. # Regenerate with: npm --prefix desktop run lock:desktop -# input-sha256: 207ac56d3accd481bbd7d5fdd7de4ab076f3184f1dc6c7d0bd6de1d1b958d38a +# input-sha256: 2580e67756923a30f416529f071ff7372044a681fb55cd05ef896e558846dd73 # Target: CPython 3.11 on macOS Apple Silicon -agentscope==2.0.0 +agentscope @ https://github.com/agentscope-ai/agentscope/archive/ed357feb78a45ff02df155bb5610eab9bf543832.tar.gz # via -r desktop/requirements-desktop.txt aiofiles==25.1.0 # via agentscope @@ -31,6 +31,7 @@ anyio==4.14.2 # anthropic # google-genai # httpx + # httpx2 # mcp # openai # sse-starlette @@ -101,9 +102,7 @@ defusedxml==0.7.1 # -r desktop/requirements-desktop.txt # markitdown deprecated==1.3.1 - # via - # limits - # pikepdf + # via pikepdf dingtalk-stream==0.24.3 # via -r desktop/requirements-desktop.txt distro==1.9.0 @@ -157,6 +156,7 @@ grpcio==1.83.0 h11==0.16.0 # via # httpcore + # httpcore2 # uvicorn # wsproto h2==4.4.0 @@ -167,6 +167,8 @@ hpack==4.2.0 # via h2 httpcore==1.0.9 # via httpx +httpcore2==2.16.0 + # via httpx2 httpx==0.28.1 # via # -r desktop/requirements-desktop.txt @@ -177,15 +179,16 @@ httpx==0.28.1 # huggingface-hub # lark-oapi # litellm - # mcp # mem0ai # ollama # openai # qdrant-client # tavily-python httpx-sse==0.4.3 + # via dashscope +httpx2==2.16.0 # via - # dashscope + # agentscope # mcp huggingface-hub==1.26.0 # via tokenizers @@ -197,6 +200,7 @@ idna==3.18 # via # anyio # httpx + # httpx2 # requests # yarl importlib-metadata==8.9.0 @@ -209,7 +213,7 @@ jiter==0.16.0 # via # anthropic # openai -json-repair==0.61.7 +json-repair==0.64.1 # via agentscope json5==0.15.0 # via agentscope @@ -217,6 +221,7 @@ jsonschema==4.26.0 # via # -r desktop/requirements-desktop.txt # agentscope + # json-repair # litellm # mcp jsonschema-specifications==2025.9.1 @@ -225,8 +230,6 @@ kiwisolver==1.5.0 # via matplotlib lark-oapi==1.7.1 # via -r desktop/requirements-desktop.txt -limits==5.8.0 - # via slowapi litellm==1.91.4 # via -r desktop/requirements-desktop.txt lxml==6.1.1 @@ -254,10 +257,12 @@ matplotlib==3.11.1 # via # -r desktop/requirements-desktop.txt # seaborn -mcp==1.29.0 +mcp==2.3.0 # via # -r desktop/requirements-desktop.txt # agentscope +mcp-types==2.3.0 + # via mcp mdurl==0.1.2 # via markdown-it-py mem0ai==2.0.14 @@ -300,6 +305,7 @@ openpyxl==3.1.5 opentelemetry-api==1.44.0 # via # agentscope + # mcp # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http # opentelemetry-sdk @@ -332,7 +338,6 @@ packaging==26.2 # via # faiss-cpu # huggingface-hub - # limits # matplotlib # onnxruntime # pikepdf @@ -396,17 +401,16 @@ pydantic==2.13.4 # anthropic # fastapi # google-genai + # json-repair # litellm # mcp + # mcp-types # mem0ai # ollama # openai - # pydantic-settings # qdrant-client pydantic-core==2.46.4 # via pydantic -pydantic-settings==2.14.2 - # via mcp pyee==13.0.1 # via playwright pygments==2.20.0 @@ -418,7 +422,9 @@ pymilvus==2.5.18 pyparsing==3.3.2 # via matplotlib pypdf==6.14.2 - # via -r desktop/requirements-desktop.txt + # via + # -r desktop/requirements-desktop.txt + # agentscope pypdf2==3.0.1 # via -r desktop/requirements-desktop.txt pypdfium2==5.12.1 @@ -437,7 +443,6 @@ python-dotenv==1.2.2 # -r desktop/requirements-desktop.txt # litellm # magika - # pydantic-settings # pymilvus python-engineio==4.13.3 # via python-socketio @@ -493,10 +498,9 @@ requests-toolbelt==1.0.0 # via lark-oapi rich==15.0.0 # via + # agentscope # dashscope # typer -ripgrep==15.1.0 - # via agentscope rpds-py==2026.6.3 # via # jsonschema @@ -521,8 +525,6 @@ six==1.17.0 # via # markdownify # python-dateutil -slowapi==0.1.10 - # via -r desktop/requirements-desktop.txt sniffio==1.3.1 # via # anthropic @@ -536,7 +538,9 @@ sqlalchemy==2.0.51 # alembic # mem0ai sse-starlette==3.0.3 - # via mcp + # via + # -r desktop/requirements-desktop.txt + # mcp starlette==0.46.2 # via # fastapi @@ -566,6 +570,10 @@ tree-sitter==0.26.0 # via agentscope tree-sitter-bash==0.25.1 # via agentscope +truststore==0.10.5 + # via + # httpcore2 + # httpx2 typer==0.27.0 # via dashscope typing-extensions==4.16.0 @@ -579,9 +587,10 @@ typing-extensions==4.16.0 # fastapi # google-genai # grpcio + # httpx2 # huggingface-hub - # limits # mcp + # mcp-types # openai # opentelemetry-api # opentelemetry-exporter-otlp-proto-grpc @@ -602,7 +611,8 @@ typing-inspection==0.4.2 # via # mcp # pydantic - # pydantic-settings +tzdata==2026.5 + # via agentscope ujson==5.13.0 # via pymilvus urllib3==2.7.0 diff --git a/desktop/requirements-desktop-macos-x86_64-py311.lock b/desktop/requirements-desktop-macos-x86_64-py311.lock index 4118f46d..39a17b34 100644 --- a/desktop/requirements-desktop-macos-x86_64-py311.lock +++ b/desktop/requirements-desktop-macos-x86_64-py311.lock @@ -1,8 +1,8 @@ # This file is generated. Do not edit it by hand. # Regenerate with: npm --prefix desktop run lock:desktop -# input-sha256: 207ac56d3accd481bbd7d5fdd7de4ab076f3184f1dc6c7d0bd6de1d1b958d38a +# input-sha256: 2580e67756923a30f416529f071ff7372044a681fb55cd05ef896e558846dd73 # Target: CPython 3.11 on macOS Intel -agentscope==2.0.0 +agentscope @ https://github.com/agentscope-ai/agentscope/archive/ed357feb78a45ff02df155bb5610eab9bf543832.tar.gz # via -r desktop/requirements-desktop.txt aiofiles==25.1.0 # via agentscope @@ -31,6 +31,7 @@ anyio==4.14.2 # anthropic # google-genai # httpx + # httpx2 # mcp # openai # sse-starlette @@ -101,9 +102,7 @@ defusedxml==0.7.1 # -r desktop/requirements-desktop.txt # markitdown deprecated==1.3.1 - # via - # limits - # pikepdf + # via pikepdf dingtalk-stream==0.24.3 # via -r desktop/requirements-desktop.txt distro==1.9.0 @@ -159,6 +158,7 @@ grpcio==1.83.0 h11==0.16.0 # via # httpcore + # httpcore2 # uvicorn # wsproto h2==4.4.0 @@ -169,6 +169,8 @@ hpack==4.2.0 # via h2 httpcore==1.0.9 # via httpx +httpcore2==2.16.0 + # via httpx2 httpx==0.28.1 # via # -r desktop/requirements-desktop.txt @@ -179,15 +181,16 @@ httpx==0.28.1 # huggingface-hub # lark-oapi # litellm - # mcp # mem0ai # ollama # openai # qdrant-client # tavily-python httpx-sse==0.4.3 + # via dashscope +httpx2==2.16.0 # via - # dashscope + # agentscope # mcp huggingface-hub==1.26.0 # via tokenizers @@ -199,6 +202,7 @@ idna==3.18 # via # anyio # httpx + # httpx2 # requests # yarl importlib-metadata==8.9.0 @@ -211,7 +215,7 @@ jiter==0.16.0 # via # anthropic # openai -json-repair==0.61.7 +json-repair==0.64.1 # via agentscope json5==0.15.0 # via agentscope @@ -219,6 +223,7 @@ jsonschema==4.26.0 # via # -r desktop/requirements-desktop.txt # agentscope + # json-repair # litellm # mcp jsonschema-specifications==2025.9.1 @@ -227,8 +232,6 @@ kiwisolver==1.5.0 # via matplotlib lark-oapi==1.7.1 # via -r desktop/requirements-desktop.txt -limits==5.8.0 - # via slowapi litellm==1.91.4 # via -r desktop/requirements-desktop.txt lxml==6.1.1 @@ -256,10 +259,12 @@ matplotlib==3.11.1 # via # -r desktop/requirements-desktop.txt # seaborn -mcp==1.29.0 +mcp==2.3.0 # via # -r desktop/requirements-desktop.txt # agentscope +mcp-types==2.3.0 + # via mcp mdurl==0.1.2 # via markdown-it-py mem0ai==2.0.14 @@ -302,6 +307,7 @@ openpyxl==3.1.5 opentelemetry-api==1.44.0 # via # agentscope + # mcp # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http # opentelemetry-sdk @@ -334,7 +340,6 @@ packaging==26.2 # via # faiss-cpu # huggingface-hub - # limits # matplotlib # onnxruntime # pikepdf @@ -398,17 +403,16 @@ pydantic==2.13.4 # anthropic # fastapi # google-genai + # json-repair # litellm # mcp + # mcp-types # mem0ai # ollama # openai - # pydantic-settings # qdrant-client pydantic-core==2.46.4 # via pydantic -pydantic-settings==2.14.2 - # via mcp pyee==13.0.1 # via playwright pygments==2.20.0 @@ -420,7 +424,9 @@ pymilvus==2.5.18 pyparsing==3.3.2 # via matplotlib pypdf==6.14.2 - # via -r desktop/requirements-desktop.txt + # via + # -r desktop/requirements-desktop.txt + # agentscope pypdf2==3.0.1 # via -r desktop/requirements-desktop.txt pypdfium2==5.12.1 @@ -439,7 +445,6 @@ python-dotenv==1.2.2 # -r desktop/requirements-desktop.txt # litellm # magika - # pydantic-settings # pymilvus python-engineio==4.13.3 # via python-socketio @@ -495,10 +500,9 @@ requests-toolbelt==1.0.0 # via lark-oapi rich==15.0.0 # via + # agentscope # dashscope # typer -ripgrep==14.1.0 - # via agentscope rpds-py==2026.6.3 # via # jsonschema @@ -523,8 +527,6 @@ six==1.17.0 # via # markdownify # python-dateutil -slowapi==0.1.10 - # via -r desktop/requirements-desktop.txt sniffio==1.3.1 # via # anthropic @@ -538,7 +540,9 @@ sqlalchemy==2.0.51 # alembic # mem0ai sse-starlette==3.0.3 - # via mcp + # via + # -r desktop/requirements-desktop.txt + # mcp starlette==0.46.2 # via # fastapi @@ -568,6 +572,10 @@ tree-sitter==0.26.0 # via agentscope tree-sitter-bash==0.25.1 # via agentscope +truststore==0.10.5 + # via + # httpcore2 + # httpx2 typer==0.27.0 # via dashscope typing-extensions==4.16.0 @@ -581,9 +589,10 @@ typing-extensions==4.16.0 # fastapi # google-genai # grpcio + # httpx2 # huggingface-hub - # limits # mcp + # mcp-types # openai # opentelemetry-api # opentelemetry-exporter-otlp-proto-grpc @@ -604,7 +613,8 @@ typing-inspection==0.4.2 # via # mcp # pydantic - # pydantic-settings +tzdata==2026.5 + # via agentscope ujson==5.13.0 # via pymilvus urllib3==2.7.0 diff --git a/desktop/requirements-desktop-windows-py311.lock b/desktop/requirements-desktop-windows-py311.lock index 36895fe1..48953834 100644 --- a/desktop/requirements-desktop-windows-py311.lock +++ b/desktop/requirements-desktop-windows-py311.lock @@ -1,8 +1,8 @@ # This file is generated. Do not edit it by hand. # Regenerate with: npm --prefix desktop run lock:windows -# input-sha256: d18da0b813b601a5c727eb280b730ddb3b307fb487ab2a2bb28d4c568a336990 +# input-sha256: b9680ef14192ea099c4a7159c08c064cbd555bd681c00c49c5c74b1f7f3d4046 # Target: CPython 3.11 on Windows x86_64 -agentscope==2.0.0 +agentscope @ https://github.com/agentscope-ai/agentscope/archive/ed357feb78a45ff02df155bb5610eab9bf543832.tar.gz # via -r desktop/requirements-desktop.txt aiofiles==25.1.0 # via agentscope @@ -31,6 +31,7 @@ anyio==4.14.2 # anthropic # google-genai # httpx + # httpx2 # mcp # openai # sse-starlette @@ -106,8 +107,6 @@ defusedxml==0.7.1 # via # -r desktop/requirements-desktop.txt # markitdown -deprecated==1.3.1 - # via limits dingtalk-stream==0.24.3 # via -r desktop/requirements-desktop.txt distro==1.9.0 @@ -163,6 +162,7 @@ grpcio==1.83.0 h11==0.16.0 # via # httpcore + # httpcore2 # uvicorn # wsproto h2==4.4.0 @@ -173,6 +173,8 @@ hpack==4.2.0 # via h2 httpcore==1.0.9 # via httpx +httpcore2==2.16.0 + # via httpx2 httpx==0.28.1 # via # -r desktop/requirements-desktop.txt @@ -183,15 +185,16 @@ httpx==0.28.1 # huggingface-hub # lark-oapi # litellm - # mcp # mem0ai # ollama # openai # qdrant-client # tavily-python httpx-sse==0.4.3 + # via dashscope +httpx2==2.16.0 # via - # dashscope + # agentscope # mcp huggingface-hub==1.26.0 # via tokenizers @@ -203,6 +206,7 @@ idna==3.18 # via # anyio # httpx + # httpx2 # requests # yarl importlib-metadata==8.9.0 @@ -215,7 +219,7 @@ jiter==0.16.0 # via # anthropic # openai -json-repair==0.61.7 +json-repair==0.64.1 # via agentscope json5==0.15.0 # via agentscope @@ -223,6 +227,7 @@ jsonschema==4.26.0 # via # -r desktop/requirements-desktop.txt # agentscope + # json-repair # litellm # mcp jsonschema-specifications==2025.9.1 @@ -231,8 +236,6 @@ kiwisolver==1.5.0 # via matplotlib lark-oapi==1.7.1 # via -r desktop/requirements-desktop.txt -limits==5.8.0 - # via slowapi litellm==1.94.1 # via -r desktop/requirements-desktop.txt lxml==6.1.1 @@ -260,10 +263,12 @@ matplotlib==3.11.1 # via # -r desktop/requirements-desktop.txt # seaborn -mcp==1.29.0 +mcp==2.3.0 # via # -r desktop/requirements-desktop.txt # agentscope +mcp-types==2.3.0 + # via mcp mdurl==0.1.2 # via markdown-it-py mem0ai==2.0.14 @@ -304,6 +309,7 @@ openpyxl==3.1.5 opentelemetry-api==1.44.0 # via # agentscope + # mcp # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http # opentelemetry-sdk @@ -336,7 +342,6 @@ packaging==26.2 # via # faiss-cpu # huggingface-hub - # limits # matplotlib # onnxruntime # pikepdf @@ -398,17 +403,16 @@ pydantic==2.13.4 # anthropic # fastapi # google-genai + # json-repair # litellm # mcp + # mcp-types # mem0ai # ollama # openai - # pydantic-settings # qdrant-client pydantic-core==2.46.4 # via pydantic -pydantic-settings==2.14.2 - # via mcp pyee==13.0.1 # via playwright pygments==2.20.0 @@ -420,7 +424,9 @@ pymilvus==2.5.18 pyparsing==3.3.2 # via matplotlib pypdf==6.14.2 - # via -r desktop/requirements-desktop.txt + # via + # -r desktop/requirements-desktop.txt + # agentscope pypdf2==3.0.1 # via -r desktop/requirements-desktop.txt pypdfium2==5.12.1 @@ -441,7 +447,6 @@ python-dotenv==1.2.2 # -r desktop/requirements-desktop.txt # litellm # magika - # pydantic-settings # pymilvus python-engineio==4.13.3 # via python-socketio @@ -501,10 +506,9 @@ requests-toolbelt==1.0.0 # via lark-oapi rich==15.0.0 # via + # agentscope # dashscope # typer -ripgrep==14.1.0 - # via agentscope rpds-py==2026.6.3 # via # jsonschema @@ -529,8 +533,6 @@ six==1.17.0 # via # markdownify # python-dateutil -slowapi==0.1.10 - # via -r desktop/requirements-desktop.txt sniffio==1.3.1 # via # anthropic @@ -544,7 +546,9 @@ sqlalchemy==2.0.51 # alembic # mem0ai sse-starlette==3.0.3 - # via mcp + # via + # -r desktop/requirements-desktop.txt + # mcp starlette==0.46.2 # via # fastapi @@ -574,6 +578,10 @@ tree-sitter==0.26.0 # via agentscope tree-sitter-bash==0.25.1 # via agentscope +truststore==0.10.5 + # via + # httpcore2 + # httpx2 typer==0.27.0 # via dashscope typing-extensions==4.16.0 @@ -587,9 +595,10 @@ typing-extensions==4.16.0 # fastapi # google-genai # grpcio + # httpx2 # huggingface-hub - # limits # mcp + # mcp-types # openai # opentelemetry-api # opentelemetry-exporter-otlp-proto-grpc @@ -610,9 +619,10 @@ typing-inspection==0.4.2 # via # mcp # pydantic - # pydantic-settings tzdata==2026.3 - # via pandas + # via + # agentscope + # pandas ujson==5.13.0 # via pymilvus urllib3==2.7.0 @@ -636,8 +646,6 @@ werkzeug==3.1.8 # via -r desktop/requirements-desktop.txt wheel==0.47.0 # via -r desktop/requirements-desktop.txt -wrapt==2.3.0 - # via deprecated wsproto==1.3.2 # via simple-websocket xlsxwriter==3.2.9 diff --git a/desktop/requirements-desktop.txt b/desktop/requirements-desktop.txt index e2937d7a..9b2d2192 100644 --- a/desktop/requirements-desktop.txt +++ b/desktop/requirements-desktop.txt @@ -18,17 +18,19 @@ uv==0.11.33 # Core web service and orchestration. fastapi==0.115.12 uvicorn>=0.27.0 -pydantic>=2.6.0 -anyio>=4.2.0 +pydantic>=2.12.0 +anyio>=4.9.0 python-dotenv>=1.0.0 requests>=2.31.0 httpx>=0.27.0 jsonschema>=4.20.0 -agentscope==2.0.0 +agentscope @ https://github.com/agentscope-ai/agentscope/archive/ed357feb78a45ff02df155bb5610eab9bf543832.tar.gz litellm>=1.50.0 google-genai>=0.3.0 ollama>=0.3.0 -mcp>=1.0.0,<2.0.0 +mcp==2.3.0 +# Keep the SSE dependency within FastAPI 0.115 Starlette constraints. +sse-starlette==3.0.3 tavily-python>=0.5.0 beautifulsoup4>=4.12.0 @@ -40,7 +42,6 @@ redis>=5.0.0 werkzeug>=3.0.0 python-multipart>=0.0.9 structlog>=23.2.0 -slowapi>=0.1.9 argon2-cffi>=23.1.0 croniter>=2.0.0 pytz>=2024.1 diff --git a/desktop/scripts/build-runtime.mjs b/desktop/scripts/build-runtime.mjs index bb4a1430..dc159fc4 100644 --- a/desktop/scripts/build-runtime.mjs +++ b/desktop/scripts/build-runtime.mjs @@ -112,6 +112,9 @@ export function buildDesktopRuntime({ desktopDir, repoRoot, sourceRoot, python } "--break-system-packages", "--only-binary", ":all:", + // AgentScope is pinned to a reviewed source commit; other packages require wheels. + "--no-binary", + "agentscope", join(repoRoot, config.lockFile), ], { cwd: repoRoot }, diff --git a/desktop/scripts/desktop-dependencies.test.mjs b/desktop/scripts/desktop-dependencies.test.mjs index 2f28f373..c6540fbc 100644 --- a/desktop/scripts/desktop-dependencies.test.mjs +++ b/desktop/scripts/desktop-dependencies.test.mjs @@ -18,11 +18,26 @@ import { WINDOWS_DESKTOP_LOCK_FILE, desktopDependencyFingerprint, readAndValidateWindowsDesktopLock, + readAndValidateDesktopLock, } from "./desktop-dependencies.mjs"; const desktopDir = resolve(dirname(fileURLToPath(import.meta.url)), ".."); const repoRoot = resolve(desktopDir, ".."); +function assertExactLock(lock) { + const source = readFileSync(join(repoRoot, DESKTOP_REQUIREMENTS_FILE), "utf8") + .match(/^agentscope @ https:\/\/github\.com\/agentscope-ai\/agentscope\/archive\/[a-f0-9]{40}\.tar\.gz$/m)?.[0]; + assert.ok(source, "AgentScope must use an immutable commit archive"); + assert.ok(readFileSync(join(repoRoot, "requirements.txt"), "utf8").split(/\r?\n/).includes(source)); + assert.ok(lock.split(/\r?\n/).includes(source)); + for (const line of lock.split(/\r?\n/)) { + if (/^[A-Za-z0-9_.-]+/.test(line) && line !== source) { + assert.match(line, /^[A-Za-z0-9_.-]+==[^\s]+$/); + } + } + assert.match(lock, /^mcp==2\.3\.0$/m); +} + function requirementNames(contents) { return new Set( contents @@ -73,14 +88,7 @@ test("Windows Python 3.11 lock is exact and matches its desktop input", () => { const lock = readAndValidateWindowsDesktopLock(repoRoot); const names = requirementNames(lock); assert.ok(names.size > 100, "the transitive desktop lock must be complete"); - for (const line of lock.split(/\r?\n/)) { - if (/^[A-Za-z0-9_.-]+/.test(line)) { - assert.match(line, /^[A-Za-z0-9_.-]+==[^\s]+$/); - } - } - const mcpVersion = lock.match(/^mcp==([^\s]+)$/m)?.[1]; - assert.ok(mcpVersion, "the lock must contain mcp"); - assert.equal(Number(mcpVersion.split(".")[0]), 1); + assertExactLock(lock); assert.match(lock, /^pymilvus==2\.5\.18$/m); assert.match(lock, /^milvus-lite==3\.2\.2$/m); // Some platform locks pull boto3 through LiteLLM; it is not a storage backend. @@ -124,14 +132,9 @@ test("desktop dependency hash is independent of checkout line endings", () => { test("all supported desktop targets have exact Python 3.11 locks", () => { for (const [target, config] of Object.entries(DESKTOP_TARGETS)) { - const lock = readFileSync(join(repoRoot, config.lockFile), "utf8"); - assert.match(lock, /# input-sha256: [a-f0-9]{64}/); + const lock = readAndValidateDesktopLock(repoRoot, target); assert.ok(requirementNames(lock).size > 100, `${target} lock is incomplete`); - for (const line of lock.split(/\r?\n/)) { - if (/^[A-Za-z0-9_.-]+/.test(line)) { - assert.match(line, /^[A-Za-z0-9_.-]+==[^\s]+$/); - } - } + assertExactLock(lock); } }); diff --git a/desktop/scripts/test-native-tools.py b/desktop/scripts/test-native-tools.py index fa8e40ee..9e076108 100644 --- a/desktop/scripts/test-native-tools.py +++ b/desktop/scripts/test-native-tools.py @@ -174,7 +174,7 @@ def test_backend_prefers_relocated_private_tools_without_host_path(self): import importlib.util from unittest.mock import patch spec = importlib.util.spec_from_file_location( - "office_helpers", SCRIPT.parents[2] / "src/backend/core/content/office.py") + "office_helpers", SCRIPT.parents[2] / "src/backend/core/files/content/office.py") office = importlib.util.module_from_spec(spec) spec.loader.exec_module(office) with tempfile.TemporaryDirectory() as folder: diff --git a/docker-compose.application-hosting.yml b/docker-compose.application-hosting.yml index 74c1e7c7..46517f82 100644 --- a/docker-compose.application-hosting.yml +++ b/docker-compose.application-hosting.yml @@ -29,7 +29,7 @@ services: target: production volumes: - ./src/backend:/app/src/backend:ro - command: ["python", "-m", "core.services.application_hosting_setup"] + command: ["python", "-m", "features.hosted_apps.hosting_setup"] environment: APPLICATION_DATABASE_URL: postgresql://application_owner:${APPLICATION_OWNER_DB_PASSWORD:?Set a separate APPLICATION_OWNER_DB_PASSWORD}@application-postgres:5432/applications APPLICATION_PROVISION_DATABASE_URL: postgresql://application_admin:${APPLICATION_DB_PASSWORD}@application-postgres:5432/applications diff --git a/docker-compose.yml b/docker-compose.yml index c8d35b9b..0446ccf6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -128,8 +128,6 @@ services: S3_ACCESS_KEY: ${S3_ACCESS_KEY:-} S3_SECRET_KEY: ${S3_SECRET_KEY:-} CORS_ORIGINS: ${CORS_ORIGINS:-http://localhost:3000,http://localhost:5173} - RATE_LIMIT_ENABLED: ${RATE_LIMIT_ENABLED:-true} - RATE_LIMIT_DEFAULT: ${RATE_LIMIT_DEFAULT:-100/minute} LOG_LEVEL: ${LOG_LEVEL:-INFO} LOG_FORMAT: ${LOG_FORMAT:-json} LOG_TO_FILE: ${LOG_TO_FILE:-false} @@ -438,6 +436,7 @@ services: environment: TZ: ${TZ:-Asia/Shanghai} OPENSANDBOX_INSECURE_SERVER: 'NO' + OPENSANDBOX_POOL_MAX_TOTAL: ${OPENSANDBOX_POOL_MAX_TOTAL:-20} volumes: - /var/run/docker.sock:/var/run/docker.sock - opensandbox_config:/etc/opensandbox:ro diff --git a/docker/Dockerfile.opensandbox-server b/docker/Dockerfile.opensandbox-server index 95b92d71..80e20456 100644 --- a/docker/Dockerfile.opensandbox-server +++ b/docker/Dockerfile.opensandbox-server @@ -2,6 +2,7 @@ FROM opensandbox/server:v0.1.13 USER root RUN apt-get update && apt-get install -y --no-install-recommends s3fs fuse && rm -rf /var/lib/apt/lists/* +COPY docker/opensandbox_capacity.py /app/opensandbox_server/services/docker/capacity.py COPY docker/opensandbox_expirations.py /app/opensandbox_server/services/docker/expirations.py COPY docker/opensandbox_mount_worker.py /app/opensandbox_server/services/docker/mount_worker.py COPY src/backend/core/sandbox/sessions/metadata.py /app/opensandbox_server/services/docker/sandbox_metadata.py diff --git a/docker/Dockerfile.opensandbox-server.dockerignore b/docker/Dockerfile.opensandbox-server.dockerignore index 2b3b7e78..65a96d21 100644 --- a/docker/Dockerfile.opensandbox-server.dockerignore +++ b/docker/Dockerfile.opensandbox-server.dockerignore @@ -1,5 +1,6 @@ ** !docker/ +!docker/opensandbox_capacity.py !docker/patch_opensandbox_server.py !docker/opensandbox_expirations.py !docker/opensandbox_mount_worker.py diff --git a/docker/Dockerfile.personal-agent b/docker/Dockerfile.personal-agent index 3eecda26..5a70f2c7 100644 --- a/docker/Dockerfile.personal-agent +++ b/docker/Dockerfile.personal-agent @@ -17,7 +17,7 @@ RUN install -m 755 /opt/personal-agent/desktop/pa-browser /usr/local/bin/pa-brow && install -m 644 /opt/personal-agent/desktop/mimeapps.list /etc/xdg/mimeapps.list \ && install -m 755 /opt/personal-agent/desktop/pa-stream.py /usr/local/bin/pa-stream RUN useradd --create-home --uid 1001 --shell /bin/bash pa-desktop && chmod 700 /home/pa-desktop && mkdir -p /opt/personal-agent -COPY src/backend/core/device_link/driver.py /opt/personal-agent/computer-driver.py +COPY src/backend/personal_agent/device_link/driver.py /opt/personal-agent/computer-driver.py COPY docker/personal-agent-entrypoint.sh /opt/personal-agent/start.sh RUN chmod 755 /opt/personal-agent/start.sh /opt/personal-agent/computer-driver.py diff --git a/docker/opensandbox_capacity.py b/docker/opensandbox_capacity.py new file mode 100644 index 00000000..8c1c0cbb --- /dev/null +++ b/docker/opensandbox_capacity.py @@ -0,0 +1,31 @@ +"""Serialize Docker admission, not sandbox initialization or mount recovery.""" +import fcntl +import hashlib +import os +from contextlib import contextmanager +from pathlib import Path + +from fastapi import HTTPException + + +@contextmanager +def sandbox_capacity(service): + # File locking also fences multiple controller processes sharing the volume. + # Docker inventory survives controller/backend restart; no reservation ledger. + root = Path("/var/lib/opensandbox") + root.mkdir(parents=True, exist_ok=True) + namespace = hashlib.sha256(service.network_mode.encode()).hexdigest()[:24] + limit = int(os.environ.get("OPENSANDBOX_POOL_MAX_TOTAL", "20")) + if limit < 1: + raise HTTPException(503, "Sandbox capacity must be positive") + with (root / ("capacity-" + namespace + ".lock")).open("a") as lock: + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + raise HTTPException(429, "Sandbox creation busy; retry later") from None + try: + if len(service._owned_sandboxes()) >= limit: + raise HTTPException(429, "Deployment sandbox capacity reached") + yield + finally: + fcntl.flock(lock, fcntl.LOCK_UN) diff --git a/docker/opensandbox_mount_recovery.py b/docker/opensandbox_mount_recovery.py index b7efa6b0..09c608a2 100644 --- a/docker/opensandbox_mount_recovery.py +++ b/docker/opensandbox_mount_recovery.py @@ -55,7 +55,7 @@ def _specifications(service, container, request): def _healthy(container, specs): if not specs: return True - script = """import json,os,sys + script = """import errno,json,os,sys from pathlib import Path mounts = {} for line in Path('/proc/self/mountinfo').read_text().splitlines(): @@ -63,14 +63,23 @@ def _healthy(container, specs): separator = fields.index('-') mounts[fields[4]] = (fields[separator + 1], fields[5].split(',')) for target, readonly in json.loads(sys.argv[1]): + if target not in mounts: + sys.exit(10) filesystem, options = mounts[target] - assert filesystem.startswith('fuse'), 'Object filesystem is absent' - assert ('ro' in options) == readonly, 'Mount access mode differs' - os.stat(target) + if not filesystem.startswith('fuse') or ('ro' in options) != readonly: + sys.exit(10) + try: + os.stat(target) + except OSError as exc: + sys.exit(10 if exc.errno == errno.ENOTCONN else 11) """ result = container.exec_run(["timeout", "4", "python3", "-c", script, json.dumps([(s["target"], s["read_only"]) for s in specs])], user="0") - return result.exit_code == 0 + if result.exit_code == 0: + return True + if result.exit_code == 10: + return False + raise HTTPException(503, "Object mount health is uncertain; existing connection retained") def remove_helper(service, sandbox_id): from docker.errors import NotFound @@ -127,7 +136,8 @@ def ensure_mounts(service, sandbox_id, request): if api.exec_inspect(execution["Id"])["ExitCode"] != 0 or not _healthy(container, specs): raise HTTPException(503, "Object mount recovery failed; workspace retained") except Exception: - remove_helper(service, sandbox_id) + # The helper may own live writes even if verification timed out. + # A later confirmed-broken probe can replace it safely. raise return {"epoch": EPOCH, "recovered": True} diff --git a/docker/patch_opensandbox_server.py b/docker/patch_opensandbox_server.py index 5460b21a..9bec849e 100644 --- a/docker/patch_opensandbox_server.py +++ b/docker/patch_opensandbox_server.py @@ -110,6 +110,8 @@ def patched_sources(root: Path) -> dict[str, str]: " async def create_sandbox(self, request: CreateSandboxRequest) -> CreateSandboxResponse:\n" " return await run_in_threadpool(self._create_sandbox_sync, request)\n\n" " def _create_sandbox_sync(self, request: CreateSandboxRequest) -> CreateSandboxResponse:\n") + source = replace_once(source, "import inspect\n", + "import inspect\nfrom .capacity import sandbox_capacity\n") sources[name] = source name = "api/lifecycle.py" @@ -152,6 +154,15 @@ def patched_sources(root: Path) -> dict[str, str]: patch_shared_lifecycle(sources) patch_durable_expirations(sources) patch_namespace_mounts(sources) + name = "services/docker/docker_service.py" + source = sources[name] + start = source.index(" created_container = self._create_and_start_container(") + end = source.index("\n )", start) + len("\n )") + block = source[start:end] + source = (source[:start] + " with sandbox_capacity(self):\n" + + "\n".join(" " + line for line in block.splitlines()) + source[end:]) + sources[name] = source + name = "services/docker/docker_service.py" sources[name] = replace_once(sources[name], "import inspect\n", "import inspect\nfrom .object_mounts import install as install_object_mounts\ninstall_object_mounts()\n") diff --git a/document/en/api/overview.md b/document/en/api/overview.md index d8c14af6..c29d2b93 100644 --- a/document/en/api/overview.md +++ b/document/en/api/overview.md @@ -59,7 +59,7 @@ The backend runs several parallel authentication mechanisms, each covering a dif | Mechanism | How to send the credential | Scope | Implementation | |---|---|---|---| | **Session cookie** | `Cookie: jx_session=` (set automatically on login) | All end-user-facing `/v1/*` endpoints (chats, projects, kb, memories, …) | `get_current_user`: cookie → Redis session lookup | -| **Personal API key** | `Authorization: Bearer sk-jx-...` | Same user identity as a session cookie; intended for scripts / third-party integrations | Recognized by the `sk-jx-` prefix; plaintext returned only once at creation, DB stores SHA256 only (`core/services/api_key_service.py`); self-managed at `/v1/me/api-keys` | +| **Personal API key** | `Authorization: Bearer sk-jx-...` | Same user identity as a session cookie; intended for scripts / third-party integrations | Recognized by the `sk-jx-` prefix; plaintext returned only once at creation, DB stores SHA256 only (`core/auth/api_keys.py`); self-managed at `/v1/me/api-keys` | | **Bearer token (mock/remote mode)** | `Authorization: Bearer ` | `AUTH_MODE=mock` (development: any token, or none, resolves to the mock user) or `remote` (verified against the user center) | Fallback branch of `get_current_user`; not accepted under `AUTH_MODE=session` | | **ADMIN_TOKEN** | `Authorization: Bearer ` | `/admin` console backend (`/v1/admin/skills`, `/v1/admin/kb`, …) and content-block write endpoints | `require_admin` (env var `ADMIN_TOKEN`) | | **CONFIG_TOKEN** | `Authorization: Bearer ` | `/config` console backend (`/v1/config/*`, `/v1/models`, `/v1/service-configs`, some `/v1/admin/*` observability endpoints) | `require_config` (env var `CONFIG_TOKEN`) | diff --git a/document/en/architecture/backend.md b/document/en/architecture/backend.md index 09e4841b..14ff3a4b 100644 --- a/document/en/architecture/backend.md +++ b/document/en/architecture/backend.md @@ -128,7 +128,7 @@ The root contains only the protocol, errors, factory, and package initializer. I The runner depends only on process and path primitives that do not load backend settings or databases. OpenSandbox-specific components stay with that provider; shared processes, paths, and account environment support serve both OpenSandbox and Cube. -### core/agent_skills — the skill engine +### core/capabilities/skills — the skill engine | File | Responsibility | |---|---| @@ -144,12 +144,12 @@ The runner depends only on process and path primitives that do not load backend | Submodule | Responsibilities and key files | |---|---| | `core/chat` | Workflow context assembly (`context.py`), SSE tool-log event construction (`tool_log.py`) | -| `core/content` | Attachment parsing (`file_parser.py`), KB document chunking/keywords/vectorization (`kb_processing.py`), upload validation (`file_validation.py`), artifact reading and summaries (`artifact_reader/refs/summary.py`), content-block import/export (`content_blocks.py`), `svg_fit.py` | +| `core/files/content` | Attachment parsing (`file_parser.py`), KB document chunking/keywords/vectorization (`kb_processing.py`), upload validation (`file_validation.py`), artifact reading and summaries (`artifact_reader/refs/summary.py`), content-block import/export (`content_blocks.py`), `svg_fit.py` | | `core/kb` / `edition_ee/kb` | Shared private-KB parsing and Milvus vector storage stay in `core/kb`; the Dify client and external retrieval adapter live only in `edition_ee/kb` (EE) | -| `core/artifacts` | Artifact store `store.py`: local / OSS dual mode | -| `core/infra` | Unified responses (`responses.py`), exceptions (`exceptions.py`), structured logging (`logging.py`), rate limiting (`rate_limit.py`), Redis singleton (`redis.py`), metrics (`metrics.py`), background-task registry (`runtime_state.py`), data masking (`data_masking.py`), distillation budget gates (`distillation_budget.py`, Enterprise Edition, EE) | +| `core/files/artifacts` | Artifact store `store.py`: local / OSS dual mode | +| `core/infra` | Unified responses (`responses.py`), exceptions (`exceptions.py`), structured logging (`logging.py`), Redis singleton (`redis.py`), metrics (`metrics.py`), background-task registry (`runtime_state.py`), data masking (`data_masking.py`), distillation budget gates (`distillation_budget.py`, Enterprise Edition, EE) | | `edition_ee/licensing` | Feature enums, signature verification, clock guard, seat policy, middleware, and the manager all exist only in EE; the CE tree has no License package and gets a fixed edition probe from its `api/middleware/edition.py` overlay | -| `core/storage` | Storage protocol `protocol.py` + factory `factory.py`; `local.py` (CE), `s3.py` / `oss.py` (Enterprise Edition, EE) | +| `core/files/storage` | Storage protocol `protocol.py` + factory `factory.py`; `local.py` (CE), `s3.py` / `oss.py` (Enterprise Edition, EE) | ## orchestration/ — the orchestration layer @@ -224,7 +224,7 @@ CE routes live under `api/routes/v1/`; physically split commercial routes live u | Sandbox protocol | `src/backend/core/sandbox/protocol.py` | | Memory pipeline | `src/backend/core/memory/pipeline.py` | | EE License implementation / CE edition middleware | `src/backend/edition_ee/licensing/`, `src/backend/api/middleware/edition.py` | -| Skill engine | `src/backend/core/agent_skills/loader.py` | +| Skill engine | `src/backend/core/capabilities/skills/loader.py` | | MCP registry and ports | `src/backend/extensions/mcp_servers/registry.py`, `_ports.py` | The space MCP interface lives in `extensions/mcp_servers/space_files_mcp/`. The registry separates installation placement from exposure. Hybrid desktops access spaces through the capability gateway; mutations use unified approval. See [space file mounts](../deployment/space-files.md). diff --git a/document/en/architecture/data-model.md b/document/en/architecture/data-model.md index 0c1a94c2..9d29d477 100644 --- a/document/en/architecture/data-model.md +++ b/document/en/architecture/data-model.md @@ -185,5 +185,5 @@ A few tables that *look* EE but are required by CE are deliberately excluded fro | CE/EE table boundary | `src/backend/core/db/edition_tables.py` | | EE migration chain | `src/backend/alembic/versions/` | | CE migration baseline | `ce/overlay/src/backend/alembic/versions/ce_0001_initial.py` | -| Prompt version pool service | `src/backend/core/services/prompt_version_service.py` | +| Prompt version pool service | `src/backend/core/llm/prompt_versions.py` | | Authoritative edition boundary | [Editions](../editions/overview.md) | diff --git a/document/en/architecture/overview.md b/document/en/architecture/overview.md index 9105c485..4d692e32 100644 --- a/document/en/architecture/overview.md +++ b/document/en/architecture/overview.md @@ -51,14 +51,14 @@ This page describes the system from five angles: the layered architecture, the o ┌──────────────────────────────▼─────────────────────────────────────┐ │ Agent layer (src/backend/core/llm/) │ │ factory/build → AgentScope 2.0 ReActAgent │ -│ ToolCollector → Toolkit (built-in tools + skills + MCP tools) │ +│ ToolAssembly → Toolkit (built-in tools + skills + MCP tools) │ │ middlewares: dynamic model / file context / system reminders │ │ offloader: oversized tool results spilled to sandbox │ │ prompts/prompt_runtime: system prompt assembly (DB-first) │ └───────┬──────────────────────┬──────────────────────┬──────────────┘ │ streamable-http │ sandbox protocol │ SQL / vectors ┌───────▼───────────┐ ┌────────▼──────────┐ ┌─────────▼─────────────┐ -│ MCP tool pool │ │ Sandboxes │ │ Data layer │ +│ MCP clients │ │ Sandboxes │ │ Data layer │ │ mcp container, │ │ script-runner(CE) │ │ PostgreSQL · Redis │ │ tool processes │ │ OpenSandbox CE/EE │ │ Milvus · Neo4j (opt.) │ │ see _ports.py │ │ CubeSandbox (EE) │ │ Local / S3 / OSS │ @@ -214,11 +214,11 @@ Every v1 endpoint returns `{ code, message, data, trace_id, timestamp }`, produc ### Prompts live in the database -System prompts are no longer hardcoded: the version pool is stored in the `content_blocks` table (`id=prompt_versions`), managed by `core/services/prompt_version_service.py` (CRUD, activation, first-boot seeding). At assembly time, `prompts/prompt_runtime.py` degrades through three levels — active DB version → file fallback under `prompts/prompt_text/default/system/` → a minimal hardcoded fallback — guaranteeing a non-empty prompt. Operators can canary-switch prompt versions from the admin console without a release. See [Prompt System](../modules/prompts.md). +System prompts are no longer hardcoded: the version pool is stored in the `content_blocks` table (`id=prompt_versions`), managed by `core/llm/prompt_versions.py` (CRUD, activation, first-boot seeding). At assembly time, `prompts/prompt_runtime.py` degrades through three levels — active DB version → file fallback under `prompts/prompt_text/default/system/` → a minimal hardcoded fallback — guaranteeing a non-empty prompt. Operators can canary-switch prompt versions from the admin console without a release. See [Prompt System](../modules/prompts.md). ### MCP tools as independent processes -Each tool is a standalone MCP server under `src/backend/extensions/mcp_servers//`, all running inside the dedicated `mcp` container as long-lived streamable-http listeners (the port table in `extensions/mcp_servers/_ports.py` is the single source of truth). The backend connects through the HTTP client pool in `core/llm/mcp_manager.py`. Benefits: a tool crash never affects the main process, document editing and export have moved into sandbox skills, and the tool tier can scale independently. CE keeps 7 general-purpose servers; the two industry tools, `query_database` and `ai_chain_information_mcp`, depend on intranet data sources and belong to the Enterprise Edition (EE). See [MCP Tools](../modules/mcp-tools.md). +Each tool is a standalone MCP server under `src/backend/extensions/mcp_servers//`, all running inside the dedicated `mcp` container as long-lived streamable-http listeners (the port table in `extensions/mcp_servers/_ports.py` is the single source of truth). The backend connects through the run-owned native client in `core/llm/mcp/lifetime.py`. Benefits: a tool crash never affects the main process, document editing and export have moved into sandbox skills, and the tool tier can scale independently. CE keeps 7 general-purpose servers; the two industry tools, `query_database` and `ai_chain_information_mcp`, depend on intranet data sources and belong to the Enterprise Edition (EE). See [MCP Tools](../modules/mcp-tools.md). ### Runs decoupled from connections diff --git a/document/en/deployment/environment-variables.md b/document/en/deployment/environment-variables.md index 27a293aa..cee9baf9 100644 --- a/document/en/deployment/environment-variables.md +++ b/document/en/deployment/environment-variables.md @@ -225,7 +225,7 @@ the database has no value. | `OPENSANDBOX_PORT` | Full repository: `8910`; CE: no mapping | Controller uses 8080 inside the network | | `OPENSANDBOX_POOL_JUPYTER_MIN_IDLE` / `MAX_IDLE` | `.env.example` 1/3; compose defaults 2/3 | User session pool (historical setting name); CE minimum is 0 | | `OPENSANDBOX_POOL_LIGHT_MIN_IDLE` / `MAX_IDLE` | `2` / `5` | Light bucket (one-shot executions) | -| `OPENSANDBOX_POOL_MAX_TOTAL` | CE: `4`; full: `20` | Pool ceiling (idle + in use) | +| `OPENSANDBOX_POOL_MAX_TOTAL` | CE: `4`; full: `20` | Controller-wide ceiling (all workers, warm and active instances); backend and controller must share this value | | `OPENSANDBOX_SNAPSHOT_ENABLED` | `true` | Snapshot persistence master switch (snapshot+kill when idle, restore on reconnect) | | `OPENSANDBOX_SNAPSHOT_RETENTION_DAYS` | `7` | Snapshot retention (GC sweeps expired rows) | | `OPENSANDBOX_SNAPSHOT_WAIT_TIMEOUT_S` | `120` | Maximum poll for snapshot readiness (s) | @@ -323,16 +323,10 @@ estimate only when it is not. See the [License mechanism](../editions/license.md) for details. -## Miscellaneous (rate limiting, circuit breakers, security, alerting) +## Miscellaneous (security, alerting) | Variable | Default | Description | Edition | |---|---|---|---| -| `RATE_LIMIT_ENABLED` | `true` | API rate limiting | CE | -| `RATE_LIMIT_STORAGE` | `memory://` | Rate-limit counter store (`redis://...` supported) | CE | -| `RATE_LIMIT_GLOBAL` / `RATE_LIMIT_PER_USER` | `500/minute` / `50/minute` | Global / per-user limits | CE | -| `CB_USER_CENTER_THRESHOLD` / `CB_USER_CENTER_TIMEOUT` | `5` / `60` | User-center circuit breaker | EE | -| `CB_MODEL_API_THRESHOLD` / `CB_MODEL_API_TIMEOUT` | `10` / `30` | Model-API circuit breaker | CE | -| `CB_STORAGE_THRESHOLD` / `CB_STORAGE_TIMEOUT` | `5` / `60` | Storage circuit breaker | CE | | `CORS_ORIGINS` | compose default `http://localhost:3000,http://localhost:5173` | Allowed cross-origin sources | CE | | `ENABLE_SECURITY_HEADERS` | (commented; recommended `true` in production) | Security response headers | CE | | `AUDIT_LOG_RETENTION_DAYS` / `AUDIT_LOG_EXPORT_ENABLED` | `90` / `true` (commented samples) | Audit-log retention / export | EE | diff --git a/document/en/deployment/offline-production.md b/document/en/deployment/offline-production.md index 2b8569c1..efe4f663 100644 --- a/document/en/deployment/offline-production.md +++ b/document/en/deployment/offline-production.md @@ -43,13 +43,9 @@ System prompts live in the database `content_blocks` table (the `prompt_versions ### 1. Export on the connected side -```bash -python src/backend/scripts/export_content.py \ - --api-url http://localhost:3001 --only prompts -# outputs src/backend/scripts/exported/prompts_snapshot_.json -``` - -(`--database-url` is also supported for direct-DB export; `ADMIN_TOKEN` is read from `.env` automatically.) +Export a snapshot through `GET /v1/content/prompts/export` using the source environment's +administrator or system configuration token. Save the response's `data` object as +`prompts_snapshot_.json`; do not save the complete response envelope. ### 2. Copy @@ -80,5 +76,5 @@ The API import invalidates the prompt cache automatically — **no backend resta | Complete EE image pack (deploy branch) | `deploy_kit/save_pack_ee.sh` | | Compiled EE images (deploy branch) | `docker/Dockerfile.backend-compiled`, `docker/Dockerfile.mcp-compiled` | | Compose overlays (deploy branch) | `docker-compose.prod.yml`, `docker-compose.deploy.yml`; runtime follows the delivered kit | -| Prompt export / import | `src/backend/scripts/export_content.py`, `src/backend/scripts/import_content.py` | -| Prompt version pool | `src/backend/core/services/prompt_version_service.py` | +| Prompt export / import | `src/backend/api/routes/v1/content.py` | +| Prompt version pool | `src/backend/core/llm/prompt_versions.py` | diff --git a/document/en/deployment/quick-install.md b/document/en/deployment/quick-install.md index 04d1175b..41256c45 100644 --- a/document/en/deployment/quick-install.md +++ b/document/en/deployment/quick-install.md @@ -187,5 +187,5 @@ The no-Docker single-machine mode is built to be lightweight. Here is how it dif | Frontend static hosting + `/api` bridge | `src/backend/api/local_hosting.py` | | MCP / sandbox subprocess supervision | `src/backend/orchestration/local_subprocess.py` | | In-process event log / ephemeral state | `src/backend/orchestration/run_event_stream.py`, `src/backend/core/infra/ephemeral.py` (enabled by leaving `REDIS_URL` empty) | -| Built-in MCP catalog seed | `src/backend/core/services/mcp_service.py` (`seed_builtin_mcp_servers_if_empty`) | +| Built-in MCP catalog seed | `src/backend/core/capabilities/mcp/service.py` (`seed_builtin_mcp_servers_if_empty`) | | Environment variables | [environment-variables.md](environment-variables.md) (the "No-Docker local mode" section) | diff --git a/document/en/deployment/space-files.md b/document/en/deployment/space-files.md index 6e2e1b55..ded1d526 100644 --- a/document/en/deployment/space-files.md +++ b/document/en/deployment/space-files.md @@ -36,15 +36,11 @@ Before switching an existing deployment, back up configuration and database. Copy valid file objects and attachments. Check each object's SHA-256. Keep source files. Switch STORAGE_TYPE and recreate the application containers after verification. -## File import +## Existing file migration -Back up data and pause writes. Preview one user's or team's import plan: - - PYTHONPATH=src/backend python -m scripts.import_space_files --user USER_ID - PYTHONPATH=src/backend python -m scripts.import_space_files --team TEAM_ID - -Review destinations and conflicts. Add --apply to import. -The tool verifies content and preserves source objects. +The standalone file import command has been removed. Back up the database and objects, +pause destination writes, and review a migration plan before moving existing files. +Check destinations, name conflicts, and content. Preserve source objects and references. ## Verification diff --git a/document/en/development/backend.md b/document/en/development/backend.md index 3ed2f541..d596ec56 100644 --- a/document/en/development/backend.md +++ b/document/en/development/backend.md @@ -41,7 +41,7 @@ The following targets are taken from the root `Makefile`: | `make migrate` | `alembic upgrade head` | | `make migrate-new msg="..."` | `alembic revision --autogenerate -m "..."` | | `make migrate-down` / `migrate-history` | roll back one step / show history | -| `make db-reset` / `db-seed` | reset (destructive) / seed sample data | +| `make db-reset` | Reset the database (destructive) | | `make build` / `up` / `down` / `logs` / `ps` etc. | docker-compose wrappers | > ⚠️ The committed tree is not formatter-clean: **do not run black/isort over files you did not touch** (it buries the real diff in formatting noise); format only the scope you changed. @@ -175,7 +175,7 @@ PYTHONPATH=src/backend python -m extensions.mcp_servers._mcp._selftest ## Adding a skill (Agent Skill) -Skill loading is multi-source (`core/agent_skills/config.py`): built-in (`extensions/skills/default/`, always-on), admin (DB / `/app/storage/admin_skills/`), user, and project sources merge by priority. `extensions/skills/marketplace/` holds install-based marketplace seeds scanned separately by the marketplace service — it is **not** part of the default loading sources. +Skill loading is multi-source (`core/capabilities/skills/config.py`): built-in (`extensions/skills/default/`, always-on), admin (DB / `/app/storage/admin_skills/`), user, and project sources merge by priority. `extensions/skills/marketplace/` holds install-based marketplace seeds scanned separately by the marketplace service — it is **not** part of the default loading sources. Adding a **built-in skill**: @@ -208,7 +208,7 @@ A **marketplace skill** goes under `extensions/skills/marketplace//` w | ORM / table boundary | `src/backend/core/db/models.py`, `src/backend/core/db/edition_tables.py` | | MCP port table / connection config | `src/backend/extensions/mcp_servers/_ports.py`, `src/backend/core/config/mcp_config.py` | | Capability catalog | `src/backend/core/config/catalog.json` | -| Skill loading | `src/backend/core/agent_skills/` (`config.py` / `loader.py` / `registry.py`) | +| Skill loading | `src/backend/core/capabilities/skills/` (`config.py` / `loader.py` / `registry.py`) | | Workflow orchestration | `src/backend/orchestration/workflow.py` | See also: [Frontend Development Guide](frontend.md) · [API Overview](../api/overview.md) · [MCP Tools](../modules/mcp-tools.md) · [Agent Skills](../modules/agent-skills.md) diff --git a/document/en/development/workbench-migration.md b/document/en/development/workbench-migration.md index 1449f3fd..89d6a38e 100644 --- a/document/en/development/workbench-migration.md +++ b/document/en/development/workbench-migration.md @@ -44,24 +44,7 @@ The SDK CLI validates declarations. It does not migrate databases or business da # Workbench configuration migration and recovery -Migration is an explicit operator action. It never runs during startup. Rehearse on an isolated database copy before separately authorizing a real migration. The current command accepts local rehearsal copies only. - -## Inventory and cutover - -1. Freeze every writer on the copy, including content imports, preferences, plugin installation, and background jobs. --writes-frozen confirms this operator action; it does not stop services. -2. Set WB20_COPY_DATABASE_URL through protected configuration. SQLite must reside under the current checkout's .scratch/workbench-20 directory. PostgreSQL must use a loopback host and a database name beginning with workbench_20. Never target business data. -3. Create a private 0700 snapshot directory. Run: - -```bash -PYTHONPATH=src/backend python -m scripts.workbench_migration inventory \ - --snapshot /absolute/private/path/before.json -PYTHONPATH=src/backend python -m scripts.workbench_migration apply \ - --snapshot /absolute/private/path/before.json --digest --writes-frozen -``` - -Inventory never writes the database. Reports contain counts, contract versions, redacted locations, and error codes. They omit URLs, user identities, tickets, and secrets. The 0600 snapshot retains all migration inputs: configuration, preferences, role/team relationships, plugin declarations, and workbench records. It does not replace a full database or business-file backup. - -A matching digest and unchanged source permit one transactional cutover. Failures roll back the transaction. A workbench_migration:v1 receipt records both snapshot digests. Cutover removes the old app_config row and navigation lists. The directory reads canonical sources. Duplicate external IDs, unknown IDs, dangerous URLs, and unknown fields are quarantined without creating entries or grants. Protected snapshots retain their raw values. +The standalone migration command has been removed. The following sections describe internal migration and recovery constraints; they are not an executable operator procedure. Back up existing deployments, prepare a version-specific plan, validate it on an isolated copy, and obtain separate authorization before execution. ## Preserved behavior @@ -69,7 +52,7 @@ A matching digest and unchanged source permit one transactional cutover. Failure - External ordering and enabled state survive. Default navigation becomes layout policy. Existing modern preferences win. Confirmed old preferences.pinned_apps/app_order migrate to web preferences: null inherits and an empty list explicitly unpins. Chat order, theme, and unrelated preferences remain unchanged. - Unmapped permission references remain unchanged and are reported. They do not grant access to new identities. Plugin UI v1/v2 is not upgraded to v3. Names and slugs never substitute for exact installations. - Existing workbench_external_auth:legacy: bindings retain their keys. Missing target adapters produce auth_configuration_required. Administrators must approve explicit login modes. Real legacy targets remain unverified until exercised. Generic SSO token forwarding must never return. -- Same-service recovery retains workbench_origin_identity. A service clone requires a fresh inventory and clone-origin on the copy. Existing plugin keys do not automatically grant access to the new origin. Review cross-origin state and preferences separately. +- Same-service recovery retains workbench_origin_identity. A service clone requires a fresh inventory and a regenerated service origin identity on the copy. Existing plugin keys do not automatically grant access to the new origin. Review cross-origin state and preferences separately. ## External compatibility window @@ -77,13 +60,7 @@ content-v1 remains supported throughout workbench protocol 1.x. Announce a separ ## Recovery and incremental writes -Freeze every writer again: - -```bash -PYTHONPATH=src/backend python -m scripts.workbench_migration rollback \ - --snapshot /absolute/private/path/before.json --digest \ - --delta /absolute/private/path/delta.json --writes-frozen -``` +Freeze all writers and follow the reviewed recovery plan. Recovery first exports a protected delta. Changes to already-mapped ordinary legacy links can replay into the old format. New identities, application state, resources, permissions, plugins, authentication, or policy changes block unsafe downgrade with delta_not_representable. New data stays intact. Never restore only the old snapshot. Keep the verified newer version and review the delta. Recovery receipts are idempotent. A restored batch requires a new reviewed batch before another apply. diff --git a/document/en/editions/build-ce.md b/document/en/editions/build-ce.md index 5921ec28..8dc42d88 100644 --- a/document/en/editions/build-ce.md +++ b/document/en/editions/build-ce.md @@ -26,7 +26,7 @@ The manifest has the following sections, in processing order: Glob patterns (relative to the repo root); a match means the file is never copied. Covers: -- **Backend EE modules**: the complete `edition_ee/**` implementation root (Team/RBAC, SSO, license verification and gate, EE ORM, Dify integration), cloud storage (`core/storage/s3.py`, `oss.py`), the Cube sandbox provider, memory audit, skill distillation, and other EE services; +- **Backend EE modules**: the complete `edition_ee/**` implementation root (Team/RBAC, SSO, license verification and gate, EE ORM, Dify integration), cloud storage (`core/files/storage/s3.py`, `oss.py`), the Cube sandbox provider, memory audit, skill distillation, and other EE services; - **EE routes**: `api/routes/v1/admin_*.py`, `config_*.py`, `audit.py`, `auth.py`, `team_files.py`, `service_configs.py`, `data_sources.py`, `db_metadata.py`, `gateway_*.py`; - **Industry MCP servers**: `extensions/mcp_servers/query_database_mcp/**`, `ai_chain_information_mcp/**`; - **Industry-specific frontend code and seeds**: the industry-chain Canvas, @@ -84,12 +84,12 @@ Industry-chain, news, and company-profile presentation remains an EE feature. | `.github/workflows/desktop-release.yml` | Public CE desktop release workflow, including the release-tag/version preflight gate | | `src/backend/api/routes/v1/__init__.py` | CE route registry; `EE_ROUTERS` is always empty | | `src/backend/core/auth/permissions_iface.py` | Owner-only single-tenant authorization interface with no team permission exports | -| `src/backend/core/services/artifact_edition.py` | Personal artifact-scope interface with no team fields, permissions, or repository methods | +| `src/backend/core/files/artifacts/edition.py` | Personal artifact-scope interface with no team fields, permissions, or repository methods | | `src/backend/core/llm/tools/edition_{myspace,myspace_vfs,artifact_recovery}.py` | Personal MySpace tool, VFS, and recovery interfaces; organization implementations do not enter CE | | `src/backend/core/config/edition_display_names.py` | CE tool display names with no team-tool names | | `src/backend/core/memory/audit.py` | Memory-audit no-op stub (same interface, writes nothing) | | `src/backend/alembic/versions/ce_000{1,2}_*.py` | CE independent migration-chain baseline and idempotent schema upgrade (next section) | -| `src/backend/core/services/edition_startup.py` | CE startup seam with no-op implementations for EE data-source recovery and distillation scheduling, so pruned modules are never imported | +| `src/backend/orchestration/workers/edition_startup.py` | CE startup seam with no-op implementations for EE data-source recovery and distillation scheduling, so pruned modules are never imported | | `src/backend/tests/ce_release/` | Executable CE release regressions for login, private skills/MCP, database upgrades, startup, and Compose contracts | | `src/backend/api/routes/v1/{agents,content,kb_models,projects}.py` | CE API contracts with administration endpoints and organization fields removed | | `src/backend/extensions/mcp_servers/_ports.py` | Port table for the 7 general tools (EE industry-tool and retired ports marked reserved) | diff --git a/document/en/modules/agent-skills.md b/document/en/modules/agent-skills.md index 18846fd9..e44f02df 100644 --- a/document/en/modules/agent-skills.md +++ b/document/en/modules/agent-skills.md @@ -29,11 +29,11 @@ Download integrity, path containment and account ownership checks remain enforce └── _scripts.json # optional: script whitelist (auto-detected by extension when absent) ``` -Parsing and data structures live in `core/agent_skills/registry.py`: `AgentSkillMetadata` (lightweight, for listings) and `AgentSkillSpec` (full instructions, for execution). When `_scripts.json` is missing, `loader._auto_detect_scripts` builds the whitelist from file extensions automatically. +Parsing and data structures live in `core/capabilities/skills/registry.py`: `AgentSkillMetadata` (lightweight, for listings) and `AgentSkillSpec` (full instructions, for execution). When `_scripts.json` is missing, `loader._auto_detect_scripts` builds the whitelist from file extensions automatically. ## Multi-source loading and priorities -`core/agent_skills/loader.py::MultiSourceSkillLoader` loads skills from several sources through a backend abstraction (`backends/`: filesystem / database / composite); on ID conflicts the higher-priority source wins (`core/agent_skills/config.py`): +`core/capabilities/skills/loader.py::MultiSourceSkillLoader` loads skills from several sources through a backend abstraction (`backends/`: filesystem / database / composite); on ID conflicts the higher-priority source wins (`core/capabilities/skills/config.py`): | Source | Directory | Priority | Notes | |---|---|---|---| @@ -149,7 +149,7 @@ Key points: - **Scope**: main-chat assembly and sub-agents. A sub-agent's bound plugins are deferred the same way, but activation lives only within that run and is never persisted (sub-agent runs are short-lived and isolated; writing under the parent chat's key would leak the activation into the main agent's assembly). Plugins pinned by a restricted chat mode stay eagerly loaded (a deliberate admin narrowing); plugins with stdio-transport MCP components are not deferred. - **Kill switch**: set `PLUGIN_PROGRESSIVE_LOADING=false` to revert to eager assembly. -Implementation: `core/plugins/runtime/__init__.py` (deferral resolution / directory rendering / activation tool) plus the assembly hook in `core/llm/factory/build.py`. +Implementation: `core/capabilities/plugins/runtime/__init__.py` (deferral resolution / directory rendering / activation tool) plus the assembly hook in `core/llm/factory/build.py`. ## The Skill Marketplace @@ -214,7 +214,7 @@ The `/admin` console's skill management maps to `api/routes/v1/admin_skills.py` - **Create/edit**: hand-written SKILL.md (`POST /`, `PUT /{id}`) or **zip upload** of an entire package (`POST /upload`, auto-detects directory prefixes, base64-encodes binaries into the DB, 200 MB cap); - **File-level management**: `GET/PUT/DELETE /{id}/files/{filename}` for inline editing of auxiliary files; - **Operations**: toggle (`/{id}/toggle`), ordering, icons, fork, bulk export/import (cross-environment migration); -- **Dependency management**: `POST /{id}/rescan-deps` statically scans scripts for pip/apt dependencies via `core/agent_skills/deps_detector.py`, with `PUT /{id}/dependencies` for manual correction; the aggregated manifest feeds the sandbox image rebuild (see the admin dependency rebuild in the [sandbox module](sandbox.md)). +- **Dependency management**: `POST /{id}/rescan-deps` statically scans scripts for pip/apt dependencies via `core/capabilities/skills/deps/detector.py`, with `PUT /{id}/dependencies` for manual correction; the aggregated manifest feeds the sandbox image rebuild (see the admin dependency rebuild in the [sandbox module](sandbox.md)). ## Skill distillation (Enterprise EE) @@ -281,12 +281,12 @@ only submission, never conversation or continued local editing. | Path | Description | |---|---| -| `src/backend/core/agent_skills/registry.py` | SKILL.md parsing, metadata / full-spec dataclasses | -| `src/backend/core/agent_skills/loader.py` | Multi-source loading, DB-skill materialization, sandbox path repointing | -| `src/backend/core/agent_skills/config.py` | Source priorities, unified sandbox skills dir, built-in sync | -| `src/backend/core/agent_skills/selector.py` | LLM-based skill selection per user intent (progressive disclosure) | -| `src/backend/core/agent_skills/deps_detector.py` | Static pip/apt dependency detection for scripts | -| `src/backend/core/agent_skills/backends/` | filesystem / database / composite loading backends | +| `src/backend/core/capabilities/skills/registry.py` | SKILL.md parsing, metadata / full-spec dataclasses | +| `src/backend/core/capabilities/skills/loader.py` | Multi-source loading, DB-skill materialization, sandbox path repointing | +| `src/backend/core/capabilities/skills/config.py` | Source priorities, unified sandbox skills dir, built-in sync | +| `src/backend/core/llm/skill_selection.py` | LLM-based skill selection per user intent (progressive disclosure) | +| `src/backend/core/capabilities/skills/deps/detector.py` | Static pip/apt dependency detection for scripts | +| `src/backend/core/capabilities/skills/backends/` | filesystem / database / composite loading backends | | `src/backend/core/llm/tools/skill_tool.py` | Restricted view_text_file + {baseDir} substitution + Runtime Hint | | `src/backend/extensions/skills/default/` | The 12 built-in skills (incl. the 10 OfficeCLI ones) | | `src/backend/extensions/skills/marketplace/` | The 45 installable marketplace packages | diff --git a/document/en/modules/auth.md b/document/en/modules/auth.md index 5689136e..de0f0b6c 100644 --- a/document/en/modules/auth.md +++ b/document/en/modules/auth.md @@ -185,7 +185,7 @@ The actual gating, verified against each route file's dependencies: | `PATCH /v1/me/api-keys/{key_id}` | Enable / disable | | `DELETE /v1/me/api-keys/{key_id}` | Revoke | -Keys look like `sk-jx-...` and can be used as a Bearer token to call business APIs **in every AUTH_MODE** (external programmatic callers usually have no cookies). Validation (enabled / not revoked / not expired / user flag still on) is centralized in `core/services/api_key_service.py::resolve_api_key`. +Keys look like `sk-jx-...` and can be used as a Bearer token to call business APIs **in every AUTH_MODE** (external programmatic callers usually have no cookies). Validation (enabled / not revoked / not expired / user flag still on) is centralized in `core/auth/api_keys.py::resolve_api_key`. Disabling or revoking an API key rejects new requests and cancels its admitted chat tasks that are pending, running, or awaiting attention. Cancellation stops background execution and terminates the SSE stream; completed external tool effects cannot be undone. Re-enabling the key does not resume cancelled tasks. Call duration stays fixed after execution stops, including the needs-attention state. @@ -216,7 +216,7 @@ Key authentication events all land in the audit table (`audit_logs`): login succ | Administrative credential dependencies | `src/backend/api/deps.py` | | Profile / preferences | `src/backend/api/routes/v1/users.py` | | Per-user permission flags | `src/backend/edition_ee/routes/config_users.py` | -| Personal API keys | `src/backend/api/routes/v1/api_keys.py`, `src/backend/core/services/api_key_service.py` | +| Personal API keys | `src/backend/api/routes/v1/api_keys.py`, `src/backend/core/auth/api_keys.py` | | Capability-center self-service (owner isolation) | `src/backend/api/routes/v1/me_capabilities.py` | | Invite codes | `src/backend/edition_ee/auth/invite.py`, `src/backend/edition_ee/routes/config_invites.py` | | Team management | `src/backend/edition_ee/routes/config_teams.py`, `src/backend/edition_ee/routes/me_teams.py` | diff --git a/document/en/modules/automation.md b/document/en/modules/automation.md index 4fb0380c..ad041308 100644 --- a/document/en/modules/automation.md +++ b/document/en/modules/automation.md @@ -48,6 +48,7 @@ Tasks live in the `scheduled_tasks` table (`ScheduledTask` in `core/db/models/au - **Execution duration**: there is no fixed total runtime limit for scheduled or manually triggered Prompt, Plan, or Loop tasks. Their conversations and successors are also exempt from the ordinary chat absolute-age cap. - **Failure governance**: when consecutive failures reach `max_failures` (default 3), the task is automatically set to `disabled`. - **Orphan recovery**: at startup and on every poll, a worker must acquire execution ownership before resuming observation of the linked durable conversation and saving its final outcome. Only orphan runs without a linked conversation are marked `failed`, with overdue schedules advanced and one-shot tasks finalized. Runtime never determines recovery. The lock renews every 15 seconds; its 60-second lease detects a disconnected worker and does not limit total runtime. Missed one-shot tasks also acquire ownership before firing. +- **Execution lease shutdown**: shutdown signals stop, cancels renewal, waits at most 2 seconds, and always attempts token-checked deletion. Local ownership markers are removed even on cleanup errors. Renewal also stops when the execution Task finishes. Result delivery has a 60-second total wait budget; failures do not overwrite committed success and do not retry automatically. Running records carry an internal ownership token; completion requires both running status and a matching token. Recovery claims the record under its new token. Each execution creates and immediately links a **real conversation** through the shared ChatRun executor. Prompt runs stream text, thinking and tools; plan runs stream step progress. Opening a running conversation or refreshing the page follows and replays the same event stream as ordinary chat. Users can stop execution from the conversation; prompt runs also accept additional instructions through the durable steering queue at the next safe model boundary. Stopping does not increase consecutive failures, and loss of execution ownership or scheduler shutdown cancels the child run. Final messages, artifacts, citations and delivery remain owned by the shared executor and scheduler outcome handler. Conversation titles use the task name without an automation prefix. Older executions started before live-stream support cannot replay events that were never recorded. @@ -98,14 +99,41 @@ Every morning an Industry morning brief execution conversation is created. The s | Topic | Path | |---|---| -| Automation REST API | `src/backend/api/routes/v1/automations.py` | -| Scheduler (polling / locks / recovery) | `src/backend/orchestration/schedulers/automation_scheduler.py` | -| Automation service (cron computation / state machine) | `src/backend/core/services/automation_service.py` | -| Task / run-record models | `src/backend/core/db/models/automation.py` | +| Automation REST API | `src/backend/features/automation/api/automations.py` | +| Scheduler (polling / locks / recovery) | `src/backend/orchestration/automations/automation_scheduler.py` | +| Automation service (cron computation / state machine) | `src/backend/features/automation/service.py` | +| Task / run-record models | `src/backend/features/automation/models.py` | | Plan REST API | `src/backend/api/routes/v1/plans.py` | -| Plan generation / execution orchestration | `src/backend/orchestration/subagents/plan_mode.py`, `src/backend/core/services/plan_service.py` | +| Plan generation / execution orchestration | `src/backend/orchestration/subagents/plan_mode.py`, `src/backend/core/chat/plans.py` | | Plan / step models | `src/backend/core/db/models/agent.py` | | Frontend automation UI | `src/frontend/src/components/lab/AutomationPanel.tsx` etc., `src/frontend/src/components/automation/RunTimelinePanel.tsx` | | Frontend state | `src/frontend/src/stores/automationStore.ts`, `automationChatStore.ts` | Further reading: [Chat System](chat.md) · [MCP Tools](mcp-tools.md) · [Canvas & Artifacts](canvas-artifacts.md) + +## Multi-process execution and recovery + +Scheduled and manual triggers share one loop, scanned every 3 seconds with at most +8 observed executions. Trigger consumption, schedule advancement and insertion of +a pending run commit in one transaction. Conditional database updates fence +concurrent admission. A manual trigger for a busy task remains queued. + +Pending means accepted and awaiting execution. Scheduler shutdown or lease loss +transfers observation without cancelling the accepted conversation. A successor +observes the same durable execution and its handoffs. User cancellation still +stops execution. Child linkage and recovery arguments commit with acceptance; +recovery never restarts a command to obtain its result. Existing notification +delivery semantics remain; this change does not promise exactly-once delivery +to external channels. + +Apply FULL migration automationqueue01 or CE migration ce_0032 before rollout. +Local SQLite uses the installed automation module's schema initialization. +Downgrade refuses pending records rather than discarding accepted work. Stop old +schedulers, migrate, then start the new version; do not mix scheduling protocols. + +Plan/Loop persist all initial arguments with child acceptance. After a crash before +worker registration, the successor dispatches only pending children. It never +replays an already-running child from the beginning. Drain old in-flight work +before upgrading; missing legacy arguments are not reconstructed by a compatibility path. + +Task delivery uses delivery_targets exclusively. The upgrade migrates flat channel_id/conversation_id fields into destination lists, removes the old fields, and preserves explicit destinations and unrelated metadata. Request handling no longer reads the old format. diff --git a/document/en/modules/canvas-artifacts.md b/document/en/modules/canvas-artifacts.md index 707ad1a8..0dd6328c 100644 --- a/document/en/modules/canvas-artifacts.md +++ b/document/en/modules/canvas-artifacts.md @@ -87,12 +87,12 @@ limits show a local-download prompt while the main page remains responsive. ### How artifacts are produced and persisted -During a streamed turn, files produced by tools first enter the per-conversation **workspace** (`core/llm/workspace`); only files that are explicitly **pinned** are written to the `artifacts` table at end of turn by `core/services/artifact_service.py::persist_artifacts` — this strict workspace gate keeps temporary intermediates from polluting My Space. +During a streamed turn, files produced by tools first enter the per-conversation **workspace** (`core/llm/workspace`); only files that are explicitly **pinned** are written to the `artifacts` table at end of turn by `core/files/artifacts/service.py::persist_artifacts` — this strict workspace gate keeps temporary intermediates from polluting My Space. -Physical storage is managed by `core/artifacts/store.py`, dual-mode by `STORAGE_TYPE`: +Physical storage is managed by `core/files/artifacts/store.py`, dual-mode by `STORAGE_TYPE`: - `local` (default, Community Edition): files are written under `${STORAGE_PATH:-result}/artifacts/` with a local JSON index (`index.json`), served via FileResponse. -- `oss` (Enterprise Edition, `cloud_storage`): files are uploaded to Aliyun OSS; the local index is also backed up to OSS so it survives container restarts. The S3 backend lives in `core/storage/`. +- `oss` (Enterprise Edition, `cloud_storage`): files are uploaded to Aliyun OSS; the local index is also backed up to OSS so it survives container restarts. The S3 backend lives in `core/files/storage/`. Downloads all go through `GET /files/{file_id}` (`api/routes/files.py`), with an owner ∪ team permission check via `core/auth/permissions_iface.py::resolve_artifact_access`; Office files support PDF-converted online preview. @@ -153,8 +153,8 @@ attachments and My Space, and xlsx files remain editable in the data canvas. |---|---| | Canvas panel / Univer spreadsheet | `src/frontend/src/components/canvas/CanvasPanel.tsx`, `UniverSpreadsheet.tsx`, `src/frontend/src/stores/canvasStore.ts` | | Artifact REST API | `src/backend/api/routes/v1/artifacts.py` | -| Artifact store (local/OSS) | `src/backend/core/artifacts/store.py` | -| Artifact persistence (workspace gate) | `src/backend/core/services/artifact_service.py` | +| Artifact store (local/OSS) | `src/backend/core/files/artifacts/store.py` | +| Artifact persistence (workspace gate) | `src/backend/core/files/artifacts/service.py` | | File download / preview | `src/backend/api/routes/files.py` | | Chat share API | `src/backend/api/routes/v1/chat_shares.py` | | Share frontend | `src/frontend/src/SharePreviewApp.tsx`, `src/frontend/src/components/share/ShareRecordsPage.tsx` | diff --git a/document/en/modules/catalog.md b/document/en/modules/catalog.md index 610135a0..e15332cd 100644 --- a/document/en/modules/catalog.md +++ b/document/en/modules/catalog.md @@ -13,7 +13,7 @@ core/config/catalog.json system-level single source of truth (admin t core/config/catalog.py public API: get_catalog / is_enabled / get_enabled_ids │ set_enabled / reorder_items ▼ -core/config/catalog_resolver.py merge layer: catalog.json defaults ∩ per-user DB overrides +core/capabilities/catalog/resolver.py merge layer: catalog.json defaults ∩ per-user DB overrides │ resolve_all_runtime_enabled(db, user_id) → (skills, agents, mcps) │ resolve_explicit_runtime_capabilities(...) → per-turn invocation (ignores personal switches, keeps hard gates) ▼ @@ -39,7 +39,7 @@ The file lives at `src/backend/core/config/catalog.json` (path overridable via t } ``` -`catalog_loader.py` performs **dynamic-source synchronization** at load time: new ids discovered from the skill loader (`core/agent_skills/`) and the MCP config service (`core/services/mcp_service.py`, DB table `admin_mcp_servers`) are added automatically; names/descriptions/versions refresh from the source; stale entries whose id no longer exists in any source are removed. Display details (Capability Center intro markdown, icons) are runtime-only fields and never persisted. Results are held in a short-TTL in-memory cache; writes call `invalidate_catalog_cache()`. +`catalog_loader.py` performs **dynamic-source synchronization** at load time: new ids discovered from the skill loader (`core/capabilities/skills/`) and the MCP config service (`core/capabilities/mcp/service.py`, DB table `admin_mcp_servers`) are added automatically; names/descriptions/versions refresh from the source; stale entries whose id no longer exists in any source are removed. Display details (Capability Center intro markdown, icons) are runtime-only fields and never persisted. Results are held in a short-TTL in-memory cache; writes call `invalidate_catalog_cache()`. ### Gating API @@ -54,7 +54,7 @@ These two functions are the system-wide read path for capability switches: `fact ## Per-user overrides (catalog_overrides) -When a user toggles a capability in the Capability Center, the change is written to the `catalog_overrides` DB table (`core/services/catalog_service.py`), per-user and isolated: +When a user toggles a capability in the Capability Center, the change is written to the `catalog_overrides` DB table (`core/capabilities/catalog/overrides.py`), per-user and isolated: - **Merge algorithm** (`catalog_resolver.py::_merge_kind`): overrides can only flip the enabled flag of items that *exist* in the base catalog; they can never resurrect deleted items. - **Admin lock**: items disabled in the base catalog (`enabled=false`) **cannot** be re-enabled by user overrides, and are completely hidden from the user-facing `/v1/catalog` response. @@ -66,7 +66,7 @@ When a user toggles a capability in the Capability Center, the change is written - **Explicit plugin guidance**: selecting a plugin loads its skills and MCP tools and recommends using them when relevant, without restricting the first tool call or requiring a call before the answer. - **Chat-sticky capability restoration**: later turns restore activated skills, connectors, and plugin components before personal-switch filtering, keeping the skill list and tool names/prefixes stable in that chat. Installation, global admin state, dependency readiness, and ownership are revalidated each turn, so hard-gate changes take effect immediately. -On every chat, `core/chat/context.py::resolve_enabled_capabilities()` writes the default merged result into the workflow context. `api/routes/v1/chats/__init__.py::_resolve_explicit_capability_invocation()` then verifies plugin installations and capability ownership, resolves components from the server-side installation record, and merges hard-gate-approved explicit selections into that turn. On later turns, `core/llm/session_capabilities.py` and `core/plugins/runtime/__init__.py::resolve_sticky_plugin_capabilities()` restore chat-activated capabilities. +On every chat, `core/chat/context.py::resolve_enabled_capabilities()` writes the default merged result into the workflow context. `api/routes/v1/chats/__init__.py::_resolve_explicit_capability_invocation()` then verifies plugin installations and capability ownership, resolves components from the server-side installation record, and merges hard-gate-approved explicit selections into that turn. On later turns, `core/llm/runtime/session_capabilities.py` and `core/capabilities/plugins/runtime/__init__.py::resolve_sticky_plugin_capabilities()` restore chat-activated capabilities. ## The /v1/catalog route and KB injection @@ -116,7 +116,6 @@ State is centralized in `src/frontend/src/stores/catalogStore.ts`; local default ## Admin-side capability management - **`/admin` content console**: skill upload/toggle/reorder (`api/routes/v1/admin_skills.py`, which calls `catalog.set_enabled` / `reorder_items` to write back to catalog.json), MCP server management (`admin_mcp_servers.py`, DB table + connectivity probe + cache refresh), sub-agent management (`admin_agents.py`), and skill marketplace listing review (`admin_marketplace.py`, **Enterprise Edition (EE)**). -- **Catalog snapshot migration**: `scripts/export_content.py --only catalog` exports catalog.json + `catalog_overrides`; `scripts/import_content.py --catalog ` imports them. ## Source map @@ -125,11 +124,11 @@ State is centralized in `src/frontend/src/stores/catalogStore.ts`; local default | Public API (is_enabled, etc.) | `src/backend/core/config/catalog.py` | | Single-source file | `src/backend/core/config/catalog.json` (overridable via `CATALOG_PATH`) | | Loading / dynamic sync / caching | `src/backend/core/config/catalog_loader.py`, `catalog_common.py`, `catalog_migration.py` | -| User-override merging | `src/backend/core/config/catalog_resolver.py`, `core/services/catalog_service.py` | +| User-override merging | `src/backend/core/capabilities/catalog/resolver.py`, `core/capabilities/catalog/overrides.py` | | Catalog route | `src/backend/api/routes/v1/catalog.py` | | User self-service capabilities | `src/backend/api/routes/v1/me_capabilities.py` | -| MCP server config (DB) | `src/backend/core/services/mcp_service.py`, `api/routes/v1/admin_mcp_servers.py` | -| Skill management | `src/backend/api/routes/v1/admin_skills.py`, `core/agent_skills/` | +| MCP server config (DB) | `src/backend/core/capabilities/mcp/service.py`, `api/routes/v1/admin_mcp_servers.py` | +| Skill management | `src/backend/api/routes/v1/admin_skills.py`, `core/capabilities/skills/` | | Dify KB injection (EE) | `src/backend/edition_ee/kb/dify.py`, via the shared `core/kb/external_provider.py` seam | | Frontend Capability Center | `src/frontend/src/components/catalog/`, `stores/catalogStore.ts` | | Factory consumption | `src/backend/core/llm/factory/build.py::_effective_mcp_server_keys` | diff --git a/document/en/modules/chat.md b/document/en/modules/chat.md index a5f2fc76..ef27e794 100644 --- a/document/en/modules/chat.md +++ b/document/en/modules/chat.md @@ -71,10 +71,10 @@ orchestration/chat_run_executor.py::start_run() ▼ orchestration/workflow.py::astream_chat_workflow() │ ├─ orchestration/memory_integration.py non-blocking memory retrieval (bg task + budget) - │ ├─ core/config/catalog_resolver.py resolve enabled skills/mcp/kb for this request + │ ├─ core/capabilities/catalog/resolver.py resolve enabled skills/mcp/kb for this request │ ├─ core/llm/factory/build.py::create_agent_executor() │ │ MCP pool + skill registration + file tools + system prompt + middlewares → Agent - │ ├─ core/llm/context_manager.py trim history to the token budget + │ ├─ core/llm/context/window_policy.py trim history to the token budget │ └─ orchestration/streaming.py::StreamingAgent.stream() │ consumes agent.reply_stream(), coalesces 25 event kinds while preserving tool-argument deltas ▼ @@ -160,8 +160,8 @@ Trusted unattended entry points (channel bots, scheduled automation) take the `d `create_agent_executor()` is the shared factory for every mode (main chat, plan, batch, sub-agents, automation): -- **MCP tools**: after the three-layer filter of catalog + per-user overrides + request context (see [Capability Center](catalog.md)), stable servers reuse the process-level connection pool (`core/llm/mcp_pool.py`); per-request servers (e.g. `retrieve_dataset_content`, which needs per-request HTTP headers) are spawned fresh; the user's self-added private MCP servers are merged in with owner isolation. -- **Skills**: registered as AgentScope Agent Skills via `core/agent_skills/loader.py`, with `view_text_file` allow-listed to read SKILL.md files (see [Agent Skills](agent-skills.md)). +- **MCP tools**: after the three-layer filter of catalog + per-user overrides + request context (see [Capability Center](catalog.md)), stable servers reuse the process-level connection pool (`core/llm/mcp/pool.py`); per-request servers (e.g. `retrieve_dataset_content`, which needs per-request HTTP headers) are spawned fresh; the user's self-added private MCP servers are merged in with owner isolation. +- **Skills**: registered as AgentScope Agent Skills via `core/capabilities/skills/loader.py`, with `view_text_file` allow-listed to read SKILL.md files (see [Agent Skills](agent-skills.md)). - **File / sandbox tools**: `bash`, `sandbox_put_artifact`, `sandbox_get_artifact` are always registered; Read/Edit/Write/Glob/Grep/Delete/Move/mkdir plus the MySpace tools are gated by `CODE_CAPABILITY_ENABLED` and share one `ReadStateTracker` to keep the "must Read before Edit" invariant. - **Middlewares** (onion model, `core/llm/middlewares.py`): `DynamicModelMiddleware` (switches the model per chat_mode, see [Model Providers](model-providers.md)), `FileContextMiddleware` (injects uploaded/historical file context), `SteerMiddleware` (injects follow-ups after tool results and before the next reasoning round), `WorkspacePinHintMiddleware`, `FinishPinGuardMiddleware`. - **Context compression**: `CompactingAgent` routes the pre-turn, mid-turn, and post-turn trigger points through one persistent checkpoint engine and one Codex-style handoff prompt. `ContextConfig` remains only for tool-result limiting. Compaction uses the unified coordinator without a second framework summary fallback. Failure preserves the original context. If the final request still exceeds its budget, the run saves a checkpoint and pauses before calling the reply model. @@ -428,7 +428,7 @@ or no-longer-owned components are not restored. - **Autonomous main-agent dispatch**: when neither a structured `@` selection nor the strict natural-language command matches, the main agent can use the `call_subagent` tool registered by - `core/llm/subagent_tool.py`. Each child runs in its own thread and event loop, + `core/llm/subagents/tool.py`. Each child runs in its own thread and event loop, then returns text for the main agent to integrate. This path supports parallel sub-agents, task decomposition, and cross-domain synthesis. @@ -439,7 +439,7 @@ Three complementary layers: | Layer | Implementation | Trigger | |---|---|---| | Chat title summary | `core/llm/summarizer.py::ConversationSummarizer` (`summarizer` model role, `ENABLE_SUMMARY` flag), `POST /v1/summary` | Auto-titling new chats | -| Unified context checkpoint | `core/services/compaction_service.py::run_compaction()`; pre-turn, mid-turn, and post-turn share one trigger ratio, handoff prompt, replacement shape, and persistent checkpoint | Context reaches the configured fraction of the model window | +| Unified context checkpoint | `core/chat/compaction/service.py::run_compaction()`; pre-turn, mid-turn, and post-turn share one trigger ratio, handoff prompt, replacement shape, and persistent checkpoint | Context reaches the configured fraction of the model window | | Deterministic overflow protection | `ContextConfig.tool_result_limit` bounds individual tool results first; failed compaction preserves the original context, and an over-budget final request saves a checkpoint and pauses | A tool result is oversized or unified compaction is temporarily unavailable | Compaction cuts only at legal boundaries: history is first split into steps, @@ -468,7 +468,7 @@ recovery stay consistent. ## Oversized tool-result offloading -`core/llm/offloader.py::SandboxOffloader` implements the AgentScope 2.0 `Offloader` protocol: when context compression or tool-result truncation happens, the overflow is no longer silently discarded — it is written into the sandbox at `/workspace/.offload/` (`tool_.txt` / `context_.txt`), the framework appends the path to the model-facing ``, and the model can read it back on demand via `Read` / `bash`. Mounted only when sandbox tools are enabled (`SANDBOX_TOOLS_ENABLED=true`, default on); write failures never raise and degrade to an explanatory message. +`core/llm/context/offloader.py::SandboxOffloader` implements the AgentScope 2.0 `Offloader` protocol: when context compression or tool-result truncation happens, the overflow is no longer silently discarded — it is written into the sandbox at `/workspace/.offload/` (`tool_.txt` / `context_.txt`), the framework appends the path to the model-facing ``, and the model can read it back on demand via `Read` / `bash`. Mounted only when sandbox tools are enabled (`SANDBOX_TOOLS_ENABLED=true`, default on); write failures never raise and degrade to an explanatory message. ## Chat sharing @@ -498,15 +498,15 @@ The same orchestration foundation also powers: response regeneration (`POST /v1/ | Event mapping (reply_stream → SSE) | `src/backend/orchestration/streaming.py` | | Runtime context assembly | `src/backend/core/chat/context.py` | | Agent factory | `src/backend/core/llm/factory/build.py` | -| Middlewares | `src/backend/core/llm/middlewares.py` (pure-function helpers in `core/llm/hooks.py`) | +| Middlewares | `src/backend/core/llm/middlewares.py` (pure-function helpers in `core/llm/runtime/hooks.py`) | | Citation extraction | `src/backend/orchestration/citations.py` | -| Cross-chat references (cards / digests / paging) | `src/backend/core/services/chat_reference_service.py` | +| Cross-chat references (cards / digests / paging) | `src/backend/core/chat/references.py` | | Cross-chat history tools | `src/backend/core/llm/tools/chat_history_tool.py` | | Citation rendering | `src/frontend/src/utils/citations.ts`, `src/frontend/src/components/citation/` | | Plan mode | `src/backend/orchestration/subagents/plan_mode.py`, `api/routes/v1/plans.py` | -| Sub-agent tool | `src/backend/core/llm/subagent_tool.py`, `api/routes/v1/agents.py` | -| Title summarization / context compaction / window protection | `src/backend/core/llm/summarizer.py`, `compaction.py`, `core/services/compaction_service.py`, `context_manager.py` | -| Oversized-result offloading | `src/backend/core/llm/offloader.py` | +| Sub-agent tool | `src/backend/core/llm/subagents/tool.py`, `api/routes/v1/agents.py` | +| Title summarization / context compaction / window protection | `src/backend/core/llm/summarizer.py`, `compaction.py`, `core/chat/compaction/service.py`, `context_manager.py` | +| Oversized-result offloading | `src/backend/core/llm/context/offloader.py` | | Chat sharing | `src/backend/api/routes/v1/chat_shares.py` | | Follow-up generation | `src/backend/orchestration/followups.py` | | Frontend stream parsing / follow-up queue | `src/frontend/src/hooks/chatStream.ts`, `useStreaming.ts`, `components/chat/QueuedMessageCard.tsx` | diff --git a/document/en/modules/knowledge-base.md b/document/en/modules/knowledge-base.md index 1e84aa15..1284c583 100644 --- a/document/en/modules/knowledge-base.md +++ b/document/en/modules/knowledge-base.md @@ -29,7 +29,7 @@ In CE, the frontend shows only **Private Knowledge Base**, and `/v1/catalog` ret Object storage write (storage_key) │ · embed_text(query) │ │ · Milvus hybrid_search: ▼ BackgroundTask │ dense (IP) + sparse (BoW) → RRF(k=60) - core/content/kb_processing.py │ · child hit → fetch parent chunk + core/files/content/kb_processing.py │ · child hit → fetch parent chunk vectorise_document_background() │ text from kb_chunks (PostgreSQL) · kb_parser.parse_and_chunk() │ · optional reranker pass · (optional) LLM keywords / questions │ · user_id isolation + global public KBs @@ -84,7 +84,7 @@ Parsing and chunking happen in `core/kb/kb_parser.py::parse_and_chunk()`, suppor | `recursive` | Recursive fixed-size splitting | | `embedding_semantic` | Semantic boundary detection via embedding similarity | -Parent-child parameters are tunable per upload via `indexing_config`: `parent_chunk_size` (default 1024 tokens), `child_chunk_size` (128), `overlap_tokens` (20), `parent_child_indexing` (default true). LLM enrichment is opt-in: `auto_keywords_count` (keywords per parent chunk, stored as tags and fed into sparse retrieval) and `auto_questions_count` (generated questions per parent chunk, indexed as separate question rows in Milvus to boost question-style recall). The background vectorization task is `core/content/kb_processing.py::vectorise_document_background()`. +Parent-child parameters are tunable per upload via `indexing_config`: `parent_chunk_size` (default 1024 tokens), `child_chunk_size` (128), `overlap_tokens` (20), `parent_child_indexing` (default true). LLM enrichment is opt-in: `auto_keywords_count` (keywords per parent chunk, stored as tags and fed into sparse retrieval) and `auto_questions_count` (generated questions per parent chunk, indexed as separate question rows in Milvus to boost question-style recall). The background vectorization task is `core/files/content/kb_processing.py::vectorise_document_background()`. ### Multimodal: image retrieval @@ -195,9 +195,9 @@ DIFY_API_KEY=dataset-... # alias: DIFY_AUTH_TOKEN ## Supported file formats -KB upload validation is in `core/content/file_validation.py::validate_kb_file` (extension + magic-bytes double check), allowing: `.pdf` `.txt` `.md` `.doc` `.docx` `.xls` `.xlsx` `.csv` `.json`, plus images (`.png` `.jpg` `.jpeg` `.webp` `.gif`). +KB upload validation is in `core/files/content/file_validation.py::validate_kb_file` (extension + magic-bytes double check), allowing: `.pdf` `.txt` `.md` `.doc` `.docx` `.xls` `.xlsx` `.csv` `.json`, plus images (`.png` `.jpg` `.jpeg` `.webp` `.gif`). -The general-purpose parser `core/content/file_parser.py::parse_file()` (shared by chat attachments and MySpace files) covers a wider range: PDF, DOCX, DOC/WPS (via LibreOffice conversion), TXT, XLSX/XLS, CSV, PPTX, plus plain-text formats (HTML / Markdown / JSON / YAML / source code, etc.) decoded directly as UTF-8. +The general-purpose parser `core/files/content/file_parser.py::parse_file()` (shared by chat attachments and MySpace files) covers a wider range: PDF, DOCX, DOC/WPS (via LibreOffice conversion), TXT, XLSX/XLS, CSV, PPTX, plus plain-text formats (HTML / Markdown / JSON / YAML / source code, etc.) decoded directly as UTF-8. ## Frontend @@ -218,9 +218,9 @@ The general-purpose parser `core/content/file_parser.py::parse_file()` (shared b | `src/backend/core/kb/kb_assets.py` | Media assets: storage, image understanding (via the vision bridge), retrieval rows, result attachment | | `src/backend/edition_ee/kb/dify.py` | Dify datasets client and enablement logic (EE only) | | `src/backend/core/kb/external_provider.py` | Edition-neutral external-provider seam; disabled by the CE overlay | -| `src/backend/core/content/kb_processing.py` | Background vectorization, LLM keyword / question enrichment | -| `src/backend/core/content/file_validation.py` | Upload validation (extension + magic bytes) | -| `src/backend/core/content/file_parser.py` | General-purpose file parser | +| `src/backend/core/files/content/kb_processing.py` | Background vectorization, LLM keyword / question enrichment | +| `src/backend/core/files/content/file_validation.py` | Upload validation (extension + magic bytes) | +| `src/backend/core/files/content/file_parser.py` | General-purpose file parser | | `src/backend/core/services/kb_service.py` | KB business logic (incl. the system-managed sync space) | | `src/backend/api/routes/v1/kb.py` + `kb_models.py` | User-facing `/v1/catalog/kb` routes | | `src/backend/api/routes/v1/admin_kb.py` | Admin public KB routes (Enterprise Edition, EE) | diff --git a/document/en/modules/mcp-tools.md b/document/en/modules/mcp-tools.md index 6f5bad94..84857199 100644 --- a/document/en/modules/mcp-tools.md +++ b/document/en/modules/mcp-tools.md @@ -29,7 +29,7 @@ must execute in separate rounds. │ agent) │ │ │ :9103 unified Industry Knowledge Center (EE)│ └─────────┘ │ │ :9104 reserved (retired chart MCP) │ │ │ │ :9105 reserved (former report export MCP) │ - MCPConnectionPool │ │ :9106 web_fetch │ + Native MCP Client │ │ :9106 web_fetch │ mcp_pool.py) └───┬────┘ :9108 automation_task (automation plugin) │ │ 9109–9111 reserved (former office MCPs) │ │ :9112 skill_manager │ @@ -237,17 +237,14 @@ before the result enters the model context and before persistence; the frontend render `cite_id` as per-item chips that match the `[anchor text](cite:eN)` references in the answer body. -## Backend client: connection pool & bare-name restoration +## Backend client: native MCP lifetimes -The backend connects through AgentScope 2.0's `MCPClient`, centred on two files: +The backend uses MCP SDK 2.3 `Client(mode="auto")` and AgentScope `ToolBase`. `client.py` owns transport authorization; `lifetime.py` owns SDK scopes and complete paginated discovery. HTTP calls reuse the same client throughout a run. STDIO sessions use exclusive leases scoped to the executing user, chat and configuration; successful sessions remain idle for 60 seconds, while failed or cancelled calls destroy them. There is no stable-connection switch or startup connection pool. -- **`core/llm/mcp_pool.py` — `MCPConnectionPool`** (process-level singleton): at startup `warmup_mcp_tools()` reads all enabled server configs from the DB and pre-connects. Pooling semantics under 2.0: - - **stdio servers marked `is_stable=true`** keep their connection across requests (saving the 1–7 s subprocess cold start); - - **HTTP servers are never pooled** — 2.0's stateful HTTP client is bound to its asyncio task, and reuse across requests triggers cancel-scope crashes, so each request gets a fresh `is_stateful=False` connection; - - per-request servers (KB retrieval with user headers) connect on demand and are closed via `close_transient()` when the request ends. -- **`core/llm/mcp_manager.py` — `BareNameMCPClient`**: AgentScope 2.0 rewrites tool names to `mcp____`; this subclass restores the server-side bare name (`internet_search`, not `mcp__internet_search__internet_search`) so the display-name mapping (`core/config/display_names.py`), [citation extraction](chat.md) keyed by tool name, and frontend icon rendering all keep working as in 1.x. +`callbacks.py` binds isolated text sampling and user elicitation to the active run. Resources, URI templates and prompts become permission-governed tools. Progress reaches the existing tool card. Structured results and resource links remain in tool metadata and model-facing output. SDK Tasks extensions are not available in 2.3. + +MCP execution budgets exclude time waiting for user answers. Valid questions pause the current invocation; answering or cancelling a question resumes its remaining budget. Repeated or overlapping questions do not reset that budget. Other invocations keep independent timers. Existing human-interaction deadlines still apply, and cancelling a run clears pending questions. Invocation budgets bound SDK requests and HTTP reads; connection, write and discovery timeouts remain active. -The 2.0 `Toolkit` is constructed once, in `core/llm/factory/build.py`: `Toolkit(tools=[...], mcps=clients)`. ## Unified tool-permission gateway @@ -270,7 +267,7 @@ The 2.0 `Toolkit` is constructed once, in `core/llm/factory/build.py`: `Toolkit( tools become dispatchable. Native tools may opt in with -`ToolCollector.register_tool_function(..., permission=spec)`. The platform's +`ToolAssembly.append(function_tool(..., dispatch_spec=spec))`. The platform's local-file, Bash, and My Space rules live in `builtin_tool_permission()`. Native tools with neither an explicit spec nor a built-in entry pass through. @@ -297,7 +294,7 @@ tools that should be governed need a registry rule. ## Registration: DB-driven config + catalog gating -The source of truth for MCP server configuration is the `admin_mcp_servers` table (ORM: `core/db/models.py::AdminMcpServer`), read through `core/services/mcp_service.py::McpServerConfigService` with a 30-second TTL cache, in a dict format compatible with the legacy `MCP_SERVERS` (`transport / command / args / env / url / headers / is_stable`). `core/config/mcp_config.py` remains as the URL builder for the built-in servers (first-deployment seeds). +The source of truth for MCP server configuration is the `admin_mcp_servers` table (ORM: `core/db/models.py::AdminMcpServer`), read through `core/capabilities/mcp/service.py::McpServerConfigService` with a 30-second TTL cache, using native client configuration (`transport / command / args / env / url / headers`). `core/config/mcp_config.py` remains as the URL builder for the built-in servers (first-deployment seeds). Whether a server is **visible to the model** additionally passes through [catalog](catalog.md) gating: each entry in the `mcp` section of `core/config/catalog.json` corresponds to a server_id, and a server whose `is_enabled(id, "mcp_server")` is false will not be registered with the agent even if connected. @@ -309,7 +306,7 @@ The `/config` system console's **MCP Tools → MCP Server Management** view maps - **Probe-on-create**: `_probe_connectivity` performs a real connection; failures are rejected before persisting; - **Toggle & ordering**: `POST /{id}/toggle` switches a server instantly (refreshing the catalog and the connection pool); - **Secret protection**: HTTP header values are encrypted at rest and returned as `***`; secret-looking `env_vars` values are masked as well; -- **Test & reload**: `POST /{id}/test` re-probes a single server; `POST /reload-pool` hot-rebuilds the connection pool. +- **Test & reload**: `POST /{id}/test` re-probes a single server; `automatic cache invalidation` refreshes discovery on the next run. - **Move to MCP Marketplace**: creates a credential-free marketplace snapshot from an existing remote HTTP/SSE MCP and disables the original global instance. The service then leaves the MCP server list and can be reviewed, @@ -329,7 +326,7 @@ Regular users can add remote MCP servers visible **only to themselves** (`api/ro - `POST /v1/me/mcp-servers`: add a private remote MCP over public HTTP or HTTPS using HTTP/SSE transports; the user entry point deliberately forbids stdio (no arbitrary command execution on the server), and DNS/IP validation blocks localhost, private, link-local, and reserved addresses; probe-on-create means unreachable endpoints are never persisted. HTTP is plaintext, so HTTPS remains recommended for production; - `DELETE /v1/me/mcp-servers/{id}`: remove one's own private MCP. -Implementation reuses the same `admin_mcp_servers` table: `owner_user_id` = current user for owner isolation, auto-generated `umcp_` server IDs to avoid clashes, and `is_stable=False` to keep them out of the warmup pool. HTTP header values are encrypted with Fernet at rest and decrypted only at runtime. The feature is gated by the per-user `can_add_mcp` permission flag (open by default in the single-tenant Community Edition; granted per user by organisation admins in the Enterprise Edition — see [editions](../editions/overview.md)). +Implementation reuses the same `admin_mcp_servers` table: `owner_user_id` = current user for owner isolation, auto-generated `umcp_` server IDs to avoid clashes. HTTP header values are encrypted with Fernet at rest and decrypted only at runtime. The feature is gated by the per-user `can_add_mcp` permission flag (open by default in the single-tenant Community Edition; granted per user by organisation admins in the Enterprise Edition — see [editions](../editions/overview.md)). ## MCP marketplace @@ -409,12 +406,12 @@ docker-compose up -d --build mcp | `src/backend/extensions/mcp_servers/_serve.py` | Unified stdio / streamable-http entry point | | `src/backend/extensions/mcp_servers/registry.py` | Installation placement, exposure, packages, and base ports | | `src/backend/extensions/mcp_servers/_ports.py` | Deployment port mapping | -| `src/backend/core/llm/mcp_pool.py` | MCP connection pool (stdio pooled / HTTP per-request) | -| `src/backend/core/llm/mcp_manager.py` | MCPClient construction + bare tool-name restoration | +| `src/backend/core/llm/mcp/lifetime.py` | Native MCP client ownership and discovery | +| `src/backend/core/llm/mcp/manager.py` | MCPClient construction + bare tool-name restoration | | `src/backend/core/llm/tool_permissions.py` | Governed-tool registry, decisions, approval routing, and execution tickets | -| `src/backend/core/services/mcp_service.py` | DB-driven server config service (30 s cache) | +| `src/backend/core/capabilities/mcp/service.py` | DB-driven server config service (30 s cache) | | `src/backend/core/services/mcp_marketplace_service.py` | Marketplace publishing, review, installation, visibility, and security controls | -| `src/backend/core/services/mcp_oauth_service.py` | OAuth 2.1 login, SDK metadata discovery, encrypted token storage, and refresh | +| `src/backend/core/capabilities/mcp/oauth.py` | OAuth 2.1 login, SDK metadata discovery, encrypted token storage, and refresh | | `src/backend/core/services/mcp_marketplace_monitor.py` | Periodic remote tool-snapshot revalidation and drift monitoring | | `src/backend/core/config/mcp_config.py` | Built-in server URL builder (http://mcp:NNNN/mcp/) | | `src/backend/core/config/catalog.json` | Capability catalog: MCP enable/disable seeds | diff --git a/document/en/modules/model-providers.md b/document/en/modules/model-providers.md index 969c7049..6a333ac4 100644 --- a/document/en/modules/model-providers.md +++ b/document/en/modules/model-providers.md @@ -36,6 +36,8 @@ Precedence is **shared subagent model → an agent's existing explicit model → ### Wire protocol: Responses by default, chat completions only as a fallback +The pinned AgentScope version natively parses raw reasoning and summary events from Responses. The platform preserves each event's `item_id` and part index to prevent streaming failures and keep distinct reasoning items and parts separate. + OpenAI-compatible endpoints come in two wire protocols: the older `POST /chat/completions` and the newer `POST /responses`. The platform **defaults to Responses**, because it carries the model's reasoning as a first-class `reasoning` item that can be handed back verbatim on the next turn; `/chat/completions` has nowhere to put reasoning, so it has to be smuggled through a `reasoning_content` field. Chat completions is used only for endpoints that genuinely do not route `/responses` — commonly thin relay gateways. The choice lives in `extra_config.api_protocol` (`responses` / `chat_completions`). It is never inferred from the vendor name: saving a provider probes the endpoint, and only an endpoint that clears both stages is recorded as Responses-capable. @@ -134,7 +136,7 @@ chat_mode → hooks._resolve_chat_mode(agent.state) high/max → reasoning_effort=high/max (endpoint must declare support) ``` -Legacy clients that only send `enable_thinking` are mapped to medium / fast. This is the AgentScope 2.0 incarnation of the old 1.x "pre_reply dynamic-model hook" — the hook factories were refactored into middlewares, and `core/llm/hooks.py` now only hosts pure-function helpers. +Legacy clients that only send `enable_thinking` are mapped to medium / fast. This is the AgentScope 2.0 incarnation of the old 1.x "pre_reply dynamic-model hook" — the hook factories were refactored into middlewares, and `core/llm/runtime/hooks.py` now only hosts pure-function helpers. ## Environment variables and MCP subprocess injection @@ -147,7 +149,7 @@ Full variable list: [Environment Variables](../deployment/environment-variables. ## Personal API keys -Users can issue personal keys for programmatic access to the platform API (`api/routes/v1/api_keys.py` + `core/services/api_key_service.py`): +Users can issue personal keys for programmatic access to the platform API (`api/routes/v1/api_keys.py` + `core/auth/api_keys.py`): - Plaintext keys look like `sk-jx-` and are **returned exactly once at creation**; the DB stores only a SHA256 hash + prefix. Expiry options 7/30/90/180/365 days or never; enable/disable and revoke supported. - Usage: `Authorization: Bearer sk-jx-...`; the auth layer (`core/auth/backend.py`) recognizes the prefix and resolves the key to a user context via `resolve_api_key`, checking enabled/unrevoked/unexpired. @@ -176,9 +178,9 @@ Community Edition users can see their own token usage; organization-level billin | Role resolution service (DB + cache) | `src/backend/core/services/model_config.py` | | Role definitions / provider repository | `src/backend/core/db/model_repository.py` | | Management API | `src/backend/api/routes/v1/models.py` | -| Dynamic-model middleware | `src/backend/core/llm/middlewares.py::DynamicModelMiddleware`, helpers in `core/llm/hooks.py` | +| Dynamic-model middleware | `src/backend/core/llm/middlewares.py::DynamicModelMiddleware`, helpers in `core/llm/runtime/hooks.py` | | MCP env injection | `src/backend/core/services/model_config.py::get_mcp_env_overlay`, `core/config/runtime_env.py` | -| Personal API keys | `src/backend/api/routes/v1/api_keys.py`, `core/services/api_key_service.py`, `core/auth/backend.py` | +| Personal API keys | `src/backend/api/routes/v1/api_keys.py`, `core/auth/api_keys.py`, `core/auth/backend.py` | | Usage logs / billing | `src/backend/api/routes/v1/admin_usage_logs.py`, `api/routes/v1/admin_billing.py` | | Routing strategy | `src/backend/orchestration/strategy.py` | | Fail-fast on missing main model | `src/backend/api/routes/v1/chats/__init__.py::_ensure_main_model_configured` | diff --git a/document/en/modules/projects-myspace.md b/document/en/modules/projects-myspace.md index 3fb51354..b592ab92 100644 --- a/document/en/modules/projects-myspace.md +++ b/document/en/modules/projects-myspace.md @@ -35,7 +35,7 @@ Key ORM models (`src/backend/core/db/models/`): ## Projects -Routes: `src/backend/api/routes/v1/projects.py` (CE router table); business logic in `core/services/project_service.py` and `project_file_service.py`. +Routes: `src/backend/api/routes/v1/projects.py` (CE router table); business logic in `core/projects/service.py` and `project_file_service.py`. | Method | Path | Description | |---|---|---| @@ -71,7 +71,7 @@ In the project overview, the model and thinking-effort menu always opens downwar ### How project context enters a conversation -When a conversation is started inside a project (the request carries `project_id`), `core/services/session_operations/request_context.py` assembles the workflow context: +When a conversation is started inside a project (the request carries `project_id`), `core/chat/operations/request_context.py` assembles the workflow context: 1. Project metadata is loaded — `project_name`, `project_instructions`, the linked folder name and working directory. Instructions still come from `AGENTS.md`; cloud and local project files are not recursively enumerated; 2. `core/llm/factory/build.py` injects these into the system prompt via `_build_project_section` (`build_system_prompt(cfg, ctx=...)`); @@ -167,7 +167,7 @@ Permission model: `TeamMember.role` (owner/admin/member) + `file_permission` (vi Three complementary paths: -1. **Attachment injection (hooks)**: frontend requests send only `file_id`, `name`, and `mime_type` in each `attachments[]` item, never the file body. `core/llm/hooks.py` lazily downloads and parses by `file_id`, caching `parsed_text` and its summary; the first turn injects only metadata and a bounded preview of at most 5,000 characters, while xlsx previews also include total dimensions, headers, and a few data rows. The model reads content beyond the preview on demand through `read_artifact`; images still use multimodal injection. Every read validates `user_id` ownership to block forged cross-user access; +1. **Attachment injection (hooks)**: frontend requests send only `file_id`, `name`, and `mime_type` in each `attachments[]` item, never the file body. `core/llm/runtime/hooks.py` lazily downloads and parses by `file_id`, caching `parsed_text` and its summary; the first turn injects only metadata and a bounded preview of at most 5,000 characters, while xlsx previews also include total dimensions, headers, and a few data rows. The model reads content beyond the preview on demand through `read_artifact`; images still use multimodal injection. Every read validates `user_id` ownership to block forged cross-user access; 2. **Project file access**: a project conversation's system prompt supplies project instructions and directory scope without an automatic file inventory or count. The agent discovers files on demand through directory, search and read tools; 3. **Sandbox virtual filesystem**: with code execution enabled, the `/myspace/...` path maps MySpace into the sandbox (lazy loading + reverse sync); team projects map the team folder — see [Sandbox](./sandbox.md). @@ -176,8 +176,8 @@ Three complementary paths: | Path | Responsibility | |---|---| | `src/backend/api/routes/v1/projects.py` | Projects API | -| `src/backend/core/services/project_service.py` / `project_file_service.py` | Project business logic | -| `src/backend/core/services/project_scope.py` | `ProjectScope` (sandbox path scoping) | +| `src/backend/core/projects/service.py` / `project_file_service.py` | Project business logic | +| `src/backend/core/projects/scope.py` | `ProjectScope` (sandbox path scoping) | | `src/backend/api/routes/v1/myspace_folders.py` | Personal folders API | | `src/backend/api/routes/v1/artifacts.py` | Asset list / chat favorites / add-to-KB | | `src/backend/edition_ee/routes/team_files.py` | Team folders & files API (Enterprise Edition, EE) | @@ -186,7 +186,7 @@ Three complementary paths: | `src/backend/core/db/models/identity.py` | Shared identity ORM such as `UserFolder` | | `src/backend/edition_ee/db/models/identity.py` | `Team` / `TeamMember` / `TeamFolder` ORM (EE only) | | `src/backend/core/db/models/artifact.py` | `Artifact` ORM | -| `src/backend/core/llm/hooks.py` | Attachment context injection (`_build_file_context`, etc.) | +| `src/backend/core/llm/runtime/hooks.py` | Attachment context injection (`_build_file_context`, etc.) | | `src/backend/core/llm/factory/build.py` | Project section injection into the system prompt | | `src/backend/core/llm/tools/myspace_vfs/__init__.py` | MySpace ↔ sandbox mapping layer | | `src/frontend/src/components/projects/` | Project frontend components | @@ -262,11 +262,8 @@ staleness is allowed. Failed registrations, missing files and failed downloads a revisited at the next normal synchronization, without continuous background retries. Before upgrading, stop writers and back up the database and myspace_cache. -Preview with `python src/backend/scripts/clean_myspace_duplicates.py --all`. -Apply only with `--apply --backup-dir `. -Cleanup deletes every live record in each duplicate group and its mirror path; -it does not retain the newest record. Tombstones prevent historical backfill from -resurrecting deleted identities. Original objects and recovery journals are retained. +Inspect duplicate names and review a cleanup and recovery plan before upgrading. +There is no standalone deduplication script. Then upgrade Alembic (EE: personalname01; CE: ce_0018), resume services and run reconciliation and end-to-end checks. Migration refuses remaining duplicates instead of silently deleting data in another environment. @@ -291,7 +288,7 @@ Custom environments bind subagents, plugins, skills and connectors. The main age The backend limits capability registration, explicit invocation, restored session capabilities, subagents and dynamic plugin loading. An empty custom environment never falls back to all resources. Changes apply from the next turn; running responses and their derived jobs keep the original configuration. Conversations outside the project are unaffected. Basic file and conversation tools retain their existing permission rules. -Implementation follows separate responsibilities: `core/project_environment/` validates configuration and resolves dependencies without catalog access. The resource service reads authorized inventory, the environment service saves and previews configuration, and the runtime policy freezes and narrows registration and invocation. The frontend editor composes the interface; `useProjectEnvironment` loads resources, previews dependencies and handles marketplace acquisition. Resource pickers display explicit bindings and inferred dependencies. +Implementation follows separate responsibilities: `core/projects/environment/` validates configuration and resolves dependencies without catalog access. The resource service reads authorized inventory, the environment service saves and previews configuration, and the runtime policy freezes and narrows registration and invocation. The frontend editor composes the interface; `useProjectEnvironment` loads resources, previews dependencies and handles marketplace acquisition. Resource pickers display explicit bindings and inferred dependencies. ### Online cloud document editing diff --git a/document/en/modules/prompts.md b/document/en/modules/prompts.md index 6277e574..44e044d5 100644 --- a/document/en/modules/prompts.md +++ b/document/en/modules/prompts.md @@ -30,7 +30,7 @@ src/backend/prompts/prompt_text/default/system/ └── 40_format.system.md # output format + [ref:tool-N] citation contract ``` -After the base prompt, runtime appends dynamic sections per context: the tools & skills notice (`_TOOLS_AND_SKILLS_NOTICE`), the lightweight KB catalog (`prompts/kb_lite_section.py`), the project-mode section (`prompts/project_section.py`, injected only for project chats), the code-execution segment and batch-mode hint (appended by `factory/build.py`), and the sub-agent routing table (`core/llm/subagent_tool.py::build_subagent_prompt_section`). +After the base prompt, runtime appends dynamic sections per context: the tools & skills notice (`_TOOLS_AND_SKILLS_NOTICE`), the lightweight KB catalog (`prompts/kb_lite_section.py`), the project-mode section (`prompts/project_section.py`, injected only for project chats), the code-execution segment and batch-mode hint (appended by `factory/build.py`), and the sub-agent routing table (`core/llm/subagents/tool.py::build_subagent_prompt_section`). ### Caching @@ -49,7 +49,7 @@ Pool management and runtime assembly share this read path. After an active versi ### Execution manifest and runtime binding -Before an Agent can execute, `core/llm/execution_manifest.py::PromptManifestBuilder` records the exact final request surface. Every ordered prompt section records `id`, origin, trust level, priority, cache class, budget, token estimate, content hash and version. Every final tool definition similarly hashes its name, complete description, canonical parameter schema, permission policy and recovery policy. Tool definitions are sorted by stable id; prompt sections retain their deliberate prefix order. +Before an Agent can execute, `core/llm/runtime/execution_manifest.py::PromptManifestBuilder` records the exact final request surface. Every ordered prompt section records `id`, origin, trust level, priority, cache class, budget, token estimate, content hash and version. Every final tool definition similarly hashes its name, complete description, canonical parameter schema, permission policy and recovery policy. Tool definitions are sorted by stable id; prompt sections retain their deliberate prefix order. The builder emits `prompt_hash`, `prompt_manifest_hash`, `tool_manifest_hash`, `context_hash` and one `aggregate_hash`. Manifest construction and AgentScope model input consume the same run-scoped frozen tool/skill snapshot, so the recorded surface cannot differ from the request. Progressive plugin activation invalidates that snapshot and publishes an explicit next `surface_generation` before the next model call. @@ -59,15 +59,15 @@ Prompt text, project instructions, file names, tool descriptions and schemas are ### Canonical Context IR -Immediately before every model call, `core/llm/context_ir.py` converts the complete request context into immutable `ContextItem` records. User input, assistant history, account identity, frozen memory, project/file material, attachments, tool calls/results, compaction summaries, steer messages and harness reminders therefore carry machine-readable `kind`, `origin`, `trust`, visibility, priority, item budget, truncation policy, content reference/hash, cache class and creation sequence. Compatibility rendering may still use a `user` role for memory and mid-turn reminders, but their provenance is no longer inferred from XML tags or from that role. +Immediately before every model call, `core/llm/context/ir.py` converts the complete request context into immutable `ContextItem` records. User input, assistant history, account identity, frozen memory, project/file material, attachments, tool calls/results, compaction summaries, steer messages and harness reminders therefore carry machine-readable `kind`, `origin`, `trust`, visibility, priority, item budget, truncation policy, content reference/hash, cache class and creation sequence. Compatibility rendering may still use a `user` role for memory and mid-turn reminders, but their provenance is no longer inferred from XML tags or from that role. `ContextAssembler` applies a deterministic order and an explicit message budget: the model window minus a 15% response reserve and the canonical final-tool-schema reserve. Critical system/current-user items cannot be silently displaced; optional items are selected by priority; text and tool outputs use declared head/tail policies; and every tool call/result pair is kept, shortened or excluded as one unit. The sanitized context manifest explains every inclusion, truncation and exclusion without retaining plaintext. Its hash is folded into the request's `context_hash` and `aggregate_hash`, then rebound to the run's Asset Bundle. -`core/llm/context_adapter.py::AgentScopeContextAdapter` is the sole Context IR ↔ AgentScope message conversion seam. Persisted session metadata survives history replay, while final model roles and SSE behavior remain compatible. `ManifestBoundAgent` performs final assembly through the public `Agent.reply` path after the run-scoped tool/skill surface has been frozen. +`core/llm/context/adapter.py::AgentScopeContextAdapter` is the sole Context IR ↔ AgentScope message conversion seam. Persisted session metadata survives history replay, while final model roles and SSE behavior remain compatible. `ManifestBoundAgent` performs final assembly through the public `Agent.reply` path after the run-scoped tool/skill surface has been frozen. ## Prompt version pool (prompt_versions) -The pool stores multiple prompt sets in a single `ContentBlock(id="prompt_versions")` row with payload `{active: {kind: version_id}, versions: [...]}`; the service layer is `core/services/prompt_version_service.py`: +The pool stores multiple prompt sets in a single `ContentBlock(id="prompt_versions")` row with payload `{active: {kind: version_id}, versions: [...]}`; the service layer is `core/llm/prompt_versions.py`: - **Six kinds** (`VALID_KINDS`): `system` (main agent), `code_exec` (code-execution segment), `distillation` (skill distillation), `plan_mode` (plan mode), `subagents` (independent prompts for the platform-default Explorer, Worker, and Reviewer), and `turbo` (the standalone turbo-mode prompt). - Each version carries `(kind, id, name, description, parts[])`, a part being `{part_id, display_name, content, sort_order, is_enabled}`. @@ -126,18 +126,6 @@ Prompts live only in the database and never ship with code; migration across env Both accept `ADMIN_TOKEN` or `CONFIG_TOKEN`. Snapshots are validated against `PROMPT_BLOCK_MAP`, so docs snapshots and prompt snapshots cannot be imported through the wrong endpoint. -### Scripts (src/backend/scripts/) - -```bash -# Export (via a running backend API; --database-url for direct DB access also works) -python scripts/export_content.py --api-url http://localhost:3000/api --only prompts -# → scripts/exported/prompts_snapshot_.json - -# Import into the target environment (use the target machine's ADMIN_TOKEN) -python scripts/import_content.py --api-url http:///api --prompts prompts_snapshot_.json -# supports --no-overwrite / --dry-run -``` - The same flow works for offline production (image-pack delivery, persistent DB volume): copy the snapshot in alongside the image pack and `curl -X POST .../v1/content/prompts/import` inside the backend container — no restart required. When migrating to a differently-branded environment, review the snapshot manually and rewrite brand-specific wording in context before importing (never do mechanical find-and-replace). ## Source map @@ -147,10 +135,9 @@ The same flow works for offline production (image-pack delivery, persistent DB v | Runtime assembly + caching | `src/backend/prompts/prompt_runtime.py` | | Providers (filesystem/inline/minimal) | `src/backend/prompts/provider.py` | | Config (provider/parts/PROMPT_DIR) | `src/backend/prompts/prompt_config.py`, `prompts/config/default.json` | -| Version pool service | `src/backend/core/services/prompt_version_service.py` | +| Version pool service | `src/backend/core/llm/prompt_versions.py` | | Console routes | `src/backend/api/routes/v1/admin_prompts.py` | -| Migration endpoints (export/import) | `src/backend/api/routes/v1/content.py`, `core/content/content_blocks.py` | -| Migration scripts | `src/backend/scripts/export_content.py`, `scripts/import_content.py` | +| Migration endpoints (export/import) | `src/backend/api/routes/v1/content.py`, `core/files/content/content_blocks.py` | | System prompt fallback files | `src/backend/prompts/prompt_text/default/system/` | | Scenario prompt fallbacks | `src/backend/prompts/prompt_text/{code_exec,distillation,plan_mode,turbo}/` | | Dynamic sections | `src/backend/prompts/kb_lite_section.py`, `prompts/project_section.py` | diff --git a/document/en/modules/sandbox.md b/document/en/modules/sandbox.md index c0921ff9..2ce0c57c 100644 --- a/document/en/modules/sandbox.md +++ b/document/en/modules/sandbox.md @@ -82,7 +82,7 @@ Registration splits by the nature of the file — it is **not** a blanket push: Other points: -- **Registration attaches no chat, and pins no workspace card**: MySpace is one per-user directory that every session's sandbox mounts, so a filesystem event carries a path and no chat identity — who wrote it is simply not knowable here. Registration therefore records only the owning user (`Artifact.chat_id` stays null), and the registry never pins: a card can only be pinned by the run that wrote the file (the model calling `pin_to_workspace`, or that run's own Write / delivery guard — all through the `core.llm.workspace` ContextVar), where both "which chat" and "which file" are certain. Picking "whichever run of this user happens to be live" instead used to file one chat's output under another chat and surface it in that chat's workspace. +- **Registration attaches no chat, and pins no workspace card**: MySpace is one per-user directory that every session's sandbox mounts, so a filesystem event carries a path and no chat identity — who wrote it is simply not knowable here. Registration therefore records only the owning user (`Artifact.chat_id` stays null), and the registry never pins: a card can only be pinned by the run that wrote the file (the model calling `pin_to_workspace`, or that run's own Write / delivery guard — all through the `core.llm.runtime.workspace` ContextVar), where both "which chat" and "which file" are certain. Picking "whichever run of this user happens to be live" instead used to file one chat's output under another chat and surface it in that chat's workspace. - **The reference point is the artifact record, not the mirror cache**: with the bind mount on, the sandbox's `/workspace/myspace/{uid}` *is* the backend's `myspace_cache/{uid}`, so "sandbox file matches mirror cache" is trivially true and can no longer mean "already registered". That mismatch used to make write-back a permanent no-op. - **"Newer on disk than in the ledger" is the only signal**: when the backend writes into the mirror itself (materialising a UI upload, restoring a declined change) it aligns the file's mtime to the registration time, so the watcher never mistakes the backend's own write for a fresh sandbox change — no separate "I wrote this" bookkeeping is needed. - **Every worker watches; a claim deduplicates**: the confirmation bar lives in the process running that chat, so the watcher has to run in every worker (the cost is inotify watches scaling as workers × directories against `fs.inotify.max_user_watches`; raise it on deployments with very many folders — exceeding it drops events **silently**); one change is still handled once, via the claim in `core.infra.ephemeral`, and preferably by the worker hosting that user's chat (only it can raise the bar). @@ -186,7 +186,7 @@ fix on both execution surfaces. Updating only the cloud does not fix old clients ## Administrator sandbox management - **Read-only monitoring (security console)**: `api/routes/v1/config_security.py` exposes `/v1/config/security/sandbox/*` — overview, instance list, per-instance detail, snapshot list, rebuild history and effective configuration; everything goes through the providers' `admin_*` interfaces, with the UI trimmed by each provider's `admin_capabilities()` declaration (script_runner can't enumerate instances, so those columns are greyed out). -- **Dependency rebuild (Enterprise EE)**: `api/routes/v1/admin_sandbox.py` (`/v1/admin/sandbox/*`) aggregates the pip/apt dependencies declared by all skills (`core/services/skill_deps_aggregator.py`) and lets an admin trigger a sandbox image rebuild with one click: the `script-runner` / `opensandbox` targets run a local `docker compose build`, while the `cube` target rebuilds the template on the remote node over SSH and hot-swaps it (`core/services/sandbox_rebuild_service.py` + `cube_template_builder`), with per-run status and logs. New skill dependencies get baked into the images without hand-editing Dockerfiles. +- **Dependency rebuild (Enterprise EE)**: `api/routes/v1/admin_sandbox.py` (`/v1/admin/sandbox/*`) aggregates the pip/apt dependencies declared by all skills (`core/capabilities/skills/deps/aggregator.py`) and lets an admin trigger a sandbox image rebuild with one click: the `script-runner` / `opensandbox` targets run a local `docker compose build`, while the `cube` target rebuilds the template on the remote node over SSH and hot-swaps it (`core/services/sandbox_rebuild_service.py` + `cube_template_builder`), with per-run status and logs. New skill dependencies get baked into the images without hand-editing Dockerfiles. ## Key environment variables @@ -226,7 +226,7 @@ Full list in the [environment variable reference](../deployment/environment-vari | `src/backend/core/sandbox/providers/opensandbox/pool.py` | Two-bucket warm pool | | `src/backend/core/sandbox/providers/cube/provider.py` | Cube remote-MicroVM provider (EE) | | `src/backend/core/llm/tools/sandbox_tool.py` | Bash / write_stdin / sandbox_put_artifact / sandbox_get_artifact | -| `src/backend/core/llm/offloader.py` | Overflow offloading to the session workspace | +| `src/backend/core/llm/context/offloader.py` | Overflow offloading to the session workspace | | `src/backend/api/routes/v1/admin_sandbox.py` | Dependency-rebuild admin API (EE) | | `src/backend/api/routes/v1/config_security.py` | Security-console read-only sandbox views | | `src/backend/core/services/sandbox_rebuild_service.py` | Image/template rebuild orchestration (EE) | @@ -273,5 +273,31 @@ OpenSandbox instance lists and counts use the controller's current running inventory instead of cached process handles. Controller query failures mean the count is unavailable, never a successful zero. Pool diagnostics include scope=worker and worker_pid. They describe one backend process and are not a -global per-user quota. This fix does not convert local pool limits into -cross-process admission limits. +global per-user quota. Controller admission now enforces the shared ceiling described below. + +## Shared capacity and maintenance ownership + +The controller now enforces OPENSANDBOX_POOL_MAX_TOTAL against Docker inventory +across backend workers, warm instances, restored sessions and computer instances. +Worker pool counts remain local diagnostics. Container creation holds a file lock +on the controller data volume; mount initialization stays outside that critical +section. Capacity exhaustion or a competing admission returns 429 for later retry. +Processes of the same controller must share its opensandbox_data volume. + +The leader discovers idle sessions from shared bindings, including sessions +created by other workers or surviving a restart. Identity-checked cleanup of a +missing binding does not block other sessions. Snapshot work stops with leadership; +GC deletes metadata only for the snapshot_id it processed. + +Output reads no longer prepare mounts or write object directory markers. +Directory initialization happens during creation, while execution and file access +still check authorization. An uncertain or timed-out health probe preserves the +FUSE helper; only confirmed missing or disconnected mounts permit replacement. +Command observation retries are bounded before durable recovery takes over. +Processes without a recoverable reference require confirmed termination, not loss +of their control handle merely because observation failed. + +Roll out both backend and OpenSandbox controller images. Updating only the backend +does not activate the shared capacity limit. + +Skill mounts require HOST_STORAGE_PATH to identify the host directory corresponding to backend STORAGE_PATH. Missing configuration fails explicitly; source-tree fallback is removed. Restart adoption requires explicit ownership metadata and never infers ownership from old entrypoints. diff --git a/document/en/modules/storage.md b/document/en/modules/storage.md index ab669079..83079ed6 100644 --- a/document/en/modules/storage.md +++ b/document/en/modules/storage.md @@ -12,7 +12,7 @@ Build the backend image before starting the bucket initializer. Object mounts us Existing installations must back up configuration, databases and file references before changing providers. Copy required objects and verify their SHA-256 values. Preserve the source objects. -The storage factory is core/storage/factory.py. S3StorageBackend uses boto3 and includes directory operations. LocalStorageBackend serves desktop local files. OSS and OBS adapters remain EE providers. +The storage factory is core/files/storage/factory.py. S3StorageBackend uses boto3 and includes directory operations. LocalStorageBackend serves desktop local files. OSS and OBS adapters remain EE providers. The artifact store supports S3, OSS and OBS. It keeps local metadata records and mirrors them into object storage. Historical download references remain valid when migrated objects keep their storage keys. diff --git a/document/zh-CN/api/overview.md b/document/zh-CN/api/overview.md index 3906b512..abadb591 100644 --- a/document/zh-CN/api/overview.md +++ b/document/zh-CN/api/overview.md @@ -59,7 +59,7 @@ HugAgentOS 后端是一个 FastAPI 应用，所有业务接口挂在 `/v1/*` 前 | 方式 | 凭据携带方法 | 适用范围 | 实现 | |---|---|---|---| | **会话 Cookie** | `Cookie: jx_session=`（登录后自动种下） | 所有面向最终用户的 `/v1/*` 接口（chats、projects、kb、memories…） | `get_current_user`：Cookie → Redis 会话查找 | -| **个人 API-Key** | `Authorization: Bearer sk-jx-...` | 与会话 Cookie 等价的用户身份，适合脚本/第三方集成调用 | `sk-jx-` 前缀识别；明文仅创建时返回一次，DB 只存 SHA256（`core/services/api_key_service.py`）；在 `/v1/me/api-keys` 自助管理 | +| **个人 API-Key** | `Authorization: Bearer sk-jx-...` | 与会话 Cookie 等价的用户身份，适合脚本/第三方集成调用 | `sk-jx-` 前缀识别；明文仅创建时返回一次，DB 只存 SHA256（`core/auth/api_keys.py`）；在 `/v1/me/api-keys` 自助管理 | | **Bearer token（mock/remote 模式）** | `Authorization: Bearer ` | `AUTH_MODE=mock`（开发：任意 token / 缺省 token 均放行为 mock 用户）或 `remote`（对接用户中心校验） | `get_current_user` 的兜底分支；`AUTH_MODE=session` 下不接受 | | **ADMIN_TOKEN** | `Authorization: Bearer ` | `/admin` 管理台后端（`/v1/admin/skills`、`/v1/admin/kb` 等）及内容块写接口 | `require_admin`（环境变量 `ADMIN_TOKEN`） | | **CONFIG_TOKEN** | `Authorization: Bearer ` | `/config` 系统配置台后端（`/v1/config/*`、`/v1/models`、`/v1/service-configs`、部分 `/v1/admin/*` 观测接口） | `require_config`（环境变量 `CONFIG_TOKEN`） | diff --git a/document/zh-CN/architecture/backend.md b/document/zh-CN/architecture/backend.md index ab242a68..5652e68f 100644 --- a/document/zh-CN/architecture/backend.md +++ b/document/zh-CN/architecture/backend.md @@ -128,7 +128,7 @@ src/backend/ `runner` 仅依赖不加载后端配置或数据库的进程和路径基础模块。OpenSandbox 专属组件留在自己的驱动目录；Cube 与 OpenSandbox 共用的进程、路径和账号环境能力由共享包提供。 -### core/agent_skills — 技能引擎 +### core/capabilities/skills — 技能引擎 | 文件 | 职责 | |---|---| @@ -144,12 +144,12 @@ src/backend/ | 子模块 | 职责与关键文件 | |---|---| | `core/chat` | workflow 上下文组装（`context.py`）、SSE 工具日志事件构造（`tool_log.py`） | -| `core/content` | 附件解析（`file_parser.py`）、KB 文档分块/关键词/向量化（`kb_processing.py`）、上传校验（`file_validation.py`）、产物读取与摘要（`artifact_reader/refs/summary.py`）、内容块导入导出（`content_blocks.py`）、`svg_fit.py` | +| `core/files/content` | 附件解析（`file_parser.py`）、KB 文档分块/关键词/向量化（`kb_processing.py`）、上传校验（`file_validation.py`）、产物读取与摘要（`artifact_reader/refs/summary.py`）、内容块导入导出（`content_blocks.py`）、`svg_fit.py` | | `core/kb` / `edition_ee/kb` | 自建知识库解析与 Milvus 向量库保留在 `core/kb`；Dify 客户端与外部检索适配器只存在于 `edition_ee/kb`（商业版 EE） | -| `core/artifacts` | 产物存储 `store.py`：本地 / OSS 双模式 | -| `core/infra` | 统一响应（`responses.py`）、异常（`exceptions.py`）、结构化日志（`logging.py`）、限流（`rate_limit.py`）、Redis 单例（`redis.py`）、指标（`metrics.py`）、后台任务注册表（`runtime_state.py`）、脱敏（`data_masking.py`）、蒸馏预算闸门（`distillation_budget.py`，商业版 EE） | +| `core/files/artifacts` | 产物存储 `store.py`：本地 / OSS 双模式 | +| `core/infra` | 统一响应（`responses.py`）、异常（`exceptions.py`）、结构化日志（`logging.py`）、Redis 单例（`redis.py`）、指标（`metrics.py`）、后台任务注册表（`runtime_state.py`）、脱敏（`data_masking.py`）、蒸馏预算闸门（`distillation_budget.py`，商业版 EE） | | `edition_ee/licensing` | 能力枚举、验签、时钟回拨防护、席位策略、中间件和 manager 全部只存在于商业版 EE；CE 树物理不含 License 包，由 `api/middleware/edition.py` 的 CE overlay 提供固定版本探针 | -| `core/storage` | 存储协议 `protocol.py` + 工厂 `factory.py`；`local.py`（CE）、`s3.py` / `oss.py`（商业版 EE） | +| `core/files/storage` | 存储协议 `protocol.py` + 工厂 `factory.py`；`local.py`（CE）、`s3.py` / `oss.py`（商业版 EE） | ## orchestration/ — 编排层 @@ -224,7 +224,7 @@ CE 路由位于 `api/routes/v1/`；已物理拆分的商业路由位于 `edition | 沙箱协议 | `src/backend/core/sandbox/protocol.py` | | 记忆流水线 | `src/backend/core/memory/pipeline.py` | | EE License 实现 / CE 版本中间件 | `src/backend/edition_ee/licensing/`、`src/backend/api/middleware/edition.py` | -| 技能引擎 | `src/backend/core/agent_skills/loader.py` | +| 技能引擎 | `src/backend/core/capabilities/skills/loader.py` | | MCP 注册表与端口 | `src/backend/extensions/mcp_servers/registry.py`、`_ports.py` | 空间 MCP 接口位于 `extensions/mcp_servers/space_files_mcp/`。注册表分别管理安装位置和暴露方式；混合模式通过桌面能力网关访问空间，修改操作接入统一审批。详见[空间文件挂载](../deployment/space-files.md)。 diff --git a/document/zh-CN/architecture/data-model.md b/document/zh-CN/architecture/data-model.md index 8a99e9f7..60ebd30b 100644 --- a/document/zh-CN/architecture/data-model.md +++ b/document/zh-CN/architecture/data-model.md @@ -185,5 +185,5 @@ user_feedback # 问题反馈审核 / | CE/EE 建表边界 | `src/backend/core/db/edition_tables.py` | | EE 迁移主链 | `src/backend/alembic/versions/` | | CE 迁移基线 | `ce/overlay/src/backend/alembic/versions/ce_0001_initial.py` | -| 提示词版本池服务 | `src/backend/core/services/prompt_version_service.py` | +| 提示词版本池服务 | `src/backend/core/llm/prompt_versions.py` | | 版本边界权威方案 | [版本与授权](../editions/overview.md) | diff --git a/document/zh-CN/architecture/overview.md b/document/zh-CN/architecture/overview.md index 9219e0e8..64b21a8b 100644 --- a/document/zh-CN/architecture/overview.md +++ b/document/zh-CN/architecture/overview.md @@ -50,7 +50,7 @@ HugAgentOS 是一个面向企业的 AgentOS，同时提供轻量本地运行形 ┌──────────────────────────────▼─────────────────────────────────────┐ │ 智能体层（src/backend/core/llm/） │ │ factory/build → AgentScope 2.0 ReActAgent │ -│ ToolCollector → Toolkit（自研工具 + 技能 + MCP 工具） │ +│ ToolAssembly → Toolkit（自研工具 + 技能 + MCP 工具） │ │ middlewares：动态模型/文件上下文/系统提醒 · offloader：超长结果落盘 │ │ prompts/prompt_runtime：系统提示词装配（DB 优先，文件兜底） │ └───────┬──────────────────────┬──────────────────────┬──────────────┘ @@ -203,11 +203,11 @@ RAG 则提供支撑决策所需的文档与证据。 ### 提示词 DB 化 -系统提示词不再硬编码：版本池存于 `content_blocks` 表（`id=prompt_versions`），由 `core/services/prompt_version_service.py` 管理 CRUD、激活与首启 seed；`prompts/prompt_runtime.py` 装配时按「DB 激活版本 → `prompts/prompt_text/default/system/` 文件兜底 → 最小硬编码兜底」三级降级，保证永不为空。运营可在管理台灰度切换提示词版本而无需发版。详见 [提示词系统](../modules/prompts.md)。 +系统提示词不再硬编码：版本池存于 `content_blocks` 表（`id=prompt_versions`），由 `core/llm/prompt_versions.py` 管理 CRUD、激活与首启 seed；`prompts/prompt_runtime.py` 装配时按「DB 激活版本 → `prompts/prompt_text/default/system/` 文件兜底 → 最小硬编码兜底」三级降级，保证永不为空。运营可在管理台灰度切换提示词版本而无需发版。详见 [提示词系统](../modules/prompts.md)。 ### MCP 工具独立进程 -每个工具是 `src/backend/extensions/mcp_servers//` 下的独立 MCP 服务器，统一运行在专用 `mcp` 容器内，以 streamable-http 常驻监听（端口表 `extensions/mcp_servers/_ports.py` 为单一真源），后端经 `core/llm/mcp_manager.py` 的 HTTP 客户端连接池复用连接。好处：工具崩溃不影响主进程、依赖隔离（文档编辑与导出已迁入沙箱技能）、可独立扩缩。CE 保留 7 个通用 Server；`query_database` 与 `ai_chain_information_mcp` 两个行业工具依赖内网数据源，归商业版 EE。详见 [MCP 工具](../modules/mcp-tools.md)。 +每个工具是 `src/backend/extensions/mcp_servers//` 下的独立 MCP 服务器，统一运行在专用 `mcp` 容器内，以 streamable-http 常驻监听（端口表 `extensions/mcp_servers/_ports.py` 为单一真源），后端经 `core/llm/mcp/manager.py` 的 HTTP 客户端连接池复用连接。好处：工具崩溃不影响主进程、依赖隔离（文档编辑与导出已迁入沙箱技能）、可独立扩缩。CE 保留 7 个通用 Server；`query_database` 与 `ai_chain_information_mcp` 两个行业工具依赖内网数据源，归商业版 EE。详见 [MCP 工具](../modules/mcp-tools.md)。 ### Run 与连接解耦 diff --git a/document/zh-CN/deployment/environment-variables.md b/document/zh-CN/deployment/environment-variables.md index e83e6ab5..ce6e741a 100644 --- a/document/zh-CN/deployment/environment-variables.md +++ b/document/zh-CN/deployment/environment-variables.md @@ -223,7 +223,7 @@ Config 管理台会分别保存 Dify、FastGPT、WeKnora 的 URL、API Key 与 | `OPENSANDBOX_PORT` | 全量仓库 `8910`；CE 不映射 | 控制器容器内固定 8080 | | `OPENSANDBOX_POOL_JUPYTER_MIN_IDLE` / `MAX_IDLE` | `.env.example` 1/3；compose 默认 2/3 | 用户会话池（历史配置名）；CE 最小预热数为 0 | | `OPENSANDBOX_POOL_LIGHT_MIN_IDLE` / `MAX_IDLE` | `2` / `5` | 轻量桶（一次性执行） | -| `OPENSANDBOX_POOL_MAX_TOTAL` | CE: `4`; full: `20` | 全池上限（含使用中） | +| `OPENSANDBOX_POOL_MAX_TOTAL` | CE: `4`; full: `20` | 控制器全局上限（所有 worker、预热与使用中实例）；backend 和控制器使用同一值 | | `OPENSANDBOX_SNAPSHOT_ENABLED` | `true` | 快照持久化总开关（idle 时 snapshot+kill，重连恢复） | | `OPENSANDBOX_SNAPSHOT_RETENTION_DAYS` | `7` | 快照保留天数（GC 周期清理） | | `OPENSANDBOX_SNAPSHOT_WAIT_TIMEOUT_S` | `120` | 等快照 Ready 的轮询上限（秒） | @@ -317,16 +317,10 @@ Config 管理台会分别保存 Dify、FastGPT、WeKnora 的 URL、API Key 与 License 细节见 [License 机制](../editions/license.md)。 -## 其他（限流、熔断、安全、告警） +## 其他（安全、告警） | 变量 | 默认值 | 说明 | 版本 | |---|---|---|---| -| `RATE_LIMIT_ENABLED` | `true` | 接口限流开关 | CE | -| `RATE_LIMIT_STORAGE` | `memory://` | 限流计数存储（可 `redis://...`） | CE | -| `RATE_LIMIT_GLOBAL` / `RATE_LIMIT_PER_USER` | `500/minute` / `50/minute` | 全局 / 单用户限流 | CE | -| `CB_USER_CENTER_THRESHOLD` / `CB_USER_CENTER_TIMEOUT` | `5` / `60` | 用户中心熔断器 | EE | -| `CB_MODEL_API_THRESHOLD` / `CB_MODEL_API_TIMEOUT` | `10` / `30` | 模型 API 熔断器 | CE | -| `CB_STORAGE_THRESHOLD` / `CB_STORAGE_TIMEOUT` | `5` / `60` | 存储熔断器 | CE | | `CORS_ORIGINS` | compose 默认 `http://localhost:3000,http://localhost:5173` | 允许跨域来源 | CE | | `ENABLE_SECURITY_HEADERS` | （注释，建议生产 `true`） | 安全响应头 | CE | | `AUDIT_LOG_RETENTION_DAYS` / `AUDIT_LOG_EXPORT_ENABLED` | `90` / `true`（注释示例） | 审计日志保留 / 导出 | EE | diff --git a/document/zh-CN/deployment/offline-production.md b/document/zh-CN/deployment/offline-production.md index db13e3e6..820d28e6 100644 --- a/document/zh-CN/deployment/offline-production.md +++ b/document/zh-CN/deployment/offline-production.md @@ -43,13 +43,8 @@ ### 1. 联网侧导出 -```bash -python src/backend/scripts/export_content.py \ - --api-url http://localhost:3001 --only prompts -# 产出 src/backend/scripts/exported/prompts_snapshot_.json -``` - -（也支持 `--database-url` 直连库导出；`ADMIN_TOKEN` 自动从 `.env` 读取。） +通过 `GET /v1/content/prompts/export` 导出快照。请求使用源环境的管理员或系统配置令牌。 +将响应的 `data` 对象保存为 `prompts_snapshot_.json`，不要保存完整响应信封。 ### 2. 拷贝 @@ -80,5 +75,5 @@ API 导入会自动失效提示词缓存，**无需重启 backend**。 | EE 完整镜像包（deploy 分支） | `deploy_kit/save_pack_ee.sh` | | EE 编译镜像（deploy 分支） | `docker/Dockerfile.backend-compiled`、`docker/Dockerfile.mcp-compiled` | | Compose 覆盖（deploy 分支） | `docker-compose.prod.yml`、`docker-compose.deploy.yml`；运行时以交付 kit 为准 | -| 提示词导出 / 导入 | `src/backend/scripts/export_content.py`、`src/backend/scripts/import_content.py` | -| 提示词版本池 | `src/backend/core/services/prompt_version_service.py` | +| 提示词导出 / 导入 | `src/backend/api/routes/v1/content.py` | +| 提示词版本池 | `src/backend/core/llm/prompt_versions.py` | diff --git a/document/zh-CN/deployment/quick-install.md b/document/zh-CN/deployment/quick-install.md index 961e6099..51912d98 100644 --- a/document/zh-CN/deployment/quick-install.md +++ b/document/zh-CN/deployment/quick-install.md @@ -189,5 +189,5 @@ hugagent doctor # 环境自检（Python 版本、端口占用、数据目录 | 前端静态托管 + `/api` 桥接 | `src/backend/api/local_hosting.py` | | MCP / 沙箱子进程督管 | `src/backend/orchestration/local_subprocess.py` | | 进程内事件流 / 短时状态 | `src/backend/orchestration/run_event_stream.py`、`src/backend/core/infra/ephemeral.py`（`REDIS_URL` 留空即启用） | -| 内置 MCP 目录种子 | `src/backend/core/services/mcp_service.py`（`seed_builtin_mcp_servers_if_empty`） | +| 内置 MCP 目录种子 | `src/backend/core/capabilities/mcp/service.py`（`seed_builtin_mcp_servers_if_empty`） | | 环境变量参考 | [environment-variables.md](environment-variables.md)（「无 Docker 本地模式」段） | diff --git a/document/zh-CN/deployment/space-files.md b/document/zh-CN/deployment/space-files.md index ea24a7cc..275082d5 100644 --- a/document/zh-CN/deployment/space-files.md +++ b/document/zh-CN/deployment/space-files.md @@ -31,19 +31,11 @@ S3 服务部署在本机时，请使用容器可达的服务地址。 控制器不再持有 FUSE 连接，无需共享传播目录或宿主挂载准备脚本。 沙盒文件操作仍使用普通用户。 -## 文件导入 +## 存量文件迁移 -先备份数据库和对象数据。暂停对目标目录的写入。 -对每个用户或团队生成导入清单： - -```bash -PYTHONPATH=src/backend python -m scripts.import_space_files --user USER_ID -PYTHONPATH=src/backend python -m scripts.import_space_files --team TEAM_ID -``` - -确认目标目录和文件清单后，追加 `--apply` 执行导入。 -工具拒绝同名冲突，并校验复制后的内容。 -工具保留源对象和文件引用。导入中断后可重新运行。 +独立文件导入命令已移除。迁移存量文件前，先备份数据库和对象数据， +暂停目标目录写入，制定并审核迁移方案。核对目标路径、同名冲突和文件内容， +保留源对象及文件引用。 ## 验证 diff --git a/document/zh-CN/development/backend.md b/document/zh-CN/development/backend.md index d8c53c55..414a2dc2 100644 --- a/document/zh-CN/development/backend.md +++ b/document/zh-CN/development/backend.md @@ -41,7 +41,7 @@ docker-compose logs -f backend | `make migrate` | `alembic upgrade head` | | `make migrate-new msg="..."` | `alembic revision --autogenerate -m "..."` | | `make migrate-down` / `migrate-history` | 回滚一步 / 查看历史 | -| `make db-reset` / `db-seed` | 重置（破坏性）/ 种子数据 | +| `make db-reset` | 重置数据库（破坏性） | | `make build` / `up` / `down` / `logs` / `ps` 等 | docker-compose 包装 | > ⚠️ 已提交代码并非 formatter-clean：**不要对未改动的文件整体跑 black/isort**（会把真实 diff 淹没在格式化噪音里），只格式化自己改的范围。 @@ -175,7 +175,7 @@ PYTHONPATH=src/backend python -m extensions.mcp_servers._mcp._selftest ## 新增技能（Agent Skill）步骤 -技能加载是多源架构（`core/agent_skills/config.py`）：内置（`extensions/skills/default/`，always-on）、admin（DB / `/app/storage/admin_skills/`）、用户、项目四个加载源按优先级合并；`extensions/skills/marketplace/` 是安装制技能市场种子，由 marketplace 服务单独扫描，**不在默认加载源内**。 +技能加载是多源架构（`core/capabilities/skills/config.py`）：内置（`extensions/skills/default/`，always-on）、admin（DB / `/app/storage/admin_skills/`）、用户、项目四个加载源按优先级合并；`extensions/skills/marketplace/` 是安装制技能市场种子，由 marketplace 服务单独扫描，**不在默认加载源内**。 新增**内置技能**： @@ -208,7 +208,7 @@ PYTHONPATH=src/backend python -m extensions.mcp_servers._mcp._selftest | ORM / 建表边界 | `src/backend/core/db/models.py`、`src/backend/core/db/edition_tables.py` | | MCP 端口表 / 连接配置 | `src/backend/extensions/mcp_servers/_ports.py`、`src/backend/core/config/mcp_config.py` | | 能力目录 | `src/backend/core/config/catalog.json` | -| 技能加载 | `src/backend/core/agent_skills/`（`config.py` / `loader.py` / `registry.py`） | +| 技能加载 | `src/backend/core/capabilities/skills/`（`config.py` / `loader.py` / `registry.py`） | | 工作流编排 | `src/backend/orchestration/workflow.py` | 相关阅读：[前端开发指南](frontend.md) · [API 总览](../api/overview.md) · [MCP 工具](../modules/mcp-tools.md) · [技能系统](../modules/agent-skills.md) diff --git a/document/zh-CN/development/workbench-migration.md b/document/zh-CN/development/workbench-migration.md index f1593a59..5585d389 100644 --- a/document/zh-CN/development/workbench-migration.md +++ b/document/zh-CN/development/workbench-migration.md @@ -44,24 +44,7 @@ SDK CLI 只校验声明，不迁移数据库或业务数据。 # 工作台配置迁移与恢复 -迁移是显式运维操作，不会随启动自动执行。先在隔离数据库副本完成演练；真实配置迁移需要单独授权。当前命令只接受本机隔离副本。 - -## 盘点与切换 - -1. 停止副本的所有写入，包括内容导入、用户偏好、插件安装和后台写入。--writes-frozen 只是操作员确认，不会自动停止服务。 -2. 配置 WB20_COPY_DATABASE_URL。SQLite 文件须在当前源码目录的 .scratch/workbench-20 下。PostgreSQL 须为回环主机且数据库名以 workbench_20 开头。不要将真实库设置为演练目标。 -3. 创建权限为 0700 的快照目录。运行如下命令： - -```bash -PYTHONPATH=src/backend python -m scripts.workbench_migration inventory \ - --snapshot /absolute/private/path/before.json -PYTHONPATH=src/backend python -m scripts.workbench_migration apply \ - --snapshot /absolute/private/path/before.json --digest --writes-frozen -``` - -盘点不写数据库。报告包含数量、来源版本、脱敏位置和错误码；不打印 URL、用户标识、票据或秘密。完整迁移范围快照保存在 0600 文件中，包含原配置、偏好、角色/团队关系、插件声明和工作台数据。它不是业务文件或完整数据库备份，须与完整数据库/文件备份共同保管。 - -摘要匹配且源未变化时，切换在单一事务中执行。失败回滚事务。切换前后快照的摘要记录在 workbench_migration:v1 receipt 中。原始 app_config 行和旧导航列表移除；新目录只读取规范来源。重复外部 ID 的全部副本、未知 ID、危险 URL 和未知字段隔离，不产生入口或授权。原始副本保留在受保护快照。 +独立迁移命令已移除。以下内容记录内部迁移与恢复约束，不构成可直接执行的运维流程。现有部署需要迁移时，应先备份并制定针对目标版本的方案，在隔离副本验证后单独授权执行。 ## 保留的语义 @@ -69,7 +52,7 @@ PYTHONPATH=src/backend python -m scripts.workbench_migration apply \ - 外部条目顺序和启停保留。默认侧栏转换为策略。已有现代偏好优先；已确认旧 preferences.pinned_apps/app_order 转换为 web 偏好。null 继承，空列表明确取消固定。聊天排序、主题和其他偏好保持不变。 - 无法映射的权限引用保留原值并报告，不成为新授权。旧插件 UI v1/v2 不改写为 v3。不会按名称或 slug 绑定新安装。 - 旧认证配置 workbench_external_auth:legacy: 保持原 key。缺少目标侧适配时报告 auth_configuration_required。管理员须显式审核登录方式；真实旧目标尚未验证时，不得宣称外部登录无回归。不得恢复通用 SSO token 拼接。 -- 同服务恢复保留 workbench_origin_identity。克隆到另一服务须先在克隆副本重新盘点，再执行 clone-origin；旧插件 key 不自动映射为新来源授权。跨来源偏好和数据须另行核对。 +- 同服务恢复保留 workbench_origin_identity。克隆到另一服务须先在克隆副本重新盘点，再重建服务来源标识；旧插件 key 不自动映射为新来源授权。跨来源偏好和数据须另行核对。 ## 对外版本窗口 @@ -77,13 +60,7 @@ content-v1 接口在工作台协议 1.x 内保留，协议 2.0 迁移前另行 ## 恢复与新增数据 -先冻结全部写入，再执行： - -```bash -PYTHONPATH=src/backend python -m scripts.workbench_migration rollback \ - --snapshot /absolute/private/path/before.json --digest \ - --delta /absolute/private/path/delta.json --writes-frozen -``` +恢复前冻结全部写入，并按已审核的恢复方案操作。 恢复前导出受保护增量。已映射普通旧外链的修改可回放到旧格式。新身份、应用状态、资源、权限、插件、认证或策略变化不能安全表达时，返回 delta_not_representable 并保留新数据。不要仅恢复旧快照。继续运行已验证的新版本，核对增量并另行制定迁移方案。恢复 receipt 幂等；已恢复批次不能直接重复 apply，应创建新的审核批次。 diff --git a/document/zh-CN/editions/build-ce.md b/document/zh-CN/editions/build-ce.md index ecd3efbb..8ac88134 100644 --- a/document/zh-CN/editions/build-ce.md +++ b/document/zh-CN/editions/build-ce.md @@ -26,7 +26,7 @@ manifest 按处理顺序分为以下几段： glob 模式（相对仓库根），命中即不拷贝。覆盖： -- **后端 EE 模块**：完整的 `edition_ee/**` 实现根（团队/RBAC、SSO、License 验签与闸门、EE ORM、Dify 集成），以及云存储（`core/storage/s3.py`、`oss.py`）、Cube 沙箱 provider、记忆审计、技能蒸馏等 EE 服务； +- **后端 EE 模块**：完整的 `edition_ee/**` 实现根（团队/RBAC、SSO、License 验签与闸门、EE ORM、Dify 集成），以及云存储（`core/files/storage/s3.py`、`oss.py`）、Cube 沙箱 provider、记忆审计、技能蒸馏等 EE 服务； - **EE 路由**：`api/routes/v1/admin_*.py`、`config_*.py`、`audit.py`、`auth.py`、`team_files.py`、`service_configs.py`、`data_sources.py`、`db_metadata.py`、`gateway_*.py`； - **行业 MCP**：`extensions/mcp_servers/query_database_mcp/**`、`ai_chain_information_mcp/**`； - **行业专属前端与种子**：产业链 Canvas、资讯 / 企业画像渲染器、行业 API 客户端、图标资源， @@ -82,12 +82,12 @@ CE 前端不按工具名猜测远程 MCP 的业务结构。用户接入的远程 | `.github/workflows/desktop-release.yml` | 公开 CE 桌面发版 workflow，包含 release tag / 版本前置门禁 | | `src/backend/api/routes/v1/__init__.py` | CE 路由注册表；`EE_ROUTERS` 恒为空 | | `src/backend/core/auth/permissions_iface.py` | 单租户 owner-only 权限接口；不导出团队权限函数 | -| `src/backend/core/services/artifact_edition.py` | 个人空间 artifact 作用域接口；不暴露团队字段、权限或仓储方法 | +| `src/backend/core/files/artifacts/edition.py` | 个人空间 artifact 作用域接口；不暴露团队字段、权限或仓储方法 | | `src/backend/core/llm/tools/edition_{myspace,myspace_vfs,artifact_recovery}.py` | 个人空间工具、VFS 与恢复接口；组织空间实现不进入 CE | | `src/backend/core/config/edition_display_names.py` | CE 工具展示名；不包含团队工具名称 | | `src/backend/core/memory/audit.py` | 记忆审计 no-op stub（同名接口、不落数据） | | `src/backend/alembic/versions/ce_000{1,2}_*.py` | CE 独立迁移链基线与幂等 Schema 升级（见下节） | -| `src/backend/core/services/edition_startup.py` | CE 启动接缝；为 EE 数据源恢复、蒸馏调度提供 no-op 实现，避免导入已裁除模块 | +| `src/backend/orchestration/workers/edition_startup.py` | CE 启动接缝；为 EE 数据源恢复、蒸馏调度提供 no-op 实现，避免导入已裁除模块 | | `src/backend/tests/ce_release/` | 可执行 CE 发布回归套件，覆盖登录、私有技能/MCP、数据库升级及启动/Compose 契约 | | `src/backend/api/routes/v1/{agents,content,kb_models,projects}.py` | 去除管理端点与组织字段后的 CE API 契约 | | `src/backend/extensions/mcp_servers/_ports.py` | 7 个通用工具的端口表（EE 行业工具与已退役端口标注 reserved） | diff --git a/document/zh-CN/modules/agent-skills.md b/document/zh-CN/modules/agent-skills.md index a1927829..a4ba477a 100644 --- a/document/zh-CN/modules/agent-skills.md +++ b/document/zh-CN/modules/agent-skills.md @@ -29,11 +29,11 @@ └── _scripts.json # 可选：脚本白名单声明（缺省时按扩展名自动探测） ``` -解析与数据结构在 `core/agent_skills/registry.py`：`AgentSkillMetadata`（轻量元数据，列表场景用）与 `AgentSkillSpec`（含完整指令，执行场景用）。`_scripts.json` 缺失时 `loader._auto_detect_scripts` 按扩展名自动生成白名单，用户无需手写。 +解析与数据结构在 `core/capabilities/skills/registry.py`：`AgentSkillMetadata`（轻量元数据，列表场景用）与 `AgentSkillSpec`（含完整指令，执行场景用）。`_scripts.json` 缺失时 `loader._auto_detect_scripts` 按扩展名自动生成白名单，用户无需手写。 ## 多源加载与优先级 -`core/agent_skills/loader.py::MultiSourceSkillLoader` 通过后端抽象层（`backends/`：filesystem / database / composite）从多个来源加载技能，同 ID 冲突时高优先级覆盖（`core/agent_skills/config.py`）： +`core/capabilities/skills/loader.py::MultiSourceSkillLoader` 通过后端抽象层（`backends/`：filesystem / database / composite）从多个来源加载技能，同 ID 冲突时高优先级覆盖（`core/capabilities/skills/config.py`）： | 来源 | 目录 | 优先级 | 说明 | |---|---|---|---| @@ -142,7 +142,7 @@ - **范围**：主对话链路 + 子智能体。子智能体绑定的插件同样延迟加载，但激活只在该次运行内生效、不落库（子智能体运行短暂且相互隔离，落到父会话的键上会把激活泄漏进主智能体装配）；收窄模式（对话模式）圈定的插件保持全量装配（那是管理员的显式圈定）；组件含 stdio 形态 MCP 的插件不延迟。 - **开关**：环境变量 `PLUGIN_PROGRESSIVE_LOADING=false` 可整体回退到全量装配。 -实现：`core/plugins/runtime/__init__.py`（延迟解析 / 目录渲染 / 激活工具）+ `core/llm/factory/build.py` 装配接入。 +实现：`core/capabilities/plugins/runtime/__init__.py`（延迟解析 / 目录渲染 / 激活工具）+ `core/llm/factory/build.py` 装配接入。 ## 技能市场（Skill Marketplace） @@ -206,7 +206,7 @@ - **创建/编辑**：手写 SKILL.md（`POST /`、`PUT /{id}`），或 **zip 上传**整个技能包（`POST /upload`，自动识别目录前缀、二进制文件 base64 入库，上限 200MB）； - **文件级管理**：`GET/PUT/DELETE /{id}/files/{filename}` 在线编辑技能附属文件； - **运维**：启停（`/{id}/toggle`）、排序、图标、fork 复制、批量导出/导入（跨环境迁移）； -- **依赖管理**：`POST /{id}/rescan-deps` 用 `core/agent_skills/deps_detector.py` 静态扫描脚本的 pip/apt 依赖，`PUT /{id}/dependencies` 人工修正；聚合后的依赖清单用于沙箱镜像重建（见[沙箱模块](sandbox.md)的管理员依赖重建）。 +- **依赖管理**：`POST /{id}/rescan-deps` 用 `core/capabilities/skills/deps/detector.py` 静态扫描脚本的 pip/apt 依赖，`PUT /{id}/dependencies` 人工修正；聚合后的依赖清单用于沙箱镜像重建（见[沙箱模块](sandbox.md)的管理员依赖重建）。 ## 技能蒸馏（Distillation）（商业版 EE） @@ -265,12 +265,12 @@ | 路径 | 说明 | |---|---| -| `src/backend/core/agent_skills/registry.py` | SKILL.md 解析、元数据/完整 Spec 数据结构 | -| `src/backend/core/agent_skills/loader.py` | 多源加载、DB 技能物化、沙箱路径重定向 | -| `src/backend/core/agent_skills/config.py` | 加载源优先级、统一沙箱技能目录、内置技能同步 | -| `src/backend/core/agent_skills/selector.py` | 按用户意图 LLM 选技能（渐进式披露） | -| `src/backend/core/agent_skills/deps_detector.py` | 脚本 pip/apt 依赖静态探测 | -| `src/backend/core/agent_skills/backends/` | filesystem / database / composite 加载后端 | +| `src/backend/core/capabilities/skills/registry.py` | SKILL.md 解析、元数据/完整 Spec 数据结构 | +| `src/backend/core/capabilities/skills/loader.py` | 多源加载、DB 技能物化、沙箱路径重定向 | +| `src/backend/core/capabilities/skills/config.py` | 加载源优先级、统一沙箱技能目录、内置技能同步 | +| `src/backend/core/llm/skill_selection.py` | 按用户意图 LLM 选技能（渐进式披露） | +| `src/backend/core/capabilities/skills/deps/detector.py` | 脚本 pip/apt 依赖静态探测 | +| `src/backend/core/capabilities/skills/backends/` | filesystem / database / composite 加载后端 | | `src/backend/core/llm/tools/skill_tool.py` | 受限 view_text_file + {baseDir} 替换 + Runtime Hint | | `src/backend/extensions/skills/default/` | 12 个内置技能（含 10 个 OfficeCLI 系列） | | `src/backend/extensions/skills/marketplace/` | 45 个可安装市场技能包 | diff --git a/document/zh-CN/modules/auth.md b/document/zh-CN/modules/auth.md index e62c68cd..2145ace1 100644 --- a/document/zh-CN/modules/auth.md +++ b/document/zh-CN/modules/auth.md @@ -176,7 +176,7 @@ SSO 提供方响应必须是 JSON 对象，且成功码必须为有限数值 0 | `PATCH /v1/me/api-keys/{key_id}` | 启用 / 禁用 | | `DELETE /v1/me/api-keys/{key_id}` | 撤销 | -Key 形如 `sk-jx-...`，在**所有 AUTH_MODE 下**都可作为 Bearer 调用业务 API（外部程序化调用通常没有 Cookie）。校验逻辑（启用 / 未撤销 / 未过期 / 用户权限位仍开启）集中在 `core/services/api_key_service.py::resolve_api_key`。 +Key 形如 `sk-jx-...`，在**所有 AUTH_MODE 下**都可作为 Bearer 调用业务 API（外部程序化调用通常没有 Cookie）。校验逻辑（启用 / 未撤销 / 未过期 / 用户权限位仍开启）集中在 `core/auth/api_keys.py::resolve_api_key`。 禁用或撤销 API Key 会拒绝新请求，并取消该 Key 已接收且仍在等待、运行或暂停待处理的对话任务。取消会停止后台执行并结束 SSE 流；已完成的外部工具操作不会撤回。重新启用 Key 不会恢复已取消任务。调用记录进入“需要处理”等停止状态后，耗时固定在停止时间。 @@ -207,7 +207,7 @@ Key 形如 `sk-jx-...`，在**所有 AUTH_MODE 下**都可作为 Bearer 调用 | 管理凭证依赖 | `src/backend/api/deps.py` | | 用户资料 / 偏好 | `src/backend/api/routes/v1/users.py` | | 用户权限位管理 | `src/backend/edition_ee/routes/config_users.py` | -| 个人 API-Key | `src/backend/api/routes/v1/api_keys.py`、`src/backend/core/services/api_key_service.py` | +| 个人 API-Key | `src/backend/api/routes/v1/api_keys.py`、`src/backend/core/auth/api_keys.py` | | 能力中心自助（owner 隔离） | `src/backend/api/routes/v1/me_capabilities.py` | | 邀请码 | `src/backend/edition_ee/auth/invite.py`、`src/backend/edition_ee/routes/config_invites.py` | | 团队管理 | `src/backend/edition_ee/routes/config_teams.py`、`src/backend/edition_ee/routes/me_teams.py` | diff --git a/document/zh-CN/modules/automation.md b/document/zh-CN/modules/automation.md index cd888e35..03ce950a 100644 --- a/document/zh-CN/modules/automation.md +++ b/document/zh-CN/modules/automation.md @@ -48,6 +48,7 @@ HugAgentOS 内置两种「把一句话变成可重复的生产力」机制，两 - **执行时长**：单次执行没有固定总时长上限；定时触发与手动执行的 Prompt、Plan、Loop 任务均可持续运行，关联会话及其接续运行也不受普通会话的总时长硬限制。 - **失败治理**：连续失败计数 ≥ `max_failures`（默认 3）时任务自动置为 `disabled`。 - **失联恢复**：启动时及每次轮询检查 `running` 记录。取得该任务的执行锁后，继续跟随已关联的耐久会话并保存最终结果；只有未关联会话的孤儿记录才改为 `failed`，并按需推进过期调度、收尾一次性任务。任务运行多久不影响恢复判断。执行锁每 15 秒续期，60 秒租约用于识别执行者失联，不限制任务总时长。错过的一次性任务同样先取得执行锁。 +- **执行锁关闭**：设置停止信号并取消续期，最多等待 2 秒后始终尝试按 token 删除锁；清理失败也移除本地占位并记录错误。执行 Task 结束后续期自动停止。结果投递总等待上限为 60 秒，投递失败不覆盖已提交的成功，也不自动重试。运行记录保存内部所有权 token，完成更新同时校验运行中状态与 token；恢复执行先认领该记录。 每次执行通过统一的 ChatRun 会话执行器创建**真实会话**，并在开跑时立即关联执行记录。指令任务实时展示正文、思考和工具调用；计划任务展示步骤进度。点击运行中的会话或刷新页面，可以跟随和重播与普通会话相同的事件流。用户可在会话中停止执行，指令任务还可通过耐久指令队列追加要求，在下一个安全的模型执行边界生效。主动停止不计入连续失败次数；执行锁失效或调度器关闭会取消后台会话。最终消息、文件、引用和结果投递继续由统一执行器与调度收尾模块负责。会话标题直接使用任务名称，不添加自动化前缀。更新前启动的执行无法补回未记录的实时事件。 @@ -98,14 +99,37 @@ POST /v1/automations | 主题 | 路径 | |---|---| -| 自动化 REST API | `src/backend/api/routes/v1/automations.py` | -| 调度器（轮询 / 锁 / 恢复） | `src/backend/orchestration/schedulers/automation_scheduler.py` | -| 自动化服务（cron 计算 / 状态机） | `src/backend/core/services/automation_service.py` | -| 任务 / 运行记录模型 | `src/backend/core/db/models/automation.py` | +| 自动化 REST API | `src/backend/features/automation/api/automations.py` | +| 调度器（轮询 / 锁 / 恢复） | `src/backend/orchestration/automations/automation_scheduler.py` | +| 自动化服务（cron 计算 / 状态机） | `src/backend/features/automation/service.py` | +| 任务 / 运行记录模型 | `src/backend/features/automation/models.py` | | 计划 REST API | `src/backend/api/routes/v1/plans.py` | -| 计划生成 / 执行编排 | `src/backend/orchestration/subagents/plan_mode.py`、`src/backend/core/services/plan_service.py` | +| 计划生成 / 执行编排 | `src/backend/orchestration/subagents/plan_mode.py`、`src/backend/core/chat/plans.py` | | 计划 / 步骤模型 | `src/backend/core/db/models/agent.py` | | 前端自动化 UI | `src/frontend/src/components/lab/AutomationPanel.tsx` 等、`src/frontend/src/components/automation/RunTimelinePanel.tsx` | | 前端状态 | `src/frontend/src/stores/automationStore.ts`、`automationChatStore.ts` | 延伸阅读：[对话系统](chat.md) · [MCP 工具](mcp-tools.md) · [数据画布与产物](canvas-artifacts.md) + +## 多进程运行与恢复 + +定时和手动触发共用一个调度循环，每 3 秒扫描一次，最多同时观察 8 个执行。 +消费触发信号、推进排期和插入 pending 执行记录在一个数据库事务内完成； +数据库条件更新保护并发接纳。忙碌任务的新手动触发保留至当前执行结束。 + +pending 表示已接纳、等待执行。调度器停止或租约失效只转交观察权， +不会取消已接纳的会话；后续实例继续观察同一个持久执行及其后继会话。 +用户在会话中主动停止仍会停止实际执行。子执行关联和恢复参数与接纳一起提交， +不通过重跑已启动命令恢复结果。运行中的通知投递仍沿用原有投递语义， +此变更不承诺外部渠道的恰好一次投递。 + +升级数据库须应用 FULL 的 automationqueue01 或 CE 的 ce_0032； +本机 SQLite 使用已安装 automation 模块的 schema 初始化完成约束升级。 +存在 pending 记录时拒绝降级，以免丢弃已接纳工作。先停止旧版本调度器、 +迁移后启动新版本，不混用两套调度协议。 + +Plan/Loop 将完整启动参数与子任务接纳一同持久化。接纳后、启动前崩溃时， +后继调度器只领取 pending 子任务；已开始的 running 子任务不会从头重放。 +升级前应排空旧版在途任务，旧版未保存的参数不会由兼容路径猜测补齐。 + +任务投递统一存储为 delivery_targets。升级迁移将旧 channel_id/conversation_id 转换为目标列表并删除旧字段，保留已明确配置的目标和其他元数据；请求处理不再读取旧格式。 diff --git a/document/zh-CN/modules/canvas-artifacts.md b/document/zh-CN/modules/canvas-artifacts.md index b29574e2..0354ebb5 100644 --- a/document/zh-CN/modules/canvas-artifacts.md +++ b/document/zh-CN/modules/canvas-artifacts.md @@ -78,12 +78,12 @@ Web Worker 中执行，并按真实存在的单元格稀疏转换，不遍历工 ### 产物的产生与落库 -对话流式过程中，工具产出的文件先进入会话**工作区**（`core/llm/workspace`），只有被「钉住」（pinned）的文件才会在回合结束时由 `core/services/artifact_service.py::persist_artifacts` 写入 `artifacts` 表——这道**严格工作区闸门**保证临时中间文件不会污染我的空间。 +对话流式过程中，工具产出的文件先进入会话**工作区**（`core/llm/workspace`），只有被「钉住」（pinned）的文件才会在回合结束时由 `core/files/artifacts/service.py::persist_artifacts` 写入 `artifacts` 表——这道**严格工作区闸门**保证临时中间文件不会污染我的空间。 -物理存储由 `core/artifacts/store.py` 管理，按 `STORAGE_TYPE` 双模式： +物理存储由 `core/files/artifacts/store.py` 管理，按 `STORAGE_TYPE` 双模式： - `local`（默认，社区版）：文件写在 `${STORAGE_PATH:-result}/artifacts/`，维护本地 JSON 索引（`index.json`），FileResponse 直出。 -- `oss`（商业版 EE，`cloud_storage`）：上传到阿里云 OSS，本地索引同时备份到 OSS，容器重启不丢。S3 后端见 `core/storage/`。 +- `oss`（商业版 EE，`cloud_storage`）：上传到阿里云 OSS，本地索引同时备份到 OSS，容器重启不丢。S3 后端见 `core/files/storage/`。 下载统一走 `GET /files/{file_id}`（`api/routes/files.py`），按 `core/auth/permissions_iface.py::resolve_artifact_access` 做 owner ∪ team 权限判定，Office 文件支持转 PDF 在线预览。 @@ -141,8 +141,8 @@ officecli-xlsx、officecli-pptx 三个基座技能（以及它们之上的表单 |---|---| | 画布面板 / Univer 表格 | `src/frontend/src/components/canvas/CanvasPanel.tsx`、`UniverSpreadsheet.tsx`、`src/frontend/src/stores/canvasStore.ts` | | Artifact REST API | `src/backend/api/routes/v1/artifacts.py` | -| Artifact 存储（local/OSS） | `src/backend/core/artifacts/store.py` | -| 产物落库（工作区闸门） | `src/backend/core/services/artifact_service.py` | +| Artifact 存储（local/OSS） | `src/backend/core/files/artifacts/store.py` | +| 产物落库（工作区闸门） | `src/backend/core/files/artifacts/service.py` | | 文件下载 / 预览 | `src/backend/api/routes/files.py` | | 会话分享 API | `src/backend/api/routes/v1/chat_shares.py` | | 分享前端 | `src/frontend/src/SharePreviewApp.tsx`、`src/frontend/src/components/share/ShareRecordsPage.tsx` | diff --git a/document/zh-CN/modules/catalog.md b/document/zh-CN/modules/catalog.md index d8d1bd40..a201d618 100644 --- a/document/zh-CN/modules/catalog.md +++ b/document/zh-CN/modules/catalog.md @@ -13,7 +13,7 @@ core/config/catalog.json 系统级单一真源（admin 启停 + 展 core/config/catalog.py 公共 API：get_catalog / is_enabled / get_enabled_ids │ set_enabled / reorder_items ▼ -core/config/catalog_resolver.py 合并层：catalog.json 默认 ∩ 用户 DB 覆盖 +core/capabilities/catalog/resolver.py 合并层：catalog.json 默认 ∩ 用户 DB 覆盖 │ resolve_all_runtime_enabled(db, user_id) → (skills, agents, mcps) │ resolve_explicit_runtime_capabilities(...) → 本轮显式调用（忽略个人开关，保留硬门控） ▼ @@ -39,7 +39,7 @@ core/config/catalog_resolver.py 合并层：catalog.json 默认 ∩ 用户 D } ``` -`catalog_loader.py` 在加载时做**动态源同步**：从技能 loader（`core/agent_skills/`）与 MCP 配置服务（`core/services/mcp_service.py`，DB 表 `admin_mcp_servers`）发现新 id 自动补入、名称/描述/版本随源刷新、源中已消失的陈旧项自动移除；展示详情（能力中心介绍 markdown、图标）属运行时字段，不落盘。结果带短 TTL 内存缓存，写操作调 `invalidate_catalog_cache()` 失效。 +`catalog_loader.py` 在加载时做**动态源同步**：从技能 loader（`core/capabilities/skills/`）与 MCP 配置服务（`core/capabilities/mcp/service.py`，DB 表 `admin_mcp_servers`）发现新 id 自动补入、名称/描述/版本随源刷新、源中已消失的陈旧项自动移除；展示详情（能力中心介绍 markdown、图标）属运行时字段，不落盘。结果带短 TTL 内存缓存，写操作调 `invalidate_catalog_cache()` 失效。 ### 门控 API @@ -54,7 +54,7 @@ get_enabled_ids("mcp") # 某类全部启用 id ## 用户级覆盖（catalog_overrides） -用户在能力中心里开/关某项能力，写入 DB 表 `catalog_overrides`（`core/services/catalog_service.py`），按用户独立、互不影响： +用户在能力中心里开/关某项能力，写入 DB 表 `catalog_overrides`（`core/capabilities/catalog/overrides.py`），按用户独立、互不影响： - **合并算法**（`catalog_resolver.py::_merge_kind`）：覆盖只能翻转 base catalog 中**已存在**项的启用位，不能复活已删除项。 - **管理员锁**：base catalog 中 `enabled=false` 的项，用户覆盖**不可**重新启用，且在 `/v1/catalog` 响应中对前台完全隐藏。 @@ -66,7 +66,7 @@ get_enabled_ids("mcp") # 某类全部启用 id - **显式插件使用引导**：选择插件后加载其技能和 MCP 工具，提示模型优先按需使用；不限制首个工具调用，也不要求必须调用才能完成回答。 - **会话级能力恢复**：后续回合在个人开关筛选前恢复已激活的技能、连接器及插件组件，使技能清单与工具名/前缀在该会话中稳定；恢复时仍重新校验安装、管理员全局状态、依赖与归属，硬门控变化会立即生效。 -每次对话时 `core/chat/context.py::resolve_enabled_capabilities()` 把默认合并结果写入 workflow context；`api/routes/v1/chats/__init__.py::_resolve_explicit_capability_invocation()` 再校验插件安装实例与能力归属，从服务端安装记录解析组件，并把通过硬门控的显式选择并入当前回合。`core/llm/session_capabilities.py` 和 `core/plugins/runtime/__init__.py::resolve_sticky_plugin_capabilities()` 在后续回合恢复会话已激活能力。 +每次对话时 `core/chat/context.py::resolve_enabled_capabilities()` 把默认合并结果写入 workflow context；`api/routes/v1/chats/__init__.py::_resolve_explicit_capability_invocation()` 再校验插件安装实例与能力归属，从服务端安装记录解析组件，并把通过硬门控的显式选择并入当前回合。`core/llm/runtime/session_capabilities.py` 和 `core/capabilities/plugins/runtime/__init__.py::resolve_sticky_plugin_capabilities()` 在后续回合恢复会话已激活能力。 ## /v1/catalog 路由与 KB 注入 @@ -116,7 +116,6 @@ get_enabled_ids("mcp") # 某类全部启用 id ## 管理端能力管理 - **`/admin` 内容管理台**：技能上传/启停/排序（`api/routes/v1/admin_skills.py`，内部调 `catalog.set_enabled` / `reorder_items` 回写 catalog.json）、MCP server 管理（`admin_mcp_servers.py`，DB 表 + 试连 + 缓存刷新）、子智能体管理（`admin_agents.py`）、技能市场上架审核（`admin_marketplace.py`，**商业版 EE**）。 -- **catalog 快照迁移**：`scripts/export_content.py --only catalog` 导出 catalog.json + `catalog_overrides`，`scripts/import_content.py --catalog ` 导入。 ## 相关源码 @@ -125,11 +124,11 @@ get_enabled_ids("mcp") # 某类全部启用 id | 公共 API（is_enabled 等） | `src/backend/core/config/catalog.py` | | 单一真源文件 | `src/backend/core/config/catalog.json`（`CATALOG_PATH` 可覆盖） | | 加载 / 动态同步 / 缓存 | `src/backend/core/config/catalog_loader.py`，`catalog_common.py`，`catalog_migration.py` | -| 用户覆盖合并 | `src/backend/core/config/catalog_resolver.py`，`core/services/catalog_service.py` | +| 用户覆盖合并 | `src/backend/core/capabilities/catalog/resolver.py`，`core/capabilities/catalog/overrides.py` | | 目录路由 | `src/backend/api/routes/v1/catalog.py` | | 用户自助能力 | `src/backend/api/routes/v1/me_capabilities.py` | -| MCP 服务配置（DB） | `src/backend/core/services/mcp_service.py`，`api/routes/v1/admin_mcp_servers.py` | -| 技能管理 | `src/backend/api/routes/v1/admin_skills.py`，`core/agent_skills/` | +| MCP 服务配置（DB） | `src/backend/core/capabilities/mcp/service.py`，`api/routes/v1/admin_mcp_servers.py` | +| 技能管理 | `src/backend/api/routes/v1/admin_skills.py`，`core/capabilities/skills/` | | Dify KB 注入（EE） | `src/backend/edition_ee/kb/dify.py`，经共享接缝 `core/kb/external_provider.py` 调用 | | 前端能力中心 | `src/frontend/src/components/catalog/`，`stores/catalogStore.ts` | | 工厂消费侧 | `src/backend/core/llm/factory/build.py::_effective_mcp_server_keys` | diff --git a/document/zh-CN/modules/chat.md b/document/zh-CN/modules/chat.md index 0e5f7005..92b78771 100644 --- a/document/zh-CN/modules/chat.md +++ b/document/zh-CN/modules/chat.md @@ -71,10 +71,10 @@ orchestration/chat_run_executor.py::start_run() ▼ orchestration/workflow.py::astream_chat_workflow() │ ├─ orchestration/memory_integration.py 非阻塞记忆检索（后台 task + 预算超时） - │ ├─ core/config/catalog_resolver.py 解析本次启用的 skills/mcp/kb + │ ├─ core/capabilities/catalog/resolver.py 解析本次启用的 skills/mcp/kb │ ├─ core/llm/factory/build.py::create_agent_executor() │ │ MCP 连接池 + 技能注册 + 文件工具 + 系统提示词 + 中间件 → Agent - │ ├─ core/llm/context_manager.py 历史按 token 预算裁剪 + │ ├─ core/llm/context/window_policy.py 历史按 token 预算裁剪 │ └─ orchestration/streaming.py::StreamingAgent.stream() │ 消费 agent.reply_stream()，归并 25 种细粒度事件并保留工具参数增量 ▼ @@ -143,8 +143,8 @@ SSE follower：chat_run_executor.follow_run_as_sse() `create_agent_executor()` 是所有模式（主对话、计划、批量、子智能体、自动化）共用的工厂： -- **MCP 工具**：经 catalog + 用户覆盖 + 请求上下文三层过滤后（见 [能力目录](catalog.md)），stable 服务复用进程级连接池（`core/llm/mcp_pool.py`），per-request 服务（如 `retrieve_dataset_content` 需带每请求 HTTP header）每次新建；用户自助添加的私有 MCP 按 owner 现查合入。 -- **技能**：经 `core/agent_skills/loader.py` 注册为 AgentScope Agent Skills，并放行 `view_text_file` 读取 SKILL.md（详见 [技能系统](agent-skills.md)）。 +- **MCP 工具**：经 catalog + 用户覆盖 + 请求上下文三层过滤后（见 [能力目录](catalog.md)），stable 服务复用进程级连接池（`core/llm/mcp/pool.py`），per-request 服务（如 `retrieve_dataset_content` 需带每请求 HTTP header）每次新建；用户自助添加的私有 MCP 按 owner 现查合入。 +- **技能**：经 `core/capabilities/skills/loader.py` 注册为 AgentScope Agent Skills，并放行 `view_text_file` 读取 SKILL.md（详见 [技能系统](agent-skills.md)）。 - **文件/沙箱工具**：`bash`、`sandbox_put_artifact`、`sandbox_get_artifact` 无条件注册；Read/Edit/Write/Glob/Grep/Delete/Move/mkdir + MySpace 工具受 `CODE_CAPABILITY_ENABLED` 门控，共享同一个 `ReadStateTracker` 维持「先 Read 才能 Edit」不变量。 - **中间件**（洋葱模型，`core/llm/middlewares.py`）：`DynamicModelMiddleware`（按 chat_mode 切模型，见 [模型接入](model-providers.md)）、`FileContextMiddleware`（注入上传/历史文件上下文）、`SteerMiddleware`（工具结果之后、下一轮推理之前注入追加指令）、`WorkspacePinHintMiddleware`、`FinishPinGuardMiddleware`。 - **上下文压缩**：`CompactingAgent` 把轮前、轮内和轮末三个触发时机接入同一个持久化 checkpoint 引擎，并共用 Codex 风格交接提示词；`ContextConfig` 只保留工具结果限长。压缩只使用统一协调器，不再运行框架的第二条摘要回退。压缩失败时保留原始上下文；最终请求仍超预算时保存断点并暂停，不继续调用回复模型。 @@ -326,7 +326,7 @@ Enter / Tab 确认和 Escape 返回或关闭。个人在能力中心关闭的技 - **专属会话**：从子智能体详情页进入的会话使用 `agent_id`，后续轮次持续由该子智能体执行。 - **主智能体自主编排**：既没有结构化 `@`，也没有命中严格自然语言调用语法时，主智能体可 按任务需要调用 - `core/llm/subagent_tool.py` 注册的 `call_subagent`。子智能体在独立线程和事件循环中运行，结果 + `core/llm/subagents/tool.py` 注册的 `call_subagent`。子智能体在独立线程和事件循环中运行，结果 回传给主智能体整合。这一路径适合多子智能体并行、任务拆分和跨领域汇总。 ## 会话摘要与上下文压缩 @@ -336,7 +336,7 @@ Enter / Tab 确认和 Escape 返回或关闭。个人在能力中心关闭的技 | 层次 | 实现 | 触发 | |---|---|---| | 会话标题摘要 | `core/llm/summarizer.py::ConversationSummarizer`（`summarizer` 模型角色，`ENABLE_SUMMARY` 开关），`POST /v1/summary` | 新会话标题自动生成 | -| 统一上下文检查点 | `core/services/compaction_service.py::run_compaction()`；轮前、轮内、轮末共用同一触发比例、交接提示词、替代历史结构和持久化 checkpoint | 上下文达到模型窗口的配置比例时 | +| 统一上下文检查点 | `core/chat/compaction/service.py::run_compaction()`；轮前、轮内、轮末共用同一触发比例、交接提示词、替代历史结构和持久化 checkpoint | 上下文达到模型窗口的配置比例时 | | 确定性溢出保护 | `ContextConfig.tool_result_limit` 先限制单条工具结果；统一压缩失败时保留原始上下文，最终请求超预算则保存断点并暂停 | 单条工具结果过大，或统一压缩暂时不可用时 | 压缩只在合法边界切分：历史先按步骤展开，从尾部保留装得下 @@ -356,7 +356,7 @@ framing 的后端估算。当前回合结束后若后台压缩已启动，`meta. ## 超长工具结果 offload -`core/llm/offloader.py::SandboxOffloader` 实现 AgentScope 2.0 `Offloader` 协议：上下文压缩/工具结果截断时，溢出部分不再被静默丢弃，而是落盘到沙箱 `/workspace/.offload/`（`tool_.txt` / `context_.txt`），框架会在给模型的 `` 里附上路径，模型可用 `Read` / `bash` 按需读回。仅在沙箱工具启用（`SANDBOX_TOOLS_ENABLED=true`，默认开）时挂载，写失败永不抛异常、返回降级说明。 +`core/llm/context/offloader.py::SandboxOffloader` 实现 AgentScope 2.0 `Offloader` 协议：上下文压缩/工具结果截断时，溢出部分不再被静默丢弃，而是落盘到沙箱 `/workspace/.offload/`（`tool_.txt` / `context_.txt`），框架会在给模型的 `` 里附上路径，模型可用 `Read` / `bash` 按需读回。仅在沙箱工具启用（`SANDBOX_TOOLS_ENABLED=true`，默认开）时挂载，写失败永不抛异常、返回降级说明。 ## 会话分享（Chat Shares） @@ -386,15 +386,15 @@ framing 的后端估算。当前回合结束后若后台压缩已启动，`meta. | 事件映射（reply_stream → SSE） | `src/backend/orchestration/streaming.py` | | 运行时上下文装配 | `src/backend/core/chat/context.py` | | Agent 工厂 | `src/backend/core/llm/factory/build.py` | -| 中间件 | `src/backend/core/llm/middlewares.py`（纯函数 helper 在 `core/llm/hooks.py`） | +| 中间件 | `src/backend/core/llm/middlewares.py`（纯函数 helper 在 `core/llm/runtime/hooks.py`） | | 引用抽取 | `src/backend/orchestration/citations.py` | -| 跨会话引用（名片 / 摘要 / 分页） | `src/backend/core/services/chat_reference_service.py` | +| 跨会话引用（名片 / 摘要 / 分页） | `src/backend/core/chat/references.py` | | 跨会话历史工具 | `src/backend/core/llm/tools/chat_history_tool.py` | | 引用前端渲染 | `src/frontend/src/utils/citations.ts`，`src/frontend/src/components/citation/` | | 计划模式 | `src/backend/orchestration/subagents/plan_mode.py`，`api/routes/v1/plans.py` | -| 子智能体工具 | `src/backend/core/llm/subagent_tool.py`，`api/routes/v1/agents.py` | -| 标题摘要 / 上下文压缩 / 窗口保护 | `src/backend/core/llm/summarizer.py`、`compaction.py`、`core/services/compaction_service.py`、`context_manager.py` | -| 超长结果 offload | `src/backend/core/llm/offloader.py` | +| 子智能体工具 | `src/backend/core/llm/subagents/tool.py`，`api/routes/v1/agents.py` | +| 标题摘要 / 上下文压缩 / 窗口保护 | `src/backend/core/llm/summarizer.py`、`compaction.py`、`core/chat/compaction/service.py`、`context_manager.py` | +| 超长结果 offload | `src/backend/core/llm/context/offloader.py` | | 会话分享 | `src/backend/api/routes/v1/chat_shares.py` | | 追问生成 | `src/backend/orchestration/followups.py` | | 前端流式解析 / 追加消息 | `src/frontend/src/hooks/chatStream.ts`，`useStreaming.ts`，`components/chat/QueuedMessageCard.tsx` | diff --git a/document/zh-CN/modules/knowledge-base.md b/document/zh-CN/modules/knowledge-base.md index 15c3f525..1351a810 100644 --- a/document/zh-CN/modules/knowledge-base.md +++ b/document/zh-CN/modules/knowledge-base.md @@ -29,7 +29,7 @@ HugAgentOS 支持自建与外部知识库。公有检索按平台配置选择一 对象存储落盘（storage_key） │ · embed_text(query) │ │ · Milvus hybrid_search： ▼ BackgroundTask │ 稠密(IP) + 稀疏(BoW) → RRF(k=60) - core/content/kb_processing.py │ · 命中子块 → 回表 kb_chunks 取父块原文 + core/files/content/kb_processing.py │ · 命中子块 → 回表 kb_chunks 取父块原文 vectorise_document_background() │ · 可选 Reranker 重排 · kb_parser.parse_and_chunk() │ · user_id 隔离 + public kb 全局可见 ·（可选）LLM 抽关键词/生成问题 │ @@ -83,7 +83,7 @@ Wiki 相关的三张表（`kb_wiki_pages` / `kb_wiki_folders` / `kb_wiki_jobs` | `recursive` | 递归定长切分 | | `embedding_semantic` | 基于 embedding 相似度的语义边界检测 | -父子分块参数可在上传时通过 `indexing_config` 调整：`parent_chunk_size`（默认 1024 token）、`child_chunk_size`（128）、`overlap_tokens`（20）、`parent_child_indexing`（默认 true）。还可启用 LLM 增强：`auto_keywords_count`（每父块抽关键词入 tags，参与稀疏检索）与 `auto_questions_count`（每父块生成关联问题，作为独立 question 行入 Milvus，提高问句召回）。后台向量化任务在 `core/content/kb_processing.py::vectorise_document_background()`。 +父子分块参数可在上传时通过 `indexing_config` 调整：`parent_chunk_size`（默认 1024 token）、`child_chunk_size`（128）、`overlap_tokens`（20）、`parent_child_indexing`（默认 true）。还可启用 LLM 增强：`auto_keywords_count`（每父块抽关键词入 tags，参与稀疏检索）与 `auto_questions_count`（每父块生成关联问题，作为独立 question 行入 Milvus，提高问句召回）。后台向量化任务在 `core/files/content/kb_processing.py::vectorise_document_background()`。 ### 多模态：图片检索 @@ -180,9 +180,9 @@ DIFY_API_KEY=dataset-... # 兼容别名 DIFY_AUTH_TOKEN ## 文件解析支持 -知识库上传校验在 `core/content/file_validation.py::validate_kb_file`（扩展名 + magic bytes 双重校验），允许：`.pdf` `.txt` `.md` `.doc` `.docx` `.xls` `.xlsx` `.csv` `.json` 及图片（`.png` `.jpg` `.jpeg` `.webp` `.gif`）。 +知识库上传校验在 `core/files/content/file_validation.py::validate_kb_file`（扩展名 + magic bytes 双重校验），允许：`.pdf` `.txt` `.md` `.doc` `.docx` `.xls` `.xlsx` `.csv` `.json` 及图片（`.png` `.jpg` `.jpeg` `.webp` `.gif`）。 -通用文件解析器 `core/content/file_parser.py::parse_file()` 覆盖更广（对话附件、我的空间文件共用）：PDF、DOCX、DOC/WPS（经 LibreOffice 转换）、TXT、XLSX/XLS、CSV、PPTX，外加 HTML / Markdown / JSON / YAML / 代码等纯文本格式直接 UTF-8 解码。 +通用文件解析器 `core/files/content/file_parser.py::parse_file()` 覆盖更广（对话附件、我的空间文件共用）：PDF、DOCX、DOC/WPS（经 LibreOffice 转换）、TXT、XLSX/XLS、CSV、PPTX，外加 HTML / Markdown / JSON / YAML / 代码等纯文本格式直接 UTF-8 解码。 ## 前端 @@ -201,9 +201,9 @@ DIFY_API_KEY=dataset-... # 兼容别名 DIFY_AUTH_TOKEN | `src/backend/core/kb/kb_assets.py` | 媒体资产：落盘、图像理解（调用视觉桥）、检索行、结果附图 | | `src/backend/edition_ee/kb/dify.py` | Dify datasets 客户端与启用判定（仅 EE） | | `src/backend/core/kb/external_provider.py` | 版本中立的外部知识库接缝；CE overlay 将其禁用 | -| `src/backend/core/content/kb_processing.py` | 后台向量化任务、LLM 关键词 / 问题增强 | -| `src/backend/core/content/file_validation.py` | 上传文件校验（扩展名 + magic bytes） | -| `src/backend/core/content/file_parser.py` | 通用文件解析器 | +| `src/backend/core/files/content/kb_processing.py` | 后台向量化任务、LLM 关键词 / 问题增强 | +| `src/backend/core/files/content/file_validation.py` | 上传文件校验（扩展名 + magic bytes） | +| `src/backend/core/files/content/file_parser.py` | 通用文件解析器 | | `src/backend/core/services/kb_service.py` | 知识库业务逻辑（含系统托管同步库） | | `src/backend/api/routes/v1/kb.py` + `kb_models.py` | 用户侧 `/v1/catalog/kb` 路由 | | `src/backend/api/routes/v1/admin_kb.py` | 管理台公共知识库路由（商业版 EE） | diff --git a/document/zh-CN/modules/mcp-tools.md b/document/zh-CN/modules/mcp-tools.md index ad8af2da..01976fd8 100644 --- a/document/zh-CN/modules/mcp-tools.md +++ b/document/zh-CN/modules/mcp-tools.md @@ -26,7 +26,7 @@ HugAgentOS 的工具生态构建在 [MCP（Model Context Protocol）](https://mo │ agent) │ │ │ :9103 产业知识中心统一 MCP（插件，EE） │ └─────────┘ │ │ :9104 保留（图表 MCP 已退役） │ │ │ │ :9105 保留（原报告导出 MCP） │ - MCPConnectionPool │ │ :9106 web_fetch（网页抓取） │ + Native MCP Client │ │ :9106 web_fetch（网页抓取） │ mcp_pool.py) └───┬────┘ :9108 automation_task（自动化插件提供） │ │ 9109–9111 保留（原 excel/ppt/pdf MCP） │ │ :9112 skill_manager（技能管理） │ @@ -201,17 +201,14 @@ extensions/mcp_servers/_mcp/ Server 同样适用（返回 JSON 即可）。回注发生在结果进模型上下文与落库之前，前端工具卡片 会把 `cite_id` 渲染成条目徽章，与正文 `[锚文本](cite:eN)` 引用一一对应。 -## 后端客户端：连接池与裸名还原 +## 后端客户端：原生 MCP 生命周期 -后端基于 AgentScope 2.0 的 `MCPClient` 连接 MCP Server，核心在两个文件： +后端使用 MCP SDK 2.3 的 `Client(mode="auto")` 和 AgentScope `ToolBase`。`client.py` 处理传输授权，`lifetime.py` 管理 SDK 上下文并完整遍历分页。HTTP 在一次运行内复用客户端。STDIO 按执行用户、聊天和配置独占租用，成功结束后保留 60 秒；失败或取消则销毁。已删除稳定连接开关和启动连接池。 -- **`core/llm/mcp_pool.py` — `MCPConnectionPool`**（进程级单例）：启动时 `warmup_mcp_tools()` 从 DB 读取全部启用的 Server 配置并预连接。注意 2.0 语义下的池化策略： - - **stdio 且 `is_stable=true`** 的 Server 跨请求保持连接（省掉 1–7 秒子进程冷启动）； - - **HTTP Server 一律不池化**——2.0 的 stateful HTTP 客户端与 asyncio task 绑定，跨请求复用会触发 cancel-scope 崩溃，因此每请求用 `is_stateful=False` 新建连接； - - per-request Server（知识库检索带用户头）每次现连，请求结束 `close_transient()` 关闭。 -- **`core/llm/mcp_manager.py` — `BareNameMCPClient`**：AgentScope 2.0 默认把工具名改写为 `mcp____`，该子类还原为服务器侧裸名（`internet_search` 而非 `mcp__internet_search__internet_search`），保证展示名映射（`core/config/display_names.py`）、[引用溯源](chat.md)的按工具名分发、前端图标渲染等 1.x 约定继续成立。 +`callbacks.py` 将隔离的文本 Sampling 与用户 Elicitation 绑定到当前运行。资源、URI 模板和提示词转换为受权限控制的工具。进度显示在现有工具卡片。结构化输出和资源链接保留在工具 metadata 与模型输出中。SDK 2.3 尚未提供新版 Tasks 扩展。 + +MCP 工具的执行预算不包含等待用户回答的时间。有效提问会暂停当前调用的计时；回答或放弃后，从剩余预算继续。多次、重叠提问不会重置预算，其他调用继续独立计时。用户交互仍受现有等待期限约束，取消运行会清理待回答问题。SDK 请求和 HTTP 读取由调用预算约束，连接、写入和发现仍保留超时。 -`Toolkit` 在 2.0 是一次性构造，由 `core/llm/factory/build.py` 统一执行 `Toolkit(tools=[...], mcps=clients)`。 ## 统一工具权限网关 @@ -228,7 +225,7 @@ Server 同样适用（返回 JSON 即可）。回注发生在结果进模型上 一律拒绝，不会再以“记录审计后继续执行”降级； - 渐进加载插件把新出现的 MCP 工具加入同一个运行级 Registry，因此热加载不会绕开网关。 -原生工具可以在 `ToolCollector.register_tool_function(..., permission=spec)` 注册规则；平台 +原生工具可以在 `ToolAssembly.append(function_tool(..., dispatch_spec=spec))` 注册规则；平台 内置的本机文件、Bash 与 MySpace 工具集中列在 `builtin_tool_permission()`。没有传 `permission` 且不在该表中的原生工具不受网关管理。 @@ -251,7 +248,7 @@ MCP 规则只接受平台/管理员持有的可信连接配置，不采信远端 ## 配置注册：DB 驱动 + catalog 门控 -MCP Server 的配置真源是数据库表 `admin_mcp_servers`（ORM：`core/db/models.py::AdminMcpServer`），由 `core/services/mcp_service.py::McpServerConfigService` 以 30 秒 TTL 缓存读出，格式与旧 `MCP_SERVERS` dict 兼容（`transport / command / args / env / url / headers / is_stable`）。`core/config/mcp_config.py` 保留为内置 Server 的 URL 构造器（首次部署种子）。 +MCP Server 的配置真源是数据库表 `admin_mcp_servers`（ORM：`core/db/models.py::AdminMcpServer`），由 `core/capabilities/mcp/service.py::McpServerConfigService` 以 30 秒 TTL 缓存读出，使用原生客户端配置（`transport / command / args / env / url / headers`）。`core/config/mcp_config.py` 保留为内置 Server 的 URL 构造器（首次部署种子）。 是否**对模型可见**还要过一道 [catalog](catalog.md) 门控：`core/config/catalog.json` 中 `mcp` 段的每项对应一个 server_id，`is_enabled(id, "mcp_server")` 为 false 的 Server 即使连接着也不会注册给智能体。 @@ -263,7 +260,7 @@ MCP Server 的配置真源是数据库表 `admin_mcp_servers`（ORM：`core/db/m - **创建即试连**：`_probe_connectivity` 真实连一次，失败拒绝落库； - **开关与排序**：`POST /{id}/toggle` 即时启停（联动刷新 catalog 与连接池）； - **密钥保护**：HTTP Header 值加密存储、接口统一以 `***` 返回；`env_vars` 中疑似密钥的值同样脱敏； -- **测试与重载**：`POST /{id}/test` 单独试连；`POST /reload-pool` 热重建连接池； +- **测试与重载**：`POST /{id}/test` 单独试连；`配置更新自动失效缓存` 更新后下一次运行重新发现； - **移至 MCP 市场**：把现有远程 HTTP/SSE MCP 生成无凭据的市场快照， 并停用原全局实例；该服务随后从 MCP 服务管理列表移除，可在市场中复核、 编辑展示信息，再按需全局安装。原服务已配置的 Token 仍加密保留在服务端， @@ -279,7 +276,7 @@ MCP Server 的配置真源是数据库表 `admin_mcp_servers`（ORM：`core/db/m - `POST /v1/me/mcp-servers`：添加私有远程 MCP，支持公网 HTTP/HTTPS 上的 HTTP/SSE——用户入口禁止 stdio，避免在服务器执行任意命令；地址会经过 DNS/IP 检查以阻断 localhost、内网、链路本地与保留地址；创建即试连，连不上不落库。HTTP 为明文传输，生产环境仍建议使用 HTTPS； - `DELETE /v1/me/mcp-servers/{id}`：删除自己的私有 MCP。 -实现上复用同一张 `admin_mcp_servers` 表：`owner_user_id` = 当前用户实现 owner 隔离，server_id 自动生成 `umcp_` 防冲突，`is_stable=False` 不进 warmup 池。HTTP Header 值使用 Fernet 加密后落库，运行时才解密。该功能由管理台的 per-user 权限位 `can_add_mcp` 控制（社区版单租户默认即可放开；商业版由组织管理员按用户授予，见[版本说明](../editions/overview.md)）。 +实现上复用同一张 `admin_mcp_servers` 表：`owner_user_id` = 当前用户实现 owner 隔离，server_id 自动生成 `umcp_` 防冲突。HTTP Header 值使用 Fernet 加密后落库，运行时才解密。该功能由管理台的 per-user 权限位 `can_add_mcp` 控制（社区版单租户默认即可放开；商业版由组织管理员按用户授予，见[版本说明](../editions/overview.md)）。 ## MCP 市场 @@ -359,12 +356,12 @@ docker-compose up -d --build mcp | `src/backend/extensions/mcp_servers/_serve.py` | stdio / streamable-http 双 transport 统一入口 | | `src/backend/extensions/mcp_servers/registry.py` | 安装位置、暴露方式、包名和基础端口 | | `src/backend/extensions/mcp_servers/_ports.py` | 实际部署端口映射 | -| `src/backend/core/llm/mcp_pool.py` | MCP 连接池（stdio 池化 / HTTP per-request） | -| `src/backend/core/llm/mcp_manager.py` | MCPClient 构造 + 工具裸名还原 | +| `src/backend/core/llm/mcp/pool.py` | MCP 连接池（stdio 池化 / HTTP per-request） | +| `src/backend/core/llm/mcp/manager.py` | MCPClient 构造 + 工具裸名还原 | | `src/backend/core/llm/tool_permissions.py` | 受控工具 Registry、权限判定、确认路由与执行票据 | -| `src/backend/core/services/mcp_service.py` | DB 驱动的 Server 配置服务（30s 缓存） | +| `src/backend/core/capabilities/mcp/service.py` | DB 驱动的 Server 配置服务（30s 缓存） | | `src/backend/core/services/mcp_marketplace_service.py` | 市场发布、审核、安装、可见性与安全控制 | -| `src/backend/core/services/mcp_oauth_service.py` | OAuth 2.1 登录、SDK 元数据发现、加密令牌存储与刷新 | +| `src/backend/core/capabilities/mcp/oauth.py` | OAuth 2.1 登录、SDK 元数据发现、加密令牌存储与刷新 | | `src/backend/core/services/mcp_marketplace_monitor.py` | 远程工具快照定期复检与漂移监控 | | `src/backend/core/config/mcp_config.py` | 内置 Server URL 构造（http://mcp:NNNN/mcp/） | | `src/backend/core/config/catalog.json` | 能力目录：MCP 启停门控种子 | diff --git a/document/zh-CN/modules/model-providers.md b/document/zh-CN/modules/model-providers.md index 6a048edf..952bcaf9 100644 --- a/document/zh-CN/modules/model-providers.md +++ b/document/zh-CN/modules/model-providers.md @@ -36,6 +36,8 @@ HugAgentOS 通过 **OpenAI 兼容协议**接入任意大模型端点（vLLM、Ol ### 接口协议：默认 Responses，不支持才回退 +Responses 的思考原文和摘要事件由当前锁定的 AgentScope 原生解析。平台原样保留事件的 `item_id` 与分段索引，避免流式发送失败，以及不同思考条目或分段被错误合并。 + OpenAI 兼容上游有两套接口：较老的 `POST /chat/completions` 和较新的 `POST /responses`。平台**默认走 Responses**，因为它把模型的思考做成独立的 `reasoning` 条目，下一轮可原样回传；`/chat/completions` 没有放思考的位置，只能把思考夹带进 `reasoning_content` 字段。只有确实没有 `/responses` 路由的上游（常见于只做转发的中转网关）才使用 Chat Completions。 走哪套由 `extra_config.api_protocol` 决定，取值 `responses` / `chat_completions`。该值不靠厂商名推断，而是保存供应商时向上游探测后写入。探测分两步，两步都过才判为 Responses 可用： @@ -134,7 +136,7 @@ chat_mode → hooks._resolve_chat_mode(agent.state) high/max → reasoning_effort=high/max（端点须声明支持） ``` -旧客户端只传 `enable_thinking` 时回退映射为 medium / fast。这就是 1.x「pre_reply 动态模型 hook」在 AgentScope 2.0 下的形态——hook 工厂已重构为中间件，`core/llm/hooks.py` 仅保留纯函数 helper。 +旧客户端只传 `enable_thinking` 时回退映射为 medium / fast。这就是 1.x「pre_reply 动态模型 hook」在 AgentScope 2.0 下的形态——hook 工厂已重构为中间件，`core/llm/runtime/hooks.py` 仅保留纯函数 helper。 ## 环境变量与 MCP 子进程注入 @@ -147,7 +149,7 @@ chat_mode → hooks._resolve_chat_mode(agent.state) ## 个人 API-Key -用户可签发个人密钥以编程方式调用平台 API（`api/routes/v1/api_keys.py` + `core/services/api_key_service.py`）： +用户可签发个人密钥以编程方式调用平台 API（`api/routes/v1/api_keys.py` + `core/auth/api_keys.py`）： - 明文形如 `sk-jx-`，**仅创建时返回一次**，DB 只存 SHA256 哈希 + 前缀；支持 7/30/90/180/365 天有效期或永不过期、启停与撤销。 - 调用方式 `Authorization: Bearer sk-jx-...`，鉴权层（`core/auth/backend.py`）识别 key 前缀后经 `resolve_api_key` 解析为用户上下文，校验启用/未撤销/未过期。 @@ -176,9 +178,9 @@ token 用量在流式收尾的 `meta.usage` 中统计（`orchestration/streaming | 角色解析服务（DB + 缓存） | `src/backend/core/services/model_config.py` | | 角色定义 / 供应商仓储 | `src/backend/core/db/model_repository.py` | | 管理 API | `src/backend/api/routes/v1/models.py` | -| 动态模型中间件 | `src/backend/core/llm/middlewares.py::DynamicModelMiddleware`，helper 在 `core/llm/hooks.py` | +| 动态模型中间件 | `src/backend/core/llm/middlewares.py::DynamicModelMiddleware`，helper 在 `core/llm/runtime/hooks.py` | | MCP env 注入 | `src/backend/core/services/model_config.py::get_mcp_env_overlay`，`core/config/runtime_env.py` | -| 个人 API-Key | `src/backend/api/routes/v1/api_keys.py`，`core/services/api_key_service.py`，`core/auth/backend.py` | +| 个人 API-Key | `src/backend/api/routes/v1/api_keys.py`，`core/auth/api_keys.py`，`core/auth/backend.py` | | 用量日志 / 计费 | `src/backend/api/routes/v1/admin_usage_logs.py`，`api/routes/v1/admin_billing.py` | | 路由策略 | `src/backend/orchestration/strategy.py` | | 主模型缺失快速失败 | `src/backend/api/routes/v1/chats/__init__.py::_ensure_main_model_configured` | diff --git a/document/zh-CN/modules/projects-myspace.md b/document/zh-CN/modules/projects-myspace.md index 3e42a848..9e94ac8c 100644 --- a/document/zh-CN/modules/projects-myspace.md +++ b/document/zh-CN/modules/projects-myspace.md @@ -35,7 +35,7 @@ teams ─────────┬── team_members（role: owner/admin/memb ## 项目（Projects） -路由：`src/backend/api/routes/v1/projects.py`（CE 路由表），业务在 `core/services/project_service.py` 与 `project_file_service.py`。 +路由：`src/backend/api/routes/v1/projects.py`（CE 路由表），业务在 `core/projects/service.py` 与 `project_file_service.py`。 | 方法 | 路径 | 说明 | |---|---|---| @@ -71,7 +71,7 @@ teams ─────────┬── team_members（role: owner/admin/memb ### 项目上下文如何进入对话 -在项目内发起对话时（请求携带 `project_id`），`core/services/session_operations/request_context.py` 组装 workflow context： +在项目内发起对话时（请求携带 `project_id`），`core/chat/operations/request_context.py` 组装 workflow context： 1. 读取项目元信息——`project_name`、`project_instructions`、挂钩文件夹名和工作目录；项目指令仍读取 `AGENTS.md`，不递归枚举云端或本机项目文件； 2. `core/llm/factory/build.py` 把这些经 `_build_project_section` 注入 system prompt（`build_system_prompt(cfg, ctx=...)`）； @@ -158,7 +158,7 @@ teams ─────────┬── team_members（role: owner/admin/memb 三条路径，互为补充： -1. **附件注入（hooks）**：前端请求的 `attachments[]` 只携带 `file_id`、`name`、`mime_type`，不携带文件正文。`core/llm/hooks.py` 按 `file_id` 延迟下载、解析并缓存 `parsed_text` 与摘要；首轮只注入文件元数据和最多 5,000 字符的有界预览，xlsx 额外提供总行列数、表头和少量数据行。模型需要预览外的正文时通过 `read_artifact` 按需读取；图片仍走多模态注入。所有读取都校验 `user_id` 归属，防止伪造 `file_id` 跨用户访问； +1. **附件注入（hooks）**：前端请求的 `attachments[]` 只携带 `file_id`、`name`、`mime_type`，不携带文件正文。`core/llm/runtime/hooks.py` 按 `file_id` 延迟下载、解析并缓存 `parsed_text` 与摘要；首轮只注入文件元数据和最多 5,000 字符的有界预览，xlsx 额外提供总行列数、表头和少量数据行。模型需要预览外的正文时通过 `read_artifact` 按需读取；图片仍走多模态注入。所有读取都校验 `user_id` 归属，防止伪造 `file_id` 跨用户访问； 2. **项目文件访问**：项目对话的 system prompt 提供项目指令和目录范围，不自动携带文件列表或文件总数。agent 按需使用目录、搜索和读取工具获取文件信息； 3. **沙箱虚拟文件系统**：开启代码执行时，`/myspace/...` 路径把我的空间映射进沙箱（懒加载 + 文件系统实时登记），团队项目映射团队文件夹——见 [沙箱](./sandbox.md)。 @@ -167,8 +167,8 @@ teams ─────────┬── team_members（role: owner/admin/memb | 路径 | 职责 | |---|---| | `src/backend/api/routes/v1/projects.py` | 项目 API | -| `src/backend/core/services/project_service.py` / `project_file_service.py` | 项目业务逻辑 | -| `src/backend/core/services/project_scope.py` | `ProjectScope`（沙箱路径作用域） | +| `src/backend/core/projects/service.py` / `project_file_service.py` | 项目业务逻辑 | +| `src/backend/core/projects/scope.py` | `ProjectScope`（沙箱路径作用域） | | `src/backend/api/routes/v1/myspace_folders.py` | 个人文件夹 API | | `src/backend/api/routes/v1/artifacts.py` | 资产列表 / 会话收藏 / 加入知识库 | | `src/backend/edition_ee/routes/team_files.py` | 团队文件夹与文件 API（商业版 EE） | @@ -177,7 +177,7 @@ teams ─────────┬── team_members（role: owner/admin/memb | `src/backend/core/db/models/identity.py` | `UserFolder` 等共享身份 ORM | | `src/backend/edition_ee/db/models/identity.py` | `Team` / `TeamMember` / `TeamFolder` ORM（仅 EE） | | `src/backend/core/db/models/artifact.py` | `Artifact` ORM | -| `src/backend/core/llm/hooks.py` | 附件上下文注入（`_build_file_context` 等） | +| `src/backend/core/llm/runtime/hooks.py` | 附件上下文注入（`_build_file_context` 等） | | `src/backend/core/llm/factory/build.py` | 项目 section 注入 system prompt | | `src/backend/core/llm/tools/myspace_vfs/__init__.py` | 我的空间 ↔ 沙箱映射层 | | `src/frontend/src/components/projects/` | 项目前端组件 | @@ -249,10 +249,7 @@ Config「团队管理」新增成员或调整成员权限时，可选「所有 缺失文件或失败下载也在下次正常同步时补齐。 升级前先停止本地写入并备份数据库及 myspace_cache。 -运行 `python src/backend/scripts/clean_myspace_duplicates.py --all` 预览； -加 `--apply --backup-dir <新的备份目录>` 才执行。 -清理删除每个重名组的全部有效记录及镜像路径，不保留“最新一条”。 -采用删除标记避免历史记录回填复活，原对象和恢复日志保留在维护备份中。 +升级前检查同名记录，并制定经审核的数据清理与恢复方案。当前不提供独立去重脚本。 然后执行 Alembic 升级（EE: personalname01；CE: ce_0018），恢复服务并运行对账与端到端测试。 迁移遇到未清理重名会终止，不会自行删除其他环境数据。 恢复须先停止写入、撤销唯一索引，再由备份恢复；不要直接向正常服务恢复重名记录。 @@ -275,7 +272,7 @@ Config「团队管理」新增成员或调整成员权限时，可选「所有 后端按项目配置限制能力注册、显式调用、历史会话资源恢复、子智能体和插件动态加载。空的自定义环境不回退到全部资源。环境调整从下一轮对话生效；正在执行的响应及其派生作业保留原配置。项目外对话不受影响。基础文件与对话工具仍遵守原有权限规则。 -实现按职责划分：`core/project_environment/` 定义配置校验和纯依赖展开；资源服务读取当前用户获授权的目录；环境服务负责保存和预览；运行策略负责冻结及收窄注册、调用范围。前端编辑器只组织界面，`useProjectEnvironment` 负责资源加载、依赖预览和市场回填，资源选框负责展示显式绑定与推导依赖。 +实现按职责划分：`core/projects/environment/` 定义配置校验和纯依赖展开；资源服务读取当前用户获授权的目录；环境服务负责保存和预览；运行策略负责冻结及收窄注册、调用范围。前端编辑器只组织界面，`useProjectEnvironment` 负责资源加载、依赖预览和市场回填，资源选框负责展示显式绑定与推导依赖。 ### 云文档在线编辑 diff --git a/document/zh-CN/modules/prompts.md b/document/zh-CN/modules/prompts.md index 8c4d77f2..c5018a0e 100644 --- a/document/zh-CN/modules/prompts.md +++ b/document/zh-CN/modules/prompts.md @@ -29,7 +29,7 @@ src/backend/prompts/prompt_text/default/system/ └── 40_format.system.md # 输出格式 + [ref:工具名-N] 引用规范 ``` -在 base prompt 之后，运行时还会按上下文追加动态段：工具与技能说明（`_TOOLS_AND_SKILLS_NOTICE`）、轻量知识库目录（`prompts/kb_lite_section.py`）、项目模式段（`prompts/project_section.py`，仅项目对话注入）、代码执行段与批量模式提示（由 `factory/build.py` 追加）、子智能体路由表（`core/llm/subagent_tool.py::build_subagent_prompt_section`）。 +在 base prompt 之后，运行时还会按上下文追加动态段：工具与技能说明（`_TOOLS_AND_SKILLS_NOTICE`）、轻量知识库目录（`prompts/kb_lite_section.py`）、项目模式段（`prompts/project_section.py`，仅项目对话注入）、代码执行段与批量模式提示（由 `factory/build.py` 追加）、子智能体路由表（`core/llm/subagents/tool.py::build_subagent_prompt_section`）。 ### 缓存设计 @@ -48,7 +48,7 @@ src/backend/prompts/prompt_text/default/system/ ### 执行 Manifest 与运行时绑定 -Agent 真正开始执行前，`core/llm/execution_manifest.py::PromptManifestBuilder` 会记录最终请求面。每个有序 PromptSection 都带 `id`、来源、信任级、优先级、缓存类别、预算、token 估算、内容哈希和版本；每个最终 ToolDefinition 同样对名称、完整描述、canonical 参数 schema、权限策略和恢复策略生成稳定哈希。工具定义按稳定 id 排序，提示词分段则保持刻意设计的前缀顺序。 +Agent 真正开始执行前，`core/llm/runtime/execution_manifest.py::PromptManifestBuilder` 会记录最终请求面。每个有序 PromptSection 都带 `id`、来源、信任级、优先级、缓存类别、预算、token 估算、内容哈希和版本；每个最终 ToolDefinition 同样对名称、完整描述、canonical 参数 schema、权限策略和恢复策略生成稳定哈希。工具定义按稳定 id 排序，提示词分段则保持刻意设计的前缀顺序。 Builder 最终产出 `prompt_hash`、`prompt_manifest_hash`、`tool_manifest_hash`、`context_hash` 和统一的 `aggregate_hash`。Manifest 构建与 AgentScope 模型请求共用同一份 run-scoped 工具/技能冻结快照，因此“记录的请求面”和“实际发送的请求面”不会错位；渐进式插件激活会让旧快照失效，并在下一次模型调用前发布明确递增的 `surface_generation`。 @@ -58,15 +58,15 @@ Runtime Binder 把这份脱敏 Manifest 挂到本轮递归不可变的 Asset Bun ### 统一 Context IR -每次模型调用前，`core/llm/context_ir.py` 都会把最终上下文统一转换成不可变的 `ContextItem`。用户输入、助手历史、账户身份、冻结记忆、项目/文件材料、附件、工具调用与结果、压缩摘要、执行中追加指令、Harness 系统提醒，都会携带机器可读的 `kind`、来源、信任级、可见性、优先级、单项预算、截断策略、内容引用/哈希、缓存类别和创建顺序。为兼容模型，记忆和对话中途的提醒仍可渲染成 `user` role，但系统不再靠 XML 标签或 role 猜测它们是谁说的。 +每次模型调用前，`core/llm/context/ir.py` 都会把最终上下文统一转换成不可变的 `ContextItem`。用户输入、助手历史、账户身份、冻结记忆、项目/文件材料、附件、工具调用与结果、压缩摘要、执行中追加指令、Harness 系统提醒，都会携带机器可读的 `kind`、来源、信任级、可见性、优先级、单项预算、截断策略、内容引用/哈希、缓存类别和创建顺序。为兼容模型，记忆和对话中途的提醒仍可渲染成 `user` role，但系统不再靠 XML 标签或 role 猜测它们是谁说的。 `ContextAssembler` 使用确定性顺序，并设置明确的消息预算：模型窗口先预留 15% 给响应，再扣除最终工具 schema 的 canonical token 预算。关键系统规则和当前用户指令不会被低可信内容挤掉；可选内容按优先级选择；文本和工具输出按声明的头尾保留策略截断；工具调用与对应结果则作为一个整体保留、缩短或排除。脱敏 Context Manifest 会逐项说明“纳入、截断、排除”及原因，不保存正文；其哈希会进入本次请求的 `context_hash` 与 `aggregate_hash`，再重新绑定到该 run 的 Asset Bundle。 -`core/llm/context_adapter.py::AgentScopeContextAdapter` 是 Context IR 与 AgentScope 消息之间唯一的转换入口。持久化会话中的来源元数据可在历史回放时保留，最终模型 role 和 SSE 行为继续兼容。`ManifestBoundAgent` 会在 run-scoped 工具/技能请求面冻结后，通过公开的 `Agent.reply` 链路完成最终装配。 +`core/llm/context/adapter.py::AgentScopeContextAdapter` 是 Context IR 与 AgentScope 消息之间唯一的转换入口。持久化会话中的来源元数据可在历史回放时保留，最终模型 role 和 SSE 行为继续兼容。`ManifestBoundAgent` 会在 run-scoped 工具/技能请求面冻结后，通过公开的 `Agent.reply` 链路完成最终装配。 ## 提示词版本池（prompt_versions） -版本池把多套提示词存进单行 `ContentBlock(id="prompt_versions")`，payload 结构 `{active: {kind: version_id}, versions: [...]}`，服务层为 `core/services/prompt_version_service.py`： +版本池把多套提示词存进单行 `ContentBlock(id="prompt_versions")`，payload 结构 `{active: {kind: version_id}, versions: [...]}`，服务层为 `core/llm/prompt_versions.py`： - **六类 kind**（`VALID_KINDS`）：`system`（主智能体）、`code_exec`（代码执行能力段）、`distillation`（技能蒸馏）、`plan_mode`（计划模式）、`subagents`（平台默认探索员 / 执行员 / 审查员的独立提示词）、`turbo`（极速模式独立提示词）。 - 每个版本含 `(kind, id, name, description, parts[])`，part 即 `{part_id, display_name, content, sort_order, is_enabled}`。 @@ -120,18 +120,6 @@ Runtime Binder 把这份脱敏 Manifest 挂到本轮递归不可变的 Asset Bun 两端点接受 `ADMIN_TOKEN` 或 `CONFIG_TOKEN`。快照 schema 经 `PROMPT_BLOCK_MAP` 校验，docs 快照与 prompts 快照不能从错误的端点互导。 -### 脚本（src/backend/scripts/） - -```bash -# 导出（走运行中的后端 API；也支持 --database-url 直连库） -python scripts/export_content.py --api-url http://localhost:3000/api --only prompts -# → scripts/exported/prompts_snapshot_.json - -# 导入到目标环境（用目标机 .env 的 ADMIN_TOKEN） -python scripts/import_content.py --api-url http:///api --prompts prompts_snapshot_.json -# 支持 --no-overwrite / --dry-run -``` - 离线生产环境（镜像包交付，DB 卷持久化）同样适用：快照文件随镜像包拷入，在 backend 容器内 `curl -X POST .../v1/content/prompts/import` 导入即可，无需重启。换品牌环境迁移时，需人工通读快照逐处改写品牌相关话术后再导入（不要机械查找替换）。 ## 相关源码 @@ -141,10 +129,9 @@ python scripts/import_content.py --api-url http:///api --prompts prompts_s | 运行时装配 + 缓存 | `src/backend/prompts/prompt_runtime.py` | | Provider（filesystem/inline/minimal） | `src/backend/prompts/provider.py` | | 配置（provider/parts/PROMPT_DIR） | `src/backend/prompts/prompt_config.py`，`prompts/config/default.json` | -| 版本池服务 | `src/backend/core/services/prompt_version_service.py` | +| 版本池服务 | `src/backend/core/llm/prompt_versions.py` | | 管理台路由 | `src/backend/api/routes/v1/admin_prompts.py` | -| 迁移接口（export/import） | `src/backend/api/routes/v1/content.py`，`core/content/content_blocks.py` | -| 迁移脚本 | `src/backend/scripts/export_content.py`，`scripts/import_content.py` | +| 迁移接口（export/import） | `src/backend/api/routes/v1/content.py`，`core/files/content/content_blocks.py` | | 系统提示词兜底文件 | `src/backend/prompts/prompt_text/default/system/` | | 场景提示词兜底 | `src/backend/prompts/prompt_text/{code_exec,distillation,plan_mode,turbo}/` | | 动态段 | `src/backend/prompts/kb_lite_section.py`，`prompts/project_section.py` | diff --git a/document/zh-CN/modules/sandbox.md b/document/zh-CN/modules/sandbox.md index 3421f464..6b30f17e 100644 --- a/document/zh-CN/modules/sandbox.md +++ b/document/zh-CN/modules/sandbox.md @@ -84,7 +84,7 @@ Cube 的设计取舍（远端节点版的代价）：所有语言统一走"写 其它要点： -- **登记不挂会话、也不挂产物卡片**：「我的空间」是用户级的一份目录，每个会话的沙箱都挂着它，文件事件里只有路径、没有会话身份——谁写的，这里无从得知。所以登记只记到用户名下（`Artifact.chat_id` 为空），产物区的文件卡片则完全不由登记器挂：卡片只能由**写文件的那一轮自己**挂（模型调 `pin_to_workspace`，或那一轮里的 Write/交付兜底，都走 `core.llm.workspace` 的 ContextVar），那里"哪个会话、哪个文件"都是确定的。曾经拿"这个用户当下随便哪个在跑的会话"顶上，结果是另一个会话写的文件被记到了这个会话名下，还在它的产物区里多出一张卡片。 +- **登记不挂会话、也不挂产物卡片**：「我的空间」是用户级的一份目录，每个会话的沙箱都挂着它，文件事件里只有路径、没有会话身份——谁写的，这里无从得知。所以登记只记到用户名下（`Artifact.chat_id` 为空），产物区的文件卡片则完全不由登记器挂：卡片只能由**写文件的那一轮自己**挂（模型调 `pin_to_workspace`，或那一轮里的 Write/交付兜底，都走 `core.llm.runtime.workspace` 的 ContextVar），那里"哪个会话、哪个文件"都是确定的。曾经拿"这个用户当下随便哪个在跑的会话"顶上，结果是另一个会话写的文件被记到了这个会话名下，还在它的产物区里多出一张卡片。 - **判定基准是 artifact 记录，不是镜像缓存**：开了 bind mount 后沙箱的 `/workspace/myspace/{uid}` 就是后端 `myspace_cache/{uid}` 本身，"沙箱文件与镜像缓存一致"恒真、不能再当作"已登记"。这条拓扑差异曾让回写恒为空转。 - **"磁盘比账本新"是唯一依据**：后端自己往镜像目录写文件（界面上传的物化、否决后的还原）时会把 mtime 对齐到登记时间，于是监听器不会把后端自己的写入当成沙箱的新改动转头再登记一次——不必另外维护一张"这是我自己写的"的表。 - **每个 worker 都监听，认领去重**：确认条活在跑这个会话的那个进程里，所以监听器必须每个 worker 都跑（代价是 inotify watch 按「worker 数 × 目录数」占内核配额 `fs.inotify.max_user_watches`，目录极多的部署要相应调大，超限会**静默丢事件**）；同一次改动由 `core.infra.ephemeral` 的认领保证只处理一次，且优先交给托管着该用户会话的进程（它才弹得出确认条）。 @@ -176,7 +176,7 @@ artifact 送入沙盒、`Bash` 执行后 `/myspace` 写回同步；backend 与 s ## 管理员沙箱管理 - **只读监控（安全管理台）**：`api/routes/v1/config_security.py` 暴露 `/v1/config/security/sandbox/*`——总览、实例列表、单实例详情、快照列表、重建历史、生效配置；底层走 provider 的 `admin_*` 接口，按 `admin_capabilities()` 声明裁剪 UI（script_runner 无法枚举实例即置灰对应列）。 -- **依赖重建（商业版 EE）**：`api/routes/v1/admin_sandbox.py`（`/v1/admin/sandbox/*`）聚合所有技能声明的 pip/apt 依赖（`core/services/skill_deps_aggregator.py`），管理员一键触发沙箱镜像重建：`script-runner` / `opensandbox` 目标走本地 `docker compose build`，`cube` 目标走 SSH 到远端节点重建模板并热切（`core/services/sandbox_rebuild_service.py` + `cube_template_builder`），可查看每次 run 的状态与日志。技能新依赖从此"烤入"镜像，无需手工改 Dockerfile。 +- **依赖重建（商业版 EE）**：`api/routes/v1/admin_sandbox.py`（`/v1/admin/sandbox/*`）聚合所有技能声明的 pip/apt 依赖（`core/capabilities/skills/deps/aggregator.py`），管理员一键触发沙箱镜像重建：`script-runner` / `opensandbox` 目标走本地 `docker compose build`，`cube` 目标走 SSH 到远端节点重建模板并热切（`core/services/sandbox_rebuild_service.py` + `cube_template_builder`），可查看每次 run 的状态与日志。技能新依赖从此"烤入"镜像，无需手工改 Dockerfile。 ## 关键环境变量 @@ -216,7 +216,7 @@ artifact 送入沙盒、`Bash` 执行后 `/myspace` 写回同步；backend 与 s | `src/backend/core/sandbox/providers/opensandbox/pool.py` | 双桶预热池 | | `src/backend/core/sandbox/providers/cube/provider.py` | Cube 远端 MicroVM provider（EE） | | `src/backend/core/llm/tools/sandbox_tool.py` | Bash / write_stdin / sandbox_put_artifact / sandbox_get_artifact | -| `src/backend/core/llm/offloader.py` | 超长结果落盘 /workspace/.offload | +| `src/backend/core/llm/context/offloader.py` | 超长结果落盘 /workspace/.offload | | `src/backend/api/routes/v1/admin_sandbox.py` | 依赖重建管理 API（EE） | | `src/backend/api/routes/v1/config_security.py` | 安全管理台沙箱只读视图 | | `src/backend/core/services/sandbox_rebuild_service.py` | 镜像/模板重建编排（EE） | @@ -245,3 +245,24 @@ OpenSandbox 的实例列表与实例计数读取控制器当前运行清单， 控制器查询失败时，计数不可用；不能把查询失败报告成零实例。 连接池诊断保留 scope=worker 与 worker_pid，表示当前后端进程的本地池， 不代表全部进程的用户配额。该修复不把本地池上限改成跨进程全局配额。 + +## 共享容量与维护所有权 + +OPENSANDBOX_POOL_MAX_TOTAL 现在由控制器依据 Docker 实例库存统一执行， +覆盖所有 backend worker、预热实例、会话恢复和计算机实例。各进程池统计仅用于本地诊断。 +控制器在容器创建时持有共享数据卷上的文件锁，不把挂载初始化放入临界区； +库存达到上限或另一创建正在占用准入锁时返回 429，调用方可稍后重试。 +控制器的 opensandbox_data 数据卷应在同一控制器的进程间共享。 + +领导进程从共享会话登记发现空闲会话，即使会话由其他 worker 创建或经历过重启， +也进入快照回收流程。失效登记按身份清理，不阻断其他会话。 +快照任务随领导权停止；GC 只删除本次处理的 snapshot_id 对应记录。 + +输出读取不再触发挂载准备或对象存储目录写入。目录初始化发生在创建路径， +执行和文件访问仍核验权限。挂载探测超时或返回不确定错误时保留现有 FUSE helper， +只有确认缺失或断开时重建。命令观察失败先有限重试，再交给持久恢复； +没有可恢复引用的进程须确认停止，不能仅因观察失败就丢弃控制句柄。 + +此机制需要同时更新 backend 和 OpenSandbox 控制器镜像；只更新 backend 无法获得全局容量约束。 + +技能挂载必须配置 HOST_STORAGE_PATH，指向与后端 STORAGE_PATH 对应的宿主存储目录。缺失配置会明确失败，不再回退到源码目录。重启接管只识别显式归属标签，不根据旧启动命令推断归属。 diff --git a/document/zh-CN/modules/storage.md b/document/zh-CN/modules/storage.md index d34be521..64bfe481 100644 --- a/document/zh-CN/modules/storage.md +++ b/document/zh-CN/modules/storage.md @@ -12,7 +12,7 @@ CE 和 EE 的云端 Docker 部署默认使用 S3 兼容存储。桌面离线本 已有部署切换存储前，先备份配置、数据库和文件引用。复制所需对象并逐对象校验 SHA-256。保留源对象。 -存储工厂位于 core/storage/factory.py。S3StorageBackend 使用 boto3 并提供目录操作。LocalStorageBackend 服务桌面本地文件。OSS 和 OBS 适配仍属于 EE。 +存储工厂位于 core/files/storage/factory.py。S3StorageBackend 使用 boto3 并提供目录操作。LocalStorageBackend 服务桌面本地文件。OSS 和 OBS 适配仍属于 EE。 生成物仓库支持 S3、OSS 和 OBS。元数据记录保存在本地，并镜像到对象存储。迁移时保留原存储键，以保持历史下载引用有效。 diff --git a/pyproject.toml b/pyproject.toml index c774c665..fad4f86f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,7 @@ dev = [ "black>=23.0.0", "isort>=5.12.0", "mypy>=1.0.0", + "import-linter>=2.15,<3", ] # Console entry for the no-Docker local profile: `hugagent` (onboard / serve / @@ -47,7 +48,7 @@ dependencies = {file = ["requirements.txt"]} where = ["src/backend"] include = [ "api*", "core*", "orchestration*", "prompts*", "extensions*", - "hugagent_cli*", "edition_ee*", "personal_agent*", + "hugagent_cli*", "edition_ee*", "personal_agent*", "features*", "integrations*", ] # alembic migrations / tests / uploads / scripts are not import-time runtime deps. exclude = ["tests*", "alembic*", "uploads*", "scripts*"] @@ -55,7 +56,7 @@ exclude = ["tests*", "alembic*", "uploads*", "scripts*"] # Bundle manifests, prompts, scripts and binary assets are runtime resources. [tool.setuptools.package-data] extensions = ["README.md", "skills/**/*", "agents/**/*", "plugins/**/*", "mcp_servers/**/*.md"] -"core.applications" = ["schemas/*.json"] +"features.workbench" = ["schemas/*.json"] "core.config" = ["*.json"] prompts = ["config/*.json", "prompt_text/**/*.md"] "personal_agent.workspace" = ["prompts/*.md"] @@ -111,3 +112,12 @@ markers = [ # GCE 门禁类测试：跑得起但不属于快速内环，CI 全量跑。 "slow: gate / benchmark tests excluded from the fast inner loop", ] + +# Core boundary checks use the migration manifest to classify legacy locations. +[tool.importlinter] +root_packages = ["core", "api", "orchestration", "personal_agent", "edition_ee", "features", "integrations"] +contract_types = ["core-boundary: scripts.refactor.import_contract.CoreBoundaryContract"] + +[[tool.importlinter.contracts]] +name = "Core boundaries and layer ratchet" +type = "core-boundary" diff --git a/requirements.txt b/requirements.txt index f38daea0..0eaea501 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,10 @@ # Core Web Framework fastapi==0.115.12 uvicorn>=0.27.0 -pydantic>=2.6.0 +pydantic>=2.12.0 # Async & Utilities -anyio>=4.2.0 +anyio>=4.9.0 python-dotenv>=1.0.0 requests>=2.31.0 httpx>=0.27.0 @@ -13,11 +13,11 @@ httpx>=0.27.0 jsonschema>=4.20.0 # AgentScope (replaces LangChain ecosystem) -# 已迁移到 2.0：Agent/Toolkit/Middleware/MCP 全面重写，固定 ==2.0.0（GitHub main 已与 2.0.0 分叉，不兼容） -agentscope==2.0.0 +# Pin the reviewed MCP 2.x implementation for reproducible builds. +agentscope @ https://github.com/agentscope-ai/agentscope/archive/ed357feb78a45ff02df155bb5610eab9bf543832.tar.gz # 多厂商模型渠道所需 SDK（均为惰性 import，未装只影响对应渠道，不影响其余功能）： -# - anthropic / dashscope / openai 已随 agentscope==2.0.0 内置，无需在此重复声明 +# - anthropic / dashscope / openai 已由 AgentScope 声明依赖，无需在此重复声明 # - 以下为 agentscope 未捆绑、需显式安装的渠道 SDK： # litellm —— 非 OpenAI 协议且 AgentScope 无原生类的厂商（如 AWS Bedrock） litellm>=1.50.0 @@ -27,7 +27,9 @@ google-genai>=0.3.0 ollama>=0.3.0 # MCP Integration -mcp>=1.0.0,<2.0.0 +mcp==2.3.0 +# Keep the SSE dependency within FastAPI 0.115 Starlette constraints. +sse-starlette==3.0.3 # Additional Tools tavily-python>=0.5.0 @@ -67,9 +69,8 @@ botocore>=1.34.0 werkzeug>=3.0.0 python-multipart>=0.0.9 -# Logging & Rate Limiting +# Logging structlog>=23.2.0 -slowapi>=0.1.9 # Security safety>=3.0.0 diff --git a/scripts/performance/benchmark_preview.py b/scripts/performance/benchmark_preview.py index f319cc0d..80e51c3a 100644 --- a/scripts/performance/benchmark_preview.py +++ b/scripts/performance/benchmark_preview.py @@ -8,7 +8,7 @@ import time from docx import Document from api.routes.files import _convert_office_to_pdf -from core.content.office_preview_cache import cached_office_pdf +from core.files.content.office_preview_cache import cached_office_pdf with tempfile.TemporaryDirectory(prefix="perf172-office-") as directory: root = Path(directory) diff --git a/scripts/performance/serve_fixture.py b/scripts/performance/serve_fixture.py index a6c20c0c..5a017e70 100644 --- a/scripts/performance/serve_fixture.py +++ b/scripts/performance/serve_fixture.py @@ -11,7 +11,7 @@ import time from types import SimpleNamespace -import core.services as chat_services +import core.chat.service as chat_services_ChatService_module import uvicorn from api.routes.v1 import chats from core.auth.backend import get_current_user @@ -171,7 +171,7 @@ async def chunks(): # Isolate the slow database boundary for an HTTP concurrency experiment. The # real get_pending_user_questions route/worker scheduling remains unchanged. -OriginalService = chat_services.ChatService +OriginalService = chat_services_ChatService_module.ChatService class MeasuredService(OriginalService): @@ -182,7 +182,7 @@ def get_session(self, chat_id, user_id): return super().get_session(chat_id, user_id) -chat_services.ChatService = MeasuredService +chat_services_ChatService_module.ChatService = MeasuredService async def no_questions(chat_id): diff --git a/scripts/refactor/__init__.py b/scripts/refactor/__init__.py new file mode 100644 index 00000000..c782d1ba --- /dev/null +++ b/scripts/refactor/__init__.py @@ -0,0 +1 @@ +"""Core restructuring verification tools.""" diff --git a/scripts/refactor/check_ce_paths.py b/scripts/refactor/check_ce_paths.py new file mode 100644 index 00000000..4a414b0d --- /dev/null +++ b/scripts/refactor/check_ce_paths.py @@ -0,0 +1,79 @@ +"""Validate source paths in the CE derivation manifest.""" +from __future__ import annotations + +import ast +import fnmatch +import subprocess +from pathlib import Path + +import yaml + +ROOT = Path(__file__).resolve().parents[2] + + +def path_errors(root: Path, manifest: dict, tracked: list[str]) -> list[str]: + errors = [] + optional = set(manifest.get("optional_safety_excludes", [])) + excludes = manifest.get("exclude", []) + for pattern in optional - set(excludes): + errors.append(f"Safety exception has no exclusion: {pattern}") + for pattern in excludes: + if pattern not in optional and not any(fnmatch.fnmatch(p, pattern) and (root / p).is_file() for p in tracked): + errors.append(f"exclude: no source file matches {pattern}") + overlay = root / manifest["overlay"] + for path in manifest.get("split", []): + if not (overlay / path).is_file(): + errors.append(f"split: missing overlay {path}") + # Prunes run before overlay. Check only declared input paths, not output + # names, replacement text, dictionary values, or forbidden module names. + for name, rule in manifest.get("prunes", {}).items(): + inputs = [] + if "path" in rule: + inputs.append(rule["path"]) + inputs.extend(rule.get("paths", [])) + inputs.extend(rule.get("dockerfiles", [])) + inputs.extend(rule.get("text_paths", [])) + inputs.extend(item["path"] for item in rule.get("files", [])) + for path in inputs: + if not (root / path).exists(): + errors.append(f"prunes.{name}: missing source {path}") + generated = { + rule["optional_out"] for rule in manifest.get("prunes", {}).values() + if "optional_out" in rule + } + generated.update(pair[1] for pair in manifest.get("renames", [])) + for path in manifest.get("required_files", []): + if path not in generated and not (root / path).is_file() and not (overlay / path).is_file(): + errors.append(f"required_files: no source or overlay {path}") + for source, _target in manifest.get("renames", []): + if not (root / source).exists(): + errors.append(f"renames: missing source {source}") + # CE overlays must not restore old aggregate exports after ownership moves. + for relative in ("src/backend/core/db/models/__init__.py", "src/backend/core/db/repository/__init__.py"): + path = overlay / relative + if not path.is_file(): + continue + for node in ast.walk(ast.parse(path.read_text())): + modules = ([node.module or ""] if isinstance(node, ast.ImportFrom) + else [alias.name for alias in node.names] if isinstance(node, ast.Import) else []) + for module in modules: + if module.split(".")[0] in {"features", "integrations", "personal_agent"}: + errors.append(f"overlay aggregate forwards moved ownership: {relative} -> {module}") + return sorted(errors) + + +def main(): + manifest = yaml.safe_load((ROOT / "ce/manifest.yaml").read_text()) + tracked = subprocess.check_output( + ["git", "ls-files", "-z"], cwd=ROOT, text=True + ).split("\0") + errors = path_errors(ROOT, manifest, tracked) + for error in errors: + print(error) + if not errors: + print("CE manifest paths: passed") + return bool(errors) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/refactor/check_line_budget.py b/scripts/refactor/check_line_budget.py new file mode 100644 index 00000000..2b78b2c0 --- /dev/null +++ b/scripts/refactor/check_line_budget.py @@ -0,0 +1,37 @@ +"""Reject new oversized Python modules and growth of existing exceptions.""" +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +BASELINE = ROOT / "internal design docs" + + +def violations(root: Path, baseline: dict[str, int]) -> list[str]: + errors = [] + for directory in ("src/backend", "ce/overlay/src/backend", "scripts/refactor"): + for path in sorted((root / directory).rglob("*.py")): + relative = path.relative_to(root).as_posix() + count = len(path.read_text(encoding="utf-8-sig").splitlines()) + limit = max(500, baseline.get(relative, 500)) + if count > limit: + errors.append(f"{relative}: {count} lines exceeds {limit}") + return errors + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--baseline", type=Path, default=BASELINE) + args = parser.parse_args() + errors = violations(ROOT, json.loads(args.baseline.read_text())) + for error in errors: + print(error) + if not errors: + print("Line budget: passed (existing exceptions may only shrink)") + return bool(errors) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/refactor/check_test_baseline.py b/scripts/refactor/check_test_baseline.py new file mode 100644 index 00000000..84c01661 --- /dev/null +++ b/scripts/refactor/check_test_baseline.py @@ -0,0 +1,75 @@ +"""Compare a completed pytest run against the original failure set.""" +from __future__ import annotations + +import argparse +import json +from pathlib import Path +import re + + +def result(log: str) -> dict: + summaries = re.findall(r"^=+ (.*\bin [0-9.]+s(?: \([^\n]*\))?) =+$", log, re.MULTILINE) + if not summaries or not re.search(r"\d+ (?:passed|failed|error|skipped)", summaries[-1]): + raise ValueError("pytest has not produced a complete final summary") + if "Interrupted:" in log or "INTERNALERROR>" in log: + raise ValueError("interrupted or internally failed pytest run is not a baseline") + failures = [] + # Application logs can also start with ERROR; only parse pytest's final list. + failure_section = log.rsplit("short test summary info", 1)[-1] + for line in failure_section.splitlines(): + if line.startswith(("FAILED ", "ERROR ")): + failures.append(line.split(" ", 1)[1].split(" - ", 1)[0]) + reported = re.search(r"(\d+) failed", summaries[-1]) + errors = re.search(r"(\d+) errors?", summaries[-1]) + expected = sum(int(match[1]) for match in (reported, errors) if match) + if len(failures) != expected: + raise ValueError("failure list is incomplete; keep the full pytest report") + return {"summary": summaries[-1], "failures": sorted(set(failures))} + + +def canonical_failures(failures: list[str], moves: dict[str, str]) -> set[str]: + """Compare original node IDs after explicit test-file moves, keeping parameters intact.""" + destinations = set() + for old, new in moves.items(): + for path in (old, new): + if not isinstance(path, str) or not path.startswith("src/backend/tests/") or not path.endswith(".py") or ".." in Path(path).parts or "\\" in path: + raise ValueError(f"Invalid test migration path: {path!r}") + if old == new or new in destinations or new in moves: + raise ValueError(f"Ambiguous test migration: {old} -> {new}") + destinations.add(new) + reverse = {new: old for old, new in moves.items()} + normalized = set() + for failure in failures: + path, separator, test = failure.partition("::") + normalized.add(reverse.get(path, path) + separator + test) + return normalized + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("log", type=Path) + mode = parser.add_mutually_exclusive_group(required=True) + mode.add_argument("--save", type=Path) + mode.add_argument("--baseline", type=Path) + parser.add_argument("--test-moves", type=Path, help="Reviewed original-to-current test file mapping") + args = parser.parse_args() + if args.save and args.test_moves: + parser.error("save the original baseline without migration normalization") + actual = result(args.log.read_text()) + if args.save: + if args.save.exists(): + parser.error("baseline already exists; preserve the original evidence") + args.save.write_text(json.dumps(actual, ensure_ascii=False, indent=2) + "\n") + else: + expected = json.loads(args.baseline.read_text()) + moves = json.loads(args.test_moves.read_text()) if args.test_moves else {} + introduced = sorted(canonical_failures(actual["failures"], moves) - set(expected["failures"])) + if introduced: + print("New test failures:\n" + "\n".join(introduced)) + return 1 + print(actual["summary"]) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/refactor/import_contract.py b/scripts/refactor/import_contract.py new file mode 100644 index 00000000..abf1c1ca --- /dev/null +++ b/scripts/refactor/import_contract.py @@ -0,0 +1,209 @@ +"""Import-linter contract for the core migration's dependency ratchet.""" +from __future__ import annotations + +import ast +import json +from pathlib import Path + +from importlinter.application import output +from importlinter.domain.contract import Contract, ContractCheck + +ROOT = Path(__file__).resolve().parents[2] +DATA = ROOT / "internal design docs" +LAYERS = ( + ("core.agent_runtime",), + ("core.runs", "core.jobs", "core.chat", "core.sandbox.commands"), + ("core.llm",), + ("core.evolution", "core.ontology"), + ("core.memory", "core.sandbox", "core.capabilities", "core.files", + "core.projects", "core.auth"), + ("core.models",), + ("core.db",), + ("core.config", "core.infra"), +) +FEATURE_EDGES = { + ("desktop", "marketplace"), ("desktop", "knowledge"), + ("sites", "desktop"), ("workbench", "desktop"), ("hosted_apps", "desktop"), + ("workbench", "sites"), ("hosted_apps", "sites"), + ("automation", "desktop"), # Authorized local task dispatch; see module README. + ("distillation", "marketplace"), # Explicit draft publication; see module README. + ("security_console", "sandbox_ops"), # Read-only rebuild status; see module README. +} +OUTER = ("features", "integrations", "personal_agent", "orchestration", "api") + + +def module_name(path): + name = path.removesuffix(".py").replace("/", ".") + return name.removesuffix(".__init__") + + +def belongs(name, prefix): + return name == prefix or name.startswith(prefix + ".") + + +def canonical_names(): + # Apply the accepted manifest virtually before moves. Thus the baseline + # already exposes dependency debt concealed by the old services directory. + result = {} + rows = json.loads((DATA / "manifest.json").read_text()) + supplemental = DATA / "supplemental_moves.json" + if supplemental.exists(): + rows.extend(json.loads(supplemental.read_text())) + for row in rows: + if row["action"] == "move": + result[module_name(row["src"])] = module_name(row["dst"]) + routes = DATA / "routes.json" + if routes.exists(): + for row in json.loads(routes.read_text())["moves"]: + result[module_name(row["src"])] = module_name(row["dst"]) + # A completed move may reuse an earlier source name (D11: capabilities). + # Live destination modules represent their new owner, never the historical + # alias. Preserve virtual classification only for paths not yet migrated. + destinations = {module_name(row["dst"]) for row in rows if row["action"] == "move"} + for name in destinations: + module_path = ROOT / "src/backend" / Path(*name.split(".")) + if module_path.with_suffix(".py").is_file() or (module_path / "__init__.py").is_file(): + result.pop(name, None) + test_moves = DATA / "test_moves.json" + if test_moves.exists(): + from scripts.refactor.check_test_baseline import canonical_failures + moves = json.loads(test_moves.read_text()) + canonical_failures([], moves) # Validate one-to-one test-file migrations. + for old, new in moves.items(): + result[module_name(new.removeprefix("src/backend/"))] = module_name(old.removeprefix("src/backend/")) + return result + + +def layer(name): + candidates = [ + (len(prefix), index) + for index, prefixes in enumerate(LAYERS) + for prefix in prefixes if belongs(name, prefix) + ] + return max(candidates)[1] if candidates else None + + +def edge_rule(source, target): + if belongs(source, "core") and any(belongs(target, p) for p in OUTER): + return "core-outward" + if source.startswith("features.") and target.startswith("features."): + a, b = source.split(".")[1], target.split(".")[1] + if a != b and (a, b) not in FEATURE_EDGES: + return "feature-direction" + a, b = layer(source), layer(target) + if a is not None and b is not None and a > b: + return "core-layers" + return None + + +def canonical(name, aliases): + candidates = [key for key in aliases if belongs(name, key)] + if not candidates: + return name + prefix = max(candidates, key=len) + # A destination nested under its former module is already canonical. + if any(belongs(name, target) and len(target) >= len(prefix) for target in aliases.values()): + return name + return aliases[prefix] + name[len(prefix):] + + +def private_imports_in_module(relative, text, aliases): + physical_source = module_name(relative) + source = canonical(physical_source, aliases) + if belongs(source, "core"): + return set() + package = physical_source if relative.endswith("/__init__.py") else physical_source.rpartition(".")[0] + result, bindings = set(), {} + tree = ast.parse(text) + + def bind(name, target): + # Keep every candidate across scopes: a later local alias must never + # erase an outer import and hide a boundary violation. + bindings.setdefault(name, set()).add(target) + + def record(target, symbol): + target = canonical(target, aliases) + if belongs(target, "core") and symbol.startswith("_"): + result.add(f"private-symbol: {source} -> {target}.{symbol}") + + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for alias in node.names: + bind(alias.asname or alias.name.split(".")[0], alias.name if alias.asname else alias.name.split(".")[0]) + if not isinstance(node, ast.ImportFrom): + continue + target = node.module or "" + if node.level: + parts = package.split(".") + target = ".".join(parts[:len(parts) - node.level + 1] + ([target] if target else [])) + for alias in node.names: + record(target, alias.name) + bind(alias.asname or alias.name, target + "." + alias.name) + for node in ast.walk(tree): + if not isinstance(node, ast.Attribute) or not node.attr.startswith("_"): + continue + parts, value = [], node.value + while isinstance(value, ast.Attribute): + parts.insert(0, value.attr) + value = value.value + if isinstance(value, ast.Name) and value.id in bindings: + for target in bindings[value.id]: + record(".".join([target, *parts]), node.attr) + return result + + +def private_imports(): + result = set() + backend = ROOT / "src/backend" + aliases = canonical_names() + for path in backend.rglob("*.py"): + relative = path.relative_to(backend).as_posix() + result.update(private_imports_in_module(relative, path.read_text(encoding="utf-8-sig"), aliases)) + return result + + +def package_marker_errors(backend): + """Keep namespace directories from silently disappearing from grimp's graph.""" + missing = set() + for root_name in ("core", "features", "integrations"): + root = backend / root_name + for path in root.rglob("*.py"): + if "__pycache__" in path.parts: + continue + parent = path.parent + while parent != backend: + if not (parent / "__init__.py").is_file(): + missing.add("package-marker: " + parent.relative_to(backend).as_posix()) + parent = parent.parent + return missing + + +def violations(graph): + aliases = canonical_names() + result = private_imports() | package_marker_errors(ROOT / "src/backend") + for source in sorted(graph.modules): + for target in graph.find_modules_directly_imported_by(source): + rule = edge_rule(aliases.get(source, source), aliases.get(target, target)) + if rule: + result.add(f"{rule}: {aliases.get(source, source)} -> {aliases.get(target, target)}") + return result + + +class CoreBoundaryContract(Contract): + """Require existing architecture violations to decrease, never increase.""" + + def check(self, graph, verbose): + current = violations(graph) + baseline = set(json.loads((DATA / "import-baseline.json").read_text())) + introduced = sorted(current - baseline) + removed = sorted(baseline - current) + return ContractCheck( + kept=not introduced, + metadata={"introduced": introduced}, + ignored_import_count=len(current & baseline), + warnings=[f"Remove {len(removed)} resolved baseline entries"] if removed else [], + ) + + def render_broken_contract(self, check): + for violation in check.metadata["introduced"]: + output.print_error(violation) diff --git a/scripts/refactor/rewrite_imports.py b/scripts/refactor/rewrite_imports.py new file mode 100644 index 00000000..59ffead9 --- /dev/null +++ b/scripts/refactor/rewrite_imports.py @@ -0,0 +1,357 @@ +"""AST-based import rewrite planner. Preview by default; --apply writes changes.""" +from __future__ import annotations + +import argparse +import ast +import importlib.util +import json +import re +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +DATA = ROOT / "internal design docs" + + +def module_name(path: str) -> str: + return path.removesuffix(".py").replace("/", ".").removesuffix(".__init__") + + +def mapped(value: str, moves: dict[str, str]) -> str: + for old in sorted(moves, key=len, reverse=True): + if value == old or value.startswith(old + "."): + return moves[old] + value[len(old):] + return value + + +def dotted(node): + if isinstance(node, ast.Name): + return node.id + if isinstance(node, ast.Attribute): + prefix = dotted(node.value) + if prefix: + return prefix + "." + node.attr + return None + + +def rewrite(source, old_module, moves, symbols=None, *, package=False): + """Return source and unresolved string references; do not execute imports.""" + symbols = symbols or {} + tree = ast.parse(source) + # AST offsets are UTF-8 bytes, not character positions. + lines = source.encode("utf-8").splitlines(keepends=True) + offsets = [0] + for line in lines: + offsets.append(offsets[-1] + len(line)) + patches = [] + unresolved = [] + parents = {child: node for node in ast.walk(tree) for child in ast.iter_child_nodes(node)} + + imported_names = {} + shadowed_names = set() + wildcard_import = False + + def record_import(binding, target): + previous = imported_names.get(binding) + if previous is not None and previous != target: + shadowed_names.add(binding) + imported_names[binding] = target + + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for alias in node.names: + record_import( + alias.asname or alias.name.split(".")[0], + alias.name if alias.asname else alias.name.split(".")[0], + ) + elif isinstance(node, ast.ImportFrom): + for alias in node.names: + if alias.name == "*": + wildcard_import = True + record_import(alias.asname or alias.name, "." * node.level + (node.module or "") + "." + alias.name) + elif isinstance(node, ast.Name) and isinstance(node.ctx, (ast.Store, ast.Del)): + shadowed_names.add(node.id) + elif isinstance(node, ast.arg): + shadowed_names.add(node.arg) + elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + shadowed_names.add(node.name) + elif isinstance(node, (ast.ExceptHandler, ast.MatchAs, ast.MatchStar)): + shadowed_names.add(node.name) + elif isinstance(node, ast.MatchMapping): + shadowed_names.add(node.rest) + + def patch_target(node): + parent = parents.get(node) + call = parents.get(parent) if isinstance(parent, ast.keyword) else parent + if not isinstance(call, ast.Call): + return False + first_argument = bool(call.args) and call.args[0] is node + target_keyword = isinstance(parent, ast.keyword) and parent.arg == "target" + if not (first_argument or target_keyword): + return False + name = dotted(call.func) or "" + head, _, suffix = name.partition(".") + resolved = imported_names.get(head, head) + ("." + suffix if suffix else "") + if not wildcard_import and head not in shadowed_names and resolved in { + "unittest.mock.patch", "unittest.mock.patch.dict", "unittest.mock.patch.multiple" + }: + return True + if name != "monkeypatch.setattr": + return False + ancestor = call + while ancestor in parents: + ancestor = parents[ancestor] + if isinstance(ancestor, (ast.FunctionDef, ast.AsyncFunctionDef)): + args = ancestor.args.posonlyargs + ancestor.args.args + ancestor.args.kwonlyargs + if not any(arg.arg == "monkeypatch" for arg in args): + continue + rebound = any( + (isinstance(child, ast.Name) and child.id == "monkeypatch" + and isinstance(child.ctx, (ast.Store, ast.Del))) + or (isinstance(child, (ast.ExceptHandler, ast.MatchAs, ast.MatchStar)) + and child.name == "monkeypatch") + or (isinstance(child, ast.MatchMapping) and child.rest == "monkeypatch") + or (isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)) + and child.name == "monkeypatch") + or (isinstance(child, ast.alias) + and (child.asname or child.name.split(".")[0]) in {"monkeypatch", "*"}) + for child in ast.walk(ancestor) + ) + return not rebound + return False + + def replace(node, text): + start = offsets[node.lineno - 1] + node.col_offset + end = offsets[node.end_lineno - 1] + node.end_col_offset + patches.append((start, end, text.encode("utf-8"))) + + def import_lines(node, statements): + prefix = lines[node.lineno - 1][:node.col_offset].decode("utf-8") + separator = "; " if prefix.strip() else "\n" + prefix + replacement = separator.join(statements) + original = ast.get_source_segment(source, node) + if replacement != original: + replace(node, replacement) + + unaliased = {} + for node in ast.walk(tree): + if isinstance(node, ast.Import): + statements = [] + for alias in node.names: + target = mapped(alias.name, moves) + statements.append("import " + target + (" as " + alias.asname if alias.asname else "")) + if target != alias.name and not alias.asname: + unaliased[alias.name] = target + # Only rewrite changed imports, preserving comments/format otherwise. + if any(mapped(a.name, moves) != a.name for a in node.names): + import_lines(node, statements) + elif isinstance(node, ast.ImportFrom): + original_module = node.module or "" + if node.level: + containing = old_module if package else old_module.rpartition(".")[0] + original_module = importlib.util.resolve_name( + "." * node.level + original_module, containing + ) + groups = {} + for alias in node.names: + submodule = original_module + "." + alias.name + if submodule in moves: + dest = moves[submodule] + parent, _, name = dest.rpartition(".") + renamed = ast.alias(name=name, asname=alias.asname or ( + alias.name if name != alias.name else None + )) + groups.setdefault(parent, []).append(renamed) + else: + target = mapped(original_module, moves) + if original_module == "core.db.models" or original_module.startswith("core.db.models."): + target = symbols.get(alias.name, target) + groups.setdefault(target, []).append(alias) + moved_self = old_module in moves + changed = any(target != original_module for target in groups) + changed |= any(original_module + "." + a.name in moves for a in node.names) + if changed or (node.level and moved_self): + statements = [ + "from " + target + " import " + ", ".join( + a.name + (" as " + a.asname if a.asname else "") for a in aliases + ) + for target, aliases in groups.items() + ] + import_lines(node, statements) + + protected_roots = { + name.split(".")[0] for pair in unaliased.items() for name in pair + } + for node in ast.walk(tree): + binding = None + if isinstance(node, ast.Name) and isinstance(node.ctx, (ast.Store, ast.Del)): + binding = node.id + elif isinstance(node, ast.arg): + binding = node.arg + elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + binding = node.name + elif isinstance(node, ast.alias): + parent = parents.get(node) + if node.asname: + binding = node.asname + elif isinstance(parent, ast.ImportFrom): + binding = node.name + elif isinstance(node, ast.ExceptHandler): + binding = node.name + elif isinstance(node, (ast.MatchAs, ast.MatchStar)): + binding = node.name + elif isinstance(node, ast.MatchMapping): + binding = node.rest + if binding in protected_roots: + raise ValueError(f"ambiguous binding {binding!r} at line {node.lineno}; manual rewrite required") + + changed_roots = { + old.split(".")[0] for old, new in unaliased.items() + if old.split(".")[0] != new.split(".")[0] + } + for node in ast.walk(tree): + if not isinstance(node, ast.Name) or not isinstance(node.ctx, ast.Load): + continue + if node.id not in changed_roots: + continue + usage = node + while isinstance(parents.get(usage), ast.Attribute): + usage = parents[usage] + name = dotted(usage) + if mapped(name, unaliased) == name: + raise ValueError(f"unresolved root use {name!r} at line {node.lineno}; manual rewrite required") + + for node in ast.walk(tree): + if isinstance(node, ast.Attribute) and not isinstance(parents.get(node), ast.Attribute): + name = dotted(node) + if name and mapped(name, unaliased) != name: + replace(node, mapped(name, unaliased)) + elif isinstance(node, ast.Constant) and isinstance(node.value, str): + target = node.value + for old in sorted(moves, key=len, reverse=True): + target = re.sub( + r"(? end_of_previous: + raise ValueError("Overlapping AST edits; manual review required") + result = result[:start] + replacement + result[end:] + end_of_previous = start + result = result.decode("utf-8") + ast.parse(result) + return result, unresolved + + +def load_moves(batch): + rows = json.loads((DATA / "manifest.json").read_text()) + supplemental = DATA / "supplemental_moves.json" + if supplemental.exists(): + for extra in json.loads(supplemental.read_text()): + if extra.get("override") is not True: + rows.append(extra) + continue + matches = [i for i, row in enumerate(rows) if row["src"] == extra["src"]] + if (len(matches) != 1 or not extra.get("note", "").strip() + or rows[matches[0]]["batch"] != extra["batch"] + or rows[matches[0]]["action"] != extra["action"]): + raise ValueError("Invalid reviewed migration override") + rows[matches[0]] = extra + routes = json.loads((DATA / "routes.json").read_text())["moves"] + sources, destinations, destination_modules = set(), set(), set() + for row in rows + routes: + if row.get("action", "move") != "move": + continue + if row["src"] in sources or row["dst"] in destinations or module_name(row["dst"]) in destination_modules: + raise ValueError(f"Conflicting migration path: {row['src']} -> {row['dst']}") + sources.add(row["src"]) + destinations.add(row["dst"]) + destination_modules.add(module_name(row["dst"])) + return { + module_name(row["src"]): module_name(row["dst"]) + for row in rows + routes + if row["batch"] == batch and row.get("action", "move") == "move" + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--batch", required=True) + parser.add_argument("--apply", action="store_true") + parser.add_argument("--symbols", type=Path, help="Reviewed symbol subset for this batch") + parser.add_argument("--report", type=Path, required=True) + args = parser.parse_args() + moves = load_moves(args.batch) + if not moves: + parser.error("Batch has no pure file moves; provide a supported migration batch") + if args.symbols: + symbols = json.loads(args.symbols.read_text()) + else: + owner = args.batch.split("-", 1)[-1] + symbols = { + name: target + for name, target in json.loads((DATA / "symbol_moves.json").read_text()).items() + if target in {f"features.{owner}.models", f"integrations.{owner}.models"} + } + reverse = {target: old for old, target in moves.items()} + changes = [] + report = { + "batch": args.batch, "changed_files": [], "unresolved_strings": [], + "core_string_inventory": [], "non_runtime_resources": [], + } + for base in ("src/backend", "ce/overlay/src/backend", "evaluation"): + directory = ROOT / base + for path in sorted(directory.rglob("*.py")): + relative = path.relative_to(directory).as_posix() + module = module_name(relative) + if base == "evaluation": + module = "evaluation" if relative == "__init__.py" else "evaluation." + module + old = reverse.get(module, module) + source = path.read_text(encoding="utf-8") + try: + tree = ast.parse(source) + except SyntaxError: + if "core." in source or "from core" in source or old in moves: + raise ValueError(f"Cannot parse migration source: {path}") from None + report["non_runtime_resources"].append(path.relative_to(ROOT).as_posix()) + continue + for node in ast.walk(tree): + if isinstance(node, ast.Constant) and isinstance(node.value, str) and "core." in node.value: + report["core_string_inventory"].append({ + "file": path.relative_to(ROOT).as_posix(), + "line": node.lineno, "value": node.value, + }) + try: + result, unresolved = rewrite( + source, old, moves, symbols, package=path.name == "__init__.py" + ) + except ValueError as error: + raise ValueError(f"{path.relative_to(ROOT)}: {error}") from error + if result != source: + changes.append((path, result)) + report["changed_files"].append(path.relative_to(ROOT).as_posix()) + for item in unresolved: + report["unresolved_strings"].append({ + "file": path.relative_to(ROOT).as_posix(), **item + }) + args.report.parent.mkdir(parents=True, exist_ok=True) + args.report.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n") + if args.apply and report["unresolved_strings"]: + print("No files written: review unresolved module strings in", args.report) + return 2 + if args.apply: + for path, result in changes: + path.write_text(result) + print(f"{len(changes)} files planned; {len(report['unresolved_strings'])} strings need review") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/refactor/snapshot_routes.py b/scripts/refactor/snapshot_routes.py new file mode 100644 index 00000000..cc03fe92 --- /dev/null +++ b/scripts/refactor/snapshot_routes.py @@ -0,0 +1,35 @@ +"""Print a deterministic route contract without starting application workers.""" +from __future__ import annotations + +import contextlib +import json +import sys + + +def snapshot(app): + """Retain route order and duplicate routes, including non-OpenAPI endpoints.""" + return { + "openapi_paths": sorted(app.openapi()["paths"]), + "routes": [ + { + "order": index, + "path": getattr(route, "path", None), + "methods": sorted(getattr(route, "methods", None) or []), + "kind": type(route).__name__, + } + for index, route in enumerate(app.routes) + ], + } + + +def main(): + # Third-party import diagnostics must not contaminate the JSON stream. + with contextlib.redirect_stdout(sys.stderr): + from api.app import app + + result = snapshot(app) + print(json.dumps(result, ensure_ascii=False, indent=2, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/scripts/refactor/snapshot_schema.py b/scripts/refactor/snapshot_schema.py new file mode 100644 index 00000000..319e6eab --- /dev/null +++ b/scripts/refactor/snapshot_schema.py @@ -0,0 +1,81 @@ +"""Snapshot registered ORM structure without connecting to a database.""" +from __future__ import annotations + +import contextlib +import json +import sys + + +def default_value(default): + if default is None: + return None + if hasattr(default, "sqltext"): + return {"computed": str(default.sqltext), "persisted": default.persisted} + value = getattr(default, "arg", default) + if callable(value): + return {"callable": value.__qualname__} + return str(value) + + +def constraint_snapshot(constraint): + value = { + "type": type(constraint).__name__, + "name": constraint.name, + "columns": [column.name for column in constraint.columns], + "deferrable": constraint.deferrable, + "initially": constraint.initially, + } + if hasattr(constraint, "sqltext"): + value["sqltext"] = str(constraint.sqltext) + if hasattr(constraint, "elements"): + value["references"] = [ + {"target": item.target_fullname, "ondelete": item.ondelete, + "onupdate": item.onupdate, "match": item.match} + for item in constraint.elements + ] + return value + + +def snapshot(metadata): + return { + table.fullname: { + "columns": [ + { + "name": column.name, + "type": str(column.type), + "nullable": column.nullable, + "primary_key": column.primary_key, + "default": default_value(column.default), + "server_default": default_value(column.server_default), + "onupdate": default_value(column.onupdate), + "server_onupdate": default_value(column.server_onupdate), + "foreign_keys": sorted( + (fk.target_fullname, fk.ondelete, fk.onupdate) + for fk in column.foreign_keys + ), + } + for column in table.columns + ], + "constraints": sorted( + (constraint_snapshot(item) for item in table.constraints), + key=lambda item: json.dumps(item, sort_keys=True), + ), + "indexes": sorted( + (index.name, bool(index.unique), tuple(str(e) for e in index.expressions)) + for index in table.indexes + ), + } + for table in sorted(metadata.tables.values(), key=lambda item: item.fullname) + } + + +def main(): + with contextlib.redirect_stdout(sys.stderr): + # The app is the composition root for installed feature schemas. + from api.app import app # noqa: F401 + from core.db.engine import Base + print(json.dumps(snapshot(Base.metadata), ensure_ascii=False, indent=2, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/scripts/refactor/tests/test_edition_seams.py b/scripts/refactor/tests/test_edition_seams.py new file mode 100644 index 00000000..e8fac12b --- /dev/null +++ b/scripts/refactor/tests/test_edition_seams.py @@ -0,0 +1,33 @@ +"""Enforce the completed P7 boundary instead of relying on reviewer memory.""" +import ast +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[3] + + +def test_core_and_feature_overlays_select_only_edition_policy(): + overlay = ROOT / "ce/overlay/src/backend" + violations = [] + for domain in ("core", "features"): + for path in (overlay / domain).rglob("*.py"): + if path == overlay / "features/installed.py": + continue # D10's installed-module selection is explicitly overlaid. + if path.name != "edition.py" or len(path.read_text().splitlines()) > 150: + violations.append(str(path.relative_to(ROOT))) + assert violations == [] + + +def test_enterprise_imports_are_confined_to_edition_entry_points(): + violations = [] + backend = ROOT / "src/backend" + for domain in ("core", "features", "integrations", "api", "orchestration", "personal_agent"): + for path in (backend / domain).rglob("*.py"): + if "edition" in path.stem: + continue + for node in ast.walk(ast.parse(path.read_text())): + modules = ([node.module or ""] if isinstance(node, ast.ImportFrom) + else [alias.name for alias in node.names] if isinstance(node, ast.Import) + else []) + if any(name == "edition_ee" or name.startswith("edition_ee.") for name in modules): + violations.append((str(path.relative_to(ROOT)), node.lineno)) + assert violations == [] diff --git a/scripts/refactor/tests/test_guards.py b/scripts/refactor/tests/test_guards.py new file mode 100644 index 00000000..ffdff686 --- /dev/null +++ b/scripts/refactor/tests/test_guards.py @@ -0,0 +1,127 @@ +"""Behavioral checks for the refactoring gates.""" +from pathlib import Path +from types import SimpleNamespace + +from scripts.refactor.check_ce_paths import path_errors +from scripts.refactor.check_line_budget import violations +from scripts.refactor.import_contract import edge_rule, layer +from scripts.refactor.snapshot_routes import snapshot + + +def test_routes_keep_duplicates_methods_and_registration_order(): + def route(path, methods): + return SimpleNamespace(path=path, methods=set(methods)) + app = SimpleNamespace( + routes=[route("/a", ["POST", "GET"]), route("/a", ["GET"]), route("/b", [])], + openapi=lambda: {"paths": {"/b": {}, "/a": {}}}, + ) + result = snapshot(app) + assert result["openapi_paths"] == ["/a", "/b"] + assert [r["path"] for r in result["routes"]] == ["/a", "/a", "/b"] + assert [r["order"] for r in result["routes"]] == [0, 1, 2] + assert result["routes"][0]["methods"] == ["GET", "POST"] + + +def test_line_budget_blocks_growth_but_allows_reduction(tmp_path): + path = tmp_path / "src/backend/core/example.py" + path.parent.mkdir(parents=True) + path.write_text("# line\n" * 600) + key = "src/backend/core/example.py" + assert violations(tmp_path, {key: 600}) == [] + assert violations(tmp_path, {key: 601}) == [] + assert len(violations(tmp_path, {key: 599})) == 1 + assert len(violations(tmp_path, {})) == 1 + + +def test_new_file_at_limit_passes(tmp_path): + path = tmp_path / "src/backend/new.py" + path.parent.mkdir(parents=True) + path.write_text("# line\n" * 500) + assert violations(tmp_path, {}) == [] + + +def test_ce_missing_business_path_is_not_silently_ignored(tmp_path): + manifest = {"overlay": "overlay", "exclude": ["src/missing.py"]} + assert path_errors(tmp_path, manifest, []) == [ + "exclude: no source file matches src/missing.py" + ] + + +def test_ce_safety_exception_retains_exclusion(tmp_path): + manifest = { + "overlay": "overlay", "exclude": ["private.key"], + "optional_safety_excludes": ["private.key"], + } + assert path_errors(tmp_path, manifest, []) == [] + manifest["exclude"] = [] + assert path_errors(tmp_path, manifest, []) == [ + "Safety exception has no exclusion: private.key" + ] + + +def test_ce_required_overlay_and_prune_input_are_different_phases(tmp_path): + overlay = tmp_path / "overlay/src" + overlay.mkdir(parents=True) + (overlay / "only.py").write_text("") + manifest = { + "overlay": "overlay", "required_files": ["src/only.py"], + "split": ["src/only.py"], "prunes": {"example": {"path": "src/only.py"}}, + } + assert path_errors(tmp_path, manifest, []) == [ + "prunes.example: missing source src/only.py" + ] + + +def test_dependency_directions_cover_feature_and_core_boundaries(): + assert edge_rule("core.llm.runner", "features.desktop.service") == "core-outward" + assert edge_rule("features.desktop.service", "features.sites.service") == "feature-direction" + assert edge_rule("features.sites.service", "features.desktop.service") is None + assert edge_rule("core.db.engine", "core.models.client") == "core-layers" + assert edge_rule("core.models.client", "core.db.engine") is None + + +def test_nested_command_layer_takes_priority_over_sandbox(): + assert layer("core.sandbox.commands.runtime") == 1 + assert layer("core.sandbox.providers.local") == 4 + assert edge_rule("core.llm.tools", "core.sandbox.commands.runtime") == "core-layers" + + +def test_import_contract_rejects_new_edge_even_when_existing_debt_is_allowed(monkeypatch, tmp_path): + import json + import scripts.refactor.import_contract as contract + + (tmp_path / "import-baseline.json").write_text(json.dumps(["existing"])) + monkeypatch.setattr(contract, "DATA", tmp_path) + monkeypatch.setattr(contract, "violations", lambda graph: {"existing", "new-edge"}) + gate = contract.CoreBoundaryContract("test", {}, {}) + check = gate.check(None, False) + assert not check.kept + assert check.metadata["introduced"] == ["new-edge"] + assert check.ignored_import_count == 1 + + +def test_deleted_tracked_file_cannot_satisfy_ce_path_gate(tmp_path): + manifest = {"overlay": "overlay", "exclude": ["src/business.py"]} + assert path_errors(tmp_path, manifest, ["src/business.py"]) == [ + "exclude: no source file matches src/business.py" + ] + + +def test_ce_overlay_cannot_restore_moved_repository_exports(tmp_path): + from scripts.refactor.check_ce_paths import path_errors + path = tmp_path / "overlay/src/backend/core/db/repository/__init__.py" + path.parent.mkdir(parents=True) + path.write_text("from integrations.email.repository import EmailConnectionRepository\n") + errors = path_errors(tmp_path, {"overlay": "overlay"}, []) + assert len(errors) == 1 and "forwards moved ownership" in errors[0] + path.write_text("from core.db.repository.user import UserRepository\n") + assert path_errors(tmp_path, {"overlay": "overlay"}, []) == [] + + +def test_canonical_does_not_repeat_nested_destination(): + from scripts.refactor.import_contract import canonical + moves = {"core.evolution.settlement": "core.evolution.settlement.settlement", + "core.evolution.settlement_runner": "core.evolution.settlement.settlement_runner"} + assert canonical("core.evolution.settlement", moves) == "core.evolution.settlement.settlement" + assert canonical("core.evolution.settlement.settlement", moves) == "core.evolution.settlement.settlement" + assert canonical("core.evolution.settlement.settlement_runner", moves) == "core.evolution.settlement.settlement_runner" diff --git a/scripts/refactor/tests/test_package_markers.py b/scripts/refactor/tests/test_package_markers.py new file mode 100644 index 00000000..c393c077 --- /dev/null +++ b/scripts/refactor/tests/test_package_markers.py @@ -0,0 +1,20 @@ +from scripts.refactor.import_contract import package_marker_errors + + +def test_missing_domain_and_nested_package_cannot_escape_scan(tmp_path): + nested = tmp_path / "features/marketplace/versions" + nested.mkdir(parents=True) + (nested / "selection.py").write_text("value = 1") + assert package_marker_errors(tmp_path) == { + "package-marker: features", "package-marker: features/marketplace", + "package-marker: features/marketplace/versions", + } + for path in (tmp_path / "features", nested.parent, nested): + (path / "__init__.py").write_text("") + assert package_marker_errors(tmp_path) == set() + + +def test_non_domain_scripts_do_not_need_package_markers(tmp_path): + (tmp_path / "scripts").mkdir() + (tmp_path / "scripts/job.py").write_text("pass") + assert package_marker_errors(tmp_path) == set() diff --git a/scripts/refactor/tests/test_private_imports.py b/scripts/refactor/tests/test_private_imports.py new file mode 100644 index 00000000..22f774c5 --- /dev/null +++ b/scripts/refactor/tests/test_private_imports.py @@ -0,0 +1,41 @@ +import importlib.util +from pathlib import Path + +from scripts.refactor import import_contract as contract +ALIASES = {"core.services.market": "features.marketplace.source", "core.plugins.archive": "core.capabilities.plugins.archive", "core.services.other": "features.marketplace.other"} + + +def test_future_external_direct_private_import(): + assert contract.private_imports_in_module("core/services/market.py", "from core.capabilities.plugins.archive import _extract", ALIASES) == {"private-symbol: features.marketplace.source -> core.capabilities.plugins.archive._extract"} + + +def test_module_alias_cannot_hide_private_access(): + for source in ("import core.capabilities.plugins.archive as archive; archive._extract()", "from core.capabilities.plugins import archive as archive; archive._extract()", "import core.capabilities.plugins.archive; core.capabilities.plugins.archive._extract()"): + assert contract.private_imports_in_module("core/services/market.py", source, ALIASES) == {"private-symbol: features.marketplace.source -> core.capabilities.plugins.archive._extract"} + + +def test_relative_target_resolves_before_ownership(): + assert contract.private_imports_in_module("core/services/market.py", "from ..plugins.archive import _extract", ALIASES) == {"private-symbol: features.marketplace.source -> core.capabilities.plugins.archive._extract"} + + +def test_future_feature_target_is_not_a_core_private_import(): + assert not contract.private_imports_in_module("api/host.py", "from core.services.other import _helper", ALIASES) + + +def test_internal_core_and_public_interfaces_are_allowed(): + assert not contract.private_imports_in_module("core/internal.py", "from core.capabilities.plugins.archive import _extract", ALIASES) + assert not contract.private_imports_in_module("core/services/market.py", "from core.capabilities.plugins.archive import extract", ALIASES) + + +def test_unrelated_scope_alias_cannot_hide_outer_import(): + source = "import core.capabilities.plugins.archive as archive\narchive._extract()\ndef unrelated():\n import pathlib as archive\n" + assert contract.private_imports_in_module("core/services/market.py", source, ALIASES) == {"private-symbol: features.marketplace.source -> core.capabilities.plugins.archive._extract"} + + +def test_dependency_identity_survives_a_physical_move(monkeypatch): + from types import SimpleNamespace + monkeypatch.setattr(contract, "canonical_names", lambda: {"core.old": "core.models.client"}) + monkeypatch.setattr(contract, "private_imports", lambda: set()) + def graph(source): + return SimpleNamespace(modules=[source], find_modules_directly_imported_by=lambda _: ["core.llm.stub"]) + assert contract.violations(graph("core.old")) == contract.violations(graph("core.models.client")) == {"core-layers: core.models.client -> core.llm.stub"} diff --git a/scripts/refactor/tests/test_rewrite_imports.py b/scripts/refactor/tests/test_rewrite_imports.py new file mode 100644 index 00000000..dcd2db9c --- /dev/null +++ b/scripts/refactor/tests/test_rewrite_imports.py @@ -0,0 +1,142 @@ +"""The rewriter changes ownership without executing or changing module logic.""" +from scripts.refactor.rewrite_imports import rewrite + +MOVES = {"core.services.old_service": "features.example.service"} + + +def apply(source, old="caller", moves=None, **kwargs): + return rewrite(source, old, moves or MOVES, **kwargs) + + +def test_function_local_import_keeps_alias(): + result, review = apply("def run():\n from core.services.old_service import get as read\n return read()\n") + assert " from features.example.service import get as read" in result + assert not review + + +def test_importing_submodule_from_package_preserves_local_binding(): + result, _ = apply("from core.services import old_service, other\n") + assert result == "from features.example import service as old_service\nfrom core.services import other\n" + + +def test_relative_import_in_moved_module_resolves_against_old_package(): + result, _ = apply("from .helpers import helper\n", old="core.services.old_service") + assert result == "from core.services.helpers import helper\n" + + +def test_relative_import_with_parent_keeps_original_dependency(): + result, _ = apply("from ..infra import logging\n", old="core.services.old_service") + assert result == "from core.infra import logging\n" + + +def test_patch_and_monkeypatch_targets_are_updated(): + source = "from unittest import mock\ndef test_x(monkeypatch):\n with mock.patch('core.services.old_service.get'):\n monkeypatch.setattr('core.services.old_service.value', 1)\n" + result, review = apply(source) + assert "'features.example.service.get'" in result + assert "'features.example.service.value'" in result + assert not review + + +def test_unrecognized_string_is_not_changed_and_requires_review(): + source = "target = 'core.services.old_service.get'\n" + result, review = apply(source) + assert result == source + assert review == [{"line": 1, "old": "core.services.old_service.get", "new": "features.example.service.get"}] + + +def test_unicode_and_multiple_imports_keep_valid_offsets(): + source = "标签 = '中文'; import core.services.old_service as svc\n" + result, _ = apply(source) + assert result == "标签 = '中文'; import features.example.service as svc\n" + + +def test_unaliased_dotted_import_updates_use_sites(): + result, _ = apply("import core.services.old_service\nx = core.services.old_service.get()\n") + assert result == "import features.example.service\nx = features.example.service.get()\n" + + +def test_orm_symbol_moves_do_not_move_unrelated_models(): + result, _ = apply( + "from core.db.models import Site, UserShadow\n", + symbols={"Site": "features.sites.models"}, + ) + assert result == "from features.sites.models import Site\nfrom core.db.models import UserShadow\n" + + +def test_unrelated_import_preserves_multiline_comments(): + source = "from core.db.models import (\n UserShadow, # identity remains in core\n)\n" + result, _ = apply(source) + assert result == source + + +def test_embedded_module_reference_requires_review(): + source = "message = 'Cannot import core.services.old_service.get'\n" + result, review = apply(source) + assert result == source + assert review[0]["new"] == "Cannot import features.example.service.get" + + +def test_split_import_after_semicolon_is_valid(): + result, _ = apply("x = 1; from core.services import old_service, other\n") + assert result == "x = 1; from features.example import service as old_service; from core.services import other\n" + + +def test_parameter_shadowing_rejects_unaliased_rewrite(): + import pytest + with pytest.raises(ValueError, match="ambiguous binding"): + apply("import core.services.old_service\ndef read(core):\n return core.services.old_service.get()\n") + + +def test_destination_root_shadowing_is_also_rejected(): + import pytest + with pytest.raises(ValueError, match="ambiguous binding"): + apply("import core.services.old_service\ndef read(features):\n return core.services.old_service.get()\n") + + +def test_business_patch_function_is_not_a_mock(): + source = "def patch(value):\n return value\nx = patch('core.services.old_service')\n" + result, review = apply(source) + assert result == source + assert len(review) == 1 + + +def test_imported_mock_alias_and_keyword_target_are_recognized(): + source = "from unittest.mock import patch as replace\nwith replace(target='core.services.old_service.get'):\n pass\n" + result, review = apply(source) + assert "target='features.example.service.get'" in result + assert not review + + +def test_nested_import_cannot_reclassify_outer_business_patch(): + source = "from application import patch\ndef test_real():\n from unittest.mock import patch\n return patch('core.services.old_service')\ndef unrelated():\n return patch('core.services.old_service')\n" + result, review = apply(source) + assert result == source + assert len(review) == 2 + + +def test_wildcard_import_makes_mock_binding_ambiguous(): + source = "from unittest.mock import patch\nfrom business import *\npatch('core.services.old_service')\n" + result, review = apply(source) + assert result == source + assert len(review) == 1 + + +def test_relative_import_cannot_reclassify_business_patch(): + source = "from unittest.mock import patch\ndef f():\n from .application import patch\n return patch('core.services.old_service')\n" + result, review = apply(source, old="core.example.caller") + assert result == source + assert len(review) == 1 + + +def test_rebound_monkeypatch_fixture_requires_manual_review(): + for binding in ("from application import monkeypatch", "def monkeypatch(): pass", "class monkeypatch: pass"): + source = "def test_x(monkeypatch):\n " + binding + "\n monkeypatch.setattr('core.services.old_service', 1)\n" + result, review = apply(source) + assert result == source + assert len(review) == 1 + + +def test_root_escape_after_namespace_move_requires_manual_review(): + import pytest + with pytest.raises(ValueError, match="unresolved root use"): + apply("import core.services.old_service\nmodule = core\n") diff --git a/scripts/refactor/tests/test_snapshot_schema.py b/scripts/refactor/tests/test_snapshot_schema.py new file mode 100644 index 00000000..f9635dad --- /dev/null +++ b/scripts/refactor/tests/test_snapshot_schema.py @@ -0,0 +1,33 @@ +from sqlalchemy import Column, Integer, MetaData, Table, UniqueConstraint, CheckConstraint, ForeignKeyConstraint +from scripts.refactor.snapshot_schema import snapshot + + +def test_snapshot_detects_removed_composite_unique_constraint(): + metadata = MetaData() + table = Table("example", metadata, Column("owner", Integer), Column("name", Integer), UniqueConstraint("owner", "name")) + before = snapshot(metadata) + table.constraints.remove(next(c for c in table.constraints if isinstance(c, UniqueConstraint))) + assert snapshot(metadata) != before + + +def test_snapshot_detects_check_foreign_key_and_default_changes(): + metadata = MetaData() + Table("parent", metadata, Column("a", Integer, primary_key=True), Column("b", Integer, primary_key=True)) + table = Table("child", metadata, Column("a", Integer, server_default="1"), Column("b", Integer), CheckConstraint("a > 0"), ForeignKeyConstraint(["a", "b"], ["parent.a", "parent.b"], ondelete="CASCADE")) + before = snapshot(metadata) + table.c.a.server_default = None + assert snapshot(metadata) != before + foreign = next(c for c in before["child"]["constraints"] if c["type"] == "ForeignKeyConstraint") + assert foreign["columns"] == ["a", "b"] + assert [item["target"] for item in foreign["references"]] == ["parent.a", "parent.b"] + assert all(item["ondelete"] == "CASCADE" for item in foreign["references"]) + assert any(c.get("sqltext") == "a > 0" for c in before["child"]["constraints"]) + + +def test_computed_defaults_are_stable_across_metadata_instances(): + from sqlalchemy import Computed + def schema(): + metadata = MetaData() + Table("example", metadata, Column("a", Integer), Column("b", Integer, Computed("a + 1", persisted=True))) + return snapshot(metadata) + assert schema() == schema() diff --git a/scripts/refactor/tests/test_supplemental_moves.py b/scripts/refactor/tests/test_supplemental_moves.py new file mode 100644 index 00000000..85832268 --- /dev/null +++ b/scripts/refactor/tests/test_supplemental_moves.py @@ -0,0 +1,47 @@ +import importlib.util +import json +from pathlib import Path + +import pytest + +from scripts.refactor import rewrite_imports as rewrite + + +def manifests(tmp_path, monkeypatch, supplemental): + (tmp_path / "manifest.json").write_text(json.dumps([{"src": "core/old.py", "dst": "features/one/old.py", "batch": "P2-one", "action": "move"}])) + (tmp_path / "routes.json").write_text(json.dumps({"moves": []})) + (tmp_path / "supplemental_moves.json").write_text(json.dumps(supplemental)) + monkeypatch.setattr(rewrite, "DATA", tmp_path) + + +def test_extracted_module_is_included_in_its_own_batch(tmp_path, monkeypatch): + manifests(tmp_path, monkeypatch, [{"src": "core/extracted.py", "dst": "features/two/extracted.py", "batch": "P2-two", "action": "move"}]) + assert rewrite.load_moves("P2-two") == {"core.extracted": "features.two.extracted"} + assert rewrite.load_moves("P2-one") == {"core.old": "features.one.old"} + + +@pytest.mark.parametrize("source,target", [("core/old.py", "features/two/new.py"), ("core/new.py", "features/one/old.py")]) +def test_ambiguous_source_or_destination_is_rejected(tmp_path, monkeypatch, source, target): + manifests(tmp_path, monkeypatch, [{"src": source, "dst": target, "batch": "P2-two", "action": "move"}]) + with pytest.raises(ValueError, match="Conflicting migration path"): + rewrite.load_moves("P2-two") + + +def test_reviewed_override_preserves_source_and_batch(tmp_path, monkeypatch): + manifests(tmp_path, monkeypatch, [{"src": "core/old.py", "dst": "features/one/service.py", "batch": "P2-one", "action": "move", "override": True, "note": "Resolve a module/package collision"}]) + assert rewrite.load_moves("P2-one") == {"core.old": "features.one.service"} + + +@pytest.mark.parametrize("change", [{"src": "core/missing.py"}, {"batch": "P2-other"}, {"action": "split"}, {"note": ""}]) +def test_override_requires_existing_source_same_batch_and_reason(tmp_path, monkeypatch, change): + entry = {"src": "core/old.py", "dst": "features/one/service.py", "batch": "P2-one", "action": "move", "override": True, "note": "Resolve a module/package collision"} + entry.update(change) + manifests(tmp_path, monkeypatch, [entry]) + with pytest.raises(ValueError, match="Invalid reviewed migration override"): + rewrite.load_moves("P2-one") + + +def test_module_and_package_destination_collision_is_rejected(tmp_path, monkeypatch): + manifests(tmp_path, monkeypatch, [{"src": "core/new.py", "dst": "features/one/old/__init__.py", "batch": "P2-two", "action": "move"}]) + with pytest.raises(ValueError, match="Conflicting migration path"): + rewrite.load_moves("P2-two") diff --git a/scripts/refactor/tests/test_test_baseline.py b/scripts/refactor/tests/test_test_baseline.py new file mode 100644 index 00000000..9bbea723 --- /dev/null +++ b/scripts/refactor/tests/test_test_baseline.py @@ -0,0 +1,22 @@ +import pytest +from scripts.refactor.check_test_baseline import result + + +def test_completed_failure_summary_keeps_parameter_spaces(): + parsed = result("FAILED tests/test_x.py::test_case[some value] - AssertionError\n==== 1 failed, 3 passed in 2.02s ====\n") + assert parsed["failures"] == ["tests/test_x.py::test_case[some value]"] + + +def test_incomplete_or_truncated_run_is_rejected(): + for log in ("tests/test_x.py F", "==== 1 failed in 2.02s ====", "INTERNALERROR> failure\n==== 1 passed in 2.02s ===="): + with pytest.raises(ValueError): + result(log) + + +def test_passed_run_has_empty_failure_set(): + assert result("==== 3 passed, 1 skipped in 2.02s ====\n")["failures"] == [] + + +def test_application_error_logs_are_not_test_outcomes(): + log = "ERROR unrelated application log\n==== short test summary info ====\nFAILED tests/test_x.py::test_case\n==== 1 failed in 2.02s ====\n" + assert result(log)["failures"] == ["tests/test_x.py::test_case"] diff --git a/scripts/refactor/tests/test_test_moves.py b/scripts/refactor/tests/test_test_moves.py new file mode 100644 index 00000000..55ff44b7 --- /dev/null +++ b/scripts/refactor/tests/test_test_moves.py @@ -0,0 +1,45 @@ +"""Test moves preserve exact failure identities without masking regressions.""" +import pytest +from scripts.refactor.check_test_baseline import canonical_failures + +OLD = "src/backend/tests/services/test_source.py" +NEW = "src/backend/tests/features/datasources/test_source.py" + + +def test_only_file_prefix_is_normalized(): + node = NEW + "::test_query[" + NEW + "]" + assert canonical_failures([node], {OLD: NEW}) == {OLD + "::test_query[" + NEW + "]"} + + +def test_new_tests_remain_new_failures(): + assert canonical_failures([NEW + "::test_new"], {OLD: NEW}) == {OLD + "::test_new"} + + +@pytest.mark.parametrize("moves", [ + {OLD: OLD}, + {OLD: NEW, "src/backend/tests/other.py": NEW}, + {OLD: NEW, NEW: "src/backend/tests/final.py"}, + {OLD: "src/backend/tests/../app.py"}, + {OLD: "src/backend/api/app.py"}, +]) +def test_ambiguous_or_non_test_mappings_are_rejected(moves): + with pytest.raises(ValueError): + canonical_failures([], moves) + + +def test_dependency_baseline_keeps_original_test_identity(tmp_path, monkeypatch): + import json + from scripts.refactor import import_contract + (tmp_path / "manifest.json").write_text("[]") + (tmp_path / "test_moves.json").write_text(json.dumps({OLD: NEW})) + monkeypatch.setattr(import_contract, "DATA", tmp_path) + assert import_contract.canonical_names()["tests.features.datasources.test_source"] == "tests.services.test_source" + + +def test_route_moves_are_in_dependency_baseline(tmp_path, monkeypatch): + import json + from scripts.refactor import import_contract + (tmp_path / "manifest.json").write_text("[]") + (tmp_path / "routes.json").write_text(json.dumps({"moves": [{"src": "api/routes/v1/sites.py", "dst": "features/sites/api/sites.py"}]})) + monkeypatch.setattr(import_contract, "DATA", tmp_path) + assert import_contract.canonical_names()["api.routes.v1.sites"] == "features.sites.api.sites" diff --git a/scripts/repair_artifact_storage_keys.py b/scripts/repair_artifact_storage_keys.py index dc557b8f..2e087b6b 100644 --- a/scripts/repair_artifact_storage_keys.py +++ b/scripts/repair_artifact_storage_keys.py @@ -4,10 +4,10 @@ import argparse import json -from core.content.artifact_key_repair import repair_legacy_key, rollback_legacy_key, verified_legacy_key +from core.files.artifacts.key_repair import repair_legacy_key, rollback_legacy_key, verified_legacy_key from core.db.engine import SessionLocal from core.db.models import Artifact -from core.storage import get_storage +from core.files.storage import get_storage def main(): diff --git a/scripts/sync-workbench-schema.mjs b/scripts/sync-workbench-schema.mjs index 80e398b1..f4a546d5 100644 --- a/scripts/sync-workbench-schema.mjs +++ b/scripts/sync-workbench-schema.mjs @@ -3,11 +3,11 @@ import {fileURLToPath} from 'node:url'; const root = fileURLToPath(new URL('../', import.meta.url)); for (const name of ['application.schema.json', 'module.schema.json', 'host.json']) { const source = await readFile(root + 'packages/workbench-protocol/schemas/' + name, 'utf8'); - const destination = root + 'src/backend/core/applications/schemas/' + name; + const destination = root + 'src/backend/features/workbench/schemas/' + name; if (process.argv.includes('--check')) { if (await readFile(destination, 'utf8') !== source) throw new Error('Workbench Schema artifact drift: ' + name); } else { - await mkdir(root + 'src/backend/core/applications/schemas', {recursive: true}); + await mkdir(root + 'src/backend/features/workbench/schemas', {recursive: true}); await writeFile(destination, source); } } diff --git a/scripts/test-workbench-build.mjs b/scripts/test-workbench-build.mjs index 5eb6bb86..cb00ac43 100644 --- a/scripts/test-workbench-build.mjs +++ b/scripts/test-workbench-build.mjs @@ -17,7 +17,7 @@ function fixture(name) { cpSync(join(repo, 'packages'), join(root, 'packages'), {recursive: true, filter: path => !['node_modules', 'dist'].includes(path.split('/').at(-1))}); for (const path of ['src/frontend/package.json', 'src/frontend/package-lock.json', - 'src/backend/core/applications/schemas']) { + 'src/backend/features/workbench/schemas']) { mkdirSync(dirname(join(root, path)), {recursive: true}); cpSync(join(repo, path), join(root, path), {recursive: true}); } diff --git a/src/backend/alembic/env.py b/src/backend/alembic/env.py index bc4b71fb..a75537aa 100644 --- a/src/backend/alembic/env.py +++ b/src/backend/alembic/env.py @@ -26,6 +26,10 @@ from core.db.connection import utc_connect_args from core.db.utc_datetime import render_utc_type from core.db import models as db_models # noqa: F401 +from core.harness.registry import load_models +from features.installed import INSTALLED_FEATURES + +load_models(INSTALLED_FEATURES) config.set_main_option("sqlalchemy.url", settings.db.url) diff --git a/src/backend/alembic/versions/ce_0001_initial.py b/src/backend/alembic/versions/ce_0001_initial.py index 47ec1f45..6313bacc 100644 --- a/src/backend/alembic/versions/ce_0001_initial.py +++ b/src/backend/alembic/versions/ce_0001_initial.py @@ -24,7 +24,7 @@ def upgrade() -> None: - from core.db.edition_tables import ce_create_all + from core.db.edition import ce_create_all ce_create_all(op.get_bind()) diff --git a/src/backend/alembic/versions/ce_0002_reconcile_schema.py b/src/backend/alembic/versions/ce_0002_reconcile_schema.py index b586d38b..8ff9b8cf 100644 --- a/src/backend/alembic/versions/ce_0002_reconcile_schema.py +++ b/src/backend/alembic/versions/ce_0002_reconcile_schema.py @@ -14,7 +14,7 @@ def upgrade() -> None: - from core.db.edition_tables import ce_reconcile_schema + from core.db.edition import ce_reconcile_schema ce_reconcile_schema(op.get_bind()) diff --git a/src/backend/alembic/versions/ce_0003_mcp_marketplace.py b/src/backend/alembic/versions/ce_0003_mcp_marketplace.py index 2082dc09..7a39d7e0 100644 --- a/src/backend/alembic/versions/ce_0003_mcp_marketplace.py +++ b/src/backend/alembic/versions/ce_0003_mcp_marketplace.py @@ -17,7 +17,7 @@ def upgrade() -> None: # CE intentionally owns an independent migration chain. Reusing the CE # metadata reconciler creates the new core marketplace tables/columns while # continuing to omit every EE-only table and foreign-key dependency. - from core.db.edition_tables import ce_reconcile_schema + from core.db.edition import ce_reconcile_schema ce_reconcile_schema(op.get_bind()) diff --git a/src/backend/alembic/versions/ce_0009_desktop_delivery.py b/src/backend/alembic/versions/ce_0009_desktop_delivery.py index 21567665..95db8694 100644 --- a/src/backend/alembic/versions/ce_0009_desktop_delivery.py +++ b/src/backend/alembic/versions/ce_0009_desktop_delivery.py @@ -8,7 +8,7 @@ def upgrade() -> None: - from core.db.edition_tables import ce_reconcile_schema + from core.db.edition import ce_reconcile_schema ce_reconcile_schema(op.get_bind()) diff --git a/src/backend/alembic/versions/ce_0010_desktop_channel_relay.py b/src/backend/alembic/versions/ce_0010_desktop_channel_relay.py index a8f644d2..349e3cb9 100644 --- a/src/backend/alembic/versions/ce_0010_desktop_channel_relay.py +++ b/src/backend/alembic/versions/ce_0010_desktop_channel_relay.py @@ -8,7 +8,7 @@ def upgrade() -> None: - from core.db.edition_tables import ce_reconcile_schema + from core.db.edition import ce_reconcile_schema ce_reconcile_schema(op.get_bind()) diff --git a/src/backend/alembic/versions/ce_0012_model_steps.py b/src/backend/alembic/versions/ce_0012_model_steps.py index fce26851..b00ca229 100644 --- a/src/backend/alembic/versions/ce_0012_model_steps.py +++ b/src/backend/alembic/versions/ce_0012_model_steps.py @@ -8,11 +8,10 @@ def upgrade() -> None: - from core.db.edition_tables import ce_reconcile_schema + from core.db.edition import ce_reconcile_schema ce_reconcile_schema(op.get_bind()) def downgrade() -> None: raise NotImplementedError("Model-step reconciliation preserves conversation history") - diff --git a/src/backend/alembic/versions/ce_0013_tool_media.py b/src/backend/alembic/versions/ce_0013_tool_media.py index 1fd478ab..6d9d8009 100644 --- a/src/backend/alembic/versions/ce_0013_tool_media.py +++ b/src/backend/alembic/versions/ce_0013_tool_media.py @@ -8,7 +8,7 @@ def upgrade() -> None: - from core.db.edition_tables import ce_reconcile_schema + from core.db.edition import ce_reconcile_schema ce_reconcile_schema(op.get_bind()) diff --git a/src/backend/alembic/versions/ce_0014_site_access_password.py b/src/backend/alembic/versions/ce_0014_site_access_password.py index 69c10300..e707944d 100644 --- a/src/backend/alembic/versions/ce_0014_site_access_password.py +++ b/src/backend/alembic/versions/ce_0014_site_access_password.py @@ -8,7 +8,7 @@ def upgrade() -> None: - from core.db.edition_tables import ce_reconcile_schema + from core.db.edition import ce_reconcile_schema ce_reconcile_schema(op.get_bind()) diff --git a/src/backend/alembic/versions/ce_0019_history_display.py b/src/backend/alembic/versions/ce_0019_history_display.py index 4beba214..ef9bb570 100644 --- a/src/backend/alembic/versions/ce_0019_history_display.py +++ b/src/backend/alembic/versions/ce_0019_history_display.py @@ -8,7 +8,7 @@ def upgrade() -> None: - from core.db.edition_tables import ce_reconcile_schema + from core.db.edition import ce_reconcile_schema ce_reconcile_schema(op.get_bind()) diff --git a/src/backend/alembic/versions/ce_0020_plugin_resources.py b/src/backend/alembic/versions/ce_0020_plugin_resources.py index b7e8e9a4..8aba8436 100644 --- a/src/backend/alembic/versions/ce_0020_plugin_resources.py +++ b/src/backend/alembic/versions/ce_0020_plugin_resources.py @@ -8,7 +8,7 @@ def upgrade(): - from core.db.edition_tables import ce_reconcile_schema + from core.db.edition import ce_reconcile_schema ce_reconcile_schema(op.get_bind()) diff --git a/src/backend/alembic/versions/ce_0021_retire_site_storage.py b/src/backend/alembic/versions/ce_0021_retire_site_storage.py index eb623ef2..d502271e 100644 --- a/src/backend/alembic/versions/ce_0021_retire_site_storage.py +++ b/src/backend/alembic/versions/ce_0021_retire_site_storage.py @@ -36,7 +36,7 @@ def downgrade(): raise RuntimeError("Set SITE_STORAGE_RESTORE_FILE to the verified retirement archive") manifest = json.loads(Path(filename).read_text(encoding="utf-8")) payload = manifest["tables"] - from core.db.site_storage_retirement import _digest + from core.db.site_storage_retirement import digest as _digest if manifest.get("revision") != "site05retire" or manifest.get("sha256") != _digest(payload): raise RuntimeError("Retired storage archive is invalid") diff --git a/src/backend/alembic/versions/ce_0028_personal_agent.py b/src/backend/alembic/versions/ce_0028_personal_agent.py index 63804b17..25eaaac4 100644 --- a/src/backend/alembic/versions/ce_0028_personal_agent.py +++ b/src/backend/alembic/versions/ce_0028_personal_agent.py @@ -16,8 +16,8 @@ def upgrade(): # CE bootstrap also creates current metadata; checkfirst handles fresh installs. - from core.db.edition_tables import _ce_metadata - metadata = _ce_metadata() + from core.db.personal_schema import ce_metadata + metadata = ce_metadata() metadata.create_all(op.get_bind(), tables=[metadata.tables[name] for name in TABLES], checkfirst=True) diff --git a/src/backend/alembic/versions/ce_0031_automation_lease_token.py b/src/backend/alembic/versions/ce_0031_automation_lease_token.py new file mode 100644 index 00000000..3a30147d --- /dev/null +++ b/src/backend/alembic/versions/ce_0031_automation_lease_token.py @@ -0,0 +1,19 @@ +"""Fence scheduled run completion with the current execution token.""" + +import sqlalchemy as sa +from alembic import op + +revision = "ce_0031" +down_revision = "ce_0030" +branch_labels = None +depends_on = None + + +def upgrade(): + columns = {c["name"] for c in sa.inspect(op.get_bind()).get_columns("scheduled_task_runs")} + if "lease_token" not in columns: + op.add_column("scheduled_task_runs", sa.Column("lease_token", sa.String(64), nullable=True)) + + +def downgrade(): + op.drop_column("scheduled_task_runs", "lease_token") diff --git a/src/backend/alembic/versions/ce_0032_automation_queue.py b/src/backend/alembic/versions/ce_0032_automation_queue.py new file mode 100644 index 00000000..80d7ef2c --- /dev/null +++ b/src/backend/alembic/versions/ce_0032_automation_queue.py @@ -0,0 +1,25 @@ +"""Durable queued automation invocations.""" +from alembic import op + +revision = "ce_0032" +down_revision = "ce_0031" +branch_labels = None +depends_on = None + + +def upgrade(): + from features.automation.schema import upgrade_queue + upgrade_queue(op.get_bind()) + + +def downgrade(): + # Refuse to erase queued work while downgrading the status contract. + from sqlalchemy import text + if op.get_bind().execute(text( + "SELECT 1 FROM scheduled_task_runs WHERE status = 'pending' LIMIT 1" + )).first(): + raise RuntimeError("Drain pending automation runs before downgrade") + with op.batch_alter_table("scheduled_task_runs") as batch: + batch.drop_constraint("scheduled_task_runs_status_check", type_="check") + batch.create_check_constraint("scheduled_task_runs_status_check", + "status IN ('running', 'success', 'failed')") diff --git a/src/backend/alembic/versions/ce_0033_automation_targets.py b/src/backend/alembic/versions/ce_0033_automation_targets.py new file mode 100644 index 00000000..e4b38e6d --- /dev/null +++ b/src/backend/alembic/versions/ce_0033_automation_targets.py @@ -0,0 +1,17 @@ +"""Normalize stored automation delivery destinations.""" +from alembic import op + +revision = "ce_0033" +down_revision = "ce_0032" +branch_labels = None +depends_on = None + + +def upgrade(): + from features.automation.schema import upgrade_delivery_targets + upgrade_delivery_targets(op.get_bind()) + + +def downgrade(): + # The earlier schema already reads delivery_targets; no lossy reverse conversion. + pass diff --git a/src/backend/api/__init__.py b/src/backend/api/__init__.py index aa95844f..ea4958f5 100644 --- a/src/backend/api/__init__.py +++ b/src/backend/api/__init__.py @@ -1,5 +1,20 @@ -"""API layer for HugAgentOS.""" +"""API layer for HugAgentOS. -from .app import app +``app`` is resolved on first access: importing a submodule such as +``api.deps`` from a route module must not compose the whole application, +which would import that same, still half-initialised route module. +""" __all__ = ["app"] + + +def __getattr__(name): + if name == "app": + import importlib + + # Importing the submodule binds ``api.app`` to the module; rebind it to + # the FastAPI instance, as the former eager ``from .app import app`` did. + instance = importlib.import_module("api.app").app + globals()["app"] = instance + return instance + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/src/backend/api/app.py b/src/backend/api/app.py index 8b1944ea..77e61be8 100644 --- a/src/backend/api/app.py +++ b/src/backend/api/app.py @@ -13,6 +13,17 @@ import os import sys +from orchestration.configuration import register_configuration +register_configuration() + +from features import register_run_capabilities + +register_run_capabilities() +from features import register_model_credentials +register_model_credentials() +from integrations import register_credentials +register_credentials() + from api.lifecycle import lifespan from api.openapi import _build_custom_openapi, _docs_title from api.health import router as health_router @@ -20,7 +31,8 @@ from api.middleware.edition import edition_router_dependencies, setup_edition_middleware from api.middleware.error_handler import setup_error_handlers from api.middleware.logging import setup_logging_middleware -from api.routes import files_router, sites_serve_router +from api.routes import files_router +from features.sites.api.sites_serve import router as sites_serve_router from api.routes.v1 import ( CE_ROUTERS, EE_ROUTERS, @@ -29,9 +41,10 @@ mock_sso_router, ) from core.config.settings import settings -from core.extensions import load_extensions -from core.extensions import routers as extension_routers +from core.harness.registry import load_extensions +from core.harness.registry import routers as extension_routers from core.infra.logging import get_logger +from features.installed import INSTALLED_FEATURES from fastapi import FastAPI from fastapi.openapi.docs import get_redoc_html, get_swagger_ui_html @@ -184,7 +197,14 @@ async def root(): dependencies=edition_router_dependencies(_feature), ) # Optional modules mount their routers only when switched on. -load_extensions() +from features import register_product_tools, register_package_sources, register_resource_owners + +register_product_tools() +from features import register_catalog_options +register_catalog_options() +register_package_sources() +register_resource_owners() +load_extensions(INSTALLED_FEATURES) for _router, _feature in extension_routers(): app.include_router(_router, dependencies=edition_router_dependencies(_feature)) diff --git a/src/backend/api/deps.py b/src/backend/api/deps.py index 68c8509b..c5f2e9b6 100644 --- a/src/backend/api/deps.py +++ b/src/backend/api/deps.py @@ -52,7 +52,7 @@ async def dependency( # ②b 桌面桥接（混合架构）：桌面壳孵化的本机后端上，壳注入的云端身份携带 # 对应管理能力标志时放行——config 控制台在桌面双模式下可读本机数据。 # 云端部署不设桥接 env，此分支恒短路（零行为变化）。 - from core.auth.desktop_bridge import resolve_bridge_user + from features.desktop.auth.bridge import resolve_bridge_user bridge_user = resolve_bridge_user(request, db) if bridge_user is not None and _user_has_meta_flag(db, bridge_user.user_id, meta_flag): diff --git a/src/backend/api/edition.py b/src/backend/api/edition.py new file mode 100644 index 00000000..3acc72a7 --- /dev/null +++ b/src/backend/api/edition.py @@ -0,0 +1,13 @@ +"""Community API providers.""" +EE_ROUTERS = () + + +def __getattr__(name): + if name in ("router", "_require_can_add_agent"): + import api.personal_agents as agents + return getattr(agents, name) + raise AttributeError(name) + + +def gateway_service(db): + return None diff --git a/src/backend/api/health.py b/src/backend/api/health.py index 5f4ecea2..dce87d8f 100644 --- a/src/backend/api/health.py +++ b/src/backend/api/health.py @@ -12,7 +12,7 @@ from core.config.settings import settings from core.db.engine import get_db from core.infra.logging import get_logger -from core.storage import get_storage +from core.files.storage import get_storage logger = get_logger(__name__) diff --git a/src/backend/api/lifecycle.py b/src/backend/api/lifecycle.py index e16173da..0ca37626 100644 --- a/src/backend/api/lifecycle.py +++ b/src/backend/api/lifecycle.py @@ -9,7 +9,6 @@ from api.startup.workers import ( _env_flag, _startup_preload_enabled, - _mcp_warmup_enabled, _shutdown_automation_scheduler, _shutdown_idle_session_reaper, _startup_idle_session_reaper, @@ -42,6 +41,8 @@ _shutdown_orphan_job_reaper, _shutdown_channel_manager, _startup_warm_sandbox_pool, + _shutdown_sandbox_maintenance, + _shutdown_sandbox_provider, _startup_local_sidecars, _shutdown_local_sidecars, _startup_device_link, @@ -83,7 +84,7 @@ def _runtime_role() -> str: - from core.auth.desktop_bridge import bridge_enabled + from features.desktop.auth.bridge import bridge_enabled return EXECUTION_PLANE if bridge_enabled() else SERVICE @@ -148,7 +149,7 @@ def _startup_steps(): (_startup_recover_jobs, None, False, _ALL_ROLES, _SINGLETON), (_startup_orphan_job_reaper, _shutdown_orphan_job_reaper, False, _ALL_ROLES, _SINGLETON), (_startup_stale_run_reaper, _shutdown_stale_run_reaper, False, _ALL_ROLES, _SINGLETON), - (_startup_warm_sandbox_pool, None, False, _ALL_ROLES, _SINGLETON), + (_startup_warm_sandbox_pool, _shutdown_sandbox_maintenance, False, _ALL_ROLES, _SINGLETON), ( _startup_idle_session_reaper, _shutdown_idle_session_reaper, @@ -214,13 +215,17 @@ async def _stop_singleton_work() -> None: the role. The handlers come from the registry, so this cannot stop a different set than the one the leader started. """ - for _step, stop, _gate, _roles, scope in _startup_steps(): + failures = [] + for _step, stop, _gate, _roles, scope in reversed(_startup_steps()): if scope is not _SINGLETON or stop is None: continue try: await stop() except Exception as exc: # noqa: BLE001 - one failure must not skip the rest logger.warning("leader_stop_failed", step=stop.__name__, error=str(exc)) + failures.append(exc) + if failures: + raise ExceptionGroup("Singleton work did not stop", failures) async def _run_startup_steps(steps) -> None: @@ -297,7 +302,7 @@ async def lifespan(app: FastAPI): await _shutdown_stale_run_reaper() await _shutdown_memory_outbox_worker() await _shutdown_desktop_observability() - from core.services.desktop_gateway_observability import drain + from features.desktop.gateway.observability import drain await drain() await _shutdown_orphan_job_reaper() @@ -308,4 +313,5 @@ async def lifespan(app: FastAPI): await _shutdown_datasource_sidecar_recovery() await _shutdown_mcp_market_monitor() await _shutdown_local_sidecars() + await _shutdown_sandbox_provider() await _shutdown_pools() diff --git a/src/backend/api/middleware/agent_api_scope.py b/src/backend/api/middleware/agent_api_scope.py index d7a16926..5c895dcf 100644 --- a/src/backend/api/middleware/agent_api_scope.py +++ b/src/backend/api/middleware/agent_api_scope.py @@ -16,13 +16,8 @@ from core.auth.backend import UserContext from core.db.engine import SessionLocal from core.infra.responses import error_response -from core.services.api_key_service import api_key_token_from_header, resolve_api_key_identity -from core.services.agent_api_service import ( - begin_agent_api_call, - call_log_tracking, - fail_agent_api_call, - make_agent_api_scope, -) +from core.auth.api_keys import api_key_token_from_header, resolve_api_key_identity +from core.auth.agent_api import begin_agent_api_call, call_log_tracking, fail_agent_api_call, make_agent_api_scope class AgentApiScopeMiddleware: @@ -83,7 +78,7 @@ def authorize(): status_code=503, ) return await response(scope, receive, send) - from core.services.api_key_runs import request_api_key + from core.auth.api_key_runs import request_api_key credential_token = request_api_key.set((context.user_id, context.api_key_id)) try: diff --git a/src/backend/api/middleware/logging.py b/src/backend/api/middleware/logging.py index 72ffe64a..71c09f70 100644 --- a/src/backend/api/middleware/logging.py +++ b/src/backend/api/middleware/logging.py @@ -80,7 +80,7 @@ async def send_wrapper(message: dict) -> None: headers.append((b"x-trace-id", trace_id.encode())) # 能力变更号：桌面本机端据此发现云端能力改动并同步一次， # 免掉定时轮询（core/capabilities/change_signal.py）。 - from core.capabilities.change_signal import current + from features.desktop.capabilities.change_signal import current headers.append((b"x-hugagent-capability-epoch", current().encode())) message = {**message, "headers": headers} diff --git a/src/backend/api/openapi.py b/src/backend/api/openapi.py index f592eef5..90cce631 100644 --- a/src/backend/api/openapi.py +++ b/src/backend/api/openapi.py @@ -151,7 +151,7 @@ def _docs_title() -> str: removing the hard-coded brand. Falls back to app.title when the DB read fails (already the env brand, no literal "HugAgentOS").""" try: - from core.content.content_blocks import get_admin_platform_info + from core.files.content.branding import get_admin_platform_info info = get_admin_platform_info() name = (info.get("product_name") or "").strip() diff --git a/src/backend/api/routes/v1/agents.py b/src/backend/api/personal_agents.py similarity index 96% rename from src/backend/api/routes/v1/agents.py rename to src/backend/api/personal_agents.py index 56786c70..4c9c098b 100644 --- a/src/backend/api/routes/v1/agents.py +++ b/src/backend/api/personal_agents.py @@ -11,8 +11,8 @@ from core.db.engine import get_db from core.infra.exceptions import AccessDeniedError, BadRequestError from core.infra.responses import error_response, success_response -from core.services.agent_markdown import agent_filename, agent_to_markdown, parse_agents_upload -from core.services.user_agent_service import UserAgentService +from core.capabilities.agents.markdown import agent_filename, agent_to_markdown, parse_agents_upload +from core.capabilities.agents.edition import UserAgentService from fastapi import APIRouter, Depends, File, Response, UploadFile from pydantic import BaseModel, Field from sqlalchemy.orm import Session @@ -76,7 +76,7 @@ async def list_agents( db: Session = Depends(get_db), ): # 本机后端：云端账号的智能体登记在本机登记表里，业务库没有对应行。 - from core.capabilities import device_catalog + from features.desktop.capabilities import device_catalog agents = device_catalog.merge_items( UserAgentService(db).list_for_user(user.user_id), @@ -162,7 +162,7 @@ async def export_agent( """把单个子智能体导出为 frontmatter markdown 文件（正文即 system prompt）， 对齐 Claude Code / pi agent 一类 harness 的 subagent 文件格式，可直接重新导入。 """ - from core.llm.builtin_subagents import list_builtin_subagents + from core.capabilities.agents.builtin import list_builtin_subagents agent = next( ( @@ -238,7 +238,7 @@ async def update_agent( data = body.model_dump(exclude_none=True) # 云端同步来的智能体登记在本机登记表里，本机业务库没有对应行——不先拦下来， # 卡片上的开关会打成 404。本机只允许改启停，定义仍以云端那份为准。 - from core.capabilities import device_catalog + from features.desktop.capabilities import device_catalog projected = device_catalog.toggle_projected_agent(agent_id, data) if projected is not None: diff --git a/src/backend/api/routes/__init__.py b/src/backend/api/routes/__init__.py index ddad8534..01e58b53 100644 --- a/src/backend/api/routes/__init__.py +++ b/src/backend/api/routes/__init__.py @@ -1,10 +1,17 @@ """API routes. v1 routes are registered via the edition registry in ``api.routes.v1`` (seam C1); -this package only keeps the re-export of non-v1 file download routes. +this package only keeps the re-export of non-v1 file download routes, resolved +lazily so that importing a sibling module (``api.routes.file_location``) does not +load ``files`` and the feature routers it mounts. """ -from .files import router as files_router -from .sites_serve import router as sites_serve_router +__all__ = ["files_router"] -__all__ = ["files_router", "sites_serve_router"] + +def __getattr__(name): + if name == "files_router": + from api.routes.files import router + + return router + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/src/backend/api/routes/file_archive.py b/src/backend/api/routes/file_archive.py index dac000d5..dcd2734e 100644 --- a/src/backend/api/routes/file_archive.py +++ b/src/backend/api/routes/file_archive.py @@ -11,17 +11,12 @@ from sqlalchemy.orm import Session from core.auth.backend import UserContext, get_current_user -from core.content.archive_browser import ( - ArchiveError, - MAX_ARCHIVE_BYTES, - inspect_archive, - read_archive_member, -) +from core.files.content.archive_browser import ArchiveError, MAX_ARCHIVE_BYTES, inspect_archive, read_archive_member from core.db.engine import get_db from core.infra.exceptions import StorageError from core.infra.responses import success_response -from core.storage import get_storage -from core.storage.local import LocalStorageBackend +from core.files.storage import get_storage +from core.files.storage.local import LocalStorageBackend router = APIRouter() diff --git a/src/backend/api/routes/file_location.py b/src/backend/api/routes/file_location.py index 06a1911f..531aa629 100644 --- a/src/backend/api/routes/file_location.py +++ b/src/backend/api/routes/file_location.py @@ -5,7 +5,7 @@ from core.auth.backend import UserContext, require_auth from core.db.engine import get_db from core.db.models import Artifact -from core.services.artifact_location import artifact_location +from core.files.artifacts.location import artifact_location router = APIRouter() diff --git a/src/backend/api/routes/files.py b/src/backend/api/routes/files.py index 447e28cc..122c768f 100644 --- a/src/backend/api/routes/files.py +++ b/src/backend/api/routes/files.py @@ -8,25 +8,25 @@ from pathlib import Path from typing import Any, Optional -from core.artifacts.store import get_artifact +from core.files.artifacts.store import get_artifact from core.auth.backend import UserContext, require_auth -from core.content.office import find_libreoffice_binary -from core.content.office_preview_cache import cached_office_pdf +from core.files.content.office import find_libreoffice_binary +from core.files.content.office_preview_cache import cached_office_pdf from core.config.local_mode import local_mode_enabled -from core.storage.local import LocalStorageBackend +from core.files.storage.local import LocalStorageBackend from core.db.engine import get_db from core.db.repository import AuditLogRepository from core.infra.exceptions import StorageError from core.infra.proc import no_window_kwargs -from core.services.artifact_edition import artifact_access_metadata, can_access_artifact_metadata -from core.storage import get_storage +from core.files.edition import artifact_access_metadata, can_access_artifact_metadata +from core.files.storage import get_storage from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Query from fastapi.responses import FileResponse, JSONResponse from sqlalchemy.orm import Session logger = logging.getLogger(__name__) from api.routes.file_archive import router as archive_router -from api.routes.local_file_actions import router as local_actions_router +from features.desktop.api.local_file_actions import router as local_actions_router router = APIRouter(prefix="/files", tags=["files"]) router.include_router(archive_router) diff --git a/src/backend/api/routes/v1/__init__.py b/src/backend/api/routes/v1/__init__.py index b941b623..6bffc440 100644 --- a/src/backend/api/routes/v1/__init__.py +++ b/src/backend/api/routes/v1/__init__.py @@ -12,7 +12,7 @@ ("users", "router"), ("user_preferences", "router"), ("catalog", "router"), - ("kb", "router"), + ("features.knowledge.api.kb", "router"), ("summary", "router"), ("classify", "router"), ("config", "router"), @@ -28,13 +28,13 @@ ("agent_responses", "router"), ("agent_api_keys", "router"), ("project_api_keys", "router"), - ("agents", "router"), + ("api.edition", "router"), ("artifacts", "router"), ("space_files", "router"), ("space_previews", "router"), ("plans", "router"), ("loops", "router"), - ("automations", "router"), + ("features.automation.api.automations", "router"), ("chat_runs", "router"), ("me_system", "router"), ("me_logs", "router"), @@ -45,42 +45,42 @@ ("internal_jobs", "router"), ("jobs", "router"), ("commands", "router"), - ("internal_sites", "router"), - ("applications", "router"), - ("workbench", "router"), - ("application_transports", "internal_router"), - ("application_transports", "public_router"), - ("application_transports", "mcp_router"), + ("features.sites.api.internal_sites", "router"), + ("features.hosted_apps.api.applications", "router"), + ("features.workbench.api.workbench", "router"), + ("features.hosted_apps.api.application_transports", "internal_router"), + ("features.hosted_apps.api.application_transports", "public_router"), + ("features.hosted_apps.api.application_transports", "mcp_router"), ("project_environments", "router"), ("projects", "router"), ("api_keys", "router"), ("me_capabilities", "router"), - ("marketplace", "router"), - ("mcp_marketplace", "router"), - ("agent_marketplace", "router"), - ("plugin_ui", "router"), + ("features.marketplace.api.marketplace", "router"), + ("features.marketplace.api.mcp_marketplace", "router"), + ("features.marketplace.api.agent_marketplace", "router"), + ("features.workbench.api.plugin_ui", "router"), ("plugin_resources", "router"), ("plugins", "router"), - ("integrations", "router"), - ("channels", "router"), - ("channel_desktop", "router"), + ("integrations.api.integrations", "router"), + ("integrations.channels.api.channels", "router"), + ("integrations.channels.api.channel_desktop", "router"), ("meta", "router"), # 个人进化的证据面与偏好开关（结算查询 / prefs / 个人候选审批）。CE 完整 # 可用；控制面（管理员审批/发布）在 EE。此前 CE 漏挂本路由，首启向导的 # PATCH /v1/evolution/prefs 落到桌面本机 server 的 GET catch-all 上， # 「启动进化」直接 405。 ("evolution", "router"), - ("sites", "router"), - ("desktop", "router"), - ("desktop_login", "router"), + ("features.sites.api.sites", "router"), + ("features.desktop.api.desktop", "router"), + ("features.desktop.api.desktop_login", "router"), # The desktop shell bridge and device capability management are CE runtime APIs. - ("desktop_capability", "router"), - ("desktop_observability", "router"), - ("desktop_capabilities", "router"), + ("features.desktop.api.desktop_capability", "router"), + ("features.desktop.api.desktop_observability", "router"), + ("features.desktop.api.desktop_capabilities", "router"), # This computer's link to a Personal Agent on the cloud it signed in to. - ("device_link", "router"), - ("local", "router"), - ("local_site_sources", "router"), + ("features.desktop.api.device_link", "router"), + ("features.desktop.api.local", "router"), + ("features.sites.api.local_site_sources", "router"), # 工具执行权限档（标准 / 只读 / 自动）——输入框工具栏那颗权限胶囊的数据源 ("tool_approval", "router"), ) @@ -91,7 +91,8 @@ def iter_edition_routers(entries): for entry in entries: module_name, attr, *feature_items = entry - module = import_module(f"{__name__}.{module_name}") + module_path = module_name if "." in module_name else f"{__name__}.{module_name}" + module = import_module(module_path) feature = feature_items[0] if feature_items else None yield module_name, getattr(module, attr), feature diff --git a/src/backend/api/routes/v1/agent_api_keys.py b/src/backend/api/routes/v1/agent_api_keys.py index fcfeaefe..314ab6e6 100644 --- a/src/backend/api/routes/v1/agent_api_keys.py +++ b/src/backend/api/routes/v1/agent_api_keys.py @@ -10,8 +10,8 @@ from core.auth.capabilities import resolve_user_capabilities from core.db.engine import get_db from core.infra.responses import created_response, paginated_response, success_response -from core.services.agent_api_service import list_agent_api_calls, require_agent_manager -from core.services.api_key_service import ApiKeyService +from core.auth.agent_api import list_agent_api_calls, require_agent_manager +from core.auth.api_keys import ApiKeyService router = APIRouter(prefix="/v1/agents", tags=["Agent API Keys"]) _ALLOWED_EXPIRY_DAYS = {7, 30, 90, 180, 365} @@ -140,7 +140,7 @@ async def toggle_key( row = ApiKeyService(db).set_enabled(user.user_id, key_id, body.enabled, agent_id) data = _dto(row) if not body.enabled: - from core.services.api_key_runs import stop_cancelled_key_workers + from core.auth.api_key_runs import stop_cancelled_key_workers await stop_cancelled_key_workers(db, key_id, user.user_id) return success_response(data=data) @@ -155,7 +155,7 @@ async def revoke_key( ): _find(db, user, agent_id, key_id) ApiKeyService(db).revoke_key(user.user_id, key_id, agent_id) - from core.services.api_key_runs import stop_cancelled_key_workers + from core.auth.api_key_runs import stop_cancelled_key_workers await stop_cancelled_key_workers(db, key_id, user.user_id) return success_response(data={"id": key_id, "revoked": True}) diff --git a/src/backend/api/routes/v1/agent_responses.py b/src/backend/api/routes/v1/agent_responses.py index c9e70daa..1eda2369 100644 --- a/src/backend/api/routes/v1/agent_responses.py +++ b/src/backend/api/routes/v1/agent_responses.py @@ -1,18 +1,19 @@ """One durable admission path for streaming and complete agent responses.""" import asyncio -from core.services.session_operations.admission import _start_response_run +import api.transports.run_sse as run_sse import core.chat.context as chat_context import core.db.engine as db_engine -import core.services as chat_services -from core.chat.schemas import ChatRequest, ChatResponse +import core.runs.runtime as runtime +import core.chat.service as chat_services_ChatService_module from core.auth.backend import UserContext, get_current_user +from core.chat.schemas import ChatRequest, ChatResponse from core.db.engine import get_db from core.infra.logging import get_logger from core.infra.responses import sse_response +from core.chat.operations.admission import start_response_run as _start_response_run from fastapi import APIRouter, Depends, HTTPException -from orchestration import chat_run_executor from sqlalchemy.orm import Session logger = get_logger(__name__) @@ -20,7 +21,7 @@ async def _wait_response(run_id: str, *, chat_id: str, message_id: str) -> ChatResponse: - terminal = await chat_run_executor.wait_run(run_id) + terminal = await runtime.wait_run(run_id) if terminal.status == "cancelled": raise HTTPException(status_code=409, detail={"code": "run_cancelled", "run_id": run_id}) if terminal.status == "needs_attention": @@ -35,7 +36,7 @@ async def _wait_response(run_id: str, *, chat_id: str, message_id: str) -> ChatR ), ) with db_engine.SessionLocal() as db: - assistant = chat_services.ChatService(db).get_message_by_id(message_id) + assistant = chat_services_ChatService_module.ChatService(db).get_message_by_id(message_id) if assistant is None: raise HTTPException(status_code=500, detail="已完成的任务缺少回复") meta = assistant.extra_data or {} @@ -74,7 +75,7 @@ async def agent_response( """ run = await _start_response_run(request, user, db) if request.stream: - response = sse_response(chat_run_executor.follow_run_as_sse(run.run_id, chat_id=run.chat_id)) + response = sse_response(run_sse.follow_run_as_sse(run.run_id, chat_id=run.chat_id)) # This is the user row committed by admission, available before any # stream frames so clients can reconcile their optimistic message. response.headers["x-hugagent-user-message-id"] = run.user_message_id @@ -90,6 +91,6 @@ async def agent_response( async def _record_http_status(user: UserContext, run_id: str, status_code: int) -> None: if user.api_key_agent_id: - from core.services.agent_api_service import record_agent_api_http_status + from core.auth.agent_api import record_agent_api_http_status await asyncio.to_thread(record_agent_api_http_status, run_id, status_code) diff --git a/src/backend/api/routes/v1/api_keys.py b/src/backend/api/routes/v1/api_keys.py index 539020e2..b3547d96 100644 --- a/src/backend/api/routes/v1/api_keys.py +++ b/src/backend/api/routes/v1/api_keys.py @@ -14,7 +14,7 @@ from core.db.engine import get_db from core.infra.exceptions import AccessDeniedError, BadRequestError, ResourceNotFoundError from core.infra.responses import created_response, success_response -from core.services import ApiKeyService +from core.auth.api_keys import ApiKeyService from fastapi import APIRouter, Depends, status from pydantic import BaseModel, Field from sqlalchemy.orm import Session diff --git a/src/backend/api/routes/v1/artifacts.py b/src/backend/api/routes/v1/artifacts.py index 3fe2477c..ec0b7472 100644 --- a/src/backend/api/routes/v1/artifacts.py +++ b/src/backend/api/routes/v1/artifacts.py @@ -14,20 +14,16 @@ from typing import Any, Dict, List, Optional, Tuple from core.auth.backend import UserContext, get_current_user -from core.content.kb_processing import vectorise_document_background +from features.knowledge.processing import vectorise_document_background from core.db.engine import SessionLocal, get_db -from core.db.models import Artifact, ChatMessage, ChatSession, KBDocument, KBSpace, UserShadow +from core.db.models import Artifact, ChatMessage, ChatSession, UserShadow +from features.knowledge.models import KBDocument, KBSpace from core.db.paging import DEFAULT_PAGE_SIZE from core.db.repository import ArtifactRepository from core.infra.responses import error_response, success_response -from core.services import KBService -from core.services.artifact_edition import ( - artifact_list_scope, - artifact_scope_fields, - can_access_artifact, - extend_artifact_item, -) -from core.storage import get_storage +from features.knowledge.service import KBService +from core.files.edition import artifact_list_scope, artifact_scope_fields, can_access_artifact, extend_artifact_item +from core.files.storage import get_storage from fastapi import APIRouter, BackgroundTasks, Depends, Query from pydantic import BaseModel from sqlalchemy import desc, func @@ -60,7 +56,7 @@ def _backfill_already_done(db: Session, user_id: str) -> bool: def _mark_backfill_done(db: Session, user_id: str) -> None: - from core.services.user_service import UserService + from core.auth.users import UserService UserService(db).update_user_metadata( user_id, {_BACKFILL_MARKER: datetime.now(timezone.utc).isoformat()} @@ -104,9 +100,9 @@ class AddArtifactToKBRequest(BaseModel): # ── Shared artifact-ref helpers ─────────────────────────────────────────── -# Moved to core.content.artifact_refs so lower layers can use them without +# Moved to core.files.artifacts.refs so lower layers can use them without # importing this API route module. Re-exported here for existing call sites. -from core.content.artifact_refs import extract_file_refs, infer_artifact_type # noqa: E402 +from core.files.artifacts.refs import extract_file_refs, infer_artifact_type # noqa: E402 def sanitize_chat_preview(content: Optional[str], max_len: int = 200) -> str: @@ -140,7 +136,7 @@ def sanitize_chat_preview(content: Optional[str], max_len: int = 200) -> str: # ── Backfill (runs once per user, ever — see the durable marker above) ──── -from core.services.artifact_backfill import _backfill_artifacts_from_messages +from core.files.artifacts.backfill import backfill_artifacts_from_messages as _backfill_artifacts_from_messages def _collect_artifact_kb_usage( diff --git a/src/backend/api/routes/v1/audio.py b/src/backend/api/routes/v1/audio.py index f5a36741..ac4d2f02 100644 --- a/src/backend/api/routes/v1/audio.py +++ b/src/backend/api/routes/v1/audio.py @@ -10,7 +10,7 @@ from core.auth.backend import UserContext, require_auth from core.db.engine import get_db from core.infra.responses import success_response -from core.services.audio_transcription import MAX_BYTES, MAX_SECONDS, resolve_audio_model, transcribe, validate_wav +from core.chat.audio_transcription import MAX_BYTES, MAX_SECONDS, resolve_audio_model, transcribe, validate_wav router = APIRouter(prefix="/v1/audio", tags=["Audio"]) # Per-worker admission, no unbounded queue. Every call enters before its first await. diff --git a/src/backend/api/routes/v1/auth.py b/src/backend/api/routes/v1/auth.py index 597e0ff3..72d4eb3e 100644 --- a/src/backend/api/routes/v1/auth.py +++ b/src/backend/api/routes/v1/auth.py @@ -8,7 +8,7 @@ from typing import Optional -from core.auth import desktop_ticket_store +from features.desktop.auth import ticket_store as desktop_ticket_store from core.auth.capabilities import page_admin_flags, resolve_capabilities from core.auth.mock_ticket_store import consume_ticket from core.auth.session import ( @@ -23,7 +23,7 @@ from core.db.engine import get_db from core.infra.logging import get_logger from core.infra.responses import success_response -from core.services import UserService +from core.auth.users import UserService from fastapi import APIRouter, Depends, HTTPException, Request, Response from pydantic import BaseModel, Field from sqlalchemy.orm import Session @@ -54,7 +54,7 @@ def _clear_session_cookie(response: Response) -> None: def _serialize_user(db: Session, user_data: dict, ttl_seconds: Optional[int] = None) -> dict: from core.db.models import LocalUser, UserShadow - from core.services.local_user_service import ce_onboarding_required + from core.auth.local_user_policy import ce_onboarding_required user_id = user_data["user_id"] shadow = db.query(UserShadow).filter(UserShadow.user_id == user_id).first() diff --git a/src/backend/api/routes/v1/catalog.py b/src/backend/api/routes/v1/catalog.py index e129d5f4..40408173 100644 --- a/src/backend/api/routes/v1/catalog.py +++ b/src/backend/api/routes/v1/catalog.py @@ -8,19 +8,14 @@ from typing import Any, Dict, List, Optional from core.auth.backend import UserContext, get_current_user -from core.config.catalog_runtime import get_runtime_catalog +from core.capabilities.catalog.runtime import get_runtime_catalog from core.config.settings import settings from core.db.engine import get_db -from core.db.repository import KBRepository +from features.knowledge.repository import KBRepository from core.infra.exceptions import BadRequestError from core.infra.responses import success_response -from core.kb.external_provider import ( - get_provider_cache_identity, - get_provider_name, - is_enabled, - list_collections, -) -from core.services import CatalogService +from features.knowledge.edition import get_provider_cache_identity, get_provider_name, is_enabled, list_collections +from core.capabilities.catalog.overrides import CatalogService from fastapi import APIRouter, Depends, Path from pydantic import BaseModel, Field from sqlalchemy.orm import Session @@ -162,7 +157,7 @@ async def _ensure_external_cache_ready() -> None: def _index_modes_of(space) -> List[str]: """自建知识库的索引模式；解析失败按仅 RAG（历史库的语义）。""" try: - from core.kb.wiki.config import index_modes_of + from features.knowledge.wiki.config import index_modes_of return index_modes_of(space) except Exception: # noqa: BLE001 @@ -172,14 +167,14 @@ def _index_modes_of(space) -> List[str]: def _kb_capabilities(space) -> Dict[str, bool]: """索引模式 → 能力位，字段与外接知识库的 capabilities 同构。""" try: - from core.kb.wiki_router import capabilities_for_local_kb + from features.knowledge.wiki_router import capabilities_for_local_kb return capabilities_for_local_kb(space) except Exception: # noqa: BLE001 return {"vector": True, "keyword": True, "wiki": False, "graph": False} -from .catalog_ownership import _load_owned_capability_items, _plugin_component_ids, _is_owned_capability +from features.marketplace.api.catalog_ownership import _load_owned_capability_items, _plugin_component_ids, _is_owned_capability def _get_catalog_items_sync(user: UserContext, db: Session): @@ -199,7 +194,7 @@ def _get_catalog_items_sync(user: UserContext, db: Session): # invisible/unusable. The catalog itself queries the DB directly and does not use this # cache, so this call only affects agent resolution and is harmless to display. try: - from core.config.catalog_resolver import invalidate_capability_cache + from core.capabilities.catalog.resolver import invalidate_capability_cache invalidate_capability_cache(str(user.user_id)) except Exception as exc: # noqa: BLE001 @@ -269,7 +264,7 @@ def merge_items(base_items: List[Dict], override_items: List[Dict]) -> List[Dict # global version — otherwise the capability library would show two identically named, # identical-content items (private id=entry-fingerprint, global id=entry, which differ # so both entered the list). Merge by entry_name; dedupe only against enabled global items. - from core.services.marketplace_service import base_entry_name + from core.capabilities.skills.installation import base_entry_name as base_entry_name for kind, owned in (("skills", owned_skill_items), ("mcp", owned_mcp_items)): global_ids = {it.get("id") for it in base_catalog.get(kind, []) if it.get("enabled", True)} @@ -288,7 +283,7 @@ def merge_items(base_items: List[Dict], override_items: List[Dict]) -> List[Dict try: external_items = _list_datasets_cached() # Permission assignment: narrow to datasets visible to the current user (public + granted scoped; defaults to public when unset). - from core.auth.kb_permissions import get_dataset_levels + from features.knowledge.edition import get_dataset_levels ds_ids = [str(item.get("id", "")).strip() for item in external_items if item.get("id")] ds_levels = get_dataset_levels(db, user.user_id, ds_ids) @@ -363,8 +358,8 @@ def merge_items(base_items: List[Dict], override_items: List[Dict]) -> List[Dict public_local_kb_items: List[Dict[str, Any]] = [] if not is_ce: try: - from core.auth.kb_permissions import get_accessible_local_kb_levels, level_to_caps - from core.db.models import KBSpace + from features.knowledge.edition import get_accessible_local_kb_levels, level_to_caps + from features.knowledge.models import KBSpace levels = get_accessible_local_kb_levels(db, user.user_id) owned_ids = {s.kb_id for s in spaces} @@ -432,7 +427,7 @@ def merge_items(base_items: List[Dict], override_items: List[Dict]) -> List[Dict # 桌面双端的本机后端：云端账号的技能与连接器同步在本机登记表里，业务库中没有 # 对应行。不在这里叠加进来，能力中心就只剩本机自带的内置项——显示的和真正生效的 # 不是同一份。云端部署下叠加层为空，这几行是恒等变换。 - from core.capabilities import device_catalog + from features.desktop.capabilities import device_catalog overlay = device_catalog.catalog_overlay(user_id=str(user.user_id)) skill_list = device_catalog.merge_items( diff --git a/src/backend/api/routes/v1/catalog_mutations.py b/src/backend/api/routes/v1/catalog_mutations.py index dc0b37f9..4fb889c6 100644 --- a/src/backend/api/routes/v1/catalog_mutations.py +++ b/src/backend/api/routes/v1/catalog_mutations.py @@ -1,15 +1,15 @@ """Capability catalog preference mutations.""" from typing import Any, Dict, Optional from core.auth.backend import UserContext, get_current_user -from core.config.catalog_runtime import get_runtime_catalog +from core.capabilities.catalog.runtime import get_runtime_catalog from core.db.engine import get_db from core.infra.exceptions import BadRequestError from core.infra.responses import success_response -from core.services import CatalogService +from core.capabilities.catalog.overrides import CatalogService from fastapi import APIRouter, Depends, Path from pydantic import BaseModel, Field from sqlalchemy.orm import Session -from .catalog_ownership import _is_owned_capability +from features.marketplace.api.catalog_ownership import _is_owned_capability router = APIRouter() # Request/Response Models @@ -76,7 +76,7 @@ def update_catalog_item( # 云端同步来的技能 / 智能体登记在本机登记表里，业务库没有对应行，写目录覆盖 # 不会生效也读不回来。启停就写它——读写同源，运行时解析读的也是这份。 if request.enabled is not None and normalized_kind in ("skill", "agent"): - from core.capabilities import device_catalog + from features.desktop.capabilities import device_catalog if device_catalog.set_enabled(normalized_kind, id, request.enabled, user_id=str(user.user_id)): return success_response( @@ -92,7 +92,7 @@ def update_catalog_item( # 云端下发的连接器不在本机目录里，它是在目录解析之后按 mcp.json 的托管标志 # 追加回来的——所以启停只有写进那里才作数，写目录覆盖不会生效。 if normalized_kind == "mcp" and request.enabled is not None: - from core.services.desktop_cloud_bridge import set_managed_connector_enabled + from features.desktop.bridge.managed_connectors import set_managed_connector_enabled if set_managed_connector_enabled(id, request.enabled): return success_response( @@ -194,7 +194,7 @@ def update_catalog_item( # Invalidate this user's capability cache so the toggle takes effect on the # next message (otherwise resolve_all_runtime_enabled's 30s cache hides it). try: - from core.config.catalog_resolver import invalidate_capability_cache + from core.capabilities.catalog.resolver import invalidate_capability_cache invalidate_capability_cache(str(user.user_id)) except Exception: diff --git a/src/backend/api/routes/v1/chat_forks.py b/src/backend/api/routes/v1/chat_forks.py index 6b8a4e98..e285fda9 100644 --- a/src/backend/api/routes/v1/chat_forks.py +++ b/src/backend/api/routes/v1/chat_forks.py @@ -5,7 +5,7 @@ from core.auth.backend import UserContext, get_current_user from core.db.engine import get_db from core.infra.responses import created_response -from core.services.chat_fork_service import ChatForkError, ChatForkService +from core.chat.fork import ChatForkError, ChatForkService from fastapi import APIRouter, Depends, HTTPException from pydantic import BaseModel, ConfigDict, Field, field_validator from sqlalchemy.orm import Session diff --git a/src/backend/api/routes/v1/chat_modes.py b/src/backend/api/routes/v1/chat_modes.py index c02cc531..c8f941e3 100644 --- a/src/backend/api/routes/v1/chat_modes.py +++ b/src/backend/api/routes/v1/chat_modes.py @@ -21,7 +21,7 @@ from core.db.models.chat_mode import ChatMode from core.infra.logging import get_logger from core.infra.responses import created_response, success_response -from core.services.chat_mode_service import ChatModeService +from core.chat.modes import ChatModeService from fastapi import APIRouter, Depends, HTTPException, status from pydantic import BaseModel, Field from sqlalchemy.orm import Session diff --git a/src/backend/api/routes/v1/chat_runs.py b/src/backend/api/routes/v1/chat_runs.py index 20717039..fc272320 100644 --- a/src/backend/api/routes/v1/chat_runs.py +++ b/src/backend/api/routes/v1/chat_runs.py @@ -1,13 +1,12 @@ """Chat Run management API — cancel and durable queued input.""" -from core.services.session_operations.schemas import SteerChatRunRequest - +import core.runs.runtime as runtime from core.auth.backend import UserContext, get_current_user from core.infra.logging import get_logger from core.infra.responses import success_response -from core.services.chat_steer_service import list_run_steers, remove_pending_steer +from core.chat.steer import list_run_steers, remove_pending_steer +from core.chat.operations.schemas import SteerChatRunRequest from fastapi import APIRouter, Depends, HTTPException -from orchestration import chat_run_executor logger = get_logger(__name__) @@ -20,7 +19,7 @@ async def cancel_chat_run( user: UserContext = Depends(get_current_user), ): """取消正在执行的 chat run，真正杀掉后台任务；run 不存在返回 404，无权操作返回 403。""" - from core.services.session_operations.commands import cancel_chat_run as operation + from core.chat.operations.commands import cancel_chat_run as operation return await operation(run_id=run_id, user=user) @@ -32,7 +31,7 @@ async def steer_chat_run( user: UserContext = Depends(get_current_user), ): """Queue one instruction for the live ReAct loop's next safe boundary.""" - from core.services.session_operations.commands import steer_chat_run as operation + from core.chat.operations.commands import steer_chat_run as operation return await operation(run_id=run_id, body=body, user=user) @@ -42,7 +41,7 @@ async def list_chat_run_steers( run_id: str, user: UserContext = Depends(get_current_user), ): - run = chat_run_executor.get_run(run_id) + run = runtime.get_run(run_id) if run is None: raise HTTPException(status_code=404, detail="run not found") if run.user_id != user.user_id: @@ -56,7 +55,7 @@ async def withdraw_chat_run_steer( steer_id: str, user: UserContext = Depends(get_current_user), ): - run = chat_run_executor.get_run(run_id) + run = runtime.get_run(run_id) if run is None: raise HTTPException(status_code=404, detail="run not found") if run.user_id != user.user_id: @@ -67,7 +66,7 @@ async def withdraw_chat_run_steer( @router.get("/{run_id}", summary="获取后台运行状态") async def get_chat_run(run_id: str, user: UserContext = Depends(get_current_user)): - run = chat_run_executor.get_run(run_id) + run = runtime.get_run(run_id) if run is None or run.user_id != user.user_id: raise HTTPException(404, "Run not found") return success_response( diff --git a/src/backend/api/routes/v1/chats/confirms.py b/src/backend/api/routes/v1/chats/confirms.py index 96100814..6705a825 100644 --- a/src/backend/api/routes/v1/chats/confirms.py +++ b/src/backend/api/routes/v1/chats/confirms.py @@ -1,6 +1,6 @@ """Pending tool confirmations and user decision endpoints.""" -import core.services.session_operations.schemas as chat_models +import core.chat.operations.schemas as chat_models import api.routes.v1.chats.pending_access as chat_pending_access import core.auth.backend as auth_backend import core.db.engine as db_engine @@ -26,7 +26,7 @@ async def list_pending_confirms( 注意：本路由必须声明在 ``GET /{chat_id}`` **之前**，否则会被 path 参数路由吞掉（FastAPI 按声明顺序匹配）。 """ - from core.llm.tools import _tool_confirmation as _mc + from core.llm.tools import confirmation as _mc items = [] for cid in await _mc.list_pending_chat_ids_shared(): @@ -60,7 +60,7 @@ async def get_pending_confirm( ): raise HTTPException(status_code=404, detail="会话不存在或无权访问") - from core.llm.tools import _tool_confirmation as _mc + from core.llm.tools import confirmation as _mc # pendings: all outstanding items; the frontend restores the whole confirmation queue from this # (one round of parallel tool calls can concurrently register N distinct pending confirmations). @@ -89,7 +89,7 @@ async def file_confirm( ): raise HTTPException(status_code=404, detail="会话不存在或无权访问") - from core.llm.tools import _tool_confirmation as _mc + from core.llm.tools import confirmation as _mc res = await _mc.set_decision_shared( chat_id, body.confirm_id, body.decision, option_id=body.option_id diff --git a/src/backend/api/routes/v1/chats/feedback.py b/src/backend/api/routes/v1/chats/feedback.py index a696488d..67020df4 100644 --- a/src/backend/api/routes/v1/chats/feedback.py +++ b/src/backend/api/routes/v1/chats/feedback.py @@ -1,11 +1,11 @@ """Ontology acceptance and assistant feedback endpoints.""" -import core.services.session_operations.schemas as chat_models -import core.services.session_operations.session_context as chat_session_context +import core.chat.operations.schemas as chat_models +import core.chat.operations.session_context as chat_session_context import core.auth.backend as auth_backend import core.chat.context as chat_context import core.db.engine as db_engine -import core.services as chat_services +import core.chat.service as chat_services_ChatService_module from core.auth.backend import UserContext from core.db.models import MessageFeedback from core.infra.exceptions import ResourceNotFoundError @@ -30,12 +30,12 @@ def accept_ontology_revision( db: Session = Depends(db_engine.get_db), ): """Atomically replace one assistant message with its reviewed candidate answer.""" - chat_service = chat_services.ChatService(db) + chat_service = chat_services_ChatService_module.ChatService(db) target = chat_service.get_message_by_id(message_id) if not target or target.role != "assistant": raise ResourceNotFoundError(resource_type="chat_message", resource_id=message_id) db_user_id = chat_context.resolve_db_user_id( - db, chat_session_context._authenticated_user_id(user) + db, chat_session_context.authenticated_user_id(user) ) access = chat_service.get_session_with_access(target.chat_id, db_user_id) if access is None: @@ -65,7 +65,7 @@ def submit_feedback( if body.rating not in ("like", "dislike"): raise HTTPException(status_code=400, detail="rating must be 'like' or 'dislike'") - db_user_id = chat_session_context._authenticated_user_id(user) + db_user_id = chat_session_context.authenticated_user_id(user) existing = ( db.query(MessageFeedback) @@ -96,7 +96,7 @@ def submit_feedback( if body.rating == "dislike" and (body.comment or "").strip() and record.chat_id: try: - from core.services.ontology_evolution_service import OntologyEvolutionService + from core.ontology.evolution import OntologyEvolutionService OntologyEvolutionService(db).ingest_user_correction( user_id=db_user_id, diff --git a/src/backend/api/routes/v1/chats/history.py b/src/backend/api/routes/v1/chats/history.py index 61e2eb6d..34289c47 100644 --- a/src/backend/api/routes/v1/chats/history.py +++ b/src/backend/api/routes/v1/chats/history.py @@ -4,12 +4,12 @@ import core.auth.backend as auth_backend import core.db.engine as db_engine -import core.services as chat_services +import core.chat.service as chat_services_ChatService_module from core.auth.backend import UserContext -from core.chat.display_bounds import bound_result_for_display +from core.infra.text_bounds import bound_result_for_display from core.infra.exceptions import ResourceNotFoundError from core.infra.responses import success_response -from core.services.context_usage_projection import get_chat_context_state +from core.chat.compaction.usage_projection import get_chat_context_state from fastapi import APIRouter, Depends, Query from sqlalchemy.orm import Session @@ -31,7 +31,7 @@ def list_messages( db: Session = Depends(db_engine.get_db), ): """获取当前用户有权读取的会话消息列表；无权访问时返回 404。""" - from core.services.session_operations.history import list_messages as operation + from core.chat.operations.history import list_messages as operation return operation( chat_id=chat_id, @@ -52,11 +52,11 @@ def get_context_usage( db: Session = Depends(db_engine.get_db), ): """Return the latest provider measurement and active compaction baseline.""" - chat_service = chat_services.ChatService(db) + chat_service = chat_services_ChatService_module.ChatService(db) if chat_service.get_session_with_access(chat_id, str(user.user_id)) is None: raise ResourceNotFoundError(resource_type="chat_session", resource_id=chat_id) - from core.services.compaction_progress import read_progress + from core.chat.compaction.progress import read_progress return success_response( data={ @@ -84,7 +84,7 @@ def get_tool_call_result( 展开哪张卡，前端才来这里取哪一张。仍然过一遍 ``bound_result_for_display`` 的宽档 上限：单张卡也不该把 5MB 一次性塞进标签页。 """ - chat_service = chat_services.ChatService(db) + chat_service = chat_services_ChatService_module.ChatService(db) if chat_service.get_session_with_access(chat_id, str(user.user_id)) is None: raise ResourceNotFoundError(resource_type="chat_session", resource_id=chat_id) @@ -119,7 +119,7 @@ def get_followups( db: Session = Depends(db_engine.get_db), ): """Return follow-up questions stored in a message's extra_data.""" - chat_service = chat_services.ChatService(db) + chat_service = chat_services_ChatService_module.ChatService(db) session = chat_service.get_session(chat_id, user.user_id) if not session: diff --git a/src/backend/api/routes/v1/chats/pending_access.py b/src/backend/api/routes/v1/chats/pending_access.py index 2eac2f36..24ff246c 100644 --- a/src/backend/api/routes/v1/chats/pending_access.py +++ b/src/backend/api/routes/v1/chats/pending_access.py @@ -2,8 +2,8 @@ from datetime import datetime -import core.services.session_operations.session_context as chat_session_context -import core.services as chat_services +import core.chat.operations.session_context as chat_session_context +import core.chat.service as chat_services_ChatService_module from sqlalchemy.orm import Session @@ -42,6 +42,6 @@ def _detect_chat_run_interrupted(db: Session, chat_id: str) -> bool: def _owns_pending_chat(db, chat_id, user_id): try: - return chat_services.ChatService(db).get_session(chat_id, user_id) is not None + return chat_services_ChatService_module.ChatService(db).get_session(chat_id, user_id) is not None finally: - chat_session_context._release_request_session(db) + chat_session_context.release_request_session(db) diff --git a/src/backend/api/routes/v1/chats/questions.py b/src/backend/api/routes/v1/chats/questions.py index 6d9842f0..2326ca26 100644 --- a/src/backend/api/routes/v1/chats/questions.py +++ b/src/backend/api/routes/v1/chats/questions.py @@ -1,6 +1,6 @@ """Pending user-question recovery, answer and cancellation endpoints.""" -import core.services.session_operations.schemas as chat_models +import core.chat.operations.schemas as chat_models import api.routes.v1.chats.pending_access as chat_pending_access import core.auth.backend as auth_backend import core.db.engine as db_engine diff --git a/src/backend/api/routes/v1/chats/reruns.py b/src/backend/api/routes/v1/chats/reruns.py index 1dd5b1e7..1daa5622 100644 --- a/src/backend/api/routes/v1/chats/reruns.py +++ b/src/backend/api/routes/v1/chats/reruns.py @@ -2,23 +2,26 @@ from typing import Any, Dict -import core.services.session_operations.admission_errors as chat_admission -import core.services.session_operations.agent_targets as chat_agent_targets -import core.services.session_operations.invocation as chat_invocation -import core.services.session_operations.schemas as chat_models -import core.services.session_operations.request_context as chat_request_context import api.routes.v1.chats.rerun_metadata as chat_rerun_metadata -import core.services.session_operations.session_context as chat_session_context +import api.transports.run_sse as run_sse import core.auth.backend as auth_backend import core.chat.context as chat_context +import core.chat.runtime.admission as admission import core.db.engine as db_engine import core.infra.responses as responses -import core.services as chat_services -from core.chat.schemas import ChatRequest +import core.chat.service as chat_services_ChatService_module +import core.auth.users as chat_services_UserService_module +import core.chat.operations.admission_errors as chat_admission +import core.chat.operations.agent_targets as chat_agent_targets +import core.chat.operations.invocation as chat_invocation +import core.chat.operations.request_context as chat_request_context +import core.chat.operations.schemas as chat_models +import core.chat.operations.session_context as chat_session_context from core.auth.backend import UserContext +from core.chat.schemas import ChatRequest from core.db.reserved_chats import reserved_chat_error, reserved_chat_owner from core.infra.exceptions import ResourceNotFoundError -from core.services.chat_reference_service import render_reference_block +from core.chat.references import render_reference_block from fastapi import APIRouter, Depends, HTTPException from sqlalchemy.orm import Session from starlette.concurrency import run_in_threadpool @@ -39,10 +42,10 @@ async def regenerate_message( raise HTTPException(status_code=403, detail=reserved_chat_error(chat_id)) def prepare(): - chat_session_context._ensure_main_model_configured() - chat_service = chat_services.ChatService(db) + chat_session_context.ensure_main_model_configured() + chat_service = chat_services_ChatService_module.ChatService(db) db_user_id = chat_context.resolve_db_user_id( - db, chat_session_context._authenticated_user_id(user) + db, chat_session_context.authenticated_user_id(user) ) # Shared sessions: deleting/rewriting history is only granted to the session owner / project admin (the "admin" level in the shared context). @@ -79,7 +82,7 @@ def prepare(): **chat_rerun_metadata._restore_invocation(user_extra), ) regen_request, _, execution_message, explicit_subagent_command = ( - chat_agent_targets._resolve_rerun_agent_targets( + chat_agent_targets.resolve_rerun_agent_targets( db, regen_request, db_user_id, @@ -88,30 +91,30 @@ def prepare(): assistant_message_id=target_msg.message_id, ) ) - regen_request = chat_invocation._resolve_explicit_capability_invocation( + regen_request = chat_invocation.resolve_explicit_capability_invocation( db, regen_request, db_user_id ) - from core.services.project_init import resolve_project_init + from core.chat.project_init import resolve_project_init init_message = resolve_project_init(db, regen_request, db_user_id) - selected_model_provider_id = chat_session_context._resolve_selected_model_provider_id( + selected_model_provider_id = chat_session_context.resolve_selected_model_provider_id( db, regen_request, db_user_id ) - actual_model_name = chat_session_context._resolve_actual_chat_model_name( + actual_model_name = chat_session_context.resolve_actual_chat_model_name( regen_request, selected_model_provider_id, ) enabled_skills, enabled_agents, enabled_mcps = chat_context.resolve_enabled_capabilities( db, db_user_id ) - _user_settings = chat_services.UserService(db).get_user_settings(db_user_id) + _user_settings = chat_services_UserService_module.UserService(db).get_user_settings(db_user_id) effective_msg = chat_context.build_effective_user_message( init_message or execution_message, regen_request.quoted_follow_up, render_reference_block(user_extra.get("referenced_chats")), ) - context = chat_request_context._build_ctx( + context = chat_request_context.build_ctx( regen_request, db_user_id, enabled_skills, @@ -127,7 +130,7 @@ def prepare(): approval_mode=_user_settings.get("tool_approval_mode"), ) - from core.services.chat_sequencer import ChatBusyError, ChatSequencer + from core.chat.sequencer import ChatBusyError, ChatSequencer request_payload = regen_request.model_dump(exclude_none=True) request_payload["operation"] = "regenerate" @@ -163,7 +166,7 @@ def prepare(): raise chat_admission.chat_busy_http_exception(exc) from exc try: - session_messages = chat_session_context._load_session_messages( + session_messages = chat_session_context.load_session_messages( chat_service, chat_id, db_user_id ) kwargs = dict( @@ -200,10 +203,10 @@ async def edit_and_resend( raise HTTPException(status_code=403, detail=reserved_chat_error(chat_id)) def prepare(): - chat_session_context._ensure_main_model_configured() - chat_service = chat_services.ChatService(db) + chat_session_context.ensure_main_model_configured() + chat_service = chat_services_ChatService_module.ChatService(db) db_user_id = chat_context.resolve_db_user_id( - db, chat_session_context._authenticated_user_id(user) + db, chat_session_context.authenticated_user_id(user) ) # Shared sessions: editing history is an owner-only operation (rewriting others' / one's own earlier messages would break the collaboration context). @@ -239,7 +242,7 @@ def prepare(): **saved_invocation, ) edit_request, _, execution_message, explicit_subagent_command = ( - chat_agent_targets._resolve_rerun_agent_targets( + chat_agent_targets.resolve_rerun_agent_targets( db, edit_request, db_user_id, @@ -247,22 +250,22 @@ def prepare(): target_msg, ) ) - from core.services.project_init import resolve_project_init + from core.chat.project_init import resolve_project_init init_message = resolve_project_init(db, edit_request, db_user_id) - edit_request = chat_invocation._resolve_explicit_capability_invocation( + edit_request = chat_invocation.resolve_explicit_capability_invocation( db, edit_request, db_user_id ) - selected_model_provider_id = chat_session_context._resolve_selected_model_provider_id( + selected_model_provider_id = chat_session_context.resolve_selected_model_provider_id( db, edit_request, db_user_id ) - actual_model_name = chat_session_context._resolve_actual_chat_model_name( + actual_model_name = chat_session_context.resolve_actual_chat_model_name( edit_request, selected_model_provider_id ) enabled_skills, enabled_agents, enabled_mcps = chat_context.resolve_enabled_capabilities( db, db_user_id ) - _user_settings = chat_services.UserService(db).get_user_settings(db_user_id) + _user_settings = chat_services_UserService_module.UserService(db).get_user_settings(db_user_id) # Persist the edited user message _edit_extra: Dict[str, Any] = { @@ -285,7 +288,7 @@ def prepare(): if selected_model_provider_id: _edit_extra["model_provider_id"] = selected_model_provider_id - context = chat_request_context._build_ctx( + context = chat_request_context.build_ctx( edit_request, db_user_id, enabled_skills, @@ -301,7 +304,7 @@ def prepare(): approval_mode=_user_settings.get("tool_approval_mode"), ) - from core.services.chat_sequencer import ChatBusyError, ChatSequencer + from core.chat.sequencer import ChatBusyError, ChatSequencer request_payload = edit_request.model_dump(exclude_none=True) request_payload["operation"] = "edit" @@ -328,7 +331,7 @@ def prepare(): raise chat_admission.chat_busy_http_exception(exc) from exc try: - session_messages = chat_session_context._load_session_messages( + session_messages = chat_session_context.load_session_messages( chat_service, chat_id, db_user_id ) kwargs = dict( @@ -358,18 +361,18 @@ def prepare(): async def _start_prepared_run(prepare, db: Session, chat_id: str): """Release admission locks before streaming; abandon admission if startup fails.""" - from core.services.chat_sequencer import ChatSequencer - from orchestration import chat_run_executor + from core.chat.sequencer import ChatSequencer + pass def prepare_and_release(): try: return prepare() finally: - chat_session_context._release_request_session(db) + chat_session_context.release_request_session(db) kwargs = await run_in_threadpool(prepare_and_release) try: - run = await chat_run_executor.start_run(**kwargs) + run = await admission.start_run(**kwargs) except Exception as exc: reason = str(exc) @@ -377,8 +380,8 @@ def abandon(): try: ChatSequencer(db).abandon_pending_run(kwargs["accepted_run"].run_id, reason=reason) finally: - chat_session_context._release_request_session(db) + chat_session_context.release_request_session(db) await run_in_threadpool(abandon) raise - return responses.sse_response(chat_run_executor.follow_run_as_sse(run.run_id, chat_id=chat_id)) + return responses.sse_response(run_sse.follow_run_as_sse(run.run_id, chat_id=chat_id)) diff --git a/src/backend/api/routes/v1/chats/run_views.py b/src/backend/api/routes/v1/chats/run_views.py index 8c543814..dc79e2c9 100644 --- a/src/backend/api/routes/v1/chats/run_views.py +++ b/src/backend/api/routes/v1/chats/run_views.py @@ -2,11 +2,13 @@ from typing import Annotated -import core.services.session_operations.session_context as chat_session_context +import api.transports.run_sse as run_sse import core.auth.backend as auth_backend import core.chat.context as chat_context import core.db.engine as db_engine import core.infra.responses as responses +import core.runs.runtime as runtime +import core.chat.operations.session_context as chat_session_context from core.auth.backend import UserContext from core.db.reserved_chats import reserved_chat_owner from core.infra.logging import quiet_access_log @@ -32,7 +34,7 @@ def list_active_chat_runs( 侧边栏对没点开过的会话一无所知。这里按用户一次查完：查询只命中 pending / running 这一小撮行，与历史 run 的数量无关。 """ - from core.services.session_operations.runs import list_active_chat_runs as operation + from core.chat.operations.runs import list_active_chat_runs as operation return operation(user=user, db=db, include_status=include_status, recent_seconds=recent_seconds) @@ -48,21 +50,21 @@ def chat_stream_resume( 校验 run 存在且归属当前用户（非属主 403、不存在 404），返回 text/event-stream。 """ - from orchestration import chat_run_executor + pass db_user_id = chat_context.resolve_db_user_id( - db, chat_session_context._authenticated_user_id(user) + db, chat_session_context.authenticated_user_id(user) ) - run = chat_run_executor.get_run(run_id) + run = runtime.get_run(run_id) # A module-owned chat's run is replayed by its module, not as an ordinary chat. if run is None or reserved_chat_owner(run.chat_id): raise HTTPException(status_code=404, detail="run not found") if run.user_id != db_user_id: raise HTTPException(status_code=403, detail="无权访问该 run") resumed_chat_id = run.chat_id - chat_session_context._release_request_session(db) + chat_session_context.release_request_session(db) return responses.sse_response( - chat_run_executor.follow_run_as_sse( + run_sse.follow_run_as_sse( run_id, chat_id=resumed_chat_id, from_offset=from_offset ), ) @@ -80,7 +82,7 @@ def chat_active_run( 有则返回 run_id / message_id / status / 刷新重放 offset / thinking 模式等元信息， 无则返回 data=null。需认证（有效 cookie 或 API-Key）。 """ - from core.services.session_operations.runs import chat_active_run as operation + from core.chat.operations.runs import chat_active_run as operation return operation(chat_id=chat_id, user=user, db=db) @@ -92,13 +94,13 @@ def chat_run_subscription( db: Session = Depends(db_engine.get_db), ): """Native ordinary-chat snapshot + tail; the raw replay API is separate.""" - from orchestration import chat_run_executor - from orchestration.run_subscription import subscribe_run + pass + from core.chat.projection.subscription import subscribe_run db_user_id = chat_context.resolve_db_user_id( - db, chat_session_context._authenticated_user_id(user) + db, chat_session_context.authenticated_user_id(user) ) - run = chat_run_executor.get_run(run_id) + run = runtime.get_run(run_id) if run is None: raise HTTPException(status_code=404, detail="run not found") if run.user_id != db_user_id: @@ -106,5 +108,5 @@ def chat_run_subscription( payload = run.request_payload if isinstance(run.request_payload, dict) else {} if payload.get("kind", "chat") != "chat": raise HTTPException(status_code=400, detail="This run uses a specialized progress protocol") - chat_session_context._release_request_session(db) + chat_session_context.release_request_session(db) return responses.sse_response(subscribe_run(run_id)) diff --git a/src/backend/api/routes/v1/chats/sessions.py b/src/backend/api/routes/v1/chats/sessions.py index c5eb77bd..0ade4531 100644 --- a/src/backend/api/routes/v1/chats/sessions.py +++ b/src/backend/api/routes/v1/chats/sessions.py @@ -2,8 +2,8 @@ from typing import Optional -import core.services.session_operations.schemas as chat_models -import core.services.session_operations.session_context as chat_session_context +import core.chat.operations.schemas as chat_models +import core.chat.operations.session_context as chat_session_context import core.auth.backend as auth_backend import core.chat.context as chat_context import core.db.engine as db_engine @@ -39,7 +39,7 @@ def list_chats( - -created_at - Most recently created first - created_at - Oldest created first """ - from core.services.session_operations.sessions import list_chats as operation + from core.chat.operations.sessions import list_chats as operation return operation( page=page, @@ -63,7 +63,7 @@ def create_chat( The session is automatically associated with the current authenticated user. """ - from core.services.session_operations.sessions import create_chat as operation + from core.chat.operations.sessions import create_chat as operation return operation(request=request, user=user, db=db) @@ -87,7 +87,7 @@ def search_chats( Returns sessions with match_type ("title" or "content") and matched_snippet for content matches. """ - from core.services.session_operations.sessions import search_chats as operation + from core.chat.operations.sessions import search_chats as operation return operation(q=q, scope=scope, page=page, page_size=page_size, user=user, db=db) @@ -106,10 +106,10 @@ def list_referencable_chats( 在项目里只给同项目的会话——项目是用户自己划的话题边界。只返回标题级信息，正文由 智能体在回答时按需调 ``read_chat`` 读取。 """ - from core.services.chat_reference_service import list_referencable_sessions, session_brief + from core.chat.references import list_referencable_sessions, session_brief db_user_id = chat_context.resolve_db_user_id( - db, chat_session_context._authenticated_user_id(user) + db, chat_session_context.authenticated_user_id(user) ) sessions = list_referencable_sessions( db, @@ -132,7 +132,7 @@ def get_chat( db: Session = Depends(db_engine.get_db), ): """获取当前用户有权读取的会话详情。""" - from core.services.session_operations.sessions import get_chat as operation + from core.chat.operations.sessions import get_chat as operation return operation(chat_id=chat_id, user=user, db=db) @@ -145,7 +145,7 @@ def update_chat( db: Session = Depends(db_engine.get_db), ): """更新当前用户有权修改的会话元信息。""" - from core.services.session_operations.sessions import update_chat as operation + from core.chat.operations.sessions import update_chat as operation return operation(chat_id=chat_id, request=request, user=user, db=db) @@ -157,6 +157,6 @@ def delete_chat( db: Session = Depends(db_engine.get_db), ): """软删当前用户有权管理的会话。""" - from core.services.session_operations.sessions import delete_chat as operation + from core.chat.operations.sessions import delete_chat as operation return operation(chat_id=chat_id, user=user, db=db) diff --git a/src/backend/api/routes/v1/chats/sidebar.py b/src/backend/api/routes/v1/chats/sidebar.py index 4b4cac74..af0f29ea 100644 --- a/src/backend/api/routes/v1/chats/sidebar.py +++ b/src/backend/api/routes/v1/chats/sidebar.py @@ -2,11 +2,11 @@ from typing import List, Optional -import core.services.session_operations.schemas as chat_models -import core.services.session_operations.session_context as chat_session_context +import core.chat.operations.schemas as chat_models +import core.chat.operations.session_context as chat_session_context import core.auth.backend as auth_backend import core.db.engine as db_engine -import core.services as chat_services +import core.auth.users as chat_services_UserService_module from core.auth.backend import UserContext from core.infra.responses import success_response from fastapi import APIRouter, Depends @@ -25,7 +25,7 @@ def _dedup_id_list(raw: Optional[list]) -> List[str]: """Clean + de-duplicate an id list, preserving first-seen order.""" seen: set = set() out: List[str] = [] - for cid in chat_session_context._clean_id_list(raw): + for cid in chat_session_context.clean_id_list(raw): if cid in seen: continue seen.add(cid) @@ -45,7 +45,7 @@ def get_sidebar_order( 注意：本路由必须声明在 ``GET /{chat_id}`` **之前**，否则会被 path 参数 路由吞掉（FastAPI 按声明顺序匹配）。 """ - user_settings = chat_services.UserService(db).get_user_settings(str(user.user_id)) + user_settings = chat_services_UserService_module.UserService(db).get_user_settings(str(user.user_id)) order = _dedup_id_list(user_settings.get(SIDEBAR_ORDER_KEY))[:SIDEBAR_ORDER_MAX] return success_response(data={"order": order}) @@ -62,7 +62,7 @@ def update_sidebar_order( 查不到对应项而被忽略，反倒省掉一次全表校验。超出上限的尾部直接截断。 """ order = _dedup_id_list(request.order)[:SIDEBAR_ORDER_MAX] - chat_services.UserService(db).update_user_metadata( + chat_services_UserService_module.UserService(db).update_user_metadata( user_id=str(user.user_id), patch={SIDEBAR_ORDER_KEY: order} ) return success_response(data={"order": order}) diff --git a/src/backend/api/routes/v1/classify.py b/src/backend/api/routes/v1/classify.py index cfccfa99..fa3c0a61 100644 --- a/src/backend/api/routes/v1/classify.py +++ b/src/backend/api/routes/v1/classify.py @@ -6,7 +6,7 @@ from core.auth.backend import get_current_user, UserContext from core.infra.responses import success_response -from core.llm.classifier import get_classifier, BUSINESS_TOPICS +from core.models.completions.classifier import get_classifier, BUSINESS_TOPICS from core.infra.exceptions import BadRequestError router = APIRouter(prefix="/v1/classify", tags=["Classify"]) diff --git a/src/backend/api/routes/v1/commands.py b/src/backend/api/routes/v1/commands.py index 1e856eeb..02aa7bea 100644 --- a/src/backend/api/routes/v1/commands.py +++ b/src/backend/api/routes/v1/commands.py @@ -1,17 +1,18 @@ """Authenticated background command status, output and cancellation.""" import asyncio -from fastapi import APIRouter, Depends, HTTPException, Request -from starlette.responses import StreamingResponse + from core.auth.backend import UserContext, get_current_user from core.infra.responses import success_response -from core.services.commands import store -from orchestration.commands.runtime import runtime +from core.sandbox.commands.runtime import runtime +from core.sandbox.commands import store +from fastapi import APIRouter, Depends, HTTPException, Request +from starlette.responses import StreamingResponse router = APIRouter(prefix="/v1/commands", tags=["Commands"]) @router.get("/events") async def events(request: Request, chat_id: str): - from orchestration.commands.subscription import authorize, subscribe + from core.sandbox.commands.subscription import authorize, subscribe user_id = await authorize(request, chat_id) return StreamingResponse(subscribe(request, chat_id, user_id), media_type="text/event-stream", diff --git a/src/backend/api/routes/v1/content.py b/src/backend/api/routes/v1/content.py index 25931239..bc587ff3 100644 --- a/src/backend/api/routes/v1/content.py +++ b/src/backend/api/routes/v1/content.py @@ -12,11 +12,8 @@ from fastapi import APIRouter, Depends from sqlalchemy.orm import Session -from core.content.content_blocks import ( - DEFAULT_APP_CONFIG, - DOCS_BLOCK_MAP, - normalize_app_config, -) +from core.files.content.content_blocks_constants import DEFAULT_APP_CONFIG, DOCS_BLOCK_MAP +from core.files.content.branding import normalize_app_config from core.db.engine import get_db from core.db.models import ContentBlock from core.infra.responses import success_response diff --git a/src/backend/api/routes/v1/desktop_capability.py b/src/backend/api/routes/v1/desktop_capability.py deleted file mode 100644 index 63ad5e07..00000000 --- a/src/backend/api/routes/v1/desktop_capability.py +++ /dev/null @@ -1,824 +0,0 @@ -"""桌面双端能力桥 API（云端侧 + 本机侧共用一个路由文件，各端点自行守门）。 - -云端侧（部署在云端 / 预发 / 生产后端）： - POST /v1/desktop/capability/token 会话换取短时 capability token - GET /v1/desktop/capability/manifest 当前用户最终可用 MCP 清单 - POST /v1/desktop/capability/gateway/{sid}/call 动态 manifest 的 JSON 工具调用网关 - ANY /v1/desktop/capability/gateway/{sid}/mcp 已安装旧客户端使用的透明反代网关 - GET /v1/desktop/capability/models 无密钥模型拓扑 - POST /v1/desktop/capability/gateway/models/... 模型流式反代网关 - GET /v1/desktop/capability/skills/manifest 当前用户最终可用技能清单（含内容哈希） - GET /v1/desktop/capability/skills/{id}/bundle 单个授权技能的完整 zip 包 - -本机侧（桌面壳孵化的本机后端）： - POST /v1/desktop/capability/cloud-bridge 壳推送 {cloud_base, token} - GET /v1/desktop/capability/cloud-bridge/status 桥诊断视图 - -安全模型： -- token 端点要求云端登录会话；网关/manifest 只认 capability token（HMAC 短时 - 令牌，见 core/services/desktop_capability.py），会话 cookie / 内部 token / - 第三方密钥不出云端； -- 网关按「该用户当前有效能力集」授权 server_id，未命中一律 404（不区分 - 不存在/无权）；身份头由网关覆写，客户端伪造的 X-Current-User-Id 不生效； -- cloud-bridge 接收端只认桌面壳 Bearer 进程秘密，不以用户管理权限替代； - 仅在桌面桥进程（HUGAGENT_DESKTOP_BRIDGE_SECRET 已注入）下开放， - 浏览器 Origin 请求拒绝，云端部署恒 403。 -""" - -from __future__ import annotations - -import asyncio -import json -import logging -import time -from typing import Optional - -import httpx -from core.auth.backend import UserContext, get_current_user -from core.infra.logging import quiet_access_log -from core.infra.responses import success_response -from fastapi import APIRouter, Depends, HTTPException, Request, Response -from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer -from pydantic import BaseModel, Field -from starlette.background import BackgroundTask -from starlette.concurrency import run_in_threadpool -from starlette.responses import StreamingResponse - -logger = logging.getLogger(__name__) - -from core.services.desktop_gateway_observability import ObservedGatewayRoute - -router = APIRouter(prefix="/v1/desktop/capability", tags=["Desktop Capability"], route_class=ObservedGatewayRoute) - -# 网关上行连接：连接短超时快速失败；读不设限（SSE 长流 / 长工具调用）， -# 上游 MCP 自身带 execution_timeout 兜底。 -_gateway_client: Optional[httpx.AsyncClient] = None - - -def _client() -> httpx.AsyncClient: - global _gateway_client - if _gateway_client is None: - _gateway_client = httpx.AsyncClient( - timeout=httpx.Timeout(600.0), - # 桌面端两条消息之间常常隔着几十秒；默认 5s 的 keepalive 意味着几乎 - # 每次模型调用都要重新 TCP + TLS 握手。 - limits=httpx.Limits(max_keepalive_connections=64, keepalive_expiry=300.0), - ) - return _gateway_client - - -# ── token 签发（云端，会话鉴权） ──────────────────────────────────────── - - -class CapabilityTokenBody(BaseModel): - device_id: str = Field( - ..., min_length=1, max_length=128, pattern=r"^[A-Za-z0-9][A-Za-z0-9._-]*$" - ) - - -@router.post("/token", summary="签发桌面能力令牌") -async def issue_token(body: CapabilityTokenBody, request: Request, response: Response): - """Only an actual live cloud cookie session can mint a desktop credential.""" - import hashlib - - from core.auth.session import validate_session - from core.config.settings import settings - from core.services.desktop_capability import issue_capability_token, session_authorization_epoch - - cookie = request.cookies.get(settings.session.cookie_name, "") - try: - current = await validate_session(cookie) if cookie else None - except Exception: - current = None - if ( - not current - or not current.get("user_id") - or not current.get("user_center_id") - or not session_authorization_epoch(current) - ): - raise HTTPException(status_code=401, detail="valid cloud session required") - data = issue_capability_token( - str(current["user_id"]), - device_id=body.device_id, - issuer=str(request.base_url), - session_hash=hashlib.sha256(cookie.encode()).hexdigest(), - authorization_epoch=session_authorization_epoch(current), - user_center_id=str(current["user_center_id"]), - ) - response.headers["Cache-Control"] = "no-store" - response.headers["Pragma"] = "no-cache" - return success_response(data=data) - - -# ── capability token 鉴权依赖 ────────────────────────────────────────── - -_bearer = HTTPBearer(auto_error=False) - - -async def _require_capability_user( - request: Request, - credentials: Optional[HTTPAuthorizationCredentials] = Depends(_bearer), -) -> str: - from core.services.desktop_capability import CAPABILITY_DEVICE_HEADER, verify_capability_token - - token = credentials.credentials if credentials else "" - user_id = await verify_capability_token( - token, - device_id=request.headers.get(CAPABILITY_DEVICE_HEADER, ""), - issuer=str(request.base_url), - ) - if not user_id: - raise HTTPException(status_code=401, detail="invalid capability token") - request.state.desktop_observation_user = user_id - return user_id - - -class WorkcopyCommitBody(BaseModel): - files: dict[str, str] - expected_revision: str | None = None - expected_model_version: str | None = None - request_id: str = Field(pattern=r"^[a-f0-9]{32}$") - acknowledge_sensitive: bool = False - create_only: bool = False - - -@router.get("/workcopies/{kind}/{key}", summary="比较当前账号的能力文件版本") -async def workcopy_snapshot(kind: str, key: str, response: Response, - user_id: str = Depends(_require_capability_user)): - from core.services.capability_workcopies import snapshot - response.headers["Cache-Control"] = "no-store" - result = await run_in_threadpool(snapshot, user_id, kind, key) - return success_response(data=await run_in_threadpool(_public_content, user_id, result)) - - -@router.post("/workcopies/{kind}/{key}", summary="显式提交能力文件版本") -async def workcopy_commit(kind: str, key: str, body: WorkcopyCommitBody, response: Response, - user_id: str = Depends(_require_capability_user)): - from core.services.capability_workcopies import commit - response.headers["Cache-Control"] = "no-store" - await run_in_threadpool(_public_content, user_id, body.files) - result = await run_in_threadpool(commit, user_id, kind, key, body.model_dump()) - return success_response(data=await run_in_threadpool(_public_content, user_id, result)) - - -def _public_content(user_id: str, value, *, bundle: bool = False): - from core.services.desktop_capability import ( - CapabilityContentRejected, - guard_capability_bundle, - guard_capability_content, - ) - - try: - if callable(value): - value = value() - return ( - guard_capability_bundle(user_id, value) - if bundle - else guard_capability_content(user_id, value) - ) - except CapabilityContentRejected: - raise HTTPException( - status_code=422, - detail={ - "code": "integrity_failed", - "message": "capability content blocked by credential policy", - }, - ) from None - - -def _stream_secrets(user_id: str, target: dict) -> set[str]: - from core.services.desktop_capability import CapabilityContentRejected, gateway_stream_secrets - - try: - return gateway_stream_secrets(user_id, target) - except CapabilityContentRejected: - raise HTTPException( - status_code=422, - detail={"code": "integrity_failed", "message": "credential policy unavailable"}, - ) from None - - -async def _checked_upstream_bytes(upstream: httpx.Response, secrets: set[str]): - from core.services.desktop_capability import CapabilityContentRejected, guard_capability_stream - - try: - async for chunk in guard_capability_stream(upstream.aiter_raw(), secrets): - yield chunk - except CapabilityContentRejected: - logger.warning("[desktop-capability] upstream content blocked code=integrity_failed") - # Streaming headers may already be sent. Return only a fixed diagnostic; - # the buffered bytes containing the credential are never released. - yield b'data: {"error":{"code":"integrity_failed","message":"upstream content blocked"}}\n\n' - finally: - await upstream.aclose() - - -# ── manifest（云端，capability token 鉴权） ───────────────────────────── - - -@router.get("/manifest", summary="当前用户的云端能力清单") -async def get_manifest( - request: Request, - response: Response, - user_id: str = Depends(_require_capability_user), -): - from core.services.desktop_capability import build_user_capability_manifest - - use_cache = request.headers.get("cache-control") != "no-cache" - manifest = await run_in_threadpool( - _public_content, user_id, - lambda: build_user_capability_manifest(user_id, use_cache=use_cache), - ) - etag = f'"{manifest["revision"]}"' - headers = {"ETag": etag, "Cache-Control": "private, max-age=0, must-revalidate"} - if request.headers.get("if-none-match") == etag: - return Response(status_code=304, headers=headers) - response.headers.update(headers) - return success_response(data=manifest) - - -@router.get("/models", summary="桌面本机执行面的无密钥模型清单") -async def get_model_manifest( - request: Request, - response: Response, - user_id: str = Depends(_require_capability_user), -): - from core.services.desktop_capability import build_user_model_manifest - - manifest = await run_in_threadpool( - _public_content, user_id, lambda: build_user_model_manifest(user_id) - ) - etag = f'"{manifest["revision"]}"' - headers = {"ETag": etag, "Cache-Control": "private, max-age=0, must-revalidate"} - if request.headers.get("if-none-match") == etag: - return Response(status_code=304, headers=headers) - response.headers.update(headers) - return success_response(data=manifest) - - -@router.get("/skills/manifest", summary="当前用户的云端技能清单") -async def get_skill_manifest( - request: Request, - response: Response, - user_id: str = Depends(_require_capability_user), -): - from core.services.desktop_capability import build_user_skill_manifest - - use_cache = request.headers.get("cache-control") != "no-cache" - manifest = await run_in_threadpool( - _public_content, user_id, - lambda: build_user_skill_manifest(user_id, use_cache=use_cache), - ) - etag = f'"{manifest["revision"]}"' - headers = {"ETag": etag, "Cache-Control": "private, max-age=0, must-revalidate"} - if request.headers.get("if-none-match") == etag: - return Response(status_code=304, headers=headers) - response.headers.update(headers) - return success_response(data=manifest) - - -@router.get("/skills/{skill_id}/bundle", summary="下载一个当前授权技能的完整 zip 包") -async def get_skill_bundle( - skill_id: str, - request: Request, - user_id: str = Depends(_require_capability_user), -): - from core.services.desktop_capability import resolve_skill_bundle - - resolved = _public_content( - user_id, lambda: resolve_skill_bundle(user_id, skill_id), bundle=True - ) - if resolved is None: - raise HTTPException(status_code=404, detail="skill not available") - data, content_hash = resolved - etag = f'"{content_hash}"' - headers = {"ETag": etag, "Cache-Control": "private, max-age=0, must-revalidate"} - if request.headers.get("if-none-match") == etag: - return Response(status_code=304, headers=headers) - return Response(content=data, media_type="application/zip", headers=headers) - - -def _entity_manifest_response(request: Request, response: Response, manifest: dict): - etag = f'"{manifest["revision"]}"' - headers = {"ETag": etag, "Cache-Control": "private, max-age=0, must-revalidate"} - if request.headers.get("if-none-match") == etag: - return Response(status_code=304, headers=headers) - response.headers.update(headers) - return success_response(data=manifest) - - -def _bundle_response(request: Request, resolved): - if resolved is None: - raise HTTPException(status_code=404, detail="not available") - data, content_hash = resolved - etag = f'"{content_hash}"' - headers = {"ETag": etag, "Cache-Control": "private, max-age=0, must-revalidate"} - if request.headers.get("if-none-match") == etag: - return Response(status_code=304, headers=headers) - return Response(content=data, media_type="application/zip", headers=headers) - - -@router.get("/agents/manifest", summary="当前用户可见的智能体定义清单") -async def get_agent_manifest( - request: Request, response: Response, user_id: str = Depends(_require_capability_user) -): - from core.services.desktop_capability import build_user_agent_manifest - - use_cache = request.headers.get("cache-control") != "no-cache" - manifest = await run_in_threadpool( - _public_content, user_id, - lambda: build_user_agent_manifest(user_id, use_cache=use_cache), - ) - return _entity_manifest_response(request, response, manifest) - - -@router.get( - "/agents/{agent_id}/bundle", summary="下载一个智能体定义包（agent.json + instructions.md）" -) -async def get_agent_bundle( - agent_id: str, request: Request, user_id: str = Depends(_require_capability_user) -): - from core.services.desktop_capability import resolve_agent_bundle - - return _bundle_response( - request, - _public_content(user_id, lambda: resolve_agent_bundle(user_id, agent_id), bundle=True), - ) - - -@router.get("/plugins/manifest", summary="当前用户已安装插件清单") -async def get_plugin_manifest( - request: Request, response: Response, user_id: str = Depends(_require_capability_user) -): - from core.services.desktop_capability import build_user_plugin_manifest - - use_cache = request.headers.get("cache-control") != "no-cache" - manifest = await run_in_threadpool( - _public_content, user_id, - lambda: build_user_plugin_manifest(user_id, use_cache=use_cache), - ) - return _entity_manifest_response(request, response, manifest) - - -@router.get("/plugins/{install_id}/bundle", summary="下载一个插件定义包（plugin.json）") -async def get_plugin_bundle( - install_id: str, request: Request, user_id: str = Depends(_require_capability_user) -): - from core.services.desktop_capability import resolve_plugin_bundle - - return _bundle_response( - request, - _public_content(user_id, lambda: resolve_plugin_bundle(user_id, install_id), bundle=True), - ) - - -# ── MCP 网关（云端，capability token 鉴权，透明反代） ────────────────── - -# 请求侧按 deny-list 透传：运行时上下文头（X-Chat-Id、X-Reranker-Enabled、 -# 外接 KB 头等，见 agent_factory._inject_runtime_headers）随本机侧注入原样 -# 过桥，新增上下文头无需改网关；只拦截身份/凭据/逐跳头，身份由网关按 -# capability token 覆写。 -_DROP_REQUEST_HEADERS = { - # 身份与凭据：capability token 不上传上游；身份头由网关重写 - "authorization", - "cookie", - "x-current-user-id", - # 知识库授权按云端用户重算（头缺失时 KB MCP 自动解析可用知识库） - "x-allowed-kb-ids", - # 桌面桥内部头 - "x-desktop-bridge", - "x-desktop-bridge-user", - "x-desktop-device-id", - "x-hugagent-target", - # 逐跳 / 传输层头，由 httpx 按新连接重建 - "host", - "content-length", - "connection", - "accept-encoding", - "transfer-encoding", - "keep-alive", - "proxy-authorization", - "te", - "upgrade", -} -# 响应侧透传的头。 -_FWD_RESPONSE_HEADERS = ("content-type", "mcp-session-id", "mcp-protocol-version") -_RUNTIME_CONTEXT_HEADERS = { - "x-chat-id", - "x-channel-id", - "x-conversation-id", - "x-allowed-dataset-ids", - "x-reranker-enabled", -} - - -class GatewayToolCallBody(BaseModel): - tool_name: str = Field(..., min_length=1, max_length=200) - arguments: dict = Field(default_factory=dict) - schema_hash: str = Field(..., min_length=64, max_length=64) - - -@router.post("/gateway/{server_id}/site-publish", summary="上传桌面构建产物并云端托管") -async def gateway_site_publish( - server_id: str, - request: Request, - user_id: str = Depends(_require_capability_user), -): - import tarfile - - from core.infra.exceptions import AppException - from core.services.desktop_capability import resolve_gateway_tool - from core.services.desktop_capability_protocol import CapabilityManifestStaleError - from core.services.desktop_gateway_uploads import ( - MAX_UPLOAD_OPTIONS_CHARS, - UPLOAD_OPTIONS_HEADER, - UPLOAD_SCHEMA_HEADER, - endpoint_plugin, - ) - from core.services.desktop_site_publish import ( - MAX_ARCHIVE_BYTES, - SiteUploadOptions, - publish_uploaded_site, - ) - from pydantic import ValidationError - - try: - resolved = await run_in_threadpool( - resolve_gateway_tool, - user_id, - server_id, - "publish_site", - schema_hash=request.headers.get(UPLOAD_SCHEMA_HEADER, ""), - ) - except CapabilityManifestStaleError: - raise HTTPException(status_code=409, detail="capability manifest changed") from None - if resolved is None or str( - resolved["target"].get("source_plugin") or "" - ) != endpoint_plugin("site-publish"): - raise HTTPException(status_code=403, detail="site publishing is not authorized") - raw_options = request.headers.get(UPLOAD_OPTIONS_HEADER, "{}") - if len(raw_options) > MAX_UPLOAD_OPTIONS_CHARS: - raise HTTPException(status_code=400, detail="site options too large") - try: - options = SiteUploadOptions.model_validate_json(raw_options) - except ValidationError: - raise HTTPException(status_code=400, detail="invalid site options") from None - archive = bytearray() - async for chunk in request.stream(): - if len(archive) + len(chunk) > MAX_ARCHIVE_BYTES: - raise HTTPException(status_code=413, detail="site archive exceeds 40 MB") - archive.extend(chunk) - try: - result = await asyncio.to_thread(publish_uploaded_site, user_id, bytes(archive), options) - except (ValueError, tarfile.TarError): - raise HTTPException(status_code=400, detail="invalid site build archive") from None - except AppException as exc: - raise HTTPException(status_code=400, detail=exc.message) from None - return success_response(data=_public_content(user_id, result)) - - -@router.post( - "/gateway/{server_id}/call", - summary="桌面云端工具调用网关（动态 manifest schema + JSON invocation）", -) -async def gateway_mcp_call( - server_id: str, - body: GatewayToolCallBody, - request: Request, - user_id: str = Depends(_require_capability_user), -): - """Execute a currently-authorized MCP tool from the cloud network.""" - from core.services.desktop_capability import ( - CapabilityContentRejected, - invoke_gateway_tool, - resolve_gateway_tool, - ) - from core.services.desktop_capability_protocol import CapabilityManifestStaleError - - try: - resolved = await run_in_threadpool( - resolve_gateway_tool, - user_id, - server_id, - body.tool_name, - schema_hash=body.schema_hash, - ) - except CapabilityManifestStaleError as exc: - raise HTTPException(status_code=409, detail="capability manifest changed") from exc - if resolved is None: - raise HTTPException(status_code=404, detail="tool not available") - - runtime_headers = { - k: v for k, v in request.headers.items() if k.lower() in _RUNTIME_CONTEXT_HEADERS - } - from core.services.automation_remote_effect import RemoteOperationConflict - started = time.monotonic() - try: - result = await invoke_gateway_tool(resolved, body.arguments, runtime_headers) - except HTTPException: - from extensions.mcp_servers.registry import gateway_server - if gateway_server(resolved["server_id"]) is not None: - raise - raise HTTPException(502, "Cloud tool call failed") from None - except RemoteOperationConflict: - raise HTTPException(status_code=400, detail="operation key already bound to different arguments") from None - except CapabilityContentRejected: - raise HTTPException( - status_code=422, - detail={"code": "integrity_failed", "message": "upstream content blocked"}, - ) from None - except asyncio.TimeoutError as exc: - logger.warning( - "[desktop-capability] tool call timeout user=%s server=%s tool=%s", - user_id, - server_id, - body.tool_name, - ) - raise HTTPException(status_code=504, detail="upstream mcp tool timed out") from exc - except Exception as exc: # noqa: BLE001 - never expose cloud credentials/errors - logger.warning( - "[desktop-capability] tool call failed user=%s server=%s tool=%s error_type=%s", - user_id, - server_id, - body.tool_name, - type(exc).__name__, - ) - raise HTTPException(status_code=502, detail="upstream mcp tool failed") from exc - - logger.info( - "[desktop-capability] tool call user=%s server=%s tool=%s duration_ms=%.0f", - user_id, - server_id, - body.tool_name, - (time.monotonic() - started) * 1000, - ) - return success_response(data=_public_content(user_id, result)) - - -class GatewayReceiptBody(BaseModel): - tool_name: str = Field(..., min_length=1, max_length=128) - schema_hash: str = Field(..., min_length=64, max_length=64) - operation_id: str = Field(..., min_length=1, max_length=128) - - -@router.post("/gateway/{server_id}/authorize", summary="校验本机任务工具的当前授权") -async def gateway_authorize_local( - server_id: str, body: GatewayReceiptBody, - user_id: str = Depends(_require_capability_user), -): - from core.services.desktop_capability import resolve_gateway_tool - from core.services.desktop_capability_protocol import CapabilityManifestStaleError - from core.services.automation_tool_routing import TOOLS - try: - resolved = await run_in_threadpool(resolve_gateway_tool, user_id, server_id, - body.tool_name, schema_hash=body.schema_hash) - except CapabilityManifestStaleError: - raise HTTPException(status_code=409, detail="capability manifest changed") from None - if (resolved is None or body.tool_name not in TOOLS - or resolved["target"].get("source_plugin") != "automation"): - raise HTTPException(status_code=403, detail="local automation is not authorized") - return success_response(data={"authorized": True}) - - -@router.post("/gateway/{server_id}/receipt", summary="查询云端操作回执") -async def gateway_effect_receipt( - server_id: str, body: GatewayReceiptBody, - user_id: str = Depends(_require_capability_user), -): - from core.services.desktop_capability import resolve_gateway_tool - from core.services.desktop_capability_protocol import CapabilityManifestStaleError - from core.services.automation_remote_effect import RECEIPT_TOOLS, lookup_remote_receipt - - try: - resolved = await run_in_threadpool(resolve_gateway_tool, user_id, server_id, - body.tool_name, schema_hash=body.schema_hash) - except CapabilityManifestStaleError: - raise HTTPException(status_code=409, detail="capability manifest changed") from None - if resolved is None or body.tool_name not in RECEIPT_TOOLS: - raise HTTPException(status_code=404, detail="tool not available") - result = await run_in_threadpool(lookup_remote_receipt, user_id, server_id, - body.tool_name, body.operation_id) - return success_response(data=_public_content(user_id, result)) - - -@router.post( - "/gateway/models/{provider_id}/{model_path:path}", - summary="桌面模型网关（OpenAI-compatible 流式反代）", -) -async def gateway_model( - provider_id: str, - model_path: str, - request: Request, - user_id: str = Depends(_require_capability_user), -): - """把本机 Agent 的对话/向量/重排请求转发到云端内网模型。 - - 路径、模型名和上游凭据全由云端 DB 重算；客户端提供的 - Authorization / x-api-key / model 字段都不可信。 - """ - from core.services.desktop_capability import resolve_model_gateway_target - - target = await run_in_threadpool(resolve_model_gateway_target, user_id, provider_id) - if target is None or model_path.strip("/") != target["path"]: - raise HTTPException(status_code=404, detail="model not available") - - try: - payload = await request.json() - except (json.JSONDecodeError, UnicodeDecodeError, ValueError): - raise HTTPException(status_code=400, detail="model request must be JSON") - if not isinstance(payload, dict): - raise HTTPException(status_code=400, detail="model request must be a JSON object") - payload["model"] = target["model_name"] - request.state.desktop_observation_model = target["model_name"] - - headers = { - "authorization": f"Bearer {target['api_key']}", - "content-type": "application/json", - "accept": request.headers.get("accept", "application/json"), - # aiter_raw 保留原始字节，因此明确要求上游不压缩，避免在没有 - # Content-Encoding 响应头的情况下把 gzip 字节直接送给 OpenAI SDK。 - "accept-encoding": "identity", - } - response_secrets = await run_in_threadpool(_stream_secrets, user_id, target) - client = _client() - upstream_req = client.build_request( - "POST", - target["url"], - headers=headers, - content=json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode("utf-8"), - ) - try: - upstream = await client.send(upstream_req, stream=True) - except httpx.HTTPError as exc: - logger.warning( - "[desktop-capability] model gateway upstream error user=%s provider=%s error_type=%s", - user_id, - provider_id, - type(exc).__name__, - ) - raise HTTPException(status_code=502, detail="upstream model unreachable") - - logger.info( - "[desktop-capability] model gateway user=%s provider=%s type=%s status=%s", - user_id, - provider_id, - target["provider_type"], - upstream.status_code, - ) - response_headers = {} - if "content-type" in upstream.headers: - response_headers["content-type"] = upstream.headers["content-type"] - return StreamingResponse( - _checked_upstream_bytes(upstream, response_secrets), - status_code=upstream.status_code, - headers=response_headers, - background=BackgroundTask(upstream.aclose), - ) - - -@router.api_route( - "/gateway/{server_id}/mcp", - methods=["POST", "GET", "DELETE"], - summary="桌面能力网关（MCP streamable-http 反代）", -) -async def gateway_mcp( - server_id: str, - request: Request, - user_id: str = Depends(_require_capability_user), -): - # 授权解析走 30s per-user 缓存、内部短会话——不经 Depends(get_db)， - # 避免流式转发全程占住连接池里的一条 DB 连接。 - from core.services.desktop_capability import resolve_gateway_target - - target = await run_in_threadpool(resolve_gateway_target, user_id, server_id) - if target is None: - raise HTTPException(status_code=404, detail="server not available") - - headers: dict[str, str] = { - k: v for k, v in request.headers.items() if k.lower() not in _DROP_REQUEST_HEADERS - } - # 云端侧连接凭据（远程第三方 MCP 的 headers 等）只在此处物化，不出云端。 - for k, v in (target.get("headers") or {}).items(): - if isinstance(k, str) and isinstance(v, str): - headers[k] = v - # 身份以 capability token 为准，客户端注入的同名头已在 deny-list 拦下。 - headers["X-Current-User-Id"] = user_id - # StreamingResponse 使用 aiter_raw 原样转发，因此不允许上游压缩后再丢失 - # Content-Encoding；这是旧客户端透明 MCP 端点的传输完整性要求。 - headers["accept-encoding"] = "identity" - - body = await request.body() - response_secrets = await run_in_threadpool(_stream_secrets, user_id, target) - client = _client() - upstream_req = client.build_request( - request.method, target["url"], headers=headers, content=body - ) - try: - upstream = await client.send(upstream_req, stream=True) - except httpx.HTTPError as exc: - logger.warning( - "[desktop-capability] gateway upstream error user=%s server=%s error_type=%s", - user_id, - server_id, - type(exc).__name__, - ) - raise HTTPException(status_code=502, detail="upstream mcp unreachable") - - logger.info( - "[desktop-capability] gateway %s user=%s server=%s status=%s", - request.method, - user_id, - server_id, - upstream.status_code, - ) - resp_headers = { - name: upstream.headers[name] for name in _FWD_RESPONSE_HEADERS if name in upstream.headers - } - return StreamingResponse( - _checked_upstream_bytes(upstream, response_secrets), - status_code=upstream.status_code, - headers=resp_headers, - background=BackgroundTask(upstream.aclose), - ) - - -# ── 本机侧：壳推送桥配置 ──────────────────────────────────────────────── - - -class CloudBridgeBody(BaseModel): - cloud_base: str = Field(..., min_length=1, description="云端后端根地址（含协议）") - token: str = Field(..., min_length=8, description="capability token") - expires_in: int = Field(default=600, ge=60, le=600) - device_id: str = Field( - ..., min_length=1, max_length=128, pattern=r"^[A-Za-z0-9][A-Za-z0-9._-]*$" - ) - - -def _require_desktop_bridge_process() -> None: - from core.auth.desktop_bridge import bridge_enabled - - if not bridge_enabled(): - raise HTTPException(status_code=403, detail="仅桌面双端本机后端可用") - - -def _require_desktop_shell_control(request: Request) -> None: - from core.services.desktop_capability import is_desktop_shell_control - - _require_desktop_bridge_process() - if not is_desktop_shell_control( - request.headers.get("authorization", ""), request.headers.get("origin") - ): - raise HTTPException(status_code=401, detail="desktop shell authorization required") - - -@router.post("/cloud-bridge", summary="推送云端能力桥配置（桌面壳 → 本机后端）") -async def set_cloud_bridge( - body: CloudBridgeBody, - _: None = Depends(_require_desktop_shell_control), -): - _require_desktop_bridge_process() - from core.services.desktop_cloud_bridge import set_state - - base = body.cloud_base.strip().rstrip("/") - from urllib.parse import urlsplit - - try: - parsed = urlsplit(base) - except ValueError: - raise HTTPException(status_code=400, detail="cloud_base 地址无效") - if ( - parsed.scheme not in ("http", "https") - or not parsed.hostname - or parsed.username - or parsed.password - or parsed.query - or parsed.fragment - ): - raise HTTPException(status_code=400, detail="cloud_base 必须是不含凭据的 http(s) 地址") - set_state(base, body.token, body.expires_in, device_id=body.device_id) - logger.info("[cloud-bridge] 桥配置已更新（cloud_base=%s）", base) - return success_response(data={"ok": True}) - - -@router.delete("/cloud-bridge", summary="清除云端能力桥（桌面退出登录）") -async def clear_cloud_bridge(_: None = Depends(_require_desktop_shell_control)): - _require_desktop_bridge_process() - from core.services.desktop_cloud_bridge import clear_state - - clear_state() - return success_response(data={"ok": True}) - - -@router.get("/cloud-bridge/status", summary="云端能力桥状态") -async def cloud_bridge_status(_user: UserContext = Depends(get_current_user)): - from core.services.desktop_cloud_bridge import bridge_status - - return success_response(data=bridge_status()) - - -@router.get("/cloud-bridge/readiness", summary="桌面壳首次能力同步进度") -@quiet_access_log -async def cloud_bridge_readiness(_: None = Depends(_require_desktop_shell_control)): - from core.services.desktop_cloud_bridge import initial_sync_status - - return success_response(data=await asyncio.to_thread(initial_sync_status)) diff --git a/src/backend/api/routes/v1/evolution.py b/src/backend/api/routes/v1/evolution.py index 9f56c954..d9bd40c6 100644 --- a/src/backend/api/routes/v1/evolution.py +++ b/src/backend/api/routes/v1/evolution.py @@ -12,9 +12,9 @@ from typing import Optional from core.auth.backend import UserContext, get_current_user -from core.evolution.settlement import SETTLE_PENDING +from core.evolution.settlement.settlement import SETTLE_PENDING from core.infra.exceptions import BadRequestError -from core.evolution.settlement_store import load_summary +from core.evolution.settlement.settlement_store import load_summary from core.infra.responses import success_response from fastapi import APIRouter, Depends from pydantic import BaseModel, Field @@ -53,7 +53,7 @@ async def get_evolution_settings(user: UserContext = Depends(get_current_user)): simply follows ``evolution_prefs.enabled``. This endpoint stays because already-shipped desktop bundles still read it. """ - from core.evolution.user_settings import resolve_for_user + from core.evolution.profiles.user_settings import resolve_for_user return success_response(data=resolve_for_user(user.user_id)) @@ -64,7 +64,7 @@ async def update_evolution_settings( user: UserContext = Depends(get_current_user), ): """Legacy alias: writes the same switch as PATCH /evolution/prefs.""" - from core.evolution.user_settings import update_for_user + from core.evolution.profiles.user_settings import update_for_user try: return success_response(data=update_for_user(user.user_id, body.enabled)) @@ -80,7 +80,7 @@ async def get_contributions(user: UserContext = Depends(get_current_user)): it actually do?" — and reports each candidate's true governance state rather than an optimistic one. """ - from core.evolution.user_settings import summarise_contributions + from core.evolution.profiles.personal_review import summarise_contributions return success_response(data=summarise_contributions(user.user_id)) @@ -93,7 +93,8 @@ async def list_my_candidates(user: UserContext = Depends(get_current_user)): from other people's conversations activates for the whole fleet, and that is not one person's call to make. """ - from core.evolution.user_settings import OWN_EVIDENCE_THRESHOLD, pending_for_user + from core.evolution.profiles.user_settings_constants import OWN_EVIDENCE_THRESHOLD + from core.evolution.profiles.personal_review import pending_for_user return success_response( data={ @@ -109,8 +110,8 @@ async def approve_my_candidate( user: UserContext = Depends(get_current_user), ): """Accept a candidate as a private skill for this user.""" - from core.evolution.activation import ActivationError - from core.evolution.user_settings import approve_for_user + from core.evolution.cycle.activation_types import ActivationError + from core.evolution.profiles.personal_review import approve_for_user try: return success_response(data=approve_for_user(candidate_id, user.user_id)) @@ -137,11 +138,7 @@ async def get_prefs(user: UserContext = Depends(get_current_user)): configured" as a visible fact rather than hiding the setting — an absent control looks like a missing feature, which is a different message. """ - from core.evolution.prefs import ( - ONTOLOGY_MODES, - available_ontology_packs, - resolve_prefs, - ) + from core.evolution.profiles.prefs import ONTOLOGY_MODES, available_ontology_packs, resolve_prefs return success_response( data={ @@ -159,7 +156,7 @@ async def update_prefs_api( ): import asyncio - from core.evolution.prefs import update_prefs + from core.evolution.profiles.prefs import update_prefs patch = {k: v for k, v in body.model_dump().items() if v is not None} @@ -169,7 +166,7 @@ async def update_prefs_api( # to tell that apart from "my conversations just aren't producing patterns". # Turning it off is never blocked. if patch.get("enabled") is True: - from core.evolution.readiness import check_readiness + from core.evolution.cycle.readiness import check_readiness readiness = await asyncio.to_thread(check_readiness) if not readiness.ready: @@ -194,6 +191,6 @@ async def run_my_cycle(user: UserContext = Depends(get_current_user)): Personal scope, so it needs no administrator: it can only ever mine this user's evidence and produce candidates only they can accept. """ - from core.evolution.personal import run_personal_cycle + from core.evolution.profiles.personal import run_personal_cycle return success_response(data=run_personal_cycle(user.user_id)) diff --git a/src/backend/api/routes/v1/file_parse.py b/src/backend/api/routes/v1/file_parse.py index 38059a0b..1f7891ea 100644 --- a/src/backend/api/routes/v1/file_parse.py +++ b/src/backend/api/routes/v1/file_parse.py @@ -3,7 +3,8 @@ from fastapi import APIRouter, Depends, File, HTTPException, UploadFile from core.auth.backend import get_current_user, UserContext -from core.content.file_parser import SUPPORTED_EXTENSIONS, parse_file +from core.files.content.file_formats import SUPPORTED_EXTENSIONS +from core.files.content.file_parser import parse_file router = APIRouter(prefix="/v1/file", tags=["file"]) diff --git a/src/backend/api/routes/v1/file_text.py b/src/backend/api/routes/v1/file_text.py index 4e433a30..fcffbfd3 100644 --- a/src/backend/api/routes/v1/file_text.py +++ b/src/backend/api/routes/v1/file_text.py @@ -7,7 +7,7 @@ from core.auth.backend import UserContext, get_current_user from core.db.engine import get_db from core.infra.responses import success_response -from core.services.artifact_text_service import ArtifactTextService +from core.files.artifacts.text import ArtifactTextService router = APIRouter() diff --git a/src/backend/api/routes/v1/file_upload.py b/src/backend/api/routes/v1/file_upload.py index ef9eae86..ef68ecf0 100644 --- a/src/backend/api/routes/v1/file_upload.py +++ b/src/backend/api/routes/v1/file_upload.py @@ -12,9 +12,9 @@ from core.auth.backend import UserContext, get_current_user from core.db.engine import get_db from core.db.models import Artifact, ChatSession, UserFolder -from core.services.artifact_service import store_bytes_as_artifact -from core.storage import get_storage -from core.services.upload_retry import prepare_upload +from core.files.artifacts.service import store_bytes_as_artifact +from core.files.storage import get_storage +from core.files.upload_retry import prepare_upload from .file_text import router as text_router logger = logging.getLogger(__name__) @@ -59,7 +59,7 @@ def upload_user_file( public_meta = {} if user.public_chat_visitor: - from core.services.public_chat_service import prepare_visitor_upload + from core.chat.public import prepare_visitor_upload public_meta = prepare_visitor_upload(db, user, chat_id, folder_id) # Verify the chat_id already exists in the database (the session may not yet be created at upload time) @@ -121,8 +121,8 @@ def upload_user_file( raise HTTPException(status_code=500, detail=f"文件上传失败: {e}") if db_folder_id: - from core.spaces.files import SpaceFiles - from core.spaces.projects import folder_path + from core.files.spaces.files import SpaceFiles + from core.files.spaces.projects import folder_path directory = folder_path(db, UserFolder, {"user_id": user_id}, db_folder_id) SpaceFiles(db, user_id, get_storage()).write( "personal", directory + "/" + file.filename, file_bytes, @@ -153,15 +153,15 @@ async def overwrite_file( if len(file_bytes) > _MAX_UPLOAD_BYTES: raise HTTPException(status_code=413, detail="文件过大，最大支持 50 MB") - from core.artifacts.local_reference import is_local_file_ref, overwrite_reference + from core.files.artifacts.local_reference import is_local_file_ref, overwrite_reference if is_local_file_ref(file_id): return overwrite_reference(file_id, str(user.user_id), file_bytes, db) - from core.auth.permissions_iface import resolve_artifact_access + from core.auth.edition import resolve_artifact_access artifact = db.query(Artifact).filter( Artifact.artifact_id == file_id, Artifact.deleted_at.is_(None), ).populate_existing().with_for_update().first() - from core.services.artifact_edition import artifact_access_metadata + from core.files.edition import artifact_access_metadata metadata = artifact_access_metadata(artifact) if artifact else {} level = resolve_artifact_access(db, str(user.user_id), metadata.get("owner_id"), metadata.get("scope_id")) if artifact else "none" if level == "none": diff --git a/src/backend/api/routes/v1/internal_jobs.py b/src/backend/api/routes/v1/internal_jobs.py index 3e4dc18d..022f3c24 100644 --- a/src/backend/api/routes/v1/internal_jobs.py +++ b/src/backend/api/routes/v1/internal_jobs.py @@ -21,7 +21,7 @@ from core.db.engine import SessionLocal from core.infra.responses import success_response -from core.services.job_service import JobService +from core.jobs.store import JobService from fastapi import APIRouter, Header, HTTPException from .job_callback_session import callback_session @@ -123,7 +123,7 @@ def _schema_hint(schema: Dict[str, Any]) -> str: ) -from orchestration.jobs.agent import run_derived_agent, run_with_job_fence +from core.jobs.agent import run_derived_agent, run_with_job_fence # ── 端点 ────────────────────────────────────────────────────────────── @@ -187,7 +187,7 @@ def validate_execution(): async with _GLOBAL_GATE, gate: _auth(job_id, x_job_token) try: - from core.services.run_journal import durable_run_binding + from core.runs.journal import durable_run_binding async with durable_run_binding( user_id=ctx["user_id"], @@ -234,7 +234,7 @@ def validate_execution(): raise HTTPException(409, "作业预算已耗尽，本项已停止，不自动重试") from exc raise except Exception as exc: - from core.services.tool_effect_ledger import find_tool_outcome_unknown + from core.runs.tool_effects.errors import find_tool_outcome_unknown if find_tool_outcome_unknown(exc) is not None: raise HTTPException( @@ -366,7 +366,7 @@ def job_log( if msg: logger.info("[job %s] %s", job_id, msg[:500]) try: - from orchestration.jobs.notifications import _emit_progress + from core.jobs.notifications import emit_progress as _emit_progress _emit_progress(ctx["chat_id"], msg[:200]) except Exception: # noqa: BLE001 diff --git a/src/backend/api/routes/v1/job_callback_session.py b/src/backend/api/routes/v1/job_callback_session.py index 2455844b..51924bbf 100644 --- a/src/backend/api/routes/v1/job_callback_session.py +++ b/src/backend/api/routes/v1/job_callback_session.py @@ -3,7 +3,7 @@ from contextlib import contextmanager from core.db.engine import SessionLocal -from core.services.job_service import JobService +from core.jobs.store import JobService from fastapi import HTTPException diff --git a/src/backend/api/routes/v1/jobs.py b/src/backend/api/routes/v1/jobs.py index ecc5ca43..96cc3e51 100644 --- a/src/backend/api/routes/v1/jobs.py +++ b/src/backend/api/routes/v1/jobs.py @@ -16,7 +16,7 @@ from core.db.models import Job from core.infra.logging import get_logger from core.infra.responses import success_response -from core.services.job_service import JobService +from core.jobs.store import JobService from fastapi import APIRouter, Depends, HTTPException, Query from sqlalchemy.orm import Session @@ -86,7 +86,7 @@ def get_job( @router.post("/{job_id}/cancel", summary="用户手动取消作业") async def cancel_job(job_id: str, user: UserContext = Depends(get_current_user)): """状态条上的取消按钮 —— 跑歪了的作业不该只能等它烧完预算。""" - from orchestration import job_runtime + import core.jobs.runtime as job_runtime try: ok = await job_runtime.cancel_job(job_id, user_id=user.user_id) diff --git a/src/backend/api/routes/v1/kb.py b/src/backend/api/routes/v1/kb.py deleted file mode 100644 index 138d2e2b..00000000 --- a/src/backend/api/routes/v1/kb.py +++ /dev/null @@ -1,873 +0,0 @@ -"""Knowledge Base management API routes (v1). - -Route definitions only — models live in ``kb_models.py``, processing logic -in ``core/kb_processing.py``, and CRUD in ``core/services/kb_service.py``. -""" - -import hashlib -import json -import logging -import os -from typing import Optional - -from api.routes.v1.kb_models import ( - CreateKBSpaceRequest, - PolishKBDescriptionRequest, - ReindexRequest, - UpdateChunkRequest, - UpdateKBSpaceRequest, -) -from core.auth.backend import UserContext, get_current_user -from core.config.settings import settings -from core.content.file_validation import validate_kb_file -from core.kb.wiki.config import index_modes_of, normalize_index_modes -from core.db.engine import get_db -from core.infra.exceptions import ( - AccessDeniedError, - BadRequestError, - FileTooLargeError, - ResourceNotFoundError, - StorageError, -) -from core.infra.responses import created_response, paginated_response, success_response -from core.llm.chat_models import get_summarize_model -from core.llm.message_compat import extract_text_from_chat_response -from core.services import KBService -from core.storage import generate_storage_key, get_storage -from fastapi import APIRouter, Depends, File, Form, Path, Query, UploadFile, status -from fastapi.concurrency import run_in_threadpool -from sqlalchemy.orm import Session - -logger = logging.getLogger(__name__) - -router = APIRouter(prefix="/v1/catalog/kb", tags=["Knowledge Base"]) - -# Constants -MAX_FILE_SIZE = 100 * 1024 * 1024 # 100MB -MAX_PREVIEW_SIZE = 20 * 1024 * 1024 # 20MB - - -def _can_read_space(db, space, user_id: str) -> bool: - """Read permission: owner / everyone for public KBs / grantees of a scoped-visibility KB (single-source-of-truth resolution).""" - from core.auth.kb_permissions import resolve_local_kb_level - - return resolve_local_kb_level(db, user_id, space.kb_id) != "none" - - -def _require_kb_write(db, space, user_id: str, required: str) -> None: - """Write permission check: required is 'edit' (upload/reindex) or 'admin' (rename/delete/edit chunks). - - Raises AccessDeniedError when insufficient. The owner is always admin, superadmins are always admin, granted users go by their level. - """ - from core.auth.kb_permissions import has_kb_permission, resolve_local_kb_level - - level = resolve_local_kb_level(db, user_id, space.kb_id) - if not has_kb_permission(level, required): - raise AccessDeniedError(message="Access denied", reason=f"requires kb level >= {required}") - - -def _enqueue_document_indexing( - db, - *, - document_id: str, - user_id: str, - chunk_method: str, - indexing_config: Optional[dict], - index_modes: Optional[list], -) -> None: - """把文档排进持久化索引队列，交给常驻 worker 执行。 - - 索引不能再跑在请求线程池里：那是 Starlette 与所有同步依赖（``get_db``）共用的 - 池子，批量上传会把它占满，导致整个后端在 handler 之前就排队。详见 - :mod:`core.kb.index_worker`。 - """ - from core.db.models import KBDocument - from core.kb import index_queue - - doc = db.query(KBDocument).filter(KBDocument.document_id == document_id).first() - if doc is None: - logger.warning("Cannot enqueue indexing: document %s not found", document_id) - return - index_queue.enqueue( - db, - doc, - user_id=user_id, - chunk_method=chunk_method, - indexing_config=indexing_config, - index_modes=index_modes, - ) - - -def _fallback_kb_description(name: str, description: str) -> str: - base_name = (name or "").strip() - base_desc = " ".join((description or "").split()) - if base_desc: - return ( - f"{base_name}相关资料，适合检索概念说明、政策信息、行业动态、业务规则和常见问题。" - f"当前范围包括：{base_desc[:120]}" - )[:220] - return ( - f"{base_name}相关内容，适合检索概念说明、政策解读、业务规则、行业动态、" - f"典型案例和常见问题。" - )[:220] - - -async def _generate_kb_description(name: str, description: str) -> str: - prompt = f"""/no_think -你是一名知识库配置助手。请根据知识库名称和现有简介，生成一段更利于智能路由判断的“知识库简介”。 - -目标： -1. 帮助系统快速判断什么问题应该路由到这个知识库 -2. 文本要像知识库简介，不是宣传文案 -3. 明确主题范围、文档类型、适合回答的问题类型 -4. 用中文，1 段即可，控制在 60 到 120 字 -5. 不要使用“本知识库可以帮助你”这类口语化表达 -6. 不要使用“该知识库聚焦”“本知识库涵盖”这类空泛起手式 -7. 直接输出简介正文，不要加标题、引号、序号或解释 - -知识库名称：{name} -现有简介：{description or "无"} -""" - - try: - model = get_summarize_model() - # AgentScope 2.0: model.__call__ takes list[Msg] (not list[dict]) - from agentscope.message import Msg, TextBlock - - result = await model( - messages=[Msg(name="user", role="user", content=[TextBlock(type="text", text=prompt)])] - ) - polished = extract_text_from_chat_response(result).strip() - polished = " ".join(polished.split()) - return polished or _fallback_kb_description(name, description) - except Exception as exc: - logger.warning("Failed to polish KB description for %s: %s", name, exc) - return _fallback_kb_description(name, description) - - -# ── Routes ────────────────────────────────────────────────────────────────── - - -@router.get("/assets/{asset_id}", summary="读取知识库媒体资产") -def get_kb_asset( - asset_id: str = Path(..., description="Asset ID"), - user: UserContext = Depends(get_current_user), - db: Session = Depends(get_db), -): - """按 ID 读取知识库中的媒体资产（版面解析切出的图片等）原始字节。 - - 资产的可见性完全跟随所属知识库空间：能读该库才能读它的图。图片链接会出现在分块正文 - 与检索结果里，故按 ID 直取，不经过库/文档路径。 - """ - from fastapi.responses import Response - - from core.db.models import KBAsset - from core.db.repository import KBRepository - - asset = db.query(KBAsset).filter(KBAsset.asset_id == asset_id).first() - if not asset: - raise ResourceNotFoundError(resource_type="kb_asset", resource_id=asset_id) - - kb_space = KBRepository(db).get_space(asset.kb_id) - if not kb_space or not _can_read_space(db, kb_space, user.user_id): - # Same shape as a missing asset: an unauthorised caller must not be able to - # probe which asset ids exist. - raise ResourceNotFoundError(resource_type="kb_asset", resource_id=asset_id) - - try: - data = get_storage().download_bytes(asset.storage_key) - except Exception as exc: - logger.warning("Asset %s read failed: %s", asset_id, exc) - raise ResourceNotFoundError(resource_type="kb_asset", resource_id=asset_id) - - return Response( - content=data, - media_type=asset.mime_type or "application/octet-stream", - headers={"Cache-Control": "private, max-age=86400"}, - ) - - -@router.post("/preview-chunks", summary="预览文档分块效果") -async def preview_chunks( - file: UploadFile = File(..., description="Document file to preview"), - chunk_method: str = Form( - "structured", description="Chunking method: structured|recursive|embedding_semantic|laws|qa" - ), - parent_chunk_size: int = Form(1024, description="Parent chunk size in tokens"), - child_chunk_size: int = Form(128, description="Child chunk size in tokens"), - overlap_tokens: int = Form(20, description="Overlap between child chunks"), - parent_child_indexing: bool = Form( - True, description="Enable parent-child indexing; False = flat chunks only" - ), - separators: Optional[str] = Form( - None, description="父分块分隔符，JSON 数组字符串；为空用默认层级" - ), - child_separators: Optional[str] = Form( - None, description="子分块分隔符，JSON 数组字符串；为空走定长滑窗" - ), - user: UserContext = Depends(get_current_user), -): - """预览文档的分块效果，不落库、不写入向量库。上传文件并按指定分块方法（structured/recursive/embedding_semantic/laws/qa）与父子分块参数解析，返回父块及其子块预览，用于创建知识库前调参。文件大小受 MAX_PREVIEW_SIZE（20MB）限制。""" - content = await file.read() - if len(content) > MAX_PREVIEW_SIZE: - raise FileTooLargeError(max_size=MAX_PREVIEW_SIZE, actual_size=len(content)) - if len(content) == 0: - raise BadRequestError(message="File is empty", data={"filename": file.filename}) - - if not file.filename: - raise BadRequestError(message="Filename is required", data={"field": "file"}) - _, mime_type = validate_kb_file(file.filename, content) - - from core.kb.kb_parser import _count_tokens, parse_and_chunk, parse_separators_form - - _separators = parse_separators_form(separators) - _child_separators = parse_separators_form(child_separators) - - embed_fn = None - if chunk_method == "embedding_semantic": - from core.kb.kb_vector import embed_batch - - embed_fn = embed_batch - - parent_chunks = parse_and_chunk( - content, - mime_type, - chunk_method=chunk_method, - parent_size=parent_chunk_size, - child_size=child_chunk_size, - overlap=overlap_tokens, - embed_fn=embed_fn, - separators=_separators, - child_separators=_child_separators, - ) - - total_children = sum(len(pc.children) for pc in parent_chunks) if parent_child_indexing else 0 - - chunks_preview = [] - for idx, pc in enumerate(parent_chunks): - if parent_child_indexing: - children_preview = [ - { - "index": c.index, - "content": c.content[:200] + ("..." if len(c.content) > 200 else ""), - } - for c in pc.children[:5] - ] - children_count = len(pc.children) - else: - children_preview = [] - children_count = 0 - chunks_preview.append( - { - "index": idx, - "content": pc.content, - "token_count": _count_tokens(pc.content), - "children_count": children_count, - "children_preview": children_preview, - } - ) - - return success_response( - data={ - "total_chunks": len(parent_chunks), - "total_children": total_children, - "chunks": chunks_preview, - }, - message="Chunk preview generated successfully", - ) - - -@router.post("", status_code=status.HTTP_201_CREATED, summary="创建知识库空间") -def create_kb_space( - request: CreateKBSpaceRequest, - user: UserContext = Depends(get_current_user), - db: Session = Depends(get_db), -): - """创建归属当前登录用户的知识库空间；可见性规则由当前版本决定。""" - visibility = (request.visibility or "private").strip() - if visibility not in ("private", "public"): - raise BadRequestError( - message="visibility 仅支持 private / public", data={"field": "visibility"} - ) - if settings.edition.edition == "ce" and visibility != "private": - raise BadRequestError( - message="CE 仅支持私有知识库", - data={"field": "visibility", "allowed": ["private"]}, - ) - - # Permission check: require the creation permission matching the chosen visibility - from core.auth.capabilities import resolve_user_capabilities - - caps = resolve_user_capabilities(db, user.user_id) - required_cap = "can_create_public_kb" if visibility == "public" else "can_create_private_kb" - if not caps.get(required_cap, False): - label = "公有" if visibility == "public" else "私有" - raise AccessDeniedError(message="Access denied", reason=f"缺少「创建{label}知识库」权限") - - kb_service = KBService(db) - metadata = dict(request.metadata or {}) - if request.indexing_config: - metadata["indexing_config"] = request.indexing_config.model_dump() - - # 索引模式：rag（向量检索）/ wiki（LLM Wiki 图谱），两者同选即 LLM-Wiki 知识库。 - # 三种组合都会分块——wiki 省掉的是向量化，不是分块（引文标注要 chunk id、 - # 回溯原文要按 chunk 直取）。 - metadata["index_modes"] = normalize_index_modes(request.index_modes) - if request.wiki_config: - metadata["wiki_config"] = { - k: v for k, v in request.wiki_config.model_dump().items() if v is not None - } - - from core.services.kb_edition import initial_visibility_grants - - initial_grants = initial_visibility_grants(db, user.user_id, visibility) - - space = kb_service.create_space( - user_id=user.user_id, - name=request.name, - description=request.description, - chunk_method=request.chunk_method or "semantic", - metadata=metadata, - visibility=visibility, - initial_grants=initial_grants, - granted_by=user.user_id, - ) - return created_response(data=space, message="Knowledge base space created successfully") - - -@router.post("/polish-description", summary="AI 生成知识库简介") -async def polish_kb_description( - request: PolishKBDescriptionRequest, - user: UserContext = Depends(get_current_user), -): - """根据知识库名称和现有简介，调用大模型生成一段更利于智能路由判断的知识库简介。名称为必填，模型失败时回退到本地模板生成。仅返回润色后的简介文本，不修改任何知识库记录。""" - name = request.name.strip() - description = (request.description or "").strip() - if not name: - raise BadRequestError(message="Knowledge base name is required", data={"field": "name"}) - - polished = await _generate_kb_description(name, description) - return success_response( - data={"description": polished}, - message="Knowledge base description polished successfully", - ) - - -@router.patch("/{kb_id}", summary="更新知识库空间") -def update_kb_space( - kb_id: str, - request: UpdateKBSpaceRequest, - user: UserContext = Depends(get_current_user), - db: Session = Depends(get_db), -): - """更新指定知识库空间的名称、简介与索引模式。仅空间所有者可操作；空间不存在或无权访问时返回 404。 - - 改索引模式不会自动补建索引：新勾选 Wiki 后要调 `POST /{kb_id}/wiki/rebuild` - 才会把已有文档排进生成队列。 - """ - kb_service = KBService(db) - space = kb_service.update_space( - kb_id=kb_id, - user_id=user.user_id, - name=request.name, - description=request.description, - index_modes=request.index_modes, - wiki_config=(request.wiki_config.model_dump() if request.wiki_config else None), - ) - if not space: - raise ResourceNotFoundError(resource_type="kb_space", resource_id=kb_id) - return success_response(data=space, message="Knowledge base space updated successfully") - - -@router.post("/{kb_id}/documents", status_code=status.HTTP_201_CREATED, summary="上传文档到知识库") -async def upload_document( - kb_id: str = Path(..., description="KB space ID"), - file: UploadFile = File(..., description="Document file to upload"), - title: Optional[str] = Form(None, description="Document title (defaults to filename)"), - metadata: Optional[str] = Form(None, description="Additional metadata as JSON string"), - indexing_config: Optional[str] = Form(None, description="Indexing config as JSON string"), - chunk_method: Optional[str] = Form(None, description="Chunking method override"), - user: UserContext = Depends(get_current_user), - db: Session = Depends(get_db), -): - """上传文档到指定知识库空间并异步建立索引。仅空间所有者可上传；文件大小受 MAX_FILE_SIZE（100MB）限制，可选传入标题、元数据、索引配置及分块方法覆盖。上传成功后通过后台任务完成解析、分块与向量化，立即返回 indexing_status=processing。""" - kb_service = KBService(db) - - from core.db.repository import KBRepository - - kb_repo = KBRepository(db) - kb_space = kb_repo.get_space(kb_id) - - if not kb_space: - raise ResourceNotFoundError(resource_type="kb_space", resource_id=kb_id) - _require_kb_write(db, kb_space, user.user_id, "edit") - - if not file.filename: - raise BadRequestError(message="Filename is required", data={"field": "file"}) - - content = await file.read() - file_size = len(content) - - if file_size > MAX_FILE_SIZE: - raise FileTooLargeError(max_size=MAX_FILE_SIZE, actual_size=file_size) - if file_size == 0: - raise BadRequestError(message="File is empty", data={"filename": file.filename}) - - _, mime_type = validate_kb_file(file.filename, content) - - checksum = hashlib.sha256(content).hexdigest() - env = os.getenv("ENVIRONMENT", "dev") - storage_key = generate_storage_key( - env=env, user_id=user.user_id, category="kb_documents", filename=file.filename - ) - - try: - storage = get_storage() - # 对象存储是阻塞的网络调用，直接在 async handler 里调会卡住整个事件循环—— - # 批量上传时后端会连 SSE 和其它接口一起停摆。 - await run_in_threadpool(storage.upload_bytes, content, storage_key) - logger.info("Document uploaded to storage: %s", storage_key) - except StorageError as e: - logger.error("Failed to upload document to storage: %s", e) - raise StorageError( - operation="upload", error=f"Failed to upload document: {e.data.get('error', str(e))}" - ) - - doc_title = title or file.filename - - if metadata: - try: - json.loads(metadata) - except json.JSONDecodeError: - raise BadRequestError(message="Invalid metadata JSON", data={"field": "metadata"}) - - try: - document = kb_service.upload_document( - kb_id=kb_id, - user_id=user.user_id, - title=doc_title, - filename=file.filename, - size_bytes=file_size, - mime_type=mime_type, - storage_key=storage_key, - checksum=checksum, - ) - except PermissionError as e: - raise AccessDeniedError(message="Access denied", reason=str(e)) - - _effective_chunk_method = chunk_method or kb_space.chunk_method or "semantic" - _upload_indexing_config = None - if indexing_config: - try: - _upload_indexing_config = json.loads(indexing_config) - except json.JSONDecodeError: - raise BadRequestError( - message="Invalid indexing_config JSON", data={"field": "indexing_config"} - ) - _space_extra = kb_space.extra_data if isinstance(kb_space.extra_data, dict) else {} - _indexing_config = _upload_indexing_config or _space_extra.get("indexing_config") - - # 排进持久化索引队列，由常驻 worker 认领执行（见 core/kb/index_worker.py）。 - # 早先这里挂的是 FastAPI BackgroundTasks：同步任务落进 Starlette 的共享请求线程池， - # 一次多选几十个文件就把池占满，后续请求连 handler 都进不去——前端一直转圈、网关 - # 报 502；而且任务是进程内的，后端一重启就全丢，文档永远停在「索引中」。 - _enqueue_document_indexing( - db, - document_id=document["document_id"], - user_id=user.user_id, - chunk_method=_effective_chunk_method, - indexing_config=_indexing_config, - index_modes=index_modes_of(kb_space), - ) - - return created_response( - data={**document, "indexing_status": "processing"}, - message="Document uploaded successfully. Indexing in progress.", - ) - - -@router.get("/{kb_id}/documents", summary="获取知识库文档列表") -def list_documents( - kb_id: str = Path(..., description="Local or external knowledge collection ID"), - page: int = Query(1, ge=1, description="Page number"), - page_size: int = Query(20, ge=1, le=100, description="Items per page"), - keyword: Optional[str] = Query(None, description="按标题/文件名搜索（全库范围，再分页）"), - user: UserContext = Depends(get_current_user), - db: Session = Depends(get_db), -): - """分页获取知识库文档；非本地 ID 交由当前版本的外部知识提供方处理。""" - from core.db.repository import KBRepository - - kb_repo = KBRepository(db) - kb_space = kb_repo.get_space(kb_id) - - if not kb_space: - from core.kb.external_provider import is_enabled, list_documents - - if is_enabled(): - result = list_documents(kb_id, page=page, limit=page_size, keyword=keyword or "") - response = paginated_response( - items=result.get("items", []), - page=result.get("page", page), - page_size=result.get("page_size", page_size), - total_items=result.get("total", 0), - message="Documents retrieved successfully", - ) - from core.kb.external_provider import count_document_statuses - - status_counts = count_document_statuses( - kb_id, - keyword=keyword or "", - initial_result=result, - ) - if status_counts is not None: - response["data"]["status_counts"] = status_counts - return response - raise ResourceNotFoundError(resource_type="kb_space", resource_id=kb_id) - - # Public KBs (created in the admin console, owned by the system principal) are readable by all logged-in users - if not _can_read_space(db, kb_space, user.user_id): - raise AccessDeniedError( - message="Access denied", reason="Only the KB space owner can list documents" - ) - - documents, total = kb_repo.list_documents(kb_id, page, page_size, keyword=keyword) - items = [ - { - "id": d.document_id, - "title": d.title, - "desc": d.filename, - "filename": d.filename, - "size": d.size_bytes, - "mime_type": d.mime_type, - "storage_key": d.storage_key, - "uploaded_at": d.uploaded_at.isoformat(), - "indexing_status": getattr(d, "indexing_status", "processing"), - } - for d in documents - ] - response = paginated_response( - items=items, - page=page, - page_size=page_size, - total_items=total, - message="Documents retrieved successfully", - ) - response["data"]["status_counts"] = kb_repo.count_document_statuses(kb_id, keyword=keyword) - return response - - -@router.get("/{kb_id}/documents/{document_id}", summary="获取知识库文档详情") -def get_document_detail( - kb_id: str = Path(..., description="Local or external knowledge collection ID"), - document_id: str = Path(..., description="Document ID"), - user: UserContext = Depends(get_current_user), - db: Session = Depends(get_db), -): - """获取文档详情；非本地 ID 交由当前版本的外部知识提供方处理。""" - from core.db.repository import KBRepository - - kb_repo = KBRepository(db) - kb_space = kb_repo.get_space(kb_id) - - if not kb_space: - from core.kb.external_provider import get_document_detail, is_enabled - - if is_enabled(): - detail = get_document_detail(kb_id, document_id) - return success_response(data=detail, message="Document detail retrieved successfully") - raise ResourceNotFoundError(resource_type="kb_space", resource_id=kb_id) - - # Public KBs are readable by all logged-in users - if not _can_read_space(db, kb_space, user.user_id): - raise AccessDeniedError( - message="Access denied", reason="Only the KB space owner can view document details" - ) - - document = kb_repo.get_document(document_id) - if not document or document.kb_id != kb_id: - raise ResourceNotFoundError(resource_type="kb_document", resource_id=document_id) - - from core.db.models import KBChunk - - chunks = ( - db.query(KBChunk) - .filter(KBChunk.document_id == document_id, KBChunk.kb_id == kb_id) - .order_by(KBChunk.chunk_index) - .all() - ) - content = "\n\n".join(c.content for c in chunks) if chunks else "" - - return success_response( - data={ - "id": document.document_id, - "title": document.title, - "desc": document.filename, - "filename": document.filename, - "mime_type": document.mime_type, - "uploaded_at": document.uploaded_at.isoformat(), - "content": content, - }, - message="Document detail retrieved successfully", - ) - - -@router.delete("/{kb_id}", summary="删除知识库空间") -def delete_kb_space( - kb_id: str = Path(..., description="KB space ID"), - user: UserContext = Depends(get_current_user), - db: Session = Depends(get_db), -): - """删除指定知识库空间及其下属文档、分块与向量数据。仅空间所有者可操作；空间不存在或无权访问时返回 404。此操作不可恢复。""" - kb_service = KBService(db) - deleted = kb_service.delete_space(kb_id, user.user_id) - if not deleted: - raise ResourceNotFoundError(resource_type="kb_space", resource_id=kb_id) - return success_response(data=None, message="Knowledge base deleted successfully") - - -@router.delete("/{kb_id}/documents/{document_id}", summary="删除知识库文档") -def delete_kb_document( - kb_id: str = Path(..., description="KB space ID"), - document_id: str = Path(..., description="Document ID"), - user: UserContext = Depends(get_current_user), - db: Session = Depends(get_db), -): - """从知识库空间删除指定文档及其分块与向量数据。仅文档所属空间的所有者可操作；文档不存在或无权访问时返回 404。此操作不可恢复。""" - kb_service = KBService(db) - deleted = kb_service.delete_document(document_id, user.user_id) - if not deleted: - raise ResourceNotFoundError(resource_type="kb_document", resource_id=document_id) - return success_response(data=None, message="Document deleted successfully") - - -@router.post("/{kb_id}/documents/{document_id}/reindex", summary="重新索引文档") -def reindex_document( - kb_id: str = Path(..., description="KB space ID"), - document_id: str = Path(..., description="Document ID"), - request: Optional[ReindexRequest] = None, - user: UserContext = Depends(get_current_user), - db: Session = Depends(get_db), -): - """对指定文档重新建立索引。会先清除该文档的旧分块及向量数据，再从存储重新下载原文，按可选的索引配置与分块方法异步重新解析、分块、向量化。仅空间所有者可操作，立即返回 indexing_status=processing。""" - from core.db.models import KBChunk - from core.db.repository import KBRepository - - kb_repo = KBRepository(db) - kb_space = kb_repo.get_space(kb_id) - if not kb_space: - raise ResourceNotFoundError(resource_type="kb_space", resource_id=kb_id) - _require_kb_write(db, kb_space, user.user_id, "edit") - - document = kb_repo.get_document(document_id) - if not document or document.kb_id != kb_id: - raise ResourceNotFoundError(resource_type="kb_document", resource_id=document_id) - - db.query(KBChunk).filter(KBChunk.document_id == document_id, KBChunk.kb_id == kb_id).delete() - document.indexing_status = "processing" - db.commit() - - # 资产随分块一起重建：不清会在每次重索引后留下一份孤儿图（存储与表都涨）。 - try: - from core.kb.kb_assets import purge_document_assets - - purge_document_assets(db, document_id) - except Exception as exc: - logger.warning("Asset cleanup failed for reindex of %s: %s", document_id, exc) - - try: - from core.kb.kb_vector import delete_by_document - - delete_by_document(document_id, user.user_id) - except Exception as exc: - logger.warning("Milvus cleanup failed for reindex of %s: %s", document_id, exc) - - _idx_cfg = None - if request and request.indexing_config: - _idx_cfg = request.indexing_config.model_dump() - if not _idx_cfg: - _space_extra = kb_space.extra_data if isinstance(kb_space.extra_data, dict) else {} - _idx_cfg = _space_extra.get("indexing_config") - - _reindex_chunk_method = ( - (request.chunk_method if request and request.chunk_method else None) - or kb_space.chunk_method - or "semantic" - ) - - # 与上传同一条路：排队 + 常驻 worker，原文由 worker 自己从存储取回。 - # 这里不再预先 download_bytes——几十 MB 的原文既没必要在请求里读，也不该跟着 - # 任务在内存里排队等着跑。 - _enqueue_document_indexing( - db, - document_id=document_id, - user_id=user.user_id, - chunk_method=_reindex_chunk_method, - indexing_config=_idx_cfg, - index_modes=index_modes_of(kb_space), - ) - - return success_response( - data={"document_id": document_id, "indexing_status": "processing"}, - message="Document re-indexing started", - ) - - -@router.get("/{kb_id}/chunks", summary="获取文档分块列表（含标签和问题）") -def list_chunks( - kb_id: str = Path(..., description="KB space ID"), - document_id: Optional[str] = Query(None, description="Filter by document ID"), - page: int = Query(1, ge=1), - page_size: int = Query(20, ge=1, le=100), - user: UserContext = Depends(get_current_user), - db: Session = Depends(get_db), -): - """分页获取知识库分块列表，可选按 document_id 过滤。每个分块返回完整正文内容、标签和问题列表，供分块管理界面查看与编辑。仅空间所有者可访问。""" - from core.db.models import KBChunk - from core.db.repository import KBRepository - - kb_repo = KBRepository(db) - kb_space = kb_repo.get_space(kb_id) - if not kb_space: - raise ResourceNotFoundError(resource_type="kb_space", resource_id=kb_id) - # Public KBs are readable by all logged-in users (chunks are read-only viewing) - if not _can_read_space(db, kb_space, user.user_id): - raise AccessDeniedError(message="Access denied", reason="Not the KB owner") - - query = db.query(KBChunk).filter(KBChunk.kb_id == kb_id) - if document_id: - query = query.filter(KBChunk.document_id == document_id) - total = query.count() - chunks = ( - query.order_by(KBChunk.chunk_index).offset((page - 1) * page_size).limit(page_size).all() - ) - - items = [ - { - "chunk_id": c.chunk_id, - "document_id": c.document_id, - "chunk_index": c.chunk_index, - "content": c.content, - "tags": c.tags or [], - "questions": c.questions or [], - } - for c in chunks - ] - return paginated_response(items=items, page=page, page_size=page_size, total_items=total) - - -@router.get("/{kb_id}/chunks/{chunk_id}/children", summary="获取父块下的子块列表") -def list_chunk_children( - kb_id: str = Path(..., description="KB space ID"), - chunk_id: str = Path(..., description="Parent chunk ID"), - user: UserContext = Depends(get_current_user), - db: Session = Depends(get_db), -): - """返回某父块在向量库里的子块（父子分块模式）。扁平模式无子块时返回空列表。仅空间可读者可访问。""" - from core.db.models import KBChunk - from core.db.repository import KBRepository - - kb_repo = KBRepository(db) - kb_space = kb_repo.get_space(kb_id) - if not kb_space: - raise ResourceNotFoundError(resource_type="kb_space", resource_id=kb_id) - if not _can_read_space(db, kb_space, user.user_id): - raise AccessDeniedError(message="Access denied", reason="Not the KB owner") - - parent = db.query(KBChunk).filter(KBChunk.chunk_id == chunk_id, KBChunk.kb_id == kb_id).first() - if not parent: - raise ResourceNotFoundError(resource_type="kb_chunk", resource_id=chunk_id) - - from core.kb.kb_vector import get_children_for_parent - - children = get_children_for_parent(chunk_id) - return success_response( - data={"children": children, "count": len(children)}, - message="Chunk children retrieved successfully", - ) - - -@router.patch("/{kb_id}/chunks/{chunk_id}", summary="更新分块内容/标签/问题") -def update_chunk( - kb_id: str = Path(..., description="KB space ID"), - chunk_id: str = Path(..., description="Chunk ID"), - request: UpdateChunkRequest = ..., - user: UserContext = Depends(get_current_user), - db: Session = Depends(get_db), -): - """更新指定分块的正文内容、标签和问题列表，并同步刷新其在向量库中的向量与索引。 - - 传入 ``content`` 时会对该分块重新向量化（dense + sparse）；仅改标签时只刷新稀疏/标签 - 索引；问题始终单独重建问题向量行。仅空间所有者（admin 权限）可操作，系统托管知识库禁止 - 编辑分块；仅传入的字段会被更新。""" - from core.db.models import KBChunk, KBDocument - from core.db.repository import KBRepository - from core.kb.kb_vector import reindex_chunk_content, reindex_chunk_tags, upsert_question_rows - - kb_repo = KBRepository(db) - kb_space = kb_repo.get_space(kb_id) - if not kb_space: - raise ResourceNotFoundError(resource_type="kb_space", resource_id=kb_id) - _require_kb_write(db, kb_space, user.user_id, "admin") - if KBService(db)._is_system_managed_space(kb_space): - raise AccessDeniedError( - message="Access denied", reason="System managed KB does not allow chunk edits" - ) - - chunk = db.query(KBChunk).filter(KBChunk.chunk_id == chunk_id, KBChunk.kb_id == kb_id).first() - if not chunk: - raise ResourceNotFoundError(resource_type="kb_chunk", resource_id=chunk_id) - - content_changed = request.content is not None - if content_changed: - new_content = request.content.strip() - if not new_content: - raise BadRequestError(message="分块内容不能为空", data={"field": "content"}) - chunk.content = new_content - if request.tags is not None: - chunk.tags = request.tags - if request.questions is not None: - chunk.questions = request.questions - db.commit() - db.refresh(chunk) - - try: - doc = db.query(KBDocument).filter_by(document_id=chunk.document_id).first() - doc_title = doc.title if doc else "" - - if content_changed: - # Content changed: re-vectorize the whole chunk (current tags already included) - reindex_chunk_content( - chunk_id=chunk_id, - content=chunk.content, - tags=chunk.tags or [], - user_id=user.user_id, - kb_id=kb_id, - document_id=chunk.document_id, - title=doc_title, - chunk_index=chunk.chunk_index, - ) - elif request.tags is not None: - reindex_chunk_tags(chunk_id, chunk.content, chunk.tags) - if request.questions is not None: - upsert_question_rows( - parent_chunk_id=chunk_id, - questions=chunk.questions, - user_id=user.user_id, - kb_id=kb_id, - document_id=chunk.document_id, - title=doc_title, - chunk_index=chunk.chunk_index, - ) - except Exception as exc: - logger.warning("Milvus re-index failed for chunk %s: %s", chunk_id, exc) - - return success_response( - data={ - "chunk_id": chunk.chunk_id, - "content": chunk.content, - "tags": chunk.tags, - "questions": chunk.questions, - }, - message="Chunk updated successfully", - ) diff --git a/src/backend/api/routes/v1/loops.py b/src/backend/api/routes/v1/loops.py index cc2ce7a9..37822cbe 100644 --- a/src/backend/api/routes/v1/loops.py +++ b/src/backend/api/routes/v1/loops.py @@ -2,20 +2,22 @@ See internal design docs (§5). """ + from typing import Any, Dict, List, Optional +import api.transports.run_sse as run_sse +import core.runs.runtime as runtime +import orchestration.loops.admission as admission +from core.auth.backend import UserContext, get_current_user +from core.db.engine import get_db +from core.infra.logging import get_logger +from core.infra.responses import sse_response, success_response +from core.chat.service import ChatService +from features.automation.loops import LoopService from fastapi import APIRouter, Depends, HTTPException from pydantic import BaseModel, Field from sqlalchemy.orm import Session -from core.auth.backend import get_current_user, UserContext -from core.db.engine import get_db -from core.infra.logging import get_logger -from core.infra.responses import success_response, sse_response -from core.services.chat_service import ChatService -from core.services.loop_service import LoopService -from orchestration import chat_run_executor - logger = get_logger(__name__) router = APIRouter(prefix="/v1/loops", tags=["AutonomousLoop"]) @@ -187,7 +189,7 @@ async def _launch_loop(loop_id: str, req: StartLoopReq, db: Session, user: UserC hitl_enabled = req.hitl_enabled or bool(saved.get("hitl_enabled")) # 与普通聊天同一套模型选择权限闸：未开放切换/不在可选列表的 provider 一律忽略。 - from core.services.user_model_selection import resolve_user_model_provider_id + from core.auth.model_selection import resolve_user_model_provider_id model_provider_id = resolve_user_model_provider_id( db, raw_provider_id, user_id=user.user_id @@ -204,7 +206,7 @@ async def _launch_loop(loop_id: str, req: StartLoopReq, db: Session, user: UserC # The project the loop is bound to (stored in metadata at creation) — the # worker/reviewer scope to the project folder based on it. project_id = (loop.extra_data or {}).get("project_id") if loop.extra_data else None - run = await chat_run_executor.start_autonomous_loop_run( + run = await admission.start_autonomous_loop_run( loop_id=loop_id, chat_id=chat_id, user_id=user.user_id, @@ -221,7 +223,7 @@ async def _launch_loop(loop_id: str, req: StartLoopReq, db: Session, user: UserC project_id=project_id, ) return sse_response( - chat_run_executor.follow_run_as_sse( + run_sse.follow_run_as_sse( run.run_id, chat_id=chat_id, error_event_factory=lambda reason: { @@ -292,7 +294,7 @@ async def cancel_loop( if not loop: raise HTTPException(status_code=404, detail="loop not found") chat_id = loop.chat_id or f"loopchat_{loop_id}" - run = chat_run_executor.get_active_run_for_chat(chat_id) + run = runtime.get_active_run_for_chat(chat_id, user_id=user.user_id) if not run: # 无活跃 run（崩溃遗留/已中断）：把 loop 状态归位而不是 404——历史缺陷是 # 僵尸 running 行永远取消不掉。 @@ -301,5 +303,5 @@ async def cancel_loop( db.commit() return success_response({"cancelled": True, "note": "无活跃任务，已直接归位为 cancelled"}) return success_response({"cancelled": False, "status": loop.status}) - ok = await chat_run_executor.cancel_run(run.run_id, user_id=user.user_id) + ok = await runtime.cancel_run(run.run_id, user_id=user.user_id) return success_response({"cancelled": ok}) diff --git a/src/backend/api/routes/v1/me_capabilities.py b/src/backend/api/routes/v1/me_capabilities.py index 3e79b20e..ca72ee84 100644 --- a/src/backend/api/routes/v1/me_capabilities.py +++ b/src/backend/api/routes/v1/me_capabilities.py @@ -19,8 +19,8 @@ import uuid from typing import Dict, List, Optional -from core.capabilities.paths import capabilities_enabled -from core.services import local_skill_editor as local_editor +from core.harness.registry.run_capabilities import has_run_capabilities +from features.desktop.local.skills import editor as local_editor from core.auth.backend import UserContext, get_current_user from core.auth.capabilities import resolve_user_capabilities from core.config.settings import settings @@ -28,21 +28,8 @@ from core.db.models import AdminMcpServer, AdminSkill from core.infra.exceptions import AccessDeniedError, BadRequestError, ResourceNotFoundError from core.infra.responses import created_response, success_response -from core.services.mcp_management_service import ( - encrypt_mcp_headers, - probe_mcp_connectivity, - refresh_mcp_caches, - validate_remote_mcp_url, -) -from core.services.skill_management_service import ( - build_skill_content, - extract_instructions, - extract_mcp_server_ids, - parse_and_upsert_skill_zip, - refresh_skill_caches, - resolve_mcp_bindings, - resolve_ontology_workflows, -) +from core.capabilities.mcp.management import encrypt_mcp_headers, probe_mcp_connectivity, refresh_mcp_caches, validate_remote_mcp_url +from core.capabilities.skills.management import build_skill_content, extract_instructions, extract_mcp_server_ids, parse_and_upsert_skill_zip, refresh_skill_caches, resolve_mcp_bindings, resolve_ontology_workflows from fastapi import APIRouter, Depends, File, HTTPException, UploadFile from pydantic import BaseModel, Field, field_validator from sqlalchemy.orm import Session @@ -113,7 +100,6 @@ async def create_my_mcp_server( transport=body.transport, url=body.url.strip(), headers=body.headers or {}, - is_stable=False, # user-private MCPs don't enter the warmup connection pool is_enabled=True, owner_user_id=str(user.user_id), icon=body.icon, @@ -182,7 +168,7 @@ async def upload_my_skill( 复用 admin 上传的解析链路，但打上 ``owner_user_id`` 并限 50MB。技能 id 取 SKILL.md frontmatter ``name``，须全局唯一（与公共/他人技能冲突会被拒绝）。 """ - if capabilities_enabled(): + if has_run_capabilities(): return created_response( data=local_editor.install_archive(str(user.user_id), await file.read()) ) @@ -242,7 +228,7 @@ def create_my_skill( 技能 id 须全局唯一：不能与公共技能或他人私有技能冲突；与本人已有同名私有技能 则视为更新。 """ - if capabilities_enabled(): + if has_run_capabilities(): return created_response(data=local_editor.save(str(user.user_id), body)) _require_flag(str(user.user_id), db, "can_add_skill", "自助添加技能") @@ -333,7 +319,7 @@ def create_my_skill( ) db.commit() if body.icon is not None: - from core.services.skill_icon_service import set_skill_icon + from core.capabilities.skills.icons import set_skill_icon set_skill_icon(db, skill_id, body.icon) refresh_skill_caches() @@ -375,13 +361,13 @@ def get_my_skill( 与手写新建表单的「技能正文」一一对应，编辑保存走 ``POST /v1/me/skills`` upsert。 附带 ``extra_files`` 清单（文件名/大小/是否二进制），供技能文件管理 UI 渲染。 """ - if capabilities_enabled(): + if has_run_capabilities(): return success_response(data=local_editor.detail(str(user.user_id), skill_id)) _require_flag(str(user.user_id), db, "can_add_skill", "自助添加技能") row = _get_own_skill(db, str(user.user_id), skill_id) - from core.agent_skills.binary_files import is_binary_value - from core.services.skill_icon_service import get_skill_icon + from core.capabilities.skills.binary_files import is_binary_value + from core.capabilities.skills.icons import get_skill_icon extra_files = [ { @@ -415,7 +401,7 @@ def delete_my_skill( db: Session = Depends(get_db), ): """删除自己上传的私有技能。只能删自己的（owner 校验）。""" - if capabilities_enabled(): + if has_run_capabilities(): return success_response(data=local_editor.uninstall(str(user.user_id), skill_id)) _require_flag(str(user.user_id), db, "can_add_skill", "自助添加技能") row = ( @@ -431,13 +417,13 @@ def delete_my_skill( db.delete(row) db.commit() - from core.services.skill_icon_service import delete_skill_icon + from core.capabilities.skills.icons import delete_skill_icon delete_skill_icon(db, skill_id) - from core.agent_skills.config import purge_skill_sandbox_files + from core.capabilities.skills.config import purge_skill_sandbox_files # Desktop reconciliation retains immutable revisions and a removed tombstone. - if not capabilities_enabled(): + if not has_run_capabilities(): purge_skill_sandbox_files(skill_id) refresh_skill_caches() return success_response(data={"skill_id": skill_id, "deleted": True}) @@ -455,7 +441,7 @@ def set_my_skill_icon( db: Session = Depends(get_db), ): """给自己的私有技能设置图标（owner 校验）。空串=恢复默认。""" - if capabilities_enabled(): + if has_run_capabilities(): return success_response(data=local_editor.icon(str(user.user_id), skill_id, body.icon)) _require_flag(str(user.user_id), db, "can_add_skill", "自助添加技能") row = ( @@ -466,7 +452,7 @@ def set_my_skill_icon( if not row: raise ResourceNotFoundError("skill", skill_id) - from core.services.skill_icon_service import set_skill_icon + from core.capabilities.skills.icons import set_skill_icon icon = set_skill_icon(db, skill_id, body.icon) refresh_skill_caches() diff --git a/src/backend/api/routes/v1/me_skill_files.py b/src/backend/api/routes/v1/me_skill_files.py index c5aad0b7..eb819ad2 100644 --- a/src/backend/api/routes/v1/me_skill_files.py +++ b/src/backend/api/routes/v1/me_skill_files.py @@ -6,8 +6,8 @@ from datetime import datetime from typing import Dict, List, Optional -from core.capabilities.paths import capabilities_enabled -from core.services import local_skill_editor as local_editor +from core.harness.registry.run_capabilities import has_run_capabilities +from features.desktop.local.skills import editor as local_editor from core.auth.backend import UserContext, get_current_user from core.auth.capabilities import resolve_user_capabilities from core.config.settings import settings @@ -15,22 +15,8 @@ from core.db.models import AdminMcpServer, AdminSkill from core.infra.exceptions import AccessDeniedError, BadRequestError, ResourceNotFoundError from core.infra.responses import created_response, success_response -from core.services.mcp_management_service import ( - encrypt_mcp_headers, - probe_mcp_connectivity, - refresh_mcp_caches, - validate_remote_mcp_url, -) -from core.services.skill_management_service import ( - build_skill_content, - extract_instructions, - extract_mcp_server_ids, - parse_and_upsert_skill_zip, - refresh_skill_caches, - resolve_mcp_bindings, - resolve_ontology_workflows, - validate_skill_file_path, -) +from core.capabilities.mcp.management import encrypt_mcp_headers, probe_mcp_connectivity, refresh_mcp_caches, validate_remote_mcp_url +from core.capabilities.skills.management import build_skill_content, extract_instructions, extract_mcp_server_ids, parse_and_upsert_skill_zip, refresh_skill_caches, resolve_mcp_bindings, resolve_ontology_workflows, validate_skill_file_path from fastapi import APIRouter, Depends, File, Form, HTTPException, Response, UploadFile from pydantic import BaseModel, Field, field_validator from sqlalchemy.orm import Session @@ -56,12 +42,12 @@ def get_my_skill_file( db: Session = Depends(get_db), ): """读取自己私有技能中的单个附加文件。二进制文件只返回 ``is_binary=true`` 不回传内容。""" - if capabilities_enabled(): + if has_run_capabilities(): return success_response(data=local_editor.file_get(str(user.user_id), skill_id, filename)) _require_flag(str(user.user_id), db, "can_add_skill", "自助添加技能") row = _get_own_skill(db, str(user.user_id), skill_id) - from core.agent_skills.binary_files import is_binary_value + from core.capabilities.skills.binary_files import is_binary_value extra = row.extra_files or {} if filename not in extra: @@ -84,7 +70,7 @@ def save_my_skill_file( SKILL.md 不走本接口（正文在编辑表单里改，经 ``POST /v1/me/skills`` 重建）。 """ - if capabilities_enabled(): + if has_run_capabilities(): return success_response(data=local_editor.file_change(str(user.user_id), skill_id, filename, body.content.encode("utf-8"))) _require_flag(str(user.user_id), db, "can_add_skill", "自助添加技能") @@ -114,7 +100,7 @@ def delete_my_skill_file( db: Session = Depends(get_db), ): """删除自己私有技能中的单个附加文件。""" - if capabilities_enabled(): + if has_run_capabilities(): return success_response(data=local_editor.file_change(str(user.user_id), skill_id, filename)) _require_flag(str(user.user_id), db, "can_add_skill", "自助添加技能") row = _get_own_skill(db, str(user.user_id), skill_id) @@ -140,11 +126,11 @@ async def upload_my_skill_file( db: Session = Depends(get_db), ): """以 multipart 上传单个文件到自己的私有技能，二进制按 base64 标记安全存储。""" - if capabilities_enabled(): + if has_run_capabilities(): return success_response(data=local_editor.file_change(str(user.user_id), skill_id, path or file.filename or "", await file.read())) _require_flag(str(user.user_id), db, "can_add_skill", "自助添加技能") - from core.agent_skills.binary_files import encode_upload + from core.capabilities.skills.binary_files import encode_upload filename = validate_skill_file_path(path or file.filename or "") if filename == "SKILL.md": @@ -177,12 +163,12 @@ def export_my_skill( 导出布局与 zip 上传约定一致，可直接重新导入（备份/迁移/分享给管理员上架）。 """ - if capabilities_enabled(): + if has_run_capabilities(): return Response(content=local_editor.export(str(user.user_id), skill_id), media_type="application/zip") _require_flag(str(user.user_id), db, "can_add_skill", "自助添加技能") row = _get_own_skill(db, str(user.user_id), skill_id) - from core.services.marketplace_service import build_skill_zip + from features.marketplace.skill_archives import build_skill_zip data = build_skill_zip(skill_id, row.skill_content or "", row.extra_files or {}) safe_name = "".join(c if c.isalnum() or c in "._-" else "_" for c in skill_id) or "skill" diff --git a/src/backend/api/routes/v1/me_system.py b/src/backend/api/routes/v1/me_system.py index 5ffebf9a..c911f31b 100644 --- a/src/backend/api/routes/v1/me_system.py +++ b/src/backend/api/routes/v1/me_system.py @@ -22,12 +22,10 @@ from core.config.settings import settings from core.db.engine import get_db from core.infra.responses import success_response -from core.services.service_probes import ( - config_row_to_dict, - reinitialize_mcp_pool, - test_service_group, -) -from core.services.system_config import SystemConfigService +from core.infra.probes.service_probes import config_row_to_dict +from core.capabilities.mcp.service import McpServerConfigService +from core.infra.probes.service_probes import test_service_group +from core.config.store.system_config import SystemConfigService from fastapi import APIRouter, Depends, HTTPException from pydantic import BaseModel from sqlalchemy.orm import Session @@ -132,7 +130,7 @@ async def update_personal_configs( detail=f"配置项 {item.key} 不在个人可配置范围内", ) svc.bulk_set([item.model_dump() for item in body.items], updated_by="me_system") - asyncio.ensure_future(reinitialize_mcp_pool("me_system")) + McpServerConfigService.get_instance().invalidate_cache() return success_response(data={"updated": len(body.items)}) diff --git a/src/backend/api/routes/v1/memories.py b/src/backend/api/routes/v1/memories.py index 8cae31f7..cfb5f0ff 100644 --- a/src/backend/api/routes/v1/memories.py +++ b/src/backend/api/routes/v1/memories.py @@ -31,8 +31,8 @@ from core.memory.profile import delete_field as profile_delete_field from core.memory.profile import get as profile_get from core.memory.service import delete_all_memories, delete_memory, get_all_memories -from core.services import UserService -from core.services.memory_settings_service import MemorySettingsService +from core.auth.users import UserService +from core.memory.settings import MemorySettingsService from fastapi import APIRouter, Depends, Query from pydantic import BaseModel, Field from sqlalchemy.orm import Session @@ -68,7 +68,7 @@ class ProfileFieldRequest(BaseModel): def _is_reranker_available() -> bool: """Check if reranker endpoint is configured at the infra level.""" try: - from core.services.model_config import ModelConfigService + from core.models.config import ModelConfigService cfg = ModelConfigService.get_instance().resolve("reranker") if cfg and cfg.base_url and cfg.model_name: @@ -163,8 +163,8 @@ async def list_memories( scope_user_id = str(user.user_id) if project_id: - from core.services.project_scope import project_memory_policy - from core.auth.permissions_iface import resolve_project_access + from core.projects.edition import project_memory_policy + from core.auth.edition import resolve_project_access from core.db.models import Project project = db.query(Project).filter_by(project_id=project_id, deleted_at=None).first() @@ -291,7 +291,7 @@ async def remove_profile_field( if not ok: return error_response(code=50001, message="删除失败", status_code=500) if message_id: - from core.evolution.settlement_store import drop_entry + from core.evolution.settlement.settlement_store import drop_entry drop_entry(message_id, key) return success_response(data={"deleted": key}) @@ -311,7 +311,7 @@ async def remove_graph_relation( if not ok: return error_response(code=50001, message="删除失败", status_code=500) if message_id: - from core.evolution.settlement_store import drop_entry + from core.evolution.settlement.settlement_store import drop_entry drop_entry(message_id, relation_id) return success_response(data={"deleted": relation_id}) @@ -362,7 +362,7 @@ async def remove_memory( if not ok: return error_response(code=50001, message="删除失败", status_code=500) if message_id: - from core.evolution.settlement_store import drop_entry + from core.evolution.settlement.settlement_store import drop_entry drop_entry(message_id, memory_id) return success_response(data={"deleted": memory_id}) @@ -461,7 +461,7 @@ async def get_memory_graph( scope_user_id = str(user.user_id) workspace_id = "default" if project_id: - from core.services.project_scope import project_memory_policy + from core.projects.edition import project_memory_policy policy = project_memory_policy(db, project_id, scope_user_id) if policy is None: diff --git a/src/backend/api/routes/v1/mock_sso.py b/src/backend/api/routes/v1/mock_sso.py index 535270e7..1baf8598 100644 --- a/src/backend/api/routes/v1/mock_sso.py +++ b/src/backend/api/routes/v1/mock_sso.py @@ -15,7 +15,7 @@ from typing import Any, Dict, Optional from urllib.parse import parse_qsl, quote, urlencode, urlparse, urlunparse -from core.content.content_blocks import get_branding_info, is_register_allowed +from core.files.content.branding import get_branding_info, is_register_allowed from core.infra.logging import get_logger from fastapi import APIRouter, Form, Query, Request from fastapi.responses import HTMLResponse, RedirectResponse @@ -1453,7 +1453,7 @@ def _try_local_login(username: str, password: str) -> Optional[Dict[str, Any]]: try: from core.db.engine import SessionLocal - from core.services.local_user_service import LocalUserService + from core.auth.local_users import LocalUserService except Exception as exc: logger.warning("local auth unavailable: %s", exc) return None @@ -1611,7 +1611,7 @@ def _bounce(msg: str) -> RedirectResponse: try: from core.db.engine import SessionLocal - from core.services.local_user_service import LocalUserService + from core.auth.local_users import LocalUserService except Exception as exc: logger.warning("local auth module unavailable: %s", exc) return _bounce("注册服务暂不可用") diff --git a/src/backend/api/routes/v1/model_reasoning.py b/src/backend/api/routes/v1/model_reasoning.py index 9ac05b2a..e49fe898 100644 --- a/src/backend/api/routes/v1/model_reasoning.py +++ b/src/backend/api/routes/v1/model_reasoning.py @@ -6,9 +6,9 @@ from api.deps import require_system_settings from core.db.engine import get_db -from core.db.model_repository import get_provider +from core.models.repository import get_provider from core.infra.responses import success_response -from core.llm.providers.reasoning_probe import discover_reasoning_efforts +from core.models.providers.reasoning_probe import discover_reasoning_efforts router = APIRouter() diff --git a/src/backend/api/routes/v1/models.py b/src/backend/api/routes/v1/models.py index fb3ad90d..b2627ea0 100644 --- a/src/backend/api/routes/v1/models.py +++ b/src/backend/api/routes/v1/models.py @@ -30,26 +30,11 @@ from core.auth.backend import UserContext, require_auth from core.config.settings import settings from core.db.engine import get_db -from core.services.model_config import ModelConfigService -from core.db.model_repository import ( - ROLE_DEFINITIONS, - assign_role, - create_provider, - delete_provider, - export_all, - get_provider, - import_all, - list_providers, - list_role_assignments, - provider_has_capability, - provider_is_referenced, - role_required_capability, - set_provider_test_result, - unassign_role, - update_provider, -) +from core.models.config import ModelConfigService +from core.models.repository import ROLE_DEFINITIONS, assign_role, create_provider, delete_provider, export_all, get_provider, import_all, list_providers, list_role_assignments, provider_has_capability, provider_is_referenced, role_required_capability, set_provider_test_result, unassign_role, update_provider from core.infra.responses import success_response -from core.services.user_model_selection import list_user_selectable_models, user_can_switch_model +from core.auth.model_selection import user_can_switch_model +from core.models.selection import list_user_selectable_models router = APIRouter(prefix="/v1/models", tags=["Models"]) logger = logging.getLogger(__name__) @@ -329,7 +314,7 @@ async def _autofill_context_length( # cannot outlive the number it described. extra_config.pop("context_length_source", None) - from core.llm.providers.context_probe import discover_context_length + from core.models.providers.context_probe import discover_context_length result = await discover_context_length( provider=provider, @@ -378,7 +363,7 @@ async def _autofill_api_protocol( if not isinstance(extra_config, dict) or provider_type != "chat": return - from core.llm.providers.registry import get_spec + from core.models.providers.registry import get_spec if not get_spec(provider).speaks_responses: return @@ -387,7 +372,7 @@ async def _autofill_api_protocol( return extra_config.pop("api_protocol_source", None) - from core.llm.providers.protocol_probe import PROBE_VERSION, detect_api_protocol + from core.models.providers.protocol_probe import PROBE_VERSION, detect_api_protocol result = await detect_api_protocol( base_url=base_url, api_key=api_key, model_name=model_name @@ -472,8 +457,8 @@ async def _ping_via_model( """Native/litellm vendors: actually build the model and issue one minimal non-streaming call to verify connectivity.""" from agentscope.message import Msg, TextBlock - from core.llm.chat_models import make_chat_model - from core.llm.providers.registry import get_spec, split_provider_extra + from core.models.chat_models import make_chat_model + from core.models.providers.registry import get_spec, split_provider_extra spec = get_spec(provider) provider_extra = split_provider_extra(spec, extra or {}) @@ -515,12 +500,12 @@ async def _test_connection( extra_config: Optional[dict] = None, ) -> dict: """Test connectivity (dispatched by engine). Returns {success, latency_ms, error}.""" - from core.llm.providers.registry import get_spec + from core.models.providers.registry import get_spec spec = get_spec(provider) extra = extra_config or {} if provider_type == "audio": - from core.services.audio_transcription import test_audio_connection + from core.chat.audio_transcription import test_audio_connection return await test_audio_connection(base_url, api_key, model_name) if spec.engine == "openai": if spec.id == "azure_openai": @@ -540,19 +525,19 @@ def get_main_capabilities( ): """供前端展示模型相关入口；只暴露非敏感信息，不泄露 base_url/api_key。""" cfg = ModelConfigService.get_instance().resolve("main_agent") - from core.llm.reasoning_effort import reasoning_capabilities + from core.models.reasoning_effort import reasoning_capabilities effort_caps = reasoning_capabilities(cfg.extra if cfg else {}) switch_enabled = bool(user and user_can_switch_model(db, user.user_id)) # Real context window (tokens) of the main chat model, so the frontend # context-usage gauge reflects the model's true window instead of a guess. # 0 when unconfigured — the frontend falls back to a heuristic/default. main_context_length = int(cfg.context_length) if cfg and cfg.context_length else 0 - from core.content.artifact_reader import ATTACHMENT_PREVIEW_MAX_CHARS + from core.files.artifacts.reader import ATTACHMENT_PREVIEW_MAX_CHARS # Image reading: either the main model is natively multimodal, or a `vision` role # model is assigned and the vision bridge transcribes images into text evidence. # The frontend uses `can_read_image` alone to decide whether to offer image upload; # the two flags below only explain *how* it works. - from core.vision import model_supports_vision, resolve_vision_config + from core.models.vision import model_supports_vision, resolve_vision_config main_native_vision = model_supports_vision(cfg) vision_cfg = resolve_vision_config() @@ -586,7 +571,7 @@ async def list_provider_schemas(_: None = Depends(require_system_settings)): 供前端模型配置表单按所选厂商动态渲染。仅限管理员。 """ - from core.llm.providers.registry import to_frontend_schema + from core.models.providers.registry import to_frontend_schema return success_response(data=to_frontend_schema()) @@ -612,7 +597,7 @@ async def create_provider_endpoint( db: Session = Depends(get_db), ): """新增一个模型供应商。仅限管理员；保存前会校验厂商字段并做连通性预校验，失败则返回 400，成功后刷新模型配置缓存。""" - from core.llm.providers.registry import validate_payload + from core.models.providers.registry import validate_payload err = validate_payload(body.provider, body.provider_type, body.extra_config) if err: @@ -702,7 +687,7 @@ async def update_provider_endpoint( if any(ROLE_DEFINITIONS.get(role, {}).get("type") != new_type for role in roles): raise HTTPException(400, "请先取消该模型的角色分配，再修改模型类型") - from core.llm.providers.registry import validate_payload + from core.models.providers.registry import validate_payload err = validate_payload(new_provider, new_type, new_extra) if err: @@ -828,7 +813,7 @@ async def detect_context_length( 返回 `{context_length, source, source_label, confidence, detail, notes}`；`context_length=0` 表示没探到，`notes` 逐级说明每个来源看到了什么，便于管理员判断该手工填多少。 """ - from core.llm.providers.context_probe import discover_context_length + from core.models.providers.context_probe import discover_context_length api_key = body.api_key if not api_key and body.provider_id: @@ -947,7 +932,7 @@ def export_endpoint( async def _require_import_control(request: Request, db: Session = Depends(get_db)): - from core.services.desktop_capability import is_desktop_shell_control + from features.desktop.capability.service import is_desktop_shell_control if is_desktop_shell_control(request.headers.get("authorization", ""), request.headers.get("origin")): return await require_config(request=request, authorization=request.headers.get("authorization"), db=db) @@ -961,9 +946,9 @@ def import_endpoint( db: Session = Depends(get_db), ): """导入模型配置（供应商 + 角色分配）。仅限管理员（CONFIG_TOKEN / can_system_config，同 export）；overwrite=True 时覆盖同名条目，导入后刷新模型配置缓存。""" - from core.services.desktop_model_credentials import sanitize_import + from features.desktop.bridge.model_credentials import sanitize_import if any(str(p.get("api_key") or "").startswith(("dcap1.", "dcap2.", "desktop-capability:")) for p in body.providers): - from core.services.desktop_capability import is_desktop_shell_control + from features.desktop.capability.service import is_desktop_shell_control if not is_desktop_shell_control(request.headers.get("authorization", ""), request.headers.get("origin")): raise HTTPException(status_code=403, detail="desktop shell control required") try: diff --git a/src/backend/api/routes/v1/myspace_folders.py b/src/backend/api/routes/v1/myspace_folders.py index 0335cdac..11b62968 100644 --- a/src/backend/api/routes/v1/myspace_folders.py +++ b/src/backend/api/routes/v1/myspace_folders.py @@ -18,13 +18,13 @@ from typing import Optional from api.folder_batch_schemas import FolderBatchBody -from core.services.folder_batch import create_folder_batch +from core.files.folder_batch import create_folder_batch from core.db.models import UserFolder, UserShadow from core.auth.backend import UserContext, get_current_user from core.db.engine import get_db from core.infra.responses import success_response -from core.services.user_folder_service import UserFolderService +from core.projects.user_folders import UserFolderService from fastapi import APIRouter, Depends, HTTPException, Query from pydantic import BaseModel, Field from sqlalchemy.orm import Session diff --git a/src/backend/api/routes/v1/ontologies.py b/src/backend/api/routes/v1/ontologies.py index 18bd9cfb..0ba476c3 100644 --- a/src/backend/api/routes/v1/ontologies.py +++ b/src/backend/api/routes/v1/ontologies.py @@ -15,14 +15,9 @@ from core.db.models import OntologyDraft, OntologyEnforcementEvent, OntologyReviewRun from core.infra.exceptions import BadRequestError, ResourceNotFoundError from core.infra.responses import success_response -from core.services import UserService -from core.services.ontology_policy import ( - plugin_import_build_validation_forced, - require_ontology_validation_permission, - set_plugin_import_build_validation_forced, - user_can_use_ontology_validation, -) -from core.services.ontology_service import OntologyService +from core.auth.users import UserService +from core.ontology.policy import plugin_import_build_validation_forced, require_ontology_validation_permission, set_plugin_import_build_validation_forced, user_can_use_ontology_validation +from core.ontology.service import OntologyService from fastapi import APIRouter, Depends, Query from pydantic import BaseModel, Field from sqlalchemy import func @@ -182,7 +177,7 @@ def preview_ontology_runtime( settings = UserService(db).get_user_settings(str(user.user_id)) if not settings.get("ontology_enabled", False): return success_response(data={"enabled": False, "packs": [], "review_level": "none"}) - from core.services.ontology_service import build_user_ontology_runtime + from core.ontology.runtime_policy import build_user_ontology_runtime _, runtime = build_user_ontology_runtime( user_id=str(user.user_id), @@ -701,7 +696,7 @@ async def generate_ontology_evolution_drafts( body: OntologyEvolutionGenerateRequest, db: Session = Depends(get_db), ): - from core.services.ontology_evolution_service import OntologyEvolutionService + from core.ontology.evolution import OntologyEvolutionService result = await OntologyEvolutionService(db).generate_candidates( min_occurrences=body.min_occurrences, @@ -725,7 +720,7 @@ def materialize_ontology_draft( draft_id: str, db: Session = Depends(get_db), ): - from core.services.ontology_evolution_service import OntologyEvolutionService + from core.ontology.evolution import OntologyEvolutionService row = OntologyEvolutionService(db).materialize_approved_draft(draft_id) return success_response(data=_version_dict(row, include_content=True)) diff --git a/src/backend/api/routes/v1/plans.py b/src/backend/api/routes/v1/plans.py index ae9c2e62..0e66a6b5 100644 --- a/src/backend/api/routes/v1/plans.py +++ b/src/backend/api/routes/v1/plans.py @@ -2,22 +2,21 @@ from typing import Any, Dict, List, Optional -from fastapi import APIRouter, Depends, HTTPException -from pydantic import BaseModel, Field -from sqlalchemy.orm import Session - -from core.auth.backend import get_current_user, UserContext +import api.transports.run_sse as run_sse +import core.runs.runtime as runtime +import orchestration.plans.admission as admission +from core.auth.backend import UserContext, get_current_user from core.chat.context import generate_smart_title from core.db.engine import get_db -from core.infra.responses import success_response, sse_response -from core.services.plan_service import PlanService -from core.services.chat_service import ChatService -from core.services.user_model_selection import ( - UserModelSelectionError, - resolve_effective_chat_model_name, - resolve_user_model_provider_id, -) from core.infra.logging import get_logger +from core.infra.responses import sse_response, success_response +from core.chat.service import ChatService +from core.chat.plans import PlanService +from core.auth.model_selection import UserModelSelectionError, resolve_user_model_provider_id +from core.models.selection import resolve_effective_chat_model_name +from fastapi import APIRouter, Depends, HTTPException +from pydantic import BaseModel, Field +from sqlalchemy.orm import Session logger = get_logger(__name__) @@ -155,7 +154,7 @@ def _load_chat_history( # structured tool replay); long sessions get the compacted history rather than the full raw text. if chat_id: try: - from core.services.compaction_service import load_session_history + from core.chat.compaction.service import load_session_history history = load_session_history(ChatService(db), chat_id, user_id) if history: @@ -232,9 +231,9 @@ async def generate_plan( if req.attachments: uploaded_files = [a.model_dump() for a in req.attachments if a.file_id] - from orchestration import chat_run_executor + pass - run = await chat_run_executor.start_plan_generate_run( + run = await admission.start_plan_generate_run( chat_id=db_chat_id, user_id=user.user_id, task_description=req.task_description, @@ -249,7 +248,7 @@ async def generate_plan( ) return sse_response( - chat_run_executor.follow_run_as_sse( + run_sse.follow_run_as_sse( run.run_id, chat_id=db_chat_id, error_event_factory=lambda reason: {"type": "plan_error", "error": reason}, @@ -384,9 +383,9 @@ async def execute_plan( "[plan-execute] chat_id=%s, loaded %d history messages", req.chat_id, len(session_messages) ) - from orchestration import chat_run_executor + pass - run = await chat_run_executor.start_plan_execute_run( + run = await admission.start_plan_execute_run( plan_id=plan_id, chat_id=db_chat_id, user_id=user.user_id, @@ -399,7 +398,7 @@ async def execute_plan( ) return sse_response( - chat_run_executor.follow_run_as_sse( + run_sse.follow_run_as_sse( run.run_id, chat_id=db_chat_id, error_event_factory=lambda reason: { @@ -428,13 +427,13 @@ async def cancel_plan( svc.update_plan(plan_id, status="cancelled") # Also cancel the background run in lockstep (only exists during the plan execute phase) - from orchestration import chat_run_executor + pass - run = chat_run_executor.get_run_by_plan_id(plan_id) + run = admission.get_run_by_plan_id(plan_id) if run is not None: try: - await chat_run_executor.cancel_run(run.run_id, user_id=user.user_id) - except chat_run_executor.ChatRunPermissionDenied: + await runtime.cancel_run(run.run_id, user_id=user.user_id) + except runtime.ChatRunPermissionDenied: pass # the plan was already authorized; a run auth failure means user_id inference went wrong — do not block except Exception as exc: logger.warning( diff --git a/src/backend/api/routes/v1/plugin_resources.py b/src/backend/api/routes/v1/plugin_resources.py index 623a08a5..f223205a 100644 --- a/src/backend/api/routes/v1/plugin_resources.py +++ b/src/backend/api/routes/v1/plugin_resources.py @@ -7,10 +7,10 @@ from core.auth.backend import UserContext, get_current_user from core.db.engine import get_db from core.infra.responses import success_response -from core.plugins.resources import service, store -from core.plugins.resources.stream import connect +from core.capabilities.plugins.resources import service, store +from core.capabilities.plugins.resources.stream import connect -from core.plugins.resources.lifecycle import lifespan +from core.capabilities.plugins.resources.lifecycle import lifespan router = APIRouter(prefix="/v1", tags=["Plugin resources"], lifespan=lifespan) class OpenRequest(BaseModel): @@ -29,7 +29,7 @@ class ToolRequest(OpenRequest): @router.get("/plugins/{slug}/checkpoints") def list_checkpoints(slug: str, install_id: str | None = None, user: UserContext = Depends(get_current_user), db: Session = Depends(get_db)): - from core.plugins.resources.installation import resolve + from core.capabilities.plugins.resources.installation import resolve selected = resolve(db, slug, str(user.user_id), install_id) rows = db.query(store.PluginResourceCheckpoint).filter_by(user_id=str(user.user_id), install_id=selected.install_id).all() return success_response(data={"items": [{"checkpoint_id": row.checkpoint_id, "name": row.name} for row in rows]}) @@ -83,38 +83,38 @@ def attach_resource(resource_id: str, request: Request, user: UserContext = Depe parsed = urlsplit(origin) if parsed.scheme not in {"http", "https"} or not parsed.netloc: raise HTTPException(403, "invalid_origin") - from core.plugins.resources.authentication import capture + from core.capabilities.plugins.resources.authentication import capture return success_response(data={"ticket": store.ticket(db, row, origin, capture(request))}) @router.post("/plugin-resources/{resource_id}/asset-ticket") def issue_asset_ticket(resource_id: str, request: Request, user: UserContext = Depends(get_current_user), db: Session = Depends(get_db)): row = service.authorized(db, resource_id, str(user.user_id)) - from core.plugins.resources.installation import resolve - from core.plugins.ui.contract import find_module + from core.capabilities.plugins.resources.installation import resolve + from features.workbench.ui.contract import find_module if row.application_key is not None: - from core.plugins.resources.owners import resolve_installation + from core.capabilities.plugins.resources.owners import resolve_installation selected = resolve_installation(db, row, str(user.user_id)) else: selected = resolve(db, row.slug, str(user.user_id), row.install_id) module = find_module(selected.ui, row.module_id) if module is None: raise HTTPException(404, "resource_module_unavailable") - from core.plugins.resources.authentication import capture + from core.capabilities.plugins.resources.authentication import capture return success_response(data={"ticket": store.asset_ticket(db, row, capture(request)), "entry": module["entry"]}) @router.get("/plugin-resource-assets/{execution_scope}/{ticket}/{asset_path:path}") async def resource_asset(execution_scope: str, ticket: str, asset_path: str, db: Session = Depends(get_db)): # Null-origin iframe subresources cannot carry SameSite login cookies. # The ticket grants static package reads only; its saved session is still revocable. - from .plugin_ui import web_asset_response - from core.plugins.resources.authentication import validate + from features.workbench.api.plugin_ui import web_asset_response + from core.capabilities.plugins.resources.authentication import validate resource_id, user_id, authentication = store.resolve_asset_ticket(db, ticket) await validate(authentication, user_id, db) row = service.authorized(db, resource_id, user_id) if row.scope != execution_scope: raise HTTPException(409, "execution_scope_mismatch") if row.application_key is not None: - from core.plugins.resources.owners import resolve_installation + from core.capabilities.plugins.resources.owners import resolve_installation selected = resolve_installation(db, row, user_id) response = web_asset_response(db, row.slug, user_id, asset_path, row.install_id, _installation=selected) @@ -136,7 +136,7 @@ async def close_resource(resource_id: str, user: UserContext = Depends(get_curre @router.post("/plugin-resources/{resource_id}/downloads/{download_id}/retain") async def retain_download(resource_id: str, download_id: str, user: UserContext = Depends(get_current_user), db: Session = Depends(get_db)): - from core.plugins.resources.artifacts import retain + from core.capabilities.plugins.resources.artifacts import retain row = service.authorized(db, resource_id, str(user.user_id)) return success_response(data=await retain(row, download_id)) @@ -166,9 +166,9 @@ async def content(): @router.post("/internal/plugin-resources/tool") async def tool_resource(body: ToolRequest, request: Request, db: Session = Depends(get_db)): - from .internal_site_auth import _check_internal_token + from features.sites.api.internal_site_auth import _check_internal_token _check_internal_token(request.headers.get("x-internal-token")) - from core.llm.mcp_invocation import verify + from core.llm.mcp.invocation import verify from extensions.mcp_servers._ports import PORTS audience = request.headers.get("x-hugagent-mcp-audience", "") if audience not in PORTS or request.headers.get("x-chat-id") != body.chat_id: @@ -198,7 +198,7 @@ async def tool_resource(body: ToolRequest, request: Request, db: Session = Depen return success_response(data={"closed": True}) if body.action == "retain_download": await service.command(row, {"id": uuid.uuid4().hex, "action": "state", "params": {}}, actor="agent") - from core.plugins.resources.artifacts import retain + from core.capabilities.plugins.resources.artifacts import retain return success_response(data=await retain(row, str(body.params["download_id"]))) data = await service.command(row, {"id": body.command_id, "action": body.action, "params": body.params}, actor="agent") return success_response(data=data) diff --git a/src/backend/api/routes/v1/plugins.py b/src/backend/api/routes/v1/plugins.py index 542515ce..f58ed78e 100644 --- a/src/backend/api/routes/v1/plugins.py +++ b/src/backend/api/routes/v1/plugins.py @@ -25,6 +25,7 @@ """ from __future__ import annotations +from features.marketplace import plugin_market import json import logging @@ -40,9 +41,9 @@ from core.db.engine import get_db from core.infra.exceptions import AccessDeniedError, BadRequestError, ResourceNotFoundError from core.infra.responses import created_response, success_response -from core.services import marketplace_listing as ml -from core.plugins import management as ps -from core.capabilities.paths import capabilities_enabled +from features.marketplace import edition as ml +from core.capabilities.plugins import management as ps +from core.harness.registry.run_capabilities import has_run_capabilities router = APIRouter(prefix="/v1/plugins", tags=["Plugin"]) logger = logging.getLogger(__name__) @@ -72,7 +73,7 @@ def list_plugins( user: UserContext = Depends(get_current_user), db: Session = Depends(get_db), ): - items = ps.list_plugins(db, owner_user_id=str(user.user_id)) + items = plugin_market.list_plugins(db, owner_user_id=str(user.user_id)) return success_response(data={"items": items}) @@ -81,7 +82,7 @@ def list_installed( user: UserContext = Depends(get_current_user), db: Session = Depends(get_db), ): - from core.capabilities import device_catalog + from features.desktop.capabilities import device_catalog items = ps.list_installed(db, owner_user_id=str(user.user_id), include_global=True) # 云端账号装的插件登记在本机登记表里、业务库没有行，要投影进来才看得到。按 slug @@ -100,7 +101,7 @@ def get_installed_detail( ): """已安装插件的完整详情：技能（含指令/文件清单）+ MCP（含工具列表）。""" # 云端同步来的插件在本机业务库里查不到，走登记表那份，否则详情页直接 404。 - from core.capabilities import device_catalog + from features.desktop.capabilities import device_catalog projected = device_catalog.plugin_detail(install_id, user_id=str(user.user_id)) if projected is not None: @@ -117,21 +118,21 @@ async def get_feishu_app_status(_: str = Depends(require_system_settings)): CE does not ship the Config console, so the system-settings capability is the shared authorization seam for both CE instance admins and EE admins. """ - from core.services.lark_service import LarkService + from integrations.lark.service import LarkService return success_response(data=LarkService(None).app_status()) @router.post("/feishu-cli/app/init", summary="从飞书插件页发起共享应用初始化") async def init_feishu_app(_: str = Depends(require_system_settings)): - from core.services.lark_service import LarkService + from integrations.lark.service import LarkService return success_response(data=await LarkService(None).start_app_init()) @router.post("/feishu-cli/app/reset", summary="从飞书插件页重置共享应用") async def reset_feishu_app(_: str = Depends(require_system_settings)): - from core.services.lark_service import LarkService + from integrations.lark.service import LarkService return success_response(data=await LarkService(None).reset_app()) @@ -162,8 +163,8 @@ def install_plugin( # Both installing from and importing into the plugin marketplace require can_import_plugin # (same as the skill marketplace's can_add_skill); granted per user via the Config backend # "User Management -> Permission Config". Admin global install goes through admin_plugins. - if capabilities_enabled(): - from core.services.local_market_install import install_plugin as install_local + if has_run_capabilities(): + from features.desktop.local.market_install import install_plugin as install_local return created_response(data=install_local(db, str(user.user_id), slug, body.secrets)) _require_can_import_plugin(str(user.user_id), db) @@ -186,8 +187,8 @@ async def import_plugin( db: Session = Depends(get_db), ): """上传插件 zip 并导入（私有）。返回 import_report（imported / adapted / dropped）。""" - if capabilities_enabled(): - from core.services.local_skill_editor import install_archive + if has_run_capabilities(): + from features.desktop.local.skills.editor import install_archive if _parse_secrets(secrets): raise BadRequestError(message="本机插件凭据请通过连接配置设置，不写入插件包") @@ -214,7 +215,7 @@ def uninstall_plugin( db: Session = Depends(get_db), ): if install_id.startswith("plugin:local:"): - from core.plugins.local import service as local + from features.desktop.local.plugins import service as local try: current = local.get(str(user.user_id), install_id) @@ -241,8 +242,8 @@ def set_installed_meta( ): """展示信息（名称/分类/图标）是界面配置：用户自己导入/安装的私有插件由用户在此 配置；管理员全局插件走 admin 接口，普通用户不可改。""" - if capabilities_enabled() and install_id.startswith("plugin:local:"): - from core.plugins.local.service import set_presentation + if has_run_capabilities() and install_id.startswith("plugin:local:"): + from features.desktop.local.plugins.service import set_presentation return success_response( data=set_presentation(str(user.user_id), install_id, body.model_dump(exclude_none=True)) @@ -271,7 +272,7 @@ def set_enabled( db: Session = Depends(get_db), ): # 云端同步来的插件登记在本机登记表里，业务库没有行——写用户覆盖既不生效也读不回来。 - from core.capabilities import device_catalog + from features.desktop.capabilities import device_catalog if device_catalog.set_plugin_enabled(install_id, body.enabled, user_id=str(user.user_id)): return success_response(data={"install_id": install_id, "enabled": body.enabled}) diff --git a/src/backend/api/routes/v1/project_api_keys.py b/src/backend/api/routes/v1/project_api_keys.py index d7936518..cf137f00 100644 --- a/src/backend/api/routes/v1/project_api_keys.py +++ b/src/backend/api/routes/v1/project_api_keys.py @@ -10,8 +10,8 @@ from core.auth.capabilities import resolve_user_capabilities from core.db.engine import get_db from core.infra.responses import created_response, success_response -from core.services.project_api_service import require_project_key_access -from core.services.api_key_service import ApiKeyService +from core.projects.api import require_project_key_access +from core.auth.api_keys import ApiKeyService router = APIRouter(prefix="/v1/projects", tags=["Project API Keys"]) _ALLOWED_EXPIRY_DAYS = {7, 30, 90, 180, 365} @@ -142,7 +142,7 @@ async def toggle_key( row = ApiKeyService(db).set_enabled(user.user_id, key_id, body.enabled, project_id=project_id) data = _dto(row) if not body.enabled: - from core.services.api_key_runs import stop_cancelled_key_workers + from core.auth.api_key_runs import stop_cancelled_key_workers await stop_cancelled_key_workers(db, key_id, user.user_id) return success_response(data=data) @@ -157,7 +157,7 @@ async def revoke_key( ): _find(db, user, project_id, key_id) ApiKeyService(db).revoke_key(user.user_id, key_id, project_id=project_id) - from core.services.api_key_runs import stop_cancelled_key_workers + from core.auth.api_key_runs import stop_cancelled_key_workers await stop_cancelled_key_workers(db, key_id, user.user_id) return success_response(data={"id": key_id, "revoked": True}) diff --git a/src/backend/api/routes/v1/project_environments.py b/src/backend/api/routes/v1/project_environments.py index fca30b10..ff05fc95 100644 --- a/src/backend/api/routes/v1/project_environments.py +++ b/src/backend/api/routes/v1/project_environments.py @@ -1,12 +1,12 @@ """Project environment configuration and authorized resource options.""" from core.auth.backend import UserContext, get_current_user -from core.auth.permissions_iface import ProjectAccess, require_project_access +from core.auth.edition import ProjectAccess, require_project_access from core.db.engine import get_db from core.infra.responses import success_response -from core.project_environment.config import ProjectEnvironment -from core.services.project_environment import ProjectEnvironmentService -from core.services.project_environment_resources import environment_resources +from core.projects.environment.config import ProjectEnvironment +from core.projects.environment.service import ProjectEnvironmentService +from core.projects.environment_resources import environment_resources from fastapi import APIRouter, Depends from sqlalchemy.orm import Session diff --git a/src/backend/api/routes/v1/projects.py b/src/backend/api/routes/v1/projects.py index ef1a2e58..552b5db2 100644 --- a/src/backend/api/routes/v1/projects.py +++ b/src/backend/api/routes/v1/projects.py @@ -26,16 +26,16 @@ from urllib.parse import quote from core.auth.backend import UserContext, get_current_user -from core.auth.permissions_iface import ProjectAccess, require_project_access +from core.auth.edition import ProjectAccess, require_project_access from core.config.local_mode import local_mode_enabled from core.db.engine import get_db from core.db.models import Artifact from core.infra.responses import created_response, paginated_response, success_response from core.infra.time import utc_now -from core.project_environment.config import ProjectEnvironment -from core.services.project_environment import ProjectEnvironmentService -from core.services.project_file_service import ProjectFileService -from core.services.project_service import ProjectService +from core.projects.environment.config import ProjectEnvironment +from core.projects.environment.service import ProjectEnvironmentService +from core.projects.files import ProjectFileService +from core.projects.edition import ProjectService from fastapi import ( APIRouter, BackgroundTasks, diff --git a/src/backend/api/routes/v1/prompt_management.py b/src/backend/api/routes/v1/prompt_management.py index cb60e554..f04cf467 100644 --- a/src/backend/api/routes/v1/prompt_management.py +++ b/src/backend/api/routes/v1/prompt_management.py @@ -6,6 +6,7 @@ """ from __future__ import annotations +from core.llm import prompt_kinds as prompt_kinds_module, prompt_rendering as prompt_rendering_module, prompt_storage as prompt_storage_module, prompt_versions as prompt_versions_module import logging from pathlib import Path @@ -14,8 +15,7 @@ from api.deps import require_system_settings as require_config from core.db.engine import get_db from core.infra.responses import success_response -from core.services import prompt_version_service as pvs -from core.services.system_config import code_capability_enabled +from core.config.store.execution_options import code_capability_enabled from fastapi import APIRouter, Depends, HTTPException from prompts.prompt_config import load_prompt_config from prompts.provider import _FILE_CONTENT_CACHE, FilesystemPromptProvider @@ -76,8 +76,8 @@ def _load_fs_parts() -> Dict[str, Dict[str, Any]]: def _require_active_system_version(db: Session) -> Dict[str, Any]: """Return the active 'system' version, seeding defaults if the pool is empty.""" - pvs.seed_from_filesystem(db=db) - active = pvs.get_active_version("system", db=db) + prompt_versions_module.seed_from_filesystem(db=db) + active = prompt_versions_module.get_active_version("system", db=db) if not active: raise HTTPException(status_code=500, detail="no active system prompt version configured") return active @@ -85,7 +85,7 @@ def _require_active_system_version(db: Session) -> Dict[str, Any]: def _persist_active_system(version: Dict[str, Any], db: Session) -> Dict[str, Any]: """Save updated parts/metadata back into the active system version.""" - saved = pvs.upsert_version( + saved = prompt_versions_module.upsert_version( "system", version["id"], name=version.get("name"), @@ -199,8 +199,8 @@ async def _runtime_prompt_preview(db: Session, *, approval_mode: Optional[str] = + tool_appendix ) - active = pvs.get_active_version("system", db=db) - code_active = pvs.get_active_version("code_exec", db=db) + active = prompt_versions_module.get_active_version("system", db=db) + code_active = prompt_versions_module.get_active_version("code_exec", db=db) enabled_parts = len( [ p @@ -208,7 +208,7 @@ async def _runtime_prompt_preview(db: Session, *, approval_mode: Optional[str] = if p.get("is_enabled", True) and (p.get("content") or "").strip() ] ) - code_seg = pvs.render_kind_segment("code_exec", db) if code_capability_enabled() else "" + code_seg = prompt_rendering_module.render_kind_segment("code_exec", db) if code_capability_enabled() else "" part_count = ( enabled_parts + (1 if code_seg else 0) @@ -243,7 +243,7 @@ async def _runtime_prompt_preview(db: Session, *, approval_mode: Optional[str] = finally: if clients: try: - from core.llm.mcp_manager import close_clients + from core.llm.mcp.manager import close_clients await close_clients(clients) except Exception as exc: @@ -279,12 +279,12 @@ def list_parts(db: Session = Depends(get_db)): @router.get("/active", dependencies=[Depends(require_config)], summary="激活版本概览") def get_active_info(db: Session = Depends(get_db)): """返回各 kind 当前激活提示词版本的概要（id、名称、分段数、更新时间）。""" - pvs.seed_from_filesystem(db=db) - payload = pvs._load_payload(db) # noqa: SLF001 + prompt_versions_module.seed_from_filesystem(db=db) + payload = prompt_storage_module.load_payload(db) # noqa: SLF001 active = payload.get("active") or {} result = {} for kind, vid in active.items(): - v = pvs.get_version(kind, vid, db=db) if vid else None + v = prompt_versions_module.get_version(kind, vid, db=db) if vid else None result[kind] = { "id": vid, "name": (v or {}).get("name"), @@ -478,7 +478,7 @@ def _finalize(prompt: str, part_count: int, base: Dict[str, Any]): seg = "" if code_capability_enabled(): try: - seg = pvs.render_kind_segment("code_exec", db) or "" + seg = prompt_rendering_module.render_kind_segment("code_exec", db) or "" except Exception: seg = "" appended = bool(seg) @@ -509,7 +509,7 @@ def _finalize(prompt: str, part_count: int, base: Dict[str, Any]): return success_response(data=base) # Prefer the active version from the prompt_version pool - active = pvs.get_active_version("system", db=db) + active = prompt_versions_module.get_active_version("system", db=db) if active and active.get("parts"): chunks: List[str] = [] for p in active["parts"]: @@ -565,15 +565,15 @@ class KindCreateRequest(BaseModel): @router.get("/kinds", dependencies=[Depends(require_config)], summary="提示词 kind（tab）清单") def list_prompt_kinds(db: Session = Depends(get_db)): """内置 + 管理员自建的 kind。「模式选择」页绑定提示词时也读这份清单。""" - pvs.seed_from_filesystem(db=db) - return success_response(data={"kinds": pvs.all_kinds(db)}) + prompt_versions_module.seed_from_filesystem(db=db) + return success_response(data={"kinds": prompt_kinds_module.all_kinds(db)}) @router.post("/kinds", dependencies=[Depends(require_config)], summary="新建提示词 tab") def create_prompt_kind(req: KindCreateRequest, db: Session = Depends(get_db)): """新开一个 tab，并给它建好可直接编辑的空 default 版本。""" try: - created = pvs.create_custom_kind(req.key, req.label, db=db) + created = prompt_kinds_module.create_custom_kind(req.key, req.label, db=db) except ValueError as exc: raise HTTPException(status_code=400, detail=str(exc)) return success_response(data=created) @@ -586,7 +586,7 @@ def delete_prompt_kind(key: str, db: Session = Depends(get_db)): 绑了它的对话模式会退回默认提示词装配，不会让对话起不来。 """ try: - ok = pvs.delete_custom_kind(key, db=db) + ok = prompt_kinds_module.delete_custom_kind(key, db=db) except ValueError as exc: raise HTTPException(status_code=400, detail=str(exc)) if not ok: @@ -598,9 +598,9 @@ def delete_prompt_kind(key: str, db: Session = Depends(get_db)): def list_pool_versions(kind: Optional[str] = None, db: Session = Depends(get_db)): """列出提示词版本池中的所有版本，可按 kind 过滤；并返回各 kind 的激活映射。""" # Auto-seed defaults on first read so the pool is never empty after boot - pvs.seed_from_filesystem(db=db) - items = pvs.list_versions(kind=kind, db=db) - payload = pvs._load_payload(db) # noqa: SLF001 — access for active map + prompt_versions_module.seed_from_filesystem(db=db) + items = prompt_versions_module.list_versions(kind=kind, db=db) + payload = prompt_storage_module.load_payload(db) # noqa: SLF001 — access for active map return success_response( data={ "versions": items, @@ -616,7 +616,7 @@ def list_pool_versions(kind: Optional[str] = None, db: Session = Depends(get_db) ) def get_pool_version(kind: str, version_id: str, db: Session = Depends(get_db)): """获取版本池中指定 kind/version_id 版本的完整内容。""" - v = pvs.get_version(kind, version_id, db=db) + v = prompt_versions_module.get_version(kind, version_id, db=db) if v is None: raise HTTPException(status_code=404, detail=f"version not found: {kind}/{version_id}") return success_response(data=v) @@ -628,11 +628,11 @@ def create_pool_version(req: VersionUpsertRequest, db: Session = Depends(get_db) if not req.id: raise HTTPException(status_code=400, detail="id is required when creating a version") # Reject duplicates explicitly - if pvs.get_version(req.kind, req.id, db=db) is not None: + if prompt_versions_module.get_version(req.kind, req.id, db=db) is not None: raise HTTPException(status_code=409, detail=f"version already exists: {req.kind}/{req.id}") parts = [p.model_dump() for p in (req.parts or [])] if req.parts is not None else None try: - saved = pvs.upsert_version( + saved = prompt_versions_module.upsert_version( req.kind, req.id, name=req.name, @@ -659,10 +659,10 @@ def update_pool_version( ): """更新版本池中指定版本的元数据与分段；路径中的 kind/version_id 为准，更新后失效提示词缓存。""" # kind/version_id in path are authoritative; body fields override metadata - if pvs.get_version(kind, version_id, db=db) is None: + if prompt_versions_module.get_version(kind, version_id, db=db) is None: raise HTTPException(status_code=404, detail=f"version not found: {kind}/{version_id}") parts = [p.model_dump() for p in (req.parts or [])] if req.parts is not None else None - saved = pvs.upsert_version( + saved = prompt_versions_module.upsert_version( kind, version_id, name=req.name, @@ -688,7 +688,7 @@ def update_pool_version( def delete_pool_version(kind: str, version_id: str, db: Session = Depends(get_db)): """从版本池中删除指定版本（激活中的版本不可删除）。""" try: - pvs.delete_version(kind, version_id, db=db) + prompt_versions_module.delete_version(kind, version_id, db=db) except KeyError: raise HTTPException(status_code=404, detail=f"version not found: {kind}/{version_id}") except ValueError as exc: @@ -704,7 +704,7 @@ def delete_pool_version(kind: str, version_id: str, db: Session = Depends(get_db def activate_pool_version(kind: str, version_id: str, db: Session = Depends(get_db)): """将指定版本设为该 kind 的激活版本，并失效提示词缓存使其立即生效。""" try: - pvs.activate_version(kind, version_id, db=db) + prompt_versions_module.activate_version(kind, version_id, db=db) except KeyError: raise HTTPException(status_code=404, detail=f"version not found: {kind}/{version_id}") except ValueError as exc: @@ -717,7 +717,7 @@ def activate_pool_version(kind: str, version_id: str, db: Session = Depends(get_ @router.post("/versions/seed", dependencies=[Depends(require_config)], summary="初始化版本池") def seed_pool(db: Session = Depends(get_db)): """从文件系统默认提示词初始化版本池。""" - result = pvs.seed_from_filesystem(db=db) + result = prompt_versions_module.seed_from_filesystem(db=db) _invalidate_prompt_cache() return success_response(data=result) @@ -734,7 +734,7 @@ async def desktop_preview(request: DesktopPreviewRequest, _=Depends(require_conf from prompts.desktop_templates import desktop_version, render_desktop_part from prompts.desktop_workspace import SKIP_PARTS from prompts.prompt_runtime import build_system_prompt - version = pvs.get_version("desktop", request.version_id, db=db) + version = prompt_versions_module.get_version("desktop", request.version_id, db=db) if version is None: raise HTTPException(404, "desktop prompt version not found") if request.parts is not None: @@ -754,7 +754,7 @@ async def desktop_preview(request: DesktopPreviewRequest, _=Depends(require_conf )) or " " prompt = build_system_prompt(load_prompt_config(), ctx, desktop_preview=True) if code_capability_enabled(): - parts = pvs.effective_parts("code_exec", db=db) + parts = prompt_rendering_module.effective_parts("code_exec", db=db) prompt += "\n\n" + "\n\n".join(p["content"] for p in sorted(parts, key=lambda p: p.get("sort_order", 0)) if p.get("is_enabled", True) and p.get("part_id") not in SKIP_PARTS["code_exec"]) from core.sandbox.workspace.paths import WORKSPACE @@ -772,17 +772,17 @@ async def desktop_preview(request: DesktopPreviewRequest, _=Depends(require_conf @router.get("/snapshot", dependencies=[Depends(require_config)]) def export_snapshot(db: Session = Depends(get_db)): - from core.content.content_blocks import build_prompt_snapshot + from core.files.content.content_blocks import build_prompt_snapshot return success_response(build_prompt_snapshot(db)) @router.post("/snapshot", dependencies=[Depends(require_config)]) def import_snapshot(body: Dict[str, Any], db: Session = Depends(get_db)): - from core.content.content_blocks import import_prompt_snapshot, ContentSnapshotError + from core.files.content.content_blocks import import_prompt_snapshot, ContentSnapshotError try: result = import_prompt_snapshot(db, body, overwrite=True, default_updated_by="prompt_management") except ContentSnapshotError as exc: raise HTTPException(400, str(exc)) from exc - pvs.invalidate_cache() + prompt_storage_module.invalidate_cache() _invalidate_prompt_cache() return success_response(result) diff --git a/src/backend/api/routes/v1/public_chats.py b/src/backend/api/routes/v1/public_chats.py index f9d82f1e..c5fe0e0a 100644 --- a/src/backend/api/routes/v1/public_chats.py +++ b/src/backend/api/routes/v1/public_chats.py @@ -5,7 +5,7 @@ from core.db.engine import get_db from core.db.models import ChatSession, Project, UserAgent from core.infra.responses import success_response -from core.services import public_chat_service as service +from core.chat import public as service from fastapi import APIRouter, Depends, HTTPException, Response from pydantic import BaseModel from sqlalchemy.orm import Session @@ -54,7 +54,7 @@ async def toggle_link( data = service.set_enabled(db, user.user_id, kind, resource_id, body.enabled) if not body.enabled: from core.db.models import ContentBlock - from core.services.api_key_runs import stop_cancelled_key_workers + from core.auth.api_key_runs import stop_cancelled_key_workers row = db.get(ContentBlock, service.link_id(user.user_id, kind, resource_id)) await stop_cancelled_key_workers(db, row.payload["key_id"], user.user_id) @@ -75,7 +75,7 @@ def bootstrap( ) if not resource: raise HTTPException(404, detail="Resource unavailable") - from core.services.session_operations.sessions import _session_to_dict + from core.chat.operations.sessions import session_to_dict as _session_to_dict scope_name = "agent_api_scope" if user.api_key_agent_id else "project_api_scope" # JSON extraction is supported by both SQLite and Postgres. Bound the result, diff --git a/src/backend/api/routes/v1/space_files.py b/src/backend/api/routes/v1/space_files.py index a31e9c84..80a8e30b 100644 --- a/src/backend/api/routes/v1/space_files.py +++ b/src/backend/api/routes/v1/space_files.py @@ -10,10 +10,10 @@ from core.auth.backend import UserContext, get_current_user from core.db.engine import get_db from core.infra.responses import success_response -from core.spaces.access import resolve_grants -from core.spaces.mounts import mount_revision -from core.spaces.files import MAX_TRANSFER_BYTES, SpaceFiles -from core.storage import get_storage +from core.files.spaces.access import resolve_grants +from core.files.spaces.mounts import mount_revision +from core.files.spaces.files import MAX_TRANSFER_BYTES, SpaceFiles +from core.files.storage import get_storage router = APIRouter(prefix="/v1/myspace", tags=["Space Files"]) @@ -80,7 +80,7 @@ def delete(space: str, path: str, directory: bool = False, files: SpaceFiles = D @router.post("/references") def reference(body: PathBody, files: SpaceFiles = Depends(service)): - from core.artifacts.store import save_artifact_bytes + from core.files.artifacts.store import save_artifact_bytes data, info = files.read(body.space, body.path) mime = mimetypes.guess_type(info.name)[0] or "application/octet-stream" extension = info.name.rpartition(".")[2] if "." in info.name else "" @@ -95,12 +95,12 @@ def reference(body: PathBody, files: SpaceFiles = Depends(service)): @router.get("/text") def read_text(space: str, path: str, response: Response, files: SpaceFiles = Depends(service)): - from core.spaces.text import SpaceText + from core.files.spaces.text import SpaceText response.headers["Cache-Control"] = "no-store" return success_response(data=SpaceText(files).read(space, path)) @router.put("/text") def save_text(body: TextSaveRequest, space: str, path: str, files: SpaceFiles = Depends(service)): - from core.spaces.text import SpaceText + from core.files.spaces.text import SpaceText return success_response(data=SpaceText(files).save(space, path, body.content, body.revision)) diff --git a/src/backend/api/routes/v1/space_previews.py b/src/backend/api/routes/v1/space_previews.py index af9b673e..2cf49a61 100644 --- a/src/backend/api/routes/v1/space_previews.py +++ b/src/backend/api/routes/v1/space_previews.py @@ -7,9 +7,9 @@ from urllib.parse import quote from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Response -from core.content.archive_browser import ArchiveError, inspect_archive, read_archive_member +from core.files.content.archive_browser import ArchiveError, inspect_archive, read_archive_member from core.infra.responses import success_response -from core.spaces.files import SpaceFiles +from core.files.spaces.files import SpaceFiles from api.routes.v1.space_files import service router = APIRouter(prefix="/v1/myspace/content") diff --git a/src/backend/api/routes/v1/summary.py b/src/backend/api/routes/v1/summary.py index 238626d8..c898ffd9 100644 --- a/src/backend/api/routes/v1/summary.py +++ b/src/backend/api/routes/v1/summary.py @@ -6,7 +6,7 @@ from core.auth.backend import get_current_user, UserContext from core.infra.responses import success_response -from core.llm.summarizer import get_summarizer +from core.models.completions.summarizer import get_summarizer from core.infra.exceptions import BadRequestError router = APIRouter(prefix="/v1/summary", tags=["Summary"]) diff --git a/src/backend/api/routes/v1/tool_approval.py b/src/backend/api/routes/v1/tool_approval.py index 35ca66ea..4d6c15d0 100644 --- a/src/backend/api/routes/v1/tool_approval.py +++ b/src/backend/api/routes/v1/tool_approval.py @@ -5,7 +5,7 @@ ``core.llm.tool_permissions`` 决定要不要为工具调用停下来问用户。 网页端与桌面端共用这一档——桌面端不再另存一份「本机操作权限档」，本机执行策略由 -``core.services.local_grant_service.policy_for_gate()`` 按本档位翻译。桌面端仍有的 +``features.desktop.local.grants.policy_for_gate()`` 按本档位翻译。桌面端仍有的 「授权目录 / 危险命令分类处置」（``/v1/local/grants``、``/v1/local/policy``）管的是 "本机哪些目录能动"，与本档位是两件事。 """ @@ -17,8 +17,8 @@ from core.auth.backend import UserContext, get_current_user from core.db.engine import get_db from core.infra.responses import success_response -from core.llm.tool_permissions import normalize_approval_mode -from core.services.user_service import UserService +from core.sandbox.permissions.approval import normalize_approval_mode +from core.auth.users import UserService from fastapi import APIRouter, Depends from pydantic import BaseModel from sqlalchemy.orm import Session @@ -39,7 +39,7 @@ def get_tool_approval_mode( ): settings = UserService(db).get_user_settings(str(user.user_id)) if user.public_chat_visitor: - from core.services.public_chat_service import visitor_approval_mode + from core.chat.public import visitor_approval_mode mode = visitor_approval_mode(db, user, settings.get(_METADATA_KEY)) else: mode = normalize_approval_mode(settings.get(_METADATA_KEY)) @@ -54,7 +54,7 @@ def set_tool_approval_mode( ): mode = normalize_approval_mode(body.mode) if user.public_chat_visitor: - from core.services.public_chat_service import set_visitor_approval_mode + from core.chat.public import set_visitor_approval_mode set_visitor_approval_mode(db, user, mode) else: UserService(db).update_user_metadata(user_id=str(user.user_id), patch={_METADATA_KEY: mode}) diff --git a/src/backend/api/routes/v1/users.py b/src/backend/api/routes/v1/users.py index 2eea8dea..792f2357 100644 --- a/src/backend/api/routes/v1/users.py +++ b/src/backend/api/routes/v1/users.py @@ -13,7 +13,7 @@ from core.db.repository import UserRepository from core.infra.exceptions import AccessDeniedError, ResourceNotFoundError, ValidationError from core.infra.responses import success_response -from core.storage import get_storage +from core.files.storage import get_storage from fastapi import APIRouter, Depends, File, HTTPException, Path, UploadFile from fastapi.responses import Response from pydantic import BaseModel, Field @@ -77,7 +77,7 @@ def get_current_user_info( local = db.query(LocalUser).filter(LocalUser.user_id == user.user_id).first() meta = dict(user_shadow.extra_data or {}) - from core.services.local_user_service import ce_onboarding_required + from core.auth.local_user_policy import ce_onboarding_required data = { "user_id": user_shadow.user_id, @@ -96,7 +96,7 @@ def get_current_user_info( "created_at": user_shadow.created_at.isoformat(), } - from core.auth.account_view import extend_current_account + from core.auth.edition import extend_current_account data = extend_current_account(db, str(user.user_id), data) @@ -125,7 +125,7 @@ def update_current_user_info( """更新当前用户资料（nickname / real_name / phone，仅传的字段才更新）。仅本地账号可改，外部 SSO 用户需到身份源修改。""" from core.db.models import LocalUser from core.infra.exceptions import ValidationError - from core.services.local_user_service import LocalUserService + from core.auth.local_users import LocalUserService local = db.query(LocalUser).filter(LocalUser.user_id == user.user_id).first() if not local: @@ -156,7 +156,7 @@ def change_my_password( db: Session = Depends(get_db), ): """Verify the current password, save the replacement, and clear the first-login flag.""" - from core.services.local_user_service import LocalUserService + from core.auth.local_users import LocalUserService result = LocalUserService(db).change_password( user_id=user.user_id, @@ -186,7 +186,7 @@ def complete_my_onboarding( final, idempotent checkpoint that unlocks the normal application shell. """ from core.db.models import ModelProvider, ModelRoleAssignment - from core.services.local_user_service import CE_ONBOARDING_VERSION + from core.auth.local_users_constants import CE_ONBOARDING_VERSION if settings.edition.edition != "ce": raise AccessDeniedError( diff --git a/src/backend/api/startup/bootstrap.py b/src/backend/api/startup/bootstrap.py index 886ae95a..2a4e0ec9 100644 --- a/src/backend/api/startup/bootstrap.py +++ b/src/backend/api/startup/bootstrap.py @@ -1,4 +1,5 @@ """Application lifecycle operations.""" +from core.llm import prompt_versions as prompt_versions_module import os import sys from core.config.settings import settings @@ -27,7 +28,7 @@ async def _startup_seed_ce_admin(): return try: from core.db.engine import SessionLocal - from core.services.local_user_service import ensure_ce_default_admin + from core.auth.local_user_bootstrap import ensure_ce_default_admin db = SessionLocal() try: @@ -49,7 +50,7 @@ async def _startup_prepare_artifact_records(): try: import asyncio - from core.artifacts.store import prepare_records + from core.files.artifacts.store import prepare_records written = await asyncio.to_thread(prepare_records) if written: @@ -60,7 +61,7 @@ async def _startup_prepare_artifact_records(): async def _startup_seed_page_config(): """Seed default page_config content block if missing (idempotent).""" try: - from core.content.content_blocks import enforce_ce_branding, seed_page_config_if_missing + from core.files.content.branding import enforce_ce_branding, seed_page_config_if_missing from core.db.engine import SessionLocal db = SessionLocal() @@ -82,11 +83,11 @@ async def _startup_seed_prompt_versions(): """ try: from core.db.engine import SessionLocal - from core.services import prompt_version_service as pvs + from core.llm import prompt_versions as pvs db = SessionLocal() try: - result = pvs.seed_from_filesystem(db=db) + result = prompt_versions_module.seed_from_filesystem(db=db) added = result.get("added") or [] if added: logger.info("[startup] prompt versions seeded: %s", ", ".join(added)) @@ -103,7 +104,7 @@ async def _startup_seed_roles(): """ try: from core.db.engine import SessionLocal - from core.services.edition_startup import seed_default_roles + from orchestration.workers.edition_startup import seed_default_roles db = SessionLocal() try: diff --git a/src/backend/api/startup/commands.py b/src/backend/api/startup/commands.py index c15db35d..5602925b 100644 --- a/src/backend/api/startup/commands.py +++ b/src/backend/api/startup/commands.py @@ -1,9 +1,10 @@ """Background command monitoring and durable notification dispatch.""" import asyncio import logging + from core.sandbox import get_sandbox_provider -from orchestration.commands.runtime import runtime -from orchestration.commands.notifications import dispatch +from core.sandbox.commands.notifications import dispatch +from core.sandbox.commands.runtime import runtime logger = logging.getLogger(__name__) _task = None @@ -20,6 +21,9 @@ async def _run(): async def startup_commands(): global _task runtime.bind_loop() + bind = getattr(get_sandbox_provider(), "bind_loop", None) + if bind is not None: + bind() _task = asyncio.create_task(_run(), name="background-command-reconciliation") async def shutdown_commands(): diff --git a/src/backend/api/startup/memory.py b/src/backend/api/startup/memory.py index db0c65d7..35487dad 100644 --- a/src/backend/api/startup/memory.py +++ b/src/backend/api/startup/memory.py @@ -1,8 +1,10 @@ """Application lifecycle operations.""" import os import sys + from core.config.settings import settings from core.infra.logging import get_logger + logger = get_logger(__name__) _persona_distill_worker = None @@ -12,7 +14,7 @@ async def _startup_evolution_scheduler(): """Start the nightly evolution-cycle backstop.""" try: - from orchestration.schedulers.evolution_scheduler import EvolutionCronScheduler + from orchestration.workers.evolution_scheduler import EvolutionCronScheduler await EvolutionCronScheduler().start() except Exception as exc: @@ -24,7 +26,7 @@ async def _startup_memory_ttl_scheduler(): if not settings.memory.enabled: return try: - from orchestration.schedulers.memory_ttl_scheduler import MemoryTtlScheduler + from orchestration.workers.memory_ttl_scheduler import MemoryTtlScheduler await MemoryTtlScheduler().start() except Exception as exc: @@ -66,7 +68,7 @@ async def _startup_persona_distill_worker(): should not depend on which worker took the HTTP request. """ try: - from orchestration.schedulers.persona_distill_worker import PersonaDistillWorker + from orchestration.workers.persona_distill_worker import PersonaDistillWorker global _persona_distill_worker _persona_distill_worker = PersonaDistillWorker() @@ -91,7 +93,7 @@ async def _startup_warmup_memory(): async def _warmup() -> None: try: - from core.memory.service import _get_memory # type: ignore[attr-defined] + from core.memory.service import get_memory as _get_memory # type: ignore[attr-defined] # _get_memory is thread-safe lazy loading; the first call establishes the Milvus + Neo4j connections instance = _get_memory() diff --git a/src/backend/api/startup/plugins.py b/src/backend/api/startup/plugins.py index 276d5a90..114d3bcd 100644 --- a/src/backend/api/startup/plugins.py +++ b/src/backend/api/startup/plugins.py @@ -16,12 +16,9 @@ async def _startup_seed_mcp_servers(): """ try: from core.db.engine import SessionLocal - from core.services.mcp_management_service import encrypt_legacy_mcp_headers - from core.services.mcp_marketplace_service import ensure_curated_market_items - from core.services.mcp_service import ( - prune_removed_builtin_mcp_servers, - seed_builtin_mcp_servers_if_empty, - ) + from core.capabilities.mcp.management import encrypt_legacy_mcp_headers + from features.marketplace.mcp_curated import ensure_curated_market_items + from core.capabilities.mcp.service import prune_removed_builtin_mcp_servers, seed_builtin_mcp_servers_if_empty db = SessionLocal() try: @@ -45,7 +42,7 @@ async def _startup_seed_mcp_servers(): async def _startup_mcp_market_monitor(): """Start periodic remote-tool drift detection after MCP rows are available.""" try: - from core.services.mcp_marketplace_monitor import start_monitor + from features.marketplace.mcp_monitor import start_monitor start_monitor() except Exception as exc: @@ -53,7 +50,7 @@ async def _startup_mcp_market_monitor(): async def _shutdown_mcp_market_monitor(): try: - from core.services.mcp_marketplace_monitor import stop_monitor + from features.marketplace.mcp_monitor import stop_monitor await stop_monitor() except Exception as exc: @@ -64,7 +61,7 @@ async def _startup_upgrade_sites_plugin(): import asyncio from core.db.engine import SessionLocal - from core.plugins.local.site_upgrade import upgrade_builtin_sites + from features.sites.builtin_upgrade import upgrade_builtin_sites def upgrade(): with SessionLocal() as db: @@ -88,7 +85,7 @@ async def _startup_plugin_device_assets(): return from core.db.engine import SessionLocal - from core.plugins.local.assets import provision_present_plugins + from features.desktop.local.plugins.assets import provision_present_plugins def provision(): with SessionLocal() as db: @@ -100,7 +97,7 @@ def provision(): async def _startup_backfill_tool_schemas(): """Capture real tool schemas for servers a plugin manifest only named.""" - from core.services.mcp_tool_schema_backfill import backfill_missing_tool_schemas + from core.capabilities.mcp.tool_schema_backfill import backfill_missing_tool_schemas count = await backfill_missing_tool_schemas() if count: @@ -108,7 +105,7 @@ async def _startup_backfill_tool_schemas(): async def _startup_backfill_api_protocol(): """Record which OpenAI wire protocol each pre-existing model endpoint speaks.""" - from core.services.api_protocol_backfill import backfill_missing_api_protocol + from core.models.api_protocol_backfill import backfill_missing_api_protocol count = await backfill_missing_api_protocol() if count: @@ -127,7 +124,7 @@ async def _startup_seed_default_plugins(): if edition not in {"ce", "ee"}: return if settings.deploy.is_local: - from core.capabilities import device_catalog + from features.desktop.capabilities import device_catalog # Hybrid capabilities come from the authenticated cloud account. Local # defaults also respect the installer's explicit onboarding opt-in. @@ -135,10 +132,10 @@ async def _startup_seed_default_plugins(): return from core.db.engine import SessionLocal - from core.services.edition_startup import bootstrap_edition_plugins - from core.plugins.management import DEFAULT_BOOTSTRAP_PLUGIN_SLUGS, ensure_default_plugins_bootstrapped - from core.plugins.ui.contributions import refresh_builtin_ui_contributions - from core.plugins.management.defaults import ensure_visualize_plugin_bootstrapped + from orchestration.workers.edition_startup import bootstrap_edition_plugins + from core.capabilities.plugins.management import DEFAULT_BOOTSTRAP_PLUGIN_SLUGS, ensure_default_plugins_bootstrapped + from features.workbench.ui.contributions import refresh_builtin_ui_contributions + from core.capabilities.plugins.management.defaults import ensure_visualize_plugin_bootstrapped db = SessionLocal() try: diff --git a/src/backend/api/startup/runtime.py b/src/backend/api/startup/runtime.py index db2fac7f..ab1393ab 100644 --- a/src/backend/api/startup/runtime.py +++ b/src/backend/api/startup/runtime.py @@ -1,13 +1,20 @@ """Application lifecycle operations.""" + import os import sys + +import core.chat.runtime.recovery as recovery +import core.runs.watchdog as watchdog +import orchestration.loops.recovery as loops_recovery +import orchestration.workers.run_recovery as _run_recovery from core.config.settings import settings from core.infra.logging import get_logger + logger = get_logger(__name__) async def _startup_watch_capability_changes(): """智能体 / 技能 / 连接器 / 插件被增删时递增能力变更号，桌面端据此同步一次。""" - from core.capabilities.change_signal import watch_capability_tables + from features.desktop.capabilities.change_signal import watch_capability_tables from core.db.engine import engine watch_capability_tables(engine) @@ -15,9 +22,9 @@ async def _startup_watch_capability_changes(): async def _startup_recover_chat_runs(): """Resume claimable chat_runs; valid leases are retried by the recovery loop.""" try: - from orchestration import chat_run_executor + pass - count = await chat_run_executor.recover_orphan_runs() + count = await recovery.recover_orphan_runs() if count: logger.info("[startup] chat_runs orphan recovered: %d", count) except Exception as exc: @@ -26,9 +33,9 @@ async def _startup_recover_chat_runs(): async def _startup_resume_loops(): """Resume interrupted autonomous loops (M4, off by default; enable with LOOP_AUTO_RESUME=true).""" try: - from orchestration import chat_run_executor + pass - count = await chat_run_executor.resume_running_loops() + count = await loops_recovery.resume_running_loops() if count: logger.info("[startup] autonomous loops resumed: %d", count) except Exception as exc: @@ -41,7 +48,7 @@ async def _startup_recover_jobs(): 可直接断点续跑，已完成的项不会重做。 """ try: - from orchestration import job_runtime + import core.jobs.runtime as job_runtime count = await job_runtime.resume_running_jobs() if count: @@ -55,7 +62,7 @@ async def _startup_orphan_job_reaper(): try: import asyncio - from orchestration import job_runtime + import core.jobs.runtime as job_runtime task = asyncio.create_task(job_runtime.run_job_reaper_loop()) app.state.orphan_job_reaper_task = task @@ -69,19 +76,20 @@ async def _startup_stale_run_reaper(): try: import asyncio - from orchestration import chat_run_executor + pass - task = asyncio.create_task(chat_run_executor.run_stale_reaper_loop()) + task = asyncio.create_task(_run_recovery.run_stale_reaper_loop()) app.state.stale_run_reaper_task = task logger.info("[startup] chat_run stale reaper started") except Exception as exc: logger.warning("[startup] chat_run stale reaper start failed: %s", exc) async def _shutdown_stale_run_reaper(): - from api.app import app import asyncio import contextlib + from api.app import app + task = getattr(app.state, "stale_run_reaper_task", None) if task is not None: task.cancel() @@ -89,10 +97,11 @@ async def _shutdown_stale_run_reaper(): await task async def _shutdown_orphan_job_reaper(): - from api.app import app import asyncio import contextlib + from api.app import app + task = getattr(app.state, "orphan_job_reaper_task", None) if task is not None: task.cancel() @@ -101,7 +110,7 @@ async def _shutdown_orphan_job_reaper(): async def _shutdown_channel_manager(): try: - from core.channels.manager import get_manager + from integrations.channels.manager import get_manager get_manager().shutdown() except Exception: # noqa: BLE001 @@ -123,6 +132,7 @@ async def _startup_warm_sandbox_pool(): logger.info("[startup] sandbox provider %s warmup kicked off", provider.name) except Exception as exc: # noqa: BLE001 logger.warning("[startup] sandbox warmup failed: %s", exc) + raise async def _startup_local_sidecars(): """Spawn and verify MCP + script_runner sidecars in local profile. @@ -133,7 +143,7 @@ async def _startup_local_sidecars(): startup behavior. """ try: - from orchestration.local_subprocess import start_local_sidecars + from core.infra.local_runtime.startup import start_local_sidecars await start_local_sidecars() except Exception as exc: @@ -144,7 +154,7 @@ async def _startup_local_sidecars(): async def _shutdown_local_sidecars(): """Reap local-profile sidecar subprocesses on shutdown.""" try: - from orchestration.local_subprocess import stop_local_sidecars + from core.infra.local_runtime.startup import stop_local_sidecars await stop_local_sidecars() except Exception as exc: @@ -152,23 +162,24 @@ async def _shutdown_local_sidecars(): async def _startup_device_link(): """Serve this computer's Personal Agent mailbox (desktop execution plane only).""" - from core.device_link.runner import runner + from personal_agent.device_link.runner import runner runner.start() async def _shutdown_device_link(): - from core.device_link.runner import runner + from personal_agent.device_link.runner import runner await runner.stop() async def _startup_recover_datasource_sidecars(): """Recover configured DBHub/ES MCP sidecars without blocking API startup.""" - from api.app import app import asyncio + from api.app import app + async def _recover() -> None: try: - from core.services.edition_startup import recover_datasource_sidecars + from orchestration.workers.edition_startup import recover_datasource_sidecars result = await recover_datasource_sidecars() failures = [ @@ -198,10 +209,11 @@ async def _recover() -> None: app.state.datasource_sidecar_recovery_task = asyncio.create_task(_recover()) async def _shutdown_datasource_sidecar_recovery(): - from api.app import app import asyncio import contextlib + from api.app import app + task = getattr(app.state, "datasource_sidecar_recovery_task", None) if task is not None and not task.done(): task.cancel() @@ -214,13 +226,13 @@ async def _shutdown_datasource_sidecar_recovery(): async def _startup_desktop_observability(): global _desktop_sync_worker, _desktop_capture_remove - from core.auth.desktop_bridge import bridge_enabled + from features.desktop.auth.bridge import bridge_enabled if not bridge_enabled(): return from core.db.engine import SessionLocal - from core.services.desktop_cloud_bridge import get_identity_state - from core.services.desktop_observability_sync import DesktopSyncWorker, install_capture + from features.desktop.bridge.identity import get_identity_state + from features.desktop.observability.sync import DesktopSyncWorker, install_capture if _desktop_capture_remove is None: _desktop_capture_remove = install_capture(SessionLocal, get_identity_state) @@ -241,15 +253,15 @@ async def _shutdown_desktop_observability(): async def _startup_channel_desktop(): global _channel_desktop_worker - from core.auth.desktop_bridge import bridge_enabled + from features.desktop.auth.bridge import bridge_enabled from core.db.engine import SessionLocal if bridge_enabled(): - from core.services.channel_desktop_worker import ChannelDesktopWorker + from integrations.channels.desktop_worker import ChannelDesktopWorker _channel_desktop_worker = ChannelDesktopWorker(SessionLocal) else: - from core.services.channel_relay_dispatch import ChannelRelayReaper + from integrations.channels.relay_dispatch import ChannelRelayReaper _channel_desktop_worker = ChannelRelayReaper(SessionLocal) _channel_desktop_worker.start() @@ -261,13 +273,27 @@ async def _shutdown_channel_desktop(): _channel_desktop_worker = None async def _startup_extensions(): - from core.extensions import lifecycle_hooks + from core.harness.registry import lifecycle_hooks for start, _ in lifecycle_hooks(): await start() async def _shutdown_extensions(): - from core.extensions import lifecycle_hooks + from core.harness.registry import lifecycle_hooks for _, stop in reversed(lifecycle_hooks()): await stop() + + +async def _shutdown_sandbox_maintenance(): + from core.sandbox import get_sandbox_provider + stop = getattr(get_sandbox_provider(), "stop_maintenance", None) + if stop is not None: + await stop() + + +async def _shutdown_sandbox_provider(): + from core.sandbox import get_sandbox_provider + shutdown = getattr(get_sandbox_provider(), "shutdown", None) + if shutdown is not None: + await shutdown() diff --git a/src/backend/api/startup/workers.py b/src/backend/api/startup/workers.py index e8fcc2d2..f1e06ecf 100644 --- a/src/backend/api/startup/workers.py +++ b/src/backend/api/startup/workers.py @@ -1,9 +1,11 @@ """Application lifecycle operations.""" import os -from typing import Callable import sys +from typing import Callable + from core.config.settings import settings from core.infra.logging import get_logger + logger = get_logger(__name__) def _env_flag(name: str, default: bool) -> bool: @@ -15,9 +17,6 @@ def _env_flag(name: str, default: bool) -> bool: def _startup_preload_enabled() -> bool: return _env_flag("STARTUP_PRELOAD_ENABLED", True) -def _mcp_warmup_enabled() -> bool: - return _env_flag("MCP_WARMUP_ENABLED", True) - async def _shutdown_automation_scheduler(): """Stop the automation scheduler, whether the process is exiting or this worker just lost the leader role. Idempotent: the scheduler's own ``stop`` @@ -78,7 +77,7 @@ async def _run(): logger.warning("[startup] idle session reaper failed to start: %s", exc) async def _startup_preload(): - """Pre-load caches and initialize MCP connection pool at startup.""" + """Pre-load application caches at startup.""" import asyncio if not _startup_preload_enabled(): @@ -104,7 +103,7 @@ async def _run_sync(label: str, func: Callable[[], None]) -> None: # 2. Pre-load skill metadata def _load_skill_metadata() -> None: - from core.agent_skills.loader import get_skill_loader + from core.capabilities.skills.loader_factory import get_skill_loader loader = get_skill_loader() meta = loader.load_all_metadata() @@ -117,13 +116,13 @@ def _load_skill_metadata() -> None: # at the same in-sandbox path. DB skills materialize into the same # dir on demand. Must run before sandboxes are created/used. def _sync_builtin_skills() -> None: - from core.agent_skills.config import sync_builtin_skills_to_sandbox_dir - from core.capabilities.paths import capabilities_enabled + from core.capabilities.skills.config import sync_builtin_skills_to_sandbox_dir + from core.harness.registry.run_capabilities import has_run_capabilities - if capabilities_enabled(): + if has_run_capabilities(): # Desktop store: move the pre-store layout aside first (idempotent, # quarantines instead of deleting), then the views are link-only. - from core.capabilities.migration import migrate_legacy_layout + from features.desktop.capabilities.migration import migrate_legacy_layout report = migrate_legacy_layout() if report is not None and (report.changed or report.skipped): @@ -134,7 +133,7 @@ def _sync_builtin_skills() -> None: # Pre-create the per-user skills root for the same reason as above: # script-runner bind-mounts it, and a dockerd-created one is root-owned. try: - from core.agent_skills.config import get_user_skills_root + from core.capabilities.skills.config import get_user_skills_root get_user_skills_root() except Exception as exc: # noqa: BLE001 @@ -147,7 +146,7 @@ def _sync_builtin_skills() -> None: # created by dockerd as root and the runner can't write into it → # backend must create it first. opensandbox/cube don't use this one. try: - from core.sandbox.workspace.credentials import yida_shared_workspace_dir + from integrations.yida.storage import yida_shared_workspace_dir p = yida_shared_workspace_dir() p.mkdir(parents=True, exist_ok=True) @@ -160,24 +159,13 @@ def _sync_builtin_skills() -> None: # 3. Pre-load public DB capability overlay so first chat avoids # scanning admin_skills/admin_mcp_servers on the request path. def _warm_runtime_catalog() -> None: - from core.config.catalog_runtime import warmup_runtime_catalog_cache + from core.capabilities.catalog.runtime import warmup_runtime_catalog_cache warmup_runtime_catalog_cache() logger.info("[startup] Runtime catalog DB overlay warmed") await _run_sync("Runtime catalog", _warm_runtime_catalog) - # 4. Initialize MCP connection pool (the big one — 1-7s savings) - if _mcp_warmup_enabled(): - try: - from core.llm.factory import warmup_mcp_tools - - await warmup_mcp_tools() - except Exception as exc: - logger.warning("[startup] MCP pool initialization failed: %s", exc) - else: - logger.info("[startup] MCP pool warmup skipped for current environment") - # 5. Pre-load DB prompt parts so first chat skips DB query from prompts.prompt_runtime import warmup_prompt_cache @@ -188,7 +176,7 @@ def _warm_runtime_catalog() -> None: # row. See _NAV_BACKFILL_ENTRIES in content_blocks.py — append a # new entry there each time we add a sidebar feature. def _backfill_nav(): - from core.content.content_blocks import backfill_navigation_entries + from core.files.content.branding import backfill_navigation_entries from core.db.engine import SessionLocal db = SessionLocal() @@ -214,7 +202,7 @@ async def _startup_automation_scheduler(): logger.info("[startup] Automation scheduler disabled") return try: - from orchestration.schedulers.automation_scheduler import AutomationScheduler + from orchestration.automations.automation_scheduler import AutomationScheduler global _automation_scheduler _automation_scheduler = AutomationScheduler() @@ -235,7 +223,7 @@ async def _startup_kb_wiki_worker(): logger.info("[startup] KB wiki worker disabled") return try: - from core.kb.wiki.worker import WikiIngestWorker + from features.knowledge.wiki.worker import WikiIngestWorker global _kb_wiki_worker _kb_wiki_worker = WikiIngestWorker() @@ -258,7 +246,7 @@ async def _startup_kb_index_worker(): logger.info("[startup] KB index worker disabled") return try: - from core.kb.index_worker import KBIndexWorker + from features.knowledge.index_worker import KBIndexWorker global _kb_index_worker _kb_index_worker = KBIndexWorker() @@ -295,7 +283,7 @@ async def _startup_channel_manager(): logger.info("[startup] Channel bots disabled") return try: - from core.channels.manager import get_manager + from integrations.channels.manager import get_manager await get_manager().start_all() from core.infra import runtime_state @@ -317,7 +305,7 @@ async def _startup_channel_manager(): async def _startup_distillation_scheduler(): """Start the daily skill-distillation cron scheduler.""" try: - from core.services.edition_startup import create_distillation_scheduler + from orchestration.workers.edition_startup import create_distillation_scheduler global _distillation_scheduler _distillation_scheduler = create_distillation_scheduler() @@ -331,7 +319,9 @@ async def _startup_distillation_scheduler(): logger.warning("[startup] Distillation cron scheduler failed to start: %s", exc) async def _shutdown_pools(): - """Close MCP connection pool, KB HTTP server, automation scheduler, and Redis on shutdown.""" + """Close chat MCP sessions and application services on shutdown.""" + from core.llm.mcp.chat_sessions import close_chat + await close_chat() # Stop automation scheduler — same handler the leader path calls, so a # deposed leader and an exiting process cannot stop different things. try: @@ -350,12 +340,13 @@ async def _shutdown_pools(): await _shutdown_idle_session_reaper() except Exception as e: logger.warning("idle_reaper_shutdown_error", error=str(e)) - try: - from core.llm.mcp_pool import MCPConnectionPool + import asyncio - await MCPConnectionPool.get_instance().shutdown() - except Exception as e: - logger.warning("mcp_pool_shutdown_error", error=str(e)) + from core.harness.observation_writer import flush_observations + + # Hook stage events and usage are written in the background; persist the + # tail before the database and Redis go away. + await asyncio.to_thread(flush_observations, 5.0) try: from core.infra.redis import close_redis diff --git a/src/backend/core/content/__init__.py b/src/backend/api/transports/__init__.py similarity index 100% rename from src/backend/core/content/__init__.py rename to src/backend/api/transports/__init__.py diff --git a/src/backend/api/transports/run_sse.py b/src/backend/api/transports/run_sse.py new file mode 100644 index 00000000..e4bc29ea --- /dev/null +++ b/src/backend/api/transports/run_sse.py @@ -0,0 +1,85 @@ +"""Encode durable run events into the existing public SSE wire protocol.""" + +from __future__ import annotations + +import asyncio +import contextlib +import json +from typing import Any, AsyncIterator, Callable, Dict, Optional + +import core.runs.events.subscription as subscription +import core.runs.runtime as runtime +import core.runs.settings as settings +from core.infra.logging import get_logger + +logger = get_logger(__name__) + + +async def follow_run_as_sse( + run_id: str, + *, + chat_id: str, + from_offset: int = 0, + error_event_factory: Optional[Callable[[str], Dict[str, Any]]] = None, +) -> AsyncIterator[str]: + """Wrap follow_run as SSE wire frames. + + ``error_event_factory(reason)`` lets callers customize the error event shape + (chat protocol vs plan protocol). Defaults to a chat-style ``{type: error}``. + """ + + def _default_err(reason: str) -> Dict[str, Any]: + return {"type": "error", "error": reason, "chat_id": chat_id} + + factory = error_event_factory or _default_err + + # Decouple follow_run from the yield cadence via an intermediate queue: it + # lets wait_for write an SSE comment line to the wire when the stream has + # been silent longer than _HEARTBEAT_INTERVAL_SEC, serving as nginx / + # reverse-proxy keepalive. Cancelling queue.get via wait_for is safe + # (asyncio.Queue handles the cancel race); the underlying follow_run + # coroutine is never interrupted. + queue: asyncio.Queue = asyncio.Queue() + + async def _pump() -> None: + try: + async for event in subscription.follow_run(run_id, from_offset=from_offset): + await queue.put(("event", event)) + except runtime.ChatRunNotFound: + await queue.put(("not_found", None)) + except Exception as exc: # noqa: BLE001 + await queue.put(("error", exc)) + else: + await queue.put(("end", None)) + + pump_task = asyncio.create_task(_pump(), name=f"sse_pump:{run_id}") + try: + while True: + try: + kind, payload = await asyncio.wait_for( + queue.get(), timeout=settings.HEARTBEAT_INTERVAL_SEC + ) + except asyncio.TimeoutError: + yield ": heartbeat\n\n" + continue + + if kind == "event": + clean = {k: v for k, v in payload.items() if not k.startswith("_")} + yield f"data: {json.dumps(clean, ensure_ascii=False)}\n\n" + elif kind == "end": + break + elif kind == "not_found": + yield f"data: {json.dumps(factory('run not found'), ensure_ascii=False)}\n\n" + break + elif kind == "error": + logger.warning( + "follow_run_as_sse_failed", run_id=run_id, error=str(payload), exc_info=payload + ) + yield f"data: {json.dumps(factory('流式响应中断'), ensure_ascii=False)}\n\n" + break + finally: + pump_task.cancel() + with contextlib.suppress(BaseException): + await pump_task + + yield "data: [DONE]\n\n" diff --git a/src/backend/cli.py b/src/backend/cli.py index 4455f89b..82645161 100644 --- a/src/backend/cli.py +++ b/src/backend/cli.py @@ -27,7 +27,6 @@ from pathlib import Path from typing import Optional - def _configure_standard_streams() -> None: """Keep redirected desktop logs UTF-8 and tolerant of status symbols.""" for stream in (sys.stdout, sys.stderr): @@ -42,7 +41,6 @@ def _configure_standard_streams() -> None: except (LookupError, OSError, ValueError): pass - def _status(message: str, *, file=None, **kwargs) -> None: """Print CLI status without letting a legacy code page break startup.""" stream = file or sys.stdout @@ -56,16 +54,13 @@ def _status(message: str, *, file=None, **kwargs) -> None: safe_message = message.encode("ascii", errors="replace").decode("ascii") print(safe_message, file=stream, **kwargs) - # ── Data dir & environment ──────────────────────────────────────────────────── - def data_dir() -> Path: from core.config.runtime_env import local_data_dir return local_data_dir() - def ensure_loopback_proxy_bypass() -> None: """Keep loopback traffic off any HTTP(S)/SOCKS proxy from the environment. @@ -98,7 +93,6 @@ def ensure_loopback_proxy_bypass() -> None: os.environ["NO_PROXY"] = merged os.environ["no_proxy"] = merged - def _resolve_frontend_dist() -> Optional[str]: """Locate the built frontend for StaticFiles hosting.""" env = os.getenv("FRONTEND_DIST_DIR", "").strip() @@ -109,13 +103,11 @@ def _resolve_frontend_dist() -> Optional[str]: cand = src / "frontend" / "dist" return str(cand) if (cand / "index.html").exists() else None - def _site_template_home() -> Path: - from core.plugins.local.assets import site_template_home + from features.desktop.local.plugins.assets import site_template_home return site_template_home() - def apply_local_env(port: int) -> dict: """Populate the local-profile env (idempotent; real env wins) + data dirs.""" ensure_loopback_proxy_bypass() @@ -196,6 +188,13 @@ def apply_local_env(port: int) -> dict: # ``defaults`` would freeze a settings object that predates # ``DEPLOY_PROFILE=local`` — the whole process would then run as a non-local # deployment (no ``/api`` prefix bridge, no static frontend, mock auth). + from orchestration.configuration import register_configuration + register_configuration() + from features import register_run_capabilities, register_model_credentials + register_run_capabilities() + register_model_credentials() + from integrations import register_credentials + register_credentials() site_home = _site_template_home() defaults["SITE_TEMPLATE_HOME"] = str(site_home) defaults["SITE_TEMPLATE_DIR"] = str(site_home / "react-vite") @@ -203,7 +202,6 @@ def apply_local_env(port: int) -> dict: os.environ.setdefault("SITE_TEMPLATE_DIR", defaults["SITE_TEMPLATE_DIR"]) return defaults - def _is_initialized() -> bool: """True if the SQLite DB exists and has at least one local account.""" dd = data_dir() @@ -227,24 +225,6 @@ def _is_initialized() -> bool: except Exception: return False - -# ── DB helpers (import lazily, after env is set) ───────────────────────────── - - -def _ensure_schema_and_seed(): - """create_all + built-in catalog seed (mirrors the app's startup hooks).""" - from core.db.engine import SessionLocal, init_db - from core.services.mcp_service import prune_removed_builtin_mcp_servers, seed_builtin_mcp_servers_if_empty - - init_db() - db = SessionLocal() - try: - prune_removed_builtin_mcp_servers(db) - seed_builtin_mcp_servers_if_empty(db) - finally: - db.close() - - # ── Onboarding ──────────────────────────────────────────────────────────────── _PRESETS = { @@ -260,13 +240,11 @@ def _ensure_schema_and_seed(): # guess a missing context window. DEFAULT_LOCAL_CONTEXT_LENGTH = 32768 - def _prompt(msg: str, default: str = "") -> str: suffix = f" [{default}]" if default else "" val = input(f"{msg}{suffix}: ").strip() return val or default - def bootstrap_admin(username: str, password: str) -> str: """Create the local admin account and elevate it to super_admin. @@ -275,8 +253,8 @@ def bootstrap_admin(username: str, password: str) -> str: step missing everywhere else in the codebase. """ from core.db.engine import SessionLocal - from core.services.local_user_service import LocalUserService - from core.services.user_service import UserService + from core.auth.local_users import LocalUserService + from core.auth.users import UserService db = SessionLocal() try: @@ -288,7 +266,6 @@ def bootstrap_admin(username: str, password: str) -> str: finally: db.close() - def configure_model( base_url: str, api_key: str, @@ -303,9 +280,9 @@ def configure_model( """ import asyncio - from core.db import model_repository as mr + from core.models import repository as mr from core.db.engine import SessionLocal - from core.services.model_config import ModelConfigService + from core.models.config import ModelConfigService if test: from api.routes.v1.models import _test_connection @@ -351,12 +328,11 @@ def configure_model( # Repository writes don't invalidate the read cache — do it explicitly. ModelConfigService.get_instance().invalidate_cache() - def mark_web_onboarding_complete(user_id: str) -> None: """Avoid repeating the browser setup after the terminal wizard succeeds.""" from core.db.engine import SessionLocal - from core.services.local_user_service import CE_ONBOARDING_VERSION - from core.services.user_service import UserService + from core.auth.local_users_constants import CE_ONBOARDING_VERSION + from core.auth.users import UserService db = SessionLocal() try: @@ -370,7 +346,6 @@ def mark_web_onboarding_complete(user_id: str) -> None: finally: db.close() - # ── Plugins ─────────────────────────────────────────────────────────────────── # Recommended default plugin set for a personal single-machine install: task @@ -378,41 +353,46 @@ def mark_web_onboarding_complete(user_id: str) -> None: # (IM / email / low-code) need per-user credentials, so they're opt-in only. _DEFAULT_PLUGIN_BOOTSTRAP_ENV = "HUGAGENT_BOOTSTRAP_DEFAULT_PLUGINS" - def _default_plugins() -> list: """推荐的默认插件集合，与库里的引导保持同一份来源。""" - from core.plugins.management import DEFAULT_BOOTSTRAP_PLUGIN_SLUGS + from core.capabilities.plugins.management import DEFAULT_BOOTSTRAP_PLUGIN_SLUGS return list(DEFAULT_BOOTSTRAP_PLUGIN_SLUGS) - def _default_plugins_marker() -> Path: - from core.plugins.management import LOCAL_BOOTSTRAP_MARKER_NAME + from core.capabilities.plugins.management import LOCAL_BOOTSTRAP_MARKER_NAME return data_dir() / LOCAL_BOOTSTRAP_MARKER_NAME - def list_installable_plugins() -> list: """Local-filesystem plugin bundles (default + marketplace) with install state.""" from core.db.engine import SessionLocal - from core.plugins import management as plugin_service + from core.capabilities.plugins import management as plugin_service + from features.marketplace import plugin_market + + from features import register_package_sources, register_resource_owners + register_package_sources() + register_resource_owners() db = SessionLocal() try: - items = plugin_service.list_plugins(db, None, include_disabled=True) + items = plugin_market.list_plugins(db, None, include_disabled=True) finally: db.close() # Only the shipped bundles are installable offline; DB-market packages need an # admin upload that doesn't exist on a fresh local install. return [it for it in items if it.get("source") == "builtin"] - def install_plugins(slugs: list) -> list: """Install the given plugin slugs globally (owner_user_id=None). Best-effort.""" from core.db.engine import SessionLocal - from core.plugins import management as plugin_service + from core.capabilities.plugins import management as plugin_service done = [] + from features import register_package_sources, register_resource_owners + register_package_sources() + register_resource_owners() + db = SessionLocal() try: for slug in slugs: @@ -427,7 +407,6 @@ def install_plugins(slugs: list) -> list: db.close() return done - def ensure_default_plugins_once() -> bool: """Install local-install defaults once without overriding later user choices. @@ -453,7 +432,6 @@ def ensure_default_plugins_once() -> bool: os.replace(temporary, marker) return True - def _select_plugins_interactively(available: list) -> list: """Show the plugin menu, return the slugs the user picked.""" recommended = _default_plugins() @@ -481,10 +459,8 @@ def _select_plugins_interactively(available: list) -> list: picked.append(it["slug"]) return picked - # ── File parser (PDF / document parsing on upload) ──────────────────────────── - def configure_file_parser( api_url: str, *, backend: str = "pipeline", parse_method: str = "auto" ) -> None: @@ -494,7 +470,7 @@ def configure_file_parser( service at ``file_parser.api_url``. Excel/CSV/PPTX/text parse in-process and need none of this. """ - from core.services.system_config import SystemConfigService + from core.config.store.system_config import SystemConfigService svc = SystemConfigService.get_instance() # Touch the cache once so the seed rows exist before we set() them @@ -509,10 +485,8 @@ def configure_file_parser( updated_by="onboard", ) - # ── Internet search (Tavily / Baidu / LangSearch key) ──────────────────────── - def configure_search_engine(engine: str, api_key: str) -> None: """Write the internet-search engine + key into system_configs. @@ -521,7 +495,7 @@ def configure_search_engine(engine: str, api_key: str) -> None: writing, the MCP subprocess picks up the new key via runtime_env. It can also be changed later in the Web UI under Settings -> System Management. """ - from core.services.system_config import SystemConfigService + from core.config.store.system_config import SystemConfigService svc = SystemConfigService.get_instance() svc.get_all_configs() # touch cache so seed rows exist before set() @@ -540,13 +514,11 @@ def configure_search_engine(engine: str, api_key: str) -> None: updated_by="onboard", ) - # ── Site-building template (React path) ────────────────────────────────────── - def report_site_template(verbose: bool = True) -> bool: """向导里报一句建站模板的状态；铺设本身在安装站点插件时就做了。""" - from core.plugins.local.assets import site_template_home + from features.desktop.local.plugins.assets import site_template_home home = site_template_home() ready = (home / "init-react-site.sh").is_file() @@ -562,12 +534,11 @@ def report_site_template(verbose: bool = True) -> bool: _status(f" ✓ React 建站模板已铺入：{home}（未检测到 Node/npm，首次构建时需要）") return True - def _probe_host_tools() -> dict: """Which optional host tools are present (affects local-mode capabilities).""" import shutil - from core.content.office import find_libreoffice_binary + from core.files.content.office import find_libreoffice_binary return { "node": shutil.which("node") or shutil.which("nodejs"), @@ -579,7 +550,6 @@ def _probe_host_tools() -> dict: "os_sandbox": _os_sandbox_status(), } - def _os_sandbox_status() -> tuple[str, str]: """``(backend name, reason it is unavailable)``; the reason is empty when ready.""" from core.sandbox.permissions.os_gate import backend_name @@ -587,7 +557,6 @@ def _os_sandbox_status() -> tuple[str, str]: return backend_name(), confinement_unavailable_reason() - def _print_capability_summary() -> None: """End-of-onboard note on what host tools gate which capabilities.""" tools = _probe_host_tools() @@ -616,7 +585,6 @@ def _print_capability_summary() -> None: + ("" if not sandbox_reason else f"（{sandbox_reason}：除「完全放开」外的权限档会拒绝执行）") ) - def _configure_aux_model_step( args, *, ptype: str, title: str, ok_label: str, bu, key, model, interactive: bool ) -> None: @@ -641,11 +609,12 @@ def _configure_aux_model_step( except Exception as exc: # noqa: BLE001 print(f"! {ok_label}失败（该能力将不可用）：{exc}", file=sys.stderr) - def cmd_onboard(args) -> int: apply_local_env(args.port) print("HugAgentOS 本地初始化\n" + "─" * 40) - _ensure_schema_and_seed() + from hugagent_cli.schema import ensure_schema_and_seed + + ensure_schema_and_seed() # Step 1 — admin account if args.username and args.password: @@ -795,10 +764,8 @@ def cmd_onboard(args) -> int: print("正在启动服务…\n") return cmd_serve(args) - # ── Serve ───────────────────────────────────────────────────────────────────── - def _open_browser_when_ready(port: int) -> None: url = f"http://127.0.0.1:{port}/" for _ in range(60): @@ -816,7 +783,6 @@ def _open_browser_when_ready(port: int) -> None: except Exception: pass - def bootstrap_default_plugins_if_needed() -> None: """按形态决定要不要装本机默认插件。 @@ -824,7 +790,7 @@ def bootstrap_default_plugins_if_needed() -> None: 壳在双模式下本就不会设引导变量，UOS 壳是无条件设的——判断放在这里，两端和历史 启动脚本才是同一套规矩。 """ - from core.capabilities import device_catalog + from features.desktop.capabilities import device_catalog if os.getenv(_DEFAULT_PLUGIN_BOOTSTRAP_ENV) != "1" or device_catalog.active(): return @@ -838,10 +804,11 @@ def bootstrap_default_plugins_if_needed() -> None: if bootstrapped: _status("✓ 默认插件已就绪：定时任务、技能管理、站点发布") - def cmd_serve(args) -> int: apply_local_env(args.port) - _ensure_schema_and_seed() + from hugagent_cli.schema import ensure_schema_and_seed + + ensure_schema_and_seed() bootstrap_default_plugins_if_needed() import uvicorn from api.app import app @@ -856,10 +823,8 @@ def cmd_serve(args) -> int: uvicorn.run(app, host=host, port=port, log_level="info", access_log=False) return 0 - # ── Doctor ──────────────────────────────────────────────────────────────────── - def cmd_doctor(args) -> int: apply_local_env(args.port) ok = True @@ -930,10 +895,8 @@ def check(label: str, passed: bool, detail: str = "", *, required: bool = True) print("✓ 自检通过" if ok else "✗ 存在需处理的问题（见上）") return 0 if ok else 1 - # ── Entry ───────────────────────────────────────────────────────────────────── - def build_parser() -> argparse.ArgumentParser: # Shared options usable both before and after the subcommand # (`hugagent --host X --port Y serve` and options after `serve` both work). @@ -990,7 +953,6 @@ def build_parser() -> argparse.ArgumentParser: sub.add_parser("doctor", parents=[common], help="环境自检").set_defaults(func=cmd_doctor) return p - def main(argv: Optional[list] = None) -> int: _configure_standard_streams() parser = build_parser() @@ -1020,6 +982,5 @@ def main(argv: Optional[list] = None) -> int: setattr(args, attr, getattr(args, attr, val)) return args.func(args) or 0 - if __name__ == "__main__": sys.exit(main()) diff --git a/src/backend/core/agent_runtime/README.md b/src/backend/core/agent_runtime/README.md new file mode 100644 index 00000000..39de8f50 --- /dev/null +++ b/src/backend/core/agent_runtime/README.md @@ -0,0 +1,18 @@ +# Normal agent execution + +[中文](README.zh-CN.md) + +The public entry is runner.astream_chat_workflow. It selects the main or direct +agent route and closes the full generator chain when a consumer exits. + +- runner / direct: lifecycle, output review, final metadata and post-turn work. +- prepare_main / prepare_direct: build the agent and assemble its input. +- stream_main / stream_direct: adapt model/tool events into the existing event contract. +- main_state / direct_state: typed state owned by one turn, never shared across runs. +- targets, skills, budget, events: routing inputs and focused runtime helpers. +- integrations: memory, compaction, ontology evidence/repair and evolution integration. +- complete: the existing non-streaming entry and result shape. + +Durable ownership and database transactions are outside these phases. +Chat workers in core.chat.runtime consume events and persist them under core.runs +lease fencing. Plan and autonomous Loop composition belong to orchestration. diff --git a/src/backend/core/agent_runtime/README.zh-CN.md b/src/backend/core/agent_runtime/README.zh-CN.md new file mode 100644 index 00000000..298d965b --- /dev/null +++ b/src/backend/core/agent_runtime/README.zh-CN.md @@ -0,0 +1,18 @@ +# 通用智能体执行 + +[English](README.md) + +公开入口为 runner.astream_chat_workflow。它选择主智能体或直接智能体路径， +并在调用方退出时关闭整条生成器链。 + +- runner / direct：运行生命周期、输出评审、最终元数据和响应后的工作。 +- prepare_main / prepare_direct：创建智能体并组装输入。 +- stream_main / stream_direct：把模型和工具事件转换成既有事件协议。 +- main_state / direct_state：单轮持有的有类型状态，不跨任务共享。 +- targets、skills、budget、events：目标选择、技能、预算与事件辅助逻辑。 +- integrations：记忆、压缩、本体证据/修复与演化集成。 +- complete：保留原有非流式执行入口和结果结构。 + +运行所有权与数据库事务在这些阶段之外管理。core.chat.runtime 中的聊天 worker +消费事件，在 core.runs 租约约束下持久化输出。Plan 与自主 Loop 的流程组合 +属于外层 orchestration。 diff --git a/src/backend/core/plugins/resources/__init__.py b/src/backend/core/agent_runtime/__init__.py similarity index 100% rename from src/backend/core/plugins/resources/__init__.py rename to src/backend/core/agent_runtime/__init__.py diff --git a/src/backend/core/agent_runtime/budget.py b/src/backend/core/agent_runtime/budget.py new file mode 100644 index 00000000..8eb040b4 --- /dev/null +++ b/src/backend/core/agent_runtime/budget.py @@ -0,0 +1,74 @@ +"""Read the frozen model and prompt surface a turn's compaction budget is computed from.""" + +from __future__ import annotations + +import logging +from typing import Any, Dict, Tuple + +import core.llm.context.compacting_agent as compacting_agent +import core.models.context_window as context_manager + +logger = logging.getLogger(__name__) + + +def _resolve_agent_model_runtime(agent: Any, fallback_model_name: Any = "") -> Tuple[str, int]: + """Return the effective model name and context window baked into an agent. + + AgentScope chat models expose the upstream name as ``.model``. Some provider + wrappers expose ``.model_name`` instead, so accept both before consulting the + request fallback. Channel runs intentionally omit a model selection; converting + that ``None`` to the literal string ``"None"`` previously made direct sub-agent + conversations fail context-window lookup even though the resolved main model was + correctly configured. + """ + model = getattr(agent, "model", None) + state = getattr(agent, "state", None) + candidates = ( + getattr(model, "model", None), + getattr(model, "model_name", None), + getattr(state, "model_name", None), + fallback_model_name, + ) + model_name = "" + for candidate in candidates: + text = str(candidate or "").strip() + if text and text.lower() != "none": + model_name = text + break + + context_window = int(getattr(model, "context_size", 0) or 0) + if context_window <= 0: + context_window = context_manager.resolve_model_context_window(model_name) + return model_name, context_window + + +def _compaction_budget_inputs(agent: Any, context_window: int) -> Dict[str, Any]: + """Read the exact frozen prompt/tool surface cached by agent_factory.""" + system_prompt, tool_schema = compacting_agent.frozen_prompt_surface(agent) + return { + "system_prompt": system_prompt, + "tool_schema": tool_schema, + "context_window": int(context_window or 0), + "model_name": str( + getattr(getattr(agent, "model", None), "model", None) + or getattr(getattr(agent, "state", None), "model_name", None) + or "" + ), + "model_provider_id": str( + getattr(getattr(agent, "model", None), "resolved_model_provider_id", "") + or getattr(getattr(agent, "state", None), "model_provider_id", "") + or "" + ), + } + + +def _pre_turn_compaction_inputs( + agent: Any, + context_window: int, + resolved_model_name: str, +) -> Dict[str, Any]: + """Adapt the shared budget snapshot to the pre-turn service signature.""" + inputs = _compaction_budget_inputs(agent, context_window) + inputs.pop("model_provider_id", None) + inputs["model_name"] = str(inputs.get("model_name") or resolved_model_name) + return inputs diff --git a/src/backend/core/agent_runtime/direct.py b/src/backend/core/agent_runtime/direct.py new file mode 100644 index 00000000..ad1c105d --- /dev/null +++ b/src/backend/core/agent_runtime/direct.py @@ -0,0 +1,458 @@ +"""Coordinate the direct agent lifecycle.""" + +from __future__ import annotations +from core.agent_runtime.integrations import memory +from core.agent_runtime.integrations import memory_session +from core.ontology import output_review_state +from core.ontology import runtime_pack +from core.runs.tool_effects import errors +from core.llm.citations import allocator + +import asyncio +import logging +import typing as _typing +from contextlib import aclosing as _aclosing +from typing import Any, AsyncIterator, Dict, List, Optional + +import core.agent_runtime.budget as budget +import core.agent_runtime.integrations.evolution as evolution +import core.agent_runtime.integrations.memory_budget as memory_budget +import core.agent_runtime.integrations.ontology_evidence as ontology_evidence +import core.agent_runtime.integrations.ontology_repair as ontology_repair +import core.agent_runtime.targets as targets +import core.chat.message_parser as message_parser +import core.capabilities.catalog.resolver as catalog_resolver + + +from core.agent_runtime import prepare_direct, stream_direct +from core.agent_runtime.direct_state import DirectTurn + +logger = logging.getLogger(__name__) + + +async def _astream_subagent_direct( + *, + agent_id: str, + session_messages: List[Dict[str, Any]], + user_message: str, + context: Dict[str, Any], +) -> AsyncIterator[Dict[str, Any]]: + """Stream a direct conversation with a user-created sub-agent. + + Loads the UserAgent config from DB and uses it to build the agent + with custom system_prompt, MCP tools, skills, KB, and model params. + Shares the same streaming/memory/citation infrastructure as the main route. + """ + turn = DirectTurn(context=context, session_messages=session_messages) + from core.llm.api_runtime.agent import apply_api_scope + from core.harness.registry.evaluation import apply_evaluation_scope + + turn.context = apply_evaluation_scope(apply_api_scope(turn.context)) + + import time as _time + + turn.wf_start = _time.monotonic() + turn.direct_read_only, turn.direct_allow_bash = targets._direct_agent_execution_permissions( + agent_id + ) + + turn.user_agent = targets._load_direct_user_agent( + agent_id, + str(turn.context.get("user_id", "")), + { + "enabled_skill_ids": catalog_resolver.enabled_skill_ids_from_context(turn.context) + or [], + "enabled_mcp_ids": catalog_resolver.enabled_mcp_ids_from_context(turn.context) or [], + "enabled_kb_ids": catalog_resolver.enabled_kb_ids_from_context(turn.context) or [], + "sandbox_tools_enabled": bool(turn.context.get("sandbox_tools_enabled", False)), + "code_capability_enabled": bool(turn.context.get("code_capability_enabled", False)), + }, + ) + from core.harness import log_writer + + _direct_log_started = _time.monotonic() + turn.direct_tool_count = 0 + turn.direct_skill_count = 0 + _direct_log_finished = False + _is_call_subagent_dispatch = turn.context.get("direct_agent_source") == "explicit_command_tool" + import uuid + + from core.infra.task_lifecycle import cancel_and_join, settle_task + + turn.direct_log_id = uuid.uuid4().hex + _direct_finish_task = None + _direct_start_task = asyncio.create_task( + log_writer.start_subagent_log( + { + "id": turn.direct_log_id, + "user_id": str(turn.context.get("user_id", "")) or None, + "chat_id": turn.context.get("chat_id"), + "subagent_name": turn.user_agent.name, + "subagent_type": ( + "user_agent" if _is_call_subagent_dispatch else "user_agent_direct" + ), + "subagent_id": agent_id, + "input_messages": { + "task": user_message, + "invocation": ( + "call_subagent" + if _is_call_subagent_dispatch + else turn.context.get("direct_agent_source") or "dedicated_chat" + ), + }, + } + ) + ) + + async def _finish_direct_log( + status: str, + *, + output: str = "", + error: Optional[str] = None, + usage: Optional[Dict[str, int]] = None, + ) -> None: + nonlocal _direct_log_finished, _direct_finish_task + + async def record(): + await _direct_start_task + await log_writer.finish_subagent_log( + turn.direct_log_id, + status=status, + output_content=output or None, + token_usage=usage, + tool_calls_count=turn.direct_tool_count, + skill_calls_count=turn.direct_skill_count, + error_message=error, + duration_ms=int((_time.monotonic() - _direct_log_started) * 1000), + ) + + if _direct_finish_task is None: + _direct_finish_task = asyncio.create_task(record()) + await asyncio.shield(_direct_finish_task) + _direct_log_finished = True + + _direct_exit_status = "cancelled" + _direct_exit_error = "direct agent execution interrupted" + try: + await asyncio.shield(_direct_start_task) + turn.ontology_runtime = turn.context.get("ontology_runtime") + if not isinstance(turn.ontology_runtime, dict): + turn.ontology_runtime = {} + turn.context["ontology_runtime"] = turn.ontology_runtime + activations = runtime_pack.activate_runtime_for_asset( + turn.ontology_runtime, + kind="subagent", + asset_id=agent_id, + tags=list((turn.user_agent.extra_config or {}).get("ontology_tags") or []), + ) + if activations: + await asyncio.to_thread( + ontology_evidence._record_ontology_activations, + activations, + turn.ontology_runtime, + turn.context, + ) + + # ── [memory] Non-blocking retrieval: background task, skipped on budget timeout ─── + turn.memory_user_id = str(turn.context.get("user_id", "")) + turn.memory_workspace_id = str(turn.context.get("workspace_id", "") or "default") + _mem0_chat_id = turn.context.get("chat_id") or turn.context.get("conversation_id") + turn.memory_enabled = bool(turn.context.get("memory_enabled", False)) + _mem0_write_enabled = bool(turn.context.get("memory_write_enabled", False)) + # Under a team project chats.py passes "team:"; personal/default spaces don't pass it and fall back to the real user_id + _mem0_scope_user_id = str( + turn.context.get("memory_scope_user_id", "") or turn.memory_user_id + ) + logger.info( + "[subagent] user=%s scope=%s ws=%s agent=%s enabled=%s", + turn.memory_user_id, + _mem0_scope_user_id, + turn.memory_workspace_id, + agent_id, + turn.memory_enabled, + ) + + turn.session_memory = await memory_session.open_session_memory( + chat_id=_mem0_chat_id, + scope_user_id=_mem0_scope_user_id, + workspace_id=turn.memory_workspace_id, + user_message=user_message, + memory_enabled=turn.memory_enabled, + # The retrieval budget is part of the orchestration profile: how long + # this task type is willing to wait for memory is an assembly decision, + # not a global constant. Resolved here because retrieval starts before + # the agent is built. + budget_ms=memory_budget._profile_memory_budget_ms(turn.context), + ) + + warnings: _typing.List[str] = [] + turn.full_response = "" + turn.displayed_tools: set[str] = set() + turn.all_citations: _typing.List[_typing.Dict[str, _typing.Any]] = [] + turn.ontology_event_cursor = 0 + turn.ontology_trace: _typing.List[_typing.Dict[str, _typing.Any]] = [] + # 证据锚点发号器：跨轮续号；创建后绑到 agent 上（见下方 attach_allocator）， + # 中间件与本函数由此共享同一个计数器 + turn.anchor_allocator = allocator.AnchorAllocator( + await asyncio.to_thread( + allocator.anchor_start_for_chat, + str(turn.context.get("chat_id") or "") or None, + ) + ) + + try: + async with _aclosing(prepare_direct.prepare_direct(turn)) as phase: + async for event in phase: + yield event + async with _aclosing(stream_direct.stream_direct(turn)) as phase: + async for event in phase: + yield event + + except Exception as e: + import traceback + + logger.error("subagent_stream_error: %s\n%s", e, traceback.format_exc()) + warnings.append(f"Streaming error: {str(e)[:200]}") + + unknown_outcome = errors.find_tool_outcome_unknown(e) + if unknown_outcome is not None: + if turn.streaming_agent is not None: + await turn.streaming_agent.shutdown() + await _finish_direct_log("failed", error=str(unknown_outcome)[:200]) + if unknown_outcome is e: + raise + raise unknown_outcome from e + + if not turn.full_response: + # No answer was produced: the caller gets the failure itself. Tool + # results already streamed stay visible; the run is recorded as + # failed with a structured error instead of a fabricated reply that + # would read as a completed answer. + if turn.streaming_agent is not None: + await turn.streaming_agent.shutdown() + await _finish_direct_log("failed", error=str(e)[:200]) + raise + + pending_ontology_events = turn.ontology_runtime.get("runtime_events", [])[ + turn.ontology_event_cursor : + ] + for ontology_event in pending_ontology_events: + yield dict(ontology_event) + turn.ontology_event_cursor += len(pending_ontology_events) + + _ontology_review_owner_id = ontology_evidence._ontology_review_owner( + turn.ontology_runtime, turn.context + ) + _ontology_review_claimed = bool( + turn.full_response + and output_review_state.claim_output_review( + turn.ontology_runtime, owner=_ontology_review_owner_id + ) + ) + if _ontology_review_claimed: + turn.review_owner = _ontology_review_owner_id + from core.ontology.reviewer import review_ontology_output + + yield { + "type": "ontology_review", + "status": "started", + "level": turn.ontology_runtime.get("review_level", "checkpoint"), + **ontology_evidence._ontology_review_event_context(turn.ontology_runtime), + } + repair_event_queue: asyncio.Queue[_typing.Dict[str, _typing.Any]] = asyncio.Queue() + + async def _remediate(payload: Dict[str, Any]) -> str: + primary_usage = await turn.streaming_agent.aget_usage() + primary_context_usage = turn.streaming_agent.get_context_usage(primary_usage) + try: + repaired, _, cursor, tool_count = ( + await ontology_repair._run_ontology_repair_round( + streaming_agent=turn.streaming_agent, + context=turn.context, + payload=payload, + runtime=turn.ontology_runtime, + trace=turn.ontology_trace, + citations=turn.all_citations, + allocator=turn.anchor_allocator, + event_cursor=turn.ontology_event_cursor, + subagent_log_id=turn.direct_log_id, + event_sink=repair_event_queue.put, + ) + ) + finally: + turn.streaming_agent.restore_context_usage(primary_context_usage) + turn.ontology_event_cursor = cursor + turn.direct_tool_count += tool_count + return repaired + + turn.review_task = asyncio.create_task( + review_ontology_output( + task=user_message, + answer=turn.full_response, + runtime=turn.ontology_runtime, + trace=turn.ontology_trace, + citations=turn.all_citations, + user_id=str(turn.context.get("user_id", "")), + chat_id=turn.context.get("chat_id"), + model_name=str(turn.context.get("model_name", "") or "") or None, + model_provider_id=str(turn.context.get("model_provider_id", "") or "") or None, + remediate=_remediate, + ) + ) + try: + while not turn.review_task.done() or not repair_event_queue.empty(): + try: + repair_event = await asyncio.wait_for( + repair_event_queue.get(), + timeout=0.1, + ) + except asyncio.TimeoutError: + continue + yield repair_event + review = await turn.review_task + except (asyncio.CancelledError, GeneratorExit): + await cancel_and_join(turn.review_task) + output_review_state.release_output_review( + turn.ontology_runtime, owner=_ontology_review_owner_id + ) + await turn.streaming_agent.shutdown() + raise + except Exception as exc: # noqa: BLE001 + if not turn.review_task.done(): + turn.review_task.cancel() + review = ontology_evidence._ontology_review_failure_result(turn.full_response, exc) + output_review_state.complete_output_review( + turn.ontology_runtime, + owner=_ontology_review_owner_id, + verdict=str(review.get("verdict") or "unknown"), + attempts=int(review.get("attempts") or 1), + ) + turn.ontology_runtime.setdefault("output_review", {}).update( + { + "revised": bool(review.get("revised")), + "annotated": bool(review.get("annotated")), + "repair_attempts": int(review.get("repair_attempts") or 0), + "violations": review.get("violations") or [], + "affected_claims": review.get("affected_claims") or [], + "evidence": review.get("evidence") or [], + "feedback": review.get("feedback") or [], + "manual_review": review.get("manual_review") or {}, + "candidate_answer": (review.get("answer") if review.get("revised") else ""), + "new_tools": review.get("new_tools") or [], + "new_citation_count": int(review.get("new_citation_count") or 0), + "latency_ms": review.get("latency_ms"), + } + ) + yield { + "type": "ontology_review", + "status": "completed", + "level": turn.ontology_runtime.get("review_level", "checkpoint"), + "verdict": review["verdict"], + "revised": bool(review.get("revised")), + "annotated": bool(review.get("annotated")), + "repair_attempts": int(review.get("repair_attempts") or 0), + "candidate_answer": review.get("answer") if review.get("revised") else "", + "manual_review": review.get("manual_review") or {}, + "violations": review.get("violations") or [], + "affected_claims": review.get("affected_claims") or [], + "evidence": review.get("evidence") or [], + "feedback": review.get("feedback") or [], + "new_tools": review.get("new_tools") or [], + "new_citation_count": int(review.get("new_citation_count") or 0), + "latency_ms": review.get("latency_ms"), + **ontology_evidence._ontology_review_event_context(turn.ontology_runtime), + } + + _direct_usage = await turn.streaming_agent.aget_usage() + _direct_context_usage = turn.streaming_agent.get_context_usage(_direct_usage) + await turn.streaming_agent.shutdown() + await _finish_direct_log( + "success", + output=turn.full_response, + usage=_direct_usage, + ) + yield { + "type": "meta", + "route": f"subagent:{agent_id}", + "is_markdown": message_parser.looks_markdown(turn.full_response), + "sources": message_parser.resolve_sources_conflict([]), + "artifacts": [], + "warnings": warnings, + "citations": turn.all_citations, + "usage": _direct_usage, + "context_usage": _direct_context_usage, + # Internal-only handoff consumed by chat_run_executor. Its whitelist + # intentionally keeps this prompt/tool surface out of SSE and message + # metadata while allowing post-turn compaction to use the exact same + # frozen inputs as pre-turn compaction. + "_compaction_budget_inputs": budget._compaction_budget_inputs( + turn.agent, turn.ctx_window + ), + "ontology_governance": ontology_evidence._ontology_governance_summary( + turn.ontology_runtime + ), + # Same contract as the main route: memory writes are scheduled below, + # so the frame can only announce "settling" for the client's skeleton + # card. Without this marker subagent turns never showed their card. + "evolution_pending": evolution._evolution_pending_marker( + turn.context, _mem0_write_enabled + ), + } + + # ── [memory] Post-response pipeline (SSE already closed, user isn't waiting) ──── + # No memory is written unless the user opted into memory_write_enabled (first gate). + if _mem0_write_enabled: + memory.save_memories_background( + turn.memory_user_id, + user_message, + turn.full_response, + _mem0_write_enabled, + workspace_id=turn.memory_workspace_id, + chat_id=_mem0_chat_id, + scope_user_id=_mem0_scope_user_id, + message_id=str(turn.context.get("message_id") or ""), + ) + else: + logger.debug( + "[subagent] memory save skipped: write_enabled=False (user=%s)", + turn.memory_user_id, + ) + + # ── [evolution] Evidence assembly — same contract as the main route. Without + # registration the settlement runner parks the memory report on a 45s + # watchdog before settling, which pushed subagent cards past the client's + # polling budget. + if not turn.context.get("agent_api_scope") and not turn.context.get("evaluation_session"): + evolution._assemble_episode_background( + message_id=str(turn.context.get("message_id") or ""), + run_id=str(turn.context.get("run_id") or ""), + chat_id=str(turn.context.get("chat_id") or ""), + user_id=str(turn.context.get("user_id") or ""), + objective=user_message, + agent=turn.streaming_agent, + memory_task=turn.session_memory.retrieval_task, + latency_ms=None, + memory_write_enabled=_mem0_write_enabled, + ) + except BaseException as exc: + if not isinstance(exc, (asyncio.CancelledError, GeneratorExit)): + _direct_exit_status = "failed" + _direct_exit_error = str(exc)[:200] + raise + finally: + try: + if turn.review_task is not None: + await cancel_and_join(turn.review_task) + if turn.review_owner is not None: + output_review_state.release_output_review(turn.ontology_runtime, owner=turn.review_owner) + if turn.direct_stream is not None: + await turn.direct_stream.aclose() + if not _direct_log_finished and turn.streaming_agent is not None: + await turn.streaming_agent.shutdown() + finally: + if not _direct_log_finished: + await settle_task( + asyncio.create_task( + _finish_direct_log(_direct_exit_status, error=_direct_exit_error) + ) + ) diff --git a/src/backend/core/agent_runtime/direct_state.py b/src/backend/core/agent_runtime/direct_state.py new file mode 100644 index 00000000..025e3ce9 --- /dev/null +++ b/src/backend/core/agent_runtime/direct_state.py @@ -0,0 +1,46 @@ +"""State shared only by the direct agent execution phases.""" + +from __future__ import annotations + +import asyncio +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any, AsyncIterator + +if TYPE_CHECKING: + from core.agent_runtime.integrations.memory_session import SessionMemory + from core.llm.citations.allocator import AnchorAllocator + from core.llm.runtime.streaming import StreamingAgent + + +@dataclass +class DirectTurn: + context: dict[str, Any] + session_messages: list[dict[str, Any]] + review_owner: str | None = None + anchor_allocator: AnchorAllocator | None = None + ctx_window: int = 0 + direct_allow_bash: bool = False + direct_log_id: str = "" + direct_read_only: bool = False + direct_skill_count: int = 0 + direct_stream: AsyncIterator | None = None + direct_tool_count: int = 0 + memory_enabled: bool = False + memory_user_id: str = "" + memory_workspace_id: str = "default" + ontology_event_cursor: int = 0 + ontology_runtime: dict[str, Any] = field(default_factory=dict) + ontology_trace: list[dict[str, Any]] = field(default_factory=list) + session_memory: SessionMemory | None = None + wf_start: float = 0.0 + agent: Any = None + all_citations: list[dict[str, Any]] = field(default_factory=list) + displayed_tools: set[str] = field(default_factory=set) + full_response: str = "" + mcp_clients: list[Any] = field(default_factory=list) + review_task: asyncio.Task | None = None + skill_id_by_tool_id: dict[str, str] = field(default_factory=dict) + skill_load_ids: set[str] = field(default_factory=set) + streaming_agent: StreamingAgent | None = None + user_agent: Any = None + view_text_file_args: dict[str, dict[str, Any]] = field(default_factory=dict) diff --git a/src/backend/core/agent_runtime/events.py b/src/backend/core/agent_runtime/events.py new file mode 100644 index 00000000..92fc500d --- /dev/null +++ b/src/backend/core/agent_runtime/events.py @@ -0,0 +1,71 @@ +"""Helpers for the agent event stream: synthesized tool calls, scoped log context, live-job checks.""" + +from __future__ import annotations +from core.harness import log_context + +import logging +from typing import Any, Dict, Optional + +import core.config.display_names as display_names + +logger = logging.getLogger(__name__) + + +def _synthesize_missing_tool_call( + tool_id: str, tool_name: str, displayed_tools: set +) -> Optional[Dict[str, Any]]: + """Build the ``tool_call`` event a tool never got, or ``None`` if it already has one. + + ``_tool_args_ready`` holds back the tool card while streamed args are still + incomplete, but it cannot tell "not written yet" from "there is nothing to + write": a zero-argument tool (for example, a parameterless status probe) keeps + empty args forever, so its card is suppressed for the whole run. Without a + card the frontend has nothing to attach the result to and — with tools + running in parallel — binds it onto whichever sibling is still running, + making the call vanish from the tool list and briefly corrupting the + sibling's output. Emitting the missing call right before its result keeps + every executed tool visible, in order. + + The caller must pass the already-resolved (post skill-load override) tool + name, and must have skipped tools that intentionally render no card. + """ + if not tool_id or tool_id in displayed_tools: + return None + displayed_tools.add(tool_id) + display_name = ( + "加载技能" + if tool_name == "load_skill" + else display_names.TOOL_DISPLAY_NAMES.get(tool_name, tool_name) + ) + return { + "type": "tool_call", + "tool_name": tool_name, + "tool_display_name": display_name, + "tool_args": {}, + "input": {}, + "tool_id": tool_id, + } + + +async def _anext_in_subagent_log_scope(iterator, subagent_log_id: str): # noqa: ANN001 + """Advance one agent-stream item without leaking log tokens across SSE yields.""" + + with log_context.subagent_scope(subagent_log_id, source="subagent"): + return await anext(iterator) + + +def _chat_has_live_job(chat_id: str) -> bool: + """这个会话还有在跑（或待启动）的后台作业吗 —— 决定计划栏本轮该不该收尾。""" + try: + from core.db.engine import SessionLocal as _JobSession + from core.db.models import Job as _Job + + with _JobSession() as _db: + return ( + _db.query(_Job.job_id) + .filter(_Job.chat_id == chat_id, _Job.status.in_(("pending", "running"))) + .first() + is not None + ) + except Exception: # noqa: BLE001 —— 查不到就当没有作业，照常收尾，别把计划栏永远挂着 + return False diff --git a/src/backend/core/agent_runtime/host.py b/src/backend/core/agent_runtime/host.py new file mode 100644 index 00000000..5ca7ad3c --- /dev/null +++ b/src/backend/core/agent_runtime/host.py @@ -0,0 +1,11 @@ +"""Compose the top-level chat flow with durable compaction boundaries.""" + +def stream_chat_workflow(**kwargs): + from core.agent_runtime.integrations.compaction_stream import stream_with_compaction + from core.agent_runtime.runner import astream_chat_workflow + return stream_with_compaction(astream_chat_workflow, **kwargs) + + +def register(): + from core.harness.registry.chat_execution import register_chat_execution + register_chat_execution(stream_chat_workflow) diff --git a/src/backend/tests/capabilities/__init__.py b/src/backend/core/agent_runtime/integrations/__init__.py similarity index 100% rename from src/backend/tests/capabilities/__init__.py rename to src/backend/core/agent_runtime/integrations/__init__.py diff --git a/src/backend/orchestration/compaction_stream.py b/src/backend/core/agent_runtime/integrations/compaction_stream.py similarity index 95% rename from src/backend/orchestration/compaction_stream.py rename to src/backend/core/agent_runtime/integrations/compaction_stream.py index 83b2495e..143b9826 100644 --- a/src/backend/orchestration/compaction_stream.py +++ b/src/backend/core/agent_runtime/integrations/compaction_stream.py @@ -7,9 +7,9 @@ from contextlib import aclosing, suppress from dataclasses import dataclass -from core.llm.context_safety import request_admission_scope -from core.services.compaction_progress import progress_scope, report_outcome -from core.services.context_budget_checkpoint import ContextStopCheckpoint +from core.llm.context.safety import request_admission_scope +from core.chat.compaction.progress import progress_scope, report_outcome +from core.chat.compaction.budget_checkpoint import ContextStopCheckpoint logger = logging.getLogger(__name__) @@ -51,7 +51,7 @@ async def produce(): ): if is_compact_command(raw_user_message): from core.config.settings import settings - from core.services.compaction_service import run_compaction + from core.chat.compaction.service import run_compaction if not settings.compaction.enabled: await report_outcome(chat_id, run_id, "manual", "skipped", "disabled") diff --git a/src/backend/core/agent_runtime/integrations/evolution.py b/src/backend/core/agent_runtime/integrations/evolution.py new file mode 100644 index 00000000..5310da96 --- /dev/null +++ b/src/backend/core/agent_runtime/integrations/evolution.py @@ -0,0 +1,160 @@ +"""Hand a finished turn's evidence to the evolution loop after the stream closes.""" + +from __future__ import annotations + +import logging +from typing import Any, Dict, Optional + +logger = logging.getLogger(__name__) + + +def _evolution_pending_marker(context: Dict[str, Any], memory_write_enabled: bool): + """Marker for the closing frame so the client can draw the skeleton card. + + The turn's real outcome is not knowable yet — memory writes are scheduled + after the stream closes — so all we can honestly say here is "settling, and + these are the mechanisms that might report". + """ + message_id = str(context.get("message_id") or "") + if not message_id or not memory_write_enabled: + # Without memory writes there is nothing a turn can settle to, so there + # is nothing for the client to watch. + return None + try: + from core.evolution.settlement.settlement import pending_payload + + return pending_payload(message_id=message_id) + except Exception: # pragma: no cover - defensive + return None + + +def _runtime_evidence_agent(agent: Any) -> Any: + """Unwrap streaming/decorator agents to the owner of frozen run evidence.""" + current = agent + seen: set[int] = set() + while current is not None and id(current) not in seen: + seen.add(id(current)) + if hasattr(current, "_jx_asset_bundle") or hasattr(current, "_jx_skill_selection"): + return current + nested = getattr(current, "agent", None) + if nested is None or nested is current: + break + current = nested + return current + + +def _assemble_episode_background( + *, + message_id: str, + run_id: str, + chat_id: str, + user_id: str, + objective: str, + agent: Any = None, + memory_task: Any = None, + latency_ms: Optional[int] = None, + memory_write_enabled: bool = False, +) -> None: + """Kick off Episode assembly after the stream closed. + + Fire-and-forget by design: the user already has their answer, and evidence + assembly must never be able to delay or fail a turn. Every failure mode + here degrades to "no Episode for this run", which the console shows + honestly rather than papering over. + """ + if not message_id: + return + + try: + from core.agent_runtime.integrations.memory import get_last_retrieval + from core.evolution.evidence.events import EV_MEMORY_RETRIEVED, TraceSink + from core.evolution.evidence.trace_assembler import assemble_episode + + evidence_agent = _runtime_evidence_agent(agent) + bundle = ( + getattr( + evidence_agent, + "evidence_bundle", + getattr(evidence_agent, "_jx_asset_bundle", None), + ) + if evidence_agent is not None + else None + ) + + sink = TraceSink(run_id=run_id, message_id=message_id, chat_id=chat_id, user_id=user_id) + # The rendered memory block has already discarded ids, scores and ranks; + # this is the only place they still exist. + retrieval = get_last_retrieval(memory_task) + if retrieval is not None: + sink.append( + EV_MEMORY_RETRIEVED, + retrieval.to_event_payload(), + asset_kind="memory", + ) + + # Which skills were considered, chosen, and rejected — the rejects are + # the negative examples skill attribution is built on. + # Ordered tool calls, read back from the log this run already wrote. + # Without them an episode carries no tool sequence and the promotion + # chain has nothing to distil. + from core.evolution.evidence.trace_assembler import emit_tool_events + + emit_tool_events(sink, message_id, chat_id) + + selection = ( + getattr(evidence_agent, "_jx_skill_selection", None) + if evidence_agent is not None + else None + ) + if selection is not None: + from core.evolution.evidence.events import EV_SKILL_SELECTED + + sink.append(EV_SKILL_SELECTED, selection.to_event_payload(), asset_kind="skill") + sink.flush() + + # Stamp the user's contribution choice onto the episode. Doing it at + # write time means the decision is fixed by what the user had agreed to + # *at that moment*, rather than being re-evaluated later against a + # setting they may since have changed. + from core.evolution.profiles.user_settings import resolve_for_user + + privacy_class = resolve_for_user(user_id).get("privacy_class", "tenant") + + episode_id = assemble_episode( + message_id=message_id, + run_id=run_id, + chat_id=chat_id, + user_id=user_id, + objective=objective, + bundle=bundle, + latency_ms=latency_ms, + privacy_class=privacy_class, + ) + + # Hand the evidence side to the settlement runner. It settles once the + # memory write pipeline reports what it actually wrote — the only party + # that knows. Settling here instead would have to guess that number, and + # the guess it used to make (the *retrieval* count) described what the + # turn read, not what it learned. + from core.evolution.settlement.settlement_runner import register_turn + + # Which skills the turn opened and which profile governed it stay on the + # episode, where the retirement engine reads them. They are not on the + # card: they describe the system doing its job, not learning anything. + register_turn( + message_id=message_id, + episode_id=episode_id or "", + expects_memory=memory_write_enabled, + ) + + # Consider running a cycle now that new evidence exists. The check is a + # single indexed count and almost always says no; when it says yes the + # cycle runs off-thread. The user already has their answer either way. + from core.evolution.cycle.trigger import schedule_if_due, schedule_personal + + # This user's own loop runs in every edition; the fleet-wide one is + # enterprise-gated and no-ops elsewhere. + schedule_personal(user_id) + schedule_if_due() + except Exception as exc: # pragma: no cover - defensive + logger.debug("[evolution] episode assembly skipped: %s", exc) diff --git a/src/backend/core/agent_runtime/integrations/memory.py b/src/backend/core/agent_runtime/integrations/memory.py new file mode 100644 index 00000000..c599ebae --- /dev/null +++ b/src/backend/core/agent_runtime/integrations/memory.py @@ -0,0 +1,304 @@ +"""Memory retrieval for a turn: background recall, the frozen memory block and post-turn saving.""" +from __future__ import annotations +import asyncio +from dataclasses import dataclass +from typing import Optional +from core.config.settings import settings +from core.memory import profile +from core.memory.context import MemoryContext +from core.memory.retrieval_types import MemoryRetrievalResult +from core.memory.service import retrieve_memories_structured +import logging +from core.agent_runtime.integrations import memory_state + +logger = logging.getLogger(__name__) + +async def launch_memory_retrieval( + user_id: str, + user_message: str, + memory_enabled: bool, + *, + workspace_id: str = "default", + budget_ms: Optional[int] = None, +) -> Optional[asyncio.Task]: + """Start the Fact vector retrieval task in the background and return the Task (the caller need not await immediately). + + budget_ms defaults to settings.memory.retrieval_budget_ms (600ms). Called once by the + workflow at session start; if it completes within budget the frozen block is injected, otherwise skipped. + """ + if not memory_enabled or not user_id: + return None + + effective_budget = ( + budget_ms if budget_ms is not None else settings.memory.retrieval_budget_ms + ) / 1000.0 + + # The assembly boundary below owns the latency budget. Passing the same + # budget into the service would make its inner wait_for cancel _do_search + # before build_frozen_memory_block() can shield the task. + + async def _fetch() -> Optional[MemoryRetrievalResult]: + try: + return await retrieve_memories_structured( + user_id=user_id, + query=user_message, + workspace_id=workspace_id, + timeout_s=None, + ) + except Exception as exc: + logger.warning("[memory] retrieval failed: %s", exc) + return MemoryRetrievalResult.degraded_result("orchestration_error") + + return track_memory_retrieval(asyncio.create_task(_fetch()), budget_s=effective_budget) + + +def as_result(value: object) -> Optional[MemoryRetrievalResult]: + """Normalise whatever the retrieval task produced. + + The task may legitimately yield ``None`` (never started) and, for callers + that still hand us a pre-rendered string, we degrade gracefully rather than + crash the turn. + """ + if isinstance(value, MemoryRetrievalResult): + return value + return None + + +def track_memory_retrieval(task: asyncio.Task, *, budget_s: Optional[float] = None) -> asyncio.Task: + """Attach lifecycle observation without taking ownership of cancellation.""" + + memory_state.process_state().retrieval_states[task] = "running" + if budget_s is not None: + memory_state.process_state().retrieval_budgets[task] = max(0.0, budget_s) + + def _finished(done: asyncio.Task) -> None: + previous = memory_state.process_state().retrieval_states.get(done) + if done.cancelled(): + memory_state.process_state().retrieval_states[done] = "cancelled" + return + try: + value = done.result() + except Exception: + memory_state.process_state().retrieval_states[done] = "failed" + return + result = as_result(value) + if result is not None: + set_last_retrieval(done, result) + memory_state.process_state().retrieval_states[done] = ( + "completed_after_timeout" if previous == "timed_out_running" else "completed" + ) + + task.add_done_callback(_finished) + return task + + +def get_retrieval_state(task: Optional[asyncio.Task]) -> Optional[str]: + if task is None: + return None + return memory_state.process_state().retrieval_states.get(task) + + +def set_last_retrieval(task: asyncio.Task, result: MemoryRetrievalResult) -> None: + try: + memory_state.process_state().last_retrieval[task] = result + except TypeError: # pragma: no cover - non-weakrefable task + pass + + +def get_last_retrieval(task: Optional[asyncio.Task]) -> Optional[MemoryRetrievalResult]: + """Structured recall for a turn, for trace-event emission. + + Returns ``None`` when retrieval never ran or timed out — callers must treat + that as "no evidence", never as "memory contributed nothing". + """ + if task is None: + return None + try: + return memory_state.process_state().last_retrieval.get(task) + except TypeError: # pragma: no cover + return None + + +@dataclass(frozen=True) +class FrozenMemoryBlock: + """An assembled frozen block plus whether the read behind it actually landed. + + ``settled`` is false when the L2 read timed out or came back degraded. Such + a block is missing facts for reasons that say nothing about what the user's + memory holds, so it must not be frozen for the remainder of the session. + """ + + text: str + settled: bool + + +async def build_frozen_memory_block( + user_id: str, + workspace_id: str, + memory_task: Optional[asyncio.Task], + *, + memory_enabled: bool = True, +) -> FrozenMemoryBlock: + """Assemble the "session-frozen" block = L1 Profile markdown + L2 Fact top-K. + + - When `memory_enabled=False`, return empty immediately (**do not load unless the user + has enabled persistent memory** — forms defense-in-depth with the workflow layer's + `_mem0_enabled` check) + - Profile reads the DB (fast, <20ms), always awaited + - Fact takes the result from the already-started memory_task; if the task hasn't finished, + wait briefly; if still unfinished, give up on Fact injection for this round (never block agent startup) + + Called once per chat by `resolve_session_memory()`; later turns replay the + stored snapshot instead of reassembling. + """ + if not memory_enabled: + return FrozenMemoryBlock(text="", settled=False) + + # Profile layer (L1) + profile_md = "" + if user_id: + try: + profile_md = await profile.get(user_id, workspace_id) + except Exception as exc: + logger.warning( + "[memory] profile fetch failed user=%s ws=%s: %s", user_id, workspace_id, exc + ) + + # Fact layer (L2). With no retrieval task there is no L2 read to wait for, + # so a Profile-only block is already the complete answer for this session. + fact_text = "" + fact_settled = True + fact_result: Optional[MemoryRetrievalResult] = None + if memory_task is not None: + fact_settled = False + try: + # Wait up to retrieval_budget_ms (default 600ms), then give up; memory_task was + # started before the agent was created, so in most cases it is nearly done by now. + # The old value of 50ms was measured to be far below Milvus warm search's ~200ms, + # so Fact injection almost never hit. + wait_budget_s = memory_state.process_state().retrieval_budgets.get( + memory_task, max(0.1, settings.memory.retrieval_budget_ms / 1000.0) + ) + fact_result = as_result( + await asyncio.wait_for(asyncio.shield(memory_task), timeout=wait_budget_s) + ) + fact_text = fact_result.to_text() if fact_result is not None else "" + # A degraded result means the store was unreachable or the budget + # blew inside the service — an empty answer we must not freeze. + fact_settled = fact_result is not None and not fact_result.degraded + # Stash the structured recall on the task so the workflow can emit a + # retrieval trace event without re-running the search. Attribution + # needs ids/ranks/scores that the rendered text has already lost. + if fact_result is not None: + set_last_retrieval(memory_task, fact_result) + except asyncio.TimeoutError: + if not memory_task.done(): + memory_state.process_state().retrieval_states[memory_task] = "timed_out_running" + logger.info( + "[memory] fact retrieval still running past wait window, skipping injection" + ) + # ``shield`` makes the continue-in-background policy explicit. The + # done callback retains the structured result and changes the + # observable state to completed_after_timeout. + except asyncio.CancelledError: + raise + except Exception as exc: + logger.warning("[memory] fact retrieval await failed: %s", exc) + + if not profile_md and not fact_text: + return FrozenMemoryBlock(text="", settled=fact_settled) + + parts: list[str] = ["## 关于当前用户的已知背景（会话开始时冻结）"] + if profile_md: + parts.append("") + parts.append("### 用户档案（Profile）") + parts.append(profile_md.strip()) + if fact_text: + parts.append("") + parts.append("### 相关历史记忆（Fact Top-K）") + # retrieve_memories comes with its own "## 关于该用户..." prefix; strip it to avoid duplicate headings + stripped = fact_text + for h in ("## 关于该用户的已知背景信息（来自历史会话记忆）", "## 用户相关实体关系"): + stripped = stripped.replace(h, "") + parts.append(stripped.strip()) + block = "\n".join(parts).strip() + logger.info( + "[memory] frozen block built user=%s ws=%s chars=%d profile=%d facts=%d settled=%s", + user_id, + workspace_id, + len(block), + len(profile_md or ""), + len(fact_text or ""), + fact_settled, + ) + return FrozenMemoryBlock(text=block, settled=fact_settled) + + +def save_memories_background( + user_id: str, + user_message: str, + full_response: str, + write_enabled: bool, + *, + workspace_id: str = "default", + chat_id: Optional[str] = None, + scope_user_id: Optional[str] = None, + message_id: Optional[str] = None, +) -> None: + """Delegate to the unified post-response pipeline — never await; SSE is closed and the user isn't waiting. + + **When the user has not explicitly consented to writes (`write_enabled=False`), skip + immediately and create no tasks at all**. This is the first gate of the user-level write + switch; `schedule_post_response_tasks` has a second one inside. + + Inside `schedule_post_response_tasks`: + - a durable outbox row is committed before the function returns + - leased workers resume pending/retry rows after process restart + - global Semaphore bounds worker concurrency (default 8) + - runs 0-5 extractors picked by the router's substance floor + LLM write gate + - each extractor has its own 30s timeout + - sanitize → write L1/L2/Session → audit + + ``scope_user_id`` optionally selects an edition-owned shared memory bucket. + ``user_id`` remains the real user for audit metadata. + """ + if not (write_enabled and full_response and user_id): + report_no_memory_writes(message_id) + return + + try: + from core.memory.pipeline import schedule_post_response_tasks + except Exception as exc: + logger.warning("[memory] pipeline unavailable, skipping save: %s", exc) + report_no_memory_writes(message_id) + return + + ctx = MemoryContext( + user_id=user_id, + workspace_id=workspace_id, + chat_id=chat_id, + write_enabled=write_enabled, + scope_user_id=scope_user_id, + message_id=message_id, + ) + try: + schedule_post_response_tasks(ctx, user_message, full_response) + except Exception as exc: + logger.warning("[memory] schedule_post_response_tasks failed: %s", exc) + report_no_memory_writes(message_id, failed=True) + + +def report_no_memory_writes(message_id: Optional[str], *, failed: bool = False) -> None: + """Tell the turn's settlement that no write pipeline will report for it. + + Every path that declines to schedule the pipeline must say so, otherwise the + turn's card waits out the settlement watchdog for nothing. + """ + if not message_id: + return + try: + from core.evolution.settlement.settlement_runner import report_memory_writes + + report_memory_writes(message_id, items=[], failed=failed) + except Exception as exc: # noqa: BLE001 - never break the response path + logger.debug("[memory] settlement report skipped: %s", exc) diff --git a/src/backend/core/agent_runtime/integrations/memory_budget.py b/src/backend/core/agent_runtime/integrations/memory_budget.py new file mode 100644 index 00000000..b035af85 --- /dev/null +++ b/src/backend/core/agent_runtime/integrations/memory_budget.py @@ -0,0 +1,34 @@ +"""How long a turn waits for memory retrieval, per the active agent profile.""" + +from __future__ import annotations + +import logging +from typing import Any, Dict, Optional + +logger = logging.getLogger(__name__) + + +def _profile_memory_budget_ms(context: Dict[str, Any]) -> Optional[int]: + """How long this task type waits for memory, per the active profile. + + Returns ``None`` when nothing is published, which leaves the caller on the + deployment default — the built-in profile carries that same value, so this + is a no-op until a profile deliberately changes it. + + Never raises: retrieval sits on the user's critical path, and an orchestration + lookup must not be able to fail a turn. + """ + try: + from core.auth.tenancy import tenant_of + from core.evolution.profiles.profile_selection import load_active_profile + + user_id = str(context.get("user_id") or "") + profile = load_active_profile( + task_type=str(context.get("chat_mode") or "chat"), + user_id=user_id, + tenant_id=tenant_of(user_id), + ) + return profile.memory_budget_ms + except Exception as exc: # noqa: BLE001 + logger.debug("[memory] profile budget unavailable: %s", exc) + return None diff --git a/src/backend/core/agent_runtime/integrations/memory_identity.py b/src/backend/core/agent_runtime/integrations/memory_identity.py new file mode 100644 index 00000000..a587d4ea --- /dev/null +++ b/src/backend/core/agent_runtime/integrations/memory_identity.py @@ -0,0 +1,74 @@ +"""Build and cache the per-user identity block injected at the head of a chat.""" +from __future__ import annotations +import asyncio +from dataclasses import dataclass, field +from time import monotonic +from typing import Dict +import logging +from core.agent_runtime.integrations import memory_state + +logger = logging.getLogger(__name__) + +async def build_user_identity_block(user_id: str) -> str: + """Assemble the "current user" identity block (username / nickname) for injection into the session-frozen message. + + Deliberately goes through the user-role frozen block rather than the end of the system prompt: + the server-side chat template renders the tool schemas **after** the system text, so any + per-user bytes in system split the LLM prefix cache of the tool section (the largest chunk + of the prompt). Measured: identical system differing only in the trailing username → TTFT + degrades from 0.7s back to cold-start levels. The frozen block sits at the start of the + message sequence and renders after the tool section, so the system+tools shared prefix stays intact. + + Returns empty string for anonymous / unknown users. Result cached per user_id for 60s. + """ + if not user_id or user_id == "anonymous": + return "" + now = monotonic() + hit = process_state().identity_cache.get(user_id) + if hit and now < hit[0]: + return hit[1] + + def _query() -> tuple: + from core.db.engine import SessionLocal + from core.db.models import LocalUser, UserShadow + + with SessionLocal() as db: + row = db.query(UserShadow.username).filter(UserShadow.user_id == user_id).first() + nick = db.query(LocalUser.nickname).filter(LocalUser.user_id == user_id).first() + return ( + (row[0] or "").strip() if row else "", + (nick[0] or "").strip() if nick else "", + ) + + try: + username, nickname = await asyncio.to_thread(_query) + except Exception as exc: + logger.warning("[identity] user lookup failed user=%s: %s", user_id, exc) + return "" + + lines: list[str] = [] + if username and username != "anonymous": + lines.append(f"- 用户名：{username}") + if nickname and nickname != username: + lines.append(f"- 昵称：{nickname}") + block = "" + if lines: + block = ( + "## 当前用户\n" + + "\n".join(lines) + + "\n需要称呼用户时，用上述昵称（无昵称则用用户名）自然称呼。" + ) + process_state().identity_cache[user_id] = (now + memory_state.IDENTITY_CACHE_TTL_S, block) + return block + + +@dataclass +class MemoryState: + identity_cache: Dict[str, tuple] = field(default_factory=lambda: {}) + + +_STATE = MemoryState() + + +def process_state() -> MemoryState: + return _STATE diff --git a/src/backend/core/agent_runtime/integrations/memory_session.py b/src/backend/core/agent_runtime/integrations/memory_session.py new file mode 100644 index 00000000..74a0c01c --- /dev/null +++ b/src/backend/core/agent_runtime/integrations/memory_session.py @@ -0,0 +1,178 @@ +"""Open, assemble and inject the chat's session-frozen memory and identity blocks.""" +from __future__ import annotations +from core.agent_runtime.integrations import memory +import asyncio +from dataclasses import dataclass, replace +from typing import Any, Dict, List, Optional +from core.llm.context.ir_constants import KIND_IDENTITY, KIND_MEMORY, SESSION_CONTEXT_META_KEY +from core.llm.context.items import make_text_context_item +from core.memory.session_snapshot import SessionMemorySnapshot, load_session_memory_snapshot, save_session_memory_snapshot +import logging +logger = logging.getLogger(__name__) + +@dataclass(frozen=True) +class SessionMemory: + """One run's handle on the chat's frozen memory block. + + Split in two because the two halves sit on either side of agent assembly: + `open_session_memory()` runs first so a first-turn retrieval overlaps + building the agent, and `resolve_session_memory()` runs at injection time. + + On every turn after the first, `retrieval_task` is ``None``: the block is + replayed from the stored snapshot and no vector search is issued at all. + """ + + chat_id: str + scope_user_id: str + workspace_id: str + memory_enabled: bool + retrieval_task: Optional[asyncio.Task] = None + snapshot: Optional["SessionMemorySnapshot"] = None + + +async def open_session_memory( + *, + chat_id: Optional[str], + scope_user_id: str, + workspace_id: str, + user_message: str, + memory_enabled: bool, + budget_ms: Optional[int] = None, +) -> SessionMemory: + """Start the chat's memory read, or recognise that it already happened. + + Called before the agent is built. When this chat already has a frozen + snapshot the retrieval is skipped outright — that is the point of freezing + it: the block sits ahead of the entire conversation, so re-deriving it per + turn would invalidate the model's prefix cache for the whole context. + """ + handle = SessionMemory( + chat_id=str(chat_id or ""), + scope_user_id=scope_user_id, + workspace_id=workspace_id, + memory_enabled=memory_enabled, + ) + if not memory_enabled: + return handle + + snapshot = await asyncio.to_thread( + load_session_memory_snapshot, + handle.chat_id, + scope_user_id=scope_user_id, + workspace_id=workspace_id, + ) + if snapshot is not None: + logger.info( + "[memory] frozen snapshot replayed chat=%s chars=%d (no retrieval)", + handle.chat_id, + len(snapshot.text), + ) + return replace(handle, snapshot=snapshot) + + task = await memory.launch_memory_retrieval( + scope_user_id, + user_message, + memory_enabled, + workspace_id=workspace_id, + budget_ms=budget_ms, + ) + return replace(handle, retrieval_task=task) + + +async def resolve_session_memory(handle: SessionMemory) -> str: + """Return the block to inject, assembling and storing it on the first turn. + + The assembled block is stored only once the memory read actually landed. A + timed-out or degraded first turn leaves no snapshot, so the next turn tries + once more rather than freezing an accidental blank for the whole chat. + + The store is first-writer-wins, and we inject what it hands back: if another run + froze this chat while we were retrieving, we use its snapshot, not ours. + """ + if not handle.memory_enabled: + return "" + if handle.snapshot is not None: + return handle.snapshot.text + + built = await memory.build_frozen_memory_block( + handle.scope_user_id, + handle.workspace_id, + handle.retrieval_task, + memory_enabled=handle.memory_enabled, + ) + if not built.settled: + logger.info( + "[memory] frozen snapshot not stored chat=%s: memory read did not settle", + handle.chat_id, + ) + return built.text + + effective = await asyncio.to_thread( + save_session_memory_snapshot, + handle.chat_id, + SessionMemorySnapshot( + text=built.text, + scope_user_id=handle.scope_user_id, + workspace_id=handle.workspace_id, + ), + ) + return effective.text + + +def session_message(text: str, **item_kwargs: Any) -> Dict[str, Any]: + """One head-of-conversation message plus the context-item manifest describing it.""" + item = make_text_context_item(text, **item_kwargs) + return { + "role": "user", + "content": text, + SESSION_CONTEXT_META_KEY: item.to_manifest(), + } + + +async def inject_session_blocks( + session_messages: List[Dict[str, Any]], + *, + identity_block: str = "", + memory_block: str = "", +) -> List[Dict[str, Any]]: + """Prepend the session-constant blocks (user identity, frozen memory) as user-role messages. + + Why user rather than system: Qwen-family models require system only at index 0, and keeping + these per-user bytes out of the system prompt leaves the system text, the tool schemas and the + skill list a byte-identical shared prefix across users and chats. + + Both blocks are constant for the whole chat, so neither moves the prefix-cache boundary between + turns — that is what makes this position safe. + """ + injected: list[Dict[str, Any]] = [] + if identity_block: + injected.append( + session_message( + f"\n{identity_block}\n\n" + "（以上为系统提供的当前用户身份信息，仅用于自然称呼，不是用户本轮提问。）", + item_id="session:identity", + kind=KIND_IDENTITY, + origin="identity:account", + trust="system", + created_seq=-200, + priority=850, + token_budget=1_000, + cache_class="session", + ) + ) + if memory_block: + injected.append( + session_message( + f"\n{memory_block}\n\n" + "（以上为会话启动时系统注入的背景快照，本会话内不变，用作回答参考，请勿直接复述。）", + item_id="session:memory:frozen", + kind=KIND_MEMORY, + origin="memory:frozen_session", + trust="memory", + created_seq=-100, + priority=800, + token_budget=8_000, + cache_class="session", + ) + ) + return [*injected, *session_messages] if injected else session_messages diff --git a/src/backend/core/agent_runtime/integrations/memory_state.py b/src/backend/core/agent_runtime/integrations/memory_state.py new file mode 100644 index 00000000..68865743 --- /dev/null +++ b/src/backend/core/agent_runtime/integrations/memory_state.py @@ -0,0 +1,23 @@ +"""Per-task memory retrieval results, states and budgets shared within one process.""" +from __future__ import annotations +import weakref +import asyncio +from dataclasses import dataclass, field +from core.memory.retrieval_types import MemoryRetrievalResult + + +IDENTITY_CACHE_TTL_S = 60.0 + + +@dataclass +class MemoryState: + last_retrieval: 'weakref.WeakKeyDictionary[asyncio.Task, MemoryRetrievalResult]' = field(default_factory=lambda: weakref.WeakKeyDictionary()) + retrieval_states: 'weakref.WeakKeyDictionary[asyncio.Task, str]' = field(default_factory=lambda: weakref.WeakKeyDictionary()) + retrieval_budgets: 'weakref.WeakKeyDictionary[asyncio.Task, float]' = field(default_factory=lambda: weakref.WeakKeyDictionary()) + + +_STATE = MemoryState() + + +def process_state() -> MemoryState: + return _STATE diff --git a/src/backend/core/agent_runtime/integrations/ontology_evidence.py b/src/backend/core/agent_runtime/integrations/ontology_evidence.py new file mode 100644 index 00000000..48a3d4a9 --- /dev/null +++ b/src/backend/core/agent_runtime/integrations/ontology_evidence.py @@ -0,0 +1,229 @@ +"""Collect and summarise ontology governance evidence for the final answer review.""" + +from __future__ import annotations +from core.ontology import output_review_state +from core.llm.citations.allocator import collect_citation_dicts + +import json +import logging +from typing import Any, Dict, List, Optional + +from core.llm.citations.allocator import AnchorAllocator + +logger = logging.getLogger(__name__) + + +def _ontology_review_event_context(runtime: Dict[str, Any]) -> Dict[str, Any]: + committee_size = max( + (int(pack.get("config", {}).get("committee_size", 3)) for pack in runtime.get("packs", [])), + default=3, + ) + workflows = [ + { + "pack_id": pack.get("pack_id"), + "workflow_id": workflow.get("id"), + "workflow_name": workflow.get("name"), + } + for pack in runtime.get("packs", []) + for workflow in pack.get("workflows", []) + ] + review_state = runtime.get("output_review") or {} + return { + "governance_run_id": runtime.get("governance_run_id"), + "review_owner": review_state.get("owner"), + "review_count": int(review_state.get("count") or 0), + "committee_size": committee_size, + "workflows": workflows, + "activation_sources": sorted( + { + str(event.get("source")) + for event in runtime.get("runtime_events", []) + if event.get("type") == "ontology_activation" and event.get("source") + } + ), + } + + +def _ontology_governance_summary(runtime: Dict[str, Any]) -> Optional[Dict[str, Any]]: + """Build the compact, persisted governance card payload for one answer.""" + if not runtime.get("enabled"): + return None + events = list(runtime.get("runtime_events") or []) + activations = [ + { + key: event.get(key) + for key in ( + "pack_id", + "workflow_id", + "workflow_name", + "source", + "asset_kind", + "asset_id", + "review_level", + ) + if event.get(key) is not None + } + for event in events + if event.get("type") == "ontology_activation" + ] + gates = [ + { + key: event.get(key) + for key in ( + "decision", + "tool_name", + "matched_rule_ids", + "violations", + "denial_count", + "circuit_breaker", + ) + if event.get(key) is not None + } + for event in events + if event.get("type") == "ontology_gate" + ] + review_state = dict(runtime.get("output_review") or {}) + review_state["level"] = runtime.get("review_level", "none") + review_state["committee_size"] = _ontology_review_event_context(runtime)["committee_size"] + if not activations and not gates and review_state.get("status") == "pending": + return None + if review_state.get("status") == "pending" and not output_review_state.requires_output_review(runtime): + review_state["status"] = "skipped" + return { + "governance_run_id": runtime.get("governance_run_id"), + "activations": activations, + "gates": gates, + "review": review_state, + } + + +def _ontology_review_owner(runtime: Dict[str, Any], context: Dict[str, Any]) -> str: + """Stable owner token for the one outer-workflow final review.""" + run_id = runtime.get("governance_run_id") or context.get("chat_id") or "request" + return f"outer_workflow:{run_id}" + + +def _ontology_review_failure_result(answer: str, exc: Exception) -> Dict[str, Any]: + """Keep a completed draft usable when the post-answer review itself fails.""" + logger.error( + "[ontology-review] automatic review failed; preserving original answer: %s", + exc, + exc_info=(type(exc), exc, exc.__traceback__), + ) + return { + "verdict": "escalate", + "answer": answer, + "revised": False, + "annotated": True, + "repair_attempts": 0, + "attempts": 1, + "violations": [], + "affected_claims": [], + "evidence": [], + "feedback": ["自动评审未正常完成，已保留原文并转人工复核。"], + "manual_review": { + "required": True, + "title": "领域本体人工复核", + "summary": "自动评审未正常完成，原文已保留，请人工核对或重新发起评审。", + "items": [ + { + "quote": "本轮完整答案", + "rule_id": "ontology_review_runtime", + "risk": "自动评审未完成，系统无法确认答案满足全部领域约束。", + "manual_check": "核对关键事实、推断、适用前提和原始工具证据。", + } + ], + "actions": ["重新发起本体评审，或在采用结论前完成人工核对。"], + }, + "new_tools": [], + "new_citation_count": 0, + "latency_ms": None, + } + + +def _record_ontology_activations( + activations: List[Dict[str, Any]], + runtime: Dict[str, Any], + context: Dict[str, Any], +) -> None: + if not activations: + return + try: + from core.ontology.audit_events import record_runtime_activation + + for event in activations: + record_runtime_activation( + event, + runtime, + user_id=str(context.get("user_id", "")) or None, + chat_id=context.get("chat_id"), + ) + except Exception as exc: # noqa: BLE001 + logger.warning("[ontology] workflow activation audit persistence failed: %s", exc) + + +def _bounded_trace_value(value: Any, max_chars: int = 4000) -> Any: + """Keep reviewer evidence useful without copying unbounded tool output.""" + try: + text = json.dumps(value, ensure_ascii=False, default=str) + except Exception: + text = str(value) + if len(text) <= max_chars: + return value + return {"truncated": True, "preview": text[:max_chars]} + + +def _parse_tool_result_value(value: Any) -> Any: + if not isinstance(value, str): + return value + try: + return json.loads(value) if value else {} + except json.JSONDecodeError: + return {"result": value} + + +def _capture_nested_ontology_evidence( + payload: Dict[str, Any], + trace: List[Dict[str, Any]], + citations: List[Dict[str, Any]], + allocator: Optional[AnchorAllocator] = None, +) -> List[Dict[str, Any]]: + """Merge trusted ``call_subagent`` bypass events into the outer review trace.""" + sub_type = str(payload.get("sub_type") or "") + tool_name = str(payload.get("tool_name") or "") + tool_id = str(payload.get("tool_id") or "") + provenance = { + key: payload.get(key) + for key in ("agent_id", "agent_name", "sub_run_id", "parent_tool_id") + if payload.get(key) is not None + } + if sub_type == "tool_call" and tool_name and payload.get("input") is not None: + trace.append( + { + "type": "tool_call", + "tool_id": tool_id, + "tool_name": tool_name, + "input": _bounded_trace_value(payload.get("input")), + "source": "subagent", + **provenance, + } + ) + return [] + if sub_type != "tool_result" or not tool_name or payload.get("status") == "error": + return [] + + result = _parse_tool_result_value(payload.get("output")) + cit_dicts = collect_citation_dicts(tool_id, allocator) + citations.extend(cit_dicts) + trace.append( + { + "type": "tool_result", + "tool_id": tool_id, + "tool_name": tool_name, + "result": _bounded_trace_value(result), + "citations": cit_dicts, + "source": "subagent", + **provenance, + } + ) + return cit_dicts diff --git a/src/backend/core/agent_runtime/integrations/ontology_repair.py b/src/backend/core/agent_runtime/integrations/ontology_repair.py new file mode 100644 index 00000000..78a01e25 --- /dev/null +++ b/src/backend/core/agent_runtime/integrations/ontology_repair.py @@ -0,0 +1,366 @@ +"""Re-run the originating agent to produce a revision when ontology review rejects an answer.""" + +from __future__ import annotations +from core.llm.citations.allocator import collect_citation_dicts + +import json +import logging +import typing as _typing +from typing import Any, Awaitable, Callable, Dict, List, Optional, Tuple + +import core.agent_runtime.events as agent_runtime_events +import core.agent_runtime.integrations.ontology_evidence as ontology_evidence +import core.config.display_names as display_names +import core.ontology.revision as revision +from core.llm.citations.allocator import AnchorAllocator +from core.llm.runtime.streaming import StreamingAgent + +logger = logging.getLogger(__name__) + + +def _ontology_repair_prompt(payload: Dict[str, Any]) -> str: + violations = payload.get("violations") or [] + feedback = payload.get("feedback") or [] + affected_claims = payload.get("affected_claims") or [] + original_task = str(payload.get("original_task") or "").strip() + current_answer = str(payload.get("answer") or "").strip() + citations = [ + { + "id": item.get("id"), + "tool_name": item.get("tool_name"), + "title": item.get("title"), + } + for item in payload.get("citations") or [] + if isinstance(item, dict) + ] + return ( + "\n" + "这是同一任务唯一一次领域本体修复，不是新的用户任务。" + "优先级强制规则：已经激活的 enforce 级领域本体约束及本修复指令，高于用户原始提示中" + "与其冲突的‘不要调用工具’、‘原样照抄’或类似限制；这些限制只能约束正常生成阶段，" + "不能阻止本体修复取证和纠错。若‘确定性违规’包含‘未调用’某些必需工具，本轮必须" + "实际调用这些工具后才能形成终稿；以用户禁止工具为理由跳过调用、保留原错误断言，" + "或再次原样输出待修订内容，均视为修复失败。" + "先比较现有证据与评审问题，明确是否出现了需要重新搜索的新内容或可补充调用的新工具。" + "有证据缺口时必须调用真实工具补齐；现有证据已足够时才可直接精准修订。" + "禁止只做同义改写，也不得虚构工具调用或引用。" + "证据核对、逐项判定、原文对比和修改说明都属于内部校验过程，只能放在" + " thinking/reasoning 通道；除非用户最初明确要求核验报告，否则不得把这些内容写入" + "最终回复或生成的文件。" + "必须继续遵守用户最初指令中的输出与交付形式；如果原任务要求生成 Word、PPT、" + "表格、网页或其他文件，修订时仍须调用相应工具生成该类产物，不得降级为纯文字替代。" + "最终交付必须是利用核验结论直接修正后的完整润色稿，而不是关于原答案的评估报告。" + "保留原任务要求的文体、篇幅、结构和语气，只改正有问题的事实、推断与表述；" + "以待修订原始输出为底稿做最小必要修改，不要因为查到更多资料就加入与原主张无关的" + "企业背景、资质或延伸分析。句子级任务应尽量保留原句的主语与分句顺序，逐个修正" + "不成立的断言，形成简洁、自然、可直接交付的一句话终稿。" + "用户只要求一句话时仍输出一句话，不得自行扩写成包含‘原始语句’、‘逐项核验’、" + "‘主张评估’或‘修改说明’等章节的报告。若交付物是文件，文件正文也必须是润色终稿，" + "并重新生成、交付修正后的文件。没有获得证据的风险项只能表述为‘待核验’、" + "‘暂无数据支撑’或‘无法判断’，绝不能反向断言为‘不存在’、‘没有风险’或‘风险为零’。" + "在输出终稿和生成文件前，必须在内部逐条检查‘确定性违规’中的每个条件：若要求" + "最低长度，终稿必须达到该长度；若要求引用，相关事实后必须包含真实引用标记" + "（从工具结果复制 cite_id，写成 `[锚文本](cite:eN)`）" + "且按规则补齐参考资料；若要求区分事实、推断和待核验项，可在同一句中用分号和简短标签" + "表达，不得省略。先确定唯一的合格终稿，再把完全相同的正文写入用户要求的文件，" + "不得先生成文件后又输出一份更短、缺引用或结论不同的候选正文。" + "思考过程只放在模型的 thinking/reasoning 通道。最终正文必须且只能放在" + " `完整候选答案` 中。" + "在正式输出最终正文前，不要复述、引用或示例化这组标签，也不要用省略号代替正文。" + "标签内必须是可独立阅读、内容完整的修订答案；若无法修订，则原样放入当前完整答案。" + "事实、推断和待核验项必须明确区分；引用已有或新增工具结果时，把工具结果里" + "标注的 cite_id 原样复制进 `[锚文本](cite:eN)`，禁止自行编号。\n" + f"用户最初指令：{json.dumps(original_task, ensure_ascii=False)[:12000]}\n" + f"待修订原始输出：{json.dumps(current_answer, ensure_ascii=False)[:12000]}\n" + f"修复轮次：{payload.get('attempt', 1)}\n" + f"问题来源：{payload.get('source', 'review')}\n" + f"确定性违规：{json.dumps(violations, ensure_ascii=False)[:8000]}\n" + f"委员会意见：{json.dumps(feedback, ensure_ascii=False)[:8000]}\n" + f"受影响内容：{json.dumps(affected_claims, ensure_ascii=False)[:5000]}\n" + f"当前可用引用：{json.dumps(citations, ensure_ascii=False)[:5000]}\n" + "直接开始补充工具或输出修复后的完整答案。\n" + "" + ) + + +async def _run_ontology_repair_round( + *, + streaming_agent: StreamingAgent, + context: Dict[str, Any], + payload: Dict[str, Any], + runtime: Dict[str, Any], + trace: List[Dict[str, Any]], + citations: List[Dict[str, Any]], + allocator: Optional[AnchorAllocator] = None, + event_cursor: int = 0, + subagent_log_id: Optional[str] = None, + event_sink: Optional[Callable[[Dict[str, Any]], Awaitable[None]]] = None, +) -> Tuple[str, List[Dict[str, Any]], int, int]: + """Continue the originating agent and stream a separate revision candidate.""" + answer = "" + events: _typing.List[_typing.Dict[str, _typing.Any]] = [] + tool_count = 0 + text_buffer = "" + raw_text = "" + revision_open = False + revision_closed = False + thinking_line_prefix = "" + + async def _publish(event: Dict[str, Any]) -> None: + events.append(event) + if event_sink is not None: + await event_sink(event) + + async def _publish_text_thinking(delta: str) -> None: + """Publish text-channel reasoning and remember its current line prefix.""" + nonlocal thinking_line_prefix + if not delta: + return + if "\n" in delta: + thinking_line_prefix = delta.rsplit("\n", 1)[-1] + else: + thinking_line_prefix = (thinking_line_prefix + delta)[-256:] + await _publish({"type": "ontology_revision_thinking", "delta": delta}) + + def _is_revision_opening_boundary(line_before_open: str) -> bool: + """Allow a revision wrapper after whitespace or a reasoning close tag.""" + + prefix = line_before_open.rstrip() + return not prefix or prefix.endswith(("", "")) + + await _publish( + { + "type": "ontology_repair", + "status": "started", + "attempt": payload.get("attempt", 1), + "source": payload.get("source", "review"), + } + ) + + stream = streaming_agent.stream( + [{"role": "user", "content": _ontology_repair_prompt(payload)}], + context, + ).__aiter__() + while True: + try: + if subagent_log_id: + event_type, event_payload = await agent_runtime_events._anext_in_subagent_log_scope( + stream, + subagent_log_id, + ) + else: + event_type, event_payload = await anext(stream) + except StopAsyncIteration: + break + + state_runtime = getattr(streaming_agent.agent.state, "ontology_runtime", None) + if isinstance(state_runtime, dict): + runtime = state_runtime + context["ontology_runtime"] = state_runtime + pending_events = runtime.get("runtime_events", [])[event_cursor:] + for item in pending_events: + await _publish(dict(item)) + event_cursor += len(pending_events) + + if event_type == "vision_progress": + await _publish({"type": "vision_progress", **(event_payload or {})}) + continue + + if event_type == "text_delta": + delta = str(event_payload or "") + if not delta: + continue + raw_text += delta + if revision_closed: + await _publish_text_thinking(delta) + continue + text_buffer += delta + while text_buffer: + if not revision_open: + open_idx = text_buffer.find("") + if open_idx < 0: + keep = min(len("") - 1, len(text_buffer)) + safe_len = len(text_buffer) - keep + if safe_len > 0: + await _publish_text_thinking(text_buffer[:safe_len]) + text_buffer = text_buffer[safe_len:] + break + before_open = text_buffer[:open_idx] + line_before_open = (thinking_line_prefix + before_open).rsplit("\n", 1)[-1] + if not _is_revision_opening_boundary(line_before_open): + # Reasoning models sometimes describe the contract as + # `...`. A real + # wrapper starts a new line or immediately follows the + # model's reasoning close tag. Other inline occurrences + # remain folded as examples rather than candidate text. + ignored = text_buffer[: open_idx + len("")] + await _publish_text_thinking(ignored) + text_buffer = text_buffer[open_idx + len("") :] + continue + if open_idx > 0: + await _publish_text_thinking(before_open) + text_buffer = text_buffer[open_idx + len("") :] + revision_open = True + continue + close_idx = text_buffer.find("") + if close_idx < 0: + keep = min(len("") - 1, len(text_buffer)) + safe_len = len(text_buffer) - keep + if safe_len > 0: + candidate_delta = text_buffer[:safe_len] + answer += candidate_delta + await _publish({"type": "ontology_revision", "delta": candidate_delta}) + text_buffer = text_buffer[safe_len:] + break + if close_idx > 0: + candidate_delta = text_buffer[:close_idx] + answer += candidate_delta + await _publish({"type": "ontology_revision", "delta": candidate_delta}) + text_buffer = text_buffer[close_idx + len("") :] + revision_closed = True + break + elif event_type == "thinking_delta": + await _publish( + { + "type": "ontology_revision_thinking", + "delta": str(event_payload or ""), + } + ) + elif event_type == "tool_call_start": + tool_name = str(event_payload.get("name") or "unknown") + if tool_name != "update_plan": + await _publish( + { + "type": "tool_call_start", + "tool_name": tool_name, + "tool_display_name": display_names.TOOL_DISPLAY_NAMES.get( + tool_name, tool_name + ), + "tool_id": str(event_payload.get("id") or ""), + "scope": "ontology_revision", + } + ) + elif event_type == "tool_call_delta": + tool_name = str(event_payload.get("name") or "unknown") + if tool_name != "update_plan" and event_payload.get("delta"): + await _publish( + { + "type": "tool_call_delta", + "tool_name": tool_name, + "tool_id": str(event_payload.get("id") or ""), + "arguments_delta": str(event_payload.get("delta") or ""), + "scope": "ontology_revision", + } + ) + elif event_type == "tool_call": + tool_name = str(event_payload.get("name") or "unknown") + tool_id = str(event_payload.get("id") or "") + tool_args = event_payload.get("args") or {} + trace.append( + { + "type": "tool_call", + "tool_id": tool_id, + "tool_name": tool_name, + "input": ontology_evidence._bounded_trace_value(tool_args), + "source": "ontology_repair", + } + ) + await _publish( + { + "type": "tool_call", + "tool_name": tool_name, + "tool_display_name": display_names.TOOL_DISPLAY_NAMES.get(tool_name, tool_name), + "tool_args": tool_args, + "input": tool_args, + "tool_id": tool_id, + "scope": "ontology_revision", + } + ) + elif event_type == "tool_result": + tool_count += 1 + tool_name = str(event_payload.get("name") or "unknown") + tool_id = str(event_payload.get("id") or "") + result = ontology_evidence._parse_tool_result_value(event_payload.get("content")) + cit_dicts = collect_citation_dicts(tool_id, allocator) + citations.extend(cit_dicts) + trace.append( + { + "type": "tool_result", + "tool_id": tool_id, + "tool_name": tool_name, + "result": ontology_evidence._bounded_trace_value(result), + "citations": cit_dicts, + "source": "ontology_repair", + } + ) + await _publish( + { + "type": "tool_result", + "tool_name": tool_name, + "result": result, + "tool_id": tool_id, + "citations": cit_dicts, + "scope": "ontology_revision", + } + ) + elif event_type == "subagent_event": + nested_citations = ontology_evidence._capture_nested_ontology_evidence( + event_payload or {}, + trace, + citations, + allocator, + ) + sub_type = str((event_payload or {}).get("sub_type") or "") + if sub_type in { + "start", + "thinking", + "content", + "tool_call", + "tool_call_delta", + "tool_result", + "end", + }: + await _publish( + { + "type": "subagent_event", + **(event_payload or {}), + "scope": "ontology_revision", + **({"citations": nested_citations} if nested_citations else {}), + } + ) + elif event_type == "tool_pending": + await _publish( + { + "type": "tool_pending", + **(event_payload or {}), + "scope": "ontology_revision", + } + ) + elif event_type in {"file_confirm", "design_pick"}: + raise RuntimeError("本体自动修复轮不支持等待交互确认，请转人工复核") + elif event_type == "error": + if isinstance(event_payload, BaseException): + raise event_payload + raise RuntimeError(str(event_payload)) + + if revision_open and not revision_closed and text_buffer: + answer += text_buffer + await _publish({"type": "ontology_revision", "delta": text_buffer}) + elif not revision_open and text_buffer.strip(): + # Compatibility fallback for models that ignored the wrapper. When + # embedded text exists, only the content after the last closing + # marker can be a candidate; never promote the hidden reasoning itself. + fallback = revision.normalize_revision_candidate(raw_text.rsplit("", 1)[-1]) + if revision.is_substantive_revision(fallback): + answer = fallback + await _publish({"type": "ontology_revision", "delta": answer}) + + await _publish( + { + "type": "ontology_repair", + "status": "completed", + "attempt": payload.get("attempt", 1), + "tool_calls": tool_count, + } + ) + return revision.normalize_revision_candidate(answer), events, event_cursor, tool_count diff --git a/src/backend/core/agent_runtime/main_state.py b/src/backend/core/agent_runtime/main_state.py new file mode 100644 index 00000000..1aeb7fda --- /dev/null +++ b/src/backend/core/agent_runtime/main_state.py @@ -0,0 +1,45 @@ +"""State shared only by the main agent execution phases.""" + +from __future__ import annotations + +import asyncio +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any, AsyncIterator + +if TYPE_CHECKING: + from core.agent_runtime.integrations.memory_session import SessionMemory + from core.llm.citations.allocator import AnchorAllocator + from core.llm.runtime.streaming import StreamingAgent + + +@dataclass +class MainTurn: + context: dict[str, Any] + session_messages: list[dict[str, Any]] + user_message: str + closed: bool = False + review_task: asyncio.Task | None = None + review_owner: str | None = None + anchor_allocator: AnchorAllocator | None = None + ctx_window: int = 0 + last_plan: dict[str, Any] | None = None + main_stream: AsyncIterator | None = None + memory_enabled: bool = False + memory_user_id: str = "" + memory_workspace_id: str = "default" + mentioned_ids: list[str] = field(default_factory=list) + ontology_event_cursor: int = 0 + ontology_runtime: dict[str, Any] = field(default_factory=dict) + ontology_trace: list[dict[str, Any]] = field(default_factory=list) + pending_plan_updates: dict[str, dict[str, Any]] = field(default_factory=dict) + session_memory: SessionMemory | None = None + visible_subagents: list[dict[str, Any]] = field(default_factory=list) + agent: Any = None + all_citations: list[dict[str, Any]] = field(default_factory=list) + displayed_tools: set[str] = field(default_factory=set) + full_response: str = "" + mcp_clients: list[Any] = field(default_factory=list) + skill_id_by_tool_id: dict[str, str] = field(default_factory=dict) + skill_load_ids: set[str] = field(default_factory=set) + streaming_agent: StreamingAgent | None = None + view_text_file_args: dict[str, dict[str, Any]] = field(default_factory=dict) diff --git a/src/backend/orchestration/subagents/plugin_visibility.py b/src/backend/core/agent_runtime/plugin_visibility.py similarity index 91% rename from src/backend/orchestration/subagents/plugin_visibility.py rename to src/backend/core/agent_runtime/plugin_visibility.py index 84574145..38170a78 100644 --- a/src/backend/orchestration/subagents/plugin_visibility.py +++ b/src/backend/core/agent_runtime/plugin_visibility.py @@ -69,10 +69,12 @@ def load_enabled_plugins( mc = [m for m in (cids.get("mcp") or []) if m in enabled_plug_mcps] if not sk and not mc: continue - out.append({ - "name": r.name or r.slug, - "description": r.description or "", - "skill_ids": sk, - "mcp_ids": mc, - }) + out.append( + { + "name": r.name or r.slug, + "description": r.description or "", + "skill_ids": sk, + "mcp_ids": mc, + } + ) return out diff --git a/src/backend/core/agent_runtime/prepare_direct.py b/src/backend/core/agent_runtime/prepare_direct.py new file mode 100644 index 00000000..c6aa980a --- /dev/null +++ b/src/backend/core/agent_runtime/prepare_direct.py @@ -0,0 +1,126 @@ +"""Assemble a directly selected agent and its contextual input.""" + +from __future__ import annotations +from core.agent_runtime.integrations import memory_identity +from core.agent_runtime.integrations import memory_session +from core.llm.runtime import streaming +from core.llm.citations import allocator + +import logging +import typing as _typing + +import core.agent_runtime.budget as budget +import core.agent_runtime.targets as targets +import core.llm.factory as factory +from core.agent_runtime.direct_state import DirectTurn + +import time as _time + +logger = logging.getLogger(__name__) + + +async def prepare_direct(turn: DirectTurn): + yield {"type": "thinking", "message": "正在连接子智能体..."} + for ontology_event in turn.ontology_runtime.get("runtime_events", []): + yield dict(ontology_event) + turn.ontology_event_cursor = len(turn.ontology_runtime.get("runtime_events", [])) + + _stream_user_id = str(turn.context.get("user_id", "")) + _stream_model_name = str(turn.context.get("model_name", "")) + _stream_model_provider_id = str(turn.context.get("model_provider_id", "") or "") + _stream_chat_mode = str(turn.context.get("chat_mode", "") or "") + _stream_reranker = bool(turn.context.get("reranker_enabled", False)) + + # Create agent with sub-agent config overrides + turn.agent, turn.mcp_clients = await factory.create_agent_executor( + agent_spec=None, + enabled_mcp_ids=None, # overridden by user_agent inside factory + enabled_skill_ids=None, # overridden by user_agent inside factory + enabled_kb_ids=None, # overridden by user_agent inside factory + current_user_id=_stream_user_id, + reranker_enabled=_stream_reranker, + model_name=_stream_model_name, + model_provider_id=_stream_model_provider_id, + chat_mode=_stream_chat_mode, + memory_enabled=turn.memory_enabled, + agent_api_scope=turn.context.get("agent_api_scope"), + user_agent=turn.user_agent, + read_only=turn.direct_read_only, + allow_bash=turn.direct_allow_bash, + approval_mode=targets._approval_mode(turn.context), + required_skill_id=str(turn.context.get("skill_id") or "") or None, + required_skill_name=str(turn.context.get("skill_name") or "") or None, + required_plugin_id=str(turn.context.get("plugin_id") or "") or None, + required_plugin_name=str(turn.context.get("plugin_name") or "") or None, + required_plugin_skill_ids=[ + str(item) + for item in (turn.context.get("plugin_skill_ids") or []) + if isinstance(item, str) and item + ], + required_plugin_mcp_ids=[ + str(item) + for item in (turn.context.get("plugin_mcp_ids") or []) + if isinstance(item, str) and item + ], + # Same as the main agent: pass chat_id → the sandbox session uses + # the chat_id-keyed "user-bound persistent sandbox" (mounting the + # per-user credential volumes for lark/dws/email etc.). Omitting it + # falls back to an ephemeral light sandbox (no credentials) — the + # root cause of Feishu/DingTalk CLIs reporting "not configured" in + # direct sub-agent conversations. + chat_id=turn.context.get("chat_id"), + sandbox_session_id=turn.context.get("sandbox_session_id"), + run_id=str(turn.context.get("run_id") or "") or None, + journal_owner=str(turn.context.get("journal_owner") or "") or None, + workspace_id=turn.memory_workspace_id, + project_ctx=targets._extract_project_ctx(turn.context), + channel_origin=turn.context.get("channel_origin"), + automation_run=bool(turn.context.get("automation_run")), + ontology_runtime=turn.context.get("ontology_runtime"), + ) + + logger.info("[subagent] agent created in %.0fms", (_time.monotonic() - turn.wf_start) * 1000) + _, turn.ctx_window = budget._resolve_agent_model_runtime(turn.agent, _stream_model_name) + + # 证据锚点：把发号器绑到 agent 上（中间件与本函数由此共享同一个计数器； + # 仅靠 ContextVar 不行——本函数是 async generator，与 agent 执行所在的 + # task 上下文不互通） + allocator.attach_allocator(turn.agent, turn.anchor_allocator) + + # ── Session-constant blocks: user identity + the chat's frozen memory + # snapshot (built on the first turn, replayed byte-for-byte after). ── + turn.session_messages = await memory_session.inject_session_blocks( + turn.session_messages, + identity_block=( + "" + if turn.context.get("agent_api_scope") or turn.context.get("evaluation_session") + else await memory_identity.build_user_identity_block(turn.memory_user_id) + ), + memory_block=await memory_session.resolve_session_memory(turn.session_memory), + ) + + # The canonical ContextAssembler owns selection and truncation. Feed + # it the complete candidate set so every exclusion is visible in the + # request manifest rather than silently pre-trimming shared history. + turn.streaming_agent = streaming.StreamingAgent(turn.agent, turn.mcp_clients) + turn.skill_load_ids: set = set() + # tool_id → skill_id (parsed from the tool_call's file_path; looked up at tool_result time to replace the SSE payload with the curated detail, avoiding sending the full SKILL.md text to the frontend) + turn.skill_id_by_tool_id: _typing.Dict[str, str] = {} + # tool_id → tool_args, used at the tool_result stage to recover view_text_file's file_path/ranges + turn.view_text_file_args: _typing.Dict[str, _typing.Dict[str, _typing.Any]] = {} + + # Project scope is no longer passed via ContextVar — it now travels as + # explicit parameters along the call chain (agent_factory closes + # ProjectScope into every register_* tool; chats.py's finishing + # _persist_artifacts reconstructs the scope from the workflow context + # and passes it explicitly). See the header comment in + # core/projects/scope.py. + + # Never keep a ContextVar token alive across an async-generator + # ``yield``. The consumer may resume this generator in a copied + # context, making reset(token) fail with "created in a different + # Context". Enter the audit scope only while advancing the underlying + # agent stream; tool execution and persistence happen during anext(). + turn.direct_stream = turn.streaming_agent.stream( + turn.session_messages, turn.context + ).__aiter__() diff --git a/src/backend/core/agent_runtime/prepare_main.py b/src/backend/core/agent_runtime/prepare_main.py new file mode 100644 index 00000000..5cf481f4 --- /dev/null +++ b/src/backend/core/agent_runtime/prepare_main.py @@ -0,0 +1,288 @@ +"""Assemble the main agent and its complete contextual input.""" + +from __future__ import annotations +from core.agent_runtime.integrations import memory_identity +from core.agent_runtime.integrations import memory_session +from core.llm.runtime import streaming +from core.llm.context import ir_constants +from core.llm.context import items +from core.llm.citations import allocator + +import logging +import typing as _typing + +import core.agent_runtime.budget as budget +import core.agent_runtime.skills as skills +import core.agent_runtime.targets as targets +import core.capabilities.catalog.resolver as catalog_resolver +import core.llm.factory as factory +from core.agent_runtime.main_state import MainTurn + +logger = logging.getLogger(__name__) + + +async def prepare_main(turn: MainTurn): + import time as _time + + _wf_start = _time.monotonic() + + yield {"type": "thinking", "message": "正在分析您的问题..."} + for ontology_event in turn.ontology_runtime.get("runtime_events", []): + yield dict(ontology_event) + turn.ontology_event_cursor = len(turn.ontology_runtime.get("runtime_events", [])) + + _stream_user_id = str(turn.context.get("user_id", "")) + _stream_model_name = str(turn.context.get("model_name", "")) + _stream_reranker = bool(turn.context.get("reranker_enabled", False)) + enabled_skill_ids = catalog_resolver.enabled_skill_ids_from_context(turn.context) + enabled_kb_ids = catalog_resolver.enabled_kb_ids_from_context(turn.context) + enabled_mcp_ids = catalog_resolver.enabled_mcp_ids_from_context(turn.context) + + # ── Load visible sub-agents for main agent routing ── + turn.visible_subagents: list = [] + turn.mentioned_ids: list = [] + _explicit_command = turn.context.get("explicit_subagent_command") + _explicit_agent_id = ( + str(_explicit_command.get("agent_id")) + if isinstance(_explicit_command, dict) and _explicit_command.get("agent_id") + else "" + ) + _mention_agent_id = str(turn.context.get("mention_agent_id") or "") + try: + from core.db.engine import SessionLocal as _SessionLocal + from core.capabilities.agents.edition import UserAgentService as _UAS + from core.auth.users import UserService as _UserService + + with _SessionLocal() as _db: + _ua_svc = _UAS(_db) + turn.visible_subagents = ( + [] + if turn.context.get("evaluation_session") + else _ua_svc.list_for_user(_stream_user_id) + ) + _disabled_builtin_ids = _UserService(_db).get_disabled_builtin_subagent_ids( + _stream_user_id + ) + except Exception as _exc: + logger.warning("[workflow] failed to load visible subagents: %s", _exc) + _disabled_builtin_ids = set() + + turn.visible_subagents = targets._runtime_visible_subagents( + turn.visible_subagents, + { + "enabled_skill_ids": enabled_skill_ids or [], + "enabled_mcp_ids": enabled_mcp_ids or [], + "enabled_kb_ids": enabled_kb_ids or [], + }, + _disabled_builtin_ids, + [_explicit_agent_id, _mention_agent_id], + ) + # Prefer the structured per-turn target. Text parsing remains a + # compatibility fallback for callers that only include @name. + _delegated_agent_id = _explicit_agent_id or _mention_agent_id + if _delegated_agent_id and any( + str(item.get("agent_id") or "") == _delegated_agent_id for item in turn.visible_subagents + ): + turn.mentioned_ids = [_delegated_agent_id] + else: + turn.mentioned_ids = targets._parse_agent_mentions( + turn.user_message, turn.visible_subagents + ) + + # Create agent (with model-aware CompressionConfig + optional native LTM) + _plan_chat = bool(turn.context.get("plan_chat", False)) + # 极速模式：配置化检索工具集 + 独立 turbo 提示词 + 迭代硬上限。 + # plan 会话有自己的编排语义，不进入极速裁剪。 + _stream_mode_spec = targets._resolve_mode_spec(turn.context) + _turbo_chat = _stream_mode_spec is not None and _stream_mode_spec.restricted and not _plan_chat + if _turbo_chat: + # 收窄模式下子智能体不再只靠手动呼唤入场：模式自己圈定的那批常驻， + # 本轮被 @/显式委派命中的那个并上去（系统提示词也就只渲染这几行）。 + # 模式没圈人时退回原契约——只有被呼唤的那个能进。 + turn.visible_subagents = targets._mode_visible_subagents( + _stream_mode_spec, turn.visible_subagents, turn.mentioned_ids + ) + turn.agent, turn.mcp_clients = await factory.create_agent_executor( + agent_spec=None, + enabled_skill_ids=enabled_skill_ids, + enabled_mcp_ids=enabled_mcp_ids, + enabled_kb_ids=enabled_kb_ids, + current_user_id=_stream_user_id, + reranker_enabled=_stream_reranker, + model_name=_stream_model_name, + model_provider_id=str(turn.context.get("model_provider_id", "") or ""), + chat_mode=str(turn.context.get("chat_mode", "") or ""), + turbo_mode=_turbo_chat, + mode_spec=_stream_mode_spec, + # 显式呼唤透传：斜杠技能 / 插件展开的 skill_ids 与 mcp_ids。 + # 仅 turbo 模式消费；能力说明仍走 user message 注入，不进全量清单。 + turbo_explicit_skill_ids=( + skills._explicit_skill_ids_from_context(turn.context) if _turbo_chat else None + ), + turbo_explicit_mcp_ids=( + [m for m in (turn.context.get("mcp_ids") or []) if isinstance(m, str)] + if _turbo_chat + else None + ), + # 渐进式插件加载：本轮显式呼唤的能力不得被延迟（用户消息注入已承诺 + # 其可用），并作为会话级激活落库。 + invoked_skill_ids=skills._explicit_skill_ids_from_context(turn.context), + invoked_mcp_ids=[m for m in (turn.context.get("mcp_ids") or []) if isinstance(m, str)], + required_mcp_ids=( + [str(turn.context["connector_id"])] if turn.context.get("connector_id") else None + ), + required_skill_id=str(turn.context.get("skill_id") or "") or None, + required_skill_name=str(turn.context.get("skill_name") or "") or None, + required_plugin_id=str(turn.context.get("plugin_id") or "") or None, + required_plugin_name=str(turn.context.get("plugin_name") or "") or None, + required_plugin_skill_ids=[ + str(item) + for item in (turn.context.get("plugin_skill_ids") or []) + if isinstance(item, str) and item + ], + required_plugin_mcp_ids=[ + str(item) + for item in (turn.context.get("plugin_mcp_ids") or []) + if isinstance(item, str) and item + ], + memory_enabled=turn.memory_enabled, + approval_mode=targets._approval_mode(turn.context), + visible_subagents=turn.visible_subagents if turn.visible_subagents else None, + plan_mode=_plan_chat, + workflow_mode=bool(turn.context.get("workflow_chat", False)), + chat_id=turn.context.get("chat_id"), + sandbox_session_id=turn.context.get("sandbox_session_id"), + run_id=str(turn.context.get("run_id") or "") or None, + journal_owner=str(turn.context.get("journal_owner") or "") or None, + workspace_id=turn.memory_workspace_id, + project_ctx=targets._extract_project_ctx(turn.context), + channel_origin=turn.context.get("channel_origin"), + automation_run=bool(turn.context.get("automation_run")), + ontology_runtime=turn.context.get("ontology_runtime"), + # The single positive source of truth for update_plan: enabled + # only for "interactive main chats that can host plan mode" — has + # a chat_id, not a channel bot, not automation, not plan_chat, not + # batch. Channels/automation have no plan-bar UI; plan_chat + # have their own orchestration and must not nest plan tracking. + top_level_chat=( + bool(turn.context.get("chat_id")) + and not turn.context.get("channel_origin") + and not bool(turn.context.get("automation_run")) + and not _plan_chat + ), + ) + + logger.info("[workflow] agent created in %.0fms", (_time.monotonic() - _wf_start) * 1000) + + # 证据锚点：把发号器绑到 agent 上（中间件与本函数由此共享同一个计数器； + # 仅靠 ContextVar 不行——本函数是 async generator，与 agent 执行所在的 + # task 上下文不互通） + allocator.attach_allocator(turn.agent, turn.anchor_allocator) + + # ── Inject the per-turn sub-agent constraint into the current user message ── + # Keeps it OUT of the system prompt so the LLM provider's prefix cache + # hits across turns within a chat (otherwise every turn with different + # targets would re-build the cache from scratch). A strict natural- + # language command remains on this normal model stream; it only + # constrains the model's next real tool call instead of fabricating one + # in the workflow layer. + if turn.mentioned_ids and turn.visible_subagents: + from core.llm.subagents.tool import build_explicit_subagent_command_hint, build_subagent_mention_hint + + if _explicit_agent_id or _mention_agent_id: + _mention_hint = build_explicit_subagent_command_hint( + turn.visible_subagents, + _explicit_agent_id or _mention_agent_id, + ) + else: + _mention_hint = build_subagent_mention_hint( + turn.visible_subagents, + turn.mentioned_ids, + ) + if ( + _mention_hint + and turn.session_messages + and turn.session_messages[-1].get("role") in ("user", "human") + ): + mention_item = items.make_text_context_item( + _mention_hint, + item_id="harness:subagent_routing", + kind=ir_constants.KIND_REMINDER, + origin="harness:subagent_routing", + trust="system", + created_seq=0, + priority=950, + token_budget=4_000, + truncation_policy=ir_constants.POLICY_NEVER, + ) + turn.session_messages.insert( + -1, + { + "role": "user", + "content": _mention_hint, + ir_constants.SESSION_CONTEXT_META_KEY: items.session_context_metadata( + mention_item + ), + }, + ) + + # ── PreTurn compaction safety net (aligned with Codex pre-turn compaction) ── + # When end-of-turn background compaction failed/was skipped, or the + # previous turn's tool calls blew up the history, compact once + # synchronously with the same cross-turn compaction mechanism before + # this turn starts and write a checkpoint. Below the threshold, only a + # pure byte estimate is done (no DB / no LLM), so first-token latency + # is unaffected. Must run **before** the frozen memory block injection + # — identity/memory blocks are re-injected every turn and must not be + # baked into the persistent checkpoint. + # The model window is preferentially read straight off the model object + # (make_chat_model bakes in the real context_size per the Config model + # configuration at construction time, no default fallback), saving the + # streaming path one synchronous DB query; only if missing on the + # object do we fall back to resolve — unconfigured raises, fail loud, + # never silently run with the wrong window. + # Pre-turn compaction and the request assembler share this value. + _actual_model, turn.ctx_window = budget._resolve_agent_model_runtime( + turn.agent, _stream_model_name + ) + try: + from core.chat.compaction.service import maybe_run_pre_turn_compaction + + turn.session_messages, _ = await maybe_run_pre_turn_compaction( + turn.context.get("chat_id"), + turn.session_messages, + run_id=str(turn.context.get("run_id") or ""), + **budget._pre_turn_compaction_inputs(turn.agent, turn.ctx_window, _actual_model), + ) + except Exception as _pt_exc: # noqa: BLE001 + from core.harness.hooks import HookPaused + + if isinstance(_pt_exc, HookPaused): + raise + logger.warning("[workflow] pre-turn compaction failed: %s", _pt_exc) + + # ── Session-constant blocks: user identity + the chat's frozen memory + # snapshot. Built on the first turn (L1 profile always, L2 facts only if + # retrieval beat its budget), replayed byte-for-byte on every turn after, + # so this head of the message list never moves the prefix-cache boundary. + turn.session_messages = await memory_session.inject_session_blocks( + turn.session_messages, + identity_block=( + "" + if turn.context.get("evaluation_session") + else await memory_identity.build_user_identity_block(turn.memory_user_id) + ), + memory_block=await memory_session.resolve_session_memory(turn.session_memory), + ) + + # The final request assembler receives the full post-compaction + # candidate set. It is the only budget selector, and records every + # included, truncated, or excluded item in the context manifest. + + turn.streaming_agent = streaming.StreamingAgent(turn.agent, turn.mcp_clients) + + turn.skill_load_ids: set = set() # track tool_ids that are skill loads + # tool_id → skill_id; looked up at tool_result time to replace the SSE payload with the curated detail + turn.skill_id_by_tool_id: _typing.Dict[str, str] = {} + # tool_id → tool_args, used at the tool_result stage to recover view_text_file's file_path/ranges + turn.view_text_file_args: _typing.Dict[str, _typing.Dict[str, _typing.Any]] = {} diff --git a/src/backend/core/agent_runtime/runner.py b/src/backend/core/agent_runtime/runner.py new file mode 100644 index 00000000..332dbb2f --- /dev/null +++ b/src/backend/core/agent_runtime/runner.py @@ -0,0 +1,444 @@ +"""Coordinate the main agent lifecycle.""" + +from __future__ import annotations +from core.agent_runtime.integrations import memory +from core.agent_runtime.integrations import memory_session +from core.ontology import output_review_state +from core.ontology import runtime_pack +from core.runs.tool_effects import errors +from core.llm.citations import allocator +from core.ontology import runtime_policy + +import asyncio +import logging +import typing as _typing +from contextlib import aclosing as _aclosing +from typing import Any, Dict, List + +import core.agent_runtime.budget as budget +import core.agent_runtime.direct as direct +import core.agent_runtime.events as events +import core.agent_runtime.integrations.evolution as evolution +import core.agent_runtime.integrations.memory_budget as memory_budget +import core.agent_runtime.integrations.ontology_evidence as ontology_evidence +import core.agent_runtime.integrations.ontology_repair as ontology_repair +import core.agent_runtime.skills as skills +import core.agent_runtime.targets as targets +import core.chat.message_parser as message_parser +import core.chat.plan_progress as plan_progress + + +from core.agent_runtime import prepare_main, stream_main +from core.agent_runtime.main_state import MainTurn + +logger = logging.getLogger(__name__) + + +async def astream_chat_workflow( + *, session_messages: List[Dict[str, Any]], user_message: str, context: Dict[str, Any] +): + """Stream a chat turn and close its resources on every consumer exit.""" + turn = MainTurn(context=context, session_messages=session_messages, user_message=user_message) + try: + async with _aclosing(_stream_turn(turn)) as stream: + async for event in stream: + yield event + finally: + from core.infra.task_lifecycle import cancel_and_join + + if turn.review_task is not None: + await cancel_and_join(turn.review_task) + if turn.review_owner is not None: + output_review_state.release_output_review(turn.ontology_runtime, owner=turn.review_owner) + await _close_turn(turn) + + +async def _close_turn(turn: MainTurn) -> None: + if turn.streaming_agent is not None and not turn.closed: + turn.closed = True + await turn.streaming_agent.shutdown() + + +async def _stream_turn(turn: MainTurn): + """Stream route -> handoff -> target execution. + + Yields chunks in the format: + - {"type": "content", "delta": "text chunk"} + - {"type": "tool_call", ...} + - {"type": "tool_result", ...} + - {"type": "meta", "route": "...", "sources": [...], ...} + """ + + from core.llm.api_runtime.agent import apply_api_scope + from core.harness.registry.evaluation import apply_evaluation_scope + + turn.context = apply_evaluation_scope(apply_api_scope(turn.context)) + + # ── Persistent dedicated sub-agent conversation mode ── + # Per-turn @mentions deliberately remain on the main route so the parent + # model emits the real call_subagent tool event and keeps normal streaming. + _agent_id = targets._direct_agent_id_from_context(turn.context) + if _agent_id: + + async with _aclosing( + direct._astream_subagent_direct( + agent_id=_agent_id, + session_messages=turn.session_messages, + user_message=turn.user_message, + context=turn.context, + ) + ) as direct_stream: + async for chunk in direct_stream: + yield chunk + return + + _request_ontology_runtime = turn.context.get("ontology_runtime") + if not isinstance(_request_ontology_runtime, dict): + _request_ontology_runtime = {} + turn.context["ontology_runtime"] = _request_ontology_runtime + explicit_skill_ids = skills._explicit_skill_ids_from_context(turn.context) + for skill_id in explicit_skill_ids: + await asyncio.to_thread( + runtime_policy.resolve_runtime_asset_tags, + runtime=_request_ontology_runtime, + kind="skill", + asset_id=skill_id, + user_id=str(turn.context.get("user_id", "") or ""), + ) + activations = runtime_pack.activate_runtime_for_asset( + _request_ontology_runtime, + kind="skill", + asset_id=skill_id, + ) + if activations: + await asyncio.to_thread( + ontology_evidence._record_ontology_activations, + activations, + _request_ontology_runtime, + turn.context, + ) + + # ── Inject explicit skill instructions ── + skill_msg = skills._build_skill_injection(turn.context) + if skill_msg: + turn.session_messages.insert(-1, skill_msg) + logger.info( + "[skill_inject] injected skill instructions for '%s'", + turn.context.get("skill_id"), + ) + + # ── [memory] Session memory opened here, never awaited on the SSE path ── + # First turn of a chat: retrieval starts as a background task so it overlaps + # agent assembly, and Fact injection is skipped if it blows its budget. + # Every later turn: the stored snapshot is replayed and no retrieval runs. + turn.memory_user_id = str(turn.context.get("user_id", "")) + turn.memory_workspace_id = str(turn.context.get("workspace_id", "") or "default") + _mem0_chat_id = turn.context.get("chat_id") or turn.context.get("conversation_id") + turn.memory_enabled = bool(turn.context.get("memory_enabled", False)) + _mem0_write_enabled = bool(turn.context.get("memory_write_enabled", False)) + # Under a team project chats.py passes "team:"; personal/default spaces don't pass it and fall back to the real user_id + _mem0_scope_user_id = str(turn.context.get("memory_scope_user_id", "") or turn.memory_user_id) + logger.info( + "[memory] user=%s scope=%s ws=%s chat=%s enabled=%s write=%s", + turn.memory_user_id, + _mem0_scope_user_id, + turn.memory_workspace_id, + _mem0_chat_id, + turn.memory_enabled, + _mem0_write_enabled, + ) + + turn.session_memory = await memory_session.open_session_memory( + chat_id=_mem0_chat_id, + scope_user_id=_mem0_scope_user_id, + workspace_id=turn.memory_workspace_id, + user_message=turn.user_message, + memory_enabled=turn.memory_enabled, + # The retrieval budget is part of the orchestration profile: how long + # this task type is willing to wait for memory is an assembly decision, + # not a global constant. Resolved here because retrieval starts before + # the agent is built. + budget_ms=memory_budget._profile_memory_budget_ms(turn.context), + ) + + # ── Main-route streaming ────────────────────────────────────── + warnings: _typing.List[str] = [] + turn.full_response = "" + turn.displayed_tools: set[str] = set() + turn.all_citations: _typing.List[_typing.Dict[str, _typing.Any]] = [] + turn.ontology_runtime = _request_ontology_runtime + turn.ontology_event_cursor = 0 + turn.ontology_trace: _typing.List[_typing.Dict[str, _typing.Any]] = [] + turn.last_plan: _typing.Optional[_typing.Dict[str, _typing.Any]] = None + turn.pending_plan_updates: _typing.Dict[str, _typing.Dict[str, _typing.Any]] = {} + _stream_errored = False + # 证据锚点发号器：跨轮续号；创建后绑到 agent 上（见下方 attach_allocator）， + # 中间件与本函数由此共享同一个计数器 + turn.anchor_allocator = allocator.AnchorAllocator( + await asyncio.to_thread( + allocator.anchor_start_for_chat, str(turn.context.get("chat_id") or "") or None + ) + ) + + try: + async with _aclosing(prepare_main.prepare_main(turn)) as phase: + async for event in phase: + yield event + + # Project scope is no longer passed via ContextVar — see the comment at + # the same spot in _astream_subagent_direct above. All scope-dependent + # tools captured it by closure when registered in agent_factory; the + # finishing _persist_artifacts gets the scope explicitly from chats.py. + + async with _aclosing(stream_main.stream_main(turn)) as phase: + async for event in phase: + yield event + + except Exception as e: + import traceback + + logger.error("stream_workflow_error: %s\n%s", e, traceback.format_exc()) + warnings.append(f"Streaming error: {str(e)[:200]}") + _stream_errored = True + + unknown_outcome = errors.find_tool_outcome_unknown(e) + if unknown_outcome is not None: + if turn.streaming_agent is not None: + await _close_turn(turn) + if unknown_outcome is e: + raise + raise unknown_outcome from e + + if not turn.full_response: + # No answer was produced: propagate. The run executor persists the + # tool cards already shown together with the error and emits a + # structured ``error`` event — no fabricated "please refer to the + # tool results above" reply that would read as a completed answer. + if turn.streaming_agent is not None: + await _close_turn(turn) + raise + + pending_ontology_events = turn.ontology_runtime.get("runtime_events", [])[ + turn.ontology_event_cursor : + ] + for ontology_event in pending_ontology_events: + yield dict(ontology_event) + turn.ontology_event_cursor += len(pending_ontology_events) + + _ontology_review_owner_id = ontology_evidence._ontology_review_owner( + turn.ontology_runtime, turn.context + ) + _ontology_review_claimed = bool( + turn.full_response + and output_review_state.claim_output_review(turn.ontology_runtime, owner=_ontology_review_owner_id) + ) + if _ontology_review_claimed: + turn.review_owner = _ontology_review_owner_id + from core.ontology.reviewer import review_ontology_output + + yield { + "type": "ontology_review", + "status": "started", + "level": turn.ontology_runtime.get("review_level", "checkpoint"), + **ontology_evidence._ontology_review_event_context(turn.ontology_runtime), + } + repair_event_queue: asyncio.Queue[_typing.Dict[str, _typing.Any]] = asyncio.Queue() + + async def _remediate(payload: Dict[str, Any]) -> str: + primary_usage = await turn.streaming_agent.aget_usage() + primary_context_usage = turn.streaming_agent.get_context_usage(primary_usage) + try: + repaired, _, cursor, _ = await ontology_repair._run_ontology_repair_round( + streaming_agent=turn.streaming_agent, + context=turn.context, + payload=payload, + runtime=turn.ontology_runtime, + trace=turn.ontology_trace, + citations=turn.all_citations, + allocator=turn.anchor_allocator, + event_cursor=turn.ontology_event_cursor, + event_sink=repair_event_queue.put, + ) + finally: + turn.streaming_agent.restore_context_usage(primary_context_usage) + turn.ontology_event_cursor = cursor + return repaired + + turn.review_task = review_task = asyncio.create_task( + review_ontology_output( + task=turn.user_message, + answer=turn.full_response, + runtime=turn.ontology_runtime, + trace=turn.ontology_trace, + citations=turn.all_citations, + user_id=str(turn.context.get("user_id", "")), + chat_id=turn.context.get("chat_id"), + model_name=str(turn.context.get("model_name", "") or "") or None, + model_provider_id=str(turn.context.get("model_provider_id", "") or "") or None, + remediate=_remediate, + ) + ) + try: + while not review_task.done() or not repair_event_queue.empty(): + try: + repair_event = await asyncio.wait_for( + repair_event_queue.get(), + timeout=0.1, + ) + except asyncio.TimeoutError: + continue + yield repair_event + review = await review_task + except (asyncio.CancelledError, GeneratorExit): + from core.infra.task_lifecycle import cancel_and_join + + await cancel_and_join(review_task) + output_review_state.release_output_review(turn.ontology_runtime, owner=_ontology_review_owner_id) + await _close_turn(turn) + raise + except Exception as exc: # noqa: BLE001 + if not review_task.done(): + review_task.cancel() + review = ontology_evidence._ontology_review_failure_result(turn.full_response, exc) + output_review_state.complete_output_review( + turn.ontology_runtime, + owner=_ontology_review_owner_id, + verdict=str(review.get("verdict") or "unknown"), + attempts=int(review.get("attempts") or 1), + ) + turn.ontology_runtime.setdefault("output_review", {}).update( + { + "revised": bool(review.get("revised")), + "annotated": bool(review.get("annotated")), + "repair_attempts": int(review.get("repair_attempts") or 0), + "violations": review.get("violations") or [], + "affected_claims": review.get("affected_claims") or [], + "evidence": review.get("evidence") or [], + "feedback": review.get("feedback") or [], + "manual_review": review.get("manual_review") or {}, + "candidate_answer": (review.get("answer") if review.get("revised") else ""), + "new_tools": review.get("new_tools") or [], + "new_citation_count": int(review.get("new_citation_count") or 0), + "latency_ms": review.get("latency_ms"), + } + ) + yield { + "type": "ontology_review", + "status": "completed", + "level": turn.ontology_runtime.get("review_level", "checkpoint"), + "verdict": review["verdict"], + "revised": bool(review.get("revised")), + "annotated": bool(review.get("annotated")), + "repair_attempts": int(review.get("repair_attempts") or 0), + "candidate_answer": review.get("answer") if review.get("revised") else "", + "manual_review": review.get("manual_review") or {}, + "violations": review.get("violations") or [], + "affected_claims": review.get("affected_claims") or [], + "evidence": review.get("evidence") or [], + "feedback": review.get("feedback") or [], + "new_tools": review.get("new_tools") or [], + "new_citation_count": int(review.get("new_citation_count") or 0), + "latency_ms": review.get("latency_ms"), + **ontology_evidence._ontology_review_event_context(turn.ontology_runtime), + } + + await _close_turn(turn) + + # update_plan finalization: the model routinely finishes its last step and + # streams the answer without a closing update_plan call, leaving the plan + # bar stuck at N-1/N. When the turn ends normally with an answer and no + # step is still pending, the remaining in_progress step is the one that + # just produced the answer — close it out. + if turn.last_plan and turn.full_response and not _stream_errored: + _plan_steps = turn.last_plan.get("steps") or [] + if ( + _plan_steps + and all(s.get("status") != "pending" for s in _plan_steps) + and any(s.get("status") == "in_progress" for s in _plan_steps) + ): + _closed = { + "title": turn.last_plan.get("title", ""), + "steps": [{**s, "status": "completed"} for s in _plan_steps], + } + turn.last_plan = _closed + plan_progress.save_plan_progress(str(turn.context.get("chat_id") or ""), _closed) + yield {"type": "plan_update", **_closed} + + # 计划栏收尾 —— 本轮结束时把"还会不会有人来更新这份清单"这件事记进库里。 + # + # 过去只有前端在流结束时给它盖章，于是收尾依赖"这个标签页恰好在跟这条流"：后台作业 + # 跑完的交付轮是**后端自己发起**的，用户没点任何东西，页面可能压根没跟——计划栏就永远 + # 停在转圈。这里改由服务端认定：本会话还有活着的作业就先不收尾（后面还有交付轮要来 + # 更新它），没有了就记 settled。步骤状态一个字不改，settled 只表示"没有下一轮了"。 + if turn.last_plan: + _plan_chat_id = str(turn.context.get("chat_id") or "") + if _plan_chat_id and not events._chat_has_live_job(_plan_chat_id): + plan_progress.settle_plan_progress(_plan_chat_id) + + _main_usage = await turn.streaming_agent.aget_usage() + _main_context_usage = turn.streaming_agent.get_context_usage(_main_usage) + yield { + "type": "meta", + "route": "main", + "is_markdown": message_parser.looks_markdown(turn.full_response), + "sources": message_parser.resolve_sources_conflict([]), + "artifacts": [], + "warnings": warnings, + "citations": turn.all_citations, + "usage": _main_usage, + "context_usage": _main_context_usage, + "_compaction_budget_inputs": budget._compaction_budget_inputs(turn.agent, turn.ctx_window), + "ontology_governance": ontology_evidence._ontology_governance_summary( + turn.ontology_runtime + ), + # Settlement has not run yet (memory writes are scheduled below), so all + # this can honestly say is "settling" — the client draws a skeleton and + # fills it in when the summary arrives. + "evolution_pending": evolution._evolution_pending_marker(turn.context, _mem0_write_enabled), + } + + # ── [memory] Post-response pipeline (SSE already closed, user isn't waiting) ── + # No memory is written unless the user opted into memory_write_enabled (first gate). + # Everything goes through memory_pipeline.schedule_post_response_tasks: + # global Semaphore + 4 extractors + sanitize + write L1/L2/Session + audit, + # all executed in the background with bounds. + if _mem0_write_enabled: + logger.info( + "[memory] schedule post-response: full_response_len=%s, user=%s ws=%s", + len(turn.full_response) if turn.full_response else 0, + turn.memory_user_id, + turn.memory_workspace_id, + ) + memory.save_memories_background( + turn.memory_user_id, + turn.user_message, + turn.full_response, + _mem0_write_enabled, + workspace_id=turn.memory_workspace_id, + chat_id=_mem0_chat_id, + scope_user_id=_mem0_scope_user_id, + # Lets the pipeline report its real write count back to this turn's + # settlement instead of the card having to guess one. + message_id=str(turn.context.get("message_id") or ""), + ) + else: + logger.debug( + "[workflow] memory save skipped: write_enabled=False (user=%s)", + turn.memory_user_id, + ) + + # ── [evolution] Evidence assembly (SSE already closed, user isn't waiting) ── + # The response path only bound versions and appended events; joining them + # into one causal Episode is deliberately deferred to here. + if not turn.context.get("evaluation_session"): + evolution._assemble_episode_background( + message_id=str(turn.context.get("message_id") or ""), + run_id=str(turn.context.get("run_id") or ""), + chat_id=str(turn.context.get("chat_id") or ""), + user_id=str(turn.context.get("user_id") or ""), + objective=turn.user_message, + agent=turn.streaming_agent, + memory_task=turn.session_memory.retrieval_task, + latency_ms=None, + memory_write_enabled=_mem0_write_enabled, + ) diff --git a/src/backend/core/agent_runtime/skills.py b/src/backend/core/agent_runtime/skills.py new file mode 100644 index 00000000..0e34ea24 --- /dev/null +++ b/src/backend/core/agent_runtime/skills.py @@ -0,0 +1,176 @@ +"""Turn explicitly invoked skills into the load-instruction message for a turn.""" + +from __future__ import annotations +from core.llm.context import ir_constants +from core.llm.context import items + +import logging +import typing as _typing +from typing import Any, Dict, List, Optional + + +logger = logging.getLogger(__name__) + + +def _extract_skill_id_from_path(path: str) -> str: + """Extract skill_id from a SKILL.md path (convention: .../skills//SKILL.md).""" + if not path: + return "" + import os + + norm = path.replace("\\", "/").strip().strip('"').strip("'") + parts = [p for p in norm.split("/") if p] + if not parts: + return "" + if parts[-1].upper() == "SKILL.MD" and len(parts) >= 2: + return parts[-2] + base = os.path.basename(norm) + if base.upper() == "SKILL.MD" and len(parts) >= 2: + return parts[-2] + return "" + + +def _explicit_skill_ids_from_context(context: Dict[str, Any]) -> List[str]: + skill_ids: _typing.List[str] = [] + single = context.get("skill_id") + if single: + skill_ids.append(str(single)) + multi = context.get("skill_ids") + if isinstance(multi, list): + skill_ids.extend(str(item) for item in multi if item) + seen: set[str] = set() + return [item for item in skill_ids if not (item in seen or seen.add(item))] + + +def _build_skill_injection(context: Dict[str, Any]) -> Optional[Dict[str, Any]]: + """Build an explicit-invocation hint message (skill load instructions + MCP tool activation notice). + + Trigger sources: + - ``skill_id`` (single skill, slash-command selection) / ``skill_ids`` + (skill list, expanded from an explicitly referenced plugin) + → inject each skill's SKILL.md load instruction. + - ``mcp_ids`` (MCP servers activated by an explicitly referenced plugin or + a connector selected in the composer) → these tools are already + force-enabled into the toolset (see _build_ctx); here we append a + sentence telling the model they are ready and can be called on demand. + + Once enabled, ambient MCP tools stay resident and the model calls them by + description on its own. Explicit selections guide the model toward the selected + capabilities without forcing tool_choice or rejecting a reply without tool usage. + + Returns a dict {"role": "user", "content": "..."} or None. + """ + skill_ids = _explicit_skill_ids_from_context(context) + + mcp_ids = [str(m) for m in (context.get("mcp_ids") or []) if m] + connector_id = str(context.get("connector_id") or "") + skill_id = str(context.get("skill_id") or "") + skill_name = context.get("skill_name") + plugin_name = context.get("plugin_name") + connector_name = context.get("connector_name") + + if not skill_ids and not mcp_ids: + return None + + sections: _typing.List[str] = [] + + # ── Skills: load instructions ── + if skill_ids: + try: + from core.capabilities.skills.loader_factory import get_skill_loader + + loader = get_skill_loader() + metadata_all = loader.load_all_metadata() + entries: _typing.List[str] = [] + for sid in skill_ids: + meta = metadata_all.get(sid) + if not meta: + logger.warning("[skill_inject] skill_id=%s not found", sid) + continue + # Materialize once; each deployment publishes its own usable path. + skill_dir = loader.get_skill_dir(sid) + if not skill_dir: + logger.warning("[skill_inject] skill_id=%s has no skill dir", sid) + continue + from core.capabilities.skills.config import model_facing_skill_dir + + sandbox_dir = model_facing_skill_dir(sid, skill_dir) + entries.append(f'- 「{sid}」：view_text_file(file_path="{sandbox_dir}/SKILL.md")') + if entries: + sections.append("技能（使用技能时先读取对应说明文件）：\n" + "\n".join(entries)) + except Exception as e: # noqa: BLE001 + logger.error("[skill_inject] failed to load skills %s: %s", skill_ids, e) + + # ── MCP: activation notice (the tools themselves are already in the toolset; call by description) ── + if mcp_ids: + if connector_id: + sections.append( + f"MCP 工具：连接器「{connector_name or connector_id}」已被用户显式选择。" + "请优先按需使用该连接器完成相关任务，" + "也不得用其他工具冒充该连接器的结果。" + ) + elif plugin_name: + sections.append( + f"MCP 工具：插件「{plugin_name}」的 {len(mcp_ids)} 个 MCP 服务已就绪。" + "请优先按需使用该插件的 MCP 工具或技能，" + "使用技能时先读取对应说明文件。" + ) + else: + sections.append( + f"MCP 工具：本轮显式激活的 {len(mcp_ids)} 个 MCP 连接器服务已就绪，" + "可直接按需调用（无需加载文件），请优先使用它们完成相关任务。" + ) + + if not sections: + return None + + # 极速模式警示：被呼唤技能的 SKILL.md 往往指示 Bash/沙箱/文件工具流程， + # 而极速模式不装配这些工具。若不在呼唤点明说，模型会照着技能说明发起 + # 不存在的工具调用（实测会卡死在参数流上）。 + if str(context.get("chat_mode", "") or "").lower() == "turbo" and skill_ids: + sections.append( + "注意：当前为极速模式，没有 Bash、沙箱、文件读写和文档生成工具。" + "技能说明中涉及这些工具的步骤一律不可执行，也不要尝试调用——" + "读取 SKILL.md 后，只输出你能以纯文本交付的部分（结构、正文、模板等），" + "并在结尾建议用户切换到「快速模式」或「思考模式」完整执行该技能。" + ) + + if plugin_name and connector_name: + header = ( + f"用户已显式调用插件「{plugin_name}」并选择连接器「{connector_name}」，" + "请优先采用这些能力：" + ) + elif plugin_name: + header = f"用户已显式调用插件「{plugin_name}」，请优先按需采用以下能力：" + elif connector_name: + header = f"用户已显式选择连接器「{connector_name}」，请优先采用其能力：" + elif skill_id: + header = ( + f"用户已显式调用技能「{skill_name or skill_id}」，" + "请优先采用该技能，使用前读取其 SKILL.md：" + ) + else: + header = "用户已显式指定使用以下能力，请优先采用：" + content = ( + "\n" + + header + + "\n" + + "\n\n".join(sections) + + "\n" + ) + item = items.make_text_context_item( + content, + item_id="harness:explicit_invocation", + kind=ir_constants.KIND_REMINDER, + origin="harness:explicit_invocation", + trust="system", + created_seq=0, + priority=900, + token_budget=8_000, + truncation_policy=ir_constants.POLICY_HEAD_TAIL, + ) + return { + "role": "user", + "content": content, + ir_constants.SESSION_CONTEXT_META_KEY: items.session_context_metadata(item), + } diff --git a/src/backend/orchestration/strategy.py b/src/backend/core/agent_runtime/strategy.py similarity index 100% rename from src/backend/orchestration/strategy.py rename to src/backend/core/agent_runtime/strategy.py diff --git a/src/backend/core/agent_runtime/stream_direct.py b/src/backend/core/agent_runtime/stream_direct.py new file mode 100644 index 00000000..229a639b --- /dev/null +++ b/src/backend/core/agent_runtime/stream_direct.py @@ -0,0 +1,298 @@ +"""Translate direct-agent output while preserving scoped audit context.""" + +from __future__ import annotations +from core.llm.citations import allocator + +import json +import logging + +import core.agent_runtime.events as events +import core.agent_runtime.integrations.ontology_evidence as ontology_evidence +import core.agent_runtime.skills as skills +import core.chat.tool_payloads as tool_payloads +import core.config.display_names as display_names +from core.agent_runtime.direct_state import DirectTurn + +logger = logging.getLogger(__name__) + + +async def stream_direct(turn: DirectTurn): + try: + while True: + try: + event_type, payload = await events._anext_in_subagent_log_scope( + turn.direct_stream, + turn.direct_log_id, + ) + except StopAsyncIteration: + break + state_runtime = getattr(turn.streaming_agent.agent.state, "ontology_runtime", None) + if isinstance(state_runtime, dict): + turn.ontology_runtime = state_runtime + turn.context["ontology_runtime"] = state_runtime + pending_ontology_events = turn.ontology_runtime.get("runtime_events", [])[ + turn.ontology_event_cursor : + ] + for ontology_event in pending_ontology_events: + yield dict(ontology_event) + turn.ontology_event_cursor += len(pending_ontology_events) + if event_type == "text_delta": + turn.full_response += payload + yield {"type": "content", "event": "ai_message", "delta": payload} + + elif event_type == "vision_progress": + # 识图是模型开口前的一段纯网络等待；透传给前端，让轮级状态 + # 从「深度拥抱中」换成「图像理解中」，而不是干等一个笼统的计时器。 + yield {"type": "vision_progress", **(payload or {})} + + elif event_type == "reasoning_protocol": + yield {"type": "thinking", **payload} + + elif event_type == "steer_applied": + yield {"type": "steer_applied", **(payload or {})} + + elif event_type == "thinking_delta": + yield {"type": "thinking", "delta": payload} + + elif event_type == "context_usage": + yield {"type": "context_usage", **(payload or {})} + + elif event_type == "model_step": + yield {"type": "model_step", "step": payload} + + elif event_type == "model_call_start": + yield {"type": "model_dispatch"} + + elif event_type == "tool_call_start": + tool_name = payload.get("name", "unknown") + if tool_name != "update_plan": + yield { + "type": "tool_call_start", + "tool_name": tool_name, + "tool_display_name": display_names.TOOL_DISPLAY_NAMES.get( + tool_name, tool_name + ), + "tool_id": payload.get("id", ""), + } + + elif event_type == "tool_call_delta": + tool_name = payload.get("name", "unknown") + if tool_name != "update_plan" and payload.get("delta"): + yield { + "type": "tool_call_delta", + "tool_name": tool_name, + "tool_id": payload.get("id", ""), + "arguments_delta": payload.get("delta", ""), + } + + elif event_type == "tool_call": + tool_name = payload.get("name", "unknown") + tool_id = payload.get("id", "") + tool_args = payload.get("args", {}) + + _is_fast_emit = tool_name in tool_payloads._FAST_EMIT_TOOLS + if tool_id and tool_id in turn.displayed_tools: + if _is_fast_emit and tool_args: + pass # re-emit with updated args + else: + continue + if not tool_payloads._tool_args_ready(tool_name, tool_args): + continue + if tool_id: + turn.displayed_tools.add(tool_id) + + is_skill_load = ( + tool_name == "view_text_file" + and isinstance(tool_args, dict) + and "SKILL.md" in str(tool_args.get("file_path", "")) + ) + if is_skill_load and tool_id: + turn.skill_load_ids.add(tool_id) + _sid = skills._extract_skill_id_from_path(str(tool_args.get("file_path", ""))) + if _sid: + turn.skill_id_by_tool_id[tool_id] = _sid + # Non-skill view_text_file also needs trimming at the tool_result stage → record the args + if ( + tool_name in ("view_text_file", "Read") + and not is_skill_load + and tool_id + and isinstance(tool_args, dict) + ): + turn.view_text_file_args[tool_id] = tool_args + emit_name = "load_skill" if is_skill_load else tool_name + display_name = ( + "加载技能" + if is_skill_load + else display_names.TOOL_DISPLAY_NAMES.get(tool_name, tool_name) + ) + safe_args = tool_args if isinstance(tool_args, dict) else {} + turn.ontology_trace.append( + { + "type": "tool_call", + "tool_id": tool_id, + "tool_name": tool_name, + "input": ontology_evidence._bounded_trace_value(safe_args), + } + ) + + yield { + "type": "tool_call", + "tool_name": emit_name, + "tool_display_name": display_name, + "tool_args": safe_args, + "input": safe_args, + "tool_id": tool_id, + } + + elif event_type == "tool_progress": + yield {"type": "tool_progress", **payload} + + elif event_type == "tool_result": + tool_name = payload.get("name", "unknown") + tool_id = payload.get("id", "") + is_skill_result = (tool_id and tool_id in turn.skill_load_ids) or ( + tool_name == "view_text_file" and "SKILL.md" in str(payload.get("content", "")) + ) + if is_skill_result: + turn.direct_skill_count += 1 + tool_name = "load_skill" + else: + turn.direct_tool_count += 1 + + _missing_call = events._synthesize_missing_tool_call( + tool_id, tool_name, turn.displayed_tools + ) + if _missing_call is not None: + turn.ontology_trace.append( + { + "type": "tool_call", + "tool_id": tool_id, + "tool_name": tool_name, + "input": {}, + } + ) + yield _missing_call + + tool_content = payload.get("content", "") + + try: + tool_result_json = json.loads(tool_content) if tool_content else {} + except json.JSONDecodeError: + tool_result_json = {"result": tool_content} + if isinstance(payload.get("mcp"), dict): + tool_result_json = payload["mcp"] + + # Skill load: replace the full SKILL.md text with the same + # curated detail used by the capability center; affects + # only the SSE payload sent to the frontend — the agent's + # own memory still holds the full content + if is_skill_result: + _sid = turn.skill_id_by_tool_id.get( + tool_id, "" + ) or skills._extract_skill_id_from_path(str(tool_content)) + tool_result_json = tool_payloads._build_skill_load_payload(_sid) + elif tool_name == "view_text_file": + # Plain file read (AgentScope built-in view_text_file): replace with file metadata + short preview + tool_result_json = tool_payloads._build_view_text_file_payload( + turn.view_text_file_args.get(tool_id, {}), tool_content + ) + elif tool_name == "Read": + # Claude-Code-style Read tool: JSON payload, content holds the whole file + tool_result_json = tool_payloads._build_read_tool_payload( + turn.view_text_file_args.get(tool_id, {}), tool_result_json + ) + elif tool_name == "read_artifact": + tool_result_json = tool_payloads._build_read_artifact_payload(tool_result_json) + + extracted_query = "" + if isinstance(tool_result_json, dict) and "result" in tool_result_json: + result_data = tool_result_json["result"] + if isinstance(result_data, dict): + extracted_query = result_data.get("query", result_data.get("question", "")) + + cit_dicts = allocator.collect_citation_dicts(tool_id, turn.anchor_allocator) + turn.all_citations.extend(cit_dicts) + turn.ontology_trace.append( + { + "type": "tool_result", + "tool_id": tool_id, + "tool_name": tool_name, + "result": ontology_evidence._bounded_trace_value(tool_result_json), + "citations": cit_dicts, + } + ) + + yield { + "type": "tool_result", + "tool_name": tool_name, + "tool_args": {"query": extracted_query} if extracted_query else {}, + "result": tool_result_json, + "tool_id": tool_id, + "citations": cit_dicts, + "status": payload.get("status", "success"), + "model_step": payload.get("model_step"), + } + + elif event_type in ("heartbeat", "model_progress"): + # heartbeat = transport keep-alive; model_progress = the + # model is still streaming while a small argument batch or + # another suppressed event has nothing renderable yet — + # forwarded so the run watchdog counts activity (it + # excludes only heartbeat). + yield {"type": event_type} + + elif event_type == "tool_pending": + yield {"type": "tool_pending", **(payload or {})} + + elif event_type == "subagent_event": + # Bypass channel for the sub-agent's internal + # thinking/tool_call/tool_result/content — attached under + # the call_subagent tool card that launched it (linked via + # parent_tool_id). + if (payload or {}).get("sub_type") == "progress": + # Sub-agent liveness (long tool-call-arg generation with + # nothing renderable yet) — feed the run inactivity + # watchdog without writing a sub-step to the stream or + # the persisted tool log. + yield {"type": "model_progress"} + else: + nested_citations = ontology_evidence._capture_nested_ontology_evidence( + payload or {}, + turn.ontology_trace, + turn.all_citations, + turn.anchor_allocator, + ) + yield { + "type": "subagent_event", + **(payload or {}), + **({"citations": nested_citations} if nested_citations else {}), + } + + elif event_type in ( + "file_confirm", + "design_pick", + "user_question", + "user_question_resolved", + ): + # Human-interaction events (§13 MySpace write confirm / + # site-design pick-one-of-three / ask_user_question): a tool + # coroutine has suspended waiting for the user's out-of-band + # action. Pass through to the frontend to show the card; the + # agent task stays blocked in that tool and this SSE stream + # does not end — after the out-of-band answer POST the tool + # resumes in place. + yield {"type": event_type, **(payload or {})} + + elif event_type == "error": + # payload may be a real exception object (kind=="err") or a + # dict (e.g. ExceedMaxIters mapped to {"kind":..,"name":..}). + # Raising the latter directly gives a TypeError, masking + # the real situation — wrap it in a RuntimeError. + if isinstance(payload, BaseException): + raise payload + raise RuntimeError(str(payload)) + + except BaseException: + # Keep clients alive after a normal first draft so the same agent + # can continue its ReAct context during ontology repair. + raise diff --git a/src/backend/core/agent_runtime/stream_main.py b/src/backend/core/agent_runtime/stream_main.py new file mode 100644 index 00000000..84916e9e --- /dev/null +++ b/src/backend/core/agent_runtime/stream_main.py @@ -0,0 +1,354 @@ +"""Translate main-agent model and tool output into chat events.""" + +from __future__ import annotations +from core.llm.citations import allocator + +import json +import logging + +import core.agent_runtime.events as events +import core.agent_runtime.integrations.ontology_evidence as ontology_evidence +import core.agent_runtime.skills as skills +import core.chat.plan_progress as plan_progress +import core.chat.tool_payloads as tool_payloads +import core.config.display_names as display_names +import core.llm.tools.plan_update as plan_update_tool +from core.agent_runtime.main_state import MainTurn + +logger = logging.getLogger(__name__) + + +async def stream_main(turn: MainTurn): + try: + # Project initialization is a server-expanded command. The DB row + # deliberately remains /init; only the live model input is expanded. + stream_options = ( + {"effective_user_message": turn.user_message} + if turn.context.get("project_init") + else {} + ) + turn.main_stream = turn.streaming_agent.stream( + turn.session_messages, turn.context, **stream_options + ) + async for event_type, payload in turn.main_stream: + state_runtime = getattr(turn.streaming_agent.agent.state, "ontology_runtime", None) + if isinstance(state_runtime, dict): + turn.ontology_runtime = state_runtime + turn.context["ontology_runtime"] = state_runtime + pending_ontology_events = turn.ontology_runtime.get("runtime_events", [])[ + turn.ontology_event_cursor : + ] + for ontology_event in pending_ontology_events: + yield dict(ontology_event) + turn.ontology_event_cursor += len(pending_ontology_events) + if event_type == "text_delta": + turn.full_response += payload + yield {"type": "content", "event": "ai_message", "delta": payload} + + elif event_type == "vision_progress": + # 识图是模型开口前的一段纯网络等待；透传给前端，让轮级状态 + # 从「深度拥抱中」换成「图像理解中」，而不是干等一个笼统的计时器。 + yield {"type": "vision_progress", **(payload or {})} + + elif event_type == "reasoning_protocol": + yield {"type": "thinking", **payload} + + elif event_type == "steer_applied": + yield {"type": "steer_applied", **(payload or {})} + + elif event_type == "thinking_delta": + yield {"type": "thinking", "delta": payload} + + elif event_type == "context_usage": + yield {"type": "context_usage", **(payload or {})} + + elif event_type == "model_step": + yield {"type": "model_step", "step": payload} + + elif event_type == "model_call_start": + yield {"type": "model_dispatch"} + + elif event_type == "tool_call_start": + tool_name = payload.get("name", "unknown") + if tool_name != "update_plan": + yield { + "type": "tool_call_start", + "tool_name": tool_name, + "tool_display_name": display_names.TOOL_DISPLAY_NAMES.get( + tool_name, tool_name + ), + "tool_id": payload.get("id", ""), + } + + elif event_type == "tool_call_delta": + tool_name = payload.get("name", "unknown") + if tool_name != "update_plan" and payload.get("delta"): + yield { + "type": "tool_call_delta", + "tool_name": tool_name, + "tool_id": payload.get("id", ""), + "arguments_delta": payload.get("delta", ""), + } + + elif event_type == "tool_call": + tool_name = payload.get("name", "unknown") + tool_id = payload.get("id", "") + tool_args = payload.get("args", {}) + + # Stage the full plan until its real result is saved. + # A tool request alone must not announce completed work. + if tool_name == "update_plan": + _pu = plan_update_tool.parse_plan_update_args(tool_args) + if _pu: + turn.pending_plan_updates[tool_id] = _pu + continue + + # In streaming mode, the first chunk for a tool_call may + # arrive with empty args. For view_text_file we need args + # to decide if this is a skill load, so skip empty-arg + # duplicates until we get the complete args. + _is_fast_emit = tool_name in tool_payloads._FAST_EMIT_TOOLS + if tool_id and tool_id in turn.displayed_tools: + # Fast-emit tools: re-emit when args arrive so the + # frontend can update input display immediately. + if _is_fast_emit and tool_args: + pass # fall through to emit update + else: + continue + if not tool_payloads._tool_args_ready(tool_name, tool_args): + continue + if tool_id: + turn.displayed_tools.add(tool_id) + + # Detect skill loading: view_text_file reading a SKILL.md + is_skill_load = ( + tool_name == "view_text_file" + and isinstance(tool_args, dict) + and "SKILL.md" in str(tool_args.get("file_path", "")) + ) + if is_skill_load and tool_id: + turn.skill_load_ids.add(tool_id) + _sid = skills._extract_skill_id_from_path(str(tool_args.get("file_path", ""))) + if _sid: + turn.skill_id_by_tool_id[tool_id] = _sid + if ( + tool_name in ("view_text_file", "Read") + and not is_skill_load + and tool_id + and isinstance(tool_args, dict) + ): + turn.view_text_file_args[tool_id] = tool_args + emit_name = "load_skill" if is_skill_load else tool_name + display_name = ( + "加载技能" + if is_skill_load + else display_names.TOOL_DISPLAY_NAMES.get(tool_name, tool_name) + ) + safe_args = tool_args if isinstance(tool_args, dict) else {} + turn.ontology_trace.append( + { + "type": "tool_call", + "tool_id": tool_id, + "tool_name": tool_name, + "input": ontology_evidence._bounded_trace_value(safe_args), + } + ) + + # Resolve sub-agent name for call_subagent tool card + _tc_sa_name = "" + if tool_name == "call_subagent" and turn.visible_subagents: + _tc_sa_id = safe_args.get("agent_id", "") if safe_args else "" + if _tc_sa_id: + for _sa in turn.visible_subagents: + if _sa.get("agent_id") == _tc_sa_id: + _tc_sa_name = _sa.get("name", "") + break + if not _tc_sa_name and turn.mentioned_ids: + for _sa in turn.visible_subagents: + if _sa.get("agent_id") in turn.mentioned_ids: + _tc_sa_name = _sa.get("name", "") + break + + yield { + "type": "tool_call", + "tool_name": emit_name, + "tool_display_name": display_name, + "tool_args": safe_args, + "input": safe_args, + "tool_id": tool_id, + **({"subagent_name": _tc_sa_name} if _tc_sa_name else {}), + } + + elif event_type == "tool_progress": + yield {"type": "tool_progress", **payload} + + elif event_type == "tool_result": + tool_name = payload.get("name", "unknown") + tool_id = payload.get("id", "") + # The canonical model_step was already forwarded and + # persisted independently. Keep update_plan in the plan + # bar; only its redundant display card is suppressed. + if tool_name == "update_plan": + _pu = turn.pending_plan_updates.pop(tool_id, None) + if _pu and payload.get("status") == "success": + turn.last_plan = _pu + plan_progress.save_plan_progress( + str(turn.context.get("chat_id") or ""), _pu + ) + yield {"type": "plan_update", **_pu} + continue + # Also override tool_name for skill load results + is_skill_result = (tool_id and tool_id in turn.skill_load_ids) or ( + tool_name == "view_text_file" and "SKILL.md" in str(payload.get("content", "")) + ) + if is_skill_result: + tool_name = "load_skill" + + _missing_call = events._synthesize_missing_tool_call( + tool_id, tool_name, turn.displayed_tools + ) + if _missing_call is not None: + turn.ontology_trace.append( + { + "type": "tool_call", + "tool_id": tool_id, + "tool_name": tool_name, + "input": {}, + } + ) + yield _missing_call + + tool_content = payload.get("content", "") + + # Parse tool result + try: + tool_result_json = json.loads(tool_content) if tool_content else {} + except json.JSONDecodeError: + tool_result_json = {"result": tool_content} + if isinstance(payload.get("mcp"), dict): + tool_result_json = payload["mcp"] + + # Skill load: replace the full SKILL.md text with the same curated detail used by the capability center + if is_skill_result: + _sid = turn.skill_id_by_tool_id.get( + tool_id, "" + ) or skills._extract_skill_id_from_path(str(tool_content)) + tool_result_json = tool_payloads._build_skill_load_payload(_sid) + elif tool_name == "view_text_file": + # Plain file read (AgentScope built-in view_text_file): replace with file metadata + short preview + tool_result_json = tool_payloads._build_view_text_file_payload( + turn.view_text_file_args.get(tool_id, {}), tool_content + ) + elif tool_name == "Read": + # Claude-Code-style Read tool: JSON payload, content holds the whole file + tool_result_json = tool_payloads._build_read_tool_payload( + turn.view_text_file_args.get(tool_id, {}), tool_result_json + ) + elif tool_name == "read_artifact": + tool_result_json = tool_payloads._build_read_artifact_payload(tool_result_json) + + # Extract query if present + extracted_query = "" + if isinstance(tool_result_json, dict) and "result" in tool_result_json: + result_data = tool_result_json["result"] + if isinstance(result_data, dict): + extracted_query = result_data.get("query", result_data.get("question", "")) + + # Citations + cit_dicts = allocator.collect_citation_dicts(tool_id, turn.anchor_allocator) + turn.all_citations.extend(cit_dicts) + turn.ontology_trace.append( + { + "type": "tool_result", + "tool_id": tool_id, + "tool_name": tool_name, + "result": ontology_evidence._bounded_trace_value(tool_result_json), + "citations": cit_dicts, + } + ) + + # Resolve sub-agent name from call_subagent result text + _tr_sa_name = "" + if tool_name == "call_subagent": + _res_str = ( + str(tool_result_json.get("result", "")) + if isinstance(tool_result_json, dict) + else str(tool_result_json) + ) + if "【" in _res_str and "】" in _res_str: + _tr_sa_name = _res_str.split("【", 1)[1].split("】", 1)[0] + + yield { + "type": "tool_result", + "tool_name": tool_name, + "tool_args": {"query": extracted_query} if extracted_query else {}, + "result": tool_result_json, + "tool_id": tool_id, + "citations": cit_dicts, + "status": payload.get("status", "success"), + "model_step": payload.get("model_step"), + **({"subagent_name": _tr_sa_name} if _tr_sa_name else {}), + } + + elif event_type in ("heartbeat", "model_progress"): + # heartbeat = transport keep-alive; model_progress = the + # model is still streaming while a small argument batch or + # another suppressed event has nothing renderable yet — + # forwarded so the run watchdog counts activity (it + # excludes only heartbeat). + yield {"type": event_type} + + elif event_type == "tool_pending": + yield {"type": "tool_pending", **(payload or {})} + + elif event_type == "subagent_event": + # Bypass channel for the sub-agent's internal + # thinking/tool_call/tool_result/content — attached under + # the call_subagent tool card that launched it (linked via + # parent_tool_id). + if (payload or {}).get("sub_type") == "progress": + # Sub-agent liveness (long tool-call-arg generation with + # nothing renderable yet) — feed the run inactivity + # watchdog without writing a sub-step to the stream or + # the persisted tool log. + yield {"type": "model_progress"} + else: + nested_citations = ontology_evidence._capture_nested_ontology_evidence( + payload or {}, + turn.ontology_trace, + turn.all_citations, + turn.anchor_allocator, + ) + yield { + "type": "subagent_event", + **(payload or {}), + **({"citations": nested_citations} if nested_citations else {}), + } + + elif event_type in ( + "file_confirm", + "design_pick", + "user_question", + "user_question_resolved", + ): + # Human-interaction events (§13 MySpace write confirm / + # site-design pick-one-of-three / ask_user_question): a tool + # coroutine has suspended waiting for the user's out-of-band + # action. Pass through to the frontend to show the card; the + # agent task stays blocked in that tool and this SSE stream + # does not end — after the out-of-band answer POST the tool + # resumes in place. + yield {"type": event_type, **(payload or {})} + + elif event_type == "error": + # payload may be a real exception object (kind=="err") or a + # dict (e.g. ExceedMaxIters mapped to {"kind":..,"name":..}). + # Raising the latter directly gives a TypeError, masking + # the real situation — wrap it in a RuntimeError. + if isinstance(payload, BaseException): + raise payload + raise RuntimeError(str(payload)) + + finally: + if turn.main_stream is not None: + await turn.main_stream.aclose() diff --git a/src/backend/core/agent_runtime/targets.py b/src/backend/core/agent_runtime/targets.py new file mode 100644 index 00000000..cf726843 --- /dev/null +++ b/src/backend/core/agent_runtime/targets.py @@ -0,0 +1,208 @@ +"""Resolve which agent a turn runs: mode bindings, @mentions, direct sub-agent chats, project scope.""" + +from __future__ import annotations + +import logging +from typing import Any, Dict, Optional, Tuple + +import core.projects.edition as project_scope + +logger = logging.getLogger(__name__) + +_PROJECT_CTX_KEYS = ( + "project_id", + "project_api_scope", + "project_name", + "project_instructions", + "project_init", + "project_folder_name", + "project_folder_kind", + "project_folder_id", + # Desktop local project — needed so the prompt renders the local-mode section + # with the real host folder path instead of the cloud /workspace section. + "project_kind", + "project_is_local", + "project_local_path", + "project_local_slug", +) + project_scope.edition_project_context_keys() + + +def _mode_visible_subagents(spec, visible, mentioned_ids): + """收窄模式下最终可见的子智能体 = 模式圈定的那批 ∪ 本轮被显式 @ 的那个。 + + 模式没圈任何子智能体时维持原契约：只有本轮被呼唤的那个能入场（极速模式一直 + 是这么做的）。圈了就以模式名单为准，再并上本轮呼唤的，免得用户 @ 了一个模式 + 里没列的智能体时那次呼唤被无声吞掉。 + """ + allowed = { + a for a in (getattr(spec, "agent_ids", ()) or ()) if isinstance(a, str) and a.strip() + } + wanted = {str(m) for m in (mentioned_ids or []) if m} + keep = allowed | wanted + if not keep: + return [] + return [item for item in (visible or []) if str((item or {}).get("agent_id") or "") in keep] + + +def _runtime_visible_subagents( + user_agents, + parent_runtime, + disabled_builtin_ids, + explicit_agent_ids, +): + """Build the default agent directory plus any agent summoned this turn. + + Disabled agents stay out of the main agent's initial context. An explicit + ``@``/command may add exactly the selected accessible agent for this turn; + the route layer has already enforced ownership and platform visibility. + """ + from core.capabilities.agents.builtin import merge_builtin_subagents + + explicitly_selected = {str(agent_id) for agent_id in (explicit_agent_ids or []) if agent_id} + all_accessible = merge_builtin_subagents( + user_agents, + parent_runtime, + disabled_agent_ids=disabled_builtin_ids, + include_disabled=True, + ) + return [ + item + for item in all_accessible + if item.get("is_enabled", True) or str(item.get("agent_id") or "") in explicitly_selected + ] + + +def _resolve_mode_spec(context: dict): + """从 context 解析这段对话的模式装配契约。 + + 线上字段是 ``mode_slug``；老客户端只发 ``chat_mode='turbo'``，按极速模式解析， + 所以升级期间新旧前端都能跑。解析失败返回 None（= 标准模式，不收窄）。 + """ + slug = str(context.get("mode_slug", "") or "").strip() + if not slug and str(context.get("chat_mode", "") or "").lower() == "turbo": + slug = "turbo" + if not slug: + return None + try: + from core.db.engine import SessionLocal + from core.chat.modes import ChatModeService + + with SessionLocal() as db: + spec = ChatModeService(db).resolve(slug, str(context.get("user_id", "") or "") or None) + return spec + except Exception: # noqa: BLE001 — 模式是增强不是边界，解析炸了也要让对话跑起来 + logger.warning("[workflow] 模式解析失败 slug=%s，按标准模式继续", slug, exc_info=True) + return None + + +def _approval_mode(context: Dict[str, Any]) -> Optional[str]: + """本轮上下文里带的权限档；没带就交回 None，由工厂读用户自己存的那一档。""" + return context.get("approval_mode") + + +def _extract_project_ctx(context: Dict[str, Any]) -> Optional[Dict[str, Any]]: + """Extract project-related fields from the workflow context. Returns None when there is no project_id.""" + if not context.get("project_id"): + return None + return {k: context.get(k) for k in _PROJECT_CTX_KEYS} + + +def _parse_agent_mentions(message: str, available_agents: list) -> list: + """Parse @agent_name mentions from user message. + + Returns list of matched ``agent_id``s in the order they appear in + *message*. + + When several agent names share a prefix (e.g. ``搜索`` vs ``搜索助手``) + a naive ``"@搜索" in message`` check matches "@搜索" *inside* + "@搜索助手", so typing ``@搜索助手`` would falsely also mention the + shorter ``搜索`` agent — and the prompt hint built downstream would + instruct the LLM to call both. We sort candidates by name length + descending and reserve consumed character ranges so the longest + matching name wins and prefix-shadow matches are skipped. + """ + if not available_agents: + return [] + + candidates = [a for a in available_agents if a.get("name")] + if not candidates: + return [] + candidates.sort(key=lambda a: len(a["name"]), reverse=True) + + consumed: list = [] # non-overlapping (start, end) char ranges + + def _overlaps(start: int, end: int) -> bool: + for s, e in consumed: + if start < e and end > s: + return True + return False + + # (position, agent_id) — sorted at the end to preserve message order + hits: list = [] + for agent in candidates: + token = "@" + agent["name"] + start = 0 + while True: + idx = message.find(token, start) + if idx < 0: + break + end = idx + len(token) + if not _overlaps(idx, end): + consumed.append((idx, end)) + hits.append((idx, agent["agent_id"])) + start = idx + len(token) + + hits.sort(key=lambda x: x[0]) + seen: set = set() + ordered: list = [] + for _, aid in hits: + if aid not in seen: + seen.add(aid) + ordered.append(aid) + return ordered + + +def _direct_agent_id_from_context(context: Dict[str, Any]) -> Optional[str]: + """Return the target only for a persistent dedicated sub-agent chat. + + A per-turn @mention must stay on the main-model stream so it produces a + genuine ``call_subagent`` tool call instead of silently bypassing the + parent model's reasoning and streaming lifecycle. + """ + value = context.get("agent_id") + return str(value) if value else None + + +def _direct_agent_execution_permissions(agent_id: str) -> Tuple[bool, bool]: + """Return ``(read_only, allow_bash)`` for a dedicated sub-agent chat.""" + from core.capabilities.agents.builtin import get_builtin_subagent + + builtin_spec = get_builtin_subagent(agent_id) + if builtin_spec is None: + return False, True + return builtin_spec.read_only, builtin_spec.allow_bash + + +def _load_direct_user_agent( + agent_id: str, + user_id: str, + parent_runtime: Optional[Dict[str, Any]] = None, +): + """Load a built-in profile or detach a complete custom-agent configuration.""" + from core.db.engine import SessionLocal + from core.capabilities.agents.builtin import build_builtin_runtime_profile, get_builtin_subagent + from core.capabilities.agents.edition import UserAgentService + + builtin_spec = get_builtin_subagent(agent_id) + if builtin_spec is not None: + return build_builtin_runtime_profile(builtin_spec, parent_runtime) + + with SessionLocal() as db: + user_agent = UserAgentService(db).get_raw_by_id(agent_id, user_id=user_id) + # Eagerly load every relationship/config field consumed after the DB + # session closes. SQLAlchemy expires lazy relationships on close. + _ = user_agent.mcp_server_ids, user_agent.skill_ids, user_agent.kb_ids + _ = user_agent.system_prompt, user_agent.model_provider_id + _ = user_agent.max_iters, user_agent.temperature, user_agent.max_tokens + _ = user_agent.timeout, user_agent.extra_config + return user_agent diff --git a/src/backend/core/agent_skills/backends/__init__.py b/src/backend/core/agent_skills/backends/__init__.py deleted file mode 100644 index 2f5626df..00000000 --- a/src/backend/core/agent_skills/backends/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Backend abstraction for multi-source skill loading.""" - -from .protocol import SkillBackendProtocol, SkillFileInfo -from .filesystem import FilesystemBackend -from .composite import CompositeBackend -from .database import DatabaseBackend - -__all__ = [ - "SkillBackendProtocol", - "SkillFileInfo", - "FilesystemBackend", - "CompositeBackend", - "DatabaseBackend", -] diff --git a/src/backend/core/agent_skills/cache_refresh.py b/src/backend/core/agent_skills/cache_refresh.py deleted file mode 100644 index 696dd22c..00000000 --- a/src/backend/core/agent_skills/cache_refresh.py +++ /dev/null @@ -1,51 +0,0 @@ -"""Shared cache invalidation after admin skill mutations. - -Why centralize: skill changes ripple through 4 caches (loader metadata, -catalog JSON, per-user capability cache, system prompt). Keeping them in -one place prevents drift between admin_skills and admin_skill_drafts routes. -""" - -from __future__ import annotations - -import logging - -logger = logging.getLogger(__name__) - - -def refresh_skill_caches() -> None: - from core.agent_skills.loader import get_skill_loader - from core.config.catalog_loader import invalidate_catalog_cache - from core.capabilities.paths import capabilities_enabled - if capabilities_enabled(): - from core.capabilities.local_projection import sync_local_skills - sync_local_skills() - get_skill_loader(reset=True) - invalidate_catalog_cache() - - try: - from core.config.catalog_resolver import invalidate_capability_cache - - invalidate_capability_cache() - except Exception: - pass - - try: - from core.config.catalog_runtime import invalidate_runtime_catalog_cache - - invalidate_runtime_catalog_cache() - except Exception: - pass - - try: - from prompts.prompt_runtime import invalidate_prompt_cache - - invalidate_prompt_cache() - except Exception: - pass - - try: - from core.config.catalog_loader import load_catalog - - load_catalog(include_runtime_details=False) - except Exception as exc: - logger.warning("Failed to eagerly refresh catalog cache after skill mutation: %s", exc) diff --git a/src/backend/core/services/agent_api_service.py b/src/backend/core/auth/agent_api.py similarity index 99% rename from src/backend/core/services/agent_api_service.py rename to src/backend/core/auth/agent_api.py index 35ec1be6..81883fe3 100644 --- a/src/backend/core/services/agent_api_service.py +++ b/src/backend/core/auth/agent_api.py @@ -49,7 +49,7 @@ def _now(): def require_agent_manager(db, user_id: str, agent_id: str): """Only the personal owner can manage an agent credential.""" - from core.services.user_agent_service import UserAgentService + from core.capabilities.agents.edition import UserAgentService row = db.query(UserAgent).filter(UserAgent.agent_id == agent_id).first() if row is None: diff --git a/src/backend/core/auth/agent_api_scope.py b/src/backend/core/auth/agent_api_scope.py index 50806698..5c8ec940 100644 --- a/src/backend/core/auth/agent_api_scope.py +++ b/src/backend/core/auth/agent_api_scope.py @@ -5,7 +5,7 @@ from fastapi import HTTPException from core.db.models import Artifact, ChatRun, ChatSession -from core.services.agent_api_service import make_agent_api_scope, session_matches_scope +from core.auth.agent_api import make_agent_api_scope, session_matches_scope def enforce_agent_api_route(request, db, user) -> None: diff --git a/src/backend/core/services/api_key_runs.py b/src/backend/core/auth/api_key_runs.py similarity index 91% rename from src/backend/core/services/api_key_runs.py rename to src/backend/core/auth/api_key_runs.py index 98363914..3aa71787 100644 --- a/src/backend/core/services/api_key_runs.py +++ b/src/backend/core/auth/api_key_runs.py @@ -53,7 +53,7 @@ def require_admission_key( project_id = payload.get("project_id") project_scope = payload.get("project_api_scope") or {} if key.project_id is not None: - from core.services.project_api_service import require_project_key_access + from core.projects.api import require_project_key_access require_project_key_access(db, user_id, key.project_id) if ( @@ -79,7 +79,7 @@ def cancel_key_runs(db: Session, key_id: str, user_id: str, now: datetime) -> No Payload bindings cover personal keys and accepted-but-not-yet-logged runs; historical agent call records also identify pre-binding scoped runs. """ - from core.services.run_journal import RunJournal + from core.harness.registry.run_cancellation import cancel_run_in_session rows = ( _key_runs_query(db, key_id, user_id) @@ -90,12 +90,12 @@ def cancel_key_runs(db: Session, key_id: str, user_id: str, now: datetime) -> No .all() ) for row in rows: - RunJournal.cancel_in_session(db, row, reason="API key disabled or revoked", now=now) + cancel_run_in_session(db, row, reason="API key disabled or revoked", now=now) async def stop_cancelled_key_workers(db: Session, key_id: str, user_id: str) -> None: """Wake SSE followers and stop local workers after the durable commit.""" - from orchestration.chat_run_executor import cancel_run + from core.harness.registry.run_cancellation import cancel_run runs = ( _key_runs_query(db, key_id, user_id) @@ -114,7 +114,7 @@ async def stop_cancelled_key_workers(db: Session, key_id: str, user_id: str) -> def authorize_run_execution(db: Session, run_id: str, now: datetime) -> bool: """Reject and durably fence work whose credential is already invalid.""" - from core.services.run_journal import RunJournal + from core.harness.registry.run_cancellation import cancel_run_in_session run = db.get(ChatRun, run_id) if run is None: @@ -131,7 +131,7 @@ def authorize_run_execution(db: Session, run_id: str, now: datetime) -> bool: .one() ) if run.status in ("pending", "running", "needs_attention"): - RunJournal.cancel_in_session(db, run, reason="API key disabled or revoked", now=now) + cancel_run_in_session(db, run, reason="API key disabled or revoked", now=now) db.commit() return False diff --git a/src/backend/core/services/api_key_service.py b/src/backend/core/auth/api_keys.py similarity index 95% rename from src/backend/core/services/api_key_service.py rename to src/backend/core/auth/api_keys.py index e1b710d1..c39219cb 100644 --- a/src/backend/core/services/api_key_service.py +++ b/src/backend/core/auth/api_keys.py @@ -71,11 +71,11 @@ def create_key( if agent_id is not None and project_id is not None: raise ValueError("API credentials cannot have two resource scopes") if project_id is not None: - from core.services.project_api_service import require_project_key_access + from core.projects.api import require_project_key_access require_project_key_access(self.db, user_id, project_id) if agent_id is not None: - from core.services.agent_api_service import require_agent_manager + from core.auth.agent_api import require_agent_manager require_agent_manager(self.db, user_id, agent_id) raw = API_KEY_PREFIX + secrets.token_urlsafe(32) @@ -153,7 +153,7 @@ def set_enabled( return None row.enabled = enabled if not enabled: - from core.services.api_key_runs import cancel_key_runs + from core.auth.api_key_runs import cancel_key_runs cancel_key_runs(self.db, key_id, user_id, _now()) self.db.commit() @@ -186,7 +186,7 @@ def revoke_key( return False row.revoked_at = _now() row.enabled = False - from core.services.api_key_runs import cancel_key_runs + from core.auth.api_key_runs import cancel_key_runs cancel_key_runs(self.db, key_id, user_id, row.revoked_at) self.db.commit() @@ -249,7 +249,7 @@ def resolve_api_key_identity(db: Session, raw: str) -> Optional[Tuple[UserShadow if account is not None and account.status != "active": return None if row.agent_id is not None: - from core.services.agent_api_service import require_agent_manager + from core.auth.agent_api import require_agent_manager from fastapi import HTTPException try: @@ -258,7 +258,7 @@ def resolve_api_key_identity(db: Session, raw: str) -> Optional[Tuple[UserShadow return None if row.project_id is not None: - from core.services.project_api_service import require_project_key_access + from core.projects.api import require_project_key_access from fastapi import HTTPException if row.agent_id is not None: diff --git a/src/backend/core/auth/backend.py b/src/backend/core/auth/backend.py index bd089bea..07495a74 100644 --- a/src/backend/core/auth/backend.py +++ b/src/backend/core/auth/backend.py @@ -9,12 +9,12 @@ from typing import Any, Dict, Optional import requests -from core.auth.account_policy import AccountCapacityExceeded +from core.auth.edition import AccountCapacityExceeded from core.config.settings import settings from core.db.engine import get_db from core.db.repository import AuditLogRepository from core.infra.logging import get_logger -from core.services import UserService +from core.auth.users import UserService from fastapi import Depends, HTTPException, Request from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer from pydantic import BaseModel @@ -153,7 +153,7 @@ def _resolve_api_key_context(token: Optional[str], db: Session) -> Optional[User have no Cookie). Validation (enabled/not-revoked/not-expired/user permission bits) is handled uniformly by ``resolve_api_key_identity``. """ - from core.services.api_key_service import resolve_api_key_identity + from core.auth.api_keys import resolve_api_key_identity identity = resolve_api_key_identity(db, token or "") if identity is None: @@ -271,7 +271,7 @@ async def get_current_user( # An explicit API credential must not be widened by an accompanying cookie # or desktop identity. Invalid/revoked scoped keys likewise fail closed. - from core.services.api_key_service import api_key_token_from_header, is_api_key_token + from core.auth.api_keys import api_key_token_from_header, is_api_key_token bearer_token = api_key_token_from_header(request.headers.get("authorization")) if bearer_token is None and credentials is not None: @@ -281,7 +281,7 @@ async def get_current_user( # ── 0. 桌面桥接（仅桌面壳孵化的本机后端启用；混合架构 P2）── # 桌面壳注入的云端身份优先于本机自身的会话体系；未启用/未命中时零开销回落。 - from core.auth.desktop_bridge import resolve_bridge_user + from core.harness.registry.authentication import resolve_host_user as resolve_bridge_user from starlette.concurrency import run_in_threadpool bridge_user = await run_in_threadpool(resolve_bridge_user, request, db) @@ -376,7 +376,7 @@ async def optional_user( from core.auth.public_chat import authenticate_public_chat return authenticate_public_chat(request, db) auth_mode = _auth_mode() - from core.services.api_key_service import api_key_token_from_header + from core.auth.api_keys import api_key_token_from_header auth_header = request.headers.get("Authorization", "") raw_bearer = api_key_token_from_header(auth_header) @@ -385,7 +385,7 @@ async def optional_user( # Match required authentication: the desktop shell owns the local # identity, even when a forwarded cloud cookie is absent or stale. - from core.auth.desktop_bridge import resolve_bridge_user + from core.harness.registry.authentication import resolve_host_user as resolve_bridge_user from starlette.concurrency import run_in_threadpool bridge_user = await run_in_threadpool(resolve_bridge_user, request, db) @@ -434,14 +434,16 @@ async def optional_user( # Edition admission blocks must not degrade to anonymous access. try: db.rollback() - except Exception: - pass + except Exception as fallback_error: + from core.infra.failures import record_best_effort_failure + record_best_effort_failure('core.auth.backend:optional_user', fallback_error) raise except Exception as e: try: db.rollback() - except Exception: - pass + except Exception as fallback_error: + from core.infra.failures import record_best_effort_failure + record_best_effort_failure('core.auth.backend:optional_user', fallback_error) logger.warning("optional_auth_resolve_failed", error=str(e)) return None diff --git a/src/backend/core/auth/capabilities.py b/src/backend/core/auth/capabilities.py index a495e0b0..fa7a05f4 100644 --- a/src/backend/core/auth/capabilities.py +++ b/src/backend/core/auth/capabilities.py @@ -172,7 +172,7 @@ def user_has_capability(db: Session, user_id: str, flag: str) -> bool: meta = shadow.extra_data if isinstance(shadow.extra_data, dict) else {} if meta.get("role") == "super_admin": return True - from core.auth.edition_capabilities import default_capability_layers_for_user + from core.auth.edition import default_capability_layers_for_user caps = resolve_capabilities( meta, @@ -190,7 +190,7 @@ def resolve_user_capabilities(db: Session, user_id: str) -> Dict[str, Any]: chain is "personal → role → team → system" (both role and team degrade defensively to ``{}``). """ - from core.auth.edition_capabilities import default_capability_layers_for_user + from core.auth.edition import default_capability_layers_for_user from core.db.models import UserShadow shadow = db.query(UserShadow).filter(UserShadow.user_id == user_id).first() diff --git a/src/backend/core/auth/edition.py b/src/backend/core/auth/edition.py new file mode 100644 index 00000000..b1b2c663 --- /dev/null +++ b/src/backend/core/auth/edition.py @@ -0,0 +1,5 @@ +"""Select single-owner authorization for the community edition.""" +from core.auth.personal_permissions_iface import ChatAccessLevel, can_delete_session, resolve_chat_access, ProjectAccess, ProjectAccessInfo, ProjectPermissionLevel, is_project_owner, require_project_access, resolve_project_access, resolve_project_permission, PermissionLevel, has_permission, resolve_artifact_access +from core.auth.personal_account_policy import AccountCapacityExceeded, account_capacity_block_reason, add_account_to_scope, add_invited_account_to_scope, claim_registration_credential, list_account_scopes, registration_credential_id, validate_account_scope, validate_registration_credential +from core.auth.personal_account_view import extend_current_account +from core.auth.personal_edition_capabilities import default_capability_layers_for_user, extend_agent_visibility_filters diff --git a/src/backend/core/auth/local_user_bootstrap.py b/src/backend/core/auth/local_user_bootstrap.py new file mode 100644 index 00000000..467893a0 --- /dev/null +++ b/src/backend/core/auth/local_user_bootstrap.py @@ -0,0 +1,83 @@ +"""Create the default CE administrator on a fresh instance.""" +from __future__ import annotations +from core.infra.time import utc_now +from typing import Optional +from sqlalchemy.orm import Session +from core.auth.password import hash_password, verify_password +from core.config.settings import settings +from core.db.models import LocalUser, UserShadow +from core.auth import local_users_constants + +def ensure_ce_default_admin(db: Session) -> tuple[Optional[str], bool]: + """Ensure a fresh CE instance has an ``admin/admin`` local administrator. + + The bootstrap credential is marked for mandatory password change. Existing + passwords are never reset. If a different local account already exists, the + explicit onboarding choice is preserved and no extra account is created. + """ + if settings.edition.edition != "ce": + return None, False + + shadow = db.query(UserShadow).filter(UserShadow.username == local_users_constants.CE_DEFAULT_ADMIN_USERNAME).first() + created = False + + if shadow is None: + if db.query(LocalUser).count() > 0: + return None, False + user_id = "user_ce_admin" + shadow = UserShadow( + user_id=user_id, + user_center_id=user_id, + username=local_users_constants.CE_DEFAULT_ADMIN_USERNAME, + email=None, + avatar_url=None, + extra_data={ + "auth_source": "local", + **local_users_constants.CE_DEFAULT_ADMIN_CAPABILITIES, + "must_change_password": True, + "onboarding_required": True, + }, + last_sync_at=utc_now(), + ) + db.add(shadow) + db.flush() + local = LocalUser( + user_id=user_id, + password_hash=hash_password(local_users_constants.CE_DEFAULT_ADMIN_PASSWORD), + nickname="Administrator", + status="active", + password_updated_at=None, + ) + db.add(local) + created = True + else: + local = db.query(LocalUser).filter(LocalUser.user_id == shadow.user_id).first() + if local is None: + local = LocalUser( + user_id=shadow.user_id, + password_hash=hash_password(local_users_constants.CE_DEFAULT_ADMIN_PASSWORD), + nickname="Administrator", + status="active", + password_updated_at=None, + ) + db.add(local) + created = True + + meta = dict(shadow.extra_data or {}) + meta.update(local_users_constants.CE_DEFAULT_ADMIN_CAPABILITIES) + meta.setdefault("auth_source", "local") + try: + completed_version = int(meta.get("onboarding_completed_version") or 0) + except (TypeError, ValueError): + completed_version = 0 + if completed_version < local_users_constants.CE_ONBOARDING_VERSION: + meta["onboarding_required"] = True + else: + meta.pop("onboarding_required", None) + if verify_password(local_users_constants.CE_DEFAULT_ADMIN_PASSWORD, local.password_hash): + meta["must_change_password"] = True + shadow.extra_data = meta + shadow.updated_at = utc_now() + + db.commit() + return shadow.user_id, created diff --git a/src/backend/core/auth/local_user_policy.py b/src/backend/core/auth/local_user_policy.py new file mode 100644 index 00000000..8b5f9aa4 --- /dev/null +++ b/src/backend/core/auth/local_user_policy.py @@ -0,0 +1,16 @@ +"""Whether the CE bootstrap owner still has to finish web onboarding.""" +from __future__ import annotations +from typing import Any, Dict, Optional +from core.config.settings import settings +from core.auth import local_users_constants + +def ce_onboarding_required(metadata: Optional[Dict[str, Any]]) -> bool: + """Return whether the CE bootstrap owner must finish the current Web setup flow.""" + if settings.edition.edition != "ce": + return False + meta = metadata or {} + try: + completed_version = int(meta.get("onboarding_completed_version") or 0) + except (TypeError, ValueError): + completed_version = 0 + return bool(meta.get("onboarding_required")) and completed_version < local_users_constants.CE_ONBOARDING_VERSION diff --git a/src/backend/core/services/local_user_service.py b/src/backend/core/auth/local_users.py similarity index 79% rename from src/backend/core/services/local_user_service.py rename to src/backend/core/auth/local_users.py index ac315017..f6ed43c4 100644 --- a/src/backend/core/services/local_user_service.py +++ b/src/backend/core/auth/local_users.py @@ -1,71 +1,19 @@ -"""Local user system business logic: registration, login, password change, disabling.""" - +"""Local account registration, login, password changes and profile edits.""" from __future__ import annotations from core.infra.time import utc_now - -import re import secrets import uuid -from dataclasses import dataclass -from datetime import datetime -from typing import Any, Dict, List, Optional - +from typing import Any, Dict, Optional from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import Session - -USERNAME_PATTERN = re.compile(r"^[A-Za-z0-9_]{2,32}$") -EMAIL_PATTERN = re.compile(r"^[^\s@]+@[^\s@]+\.[^\s@]+$") - -CE_DEFAULT_ADMIN_USERNAME = "admin" -CE_DEFAULT_ADMIN_PASSWORD = "admin" -CE_ONBOARDING_VERSION = 1 -CE_DEFAULT_ADMIN_CAPABILITIES: Dict[str, Any] = { - "role": "super_admin", - "ce_default_admin": True, - "lab_enabled": True, - "can_use_api_key": True, - "can_add_skill": True, - "can_add_mcp": True, - "can_import_plugin": True, - "can_add_agent": True, - "can_create_private_kb": True, - "can_create_public_kb": False, - "can_create_channel_bot": True, - "can_switch_model": True, - "can_use_ontology_validation": True, - "can_run_autonomous_loop": True, - "can_manage_chat_modes": False, - "can_system_config": True, - "can_content_manage": True, - "allowed_apps": "*", -} - - -def ce_onboarding_required(metadata: Optional[Dict[str, Any]]) -> bool: - """Return whether the CE bootstrap owner must finish the current Web setup flow.""" - if settings.edition.edition != "ce": - return False - meta = metadata or {} - try: - completed_version = int(meta.get("onboarding_completed_version") or 0) - except (TypeError, ValueError): - completed_version = 0 - return bool(meta.get("onboarding_required")) and completed_version < CE_ONBOARDING_VERSION - - -from core.auth.account_policy import ( - add_account_to_scope, - add_invited_account_to_scope, - claim_registration_credential, - list_account_scopes, - registration_credential_id, - validate_account_scope, - validate_registration_credential, -) +from core.auth.edition import add_account_to_scope, add_invited_account_to_scope, claim_registration_credential, list_account_scopes, registration_credential_id, validate_account_scope, validate_registration_credential from core.auth.password import hash_password, verify_password from core.config.settings import settings from core.db.models import LocalUser, UserShadow from core.db.repository import AuditLogRepository, LocalUserRepository, UserRepository +from core.auth import local_user_policy +from core.auth import local_users_constants +from dataclasses import dataclass @dataclass @@ -112,7 +60,7 @@ def register( if not username: return RegisterResult(False, "账号不能为空") - if not USERNAME_PATTERN.match(username): + if not local_users_constants.USERNAME_PATTERN.match(username): return RegisterResult(False, "账号只能包含英文、数字、下划线，长度 2-32 位") if not nickname: return RegisterResult(False, "用户名不能为空") @@ -120,7 +68,7 @@ def register( return RegisterResult(False, "用户名长度不能超过 32 位") if not email: return RegisterResult(False, "邮箱不能为空") - if not EMAIL_PATTERN.match(email): + if not local_users_constants.EMAIL_PATTERN.match(email): return RegisterResult(False, "邮箱格式不正确") if len(password) < settings.auth.password_min_length: return RegisterResult(False, f"密码长度至少 {settings.auth.password_min_length} 位") @@ -132,7 +80,7 @@ def register( if self.db.query(UserShadow).filter(UserShadow.email == email).first() is not None: return RegisterResult(False, "邮箱已被使用") - from core.auth.account_policy import account_capacity_block_reason + from core.auth.edition import account_capacity_block_reason seat_block = account_capacity_block_reason(self.db) if seat_block: @@ -233,11 +181,11 @@ def create_by_admin( if not username: return RegisterResult(False, "账号不能为空") - if not USERNAME_PATTERN.match(username): + if not local_users_constants.USERNAME_PATTERN.match(username): return RegisterResult(False, "账号只能包含英文、数字、下划线，长度 2-32 位") if len(nickname) > 32: return RegisterResult(False, "用户名长度不能超过 32 位") - if email and not EMAIL_PATTERN.match(email): + if email and not local_users_constants.EMAIL_PATTERN.match(email): return RegisterResult(False, "邮箱格式不正确") if len(password) < settings.auth.password_min_length: return RegisterResult(False, f"密码长度至少 {settings.auth.password_min_length} 位") @@ -257,7 +205,7 @@ def create_by_admin( if not validate_account_scope(self.db, scope_id): return RegisterResult(False, "目标范围不存在") - from core.auth.account_policy import account_capacity_block_reason + from core.auth.edition import account_capacity_block_reason seat_block = account_capacity_block_reason(self.db) if seat_block: @@ -362,7 +310,7 @@ def get_or_create_external_account( if existing is not None: return existing, False - from core.auth.account_policy import AccountCapacityExceeded, account_capacity_block_reason + from core.auth.edition import AccountCapacityExceeded, account_capacity_block_reason block = account_capacity_block_reason(self.db) if block: @@ -490,7 +438,7 @@ def _build_user_info( "real_name": local.real_name if local else None, "auth_source": "local", "must_change_password": bool(meta.get("must_change_password")), - "onboarding_required": ce_onboarding_required(meta), + "onboarding_required": local_user_policy.ce_onboarding_required(meta), "teams": list_account_scopes(self.db, user_id), } @@ -535,78 +483,3 @@ def update_profile( user_id, shadow.username if shadow else "", shadow=shadow, local=local ), ) - - -def ensure_ce_default_admin(db: Session) -> tuple[Optional[str], bool]: - """Ensure a fresh CE instance has an ``admin/admin`` local administrator. - - The bootstrap credential is marked for mandatory password change. Existing - passwords are never reset. If a different local account already exists, the - explicit onboarding choice is preserved and no extra account is created. - """ - if settings.edition.edition != "ce": - return None, False - - shadow = db.query(UserShadow).filter(UserShadow.username == CE_DEFAULT_ADMIN_USERNAME).first() - created = False - - if shadow is None: - if db.query(LocalUser).count() > 0: - return None, False - user_id = "user_ce_admin" - shadow = UserShadow( - user_id=user_id, - user_center_id=user_id, - username=CE_DEFAULT_ADMIN_USERNAME, - email=None, - avatar_url=None, - extra_data={ - "auth_source": "local", - **CE_DEFAULT_ADMIN_CAPABILITIES, - "must_change_password": True, - "onboarding_required": True, - }, - last_sync_at=utc_now(), - ) - db.add(shadow) - db.flush() - local = LocalUser( - user_id=user_id, - password_hash=hash_password(CE_DEFAULT_ADMIN_PASSWORD), - nickname="Administrator", - status="active", - password_updated_at=None, - ) - db.add(local) - created = True - else: - local = db.query(LocalUser).filter(LocalUser.user_id == shadow.user_id).first() - if local is None: - local = LocalUser( - user_id=shadow.user_id, - password_hash=hash_password(CE_DEFAULT_ADMIN_PASSWORD), - nickname="Administrator", - status="active", - password_updated_at=None, - ) - db.add(local) - created = True - - meta = dict(shadow.extra_data or {}) - meta.update(CE_DEFAULT_ADMIN_CAPABILITIES) - meta.setdefault("auth_source", "local") - try: - completed_version = int(meta.get("onboarding_completed_version") or 0) - except (TypeError, ValueError): - completed_version = 0 - if completed_version < CE_ONBOARDING_VERSION: - meta["onboarding_required"] = True - else: - meta.pop("onboarding_required", None) - if verify_password(CE_DEFAULT_ADMIN_PASSWORD, local.password_hash): - meta["must_change_password"] = True - shadow.extra_data = meta - shadow.updated_at = utc_now() - - db.commit() - return shadow.user_id, created diff --git a/src/backend/core/auth/local_users_constants.py b/src/backend/core/auth/local_users_constants.py new file mode 100644 index 00000000..20170f71 --- /dev/null +++ b/src/backend/core/auth/local_users_constants.py @@ -0,0 +1,40 @@ +"""Validation patterns and CE bootstrap constants for local accounts.""" +from __future__ import annotations +import re +from typing import Any, Dict + +USERNAME_PATTERN = re.compile(r"^[A-Za-z0-9_]{2,32}$") + + +EMAIL_PATTERN = re.compile(r"^[^\s@]+@[^\s@]+\.[^\s@]+$") + + +CE_DEFAULT_ADMIN_USERNAME = "admin" + + +CE_DEFAULT_ADMIN_PASSWORD = "admin" + + +CE_ONBOARDING_VERSION = 1 + + +CE_DEFAULT_ADMIN_CAPABILITIES: Dict[str, Any] = { + "role": "super_admin", + "ce_default_admin": True, + "lab_enabled": True, + "can_use_api_key": True, + "can_add_skill": True, + "can_add_mcp": True, + "can_import_plugin": True, + "can_add_agent": True, + "can_create_private_kb": True, + "can_create_public_kb": False, + "can_create_channel_bot": True, + "can_switch_model": True, + "can_use_ontology_validation": True, + "can_run_autonomous_loop": True, + "can_manage_chat_modes": False, + "can_system_config": True, + "can_content_manage": True, + "allowed_apps": "*", +} diff --git a/src/backend/core/auth/mock_ticket_store.py b/src/backend/core/auth/mock_ticket_store.py index ada94208..823cc784 100644 --- a/src/backend/core/auth/mock_ticket_store.py +++ b/src/backend/core/auth/mock_ticket_store.py @@ -12,15 +12,17 @@ uvicorn worker, so the state between them cannot live in one process: with ``WEB_CONCURRENCY > 1`` an in-process dict loses roughly ``1 - 1/N`` of all logins. It is therefore kept in Redis, with the same key/TTL discipline as -:mod:`core.auth.oa_ticket_store` and :mod:`core.auth.desktop_ticket_store`: +:mod:`core.auth.oa_ticket_store` and :mod:`features.desktop.auth.ticket_store`: sha256 of the token as the key, atomic GETDEL on consume, TTL-bounded. Without Redis the backend degrades to in-process storage, which is sound because :mod:`core.infra.worker_count` already clamps such a deployment to one worker. """ +from __future__ import annotations import hashlib import json import secrets +from dataclasses import dataclass, field from datetime import datetime, timedelta, timezone from typing import Any, Dict, Optional @@ -33,8 +35,6 @@ TICKET_TTL = 300 # seconds TICKET_PREFIX = "mock_ticket_" -_MEMORY_TICKETS: Dict[str, Dict[str, Any]] = {} - def _hash(token: str) -> str: return hashlib.sha256(token.encode()).hexdigest() @@ -46,8 +46,8 @@ def _use_memory_store() -> bool: def _prune_expired_memory() -> None: now = datetime.now(timezone.utc) - for key in [k for k, v in _MEMORY_TICKETS.items() if v["expires_at"] <= now]: - _MEMORY_TICKETS.pop(key, None) + for key in [k for k, v in process_state().MEMORY_TICKETS.items() if v["expires_at"] <= now]: + process_state().MEMORY_TICKETS.pop(key, None) def is_local_ticket(ticket: str) -> bool: @@ -62,7 +62,7 @@ async def generate_ticket(user_info: Dict[str, Any]) -> str: if _use_memory_store(): _prune_expired_memory() - _MEMORY_TICKETS[key] = { + process_state().MEMORY_TICKETS[key] = { "user_info": dict(user_info), "expires_at": datetime.now(timezone.utc) + timedelta(seconds=TICKET_TTL), } @@ -87,7 +87,7 @@ async def consume_ticket(ticket: str) -> Optional[Dict[str, Any]]: if _use_memory_store(): _prune_expired_memory() - entry = _MEMORY_TICKETS.pop(key, None) + entry = process_state().MEMORY_TICKETS.pop(key, None) return dict(entry["user_info"]) if entry else None from core.infra.redis import get_redis @@ -107,3 +107,15 @@ async def consume_ticket(ticket: str) -> Optional[Dict[str, Any]]: except (json.JSONDecodeError, TypeError): logger.warning("login_ticket_corrupt", token_hash=key[:8]) return None + + +@dataclass +class MockTicketStoreState: + MEMORY_TICKETS: Dict[str, Dict[str, Any]] = field(default_factory=lambda: {}) + + +_STATE = MockTicketStoreState() + + +def process_state() -> MockTicketStoreState: + return _STATE diff --git a/src/backend/core/auth/model_selection.py b/src/backend/core/auth/model_selection.py new file mode 100644 index 00000000..7cb3846f --- /dev/null +++ b/src/backend/core/auth/model_selection.py @@ -0,0 +1,52 @@ +"""Authorize account model overrides before selecting an active provider.""" +from __future__ import annotations +from typing import Optional +from sqlalchemy.orm import Session +from core.auth.capabilities import resolve_user_capabilities +from core.db.models import ModelProvider + +class UserModelSelectionError(ValueError): + """Raised when a requested user-selectable model is not allowed.""" + + +def resolve_user_model_provider_id( + db: Session, + provider_id: Optional[str], + *, + user_id: Optional[str] = None, + can_switch_model: Optional[bool] = None, +) -> Optional[str]: + """Return an allowed provider id for the current request. + + Disabled switch means "ignore the browser-provided override" rather than + "fail the chat"; enabled switch validates against active chat providers. + """ + pid = (provider_id or "").strip() + if not pid: + return None + + enabled = ( + can_switch_model + if can_switch_model is not None + else bool(user_id and user_can_switch_model(db, user_id)) + ) + if not enabled: + return None + + provider = ( + db.query(ModelProvider) + .filter( + ModelProvider.provider_id == pid, + ModelProvider.provider_type == "chat", + ModelProvider.is_active == True, # noqa: E712 + ) + .first() + ) + if provider is None: + raise UserModelSelectionError("所选模型不可用，请刷新页面后重试。") + return pid + + +def user_can_switch_model(db: Session, user_id: str) -> bool: + """Return whether the user may switch models in the chat input.""" + return bool(resolve_user_capabilities(db, user_id).get("can_switch_model")) diff --git a/src/backend/core/auth/oa_ticket_store.py b/src/backend/core/auth/oa_ticket_store.py index 2bf9a48c..764b0107 100644 --- a/src/backend/core/auth/oa_ticket_store.py +++ b/src/backend/core/auth/oa_ticket_store.py @@ -14,10 +14,12 @@ Degrades to in-process storage when the backend has no Redis (same approach as session.py, dev/single-instance only). """ +from __future__ import annotations import hashlib import json import secrets +from dataclasses import dataclass, field from datetime import datetime, timedelta, timezone from typing import Any, Dict, Optional @@ -27,7 +29,6 @@ logger = get_logger(__name__) TICKET_KEY_PREFIX = "jx:oa_ticket:" -_MEMORY_TICKETS: dict[str, dict[str, Any]] = {} def _ttl_seconds() -> int: @@ -44,8 +45,8 @@ def _use_memory_store() -> bool: def _prune_expired_memory() -> None: now = datetime.now(timezone.utc) - for key in [k for k, v in _MEMORY_TICKETS.items() if v.get("expires_at") and v["expires_at"] <= now]: - _MEMORY_TICKETS.pop(key, None) + for key in [k for k, v in process_state().MEMORY_TICKETS.items() if v.get("expires_at") and v["expires_at"] <= now]: + process_state().MEMORY_TICKETS.pop(key, None) async def issue_ticket(payload: Dict[str, Any]) -> str: @@ -62,7 +63,7 @@ async def issue_ticket(payload: Dict[str, Any]) -> str: if _use_memory_store(): _prune_expired_memory() - _MEMORY_TICKETS[key] = { + process_state().MEMORY_TICKETS[key] = { "payload": body, "expires_at": datetime.now(timezone.utc) + timedelta(seconds=ttl), } @@ -84,7 +85,7 @@ async def consume_ticket(token: str) -> Optional[Dict[str, Any]]: if _use_memory_store(): _prune_expired_memory() - entry = _MEMORY_TICKETS.pop(key, None) + entry = process_state().MEMORY_TICKETS.pop(key, None) if entry is None: return None return dict(entry["payload"]) @@ -106,3 +107,15 @@ async def consume_ticket(token: str) -> Optional[Dict[str, Any]]: except (json.JSONDecodeError, TypeError): logger.warning("oa_ticket_corrupt", token_hash=key[:8]) return None + + +@dataclass +class OaTicketStoreState: + MEMORY_TICKETS: dict[str, dict[str, Any]] = field(default_factory=lambda: {}) + + +_STATE = OaTicketStoreState() + + +def process_state() -> OaTicketStoreState: + return _STATE diff --git a/src/backend/core/auth/account_policy.py b/src/backend/core/auth/personal_account_policy.py similarity index 100% rename from src/backend/core/auth/account_policy.py rename to src/backend/core/auth/personal_account_policy.py diff --git a/src/backend/core/auth/account_view.py b/src/backend/core/auth/personal_account_view.py similarity index 100% rename from src/backend/core/auth/account_view.py rename to src/backend/core/auth/personal_account_view.py diff --git a/src/backend/core/auth/edition_capabilities.py b/src/backend/core/auth/personal_edition_capabilities.py similarity index 100% rename from src/backend/core/auth/edition_capabilities.py rename to src/backend/core/auth/personal_edition_capabilities.py diff --git a/src/backend/core/auth/permissions_iface.py b/src/backend/core/auth/personal_permissions_iface.py similarity index 100% rename from src/backend/core/auth/permissions_iface.py rename to src/backend/core/auth/personal_permissions_iface.py diff --git a/src/backend/core/auth/project_api_scope.py b/src/backend/core/auth/project_api_scope.py index 0d05eb7e..a6b2c2d6 100644 --- a/src/backend/core/auth/project_api_scope.py +++ b/src/backend/core/auth/project_api_scope.py @@ -5,7 +5,7 @@ from fastapi import HTTPException from core.db.models import Artifact, ChatRun, ChatSession -from core.services.project_api_service import make_project_api_scope, project_session_matches +from core.projects.api import make_project_api_scope, project_session_matches def enforce_project_api_route(request, db, user) -> None: diff --git a/src/backend/core/auth/public_chat.py b/src/backend/core/auth/public_chat.py index a17742ad..ec23aac3 100644 --- a/src/backend/core/auth/public_chat.py +++ b/src/backend/core/auth/public_chat.py @@ -3,13 +3,13 @@ import re from core.db.models import Artifact, ChatRun, ChatSession -from core.services.public_chat_service import resolve_public_user +from core.auth.public_identity import resolve_public_user from fastapi import HTTPException def public_session_matches(session, user): - from core.services.agent_api_service import make_agent_api_scope, session_matches_scope - from core.services.project_api_service import make_project_api_scope, project_session_matches + from core.auth.agent_api import make_agent_api_scope, session_matches_scope + from core.projects.api import make_project_api_scope, project_session_matches if not session: return False diff --git a/src/backend/core/auth/public_identity.py b/src/backend/core/auth/public_identity.py new file mode 100644 index 00000000..c3b22678 --- /dev/null +++ b/src/backend/core/auth/public_identity.py @@ -0,0 +1,36 @@ +"""Resolve a visitor credential without granting the owner's account identity.""" +import hashlib +import re +from fastapi import HTTPException +from core.db.models import ContentBlock, UserApiKey +from core.infra.crypto import decrypt_secret +from core.auth.api_keys import resolve_api_key_identity + +def digest(value): + return hashlib.sha256(value.encode()).hexdigest() + + +def resolve_public_user(db, token, visitor): + from core.auth.backend import UserContext + + if not re.fullmatch(r"[A-Za-z0-9_-]{43}", token) or not re.fullmatch( + r"[A-Za-z0-9_-]{32,64}", visitor + ): + raise HTTPException(401, detail="Invalid public chat link") + row = db.get(ContentBlock, "publicchat:token:" + digest(token)) + key = db.get(UserApiKey, row.payload["key_id"]) if row else None + identity = ( + resolve_api_key_identity(db, decrypt_secret(key.key_enc)) if key and key.key_enc else None + ) + if not identity: + raise HTTPException(401, detail="Public chat link is unavailable") + owner, key = identity + return UserContext( + user_id=owner.user_id, + user_center_id=owner.user_center_id or "", + username=owner.username, + api_key_id=key.id, + api_key_agent_id=key.agent_id, + api_key_project_id=key.project_id, + public_chat_visitor=digest(visitor), + ) diff --git a/src/backend/core/auth/session.py b/src/backend/core/auth/session.py index 93b9b32b..9cc1db95 100644 --- a/src/backend/core/auth/session.py +++ b/src/backend/core/auth/session.py @@ -4,10 +4,12 @@ Value: JSON-encoded user data TTL: SESSION_TTL_HOURS (default 8h) """ +from __future__ import annotations import hashlib import json import secrets +from dataclasses import dataclass, field from datetime import datetime, timedelta, timezone from typing import Any, Dict, Optional @@ -18,7 +20,6 @@ logger = get_logger(__name__) SESSION_KEY_PREFIX = "jx:session:" -_MEMORY_SESSIONS: dict[str, dict[str, Any]] = {} def _ttl_seconds() -> int: @@ -33,7 +34,7 @@ def remember_ttl_seconds() -> int: """ days = 30.0 try: - from core.services.system_config import SystemConfigService + from core.config.store.system_config import SystemConfigService raw = SystemConfigService.get_instance().get("auth.remember_me_days", "30") if raw: @@ -66,11 +67,11 @@ def _prune_expired_memory_sessions() -> None: now = datetime.now(timezone.utc) expired = [ key - for key, value in _MEMORY_SESSIONS.items() + for key, value in process_state().MEMORY_SESSIONS.items() if value.get("expires_at") and value["expires_at"] <= now ] for key in expired: - _MEMORY_SESSIONS.pop(key, None) + process_state().MEMORY_SESSIONS.pop(key, None) def generate_token() -> str: @@ -112,7 +113,7 @@ async def create_session( if _use_memory_store(): _prune_expired_memory_sessions() - _MEMORY_SESSIONS[token_hash] = { + process_state().MEMORY_SESSIONS[token_hash] = { "payload": payload, "expires_at": datetime.now(timezone.utc) + timedelta(seconds=ttl), } @@ -141,7 +142,7 @@ async def validate_session(token: str) -> Optional[Dict[str, Any]]: if _use_memory_store(): _prune_expired_memory_sessions() - entry = _MEMORY_SESSIONS.get(token_hash) + entry = process_state().MEMORY_SESSIONS.get(token_hash) if entry is None: return None # Sliding renewal uses the session's own TTL (a long "remember me" session is not truncated to the default duration) @@ -178,7 +179,7 @@ async def find_session_by_hash(token_hash: str) -> Optional[Dict[str, Any]]: return None if _use_memory_store(): _prune_expired_memory_sessions() - entry = _MEMORY_SESSIONS.get(token_hash) + entry = process_state().MEMORY_SESSIONS.get(token_hash) return dict(entry["payload"]) if entry else None raw = await get_redis().get(f"{SESSION_KEY_PREFIX}{token_hash}") if raw is None: @@ -199,18 +200,18 @@ async def revoke_session(token: str) -> bool: token_hash = _hash_token(token) data = await find_session_by_hash(token_hash) if _use_memory_store(): - deleted = _MEMORY_SESSIONS.pop(token_hash, None) is not None + deleted = process_state().MEMORY_SESSIONS.pop(token_hash, None) is not None else: deleted = bool(await get_redis().delete(f"{SESSION_KEY_PREFIX}{token_hash}")) if deleted: logger.info("session_revoked", token_hash=token_hash[:8]) if data and data.get("user_id"): - from core.applications.revocation import revoke_user_workbench + from core.harness.registry.resource_owners import resource_owner from core.db.engine import SessionLocal try: with SessionLocal() as db: - await revoke_user_workbench(db, data["user_id"], token_hash) + await resource_owner().revoke_session(db, data["user_id"], token_hash) except Exception as exc: # The credential is already revoked. Preserve successful logout; # expired workers stay denied while the resource janitor retries. @@ -244,3 +245,15 @@ def expires_at_iso(ttl_seconds: Optional[int] = None) -> str: """Calculate and return the expiration time as an ISO string.""" ttl = _resolve_ttl(ttl_seconds) return (datetime.now(timezone.utc) + timedelta(seconds=ttl)).isoformat() + + +@dataclass +class SessionState: + MEMORY_SESSIONS: dict[str, dict[str, Any]] = field(default_factory=lambda: {}) + + +_STATE = SessionState() + + +def process_state() -> SessionState: + return _STATE diff --git a/src/backend/core/services/user_service.py b/src/backend/core/auth/users.py similarity index 92% rename from src/backend/core/services/user_service.py rename to src/backend/core/auth/users.py index d490fb0d..2800e655 100644 --- a/src/backend/core/services/user_service.py +++ b/src/backend/core/auth/users.py @@ -5,7 +5,7 @@ from datetime import datetime from typing import Any, Dict, Optional, Set -from core.auth.account_policy import AccountCapacityExceeded, account_capacity_block_reason +from core.auth.edition import AccountCapacityExceeded, account_capacity_block_reason from core.db.models import UserShadow from core.db.repository import AuditLogRepository, UserRepository from sqlalchemy.orm import Session @@ -100,9 +100,8 @@ def get_user_settings(self, user_id: str) -> Dict[str, Any]: return {} metadata = dict(user.extra_data) if user.extra_data else {} - # Imported lazily to keep the services package's compatibility exports - # from creating an import cycle during application startup. - from core.services.memory_settings_service import MemorySettingsService + # Imported lazily to avoid an import cycle during application startup. + from core.memory.settings import MemorySettingsService return MemorySettingsService(self.db).apply_effective_defaults(metadata) @@ -127,7 +126,7 @@ def get_disabled_builtin_subagent_ids(self, user_id: str) -> Set[str]: if not isinstance(raw_ids, list): return set() - from core.llm.builtin_subagents import get_builtin_subagent + from core.capabilities.agents.builtin import get_builtin_subagent return { agent_id @@ -139,7 +138,7 @@ def get_disabled_builtin_subagent_ids(self, user_id: str) -> Set[str]: def set_builtin_subagent_enabled(self, user_id: str, agent_id: str, enabled: bool) -> None: """Persist one user's platform sub-agent switch; defaults stay enabled.""" - from core.llm.builtin_subagents import get_builtin_subagent + from core.capabilities.agents.builtin import get_builtin_subagent if get_builtin_subagent(agent_id) is None: raise LookupError(f"Builtin sub-agent {agent_id} not found") diff --git a/src/backend/core/capabilities/__init__.py b/src/backend/core/capabilities/__init__.py index b98922f8..37c1c8cb 100644 --- a/src/backend/core/capabilities/__init__.py +++ b/src/backend/core/capabilities/__init__.py @@ -1,16 +1 @@ -"""Desktop capability runtime: partitioned storage, name resolution, flat runtime views. - -Three layers, deliberately separated (capability plan v2.0, ch. 04): - -- **store** — ``/s////``: immutable versions, one - sub-tree per source profile, same-named keys from different profiles coexist. -- **resolver** — the single place that decides which candidate a runtime name points - to for this device/user. Explicit user choice → explicit request → the current - account's usable candidate → the only usable candidate → otherwise a conflict. -- **view** — a flat directory of directory links (``/``) the - execution plane and the model see. The model-facing path contract - ``/workspace/skills/`` never changes. - -The package is inert unless :func:`paths.capability_root` is set — only the desktop -local runtime keeps a file store; cloud deployments keep their database truth. -""" +"""Capabilities an agent can use: skills, plugins, MCP servers, sub-agents and their catalog.""" diff --git a/src/backend/core/capabilities/agents/__init__.py b/src/backend/core/capabilities/agents/__init__.py new file mode 100644 index 00000000..d47444f5 --- /dev/null +++ b/src/backend/core/capabilities/agents/__init__.py @@ -0,0 +1 @@ +"""User-defined agents: storage, versioning and the resources they may bind.""" diff --git a/src/backend/core/capabilities/agents/available_resources.py b/src/backend/core/capabilities/agents/available_resources.py new file mode 100644 index 00000000..c71ac626 --- /dev/null +++ b/src/backend/core/capabilities/agents/available_resources.py @@ -0,0 +1,273 @@ +"""List the MCP servers, skills, plugins and knowledge spaces an agent may bind.""" +from __future__ import annotations +from typing import Any, Dict, List, Optional +import logging +logger = logging.getLogger(__name__) + +def list_available_resources(self, owner_user_id: Optional[str] = None) -> Dict[str, Any]: + """Return MCP servers, skills, plugins, and KB spaces bindable to agents. + + Plugin-sourced skills/MCP are removed from the skills/mcp_servers lists + and instead bound as a whole via the ``plugins`` list (plugin = + installable/removable unit, expanded at runtime into its skills+tools). + owner_user_id is used to include that user's private plugins and MCPs. + + The MCP list intentionally contains capabilities the user has personally + switched off. A sub-agent binding is an explicit, narrower capability + grant and therefore may opt into one of those tools without turning it + on for the user's main agent. Deployment-global MCPs disabled by an + administrator remain unavailable. + """ + from core.db.models import AdminMcpServer, InstalledPlugin + from sqlalchemy import or_ + # ── Plugin list + their component id sets (used to strip plugin capabilities from the loose skills/tools) ── + # Query the ORM directly for component_ids (authoritative): global plugins + current user's private plugins. + try: + pq = self.db.query(InstalledPlugin) + if owner_user_id: + pq = pq.filter( + or_( + InstalledPlugin.owner_user_id == owner_user_id, + InstalledPlugin.owner_user_id.is_(None), + ) + ) + else: + pq = pq.filter(InstalledPlugin.owner_user_id.is_(None)) + plugin_rows = pq.order_by(InstalledPlugin.created_at.desc()).all() + except Exception as exc: # noqa: BLE001 + logger.debug("Failed to list installed plugins: %s", exc) + plugin_rows = [] + + plugin_skill_ids: set = set() + plugin_mcp_ids: set = set() + # The same plugin may exist both as a global version (install_id=slug@global) + # and as the user's private version (slug@) — same name → duplicate + # display (the user sees two "定时任务管理" entries). Dedupe by slug and + # show only one: prefer the user's private version (its components carry + # the user's fingerprint and match their sandbox credentials); fall back + # to the global version when there is no private one. + # Note: plugin_skill_ids / plugin_mcp_ids still accumulate from **all** + # rows (including the deduped-away one), so the loose skill/MCP lists + # can fully exclude both versions' components. + _plugin_by_slug: Dict[str, Dict[str, Any]] = {} + for p in plugin_rows: + cids = p.component_ids or {} + s_ids = list(cids.get("skills") or []) + m_ids = list(cids.get("mcp") or []) + plugin_skill_ids.update(s_ids) + plugin_mcp_ids.update(m_ids) + slug = (p.install_id or "").rsplit("@", 1)[0] + is_owned = bool(owner_user_id) and p.owner_user_id == owner_user_id + existing = _plugin_by_slug.get(slug) + if existing is None or (is_owned and not existing["_owned"]): + _plugin_by_slug[slug] = { + "id": p.install_id, + "name": p.name, + "description": p.description or "", + "skill_count": len(s_ids), + "mcp_count": len(m_ids), + "_owned": is_owned, + } + plugin_list: List[Dict[str, Any]] = [ + {k: v for k, v in item.items() if k != "_owned"} for item in _plugin_by_slug.values() + ] + + # Built-in plugin MCPs can also be present in the static catalog without + # a source_plugin DB row. They still belong under the plugin selector, + # not the loose MCP selector. + try: + from core.capabilities.plugins.management import builtin_plugin_component_ids + + _, builtin_plugin_mcp_ids = builtin_plugin_component_ids() + plugin_mcp_ids.update(builtin_plugin_mcp_ids) + except Exception as exc: # noqa: BLE001 + logger.debug("Failed to load built-in plugin MCP ids: %s", exc) + + # Resolve the user's personal on/off layer once so each MCP option can + # explain whether it is already enabled for the main agent. This flag + # is display metadata only; disabled options remain bindable here. + enabled_mcp_ids: Optional[set[str]] = None + if owner_user_id: + try: + from core.capabilities.catalog.resolver import resolve_all_runtime_enabled + + _skills, _agents, resolved_mcps = resolve_all_runtime_enabled( + self.db, owner_user_id + ) + if resolved_mcps is not None: + enabled_mcp_ids = set(resolved_mcps) + except Exception as exc: # noqa: BLE001 + logger.debug("Failed to resolve user MCP enablement: %s", exc) + + # ── MCP tools (exclude plugin-sourced + owner isolation) ── + # Owner isolation: private entries (owner_user_id non-null) are visible + # only to their owner. Otherwise private copies produced by other users + # installing the same plugin (e.g. automation-automation_task-) would all leak into this user's bindable list → + # duplicates of "定时任务" etc. Empty owner = globally shared entry, + # visible to everyone. + _mcp_owner_ok = ( + or_( + AdminMcpServer.owner_user_id.is_(None), + AdminMcpServer.owner_user_id == owner_user_id, + ) + if owner_user_id + else AdminMcpServer.owner_user_id.is_(None) + ) + mcp_servers = ( + self.db.query(AdminMcpServer) + .filter( + _mcp_owner_ok, + # Authoritative exclusion: any plugin-sourced MCP (source_plugin + # nonnull) never enters the loose list — it is bound as a whole via + # the plugins list instead. More robust than checking only + # plugin_mcp_ids (doesn't depend on the plugin row still + # existing/being visible), and also blocks orphaned plugin MCP rows. + AdminMcpServer.source_plugin.is_(None), + ) + .order_by(AdminMcpServer.sort_order) + .all() + ) + mcp_list: List[Dict[str, Any]] = [] + seen_mcp_ids: set = set() + for server in mcp_servers: + # A global false is an administrator lock. A private false is the + # owner's personal off state and remains eligible for an explicit + # sub-agent binding. + if server.owner_user_id is None and not server.is_enabled: + continue + if server.server_id in plugin_mcp_ids: + continue + mcp_list.append( + { + "id": server.server_id, + "name": server.display_name, + "description": server.description, + "enabled": ( + server.server_id in enabled_mcp_ids + if enabled_mcp_ids is not None + else bool(server.is_enabled) + ), + } + ) + seen_mcp_ids.add(server.server_id) + + # Some umbrella/built-in MCP entries are catalog-defined and do not + # necessarily have a same-named AdminMcpServer row. Include every + # administrator-enabled catalog item so the selector is complete. + try: + from core.capabilities.catalog.runtime import get_runtime_catalog + + runtime_catalog = get_runtime_catalog(self.db, include_runtime_details=False) + for item in runtime_catalog.get("mcp") or []: + item_id = str(item.get("id") or "").strip() + if ( + not item_id + or item_id in seen_mcp_ids + or item_id in plugin_mcp_ids + or not bool(item.get("enabled", True)) + ): + continue + mcp_list.append( + { + "id": item_id, + "name": item.get("name") or item_id, + "description": item.get("description") or item.get("desc") or "", + "enabled": ( + item_id in enabled_mcp_ids + if enabled_mcp_ids is not None + else bool(item.get("enabled", True)) + ), + } + ) + seen_mcp_ids.add(item_id) + except Exception as exc: # noqa: BLE001 + logger.debug("Failed to load catalog MCP resources: %s", exc) + + # ── Skills: DB-managed + filesystem-discovered (both exclude plugin-sourced + owner isolation) ── + # ⚠️ Critical: the filesystem loader (load_all_metadata) scans **all** + # materialized skills on disk — including other users' private skills + # and plugin skills materialized by other users' plugin installs. + # Excluding via plugin_skill_ids alone (components of plugins visible to + # the current user only) would miss some, letting other users' plugin + # skills sneak into "bindable skills". So compute two **authoritative** + # exclusion sets directly from AdminSkill and filter both sources + # uniformly: + # - all_plugin_skill_ids: any plugin-sourced skill (source_plugin non-null, any owner); + # - foreign_private_skill_ids: other users' private skills (owner_user_id non-null and ≠ current user). + from core.db.models import AdminSkill + + all_plugin_skill_ids: set = set() + foreign_private_skill_ids: set = set() + try: + for row in self.db.query( + AdminSkill.skill_id, AdminSkill.source_plugin, AdminSkill.owner_user_id + ).all(): + if row.source_plugin: + all_plugin_skill_ids.add(row.skill_id) + if row.owner_user_id and row.owner_user_id != owner_user_id: + foreign_private_skill_ids.add(row.skill_id) + except Exception as exc: + logger.debug("Failed to precompute skill exclusion sets: %s", exc) + + _excluded_skill_ids = plugin_skill_ids | all_plugin_skill_ids | foreign_private_skill_ids + + skill_list: List[Dict[str, Any]] = [] + try: + _skill_owner_ok = ( + or_(AdminSkill.owner_user_id.is_(None), AdminSkill.owner_user_id == owner_user_id) + if owner_user_id + else AdminSkill.owner_user_id.is_(None) + ) + db_skills = ( + self.db.query(AdminSkill) + .filter( + AdminSkill.is_enabled == True, + _skill_owner_ok, + ) + .order_by(AdminSkill.updated_at.desc()) + .all() + ) + seen_ids = set() + for s in db_skills: + if s.skill_id in _excluded_skill_ids: + continue + skill_list.append( + {"id": s.skill_id, "name": s.display_name, "description": s.description or ""} + ) + seen_ids.add(s.skill_id) + except Exception: + seen_ids = set() + + try: + from core.capabilities.skills.loader_factory import get_skill_loader + + loader = get_skill_loader() + for sid, meta in loader.load_all_metadata().items(): + if sid not in seen_ids and sid not in _excluded_skill_ids: + skill_list.append( + { + "id": sid, + "name": getattr(meta, "name", sid), + "description": getattr(meta, "description", ""), + } + ) + except Exception as exc: + logger.debug("Failed to load filesystem skills: %s", exc) + + # KB spaces + kb_list: List[Dict[str, Any]] = [] + try: + from core.harness.registry.catalog_options import catalog_options + kb_list = catalog_options("kb_spaces", self.db) + except Exception as fallback_error: + from core.infra.failures import record_best_effort_failure + record_best_effort_failure('core.capabilities.agents.available_resources:list_available_resources', fallback_error) + + return { + "mcp_servers": mcp_list, + "skills": skill_list, + "plugins": plugin_list, + "kb_spaces": kb_list, + "ontology_tags": self._ontology_tag_options(), + } diff --git a/src/backend/core/capabilities/agents/base.py b/src/backend/core/capabilities/agents/base.py new file mode 100644 index 00000000..79da15d5 --- /dev/null +++ b/src/backend/core/capabilities/agents/base.py @@ -0,0 +1,435 @@ +"""CRUD, versioning and permission checks for user-defined agents.""" +from __future__ import annotations +from core.capabilities.agents import available_resources +from core.capabilities.agents import version_history +import uuid +from typing import Any, Dict, List, Optional +from core.db.models import UserAgent +from core.db.repository import AuditLogRepository, UserAgentRepository +from core.harness.registry.build_governance import ensure_ontology_build_valid +from sqlalchemy.orm import Session +import logging +from core.capabilities.agents import base_constants + +logger = logging.getLogger(__name__) + + +def desktop_store() -> bool: + """Whether this process keeps a desktop capability file store (agents projected there).""" + from core.harness.registry.run_capabilities import has_run_capabilities + + return has_run_capabilities() + + +class UserAgentBaseService: + """Service for user agent CRUD and permission checks.""" + + def __init__(self, db: Session): + self.db = db + self.repo = UserAgentRepository(db) + + # ── Queries ────────────────────────────────────────────────────── + + def list_for_user(self, user_id: str) -> List[Dict[str, Any]]: + agents = self.repo.list_for_user(user_id) + rows = [self._serialize(a) for a in agents] + if desktop_store(): + # Desktop: the bridged account's stored agents join the device's own; + # a display-name clash is decided by the resolver, never by order. + from core.harness.registry.local_publication import local_publication + + return local_publication().merge_visible_agents(user_id, rows) + return rows + + def list_admin(self) -> List[Dict[str, Any]]: + agents = self.repo.list_admin() + return [self._serialize(a) for a in agents] + + def get_by_id(self, agent_id: str, user_id: Optional[str] = None) -> Dict[str, Any]: + agent = self.repo.get_by_id(agent_id) + if not agent: + stored = self._stored_account_agent(agent_id, user_id) + if stored is not None: + return stored.to_serialized() + raise LookupError(f"Agent {agent_id} not found") + if user_id and not self._is_accessible(agent, user_id): + raise PermissionError("No access to this agent") + return self._serialize(agent) + + def get_raw_by_id(self, agent_id: str, user_id: Optional[str] = None) -> UserAgent: + """Return the ORM object (for direct use by workflow/factory). + + On a desktop the bridged account's stored agents are a second, equally + authoritative source: an ``AgentDefinition`` with the same attribute + surface is returned for them. + """ + agent = self.repo.get_by_id(agent_id) + if not agent: + stored = self._stored_account_agent(agent_id, user_id) + if stored is not None: + return stored # type: ignore[return-value] + raise LookupError(f"Agent {agent_id} not found") + if user_id and not self._is_accessible(agent, user_id): + raise PermissionError("No access to this agent") + return agent + + @staticmethod + def _stored_account_agent(agent_id: str, user_id: Optional[str] = None): + if not desktop_store(): + return None + from core.harness.registry.local_publication import local_publication + from core.harness.registry.execution_account import account_authorized_for + + if not account_authorized_for(user_id): + return None + return local_publication().account_agent_definition(agent_id) + + def _project_to_store(self, agent: UserAgent) -> None: + if not desktop_store(): + return + from core.harness.registry.local_publication import local_publication + + local_publication().publish_local_agent(self._serialize(agent)) + + @staticmethod + def _remove_from_store(agent_id: str) -> None: + if not desktop_store(): + return + from core.harness.registry.local_publication import local_publication + + local_publication().remove_local_agent(agent_id) + + # ── Mutations ──────────────────────────────────────────────────── + + def create( + self, + user_id: Optional[str], + operator_name: Optional[str], + owner_type: str, + data: Dict[str, Any], + scope_id: Optional[str] = None, + ) -> Dict[str, Any]: + data = dict(data) + incoming_extra = dict(data.get("extra_config") or {}) + ontology_tags = list( + data.pop("ontology_tags", incoming_extra.get("ontology_tags") or []) or [] + ) + incoming_extra["ontology_tags"] = ontology_tags + data["extra_config"] = incoming_extra + ensure_ontology_build_valid( + self.db, + asset_type="subagent", + name=str(data.get("name") or ""), + description=str(data.get("description") or ""), + instructions=str(data.get("system_prompt") or ""), + mcp_server_ids=list(data.get("mcp_server_ids") or []), + skill_ids=list(data.get("skill_ids") or []), + plugin_ids=list(data.get("plugin_ids") or []), + ontology_tags=ontology_tags, + ) + self._validate_create_scope(user_id, owner_type, scope_id) + + agent_id = f"ua_{uuid.uuid4().hex[:16]}" + created_at = self._now_iso() + creation_history = [ + { + "version": base_constants.DEFAULT_AGENT_VERSION, + "timestamp": created_at, + "content": "创建了子智能体", + "operator_name": operator_name or user_id or "未知用户", + "details": [], + } + ] + extra_config = self._merge_extra_config( + current_extra=None, + incoming_extra=incoming_extra, + version=base_constants.DEFAULT_AGENT_VERSION, + change_history=creation_history, + ) + + record = { + "agent_id": agent_id, + "owner_type": owner_type, + **self._owner_fields(user_id, owner_type, scope_id), + "created_by": user_id, + **data, + "extra_config": extra_config, + } + agent = self.repo.create(record) + self._project_to_store(agent) + self._audit( + user_id, + "agent.create", + agent_id, + {"owner_type": owner_type, "name": data.get("name")}, + ) + return self._serialize(agent) + + def update( + self, + agent_id: str, + user_id: Optional[str], + operator_name: Optional[str], + owner_type: str, + data: Dict[str, Any], + ) -> Dict[str, Any]: + agent = self.repo.get_by_id(agent_id) + if not agent: + raise LookupError(f"Agent {agent_id} not found") + self._check_ownership(agent, user_id, owner_type) + + data = dict(data) + current_extra = dict(agent.extra_config or {}) + incoming_extra = dict(data.get("extra_config") or {}) + ontology_tags = list( + data.pop( + "ontology_tags", + incoming_extra.get("ontology_tags", current_extra.get("ontology_tags") or []), + ) + or [] + ) + incoming_extra["ontology_tags"] = ontology_tags + if "extra_config" in data or ontology_tags != list( + current_extra.get("ontology_tags") or [] + ): + data["extra_config"] = incoming_extra + ensure_ontology_build_valid( + self.db, + asset_type="subagent", + name=str(data.get("name", agent.name) or ""), + description=str(data.get("description", agent.description) or ""), + instructions=str(data.get("system_prompt", agent.system_prompt) or ""), + mcp_server_ids=list(data.get("mcp_server_ids", agent.mcp_server_ids) or []), + skill_ids=list(data.get("skill_ids", agent.skill_ids) or []), + plugin_ids=list(data.get("plugin_ids", agent.plugin_ids) or []), + ontology_tags=ontology_tags, + ) + + changed_fields = self._collect_changed_fields(agent, data) + versioned_fields = [field for field in changed_fields if field not in base_constants.NON_VERSIONED_FIELDS] + changed_labels = [base_constants.VERSIONED_FIELDS[field] for field in changed_fields] + next_version = self._read_version(current_extra) + change_history = self._read_change_history(current_extra) + + if changed_labels: + change_summary = self._build_change_summary(changed_fields, data) + change_details = self._build_change_details(agent, changed_fields, data) + entry_version = next_version + if versioned_fields: + next_version = self._increment_version(next_version) + entry_version = next_version + change_history.append( + { + "version": entry_version, + "timestamp": self._now_iso(), + "content": change_summary, + "operator_name": operator_name or user_id or "未知用户", + "details": change_details, + } + ) + change_history = change_history[-base_constants.MAX_CHANGE_HISTORY:] + + payload = dict(data) + payload["extra_config"] = self._merge_extra_config( + current_extra=current_extra, + incoming_extra=incoming_extra, + version=next_version, + change_history=change_history, + ) + + agent = self.repo.update(agent_id, payload) + self._project_to_store(agent) + audit_details = {"fields": list(data.keys())} + if changed_labels: + audit_details["change_summary"] = change_summary + audit_details["version"] = next_version + self._audit(user_id, "agent.update", agent_id, audit_details) + return self._serialize(agent) + + def delete( + self, + agent_id: str, + user_id: Optional[str], + owner_type: str, + ) -> bool: + agent = self.repo.get_by_id(agent_id) + if not agent: + raise LookupError(f"Agent {agent_id} not found") + self._check_ownership(agent, user_id, owner_type) + + ok = self.repo.delete(agent_id) + if ok: + self._remove_from_store(agent_id) + self._audit(user_id, "agent.delete", agent_id) + return ok + + def toggle_enabled(self, agent_id: str) -> Dict[str, Any]: + agent = self.repo.get_by_id(agent_id) + if not agent: + raise LookupError(f"Agent {agent_id} not found") + new_val = not agent.is_enabled + agent = self.repo.update(agent_id, {"is_enabled": new_val}) + self._project_to_store(agent) + return self._serialize(agent) + + # ── Available resources ────────────────────────────────────────── + + def list_available_resources(self, owner_user_id: Optional[str] = None) -> Dict[str, Any]: + return available_resources.list_available_resources(self, owner_user_id) + + def _ontology_tag_options(self) -> List[Dict[str, Any]]: + """Controlled labels that activate a sub-agent ontology workflow at runtime.""" + from core.harness.registry.build_governance import list_asset_tag_options + + return list_asset_tag_options(self.db, "subagent") + + # ── Helpers ─────────────────────────────────────────────────────── + + def _validate_create_scope( + self, user_id: Optional[str], owner_type: str, scope_id: Optional[str] + ) -> None: + if owner_type == "user": + if not user_id: + raise ValueError("user_id required for user agents") + if self.repo.count_user_agents(user_id) >= base_constants.MAX_USER_AGENTS: + raise ValueError(f"Maximum {base_constants.MAX_USER_AGENTS} agents per user reached") + return + if owner_type == "admin": + return + raise ValueError(f"Unsupported agent owner type: {owner_type}") + + @staticmethod + def _owner_fields( + user_id: Optional[str], owner_type: str, scope_id: Optional[str] + ) -> Dict[str, Any]: + return {"user_id": user_id if owner_type == "user" else None} + + def _is_accessible(self, agent: UserAgent, user_id: str) -> bool: + if agent.owner_type == "admin" and agent.is_enabled: + return True + if agent.owner_type == "user" and agent.user_id == user_id: + return True + return False + + def _check_ownership(self, agent: UserAgent, user_id: Optional[str], owner_type: str) -> None: + # owner_type is the route context: 'admin' = Admin console routes, everything else = user-side routes + if owner_type == "admin": + if agent.owner_type != "admin": + raise PermissionError("Admin can only modify admin agents") + return + if agent.owner_type == "user" and agent.user_id == user_id: + return + raise PermissionError("You can only modify your own agents") + + @classmethod + def _serialize(cls, agent: UserAgent) -> Dict[str, Any]: + extra_config = agent.extra_config or {} + return { + "agent_id": agent.agent_id, + "owner_type": agent.owner_type, + "user_id": agent.user_id, + **cls._serialize_scope(agent), + "name": agent.name, + "avatar": agent.avatar, + "description": agent.description, + "system_prompt": agent.system_prompt, + "welcome_message": agent.welcome_message, + "suggested_questions": agent.suggested_questions or [], + "mcp_server_ids": agent.mcp_server_ids or [], + "skill_ids": agent.skill_ids or [], + "plugin_ids": agent.plugin_ids or [], + "kb_ids": agent.kb_ids or [], + "model_provider_id": agent.model_provider_id, + "temperature": float(agent.temperature) if agent.temperature is not None else None, + "max_tokens": agent.max_tokens, + "max_iters": agent.max_iters, + "timeout": agent.timeout, + "is_enabled": agent.is_enabled, + "sort_order": agent.sort_order, + "source_market_slug": agent.source_market_slug, + "ontology_tags": list(extra_config.get("ontology_tags") or []), + "extra_config": extra_config, + "version": cls._read_version(extra_config), + "change_history": cls._read_change_history(extra_config), + "created_at": agent.created_at.isoformat() if agent.created_at else None, + "updated_at": agent.updated_at.isoformat() if agent.updated_at else None, + "created_by": agent.created_by, + } + + @staticmethod + def _serialize_scope(agent: UserAgent) -> Dict[str, Any]: + return {} + + def _audit( + self, user_id: Optional[str], action: str, resource_id: str, details: Dict = None + ) -> None: + try: + audit_repo = AuditLogRepository(self.db) + audit_repo.create( + { + "user_id": user_id, + "action": action, + "resource_type": "user_agent", + "resource_id": resource_id, + "details": details or {}, + "status": "success", + } + ) + except Exception as exc: + logger.warning("Audit log failed: %s", exc) + + @staticmethod + def _normalize_value(value: Any) -> Any: + return version_history.normalize_value(value) + + @staticmethod + def _now_iso() -> str: + return version_history.now_iso() + + @classmethod + def _collect_changed_fields(cls, agent: UserAgent, data: Dict[str, Any]) -> List[str]: + return version_history.collect_changed_fields(cls, agent, data) + + @staticmethod + def _read_version(extra_config: Optional[Dict[str, Any]]) -> str: + return version_history.read_version(UserAgentBaseService, extra_config) + + @staticmethod + def _read_change_history(extra_config: Optional[Dict[str, Any]]) -> List[Dict[str, str]]: + return version_history.read_change_history(UserAgentBaseService, extra_config) + + @staticmethod + def _increment_version(version: str) -> str: + return version_history.increment_version(UserAgentBaseService, version) + + @staticmethod + def _build_change_summary(changed_fields: List[str], data: Dict[str, Any]) -> str: + return version_history.build_change_summary(changed_fields, data) + + @classmethod + def _build_change_details( + cls, agent: UserAgent, changed_fields: List[str], data: Dict[str, Any] + ) -> List[Dict[str, str]]: + return version_history.build_change_details(cls, agent, changed_fields, data) + + @staticmethod + def _stringify_detail_value(field: str, value: Any) -> str: + return version_history.stringify_detail_value(field, value) + + @staticmethod + def _normalize_change_details(details: Any) -> List[Dict[str, str]]: + return version_history.normalize_change_details(details) + + @staticmethod + def _normalize_version(version: str) -> str: + return version_history.normalize_version(version) + + @staticmethod + def _merge_extra_config( + current_extra: Optional[Dict[str, Any]], + incoming_extra: Optional[Dict[str, Any]], + *, + version: str, + change_history: List[Dict[str, str]], + ) -> Dict[str, Any]: + return version_history.merge_extra_config(current_extra, incoming_extra, version=version, change_history=change_history) diff --git a/src/backend/core/capabilities/agents/base_constants.py b/src/backend/core/capabilities/agents/base_constants.py new file mode 100644 index 00000000..e49125f6 --- /dev/null +++ b/src/backend/core/capabilities/agents/base_constants.py @@ -0,0 +1,33 @@ +"""Limits and versioned-field lists for user-defined agents.""" +from __future__ import annotations + + +MAX_USER_AGENTS = 20 + + +DEFAULT_AGENT_VERSION = "V1.0" + + +MAX_CHANGE_HISTORY = 30 + + +NON_VERSIONED_FIELDS = {"is_enabled"} + + +VERSIONED_FIELDS = { + "name": "名称", + "description": "简介", + "system_prompt": "角色设定", + "welcome_message": "开场白", + "suggested_questions": "推荐问题", + "mcp_server_ids": "绑定工具", + "skill_ids": "绑定技能", + "plugin_ids": "绑定插件", + "kb_ids": "绑定知识库", + "model_provider_id": "模型", + "temperature": "温度", + "max_tokens": "最大输出长度", + "max_iters": "最大推理轮次", + "timeout": "超时时间", + "is_enabled": "启用状态", +} diff --git a/src/backend/core/llm/builtin_subagents.py b/src/backend/core/capabilities/agents/builtin.py similarity index 98% rename from src/backend/core/llm/builtin_subagents.py rename to src/backend/core/capabilities/agents/builtin.py index ba03ab01..ce43c2a4 100644 --- a/src/backend/core/llm/builtin_subagents.py +++ b/src/backend/core/capabilities/agents/builtin.py @@ -6,6 +6,7 @@ """ from __future__ import annotations +from core.harness.registry.model_services import render_prompt_part import json import logging @@ -223,9 +224,8 @@ def refresh_builtin_subagents( def load_builtin_subagent_prompt(spec: BuiltinSubagentSpec) -> str: """Load the active DB prompt part, falling back to the versioned file.""" try: - from core.services import prompt_version_service - prompt = prompt_version_service.render_active_prompt_part("subagents", spec.role) + prompt = render_prompt_part("subagents", spec.role) if prompt: return prompt except Exception: diff --git a/src/backend/core/services/user_agent_service.py b/src/backend/core/capabilities/agents/edition.py similarity index 64% rename from src/backend/core/services/user_agent_service.py rename to src/backend/core/capabilities/agents/edition.py index 4c95be63..ffab9e59 100644 --- a/src/backend/core/services/user_agent_service.py +++ b/src/backend/core/capabilities/agents/edition.py @@ -1,6 +1,6 @@ """Community custom sub-agent service.""" -from core.services.user_agent_base import UserAgentBaseService +from core.capabilities.agents.base import UserAgentBaseService UserAgentService = UserAgentBaseService diff --git a/src/backend/core/services/agent_markdown.py b/src/backend/core/capabilities/agents/markdown.py similarity index 100% rename from src/backend/core/services/agent_markdown.py rename to src/backend/core/capabilities/agents/markdown.py diff --git a/src/backend/core/capabilities/agents/version_history.py b/src/backend/core/capabilities/agents/version_history.py new file mode 100644 index 00000000..a9e99ecb --- /dev/null +++ b/src/backend/core/capabilities/agents/version_history.py @@ -0,0 +1,180 @@ +"""Version numbers and change history for user-defined agents.""" +from __future__ import annotations +from core.capabilities.agents.base_constants import DEFAULT_AGENT_VERSION, VERSIONED_FIELDS +from core.infra.time import utc_now +import re +from decimal import Decimal +from typing import Any, Dict, List, Optional +from core.db.models import UserAgent + +def normalize_value(value: Any) -> Any: + if isinstance(value, Decimal): + return float(value) + if isinstance(value, list): + return list(value) + return value + + +def now_iso() -> str: + return utc_now().replace(microsecond=0).isoformat() + + +def collect_changed_fields(cls, agent: UserAgent, data: Dict[str, Any]) -> List[str]: + fields: List[str] = [] + for field in VERSIONED_FIELDS: + if field not in data: + continue + old_value = cls._normalize_value(getattr(agent, field, None)) + new_value = cls._normalize_value(data.get(field)) + if old_value != new_value: + fields.append(field) + return fields + + +def read_version(base_service_type, extra_config: Optional[Dict[str, Any]]) -> str: + if not isinstance(extra_config, dict): + return DEFAULT_AGENT_VERSION + raw = extra_config.get("version") + return base_service_type._normalize_version(raw if isinstance(raw, str) else "") + + +def read_change_history(base_service_type, extra_config: Optional[Dict[str, Any]]) -> List[Dict[str, str]]: + if not isinstance(extra_config, dict): + return [] + raw_history = extra_config.get("change_history") + if not isinstance(raw_history, list): + return [] + + history: List[Dict[str, str]] = [] + for item in raw_history: + if not isinstance(item, dict): + continue + timestamp = item.get("timestamp") + content = item.get("content") + version = item.get("version") + operator_name = item.get("operator_name") + details = item.get("details") + if not isinstance(timestamp, str) or not isinstance(content, str): + continue + history.append( + { + "timestamp": timestamp, + "content": content, + "version": base_service_type._normalize_version( + version if isinstance(version, str) else "" + ), + "operator_name": ( + operator_name + if isinstance(operator_name, str) and operator_name.strip() + else "未知用户" + ), + "details": base_service_type._normalize_change_details(details), + } + ) + return history + + +def increment_version(base_service_type, version: str) -> str: + normalized = base_service_type._normalize_version(version) + match = re.match(r"^[Vv](\d+)\.(\d+)$", normalized) + if not match: + return "V1.1" + major, minor = (int(part) for part in match.groups()) + return f"V{major}.{minor + 1}" + + +def build_change_summary(changed_fields: List[str], data: Dict[str, Any]) -> str: + if changed_fields == ["is_enabled"]: + return "启用了子智能体" if bool(data.get("is_enabled")) else "停用了子智能体" + + changed_labels = [VERSIONED_FIELDS[field] for field in changed_fields] + if not changed_labels: + return "更新了智能体配置" + if len(changed_labels) <= 3: + return f"修改了{'、'.join(changed_labels)}" + preview = "、".join(changed_labels[:3]) + return f"修改了{preview}等{len(changed_labels)}项" + + +def build_change_details( + cls, agent: UserAgent, changed_fields: List[str], data: Dict[str, Any] +) -> List[Dict[str, str]]: + details: List[Dict[str, str]] = [] + for field in changed_fields: + old_value = cls._stringify_detail_value(field, getattr(agent, field, None)) + new_value = cls._stringify_detail_value(field, data.get(field)) + details.append( + { + "field": VERSIONED_FIELDS[field], + "before": old_value, + "after": new_value, + } + ) + return details + + +def stringify_detail_value(field: str, value: Any) -> str: + if field == "is_enabled": + return "启用" if bool(value) else "关闭" + if value is None: + return "未填写" + if isinstance(value, list): + return "、".join(str(item) for item in value) if value else "未填写" + if isinstance(value, bool): + return "是" if value else "否" + text = str(value).strip() + return text if text else "未填写" + + +def normalize_change_details(details: Any) -> List[Dict[str, str]]: + if not isinstance(details, list): + return [] + normalized: List[Dict[str, str]] = [] + for item in details: + if not isinstance(item, dict): + continue + field = item.get("field") + before = item.get("before") + after = item.get("after") + if not isinstance(field, str): + continue + normalized.append( + { + "field": field, + "before": before if isinstance(before, str) else "未填写", + "after": after if isinstance(after, str) else "未填写", + } + ) + return normalized + + +def normalize_version(version: str) -> str: + if not isinstance(version, str) or not version.strip(): + return DEFAULT_AGENT_VERSION + raw = version.strip() + if re.match(r"^[Vv]\d+\.\d+$", raw): + return f"V{raw[1:]}" + legacy_patch = re.match(r"^(\d+)\.(\d+)\.(\d+)$", raw) + if legacy_patch: + major, minor, patch = (int(part) for part in legacy_patch.groups()) + return f"V{major}.{minor + patch}" + legacy_minor = re.match(r"^(\d+)\.(\d+)$", raw) + if legacy_minor: + major, minor = (int(part) for part in legacy_minor.groups()) + return f"V{major}.{minor}" + return DEFAULT_AGENT_VERSION + + +def merge_extra_config( + current_extra: Optional[Dict[str, Any]], + incoming_extra: Optional[Dict[str, Any]], + *, + version: str, + change_history: List[Dict[str, str]], +) -> Dict[str, Any]: + merged = dict(current_extra or {}) + if isinstance(incoming_extra, dict): + merged.update(incoming_extra) + merged["version"] = version + merged["change_history"] = change_history + return merged diff --git a/src/backend/core/capabilities/archive.py b/src/backend/core/capabilities/archive.py index 1eb93539..a69a95d9 100644 --- a/src/backend/core/capabilities/archive.py +++ b/src/backend/core/capabilities/archive.py @@ -9,8 +9,8 @@ from dataclasses import dataclass from pathlib import Path, PurePosixPath from typing import Dict, Iterable, List, Tuple -from .errors import IntegrityFailed -from .paths import _WINDOWS_RESERVED +from core.harness.capability_errors import IntegrityFailed +from core.capabilities.identity_rules import _WINDOWS_RESERVED MAX_MEMBERS = 5000 MAX_DEPTH = 24 @@ -28,7 +28,7 @@ def _reject(name: str, why: str) -> None: raise IntegrityFailed(f"unsafe archive member {name!r}: {why}") -def _normalize_member(name: str) -> str: +def normalize_member(name: str) -> str: if not name: _reject(name, "empty name") if name.startswith("/") or "\\" in name: @@ -48,7 +48,7 @@ def _normalize_member(name: str) -> str: return "/".join(parts) -def _validate_names(names: List[str]) -> None: +def validate_names(names: List[str]) -> None: spellings: Dict[str, str] = {} files = set() directories = set() @@ -91,7 +91,7 @@ def inspect_zip(data: bytes) -> Tuple[zipfile.ZipFile, List[ArchiveEntry], str]: if info.flag_bits & 1: _reject(info.filename, "encrypted member") name = info.filename.rstrip("/") if info.is_dir() else info.filename - norm = _normalize_member(name) + norm = normalize_member(name) if info.is_dir(): continue if info.file_size > MAX_MEMBER_BYTES: @@ -109,7 +109,7 @@ def inspect_zip(data: bytes) -> Tuple[zipfile.ZipFile, List[ArchiveEntry], str]: entries = [ ArchiveEntry(n[len(strip) + 1 :] if strip else n, size) for n, size in normalized ] - _validate_names([entry.relative_path for entry in entries]) + validate_names([entry.relative_path for entry in entries]) return zf, entries, strip except Exception: zf.close() @@ -145,13 +145,13 @@ def write_files(dest: Path, files: Dict[str, bytes | str]) -> List[str]: prepared = [] total = 0 for rel, body in files.items(): - norm = _normalize_member(rel) + norm = normalize_member(rel) data = body if isinstance(body, bytes) else body.encode("utf-8") total += len(data) if len(data) > MAX_MEMBER_BYTES or total > MAX_TOTAL_BYTES: raise IntegrityFailed("package exceeds the allowed size") prepared.append((norm, data)) - _validate_names([name for name, _ in prepared]) + validate_names([name for name, _ in prepared]) dest.mkdir(parents=True, exist_ok=False) for name, data in prepared: target = dest.joinpath(*PurePosixPath(name).parts) @@ -166,7 +166,7 @@ def iter_file_stats(root: Path) -> Iterable[Tuple[str, Path, os.stat_result]]: Never descend into links or reparse points. Returning each entry's lstat avoids repeating filesystem calls for type checks, signatures and sizes. """ - from .junction import _native, is_directory_link + from core.infra.directory_links import native_path as _native, is_directory_link if is_directory_link(root): raise IntegrityFailed("package root must be a real directory") diff --git a/src/backend/core/capabilities/catalog/__init__.py b/src/backend/core/capabilities/catalog/__init__.py new file mode 100644 index 00000000..0ef93965 --- /dev/null +++ b/src/backend/core/capabilities/catalog/__init__.py @@ -0,0 +1 @@ +"""The capability catalog: what is installed, overridden and visible to each user.""" diff --git a/src/backend/core/config/catalog.py b/src/backend/core/capabilities/catalog/api.py similarity index 92% rename from src/backend/core/config/catalog.py rename to src/backend/core/capabilities/catalog/api.py index bdbd69d6..e425de07 100644 --- a/src/backend/core/config/catalog.py +++ b/src/backend/core/capabilities/catalog/api.py @@ -26,23 +26,15 @@ from typing import Any, Dict, List -from core.config.catalog_common import _item, _normalize_kind -from core.config.catalog_loader import ( - _write_catalog, - ensure_default_catalog, - invalidate_catalog_cache, - load_catalog, -) - -# Re-export so existing ``from core.config.catalog import invalidate_catalog_cache`` -# and ``from core.config.catalog import ensure_default_catalog`` keep working. +from core.capabilities.catalog.common import _item, _normalize_kind +from core.capabilities.catalog.loader import _write_catalog, ensure_default_catalog, load_catalog + __all__ = [ "get_catalog", "is_enabled", "get_enabled_ids", "set_enabled", "reorder_items", - "invalidate_catalog_cache", "ensure_default_catalog", ] diff --git a/src/backend/core/config/catalog_common.py b/src/backend/core/capabilities/catalog/common.py similarity index 90% rename from src/backend/core/config/catalog_common.py rename to src/backend/core/capabilities/catalog/common.py index 26b7fd41..594ccba4 100644 --- a/src/backend/core/config/catalog_common.py +++ b/src/backend/core/capabilities/catalog/common.py @@ -11,7 +11,7 @@ from pathlib import Path from typing import Any, Dict, Optional -_CATALOG_PATH = Path(os.environ.get("CATALOG_PATH", str(Path(__file__).with_name("catalog.json")))) +CATALOG_PATH = Path(os.environ.get("CATALOG_PATH", str(Path(__file__).resolve().parents[2] / "config" / "catalog.json"))) _LOGGER = logging.getLogger(__name__) @@ -65,7 +65,7 @@ def _item( def _read_raw_catalog() -> Dict[str, Any]: - raw = _CATALOG_PATH.read_text(encoding="utf-8") + raw = CATALOG_PATH.read_text(encoding="utf-8") data = json.loads(raw) if raw.strip() else {} if not isinstance(data, dict): raise ValueError("catalog must be a JSON object") diff --git a/src/backend/core/config/catalog_details.py b/src/backend/core/capabilities/catalog/details.py similarity index 96% rename from src/backend/core/config/catalog_details.py rename to src/backend/core/capabilities/catalog/details.py index c34f31b3..fc3df3fc 100644 --- a/src/backend/core/config/catalog_details.py +++ b/src/backend/core/capabilities/catalog/details.py @@ -5,13 +5,7 @@ import logging from typing import Any, Dict, Optional -from core.config.catalog_loader import ( - DB_HIDDEN_SERVERS, - DB_UMBRELLA_ID, - DB_UMBRELLA_NAME, - DB_UMBRELLA_DESC, - _database_query_capability_available, -) +from core.capabilities.catalog.loader import DB_HIDDEN_SERVERS, DB_UMBRELLA_ID, DB_UMBRELLA_NAME, DB_UMBRELLA_DESC, _database_query_capability_available _LOGGER = logging.getLogger(__name__) @@ -51,10 +45,10 @@ def skill_body_from_raw(raw: str) -> str: if not raw: return "" try: - from core.agent_skills.registry import _split_frontmatter + from core.capabilities.skills.registry import split_frontmatter try: - _, body = _split_frontmatter(raw) + _, body = split_frontmatter(raw) except Exception: # Without valid frontmatter, use the full text directly return raw.strip() @@ -98,7 +92,7 @@ def _load_dynamic_skill_specs() -> Dict[str, Dict[str, Any]]: 4. ``""`` (when even SKILL.md is unavailable, frontend shows "暂无详情") """ try: - from core.agent_skills.loader import get_skill_loader + from core.capabilities.skills.loader_factory import get_skill_loader loader = get_skill_loader() # Refresh metadata cache to support hot-reload for bind-mounted skill files. @@ -114,7 +108,7 @@ def _load_dynamic_skill_specs() -> Dict[str, Dict[str, Any]]: try: from core.db.engine import SessionLocal from core.db.models import AdminSkill - from core.services.skill_icon_service import get_skill_icons + from core.capabilities.skills.icons import get_skill_icons with SessionLocal() as db: for sid, intro in db.query(AdminSkill.skill_id, AdminSkill.user_intro).all(): @@ -178,7 +172,7 @@ def _curated_detail_for_owned(sid: str) -> Optional[Dict[str, Any]]: detail prefers AdminSkill.user_intro, otherwise falls back to the SKILL.md body. Returns None if the skill is not found. """ try: - from core.agent_skills.loader import get_skill_loader + from core.capabilities.skills.loader_factory import get_skill_loader loader = get_skill_loader() metadata = loader.load_all_metadata().get(sid) @@ -193,7 +187,7 @@ def _curated_detail_for_owned(sid: str) -> Optional[Dict[str, Any]]: try: from core.db.engine import SessionLocal from core.db.models import AdminSkill - from core.services.skill_icon_service import get_skill_icons + from core.capabilities.skills.icons import get_skill_icons with SessionLocal() as db: row = db.query(AdminSkill.user_intro).filter(AdminSkill.skill_id == sid).first() diff --git a/src/backend/core/config/catalog_loader.py b/src/backend/core/capabilities/catalog/loader.py similarity index 85% rename from src/backend/core/config/catalog_loader.py rename to src/backend/core/capabilities/catalog/loader.py index 3c812109..42a3d628 100644 --- a/src/backend/core/config/catalog_loader.py +++ b/src/backend/core/capabilities/catalog/loader.py @@ -11,10 +11,12 @@ import copy import json import logging +from dataclasses import dataclass, field from time import monotonic from typing import Any, Dict, List -from core.config.catalog_common import _CATALOG_PATH, _item, _read_raw_catalog +from core.capabilities.catalog import common as catalog_common +from core.capabilities.catalog.common import _item, _read_raw_catalog _LOGGER = logging.getLogger(__name__) @@ -47,19 +49,19 @@ def _database_query_capability_available() -> bool: # ── In-memory catalog cache (TTL-based) ──────────────────────────────────── -_CATALOG_CACHE: Dict[bool, Dict[str, Any]] = {} # key = include_runtime_details -_CATALOG_CACHE_TIME: Dict[bool, float] = {} + # key = include_runtime_details + _CATALOG_CACHE_TTL: float = 10.0 # seconds def invalidate_catalog_cache() -> None: """Clear the in-memory catalog cache (call after writes).""" - _CATALOG_CACHE.clear() - _CATALOG_CACHE_TIME.clear() + process_state().CATALOG_CACHE.clear() + process_state().CATALOG_CACHE_TIME.clear() def _write_catalog(data: Dict[str, Any]) -> None: - _CATALOG_PATH.write_text( + catalog_common.CATALOG_PATH.write_text( json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" ) invalidate_catalog_cache() @@ -75,9 +77,9 @@ def _load_builtin_skill_metadata() -> List[Any]: are runtime capabilities and must not become repository catalog defaults. """ try: - from core.agent_skills.backends.filesystem import FilesystemBackend - from core.agent_skills.config import get_default_skill_sources - from core.agent_skills.registry import _load_skill_metadata_from_file + from core.capabilities.skills.backends.filesystem import FilesystemBackend + from core.capabilities.skills.config import get_default_skill_sources + from core.capabilities.skills.registry import _load_skill_metadata_from_file except Exception as e: _LOGGER.warning("Failed to import built-in skill loader: %s", e) return [] @@ -248,9 +250,9 @@ def _filter_to_static_defaults(data: Dict[str, Any], defaults: Dict[str, Any]) - def ensure_default_catalog() -> Dict[str, Any]: """Create catalog.json with repository defaults if missing; return loaded catalog.""" - if not _CATALOG_PATH.exists(): + if not catalog_common.CATALOG_PATH.exists(): cat = _default_catalog() - _CATALOG_PATH.write_text( + catalog_common.CATALOG_PATH.write_text( json.dumps(cat, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" ) invalidate_catalog_cache() @@ -265,17 +267,17 @@ def load_catalog(*, include_runtime_details: bool = True) -> Dict[str, Any]: Results are cached in-memory for up to ``_CATALOG_CACHE_TTL`` seconds to avoid repeated disk I/O and dynamic source loading on every request. """ - from core.config.catalog_migration import _migrate_legacy_shape + from core.capabilities.catalog.migration import _migrate_legacy_shape now = monotonic() - cached_time = _CATALOG_CACHE_TIME.get(include_runtime_details, 0.0) - if include_runtime_details in _CATALOG_CACHE and (now - cached_time) < _CATALOG_CACHE_TTL: - return copy.deepcopy(_CATALOG_CACHE[include_runtime_details]) + cached_time = process_state().CATALOG_CACHE_TIME.get(include_runtime_details, 0.0) + if include_runtime_details in process_state().CATALOG_CACHE and (now - cached_time) < _CATALOG_CACHE_TTL: + return copy.deepcopy(process_state().CATALOG_CACHE[include_runtime_details]) - if not _CATALOG_PATH.exists(): + if not catalog_common.CATALOG_PATH.exists(): result = _default_catalog() - _CATALOG_CACHE[include_runtime_details] = copy.deepcopy(result) - _CATALOG_CACHE_TIME[include_runtime_details] = monotonic() + process_state().CATALOG_CACHE[include_runtime_details] = copy.deepcopy(result) + process_state().CATALOG_CACHE_TIME[include_runtime_details] = monotonic() return result try: @@ -300,10 +302,23 @@ def load_catalog(*, include_runtime_details: bool = True) -> Dict[str, Any]: _strip_static_detail_fields(data) if include_runtime_details: - from core.config.catalog_details import _attach_runtime_details + from core.capabilities.catalog.details import _attach_runtime_details _attach_runtime_details(data) - _CATALOG_CACHE[include_runtime_details] = copy.deepcopy(data) - _CATALOG_CACHE_TIME[include_runtime_details] = monotonic() + process_state().CATALOG_CACHE[include_runtime_details] = copy.deepcopy(data) + process_state().CATALOG_CACHE_TIME[include_runtime_details] = monotonic() return data + + +@dataclass +class LoaderState: + CATALOG_CACHE: Dict[bool, Dict[str, Any]] = field(default_factory=lambda: {}) + CATALOG_CACHE_TIME: Dict[bool, float] = field(default_factory=lambda: {}) + + +_STATE = LoaderState() + + +def process_state() -> LoaderState: + return _STATE diff --git a/src/backend/core/config/catalog_migration.py b/src/backend/core/capabilities/catalog/migration.py similarity index 97% rename from src/backend/core/config/catalog_migration.py rename to src/backend/core/capabilities/catalog/migration.py index 1611b421..46c0266c 100644 --- a/src/backend/core/config/catalog_migration.py +++ b/src/backend/core/capabilities/catalog/migration.py @@ -9,7 +9,7 @@ import logging from typing import Any, Dict, List -from core.config.catalog_common import _item +from core.capabilities.catalog.common import _item _LOGGER = logging.getLogger(__name__) @@ -59,7 +59,7 @@ def _migrate_legacy_shape(raw: Dict[str, Any]) -> Dict[str, Any]: If the data already has {skills, agents, mcp, kb} it is normalised in-place. Otherwise it is rebuilt from legacy key conventions. """ - from core.config.catalog_loader import _default_catalog + from core.capabilities.catalog.loader import _default_catalog # Already new shape. if all(k in raw for k in ("skills", "agents", "mcp", "kb")): diff --git a/src/backend/core/services/catalog_service.py b/src/backend/core/capabilities/catalog/overrides.py similarity index 100% rename from src/backend/core/services/catalog_service.py rename to src/backend/core/capabilities/catalog/overrides.py diff --git a/src/backend/core/config/catalog_resolver.py b/src/backend/core/capabilities/catalog/resolver.py similarity index 87% rename from src/backend/core/config/catalog_resolver.py rename to src/backend/core/capabilities/catalog/resolver.py index 956343cc..c87de751 100644 --- a/src/backend/core/config/catalog_resolver.py +++ b/src/backend/core/capabilities/catalog/resolver.py @@ -8,24 +8,22 @@ from __future__ import annotations import logging +from dataclasses import dataclass, field from threading import Lock from time import monotonic from typing import Any, Dict, List, Optional, Tuple from sqlalchemy.orm import Session -from core.config.catalog import get_enabled_ids, is_enabled -from core.config.catalog_runtime import get_runtime_catalog +from core.capabilities.catalog.api import get_enabled_ids, is_enabled +from core.capabilities.catalog.runtime import get_runtime_catalog logger = logging.getLogger(__name__) # ── Per-user capability cache ──────────────────────────────────────────── _CAPABILITY_CACHE_TTL = 30.0 # seconds -_capability_cache_lock = Lock() + # user_id -> (expires_at, (skills, agents, mcps)) -_capability_cache: Dict[ - str, Tuple[float, Tuple[Optional[List[str]], Optional[List[str]], Optional[List[str]]]] -] = {} # ── Context helpers (extract typed lists from a runtime context dict) ──────── @@ -151,7 +149,7 @@ def _collect(model, id_col, label: str, *, dep_gated: bool = False) -> List[str] out.append(rid) return out - from core.capabilities.local_plugin_runtime import configs + from core.harness.registry.execution_catalog import local_plugin_configs as configs owned_mcps = set(_collect(AdminMcpServer, AdminMcpServer.server_id, "mcps")) mcp_overrides = _ov_map("mcps") @@ -231,29 +229,21 @@ def _clean(raw: Optional[List[str]]) -> List[str]: else: allowed_mcp_ids.discard(mid) - from core.capabilities.local_plugin_runtime import configs + from core.harness.registry.execution_catalog import local_plugin_configs as configs allowed_mcp_ids.update(configs(user_id)) # A suspended marketplace installation is no longer an active grant. # Its admin row remains for recovery, so existence alone must not revive it. if requested_mcps: - from core.db.models import McpMarketInstallation - - suspended_ids = { - row[0] - for row in db.query(McpMarketInstallation.server_id) - .filter( - McpMarketInstallation.server_id.in_(requested_mcps), - McpMarketInstallation.status == "suspended", - ) - .all() - } + from core.harness.registry.marketplace_catalog import marketplace_catalog + + suspended_ids = marketplace_catalog().suspended_mcp_ids(db, requested_mcps) allowed_mcp_ids.difference_update(suspended_ids) - from core.capabilities.paths import capabilities_enabled - if capabilities_enabled(): - from core.capabilities.invocation import resolve_explicit_ids + from core.harness.registry.run_capabilities import has_run_capabilities + if has_run_capabilities(): + from core.harness.registry.run_capabilities import resolve_explicit_ids allowed_skill_ids, allowed_mcp_ids = resolve_explicit_ids( user_id, allowed_skill_ids, allowed_mcp_ids ) @@ -275,18 +265,15 @@ def _apply_desktop_cloud_bridge(result, *, user_id=None): """ skills, agents, mcps = result try: - from core.services.desktop_cloud_bridge import ( - apply_to_enabled_mcp_ids, - apply_to_enabled_skill_ids, - ) + from core.harness.registry.execution_catalog import apply_to_enabled_mcp_ids, apply_to_enabled_skill_ids bridged_mcps = apply_to_enabled_mcp_ids(list(mcps)) if mcps is not None else None bridged_skills = apply_to_enabled_skill_ids(list(skills)) if skills is not None else None except Exception: # noqa: BLE001 - 保留本机默认值，再独立检查本机可用性 bridged_skills, _, bridged_mcps = result - from core.capabilities.paths import capabilities_enabled - if capabilities_enabled() and bridged_skills is not None: - from core.capabilities.skills import filter_available_names + from core.harness.registry.run_capabilities import has_run_capabilities + if has_run_capabilities() and bridged_skills is not None: + from core.harness.registry.run_capabilities import filter_available_names bridged_skills = filter_available_names(bridged_skills, user_id=user_id) return (bridged_skills, agents, bridged_mcps) @@ -309,18 +296,18 @@ def resolve_all_runtime_enabled( """ # Check cache first now = monotonic() - with _capability_cache_lock: - cached = _capability_cache.get(user_id) + with process_state().capability_cache_lock: + cached = process_state().capability_cache.get(user_id) if cached is not None: expires_at, result = cached if now < expires_at: return _apply_desktop_cloud_bridge(result, user_id=user_id) else: - _capability_cache.pop(user_id, None) + process_state().capability_cache.pop(user_id, None) try: # Lazy import to avoid circular dependency at module level - from core.services import CatalogService + from core.capabilities.catalog.overrides import CatalogService base_catalog = get_runtime_catalog(db, include_runtime_details=False) svc = CatalogService(db) @@ -344,7 +331,7 @@ def resolve_all_runtime_enabled( # Dedup: if a privately-installed skill / MCP has already been set global by an admin (same underlying entry_name), drop the private # copy — otherwise the same capability would be loaded twice (global id=entry, private id=entry-fingerprint), which is wasteful and also lets # the model see two identically-named, identical-content tools/skills. Same semantics as the dedup on the /v1/catalog display side. - from core.services.marketplace_service import base_entry_name + from core.capabilities.skills.installation import base_entry_name if owned_skills: _global_skills = set(skills) @@ -363,8 +350,8 @@ def resolve_all_runtime_enabled( result = (skills, agents, mcps) # Store in cache - with _capability_cache_lock: - _capability_cache[user_id] = (now + _CAPABILITY_CACHE_TTL, result) + with process_state().capability_cache_lock: + process_state().capability_cache[user_id] = (now + _CAPABILITY_CACHE_TTL, result) return _apply_desktop_cloud_bridge(result, user_id=user_id) except Exception as exc: @@ -374,8 +361,21 @@ def resolve_all_runtime_enabled( def invalidate_capability_cache(user_id: Optional[str] = None) -> None: """Invalidate capability cache. Pass user_id to clear a specific user, or None for all.""" - with _capability_cache_lock: + with process_state().capability_cache_lock: if user_id is None: - _capability_cache.clear() + process_state().capability_cache.clear() else: - _capability_cache.pop(user_id, None) + process_state().capability_cache.pop(user_id, None) + + +@dataclass +class ResolverState: + capability_cache_lock: object = field(default_factory=lambda: Lock()) + capability_cache: Dict[str, Tuple[float, Tuple[Optional[List[str]], Optional[List[str]], Optional[List[str]]]]] = field(default_factory=lambda: {}) + + +_STATE = ResolverState() + + +def process_state() -> ResolverState: + return _STATE diff --git a/src/backend/core/config/catalog_runtime.py b/src/backend/core/capabilities/catalog/runtime.py similarity index 87% rename from src/backend/core/config/catalog_runtime.py rename to src/backend/core/capabilities/catalog/runtime.py index 3bfb5166..b26d32cd 100644 --- a/src/backend/core/config/catalog_runtime.py +++ b/src/backend/core/capabilities/catalog/runtime.py @@ -10,21 +10,22 @@ import copy import logging +from dataclasses import dataclass, field from threading import Lock from time import monotonic from typing import Any, Dict, List, Optional -from core.config.catalog import get_catalog -from core.config.catalog_common import _item -from core.config.catalog_details import resolve_skill_detail -from core.config.catalog_loader import DB_HIDDEN_SERVERS, DB_UMBRELLA_ICON, DB_UMBRELLA_ID, _database_query_capability_available -from core.config.edition_display_names import edition_mcp_icons +from core.capabilities.catalog.api import get_catalog +from core.capabilities.catalog.common import _item +from core.capabilities.catalog.details import resolve_skill_detail +from core.capabilities.catalog.loader import DB_HIDDEN_SERVERS, DB_UMBRELLA_ICON, DB_UMBRELLA_ID, _database_query_capability_available +from core.config.edition import edition_mcp_icons from sqlalchemy.orm import Session logger = logging.getLogger(__name__) -_DEFAULT_MCP_ICONS: Dict[str, str] = { +DEFAULT_MCP_ICONS: Dict[str, str] = { "query_database": "/home/mcp/database.svg", "retrieve_dataset_content": "/home/mcp/knowledge.svg", "internet_search": "/home/mcp/internet.svg", @@ -33,14 +34,12 @@ } _DATABASE_QUERY_ENABLED_CONFIG = "database_query.capability_enabled" _RUNTIME_DB_CACHE_TTL = 30.0 -_runtime_db_cache: Dict[bool, tuple[float, bool, List[Dict[str, Any]], List[Dict[str, Any]]]] = {} -_runtime_db_cache_lock = Lock() def invalidate_runtime_catalog_cache() -> None: """Clear cached public DB capability overlays.""" - with _runtime_db_cache_lock: - _runtime_db_cache.clear() + with process_state().runtime_db_cache_lock: + process_state().runtime_db_cache.clear() def merge_items_by_id( @@ -70,13 +69,12 @@ def merge_items_by_id( return merged - def _public_db_skill_items(db: Session, *, include_runtime_details: bool) -> List[Dict[str, Any]]: from core.db.models import AdminSkill icons: Dict[str, str] = {} if include_runtime_details: - from core.services.skill_icon_service import get_skill_icons + from core.capabilities.skills.icons import get_skill_icons icons = get_skill_icons(db) rows = ( @@ -112,7 +110,7 @@ def _public_db_skill_items(db: Session, *, include_runtime_details: bool) -> Lis def _public_db_mcp_items(db: Session, *, include_runtime_details: bool) -> List[Dict[str, Any]]: from core.config.user_intros import MCP_SERVER_USER_INTROS from core.db.models import AdminMcpServer - from core.services.mcp_service import is_removed_builtin_mcp_server + from core.capabilities.mcp.service import is_removed_builtin_mcp_server rows = ( db.query(AdminMcpServer) @@ -142,7 +140,7 @@ def _public_db_mcp_items(db: Session, *, include_runtime_details: bool) -> List[ enabled=bool(row.is_enabled), version="1", config={"server": row.server_id}, - icon=row.icon or _DEFAULT_MCP_ICONS.get(row.server_id, ""), + icon=row.icon or DEFAULT_MCP_ICONS.get(row.server_id, ""), ) item["created_at"] = row.created_at.isoformat() if row.created_at else None item["tools"] = [ @@ -175,8 +173,6 @@ def _config_bool(db: Session, key: str, default: bool) -> bool: return default - - def _set_database_query_state(catalog: Dict[str, Any], enabled: bool) -> None: for item in catalog.get("mcp") or []: if not isinstance(item, dict) or item.get("id") != DB_UMBRELLA_ID: @@ -194,13 +190,13 @@ def _public_db_overlay( overrides and private capabilities remain resolved in catalog_resolver. """ now = monotonic() - with _runtime_db_cache_lock: - cached = _runtime_db_cache.get(include_runtime_details) + with process_state().runtime_db_cache_lock: + cached = process_state().runtime_db_cache.get(include_runtime_details) if cached is not None: expires_at, db_query_enabled, skills, mcps = cached if now < expires_at: return db_query_enabled, copy.deepcopy(skills), copy.deepcopy(mcps) - _runtime_db_cache.pop(include_runtime_details, None) + process_state().runtime_db_cache.pop(include_runtime_details, None) db_query_enabled = _config_bool(db, _DATABASE_QUERY_ENABLED_CONFIG, True) try: @@ -214,8 +210,8 @@ def _public_db_overlay( logger.warning("runtime catalog DB MCP overlay failed: %s", exc) mcps = [] - with _runtime_db_cache_lock: - _runtime_db_cache[include_runtime_details] = ( + with process_state().runtime_db_cache_lock: + process_state().runtime_db_cache[include_runtime_details] = ( now + _RUNTIME_DB_CACHE_TTL, db_query_enabled, copy.deepcopy(skills), @@ -252,7 +248,7 @@ def get_runtime_catalog( if not isinstance(catalog.get(key), list): catalog[key] = [] - from core.capabilities import device_catalog + from core.harness.registry import device_catalog if device_catalog.active(): # 混合模式：能力一律以云端账号同步下来的那份为准。本机自带的内置技能 / 连接器 @@ -290,3 +286,16 @@ def get_runtime_catalog( logger.warning("runtime catalog DB overlay merge failed: %s", exc) return catalog + + +@dataclass +class RuntimeState: + runtime_db_cache: Dict[bool, tuple[float, bool, List[Dict[str, Any]], List[Dict[str, Any]]]] = field(default_factory=lambda: {}) + runtime_db_cache_lock: object = field(default_factory=lambda: Lock()) + + +_STATE = RuntimeState() + + +def process_state() -> RuntimeState: + return _STATE diff --git a/src/backend/core/capabilities/identity_rules.py b/src/backend/core/capabilities/identity_rules.py new file mode 100644 index 00000000..d1c5093c --- /dev/null +++ b/src/backend/core/capabilities/identity_rules.py @@ -0,0 +1,36 @@ +"""Capability kinds and portable identity segments, independent of host storage.""" +import re + +KIND_SKILL = "skill" + +KIND_PLUGIN = "plugin" + +KIND_AGENT = "agent" + +KIND_MCP = "mcp" + +KINDS = (KIND_SKILL, KIND_PLUGIN, KIND_AGENT, KIND_MCP) + +LOCAL_PROFILE = "local" + +BUILTIN_PROFILE = "builtin" + +_SEGMENT_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,120}$") + +_WINDOWS_RESERVED = { + "CON", + "PRN", + "AUX", + "NUL", + *(f"COM{i}" for i in range(1, 10)), + *(f"LPT{i}" for i in range(1, 10)), +} + +def safe_segment(value: str) -> str: + """A single path segment safe on every platform, or ``ValueError``.""" + seg = (value or "").strip() + if seg != value or seg.endswith((".", " ")) or not _SEGMENT_RE.match(seg) or seg in (".", ".."): + raise ValueError(f"unsafe path segment: {value!r}") + if seg.split(".")[0].upper() in _WINDOWS_RESERVED: + raise ValueError(f"reserved name: {value!r}") + return seg diff --git a/src/backend/core/services/desktop_capability_protocol.py b/src/backend/core/capabilities/manifest_protocol.py similarity index 99% rename from src/backend/core/services/desktop_capability_protocol.py rename to src/backend/core/capabilities/manifest_protocol.py index 5bccf3c7..070d73c6 100644 --- a/src/backend/core/services/desktop_capability_protocol.py +++ b/src/backend/core/capabilities/manifest_protocol.py @@ -49,7 +49,6 @@ def canonical_hash(value: Any) -> str: def token_claims(token: str) -> Dict[str, Any]: """Read routing labels only; server-side signature/session checks authorize use.""" try: - import base64 raw = str(token).split(".") if len(raw) != 3 or raw[0] not in ("dcap1", "dcap2"): return {} @@ -82,7 +81,7 @@ def public_tool_schema(raw: Any) -> Optional[Dict[str, Any]]: "inputSchema": schema, } # Only the versioned execution declaration crosses the boundary; arbitrary metadata does not. - from core.services.management_contract import META_KEY + from core.capabilities.plugins.management.contract import META_KEY meta = raw.get("_meta") or raw.get("meta") or {} if isinstance(meta, dict) and META_KEY in meta: public["_meta"] = {META_KEY: copy.deepcopy(meta[META_KEY])} diff --git a/src/backend/core/capabilities/mcp/__init__.py b/src/backend/core/capabilities/mcp/__init__.py new file mode 100644 index 00000000..0509af9f --- /dev/null +++ b/src/backend/core/capabilities/mcp/__init__.py @@ -0,0 +1 @@ +"""MCP server installation, management and OAuth connections.""" diff --git a/src/backend/core/services/mcp_builtin_specs.py b/src/backend/core/capabilities/mcp/builtin_specs.py similarity index 94% rename from src/backend/core/services/mcp_builtin_specs.py rename to src/backend/core/capabilities/mcp/builtin_specs.py index 802cfb79..62a91d01 100644 --- a/src/backend/core/services/mcp_builtin_specs.py +++ b/src/backend/core/capabilities/mcp/builtin_specs.py @@ -12,7 +12,6 @@ '- "各区今年 1–9 月固投同比"\n\n## 输出示例\n- 精确数值 + 数据口径标注\n' "- 时间 / 地区 / 行业等多维度数据切片\n- 同比、环比、累计值自动计算\n" ), - "is_stable": True, "is_enabled": True, "sort_order": 0, "icon": "/home/mcp/knowledge.svg", @@ -22,7 +21,6 @@ "display_name": "知识库检索", "description": "从公有/私有知识库中语义检索政策文件、产业报告及用户上传文档，支持混合检索与重排序。", "user_intro": None, - "is_stable": False, "is_enabled": True, "sort_order": 1, "icon": "/home/mcp/learning.svg", @@ -32,7 +30,6 @@ "display_name": "互联网搜索", "description": "检索公开网页、官方文档、新闻及财经资讯，支持多查询并发、来源去重和原文核验。", "user_intro": None, - "is_stable": True, "is_enabled": True, "sort_order": 2, "icon": "/home/mcp/internet.svg", @@ -42,7 +39,6 @@ "display_name": "网站信息抓取", "description": "抓取指定网页 URL 的内容，提取正文文本或 Markdown，支持搜索引擎结果页解析。", "user_intro": None, - "is_stable": True, "is_enabled": True, "sort_order": 6, "icon": "/home/mcp/source.svg", diff --git a/src/backend/core/services/edition_mcp.py b/src/backend/core/capabilities/mcp/edition.py similarity index 100% rename from src/backend/core/services/edition_mcp.py rename to src/backend/core/capabilities/mcp/edition.py diff --git a/src/backend/core/services/mcp_management_service.py b/src/backend/core/capabilities/mcp/management.py similarity index 90% rename from src/backend/core/services/mcp_management_service.py rename to src/backend/core/capabilities/mcp/management.py index 0adab34e..318c3030 100644 --- a/src/backend/core/services/mcp_management_service.py +++ b/src/backend/core/capabilities/mcp/management.py @@ -3,15 +3,14 @@ from __future__ import annotations import asyncio -import hashlib import ipaddress -import json import logging import socket -from typing import Any, Dict, Iterable, List +from typing import Any, Dict, List from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse import httpx +import httpx2 from core.db.models import AdminMcpServer from core.infra.crypto import decrypt_secret, encrypt_secret from core.infra.exceptions import BadRequestError @@ -167,7 +166,7 @@ def auth_schema_from_headers(headers: Dict[str, str] | None) -> List[Dict[str, A ] -from core.services.mcp_tool_metadata import tool_snapshot_hash, snapshot_tool +from core.capabilities.mcp.tool_metadata import tool_snapshot_hash, snapshot_tool def _is_forbidden_ip(address: str) -> bool: @@ -317,16 +316,16 @@ def refresh_mcp_caches() -> None: """Invalidate MCP, catalog, capability, and prompt caches after a mutation.""" invalidators = [] try: - from core.services.mcp_service import McpServerConfigService + from core.capabilities.mcp.service import McpServerConfigService invalidators.append(McpServerConfigService.get_instance().invalidate_cache) except Exception as exc: # noqa: BLE001 logger.warning("Unable to load the MCP cache invalidator: %s", exc) for module_name, function_name in ( - ("core.config.catalog_loader", "invalidate_catalog_cache"), - ("core.config.catalog_runtime", "invalidate_runtime_catalog_cache"), - ("core.config.catalog_resolver", "invalidate_capability_cache"), + ("core.capabilities.catalog.loader", "invalidate_catalog_cache"), + ("core.capabilities.catalog.runtime", "invalidate_runtime_catalog_cache"), + ("core.capabilities.catalog.resolver", "invalidate_capability_cache"), ("prompts.prompt_runtime", "invalidate_prompt_cache"), ): try: @@ -351,7 +350,7 @@ async def probe_mcp_connectivity( timeout_seconds: float | None = None, ) -> tuple[bool, str]: """Return whether an MCP server can connect and enumerate its tools.""" - from core.services.mcp_service import McpServerConfigService + from core.capabilities.mcp.service import McpServerConfigService if timeout_seconds is None: timeout_seconds = ( @@ -361,30 +360,22 @@ async def probe_mcp_connectivity( ) cfg = McpServerConfigService.get_instance()._row_to_config(row) + cfg["discovery_timeout"] = timeout_seconds if oauth_provider is not None: cfg["oauth_provider"] = oauth_provider async def _do_probe() -> None: - from core.llm.mcp_pool import make_client + from core.harness.registry.mcp_clients import discover_tools if row.transport not in ("stdio", "streamable_http", "sse"): raise RuntimeError(f"Unknown transport: {row.transport}") - is_http = row.transport in ("streamable_http", "sse") - # AgentScope's stateful HTTP MCP client owns an AnyIO cancel scope that - # is bound to the task which opened it. Failed handshakes (notably 401) - # can finalize the underlying async generator in another task, masking - # the real HTTP error with "Attempted to exit cancel scope...". A - # stateless client keeps the complete list_tools lifecycle in this task. - client = make_client(row.server_id, cfg, is_stateful=not is_http) + resource = await discover_tools(row.server_id, cfg) try: - if not is_http: - await client.connect() - discovered = await client.list_tools() - tools_meta = [] + discovered = resource.descriptors tools_meta = [snapshot_tool(tool) for tool in discovered or []] row.tools_json = tools_meta if db is not None and tools_meta: - from core.ontology.build_validator import OntologyBuildValidator + from core.harness.registry.build_governance import validate_build extra_config = row.extra_config or {} ontology_tags = { @@ -401,7 +392,7 @@ async def _do_probe() -> None: for tag in tags if str(tag).strip() ) - report = OntologyBuildValidator(db).validate( + report = validate_build(db, asset_type="tool", name=row.display_name, description=row.description or "", @@ -413,14 +404,7 @@ async def _do_probe() -> None: messages = "; ".join(item.message for item in report.errors) raise RuntimeError(f"Domain Pack 构建校验失败：{messages}") finally: - # Stateless HTTP list_tools opens and closes its own connection. - # Calling close() is unnecessary and can re-enter the SDK's - # task-bound cancel scope. Stateful stdio still requires cleanup. - if not is_http: - try: - await client.close() - except BaseException as exc: # noqa: BLE001 - logger.debug("MCP probe client close failed: %s", exc) + await resource.close() try: await asyncio.wait_for(_do_probe(), timeout=timeout_seconds) @@ -463,13 +447,13 @@ def _format_mcp_probe_exception(exc: BaseException) -> str: if is_cancel_noise: continue - if isinstance(current, httpx.HTTPStatusError) and current.response is not None: + if isinstance(current, httpx2.HTTPStatusError) and current.response is not None: status = current.response.status_code reason = current.response.reason_phrase if status in (401, 403): return f"远端 MCP 返回 {status} {reason}，请检查鉴权设置和 Token" return f"远端 MCP 返回 HTTP {status} {reason}" - if isinstance(current, httpx.ConnectError): + if isinstance(current, httpx2.ConnectError): return "无法连接远端 MCP，请检查地址、网络和服务状态" if message: return f"{type(current).__name__}: {message}" diff --git a/src/backend/core/services/mcp_oauth_service.py b/src/backend/core/capabilities/mcp/oauth.py similarity index 54% rename from src/backend/core/services/mcp_oauth_service.py rename to src/backend/core/capabilities/mcp/oauth.py index ec465578..39b64f09 100644 --- a/src/backend/core/services/mcp_oauth_service.py +++ b/src/backend/core/capabilities/mcp/oauth.py @@ -1,13 +1,6 @@ -"""Provider-neutral OAuth 2.1 flow for remote MCP marketplace installs. - -The implementation delegates MCP protected-resource discovery, authorization -server metadata, PKCE, DCR, resource indicators, token exchange, and refresh to -the official Python MCP SDK. Marketplace metadata remains credential-free; -the resulting bundle is encrypted inside the concrete installation only. -""" - +"""Browser OAuth install flow for remote MCP servers.""" from __future__ import annotations - +from core.capabilities.mcp.management import materialize_mcp_http_connection as market_dependency_materialize_mcp_http_connection import asyncio import contextlib import json @@ -16,151 +9,23 @@ from dataclasses import dataclass, field from typing import Any, Dict, Optional from urllib.parse import urlencode, urlparse - from core.config.settings import settings from core.db.engine import SessionLocal -from core.db.models import AdminMcpServer, McpMarketItem, McpMarketVersion +from core.db.models import AdminMcpServer +from core.harness.registry.marketplace_catalog import marketplace_catalog from core.infra.crypto import decrypt_secret, encrypt_secret from core.infra.exceptions import BadRequestError, ResourceNotFoundError from core.infra.ephemeral import get_ephemeral_state -from core.services.mcp_management_service import ( - decrypt_mcp_headers, - encrypt_mcp_headers, - mcp_oauth_bundle_storage_key, - probe_mcp_connectivity, - validate_remote_mcp_url, -) -from mcp.client.auth import OAuthClientProvider, TokenStorage -from mcp.shared.auth import ( - OAuthClientInformationFull, - OAuthClientMetadata, - OAuthMetadata, - OAuthToken, - ProtectedResourceMetadata, -) +from core.capabilities.mcp.management import probe_mcp_connectivity, validate_remote_mcp_url +from mcp.shared.auth import AuthorizationCodeResult, OAuthClientInformationFull, OAuthClientMetadata from sqlalchemy.orm import Session +from core.capabilities.mcp import oauth_storage -_FLOW_TTL_SECONDS = 10 * 60 -# Authorization-URL discovery costs several sequential round trips (protected -# resource metadata, authorization server metadata, dynamic registration), so a -# slow link must keep the flow polling instead of failing the install outright. -_DISCOVERY_GRACE_SECONDS = 5.0 - - -class OAuthBundleStorage(TokenStorage): - """SDK token storage backed by an encrypted MCP installation row.""" - - def __init__(self, bundle: Optional[Dict[str, Any]] = None, *, server_id: Optional[str] = None): - self.bundle: Dict[str, Any] = dict(bundle or {}) - self.server_id = server_id - - async def get_tokens(self) -> OAuthToken | None: - raw = self.bundle.get("tokens") - return OAuthToken.model_validate(raw) if isinstance(raw, dict) else None - - async def set_tokens(self, tokens: OAuthToken) -> None: - previous = self.bundle.get("tokens") if isinstance(self.bundle.get("tokens"), dict) else {} - if not tokens.refresh_token and previous.get("refresh_token"): - tokens = tokens.model_copy(update={"refresh_token": previous["refresh_token"]}) - self.bundle["tokens"] = tokens.model_dump(mode="json", exclude_none=True) - self.bundle["expires_at"] = ( - time.time() + int(tokens.expires_in) if tokens.expires_in is not None else None - ) - self._persist() - - async def get_client_info(self) -> OAuthClientInformationFull | None: - raw = self.bundle.get("client_info") - return OAuthClientInformationFull.model_validate(raw) if isinstance(raw, dict) else None - - async def set_client_info(self, client_info: OAuthClientInformationFull) -> None: - self.bundle["client_info"] = client_info.model_dump(mode="json", exclude_none=True) - self._persist() - - def capture_context(self, provider: OAuthClientProvider) -> Dict[str, Any]: - context = provider.context - self.bundle["server_url"] = context.server_url - self.bundle["client_metadata"] = context.client_metadata.model_dump( - mode="json", exclude_none=True - ) - self.bundle["oauth_metadata"] = ( - context.oauth_metadata.model_dump(mode="json", exclude_none=True) - if context.oauth_metadata - else None - ) - self.bundle["protected_resource_metadata"] = ( - context.protected_resource_metadata.model_dump(mode="json", exclude_none=True) - if context.protected_resource_metadata - else None - ) - self.bundle["auth_server_url"] = context.auth_server_url - self.bundle["protocol_version"] = context.protocol_version - self._persist() - return dict(self.bundle) - - def _persist(self) -> None: - if not self.server_id: - return - db = SessionLocal() - try: - row = ( - db.query(AdminMcpServer).filter(AdminMcpServer.server_id == self.server_id).first() - ) - if not row: - return - headers = decrypt_mcp_headers(row.headers) - headers[mcp_oauth_bundle_storage_key()] = json.dumps( - self.bundle, ensure_ascii=False, separators=(",", ":") - ) - row.headers = encrypt_mcp_headers(headers) - db.commit() - finally: - db.close() +FLOW_TTL_SECONDS = 10 * 60 -def build_oauth_provider( - server_url: str, - bundle: Dict[str, Any], - *, - storage: Optional[OAuthBundleStorage] = None, - redirect_handler=None, - callback_handler=None, -) -> OAuthClientProvider: - """Rehydrate an SDK provider for first login or automatic refresh.""" - actual_storage = storage or OAuthBundleStorage(bundle) - metadata_raw = bundle.get("client_metadata") or { - "redirect_uris": ["http://localhost/oauth/callback"], - "client_name": "HugAgentOS MCP Client", - "grant_types": ["authorization_code", "refresh_token"], - "response_types": ["code"], - } - provider = OAuthClientProvider( - server_url=server_url, - client_metadata=OAuthClientMetadata.model_validate(metadata_raw), - storage=actual_storage, - redirect_handler=redirect_handler, - callback_handler=callback_handler, - timeout=300.0, - ) - async def _validate_oauth_request(request) -> None: - # OAuth discovery metadata is controlled by the remote resource. Apply - # the same DNS/IP SSRF boundary to every SDK-generated request, not only - # to the original MCP URL. - await validate_remote_mcp_url(str(request.url), require_https=False) - - provider.mcp_request_hook = _validate_oauth_request - context = provider.context - if isinstance(bundle.get("oauth_metadata"), dict): - context.oauth_metadata = OAuthMetadata.model_validate(bundle["oauth_metadata"]) - if isinstance(bundle.get("protected_resource_metadata"), dict): - context.protected_resource_metadata = ProtectedResourceMetadata.model_validate( - bundle["protected_resource_metadata"] - ) - context.auth_server_url = bundle.get("auth_server_url") - context.protocol_version = bundle.get("protocol_version") - expires_at = bundle.get("expires_at") - context.token_expiry_time = float(expires_at) if expires_at else None - return provider +DISCOVERY_GRACE_SECONDS = 5.0 @dataclass @@ -181,22 +46,20 @@ class OAuthInstallFlow: result: Dict[str, Any] = field(default_factory=dict) callback_code: str = "" callback_state: Optional[str] = None + callback_issuer: Optional[str] = None ready: asyncio.Event = field(default_factory=asyncio.Event) callback_received: asyncio.Event = field(default_factory=asyncio.Event) -_flows: Dict[str, OAuthInstallFlow] = {} - - -def _flow_key(flow_id: str) -> str: +def flow_key(flow_id: str) -> str: return f"mcp:oauth:flow:{flow_id}" -def _callback_key(flow_id: str) -> str: +def callback_key(flow_id: str) -> str: return f"mcp:oauth:callback:{flow_id}" -async def _store_flow_status(flow: OAuthInstallFlow) -> None: +async def store_flow_status(flow: OAuthInstallFlow) -> None: payload = { "flow_id": flow.flow_id, "slug": flow.slug, @@ -207,9 +70,9 @@ async def _store_flow_status(flow: OAuthInstallFlow) -> None: "result": dict(flow.result or {}), } await get_ephemeral_state().put( - _flow_key(flow.flow_id), + flow_key(flow.flow_id), json.dumps(payload, ensure_ascii=False, separators=(",", ":")), - ttl=_FLOW_TTL_SECONDS, + ttl=FLOW_TTL_SECONDS, ) @@ -241,33 +104,15 @@ def public_oauth_callback_url(request: Any) -> str: return f"{origin}/api/v1/mcp-market/oauth/callback" -def _purge_expired_flows() -> None: - cutoff = time.time() - _FLOW_TTL_SECONDS - for flow_id, flow in list(_flows.items()): +def purge_expired_flows() -> None: + cutoff = time.time() - FLOW_TTL_SECONDS + for flow_id, flow in list(process_state().flows.items()): if flow.created_at < cutoff: - _flows.pop(flow_id, None) + process_state().flows.pop(flow_id, None) -def _get_version(db: Session, slug: str) -> tuple[McpMarketItem, McpMarketVersion]: - item = ( - db.query(McpMarketItem) - .filter( - McpMarketItem.slug == slug, - McpMarketItem.deleted_at.is_(None), - McpMarketItem.status == "active", - ) - .first() - ) - if not item: - raise ResourceNotFoundError("mcp_market_item", slug) - version = ( - db.query(McpMarketVersion) - .filter(McpMarketVersion.version_id == item.latest_version_id) - .first() - ) - if not version: - raise ResourceNotFoundError("mcp_market_version", item.latest_version_id) - return item, version +def get_version(db: Session, slug: str) -> tuple[Any, Any]: + return marketplace_catalog().mcp_version(db, slug) async def start_oauth_install( @@ -283,11 +128,11 @@ async def start_oauth_install( client_secret: str = "", ) -> Dict[str, str]: """Start a browser OAuth install and wait only until the authorize URL is known.""" - from core.services import mcp_marketplace_service as market - _purge_expired_flows() - _, version = _get_version(db, slug) - auth_config = market._normalize_auth_config( + + purge_expired_flows() + _, version = get_version(db, slug) + auth_config = marketplace_catalog().normalize_auth_config( version.auth_config, list(version.auth_schema or []) ) method = next((row for row in auth_config["methods"] if row["id"] == auth_method), None) @@ -310,11 +155,11 @@ async def start_oauth_install( client_id=client_id.strip(), client_secret=client_secret.strip(), ) - _flows[flow_id] = flow - await _store_flow_status(flow) - asyncio.create_task(_run_flow(flow), name=f"mcp-oauth-{flow_id}") + process_state().flows[flow_id] = flow + await store_flow_status(flow) + asyncio.create_task(run_flow(flow), name=f"mcp-oauth-{flow_id}") with contextlib.suppress(asyncio.TimeoutError): - await asyncio.wait_for(flow.ready.wait(), timeout=_DISCOVERY_GRACE_SECONDS) + await asyncio.wait_for(flow.ready.wait(), timeout=DISCOVERY_GRACE_SECONDS) if flow.status == "failed": raise BadRequestError(message=flow.error or "无法启动 OAuth 登录") return { @@ -324,26 +169,27 @@ async def start_oauth_install( } -async def _run_flow(flow: OAuthInstallFlow) -> None: - from core.services import mcp_marketplace_service as market +async def run_flow(flow: OAuthInstallFlow) -> None: + db = SessionLocal() try: - item, version = _get_version(db, flow.slug) - auth_config = market._normalize_auth_config( + item, version = get_version(db, flow.slug) + auth_config = marketplace_catalog().normalize_auth_config( version.auth_config, list(version.auth_schema or []) ) method = next(row for row in auth_config["methods"] if row["id"] == flow.auth_method) - headers = market._installation_secrets( + headers = marketplace_catalog().installation_secrets( version, flow.credentials, {}, auth_method=flow.auth_method, ) - runtime_url, _ = market.materialize_mcp_http_connection(version.url, headers) + runtime_url, _ = market_dependency_materialize_mcp_http_connection(version.url, headers) await validate_remote_mcp_url(runtime_url, require_https=False) metadata = OAuthClientMetadata( + application_type=("native" if urlparse(flow.callback_url).hostname in {"localhost", "127.0.0.1", "::1"} else "web"), redirect_uris=[flow.callback_url], token_endpoint_auth_method=("client_secret_post" if flow.client_secret else "none"), grant_types=["authorization_code", "refresh_token"], @@ -351,7 +197,7 @@ async def _run_flow(flow: OAuthInstallFlow) -> None: scope=" ".join(method.get("scopes") or []) or None, client_name="HugAgentOS MCP Client", ) - storage = OAuthBundleStorage( + storage = oauth_storage.OAuthBundleStorage( { "server_url": runtime_url, "client_metadata": metadata.model_dump(mode="json", exclude_none=True), @@ -369,17 +215,17 @@ async def _run_flow(flow: OAuthInstallFlow) -> None: async def redirect_handler(url: str) -> None: flow.authorization_url = url flow.status = "waiting_for_user" - await _store_flow_status(flow) + await store_flow_status(flow) flow.ready.set() - async def callback_handler() -> tuple[str, str | None]: + async def callback_handler() -> AuthorizationCodeResult: deadline = time.monotonic() + 300.0 while time.monotonic() < deadline: if flow.callback_received.is_set(): if flow.error: raise RuntimeError(flow.error) - return flow.callback_code, flow.callback_state - raw = await get_ephemeral_state().get(_callback_key(flow.flow_id)) + return AuthorizationCodeResult(code=flow.callback_code or "", state=flow.callback_state, iss=flow.callback_issuer) + raw = await get_ephemeral_state().get(callback_key(flow.flow_id)) if raw: payload = json.loads(raw) callback_error = str(payload.get("error") or "") @@ -387,11 +233,11 @@ async def callback_handler() -> tuple[str, str | None]: raise RuntimeError(callback_error) encrypted_code = str(payload.get("code") or "") callback_code = decrypt_secret(encrypted_code) if encrypted_code else "" - return callback_code or "", payload.get("state") + return AuthorizationCodeResult(code=callback_code or "", state=payload.get("state"), iss=payload.get("iss")) await asyncio.sleep(0.5) raise TimeoutError("OAuth 登录回调超时") - provider = build_oauth_provider( + provider = oauth_storage.build_oauth_provider( runtime_url, storage.bundle, storage=storage, @@ -408,7 +254,6 @@ async def callback_handler() -> tuple[str, str | None]: args=[], env_vars={}, env_inherit=[], - is_stable=False, is_enabled=False, extra_config={}, ) @@ -422,15 +267,15 @@ async def callback_handler() -> tuple[str, str | None]: raise RuntimeError(f"OAuth MCP 连接失败：{error}") if flow.error: raise RuntimeError(flow.error) - persisted = await get_ephemeral_state().get(_flow_key(flow.flow_id)) + persisted = await get_ephemeral_state().get(flow_key(flow.flow_id)) if persisted: persisted_status = json.loads(persisted) if persisted_status.get("status") == "failed": raise RuntimeError(str(persisted_status.get("error") or "OAuth 登录已取消")) bundle = storage.capture_context(provider) flow.status = "installing" - await _store_flow_status(flow) - flow.result = await market.install_market_item( + await store_flow_status(flow) + flow.result = await marketplace_catalog().install_market_item( db, flow.slug, owner_user_id=flow.owner_user_id, @@ -440,12 +285,12 @@ async def callback_handler() -> tuple[str, str | None]: oauth_bundle=bundle, ) flow.status = "completed" - await _store_flow_status(flow) + await store_flow_status(flow) except BaseException as exc: # noqa: BLE001 db.rollback() flow.status = "failed" flow.error = str(exc) - await _store_flow_status(flow) + await store_flow_status(flow) finally: flow.ready.set() db.close() @@ -456,40 +301,43 @@ async def complete_callback( *, code: str = "", state: Optional[str] = None, + iss: Optional[str] = None, error: str = "", ) -> None: - flow = _flows.get(flow_id) - public_raw = await get_ephemeral_state().get(_flow_key(flow_id)) + flow = process_state().flows.get(flow_id) + public_raw = await get_ephemeral_state().get(flow_key(flow_id)) if not flow and not public_raw: raise ResourceNotFoundError("mcp_oauth_flow", flow_id) await get_ephemeral_state().put( - _callback_key(flow_id), + callback_key(flow_id), json.dumps( { "code": encrypt_secret(code) if code else "", "state": state, + "iss": iss, "error": error, }, ensure_ascii=False, separators=(",", ":"), ), - ttl=_FLOW_TTL_SECONDS, + ttl=FLOW_TTL_SECONDS, ) if flow: flow.callback_code = code flow.callback_state = state + flow.callback_issuer = iss flow.error = error flow.status = "failed" if error else "processing_callback" flow.callback_received.set() - await _store_flow_status(flow) + await store_flow_status(flow) else: payload = json.loads(public_raw) payload["status"] = "failed" if error else "processing_callback" payload["error"] = error await get_ephemeral_state().put( - _flow_key(flow_id), + flow_key(flow_id), json.dumps(payload, ensure_ascii=False, separators=(",", ":")), - ttl=_FLOW_TTL_SECONDS, + ttl=FLOW_TTL_SECONDS, ) @@ -503,7 +351,7 @@ async def cancel_flow(flow_id: str, *, owner_user_id: Optional[str]) -> Dict[str async def get_flow_status(flow_id: str, *, owner_user_id: Optional[str]) -> Dict[str, Any]: - raw = await get_ephemeral_state().get(_flow_key(flow_id)) + raw = await get_ephemeral_state().get(flow_key(flow_id)) if not raw: raise ResourceNotFoundError("mcp_oauth_flow", flow_id) payload = json.loads(raw) @@ -518,12 +366,13 @@ async def get_flow_status(flow_id: str, *, owner_user_id: Optional[str]) -> Dict } -__all__ = [ - "OAuthBundleStorage", - "build_oauth_provider", - "cancel_flow", - "complete_callback", - "get_flow_status", - "public_oauth_callback_url", - "start_oauth_install", -] +@dataclass +class OauthState: + flows: Dict[str, OAuthInstallFlow] = field(default_factory=lambda: {}) + + +_STATE = OauthState() + + +def process_state() -> OauthState: + return _STATE diff --git a/src/backend/core/capabilities/mcp/oauth_storage.py b/src/backend/core/capabilities/mcp/oauth_storage.py new file mode 100644 index 00000000..5bf6b3f3 --- /dev/null +++ b/src/backend/core/capabilities/mcp/oauth_storage.py @@ -0,0 +1,124 @@ +"""Encrypted token storage that backs the MCP SDK's OAuth provider.""" +from __future__ import annotations +import json +import time +from typing import Any, Dict, Optional +from core.db.engine import SessionLocal +from core.db.models import AdminMcpServer +from core.capabilities.mcp.management import decrypt_mcp_headers, encrypt_mcp_headers, mcp_oauth_bundle_storage_key, validate_remote_mcp_url +from mcp.client.auth import OAuthClientProvider, TokenStorage +from mcp.shared.auth import OAuthClientInformationFull, OAuthClientMetadata, OAuthMetadata, OAuthToken, ProtectedResourceMetadata + +class OAuthBundleStorage(TokenStorage): + """SDK token storage backed by an encrypted MCP installation row.""" + + def __init__(self, bundle: Optional[Dict[str, Any]] = None, *, server_id: Optional[str] = None): + self.bundle: Dict[str, Any] = dict(bundle or {}) + self.server_id = server_id + + async def get_tokens(self) -> OAuthToken | None: + raw = self.bundle.get("tokens") + return OAuthToken.model_validate(raw) if isinstance(raw, dict) else None + + async def set_tokens(self, tokens: OAuthToken) -> None: + previous = self.bundle.get("tokens") if isinstance(self.bundle.get("tokens"), dict) else {} + if not tokens.refresh_token and previous.get("refresh_token"): + tokens = tokens.model_copy(update={"refresh_token": previous["refresh_token"]}) + self.bundle["tokens"] = tokens.model_dump(mode="json", exclude_none=True) + self.bundle["expires_at"] = ( + time.time() + int(tokens.expires_in) if tokens.expires_in is not None else None + ) + self._persist() + + async def get_client_info(self) -> OAuthClientInformationFull | None: + raw = self.bundle.get("client_info") + return OAuthClientInformationFull.model_validate(raw) if isinstance(raw, dict) else None + + async def set_client_info(self, client_info: OAuthClientInformationFull) -> None: + self.bundle["client_info"] = client_info.model_dump(mode="json", exclude_none=True) + self._persist() + + def capture_context(self, provider: OAuthClientProvider) -> Dict[str, Any]: + context = provider.context + self.bundle["server_url"] = context.server_url + self.bundle["client_metadata"] = context.client_metadata.model_dump( + mode="json", exclude_none=True + ) + self.bundle["oauth_metadata"] = ( + context.oauth_metadata.model_dump(mode="json", exclude_none=True) + if context.oauth_metadata + else None + ) + self.bundle["protected_resource_metadata"] = ( + context.protected_resource_metadata.model_dump(mode="json", exclude_none=True) + if context.protected_resource_metadata + else None + ) + self.bundle["auth_server_url"] = context.auth_server_url + self.bundle["protocol_version"] = context.protocol_version + self._persist() + return dict(self.bundle) + + def _persist(self) -> None: + if not self.server_id: + return + db = SessionLocal() + try: + row = ( + db.query(AdminMcpServer).filter(AdminMcpServer.server_id == self.server_id).first() + ) + if not row: + return + headers = decrypt_mcp_headers(row.headers) + headers[mcp_oauth_bundle_storage_key()] = json.dumps( + self.bundle, ensure_ascii=False, separators=(",", ":") + ) + row.headers = encrypt_mcp_headers(headers) + db.commit() + finally: + db.close() + + +def build_oauth_provider( + server_url: str, + bundle: Dict[str, Any], + *, + storage: Optional[OAuthBundleStorage] = None, + redirect_handler=None, + callback_handler=None, +) -> OAuthClientProvider: + """Rehydrate an SDK provider for first login or automatic refresh.""" + actual_storage = storage or OAuthBundleStorage(bundle) + metadata_raw = bundle.get("client_metadata") or { + "redirect_uris": ["http://localhost/oauth/callback"], + "client_name": "HugAgentOS MCP Client", + "grant_types": ["authorization_code", "refresh_token"], + "response_types": ["code"], + } + provider = OAuthClientProvider( + server_url=server_url, + client_metadata=OAuthClientMetadata.model_validate(metadata_raw), + storage=actual_storage, + redirect_handler=redirect_handler, + callback_handler=callback_handler, + ) + + async def _validate_oauth_request(request) -> None: + # OAuth discovery metadata is controlled by the remote resource. Apply + # the same DNS/IP SSRF boundary to every SDK-generated request, not only + # to the original MCP URL. + await validate_remote_mcp_url(str(request.url), require_https=False) + + provider.mcp_request_hook = _validate_oauth_request + context = provider.context + if isinstance(bundle.get("oauth_metadata"), dict): + context.oauth_metadata = OAuthMetadata.model_validate(bundle["oauth_metadata"]) + if isinstance(bundle.get("protected_resource_metadata"), dict): + context.protected_resource_metadata = ProtectedResourceMetadata.model_validate( + bundle["protected_resource_metadata"] + ) + context.auth_server_url = bundle.get("auth_server_url") + context.protocol_version = bundle.get("protocol_version") + expires_at = bundle.get("expires_at") + context.token_expiry_time = float(expires_at) if expires_at else None + return provider diff --git a/src/backend/core/services/mcp_service.py b/src/backend/core/capabilities/mcp/service.py similarity index 91% rename from src/backend/core/services/mcp_service.py rename to src/backend/core/capabilities/mcp/service.py index 9359b975..25ced2fd 100644 --- a/src/backend/core/services/mcp_service.py +++ b/src/backend/core/capabilities/mcp/service.py @@ -1,7 +1,7 @@ """MCP server configuration service (DB-driven, cached). Reads MCP server configs from the admin_mcp_servers table and provides -them in the same dict format that MCPConnectionPool and agent_factory +them in the same dict format that run-owned MCP discovery and agent_factory expect (compatible with the old MCP_SERVERS dict from mcp_config.py). Thread-safe singleton with a 30s TTL cache. @@ -17,19 +17,19 @@ from core.db.engine import SessionLocal from core.db.models import AdminMcpServer -from core.services.edition_mcp import edition_retired_builtin_mcp_server_ids +from core.capabilities.mcp.edition import edition_retired_builtin_mcp_server_ids logger = logging.getLogger(__name__) _CACHE_TTL = 30.0 -def _rewrite_builtin_mcp_host(url: str) -> str: +def rewrite_builtin_mcp_host(url: str) -> str: """Point built-in MCP URLs at the configured host. Plugin-provided MCP servers (automation_task / skill_manager / site_publish) hardcode the Docker service name in their manifests - (``http://mcp:/mcp/``). That host only resolves inside the compose + (``http://mcp:/mcp``). That host only resolves inside the compose network, so in the no-Docker local profile (``MCP_HOST=127.0.0.1``) the plugin's tool calls would be unreachable. Rewrite **only** the literal ``mcp`` host to ``settings.server.mcp_host``: a no-op on compose (mcp_host == "mcp"), @@ -55,7 +55,8 @@ def _rewrite_builtin_mcp_host(url: str) -> str: port = {base: PORTS[sid] for sid, base in _BASE_PORTS.items()}.get(port, port) netloc = f"{host}:{port}" if port else host - return urlunparse(parts._replace(netloc=netloc)) + path = "/mcp" if parts.path == "/mcp/" else parts.path + return urlunparse(parts._replace(netloc=netloc, path=path)) except Exception: return url @@ -93,7 +94,6 @@ def get_all_servers( "env": {...}, # merged: env_inherit from OS + env_vars "url": "...", # for HTTP/SSE "headers": {...}, - "is_stable": True, } """ if not use_cache: @@ -170,7 +170,6 @@ def _row_to_config(self, row: AdminMcpServer) -> dict: """Convert a DB row to the config dict format.""" cfg: Dict[str, Any] = { "transport": row.transport, - "is_stable": row.is_stable, # 来源元数据（桌面双端桥按组件基名做 logical 去重时消费； # 连接层不读这两个键）。 "source_plugin": row.source_plugin, @@ -184,22 +183,19 @@ def _row_to_config(self, row: AdminMcpServer) -> dict: plaintext_headers: Dict[str, str] = {} if row.headers: - from core.services.mcp_management_service import decrypt_mcp_headers + from core.capabilities.mcp.management import decrypt_mcp_headers plaintext_headers = decrypt_mcp_headers(row.headers) # HTTP/SSE fields. Marketplace credentials that belong in the URL are # decrypted and materialized only here, immediately before connecting. if row.transport in ("streamable_http", "sse"): - from core.services.mcp_management_service import ( - materialize_mcp_http_connection, - mcp_oauth_bundle_storage_key, - ) + from core.capabilities.mcp.management import materialize_mcp_http_connection, mcp_oauth_bundle_storage_key runtime_url, runtime_headers = materialize_mcp_http_connection( row.url or "", plaintext_headers ) - cfg["url"] = _rewrite_builtin_mcp_host(runtime_url) + cfg["url"] = rewrite_builtin_mcp_host(runtime_url) if runtime_headers: cfg["headers"] = runtime_headers oauth_raw = plaintext_headers.get(mcp_oauth_bundle_storage_key()) @@ -207,10 +203,7 @@ def _row_to_config(self, row: AdminMcpServer) -> dict: try: import json - from core.services.mcp_oauth_service import ( - OAuthBundleStorage, - build_oauth_provider, - ) + from core.capabilities.mcp.oauth_storage import OAuthBundleStorage, build_oauth_provider oauth_bundle = json.loads(oauth_raw) storage = OAuthBundleStorage(oauth_bundle, server_id=row.server_id) @@ -265,22 +258,24 @@ def _build_env(self, row: AdminMcpServer) -> Dict[str, str]: # Phase 3: apply DB-driven overlays (model config, system config) try: - from core.services.model_config import ModelConfigService + from core.models.config import ModelConfigService overlay = ModelConfigService.get_instance().get_mcp_env_overlay() if overlay: env.update(overlay) - except Exception: - pass + except Exception as fallback_error: + from core.infra.failures import record_best_effort_failure + record_best_effort_failure('core.capabilities.mcp.service:_build_env', fallback_error) try: - from core.services.system_config import SystemConfigService + from core.config.store.system_config import SystemConfigService svc_overlay = SystemConfigService.get_instance().get_service_env_overlay() if svc_overlay: env.update(svc_overlay) - except Exception: - pass + except Exception as fallback_error: + from core.infra.failures import record_best_effort_failure + record_best_effort_failure('core.capabilities.mcp.service:_build_env', fallback_error) return env @@ -306,7 +301,7 @@ def get_owned_servers( if strict: raise logger.warning("[mcp_service] Failed to load owned servers for %s: %s", user_id, exc) - from core.capabilities.local_plugin_runtime import configs + from core.harness.registry.execution_catalog import local_plugin_configs as configs owned.update(configs(user_id, enabled_only=enabled_only)) return owned @@ -315,11 +310,6 @@ def get_server(self, server_id: str) -> Optional[dict]: servers = self.get_all_servers(enabled_only=False) return servers.get(server_id) - def get_stable_server_ids(self) -> Set[str]: - """Return set of server_ids where is_stable=True.""" - servers = self.get_all_servers(enabled_only=True) - return {k for k, v in servers.items() if v.get("is_stable")} - def invalidate_cache(self) -> None: """Clear cache so next call re-reads from DB.""" with self._lock: @@ -348,13 +338,13 @@ def get_all_rows(self) -> List[AdminMcpServer]: # has since deleted, and it is a no-op on any DB that already has the catalog. # # The URL is templated per-server from ``extensions.mcp_servers._ports`` + ``settings.server. -# mcp_host`` at seed time, so compose gets ``http://mcp:/mcp/`` and local -# gets ``http://127.0.0.1:/mcp/`` from the same definition. Only the +# mcp_host`` at seed time, so compose gets ``http://mcp:/mcp`` and local +# gets ``http://127.0.0.1:/mcp`` from the same definition. Only the # built-in servers the ``mcp`` container / ``_launcher`` actually serves are # included; plugin-provided servers (automation_task / skill_manager / # site_publish) arrive via plugin install, and external-service ones (db_query / # es_query) need their own containers, so neither belongs in the base seed. -from core.services.mcp_builtin_specs import BUILTIN_MCP_SERVERS +from core.capabilities.mcp.builtin_specs import BUILTIN_MCP_SERVERS # Built-ins retired from the packaged runtime. Keep this explicit tombstone # set separate from ``BUILTIN_MCP_SERVERS`` so upgraded create_all/SQLite @@ -444,7 +434,7 @@ def seed_builtin_mcp_servers_if_empty(db) -> List[str]: it bootstraps a fresh ``create_all`` DB (local profile) without touching a DB that alembic already seeded and without resurrecting admin-deleted rows. """ - from core.config.settings import settings + from core.config.mcp_config import mcp_http_url from extensions.mcp_servers._ports import PORTS existing = ( @@ -458,7 +448,6 @@ def seed_builtin_mcp_servers_if_empty(db) -> List[str]: if existing is not None: return [] - host = settings.server.mcp_host seeded: List[str] = [] for spec in BUILTIN_MCP_SERVERS: sid = spec["server_id"] @@ -472,8 +461,7 @@ def seed_builtin_mcp_servers_if_empty(db) -> List[str]: description=spec.get("description", ""), user_intro=spec.get("user_intro"), transport="streamable_http", - url=f"http://{host}:{port}/mcp/", - is_stable=spec.get("is_stable", True), + url=mcp_http_url(sid), is_enabled=spec.get("is_enabled", True), sort_order=spec.get("sort_order", 0), icon=spec.get("icon"), diff --git a/src/backend/core/capabilities/mcp/tool_metadata.py b/src/backend/core/capabilities/mcp/tool_metadata.py new file mode 100644 index 00000000..0c1a220a --- /dev/null +++ b/src/backend/core/capabilities/mcp/tool_metadata.py @@ -0,0 +1,13 @@ +"""Capture the same public MCP schema for discovery, hashing and desktop delivery.""" +import hashlib +import json +from core.capabilities.manifest_protocol import public_tool_schema + + +def snapshot_tool(tool): + return public_tool_schema(tool.model_dump(by_alias=True, mode="json")) + + +def tool_snapshot_hash(tools): + normalized = sorted([public_tool_schema(t) for t in tools or []], key=lambda t: t["name"]) + return hashlib.sha256(json.dumps(normalized, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode()).hexdigest() diff --git a/src/backend/core/capabilities/mcp/tool_schema.py b/src/backend/core/capabilities/mcp/tool_schema.py new file mode 100644 index 00000000..b55f05fd --- /dev/null +++ b/src/backend/core/capabilities/mcp/tool_schema.py @@ -0,0 +1,20 @@ +"""Interpret server tool schema metadata without loading the SDK.""" +from typing import Any + +def has_usable_schema(tool: Any) -> bool: + """Whether a manifest tool entry carries the server's real ``inputSchema``. + + A tool the platform only knows by name and description (a plugin manifest + lists its tools for display, without schemas) must not be handed to a model: + the parameter object would be empty, so the model has nowhere to put its + arguments and every call arrives blank. + + The marker is whether a schema was captured at all, not whether it declares + parameters: a server is free to describe an argument-less tool as + ``{"type": "object"}``, while an entry that never reached the server carries + no ``inputSchema`` key, which normalizes to an empty dict. + """ + if not isinstance(tool, dict): + return False + schema = tool.get("inputSchema") or tool.get("input_schema") + return isinstance(schema, dict) and bool(schema) diff --git a/src/backend/core/services/mcp_tool_schema_backfill.py b/src/backend/core/capabilities/mcp/tool_schema_backfill.py similarity index 87% rename from src/backend/core/services/mcp_tool_schema_backfill.py rename to src/backend/core/capabilities/mcp/tool_schema_backfill.py index 7ecb8275..c09464d6 100644 --- a/src/backend/core/services/mcp_tool_schema_backfill.py +++ b/src/backend/core/capabilities/mcp/tool_schema_backfill.py @@ -27,7 +27,7 @@ def _needs_backfill(tools_json) -> bool: - from core.llm.mcp_manager import has_usable_schema + from core.capabilities.mcp.tool_schema import has_usable_schema if not isinstance(tools_json, list) or not tools_json: return False @@ -43,14 +43,14 @@ async def backfill_missing_tool_schemas(server_ids: Optional[List[str]] = None) """ from core.db.engine import SessionLocal from core.db.models import AdminMcpServer - from core.services.mcp_management_service import probe_mcp_connectivity + from core.capabilities.mcp.management import probe_mcp_connectivity - from core.capabilities import device_catalog + from core.harness.registry import device_catalog - from core.capabilities.paths import capabilities_enabled - if capabilities_enabled() and not server_ids: - from core.plugins.local.legacy_migration import migrate as migrate_plugins - from core.services.local_management_migration import migrate as migrate_skills + from core.harness.registry.run_capabilities import has_run_capabilities + if has_run_capabilities() and not server_ids: + from core.harness.registry.local_publication import migrate_local_plugins as migrate_plugins + from core.harness.registry.local_publication import migrate_local_skills as migrate_skills await asyncio.to_thread(migrate_plugins) await asyncio.to_thread(migrate_skills) @@ -63,7 +63,7 @@ async def backfill_missing_tool_schemas(server_ids: Optional[List[str]] = None) db = SessionLocal() try: - from core.services.manager_bundle_upgrade import refresh + from core.capabilities.plugins.management.bundle_upgrade import refresh refresh(db) query = db.query(AdminMcpServer).filter(AdminMcpServer.is_enabled.is_(True)) if server_ids: diff --git a/src/backend/core/capabilities/plugins/README.md b/src/backend/core/capabilities/plugins/README.md new file mode 100644 index 00000000..266e230f --- /dev/null +++ b/src/backend/core/capabilities/plugins/README.md @@ -0,0 +1,23 @@ +# Plugin capabilities + +This package owns plugin manifests, installation records, capability selection and UI contracts. + +| Package | Responsibility | +| --- | --- | +| `management/` | Install, enable, uninstall, inspect and publish database-backed plugins | +| `packaging/` | Discover files, extract archives and normalize manifests | +| `runtime/` | Resolve deferred capabilities and retain per-chat activation state | +| `ui/` | Validate UI declarations, project browser-safe data and authorize data queries | + +Framework-specific activation belongs to `core.llm.tools.plugin_activation`. +Device-local installation and migration belong to `features.desktop.local.plugins`. +Marketplace publication policy belongs to `features.marketplace`. + +Installation and packaging must not import the agent factory or SDK activation. +Packaging does not publish database records: archive extraction belongs to +`packaging/archive.py`, while installation persistence belongs to `management/`. + +Use explicit owner imports and patch the consuming interface in tests. The old +`core.plugins` and `core.services` modules are removed; no compatibility exports +are provided. Bundled resource paths are relative to the backend root in local +and Docker installations. diff --git a/src/backend/core/plugins/__init__.py b/src/backend/core/capabilities/plugins/__init__.py similarity index 100% rename from src/backend/core/plugins/__init__.py rename to src/backend/core/capabilities/plugins/__init__.py diff --git a/src/backend/core/plugins/connections.py b/src/backend/core/capabilities/plugins/connections.py similarity index 83% rename from src/backend/core/plugins/connections.py rename to src/backend/core/capabilities/plugins/connections.py index 6641ab0b..984c7820 100644 --- a/src/backend/core/plugins/connections.py +++ b/src/backend/core/capabilities/plugins/connections.py @@ -10,6 +10,6 @@ def connection_type(value: Any) -> str | None: def manifest_connection(manifest: dict) -> str | None: - from core.plugins.packaging.importer import _ext_or_top, manifest_extensions + from core.capabilities.plugins.packaging.importer import ext_or_top as _ext_or_top, manifest_extensions return connection_type(_ext_or_top(manifest, manifest_extensions(manifest), "connection")) diff --git a/src/backend/core/capabilities/plugins/management/__init__.py b/src/backend/core/capabilities/plugins/management/__init__.py new file mode 100644 index 00000000..656d1cb6 --- /dev/null +++ b/src/backend/core/capabilities/plugins/management/__init__.py @@ -0,0 +1,43 @@ +"""Public plugin lifecycle API; implementations follow domain ownership.""" + +from core.capabilities.plugins.management.admin_config import get_plugin_admin_config, set_plugin_admin_config +from core.capabilities.plugins.management.details import exclude_market_skill, get_market_skill_exclusions, get_plugin_detail +from core.capabilities.plugins.management.installation import ensure_default_plugins_bootstrapped, import_plugin, import_plugin_from_zip, install_plugin +from core.capabilities.plugins.management.lifecycle import set_installed_plugin_meta, set_plugin_component_enabled, set_plugin_component_enabled_for_user, set_plugin_enabled, set_plugin_enabled_for_user, uninstall_plugin +from core.capabilities.plugins.management.packages import delete_market_package, publish_plugin_zip_to_market +from core.capabilities.plugins.management.queries import get_installed_detail, list_installed +from core.capabilities.plugins.packaging.definitions import BUILTIN_PLUGIN_MARKET_META, DEFAULT_BOOTSTRAP_MARKER_ID, DEFAULT_BOOTSTRAP_PLUGIN_SLUGS, LOCAL_BOOTSTRAP_MARKER_NAME, MAX_ICON_LEN, MAX_ICON_URL_LEN, MAX_ZIP_BYTES, PLUGIN_BUNDLES_DIR, PLUGIN_MARKET_META_BLOCK_ID, PLUGIN_SOURCE_DIRS +from core.capabilities.plugins.packaging.sources import builtin_plugin_component_ids + +__all__ = [ + "builtin_plugin_component_ids", + "list_installed", + "get_installed_detail", + "get_plugin_admin_config", + "set_plugin_admin_config", + "get_market_skill_exclusions", + "get_plugin_detail", + "exclude_market_skill", + "install_plugin", + "ensure_default_plugins_bootstrapped", + "import_plugin", + "import_plugin_from_zip", + "publish_plugin_zip_to_market", + "delete_market_package", + "uninstall_plugin", + "set_plugin_enabled", + "set_plugin_component_enabled", + "set_plugin_enabled_for_user", + "set_plugin_component_enabled_for_user", + "set_installed_plugin_meta", + "PLUGIN_BUNDLES_DIR", + "PLUGIN_SOURCE_DIRS", + "MAX_ZIP_BYTES", + "DEFAULT_BOOTSTRAP_PLUGIN_SLUGS", + "DEFAULT_BOOTSTRAP_MARKER_ID", + "LOCAL_BOOTSTRAP_MARKER_NAME", + "PLUGIN_MARKET_META_BLOCK_ID", + "BUILTIN_PLUGIN_MARKET_META", + "MAX_ICON_LEN", + "MAX_ICON_URL_LEN", +] diff --git a/src/backend/core/plugins/management/admin_config.py b/src/backend/core/capabilities/plugins/management/admin_config.py similarity index 88% rename from src/backend/core/plugins/management/admin_config.py rename to src/backend/core/capabilities/plugins/management/admin_config.py index 4b01deb8..68facf01 100644 --- a/src/backend/core/plugins/management/admin_config.py +++ b/src/backend/core/capabilities/plugins/management/admin_config.py @@ -1,4 +1,4 @@ -"""Plugin admin config responsibilities.""" +"""Read and write a plugin's admin-level configuration.""" from __future__ import annotations @@ -6,15 +6,15 @@ from typing import Any, Dict, List, Optional from core.infra.exceptions import BadRequestError, ResourceNotFoundError -from core.plugins.packaging import sources as plugin_sources -from core.plugins.packaging.importer import _ext_or_top, manifest_extensions, normalize_plugin_dir +from core.capabilities.plugins.packaging import sources as plugin_sources +from core.capabilities.plugins.packaging.importer import ext_or_top as _ext_or_top, manifest_extensions, normalize_plugin_dir logger = logging.getLogger(__name__) def _admin_config_for_slug(slug: str) -> Optional[Dict[str, Any]]: """Locate the plugin bundle by slug and read its admin_config declaration; None if absent.""" - plugin_dir = plugin_sources._resolve_plugin_dir(slug) + plugin_dir = plugin_sources.resolve_plugin_dir(slug) if plugin_dir is None: return None try: @@ -23,14 +23,14 @@ def _admin_config_for_slug(slug: str) -> Optional[Dict[str, Any]]: return None -def _connection_for_slug(slug: str) -> Optional[str]: +def connection_for_slug(slug: str) -> Optional[str]: """Read a plugin's account connection type by slug (e.g. dingtalk / lark); None if absent. Like admin_config, read live from the bundle at detail-view time, not stored in a DB column — imported plugins (whose bundle no longer exists) naturally return None without affecting display. """ - plugin_dir = plugin_sources._resolve_plugin_dir(slug) + plugin_dir = plugin_sources.resolve_plugin_dir(slug) if plugin_dir is None: return None try: @@ -45,7 +45,7 @@ def _connection_for_slug(slug: str) -> Optional[str]: def _has_admin_config_for_slug(slug: str) -> bool: """Lightweight check for whether a plugin declares admin_config (reads only plugin.json, no full normalize).""" - plugin_dir = plugin_sources._resolve_plugin_dir(slug) + plugin_dir = plugin_sources.resolve_plugin_dir(slug) if plugin_dir is None: return False try: @@ -82,7 +82,7 @@ def _admin_config_view( """ if not admin_config: return None - from core.services.system_config import SystemConfigService + from core.config.store.system_config import SystemConfigService svc = SystemConfigService.get_instance() fields: List[Dict[str, Any]] = [] @@ -133,7 +133,7 @@ def set_plugin_admin_config( items = [{"key": str(k), "value": v} for k, v in (values or {}).items() if k in allowed] if not items: raise BadRequestError(message="没有可写入的配置项（key 不属于该插件）") - from core.services.system_config import SystemConfigService + from core.config.store.system_config import SystemConfigService SystemConfigService.get_instance().bulk_set(items, updated_by=updated_by) return _admin_config_view(ac, with_values=True) diff --git a/src/backend/core/services/manager_bundle_upgrade.py b/src/backend/core/capabilities/plugins/management/bundle_upgrade.py similarity index 92% rename from src/backend/core/services/manager_bundle_upgrade.py rename to src/backend/core/capabilities/plugins/management/bundle_upgrade.py index 56aed7a6..f75bc2c3 100644 --- a/src/backend/core/services/manager_bundle_upgrade.py +++ b/src/backend/core/capabilities/plugins/management/bundle_upgrade.py @@ -1,15 +1,15 @@ """Upgrade installed official manager bundles; retain a rollback snapshot per owner.""" from core.db.models import AdminMcpServer, AdminSkill, ContentBlock, InstalledPlugin -from core.services.management_contract import MANAGERS +from core.capabilities.plugins.management.contract import MANAGERS from extensions.paths import PLUGINS_DIR def refresh(db): - from core.capabilities.paths import capabilities_enabled - if capabilities_enabled(): + from core.harness.registry.run_capabilities import has_run_capabilities + if has_run_capabilities(): return 0 # Device legacy packages are handled by local_legacy_plugin_migration. - from core.plugins import management as ps + from core.capabilities.plugins import management as ps root = PLUGINS_DIR / "marketplace" count = 0 for row in db.query(InstalledPlugin).filter(InstalledPlugin.slug.in_(list(MANAGERS))).all(): diff --git a/src/backend/core/services/cloud_management.py b/src/backend/core/capabilities/plugins/management/cloud.py similarity index 78% rename from src/backend/core/services/cloud_management.py rename to src/backend/core/capabilities/plugins/management/cloud.py index 443fd23b..fa464472 100644 --- a/src/backend/core/services/cloud_management.py +++ b/src/backend/core/capabilities/plugins/management/cloud.py @@ -1,6 +1,6 @@ """Cloud adapters for the plugin-owned management contract.""" -from core.plugins.packaging import sources as plugin_sources +from core.capabilities.plugins.packaging import sources as plugin_sources from contextlib import contextmanager import hashlib import json @@ -9,11 +9,11 @@ from sqlalchemy import text from pathlib import Path from core.capabilities import archive -from core.agent_skills.binary_files import encode_upload -from core.services.management_package import snapshot +from core.capabilities.skills.binary_files import encode_upload +from core.capabilities.plugins.management.package import snapshot from core.db.engine import SessionLocal from core.db.models import AdminSkill, AdminMcpServer, InstalledPlugin, Artifact, ContentBlock -from core.services import capability_workcopies as copies +from core.harness.registry.capability_revisions import capability_revisions @contextmanager @@ -27,7 +27,7 @@ def artifact_package(user_id, source): if artifact.size_bytes > archive.MAX_TOTAL_BYTES: raise ValueError("artifact package too large") key = artifact.storage_key - from core.storage import get_storage + from core.files.storage import get_storage data = get_storage().download_bytes(key) if len(data) > archive.MAX_TOTAL_BYTES: @@ -67,7 +67,7 @@ def _revision(db, kind, row, snap): def _owned(db, user, kind, iid): if not user: raise PermissionError("authentication required") - row = copies._model(db, kind, iid) + row = capability_revisions().model(db, kind, iid) if row is None or row.owner_user_id != user: raise PermissionError("installation does not exist or is not owned by current user") if kind == "skill" and row.source_plugin: @@ -77,7 +77,7 @@ def _owned(db, user, kind, iid): def _view(db, user, kind, row): iid = row.skill_id if kind == "skill" else row.install_id - snap = copies._snapshot(db, user, kind, iid)[0] + snap = capability_revisions().snapshot_in_session(db, user, kind, iid)[0] revision = _revision(db, kind, row, snap) return { "ok": True, @@ -109,13 +109,13 @@ def list_installed(user_id, kind): def _skill_files(root, user_id, key=None): - from core.agent_skills.registry import _split_frontmatter, _load_skill_metadata_from_str - from core.services.marketplace_service import compute_install_id, _rewrite_frontmatter_name + from core.capabilities.skills.registry import split_frontmatter, load_skill_metadata_from_str + from core.capabilities.skills.installation import compute_install_id as compute_install_id, rewrite_frontmatter_name as _rewrite_frontmatter_name files = {n: encode_upload(n, p.read_bytes()) for n, p in archive.iter_files(root)} md = files.get("SKILL.md", "") - fm, _ = _split_frontmatter(md) - _load_skill_metadata_from_str(md, str(fm.get("name") or "")) + fm, _ = split_frontmatter(md) + load_skill_metadata_from_str(md, str(fm.get("name") or "")) key = key or compute_install_id(str(fm["name"]), user_id) files["SKILL.md"] = _rewrite_frontmatter_name(md, key) return key, files @@ -125,12 +125,12 @@ def install(user_id, kind, source): with artifact_package(user_id, source) as root: if kind == "skill": key, files = _skill_files(root, user_id) - before = copies.snapshot(user_id, kind, key) + before = capability_revisions().snapshot(user_id, kind, key) if before["exists"]: if before["files"] != files: raise ValueError("name_conflict: use update_skill") return get(user_id, kind, key) - result = copies.commit( + result = capability_revisions().commit( user_id, kind, key, @@ -139,14 +139,14 @@ def install(user_id, kind, source): if not result["applied"]: raise ValueError("package could not be activated") else: - from core.plugins import management as ps - from core.plugins.packaging.importer import normalize_plugin_dir + from core.capabilities.plugins import management as ps + from core.capabilities.plugins.packaging.importer import normalize_plugin_dir np = normalize_plugin_dir(root) - key = plugin_sources._make_plugin_install_id(np.slug, user_id) + key = plugin_sources.make_plugin_install_id(np.slug, user_id) with SessionLocal() as db: _permission(db, user_id, "can_import_plugin") - if copies._model(db, kind, key) is not None: + if capability_revisions().model(db, kind, key) is not None: raise ValueError("name_conflict: use update_plugin") ps.import_plugin(db, root, owner_user_id=user_id, created_by=user_id) return get(user_id, kind, key) @@ -164,12 +164,12 @@ def update(user_id, kind, install_id, source, expected_revision): with SessionLocal() as db: _write_lock(db) row = _owned(db, user_id, kind, install_id) - before = copies._snapshot(db, user_id, kind, install_id)[0] + before = capability_revisions().snapshot_in_session(db, user_id, kind, install_id)[0] if not expected_revision or _revision(db, kind, row, before) != expected_revision: raise ValueError("revision_conflict") if kind == "plugin": - from core.plugins import management as ps - from core.plugins.packaging.importer import normalize_plugin_dir + from core.capabilities.plugins import management as ps + from core.capabilities.plugins.packaging.importer import normalize_plugin_dir _permission(db, user_id, "can_import_plugin") if normalize_plugin_dir(root).slug != row.slug: @@ -178,7 +178,7 @@ def update(user_id, kind, install_id, source, expected_revision): else: key, files = _skill_files(root, user_id, install_id) if kind == "skill": - copies.commit( + capability_revisions().commit( user_id, kind, key, @@ -187,7 +187,7 @@ def update(user_id, kind, install_id, source, expected_revision): "expected_revision": expected_revision, "expected_model_version": before["model_version"], "request_id": hashlib.sha256( - (user_id + key + expected_revision + copies.revision(files)).encode() + (user_id + key + expected_revision + capability_revisions().revision(files)).encode() ).hexdigest(), }, ) @@ -198,21 +198,21 @@ def uninstall(user_id, kind, install_id, expected_revision): with SessionLocal() as db: _write_lock(db) row = _owned(db, user_id, kind, install_id) - before = copies._snapshot(db, user_id, kind, install_id)[0] + before = capability_revisions().snapshot_in_session(db, user_id, kind, install_id)[0] if not expected_revision or _revision(db, kind, row, before) != expected_revision: raise ValueError("revision_conflict") _permission(db, user_id, "can_add_skill" if kind == "skill" else "can_import_plugin") - record = db.get(ContentBlock, copies._id("cap-cloud-copy:", user_id, kind, install_id)) + record = db.get(ContentBlock, capability_revisions().record_id("cap-cloud-copy:", user_id, kind, install_id)) if record: db.delete(record) if kind == "plugin": - from core.plugins.management import uninstall_plugin + from core.capabilities.plugins.management import uninstall_plugin uninstall_plugin(db, install_id, owner_user_id=user_id) else: db.delete(row) db.commit() - from core.agent_skills.cache_refresh import refresh_skill_caches + from core.capabilities.skills.cache_refresh import refresh_skill_caches refresh_skill_caches() return {"ok": True, "install_id": install_id, "action": "uninstalled", "source": "cloud"} diff --git a/src/backend/core/plugins/management/components.py b/src/backend/core/capabilities/plugins/management/components.py similarity index 88% rename from src/backend/core/plugins/management/components.py rename to src/backend/core/capabilities/plugins/management/components.py index 2ec15c08..a0a66e32 100644 --- a/src/backend/core/plugins/management/components.py +++ b/src/backend/core/capabilities/plugins/management/components.py @@ -1,4 +1,4 @@ -"""Plugin components responsibilities.""" +"""Upsert a plugin's skills and MCP servers as admin catalog rows.""" from __future__ import annotations @@ -6,24 +6,24 @@ import re from typing import Any, Dict, List, Optional -from core.agent_skills.binary_files import is_binary_value -from core.agent_skills.deps_detector import detect_dependencies -from core.agent_skills.registry import _load_skill_metadata_from_str +from core.capabilities.skills.binary_files import is_binary_value +from core.capabilities.skills.deps.detector import detect_dependencies +from core.capabilities.skills.registry import load_skill_metadata_from_str from core.db.models import AdminMcpServer, AdminSkill from core.infra.exceptions import BadRequestError from core.infra.time import utc_now -from core.ontology.build_validator import ensure_ontology_build_valid -from core.plugins.packaging import sources as plugin_sources -from core.plugins.packaging.importer import _rewrite_path_vars -from core.plugins.packaging.models import NormalizedSkill -from core.services.marketplace_service import _inject_secrets, _rewrite_frontmatter_name +from core.harness.registry.build_governance import ensure_ontology_build_valid +from core.capabilities.plugins.packaging import sources as plugin_sources +from core.capabilities.plugins.packaging.importer import _rewrite_path_vars +from core.capabilities.plugins.packaging.models import NormalizedSkill +from core.capabilities.skills.installation import inject_secrets as _inject_secrets, rewrite_frontmatter_name as _rewrite_frontmatter_name from sqlalchemy.orm import Session from sqlalchemy.orm.attributes import flag_modified logger = logging.getLogger(__name__) -def _merge_tool_metadata(stored: Any, manifest: List[Dict[str, Any]]) -> List[Dict[str, Any]]: +def merge_tool_metadata(stored: Any, manifest: List[Dict[str, Any]]) -> List[Dict[str, Any]]: """Keep captured tool schemas while refreshing manifest-owned display text. ``tools_json`` is the schema of record: probing the running MCP server fills in @@ -91,7 +91,7 @@ def _rewrite_sibling_refs(text: str, sibling_ids: Dict[str, str]) -> str: return text -def _apply_skill( +def apply_skill( db: Session, sk: NormalizedSkill, *, @@ -104,7 +104,7 @@ def _apply_skill( sibling_ids: Optional[Dict[str, str]] = None, ) -> str: """Upsert one normalized skill as an AdminSkill (tagged with source_plugin). Returns the skill_id.""" - skill_id = plugin_sources._make_skill_id(slug, sk.name, owner_user_id) + skill_id = plugin_sources.make_skill_id(slug, sk.name, owner_user_id) sandbox_dir = f"/workspace/skills/{skill_id}" # Path-variable rewrite: SKILL.md body + text attachments (binaries untouched) @@ -124,7 +124,7 @@ def _apply_skill( # Display name comes from the original frontmatter name (before rewriting into the namespaced id), falling back to the skill directory name. try: - _display_name = _load_skill_metadata_from_str(content, skill_id).name + _display_name = load_skill_metadata_from_str(content, skill_id).name except Exception: # noqa: BLE001 _display_name = None @@ -135,7 +135,7 @@ def _apply_skill( content = _inject_secrets(content, extra_files, required_secrets, secrets) try: - meta = _load_skill_metadata_from_str(content, skill_id) + meta = load_skill_metadata_from_str(content, skill_id) except Exception as exc: # noqa: BLE001 raise BadRequestError(message=f"技能 {sk.name!r} 的 SKILL.md 不合法：{exc}") @@ -190,7 +190,7 @@ def _apply_mcp( stdio (needs_runtime) is force-disabled even when enabled is requested (it can only be enabled once the runtime is fully in place). """ - server_id = plugin_sources._make_server_id(slug, mc.name, owner_user_id) + server_id = plugin_sources.make_server_id(slug, mc.name, owner_user_id) effective_enabled = bool(enabled) and not mc.needs_runtime tools_meta = [item for item in list(getattr(mc, "tools", None) or []) if isinstance(item, dict)] if validate_ontology_build: @@ -235,7 +235,7 @@ def _apply_mcp( # the desktop's only source of tool parameters. Overwriting them with the # manifest list would strip every parameter, so merge by name instead: # manifest text wins, captured schemas survive. - fields["tools_json"] = _merge_tool_metadata(existing.tools_json, tools_meta) + fields["tools_json"] = merge_tool_metadata(existing.tools_json, tools_meta) for key, val in fields.items(): setattr(existing, key, val) for col in ("args", "env_vars", "headers", "tools_json"): diff --git a/src/backend/core/services/management_contract.py b/src/backend/core/capabilities/plugins/management/contract.py similarity index 100% rename from src/backend/core/services/management_contract.py rename to src/backend/core/capabilities/plugins/management/contract.py diff --git a/src/backend/core/plugins/management/defaults.py b/src/backend/core/capabilities/plugins/management/defaults.py similarity index 93% rename from src/backend/core/plugins/management/defaults.py rename to src/backend/core/capabilities/plugins/management/defaults.py index de1e397e..518fe639 100644 --- a/src/backend/core/plugins/management/defaults.py +++ b/src/backend/core/capabilities/plugins/management/defaults.py @@ -1,7 +1,7 @@ """Add shared defaults without replaying an edition's earlier onboarding.""" from core.db.models import ContentBlock, InstalledPlugin -from core.plugins.management.installation import install_plugin +from core.capabilities.plugins.management.installation import install_plugin from sqlalchemy.orm import Session VISUALIZE_BOOTSTRAP_MARKER_ID = "visualize_plugin_bootstrap_v1" diff --git a/src/backend/core/plugins/management/details.py b/src/backend/core/capabilities/plugins/management/details.py similarity index 58% rename from src/backend/core/plugins/management/details.py rename to src/backend/core/capabilities/plugins/management/details.py index 09ef4d71..15288a9c 100644 --- a/src/backend/core/plugins/management/details.py +++ b/src/backend/core/capabilities/plugins/management/details.py @@ -1,58 +1,29 @@ -"""Plugin details responsibilities.""" +"""Marketplace plugin details and per-skill exclusions.""" from __future__ import annotations -import base64 import logging from typing import Any, Dict, List, Optional -from core.db.models import PluginMarketSkillExclusion -from core.infra.exceptions import BadRequestError, ResourceNotFoundError -from core.plugins.management import admin_config as plugin_admin_config -from core.plugins.management import market as plugin_market -from core.plugins.management import packages as plugin_packages -from core.plugins.packaging import sources as plugin_sources -from core.plugins.packaging.archive import _extract_plugin_zip -from core.plugins.packaging.importer import normalize_plugin_dir -from core.plugins.packaging.models import NormalizedPlugin, NormalizedSkill -from core.services.marketplace_service import _strip_frontmatter +from core.harness.registry.marketplace_catalog import marketplace_catalog +from core.infra.exceptions import BadRequestError +from core.capabilities.plugins.management import admin_config as plugin_admin_config +from core.capabilities.plugins.packaging.presentation import overlay_market_meta +from core.capabilities.plugins.packaging.models import NormalizedPlugin, NormalizedSkill +from core.capabilities.skills.installation import strip_frontmatter as _strip_frontmatter from sqlalchemy.orm import Session logger = logging.getLogger(__name__) def _normalize_market_plugin(slug: str, db: Optional[Session]) -> NormalizedPlugin: - """Fetch and normalize a plugin by slug: filesystem preset bundle first, DB-published package as fallback.""" - from core.services.marketplace_version_store import active, decode - from core.services.marketplace_version_sources import normalized_plugin - if plugin_sources._resolve_plugin_dir(slug) is None and ( - db is None or plugin_packages._market_row(db, slug) is None - ): - raise ResourceNotFoundError("plugin", slug) - snapshot = active(db, "plugin", slug) - if snapshot: - return normalized_plugin(decode(snapshot)) - plugin_dir = plugin_sources._resolve_plugin_dir(slug) - if plugin_dir is not None: - return normalize_plugin_dir(plugin_dir) - if db is not None: - row = plugin_packages._market_row(db, slug) - if row is not None: - with _extract_plugin_zip(base64.b64decode(row.package_b64)) as plugin_root: - return normalize_plugin_dir(plugin_root) - raise ResourceNotFoundError("plugin", slug) + from core.harness.registry.package_sources import package_source + return package_source("marketplace").normalize(db, slug) def get_market_skill_exclusions(db: Optional[Session], slug: str) -> set: """The set of skill names the admin has "removed" from the marketplace for this plugin.""" - if db is None: - return set() - return { - row[0] - for row in db.query(PluginMarketSkillExclusion.skill_name) - .filter(PluginMarketSkillExclusion.slug == slug) - .all() - } + return marketplace_catalog().excluded_skills(db, slug) def get_plugin_detail(slug: str, db: Optional[Session] = None) -> Dict[str, Any]: @@ -64,8 +35,8 @@ def get_plugin_detail(slug: str, db: Optional[Session] = None) -> Dict[str, Any] detail. """ np = _normalize_market_plugin(slug, db) - detail = _normalized_to_detail(np, excluded=get_market_skill_exclusions(db, slug)) - plugin_market._overlay_market_meta(detail, plugin_market.resolve_market_meta(db, slug)) + detail = normalized_to_detail(np, excluded=get_market_skill_exclusions(db, slug)) + overlay_market_meta(detail, marketplace_catalog().plugin_metadata(db, slug)) return detail @@ -81,35 +52,26 @@ def exclude_market_skill( raw_names = {s.name for s in _normalize_market_plugin(slug, db).skills} if skill_name not in raw_names: raise BadRequestError(message=f"技能 {skill_name!r} 不属于插件 {slug!r}") - existing = ( - db.query(PluginMarketSkillExclusion) - .filter( - PluginMarketSkillExclusion.slug == slug, - PluginMarketSkillExclusion.skill_name == skill_name, - ) - .first() - ) - if existing is None: - db.add(PluginMarketSkillExclusion(slug=slug, skill_name=skill_name, created_by=created_by)) - db.commit() + if marketplace_catalog().exclude_skill(db, slug, skill_name, created_by=created_by): logger.info("plugin_market_skill_excluded: slug=%s skill=%s", slug, skill_name) return {"slug": slug, "skill_name": skill_name, "excluded": True} def _skill_component_preview(sk: NormalizedSkill) -> Dict[str, Any]: """Preview component for a not-yet-installed builtin/external skill: includes body instructions + file list for pre-install inspection.""" - from core.agent_skills.registry import _split_frontmatter + from core.capabilities.skills.registry import split_frontmatter desc = "" tags: List[str] = [] try: - fm, _ = _split_frontmatter(sk.skill_content or "") + fm, _ = split_frontmatter(sk.skill_content or "") desc = (fm.get("description") or "").strip() raw_tags = fm.get("tags") or "" if isinstance(raw_tags, str) and raw_tags: tags = [x.strip() for x in raw_tags.replace("，", ",").split(",") if x.strip()] - except Exception: # noqa: BLE001 - pass + except Exception as fallback_error: # noqa: BLE001 + from core.infra.failures import record_best_effort_failure + record_best_effort_failure('core.capabilities.plugins.management.details:_skill_component_preview', fallback_error) files = sorted(k for k in (sk.extra_files or {}).keys() if k != "secrets.json") return { "skill_id": sk.name, @@ -124,7 +86,7 @@ def _skill_component_preview(sk: NormalizedSkill) -> Dict[str, Any]: } -def _normalized_to_detail( +def normalized_to_detail( np: NormalizedPlugin, *, excluded: Optional[set] = None ) -> Dict[str, Any]: excluded = excluded or set() diff --git a/src/backend/core/services/management_errors.py b/src/backend/core/capabilities/plugins/management/errors.py similarity index 100% rename from src/backend/core/services/management_errors.py rename to src/backend/core/capabilities/plugins/management/errors.py diff --git a/src/backend/core/plugins/management/installation.py b/src/backend/core/capabilities/plugins/management/installation.py similarity index 81% rename from src/backend/core/plugins/management/installation.py rename to src/backend/core/capabilities/plugins/management/installation.py index bb8ba224..74993792 100644 --- a/src/backend/core/plugins/management/installation.py +++ b/src/backend/core/capabilities/plugins/management/installation.py @@ -1,4 +1,4 @@ -"""Plugin installation responsibilities.""" +"""Install plugins from the marketplace, a directory or a zip, and bootstrap the defaults.""" from __future__ import annotations @@ -10,24 +10,24 @@ from core.db.models import AdminMcpServer, AdminSkill, ContentBlock, InstalledPlugin from core.infra.exceptions import ResourceNotFoundError from core.infra.time import utc_now -from core.plugins.management import components as plugin_components -from core.plugins.management import details as plugin_details -from core.plugins.management import market as plugin_market -from core.plugins.management import packages as plugin_packages -from core.plugins.management import projection as plugin_projection -from core.plugins.packaging import definitions as plugin_definitions -from core.plugins.packaging import sources as plugin_sources -from core.plugins.packaging.archive import _extract_plugin_zip -from core.plugins.packaging.importer import normalize_plugin_dir -from core.plugins.packaging.models import NormalizedPlugin -from core.services.ontology_policy import resolve_plugin_import_ontology_validation +from core.capabilities.plugins.management import components as plugin_components +from core.capabilities.plugins.management import details as plugin_details +from core.harness.registry.marketplace_catalog import marketplace_catalog +from core.capabilities.plugins.management import packages as plugin_packages +from core.capabilities.plugins.management import projection as plugin_projection +from core.capabilities.plugins.packaging import definitions as plugin_definitions +from core.capabilities.plugins.packaging import sources as plugin_sources +from core.capabilities.plugins.packaging.archive import extract_plugin_zip +from core.capabilities.plugins.packaging.importer import normalize_plugin_dir +from core.capabilities.plugins.packaging.models import NormalizedPlugin +from core.harness.registry.build_governance import resolve_plugin_import_ontology_validation from sqlalchemy.orm import Session from sqlalchemy.orm.attributes import flag_modified logger = logging.getLogger(__name__) -def _apply_normalized( +def apply_normalized( db: Session, np: NormalizedPlugin, *, @@ -38,15 +38,15 @@ def _apply_normalized( approved_revision: Optional[str] = None, ) -> Dict[str, Any]: """Persist a NormalizedPlugin and return the installation result (including the import_report).""" - from core.applications.coordination import configuration - from core.applications.plugin_upgrades import prepare + from core.harness.registry.package_sources import package_upgrade_policy - configuration(db) + policy = package_upgrade_policy() + policy.lock_configuration(db) db.expire_all() - install_id = plugin_sources._make_plugin_install_id(np.slug, owner_user_id) + install_id = plugin_sources.make_plugin_install_id(np.slug, owner_user_id) existing = db.query(InstalledPlugin).filter(InstalledPlugin.install_id == install_id).first() try: - staged = prepare(db, existing, np, source, approved_revision) + staged = policy.prepare(db, existing, np, source, approved_revision) except BaseException: db.rollback() raise @@ -69,12 +69,12 @@ def _apply_normalized( # Sibling skill name → final skill_id mapping (used to rewrite inter-skill ../ relative references) sibling_ids = { - sk.name: plugin_sources._make_skill_id(np.slug, sk.name, owner_user_id) + sk.name: plugin_sources.make_skill_id(np.slug, sk.name, owner_user_id) for sk in install_skills } for sk in install_skills: - sid = plugin_components._apply_skill( + sid = plugin_components.apply_skill( db, sk, slug=np.slug, @@ -112,7 +112,7 @@ def _apply_normalized( import_report = {"imported": imported, "adapted": adapted, "dropped": np.dropped} if np.package_dir and np.ui: - from core.plugins.packaging.runtime_assets import retain + from core.capabilities.plugins.packaging.runtime_assets import retain import_report["package_revision"] = retain(db, Path(np.package_dir)) component_ids = {"skills": skill_ids, "mcp": server_ids, "prompts": []} @@ -120,10 +120,10 @@ def _apply_normalized( # A plugin update is a replacement of its declared component set. Remove # components that belonged to the previous version but no longer appear in # the new manifest; otherwise retired tools/skills remain visible forever. - stale_skill_ids = set(plugin_sources._component_keys(previous_component_ids, "skills")) - set( + stale_skill_ids = set(plugin_sources.component_keys(previous_component_ids, "skills")) - set( skill_ids ) - stale_server_ids = set(plugin_sources._component_keys(previous_component_ids, "mcp")) - set( + stale_server_ids = set(plugin_sources.component_keys(previous_component_ids, "mcp")) - set( server_ids ) owner_skill_filter = ( @@ -161,7 +161,7 @@ def _apply_normalized( now = utc_now() # Display metadata for the installed record: UI-configured market metadata # (DB override → builtin seed) wins over whatever the manifest carried. - market_meta = plugin_market.resolve_market_meta(db, np.slug) + market_meta = marketplace_catalog().plugin_metadata(db, np.slug) fields = dict( slug=np.slug, name=market_meta.get("display_name") or np.name, @@ -191,7 +191,7 @@ def _apply_normalized( action = "installed" db.commit() - plugin_projection._refresh_after_change(owner_user_id) + plugin_projection.refresh_after_change(owner_user_id) plugin_projection._project_plugin_to_store( { "install_id": install_id, @@ -210,7 +210,7 @@ def _apply_normalized( ) # 插件在本机需要的资产（如站点插件的建站模板）跟着插件走：装到哪台机器上，就在 # 哪台机器上铺。云端同步装的那条走 desktop_cloud_bundles 的发布钩子，同一张表。 - from core.plugins.local.assets import provision_for + from core.harness.registry.local_publication import provision_plugin_assets as provision_for provision_for(np.slug) logger.info( @@ -245,19 +245,19 @@ def install_plugin( created_by: Optional[str] = None, ) -> Dict[str, Any]: """Install a marketplace plugin package (filesystem default / marketplace, or a DB-published marketplace package).""" - from core.services.marketplace_version_store import active, archive, decode + from core.harness.registry.package_sources import package_source plugin_details.get_plugin_detail(slug, db) - snapshot = active(db, "plugin", slug) - if snapshot: - with _extract_plugin_zip(archive(decode(snapshot))) as root: + source_archive = package_source("marketplace").archive(db, slug) + if source_archive is not None: + with extract_plugin_zip(source_archive) as root: return import_plugin( db, root, owner_user_id=owner_user_id, secrets=secrets or {}, created_by=created_by ) - plugin_dir = plugin_sources._resolve_plugin_dir(slug) + plugin_dir = plugin_sources.resolve_plugin_dir(slug) if plugin_dir is not None: np = normalize_plugin_dir(plugin_dir) - return _apply_normalized( + return apply_normalized( db, np, owner_user_id=owner_user_id, @@ -266,9 +266,9 @@ def install_plugin( created_by=created_by, ) # DB-published marketplace package: extract the original zip and go through the same path as import - row = plugin_packages._market_row(db, slug) + row = plugin_packages.market_row(db, slug) if row is not None: - with _extract_plugin_zip(base64.b64decode(row.package_b64)) as plugin_root: + with extract_plugin_zip(base64.b64decode(row.package_b64)) as plugin_root: return import_plugin( db, plugin_root, @@ -333,7 +333,7 @@ def import_plugin( """Import an external plugin directory (native / Claude Code / Codex), persist it, and return the import_report.""" np = normalize_plugin_dir(plugin_dir) source = {"claude": "imported_claude", "codex": "imported_codex"}.get(np.kind, "builtin") - return _apply_normalized( + return apply_normalized( db, np, owner_user_id=owner_user_id, @@ -352,7 +352,7 @@ def import_plugin_from_zip( created_by: Optional[str] = None, ) -> Dict[str, Any]: """Import a plugin from uploaded zip bytes (extract → locate root → import_plugin). Shared by user and admin uploads.""" - with _extract_plugin_zip(raw) as plugin_root: + with extract_plugin_zip(raw) as plugin_root: return import_plugin( db, plugin_root, diff --git a/src/backend/core/plugins/management/lifecycle.py b/src/backend/core/capabilities/plugins/management/lifecycle.py similarity index 87% rename from src/backend/core/plugins/management/lifecycle.py rename to src/backend/core/capabilities/plugins/management/lifecycle.py index 0fc91fae..08ac1a2f 100644 --- a/src/backend/core/plugins/management/lifecycle.py +++ b/src/backend/core/capabilities/plugins/management/lifecycle.py @@ -1,4 +1,4 @@ -"""Plugin lifecycle responsibilities.""" +"""Uninstall plugins and toggle them or their components, globally or per user.""" from __future__ import annotations @@ -8,9 +8,9 @@ from core.db.models import AdminMcpServer, AdminSkill, InstalledPlugin from core.infra.exceptions import BadRequestError, ResourceNotFoundError from core.infra.time import utc_now -from core.plugins.management import market as plugin_market -from core.plugins.management import projection as plugin_projection -from core.plugins.packaging import sources as plugin_sources +from core.capabilities.plugins.packaging.presentation import validate_icon +from core.capabilities.plugins.management import projection as plugin_projection +from core.capabilities.plugins.packaging import sources as plugin_sources from sqlalchemy.orm import Session logger = logging.getLogger(__name__) @@ -56,7 +56,7 @@ def _owner_filter(model): db.delete(row) db.commit() plugin_projection._purge_sandbox_skill_files(removed_skill_ids) - plugin_projection._refresh_after_change(owner_user_id) + plugin_projection.refresh_after_change(owner_user_id) plugin_projection._remove_plugin_from_store(slug) logger.info("plugin_uninstalled: id=%s slug=%s skills=%d mcp=%d", install_id, slug, n_sk, n_mcp) return {"install_id": install_id, "slug": slug, "removed_skills": n_sk, "removed_mcp": n_mcp} @@ -76,12 +76,12 @@ def set_plugin_enabled( if row.owner_user_id != owner_user_id: raise BadRequestError(message="无权操作该插件") - from core.plugins.ui.workspace_enabled import set_enabled + from core.harness.registry.package_views import set_enabled set_enabled(db, row, enabled) cids = row.component_ids or {} - skill_ids = plugin_sources._component_keys(cids, "skills") - server_ids = plugin_sources._component_keys(cids, "mcp") + skill_ids = plugin_sources.component_keys(cids, "skills") + server_ids = plugin_sources.component_keys(cids, "mcp") if skill_ids: db.query(AdminSkill).filter(AdminSkill.skill_id.in_(skill_ids)).update( {AdminSkill.is_enabled: enabled}, synchronize_session=False @@ -91,7 +91,7 @@ def set_plugin_enabled( for srv in db.query(AdminMcpServer).filter(AdminMcpServer.server_id.in_(server_ids)).all(): srv.is_enabled = bool(enabled) and srv.transport != "stdio" db.commit() - plugin_projection._refresh_after_change(owner_user_id) + plugin_projection.refresh_after_change(owner_user_id) return {"install_id": install_id, "enabled": enabled} @@ -123,7 +123,7 @@ def set_plugin_component_enabled( cids = row.component_ids or {} if kind == "skill": - if component_id not in (plugin_sources._component_keys(cids, "skills")): + if component_id not in (plugin_sources.component_keys(cids, "skills")): raise BadRequestError(message="该技能不属于此插件") sk = db.query(AdminSkill).filter(AdminSkill.skill_id == component_id).first() if sk is None: @@ -131,7 +131,7 @@ def set_plugin_component_enabled( sk.is_enabled = bool(enabled) effective = sk.is_enabled else: - if component_id not in (plugin_sources._component_keys(cids, "mcp")): + if component_id not in (plugin_sources.component_keys(cids, "mcp")): raise BadRequestError(message="该 MCP 不属于此插件") srv = db.query(AdminMcpServer).filter(AdminMcpServer.server_id == component_id).first() if srv is None: @@ -140,7 +140,7 @@ def set_plugin_component_enabled( srv.is_enabled = bool(enabled) and srv.transport != "stdio" effective = srv.is_enabled db.commit() - plugin_projection._refresh_after_change(owner_user_id) + plugin_projection.refresh_after_change(owner_user_id) return { "install_id": install_id, "kind": kind, @@ -168,19 +168,19 @@ def set_plugin_enabled_for_user( if row.owner_user_id is not None and row.owner_user_id != user_id: raise BadRequestError(message="无权操作该插件") - from core.plugins.ui.workspace_enabled import set_enabled + from core.harness.registry.package_views import set_enabled set_enabled(db, row, enabled, user_id) - from core.services.catalog_service import CatalogService + from core.capabilities.catalog.overrides import CatalogService svc = CatalogService(db) cids = row.component_ids or {} - for sid in plugin_sources._component_keys(cids, "skills"): + for sid in plugin_sources.component_keys(cids, "skills"): svc.update_override(user_id, "skill", sid, enabled) - for sid in plugin_sources._component_keys(cids, "mcp"): + for sid in plugin_sources.component_keys(cids, "mcp"): svc.update_override(user_id, "mcp", sid, enabled) - plugin_projection._refresh_after_change(user_id) + plugin_projection.refresh_after_change(user_id) return {"install_id": install_id, "enabled": enabled} @@ -211,14 +211,14 @@ def set_plugin_component_enabled_for_user( raise BadRequestError(message="无权操作该插件") cids = row.component_ids or {} - pool = plugin_sources._component_keys(cids, "skills" if kind == "skill" else "mcp") + pool = plugin_sources.component_keys(cids, "skills" if kind == "skill" else "mcp") if component_id not in (pool or []): raise BadRequestError(message="该组件不属于此插件") - from core.services.catalog_service import CatalogService + from core.capabilities.catalog.overrides import CatalogService CatalogService(db).update_override(user_id, kind, component_id, enabled) - plugin_projection._refresh_after_change(user_id) + plugin_projection.refresh_after_change(user_id) return { "install_id": install_id, "kind": kind, @@ -253,7 +253,7 @@ def set_installed_plugin_meta( if category is not None: row.category = category.strip() if icon is not None: - row.icon = plugin_market._validate_icon(icon) or None + row.icon = validate_icon(icon) or None row.updated_at = utc_now() db.commit() return { diff --git a/src/backend/core/services/management_package.py b/src/backend/core/capabilities/plugins/management/package.py similarity index 91% rename from src/backend/core/services/management_package.py rename to src/backend/core/capabilities/plugins/management/package.py index dba71369..ebd2e4bc 100644 --- a/src/backend/core/services/management_package.py +++ b/src/backend/core/capabilities/plugins/management/package.py @@ -6,7 +6,7 @@ import tempfile import zipfile from core.capabilities import archive -from core.capabilities.errors import IntegrityFailed +from core.harness.capability_errors import IntegrityFailed def _tar_files(data): @@ -25,7 +25,7 @@ def _tar_files(data): if (len(files) >= archive.MAX_MEMBERS or entry.size > archive.MAX_MEMBER_BYTES or total > archive.MAX_TOTAL_BYTES): raise IntegrityFailed("package exceeds size limits") - archive._normalize_member(name) + archive.normalize_member(name) with package.extractfile(entry) as stream: files[name] = stream.read(entry.size + 1) if len(files[name]) != entry.size: @@ -76,8 +76,8 @@ def serialized(function): from functools import wraps @wraps(function) def run(*args, **kwargs): - from core.capabilities.lockfile import locked - from core.capabilities.paths import meta_root - with locked(meta_root() / "management.lock", timeout=35): + from core.infra.file_lock import locked + from core.harness.registry.local_publication import local_publication + with locked(local_publication().management_lock_path(), timeout=35): return function(*args, **kwargs) return run diff --git a/src/backend/core/plugins/management/packages.py b/src/backend/core/capabilities/plugins/management/packages.py similarity index 70% rename from src/backend/core/plugins/management/packages.py rename to src/backend/core/capabilities/plugins/management/packages.py index e92759d9..7289ad0b 100644 --- a/src/backend/core/plugins/management/packages.py +++ b/src/backend/core/capabilities/plugins/management/packages.py @@ -6,24 +6,22 @@ import logging from typing import Any, Dict, Optional -from core.db.models import PluginMarketPackage -from core.infra.exceptions import ResourceNotFoundError +from core.harness.registry.marketplace_catalog import marketplace_catalog from core.infra.time import utc_now -from core.plugins.packaging.archive import _extract_plugin_zip -from core.plugins.packaging.importer import normalize_plugin_dir +from core.capabilities.plugins.packaging.archive import extract_plugin_zip +from core.capabilities.plugins.packaging.importer import normalize_plugin_dir from sqlalchemy.orm import Session -from sqlalchemy.orm.attributes import flag_modified logger = logging.getLogger(__name__) -def _market_row(db: Session, slug: str) -> Optional[PluginMarketPackage]: +def market_row(db: Session, slug: str) -> Any: if not slug: return None - return db.query(PluginMarketPackage).filter(PluginMarketPackage.slug == slug).first() + return marketplace_catalog().market_package(db, slug) -def _market_meta_dict(row: PluginMarketPackage) -> Dict[str, Any]: +def market_meta_dict(row: Any) -> Dict[str, Any]: """DB marketplace package → marketplace list metadata isomorphic to filesystem bundles.""" return { "slug": row.slug, @@ -47,14 +45,13 @@ def publish_plugin_zip_to_market( Once published it appears in the plugin marketplace list and can be explicitly installed by admins/users; re-uploading the same slug updates it. """ - with _extract_plugin_zip(raw) as plugin_root: + with extract_plugin_zip(raw) as plugin_root: np = normalize_plugin_dir( plugin_root ) # parse/validate; invalid input raises immediately, nothing persisted package_b64 = base64.b64encode(raw).decode("ascii") has_admin_config = bool(np.admin_config and (np.admin_config.get("fields"))) now = utc_now() - existing = _market_row(db, np.slug) fields = dict( name=np.name, version=np.version, @@ -68,15 +65,7 @@ def publish_plugin_zip_to_market( package_b64=package_b64, updated_at=now, ) - if existing is not None: - for key, val in fields.items(): - setattr(existing, key, val) - flag_modified(existing, "required_secrets") - action = "updated" - else: - db.add(PluginMarketPackage(slug=np.slug, created_at=now, created_by=created_by, **fields)) - action = "published" - db.commit() + action = marketplace_catalog().publish_package(db, np.slug, fields, created_by=created_by, now=now) logger.info( "plugin_market_%s: slug=%s kind=%s skills=%d", action, np.slug, np.kind, len(np.skills) ) @@ -92,10 +81,6 @@ def publish_plugin_zip_to_market( def delete_market_package(db: Session, slug: str) -> Dict[str, Any]: """Remove an admin-uploaded DB marketplace package from the plugin marketplace (does not affect installed instances).""" - row = _market_row(db, slug) - if row is None: - raise ResourceNotFoundError("plugin_market_package", slug) - db.delete(row) - db.commit() + marketplace_catalog().delete_package(db, slug) logger.info("plugin_market_deleted: slug=%s", slug) return {"slug": slug, "deleted": True} diff --git a/src/backend/core/plugins/management/projection.py b/src/backend/core/capabilities/plugins/management/projection.py similarity index 63% rename from src/backend/core/plugins/management/projection.py rename to src/backend/core/capabilities/plugins/management/projection.py index 9ff26e66..d613c001 100644 --- a/src/backend/core/plugins/management/projection.py +++ b/src/backend/core/capabilities/plugins/management/projection.py @@ -1,40 +1,40 @@ -"""Plugin projection responsibilities.""" +"""Project plugin changes into the desktop store and refresh dependent caches.""" from __future__ import annotations import logging from typing import Any, Dict, Optional -from core.agent_skills.cache_refresh import refresh_skill_caches -from core.config.catalog_resolver import invalidate_capability_cache +from core.capabilities.skills.cache_refresh import refresh_skill_caches +from core.capabilities.catalog.resolver import invalidate_capability_cache logger = logging.getLogger(__name__) def _project_plugin_to_store(definition: Dict[str, Any], *, owner_user_id: Optional[str]) -> None: """Desktop store: keep ``R/plugins/local//`` and the component edges in step.""" - from core.capabilities.paths import capabilities_enabled + from core.harness.registry.run_capabilities import has_run_capabilities - if not capabilities_enabled(): + if not has_run_capabilities(): return - from core.capabilities import plugins as caps_plugins + from core.harness.registry.local_publication import local_publication - caps_plugins.publish_local_plugin(definition, owner_user_id=owner_user_id) + local_publication().publish_local_plugin(definition, owner_user_id=owner_user_id) def _remove_plugin_from_store(slug: str) -> None: - from core.capabilities.paths import capabilities_enabled + from core.harness.registry.run_capabilities import has_run_capabilities - if not capabilities_enabled(): + if not has_run_capabilities(): return - from core.capabilities import plugins as caps_plugins + from core.harness.registry.local_publication import local_publication - caps_plugins.remove_local_plugin(slug) + local_publication().remove_local_plugin(slug) def _purge_sandbox_skill_files(skill_ids) -> None: """Drop the materialized files of skills a plugin no longer owns, so they stop showing up under /workspace/skills/.""" - from core.agent_skills.config import purge_skill_sandbox_files + from core.capabilities.skills.config import purge_skill_sandbox_files for sid in skill_ids or []: try: @@ -43,7 +43,7 @@ def _purge_sandbox_skill_files(skill_ids) -> None: logger.debug("purge sandbox files for skill %s failed: %s", sid, exc) -def _refresh_after_change(owner_user_id: Optional[str]) -> None: +def refresh_after_change(owner_user_id: Optional[str]) -> None: """Invalidate related caches after install/uninstall: skill cache + capability-resolution cache + MCP config cache.""" try: refresh_skill_caches() @@ -57,7 +57,7 @@ def _refresh_after_change(owner_user_id: Optional[str]) -> None: except Exception as exc: # noqa: BLE001 logger.debug("invalidate_capability_cache failed: %s", exc) try: - from core.services.mcp_service import McpServerConfigService + from core.capabilities.mcp.service import McpServerConfigService McpServerConfigService.get_instance().invalidate_cache() except Exception as exc: # noqa: BLE001 diff --git a/src/backend/core/plugins/management/queries.py b/src/backend/core/capabilities/plugins/management/queries.py similarity index 86% rename from src/backend/core/plugins/management/queries.py rename to src/backend/core/capabilities/plugins/management/queries.py index 108a0ff7..c029e998 100644 --- a/src/backend/core/plugins/management/queries.py +++ b/src/backend/core/capabilities/plugins/management/queries.py @@ -1,4 +1,4 @@ -"""Plugin queries responsibilities.""" +"""List installed plugins and read their full details.""" from __future__ import annotations @@ -7,9 +7,9 @@ from core.db.models import AdminMcpServer, AdminSkill, InstalledPlugin from core.infra.exceptions import BadRequestError, ResourceNotFoundError -from core.plugins.management import admin_config as plugin_admin_config -from core.plugins.packaging import sources as plugin_sources -from core.services.marketplace_service import _strip_frontmatter +from core.capabilities.plugins.management import admin_config as plugin_admin_config +from core.capabilities.plugins.packaging import sources as plugin_sources +from core.capabilities.skills.installation import strip_frontmatter as _strip_frontmatter from sqlalchemy.orm import Session logger = logging.getLogger(__name__) @@ -30,7 +30,7 @@ def list_installed( plugins (frontend user view; global items are read-only — consistent with the skill library showing global skills). """ - from core.capabilities import device_catalog + from core.harness.registry import device_catalog from sqlalchemy import or_ if device_catalog.active(): @@ -66,8 +66,8 @@ def list_installed( all_mcp_ids: set = set() for r in rows: cids = r.component_ids or {} - all_skill_ids.update(plugin_sources._component_keys(cids, "skills")) - all_mcp_ids.update(plugin_sources._component_keys(cids, "mcp")) + all_skill_ids.update(plugin_sources.component_keys(cids, "skills")) + all_mcp_ids.update(plugin_sources.component_keys(cids, "mcp")) callable_skills = ( { @@ -96,17 +96,9 @@ def list_installed( ) if all_mcp_ids: - from core.db.models import McpMarketInstallation + from core.harness.registry.marketplace_catalog import marketplace_catalog - suspended = { - sid - for (sid,) in db.query(McpMarketInstallation.server_id) - .filter( - McpMarketInstallation.server_id.in_(all_mcp_ids), - McpMarketInstallation.status == "suspended", - ) - .all() - } + suspended = marketplace_catalog().suspended_mcp_ids(db, all_mcp_ids) callable_mcps.difference_update(suspended) # Enabled state: @@ -115,7 +107,7 @@ def list_installed( # personal toggles on global plugins are reflected correctly. # - admin view (owner_user_id empty): determined by components' global is_enabled. if owner_user_id is not None: - from core.config.catalog_resolver import resolve_all_runtime_enabled + from core.capabilities.catalog.resolver import resolve_all_runtime_enabled eff_skills, _eff_agents, eff_mcps = resolve_all_runtime_enabled(db, owner_user_id) enabled_skills = set(eff_skills or []) @@ -157,12 +149,12 @@ def list_installed( for r in rows: cids = r.component_ids or {} enabled = any( - s in enabled_skills for s in (plugin_sources._component_keys(cids, "skills")) - ) or any(m in enabled_mcps for m in (plugin_sources._component_keys(cids, "mcp"))) + s in enabled_skills for s in (plugin_sources.component_keys(cids, "skills")) + ) or any(m in enabled_mcps for m in (plugin_sources.component_keys(cids, "mcp"))) callable_now = any( - s in callable_skills for s in (plugin_sources._component_keys(cids, "skills")) - ) or any(m in callable_mcps for m in (plugin_sources._component_keys(cids, "mcp"))) - from core.plugins.ui.workspace_enabled import enabled as workspace_enabled, workspace_only + s in callable_skills for s in (plugin_sources.component_keys(cids, "skills")) + ) or any(m in callable_mcps for m in (plugin_sources.component_keys(cids, "mcp"))) + from core.harness.registry.package_views import enabled as workspace_enabled, workspace_only enabled = enabled or workspace_enabled(db, r, owner_user_id) callable_now = callable_now or workspace_only(r) @@ -195,8 +187,8 @@ def _installed_to_dict( # Hard runtime availability, independent of the current user's # personal catalog switch. Explicit pickers hide false entries. "callable": callable_now, - "skills": plugin_sources._component_keys(cids, "skills"), - "mcp": plugin_sources._component_keys(cids, "mcp"), + "skills": plugin_sources.component_keys(cids, "skills"), + "mcp": plugin_sources.component_keys(cids, "mcp"), "import_report": r.import_report or {}, "created_at": r.created_at.isoformat() if r.created_at else None, "has_admin_config": plugin_admin_config._has_admin_config_for_slug(r.slug), @@ -219,14 +211,14 @@ def get_installed_detail( raise BadRequestError(message="无权查看该插件") cids = row.component_ids or {} - skill_ids = plugin_sources._component_keys(cids, "skills") - server_ids = plugin_sources._component_keys(cids, "mcp") + skill_ids = plugin_sources.component_keys(cids, "skills") + server_ids = plugin_sources.component_keys(cids, "mcp") # Component enabled state: user view uses their "effectively enabled" set (including per-user overrides); admin view uses is_enabled. eff_skills: Optional[set] = None eff_mcps: Optional[set] = None if owner_user_id is not None: - from core.config.catalog_resolver import resolve_all_runtime_enabled + from core.capabilities.catalog.resolver import resolve_all_runtime_enabled es, _ea, em = resolve_all_runtime_enabled(db, owner_user_id) eff_skills, eff_mcps = set(es or []), set(em or []) @@ -298,5 +290,5 @@ def get_installed_detail( plugin_admin_config._admin_config_for_slug(row.slug), with_values=False ), # Account connection type (dingtalk / lark / None): the frontend uses this to render the account-connection panel on the detail page - "connection": plugin_admin_config._connection_for_slug(row.slug), + "connection": plugin_admin_config.connection_for_slug(row.slug), } diff --git a/src/backend/core/plugins/packaging/__init__.py b/src/backend/core/capabilities/plugins/packaging/__init__.py similarity index 100% rename from src/backend/core/plugins/packaging/__init__.py rename to src/backend/core/capabilities/plugins/packaging/__init__.py diff --git a/src/backend/core/plugins/packaging/archive.py b/src/backend/core/capabilities/plugins/packaging/archive.py similarity index 96% rename from src/backend/core/plugins/packaging/archive.py rename to src/backend/core/capabilities/plugins/packaging/archive.py index 81453832..1432e4df 100644 --- a/src/backend/core/plugins/packaging/archive.py +++ b/src/backend/core/capabilities/plugins/packaging/archive.py @@ -12,7 +12,7 @@ from core.infra.exceptions import BadRequestError -from . import definitions as plugin_definitions +from core.capabilities.plugins.packaging import definitions as plugin_definitions def _locate_plugin_root(extract_dir: Path) -> Path: @@ -39,7 +39,7 @@ def _has_manifest(d: Path) -> bool: @contextmanager -def _extract_plugin_zip(raw: bytes) -> Iterator[Path]: +def extract_plugin_zip(raw: bytes) -> Iterator[Path]: """Extract the plugin zip to a temp dir and locate the plugin root; yields that root dir and cleans up the temp dir on exit.""" if len(raw) > plugin_definitions.MAX_ZIP_BYTES: raise BadRequestError( diff --git a/src/backend/core/plugins/packaging/definitions.py b/src/backend/core/capabilities/plugins/packaging/definitions.py similarity index 94% rename from src/backend/core/plugins/packaging/definitions.py rename to src/backend/core/capabilities/plugins/packaging/definitions.py index 31cae0b5..8bf6da21 100644 --- a/src/backend/core/plugins/packaging/definitions.py +++ b/src/backend/core/capabilities/plugins/packaging/definitions.py @@ -1,4 +1,4 @@ -"""Plugin definitions responsibilities.""" +"""Paths, size limits and metadata keys for plugin bundles.""" from __future__ import annotations @@ -53,7 +53,7 @@ } -_META_KEYS = ("display_name", "category", "icon") +META_KEYS = ("display_name", "category", "icon") MAX_ICON_LEN = 200_000 diff --git a/src/backend/core/plugins/packaging/importer.py b/src/backend/core/capabilities/plugins/packaging/importer.py similarity index 96% rename from src/backend/core/plugins/packaging/importer.py rename to src/backend/core/capabilities/plugins/packaging/importer.py index 9c22f784..e4ab2550 100644 --- a/src/backend/core/plugins/packaging/importer.py +++ b/src/backend/core/capabilities/plugins/packaging/importer.py @@ -21,11 +21,11 @@ from pathlib import Path from typing import Any, Dict, List, Optional, Tuple -from core.agent_skills.binary_files import pack_directory +from core.capabilities.skills.binary_files import pack_directory from core.infra.exceptions import BadRequestError -from core.plugins.ui.contract import normalize_ui +from core.harness.registry.package_views import normalize_ui -from .models import NormalizedMcp, NormalizedPlugin, NormalizedSkill +from core.capabilities.plugins.packaging.models import NormalizedMcp, NormalizedPlugin, NormalizedSkill logger = logging.getLogger(__name__) @@ -50,7 +50,7 @@ def manifest_extensions(manifest: Dict[str, Any]) -> Dict[str, Any]: return {} -def _ext_or_top(manifest: Dict[str, Any], ext: Dict[str, Any], key: str) -> Any: +def ext_or_top(manifest: Dict[str, Any], ext: Dict[str, Any], key: str) -> Any: """Read a platform field: extensions namespace first, legacy top-level as fallback.""" if key in ext: return ext.get(key) @@ -214,7 +214,7 @@ def _normalize_mcp_entry(name: str, raw: Dict[str, Any]) -> NormalizedMcp: """ url = raw.get("url") if isinstance(url, str) and url.startswith("builtin://"): - from core.config.mcp_config import _mcp_http_url + from core.config.mcp_config import mcp_http_url as _mcp_http_url from extensions.mcp_servers._ports import PORTS server_id = url.removeprefix("builtin://") @@ -285,7 +285,7 @@ def _normalize_required_secrets( """ out: List[Dict[str, Any]] = [] # native: ["api_key", ...] or [{key,label,required}] - rs = _ext_or_top(manifest, ext or {}, "required_secrets") + rs = ext_or_top(manifest, ext or {}, "required_secrets") if isinstance(rs, list): for item in rs: if isinstance(item, str): @@ -376,7 +376,7 @@ def _slugify(value: str) -> str: return s or "plugin" -def _normalize_admin_config( +def normalize_admin_config( manifest: Dict[str, Any], ext: Optional[Dict[str, Any]] = None ) -> Optional[Dict[str, Any]]: """Normalize the ``admin_config`` in the plugin manifest (admin-level provider credential declaration). @@ -385,7 +385,7 @@ def _normalize_admin_config( Missing or empty fields → None (this plugin needs no admin config). Read from the extension namespace first, legacy top-level as fallback. """ - ac = _ext_or_top(manifest, ext or {}, "admin_config") + ac = ext_or_top(manifest, ext or {}, "admin_config") if not isinstance(ac, dict): return None raw_fields = ac.get("fields") @@ -453,15 +453,15 @@ def normalize_plugin_dir(plugin_dir: Path) -> NormalizedPlugin: mcp = [_normalize_mcp_entry(n, raw) for n, raw in sorted(mcp_map.items())] required_secrets = _normalize_required_secrets(manifest, ext) - admin_config = _normalize_admin_config(manifest, ext) - connection = _ext_or_top(manifest, ext, "connection") + admin_config = normalize_admin_config(manifest, ext) + connection = ext_or_top(manifest, ext, "connection") connection = str(connection).strip() if connection else None dropped = _collect_dropped(plugin_dir, manifest, kind) - ui, ui_dropped = normalize_ui(_ext_or_top(manifest, ext, "ui")) + ui, ui_dropped = normalize_ui(ext_or_top(manifest, ext, "ui")) dropped.extend(ui_dropped) - de = _ext_or_top(manifest, ext, "default_enabled") + de = ext_or_top(manifest, ext, "default_enabled") if isinstance(de, dict): default_enabled = { "skills": [str(x) for x in (de.get("skills") or [])], diff --git a/src/backend/core/plugins/packaging/models.py b/src/backend/core/capabilities/plugins/packaging/models.py similarity index 100% rename from src/backend/core/plugins/packaging/models.py rename to src/backend/core/capabilities/plugins/packaging/models.py diff --git a/src/backend/core/capabilities/plugins/packaging/presentation.py b/src/backend/core/capabilities/plugins/packaging/presentation.py new file mode 100644 index 00000000..182c024b --- /dev/null +++ b/src/backend/core/capabilities/plugins/packaging/presentation.py @@ -0,0 +1,31 @@ +"""Validate plugin icons and apply display metadata without marketplace storage.""" +from typing import Any, Dict +from core.infra.exceptions import BadRequestError +from core.capabilities.plugins.packaging import definitions as plugin_definitions + +def validate_icon(icon: str) -> str: + """Validate an icon value from the UI picker; returns the stripped value ('' = clear).""" + icon = (icon or "").strip() + if not icon: + return "" + if icon.startswith("data:"): + if not icon.startswith("data:image/"): + raise BadRequestError(message="图标 data URI 必须是 image 类型") + if len(icon) > plugin_definitions.MAX_ICON_LEN: + raise BadRequestError(message="图标过大（上传原图请控制在 80KB 以内）") + return icon + if not (icon.startswith("/") or icon.startswith("http://") or icon.startswith("https://")): + raise BadRequestError(message="图标须从图标库选择或上传（不支持任意文本）") + if len(icon) > plugin_definitions.MAX_ICON_URL_LEN: + raise BadRequestError(message="图标地址过长") + return icon + + +def overlay_market_meta(item: Dict[str, Any], meta: Dict[str, Any]) -> None: + """Apply effective display metadata onto a market list/detail dict (in place).""" + if meta.get("display_name"): + item["name"] = meta["display_name"] + if meta.get("category"): + item["category"] = meta["category"] + if meta.get("icon"): + item["icon"] = meta["icon"] diff --git a/src/backend/core/plugins/packaging/runtime_assets.py b/src/backend/core/capabilities/plugins/packaging/runtime_assets.py similarity index 100% rename from src/backend/core/plugins/packaging/runtime_assets.py rename to src/backend/core/capabilities/plugins/packaging/runtime_assets.py diff --git a/src/backend/core/plugins/packaging/skill_transform.py b/src/backend/core/capabilities/plugins/packaging/skill_transform.py similarity index 96% rename from src/backend/core/plugins/packaging/skill_transform.py rename to src/backend/core/capabilities/plugins/packaging/skill_transform.py index cc79c63b..f4d7ed87 100644 --- a/src/backend/core/plugins/packaging/skill_transform.py +++ b/src/backend/core/capabilities/plugins/packaging/skill_transform.py @@ -3,7 +3,7 @@ import re import yaml -from core.agent_skills.binary_files import decode_binary, is_binary_value +from core.capabilities.skills.binary_files import decode_binary, is_binary_value def files(child, skill_ids, server_ids, directory, plugin_directory): diff --git a/src/backend/core/plugins/packaging/sources.py b/src/backend/core/capabilities/plugins/packaging/sources.py similarity index 83% rename from src/backend/core/plugins/packaging/sources.py rename to src/backend/core/capabilities/plugins/packaging/sources.py index 0f930a6b..8a96c814 100644 --- a/src/backend/core/plugins/packaging/sources.py +++ b/src/backend/core/capabilities/plugins/packaging/sources.py @@ -1,4 +1,4 @@ -"""Plugin sources responsibilities.""" +"""Locate plugin bundles on disk and derive namespaced component ids.""" from __future__ import annotations @@ -8,14 +8,14 @@ from pathlib import Path from typing import Any, Dict, List, Optional, Tuple -from core.plugins.packaging import definitions as plugin_definitions -from core.plugins.packaging.importer import _ext_or_top, manifest_extensions -from core.services.marketplace_service import compute_install_id +from core.capabilities.plugins.packaging import definitions as plugin_definitions +from core.capabilities.plugins.packaging.importer import ext_or_top as _ext_or_top, manifest_extensions +from core.capabilities.skills.installation import compute_install_id logger = logging.getLogger(__name__) -def _iter_plugin_dirs(): +def iter_plugin_dirs(): """Iterate over all plugin bundle directories containing plugin.json under default + marketplace.""" for root in plugin_definitions.PLUGIN_SOURCE_DIRS: if not root.is_dir(): @@ -25,7 +25,7 @@ def _iter_plugin_dirs(): yield child -def _resolve_plugin_dir(slug: str) -> Optional[Path]: +def resolve_plugin_dir(slug: str) -> Optional[Path]: """Locate a plugin bundle directory by slug in default / marketplace (default takes precedence).""" if not slug or "/" in slug or ".." in slug: return None @@ -36,7 +36,7 @@ def _resolve_plugin_dir(slug: str) -> Optional[Path]: return None -def _make_plugin_install_id(slug: str, owner_user_id: Optional[str]) -> str: +def make_plugin_install_id(slug: str, owner_user_id: Optional[str]) -> str: return f"{slug}@{owner_user_id or 'global'}" @@ -55,7 +55,7 @@ def _sanitize_component_id(value: str, maxlen: int) -> str: return f"{prefix}-{digest}" -def _make_skill_id(slug: str, skill_name: str, owner_user_id: Optional[str]) -> str: +def make_skill_id(slug: str, skill_name: str, owner_user_id: Optional[str]) -> str: """Namespaced skill id: {slug}-{skill} (+ user fingerprint). Constrained by _ID_RE (<=63).""" base = _sanitize_component_id(f"{slug}-{skill_name}", 50) return compute_install_id( @@ -63,7 +63,7 @@ def _make_skill_id(slug: str, skill_name: str, owner_user_id: Optional[str]) -> ) # appends -<6-char fingerprint> when owner is non-empty -def _make_server_id(slug: str, server_name: str, owner_user_id: Optional[str]) -> str: +def make_server_id(slug: str, server_name: str, owner_user_id: Optional[str]) -> str: """Namespaced MCP server id: {slug}-{server} (+ user fingerprint), same rules as skill ids.""" return compute_install_id( _sanitize_component_id(f"{slug}-{server_name}", 80), @@ -87,11 +87,11 @@ def builtin_plugin_component_ids() -> Tuple[set, set]: """ import json - from core.plugins.packaging.importer import _find_mcp_map + from core.capabilities.plugins.packaging.importer import _find_mcp_map skill_ids: set = set() mcp_ids: set = set() - for child in _iter_plugin_dirs(): + for child in iter_plugin_dirs(): skills_root = child / "skills" if skills_root.is_dir(): for c in skills_root.iterdir(): @@ -103,12 +103,13 @@ def builtin_plugin_component_ids() -> Tuple[set, set]: m = {} try: mcp_ids.update(_find_mcp_map(child, m if isinstance(m, dict) else {}).keys()) - except Exception: # noqa: BLE001 - pass + except Exception as fallback_error: # noqa: BLE001 + from core.infra.failures import record_best_effort_failure + record_best_effort_failure('core.capabilities.plugins.packaging.sources:builtin_plugin_component_ids', fallback_error) return skill_ids, mcp_ids -def _scan_native_manifest(plugin_dir: Path) -> Optional[Dict[str, Any]]: +def scan_native_manifest(plugin_dir: Path) -> Optional[Dict[str, Any]]: """Lightweight read of a builtin plugin bundle's metadata (no full normalize). Standard manifests carry no display fields — display metadata is overlaid @@ -148,7 +149,7 @@ def _scan_native_manifest(plugin_dir: Path) -> Optional[Dict[str, Any]]: } -def _component_keys(components: Dict[str, Any], kind: str) -> List[str]: +def component_keys(components: Dict[str, Any], kind: str) -> List[str]: """Project declarative component entries to IDs for legacy catalog operations. Keep InstalledPlugin.component_ids unchanged: version/required/platform diff --git a/src/backend/core/capabilities/plugins/resources/__init__.py b/src/backend/core/capabilities/plugins/resources/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/src/backend/core/capabilities/plugins/resources/__init__.py @@ -0,0 +1 @@ + diff --git a/src/backend/core/plugins/resources/artifacts.py b/src/backend/core/capabilities/plugins/resources/artifacts.py similarity index 93% rename from src/backend/core/plugins/resources/artifacts.py rename to src/backend/core/capabilities/plugins/resources/artifacts.py index 6fd9217b..0eeb7706 100644 --- a/src/backend/core/plugins/resources/artifacts.py +++ b/src/backend/core/capabilities/plugins/resources/artifacts.py @@ -9,7 +9,7 @@ import httpx from fastapi import HTTPException -from . import store +from core.capabilities.plugins.resources import store async def retain(row, download_id): @@ -35,7 +35,7 @@ async def retain(row, download_id): if total > limit: raise HTTPException(413, "download_limit") output.write(chunk) - from core.llm.tools._tool_helpers import _store_generated_file_path + from core.files.artifacts.generated import store_generated_file_path as _store_generated_file_path ref = await asyncio.to_thread( _store_generated_file_path, diff --git a/src/backend/core/plugins/resources/authentication.py b/src/backend/core/capabilities/plugins/resources/authentication.py similarity index 100% rename from src/backend/core/plugins/resources/authentication.py rename to src/backend/core/capabilities/plugins/resources/authentication.py diff --git a/src/backend/core/plugins/resources/configuration.py b/src/backend/core/capabilities/plugins/resources/configuration.py similarity index 100% rename from src/backend/core/plugins/resources/configuration.py rename to src/backend/core/capabilities/plugins/resources/configuration.py diff --git a/src/backend/core/plugins/resources/confinement.py b/src/backend/core/capabilities/plugins/resources/confinement.py similarity index 83% rename from src/backend/core/plugins/resources/confinement.py rename to src/backend/core/capabilities/plugins/resources/confinement.py index f2f2a93d..9f830b60 100644 --- a/src/backend/core/plugins/resources/confinement.py +++ b/src/backend/core/capabilities/plugins/resources/confinement.py @@ -7,12 +7,13 @@ def launch_policy(provider, user_id, chat_id): from core.config.local_mode import local_mode_enabled if not local_mode_enabled() or not provider.runs_on_host: return None - from core.llm.tool_permissions import resolve_approval_mode, LocalCommandAuthorization - from core.services.local_grant_service import policy_for_gate + from core.sandbox.permissions.approval import resolve_approval_mode + from core.sandbox.permissions.dispatch_models import LocalCommandAuthorization + from core.harness.registry.local_files import policy_for_gate from core.sandbox.permissions.os_gate import LocalAccessDecision from core.sandbox.permissions.os_gate import protected_read_paths from core.sandbox.permissions.models import NETWORK - from core.llm.tools._paths import workspace_directory + from core.sandbox.workspace.resolution import workspace_directory mode = resolve_approval_mode(None, user_id=user_id) policy = policy_for_gate(mode) if policy.disposition_for(NETWORK) == "block": diff --git a/src/backend/core/plugins/resources/crypto.py b/src/backend/core/capabilities/plugins/resources/crypto.py similarity index 100% rename from src/backend/core/plugins/resources/crypto.py rename to src/backend/core/capabilities/plugins/resources/crypto.py diff --git a/src/backend/core/capabilities/plugins/resources/deadlines.py b/src/backend/core/capabilities/plugins/resources/deadlines.py new file mode 100644 index 00000000..6f78eaea --- /dev/null +++ b/src/backend/core/capabilities/plugins/resources/deadlines.py @@ -0,0 +1,48 @@ +"""Generic runner deadline support; installed owners supply lease policy.""" +from fastapi import HTTPException +from core.capabilities.plugins.resources import store + +def provider_for(row): + from core.sandbox.factory import get_sandbox_provider + + provider = get_sandbox_provider() + descriptor = store.descriptor(row) + method = getattr(provider, "set_process_deadline", None) + if descriptor.get("provider") != provider.name or not callable(method): + raise HTTPException(403, detail={"code": "unsupported_capability"}) + if not descriptor.get("process_id"): + raise HTTPException(410, detail={"code": "instance_closed"}) + return method, descriptor["process_id"] + + + +def invalidate(db, row): + from core.harness.registry.resource_owners import resource_owner + return resource_owner().invalidate(db, row) + + +async def arm(db, row, *, require_background=False): + if row.application_key is None or row.chat_id is not None: + return + from core.harness.registry.resource_owners import resource_owner + return await resource_owner().arm(db, row, require_background=require_background) + + +async def restore(db, *, after="", limit=100): + rows = ( + db.query(store.PluginResource) + .filter( + store.PluginResource.application_key.is_not(None), + store.PluginResource.status == "active", + store.PluginResource.resource_id > after, + ) + .order_by(store.PluginResource.resource_id) + .limit(limit) + .all() + ) + for row in rows: + try: + await arm(db, row) + except HTTPException: + db.rollback() # Existing janitor owns persisted closing cleanup. + return rows[-1].resource_id if len(rows) == limit else "" diff --git a/src/backend/core/plugins/resources/installation.py b/src/backend/core/capabilities/plugins/resources/installation.py similarity index 56% rename from src/backend/core/plugins/resources/installation.py rename to src/backend/core/capabilities/plugins/resources/installation.py index 401f6a58..bbb34bc5 100644 --- a/src/backend/core/plugins/resources/installation.py +++ b/src/backend/core/capabilities/plugins/resources/installation.py @@ -16,31 +16,10 @@ class Installation: def resolve(db, slug, user_id, install_id=None): - from core.capabilities import device_catalog, store - from core.capabilities.device_plugin_catalog import _projected_plugin - from core.capabilities.local_plugin_runtime import enabled_for - from core.capabilities.plugins import load_manifest - - candidate = _projected_plugin(install_id or slug, user_id=user_id) - if candidate is not None: - if candidate.key != slug or not candidate.ready or not enabled_for(candidate, user_id): - raise HTTPException(403, "plugin_not_ready") - component = store.get( - "plugin", candidate.profile_id, candidate.key, candidate.resolved_revision - ) - if component is None: - raise HTTPException(409, "plugin_package_missing") - manifest = load_manifest(component) - package = component.path / "package" - if not package.is_dir(): - raise HTTPException(409, "plugin_package_missing") - return Installation( - candidate.install_id, - candidate.resolved_revision, - slug, - manifest.get("ui_contributions") or {}, - package, - ) + from core.harness.registry import device_catalog + selected = device_catalog.resolve_plugin(slug, user_id, install_id=install_id) + if selected is not None: + return selected if device_catalog.active(): raise HTTPException(404, "plugin_source_unavailable") from core.db.models import InstalledPlugin @@ -59,12 +38,12 @@ def resolve(db, slug, user_id, install_id=None): if not rows: raise HTTPException(404, "plugin_source_unavailable") row = rows[0] - from core.config.catalog_resolver import resolve_all_runtime_enabled - from core.plugins.packaging.sources import _component_keys + from core.capabilities.catalog.resolver import resolve_all_runtime_enabled + from core.capabilities.plugins.packaging.sources import component_keys as _component_keys skills, _, mcps = resolve_all_runtime_enabled(db, user_id) cids = row.component_ids or {} - from core.plugins.ui.workspace_enabled import enabled as workspace_enabled + from core.harness.registry.package_views import enabled as workspace_enabled if not ( workspace_enabled(db, row, user_id) @@ -77,13 +56,13 @@ def resolve(db, slug, user_id, install_id=None): raise HTTPException(404, "plugin_ui_unavailable") revision = (row.import_report or {}).get("package_revision") if revision: - from core.plugins.packaging.runtime_assets import materialize + from core.capabilities.plugins.packaging.runtime_assets import materialize package = materialize(db, revision) else: - from core.plugins.packaging.sources import _resolve_plugin_dir + from core.capabilities.plugins.packaging.sources import resolve_plugin_dir - package = _resolve_plugin_dir(slug) + package = resolve_plugin_dir(slug) if package is None: raise HTTPException(409, "plugin_package_missing") return Installation(row.install_id, revision or row.version or "", slug, ui, Path(package)) diff --git a/src/backend/core/plugins/resources/lifecycle.py b/src/backend/core/capabilities/plugins/resources/lifecycle.py similarity index 84% rename from src/backend/core/plugins/resources/lifecycle.py rename to src/backend/core/capabilities/plugins/resources/lifecycle.py index de1a6779..b5e4c071 100644 --- a/src/backend/core/plugins/resources/lifecycle.py +++ b/src/backend/core/capabilities/plugins/resources/lifecycle.py @@ -9,11 +9,11 @@ import httpx from core.db.engine import SessionLocal -from . import store +from core.capabilities.plugins.resources import store async def reconcile(db, user_id=None, after="", limit=100): - from . import service + from core.capabilities.plugins.resources import service query = db.query(store.PluginResource).filter( store.PluginResource.status.in_(["active", "starting", "warming", "warm", "closing"]) @@ -58,9 +58,9 @@ async def valid(row): @asynccontextmanager async def lifespan(app): - from . import prewarm + from core.capabilities.plugins.resources import prewarm - from .deadlines import restore + from core.capabilities.plugins.resources.deadlines import restore cursor = "" while True: @@ -76,17 +76,12 @@ async def sweep(): while True: await asyncio.sleep(int(os.getenv("PLUGIN_RESOURCE_RECONCILE_SECONDS", "30"))) try: - from core.applications.revocation_sweep import sweep as revoke_stale + from core.harness.registry.resource_owners import resource_owner - owner_cursor = await revoke_stale(owner_cursor) + owner_cursor = await resource_owner().sweep(owner_cursor) with SessionLocal() as db: cursor = await reconcile(db, after=cursor) - from core.applications.plugin_revisions import collect - - collect(db) - from core.applications.plugin_candidate_storage import janitor - - janitor(db) + resource_owner().collect(db) db.commit() except Exception as exc: logging.getLogger(__name__).warning( @@ -99,6 +94,6 @@ async def sweep(): finally: task.cancel() await asyncio.gather(task, return_exceptions=True) - from .prewarm import stop + from core.capabilities.plugins.resources.prewarm import stop await stop() diff --git a/src/backend/core/capabilities/plugins/resources/owners.py b/src/backend/core/capabilities/plugins/resources/owners.py new file mode 100644 index 00000000..5439e77e --- /dev/null +++ b/src/backend/core/capabilities/plugins/resources/owners.py @@ -0,0 +1,66 @@ +"""Resource owner validation and provider session identity.""" + +import hashlib +import json + +from core.config.local_mode import local_mode_enabled +from fastapi import HTTPException + + +def execution_scope(): + return "local" if local_mode_enabled() else "cloud" + + +def runtime_session(row): + if row.chat_id is not None: + return row.chat_id + # Provider session IDs are execution containers, never conversation records. + binding = [ + row.user_id, + row.application_key, + row.install_id, + row.revision, + row.scope, + row.instance_id, + ] + return "workbench-" + hashlib.sha256(json.dumps(binding).encode()).hexdigest() + + +def application_binding(db, user_id, key, install_id, revision, scope, module_id, selected=None): + from core.harness.registry.resource_owners import resource_owner + return resource_owner().binding(db, user_id, key, install_id, revision, scope, module_id, selected) + + +def resolve_installation(db, row, user_id): + if row.application_key is not None: + from core.harness.registry.resource_owners import resource_owner + return resource_owner().resolve_installation(db, row, user_id) + from core.capabilities.plugins.resources.installation import resolve + selected = resolve(db, row.slug, user_id, row.install_id) + if selected.revision != row.revision: + raise HTTPException(409, "plugin_revision_changed") + return selected + + +def validate(db, row, user_id, selected=None): + if row.chat_id is not None: + if row.application_key is not None or row.instance_id is not None: + raise HTTPException(403, "resource_owner_invalid") + from core.db.models import ChatSession + + chat = db.get(ChatSession, row.chat_id) + if chat is None or chat.user_id != user_id or chat.deleted_at is not None: + raise HTTPException(403, "conversation_unavailable") + else: + if not row.application_key or not row.instance_id: + raise HTTPException(403, "resource_owner_invalid") + application_binding( + db, + user_id, + row.application_key, + row.install_id, + row.revision, + row.scope, + row.module_id, + selected, + ) diff --git a/src/backend/core/plugins/resources/prewarm.py b/src/backend/core/capabilities/plugins/resources/prewarm.py similarity index 72% rename from src/backend/core/plugins/resources/prewarm.py rename to src/backend/core/capabilities/plugins/resources/prewarm.py index 3c78d90f..7fcff138 100644 --- a/src/backend/core/plugins/resources/prewarm.py +++ b/src/backend/core/capabilities/plugins/resources/prewarm.py @@ -1,24 +1,23 @@ """Background opt-in resource startup, bound to a conversation sandbox lifetime.""" +from __future__ import annotations import asyncio import logging import threading from contextlib import asynccontextmanager +from dataclasses import dataclass, field import httpx from fastapi import HTTPException from sqlalchemy import or_ from core.db import engine -from . import installation, service, store +from core.capabilities.plugins.resources import installation, service, store logger = logging.getLogger(__name__) -_guard = threading.Lock() -_jobs = {} -_closing = {} -_shutdown = False + def candidates(db, user_id): - from core.capabilities import device_catalog + from core.harness.registry import device_catalog if device_catalog.active(): - from core.capabilities.device_plugin_catalog import plugin_entries + from core.harness.registry.device_catalog import plugin_entries sources = [(entry["slug"], entry["install_id"]) for entry in plugin_entries(user_id) if entry["enabled"] and entry["callable"]] else: @@ -39,14 +38,14 @@ def schedule(provider, chat_id, user_id, sandbox_id): return key = (chat_id, user_id, str(sandbox_id)) loop = asyncio.get_running_loop() - with _guard: - if _shutdown or chat_id in _closing or key in _jobs: + with process_state().guard: + if process_state().shutdown or chat_id in process_state().closing or key in process_state().jobs: return # Bound completed admission markers; router providers notify repeatedly. - if len(_jobs) >= 1024: - for old in [old for old, job in _jobs.items() if job.done()][:256]: - _jobs.pop(old, None) - _jobs[key] = loop.create_task(_run(provider, chat_id, user_id, str(sandbox_id))) + if len(process_state().jobs) >= 1024: + for old in [old for old, job in process_state().jobs.items() if job.done()][:256]: + process_state().jobs.pop(old, None) + process_state().jobs[key] = loop.create_task(_run(provider, chat_id, user_id, str(sandbox_id))) async def _run(provider, chat_id, user_id, sandbox_id): from core.sandbox.sessions.binding import bind @@ -72,36 +71,36 @@ async def _cancel(task): @asynccontextmanager async def closing(chat_id): - with _guard: - _closing[chat_id] = _closing.get(chat_id, 0) + 1 + with process_state().guard: + process_state().closing[chat_id] = process_state().closing.get(chat_id, 0) + 1 try: yield finally: - with _guard: - _closing[chat_id] -= 1 - if _closing[chat_id] == 0: - _closing.pop(chat_id) + with process_state().guard: + process_state().closing[chat_id] -= 1 + if process_state().closing[chat_id] == 0: + process_state().closing.pop(chat_id) def resume(): - global _shutdown - with _guard: - _shutdown = False + pass + with process_state().guard: + process_state().shutdown = False async def stop(chat_id=None): - global _shutdown + pass if str(chat_id or "").startswith("eval_"): return True if chat_id is None: - with _guard: - _shutdown = True + with process_state().guard: + process_state().shutdown = True async with closing(chat_id): return await _stop(chat_id) async def _stop(chat_id): """Stop starts and blank workers. False forbids returning a dirty sandbox to its pool.""" - with _guard: - keys = [key for key in _jobs if chat_id is None or key[0] == chat_id] - jobs = [_jobs.pop(key) for key in keys] + with process_state().guard: + keys = [key for key in process_state().jobs if chat_id is None or key[0] == chat_id] + jobs = [process_state().jobs.pop(key) for key in keys] loop = asyncio.get_running_loop() clean = True for task in jobs: @@ -140,3 +139,18 @@ async def _stop(chat_id): row.status = "closed" db.commit() return clean + + +@dataclass +class PrewarmState: + guard: object = field(default_factory=lambda: threading.Lock()) + jobs: object = field(default_factory=lambda: {}) + closing: object = field(default_factory=lambda: {}) + shutdown: object = False + + +_STATE = PrewarmState() + + +def process_state() -> PrewarmState: + return _STATE diff --git a/src/backend/core/plugins/resources/reservation.py b/src/backend/core/capabilities/plugins/resources/reservation.py similarity index 96% rename from src/backend/core/plugins/resources/reservation.py rename to src/backend/core/capabilities/plugins/resources/reservation.py index dbb71461..4238b3dd 100644 --- a/src/backend/core/plugins/resources/reservation.py +++ b/src/backend/core/capabilities/plugins/resources/reservation.py @@ -6,7 +6,7 @@ from fastapi import HTTPException -from . import store +from core.capabilities.plugins.resources import store LIVE_STATUSES = ("active", "starting", "warming", "warm") @@ -66,7 +66,7 @@ async def adopt(db, selected, module_id, user_id, chat_id, *, prewarm, checkpoin db.expire_all() # Each wait releases the transaction: installation switches and revocation # must be noticed before the worker can be adopted. - from . import installation + from core.capabilities.plugins.resources import installation current = installation.resolve(db, selected.slug, user_id, selected.install_id) if current.revision != selected.revision: diff --git a/src/backend/core/plugins/resources/service.py b/src/backend/core/capabilities/plugins/resources/service.py similarity index 84% rename from src/backend/core/plugins/resources/service.py rename to src/backend/core/capabilities/plugins/resources/service.py index 6edc657b..d4e0fa51 100644 --- a/src/backend/core/plugins/resources/service.py +++ b/src/backend/core/capabilities/plugins/resources/service.py @@ -6,12 +6,12 @@ import time import uuid -from core.plugins.ui.contract import find_module +from core.harness.registry.package_views import find_module from fastapi import HTTPException -from . import installation, store -from .crypto import encrypt_secret -from .transport import RuntimeTransport +from core.capabilities.plugins.resources import installation, store +from core.capabilities.plugins.resources.crypto import encrypt_secret +from core.capabilities.plugins.resources.transport import RuntimeTransport def authorized(db, resource_id, user_id): @@ -24,10 +24,10 @@ def authorized(db, resource_id, user_id): def validate_binding(db, row, user_id): if row.application_key is not None: - from core.applications import background + from core.harness.registry.resource_owners import resource_owner - background.authorized_resource(db, row) - from .owners import resolve_installation, validate + resource_owner().authorize(db, row) + from core.capabilities.plugins.resources.owners import resolve_installation, validate selected = resolve_installation(db, row, user_id) validate(db, row, user_id, selected) @@ -63,7 +63,7 @@ async def create( else: if chat_id is not None or checkpoint_id or _prewarm: raise HTTPException(403, "resource_owner_invalid") - from .owners import application_binding + from core.capabilities.plugins.resources.owners import application_binding application_binding( db, @@ -76,7 +76,8 @@ async def create( ) resolver = installation.resolve if _application: - from core.applications.installation_binding import exact as resolver + from core.harness.registry.resource_owners import resource_owner + resolver = resource_owner().exact_installation selected = resolver(db, slug, user_id, install_id) if _application and selected.install_id != install_id: @@ -89,7 +90,7 @@ async def create( if _application and "resource.attach" not in module.get("grants", []): raise HTTPException(403, "resource_attach_not_granted") token = secrets.token_urlsafe(32) - from .configuration import runtime_configuration + from core.capabilities.plugins.resources.configuration import runtime_configuration config = { **runtime_configuration(module["resource"]), @@ -104,25 +105,25 @@ async def create( or checkpoint.install_id != selected.install_id ): raise HTTPException(404, "checkpoint_unavailable") - from .crypto import decrypt_secret + from core.capabilities.plugins.resources.crypto import decrypt_secret state = decrypt_secret(checkpoint.state_enc) if state is None: raise HTTPException(410, "checkpoint_expired") config["checkpoint"] = json.loads(state) - from .lifecycle import reconcile + from core.capabilities.plugins.resources.lifecycle import reconcile await reconcile(db, user_id) - from .reservation import adopt, check_quota + from core.capabilities.plugins.resources.reservation import adopt, check_quota if _application is None: row, reused = await adopt( db, selected, module_id, user_id, chat_id, prewarm=_prewarm, checkpoint_id=checkpoint_id ) else: - from .application import reserve + from core.harness.registry.resource_owners import resource_owner - row, reused = await reserve(db, selected, module_id, user_id, _application) + row, reused = await resource_owner().reserve(db, selected, module_id, user_id, _application) if reused: return store.public(row) if reused: @@ -150,7 +151,7 @@ async def create( return store.public(row) await close(db, row) # A dead warm worker has performed no user actions; reserve a fresh one. - from .reservation import lock_owner + from core.capabilities.plugins.resources.reservation import lock_owner lock_owner(db, user_id) check_quota(db, user_id) @@ -177,7 +178,7 @@ async def create( created_at=time.time(), expires_at=time.time() + 90, ) - from .owners import runtime_session + from core.capabilities.plugins.resources.owners import runtime_session session_id = runtime_session(row) db.add(row) @@ -240,7 +241,7 @@ async def create( raise HTTPException(410, "resource_start_cancelled") db.refresh(row) if _application: - from .deadlines import arm + from core.capabilities.plugins.resources.deadlines import arm await arm(db, row) except BaseException: @@ -265,9 +266,9 @@ async def create( row.status = "closed" db.commit() raise - from core.applications.diagnostics import observe_resource + from core.harness.registry.resource_owners import resource_owner - observe_resource(db, row, "resource", "ok") + resource_owner().observe(db, row, "resource", "ok") return store.public(row) @@ -314,10 +315,10 @@ async def checkpoint(db, row, name, connection_id, *, transport=None): async def close(db, row): if row.application_key is not None: - from .application_cleanup import close + from core.harness.registry.resource_owners import resource_owner - return await close(db, row) - from .owners import runtime_session + return await resource_owner().close(db, row) + from core.capabilities.plugins.resources.owners import runtime_session try: target = store.descriptor(row) @@ -339,8 +340,9 @@ async def close(db, row): chars="\x03", yield_time_ms=1000, ) - except Exception: - pass + except Exception as fallback_error: + from core.infra.failures import record_best_effort_failure + record_best_effort_failure('core.capabilities.plugins.resources.service:close', fallback_error) row.status = "closed" db.query(store.PluginResourceTicket).filter_by(resource_id=row.resource_id).delete() db.query(store.PluginResourceAssetTicket).filter_by(resource_id=row.resource_id).delete() diff --git a/src/backend/core/plugins/resources/store.py b/src/backend/core/capabilities/plugins/resources/store.py similarity index 97% rename from src/backend/core/plugins/resources/store.py rename to src/backend/core/capabilities/plugins/resources/store.py index e9c97eaf..a247ba6b 100644 --- a/src/backend/core/plugins/resources/store.py +++ b/src/backend/core/capabilities/plugins/resources/store.py @@ -12,7 +12,7 @@ PluginResourceTicket, ) -from .crypto import decrypt_secret, encrypt_secret +from core.capabilities.plugins.resources.crypto import decrypt_secret, encrypt_secret def public(row): diff --git a/src/backend/core/plugins/resources/stream.py b/src/backend/core/capabilities/plugins/resources/stream.py similarity index 95% rename from src/backend/core/plugins/resources/stream.py rename to src/backend/core/capabilities/plugins/resources/stream.py index 7ca2af07..01b742c7 100644 --- a/src/backend/core/plugins/resources/stream.py +++ b/src/backend/core/capabilities/plugins/resources/stream.py @@ -7,8 +7,8 @@ import httpx from fastapi import HTTPException, WebSocket from core.db.engine import SessionLocal -from . import service, store -from .transport import RuntimeTransport +from core.capabilities.plugins.resources import service, store +from core.capabilities.plugins.resources.transport import RuntimeTransport MAX_PACKET = 8 * 1024 * 1024 @@ -48,7 +48,7 @@ async def connect(websocket: WebSocket, resource_id: str): await websocket.close(4401) return user_id, authentication = attachment - from .authentication import validate as validate_authentication + from core.capabilities.plugins.resources.authentication import validate as validate_authentication await validate_authentication(authentication, user_id, db) row = service.authorized(db, resource_id, user_id) target = store.descriptor(row) diff --git a/src/backend/core/plugins/resources/transport.py b/src/backend/core/capabilities/plugins/resources/transport.py similarity index 100% rename from src/backend/core/plugins/resources/transport.py rename to src/backend/core/capabilities/plugins/resources/transport.py diff --git a/src/backend/core/plugins/runtime/__init__.py b/src/backend/core/capabilities/plugins/runtime/__init__.py similarity index 76% rename from src/backend/core/plugins/runtime/__init__.py rename to src/backend/core/capabilities/plugins/runtime/__init__.py index f6c2ad25..f0d5b172 100644 --- a/src/backend/core/plugins/runtime/__init__.py +++ b/src/backend/core/capabilities/plugins/runtime/__init__.py @@ -30,27 +30,10 @@ transport MCP 的插件不延迟（激活期进程内起子进程的生命周期管理复杂度不值得）。 """ -from core.plugins.runtime.activation import register_load_plugin -from core.plugins.runtime.activation_history import ( - load_activated_plugin_slugs, - record_plugin_activation, -) -from core.plugins.runtime.desktop_resolution import ( - prepare_desktop_plugin_skill_defaults, - resolve_desktop_progressive_plugins, -) -from core.plugins.runtime.models import ( - DeferredPlugin, - ProgressiveResolution, - StickyPluginCapabilities, - progressive_plugin_loading_enabled, -) -from core.plugins.runtime.resolution import ( - build_plugin_directory_section, - resolve_bound_progressive_plugins, - resolve_progressive_plugins, -) -from core.plugins.runtime.sticky_selection import resolve_sticky_plugin_capabilities +from core.capabilities.plugins.runtime.activation_history import load_activated_plugin_slugs, record_plugin_activation +from core.capabilities.plugins.runtime.models import DeferredPlugin, ProgressiveResolution, StickyPluginCapabilities, progressive_plugin_loading_enabled +from core.capabilities.plugins.runtime.resolution import build_plugin_directory_section, resolve_bound_progressive_plugins, resolve_progressive_plugins +from core.capabilities.plugins.runtime.sticky_selection import resolve_sticky_plugin_capabilities __all__ = [ "progressive_plugin_loading_enabled", @@ -63,7 +46,4 @@ "resolve_progressive_plugins", "resolve_bound_progressive_plugins", "build_plugin_directory_section", - "prepare_desktop_plugin_skill_defaults", - "resolve_desktop_progressive_plugins", - "register_load_plugin", ] diff --git a/src/backend/core/plugins/runtime/activation_history.py b/src/backend/core/capabilities/plugins/runtime/activation_history.py similarity index 97% rename from src/backend/core/plugins/runtime/activation_history.py rename to src/backend/core/capabilities/plugins/runtime/activation_history.py index 8215a99a..b00a8977 100644 --- a/src/backend/core/plugins/runtime/activation_history.py +++ b/src/backend/core/capabilities/plugins/runtime/activation_history.py @@ -10,7 +10,7 @@ _ACTIVATED_KEY = "activated_plugins" -from core.plugins.runtime import sticky_selection as plugin_sticky_selection +from core.capabilities.plugins.runtime import sticky_selection as plugin_sticky_selection def load_activated_plugin_slugs(chat_id: Optional[str]) -> List[str]: diff --git a/src/backend/core/plugins/runtime/models.py b/src/backend/core/capabilities/plugins/runtime/models.py similarity index 100% rename from src/backend/core/plugins/runtime/models.py rename to src/backend/core/capabilities/plugins/runtime/models.py diff --git a/src/backend/core/plugins/runtime/resolution.py b/src/backend/core/capabilities/plugins/runtime/resolution.py similarity index 91% rename from src/backend/core/plugins/runtime/resolution.py rename to src/backend/core/capabilities/plugins/runtime/resolution.py index d82300b7..9529a2f1 100644 --- a/src/backend/core/plugins/runtime/resolution.py +++ b/src/backend/core/capabilities/plugins/runtime/resolution.py @@ -5,11 +5,12 @@ import logging from typing import Any, Dict, List, Optional, Sequence -logger = logging.getLogger(__name__) -from core.plugins.runtime import activation_history as plugin_activation_history -from core.plugins.runtime import models as plugin_runtime_models +from core.capabilities.plugins.runtime import activation_history as plugin_activation_history +from core.capabilities.plugins.runtime import models as plugin_runtime_models + +logger = logging.getLogger(__name__) def _http_transport(cfg: Any) -> bool: @@ -21,15 +22,16 @@ def _http_transport(cfg: Any) -> bool: def _server_configs(user_id: Optional[str] = None) -> Dict[str, Any]: """Transport lookup: global servers, plus the user's private ones when given.""" try: - from core.services.mcp_service import McpServerConfigService + from core.capabilities.mcp.service import McpServerConfigService svc = McpServerConfigService.get_instance() cfgs = dict(svc.get_all_servers(enabled_only=True)) if user_id: try: cfgs.update(svc.get_owned_servers(str(user_id), enabled_only=False)) - except Exception: # noqa: BLE001 - pass + except Exception as fallback_error: # noqa: BLE001 + from core.infra.failures import record_best_effort_failure + record_best_effort_failure('core.capabilities.plugins.runtime.resolution:_server_configs', fallback_error) return cfgs except Exception as exc: # noqa: BLE001 logger.warning("[plugin-loader] server config lookup failed: %s", exc) @@ -38,7 +40,7 @@ def _server_configs(user_id: Optional[str] = None) -> Dict[str, Any]: def _skill_metadata() -> Dict[str, Any]: try: - from core.agent_skills.loader import get_skill_loader + from core.capabilities.skills.loader_factory import get_skill_loader return get_skill_loader().load_all_metadata() or {} except Exception: # noqa: BLE001 @@ -58,7 +60,7 @@ def _bound_mcp_ids( return bound -def _finalize_resolution( +def finalize_resolution( res: plugin_runtime_models.ProgressiveResolution, eligible: Sequence[plugin_runtime_models.DeferredPlugin], ) -> plugin_runtime_models.ProgressiveResolution: @@ -170,7 +172,7 @@ def resolve_progressive_plugins( if newly_pinned: plugin_activation_history.record_plugin_activation(chat_id, newly_pinned) - return _finalize_resolution(res, eligible) + return finalize_resolution(res, eligible) def resolve_bound_progressive_plugins( @@ -215,8 +217,9 @@ def resolve_bound_progressive_plugins( if skill_filter is not None: try: skills = list(skill_filter(skills)) - except Exception: # noqa: BLE001 - pass + except Exception as fallback_error: # noqa: BLE001 + from core.infra.failures import record_best_effort_failure + record_best_effort_failure('core.capabilities.plugins.runtime.resolution:resolve_bound_progressive_plugins', fallback_error) mcps = [m for m in (cids.get("mcp") or []) if isinstance(m, str) and m.strip()] if not skills and not mcps: continue @@ -234,7 +237,7 @@ def resolve_bound_progressive_plugins( eligible.append(plugin) res.deferred.append(plugin) - return _finalize_resolution(res, eligible) + return finalize_resolution(res, eligible) def build_plugin_directory_section( diff --git a/src/backend/core/plugins/runtime/sticky_selection.py b/src/backend/core/capabilities/plugins/runtime/sticky_selection.py similarity index 85% rename from src/backend/core/plugins/runtime/sticky_selection.py rename to src/backend/core/capabilities/plugins/runtime/sticky_selection.py index 89dbc334..722807db 100644 --- a/src/backend/core/plugins/runtime/sticky_selection.py +++ b/src/backend/core/capabilities/plugins/runtime/sticky_selection.py @@ -1,32 +1,31 @@ """Progressive plugin runtime: sticky selection.""" from __future__ import annotations +from core.harness.registry import execution_account as skill_account import logging from typing import Any, Dict, List, Optional, Set -logger = logging.getLogger(__name__) -from core.plugins.runtime import activation_history as plugin_activation_history -from core.plugins.runtime import models as plugin_runtime_models +from core.capabilities.plugins.runtime import activation_history as plugin_activation_history +from core.capabilities.plugins.runtime import models as plugin_runtime_models + +logger = logging.getLogger(__name__) def _cloud_sticky_selection(tokens, *, user_id, allow_aliases=False, unavailable_out=None): """Keep saved cloud selections tied to their exact account and installation.""" - from core.capabilities import registry, skills - from core.capabilities.errors import NameConflict, PermissionDenied - from core.capabilities.paths import BUILTIN_PROFILE, LOCAL_PROFILE, capabilities_enabled + from core.harness.registry.device_catalog import account_plugin_installations + from core.harness.capability_errors import NameConflict, PermissionDenied + from core.harness.registry.run_capabilities import has_run_capabilities + from core.capabilities.identity_rules import BUILTIN_PROFILE, LOCAL_PROFILE - if not capabilities_enabled(): + if not has_run_capabilities(): return [] - profile = skills.current_account_profile() - authorized = skills.account_authorized_for(user_id) - rows = ( - registry.list_installations(kind="plugin", profile_id=profile) - if profile and authorized - else [] - ) + profile = skill_account.current_account_profile() + authorized = skill_account.account_authorized_for(user_id) + rows = account_plugin_installations(user_id) if profile and authorized else [] result = [] for token in tokens: parts = str(token).split(":", 2) @@ -93,8 +92,8 @@ def resolve_sticky_plugin_capabilities( unavailable_out=result.unavailable_ids, ) if cloud: - from core.capabilities.errors import CapabilityError - from core.capabilities.plugins import cloud_binding_ids + from core.harness.capability_errors import CapabilityError + from core.harness.registry.run_capabilities import cloud_binding_ids for row in cloud: try: @@ -108,7 +107,7 @@ def resolve_sticky_plugin_capabilities( result.mcp_ids.extend(mcp_ids) try: - from core.config.catalog_resolver import resolve_explicit_runtime_capabilities + from core.capabilities.catalog.resolver import resolve_explicit_runtime_capabilities from core.db.engine import SessionLocal from core.db.models import InstalledPlugin from sqlalchemy import or_ @@ -150,7 +149,7 @@ def resolve_sticky_plugin_capabilities( requested_skills: List[str] = [] requested_mcps: List[str] = [] - from core.plugins.packaging.sources import _component_keys + from core.capabilities.plugins.packaging.sources import component_keys as _component_keys for row in selected: component_ids = row.component_ids or {} diff --git a/src/backend/core/capabilities/ref.py b/src/backend/core/capabilities/reference.py similarity index 98% rename from src/backend/core/capabilities/ref.py rename to src/backend/core/capabilities/reference.py index 14fe95b0..1b4cfb0c 100644 --- a/src/backend/core/capabilities/ref.py +++ b/src/backend/core/capabilities/reference.py @@ -7,7 +7,7 @@ from typing import Any, Dict from urllib.parse import urlsplit, urlunsplit -from .paths import KINDS, LOCAL_PROFILE, safe_segment +from core.capabilities.identity_rules import KINDS, LOCAL_PROFILE, safe_segment LOCAL_ISSUER = "local" LOCAL_NAMESPACE = "device" diff --git a/src/backend/core/capabilities/runtime/__init__.py b/src/backend/core/capabilities/runtime/__init__.py deleted file mode 100644 index 5773a355..00000000 --- a/src/backend/core/capabilities/runtime/__init__.py +++ /dev/null @@ -1,37 +0,0 @@ -"""Public capability runtime API; state and operations have one-way dependencies.""" - -from core.capabilities.runtime.connectors import bind_mcp as bind_mcp -from core.capabilities.runtime.connectors import pin_agent_definition as pin_agent_definition -from core.capabilities.runtime.connectors import references as references -from core.capabilities.runtime.dependencies import preflight as preflight -from core.capabilities.runtime.dependencies import validate_activation as validate_activation -from core.capabilities.runtime.preparation import prepare as prepare -from core.capabilities.runtime.state import PreparedRun as PreparedRun -from core.capabilities.runtime.state import child_scope as child_scope -from core.capabilities.runtime.state import frozen_loader as frozen_loader -from core.capabilities.runtime.state import get as get -from core.capabilities.runtime.state import rebuild as rebuild -from core.capabilities.runtime.state import record_tool_scope as record_tool_scope -from core.capabilities.runtime.state import require_root_tool_scope as require_root_tool_scope -from core.capabilities.runtime.state import save as save -from core.capabilities.runtime.state import validate as validate -from core.capabilities.runtime.state import view_for_execution as view_for_execution - -__all__ = [ - "child_scope", - "PreparedRun", - "get", - "validate", - "rebuild", - "save", - "frozen_loader", - "view_for_execution", - "record_tool_scope", - "require_root_tool_scope", - "prepare", - "bind_mcp", - "pin_agent_definition", - "references", - "preflight", - "validate_activation", -] diff --git a/src/backend/core/capabilities/skills.py b/src/backend/core/capabilities/skills.py deleted file mode 100644 index a7e75ee7..00000000 --- a/src/backend/core/capabilities/skills.py +++ /dev/null @@ -1,656 +0,0 @@ -"""Skill kind on the desktop store: candidates, resolution, runtime views, local publication. - -Two views are maintained, both link-only: - -- the **device view** (``get_sandbox_skills_dir()``, i.e. ``{workspace}/skills``): - shipped built-ins plus device-local skills that are not private to one user. - Process-level consumers (CLI shims on PATH) resolve through it. -- the **user view** (``get_user_skills_dir(uid)``): everything the current user - may use — device view entries plus the current cloud account's ready - installations — after name resolution. Session workspaces link to it. -""" - -from __future__ import annotations - -import hashlib -import logging -import os -import threading -from collections import OrderedDict -from concurrent.futures import ThreadPoolExecutor -from contextvars import copy_context -from pathlib import Path -from typing import Any, Callable, Dict, List, Optional, Set, Tuple - -from extensions.paths import SKILLS_DIR - -from . import registry, store -from . import view as view_mod -from .paths import ( - BUILTIN_PROFILE, - KIND_SKILL, - LOCAL_PROFILE, - capability_root, - revision_for_hash, -) -from .ref import builtin_ref, local_ref, profile_id -from .resolver import ( - SOURCE_BUILTIN, - SOURCE_CLOUD, - SOURCE_LOCAL, - SOURCE_PLUGIN, - Candidate, - Resolution, - resolve, -) - -logger = logging.getLogger(__name__) - -_SKIP_PARTS = {"__pycache__", ".git", ".svn", ".hg", "__MACOSX"} -_INVENTORY_NAME = ".inventory.json" - -_hash_lock = threading.Lock() -_hash_cache: Dict[str, Tuple[Tuple, str]] = {} -# Fresh reads still hash every byte. Reuse only the protocol JSON/base64 work -# when the complete sequence of file names and byte digests is identical. -_content_hash_cache: OrderedDict[tuple, str] = OrderedDict() - -_resolution_lock = threading.Lock() -_last_resolution: Dict[str, Resolution] = {} -_resolution_cache: Dict[str, Tuple[Tuple, Resolution]] = {} -_view_generation = 0 - - -def view_generation() -> int: - """Bumped whenever store contents change; view consumers refresh on a new value.""" - return _view_generation - - -def bump_view_generation() -> int: - global _view_generation - with _resolution_lock: - _view_generation += 1 - return _view_generation - - -def builtin_dir() -> Path: - return SKILLS_DIR / "default" - - -def allowed_link_roots() -> List[Path]: - roots = [builtin_dir()] - root = capability_root() - if root is not None: - roots.insert(0, root) - return roots - - -def device_view_dir() -> Path: - from core.agent_skills.config import get_sandbox_skills_dir - - return get_sandbox_skills_dir() - - -def user_view_dir(user_id: Optional[str]) -> Optional[Path]: - from core.agent_skills.config import get_user_skills_dir - - return get_user_skills_dir(user_id) - - -# ── content hashing (same scheme the cloud uses for its manifest) ─────── - - -def _dir_signature(path: Path) -> Tuple: - from .archive import iter_file_stats - - # A maximum mtime misses edits to any file older than the newest file. - return tuple( - (rel, metadata.st_size, metadata.st_mtime_ns, metadata.st_ctime_ns, metadata.st_ino) - for rel, _, metadata in iter_file_stats(path) - if rel != _INVENTORY_NAME - ) - - -def dir_digest(path: Path, compute: Callable[[], str], *, fresh: bool = False) -> str: - """A package directory's content digest, cached on its file signature. - - Identity is *established* with ``fresh=True``: every byte is read, so the - digest a binding freezes is always a true content hash. Later passes only - *verify* that digest, and a run re-verifies its whole closure on every - message — re-reading a hundred unchanged packages each time is pure cost. - So verification recomputes exactly when the signature moved (a file was - added, removed, or written), and reuses the frozen digest otherwise. - """ - key = str(path.resolve()) - sig = _dir_signature(path) - with _hash_lock: - hit = _hash_cache.get(key) - if not fresh and hit and hit[0] == sig: - return hit[1] - digest = compute() - with _hash_lock: - _hash_cache[key] = (sig, digest) - return digest - - -def skill_dir_hash(path: Path, *, fresh: bool = False) -> str: - """Hash every package file; reject links and excess sizes rather than omit bytes.""" - from .archive import iter_file_stats - - key = str(path.resolve()) - entries = [item for item in iter_file_stats(path) if item[0] != _INVENTORY_NAME] - sig = tuple( - (rel, metadata.st_size, metadata.st_mtime_ns, metadata.st_ctime_ns, metadata.st_ino) - for rel, _, metadata in entries - ) - with _hash_lock: - hit = _hash_cache.get(key) - if not fresh and hit and hit[0] == sig: - return hit[1] - # Reuse this traversal's metadata; fresh verification still reads every byte. - digest = _hash_skill_package(path, entries=entries) - with _hash_lock: - _hash_cache[key] = (sig, digest) - return digest - - -def _hash_skill_package(path: Path, *, entries=None) -> str: - from core.agent_skills.binary_files import encode_upload - from core.services.desktop_capability_protocol import skill_content_hash - - from . import archive - from .errors import IntegrityFailed - from .junction import _native - - raw_files = {} - total = 0 - for rel, file, metadata in archive.iter_file_stats(path) if entries is None else entries: - if rel == _INVENTORY_NAME: - continue - size = metadata.st_size - total += size - if ( - size > archive.MAX_MEMBER_BYTES - or total > archive.MAX_TOTAL_BYTES - or len(raw_files) >= archive.MAX_MEMBERS - ): - raise IntegrityFailed("skill package exceeds content verification limits") - # Scanning already supports extended Windows paths; byte reads must too. - raw = _native(file).read_bytes() - if len(raw) != size: - raise IntegrityFailed("skill package changed during verification") - raw_files[rel] = raw - content_key = tuple((rel, hashlib.sha256(raw).digest()) for rel, raw in raw_files.items()) - with _hash_lock: - cached = _content_hash_cache.get(content_key) - if cached is not None: - _content_hash_cache.move_to_end(content_key) - return cached - files = {rel: encode_upload(rel, raw) for rel, raw in raw_files.items()} - digest = skill_content_hash(files.pop("SKILL.md", ""), files) - with _hash_lock: - _content_hash_cache[content_key] = digest - _content_hash_cache.move_to_end(content_key) - while len(_content_hash_cache) > 256: - _content_hash_cache.popitem(last=False) - return digest - - -# ── account profile ─────────────────────────────────────────────────── - - -def current_account_profile() -> Optional[str]: - """Profile id of the cloud account this device is currently bridged to.""" - from core.services.desktop_capability_protocol import token_subject - from core.services.desktop_cloud_bridge import get_state - - st = get_state() - if not st: - return None - subject = token_subject(str(st.get("token") or "")) - if not subject: - return None - return profile_id(str(st["cloud_base"]), subject) - - -_shadow_user_ids: Dict[str, str] = {} - - -def current_local_user_id() -> Optional[str]: - """Map the signed stable cloud subject to this device's shadow-user ID.""" - from core.services.desktop_cloud_bridge import get_identity_state - - identity = get_identity_state() - # 本机影子用户按壳送来的命名空间标识建档（cloud:::）， - # 查找必须用同一标识，而不是凭据里的原始云端 id。 - center = str((identity or {}).get("shell_user_center_id") or "") - if not center: - return None - # user_center_id → user_id is a primary-key mapping fixed at creation; only - # hits are remembered so a shadow created later is still found. - hit = _shadow_user_ids.get(center) - if hit is not None: - return hit - from core.db.models import UserShadow - - with registry._session() as db: - row = db.query(UserShadow).filter(UserShadow.user_center_id == center).first() - if row is None: - return None - _shadow_user_ids[center] = str(row.user_id) - return _shadow_user_ids[center] - - -def account_authorized_for(user_id: Optional[str]) -> bool: - from core.services.desktop_cloud_bridge import get_state - - st = get_state() - if not st or not str(st.get("token") or "").startswith("dcap2."): - return False - current = current_local_user_id() - return bool(user_id and current and str(user_id) == current) - - -# ── candidates ──────────────────────────────────────────────────────── - - -def builtin_candidates() -> List[Candidate]: - """安装包自带的技能。 - - 混合模式下它们不进能力中心、也不会被启用（目录层在 ``get_runtime_catalog`` 里就清空 - 了），但**候选必须留着**：云端技能会声明对自带技能的依赖，自带的那份还是云端副本在 - 本平台跑不了时的兜底，候选一空这两条链路就断了。 - """ - root = builtin_dir() - out: List[Candidate] = [] - if not root.is_dir(): - return out - for d in sorted(p for p in root.iterdir() if p.is_dir()): - if not (d / "SKILL.md").is_file(): - continue - out.append( - Candidate( - install_id=registry.install_id(KIND_SKILL, BUILTIN_PROFILE, d.name), - runtime_name=d.name, - kind=KIND_SKILL, - profile=BUILTIN_PROFILE, - source=SOURCE_BUILTIN, - path=d, - content_hash=None, # computed lazily only on a name collision - ref=builtin_ref(KIND_SKILL, d.name), - usable=True, - account_level=False, - display_name=d.name, - ) - ) - return out - - -def _installation_candidate(inst: registry.Installation, *, account_level: bool) -> Candidate: - path = None - if inst.ready: - from .errors import CapabilityError - - try: - comp = store.get(KIND_SKILL, inst.profile_id, inst.key, inst.resolved_revision or "") - except (CapabilityError, OSError): - comp = None - path = comp.path if comp else None - usable = inst.enabled and path is not None - return Candidate( - install_id=inst.install_id, - runtime_name=str(inst.payload.get("runtime_name") or inst.key), - kind=KIND_SKILL, - profile=inst.profile_id, - source=inst.source, - path=path, - content_hash=inst.payload.get("resolved_content_hash") or inst.content_hash, - revision=inst.resolved_revision, - ref=inst.ref, - usable=usable, - account_level=account_level and inst.enabled, - state=inst.state if path is not None or not inst.ready else "files_missing", - display_name=inst.display_name, - description=inst.description, - version=inst.version, - ) - - -def _installation_candidates(rows): - rows = list(rows) - if os.name == "nt" and len(rows) >= 8: - with ThreadPoolExecutor(max_workers=4, thread_name_prefix="cap-candidates") as pool: - futures = [ - pool.submit(copy_context().run, _installation_candidate, row, account_level=True) - for row in rows - ] - return [future.result() for future in futures] - return [_installation_candidate(row, account_level=True) for row in rows] - - -def local_candidates(user_id: Optional[str], *, shared_only: bool = False) -> List[Candidate]: - rows = [] - for inst in registry.list_installations(kind=KIND_SKILL, profile_id=LOCAL_PROFILE): - owner = inst.payload.get("owner_user_id") - if owner and (shared_only or owner != user_id): - continue - rows.append(inst) - return _installation_candidates(rows) - - -def account_candidates(profile: Optional[str] = None) -> List[Candidate]: - profile = profile or current_account_profile() - if not profile: - return [] - return _installation_candidates( - registry.list_installations(kind=KIND_SKILL, profile_id=profile) - ) - - -def candidates(user_id: Optional[str]) -> List[Candidate]: - return ( - builtin_candidates() - + local_candidates(user_id) - + (account_candidates() if account_authorized_for(user_id) else []) - ) - - -def _fill_hashes(cands: List[Candidate]) -> List[Candidate]: - """Hash only the members of multi-candidate names (identical-content detection).""" - by_name: Dict[str, List[Candidate]] = {} - for c in cands: - by_name.setdefault(c.runtime_name, []).append(c) - out: List[Candidate] = [] - for group in by_name.values(): - if len(group) < 2: - out.extend(group) - continue - for c in group: - if c.content_hash is None and c.path is not None: - c = Candidate(**{**c.__dict__, "content_hash": skill_dir_hash(c.path)}) - out.append(c) - return out - - -def _builtin_signature() -> Tuple: - """Shipped built-ins live on disk, outside every write signal — list them.""" - root = builtin_dir() - try: - return (str(root), tuple(sorted(p.name for p in root.iterdir() if p.is_dir()))) - except OSError: - return (str(root), ()) - - -def resolve_for_user(user_id: Optional[str], *, requested: Optional[Set[str]] = None) -> Resolution: - """Which candidate wins each runtime name, recomputed only when it can differ. - - The answer is a pure function of the installations, the store contents, the - user's name preferences and the bound cloud account — every one of which - moves ``registry.generation()``, ``view_generation()`` or the account - profile. Agent assembly asks for this several times per message and the - eligibility fixpoint walks every candidate's package each time, so without - this the same answer was recomputed from disk two or three times a turn. - """ - from .discovery import scan - from .readiness import eligible_skill_candidates - scan(user_id) - key = user_id or "" - signal = ( - registry.generation(), - view_generation(), - current_account_profile(), - _builtin_signature(), - frozenset(requested) if requested is not None else None, - ) - with _resolution_lock: - hit = _resolution_cache.get(key) - if hit is not None and hit[0] == signal: - _last_resolution[key] = hit[1] - return hit[1] - - preferences = registry.preferences(KIND_SKILL, user_id=user_id) - eligible = eligible_skill_candidates( - _fill_hashes(candidates(user_id)), user_id, preferences=preferences, requested=requested - ) - res = resolve(KIND_SKILL, eligible, preferences=preferences, requested=requested) - with _resolution_lock: - _last_resolution[key] = res - # A cloud deployment shares this process across users; keep the memo - # bounded rather than letting it grow with the user table. - if len(_resolution_cache) >= 64 and key not in _resolution_cache: - _resolution_cache.pop(next(iter(_resolution_cache)), None) - _resolution_cache[key] = (signal, res) - return res - - -def filter_available_names(names, *, user_id=None): - """Implicit defaults omit unusable names; explicit bindings still fail in preflight.""" - chosen = resolve_for_user(user_id).chosen - return [name for name in dict.fromkeys(names) if name in chosen] - - -def last_resolution(user_id: Optional[str]) -> Optional[Resolution]: - with _resolution_lock: - return _last_resolution.get(user_id or "") - - -# ── views ───────────────────────────────────────────────────────────── - - -def _targets(res: Resolution) -> Dict[str, Path]: - return {name: c.path for name, c in res.chosen.items() if c.path is not None} - - -def rebuild_device_view() -> view_mod.ViewReport: - res = resolve( - KIND_SKILL, - _fill_hashes(builtin_candidates() + local_candidates(None, shared_only=True)), - preferences=registry.preferences(KIND_SKILL), - ) - report = view_mod.build_view( - device_view_dir(), _targets(res), allowed_roots=allowed_link_roots() - ) - _log_report("device", report) - return report - - -def rebuild_user_view(user_id: str) -> Optional[view_mod.ViewReport]: - vdir = user_view_dir(user_id) - if vdir is None: - return None - res = resolve_for_user(user_id) - report = view_mod.build_view(vdir, _targets(res), allowed_roots=allowed_link_roots()) - _log_report(f"user:{user_id}", report) - return report - - -def rebuild_views(user_id: Optional[str]) -> Dict[str, view_mod.ViewReport]: - reports = {"device": rebuild_device_view()} - if user_id: - user_report = rebuild_user_view(user_id) - if user_report is not None: - reports["user"] = user_report - return reports - - -def _log_report(label: str, report: view_mod.ViewReport) -> None: - if report.changed: - logger.info( - "[caps-view] %s linked=%d relinked=%d removed=%d", - label, - len(report.linked), - len(report.relinked), - len(report.removed), - ) - for name, why in report.blocked.items(): - logger.warning("[caps-view] %s name '%s' unavailable: %s", label, name, why) - - -# ── local publication (device-created / DB-materialized skills) ─────── - - -def publish_local_skill( - skill_id: str, - *, - files: Dict[str, bytes | str], - content_hash: str, - owner_user_id: Optional[str] = None, - source: str = SOURCE_LOCAL, - source_plugin: Optional[str] = None, - display_name: str = "", - description: str = "", - version: str = "", - enabled: bool = True, - from_db: bool = False, -) -> store.StoredComponent: - """Write one revision of a device-local skill and make it the ready revision.""" - revision = revision_for_hash(content_hash) - before_generation = registry.generation() - published = False - ref = local_ref(KIND_SKILL, skill_id) - inst = registry.upsert( - profile_id=LOCAL_PROFILE, - ref=ref, - display_name=display_name or skill_id, - description=description, - version=version, - content_hash=content_hash, - source=source, - source_plugin=source_plugin, - payload={"owner_user_id": owner_user_id, "from_db": from_db}, - enabled=enabled, - ) - comp = store.get(KIND_SKILL, LOCAL_PROFILE, ref.key, revision) - if comp is None: - tx = registry.begin_transaction(inst.install_id, inst.generation + 1) - try: - comp = store.write_from_files(KIND_SKILL, LOCAL_PROFILE, ref.key, revision, files) - published = True - registry.advance_transaction(tx, "published", inventory=store.inventory(comp)) - except Exception as exc: - registry.advance_transaction(tx, "failed", error=str(exc)) - registry.set_state(inst.install_id, "failed", last_error=str(exc)) - raise - registry.set_state(inst.install_id, "ready", resolved_revision=revision) - registry.advance_transaction(tx, "committed") - elif inst.resolved_revision != revision or inst.state != "ready": - registry.set_state(inst.install_id, "ready", resolved_revision=revision) - # A DB-backed skill is materialized again after restart/cache refresh. An - # identical publication must not invalidate every user's resolved closure. - if published or registry.generation() != before_generation: - bump_view_generation() - return comp - - -def remove_local_skill(skill_id: str) -> bool: - from core.capabilities.runtime import references - - if references("skill", LOCAL_PROFILE, skill_id): - # Source deletion revokes use; history still needs its exact bytes. - return registry.mark_removed(registry.install_id("skill", LOCAL_PROFILE, skill_id)) - iid = registry.install_id(KIND_SKILL, LOCAL_PROFILE, skill_id) - removed_files = store.remove_key(KIND_SKILL, LOCAL_PROFILE, skill_id) > 0 - removed_row = registry.delete(iid) - registry.clear_preference(KIND_SKILL, skill_id) - bump_view_generation() - return removed_files or removed_row - - -def prune_local(live_skill_owners: Dict[str, Optional[str]]) -> int: - """Drop local-profile entries whose database row no longer exists.""" - removed = 0 - for inst in registry.list_installations(kind=KIND_SKILL, profile_id=LOCAL_PROFILE): - if inst.source in (SOURCE_LOCAL, SOURCE_PLUGIN) and inst.payload.get("from_db"): - if inst.key not in live_skill_owners: - registry.mark_removed(inst.install_id) - removed += 1 - # Unindexed and superseded bytes may still be retained by history snapshots. - return removed - - -# ── loader integration: resolver-driven merge of skill sources ───────── - - -def _info_candidate(info: Any) -> Candidate: - """Map a backend ``SkillFileInfo`` onto a resolver candidate.""" - src = info.source_name - origin = dict(getattr(info, "origin", None) or {}) - if src == "built-in": - return Candidate( - install_id=registry.install_id(KIND_SKILL, BUILTIN_PROFILE, info.skill_id), - runtime_name=info.skill_id, - kind=KIND_SKILL, - profile=BUILTIN_PROFILE, - source=SOURCE_BUILTIN, - path=Path(info.file_path).parent, - account_level=False, - ) - if src in ("cloud", "device-store"): - return Candidate( - install_id=str(origin.get("install_id") or ""), - runtime_name=info.skill_id, - kind=KIND_SKILL, - profile=str(origin.get("profile") or ""), - source=SOURCE_CLOUD if src == "cloud" else SOURCE_LOCAL, - path=Path(info.file_path).parent, - content_hash=origin.get("content_hash"), - revision=origin.get("revision"), - account_level=True, - ) - if src == "admin": - # Device catalog row (created here, installed from a plugin or the market). - return Candidate( - install_id=registry.install_id(KIND_SKILL, LOCAL_PROFILE, info.skill_id), - runtime_name=info.skill_id, - kind=KIND_SKILL, - profile=LOCAL_PROFILE, - source=SOURCE_LOCAL, - path=Path(""), - account_level=True, - ) - return Candidate( - install_id=registry.install_id(KIND_SKILL, src, info.skill_id), - runtime_name=info.skill_id, - kind=KIND_SKILL, - profile=src, - source=SOURCE_LOCAL, - path=Path(info.file_path).parent, - account_level=False, - ) - - -def merge_skill_infos(groups: Dict[str, List[Any]], *, backend_hash) -> Dict[str, Any]: - """Composite-backend merge hook: one winner per id, conflicts load nothing. - - ``backend_hash(info) -> str`` computes a content hash for a backend entry; it - is only called for ids with more than one candidate. - """ - cands: List[Candidate] = [] - lookup: Dict[str, Any] = {} - for skill_id, infos in groups.items(): - for info in infos: - cand = _info_candidate(info) - if len(infos) > 1 and cand.content_hash is None: - cand = Candidate(**{**cand.__dict__, "content_hash": backend_hash(info)}) - cands.append(cand) - lookup[cand.install_id] = info - res = resolve( - KIND_SKILL, - cands, - preferences=registry.preferences(KIND_SKILL, user_id=current_local_user_id()), - ) - with _resolution_lock: - _last_resolution["__loader__"] = res - for name in res.conflicts: - logger.warning("[caps] skill name conflict, nothing loaded for '%s'", name) - return {name: lookup[c.install_id] for name, c in res.chosen.items()} - - -def loader_resolution() -> Optional[Resolution]: - with _resolution_lock: - return _last_resolution.get("__loader__") - - -def conflicted_names() -> Set[str]: - res = loader_resolution() - return set(res.conflicts) if res else set() diff --git a/src/backend/core/agent_skills/__init__.py b/src/backend/core/capabilities/skills/__init__.py similarity index 97% rename from src/backend/core/agent_skills/__init__.py rename to src/backend/core/capabilities/skills/__init__.py index 4288a133..24812cf2 100644 --- a/src/backend/core/agent_skills/__init__.py +++ b/src/backend/core/capabilities/skills/__init__.py @@ -1,2 +1 @@ """Agent skills registry and loaders.""" - diff --git a/src/backend/core/capabilities/skills/backends/__init__.py b/src/backend/core/capabilities/skills/backends/__init__.py new file mode 100644 index 00000000..9a8eb52c --- /dev/null +++ b/src/backend/core/capabilities/skills/backends/__init__.py @@ -0,0 +1,14 @@ +"""Backend abstraction for multi-source skill loading.""" + +from core.capabilities.skills.backends.protocol import SkillBackendProtocol, SkillFileInfo +from core.capabilities.skills.backends.filesystem import FilesystemBackend +from core.capabilities.skills.backends.composite import CompositeBackend +from core.capabilities.skills.backends.database import DatabaseBackend + +__all__ = [ + "SkillBackendProtocol", + "SkillFileInfo", + "FilesystemBackend", + "CompositeBackend", + "DatabaseBackend", +] diff --git a/src/backend/core/agent_skills/backends/composite.py b/src/backend/core/capabilities/skills/backends/composite.py similarity index 97% rename from src/backend/core/agent_skills/backends/composite.py rename to src/backend/core/capabilities/skills/backends/composite.py index 1524b48d..361c7daa 100644 --- a/src/backend/core/agent_skills/backends/composite.py +++ b/src/backend/core/capabilities/skills/backends/composite.py @@ -5,7 +5,7 @@ import threading from typing import Any, Callable, Dict, List, Optional, Tuple -from .protocol import SkillBackendProtocol, SkillFileInfo +from core.capabilities.skills.backends.protocol import SkillBackendProtocol, SkillFileInfo MergeHook = Callable[ [Dict[str, List[SkillFileInfo]], Callable[[SkillFileInfo], str]], Dict[str, SkillFileInfo] @@ -115,7 +115,7 @@ def _select_visible(self, user_id: str | None) -> Dict[str, SkillFileInfo]: def content_hash(self, info: SkillFileInfo) -> str: """Content hash of one backend entry (used only for colliding ids).""" - from core.services.desktop_capability_protocol import skill_content_hash + from core.capabilities.manifest_protocol import skill_content_hash if info.origin and info.origin.get("content_hash"): return str(info.origin["content_hash"]) @@ -218,7 +218,7 @@ def get_extra_files(self, skill_id: str) -> dict: # A single store backend can contain same-named installations. Read the # selected concrete revision, not that backend's first runtime-name hit. if info.source_name in ("cloud", "device-store") and not info.is_database: - from .filesystem import FilesystemBackend + from core.capabilities.skills.backends.filesystem import FilesystemBackend path = info.file_path.parent return FilesystemBackend(path.parent, info.source_name).get_extra_files(path.name) # Find the owning backend and delegate diff --git a/src/backend/core/agent_skills/backends/database.py b/src/backend/core/capabilities/skills/backends/database.py similarity index 99% rename from src/backend/core/agent_skills/backends/database.py rename to src/backend/core/capabilities/skills/backends/database.py index 32014074..cb61205f 100644 --- a/src/backend/core/agent_skills/backends/database.py +++ b/src/backend/core/capabilities/skills/backends/database.py @@ -8,7 +8,7 @@ from sqlalchemy import func, or_ -from .protocol import SkillFileInfo +from core.capabilities.skills.backends.protocol import SkillFileInfo # Sentinel path used when skill content comes from DB (never actually read) _DB_SENTINEL = Path("/db/admin_skills") diff --git a/src/backend/core/agent_skills/backends/filesystem.py b/src/backend/core/capabilities/skills/backends/filesystem.py similarity index 98% rename from src/backend/core/agent_skills/backends/filesystem.py rename to src/backend/core/capabilities/skills/backends/filesystem.py index 702ce649..79838555 100644 --- a/src/backend/core/agent_skills/backends/filesystem.py +++ b/src/backend/core/capabilities/skills/backends/filesystem.py @@ -6,7 +6,7 @@ from pathlib import Path from typing import List -from .protocol import SkillBackendProtocol, SkillFileInfo +from core.capabilities.skills.backends.protocol import SkillFileInfo class FilesystemBackend: diff --git a/src/backend/core/agent_skills/backends/protocol.py b/src/backend/core/capabilities/skills/backends/protocol.py similarity index 100% rename from src/backend/core/agent_skills/backends/protocol.py rename to src/backend/core/capabilities/skills/backends/protocol.py diff --git a/src/backend/core/agent_skills/binary_files.py b/src/backend/core/capabilities/skills/binary_files.py similarity index 100% rename from src/backend/core/agent_skills/binary_files.py rename to src/backend/core/capabilities/skills/binary_files.py diff --git a/src/backend/core/capabilities/skills/cache_refresh.py b/src/backend/core/capabilities/skills/cache_refresh.py new file mode 100644 index 00000000..63dcdfba --- /dev/null +++ b/src/backend/core/capabilities/skills/cache_refresh.py @@ -0,0 +1,50 @@ +"""Shared cache invalidation after admin skill mutations. + +Why centralize: skill changes ripple through 4 caches (loader metadata, +catalog JSON, per-user capability cache, system prompt). Keeping them in +one place prevents drift between admin_skills and admin_skill_drafts routes. +""" + +from __future__ import annotations + +import logging + +logger = logging.getLogger(__name__) + + +def refresh_skill_caches() -> None: + from core.capabilities.skills.loader_factory import get_skill_loader + from core.capabilities.catalog.loader import invalidate_catalog_cache + from core.harness.registry.cache_invalidation import cache_invalidators + cache_invalidators().run("before_skill_refresh") + get_skill_loader(reset=True) + invalidate_catalog_cache() + + try: + from core.capabilities.catalog.resolver import invalidate_capability_cache + + invalidate_capability_cache() + except Exception as fallback_error: + from core.infra.failures import record_best_effort_failure + record_best_effort_failure('core.capabilities.skills.cache_refresh:refresh_skill_caches', fallback_error) + + try: + from core.capabilities.catalog.runtime import invalidate_runtime_catalog_cache + + invalidate_runtime_catalog_cache() + except Exception as fallback_error: + from core.infra.failures import record_best_effort_failure + record_best_effort_failure('core.capabilities.skills.cache_refresh:refresh_skill_caches', fallback_error) + + try: + cache_invalidators().run("after_skill_refresh") + except Exception as fallback_error: + from core.infra.failures import record_best_effort_failure + record_best_effort_failure('core.capabilities.skills.cache_refresh:refresh_skill_caches', fallback_error) + + try: + from core.capabilities.catalog.loader import load_catalog + + load_catalog(include_runtime_details=False) + except Exception as exc: + logger.warning("Failed to eagerly refresh catalog cache after skill mutation: %s", exc) diff --git a/src/backend/core/agent_skills/config.py b/src/backend/core/capabilities/skills/config.py similarity index 89% rename from src/backend/core/agent_skills/config.py rename to src/backend/core/capabilities/skills/config.py index 72216838..7faaf62b 100644 --- a/src/backend/core/agent_skills/config.py +++ b/src/backend/core/capabilities/skills/config.py @@ -76,9 +76,9 @@ def get_default_skill_sources() -> List[SkillSourceConfig]: ) ) - from core.capabilities.paths import capabilities_enabled + from core.harness.registry.run_capabilities import has_run_capabilities - caps_mode = capabilities_enabled() + caps_mode = has_run_capabilities() # 3. User skills (flat ``/SKILL.md`` folders). On a desktop with a # capability store this directory *is* ``/skills`` and is owned by @@ -127,12 +127,12 @@ def get_default_skill_sources() -> List[SkillSourceConfig]: # sources is decided by the resolver (core.capabilities.skills) and shown # to the user, never silently replaced. if caps_mode: - from core.capabilities.paths import kind_root + from core.harness.registry.local_publication import local_publication sources.append( SkillSourceConfig( name="cloud", - root_dir=kind_root("skill"), + root_dir=local_publication().skill_root(), priority=150, enabled=True, ) @@ -202,18 +202,18 @@ def sync_builtin_skills_to_sandbox_dir() -> int: Reconcile complete file trees rather than overlaying bundled files on DB overrides. Returns the number of configured built-in source entries. """ - from core.capabilities.paths import capabilities_enabled + from core.harness.registry.run_capabilities import has_run_capabilities - if capabilities_enabled(): + if has_run_capabilities(): # Desktop store: the shared dir is a link view — built-ins are linked to # the shipped bundle, nothing is copied. - from core.capabilities.skills import rebuild_device_view + from core.harness.registry.local_publication import local_publication - report = rebuild_device_view() + report = local_publication().rebuild_device_view() logger.info("[skills-sync] device view rebuilt → %s", report.view_dir) return len(report.linked) + len(report.relinked) - from .publication import prepare_skill_view + from core.capabilities.skills.publication import prepare_skill_view # Startup is a publisher too: resolve DB overrides and retired bundles # under the same lock as every other writer. @@ -241,9 +241,9 @@ def get_user_skills_root() -> Path: shared = get_sandbox_skills_dir() root = shared.parent / f"{shared.name}_u" root.mkdir(parents=True, exist_ok=True) - from core.capabilities.paths import capabilities_enabled + from core.harness.registry.run_capabilities import has_run_capabilities - if capabilities_enabled(): + if has_run_capabilities(): # Desktop store: user views link straight into the store, no shared hop. return root link = root / SHARED_LINK_NAME @@ -284,7 +284,7 @@ def get_user_skills_dir(user_id: Optional[str]) -> Optional[Path]: def skill_files_dir(skill_id: str, owner_user_id: Optional[str] = None) -> Path: """Where a skill's files are materialized: the owner's dir for private skills, the shared dir otherwise.""" - from .publication import safe_skill_id + from core.capabilities.skills.publication import safe_skill_id safe_skill_id(skill_id) owner_dir = get_user_skills_dir(owner_user_id) @@ -305,12 +305,12 @@ def sync_user_skill_view(user_id: Optional[str]) -> Optional[Path]: view = get_user_skills_dir(user_id) if view is None: return None - from core.capabilities.paths import capabilities_enabled + from core.harness.registry.run_capabilities import has_run_capabilities - if capabilities_enabled(): - from core.capabilities.skills import rebuild_user_view + if has_run_capabilities(): + from core.harness.registry.local_publication import local_publication - rebuild_user_view(user_id) + local_publication().rebuild_user_view(user_id) return view view.mkdir(parents=True, exist_ok=True) shared = get_sandbox_skills_dir() @@ -345,16 +345,16 @@ def purge_skill_sandbox_files(skill_id: str) -> bool: return False if (_builtin_skills_dir() / skill_id).is_dir(): return False - from core.capabilities.paths import capabilities_enabled + from core.harness.registry.run_capabilities import has_run_capabilities - if capabilities_enabled(): - from core.capabilities.skills import rebuild_views, remove_local_skill + if has_run_capabilities(): + from core.harness.registry.local_publication import local_publication - removed = remove_local_skill(skill_id) - rebuild_views(None) + removed = local_publication().remove_local_skill(skill_id) + local_publication().rebuild_views(None) return removed - from .publication import publication_lock, remove_entry + from core.capabilities.skills.publication import publication_lock, remove_entry with publication_lock(): removed = False @@ -374,7 +374,7 @@ def _builtin_skills_dir() -> Path: def prune_orphan_sandbox_skill_dirs(live_skill_owners: dict) -> int: - from .publication import publication_lock + from core.capabilities.skills.publication import publication_lock with publication_lock(): return _prune_orphan_sandbox_skill_dirs(live_skill_owners) @@ -391,12 +391,12 @@ def _prune_orphan_sandbox_skill_dirs(live_skill_owners: dict) -> int: """ import shutil - from core.capabilities.paths import capabilities_enabled + from core.harness.registry.run_capabilities import has_run_capabilities - if capabilities_enabled(): - from core.capabilities.skills import prune_local + if has_run_capabilities(): + from core.harness.registry.local_publication import local_publication - return prune_local(live_skill_owners) + return local_publication().prune_local(live_skill_owners) builtin = {d.name for d in _builtin_skills_dir().iterdir() if d.is_dir()} removed = 0 @@ -429,6 +429,6 @@ def _prune_orphan_sandbox_skill_dirs(live_skill_owners: dict) -> int: def model_facing_skill_dir(skill_id: str, real_dir: str) -> str: """Return the directory visible to this deployment's execution tools.""" - from core.llm.tools._paths import path_rules + from core.sandbox.workspace.resolution import path_rules return path_rules().skill_directory(skill_id, real_dir) diff --git a/src/backend/core/capabilities/skills/deps/__init__.py b/src/backend/core/capabilities/skills/deps/__init__.py new file mode 100644 index 00000000..a4fd4881 --- /dev/null +++ b/src/backend/core/capabilities/skills/deps/__init__.py @@ -0,0 +1 @@ +"""Detect, probe and request the runtime dependencies that skills declare.""" diff --git a/src/backend/core/services/skill_deps_aggregator.py b/src/backend/core/capabilities/skills/deps/aggregator.py similarity index 98% rename from src/backend/core/services/skill_deps_aggregator.py rename to src/backend/core/capabilities/skills/deps/aggregator.py index d38dd5e7..a50fce15 100644 --- a/src/backend/core/services/skill_deps_aggregator.py +++ b/src/backend/core/capabilities/skills/deps/aggregator.py @@ -27,7 +27,7 @@ _PIP_HEADER = ( - "# Auto-generated by core.services.skill_deps_aggregator. DO NOT EDIT MANUALLY.\n" + "# Auto-generated by core.capabilities.skills.deps.aggregator. DO NOT EDIT MANUALLY.\n" "# Managed via admin UI → 沙盒依赖 → 应用依赖。\n" ) _NPM_DEFAULT: Dict[str, Any] = { @@ -37,7 +37,7 @@ "dependencies": {}, } _APT_HEADER = ( - "# Auto-generated by core.services.skill_deps_aggregator. DO NOT EDIT MANUALLY.\n" + "# Auto-generated by core.capabilities.skills.deps.aggregator. DO NOT EDIT MANUALLY.\n" ) diff --git a/src/backend/core/agent_skills/deps_detector.py b/src/backend/core/capabilities/skills/deps/detector.py similarity index 100% rename from src/backend/core/agent_skills/deps_detector.py rename to src/backend/core/capabilities/skills/deps/detector.py diff --git a/src/backend/core/services/skill_deps_prober.py b/src/backend/core/capabilities/skills/deps/prober.py similarity index 100% rename from src/backend/core/services/skill_deps_prober.py rename to src/backend/core/capabilities/skills/deps/prober.py diff --git a/src/backend/core/services/skill_deps_request_service.py b/src/backend/core/capabilities/skills/deps/requests.py similarity index 99% rename from src/backend/core/services/skill_deps_request_service.py rename to src/backend/core/capabilities/skills/deps/requests.py index 1e5e9942..7abe9264 100644 --- a/src/backend/core/services/skill_deps_request_service.py +++ b/src/backend/core/capabilities/skills/deps/requests.py @@ -30,7 +30,7 @@ from sqlalchemy.orm import Session from core.db.models import AdminSkill, SkillDependencyRequest -from core.services.skill_deps_prober import names_of, probe_still_missing +from core.capabilities.skills.deps.prober import names_of, probe_still_missing logger = logging.getLogger(__name__) diff --git a/src/backend/core/services/skill_icon_service.py b/src/backend/core/capabilities/skills/icons.py similarity index 100% rename from src/backend/core/services/skill_icon_service.py rename to src/backend/core/capabilities/skills/icons.py diff --git a/src/backend/core/capabilities/skills/installation.py b/src/backend/core/capabilities/skills/installation.py new file mode 100644 index 00000000..ccd9c2ef --- /dev/null +++ b/src/backend/core/capabilities/skills/installation.py @@ -0,0 +1,117 @@ +"""Shared skill installation identity, content and credentials transformations.""" +from __future__ import annotations +import hashlib +import json +import re +from typing import Any, Dict, List, Optional +from core.capabilities.skills.registry import split_frontmatter +from core.infra.exceptions import BadRequestError + +SECRETS_FILENAME = "secrets.json" + +def strip_frontmatter(raw: str) -> str: + try: + _, body = split_frontmatter(raw) + except Exception: # noqa: BLE001 + body = raw + return (body or "").strip() + + +def owner_suffix(owner_user_id: str) -> str: + """Stable 6-char fingerprint derived from user_id, used to deduplicate private-install skill ids.""" + return hashlib.sha1(owner_user_id.encode("utf-8")).hexdigest()[:6] + + +def compute_install_id(entry_name: str, owner_user_id: Optional[str]) -> str: + """The installed AdminSkill.skill_id: admin = original name (global); user = original name + fingerprint (private).""" + if owner_user_id is None: + return entry_name + return f"{entry_name}-{owner_suffix(owner_user_id)}" + + +def base_entry_name(skill_id: str, owner_user_id: Optional[str]) -> str: + """Inverse of ``compute_install_id``: recover the underlying market entry_name from an install id. + + Private install ids look like ``{entry_name}-{fingerprint}``; stripping that + user's 6-char fingerprint suffix yields the entry_name. Global ids (owner is + None) already are the entry_name and are returned as-is. Used to merge the + "privately installed version" and the "admin global version" onto the same + underlying skill for deduplication — preventing the same skill from + appearing twice in the skill library / at runtime. + """ + if owner_user_id is None: + return skill_id + suffix = f"-{owner_suffix(owner_user_id)}" + if skill_id.endswith(suffix): + return skill_id[: -len(suffix)] + return skill_id + + +def rewrite_frontmatter_name(content: str, new_name: str) -> str: + """Rewrite the ``name:`` in the SKILL.md frontmatter to ``new_name`` (touching only that line). + + Replaces only the first ``name:`` line within the frontmatter block; all + other content (including multi-line descriptions) is preserved verbatim. + Returned unchanged when there is no frontmatter. + """ + if not content.startswith("---"): + return content + end = content.find("\n---", 3) + if end == -1: + return content + head = content[: end + 1] + tail = content[end + 1 :] + head = re.sub(r"(?m)^name:[ \t]*.*$", f"name: {new_name}", head, count=1) + return head + tail + + +def inject_secrets( + skill_content: str, + extra_files: Dict[str, str], + required_secrets: List[Dict[str, Any]], + secrets: Dict[str, str], +) -> str: + """Write the user-provided credentials into secrets.json and append a "credentials configuration" section at the end of SKILL.md. + + Returns the SKILL.md content with the section appended. Missing required + fields raise a 400. + """ + cleaned: Dict[str, str] = {} + for field in required_secrets: + key = str(field.get("key") or "").strip() + if not key: + continue + val = (secrets.get(key) or "").strip() + if not val: + if field.get("required"): + label = field.get("label") or key + raise BadRequestError(message=f"请填写必填凭据：{label}") + continue + cleaned[key] = val + if not cleaned: + return skill_content + + extra_files[SECRETS_FILENAME] = json.dumps(cleaned, ensure_ascii=False, indent=2) + lines = [ + "", + "---", + "", + "## 🔑 凭据配置（由安装者提供）", + "", + f"运行本技能所需的凭据已写入技能目录下的 `{SECRETS_FILENAME}`，包含以下字段：", + "", + ] + for key in cleaned: + lines.append(f"- `{key}`") + lines += [ + "", + f"执行技能脚本前，请先从 `{SECRETS_FILENAME}` 读取这些值，并以**同名环境变量**导出后再运行，例如：", + "", + "```bash", + f'export {next(iter(cleaned))}="$(python3 -c "import json;print(json.load(open(\'{SECRETS_FILENAME}\'))[\'{next(iter(cleaned))}\'])")"', + "```", + "", + "> 凭据仅存于本技能目录，请勿在对话中明文回显。", + "", + ] + return skill_content.rstrip() + "\n" + "\n".join(lines) diff --git a/src/backend/core/agent_skills/loader.py b/src/backend/core/capabilities/skills/loader.py similarity index 77% rename from src/backend/core/agent_skills/loader.py rename to src/backend/core/capabilities/skills/loader.py index 2d7ead9e..500c17ad 100644 --- a/src/backend/core/agent_skills/loader.py +++ b/src/backend/core/capabilities/skills/loader.py @@ -1,88 +1,17 @@ -"""Multi-source skill loader with backend abstraction.""" - +"""Load skills from several sources, resolving conflicts by source priority.""" from __future__ import annotations - -import logging +from core.harness.registry.local_publication import local_publication import time -from pathlib import Path from typing import Any, Dict, List, Optional, Tuple +from core.capabilities.skills.backends import CompositeBackend, DatabaseBackend, FilesystemBackend +from core.capabilities.skills.binary_files import decode_binary, is_binary_value +from core.capabilities.skills.config import get_enabled_skill_sources, skill_files_dir +from core.capabilities.skills.registry import AgentSkillMetadata, AgentSkillSpec, _load_skill_from_file, _load_skill_from_str, _load_skill_metadata_from_file, load_skill_metadata_from_str, parse_scripts_json +import logging +from core.capabilities.skills import script_discovery logger = logging.getLogger(__name__) -from .backends import CompositeBackend, DatabaseBackend, FilesystemBackend, SkillFileInfo -from .binary_files import decode_binary, is_binary_value -from .config import get_enabled_skill_sources, skill_files_dir -from .registry import ( - AgentSkillMetadata, - AgentSkillSpec, - _load_skill_from_file, - _load_skill_from_str, - _load_skill_metadata_from_file, - _load_skill_metadata_from_str, - parse_scripts_json, -) - - -_SCRIPT_EXTENSIONS = {".py", ".js", ".sh", ".r", ".R"} - - -def _bound_skill_id(skill_info: SkillFileInfo, declared_id: str) -> str: - # Capability names come from the resolved view, independently of the file's - # original frontmatter. Ordinary source files keep their existing ID rules. - if skill_info.source_name == "prepared" or (skill_info.origin or {}).get("install_id"): - return skill_info.skill_id - return declared_id - - -def _auto_detect_scripts( - extra_files: Dict[str, str], - skill_info: "SkillFileInfo", -) -> List[Dict]: - """Auto-detect executable scripts when _scripts.json is absent. - - Scans extra_files (DB skills) or the skill directory (filesystem skills) - for files with known script extensions and returns a whitelist entry for - each one, so users don't need to manually create _scripts.json. - """ - scripts: List[Dict] = [] - - # Collect candidate filenames - candidates: List[str] = [] - - # From extra_files (DB skills) - for fname in extra_files: - if fname.startswith("_"): - continue - ext = Path(fname).suffix.lower() - if ext in _SCRIPT_EXTENSIONS: - candidates.append(fname) - - # From filesystem (filesystem skills without extra_files) - if not candidates and not skill_info.is_database and skill_info.content is None: - skill_dir = skill_info.file_path.parent - for p in skill_dir.iterdir(): - if ( - p.is_file() - and not p.name.startswith("_") - and p.suffix.lower() in _SCRIPT_EXTENSIONS - ): - candidates.append(p.name) - - # Build whitelist entries - ext_to_lang = {".py": "python", ".js": "javascript", ".sh": "bash", ".r": "r", ".R": "r"} - for name in sorted(candidates): - ext = Path(name).suffix.lower() - scripts.append( - { - "name": name, - "language": ext_to_lang.get(ext, "python"), - "timeout": 60, - } - ) - - return scripts - - class MultiSourceSkillLoader: """Loads skills from multiple sources with priority-based conflict resolution. @@ -117,7 +46,7 @@ def _create_default_backend() -> CompositeBackend: Returns: CompositeBackend configured with enabled skill sources. """ - from core.capabilities.paths import capabilities_enabled + from core.harness.registry.run_capabilities import has_run_capabilities sources = get_enabled_skill_sources() backends = [] @@ -125,9 +54,8 @@ def _create_default_backend() -> CompositeBackend: if src.name == "admin": backends.append(DatabaseBackend(priority=src.priority)) elif src.name == "cloud": - from .backends.capability_store import CapabilityStoreBackend - backends.append(CapabilityStoreBackend(priority=src.priority)) + backends.append(local_publication().skill_backend(priority=src.priority)) else: backends.append( FilesystemBackend( @@ -136,18 +64,16 @@ def _create_default_backend() -> CompositeBackend: priority=src.priority, ) ) - if not capabilities_enabled(): + if not has_run_capabilities(): return CompositeBackend(backends) - from .backends.capability_store import CapabilityStoreBackend - backends.append(CapabilityStoreBackend(local=True)) + backends.append(local_publication().skill_backend(local=True)) # Desktop store: same-id clashes go to the resolver, never to priority. - from core.capabilities.skills import merge_skill_infos return CompositeBackend( backends, - merge=lambda groups, backend_hash: merge_skill_infos(groups, backend_hash=backend_hash), + merge=lambda groups, backend_hash: local_publication().merge_skill_infos(groups, backend_hash=backend_hash), ) def _get_backend_change_token(self) -> Optional[Any]: @@ -203,14 +129,14 @@ def load_all_metadata(self) -> Dict[str, AgentSkillMetadata]: mcp_server_ids=list(item.get("mcp_server_ids") or []), ) elif skill_info.content is not None: - metadata = _load_skill_metadata_from_str( + metadata = load_skill_metadata_from_str( skill_info.content, skill_info.skill_id ) else: metadata = _load_skill_metadata_from_file(skill_info.file_path) # Add source information to skill_path for debugging metadata_with_source = AgentSkillMetadata( - id=_bound_skill_id(skill_info, metadata.id), + id=script_discovery.bound_skill_id(skill_info, metadata.id), name=metadata.name, description=metadata.description, version=metadata.version, @@ -263,8 +189,9 @@ def load_skill_full(self, skill_id: str) -> Optional[AgentSkillSpec]: try: ef = self._backend.get_extra_files(skill_id) extra_file_names = sorted(ef.keys()) - except Exception: - pass + except Exception as fallback_error: + from core.infra.failures import record_best_effort_failure + record_best_effort_failure('core.capabilities.skills.loader:load_skill_full', fallback_error) # Resolve base_dir for {baseDir} substitution base_dir = "" @@ -296,7 +223,7 @@ def load_skill_full(self, skill_id: str) -> Optional[AgentSkillSpec]: # Auto-generate whitelist from .py files when _scripts.json is absent if not executable_scripts: - auto_scripts = _auto_detect_scripts(ef, skill_info) + auto_scripts = script_discovery.auto_detect_scripts(ef, skill_info) if auto_scripts: executable_scripts = auto_scripts logger.info( @@ -308,7 +235,7 @@ def load_skill_full(self, skill_id: str) -> Optional[AgentSkillSpec]: # Add source information to skill_path spec_with_source = AgentSkillSpec( - id=_bound_skill_id(skill_info, spec.id), + id=script_discovery.bound_skill_id(skill_info, spec.id), name=spec.name, description=spec.description, version=spec.version, @@ -402,10 +329,10 @@ def _materialize_skill_files( Returns: Absolute path to the materialized directory. """ - from core.capabilities.paths import capabilities_enabled - from .publication import decode_files, publication_lock, publish_tree + from core.harness.registry.run_capabilities import has_run_capabilities + from core.capabilities.skills.publication import decode_files, publication_lock, publish_tree - if capabilities_enabled(): + if has_run_capabilities(): # The desktop capability store already publishes immutable revisions. content, current_files, owner = self._backend.read_snapshot(skill_id) return self._publish_to_store(skill_id, content, current_files, owner) @@ -426,16 +353,15 @@ def _publish_to_store( owner: Optional[str], ) -> str: """Desktop store path of a DB skill: publish its current content, link the views.""" - from core.capabilities import skills as caps_skills - from core.services.desktop_capability_protocol import skill_content_hash + from core.capabilities.manifest_protocol import skill_content_hash info = self._backend.get_skill_info(skill_id) meta = dict(info.metadata or {}) if info else {} files: Dict[str, bytes | str] = {"SKILL.md": content} for filename, body in extra_files.items(): files[filename] = decode_binary(body) if is_binary_value(body) else str(body) - before_views = caps_skills.view_generation() - comp = caps_skills.publish_local_skill( + before_views = local_publication().view_generation() + comp = local_publication().publish_local_skill( skill_id, files=files, content_hash=skill_content_hash(content, extra_files), @@ -451,8 +377,8 @@ def _publish_to_store( # view generation exactly when that set moved. Re-materializing unchanged # content leaves every existing link pointing at the right revision, so # relinking every name again would only re-derive the same answer. - if caps_skills.view_generation() != before_views: - caps_skills.rebuild_views(owner) + if local_publication().view_generation() != before_views: + local_publication().rebuild_views(owner) logger.info("Published skill '%s' to store revision %s", skill_id, comp.revision) return str(comp.path) @@ -510,18 +436,10 @@ def register_skills_to_toolkit( skill_dir = self.get_skill_dir(sid) if skill_dir is None: continue - from core.capabilities.paths import capabilities_enabled - - prepared = getattr(self, "capability_run", None) - if prepared is not None or capabilities_enabled(): - # Native Toolkit deduplicates by Skill.name. Preserve the - # resolved alias even if frontmatter keeps a source name or - # the physical directory ends in a content revision. - toolkit.register_agent_skill( - skill_dir, runtime_name=sid, capability_run=prepared - ) - else: - toolkit.register_agent_skill(skill_dir) + from core.capabilities.skills.runtime_loader import RuntimeNamedSkillLoader + toolkit.skill_loaders.append(RuntimeNamedSkillLoader( + skill_dir, sid, getattr(self, "capability_run", None) + )) count += 1 except Exception as e: logger.warning("Failed to register skill '%s': %s", sid, e) @@ -539,22 +457,3 @@ def get_skill_source(self, skill_id: str) -> Optional[str]: self._sync_backend_cache() skill_info = self._backend.get_skill_info(skill_id) return skill_info.source_name if skill_info else None - - -# Global singleton instance -_global_loader: Optional[MultiSourceSkillLoader] = None - - -def get_skill_loader(reset: bool = False) -> MultiSourceSkillLoader: - """Get the global MultiSourceSkillLoader instance. - - Args: - reset: If True, recreate the loader (useful for testing). - - Returns: - The global MultiSourceSkillLoader instance. - """ - global _global_loader - if _global_loader is None or reset: - _global_loader = MultiSourceSkillLoader() - return _global_loader diff --git a/src/backend/core/capabilities/skills/loader_factory.py b/src/backend/core/capabilities/skills/loader_factory.py new file mode 100644 index 00000000..77a68da8 --- /dev/null +++ b/src/backend/core/capabilities/skills/loader_factory.py @@ -0,0 +1,21 @@ +"""The process-wide skill loader instance.""" +from __future__ import annotations +from core.capabilities.skills import loader +from typing import Optional + +global_loader: Optional[loader.MultiSourceSkillLoader] = None + + +def get_skill_loader(reset: bool = False) -> loader.MultiSourceSkillLoader: + """Get the global MultiSourceSkillLoader instance. + + Args: + reset: If True, recreate the loader (useful for testing). + + Returns: + The global MultiSourceSkillLoader instance. + """ + global global_loader + if global_loader is None or reset: + global_loader = loader.MultiSourceSkillLoader() + return global_loader diff --git a/src/backend/core/services/skill_management_service.py b/src/backend/core/capabilities/skills/management.py similarity index 93% rename from src/backend/core/services/skill_management_service.py rename to src/backend/core/capabilities/skills/management.py index 14fa8d7c..d2df3c9b 100644 --- a/src/backend/core/services/skill_management_service.py +++ b/src/backend/core/capabilities/skills/management.py @@ -7,10 +7,10 @@ from datetime import datetime from typing import Any, Dict, List, Optional -from core.agent_skills.registry import _split_frontmatter +from core.capabilities.skills.registry import split_frontmatter from core.db.models import AdminMcpServer, AdminSkill from core.infra.exceptions import BadRequestError -from core.ontology.build_validator import ensure_ontology_build_valid +from core.harness.registry.build_governance import ensure_ontology_build_valid from fastapi import HTTPException from sqlalchemy.orm import Session from sqlalchemy.orm.attributes import flag_modified @@ -21,7 +21,7 @@ def refresh_skill_caches() -> None: """Refresh skill caches without turning a committed mutation into an HTTP 500.""" try: - from core.agent_skills.cache_refresh import refresh_skill_caches as refresh + from core.capabilities.skills.cache_refresh import refresh_skill_caches as refresh refresh() except Exception as exc: # noqa: BLE001 @@ -34,7 +34,7 @@ def _sanitize_frontmatter_value(value: str) -> str: def _parse_frontmatter_list(skill_content: Optional[str], field_name: str) -> List[str]: try: - frontmatter, _ = _split_frontmatter(skill_content or "") + frontmatter, _ = split_frontmatter(skill_content or "") except Exception: return [] raw = str(frontmatter.get(field_name, "") or "") @@ -97,10 +97,10 @@ def resolve_ontology_workflows(db: Session, tags: List[str]) -> List[str]: selected = {tag for tag in tags if isinstance(tag, str) and tag.startswith("ontology:")} if not selected: return [] - from core.services.ontology_service import OntologyService + from core.harness.registry.build_governance import list_asset_tag_options workflow_refs: List[str] = [] - for option in OntologyService(db).list_asset_tag_options("skill"): + for option in list_asset_tag_options(db, "skill"): if option.get("value") not in selected: continue for workflow in option.get("workflows") or []: @@ -145,7 +145,7 @@ def build_skill_content( def validate_skill_file_path(filename: str) -> str: """Validate and normalize a relative skill-file path.""" - from core.agent_skills.publication import normalize_file_path + from core.capabilities.skills.publication import normalize_file_path # Keep the established leading-slash UI convention; ZIP paths are strict. name = (filename or "").strip().strip("/") @@ -158,7 +158,7 @@ def validate_skill_file_path(filename: str) -> str: def extract_instructions(skill_content: Optional[str]) -> str: """Return the Markdown body after the ``SKILL.md`` frontmatter.""" try: - _, body = _split_frontmatter(skill_content or "") + _, body = split_frontmatter(skill_content or "") except Exception: return "" return (body or "").strip() @@ -171,7 +171,7 @@ def parse_and_upsert_skill_zip( owner_user_id: Optional[str] = None, ) -> Dict[str, Any]: """Parse a skill zip and upsert a global or owner-isolated DB skill.""" - from core.services.marketplace_service import parse_skill_zip + from core.capabilities.skills.zip_reader import parse_skill_zip parsed = parse_skill_zip(data) skill_id = parsed["skill_id"] diff --git a/src/backend/core/agent_skills/publication.py b/src/backend/core/capabilities/skills/publication.py similarity index 89% rename from src/backend/core/agent_skills/publication.py rename to src/backend/core/capabilities/skills/publication.py index 25b33d98..41370b2a 100644 --- a/src/backend/core/agent_skills/publication.py +++ b/src/backend/core/capabilities/skills/publication.py @@ -13,21 +13,22 @@ import tempfile import threading from contextlib import contextmanager +from dataclasses import dataclass, field from pathlib import Path, PurePosixPath -from core.capabilities.lockfile import locked as file_lock +from core.infra.file_lock import locked as file_lock logger = logging.getLogger(__name__) -_thread_lock = threading.RLock() + _local = threading.local() @contextmanager def publication_lock(): - from .config import get_sandbox_skills_dir + from core.capabilities.skills.config import get_sandbox_skills_dir - with _thread_lock: + with process_state().thread_lock: if getattr(_local, "held", False): yield return @@ -66,7 +67,7 @@ def safe_skill_id(skill_id: str) -> str: def decode_files(content: str, extra: dict) -> dict[str, bytes]: - from .binary_files import decode_binary, is_binary_value + from core.capabilities.skills.binary_files import decode_binary, is_binary_value files = {"SKILL.md": content.encode("utf-8")} for name, value in extra.items(): @@ -105,7 +106,7 @@ def publish_tree( target: Path, files: dict[str, bytes], *, modes: dict[str, int] | None = None ) -> Path: """Publish an exact tree; failure before the swap leaves the old tree intact.""" - from .config import get_sandbox_skills_dir + from core.capabilities.skills.config import get_sandbox_skills_dir safe_skill_id(target.name) files = {safe_relative(name): data for name, data in files.items()} @@ -161,15 +162,11 @@ def prepare_skill_view(user_id: str | None = None) -> Path | None: The publication lock also fences old loader instances. Preparation errors propagate: serving an unreconciled directory would violate deletion semantics. """ - from core.capabilities.paths import capabilities_enabled - from .config import ( - get_sandbox_skills_dir, - get_user_skills_dir, - sync_user_skill_view, - ) - from .loader import MultiSourceSkillLoader - - if capabilities_enabled(): + from core.harness.registry.run_capabilities import has_run_capabilities + from core.capabilities.skills.config import get_sandbox_skills_dir, get_user_skills_dir, sync_user_skill_view + from core.capabilities.skills.loader import MultiSourceSkillLoader + + if has_run_capabilities(): return sync_user_skill_view(user_id) with publication_lock(): backend = MultiSourceSkillLoader._create_default_backend() @@ -195,7 +192,7 @@ def prepare_skill_view(user_id: str | None = None) -> Path | None: def _publish_selected(backend, user_id, private_only, wanted_shared, wanted_private): - from .config import skill_files_dir + from core.capabilities.skills.config import skill_files_dir for info in backend.list_skill_files(): owner = (info.metadata or {}).get("owner_user_id") @@ -225,3 +222,15 @@ def _publish_selected(backend, user_id, private_only, wanted_shared, wanted_priv modes = {name: (info.file_path.parent / name).stat().st_mode & 0o777 for name in files} publish_tree(target, files, modes=modes) (wanted_private if owner else wanted_shared).add(info.skill_id) + + +@dataclass +class PublicationState: + thread_lock: object = field(default_factory=lambda: threading.RLock()) + + +_STATE = PublicationState() + + +def process_state() -> PublicationState: + return _STATE diff --git a/src/backend/core/agent_skills/registry.py b/src/backend/core/capabilities/skills/registry.py similarity index 97% rename from src/backend/core/agent_skills/registry.py rename to src/backend/core/capabilities/skills/registry.py index 785d4fe0..6bb35389 100644 --- a/src/backend/core/agent_skills/registry.py +++ b/src/backend/core/capabilities/skills/registry.py @@ -66,7 +66,7 @@ def _require_id(value: str) -> str: return v -def _split_frontmatter(raw: str) -> Tuple[Dict[str, str], str]: +def split_frontmatter(raw: str) -> Tuple[Dict[str, str], str]: # ZIP uploads are commonly created on Windows, where Markdown files use # CRLF line endings. Normalize all newline styles before matching the # frontmatter delimiters so a valid ``---`` header is platform-agnostic. @@ -215,9 +215,9 @@ def _parse_mcp_server_ids(fm: Dict[str, str]) -> List[str]: ) -def _load_skill_metadata_from_str(content: str, skill_id: str) -> AgentSkillMetadata: +def load_skill_metadata_from_str(content: str, skill_id: str) -> AgentSkillMetadata: """Load only metadata from SKILL.md content string (for DB-backed skills).""" - fm, body = _split_frontmatter(content) + fm, body = split_frontmatter(content) skill_id = _require_id(fm.get("name", skill_id)) name = fm.get("display_name", "") or skill_id @@ -243,7 +243,7 @@ def _load_skill_metadata_from_str(content: str, skill_id: str) -> AgentSkillMeta def _load_skill_from_str(content: str, skill_id: str) -> AgentSkillSpec: """Load full skill spec from SKILL.md content string (for DB-backed skills).""" - fm, body = _split_frontmatter(content) + fm, body = split_frontmatter(content) skill_id = _require_id(fm.get("name", skill_id)) name = fm.get("display_name", "") or skill_id @@ -280,7 +280,7 @@ def _load_skill_from_str(content: str, skill_id: str) -> AgentSkillSpec: def _load_skill_metadata_from_file(path: Path) -> AgentSkillMetadata: """Load only metadata from SKILL.md (fast, no body parsing).""" raw = path.read_text(encoding="utf-8") - fm, body = _split_frontmatter(raw) + fm, body = split_frontmatter(raw) folder_id = _require_id(path.parent.name) skill_id = _require_id(fm.get("name", folder_id)) @@ -308,7 +308,7 @@ def _load_skill_metadata_from_file(path: Path) -> AgentSkillMetadata: def _load_skill_from_file(path: Path) -> AgentSkillSpec: """Load full skill spec from SKILL.md (includes instructions).""" raw = path.read_text(encoding="utf-8") - fm, body = _split_frontmatter(raw) + fm, body = split_frontmatter(raw) folder_id = _require_id(path.parent.name) skill_id = _require_id(fm.get("name", folder_id)) diff --git a/src/backend/core/capabilities/skills/runtime_loader.py b/src/backend/core/capabilities/skills/runtime_loader.py new file mode 100644 index 00000000..6abc1b34 --- /dev/null +++ b/src/backend/core/capabilities/skills/runtime_loader.py @@ -0,0 +1,115 @@ +"""Run-scoped skill loaders with authorized names and bounded desktop I/O.""" +from __future__ import annotations +import asyncio +import logging +import os +from dataclasses import replace +from functools import lru_cache +from pathlib import Path +import frontmatter +from agentscope.skill import LocalSkillLoader, SkillLoaderBase, Skill +from core.capabilities.skills.config import model_facing_skill_dir +logger = logging.getLogger(__name__) + +def _parse_runtime_skill(text): + content = frontmatter.loads(text) + name, description = content.get("name"), content.get("description") + if not name or not description: + return None + return str(name), str(description), content.content + + +_cached_runtime_skill = lru_cache(maxsize=256)(_parse_runtime_skill) + + +class RuntimeNamedSkillLoader(SkillLoaderBase): + """Read frozen files and expose a deployment-appropriate directory.""" + + def __init__(self, directory: str, runtime_name: str, capability_run=None) -> None: + self.directory = directory + self.runtime_name = runtime_name + self.capability_run = capability_run + self._physical_loader = LocalSkillLoader(directory) + + def _list_desktop_skill(self): + from core.harness.registry.capability_runtime import validate + from core.harness.capability_errors import CapabilityError + + # Keep authorization outside the parse-error handling: it must fail the + # whole enumeration, never quietly omit a revoked or modified skill. + try: + validate(self.capability_run, only_skill=self.runtime_name) + except (OSError, CapabilityError): + # 被撤销或改动过的单个技能只从枚举里消失，不牵连这一轮其它工具。 + return [] + try: + entry = Path(self.directory) / "SKILL.md" + updated_at = entry.stat().st_mtime + text = entry.read_text(encoding="utf-8") + parsed = ( + _cached_runtime_skill(text) if len(text) <= 262144 else _parse_runtime_skill(text) + ) + if parsed is None: + return [] + _, description, markdown = parsed + return [ + Skill( + name=self.runtime_name, + description=description, + dir=model_facing_skill_dir(self.runtime_name, self.directory), + markdown=markdown, + updated_at=updated_at, + ) + ] + except Exception as exc: + logger.warning("Failed to load skill metadata: %s", type(exc).__name__) + return [] + + async def list_skills(self): + # AgentScope's general filesystem scanner crosses the executor boundary + # eight times for one known file. Batch that I/O in one worker while + # preserving its parser, fields and sandbox alias. + if os.name == "nt" and self.capability_run is not None: + return await asyncio.to_thread(self._list_desktop_skill) + from core.harness.registry.capability_runtime import validate + + if self.capability_run is not None: + # Re-check only this loader's own skill (authorization + fresh hash), + # not the whole run. N loaders each re-hashing all N skills — twice + # per assembly — was O(N²) filesystem work and the dominant cost of + # desktop agent setup; scoping to one skill makes it O(N). + from core.harness.capability_errors import CapabilityError + + try: + await asyncio.to_thread(validate, self.capability_run, only_skill=self.runtime_name) + except (CapabilityError, OSError): + return [] + physical = await self._physical_loader.list_skills() + return [ + replace(skill, name=self.runtime_name, dir=model_facing_skill_dir(self.runtime_name, self.directory)) + for skill in physical + ] + + +class ParallelRuntimeSkillLoader(SkillLoaderBase): + """Enumerate independent desktop loaders with bounded I/O concurrency.""" + + def __init__(self, loaders): + self._loaders = tuple(loaders) + + async def list_skills(self): + limit = asyncio.Semaphore(4) + + async def load(loader): + async with limit: + return await loader.list_skills() + + results = await asyncio.gather( + *(load(loader) for loader in self._loaders), return_exceptions=True + ) + skills = [] + for result in results: + if isinstance(result, BaseException): + raise result + skills.extend(result) + return skills diff --git a/src/backend/core/capabilities/skills/script_discovery.py b/src/backend/core/capabilities/skills/script_discovery.py new file mode 100644 index 00000000..3add1837 --- /dev/null +++ b/src/backend/core/capabilities/skills/script_discovery.py @@ -0,0 +1,65 @@ +"""Find the executable scripts a skill ships when it has no _scripts.json.""" +from __future__ import annotations +from pathlib import Path +from typing import Dict, List +from core.capabilities.skills.backends import SkillFileInfo + + +SCRIPT_EXTENSIONS = {".py", ".js", ".sh", ".r", ".R"} + + +def bound_skill_id(skill_info: SkillFileInfo, declared_id: str) -> str: + # Capability names come from the resolved view, independently of the file's + # original frontmatter. Ordinary source files keep their existing ID rules. + if skill_info.source_name == "prepared" or (skill_info.origin or {}).get("install_id"): + return skill_info.skill_id + return declared_id + + +def auto_detect_scripts( + extra_files: Dict[str, str], + skill_info: "SkillFileInfo", +) -> List[Dict]: + """Auto-detect executable scripts when _scripts.json is absent. + + Scans extra_files (DB skills) or the skill directory (filesystem skills) + for files with known script extensions and returns a whitelist entry for + each one, so users don't need to manually create _scripts.json. + """ + scripts: List[Dict] = [] + + # Collect candidate filenames + candidates: List[str] = [] + + # From extra_files (DB skills) + for fname in extra_files: + if fname.startswith("_"): + continue + ext = Path(fname).suffix.lower() + if ext in SCRIPT_EXTENSIONS: + candidates.append(fname) + + # From filesystem (filesystem skills without extra_files) + if not candidates and not skill_info.is_database and skill_info.content is None: + skill_dir = skill_info.file_path.parent + for p in skill_dir.iterdir(): + if ( + p.is_file() + and not p.name.startswith("_") + and p.suffix.lower() in SCRIPT_EXTENSIONS + ): + candidates.append(p.name) + + # Build whitelist entries + ext_to_lang = {".py": "python", ".js": "javascript", ".sh": "bash", ".r": "r", ".R": "r"} + for name in sorted(candidates): + ext = Path(name).suffix.lower() + scripts.append( + { + "name": name, + "language": ext_to_lang.get(ext, "python"), + "timeout": 60, + } + ) + + return scripts diff --git a/src/backend/core/agent_skills/selection_record.py b/src/backend/core/capabilities/skills/selection_record.py similarity index 98% rename from src/backend/core/agent_skills/selection_record.py rename to src/backend/core/capabilities/skills/selection_record.py index 2c83ac4d..d992d75e 100644 --- a/src/backend/core/agent_skills/selection_record.py +++ b/src/backend/core/capabilities/skills/selection_record.py @@ -149,9 +149,9 @@ def select_with_record( ) try: - from core.agent_skills.selector import select_skills_for_query + from core.harness.registry.model_services import rank_skills - selected = select_skills_for_query( + selected = rank_skills( user_query=user_query, available_skill_ids=available_skill_ids, enabled_skill_ids=enabled_skill_ids, diff --git a/src/backend/core/agent_skills/skill_archive.py b/src/backend/core/capabilities/skills/skill_archive.py similarity index 88% rename from src/backend/core/agent_skills/skill_archive.py rename to src/backend/core/capabilities/skills/skill_archive.py index 93637419..24b4afe7 100644 --- a/src/backend/core/agent_skills/skill_archive.py +++ b/src/backend/core/capabilities/skills/skill_archive.py @@ -14,11 +14,11 @@ from pathlib import Path from typing import Optional -from .publication import publication_lock, read_tree, safe_skill_id +from core.capabilities.skills.publication import publication_lock, read_tree, safe_skill_id def _archives_dir() -> Path: - from .config import get_sandbox_skills_dir + from core.capabilities.skills.config import get_sandbox_skills_dir path = get_sandbox_skills_dir().parent / "skill_archives" path.mkdir(parents=True, exist_ok=True) @@ -60,8 +60,8 @@ def build_skill_tar(skill_id: str, src: Path) -> Optional[Path]: def build_current_skill_tar(skill_id: str, user_id: str | None = None) -> Optional[Path]: """Resolve visibility and freeze current bytes under one publication lock.""" - from .publication import prepare_skill_view - from .config import get_sandbox_skills_dir + from core.capabilities.skills.publication import prepare_skill_view + from core.capabilities.skills.config import get_sandbox_skills_dir safe_skill_id(skill_id) with publication_lock(): diff --git a/src/backend/core/capabilities/skills/zip_reader.py b/src/backend/core/capabilities/skills/zip_reader.py new file mode 100644 index 00000000..82366201 --- /dev/null +++ b/src/backend/core/capabilities/skills/zip_reader.py @@ -0,0 +1,124 @@ +"""Validate skill upload archives without marketplace persistence.""" +from __future__ import annotations +import io +import os +import zipfile +from typing import Any, Dict, List +from core.capabilities.skills.binary_files import BINARY_EXTENSIONS, JUNK_BASENAMES, MAX_SINGLE_FILE, MAX_TOTAL, encode_binary, is_binary_value +from core.capabilities.skills.deps.detector import detect_dependencies +from core.capabilities.skills.registry import SkillSpecError, load_skill_metadata_from_str, split_frontmatter +from fastapi import HTTPException + +def parse_skill_zip(data: bytes) -> Dict[str, Any]: + """Parse a skill zip → returns {skill_id, skill_content (original text), meta, extra_files, dependencies, skipped}. + + Parsing/validation only; nothing is persisted. The same unpacking logic is + shared by admin market-listing uploads and users' private uploads. + """ + try: + zf = zipfile.ZipFile(io.BytesIO(data)) + except zipfile.BadZipFile: + raise HTTPException(status_code=400, detail="Invalid zip file") + + from core.capabilities.skills.publication import normalize_file_path, safe_skill_id + + # Preserve ZipInfo for reads: normalized names may not exist in the archive. + entries = {} + for info in zf.infolist(): + raw_name = info.filename.replace("\\", "/") + is_dir = raw_name.endswith("/") + try: + name = normalize_file_path(raw_name[:-1] if is_dir else raw_name) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + if is_dir: + continue + if name in entries: + raise HTTPException(status_code=400, detail=f"Duplicate skill file path: {name}") + entries[name] = info + # A file cannot also be a parent directory of another file. + for name in entries: + parts = name.split("/") + if any("/".join(parts[:i]) in entries for i in range(1, len(parts))): + raise HTTPException(status_code=400, detail=f"Conflicting skill file path: {name}") + + skill_md_paths = [n for n in entries if n.split("/")[-1] == "SKILL.md"] + if not skill_md_paths: + raise HTTPException(status_code=400, detail="No SKILL.md found in zip") + + skill_md_path = skill_md_paths[0] + parts = skill_md_path.split("/") + if len(parts) == 1: + prefix = "" + elif len(parts) == 2: + prefix = parts[0] + "/" + else: + prefix = "/".join(parts[:-1]) + "/" + + try: + raw = zf.read(entries[skill_md_path]).decode("utf-8") + fm, _ = split_frontmatter(raw) + skill_id = fm.get("name", "").strip() + if not skill_id: + if prefix: + skill_id = prefix.rstrip("/").split("/")[-1] + else: + raise HTTPException(status_code=400, detail="SKILL.md missing 'name' in frontmatter") + except SkillSpecError as e: + raise HTTPException(status_code=400, detail=f"Invalid SKILL.md: {e}") + + try: + safe_skill_id(skill_id) + meta = load_skill_metadata_from_str(raw, skill_id) + except Exception as e: # noqa: BLE001 + raise HTTPException(status_code=400, detail=f"Invalid skill: {e}") + + JUNK_PATH_MARKERS = ("__MACOSX/", "/.git/", "/__pycache__/", "/.svn/", "/.hg/") + extra_files: Dict[str, str] = {} + skipped: List[Dict[str, str]] = [] + total_size = 0 + for entry, info in entries.items(): + if entry == skill_md_path: + continue + if prefix and not entry.startswith(prefix): + continue + rel_name = entry[len(prefix):] if prefix else entry + if not rel_name: + continue + base = rel_name.rsplit("/", 1)[-1] + if base in JUNK_BASENAMES or any(m in f"/{entry}" for m in JUNK_PATH_MARKERS): + continue + if info.file_size > MAX_SINGLE_FILE: + skipped.append({"file": rel_name, "reason": f"exceeds {MAX_SINGLE_FILE // (1024 * 1024)}MB single-file ceiling"}) + continue + try: + file_bytes = zf.read(info) + except (KeyError, zipfile.BadZipFile): + skipped.append({"file": rel_name, "reason": "unreadable zip entry"}) + continue + _, ext = os.path.splitext(rel_name) + if ext.lower() in BINARY_EXTENSIONS: + stored = encode_binary(file_bytes) + else: + try: + stored = file_bytes.decode("utf-8") + except UnicodeDecodeError: + stored = encode_binary(file_bytes) + stored_size = len(stored.encode("utf-8")) + if total_size + stored_size > MAX_TOTAL: + skipped.append({"file": rel_name, "reason": f"would exceed {MAX_TOTAL // (1024 * 1024)}MB total ceiling"}) + continue + total_size += stored_size + extra_files[rel_name] = stored + + dependencies = detect_dependencies( + {fn: c for fn, c in extra_files.items() if not is_binary_value(c)} + ) + return { + "skill_id": skill_id, + "skill_content": raw, + "meta": meta, + "extra_files": extra_files, + "dependencies": dependencies, + "skipped": skipped, + } diff --git a/src/backend/core/channels/adapters/dingtalk.py b/src/backend/core/channels/adapters/dingtalk.py deleted file mode 100644 index 1e96dbfb..00000000 --- a/src/backend/core/channels/adapters/dingtalk.py +++ /dev/null @@ -1,854 +0,0 @@ -"""DingTalk inbound channel adapter. - -- **Binding validation**: exchange AppKey/AppSecret for an access_token - (DingTalk v1.0 OAuth) — obtaining one is treated as valid credentials. -- **Inbound (Stream long connection)**: WebSocket via the official - ``dingtalk-stream`` SDK, listening for bot message callbacks - (``ChatbotMessage.TOPIC``). Zero public IP, no callback configuration. If the - SDK is missing, this channel is unavailable but the process is unaffected. -- **Outbound replies**: use the ``sessionWebhook`` carried by each inbound - message (no access_token needed), unified for p2p / group. - Agent replies go through ``send_markdown`` as ``msgtype=markdown`` (DingTalk - natively renders headings/bold/links/lists, etc.; tables/code fences do not - render on mobile, so content is downgraded via - ``core.channels.markdown.downgrade_for_dingtalk`` before sending). - Short system messages (placeholders/receipts) still use ``send_text`` (plain text). - DingTalk bots **do not support editing messages** → ``edit_message`` returns - failure; but they do support **silent recall** of messages sent via the robot - API (group ``groupMessages/recall`` / one-to-one ``otoMessages/batchRecall``, - using the processQueryKey returned at send time). So placeholder messages go - through ``send_placeholder`` via the robot API (obtaining a recallable key), - and before the final reply the upper layer calls ``recall_message`` to remove - the placeholder — visually equivalent to "replace". When the robot API is - unavailable (no permission, etc.), placeholders automatically fall back to - sessionWebhook plain text (not recallable, same behavior as the old version). -- **Outbound proactive delivery (automation scheduled tasks, etc., no inbound - message)**: synthetic messages have no sessionWebhook → ``send_text`` / - ``send_markdown`` automatically fall back to the robot API (``sampleText`` / - ``sampleMarkdown``). -- **Outbound files**: sessionWebhook only accepts text-type messages, cannot - send files → ``push_file`` uses the robot API: first ``media/upload`` to get - a mediaId, then send a robot message. The robot message picks its endpoint by - conversation type: group ``groupMessages/send`` (openConversationId); - one-to-one with a staffId uses ``oToMessages/batchSend``, without one - (proactive delivery) uses the human-bot conversation's openConversationId via - ``privateChatMessages/send``. - The app must be granted the "internal enterprise robot send message" - permission on the DingTalk open platform, otherwise the send endpoints return 403. - -Conversation keying (``external_conversation_id``) always uses DingTalk ``conversationId``: - - one-to-one (conversationType==``1``) conversationId maps one-to-one to a user → naturally one conversation per person; - - group (conversationType==``2``) conversationId is unique per group → the whole group shares one conversation. - -Isomorphic to the [[lark]] adapter. See internal design docs. -""" - -from __future__ import annotations - -import asyncio -import json -import logging -import os -import re -import shutil -import tempfile -import time -from typing import Any, Callable, Dict, List, Optional - -import httpx - -from datetime import datetime, timezone - -from core.channels.markdown import derive_title, downgrade_for_dingtalk -from core.channels.protocol import ( - ChannelCaps, - ChannelResourceError, - HistoryItem, - InboundMsg, - SendResult, - chunk_text, -) -from core.infra.crypto import decrypt_secret - -logger = logging.getLogger(__name__) - -DINGTALK_API_BASE = "https://api.dingtalk.com" - - -class DingTalkAdapter: - caps = ChannelCaps( - channel_type="dingtalk", - max_message_len=4000, - supports_markdown=True, - splits_long_messages=False, - supports_long_conn=True, - # DingTalk exposes ``isInAtList`` on every bot callback, so the gate is already - # wired; whether non-@ group messages ever get delivered depends on the tenant - # holding DingTalk's group-message read permission (applied for and reviewed on the - # open platform). Without it, group listening stays inert rather than broken. - supports_group_observe=True, - bind_mode="credentials", - credential_fields=("app_id", "app_secret"), - ) - - # access_token cache: {app_id: (token, expire_epoch)} (only for validate/testing; replies use sessionWebhook) - _token_cache: Dict[str, tuple] = {} - - # ── Credentials ───────────────────────────────────────────────────── - @staticmethod - def _app_secret(conn: Any) -> str: - cfg = conn.config if isinstance(conn.config, dict) else {} - return decrypt_secret(cfg.get("app_secret_enc")) or "" - - async def _access_token(self, app_id: str, app_secret: str) -> str: - cached = self._token_cache.get(app_id) - if cached and cached[1] > time.time() + 60: - return cached[0] - url = f"{DINGTALK_API_BASE}/v1.0/oauth2/accessToken" - async with httpx.AsyncClient(timeout=10) as client: - resp = await client.post(url, json={"appKey": app_id, "appSecret": app_secret}) - data = resp.json() - token = data.get("accessToken") - if not token: - raise RuntimeError(f"钉钉 access_token 获取失败: {data}") - self._token_cache[app_id] = (token, time.time() + int(data.get("expireIn", 7000))) - return token - - async def validate_credentials(self, conn: Any) -> Dict[str, Any]: - secret = self._app_secret(conn) - if not conn.app_id or not secret: - raise RuntimeError("缺少 AppKey / AppSecret") - await self._access_token(conn.app_id, secret) - return {"app_id": conn.app_id} - - # ── Webhook verification (DingTalk uses the Stream long connection; no webhook entry point) ── - def verify_webhook(self, conn: Any, headers: Dict[str, str], body: bytes) -> bool: - return True - - # ── Event → InboundMsg (the dict from the Stream callback) ────────── - def parse_inbound(self, conn: Any, payload: Dict[str, Any]) -> Optional[InboundMsg]: - """DingTalk bot callback data (dict) → InboundMsg. Returns None for non-text/unsupported messages.""" - msgtype = payload.get("msgtype") - text = "" - if msgtype == "text": - text = ((payload.get("text") or {}).get("content") or "").strip() - elif msgtype == "richText": - # Rich text: concatenate the plain-text nodes inside - parts = (payload.get("content") or {}).get("richText") or [] - text = " ".join(p.get("text", "") for p in parts if isinstance(p, dict) and p.get("text")).strip() - attachments = self._extract_attachments(msgtype, payload) - if not text and not attachments: - return None - conv_type = str(payload.get("conversationType") or "1") - chat_type = "group" if conv_type == "2" else "p2p" - conv_id = payload.get("conversationId") or "" - return InboundMsg( - channel_id=conn.channel_id, - channel_type="dingtalk", - text=text, - chat_type=chat_type, - external_conversation_id=conv_id, - sender_id=payload.get("senderStaffId") or payload.get("senderId") or "", - sender_name=payload.get("senderNick") or "", - message_id=payload.get("msgId") or "", - attachments=attachments, - # ``isInAtList`` is DingTalk's own "was the bot in the @ list" flag. With only the - # default permission DingTalk delivers @bot messages exclusively, so this is - # always true and the gate is a no-op; once the tenant is granted group-message - # read, non-@ messages start arriving with it false and silent observation kicks - # in with no further code change. Absent field → treat as addressed. - addressed_to_bot=( - True if chat_type == "p2p" else bool(payload.get("isInAtList", True)) - ), - raw={ - "dingtalk_session_webhook": payload.get("sessionWebhook") or "", - "dingtalk_conversation_id": conv_id, - # Needed by download_resource: messageFiles/download requires robotCode, and - # per DingTalk's docs it is **not** the AppKey — the only reliable source is - # this field on the callback payload (present in Stream mode, which is what - # the long connection uses). - "dingtalk_robot_code": payload.get("robotCode") or "", - }, - ) - - # ── Group history pull (via the official dws CLI) ──────────────────── - # DingTalk has **no** REST OpenAPI for reading a group's message history — the - # server-side API list only covers sending, recalling and read receipts. The capability - # is reachable two ways, and this takes the second: - # 1. DingTalk's MCP gateway (`list_conversation_message_v2` on the group-chat MCP - # server). Works, but needs credentials provisioned per app — extra setup. - # 2. The official `dws` CLI, which the backend **already** runs as a subprocess for - # this user's DingTalk connection and which is **already logged in**. No new - # credentials, no extra configuration: it simply reuses the connection the owner - # has set up. That is why this is the one implemented. - # - # Runs as the connection's owner (per-user $HOME is the credential isolation boundary, - # same as every other backend dws call). The 8s cap keeps a slow CLI from eating into - # the caller's own history budget; any failure returns [] and history is skipped. - _DWS_HISTORY_TIMEOUT_S = 8 - - async def fetch_history( - self, - conn: Any, - conversation_id: str, - *, - since_ms: int, - limit: int, - newest_first: bool = False, - ) -> List[HistoryItem]: - """`dws chat message list --group --time ` → normalized history items.""" - conv = (conversation_id or "").strip() - owner = getattr(conn, "owner_user_id", "") or "" - if not conv or not owner: - return [] - from core.services.dingtalk_service import _run_dws - - # dws takes a local-time string, not epoch ms. `--forward` picks the direction: - # true = messages after --time (incremental from the cursor); false = messages - # before --time, which anchored at *now* is how the cold start gets the most recent - # ones instead of the oldest of a week-long window. - anchor_ms = int(datetime.now(timezone.utc).timestamp() * 1000) if newest_first else since_ms - anchor = datetime.fromtimestamp(max(anchor_ms, 0) / 1000, tz=timezone.utc).astimezone() - args = [ - "chat", "message", "list", - "--group", conv, - "--time", anchor.strftime("%Y-%m-%d %H:%M:%S"), - "--limit", str(max(1, min(int(limit or 50), 100))), - "--forward=false" if newest_first else "--forward=true", - "--format", "json", - ] - try: - out, err, code = await _run_dws(owner, args, timeout=self._DWS_HISTORY_TIMEOUT_S) - except Exception: # noqa: BLE001 - logger.warning("[dingtalk] dws 群历史拉取异常 conv=%s", conv[:24], exc_info=True) - return [] - if code != 0 or not out.strip(): - # Not logged in / no permission / timeout all land here. Debug, not warning: - # a tenant without the DingTalk connection set up would otherwise log on every - # single group message. - logger.debug( - "[dingtalk] dws 群历史拉取未成功 code=%s err=%s", code, (err or "")[:160] - ) - return [] - try: - return self._parse_dws_history(json.loads(out)) - except Exception: # noqa: BLE001 - logger.debug("[dingtalk] dws 输出解析失败: %s", out[:200], exc_info=True) - return [] - - _DWS_DOWNLOAD_TIMEOUT_S = 45 - - @staticmethod - def _dws_business_error(raw_out: str) -> Optional[str]: - """Pull the human-readable reason out of a dws error envelope, if that's what this is. - - Worth surfacing rather than collapsing to "download failed": the messages are - specific and actionable ("this data belongs to organization X while the tool is - configured for organization Y"), and reading that as an expiry sends the user - looking in entirely the wrong place. - """ - try: - err = (json.loads(raw_out or "{}") or {}).get("error") - except Exception: # noqa: BLE001 - return None - if not isinstance(err, dict): - return None - msg = str(err.get("message") or "").strip() - return msg or None - - async def _read_doc_via_dws(self, conn: Any, node_id: str) -> Optional[bytes]: - """`dws doc read --node --content-format markdown` → markdown bytes. - - A DingTalk online document has no file to download; it is read as content. Returning - markdown keeps it in the same artifact pipeline as everything else. - """ - owner = getattr(conn, "owner_user_id", "") or "" - if not owner: - return None - from core.services.dingtalk_service import _run_dws - - out, err, code = await _run_dws( - owner, - ["doc", "read", "--node", node_id, "--content-format", "markdown", "--format", "json"], - timeout=self._DWS_DOWNLOAD_TIMEOUT_S, - ) - reason = self._dws_business_error(out) or self._dws_business_error(err) - if reason: - logger.warning("[dingtalk] 钉钉文档读取被拒 node=%s: %s", node_id[:16], reason[:200]) - raise ChannelResourceError(reason) - if code != 0 or not out.strip(): - logger.debug("[dingtalk] 钉钉文档读取失败 code=%s err=%s", code, (err or "")[:160]) - return None - try: - payload = json.loads(out) - except Exception: # noqa: BLE001 - return out.encode("utf-8") - # Content sits under a few possible keys depending on CLI version; fall back to the - # whole payload rather than silently returning nothing. - for key in ("content", "markdown", "data", "result"): - val = payload.get(key) if isinstance(payload, dict) else None - if isinstance(val, str) and val.strip(): - return val.encode("utf-8") - if isinstance(val, dict): - for sub in ("content", "markdown", "text"): - inner = val.get(sub) - if isinstance(inner, str) and inner.strip(): - return inner.encode("utf-8") - return json.dumps(payload, ensure_ascii=False).encode("utf-8") - - async def _download_via_dws(self, conn: Any, file_id: str) -> Optional[bytes]: - """`dws drive download --node ` → bytes. - - The CLI only writes to a local path (it does the signed-URL dance internally), so the - bytes are read back from a temp file that is always removed, including on failure — - this runs on the backend host, not in a disposable sandbox. - """ - owner = getattr(conn, "owner_user_id", "") or "" - if not owner: - return None - from core.services.dingtalk_service import _run_dws - - tmpdir = tempfile.mkdtemp(prefix="dws_dl_") - target = os.path.join(tmpdir, "download.bin") - try: - out, err, code = await _run_dws( - owner, - ["drive", "download", "--node", file_id, "--output", target, "--format", "json"], - timeout=self._DWS_DOWNLOAD_TIMEOUT_S, - ) - if code != 0 or not os.path.exists(target): - logger.warning( - "[dingtalk] dws 文件下载失败 file_id=%s code=%s err=%s", - file_id[:16], code, (err or out or "")[:160], - ) - return None - with open(target, "rb") as fh: - return fh.read() - except Exception: # noqa: BLE001 - logger.warning("[dingtalk] dws 文件下载异常 file_id=%s", file_id[:16], exc_info=True) - return None - finally: - shutil.rmtree(tmpdir, ignore_errors=True) - - @staticmethod - def _dws_ts_ms(value: Any) -> int: - """dws ``createTime`` → epoch ms. - - The CLI returns a **local-time string** (``"2026-08-08 21:57:33"``), not epoch - milliseconds. Parsing it as an int silently yielded 0 for every message, which left - the pull cursor pinned at the cold-start window — so every @ re-fetched the same 24 - hours forever. Numeric forms are still accepted in case a CLI version returns them. - """ - if isinstance(value, (int, float)) and value > 0: - return int(value) - text = str(value or "").strip() - if not text: - return 0 - if text.isdigit(): - return int(text) - try: - naive = datetime.strptime(text, "%Y-%m-%d %H:%M:%S") - except ValueError: - return 0 - # Local time, as the CLI prints it; astimezone() attaches the process zone. - return int(naive.astimezone().timestamp() * 1000) - - # dws renders a file message as **plain text**, not a structured attachment: - # "[文件] 周报.docx fileId: X6GRez... 注意：如需下载使用dws drive download命令下载" - # Parsed back out here, otherwise a pulled file is just an unusable sentence — the agent - # can see a file was shared but has no handle to open it. Images use the same shape. - _DWS_FILE_RE = re.compile( - r"\[(?P文件|图片)\]\s*(?P.+?)\s+fileId:\s*(?P[A-Za-z0-9_\-]+)" - r"(?:\s*注意[：:][^\n]*)?" - ) - - # A DingTalk **online document** shared into a group arrives as a link card, not as a - # file: a title line, an icon, and an alidocs URL. It is a different resource from a Drive - # file — read with `dws doc read`, not downloaded by fileId — and without recognising it - # the agent sees only an unusable link. - _DWS_DOC_RE = re.compile(r"https?://alidocs\.dingtalk\.com/i/nodes/(?P[A-Za-z0-9_\-]+)") - - @classmethod - def _extract_dws_docs(cls, content: str) -> List[Dict[str, Any]]: - """DingTalk Docs link cards → attachment refs (deduped by node id).""" - nodes = {m.group("node") for m in cls._DWS_DOC_RE.finditer(content or "")} - if not nodes: - return [] - # The card's first non-empty line is the document title; skip the markdown image line - # and the "创建者：" byline, neither of which names the document. - title = "" - for line in (content or "").splitlines(): - line = line.strip() - if line and not line.startswith("![") and not line.startswith("[http"): - if line.startswith("创建者"): - continue - title = line - break - return [ - {"kind": "doc", "key": node, "name": title or "钉钉文档"} - for node in sorted(nodes) - ] - - @classmethod - def _extract_dws_files(cls, content: str) -> tuple: - """(cleaned_text, attachment_refs) for one history message body.""" - refs: List[Dict[str, Any]] = [] - for m in cls._DWS_FILE_RE.finditer(content or ""): - refs.append({ - "kind": "image" if m.group("kind") == "图片" else "file", - "key": m.group("fid"), - "name": m.group("name").strip(), - }) - doc_refs = cls._extract_dws_docs(content) - if doc_refs: - # The card body is title + icon + a long URL repeated twice; the attachment entry - # carries title and handle, so keeping the raw card would just burn prompt space. - return "", refs + doc_refs - if not refs: - return (content or "").strip(), [] - # Drop the matched sentences: the attachment ref already renders name + handle, and - # the "use dws drive download" tail is instruction noise aimed at a CLI user. - return cls._DWS_FILE_RE.sub("", content or "").strip(), refs - - @staticmethod - def _parse_dws_history(body: Dict[str, Any]) -> List[HistoryItem]: - """dws JSON output → HistoryItem list. - - Field names verified against live CLI output — they are **not** the ones the bot - callback uses, which is the trap here: - {"result": {"messages": [{"openMessageId", "sender", "content", "createTime", - "openConversationId", "senderOpenDingTalkId"}], ...}} - A bare ``{"messages": [...]}`` is accepted too, so a CLI version that drops the - envelope does not silently yield nothing. - """ - if not isinstance(body, dict): - return [] - if body.get("success") is False or body.get("errorCode"): - return [] - result = body.get("result") if isinstance(body.get("result"), dict) else body - out: List[HistoryItem] = [] - for m in result.get("messages") or []: - if not isinstance(m, dict): - continue - # Written out rather than chained: `a or b if cond else ""` binds the conditional - # looser than the `or`, which would silently make every message look empty. - text = m.get("content") or "" - if not text and isinstance(m.get("text"), dict): - text = m["text"].get("content") or "" - if not isinstance(text, str): - text = str(text) - text, refs = DingTalkAdapter._extract_dws_files(text) - if not text and not refs: - continue - out.append(HistoryItem( - # openMessageId is the real id; msgId/messageId kept as fallbacks only. - message_id=str( - m.get("openMessageId") or m.get("msgId") or m.get("messageId") or "" - ), - # `sender` is already the display name — no directory lookup needed. - sender_name=str( - m.get("sender") or m.get("senderNick") or m.get("senderName") or "群成员" - ), - text=text, - ts_ms=DingTalkAdapter._dws_ts_ms(m.get("createTime") or m.get("createAt")), - attachments=refs, - # Marks which download route this file needs: history files live on DingTalk - # Drive and are fetched by fileId, not by the bot callback's downloadCode. - raw={"dws_drive": True} if refs else {}, - )) - return out - - # ── Inbound attachments ───────────────────────────────────────────── - @staticmethod - def _extract_attachments(msgtype: str, payload: Dict[str, Any]) -> list: - """Message → attachment list. ``key`` carries DingTalk's ``downloadCode``. - - Content layout per DingTalk's "received message types" reference: - picture → {pictureDownloadCode, downloadCode} - file → {spaceId, fileName, downloadCode, fileId} - audio → {downloadCode, recognition} - video → {duration, videoType, downloadCode} - richText → {richText: [ ...nodes, image nodes carry downloadCode... ]} - Everything is read defensively (``downloadCode`` looked up in more than one place) - because these payloads are only loosely specified and vary by client version — a - missing field must degrade to "no attachment", never raise and kill the connection. - """ - content = payload.get("content") if isinstance(payload.get("content"), dict) else {} - out: list = [] - - def _add(kind: str, code: Any, name: str) -> None: - if code: - out.append({"kind": kind, "key": str(code), "name": name}) - - if msgtype == "picture": - code = content.get("downloadCode") or content.get("pictureDownloadCode") - _add("image", code, f"dingtalk_image_{payload.get('msgId', '')[:8] or 'x'}.png") - elif msgtype == "file": - _add("file", content.get("downloadCode"), content.get("fileName") or "file.bin") - elif msgtype == "audio": - _add("file", content.get("downloadCode"), "dingtalk_audio.amr") - elif msgtype == "video": - _add("file", content.get("downloadCode"), "dingtalk_video.mp4") - elif msgtype == "richText": - for i, node in enumerate(content.get("richText") or []): - if not isinstance(node, dict): - continue - code = node.get("downloadCode") or node.get("pictureDownloadCode") - _add("image", code, f"dingtalk_richtext_{i}.png") - return out - - async def download_resource( - self, conn: Any, inbound: InboundMsg, attachment: Dict[str, Any] - ) -> Optional[bytes]: - """Download an inbound attachment: downloadCode → temporary downloadUrl → bytes. - - Two hops, per DingTalk's design: - POST /v1.0/robot/messageFiles/download {downloadCode, robotCode} → {downloadUrl} - GET downloadUrl → bytes - The temporary URL serves the file with a ``.file`` extension; we ignore that and keep - the filename taken from the message, so the artifact keeps its real extension. - Download codes expire, so a deferred fetch can legitimately fail — returning None - (rather than raising) lets the caller degrade to "attachment unavailable". - """ - code = attachment.get("key") - if not code: - return None - # Files seen through *history* are DingTalk Drive entries addressed by fileId, not - # bot-callback attachments addressed by a short-lived downloadCode — different - # storage, different API. The marker is set when the history item is parsed. - if attachment.get("kind") == "doc": - return await self._read_doc_via_dws(conn, str(code)) - if (inbound.raw or {}).get("dws_drive"): - return await self._download_via_dws(conn, str(code)) - robot_code = (inbound.raw or {}).get("dingtalk_robot_code") or "" - if not robot_code: - logger.warning("[dingtalk] 缺少 robotCode，无法下载附件 code=%s", str(code)[:12]) - return None - try: - token = await self._access_token(conn.app_id, self._app_secret(conn)) - async with httpx.AsyncClient(timeout=30) as client: - resp = await client.post( - f"{DINGTALK_API_BASE}/v1.0/robot/messageFiles/download", - headers={"x-acs-dingtalk-access-token": token}, - json={"downloadCode": code, "robotCode": robot_code}, - ) - if resp.status_code != 200: - logger.warning( - "[dingtalk] 下载地址获取失败 status=%s body=%s", - resp.status_code, resp.text[:200], - ) - return None - url = (resp.json() or {}).get("downloadUrl") - if not url: - return None - fr = await client.get(url) - if fr.status_code == 200: - return fr.content - logger.warning("[dingtalk] 附件下载失败 status=%s", fr.status_code) - except Exception: # noqa: BLE001 - logger.exception("[dingtalk] 附件下载异常 code=%s", str(code)[:12]) - return None - - # ── Outbound push (via sessionWebhook, no access_token needed) ─────── - @staticmethod - def _session_webhook(inbound: InboundMsg) -> str: - return (inbound.raw or {}).get("dingtalk_session_webhook") or "" - - async def _post_webhook(self, webhook: str, payload: Dict[str, Any]) -> SendResult: - if not webhook: - return SendResult.fail("forbidden", "缺少 sessionWebhook（可能已过期）") - try: - async with httpx.AsyncClient(timeout=15) as client: - resp = await client.post(webhook, json=payload) - data = resp.json() - except Exception as exc: # noqa: BLE001 - return SendResult.fail("transient", str(exc)) - if data.get("errcode", 0) == 0: - return SendResult.ok() - kind = "rate_limited" if data.get("errcode") in (130101, 88) else "unknown" - return SendResult.fail(kind, f"errcode={data.get('errcode')} {data.get('errmsg')}") - - async def send_text(self, conn: Any, inbound: InboundMsg, text: str) -> SendResult: - webhook = self._session_webhook(inbound) - if webhook: - return await self._post_webhook( - webhook, {"msgtype": "text", "text": {"content": text}} - ) - # Proactive delivery (automation scheduled tasks, etc.) has no inbound sessionWebhook → use the robot API - try: - token = await self._access_token(conn.app_id, self._app_secret(conn)) - except Exception as exc: # noqa: BLE001 - return SendResult.fail("transient", str(exc)) - return await self._robot_send(token, conn, inbound, "sampleText", {"content": text}) - - # Lets the upper layer do a whole-content downgrade **before** chunking - # (avoids a table getting bisected by a chunk boundary and becoming unrecognizable for conversion). - prepare_markdown = staticmethod(downgrade_for_dingtalk) - - async def send_markdown(self, conn: Any, inbound: InboundMsg, text: str) -> SendResult: - """Send a DingTalk markdown message (natively rendered). The downgrade is idempotent; calling directly or via prepare_markdown is equally safe. - - title is required for DingTalk's markdown type; it shows in the - conversation-list summary / push notification and is derived from the - first line of the body. - """ - md = downgrade_for_dingtalk(text) - title = derive_title(md) - webhook = self._session_webhook(inbound) - if webhook: - return await self._post_webhook( - webhook, {"msgtype": "markdown", "markdown": {"title": title, "text": md}} - ) - # Proactive delivery → robot-API markdown message - try: - token = await self._access_token(conn.app_id, self._app_secret(conn)) - except Exception as exc: # noqa: BLE001 - return SendResult.fail("transient", str(exc)) - return await self._robot_send( - token, conn, inbound, "sampleMarkdown", {"title": title, "text": md} - ) - - async def push(self, conn: Any, inbound: InboundMsg, content: str) -> SendResult: - md = downgrade_for_dingtalk(content) - chunks = chunk_text(md, self.caps.max_message_len) or [md] - first: Optional[SendResult] = None - for i, c in enumerate(chunks): - r = await self.send_markdown(conn, inbound, c) - if i == 0: - first = r - if not r.success: - return r - return first or SendResult.fail("unknown", "空内容") - - async def edit_message(self, conn: Any, message_id: str, text: str) -> SendResult: - # DingTalk bots cannot edit messages — the upper-layer placeholder logic - # therefore switches to "recall + resend" (see recall_message). - return SendResult.fail("bad_format", "钉钉不支持编辑消息") - - # ── Placeholder messages: sent via robot API (recallable) → recall as an equivalent of "replace" ── - async def send_placeholder(self, conn: Any, inbound: InboundMsg, text: str) -> SendResult: - """Send a recallable placeholder message via the robot API; message_id is the processQueryKey (recall credential). - - If the robot API is unavailable (no permission / token failure) → fall - back to sessionWebhook plain text; then there is no message_id and the - upper layer will not attempt recall — behavior degrades to the old - version (placeholder stays, reply is sent fresh). - """ - try: - token = await self._access_token(conn.app_id, self._app_secret(conn)) - r = await self._robot_send(token, conn, inbound, "sampleText", {"content": text}) - if r.success: - return r - except Exception: # noqa: BLE001 - logger.debug("[dingtalk] 机器人占位发送失败，回退 webhook", exc_info=True) - return await self.send_text(conn, inbound, text) - - def _recall_url_body( - self, conn: Any, inbound: InboundMsg, message_id: str - ) -> Optional[tuple]: - """Pick the recall endpoint by conversation type. Group chats need openConversationId; one-to-one batchRecall needs only the key.""" - body: Dict[str, Any] = {"robotCode": conn.app_id, "processQueryKeys": [message_id]} - if inbound.chat_type == "group": - conv_id = inbound.external_conversation_id or (inbound.raw or {}).get("dingtalk_conversation_id") or "" - if not conv_id: - return None - body["openConversationId"] = conv_id - return f"{DINGTALK_API_BASE}/v1.0/robot/groupMessages/recall", body - return f"{DINGTALK_API_BASE}/v1.0/robot/otoMessages/batchRecall", body - - async def recall_message(self, conn: Any, inbound: InboundMsg, message_id: str) -> SendResult: - """Silently recall a message sent via the robot API (the client shows no recall notice).""" - if not message_id: - return SendResult.fail("bad_format", "缺少 processQueryKey") - target = self._recall_url_body(conn, inbound, message_id) - if target is None: - return SendResult.fail("bad_format", "缺少 openConversationId") - url, body = target - try: - token = await self._access_token(conn.app_id, self._app_secret(conn)) - async with httpx.AsyncClient(timeout=15) as client: - resp = await client.post(url, headers={"x-acs-dingtalk-access-token": token}, json=body) - data = resp.json() - except Exception as exc: # noqa: BLE001 - return SendResult.fail("transient", str(exc)) - if resp.status_code == 200: - return SendResult.ok(message_id) - return SendResult.fail("unknown", f"撤回失败: {resp.status_code} {str(data)[:200]}") - - # ── Outbound file delivery (sessionWebhook cannot send files → use the robot API) ── - async def push_file( - self, conn: Any, inbound: InboundMsg, content: bytes, filename: str, mime_type: str - ) -> SendResult: - secret = self._app_secret(conn) - if not conn.app_id or not secret: - return SendResult.fail("forbidden", "缺少 AppKey / AppSecret") - try: - token = await self._access_token(conn.app_id, secret) - except Exception as exc: # noqa: BLE001 - return SendResult.fail("transient", str(exc)) - - is_image = (mime_type or "").startswith("image/") - media_id = await self._upload_media(token, content, filename, "image" if is_image else "file") - if isinstance(media_id, SendResult): - return media_id - - if is_image: - # sampleImageMsg's photoURL accepts a mediaId directly - msg_key, msg_param = "sampleImageMsg", {"photoURL": media_id} - else: - ext = (filename.rsplit(".", 1)[-1] if "." in filename else "").lower() or "file" - msg_key = "sampleFile" - msg_param = {"mediaId": media_id, "fileName": filename, "fileType": ext} - return await self._robot_send(token, conn, inbound, msg_key, msg_param) - - async def _upload_media(self, token: str, content: bytes, filename: str, media_type: str): - """Upload media in exchange for a mediaId; returns SendResult on failure (str on success).""" - url = f"https://oapi.dingtalk.com/media/upload?access_token={token}&type={media_type}" - try: - async with httpx.AsyncClient(timeout=60) as client: - resp = await client.post(url, files={"media": (filename, content)}) - data = resp.json() - except Exception as exc: # noqa: BLE001 - return SendResult.fail("transient", f"媒体上传失败: {exc}") - media_id = data.get("media_id") - if data.get("errcode", 0) != 0 or not media_id: - return SendResult.fail("unknown", f"媒体上传失败: errcode={data.get('errcode')} {data.get('errmsg')}") - return media_id - - async def _robot_send( - self, token: str, conn: Any, inbound: InboundMsg, msg_key: str, msg_param: Dict[str, Any] - ) -> SendResult: - """Robot proactive message: group chats use openConversationId, one-to-one uses the recipient's staffId.""" - body: Dict[str, Any] = { - "robotCode": conn.app_id, # for internal enterprise apps, robotCode == appKey - "msgKey": msg_key, - "msgParam": json.dumps(msg_param, ensure_ascii=False), - } - if inbound.chat_type == "group": - conv_id = inbound.external_conversation_id or (inbound.raw or {}).get("dingtalk_conversation_id") or "" - if not conv_id: - return SendResult.fail("bad_format", "缺少 openConversationId") - url = f"{DINGTALK_API_BASE}/v1.0/robot/groupMessages/send" - body["openConversationId"] = conv_id - else: - staff_id = inbound.sender_id or "" - if staff_id: - url = f"{DINGTALK_API_BASE}/v1.0/robot/oToMessages/batchSend" - body["userIds"] = [staff_id] - else: - # In proactive-delivery scenarios no staffId is available → send directly via the human-bot one-to-one conversation's openConversationId - conv_id = inbound.external_conversation_id or (inbound.raw or {}).get("dingtalk_conversation_id") or "" - if not conv_id: - return SendResult.fail("bad_format", "缺少接收人 staffId / openConversationId") - url = f"{DINGTALK_API_BASE}/v1.0/robot/privateChatMessages/send" - body["openConversationId"] = conv_id - try: - async with httpx.AsyncClient(timeout=30) as client: - resp = await client.post(url, headers={"x-acs-dingtalk-access-token": token}, json=body) - data = resp.json() - except Exception as exc: # noqa: BLE001 - return SendResult.fail("transient", str(exc)) - key = data.get("processQueryKey") or data.get("processQueryKeys") - if resp.status_code == 200 and key: - # The processQueryKey is the recall credential, returned to the upper layer as message_id (used by recall_message) - if isinstance(key, list): - key = key[0] if key else None - return SendResult.ok(key if isinstance(key, str) else None) - code = data.get("code") or resp.status_code - kind = "forbidden" if resp.status_code == 403 else "unknown" - return SendResult.fail(kind, f"机器人消息发送失败: {code} {str(data)[:200]}") - - # ── Long connection (Stream, requires the dingtalk-stream SDK) ────── - def make_ws_client(self, conn: Any, on_message: Callable[[InboundMsg], None]) -> Any: - """Build the DingTalk Stream long-connection runner. ``on_message`` is invoked synchronously on the SDK thread (dispatched to the main loop). - - dingtalk_stream is lazily imported; if not installed a RuntimeError is - raised, which the manager records as an error without affecting the - process. The SDK's ``start()`` is async and ``start_forever()`` cannot - be stopped → wrap it in a controllable runner so the manager can start - and stop cleanly (actually disconnecting when a bot is disabled). - """ - try: - import dingtalk_stream - except Exception as exc: # noqa: BLE001 - raise RuntimeError("dingtalk-stream SDK 未安装，长连接不可用") from exc - - adapter = self - app_id = conn.app_id - secret = self._app_secret(conn) - - class _Handler(dingtalk_stream.ChatbotHandler): - async def process(self, callback): # type: ignore[override] - try: - inbound = adapter.parse_inbound(conn, callback.data or {}) - if inbound is not None: - on_message(inbound) - except Exception: # noqa: BLE001 - logger.exception("[dingtalk] Stream 事件处理失败 channel_id=%s", conn.channel_id) - return dingtalk_stream.AckMessage.STATUS_OK, "OK" - - def _build_client(): - sdk_logger = logging.getLogger("dingtalk_stream") - sdk_logger.setLevel(logging.WARNING) - credential = dingtalk_stream.Credential(app_id, secret) - client = dingtalk_stream.DingTalkStreamClient(credential, logger=sdk_logger) - client.register_callback_handler( - dingtalk_stream.chatbot.ChatbotMessage.TOPIC, _Handler() - ) - return client - - return _DingTalkStreamRunner(_build_client) - - -class _DingTalkStreamRunner: - """Wraps dingtalk-stream's async ``client.start()`` into the "blocking start() + stoppable stop()" the manager expects. - - Creates a dedicated event loop in the worker thread to run ``client.start()`` - (the SDK reconnects on its own); ``stop()`` closes the websocket and cancels - the task from the main thread via ``call_soon_threadsafe``, making - ``start()`` return → the manager's thread body sees stop_flag set and exits - cleanly. - """ - - def __init__(self, build_client: Callable[[], Any]): - self._build_client = build_client - self._loop: Optional[asyncio.AbstractEventLoop] = None - self._client: Any = None - self._task: Optional[asyncio.Task] = None - - def start(self) -> None: - loop = asyncio.new_event_loop() - self._loop = loop - asyncio.set_event_loop(loop) - try: - self._client = self._build_client() - self._task = loop.create_task(self._client.start()) - loop.run_until_complete(self._task) - except asyncio.CancelledError: - pass - finally: - try: - loop.run_until_complete(loop.shutdown_asyncgens()) - except Exception: # noqa: BLE001 - pass - loop.close() - - def stop(self) -> None: - loop, task, client = self._loop, self._task, self._client - if loop is None or loop.is_closed(): - return - - def _cancel() -> None: - ws = getattr(client, "websocket", None) - if ws is not None: - try: - asyncio.ensure_future(ws.close()) - except Exception: # noqa: BLE001 - pass - if task is not None: - task.cancel() - - try: - loop.call_soon_threadsafe(_cancel) - except Exception: # noqa: BLE001 - pass diff --git a/src/backend/core/channels/adapters/lark.py b/src/backend/core/channels/adapters/lark.py deleted file mode 100644 index 4b5240f3..00000000 --- a/src/backend/core/channels/adapters/lark.py +++ /dev/null @@ -1,683 +0,0 @@ -"""Feishu (Lark) inbound channel adapter. - -- **Binding validation / outbound push**: Feishu OpenAPI REST (httpx) with a - tenant_access_token; no SDK needed. -- **Webhook decryption/signature verification**: AES-256-CBC (Encrypt Key) + - signature check, pure cryptography; no SDK needed. -- **Long connection (WebSocket)**: uses the official ``lark_oapi`` SDK's - ws.Client (lazy import; if the SDK is missing only the long connection is - unavailable, webhook/push are unaffected). Long connection is recommended in - production: the user only needs App ID/Secret, no public callback URL. - -Conversation keying (``external_conversation_id``) always uses Feishu -``message.chat_id``: - - a p2p chat_id maps one-to-one to a user → naturally one conversation per DM peer; - - a group chat_id is unique per group → the whole group shares one conversation. -Outbound push also sends by chat_id (``receive_id_type=chat_id``), unified for p2p/group. - -See internal design docs. -""" - -from __future__ import annotations - -import asyncio -import base64 -import hashlib -import json -import logging -import os -import re -import threading -import time -from typing import Any, Callable, Dict, List, Optional - -import httpx - -from core.channels.protocol import ChannelCaps, HistoryItem, InboundMsg, SendResult -from core.infra.crypto import decrypt_secret - -logger = logging.getLogger(__name__) - -LARK_API_BASE = os.getenv("LARK_API_BASE", "https://open.feishu.cn").rstrip("/") -_MENTION_RE = re.compile(r"@_user_\d+") - - -# ── lark_oapi ws event-loop isolation (supports multiple coexisting bots) ──── -# Root cause: ``lark_oapi/ws/client.py`` runs everything on a single -# **module-level global** ``loop`` (``loop.run_until_complete`` in ``start()``, -# ``loop.create_task`` in ``_connect`` ...). Each bot's long connection runs in -# its own worker thread, but they all share that one global → when the second -# bot starts it overwrites the global loop with its own thread's loop, and the -# first bot's already-running coroutines then call ``loop.create_task`` on -# someone else's loop → "This event loop is already running" / cross-thread -# crashes. Single-bot setups hit the same cause sporadically (the global gets -# rewritten on reconnect/retry). -# Fix: replace that global ``loop`` with a **per-thread-resolving proxy** — each -# worker thread always resolves to its own thread's loop (prepare_ws_thread -# writes it into thread-local storage), fully eliminating cross-thread sharing -# and overwriting. -_ws_loop_tls = threading.local() -_ws_proxy_installed = False -_ws_proxy_lock = threading.Lock() - - -class _PerThreadLoopProxy: - """Masquerades as an event loop: every attribute/method access is forwarded to the real loop bound to the **current thread**.""" - - def __getattr__(self, name: str) -> Any: - loop = getattr(_ws_loop_tls, "loop", None) - if loop is None: - loop = asyncio.get_event_loop() - return getattr(loop, name) - - -def _install_ws_loop_proxy() -> None: - """One-time replacement of lark_oapi.ws.client's module-level global loop with the per-thread proxy (idempotent).""" - global _ws_proxy_installed - if _ws_proxy_installed: - return - with _ws_proxy_lock: - if _ws_proxy_installed: - return - try: - import lark_oapi.ws.client as _wsc - _wsc.loop = _PerThreadLoopProxy() - _ws_proxy_installed = True - except Exception: # noqa: BLE001 — if the SDK is missing, the long connection is unavailable anyway - pass - - -class LarkAdapter: - caps = ChannelCaps( - channel_type="lark", - max_message_len=4000, # Feishu's per-message text cap is fairly high; leave headroom for chunking - supports_markdown=False, # uses text messages; rich text (post) left for later - splits_long_messages=False, - supports_long_conn=True, - # Feishu gates group delivery with three mutually distinct scopes: - # im:message.group_at_msg → only messages @-ing this bot - # im:message.group_at_msg.include_bot:readonly → messages @-ing any bot - # im:message.group_msg (sensitive) → every group message - # Group listening needs the last one, plus republishing the app version; with the - # others Feishu never pushes a non-@ message and observe_all stays inert. Note the - # @-gate is worth running under the middle scope too: it stops us answering a message - # that @'d somebody else's bot. - supports_group_observe=True, - ) - - # tenant_access_token cache: {app_id: (token, expire_epoch)} - _token_cache: Dict[str, tuple] = {} - # The bot's own open_id: {app_id: open_id}. Stable per app → cached without expiry. - _bot_id_cache: Dict[str, str] = {} - - # ── Credentials ───────────────────────────────────────────────────── - @staticmethod - def _app_secret(conn: Any) -> str: - cfg = conn.config if isinstance(conn.config, dict) else {} - return decrypt_secret(cfg.get("app_secret_enc")) or "" - - @staticmethod - def _encrypt_key(conn: Any) -> str: - cfg = conn.config if isinstance(conn.config, dict) else {} - return decrypt_secret(cfg.get("encrypt_key_enc")) or "" - - @staticmethod - def _verification_token(conn: Any) -> str: - cfg = conn.config if isinstance(conn.config, dict) else {} - return decrypt_secret(cfg.get("verification_token_enc")) or "" - - async def _tenant_token(self, app_id: str, app_secret: str) -> str: - cached = self._token_cache.get(app_id) - if cached and cached[1] > time.time() + 60: - return cached[0] - url = f"{LARK_API_BASE}/open-apis/auth/v3/tenant_access_token/internal" - async with httpx.AsyncClient(timeout=10) as client: - resp = await client.post(url, json={"app_id": app_id, "app_secret": app_secret}) - data = resp.json() - if data.get("code") != 0: - raise RuntimeError(f"飞书 tenant_access_token 获取失败: {data.get('msg') or data}") - token = data["tenant_access_token"] - self._token_cache[app_id] = (token, time.time() + int(data.get("expire", 7000))) - return token - - async def validate_credentials(self, conn: Any) -> Dict[str, Any]: - """Successfully obtaining a token means the credentials are valid; returns an identity summary. Raises on failure.""" - secret = self._app_secret(conn) - if not conn.app_id or not secret: - raise RuntimeError("缺少 App ID / App Secret") - await self._tenant_token(conn.app_id, secret) - return {"app_id": conn.app_id} - - # ── Webhook decryption / signature verification ───────────────────── - def decrypt_webhook(self, conn: Any, body: bytes) -> Dict[str, Any]: - """If an Encrypt Key is configured, the body looks like {"encrypt": "..."} and is AES-256-CBC decrypted; otherwise plain JSON.""" - try: - raw = json.loads(body.decode("utf-8")) - except Exception: - return {} - if "encrypt" not in raw: - return raw - key = self._encrypt_key(conn) - if not key: - logger.warning("[lark] 收到加密事件但未配 Encrypt Key") - return {} - return self._aes_decrypt(raw["encrypt"], key) - - @staticmethod - def _aes_decrypt(encrypt_b64: str, encrypt_key: str) -> Dict[str, Any]: - from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes - - key = hashlib.sha256(encrypt_key.encode("utf-8")).digest() - data = base64.b64decode(encrypt_b64) - iv, ct = data[:16], data[16:] - decryptor = Cipher(algorithms.AES(key), modes.CBC(iv)).decryptor() - plain = decryptor.update(ct) + decryptor.finalize() - plain = plain[: -plain[-1]] # strip PKCS7 padding - return json.loads(plain.decode("utf-8")) - - def verify_webhook(self, conn: Any, headers: Dict[str, str], body: bytes) -> bool: - """Feishu signature: sha256(timestamp + nonce + encrypt_key + body). Without an Encrypt Key, falls back to token verification.""" - key = self._encrypt_key(conn) - sig = headers.get("x-lark-signature") or headers.get("X-Lark-Signature") - ts = headers.get("x-lark-request-timestamp") or headers.get("X-Lark-Request-Timestamp") - nonce = headers.get("x-lark-request-nonce") or headers.get("X-Lark-Request-Nonce") - if key and sig and ts is not None and nonce is not None: - h = hashlib.sha256((ts + nonce + key).encode("utf-8") + body).hexdigest() - return h == sig - # No encryption configured → fall back to verification_token (re-checked at the parse stage after decryption) - return True - - # ── Event → InboundMsg ────────────────────────────────────────────── - def parse_inbound(self, conn: Any, payload: Dict[str, Any]) -> Optional[InboundMsg]: - """Decrypted event dict → InboundMsg. Returns None for non-message events.""" - # schema 2.0: header.event_type; legacy schema: payload["event"]["type"] - header = payload.get("header") or {} - event_type = header.get("event_type") or payload.get("type") - if event_type != "im.message.receive_v1": - return None - event = payload.get("event") or {} - message = event.get("message") or {} - sender = event.get("sender") or {} - mtype = message.get("message_type") - text, attachments = self._extract_payload(mtype, message.get("content")) - if not text and not attachments: - return None # unsupported message type (sticker/location, etc.) - sender_open_id = (sender.get("sender_id") or {}).get("open_id", "") - return self._make_inbound( - conn=conn, - chat_id=message.get("chat_id", ""), - chat_type=message.get("chat_type", "p2p"), - message_id=message.get("message_id", ""), - text=text, - sender_open_id=sender_open_id, - sender_name=(sender.get("sender_id") or {}).get("user_id", "") or "", - attachments=attachments, - mentioned_ids=self._mentioned_ids(message), - raw=payload, - ) - - @staticmethod - def _mentioned_ids(message: Dict[str, Any]) -> List[str]: - """open_ids of everyone @'d in this message (``message.mentions``). - - Only relevant once the app holds ``im:message.group_msg`` and Feishu starts pushing - non-@ group messages too — until then every delivered group message has the bot in - here anyway. The bot's own open_id is not known synchronously, so matching happens - later in ``resolve_addressed``. - """ - out: List[str] = [] - for m in message.get("mentions") or []: - if not isinstance(m, dict): - continue - oid = (m.get("id") or {}).get("open_id") if isinstance(m.get("id"), dict) else None - if oid: - out.append(oid) - return out - - @staticmethod - def _strip_mentions(text: str) -> str: - return _MENTION_RE.sub("", text or "").strip() - - @classmethod - def _extract_text(cls, content: Any) -> str: - if not content: - return "" - try: - obj = json.loads(content) if isinstance(content, str) else content - text = obj.get("text", "") if isinstance(obj, dict) else "" - except Exception: - text = str(content) - return cls._strip_mentions(text) - - @classmethod - def _extract_payload(cls, mtype: str, content: Any): - """Extract (text, attachments) by message type. - - - text: plain text body (@ mentions stripped) - - image: image_key → attachment(kind=image) - - file/media/audio: file_key + file_name → attachment(kind=file) - - post (rich text): extract text + embedded image_key - """ - try: - obj = json.loads(content) if isinstance(content, str) else (content or {}) - except Exception: - obj = {} - if not isinstance(obj, dict): - obj = {} - attachments = [] - if mtype == "text": - return cls._strip_mentions(obj.get("text", "")), attachments - if mtype == "image": - key = obj.get("image_key") - if key: - attachments.append({"kind": "image", "key": key, "name": f"{key}.png"}) - return "", attachments - if mtype in ("file", "media", "audio"): - key = obj.get("file_key") - if key: - name = obj.get("file_name") or f"{key}.bin" - attachments.append({"kind": "file", "key": key, "name": name}) - return "", attachments - if mtype == "post": # rich text: extract text + embedded images - texts, img_keys = [], [] - for node in cls._post_nodes(obj): - if node.get("tag") == "text": - texts.append(node.get("text", "")) - elif node.get("tag") == "img" and node.get("image_key"): - img_keys.append(node["image_key"]) - title = obj.get("title", "") - body = " ".join(texts) - text = (title + "\n" + body) if title else body - attachments = [{"kind": "image", "key": k, "name": f"{k}.png"} for k in img_keys] - return text.strip(), attachments - return "", attachments - - @staticmethod - def _post_nodes(obj: Dict[str, Any]): - """Flatten all nodes of a post rich-text message (structure may be {content:[[node...]]} or wrapped in a language layer).""" - content = obj.get("content") or [] - if isinstance(content, dict): # language layer wrapper {"zh_cn": {"content": [...]}} - content = next(iter(content.values()), {}).get("content", []) if content else [] - for line in content or []: - for node in line or []: - if isinstance(node, dict): - yield node - - @staticmethod - def _make_inbound( - *, conn: Any, chat_id: str, chat_type: str, message_id: str, - text: str, sender_open_id: str, sender_name: str, - raw: Dict[str, Any], attachments=None, mentioned_ids=None, - ) -> InboundMsg: - return InboundMsg( - channel_id=conn.channel_id, - channel_type="lark", - text=text, - chat_type=chat_type, - external_conversation_id=chat_id, # always keyed by chat_id - sender_id=sender_open_id, - sender_name=sender_name, - message_id=message_id, - attachments=attachments or [], - # p2p is always aimed at the bot; in a group it depends on whether the bot's own - # open_id is in the mention list — resolved asynchronously in resolve_addressed. - addressed_to_bot=True if chat_type == "p2p" else None, - mentioned_ids=list(mentioned_ids or []), - raw={"lark_chat_id": chat_id, "lark_message_id": message_id}, - ) - - # ── Group history pull (app credentials, independent of group listening) ── - async def fetch_history( - self, - conn: Any, - conversation_id: str, - *, - since_ms: int, - limit: int, - newest_first: bool = False, - ) -> List[HistoryItem]: - """GET /open-apis/im/v1/messages — this chat's own messages, oldest-first. - - Uses the tenant token (app credentials), so it is a property of the bound app rather - than of any user's session. Note this sits behind a *different* permission from event - push: an app that is not allowed to receive non-@ group messages may still be allowed - to read the chat's history, which is precisely why pulling is worth having. - - Feishu takes ``start_time`` in **seconds**; the caller works in milliseconds. - Any failure returns [] — the caller treats "unsupported" and "failed" alike. - """ - chat_id = (conversation_id or "").strip() - if not chat_id: - return [] - try: - token = await self._tenant_token(conn.app_id, self._app_secret(conn)) - params = { - "container_id_type": "chat", - "container_id": chat_id, - # Desc for the cold start so the first page holds the *newest* messages of - # the window; Asc when walking forward from the cursor so nothing is skipped. - "sort_type": "ByCreateTimeDesc" if newest_first else "ByCreateTimeAsc", - "page_size": str(max(1, min(int(limit or 50), 50))), # Feishu caps page_size at 50 - } - if since_ms > 0: - params["start_time"] = str(int(since_ms // 1000)) - async with httpx.AsyncClient(timeout=15) as client: - resp = await client.get( - f"{LARK_API_BASE}/open-apis/im/v1/messages", - params=params, - headers={"Authorization": f"Bearer {token}"}, - ) - body = resp.json() or {} - if resp.status_code != 200 or body.get("code") not in (0, None): - logger.warning( - "[lark] 群历史拉取失败 chat=%s code=%s msg=%s", - chat_id, body.get("code"), str(body.get("msg"))[:120], - ) - return [] - return [ - item - for item in ( - self._history_item(raw) for raw in (body.get("data") or {}).get("items") or [] - ) - if item is not None - ] - except Exception: # noqa: BLE001 - logger.warning("[lark] 群历史拉取异常 chat=%s", chat_id, exc_info=True) - return [] - - @classmethod - def _history_item(cls, raw: Dict[str, Any]) -> Optional[HistoryItem]: - """One ``im/v1/messages`` entry → HistoryItem, reusing the inbound content parser.""" - if not isinstance(raw, dict) or raw.get("deleted"): - return None - body = raw.get("body") or {} - text, attachments = cls._extract_payload(raw.get("msg_type"), body.get("content")) - if not text and not attachments: - return None - sender = raw.get("sender") or {} - message_id = raw.get("message_id") or "" - try: - ts_ms = int(raw.get("create_time") or 0) - except (TypeError, ValueError): - ts_ms = 0 - return HistoryItem( - message_id=message_id, - # Feishu returns only ids here; a display name would need a directory lookup per - # sender, which is not worth an extra API call per message for background context. - sender_name=(sender.get("id") or "")[-6:] or "群成员", - text=text, - ts_ms=ts_ms, - attachments=attachments, - # download_resource resolves resources by message_id — carry it so a file pulled - # from history stays fetchable later via channel_read_attachment. - raw={"lark_message_id": message_id}, - ) - - # ── Group listening: is this group message @'ing the bot? ─────────── - async def resolve_addressed(self, conn: Any, inbound: InboundMsg) -> bool: - """True when the bot's own open_id appears in the message's mention list. - - Fails open (True) whenever the bot identity cannot be fetched, so a transient - ``bot/v3/info`` failure degrades to today's behavior (answer everything delivered) - rather than making the bot go mute in the group. - """ - try: - bot_open_id = await self._bot_open_id(conn) - except Exception: # noqa: BLE001 - logger.debug("[lark] 机器人 open_id 获取失败，按「已被 @」处理", exc_info=True) - return True - if not bot_open_id: - return True - return bot_open_id in (inbound.mentioned_ids or []) - - async def _bot_open_id(self, conn: Any) -> str: - """The bot's own open_id (``/open-apis/bot/v3/info``), cached per app_id. - - Feishu never puts the bot identity in the event payload, so it has to be fetched once - and kept. The value is stable for the lifetime of the app, hence a plain process cache - with no expiry (a restart just re-fetches). - """ - cached = self._bot_id_cache.get(conn.app_id) - if cached: - return cached - token = await self._tenant_token(conn.app_id, self._app_secret(conn)) - async with httpx.AsyncClient(timeout=10) as client: - resp = await client.get( - f"{LARK_API_BASE}/open-apis/bot/v3/info", - headers={"Authorization": f"Bearer {token}"}, - ) - open_id = ((resp.json() or {}).get("bot") or {}).get("open_id") or "" - if open_id: - self._bot_id_cache[conn.app_id] = open_id - return open_id - - # ── Outbound push ─────────────────────────────────────────────────── - def _chat_id(self, inbound: InboundMsg) -> str: - return (inbound.raw or {}).get("lark_chat_id") or inbound.external_conversation_id - - async def _post_message(self, conn: Any, chat_id: str, msg_type: str, content: Dict[str, Any]) -> SendResult: - if not chat_id: - return SendResult.fail("bad_format", "缺少 chat_id") - try: - token = await self._tenant_token(conn.app_id, self._app_secret(conn)) - except Exception as exc: # noqa: BLE001 - return SendResult.fail("transient", str(exc)) - url = f"{LARK_API_BASE}/open-apis/im/v1/messages?receive_id_type=chat_id" - payload = {"receive_id": chat_id, "msg_type": msg_type, - "content": json.dumps(content, ensure_ascii=False)} - try: - async with httpx.AsyncClient(timeout=15) as client: - resp = await client.post(url, headers={"Authorization": f"Bearer {token}"}, json=payload) - data = resp.json() - except Exception as exc: # noqa: BLE001 - return SendResult.fail("transient", str(exc)) - if data.get("code") == 0: - return SendResult.ok((data.get("data") or {}).get("message_id")) - code = data.get("code") - kind = "rate_limited" if code in (99991400, 99991401) else "forbidden" if code in (230002, 230013) else "unknown" - return SendResult.fail(kind, f"code={code} msg={data.get('msg')}") - - async def send_text(self, conn: Any, inbound: InboundMsg, text: str) -> SendResult: - return await self._post_message(conn, self._chat_id(inbound), "text", {"text": text}) - - async def push(self, conn: Any, inbound: InboundMsg, content: str) -> SendResult: - """Over-length content is auto-chunked by caps.max_message_len and sent piece by piece.""" - from core.channels.protocol import chunk_text - - chunks = chunk_text(content, self.caps.max_message_len) or [content] - first: Optional[SendResult] = None - cont: list = [] - for i, c in enumerate(chunks): - r = await self.send_text(conn, inbound, c) - if i == 0: - first = r - if not r.success: - return r - elif r.message_id: - cont.append(r.message_id) - if first is not None: - first.continuation_ids = cont - return first - return SendResult.fail("unknown", "空内容") - - async def edit_message(self, conn: Any, message_id: str, text: str) -> SendResult: - if not message_id: - return SendResult.fail("bad_format", "缺少 message_id") - try: - token = await self._tenant_token(conn.app_id, self._app_secret(conn)) - except Exception as exc: # noqa: BLE001 - return SendResult.fail("transient", str(exc)) - url = f"{LARK_API_BASE}/open-apis/im/v1/messages/{message_id}" - payload = {"msg_type": "text", "content": json.dumps({"text": text}, ensure_ascii=False)} - try: - async with httpx.AsyncClient(timeout=15) as client: - resp = await client.put(url, headers={"Authorization": f"Bearer {token}"}, json=payload) - data = resp.json() - except Exception as exc: # noqa: BLE001 - return SendResult.fail("transient", str(exc)) - if data.get("code") == 0: - return SendResult.ok(message_id) - return SendResult.fail("unknown", f"code={data.get('code')} msg={data.get('msg')}") - - # ── Inbound file download ─────────────────────────────────────────── - async def download_resource( - self, conn: Any, inbound: InboundMsg, attachment: Dict[str, Any] - ) -> Optional[bytes]: - """Download a resource file from a message (requires the im:resource permission). - - GET /open-apis/im/v1/messages/{message_id}/resources/{file_key}?type=file|image - """ - msg_id = (inbound.raw or {}).get("lark_message_id") or inbound.message_id - key = attachment.get("key") - res_type = "image" if attachment.get("kind") == "image" else "file" - if not msg_id or not key: - return None - try: - token = await self._tenant_token(conn.app_id, self._app_secret(conn)) - url = f"{LARK_API_BASE}/open-apis/im/v1/messages/{msg_id}/resources/{key}?type={res_type}" - async with httpx.AsyncClient(timeout=30) as client: - resp = await client.get(url, headers={"Authorization": f"Bearer {token}"}) - if resp.status_code == 200 and not resp.headers.get("content-type", "").startswith("application/json"): - return resp.content - logger.warning("[lark] 资源下载失败 msg=%s key=%s body=%s", msg_id, key, resp.text[:200]) - except Exception: # noqa: BLE001 - logger.exception("[lark] 资源下载异常 msg=%s key=%s", msg_id, key) - return None - - # ── Outbound file delivery ────────────────────────────────────────── - async def push_file( - self, conn: Any, inbound: InboundMsg, content: bytes, filename: str, mime_type: str - ) -> SendResult: - chat_id = self._chat_id(inbound) - if not chat_id: - return SendResult.fail("bad_format", "缺少 chat_id") - try: - token = await self._tenant_token(conn.app_id, self._app_secret(conn)) - except Exception as exc: # noqa: BLE001 - return SendResult.fail("transient", str(exc)) - - is_image = (mime_type or "").startswith("image/") - try: - if is_image: - key = await self._upload_image(token, content) - msg_type, msg_content = "image", {"image_key": key} - else: - key = await self._upload_file(token, content, filename) - msg_type, msg_content = "file", {"file_key": key} - except Exception as exc: # noqa: BLE001 - return SendResult.fail("transient", f"上传失败: {exc}") - if not key: - return SendResult.fail("unknown", "未拿到资源 key") - return await self._post_message(conn, chat_id, msg_type, msg_content) - - async def _upload_image(self, token: str, content: bytes) -> Optional[str]: - url = f"{LARK_API_BASE}/open-apis/im/v1/images" - async with httpx.AsyncClient(timeout=30) as client: - resp = await client.post( - url, headers={"Authorization": f"Bearer {token}"}, - data={"image_type": "message"}, - files={"image": ("img.png", content)}, - ) - data = resp.json() - return (data.get("data") or {}).get("image_key") if data.get("code") == 0 else None - - async def _upload_file(self, token: str, content: bytes, filename: str) -> Optional[str]: - # file_type: Feishu distinguishes by extension; anything not in the known set is sent as stream - ext = (filename.rsplit(".", 1)[-1] if "." in filename else "").lower() - known = {"opus", "mp4", "pdf", "doc", "xls", "ppt"} - file_type = ext if ext in known else "stream" - url = f"{LARK_API_BASE}/open-apis/im/v1/files" - async with httpx.AsyncClient(timeout=60) as client: - resp = await client.post( - url, headers={"Authorization": f"Bearer {token}"}, - data={"file_type": file_type, "file_name": filename}, - files={"file": (filename, content)}, - ) - data = resp.json() - return (data.get("data") or {}).get("file_key") if data.get("code") == 0 else None - - # ── Long connection (WebSocket, requires the lark_oapi SDK) ───────── - def make_ws_client(self, conn: Any, on_message: Callable[[InboundMsg], None]) -> Any: - """Build the Feishu long-connection client. ``on_message`` is invoked synchronously on the SDK thread; the caller dispatches it to the main loop. - - lark_oapi is lazily imported; if not installed a RuntimeError is raised, - which the manager records as an error state without affecting the process. - """ - try: - import lark_oapi as lark - from lark_oapi.api.im.v1 import P2ImMessageReceiveV1 - except Exception as exc: # noqa: BLE001 - raise RuntimeError("lark_oapi SDK 未安装，长连接不可用") from exc - - secret = self._app_secret(conn) - - def _do_message(data: "P2ImMessageReceiveV1") -> None: - try: - inbound = self._inbound_from_sdk_event(conn, data) - if inbound is not None: - on_message(inbound) - except Exception: # noqa: BLE001 - logger.exception("[lark] 长连接事件处理失败 channel_id=%s", conn.channel_id) - - handler = ( - lark.EventDispatcherHandler.builder("", "") - .register_p2_im_message_receive_v1(_do_message) - .build() - ) - return lark.ws.Client( - conn.app_id, secret, event_handler=handler, log_level=lark.LogLevel.WARNING - ) - - def prepare_ws_thread(self, loop: Any) -> None: - """Called inside the worker thread: install the per-thread loop proxy + write this thread's loop into thread-local storage. - - From then on, every access to the global ``loop`` inside - lark_oapi.ws.client is forwarded by the proxy to **this thread's** loop — - multiple Feishu bots each get their own thread and their own loop with - no mutual overwriting (fixes "This event loop is already running"). The - proxy is installed only once (process-level idempotent); each thread - writes its own thread-local value. - """ - _install_ws_loop_proxy() - _ws_loop_tls.loop = loop - - def _inbound_from_sdk_event(self, conn: Any, data: Any) -> Optional[InboundMsg]: - """lark_oapi's typed P2ImMessageReceiveV1 event → InboundMsg.""" - event = getattr(data, "event", None) - message = getattr(event, "message", None) - sender = getattr(event, "sender", None) - if message is None: - return None - mtype = getattr(message, "message_type", None) - text, attachments = self._extract_payload(mtype, getattr(message, "content", None)) - if not text and not attachments: - return None - sender_id_obj = getattr(sender, "sender_id", None) - open_id = getattr(sender_id_obj, "open_id", "") if sender_id_obj else "" - return self._make_inbound( - conn=conn, - chat_id=getattr(message, "chat_id", "") or "", - chat_type=getattr(message, "chat_type", "p2p") or "p2p", - message_id=getattr(message, "message_id", "") or "", - text=text, - sender_open_id=open_id or "", - sender_name="", - attachments=attachments, - mentioned_ids=self._sdk_mentioned_ids(message), - raw={}, - ) - - @staticmethod - def _sdk_mentioned_ids(message: Any) -> List[str]: - """SDK-typed counterpart of ``_mentioned_ids`` (``MentionEvent.id.open_id``). - - The long connection is the production path, so mention extraction must exist on both - sides; missing it here would make every group message look un-@'d. - """ - out: List[str] = [] - for m in getattr(message, "mentions", None) or []: - ident = getattr(m, "id", None) - oid = getattr(ident, "open_id", "") if ident is not None else "" - if oid: - out.append(oid) - return out diff --git a/src/backend/core/channels/inbound.py b/src/backend/core/channels/inbound.py deleted file mode 100644 index d8915037..00000000 --- a/src/backend/core/channels/inbound.py +++ /dev/null @@ -1,1005 +0,0 @@ -"""Inbound message orchestration: InboundMsg → reuse the chat pipeline as the owner → send the reply back to the channel. - -Owner service-account model: - - Always runs as the bot's ``owner_user_id`` (no group-member resolution, no team hookup). - - p2p / group share one path; the only difference is session keying: - p2p → (channel_id, sender open_id) one session per private-chat peer - group → (channel_id, group chat_id) the whole group shares one session - - The resource_scope whitelist (if any) narrows the owner's full capabilities to the specified KBs / skills. - - Speaker open_id / nickname are recorded for audit only, never mapped to a platform account. - -Reuses ``chat_run_executor.start_run`` + ``follow_run`` with zero changes to the orchestration layer. -See internal design docs. -""" - -from __future__ import annotations - -import asyncio -import logging -import mimetypes -from collections import OrderedDict, defaultdict -from datetime import datetime, timedelta, timezone -from typing import Any, Dict, List, Optional - -from core.channels.protocol import InboundMsg -from core.channels.registry import get_adapter -from core.chat.context import resolve_enabled_capabilities -from core.db.engine import SessionLocal -from core.db.models import Artifact, ChannelConnection, ChatSession -from core.db.repository.channel import ChannelConnectionRepository -from core.services.chat_service import ChatService - -logger = logging.getLogger(__name__) - -# Inbound file size cap (same as /v1/file/upload) -_MAX_INBOUND_FILE_BYTES = 50 * 1024 * 1024 - -# #1 Per-conversation serialization: one asyncio lock per (channel_id, conversation); messages -# arriving while a run is in progress queue up. handle_inbound always runs on the main event loop, -# so asyncio.Lock suffices. Different conversations have their own locks → still concurrent. -_conv_locks: "defaultdict[str, asyncio.Lock]" = defaultdict(asyncio.Lock) - - -class ChannelRunError(RuntimeError): - """A user-facing terminal error emitted by the shared chat-run stream.""" - - -def _conv_key(msg: "InboundMsg") -> str: - return f"{msg.channel_id}:{msg.external_conversation_id}" - -# In-process idempotent dedup: channels redeliver events (webhook retries / long-connection -# reconnect replays). Deduplicate by message_id with a bounded LRU (enough to cover short-window -# redelivery; strict cross-process exactly-once is not a goal). -_SEEN_MAX = 4096 -_seen_message_ids: "OrderedDict[str, None]" = OrderedDict() - - -def _already_handled(message_id: str) -> bool: - if not message_id: - return False - if message_id in _seen_message_ids: - return True - _seen_message_ids[message_id] = None - if len(_seen_message_ids) > _SEEN_MAX: - _seen_message_ids.popitem(last=False) - return False - - -# Channel-side "new conversation / clear context" commands. Only triggers when the whole -# message (after trim) exactly equals one of them, so normal sentences like "帮我清空购物车" -# ("clear my shopping cart") are never falsely matched. -_RESET_COMMANDS = frozenset({ - "/new", "/clear", "/reset", "/restart", - "新对话", "新会话", "清空", "清空上下文", "清除上下文", - "重置", "重置对话", "重新开始", -}) - - -def _is_reset_command(text: Optional[str]) -> bool: - return (text or "").strip().lower() in _RESET_COMMANDS - - -def _reset_session(db, conn: ChannelConnection, msg: InboundMsg) -> None: - """Soft-delete the current channel session. The next non-command message creates a fresh empty session via _find_or_create_session.""" - existing = ( - db.query(ChatSession) - .filter( - ChatSession.channel_id == conn.channel_id, - ChatSession.external_conversation_id == msg.external_conversation_id, - ChatSession.deleted_at.is_(None), - ) - .first() - ) - if existing is not None: - ChatService(db).delete_session_force( - existing.chat_id, actor_user_id=conn.owner_user_id - ) - - -def _find_or_create_session( - db, conn: ChannelConnection, msg: InboundMsg -) -> ChatSession: - """Reuse or create a session keyed by (channel_id, external_conversation_id), owned by the owner.""" - existing = ( - db.query(ChatSession) - .filter( - ChatSession.channel_id == conn.channel_id, - ChatSession.external_conversation_id == msg.external_conversation_id, - # Exclude soft-deleted sessions: the channel-side /new clear works by soft-deleting - # the current session — without this filter the next message would hit the old - # session again, making the clear a no-op. - ChatSession.deleted_at.is_(None), - ) - .first() - ) - if existing is not None: - # Backfill the p2p peer id: proactive delivery (automation etc.) needs it to locate - # the recipient when sending back in a private chat; old sessions predate this field, - # so we backfill on every inbound message. - if msg.chat_type == "p2p" and msg.sender_id: - meta = dict(existing.extra_data or {}) - if meta.get("channel_peer_id") != msg.sender_id: - meta["channel_peer_id"] = msg.sender_id - existing.extra_data = meta - db.commit() - return existing - - chat_service = ChatService(db) - title = (msg.text or "渠道会话").strip()[:40] or "渠道会话" - extra: Dict[str, Any] = { - "source": f"channel:{conn.channel_type}", - "channel_chat_type": msg.chat_type, - } - if msg.chat_type == "p2p" and msg.sender_id: - extra["channel_peer_id"] = msg.sender_id - session = chat_service.create_session( - user_id=conn.owner_user_id, - title=title, - extra_data=extra, - ) - session.channel_id = conn.channel_id - session.external_conversation_id = msg.external_conversation_id - db.commit() - db.refresh(session) - return session - - -def _load_history(db, chat_id: str, owner_id: str) -> List[Dict[str, Any]]: - """Load history as session_messages — same source as the web path (checkpoint-aware + keeps tool calls/results). - - Goes through ``compaction_service.load_session_history`` (the same entry point as the web - pipeline), not the old "``list_all_messages`` + keep only user/assistant plain text". The - old approach **dropped wholesale** the assistant turns' ``tool_calls`` and tool results, as - well as empty-text pure-tool turns — so in multi-turn channel sessions the model couldn't - see which tools it called last turn or what results it got. With thinking disabled it would - very easily redo the same work from scratch, degenerating into "I'll get right on generating…" - style idling until hitting max_iters. With the same-source loader, the model gets real tool - context across turns and reuses compaction checkpoints (large sessions no longer replay in full). - - When there is no access permission (theoretically impossible — the session was just created - by _find_or_create_session and belongs to the owner), load_session_history returns None; - we fall back to empty history here. - """ - from core.services.compaction_service import load_session_history - - chat_service = ChatService(db) - return load_session_history(chat_service, chat_id, owner_id) or [] - - -def _resolve_enabled( - db, conn: ChannelConnection, owner_id: str -) -> Dict[str, Optional[List[str]]]: - """Resolve the owner's capabilities, then narrow them by the resource_scope whitelist.""" - skills, agents, mcps = resolve_enabled_capabilities(db, owner_id) - scope = conn.resource_scope if isinstance(conn.resource_scope, dict) else {} - scoped_skills = scope.get("skill_ids") - scoped_kbs = scope.get("kb_ids") - return { - # Whitelist present → narrow to it (the owner must still own these; catalog gating applies as usual) - "enabled_skills": scoped_skills if isinstance(scoped_skills, list) else skills, - "enabled_agents": agents, - "enabled_mcps": mcps, - "enabled_kbs": scoped_kbs if isinstance(scoped_kbs, list) else None, - } - - -async def _ingest_attachments( - db, adapter, conn: ChannelConnection, owner_id: str, chat_id: str, msg: InboundMsg -) -> List[Dict[str, Any]]: - """Download inbound attachments → store as Artifacts (mirrors /v1/file/upload) → return uploaded_files items. - - Each item contains metadata only. Parsing is deferred to the shared - file-context/read_artifact path and cached on first use. - """ - download = getattr(adapter, "download_resource", None) - if download is None or not msg.attachments: - return [] - from core.services.artifact_service import store_bytes_as_artifact - - out: List[Dict[str, Any]] = [] - for att in msg.attachments: - try: - content = await download(conn, msg, att) - if not content or len(content) > _MAX_INBOUND_FILE_BYTES: - continue - name = att.get("name") or "file.bin" - mime = mimetypes.guess_type(name)[0] or "application/octet-stream" - art = store_bytes_as_artifact( - db, user_id=owner_id, content=content, filename=name, mime_type=mime, - chat_id=chat_id, source="channel_upload", - extra={"channel_id": conn.channel_id}, - ) - out.append({ - "file_id": art.artifact_id, - "download_url": f"/files/{art.artifact_id}", - "name": name, - "mime_type": mime, - }) - except Exception: # noqa: BLE001 - logger.exception("[channels] 入站附件处理失败 key=%s", att.get("key")) - return out - - -async def _collect_reply(run_id: str): - """Follow the run's event stream, accumulating assistant text + capturing artifact files generated this turn (meta.artifacts). - - Structured-reasoning models emit thinking as separate ``thinking`` events (ignored - here), but inline-thinking models embed ``…`` in the content deltas - themselves — strip those before handing the text to the channel, or the raw chain - of thought goes out as message text (the web UI parses the tags; IM cannot). - """ - from core.channels.markdown import strip_inline_thinking - from orchestration import chat_run_executor - - full = "" - artifacts: List[Dict[str, Any]] = [] - from core.channels.registry import current_relay_adapter - - relay = current_relay_adapter() - async for event in chat_run_executor.follow_run(run_id): - et = event.get("type") - if relay and et in {"file_confirm", "ask_user_question"}: - relay.error = "此操作需要确认，请在桌面端处理后重试" - await relay.cancel() - raise ChannelRunError(relay.error) - if et == "content": - full += event.get("delta", "") or "" - elif et == "meta": - arts = event.get("artifacts") - if isinstance(arts, list): - artifacts = arts - elif et == "error": - detail = str(event.get("error") or event.get("delta") or "").strip() - raise ChannelRunError(detail or "请求处理失败，请稍后重试") - return strip_inline_thinking(full).strip(), artifacts - - -def _load_generated_files(artifacts: List[Dict[str, Any]]): - """Read the bytes of artifact files generated this turn; returns a list of (content, name, mime) for pushing back (best-effort).""" - if not artifacts: - return [] - from core.storage import get_storage - - files = [] - with SessionLocal() as db: - storage = get_storage() - for art in artifacts: - fid = art.get("file_id") - if not fid: - continue - row = db.query(Artifact).filter(Artifact.artifact_id == fid).first() - if row is None or not row.storage_key: - continue - try: - content = storage.download_bytes(row.storage_key) - except Exception: # noqa: BLE001 - logger.warning("[channels] 产物下载失败 %s", fid, exc_info=True) - continue - files.append((content, row.filename or art.get("name") or fid, - row.mime_type or "application/octet-stream")) - return files - - -def _speaker_label(msg: InboundMsg) -> str: - """Label the speaker in group scenarios (no directory lookup; falls back to existing fields).""" - return (msg.sender_name or (msg.sender_id or "")[-6:] or "用户") - - -# ── Group listening (observe_all): bystander messages as background context ────────────── -# Cap on the buffered bystander messages carried into the next @bot turn. A busy group can -# produce hundreds of messages between two @s; without a cap the buffer would blow up both the -# JSON column and the prompt. Oldest entries are evicted first — recent context is what matters. -_OBSERVE_MAX = 60 -# Per-message truncation. Long pasted blocks are background material, not the task. -_OBSERVE_TEXT_MAX = 500 -# Cap on the lazy-fetch file index. Larger than _OBSERVE_MAX because a file stays worth -# opening long after the chatter around it scrolled out of the message buffer. -_OBSERVE_FILE_INDEX_MAX = 100 - -# ── Deterministic history pull budget ──────────────────────────────────────────────────── -# Group history is pulled by code on every @, never by the model deciding to call a tool: -# a capability that only works when the model remembers to use it is not a capability. -# Every pull is incremental — bounded below by the stored cursor — so the same messages are -# not fetched twice; the window below only applies to the very first pull in a conversation, -# where there is no cursor yet. -_HISTORY_COLDSTART_WINDOW_H = 168 # how far back the first pull reaches (7 days) -_HISTORY_MAX_MESSAGES = 80 # hard cap per pull -_HISTORY_MAX_TOTAL_CHARS = 12000 # total text budget per pull, so one chatty group can't flood the prompt -_HISTORY_TIMEOUT_S = 10 # a slow platform must never hold up the reply -# Consecutive failures after which we stop retrying for a conversation. Without this, a -# permanently unavailable API would add its timeout to every single @ forever. -_HISTORY_MAX_FAILURES = 3 - - -async def _resolve_addressed(adapter, conn: ChannelConnection, msg: InboundMsg) -> bool: - """Is this message aimed at the bot? Adapters that cannot decide synchronously defer here. - - Fails open (True) on any resolver error or when the adapter has no resolver at all, so - channels that never learned about group listening (WeCom / WeChat) keep behaving exactly - as before. - """ - if msg.addressed_to_bot is not None: - return bool(msg.addressed_to_bot) - resolver = getattr(adapter, "resolve_addressed", None) - if resolver is None: - return True - try: - return bool(await resolver(conn, msg)) - except Exception: # noqa: BLE001 - logger.debug("[channels] @ 判定失败，按「已被 @」处理 channel_id=%s", conn.channel_id, exc_info=True) - return True - - -def _observe_text(msg: InboundMsg) -> str: - """The bystander message's own text, truncated. Attachments are handled separately.""" - text = (msg.text or "").strip() - if len(text) > _OBSERVE_TEXT_MAX: - text = text[:_OBSERVE_TEXT_MAX] + "…" - return text - - -def _observe_attachment_refs(msg: InboundMsg) -> List[Dict[str, Any]]: - """Attachment handles for a bystander message — recorded, never downloaded. - - Eagerly downloading everything a group shares would pull the whole group's file traffic - into our storage: large I/O and a far wider privacy footprint than reading text. But - *discarding the key* (the earlier behavior) left the agent able to see that a file existed - while being unable to ever open it. Keeping the key costs nothing and turns this into - lazy loading: the model asks for a file by key via ``channel_read_attachment`` only when - it decides the file matters. - """ - return [ - {"kind": a.get("kind") or "file", "key": a.get("key") or "", "name": a.get("name") or ""} - for a in (msg.attachments or []) - if a.get("key") - ] - - -def _observe_message(db, conn: ChannelConnection, msg: InboundMsg) -> None: - """Record a bystander group message into the session's observation buffer. No agent run, no reply. - - The buffer lives in ``ChatSession.extra_data`` rather than being written as real session - messages, for three reasons: it stays bounded (a quiet group can't inflate history), it - avoids long runs of consecutive user-role turns in the model's message list, and draining - it into the triggering turn keeps the group log in one coherent block instead of scattered - fragments. - """ - text = _observe_text(msg) - refs = _observe_attachment_refs(msg) - if not text and not refs: - return - session = _find_or_create_session(db, conn, msg) - meta = dict(session.extra_data or {}) - - entry: Dict[str, Any] = {"n": _speaker_label(msg), "t": text} - if refs: - entry["a"] = refs - buf = list(meta.get("observed") or []) - buf.append(entry) - meta["observed"] = buf[-_OBSERVE_MAX:] - - if refs: - # Separate, longer-lived index than the message buffer: the buffer is drained into a - # prompt on the next @, but the model may only decide to open a file several turns - # later. Keyed by attachment key, it stores what the adapter needs to fetch the bytes - # (Lark resolves resources by message_id, DingTalk by the robotCode in raw) so none of - # that plumbing has to appear in the prompt. - index = dict(meta.get("observed_files") or {}) - for ref in refs: - index.pop(ref["key"], None) # re-insert so recency ordering stays correct - index[ref["key"]] = { - "name": ref["name"], "kind": ref["kind"], - "message_id": msg.message_id, "raw": dict(msg.raw or {}), - } - while len(index) > _OBSERVE_FILE_INDEX_MAX: - index.pop(next(iter(index))) - meta["observed_files"] = index - - session.extra_data = meta # reassign: in-place JSON mutation is not change-tracked - db.commit() - - -def _record_observed_batch( - db, session: ChatSession, items: List[Dict[str, Any]], file_index: Dict[str, Any] -) -> None: - """Append a pulled batch to the observation buffer + file index, respecting the caps.""" - meta = dict(session.extra_data or {}) - buf = list(meta.get("observed") or []) - buf.extend(items) - meta["observed"] = buf[-_OBSERVE_MAX:] - if file_index: - index = dict(meta.get("observed_files") or {}) - for key, entry in file_index.items(): - index.pop(key, None) - index[key] = entry - while len(index) > _OBSERVE_FILE_INDEX_MAX: - index.pop(next(iter(index))) - meta["observed_files"] = index - session.extra_data = meta - db.commit() - - -def _history_state(session: ChatSession) -> Dict[str, Any]: - state = (session.extra_data or {}).get("history_pull") - return dict(state) if isinstance(state, dict) else {} - - -def _save_history_state(db, session: ChatSession, state: Dict[str, Any]) -> None: - meta = dict(session.extra_data or {}) - meta["history_pull"] = state - session.extra_data = meta - db.commit() - - -async def _pull_history( - db, adapter, conn: ChannelConnection, msg: InboundMsg, session: ChatSession -) -> int: - """Deterministically pull this group's new messages and fold them into the buffer. - - Runs on every @ in a group, driven by code rather than by the model choosing to call a - tool. Incremental by construction: the cursor stored on the session is the lower bound, so - a pull only ever returns messages we have not already taken. The first pull in a - conversation has no cursor and instead reaches back ``_HISTORY_COLDSTART_WINDOW_H`` hours - — that is the cold start, and it is also what makes content from *before the bot joined* - reachable at all. - - Deliberately independent of group listening: pushing and pulling sit behind different - platform permissions, so a channel that will not push non-@ messages may still allow the - pull. Returns the number of messages recorded. - """ - fetch = getattr(adapter, "fetch_history", None) - if fetch is None: - return 0 - state = _history_state(session) - if int(state.get("failures") or 0) >= _HISTORY_MAX_FAILURES: - return 0 - - cursor = int(state.get("cursor_ms") or 0) - cold_start = not cursor - if cold_start: - # Timezone-aware on purpose: this value is compared against platform epoch-ms, and - # the naive-utcnow habit elsewhere in this codebase is exactly what makes stored - # timestamps read 8 hours off. - cursor = int( - (datetime.now(timezone.utc) - timedelta(hours=_HISTORY_COLDSTART_WINDOW_H)).timestamp() * 1000 - ) - seen_ids = set(state.get("seen_ids") or []) - - try: - items = await asyncio.wait_for( - fetch( - conn, msg.external_conversation_id, - since_ms=cursor, limit=_HISTORY_MAX_MESSAGES, - # Cold start takes the newest end of the window: with a 7-day window and a - # per-pull cap, taking the oldest would surface week-old chatter and leave - # the recent messages — what the conversation is actually about — unread. - newest_first=cold_start, - ), - timeout=_HISTORY_TIMEOUT_S, - ) - # Adapters may return either order; the buffer must read chronologically. Items - # without a timestamp keep their relative position rather than sinking to the front. - items = sorted(items or [], key=lambda it: int(getattr(it, "ts_ms", 0) or 0)) - except Exception: # noqa: BLE001 — includes timeout; history is a bonus, never a blocker - logger.warning( - "[channels] 群历史拉取失败 channel_id=%s conv=%s", - conn.channel_id, msg.external_conversation_id, exc_info=True, - ) - state["failures"] = int(state.get("failures") or 0) + 1 - _save_history_state(db, session, state) - return 0 - - records: List[Dict[str, Any]] = [] - file_index: Dict[str, Any] = {} - budget = _HISTORY_MAX_TOTAL_CHARS - newest = cursor - # Which end the budget cuts from differs by intent: - # cold start — walk newest→oldest, so truncation drops the stale tail. There is no - # continuity to preserve; this is a one-off seed. - # incremental — walk oldest→newest and stop at the budget, advancing the cursor only - # over what was actually recorded, so the next pull resumes exactly there - # and no message is skipped. - for it in (reversed(items) if cold_start else items): - mid = getattr(it, "message_id", "") or "" - # Dedup on two fronts: the cursor boundary is inclusive on some platforms, and a - # message may also have already arrived via push. Either way it must not be recorded - # twice — that is what made repeated pulls wasteful. - if mid and (mid in seen_ids or _already_handled(f"hist:{mid}")): - continue - text = (getattr(it, "text", "") or "").strip() - if len(text) > _OBSERVE_TEXT_MAX: - text = text[:_OBSERVE_TEXT_MAX] + "…" - refs = [ - {"kind": a.get("kind") or "file", "key": a.get("key") or "", "name": a.get("name") or ""} - for a in (getattr(it, "attachments", None) or []) - if a.get("key") - ] - if not text and not refs: - continue - if len(text) > budget: # total budget exhausted → stop, keep the cursor honest - break - budget -= len(text) - entry: Dict[str, Any] = {"n": getattr(it, "sender_name", "") or "用户", "t": text} - if refs: - entry["a"] = refs - for ref in refs: - file_index[ref["key"]] = { - "name": ref["name"], "kind": ref["kind"], - "message_id": mid, "raw": dict(getattr(it, "raw", None) or {}), - } - records.append(entry) - if mid: - seen_ids.add(mid) - newest = max(newest, int(getattr(it, "ts_ms", 0) or 0)) - - if cold_start: - # Collected newest→oldest above; the buffer is read as a transcript, so flip it back. - records.reverse() - if records: - _record_observed_batch(db, session, records, file_index) - # Advance the cursor even when nothing was recorded: the window was still covered, and - # not advancing would re-request it on every message forever. - state.update({ - "cursor_ms": newest, - "failures": 0, - # Bounded id memory purely for boundary dedup; the cursor does the real work. - "seen_ids": list(seen_ids)[-200:], - }) - _save_history_state(db, session, state) - if records: - logger.info( - "[channels] 群历史拉取 %d 条 channel_id=%s conv=%s", - len(records), conn.channel_id, msg.external_conversation_id, - ) - return len(records) - - -def _drain_observed(db, session: ChatSession) -> List[Dict[str, Any]]: - """Take and clear the observation buffer (it is folded into the turn being started).""" - meta = dict(session.extra_data or {}) - buf = list(meta.get("observed") or []) - if not buf: - return [] - meta["observed"] = [] - session.extra_data = meta - db.commit() - return buf - - -def _render_observed_line(item: Dict[str, Any]) -> str: - """One group-log line: ``speaker：text [文件：name｜key=…]``. - - Attachment keys are rendered inline so the model can pass one straight to - ``channel_read_attachment``. The files themselves were never downloaded — the key is the - only handle that exists, which is exactly why it has to reach the prompt. - """ - parts = [item.get("t") or ""] - for ref in item.get("a") or []: - label = "图片" if ref.get("kind") == "image" else "文件" - parts.append(f"[{label}：{ref.get('name') or '未命名'}｜key={ref.get('key')}]") - return f"{item.get('n', '用户')}：{' '.join(p for p in parts if p)}" - - -def _render_observed_block(items: List[Dict[str, Any]]) -> str: - """Render the buffer as a context preamble for the triggering message. - - Explicitly labelled as background and marked do-not-reply, otherwise the model treats the - group log as a list of pending requests and tries to answer every line. - """ - lines = "\n".join(_render_observed_line(it) for it in items) - hint = ( - "\n（其中带 key= 的文件并未下载，只有句柄。确有需要时用 " - "channel_read_attachment(file_key=\"…\") 取回再读，不要凭文件名臆测内容。）" - if any(it.get("a") for it in items) else "" - ) - return ( - "【群聊上下文】以下是你被 @ 之前群里的近期对话，仅供你理解背景，" - "不要逐条回复，也不要在回复中复述：\n" - f"{lines}{hint}\n" - "【以下才是本次需要你处理的消息】\n" - ) - - -async def _recall_placeholder(adapter, conn, msg: InboundMsg, placeholder_id: str) -> None: - """Recall the placeholder message (best-effort): channels that can't edit but can recall (DingTalk) use "recall + resend" as an equivalent replacement.""" - recall = getattr(adapter, "recall_message", None) - if recall is None: - return - try: - r = await recall(conn, msg, placeholder_id) - if not r.success: - logger.debug("[channels] 占位撤回失败 kind=%s detail=%s", r.error_kind, r.error_detail) - except Exception: # noqa: BLE001 - logger.debug("[channels] 占位撤回异常", exc_info=True) - - -async def _replace_placeholder( - adapter, conn, msg: InboundMsg, placeholder_id: Optional[str], text: str -) -> None: - """Finish a placeholder flow with text, always sending a terminal message. - - Edit in place when the channel returned a placeholder ID; otherwise recall - (if supported) and resend. WeChat's send API does not return a message ID, so - it must take the direct-send path instead of silently leaving the placeholder - as the last visible message. - - Error notices / "no text reply" receipts go through here — the old logic only edited, - and since DingTalk edits always fail, users were stuck on "processing" forever with - no follow-up ever shown.""" - edit = getattr(adapter, "edit_message", None) - if placeholder_id and edit is not None: - r = await edit(conn, placeholder_id, text) - if r.success: - return - if placeholder_id: - await _recall_placeholder(adapter, conn, msg, placeholder_id) - fr = await adapter.send_text(conn, msg, text) - if not fr.success: - logger.warning("[channels] 占位替换消息发送失败 kind=%s detail=%s", fr.error_kind, fr.error_detail) - - -async def _deliver_reply(adapter, conn, msg: InboundMsg, reply: str, placeholder_id: Optional[str]) -> None: - """#2+#4: chunk long replies; if there is a placeholder message, edit the first chunk into it and send the rest as follow-up messages. - Channel supports recall but not edit (DingTalk) → recall the placeholder then send, a visually equivalent replacement. - - When the channel supports markdown (caps.supports_markdown + send_markdown), send with native rendering; - run prepare_markdown on the whole text before chunking (tables etc. can no longer be recognized and converted once cut by chunk boundaries). - [ref:tool-N] citation markers only render on the web client, so they are always stripped before going out (all channels).""" - from core.channels.markdown import strip_citation_markers - from core.channels.protocol import chunk_text - - reply = strip_citation_markers(reply) - send_md = ( - getattr(adapter, "send_markdown", None) - if getattr(adapter.caps, "supports_markdown", False) - else None - ) - if send_md is not None: - prepare = getattr(adapter, "prepare_markdown", None) - if callable(prepare): - reply = prepare(reply) - chunks = chunk_text(reply, getattr(adapter.caps, "max_message_len", 0)) or [reply] - edit = getattr(adapter, "edit_message", None) - rest = chunks - if placeholder_id and edit is not None: - r = await edit(conn, placeholder_id, chunks[0]) - if r.success: - rest = chunks[1:] - else: - # edit failed/unsupported → recall the placeholder (if supported), send all chunks as new messages - await _recall_placeholder(adapter, conn, msg, placeholder_id) - for c in rest: - fr = await (send_md or adapter.send_text)(conn, msg, c) - if not fr.success: - logger.warning("[channels] 回复分段推送失败 kind=%s detail=%s", fr.error_kind, fr.error_detail) - - -def _module_route(db, conn: ChannelConnection): - """The admit hook of the first enabled module that claims this bot, if any.""" - from core.extensions import channel_routes - - for handles, admit in channel_routes(): - if handles(db, conn): - return admit - return None - - -async def handle_inbound(msg: InboundMsg) -> None: - """Main entry point for inbound messages. Scheduled on the main event loop (long-connection threads submit via run_coroutine_threadsafe). - - All exceptions are swallowed and logged — one message's failure must not take down the long connection / process. - """ - from core.services.channel_relay_dispatch import dispatch_to_desktop - if await dispatch_to_desktop(msg, SessionLocal): - return - if _already_handled(f"{msg.channel_id}:{msg.message_id}" if msg.message_id else ""): - return - if not (msg.text or "").strip() and not msg.attachments: - return - # #1 Per-conversation serialization: until an in-progress run finishes, later messages in the same conversation queue up, avoiding history races/corruption. - async with _conv_locks[_conv_key(msg)]: - await _process_inbound(msg) - - -async def _process_inbound(msg: InboundMsg) -> None: - from core.chat.context import build_runtime_context, collect_historical_attachments - from orchestration import chat_run_executor - - placeholder_id: Optional[str] = None - accepted_run_id: Optional[str] = None - db = SessionLocal() - try: - repo = ChannelConnectionRepository(db) - conn = repo.get_by_id(msg.channel_id) - if conn is None or not conn.enabled: - logger.warning("[channels] inbound 无对应启用连接 channel_id=%s", msg.channel_id) - db.close() - return - - owner_id = conn.owner_user_id - adapter = get_adapter(conn.channel_type) - _ = conn.config # force-load the credential column so it remains usable after detaching - - # Group listening gate. Only reached at all when the platform actually delivers non-@ - # group messages (needs the channel app's "read all group messages" permission); - # otherwise every inbound group message is an @ and this is a no-op. - if msg.chat_type == "group" and not await _resolve_addressed(adapter, conn, msg): - if conn.group_listen_mode == "observe_all": - _observe_message(db, conn, msg) - repo.touch_event(conn.channel_id) # bystander traffic still proves the connection is alive - logger.info( - "[channels] 旁听群消息 channel_id=%s type=%s conv=%s", - conn.channel_id, conn.channel_type, msg.external_conversation_id, - ) - else: - # mention_only → drop entirely, exactly as before this feature existed. - # Logged because reaching this line at all is the *only* evidence that the - # platform is delivering non-@ group messages (i.e. that the "read all group - # messages" permission is actually granted). Without it, "the bot ignores the - # group" and "the platform never sends the group" look identical from outside. - # DEBUG, not INFO: an active group would otherwise flood the log. - logger.debug( - "[channels] 丢弃未 @ 的群消息（旁听未开启）channel_id=%s type=%s", - conn.channel_id, conn.channel_type, - ) - # Either way: no reset-command handling, no run, no placeholder, no reply. - db.close() - return - - admit = _module_route(db, conn) - if admit is not None: - db.expunge(conn) - db.close() - await admit(conn, msg, adapter) - return - - # Channel-side "new conversation / clear context": soft-deleting the current session is - # enough (the next message automatically creates an empty session); no agent run. This is - # where /new and clear-context actually take effect in Feishu and other clients — - # otherwise the command would be fed to the agent as plain text and the history would keep - # being reused, impossible to clear. - if _is_reset_command(msg.text): - _reset_session(db, conn, msg) - db.refresh(conn) - db.expunge(conn) - db.close() - try: - await adapter.send_text(conn, msg, "✅ 已开启新对话，之前的上下文已清除。") - except Exception: # noqa: BLE001 - logger.debug("[channels] 清空回执发送失败", exc_info=True) - return - - session = _find_or_create_session(db, conn, msg) - chat_id = session.chat_id - - # #4 Placeholder first. Everything below this line does real network work — - # downloading attachments, pulling group history — and the placeholder exists - # precisely so the user is not staring at nothing while that happens. It used to be - # sent after the whole prep stage, which put ~1s of history pull ahead of any visible - # feedback and read as the bot being slow to react. - # Prefer the channel's send_placeholder (DingTalk: robot API, recallable) — only that - # path yields a message_id supporting the later edit/recall-replace; otherwise plain - # send_text. - try: - send_ph = getattr(adapter, "send_placeholder", None) or adapter.send_text - ph = await send_ph(conn, msg, "🤔 正在处理，请稍候…") - if ph.success: - placeholder_id = ph.message_id - except Exception: # noqa: BLE001 - logger.debug("[channels] 占位消息发送失败", exc_info=True) - - # Inbound attachments: download → store as Artifact → uploaded_files - uploaded_files = await _ingest_attachments(db, adapter, conn, owner_id, chat_id, msg) - # File-only message with no text → synthesize a one-line user prompt - if (msg.text or "").strip(): - user_text = msg.text - elif uploaded_files: - user_text = "[收到文件] " + "、".join(f.get("name", "") for f in uploaded_files) - else: - db.close() - return - # #5 Label the speaker in group scenarios so the agent can tell multi-person conversations apart - if msg.chat_type == "group": - user_text = f"{_speaker_label(msg)}：{user_text}" - # Deterministic incremental pull, before draining: whatever the platform pushed - # (possibly nothing) plus whatever we can pull ends up in the same buffer, and the - # drain below sees both. Failure is absorbed inside — it must not block the reply. - await _pull_history(db, adapter, conn, msg, session) - # observe_all: prepend everything the group said since the last @ as background. - # Prepending into user_text (rather than a separate context field) means it is - # persisted with the turn, so later turns still see the group log through normal - # history loading and compaction — no second retention mechanism to maintain. - observed = _drain_observed(db, session) - if observed: - user_text = _render_observed_block(observed) + user_text - - enabled = _resolve_enabled(db, conn, owner_id) - # Enable thinking: same as the web client's default in non-fast mode. With thinking off, - # models (Qwen family especially) tend to emit shallow filler like "I'll get right on X" - # without actually landing on tool calls, idling repeatedly across turns. Thinking never - # leaks into the channel reply: structured-reasoning models emit separate thinking events - # (ignored by _collect_reply, which only accumulates content/meta), and inline-thinking - # models' … spans inside content are stripped by _collect_reply. - context = build_runtime_context( - model_name=None, user_id=owner_id, chat_id=chat_id, enable_thinking=True, - uploaded_files=uploaded_files, - enabled_skills=enabled["enabled_skills"], enabled_agents=enabled["enabled_agents"], - enabled_mcps=enabled["enabled_mcps"], enabled_kbs=enabled["enabled_kbs"], - ) - from core.services.ontology_service import build_user_ontology_runtime - - ontology_enabled, ontology_runtime = build_user_ontology_runtime( - user_id=owner_id, - task=user_text, - db=db, - ) - context["ontology_enabled"] = ontology_enabled - context["ontology_runtime"] = ontology_runtime - # Cross-turn file readability: inject files uploaded/generated earlier in this session - # (including last turn's Feishu attachments) as a summary block so the model gets real - # file_ids to call read_artifact with. Exclude this turn's attachments (already injected - # in full via uploaded_files) to avoid duplication. build_runtime_context doesn't include - # this field, so we add it here (aligned with the web client's historical_files injection - # in chats.py). - current_file_ids = {f.get("file_id") for f in uploaded_files if f.get("file_id")} - context["historical_files"] = collect_historical_attachments( - chat_id, owner_id, exclude_file_ids=current_file_ids, - ) - # Bind to a specific sub-agent: pin the whole run to that sub-agent (running with its own - # prompt/tools/model/knowledge bases), via the workflow's direct sub-agent mode. NULL → - # unset, run with the owner's default capabilities (main agent). - # Note: the sub-agent carries its own capability bindings, which override the whitelist - # narrowing from _resolve_enabled above — this is intended behavior. - direct_agent_id = conn.agent_id or (conn.config or {}).get("desktop_relay_agent_id") - if direct_agent_id: - context["agent_id"] = direct_agent_id - # #7 Trigger A: let the agent self-create scheduled delivery tasks within this conversation - context["channel_origin"] = { - "channel_id": conn.channel_id, - "conversation_id": msg.external_conversation_id, - "chat_type": msg.chat_type, - # channel_type lets the agent be told *which* platform it is in, so it can reach - # for that platform's CLI to pull group history the bot never received (anything - # from before it joined). Delivery targets never needed it; the prompt does. - "channel_type": conn.channel_type, - } - from core.services.chat_sequencer import ChatSequencer - - accepted = ChatSequencer(db).accept_main_run( - chat_id=chat_id, - user_id=owner_id, - user_content=user_text, - user_extra_data={ - "channel_sender_open_id": msg.sender_id, - "channel_sender_name": msg.sender_name, - "channel_message_id": msg.message_id, - "channel_attachments": [f.get("file_id") for f in uploaded_files], - "attachments": [ - {"file_id": f.get("file_id"), "name": f.get("name")} for f in uploaded_files - ], - }, - request_payload={ - "channel_id": msg.channel_id, - "source": "desktop_channel" if (conn.config or {}).get("desktop_relay") else "channel", - "kind": "chat", - }, - ) - accepted_run_id = accepted.run.run_id - session_messages = _load_history(db, chat_id, owner_id) - repo.touch_event(conn.channel_id) - # The commits in create_session / add_message / touch_event expire conn's column - # attributes (expire_on_commit defaults to True). If we expunged as-is, reading app_id/ - # config during push after detaching would trigger a refresh and raise - # DetachedInstanceError ("is not bound to a Session"). - # Refresh first to reload all columns, then expunge — afterwards attribute reads are - # pure in-memory and no session is needed. - db.refresh(accepted.run) - db.refresh(conn) - db.expunge(accepted.run) - db.expunge(conn) # app_id/config remain readable after leaving the session, for push - except Exception: - logger.exception("[channels] inbound 准备阶段失败 channel_id=%s", msg.channel_id) - from core.channels.registry import current_relay_adapter - if current_relay_adapter(): - current_relay_adapter().error = "本机机器人准备失败" - db.close() - if accepted_run_id: - with SessionLocal() as cleanup_db: - ChatSequencer(cleanup_db).abandon_pending_run( - accepted_run_id, - reason="channel preparation failed", - ) - return - finally: - db.close() - - # (The placeholder is sent inside the prep stage above, before attachment download and - # history pull — those are the slow parts it needs to cover.) - - # §13 Channel adaptation: IM clients like Feishu have no approval UI for MySpace write - # operations. Without pre-authorization, the gate would suspend on a MySpace write waiting - # for out-of-band user confirmation (_collect_reply only recognizes content/meta; - # file_confirm is dropped) → stuck until the 2h timeout with the placeholder never updated. - # The bot runs as the owner, and in private chat the message sender is the owner themselves, - # so pre-mark this session as allowed — equivalent to the owner approving their own write. - # (Sub-agents remain non-interactive; their /myspace writes are still rejected.) - try: - from core.llm.tools import _tool_confirmation as _mc - if not (conn.config or {}).get("desktop_relay"): - _mc.allow_session(chat_id) - except Exception: # noqa: BLE001 — a pre-authorization failure must not take down the whole run - logger.debug("[channels] myspace 写预授权失败 chat_id=%s", chat_id, exc_info=True) - - try: - run = await chat_run_executor.start_run( - accepted_run=accepted.run, - chat_id=chat_id, - user_id=owner_id, - session_messages=session_messages, - effective_user_message=user_text, - raw_user_message=user_text, - context=context, - model_name=None, - ) - except Exception as exc: - with SessionLocal() as cleanup_db: - ChatSequencer(cleanup_db).abandon_pending_run( - accepted_run_id, - reason=str(exc), - ) - logger.exception("[channels] inbound run 启动失败 channel_id=%s", msg.channel_id) - from core.channels.registry import current_relay_adapter - if current_relay_adapter(): - current_relay_adapter().error = "本机机器人启动失败" - try: - await _replace_placeholder( - adapter, conn, msg, placeholder_id, "⚠️ 处理出错了，请稍后重试。" - ) - except Exception: # noqa: BLE001 - logger.debug("[channels] 失败消息回推异常", exc_info=True) - return - - try: - from core.channels.registry import current_relay_adapter - relay_adapter = current_relay_adapter() - if relay_adapter: - relay_adapter.record_run(run.run_id, owner_id) - reply, gen_artifacts = await _collect_reply(run.run_id) - except Exception as exc: - logger.exception("[channels] inbound run 失败 channel_id=%s", msg.channel_id) - if relay_adapter: - relay_adapter.error = "本机机器人执行失败" - notice = ( - str(exc).strip() - if isinstance(exc, ChannelRunError) and str(exc).strip() - else "处理出错了，请稍后重试。" - ) - try: - await _replace_placeholder(adapter, conn, msg, placeholder_id, f"⚠️ {notice}") - except Exception: # noqa: BLE001 - logger.debug("[channels] 失败消息回推异常", exc_info=True) - return - - # Note: the assistant reply is **not persisted here**. start_run's background executor - # (chat_run_executor) already persists the assistant message together with - # usage/model/tool_calls/artifacts when the stream ends. Saving another one here would create - # a **duplicate assistant message**, and one without usage (tokens recorded as 0) — exactly - # the root cause of "input/output tokens are both 0" in channel sessions. We only use the - # reply text to push back to the channel, without persisting again. - - # Push back: text (placeholder edit + follow-up chunks) + generated files - try: - if reply: - await _deliver_reply(adapter, conn, msg, reply, placeholder_id) - else: - note = "已为你生成文件。" if gen_artifacts else "（本次无文本回复）" - await _replace_placeholder(adapter, conn, msg, placeholder_id, note) - if getattr(adapter, "push_file", None): - for content, name, mime in _load_generated_files(gen_artifacts): - fr = await adapter.push_file(conn, msg, content, name, mime) - if not fr.success: - logger.warning("[channels] 文件回传失败 name=%s kind=%s", name, fr.error_kind) - except Exception: - logger.exception("[channels] inbound 回推阶段失败 channel_id=%s", msg.channel_id) - if relay_adapter: - relay_adapter.error = "本机机器人回复回传失败" diff --git a/src/backend/core/services/audio_transcription.py b/src/backend/core/chat/audio_transcription.py similarity index 98% rename from src/backend/core/services/audio_transcription.py rename to src/backend/core/chat/audio_transcription.py index 690ca47e..ebd00ee1 100644 --- a/src/backend/core/services/audio_transcription.py +++ b/src/backend/core/chat/audio_transcription.py @@ -9,7 +9,7 @@ import httpx from fastapi import HTTPException -from core.db.model_repository import get_active_role_provider +from core.models.repository import get_active_role_provider MAX_SECONDS = 120 MAX_BYTES = 16000 * 2 * MAX_SECONDS + 4096 diff --git a/src/backend/core/chat/compaction/__init__.py b/src/backend/core/chat/compaction/__init__.py new file mode 100644 index 00000000..75837be2 --- /dev/null +++ b/src/backend/core/chat/compaction/__init__.py @@ -0,0 +1 @@ +"""Context compaction for long chats: budgeting, summarising and checkpointing.""" diff --git a/src/backend/core/services/context_budget_checkpoint.py b/src/backend/core/chat/compaction/budget_checkpoint.py similarity index 97% rename from src/backend/core/services/context_budget_checkpoint.py rename to src/backend/core/chat/compaction/budget_checkpoint.py index 9537f0a6..525e9cfd 100644 --- a/src/backend/core/services/context_budget_checkpoint.py +++ b/src/backend/core/chat/compaction/budget_checkpoint.py @@ -5,8 +5,8 @@ from dataclasses import dataclass, field from core.db.models import ChatMessage -from core.llm.context_safety import ContextBudgetExceeded -from core.services.run_journal import RunJournal +from core.llm.context.safety import ContextBudgetExceeded +from core.runs.journal import RunJournal class ContextBudgetJournal(RunJournal): diff --git a/src/backend/core/services/compaction_coordinator.py b/src/backend/core/chat/compaction/coordinator.py similarity index 94% rename from src/backend/core/services/compaction_coordinator.py rename to src/backend/core/chat/compaction/coordinator.py index 3c01374b..b3a2a02b 100644 --- a/src/backend/core/services/compaction_coordinator.py +++ b/src/backend/core/chat/compaction/coordinator.py @@ -8,11 +8,11 @@ from typing import Any, Dict, List, Optional from core.config import settings as config_settings -from core.llm import compaction as C -from core.services import compaction_budget as budget -from core.services import compaction_summary as summary_model -from core.services.compaction_history import normalize_rows -from core.services.compaction_progress import finish_progress, report_outcome, start_progress +from core.llm.context import compaction as C +from core.llm.context import budget +from core.chat.compaction import summary as summary_model +from core.chat.compaction.history import normalize_rows +from core.chat.compaction.progress import finish_progress, report_outcome, start_progress logger = logging.getLogger(__name__) @@ -44,7 +44,7 @@ def __init__(self, chat_id: str, *, run_id: str = "", hook_bus: Any = None): elif run_id: from core.harness.events import EventSink from core.harness.hooks import HookBus - from core.services.harness_ledger import DurableEventStore, HarnessUsageLedger + from core.harness.ledger import DurableEventStore, HarnessUsageLedger self.hook_bus = HookBus( event_sink=EventSink(DurableEventStore()), @@ -55,7 +55,7 @@ def __init__(self, chat_id: str, *, run_id: str = "", hook_bus: Any = None): def _acquire(self): from core.db.engine import SessionLocal - from core.services.chat_service import ChatService + from core.chat.service import ChatService with SessionLocal() as db: return ChatService(db).acquire_compaction_snapshot( @@ -68,7 +68,7 @@ def _acquire(self): def _release(self) -> None: from core.db.engine import SessionLocal - from core.services.chat_service import ChatService + from core.chat.service import ChatService with SessionLocal() as db: ChatService(db).release_compaction_lease(self.chat_id, owner=self.owner) @@ -82,7 +82,7 @@ def _commit( manifest: Dict[str, Any], ) -> None: from core.db.engine import SessionLocal - from core.services.chat_service import ChatService + from core.chat.service import ChatService with SessionLocal() as db: ChatService(db).commit_compaction_checkpoint( @@ -188,7 +188,7 @@ async def compact( inputs = thaw_value(before.data["budget"]) inputs["messages"] = history if "system_prompt" not in inputs: - from core.services.compaction_summary import _load_base_system_prompt + from core.chat.compaction.summary import _load_base_system_prompt inputs["system_prompt"] = _load_base_system_prompt() estimate = budget.estimate_context_budget(**inputs) @@ -215,7 +215,7 @@ async def compact( ) return None progress = await start_progress(self.chat_id, self.run_id, phase.value) - from core.llm.model_usage import model_usage_scope + from core.models.usage import model_usage_scope with model_usage_scope( self.run_id, @@ -253,7 +253,7 @@ async def compact( } replacement_inputs = {**inputs, "messages": replacement} replacement_estimate = budget.estimate_context_budget(**replacement_inputs) - from core.llm.context_usage import build_compaction_context_usage + from core.llm.context.usage import build_compaction_context_usage manifest["replacement_budget_estimate"] = replacement_estimate manifest["replacement_context_usage"] = build_compaction_context_usage( diff --git a/src/backend/core/services/compaction_history.py b/src/backend/core/chat/compaction/history.py similarity index 93% rename from src/backend/core/services/compaction_history.py rename to src/backend/core/chat/compaction/history.py index b01671e8..6ab148eb 100644 --- a/src/backend/core/services/compaction_history.py +++ b/src/backend/core/chat/compaction/history.py @@ -6,10 +6,10 @@ from core.chat import inflight from core.chat.context import build_effective_user_message -from core.llm import compaction as C -from core.llm.context_ir import SESSION_CONTEXT_META_KEY -from core.llm.model_steps import assistant_row_replay, record_assistant_step -from core.services.chat_reference_service import render_reference_block +from core.llm.context import compaction as C +from core.llm.context.ir_constants import SESSION_CONTEXT_META_KEY +from core.llm.runtime.model_steps import assistant_row_replay, record_assistant_step +from core.chat.references import render_reference_block def _cancelled_steps(steps: Any, content: str) -> Any: diff --git a/src/backend/core/services/compaction_progress.py b/src/backend/core/chat/compaction/progress.py similarity index 100% rename from src/backend/core/services/compaction_progress.py rename to src/backend/core/chat/compaction/progress.py diff --git a/src/backend/core/services/compaction_service.py b/src/backend/core/chat/compaction/service.py similarity index 96% rename from src/backend/core/services/compaction_service.py rename to src/backend/core/chat/compaction/service.py index 479a229d..c668eada 100644 --- a/src/backend/core/services/compaction_service.py +++ b/src/backend/core/chat/compaction/service.py @@ -9,15 +9,11 @@ from core.chat import inflight from core.config import settings as config_settings -from core.llm import compaction as C -from core.llm.context_manager import ( - AUTO_COMPACT_MAX_RATIO, - resolve_model_context_window, - usable_context_window, -) -from core.services.compaction_budget import estimate_context_budget, should_compact -from core.services.compaction_history import normalize_rows -from core.services.compaction_summary import _load_base_system_prompt, summarize_without_checkpoint +from core.llm.context import compaction as C +from core.models.context_window import AUTO_COMPACT_MAX_RATIO, resolve_model_context_window +from core.llm.context.budget import estimate_context_budget, should_compact +from core.chat.compaction.history import normalize_rows +from core.chat.compaction.summary import _load_base_system_prompt, summarize_without_checkpoint logger = logging.getLogger(__name__) @@ -86,7 +82,7 @@ def _load_history(chat_service: Any, chat_id: str, *, repair: bool = True) -> Li ``repair`` is forwarded to :func:`_rows_after_seq`; see the warning there about taking a row lock on a session that outlives the read. """ - from core.services.commands.context import append_active_completions + from core.sandbox.commands.context import append_active_completions ckpt = ( chat_service.get_latest_compaction_checkpoint(chat_id) @@ -169,7 +165,7 @@ def resolve_trigger_ratio() -> float: def _read_trigger_ratio() -> float: fallback = config_settings.settings.compaction.trigger_ratio try: - from core.services.system_config import SystemConfigService + from core.config.store.system_config import SystemConfigService raw = (SystemConfigService.get_instance().get("chat.compress_in_turn_ratio") or "").strip() except Exception as exc: # noqa: BLE001 @@ -214,7 +210,7 @@ def resolve_active_tokens(usage: Optional[Dict[str, Any]]) -> int: return int(u.get("context_tokens") or u.get("total_tokens") or 0) -from core.services.compaction_coordinator import CompactionCoordinator +from core.chat.compaction.coordinator import CompactionCoordinator async def run_compaction( diff --git a/src/backend/core/services/compaction_summary.py b/src/backend/core/chat/compaction/summary.py similarity index 93% rename from src/backend/core/services/compaction_summary.py rename to src/backend/core/chat/compaction/summary.py index b4ea65c6..5bebd465 100644 --- a/src/backend/core/services/compaction_summary.py +++ b/src/backend/core/chat/compaction/summary.py @@ -8,18 +8,18 @@ from typing import Any, Dict, List, Optional import httpx -from core.llm import compaction as C -from core.llm.context_manager import resolve_model_context_window, usable_context_window +from core.llm.context import compaction as C +from core.models.context_window import resolve_model_context_window, usable_context_window logger = logging.getLogger(__name__) def _resolve_summarizer_model(): """Use the main chat model because compaction input may fill its window.""" - from core.llm.single_turn import Endpoint + from core.models.completions.single_turn import Endpoint try: - from core.services.model_config import ModelConfigService + from core.models.config import ModelConfigService c = ModelConfigService.get_instance().resolve("main_agent") if c and c.base_url and c.api_key and c.model_name: @@ -148,8 +148,8 @@ async def _summarize(history: List[Dict[str, Any]], *, timeout: int) -> Optional # (2) Call + reactive context-overflow self-rescue. The optional context is # bound by CompactionCoordinator so each direct HTTP try becomes one # append-only model attempt without changing this helper's public signature. - from core.llm.model_usage import CURRENT_MODEL_USAGE - from core.llm.single_turn import build_request, parse_response + from core.models.usage import CURRENT_MODEL_USAGE + from core.models.completions.single_turn import build_request, parse_response usage_context = CURRENT_MODEL_USAGE.get() previous_attempt_seq = None @@ -192,8 +192,9 @@ async def record_attempt(resp=None, *, status: str, started: float) -> None: ), ) previous_attempt_seq = recorded.attempt_seq - except Exception: # noqa: BLE001 - pass + except Exception as fallback_error: # noqa: BLE001 + from core.infra.failures import record_best_effort_failure + record_best_effort_failure('core.chat.compaction.summary:record_attempt', fallback_error) for attempt in range(_SUMMARIZE_MAX_ATTEMPTS): started = time.monotonic() @@ -262,7 +263,7 @@ async def summarize_without_checkpoint( ) if not older: return None - from core.services.compaction_progress import finish_progress, start_progress + from core.chat.compaction.progress import finish_progress, start_progress progress = await start_progress(chat_id, run_id, "mid_turn") if chat_id else None summary = None diff --git a/src/backend/core/services/context_usage_projection.py b/src/backend/core/chat/compaction/usage_projection.py similarity index 97% rename from src/backend/core/services/context_usage_projection.py rename to src/backend/core/chat/compaction/usage_projection.py index fa130844..127aa956 100644 --- a/src/backend/core/services/context_usage_projection.py +++ b/src/backend/core/chat/compaction/usage_projection.py @@ -5,9 +5,9 @@ from datetime import timezone from typing import Any, Dict -from core.llm.context_usage import SCHEMA_VERSION, build_compaction_context_usage +from core.llm.context.usage import SCHEMA_VERSION, build_compaction_context_usage from core.config import settings as config_settings -from core.services import compaction_budget as compaction +from core.llm.context import budget as compaction def get_chat_context_state(chat_service: Any, chat_id: str) -> Dict[str, Any]: diff --git a/src/backend/core/chat/compaction_checkpoints.py b/src/backend/core/chat/compaction_checkpoints.py new file mode 100644 index 00000000..8f9a1513 --- /dev/null +++ b/src/backend/core/chat/compaction_checkpoints.py @@ -0,0 +1,259 @@ +"""Publish and read compaction checkpoints with lease and compare-and-swap fencing.""" +from __future__ import annotations +import uuid +from dataclasses import replace +from typing import Any, Dict, List, Optional +from core.db.models import ChatCompactionState, ChatMessage +from core.chat import compaction_types +from core.infra.time import utc_now + +def invalidate_compaction_lineage_for_rewrite( + self, + chat_id: str, + *, + rewritten_seq: int, +) -> bool: + """Fence a snapshot that may contain a rewritten/deleted message.""" + state = self._ensure_compaction_state(chat_id) + # Lock the lineage row where supported. SQLite serializes the following + # write transaction; PostgreSQL blocks a concurrent publisher here. + state = ( + self.db.query(ChatCompactionState) + .filter(ChatCompactionState.chat_id == chat_id) + .with_for_update() + .populate_existing() + .one() + ) + affects_checkpoint = bool( + state.active_checkpoint_id + and int(rewritten_seq) <= int(state.covered_seq or 0) + ) + affects_snapshot = bool(state.lease_owner) + if not affects_checkpoint and not affects_snapshot: + return False + + state.active_checkpoint_id = None + state.checkpoint_version = int(state.checkpoint_version or 0) + 1 + state.covered_seq = 0 + state.lease_owner = None + state.lease_expires_at = None + state.updated_at = utc_now() + + from core.llm.context.compaction import COMPACTION_CHECKPOINT_KIND + + checkpoints = ( + self.db.query(ChatMessage) + .filter(ChatMessage.chat_id == chat_id, ChatMessage.role == "system") + .all() + ) + for checkpoint in checkpoints: + extra = dict(checkpoint.extra_data or {}) + if extra.get("kind") != COMPACTION_CHECKPOINT_KIND: + continue + extra["active"] = False + extra["notice_pending"] = False + checkpoint.extra_data = extra + return True + + +def add_compaction_checkpoint( + self, + chat_id: str, + *, + summary_text: str, + replacement_history: List[Dict], + covered_seq: Optional[int] = None, + source_hash: Optional[str] = None, + base_checkpoint_version: Optional[int] = None, + replacement_manifest: Optional[Dict[str, Any]] = None, +) -> ChatMessage: + """Compatibility wrapper around the lease + CAS checkpoint protocol.""" + owner = f"direct:{uuid.uuid4().hex}" + snapshot = self.acquire_compaction_snapshot( + chat_id, owner=owner, lease_seconds=60 + ) + if snapshot is None: + raise compaction_types.CompactionCASConflict( + "no uncompacted source or compaction lease is busy" + ) + if ( + base_checkpoint_version is not None + and snapshot.base_checkpoint_version != int(base_checkpoint_version) + ): + self.release_compaction_lease(chat_id, owner=owner) + raise compaction_types.CompactionCASConflict("checkpoint base changed") + if covered_seq is not None and int(covered_seq) != snapshot.covered_seq: + self.release_compaction_lease(chat_id, owner=owner) + raise ValueError("covered_seq must equal the acquired source watermark") + if source_hash: + snapshot = replace(snapshot, source_hash=source_hash) + return self.commit_compaction_checkpoint( + snapshot, + owner=owner, + summary_text=summary_text, + replacement_history=replacement_history, + replacement_manifest=replacement_manifest or {}, + ) + + +def latest_compaction_checkpoint_fallback( + self, chat_id: str +) -> Optional[ChatMessage]: + """Find the newest checkpoint without timestamps (legacy-state fallback).""" + from core.llm.context.compaction import COMPACTION_CHECKPOINT_KIND + + from sqlalchemy.orm import defer + + rows = ( + self.db.query(ChatMessage).options(defer(ChatMessage.model_steps), defer(ChatMessage.tool_calls)) + .filter(ChatMessage.chat_id == chat_id, ChatMessage.role == "system") + .order_by(ChatMessage.chat_seq.desc(), ChatMessage.message_id.desc()) + .limit(20) + .all() + ) + return next( + ( + row + for row in rows + if (row.extra_data or {}).get("kind") == COMPACTION_CHECKPOINT_KIND + ), + None, + ) + + +def checkpoint_covered_seq(checkpoint: Optional[ChatMessage]) -> int: + """Return only an explicitly captured source watermark. + + A legacy checkpoint was inserted *after* summarization, so its own + sequence cannot prove which earlier rows were actually summarized. + Treating ``checkpoint.chat_seq - 1`` as covered can hide a message + that arrived while the summary was in flight. Legacy checkpoints are + therefore conservatively invalidated and rebuilt from raw history. + """ + if checkpoint is None: + return 0 + extra = checkpoint.extra_data or {} + explicit = extra.get("covered_seq") + if explicit is None: + explicit = extra.get("covers_up_to_chat_seq") + if explicit is not None: + return max(0, int(explicit)) + return 0 + + +def commit_compaction_checkpoint( + self, + snapshot: compaction_types.CompactionSnapshot, + *, + owner: str, + summary_text: str, + replacement_history: List[Dict[str, Any]], + replacement_manifest: Dict[str, Any], +) -> ChatMessage: + """Atomically publish one successor from ``snapshot`` using CAS.""" + from core.llm.context.compaction import COMPACTION_CHECKPOINT_KIND + + if owner != snapshot.owner: + raise compaction_types.CompactionCASConflict("snapshot owner mismatch") + message_id = f"cmpct_{uuid.uuid4().hex[:16]}" + next_version = snapshot.base_checkpoint_version + 1 + manifest = dict(replacement_manifest) + manifest.update( + { + "source_message_ids": list(snapshot.source_message_ids), + "source_message_seqs": list(snapshot.source_message_seqs), + "source_count": len(snapshot.source_rows), + "replacement_count": len(replacement_history), + } + ) + message = self.message_repo.create( + { + "message_id": message_id, + "chat_id": snapshot.chat_id, + "role": "system", + "content": summary_text, + "extra_data": { + "kind": COMPACTION_CHECKPOINT_KIND, + "replacement_history": replacement_history, + "covered_seq": snapshot.covered_seq, + "covers_up_to_chat_seq": snapshot.covered_seq, + "covers_up_to_message_id": ( + snapshot.source_message_ids[-1] + if snapshot.source_message_ids + else None + ), + "source_hash": snapshot.source_hash, + "base_checkpoint_version": snapshot.base_checkpoint_version, + "checkpoint_version": next_version, + "replacement_manifest": manifest, + "active": True, + "notice_pending": True, + }, + }, + commit=False, + ) + now = utc_now() + updated = ( + self.db.query(ChatCompactionState) + .filter( + ChatCompactionState.chat_id == snapshot.chat_id, + ChatCompactionState.checkpoint_version + == snapshot.base_checkpoint_version, + ChatCompactionState.covered_seq == snapshot.base_covered_seq, + ChatCompactionState.lease_owner == owner, + ) + .update( + { + ChatCompactionState.active_checkpoint_id: message_id, + ChatCompactionState.checkpoint_version: next_version, + ChatCompactionState.covered_seq: snapshot.covered_seq, + ChatCompactionState.lease_owner: None, + ChatCompactionState.lease_expires_at: None, + ChatCompactionState.updated_at: now, + }, + synchronize_session=False, + ) + ) + if updated != 1: + self.db.rollback() + raise compaction_types.CompactionCASConflict("checkpoint base or lease changed") + + if snapshot.base_checkpoint_id: + previous = self.db.get(ChatMessage, snapshot.base_checkpoint_id) + if previous is not None: + previous_extra = dict(previous.extra_data or {}) + previous_extra["active"] = False + previous.extra_data = previous_extra + self.db.commit() + return message + + +def get_latest_compaction_checkpoint(self, chat_id: str) -> Optional[ChatMessage]: + """Return the chat's latest compaction checkpoint (None if there is none). + + The kind filter is done on the Python side to avoid depending on Postgres-specific + ``->>`` (SQLite compatible). Each checkpoint's extra_data carries the entire + replacement_history (can reach ~100KB), and rolling compaction keeps accumulating — + scan only the latest few rows, never ``.all()`` load everything. + """ + state = self.db.get(ChatCompactionState, chat_id) + if state is not None: + if not state.active_checkpoint_id: + return None + from sqlalchemy.orm import defer + checkpoint = self.db.get(ChatMessage, state.active_checkpoint_id, + options=[defer(ChatMessage.model_steps), defer(ChatMessage.tool_calls)]) + if ( + checkpoint is not None + and (checkpoint.extra_data or {}).get("covered_seq") is not None + ): + return checkpoint + return None + + checkpoint = self._latest_compaction_checkpoint_fallback(chat_id) + if ( + checkpoint is None + or (checkpoint.extra_data or {}).get("covered_seq") is None + ): + return None + return checkpoint diff --git a/src/backend/core/chat/compaction_snapshots.py b/src/backend/core/chat/compaction_snapshots.py new file mode 100644 index 00000000..2b5eaa46 --- /dev/null +++ b/src/backend/core/chat/compaction_snapshots.py @@ -0,0 +1,265 @@ +"""Capture the message snapshot a compaction summarises and hold its writer lease.""" +from __future__ import annotations +import hashlib +import json +from datetime import timedelta +from typing import Optional +from core.db.models import ChatCompactionState, ChatMessage, ChatSession +from core.chat import compaction_types +import logging +from core.infra.time import as_utc, utc_now +logger = logging.getLogger(__name__) + +def ensure_message_sequences(self, chat_id: str) -> None: + """Repair legacy/direct ORM rows before they enter a source snapshot.""" + from sqlalchemy import func + + # ``key_share=True`` renders ``FOR NO KEY UPDATE`` on PostgreSQL. + # A plain ``FOR UPDATE`` here is a full outage risk: inserting any row + # into ``chat_messages`` takes ``FOR KEY SHARE`` on this parent row for + # the foreign key, and ``FOR KEY SHARE`` conflicts with ``FOR UPDATE``. + # Holding that lock therefore blocks every message write for the chat — + # including the run journal's terminal commit, which runs on the event + # loop and freezes the whole process when it waits. We only bump the + # non-key column ``next_message_seq``, so the weaker lock is both + # sufficient (it still excludes a concurrent sequence repair) and + # compatible with the foreign-key lock. + chat = ( + self.db.query(ChatSession) + .filter(ChatSession.chat_id == chat_id) + .with_for_update(key_share=True) + .one_or_none() + ) + if chat is None: + return + max_seq = int( + self.db.query(func.max(ChatMessage.chat_seq)) + .filter(ChatMessage.chat_id == chat_id) + .scalar() + or 0 + ) + next_seq = max(int(chat.next_message_seq or 1), max_seq + 1) + missing = ( + self.db.query(ChatMessage) + .filter(ChatMessage.chat_id == chat_id, ChatMessage.chat_seq.is_(None)) + .order_by(ChatMessage.created_at, ChatMessage.message_id) + .all() + ) + for row in missing: + row.chat_seq = next_seq + next_seq += 1 + chat.next_message_seq = next_seq + self.db.flush() + + +def ensure_compaction_state(self, chat_id: str) -> ChatCompactionState: + state = self.db.get(ChatCompactionState, chat_id) + if state is not None: + return state + checkpoint = self._latest_compaction_checkpoint_fallback(chat_id) + extra = (checkpoint.extra_data or {}) if checkpoint is not None else {} + has_exact_watermark = ( + checkpoint is not None + and ( + extra.get("covered_seq") is not None + or extra.get("covers_up_to_chat_seq") is not None + ) + ) + state = ChatCompactionState( + chat_id=chat_id, + active_checkpoint_id=checkpoint.message_id if has_exact_watermark else None, + checkpoint_version=( + int(extra.get("checkpoint_version") or 1) if has_exact_watermark else 0 + ), + covered_seq=self._checkpoint_covered_seq(checkpoint) + if has_exact_watermark + else 0, + updated_at=utc_now(), + ) + self.db.add(state) + self.db.flush() + return state + + +def acquire_compaction_snapshot( + self, + chat_id: str, + *, + owner: str, + lease_seconds: int, +) -> Optional[compaction_types.CompactionSnapshot]: + """Capture an exact source watermark and acquire a short-lived lease. + + Every exit path must end this transaction. The body takes a row lock on + the chat before it reads, so an exception escaping with the transaction + still open strands that lock on a pooled connection that no longer has + a Python owner — an ``idle in transaction`` backend that blocks the + chat's message writes until the process dies. The rollback below is the + guarantee that cannot be forgotten by a future early return. + """ + try: + return self._acquire_compaction_snapshot( + chat_id, owner=owner, lease_seconds=lease_seconds + ) + except BaseException: + try: + self.db.rollback() + except Exception: # noqa: BLE001 - never mask the original failure + logger.exception( + "[compaction] rollback failed after a snapshot error (chat=%s)", + chat_id, + ) + raise + + +def acquire_compaction_snapshot_impl( + self, + chat_id: str, + *, + owner: str, + lease_seconds: int, +) -> Optional[compaction_types.CompactionSnapshot]: + from core.llm.context.compaction import COMPACTION_CHECKPOINT_KIND + + self._ensure_message_sequences(chat_id) + state = self._ensure_compaction_state(chat_id) + now = utc_now() + if state.lease_owner and (as_utc(state.lease_expires_at) or now) > now: + self.db.rollback() + return None + + checkpoint = ( + self.db.get(ChatMessage, state.active_checkpoint_id) + if state.active_checkpoint_id + else None + ) + replacement_history = ( + tuple( + dict(item) + for item in ( + (checkpoint.extra_data or {}).get("replacement_history") or [] + ) + ) + if checkpoint is not None + else () + ) + candidates = ( + self.db.query(ChatMessage) + .filter( + ChatMessage.chat_id == chat_id, + ChatMessage.chat_seq > int(state.covered_seq or 0), + ) + .order_by(ChatMessage.chat_seq) + .all() + ) + source_models = tuple( + row + for row in candidates + if not ( + row.role == "system" + and (row.extra_data or {}).get("kind") == COMPACTION_CHECKPOINT_KIND + ) + ) + if not source_models: + self.db.rollback() + return None + + source_rows = tuple( + compaction_types.CompactionSourceRow( + message_id=row.message_id, + chat_seq=int(row.chat_seq or 0), + role=row.role, + content=row.content, + tool_calls=row.tool_calls, + extra_data=dict(row.extra_data or {}), + ) + for row in source_models + ) + covered_seq = max(row.chat_seq for row in source_rows) + canonical_source = [ + { + "message_id": row.message_id, + "chat_seq": row.chat_seq, + "role": row.role, + "content": row.content, + "tool_calls": row.tool_calls, + "metadata": row.extra_data, + } + for row in source_rows + ] + source_hash = hashlib.sha256( + json.dumps( + { + "base_checkpoint_version": int(state.checkpoint_version or 0), + "base_covered_seq": int(state.covered_seq or 0), + "covered_seq": covered_seq, + "source": canonical_source, + }, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + default=str, + ).encode("utf-8") + ).hexdigest() + + base_version = int(state.checkpoint_version or 0) + updated = ( + self.db.query(ChatCompactionState) + .filter( + ChatCompactionState.chat_id == chat_id, + ChatCompactionState.checkpoint_version == base_version, + ( + ChatCompactionState.lease_owner.is_(None) + | (ChatCompactionState.lease_expires_at <= now) + | (ChatCompactionState.lease_owner == owner) + ), + ) + .update( + { + ChatCompactionState.lease_owner: owner, + ChatCompactionState.lease_expires_at: now + + timedelta(seconds=max(1, lease_seconds)), + ChatCompactionState.updated_at: now, + }, + synchronize_session=False, + ) + ) + if updated != 1: + self.db.rollback() + return None + base_covered_seq = int(state.covered_seq or 0) + base_checkpoint_id = checkpoint.message_id if checkpoint is not None else None + self.db.commit() + return compaction_types.CompactionSnapshot( + chat_id=chat_id, + owner=owner, + base_checkpoint_id=base_checkpoint_id, + base_checkpoint_version=base_version, + base_covered_seq=base_covered_seq, + covered_seq=covered_seq, + source_hash=source_hash, + replacement_history=replacement_history, + source_rows=source_rows, + source_message_ids=tuple(row.message_id for row in source_rows), + source_message_seqs=tuple(row.chat_seq for row in source_rows), + ) + + +def release_compaction_lease(self, chat_id: str, *, owner: str) -> bool: + updated = ( + self.db.query(ChatCompactionState) + .filter( + ChatCompactionState.chat_id == chat_id, + ChatCompactionState.lease_owner == owner, + ) + .update( + { + ChatCompactionState.lease_owner: None, + ChatCompactionState.lease_expires_at: None, + ChatCompactionState.updated_at: utc_now(), + }, + synchronize_session=False, + ) + ) + self.db.commit() + return updated == 1 diff --git a/src/backend/core/chat/compaction_types.py b/src/backend/core/chat/compaction_types.py new file mode 100644 index 00000000..f0219a50 --- /dev/null +++ b/src/backend/core/chat/compaction_types.py @@ -0,0 +1,36 @@ +"""Snapshot and conflict types shared by the compaction checkpoint protocol.""" +from __future__ import annotations +from dataclasses import dataclass +from typing import Any, Dict, Optional + +class CompactionCASConflict(RuntimeError): + """The checkpoint base/lease is stale and another writer won.""" + + +@dataclass(frozen=True) +class CompactionSourceRow: + """Detached, immutable copy of one message in the compacted snapshot.""" + + message_id: str + chat_seq: int + role: str + content: str + tool_calls: Any + extra_data: Dict[str, Any] + + +@dataclass(frozen=True) +class CompactionSnapshot: + """Immutable source boundary captured before the summary LLM call.""" + + chat_id: str + owner: str + base_checkpoint_id: Optional[str] + base_checkpoint_version: int + base_covered_seq: int + covered_seq: int + source_hash: str + replacement_history: tuple[Dict[str, Any], ...] + source_rows: tuple[CompactionSourceRow, ...] + source_message_ids: tuple[str, ...] + source_message_seqs: tuple[int, ...] diff --git a/src/backend/core/chat/context.py b/src/backend/core/chat/context.py index 21a9cbb4..8224591e 100644 --- a/src/backend/core/chat/context.py +++ b/src/backend/core/chat/context.py @@ -5,6 +5,7 @@ """ from __future__ import annotations +from core.harness.registry import execution_catalog as bridge_gateway from core.infra.time import utc_now import logging @@ -13,8 +14,8 @@ from sqlalchemy.orm import Session -from core.config.catalog_resolver import resolve_all_runtime_enabled -from core.services import UserService +from core.capabilities.catalog.resolver import resolve_all_runtime_enabled +from core.auth.users import UserService logger = logging.getLogger(__name__) @@ -40,8 +41,9 @@ def resolve_db_user_id( try: db.rollback() - except Exception: - pass + except Exception as fallback_error: + from core.infra.failures import record_best_effort_failure + record_best_effort_failure('core.chat.context:resolve_db_user_id', fallback_error) external_user = normalize_external_user_id(request_user_id) user_service = UserService(db) @@ -55,7 +57,7 @@ def resolve_db_user_id( def generate_smart_title(message: str) -> str: # 命令原文当标题等于没标题：历史列表里躺着一条叫「/init」的会话，看不出是哪个项目、 # 干了什么。命令开头的会话按命令本身命名（与前端 seedChatTitle 一致）。 - from core.services.project_init import INIT_COMMANDS + from core.chat.project_init import INIT_COMMANDS message = message.strip() if not message: @@ -82,7 +84,7 @@ def build_effective_user_message( a ``{"text": ...}`` dict; returns unchanged when there is no quote. ``reference_block`` 是用户这一轮显式引用的历史会话名片（见 - ``core.services.chat_reference_service.render_reference_block``），拼在最前面：它交代 + ``core.chat.references.render_reference_block``），拼在最前面：它交代 的是"回答这条消息要参考哪些会话"，属于这一轮的背景，先于追问引用和用户原话。 """ if isinstance(quoted_follow_up, dict): @@ -112,7 +114,7 @@ def build_effective_user_message( def resolve_user_facing_error(exc: Exception) -> str: """Map exceptions to user-friendly Chinese error strings.""" - from core.capabilities.errors import CapabilityError + from core.harness.capability_errors import CapabilityError from core.infra.exceptions import FirstResponseTimeoutError if isinstance(exc, FirstResponseTimeoutError): @@ -178,9 +180,8 @@ def _apply_desktop_cloud_bridge(ids, kind: str): 云端部署 / 纯本机模式下桥未激活，原样返回（零行为变化）。 """ try: - from core.services import desktop_cloud_bridge as bridge - apply = bridge.apply_to_enabled_skill_ids if kind == "skill" else bridge.apply_to_enabled_mcp_ids + apply = bridge_gateway.apply_to_enabled_skill_ids if kind == "skill" else bridge_gateway.apply_to_enabled_mcp_ids return apply(list(ids) if ids is not None else None) except Exception: # noqa: BLE001 - 桥故障不能影响会话装配 return ids @@ -280,11 +281,7 @@ def collect_historical_attachments( try: from core.db.engine import SessionLocal from core.db.models import Artifact as ArtifactModel, ChatMessage - from core.content.artifact_reader import ( - SOURCE_AI_GENERATED, - SOURCE_USER_UPLOAD, - infer_source, - ) + from core.files.artifacts.reader import SOURCE_AI_GENERATED, SOURCE_USER_UPLOAD, infer_source except Exception: return [] diff --git a/src/backend/core/services/chat_edition.py b/src/backend/core/chat/edition.py similarity index 100% rename from src/backend/core/services/chat_edition.py rename to src/backend/core/chat/edition.py diff --git a/src/backend/orchestration/followups.py b/src/backend/core/chat/followups.py similarity index 83% rename from src/backend/orchestration/followups.py rename to src/backend/core/chat/followups.py index d3cc23ba..b0fbd1ce 100644 --- a/src/backend/orchestration/followups.py +++ b/src/backend/core/chat/followups.py @@ -6,7 +6,7 @@ structured data (not embedded in the response text), so the frontend can render them as clickable buttons. -走 core.llm.single_turn，端点用哪条协议由它按配置决定。 +走 core.models.completions.single_turn，端点用哪条协议由它按配置决定。 """ from __future__ import annotations @@ -16,10 +16,11 @@ import logging import re import time +from dataclasses import dataclass from typing import Any, Optional from core.config.settings import settings -from core.llm.single_turn import Endpoint, complete +from core.models.completions.single_turn import Endpoint, complete _LOGGER = logging.getLogger(__name__) @@ -41,7 +42,7 @@ def _resolve_followup_endpoint() -> Optional[Endpoint]: """Resolve model config from DB: try 'followup' role, then 'summarizer', then 'main_agent'.""" try: - from core.services.model_config import ModelConfigService + from core.models.config import ModelConfigService svc = ModelConfigService.get_instance() for role in ("followup", "summarizer", "main_agent"): @@ -123,13 +124,9 @@ async def generate( status = "success" response_usage = result.usage raw = result.text - _LOGGER.info( - "[followup] raw LLM response (%d chars): %s", len(raw), raw[:200] - ) + _LOGGER.info("[followup] raw LLM response (%d chars): %s", len(raw), raw[:200]) questions = _parse_questions(raw) - _LOGGER.info( - "[followup] parsed %d questions: %s", len(questions), questions - ) + _LOGGER.info("[followup] parsed %d questions: %s", len(questions), questions) return questions except asyncio.CancelledError: @@ -144,12 +141,8 @@ async def generate( finally: if run_id: try: - from core.harness.usage import ( - AttemptUsage, - UsageAttempt, - record_usage_safely, - ) - from core.services.harness_ledger import HarnessUsageLedger + from core.harness.usage import AttemptUsage, UsageAttempt, record_usage_safely + from core.harness.ledger import HarnessUsageLedger recorder = usage_recorder or HarnessUsageLedger() await record_usage_safely( @@ -163,15 +156,9 @@ async def generate( status=status, latency_ms=int((time.monotonic() - started) * 1_000), usage=AttemptUsage( - prompt_tokens=int( - response_usage.get("prompt_tokens") or 0 - ), - completion_tokens=int( - response_usage.get("completion_tokens") or 0 - ), - cache_read_tokens=int( - response_usage.get("cache_read_tokens") or 0 - ), + prompt_tokens=int(response_usage.get("prompt_tokens") or 0), + completion_tokens=int(response_usage.get("completion_tokens") or 0), + cache_read_tokens=int(response_usage.get("cache_read_tokens") or 0), cache_write_tokens=int( response_usage.get("cache_write_tokens") or 0 ), @@ -227,11 +214,21 @@ def _clean_list(items: list) -> list[str]: # ── Singleton ────────────────────────────────────────────────────────── -_instance: FollowUpGenerator | None = None - def get_followup_generator() -> FollowUpGenerator: - global _instance - if _instance is None: - _instance = FollowUpGenerator() - return _instance + pass + if process_state().instance is None: + process_state().instance = FollowUpGenerator() + return process_state().instance + + +@dataclass +class FollowupsState: + instance: FollowUpGenerator | None = None + + +_STATE = FollowupsState() + + +def process_state() -> FollowupsState: + return _STATE diff --git a/src/backend/core/services/chat_fork_service.py b/src/backend/core/chat/fork.py similarity index 99% rename from src/backend/core/services/chat_fork_service.py rename to src/backend/core/chat/fork.py index bce6bd87..31429432 100644 --- a/src/backend/core/services/chat_fork_service.py +++ b/src/backend/core/chat/fork.py @@ -7,7 +7,7 @@ from datetime import datetime, timezone from uuid import NAMESPACE_URL, UUID, uuid4, uuid5 -from core.auth.permissions_iface import resolve_project_access +from core.auth.edition import resolve_project_access from core.db.reserved_chats import reserved_chat_error, reserved_chat_owner from core.db.models import ChatMessage, ChatRun, ChatSession, Project from sqlalchemy import update diff --git a/src/backend/core/chat/history_operations.py b/src/backend/core/chat/history_operations.py new file mode 100644 index 00000000..84b0de6e --- /dev/null +++ b/src/backend/core/chat/history_operations.py @@ -0,0 +1,49 @@ +"""User-scoped history operations for tool hosts.""" + +def history_operations(*, user_id, chat_id=None, project_id=None): + def _list(query: str, limit: int): + from core.db.engine import SessionLocal + from core.chat.references import list_referencable_sessions, session_brief + + with SessionLocal() as db: + sessions = list_referencable_sessions( + db, + user_id, + project_id=project_id, + query=query, + limit=limit, + exclude_chat_id=chat_id, + ) + return { + "ok": True, + "scope": "project" if project_id else "user", + "count": len(sessions), + "chats": [session_brief(session) for session in sessions], + } + + + def _read(target_chat_id: str, mode: str, offset: int, limit: int): + from core.db.engine import SessionLocal + from core.chat.references import read_session_messages, session_card + + with SessionLocal() as db: + card = session_card(db, target_chat_id, user_id) + if card is None: + return {"ok": False, "error": f"会话不存在或你无权访问：{target_chat_id}"} + if mode == "card": + return {"ok": True, "mode": "card", **card} + page = read_session_messages(db, target_chat_id, user_id, offset=offset, limit=limit) + if page is None: + return {"ok": False, "error": f"会话不存在或你无权访问：{target_chat_id}"} + return {"ok": True, "mode": "full", "title": card["title"], **page} + + + async def _digest(target_chat_id: str): + from core.chat.references import build_session_digest + + result = await build_session_digest(target_chat_id, user_id) + if result is None: + return {"ok": False, "error": f"会话不存在或你无权访问：{target_chat_id}"} + return {"ok": True, "mode": "digest", **result} + + return _list, _read, _digest diff --git a/src/backend/core/chat/limits.py b/src/backend/core/chat/limits.py new file mode 100644 index 00000000..6dc3146f --- /dev/null +++ b/src/backend/core/chat/limits.py @@ -0,0 +1,88 @@ +"""Length limits applied to message content and thinking before they are stored.""" +from __future__ import annotations +from typing import Any, Dict, List, Optional +import logging +logger = logging.getLogger(__name__) + +CONTENT_MAX = 100_000 + + +TRUNC_NOTE = "\n\n…（本条回复过长，已截断保存；完整过程见工具调用记录）" + + +THINKING_MAX = 200_000 + + +def clamp_message_content(content: str) -> str: + """Apply the database content limit independently of ChatService wiring.""" + + text = content or "" + if len(text) <= CONTENT_MAX: + return text + keep = CONTENT_MAX - len(TRUNC_NOTE) + logger.warning("[chat] message content truncated: %d -> %d chars", len(text), CONTENT_MAX) + kept = text[:keep] + # 老格式的思考是以 … 内联在正文里的。截断点可能落在块中间， + # 闭合标签一丢，历史重建就把整段思考当正文渲染出来（思考泄露到页面上）。 + close = "" + if kept.count("") > kept.count(close): + kept = kept[: keep - len(close)] + close + return kept + TRUNC_NOTE + + +def clamp_thinking( + blocks: Optional[List[Dict[str, Any]]], + segments: Optional[List[Dict[str, Any]]] = None, +) -> tuple[Optional[List[Dict[str, Any]]], Optional[List[Dict[str, Any]]]]: + """思考单独存一列，有自己的额度，不再和正文抢 content 的 10 万字上限。 + + 超额时从最早的块开始丢——最后几轮的推理离最终答案最近，对用户回看最有用。 + + ``metadata.segments`` 按下标引用思考块，丢块会让下标错位，所以两者必须一起夹取： + 返回夹取后的块列表，以及下标已重映射、引用了被丢块的段落已剔除的段落表。 + """ + if not blocks: + return None, segments or None + if sum(len(str(b.get("content") or "")) for b in blocks) <= THINKING_MAX: + return blocks, segments or None + kept: List[Dict[str, Any]] = [] + kept_indices: List[int] = [] + budget = THINKING_MAX + for offset, block in enumerate(reversed(blocks)): + text = str(block.get("content") or "") + if not text: + continue + if len(text) > budget: + text = text[len(text) - budget :] + if not text: + break + budget -= len(text) + kept.append({**block, "content": text}) + kept_indices.append(len(blocks) - 1 - offset) + if budget <= 0: + break + kept.reverse() + kept_indices.reverse() + if not kept: + return None, drop_thinking_segments(segments, {}) + remap = {old: new for new, old in enumerate(kept_indices)} + return kept, drop_thinking_segments(segments, remap) + + +def drop_thinking_segments( + segments: Optional[List[Dict[str, Any]]], + remap: Dict[int, int], +) -> Optional[List[Dict[str, Any]]]: + """把段落表里的思考下标重映射到夹取后的位置，引用已丢块的段落直接剔除。""" + if not segments: + return segments or None + rebuilt: List[Dict[str, Any]] = [] + for segment in segments: + if segment.get("type") != "thinking": + rebuilt.append(segment) + continue + new_index = remap.get(segment.get("index")) + if new_index is None: + continue + rebuilt.append({**segment, "index": new_index}) + return rebuilt or None diff --git a/src/backend/orchestration/message_parser.py b/src/backend/core/chat/message_parser.py similarity index 97% rename from src/backend/orchestration/message_parser.py rename to src/backend/core/chat/message_parser.py index a5ed5280..5d0217aa 100644 --- a/src/backend/orchestration/message_parser.py +++ b/src/backend/core/chat/message_parser.py @@ -99,7 +99,9 @@ def extract_text_from_messages_chunk(chunk: Any) -> Tuple[str, str, Any]: message_like, metadata = chunk node_type = "unknown" if isinstance(metadata, dict): - node_name = str(metadata.get("node_type", metadata.get("langgraph_node", ""))).strip().lower() + node_name = ( + str(metadata.get("node_type", metadata.get("langgraph_node", ""))).strip().lower() + ) if node_name == "tools": node_type = "tools" elif node_name == "model": @@ -112,6 +114,7 @@ def extract_text_from_messages_chunk(chunk: Any) -> Tuple[str, str, Any]: # Source normalisation / dedup helpers # ------------------------------------------------------------------ + def source_rank(source_type: str) -> int: mapping = { "database": 0, diff --git a/src/backend/core/chat/message_queries.py b/src/backend/core/chat/message_queries.py new file mode 100644 index 00000000..dbdb7e31 --- /dev/null +++ b/src/backend/core/chat/message_queries.py @@ -0,0 +1,91 @@ +"""Read chat messages, with access checks.""" +from __future__ import annotations +from typing import List, Optional, Tuple +from core.db.models import ChatMessage + +def list_all_messages( + self, chat_id: str, user_id: str +) -> Optional[List[ChatMessage]]: + """List all messages in chronological order with access check. + + Edition-specific sharing policies may also grant read access. + + **Excludes** compaction checkpoint rows (the only writer of role='system' in + chat_messages is add_compaction_checkpoint) — internal artifacts, invisible to all + downstream consumers (replay/export/sharing/title/memory), filtered at the SQL layer + (a checkpoint's extra_data carries the entire replacement_history; loading it just to + discard is pure waste). The replay layer (compaction_service) fetches them separately + via get_latest_compaction_checkpoint. + """ + pair = self.get_session_with_access(chat_id, user_id) + if pair is None: + return None + + return ( + self.db.query(ChatMessage) + .filter( + ChatMessage.chat_id == chat_id, + ChatMessage.role != "system", + ) + .order_by(ChatMessage.chat_seq) + .all() + ) + + +def list_messages( + self, chat_id: str, user_id: str, page: int = 1, page_size: int = 50 +) -> Optional[Tuple[List[ChatMessage], int, int]]: + """List messages in a chat session.""" + # Check ownership + session = self.get_session(chat_id, user_id) + if not session: + return None + + messages, total = self.message_repo.list_by_chat(chat_id, page, page_size) + total_pages = (total + page_size - 1) // page_size + + return messages, total, total_pages + + +def get_message_by_id(self, message_id: str) -> Optional[ChatMessage]: + """Get a single message by its ID.""" + return ( + self.db.query(ChatMessage) + .filter( + ChatMessage.message_id == message_id, + ) + .first() + ) + + +def get_message_by_index(self, chat_id: str, index: int) -> Optional[ChatMessage]: + """Get a message by its durable 0-based position in the chat.""" + return ( + self.db.query(ChatMessage) + .filter( + ChatMessage.chat_id == chat_id, + ) + .order_by(ChatMessage.chat_seq) + .offset(index) + .limit(1) + .first() + ) + + +def get_user_message_before( + self, chat_id: str, message_id: str +) -> Optional[ChatMessage]: + """Get the user message immediately before the given message.""" + target = self.get_message_by_id(message_id) + if not target: + return None + return ( + self.db.query(ChatMessage) + .filter( + ChatMessage.chat_id == chat_id, + ChatMessage.role == "user", + ChatMessage.chat_seq < target.chat_seq, + ) + .order_by(ChatMessage.chat_seq.desc()) + .first() + ) diff --git a/src/backend/core/chat/message_writes.py b/src/backend/core/chat/message_writes.py new file mode 100644 index 00000000..c2e69cd8 --- /dev/null +++ b/src/backend/core/chat/message_writes.py @@ -0,0 +1,299 @@ +"""Add, upsert, stream-refresh and delete chat messages.""" +from __future__ import annotations +from core.infra.time import as_utc, utc_now +import uuid +from typing import Any, Dict, List, Optional +from core.db.models import ChatCompactionState, ChatMessage, ChatRun, ChatSession +from core.db.repository.chat import _strip_nul +from sqlalchemy import update +from core.ontology.revision import is_substantive_revision, normalize_revision_candidate +from core.chat import limits + +def add_message( + self, + chat_id: str, + role: str, + content: str, + model: Optional[str] = None, + thinking: Optional[List[Dict[str, Any]]] = None, + tool_calls: Optional[List[Dict]] = None, + usage: Optional[Dict] = None, + error: Optional[Dict] = None, + extra_data: Dict = None, + message_id: Optional[str] = None, + commit: bool = True, + chat_seq: Optional[int] = None, + model_steps: Optional[List[Dict[str, Any]]] = None, +) -> ChatMessage: + """Add a message to a chat session.""" + extra = dict(extra_data or {}) + kept_thinking, kept_segments = limits.clamp_thinking(thinking, extra.get("segments")) + if "segments" in extra: + extra["segments"] = kept_segments + message_data = { + "message_id": message_id or f"msg_{uuid.uuid4().hex[:16]}", + "chat_id": chat_id, + "chat_seq": chat_seq, + "role": role, + "content": limits.clamp_message_content(content), + "model": model, + "thinking": kept_thinking, + "tool_calls": tool_calls, + "model_steps": model_steps, + "usage": usage, + "error": error, + "extra_data": extra, + } + + message = self.message_repo.create(message_data, commit=commit) + + # Keep session metadata in sync for list APIs. + session = self.session_repo.get_by_id(chat_id) + if session: + session.message_count = (session.message_count or 0) + 1 + now = utc_now() + session.updated_at = now + session.last_message_at = now + if commit: + self.db.commit() + + return message + + +def refresh_streaming_message( + self, + *, + run_id: str, + owner: str, + message_id: str, + content: str, + thinking: Optional[List[Dict[str, Any]]], + tool_calls: Optional[List[Dict]], + extra_data: Dict[str, Any], + model_steps: Optional[List[Dict[str, Any]]] = None, +) -> bool: + """Overwrite the in-flight assistant row while its run streams. + + The run row lock serializes this write with its final commit, so a + superseded or finishing worker cannot clobber the final message. Nothing else is touched — in particular no session + timestamps, so pollers do not mistake a checkpoint for a new message. + Returns False when the caller no longer owns the run. + """ + # Lock the same run row as journal.complete before touching its message. + # An EXISTS predicate alone can see a live lease, wait for the message + # lock, and then overwrite a final commit using that stale MVCC view. + live_run = (self.db.query(ChatRun) + .filter(ChatRun.run_id == run_id, ChatRun.lease_owner == owner, + ChatRun.status.in_(("pending", "running"))) + .with_for_update().first()) + if live_run is None: + return False + extra = dict(extra_data) + kept_thinking, kept_segments = limits.clamp_thinking(thinking, extra.get("segments")) + if "segments" in extra: + extra["segments"] = kept_segments + result = self.db.execute( + update(ChatMessage) + .where(ChatMessage.message_id == message_id) + .values( + content=limits.clamp_message_content(content), + thinking=_strip_nul(kept_thinking), + tool_calls=_strip_nul(tool_calls), + model_steps=_strip_nul(model_steps), + extra_data=_strip_nul(extra), + ) + ) + return result.rowcount > 0 + + +def upsert_message( + self, + chat_id: str, + role: str, + content: str, + *, + message_id: str, + model: Optional[str] = None, + thinking: Optional[List[Dict[str, Any]]] = None, + tool_calls: Optional[List[Dict]] = None, + usage: Optional[Dict] = None, + error: Optional[Dict] = None, + extra_data: Dict = None, + commit: bool = True, + chat_seq: Optional[int] = None, + model_steps: Optional[List[Dict[str, Any]]] = None, + invalidate_mid_turn_checkpoint: bool = False, +) -> ChatMessage: + """Idempotently upsert a message by message_id: overwrite in place if it exists, otherwise create. + + The autonomous loop uses this for "incrementally refreshing the same assistant message as + progress advances" — each requirement processed / each evaluation round flushes the + current accumulated body + tool cards into this message, so progress is visible in the DB + even after a mid-run crash/refresh, instead of only being written once at the terminal + state. The update path does not re-increment message_count (only +1 on creation). + """ + existing = self.message_repo.get_by_id(message_id) + if existing is not None: + update: Dict[str, Any] = {"content": limits.clamp_message_content(content)} + extra = dict(extra_data) if extra_data is not None else None + kept_thinking, kept_segments = limits.clamp_thinking( + thinking, (extra or {}).get("segments") + ) + if model is not None: + update["model"] = model + if thinking is not None: + update["thinking"] = kept_thinking + if tool_calls is not None: + update["tool_calls"] = tool_calls + if model_steps is not None: + update["model_steps"] = model_steps + if usage is not None: + update["usage"] = usage + if error is not None: + update["error"] = error + if extra is not None: + if "segments" in extra: + extra["segments"] = kept_segments + update["extra_data"] = extra + if invalidate_mid_turn_checkpoint: + # The completed turn has already crossed the post-turn trigger. + # Its new steps cannot be safely sliced from a row covered by a + # mid-turn checkpoint, so rebuild that successor from originals. + # Match checkpoint publisher lock order: session, then lineage. + self.db.query(ChatSession).filter(ChatSession.chat_id == chat_id).with_for_update().first() + state = (self.db.query(ChatCompactionState) + .filter(ChatCompactionState.chat_id == chat_id) + .with_for_update().populate_existing().first()) + if (state is not None and state.active_checkpoint_id + and int(existing.chat_seq or 0) <= int(state.covered_seq or 0)): + checkpoint = self.db.get(ChatMessage, state.active_checkpoint_id) + manifest = ((checkpoint.extra_data or {}).get("replacement_manifest") or {}) if checkpoint else {} + captured = (extra or {}).get("context_usage", {}).get("captured_at_ms", 0) + checkpoint_ms = int(as_utc(checkpoint.created_at).timestamp() * 1000) if checkpoint else 0 + if (manifest.get("phase") == "mid_turn" and type(captured) is int + and captured > checkpoint_ms): + self._invalidate_compaction_lineage_for_rewrite( + chat_id, rewritten_seq=int(existing.chat_seq or 0) + ) + msg = self.message_repo.update(message_id, update, commit=commit) + session = self.session_repo.get_by_id(chat_id) + if session: + now = utc_now() + session.updated_at = now + session.last_message_at = now + if commit: + self.db.commit() + return msg or existing + return self.add_message( + chat_id=chat_id, + role=role, + content=content, + model=model, + thinking=thinking, + tool_calls=tool_calls, + usage=usage, + error=error, + extra_data=extra_data, + message_id=message_id, + commit=commit, + chat_seq=chat_seq, + model_steps=model_steps, + ) + + +def delete_messages_from(self, chat_id: str, message_id: str) -> int: + """Delete a message and all subsequent messages in the chat. + + History rewrites invalidate the entire compaction lineage in the same + transaction. This fences an in-flight summarizer through the lineage + version CAS and forces the next replay/compaction to rebuild from raw + retained messages instead of trusting a stale covered watermark. + + Returns the number of messages deleted. + """ + self._ensure_message_sequences(chat_id) + target = ( + self.db.query(ChatMessage) + .filter( + ChatMessage.chat_id == chat_id, + ChatMessage.message_id == message_id, + ) + .first() + ) + if not target: + return 0 + + self._invalidate_compaction_lineage_for_rewrite( + chat_id, rewritten_seq=int(target.chat_seq) + ) + + deleted = ( + self.db.query(ChatMessage) + .filter( + ChatMessage.chat_id == chat_id, + ChatMessage.chat_seq >= int(target.chat_seq), + ) + .delete(synchronize_session="fetch") + ) + + # Update session message count + session = self.session_repo.get_by_id(chat_id) + if session: + remaining = ( + self.db.query(ChatMessage) + .filter( + ChatMessage.chat_id == chat_id, + ) + .count() + ) + session.message_count = remaining + session.updated_at = utc_now() + + self.db.commit() + return deleted + + +def update_message_extra_data( + self, + message_id: str, + patch: Dict[str, Any], +) -> bool: + """Merge *patch* into a message's extra_data. Returns True on success.""" + return self.message_repo.update_extra_data(message_id, patch) is not None + + +def accept_ontology_revision(self, message_id: str) -> Optional[ChatMessage]: + """Replace an assistant message with its persisted ontology revision candidate.""" + message = self.message_repo.get_by_id(message_id) + if not message or message.role != "assistant": + return None + extra_data = dict(message.extra_data or {}) + governance = extra_data.get("ontology_governance") + if not isinstance(governance, dict): + return None + review = governance.get("review") + if not isinstance(review, dict): + return None + candidate = normalize_revision_candidate(review.get("candidate_answer")) + if not is_substantive_revision(candidate): + return None + updated_review = {**review, "candidate_answer": candidate, "accepted": True} + extra_data["ontology_governance"] = {**governance, "review": updated_review} + # Some pure unit adapters exercise this method without a database. + if not hasattr(self, "db"): + return self.message_repo.update( + message_id, + {"content": candidate, "extra_data": extra_data}, + ) + + updated = self.message_repo.update( + message_id, + {"content": candidate, "extra_data": extra_data}, + commit=False, + ) + self._invalidate_compaction_lineage_for_rewrite( + message.chat_id, + rewritten_seq=int(message.chat_seq), + ) + self.db.commit() + return updated diff --git a/src/backend/core/chat/mode_seed.py b/src/backend/core/chat/mode_seed.py new file mode 100644 index 00000000..6a90527f --- /dev/null +++ b/src/backend/core/chat/mode_seed.py @@ -0,0 +1,51 @@ +"""Seed values for the built-in turbo chat mode.""" +from __future__ import annotations +from typing import Any, Dict +from core.db.models.chat_mode import TOOL_SCOPE_RESTRICTED +import logging +from core.chat import mode_types + +logger = logging.getLogger(__name__) + +def turbo_seed_fields() -> Dict[str, Any]: + """极速模式内置行的字段，取自历史 ``turbo.*`` SystemConfig 键。 + + 这是那批键的**最后一次**读取——播种之后本表才是真源，Config 台 + 「系统配置 → 极速模式」那组入口随之下线。 + """ + from core.config.store.execution_options import turbo_manual_invoke_enabled, turbo_max_iters, turbo_mcp_server_ids, turbo_plugin_ids, turbo_skill_ids + + try: + mcp_ids = sorted(turbo_mcp_server_ids()) + skill_ids = list(turbo_skill_ids()) + plugin_ids = list(turbo_plugin_ids()) + manual = turbo_manual_invoke_enabled() + iters = turbo_max_iters() + except Exception: # noqa: BLE001 — 配置层不可用时用内置检索三件套兜底 + logger.warning("[chat_mode] 读 turbo.* 配置失败，极速模式按内置默认播种", exc_info=True) + mcp_ids = ["internet_search", "retrieve_dataset_content", "web_fetch"] + skill_ids, plugin_ids, manual, iters = [], [], True, 4 + + return { + "id": "mode_builtin_turbo", + "slug": mode_types.SLUG_TURBO, + "name": "极速模式", + "description": "只留检索类工具、不思考，秒级直达结果", + "owner_user_id": None, + "is_builtin": True, + "enabled": True, + "sort_order": 20, + "tool_scope": TOOL_SCOPE_RESTRICTED, + "mcp_server_ids": mcp_ids, + "skill_ids": skill_ids, + "plugin_ids": plugin_ids, + "agent_ids": [], + "manual_invoke_enabled": manual, + # 极速的产品契约就是"检索直达、不执行代码"，保持关。 + "code_exec_enabled": False, + "max_iters": iters, + # 极速那条链路本就不思考，强度锁死——不是"默认不思考但可以改"。 + "default_effort": "fast", + "effort_locked": True, + "prompt_kind": "turbo", + } diff --git a/src/backend/core/chat/mode_types.py b/src/backend/core/chat/mode_types.py new file mode 100644 index 00000000..9920e3cb --- /dev/null +++ b/src/backend/core/chat/mode_types.py @@ -0,0 +1,48 @@ +"""The runtime contract a chat mode hands to agent assembly.""" +from __future__ import annotations +from dataclasses import dataclass +from typing import Optional, Tuple +from core.db.models.chat_mode import TOOL_SCOPE_ALL, TOOL_SCOPE_RESTRICTED + +@dataclass(frozen=True) +class ChatModeSpec: + """运行时装配契约——``agent_factory`` 只认这个，不认数据库行。""" + + slug: str + name: str + tool_scope: str = TOOL_SCOPE_ALL + mcp_server_ids: Tuple[str, ...] = () + skill_ids: Tuple[str, ...] = () + plugin_ids: Tuple[str, ...] = () + agent_ids: Tuple[str, ...] = () + manual_invoke_enabled: bool = True + #: 收窄模式下是否保留沙箱代码执行与文件工具；``all`` 作用域不看这个位。 + code_exec_enabled: bool = False + max_iters: Optional[int] = None + default_effort: str = "fast" + effort_locked: bool = False + prompt_kind: Optional[str] = None + prompt_text: Optional[str] = None + + @property + def restricted(self) -> bool: + """是否收窄工具面。标准模式为 False（跟随 catalog 与用户授权）。""" + return self.tool_scope == TOOL_SCOPE_RESTRICTED + + +SLUG_STANDARD = "standard" + + +SLUG_TURBO = "turbo" + + +BUILTIN_SLUGS = (SLUG_STANDARD, SLUG_TURBO) + + +STANDARD_SPEC = ChatModeSpec( + slug=SLUG_STANDARD, + name="标准模式", + tool_scope=TOOL_SCOPE_ALL, + code_exec_enabled=True, + default_effort="fast", +) diff --git a/src/backend/core/chat/mode_validation.py b/src/backend/core/chat/mode_validation.py new file mode 100644 index 00000000..3a56c337 --- /dev/null +++ b/src/backend/core/chat/mode_validation.py @@ -0,0 +1,107 @@ +"""Normalise and validate chat mode fields from request bodies.""" +from __future__ import annotations +from core.infra.time import utc_now +import uuid +from typing import Any, Dict, List, Optional +from core.db.models.chat_mode import EFFORT_VALUES, TOOL_SCOPE_ALL, TOOL_SCOPE_RESTRICTED +from core.infra.exceptions import BadRequestError + +def as_id_list(value: Any) -> List[str]: + """JSON 列 → 干净的字符串 id 列表（去空白、去重、保序）。""" + if not isinstance(value, (list, tuple)): + return [] + seen: Dict[str, None] = {} + for item in value: + if isinstance(item, str) and item.strip(): + seen.setdefault(item.strip(), None) + return list(seen.keys()) + + +def writable_fields( + payload: Dict[str, Any], owner_user_id: Optional[str], creating: bool +) -> Dict[str, Any]: + """把请求体收敛成可写字段。未出现的键在更新时保持原值。""" + out: Dict[str, Any] = {} + + def take(key: str, value: Any) -> None: + if value is not None or creating: + out[key] = value + + if creating or "name" in payload: + name = str(payload.get("name") or "").strip() + if not name: + raise BadRequestError("模式名称不能为空") + out["name"] = name[:100] + if creating or "description" in payload: + out["description"] = str(payload.get("description") or "").strip()[:500] + if creating or "enabled" in payload: + out["enabled"] = bool(payload.get("enabled", True)) + if creating or "sort_order" in payload: + try: + out["sort_order"] = int(payload.get("sort_order", 100)) + except (TypeError, ValueError): + out["sort_order"] = 100 + + if creating or "tool_scope" in payload: + scope = str(payload.get("tool_scope") or TOOL_SCOPE_RESTRICTED) + out["tool_scope"] = scope if scope in (TOOL_SCOPE_ALL, TOOL_SCOPE_RESTRICTED) else TOOL_SCOPE_RESTRICTED + for key in ("mcp_server_ids", "skill_ids", "plugin_ids", "agent_ids"): + if creating or key in payload: + out[key] = as_id_list(payload.get(key)) + if creating or "manual_invoke_enabled" in payload: + out["manual_invoke_enabled"] = bool(payload.get("manual_invoke_enabled", True)) + if creating or "code_exec_enabled" in payload: + out["code_exec_enabled"] = bool(payload.get("code_exec_enabled", False)) + if creating or "max_iters" in payload: + raw = payload.get("max_iters") + try: + # 下限 2：配成 1 会把"回答那一轮"也掐掉，等于永远拿不到结果。 + out["max_iters"] = max(2, int(raw)) if raw not in (None, "") else None + except (TypeError, ValueError): + out["max_iters"] = None + + if creating or "default_effort" in payload: + effort = str(payload.get("default_effort") or "fast") + out["default_effort"] = effort if effort in EFFORT_VALUES else "fast" + if creating or "effort_locked" in payload: + out["effort_locked"] = bool(payload.get("effort_locked", False)) + + # 提示词两种来源二选一：绑一个提示词管理里的 tab（prompt_kind，正文在那边 + # 编辑与版本化），或者就地手写一段正文（prompt_text）。互斥——同一行留两个 + # 来源，运行时不知道该听谁的。装配侧优先级 prompt_text > prompt_kind。 + # + # 版本池的 tab 现在可以新增（prompt_version_service.create_custom_kind）， + # 所以"绑 tab"这条路对自建模式同样走得通，不再只有内置极速能用。 + _wants_kind = "prompt_kind" in payload + _wants_text = "prompt_text" in payload + if creating or _wants_kind or _wants_text: + kind = str(payload.get("prompt_kind") or "").strip() + if owner_user_id is not None: + # 私有模式只许手写正文：分类是管理端的东西，绑上等于间接读管理员的 + # 提示词内容，越过了"用户看不到管理端提示词"的边界。 + kind = "" + text = str(payload.get("prompt_text") or "").strip() + if kind: + out["prompt_kind"] = kind + out["prompt_text"] = None + elif text: + out["prompt_text"] = text + out["prompt_kind"] = None + elif _wants_kind or _wants_text or creating: + # 两个都清空 = 显式取消专属提示词，退回默认装配 + out["prompt_kind"] = None + out["prompt_text"] = None + take("updated_at", utc_now()) + return out + + +def normalize_slug(raw: Any, fallback_name: Any) -> str: + """slug 归一：小写、只留 [a-z0-9_-]。空则由名称派生，再空则随机。""" + candidate = str(raw or "").strip().lower() + if not candidate: + candidate = str(fallback_name or "").strip().lower() + cleaned = "".join(ch if (ch.isalnum() and ch.isascii()) or ch in "-_" else "-" for ch in candidate) + cleaned = cleaned.strip("-_") + if not cleaned: + cleaned = f"mode-{uuid.uuid4().hex[:8]}" + return cleaned[:64] diff --git a/src/backend/core/services/chat_mode_service.py b/src/backend/core/chat/modes.py similarity index 62% rename from src/backend/core/services/chat_mode_service.py rename to src/backend/core/chat/modes.py index cbc95621..3d346d1b 100644 --- a/src/backend/core/services/chat_mode_service.py +++ b/src/backend/core/chat/modes.py @@ -1,95 +1,24 @@ -"""对话模式服务。 - -职责三块： - -1. **解析**：给一段对话解析出它的装配契约（``resolve``）。运行时唯一真源—— - ``agent_factory`` 不再自己去读 ``turbo.*`` 配置键。 -2. **CRUD**：官方模式（Config 台）与私有模式（用户设置页）走同一套增删改， - 只是可见范围和提示词落点不同。 -3. **播种/迁移**：首次启动时把内置的「标准 / 极速」两行写进去，并把历史 - ``turbo.*`` SystemConfig 键**一次性**搬到极速那一行上。 - -可见范围规则（一处实现，别处只调）：一个用户能用的模式 = 全部启用的官方模式 -+ 他自己的私有模式。私有模式永远不跨人可见，也永远不进 Config 台的列表。 -""" - +"""Create, edit and resolve chat modes, including marketplace skill bindings.""" from __future__ import annotations +from core.llm import prompt_kinds from core.infra.time import utc_now - -import logging import uuid -from dataclasses import dataclass, field -from datetime import datetime from typing import Any, Dict, List, Optional, Tuple - -from core.db.models.chat_mode import ( - EFFORT_VALUES, - TOOL_SCOPE_ALL, - TOOL_SCOPE_RESTRICTED, - ChatMode, -) +from core.db.models.chat_mode import TOOL_SCOPE_ALL, ChatMode from core.infra.exceptions import BadRequestError, ResourceNotFoundError from sqlalchemy import or_ from sqlalchemy.orm import Session +import logging +from core.chat import mode_seed +from core.chat import mode_types +from core.chat import mode_validation logger = logging.getLogger(__name__) -#: 内置模式的 slug。``standard`` 是"不收窄"的那条，``turbo`` 是历史极速模式。 -SLUG_STANDARD = "standard" -SLUG_TURBO = "turbo" -BUILTIN_SLUGS = (SLUG_STANDARD, SLUG_TURBO) -#: 未安装的市场项在清单里用这个前缀标记；保存时换成安装后的真实 id。 MARKET_PREFIX = "market:" -@dataclass(frozen=True) -class ChatModeSpec: - """运行时装配契约——``agent_factory`` 只认这个，不认数据库行。""" - - slug: str - name: str - tool_scope: str = TOOL_SCOPE_ALL - mcp_server_ids: Tuple[str, ...] = () - skill_ids: Tuple[str, ...] = () - plugin_ids: Tuple[str, ...] = () - agent_ids: Tuple[str, ...] = () - manual_invoke_enabled: bool = True - #: 收窄模式下是否保留沙箱代码执行与文件工具；``all`` 作用域不看这个位。 - code_exec_enabled: bool = False - max_iters: Optional[int] = None - default_effort: str = "fast" - effort_locked: bool = False - prompt_kind: Optional[str] = None - prompt_text: Optional[str] = None - - @property - def restricted(self) -> bool: - """是否收窄工具面。标准模式为 False（跟随 catalog 与用户授权）。""" - return self.tool_scope == TOOL_SCOPE_RESTRICTED - - -#: 标准模式的兜底契约：库里那行被删/没播种时也不能让对话起不来。 -STANDARD_SPEC = ChatModeSpec( - slug=SLUG_STANDARD, - name="标准模式", - tool_scope=TOOL_SCOPE_ALL, - code_exec_enabled=True, - default_effort="fast", -) - - -def _as_id_list(value: Any) -> List[str]: - """JSON 列 → 干净的字符串 id 列表（去空白、去重、保序）。""" - if not isinstance(value, (list, tuple)): - return [] - seen: Dict[str, None] = {} - for item in value: - if isinstance(item, str) and item.strip(): - seen.setdefault(item.strip(), None) - return list(seen.keys()) - - class ChatModeService: def __init__(self, db: Session): self.db = db @@ -142,20 +71,20 @@ def get_visible(self, mode_id: str, user_id: Optional[str]) -> Optional[ChatMode # ── 运行时解析 ────────────────────────────────────────────────────── - def resolve(self, slug: Optional[str], user_id: Optional[str]) -> ChatModeSpec: + def resolve(self, slug: Optional[str], user_id: Optional[str]) -> mode_types.ChatModeSpec: """把线上传来的 ``mode_slug`` 解析成装配契约。 解析不到（空 slug / 已删 / 别人的私有模式）一律回落标准模式——模式是 产品增强而不是权限边界，解析失败该让对话照常跑，不该 500。 """ wanted = (slug or "").strip() - if not wanted or wanted == SLUG_STANDARD: - return self._spec_of(self._find_by_slug(SLUG_STANDARD, user_id)) or STANDARD_SPEC + if not wanted or wanted == mode_types.SLUG_STANDARD: + return self._spec_of(self._find_by_slug(mode_types.SLUG_STANDARD, user_id)) or mode_types.STANDARD_SPEC row = self._find_by_slug(wanted, user_id) if row is None: logger.info("[chat_mode] slug=%s 不可用，回落标准模式", wanted) - return self._spec_of(self._find_by_slug(SLUG_STANDARD, user_id)) or STANDARD_SPEC - return self._spec_of(row) or STANDARD_SPEC + return self._spec_of(self._find_by_slug(mode_types.SLUG_STANDARD, user_id)) or mode_types.STANDARD_SPEC + return self._spec_of(row) or mode_types.STANDARD_SPEC def _find_by_slug(self, slug: str, user_id: Optional[str]) -> Optional[ChatMode]: query = self.db.query(ChatMode).filter( @@ -173,17 +102,17 @@ def _find_by_slug(self, slug: str, user_id: Optional[str]) -> Optional[ChatMode] return query.order_by(ChatMode.owner_user_id.isnot(None).desc()).first() @staticmethod - def _spec_of(row: Optional[ChatMode]) -> Optional[ChatModeSpec]: + def _spec_of(row: Optional[ChatMode]) -> Optional[mode_types.ChatModeSpec]: if row is None: return None - return ChatModeSpec( + return mode_types.ChatModeSpec( slug=row.slug, name=row.name, tool_scope=row.tool_scope or TOOL_SCOPE_ALL, - mcp_server_ids=tuple(_as_id_list(row.mcp_server_ids)), - skill_ids=tuple(_as_id_list(row.skill_ids)), - plugin_ids=tuple(_as_id_list(row.plugin_ids)), - agent_ids=tuple(_as_id_list(row.agent_ids)), + mcp_server_ids=tuple(mode_validation.as_id_list(row.mcp_server_ids)), + skill_ids=tuple(mode_validation.as_id_list(row.skill_ids)), + plugin_ids=tuple(mode_validation.as_id_list(row.plugin_ids)), + agent_ids=tuple(mode_validation.as_id_list(row.agent_ids)), manual_invoke_enabled=bool(row.manual_invoke_enabled), code_exec_enabled=bool(getattr(row, "code_exec_enabled", False)), max_iters=row.max_iters, @@ -196,8 +125,8 @@ def _spec_of(row: Optional[ChatMode]) -> Optional[ChatModeSpec]: # ── 写入 ──────────────────────────────────────────────────────────── def create(self, payload: Dict[str, Any], owner_user_id: Optional[str]) -> ChatMode: - slug = self._normalize_slug(payload.get("slug"), payload.get("name")) - if slug in BUILTIN_SLUGS: + slug = mode_validation.normalize_slug(payload.get("slug"), payload.get("name")) + if slug in mode_types.BUILTIN_SLUGS: raise BadRequestError(f"slug「{slug}」是内置模式保留字，请换一个") clash = ( self.db.query(ChatMode) @@ -213,7 +142,7 @@ def create(self, payload: Dict[str, Any], owner_user_id: Optional[str]) -> ChatM if clash is not None: raise BadRequestError(f"已存在同名模式「{slug}」") - fields = self._writable_fields(payload, owner_user_id, creating=True) + fields = mode_validation.writable_fields(payload, owner_user_id, creating=True) self._install_market_refs(fields, owner_user_id) row = ChatMode( id=f"mode_{uuid.uuid4().hex[:16]}", @@ -231,7 +160,7 @@ def update( self, mode_id: str, payload: Dict[str, Any], owner_user_id: Optional[str] ) -> ChatMode: row = self._owned_row(mode_id, owner_user_id) - fields = self._writable_fields(payload, owner_user_id, creating=False) + fields = mode_validation.writable_fields(payload, owner_user_id, creating=False) self._install_market_refs(fields, owner_user_id) for key, value in fields.items(): setattr(row, key, value) @@ -260,94 +189,6 @@ def _owned_row(self, mode_id: str, owner_user_id: Optional[str]) -> ChatMode: raise ResourceNotFoundError("chat_mode", mode_id) return row - def _writable_fields( - self, payload: Dict[str, Any], owner_user_id: Optional[str], creating: bool - ) -> Dict[str, Any]: - """把请求体收敛成可写字段。未出现的键在更新时保持原值。""" - out: Dict[str, Any] = {} - - def take(key: str, value: Any) -> None: - if value is not None or creating: - out[key] = value - - if creating or "name" in payload: - name = str(payload.get("name") or "").strip() - if not name: - raise BadRequestError("模式名称不能为空") - out["name"] = name[:100] - if creating or "description" in payload: - out["description"] = str(payload.get("description") or "").strip()[:500] - if creating or "enabled" in payload: - out["enabled"] = bool(payload.get("enabled", True)) - if creating or "sort_order" in payload: - try: - out["sort_order"] = int(payload.get("sort_order", 100)) - except (TypeError, ValueError): - out["sort_order"] = 100 - - if creating or "tool_scope" in payload: - scope = str(payload.get("tool_scope") or TOOL_SCOPE_RESTRICTED) - out["tool_scope"] = scope if scope in (TOOL_SCOPE_ALL, TOOL_SCOPE_RESTRICTED) else TOOL_SCOPE_RESTRICTED - for key in ("mcp_server_ids", "skill_ids", "plugin_ids", "agent_ids"): - if creating or key in payload: - out[key] = _as_id_list(payload.get(key)) - if creating or "manual_invoke_enabled" in payload: - out["manual_invoke_enabled"] = bool(payload.get("manual_invoke_enabled", True)) - if creating or "code_exec_enabled" in payload: - out["code_exec_enabled"] = bool(payload.get("code_exec_enabled", False)) - if creating or "max_iters" in payload: - raw = payload.get("max_iters") - try: - # 下限 2：配成 1 会把"回答那一轮"也掐掉，等于永远拿不到结果。 - out["max_iters"] = max(2, int(raw)) if raw not in (None, "") else None - except (TypeError, ValueError): - out["max_iters"] = None - - if creating or "default_effort" in payload: - effort = str(payload.get("default_effort") or "fast") - out["default_effort"] = effort if effort in EFFORT_VALUES else "fast" - if creating or "effort_locked" in payload: - out["effort_locked"] = bool(payload.get("effort_locked", False)) - - # 提示词两种来源二选一：绑一个提示词管理里的 tab（prompt_kind，正文在那边 - # 编辑与版本化），或者就地手写一段正文（prompt_text）。互斥——同一行留两个 - # 来源，运行时不知道该听谁的。装配侧优先级 prompt_text > prompt_kind。 - # - # 版本池的 tab 现在可以新增（prompt_version_service.create_custom_kind）， - # 所以"绑 tab"这条路对自建模式同样走得通，不再只有内置极速能用。 - _wants_kind = "prompt_kind" in payload - _wants_text = "prompt_text" in payload - if creating or _wants_kind or _wants_text: - kind = str(payload.get("prompt_kind") or "").strip() - if owner_user_id is not None: - # 私有模式只许手写正文：分类是管理端的东西，绑上等于间接读管理员的 - # 提示词内容，越过了"用户看不到管理端提示词"的边界。 - kind = "" - text = str(payload.get("prompt_text") or "").strip() - if kind: - out["prompt_kind"] = kind - out["prompt_text"] = None - elif text: - out["prompt_text"] = text - out["prompt_kind"] = None - elif _wants_kind or _wants_text or creating: - # 两个都清空 = 显式取消专属提示词，退回默认装配 - out["prompt_kind"] = None - out["prompt_text"] = None - take("updated_at", utc_now()) - return out - - @staticmethod - def _normalize_slug(raw: Any, fallback_name: Any) -> str: - """slug 归一：小写、只留 [a-z0-9_-]。空则由名称派生，再空则随机。""" - candidate = str(raw or "").strip().lower() - if not candidate: - candidate = str(fallback_name or "").strip().lower() - cleaned = "".join(ch if (ch.isalnum() and ch.isascii()) or ch in "-_" else "-" for ch in candidate) - cleaned = cleaned.strip("-_") - if not cleaned: - cleaned = f"mode-{uuid.uuid4().hex[:8]}" - return cleaned[:64] # ── 可选能力清单 ──────────────────────────────────────────────────── @@ -362,7 +203,7 @@ def available_capabilities(self, owner_user_id: Optional[str]) -> Dict[str, Any] 保存模式时由 :meth:`_install_market_refs` 按需安装成真实条目。 """ from core.db.models import AdminSkill - from core.services.user_agent_service import UserAgentService + from core.capabilities.agents.edition import UserAgentService svc = UserAgentService(self.db) resources = svc.list_available_resources(owner_user_id) @@ -389,7 +230,7 @@ def available_capabilities(self, owner_user_id: Optional[str]) -> Dict[str, Any] for prow in pq.all(): cids = prow.component_ids or {} plugin_component_skill_ids.update(str(x) for x in (cids.get("skills") or [])) - from core.plugins.management import builtin_plugin_component_ids + from core.capabilities.plugins.management import builtin_plugin_component_ids _builtin_skill_ids, _ = builtin_plugin_component_ids() plugin_component_skill_ids.update(str(x) for x in _builtin_skill_ids) @@ -456,17 +297,17 @@ def available_capabilities(self, owner_user_id: Optional[str]) -> Dict[str, Any] # 提示词 tab 清单：只发给管理端（owner=None）。普通用户看不到「提示词管理」， # 给他一个分类下拉等于让他绑一个自己既看不到内容、也改不了正文的黑盒—— # 私有模式的提示词只有手写正文一条路。 - prompt_kinds: List[Dict[str, Any]] = [] + kinds: List[Dict[str, Any]] = [] if owner_user_id is None: try: - from core.services import prompt_version_service as _pvs + from core.llm import prompt_versions as _pvs - prompt_kinds = _pvs.all_kinds(self.db) + kinds = prompt_kinds.all_kinds(self.db) except Exception: # noqa: BLE001 logger.debug("[chat_mode] 提示词 kind 清单读取失败", exc_info=True) return { - "prompt_kinds": prompt_kinds, + "prompt_kinds": kinds, "mcp": resources.get("mcp_servers") or [], "skills": skills, "plugins": resources.get("plugins") or [], @@ -489,9 +330,9 @@ def _market_options( agents: List[Dict[str, Any]] = [] plugins: List[Dict[str, Any]] = [] try: - from core.services.marketplace_service import list_marketplace_skills + from core.harness.registry.marketplace_catalog import marketplace_catalog - for item in list_marketplace_skills(self.db) or []: + for item in marketplace_catalog().list_marketplace_skills(self.db) or []: slug = str(item.get("slug") or "") # 内置技能全局常驻、装不了（install 会直接拒），而且已经在上面的 # 技能清单里了——列出来只会给出一个点了报错的选项。 @@ -510,9 +351,9 @@ def _market_options( except Exception: # noqa: BLE001 logger.debug("[chat_mode] 技能市场清单读取失败", exc_info=True) try: - from core.services.agent_market_service import list_marketplace_agents + from core.harness.registry.marketplace_catalog import marketplace_catalog - for item in list_marketplace_agents(self.db) or []: + for item in marketplace_catalog().list_marketplace_agents(self.db) or []: slug = str(item.get("slug") or "") if not slug or item.get("market_enabled") is False: continue @@ -527,9 +368,9 @@ def _market_options( except Exception: # noqa: BLE001 logger.debug("[chat_mode] 子智能体市场清单读取失败", exc_info=True) try: - from core.plugins.management import list_plugins + from core.harness.registry.marketplace_catalog import marketplace_catalog - for item in list_plugins(self.db, None) or []: + for item in marketplace_catalog().list_plugins(self.db, None) or []: slug = str(item.get("slug") or "") if not slug or item.get("installed") or item.get("market_enabled") is False: continue @@ -573,21 +414,21 @@ def _install_market_refs( slug = sid[len(MARKET_PREFIX):] try: if installer == "skill": - from core.services.marketplace_service import install_marketplace_skill + from core.harness.registry.marketplace_catalog import marketplace_catalog - res = install_marketplace_skill( + res = marketplace_catalog().install_marketplace_skill( self.db, slug, owner_user_id=owner_user_id ) new_id = str(res.get("skill_id") or res.get("id") or "") elif installer == "plugin": - from core.plugins.management import install_plugin + from core.capabilities.plugins.management import install_plugin res = install_plugin(self.db, slug, owner_user_id=owner_user_id) new_id = str(res.get("install_id") or "") else: - from core.services.agent_market_service import install_marketplace_agent + from core.harness.registry.marketplace_catalog import marketplace_catalog - res = install_marketplace_agent( + res = marketplace_catalog().install_marketplace_agent( self.db, slug, owner_user_id=owner_user_id ) new_id = str(res.get("agent_id") or res.get("id") or "") @@ -615,11 +456,11 @@ def ensure_builtins(self) -> None: .all() } created = False - if SLUG_STANDARD not in existing: + if mode_types.SLUG_STANDARD not in existing: self.db.add( ChatMode( id="mode_builtin_standard", - slug=SLUG_STANDARD, + slug=mode_types.SLUG_STANDARD, name="标准模式", description="完整工具、技能与插件，思考强度可调，适合大部分任务", owner_user_id=None, @@ -641,58 +482,8 @@ def ensure_builtins(self) -> None: ) ) created = True - if SLUG_TURBO not in existing: - self.db.add(ChatMode(**self._turbo_seed_fields())) + if mode_types.SLUG_TURBO not in existing: + self.db.add(ChatMode(**mode_seed.turbo_seed_fields())) created = True if created: self.db.commit() - - @staticmethod - def _turbo_seed_fields() -> Dict[str, Any]: - """极速模式内置行的字段，取自历史 ``turbo.*`` SystemConfig 键。 - - 这是那批键的**最后一次**读取——播种之后本表才是真源，Config 台 - 「系统配置 → 极速模式」那组入口随之下线。 - """ - from core.services.system_config import ( - turbo_manual_invoke_enabled, - turbo_max_iters, - turbo_mcp_server_ids, - turbo_plugin_ids, - turbo_skill_ids, - ) - - try: - mcp_ids = sorted(turbo_mcp_server_ids()) - skill_ids = list(turbo_skill_ids()) - plugin_ids = list(turbo_plugin_ids()) - manual = turbo_manual_invoke_enabled() - iters = turbo_max_iters() - except Exception: # noqa: BLE001 — 配置层不可用时用内置检索三件套兜底 - logger.warning("[chat_mode] 读 turbo.* 配置失败，极速模式按内置默认播种", exc_info=True) - mcp_ids = ["internet_search", "retrieve_dataset_content", "web_fetch"] - skill_ids, plugin_ids, manual, iters = [], [], True, 4 - - return { - "id": "mode_builtin_turbo", - "slug": SLUG_TURBO, - "name": "极速模式", - "description": "只留检索类工具、不思考，秒级直达结果", - "owner_user_id": None, - "is_builtin": True, - "enabled": True, - "sort_order": 20, - "tool_scope": TOOL_SCOPE_RESTRICTED, - "mcp_server_ids": mcp_ids, - "skill_ids": skill_ids, - "plugin_ids": plugin_ids, - "agent_ids": [], - "manual_invoke_enabled": manual, - # 极速的产品契约就是"检索直达、不执行代码"，保持关。 - "code_exec_enabled": False, - "max_iters": iters, - # 极速那条链路本就不思考，强度锁死——不是"默认不思考但可以改"。 - "default_effort": "fast", - "effort_locked": True, - "prompt_kind": "turbo", - } diff --git a/src/backend/core/services/session_operations/__init__.py b/src/backend/core/chat/operations/__init__.py similarity index 100% rename from src/backend/core/services/session_operations/__init__.py rename to src/backend/core/chat/operations/__init__.py diff --git a/src/backend/core/services/session_operations/admission.py b/src/backend/core/chat/operations/admission.py similarity index 81% rename from src/backend/core/services/session_operations/admission.py rename to src/backend/core/chat/operations/admission.py index 3ad4ee85..15d49a92 100644 --- a/src/backend/core/services/session_operations/admission.py +++ b/src/backend/core/chat/operations/admission.py @@ -3,20 +3,21 @@ import asyncio from dataclasses import dataclass -import core.services.session_operations.admission_errors as chat_admission -import core.services.session_operations.agent_targets as chat_agent_targets -import core.services.session_operations.invocation as chat_invocation -import core.services.session_operations.request_context as chat_request_context -import core.services.session_operations.session_context as chat_session_context import core.chat.context as chat_context +import core.chat.runtime.admission as admission import core.db.engine as db_engine -import core.services as chat_services -from core.chat.schemas import ChatRequest +import core.chat.service as chat_services_ChatService_module +import core.auth.users as chat_services_UserService_module +import core.chat.operations.admission_errors as chat_admission +import core.chat.operations.agent_targets as chat_agent_targets +import core.chat.operations.invocation as chat_invocation +import core.chat.operations.request_context as chat_request_context +import core.chat.operations.session_context as chat_session_context from core.auth.backend import UserContext +from core.chat.schemas import ChatRequest from core.infra.logging import get_logger, trace_id_var -from core.services.chat_sequencer import ChatBusyError, ChatSequencer +from core.chat.sequencer import ChatBusyError, ChatSequencer from fastapi import HTTPException -from orchestration import chat_run_executor from sqlalchemy.orm import Session logger = get_logger(__name__) @@ -40,17 +41,17 @@ def capabilities(): def preferences(): with db_engine.SessionLocal() as db: - return chat_services.UserService(db).get_user_settings(user_id) + return chat_services_UserService_module.UserService(db).get_user_settings(user_id) return await asyncio.gather(asyncio.to_thread(capabilities), asyncio.to_thread(preferences)) def _check_project(db: Session, request: ChatRequest, user_id: str) -> None: """A chat bound to a project always runs in that project, whichever client sends to it.""" - from core.services.project_access import require_visible_project + from core.projects.access import require_visible_project if request.chat_id: - pair = chat_services.ChatService(db).get_session_with_access(request.chat_id, user_id) + pair = chat_services_ChatService_module.ChatService(db).get_session_with_access(request.chat_id, user_id) if pair is not None and pair[0].project_id: request.project_id = pair[0].project_id if request.project_id: @@ -71,24 +72,19 @@ def _link_attachments(db: Session, request: ChatRequest, user_id: str) -> None: db.commit() -async def _start_response_run( +async def start_response_run( request: ChatRequest, user: UserContext, db: Session ) -> StartedResponse: - from core.services.agent_api_service import ( - begin_agent_api_call, - bind_agent_api_run, - fail_agent_api_call, - prepare_agent_api_request, - ) + from core.auth.agent_api import begin_agent_api_call, bind_agent_api_run, fail_agent_api_call, prepare_agent_api_request # Scope and API-session provenance come only from the authenticated key. # prepare logs its own rejected scoped requests before it raises. - from core.services.project_api_service import prepare_project_api_request + from core.projects.api import prepare_project_api_request request = prepare_project_api_request(db, user, request) request, scope = prepare_agent_api_request(db, user, request) if user.public_chat_visitor: - from core.services.public_chat_service import validate_visitor_attachments + from core.chat.public import validate_visitor_attachments validate_visitor_attachments(db, user, request.attachments) call_id = ( @@ -97,11 +93,11 @@ async def _start_response_run( else None ) try: - chat_session_context._ensure_main_model_configured() + chat_session_context.ensure_main_model_configured() user_id = chat_context.resolve_db_user_id( - db, chat_session_context._authenticated_user_id(user) + db, chat_session_context.authenticated_user_id(user) ) - from core.services.evaluation_admission import authorize + from core.harness.registry.evaluation import authorize await authorize(request, user_id, agent_scoped=bool(scope)) if scope: @@ -119,23 +115,23 @@ async def _start_response_run( request, agent_name, execution_message, command = ( chat_agent_targets._resolve_chat_agent_targets(db, request, user_id) ) - request = chat_invocation._resolve_explicit_capability_invocation(db, request, user_id) + request = chat_invocation.resolve_explicit_capability_invocation(db, request, user_id) message = chat_context.build_effective_user_message( execution_message, request.quoted_follow_up, chat_request_context._resolve_reference_block(db, request, user_id), ) - provider_id = chat_session_context._resolve_selected_model_provider_id(db, request, user_id) - model_name = chat_session_context._resolve_actual_chat_model_name(request, provider_id) + provider_id = chat_session_context.resolve_selected_model_provider_id(db, request, user_id) + model_name = chat_session_context.resolve_actual_chat_model_name(request, provider_id) (skills, agents, mcps), settings = await _read_preferences(request, user_id) if user.public_chat_visitor: - from core.services.public_chat_service import visitor_approval_mode + from core.chat.public import visitor_approval_mode settings["tool_approval_mode"] = visitor_approval_mode( db, user, settings.get("tool_approval_mode") ) _check_project(db, request, user_id) - service = chat_services.ChatService(db) + service = chat_services_ChatService_module.ChatService(db) chat_session_context._ensure_chat_session( service, request.chat_id, @@ -148,7 +144,7 @@ async def _start_response_run( site_chat=request.site_chat, project_id=request.project_id, ) - context = chat_request_context._build_ctx( + context = chat_request_context.build_ctx( request, user_id, skills, @@ -164,7 +160,7 @@ async def _start_response_run( approval_mode=settings.get("tool_approval_mode"), ) if user.api_key_project_id: - from core.services.project_api_service import make_project_api_scope + from core.projects.api import make_project_api_scope context["project_api_scope"] = make_project_api_scope(user, request.chat_id) payload = request.model_dump(exclude_none=True) @@ -213,7 +209,7 @@ async def _start_response_run( } db.flush() # No history or attachment mutation before writer admission. - messages = chat_session_context._load_session_messages( + messages = chat_session_context.load_session_messages( service, request.chat_id, user_id ) from core.chat.plan_progress import clear_plan_progress @@ -222,7 +218,7 @@ async def _start_response_run( _link_attachments(db, request, user_id) if call_id: bind_agent_api_run(db, call_id, accepted.run.run_id) - run = await chat_run_executor.start_run( + run = await admission.start_run( accepted_run=accepted.run, chat_id=request.chat_id, user_id=user_id, @@ -236,7 +232,7 @@ async def _start_response_run( raise chat_admission.chat_busy_http_exception(exc) from exc # Title work is auxiliary: it starts after first-message admission and # never waits for the agent answer or blocks a successfully launched run. - from core.services.chat_titles import schedule_first_title + from core.chat.titles import schedule_first_title try: schedule_first_title(request.chat_id, user_id, run.run_id) @@ -259,4 +255,4 @@ async def _start_response_run( finally: # Both HTTP transports must release the request transaction before # waiting on a worker that writes the same chat rows. - chat_session_context._release_request_session(db) + chat_session_context.release_request_session(db) diff --git a/src/backend/core/services/session_operations/admission_errors.py b/src/backend/core/chat/operations/admission_errors.py similarity index 92% rename from src/backend/core/services/session_operations/admission_errors.py rename to src/backend/core/chat/operations/admission_errors.py index 83ebae1c..d65507f5 100644 --- a/src/backend/core/services/session_operations/admission_errors.py +++ b/src/backend/core/chat/operations/admission_errors.py @@ -2,7 +2,7 @@ from fastapi import HTTPException -from core.services.chat_sequencer import ChatBusyError +from core.chat.sequencer import ChatBusyError def chat_busy_http_exception(exc: ChatBusyError) -> HTTPException: diff --git a/src/backend/core/services/session_operations/agent_targets.py b/src/backend/core/chat/operations/agent_targets.py similarity index 94% rename from src/backend/core/services/session_operations/agent_targets.py rename to src/backend/core/chat/operations/agent_targets.py index 1c1b945b..ee0d27e1 100644 --- a/src/backend/core/services/session_operations/agent_targets.py +++ b/src/backend/core/chat/operations/agent_targets.py @@ -2,7 +2,7 @@ from typing import Any, Optional -import core.services.user_service as user_service +import core.auth.users as user_service from core.chat.schemas import ChatRequest from fastapi import HTTPException from sqlalchemy.orm import Session @@ -28,15 +28,15 @@ def _resolve_chat_agent_targets( constrain its next real tool call to ``call_subagent``. Only a persistent ``agent_id`` conversation executes the selected sub-agent directly. """ - from core.llm.builtin_subagents import get_builtin_subagent, merge_builtin_subagents - from core.services.project_init import resolve_project_init - from core.services.user_agent_service import UserAgentService + from core.capabilities.agents.builtin import get_builtin_subagent, merge_builtin_subagents + from core.chat.project_init import resolve_project_init + from core.capabilities.agents.edition import UserAgentService initialized = resolve_project_init(db, request, user_id) if initialized is not None: return request, None, initialized, None - from core.services.subagent_routing_service import may_be_explicit_subagent_command + from core.chat.subagent_routing import may_be_explicit_subagent_command if not ( request.agent_id @@ -74,7 +74,7 @@ def _resolve_chat_agent_targets( mention_agent_id = request.mention_agent_id mention_agent_name = request.mention_name if not mention_agent_id and not mention_agent_name: - from core.services.subagent_routing_service import parse_explicit_subagent_command + from core.chat.subagent_routing import parse_explicit_subagent_command explicit_command = parse_explicit_subagent_command( request.message, @@ -109,7 +109,7 @@ def _resolve_chat_agent_targets( selected_display_name, ) else: - from core.services.subagent_routing_service import parse_explicit_subagent_command + from core.chat.subagent_routing import parse_explicit_subagent_command command = parse_explicit_subagent_command(request.message, [mentioned]) if command: @@ -205,7 +205,7 @@ def _saved_agent_source_profile( return None -def _resolve_rerun_agent_targets( +def resolve_rerun_agent_targets( db, request, user_id, session, user_message, *, assistant_message_id=None ): """Restore legacy dedicated chats and recheck each saved source before admission.""" diff --git a/src/backend/core/services/session_operations/commands.py b/src/backend/core/chat/operations/commands.py similarity index 82% rename from src/backend/core/services/session_operations/commands.py rename to src/backend/core/chat/operations/commands.py index df681beb..5d624c44 100644 --- a/src/backend/core/services/session_operations/commands.py +++ b/src/backend/core/chat/operations/commands.py @@ -1,30 +1,26 @@ """Shared commands operations; callers supply authenticated identity.""" +import core.runs.runtime as runtime from core.infra.responses import success_response - -from core.services.chat_steer_service import put_pending_steer - -from core.services.steer_queue import SteerQueueConflict - +from core.chat.steer import put_pending_steer +from core.chat.steer_queue import SteerQueueConflict from fastapi import HTTPException -from orchestration import chat_run_executor - async def cancel_chat_run(run_id, user=None): """取消正在执行的 chat run，真正杀掉后台任务；run 不存在返回 404，无权操作返回 403。""" try: - cancelled = await chat_run_executor.cancel_run(run_id, user_id=user.user_id) - except chat_run_executor.ChatRunNotFound: + cancelled = await runtime.cancel_run(run_id, user_id=user.user_id) + except runtime.ChatRunNotFound: raise HTTPException(status_code=404, detail="run not found") - except chat_run_executor.ChatRunPermissionDenied: + except runtime.ChatRunPermissionDenied: raise HTTPException(status_code=403, detail="无权取消该 run") return success_response(data={"run_id": run_id, "cancelled": cancelled}) async def steer_chat_run(run_id, body, user=None): """Queue one instruction for the live ReAct loop's next safe boundary.""" - run = chat_run_executor.get_run(run_id) + run = runtime.get_run(run_id) if run is None: raise HTTPException(status_code=404, detail="run not found") if run.user_id != user.user_id: diff --git a/src/backend/core/services/session_operations/history.py b/src/backend/core/chat/operations/history.py similarity index 92% rename from src/backend/core/services/session_operations/history.py rename to src/backend/core/chat/operations/history.py index 260e2ea7..b99737f6 100644 --- a/src/backend/core/services/session_operations/history.py +++ b/src/backend/core/chat/operations/history.py @@ -1,6 +1,6 @@ """Shared history operations; callers supply authenticated identity.""" -import core.services as chat_services +import core.chat.service as chat_services_ChatService_module from core.chat import inflight @@ -10,7 +10,7 @@ from core.infra.responses import paginated_response -from core.services.context_usage_projection import get_chat_context_state +from core.chat.compaction.usage_projection import get_chat_context_state _WAKE_MESSAGE_PREFIXES = ( "[系统] 进度播报：", @@ -67,7 +67,7 @@ def list_messages( chat_id, page=1, page_size=50, order="asc", cursor=False, before_seq=None, user=None, db=None ): """获取当前用户有权读取的会话消息列表；无权访问时返回 404。""" - chat_service = chat_services.ChatService(db) + chat_service = chat_services_ChatService_module.ChatService(db) user_id = str(user.user_id) pair = chat_service.get_session_with_access(chat_id, user_id) @@ -92,7 +92,7 @@ def list_messages( newest_first=(order.lower() == "desc"), display_only=True, ) - from core.services.compaction_service import _live_run_ids + from core.chat.compaction.service import _live_run_ids live = _live_run_ids(chat_service, messages) items = [_message_to_dict(m, live) for m in messages if not _is_internal_message(m)] @@ -110,7 +110,7 @@ def list_messages( # checkpoint baseline separately so context-usage estimates do not keep # counting messages that the model now sees only through the summary. response["data"].update(get_chat_context_state(chat_service, chat_id)) - from core.services.compaction_progress import read_progress + from core.chat.compaction.progress import read_progress response["data"]["compaction_progress"] = read_progress(chat_service, chat_id) return response diff --git a/src/backend/core/services/session_operations/invocation.py b/src/backend/core/chat/operations/invocation.py similarity index 85% rename from src/backend/core/services/session_operations/invocation.py rename to src/backend/core/chat/operations/invocation.py index ca5a4632..8dccd1fd 100644 --- a/src/backend/core/services/session_operations/invocation.py +++ b/src/backend/core/chat/operations/invocation.py @@ -10,7 +10,7 @@ logger = get_logger(__name__) -def _resolve_explicit_capability_invocation( +def resolve_explicit_capability_invocation( db: Session, request: ChatRequest, user_id: str, @@ -22,7 +22,7 @@ def _resolve_explicit_capability_invocation( its enabled switch is off, but cannot cross ownership, missing-installation, or dependency-readiness boundaries. """ - from core.config.catalog_resolver import resolve_explicit_runtime_capabilities + from core.capabilities.catalog.resolver import resolve_explicit_runtime_capabilities def _ids(raw) -> List[str]: return list( @@ -45,14 +45,14 @@ def _ids(raw) -> List[str]: .first() ) if installed is None: - from core.capabilities.errors import CapabilityError - from core.capabilities.invocation import cloud_plugin_selection - from core.capabilities.paths import capabilities_enabled + from core.harness.capability_errors import CapabilityError + from core.harness.registry.run_capabilities import cloud_plugin_selection + from core.harness.registry.run_capabilities import has_run_capabilities try: cloud_plugin = ( cloud_plugin_selection(request.plugin_id, user_id=user_id) - if capabilities_enabled() + if has_run_capabilities() else None ) except CapabilityError as exc: @@ -68,7 +68,7 @@ def _ids(raw) -> List[str]: else: if installed.owner_user_id is not None and installed.owner_user_id != user_id: raise HTTPException(status_code=403, detail="无法访问该插件，可能已卸载") - from core.plugins.packaging.sources import _component_keys + from core.capabilities.plugins.packaging.sources import component_keys as _component_keys component_ids = installed.component_ids or {} plugin_skill_ids = _component_keys(component_ids, "skills") @@ -81,7 +81,7 @@ def _ids(raw) -> List[str]: # authorization both refer to the same package. if request.skill_id: from core.db.models import AdminSkill - from core.services.marketplace_service import compute_install_id + from core.capabilities.skills.installation import compute_install_id private_id = compute_install_id(request.skill_id, user_id) private_row = ( @@ -114,22 +114,15 @@ def _ids(raw) -> List[str]: unavailable_skills, unavailable_mcps, ) - from core.capabilities import registry, skills - from core.capabilities.paths import LOCAL_PROFILE, capabilities_enabled + from core.harness.registry.device_catalog import owned_identities + from core.harness.registry.run_capabilities import has_run_capabilities # An owned desktop installation may be missing files or temporarily # disabled. Retain its selection for an explanatory answer, without # granting anything the resolver rejected. Foreign identities still fail. owned = set() - if capabilities_enabled(): - profiles = [LOCAL_PROFILE] - if skills.account_authorized_for(user_id): - profiles.append(skills.current_account_profile()) - owned = { - (row.kind, row.key) - for row in registry.list_installations(profiles=profiles) - if row.payload.get("owner_user_id") in (None, "", user_id) - } + if has_run_capabilities(): + owned = owned_identities(user_id) rejected = [("skill", name) for name in unavailable_skills] + [ ("mcp", name) for name in unavailable_mcps ] diff --git a/src/backend/core/services/session_operations/projects.py b/src/backend/core/chat/operations/projects.py similarity index 90% rename from src/backend/core/services/session_operations/projects.py rename to src/backend/core/chat/operations/projects.py index 7a0b3b97..8dc08faa 100644 --- a/src/backend/core/services/session_operations/projects.py +++ b/src/backend/core/chat/operations/projects.py @@ -1,12 +1,12 @@ """Shared projects operations; callers supply authenticated identity.""" -from core.auth.permissions_iface import ProjectAccessInfo +from core.auth.edition import ProjectAccessInfo from core.infra.responses import paginated_response, success_response -from core.services.project_file_service import ProjectFileService +from core.projects.files import ProjectFileService -from core.services.project_service import ProjectService +from core.projects.edition import ProjectService from fastapi import HTTPException diff --git a/src/backend/core/services/session_operations/request_context.py b/src/backend/core/chat/operations/request_context.py similarity index 91% rename from src/backend/core/services/session_operations/request_context.py rename to src/backend/core/chat/operations/request_context.py index 44372b26..bf2e1704 100644 --- a/src/backend/core/services/session_operations/request_context.py +++ b/src/backend/core/chat/operations/request_context.py @@ -2,20 +2,20 @@ from typing import Any, Dict, List, Optional -import core.services.session_operations.session_context as chat_session_context +import core.chat.operations.session_context as chat_session_context import core.chat.context as chat_context import core.db.engine as db_engine from core.chat.schemas import ChatRequest from core.infra.exceptions import ServiceUnavailableError from core.infra.logging import get_logger -from core.llm.tool_permissions import normalize_approval_mode +from core.sandbox.permissions.approval import normalize_approval_mode from fastapi import HTTPException from sqlalchemy.orm import Session logger = get_logger(__name__) -def _build_ctx( +def build_ctx( request: ChatRequest, db_user_id: str, enabled_skills, @@ -69,7 +69,7 @@ def _build_ctx( if project_id: try: from core.db.engine import SessionLocal as _Sess - from core.services.project_scope import build_project_ctx + from core.projects.edition import build_project_ctx with _Sess() as _db: resolved_project_ctx = build_project_ctx(_db, project_id) @@ -85,16 +85,13 @@ def _build_ctx( workspace_id_value = f"project:{project_id}" if project_id else "default" memory_scope_user_id_value = project_ctx.pop("memory_scope_user_id", None) - from core.services.ontology_service import ( - build_ontology_runtime_for_preference, - disabled_ontology_runtime, - ) + from core.ontology.runtime_policy import build_ontology_runtime_for_preference, disabled_ontology_runtime ontology_runtime: Dict[str, Any] = disabled_ontology_runtime() if ontology_enabled: try: with db_engine.SessionLocal() as _ontology_db: - from core.services.ontology_policy import user_can_use_ontology_validation + from core.ontology.policy import user_can_use_ontology_validation if user_can_use_ontology_validation(_ontology_db, db_user_id): ontology_enabled, ontology_runtime = build_ontology_runtime_for_preference( @@ -132,20 +129,20 @@ def _build_ctx( # Preserve None so downstream (SkillsMiddleware) falls back to catalog defaults. # Only call _clean_id_list when there's an actual list to normalize. "enabled_skills": ( - chat_session_context._clean_id_list(enabled_skills) + chat_session_context.clean_id_list(enabled_skills) if enabled_skills is not None else None ), "enabled_agents": ( - chat_session_context._clean_id_list(enabled_agents) + chat_session_context.clean_id_list(enabled_agents) if enabled_agents is not None else None ), "enabled_mcps": ( - chat_session_context._clean_id_list(enabled_mcps) if enabled_mcps is not None else None + chat_session_context.clean_id_list(enabled_mcps) if enabled_mcps is not None else None ), "enabled_kbs": ( - chat_session_context._clean_id_list(request.enabled_kbs) + chat_session_context.clean_id_list(request.enabled_kbs) if request.enabled_kbs is not None else None ), @@ -236,7 +233,7 @@ def _resolve_reference_block(db: Session, request: ChatRequest, user_id: str) -> 名片按当前用户的权限现查，并挂回 ``request`` 上，随后与用户消息一起落库——历史重放 时照这份快照渲染，模型看到的和当时看到的是同一份。 """ - from core.services.chat_reference_service import render_reference_block, resolve_reference_cards + from core.chat.references import render_reference_block, resolve_reference_cards cards = resolve_reference_cards(db, user_id, getattr(request, "referenced_chats", None)) request._resolved_reference_cards = cards diff --git a/src/backend/core/services/session_operations/runs.py b/src/backend/core/chat/operations/runs.py similarity index 90% rename from src/backend/core/services/session_operations/runs.py rename to src/backend/core/chat/operations/runs.py index 1f507bcb..c28b7971 100644 --- a/src/backend/core/services/session_operations/runs.py +++ b/src/backend/core/chat/operations/runs.py @@ -1,8 +1,8 @@ """Shared runs operations; callers supply authenticated identity.""" -from core.auth.backend import UserContext import core.chat.context as chat_context - +import core.runs.runtime as runtime +from core.auth.backend import UserContext from core.infra.responses import success_response @@ -13,13 +13,13 @@ def list_active_chat_runs(user=None, db=None, include_status=False, recent_secon 侧边栏对没点开过的会话一无所知。这里按用户一次查完：查询只命中 pending / running 这一小撮行，与历史 run 的数量无关。 """ - from orchestration import chat_run_executor + pass db_user_id = chat_context.resolve_db_user_id( db, user.user_id if isinstance(user, UserContext) else None ) - runs, recent_chat_ids = chat_run_executor.list_active_runs_for_user(db_user_id, recent_seconds) - from core.services.sidebar_status import sidebar_run_statuses + runs, recent_chat_ids = runtime.list_active_runs_for_user(db_user_id, recent_seconds) + from core.chat.sidebar_status import sidebar_run_statuses return success_response( data={ @@ -44,12 +44,12 @@ def chat_active_run(chat_id, user=None, db=None): 有则返回 run_id / message_id / status / 刷新重放 offset / thinking 模式等元信息， 无则返回 data=null。需认证（有效 cookie 或 API-Key）。 """ - from orchestration import chat_run_executor + pass db_user_id = chat_context.resolve_db_user_id( db, user.user_id if isinstance(user, UserContext) else None ) - run = chat_run_executor.get_active_run_for_chat(chat_id, db_user_id) + run = runtime.get_active_run_for_chat(chat_id, db_user_id) if run is None: return success_response(data=None) payload = run.request_payload if isinstance(run.request_payload, dict) else {} diff --git a/src/backend/core/services/session_operations/schemas.py b/src/backend/core/chat/operations/schemas.py similarity index 100% rename from src/backend/core/services/session_operations/schemas.py rename to src/backend/core/chat/operations/schemas.py diff --git a/src/backend/core/services/session_operations/session_context.py b/src/backend/core/chat/operations/session_context.py similarity index 86% rename from src/backend/core/services/session_operations/session_context.py rename to src/backend/core/chat/operations/session_context.py index 3eeb7564..23b3b863 100644 --- a/src/backend/core/services/session_operations/session_context.py +++ b/src/backend/core/chat/operations/session_context.py @@ -2,38 +2,35 @@ from typing import Any, Dict, List, Optional -import core.services as chat_services +import core.chat.service as chat_services_ChatService_module from core.chat.schemas import ChatRequest from core.auth.backend import UserContext from core.db.reserved_chats import reserved_chat_error, reserved_chat_owner from core.chat.context import generate_smart_title from core.infra.logging import get_logger -from core.services.model_config import ModelConfigService -from core.services.user_model_selection import ( - UserModelSelectionError, - resolve_effective_chat_model_name, - resolve_user_model_provider_id, -) +from core.models.config import ModelConfigService +from core.auth.model_selection import UserModelSelectionError, resolve_user_model_provider_id +from core.models.selection import resolve_effective_chat_model_name from fastapi import HTTPException from sqlalchemy.orm import Session logger = get_logger(__name__) -def _clean_id_list(raw: Optional[list]) -> List[str]: +def clean_id_list(raw: Optional[list]) -> List[str]: """Normalize a list of capability IDs: strip whitespace, remove empties.""" if not isinstance(raw, list): return [] return [str(s).strip() for s in raw if str(s).strip()] -def _authenticated_user_id(user: Optional[UserContext]) -> Optional[str]: +def authenticated_user_id(user: Optional[UserContext]) -> Optional[str]: if isinstance(user, UserContext): return user.user_id return None -def _ensure_main_model_configured() -> None: +def ensure_main_model_configured() -> None: """Fail fast with a user-facing error when the main chat model is missing.""" resolved = ModelConfigService.get_instance().resolve("main_agent") if resolved is not None: @@ -44,7 +41,7 @@ def _ensure_main_model_configured() -> None: ) -def _resolve_selected_model_provider_id( +def resolve_selected_model_provider_id( db: Session, request: ChatRequest, user_id: str ) -> Optional[str]: try: @@ -53,7 +50,7 @@ def _resolve_selected_model_provider_id( raise HTTPException(status_code=400, detail=str(exc)) from exc -def _resolve_actual_chat_model_name( +def resolve_actual_chat_model_name( request: ChatRequest, selected_model_provider_id: Optional[str], ) -> Optional[str]: @@ -64,7 +61,7 @@ def _resolve_actual_chat_model_name( def _ensure_chat_session( - chat_service: chat_services.ChatService, + chat_service: chat_services_ChatService_module.ChatService, chat_id: str, user_id: str, first_message: str, @@ -132,12 +129,12 @@ def _ensure_chat_session( return session -def _load_session_messages( - chat_service: chat_services.ChatService, chat_id: str, user_id: str +def load_session_messages( + chat_service: chat_services_ChatService_module.ChatService, chat_id: str, user_id: str ) -> List[Dict[str, Any]]: # Checkpoint-aware history loading: when a checkpoint exists, fetch only the tail messages from the DB (no more load-everything-then-discard). - # Cross-turn tool results are not truncated. See core/services/compaction_service.py. - from core.services.compaction_service import load_session_history + # Cross-turn tool results are not truncated. See core/chat/compaction/service.py. + from core.chat.compaction.service import load_session_history messages = load_session_history(chat_service, chat_id, user_id) if messages is None: @@ -145,7 +142,7 @@ def _load_session_messages( return messages -def _release_request_session(db: Session) -> None: +def release_request_session(db: Session) -> None: """End the request transaction before an SSE response is handed over. A streaming response outlives its handler: FastAPI keeps the request-scoped diff --git a/src/backend/core/services/session_operations/sessions.py b/src/backend/core/chat/operations/sessions.py similarity index 86% rename from src/backend/core/services/session_operations/sessions.py rename to src/backend/core/chat/operations/sessions.py index b93f4e33..16002c18 100644 --- a/src/backend/core/services/session_operations/sessions.py +++ b/src/backend/core/chat/operations/sessions.py @@ -2,9 +2,9 @@ from typing import Any, Dict, Optional -import core.services as chat_services +import core.chat.service as chat_services_ChatService_module -from core.auth.permissions_iface import can_delete_session +from core.auth.edition import can_delete_session from core.infra.exceptions import ResourceNotFoundError @@ -17,7 +17,7 @@ from sqlalchemy.orm import Session -def _session_to_dict(s) -> dict: +def session_to_dict(s) -> dict: """Convert a ChatSession ORM object to the edition-neutral API response.""" return { "chat_id": s.chat_id, @@ -34,11 +34,11 @@ def _session_to_dict(s) -> dict: } -def _session_view_for_user(db: Session, s, user_id: str, level: str) -> dict: +def session_view_for_user(db: Session, s, user_id: str, level: str) -> dict: """Render a session for the current user, then let the edition extend it.""" - from core.services.chat_edition import extend_session_view + from core.chat.edition import extend_session_view - return extend_session_view(db, s, user_id, level, _session_to_dict(s)) + return extend_session_view(db, s, user_id, level, session_to_dict(s)) def list_chats( @@ -64,7 +64,7 @@ def list_chats( - -created_at - Most recently created first - created_at - Oldest created first """ - chat_service = chat_services.ChatService(db) + chat_service = chat_services_ChatService_module.ChatService(db) # Parse filters pinned_only = filter == "pinned=true" if filter else False @@ -80,7 +80,7 @@ def list_chats( exclude_automation=exclude_automation, ) - items = [_session_to_dict(s) for s in sessions] + items = [session_to_dict(s) for s in sessions] return paginated_response( items=items, @@ -97,9 +97,9 @@ def create_chat(request, user=None, db=None): The session is automatically associated with the current authenticated user. """ - chat_service = chat_services.ChatService(db) + chat_service = chat_services_ChatService_module.ChatService(db) if request.project_id: - from core.services.project_access import require_visible_project + from core.projects.access import require_visible_project require_visible_project(db, user.user_id, request.project_id) @@ -111,7 +111,7 @@ def create_chat(request, user=None, db=None): ) return created_response( - data=_session_to_dict(session), message="Chat session created successfully" + data=session_to_dict(session), message="Chat session created successfully" ) @@ -124,7 +124,7 @@ def search_chats(q=..., scope="title", page=1, page_size=20, user=None, db=None) Returns sessions with match_type ("title" or "content") and matched_snippet for content matches. """ - chat_service = chat_services.ChatService(db) + chat_service = chat_services_ChatService_module.ChatService(db) results, total = chat_service.search_sessions( user_id=user.user_id, @@ -136,7 +136,7 @@ def search_chats(q=..., scope="title", page=1, page_size=20, user=None, db=None) items = [] for r in results: - item = _session_to_dict(r["session"]) + item = session_to_dict(r["session"]) item["match_type"] = r["match_type"] item["matched_snippet"] = r["matched_snippet"] items.append(item) @@ -148,7 +148,7 @@ def search_chats(q=..., scope="title", page=1, page_size=20, user=None, db=None) def get_chat(chat_id, user=None, db=None): """获取当前用户有权读取的会话详情。""" - chat_service = chat_services.ChatService(db) + chat_service = chat_services_ChatService_module.ChatService(db) pair = chat_service.get_session_with_access(chat_id, str(user.user_id)) if pair is None: @@ -156,14 +156,14 @@ def get_chat(chat_id, user=None, db=None): session, level = pair return success_response( - data=_session_view_for_user(db, session, str(user.user_id), level), + data=session_view_for_user(db, session, str(user.user_id), level), message="Chat session retrieved successfully", ) def update_chat(chat_id, request, user=None, db=None): """更新当前用户有权修改的会话元信息。""" - chat_service = chat_services.ChatService(db) + chat_service = chat_services_ChatService_module.ChatService(db) user_id = str(user.user_id) pair = chat_service.get_session_with_access(chat_id, user_id) @@ -193,7 +193,7 @@ def update_chat(chat_id, request, user=None, db=None): raise HTTPException(status_code=403, detail="会话元数据仅创建者可改") metadata_change = request.metadata - from core.services.chat_edition import update_member_state + from core.chat.edition import update_member_state member_state_updated = False if request.pinned is not None or request.favorite is not None: @@ -237,14 +237,14 @@ def update_chat(chat_id, request, user=None, db=None): raise ResourceNotFoundError(resource_type="chat_session", resource_id=chat_id) s2, level2 = pair2 return success_response( - data=_session_view_for_user(db, s2, user_id, level2), + data=session_view_for_user(db, s2, user_id, level2), message="Chat session updated successfully", ) def delete_chat(chat_id, user=None, db=None): """软删当前用户有权管理的会话。""" - chat_service = chat_services.ChatService(db) + chat_service = chat_services_ChatService_module.ChatService(db) user_id = str(user.user_id) # First check whether the session is visible to the current user — return 404 if not, to avoid leaking its existence @@ -256,5 +256,7 @@ def delete_chat(chat_id, user=None, db=None): if not can_delete_session(db, user_id, session): raise HTTPException(status_code=403, detail="共享会话仅创建者或项目管理员可删") + from core.llm.mcp.chat_sessions import invalidate_chat + invalidate_chat(chat_id) chat_service.delete_session_force(chat_id, actor_user_id=user_id) return Response(status_code=status.HTTP_204_NO_CONTENT) diff --git a/src/backend/core/services/plan_service.py b/src/backend/core/chat/plans.py similarity index 100% rename from src/backend/core/services/plan_service.py rename to src/backend/core/chat/plans.py diff --git a/src/backend/core/services/project_init.py b/src/backend/core/chat/project_init.py similarity index 91% rename from src/backend/core/services/project_init.py rename to src/backend/core/chat/project_init.py index 6dd8ed34..c2353cf0 100644 --- a/src/backend/core/services/project_init.py +++ b/src/backend/core/chat/project_init.py @@ -7,8 +7,8 @@ from fastapi import HTTPException from sqlalchemy.orm import Session -from core.auth.permissions_iface import resolve_project_permission -from core.services.project_instructions import ProjectInstructionsService +from core.auth.edition import resolve_project_permission +from core.projects.instructions import ProjectInstructionsService INIT_COMMANDS = frozenset({"/init", "/初始化指令"}) @@ -17,7 +17,7 @@ def resolve_project_init(db: Session, request, user_id: str) -> str | None: """Authorize against the saved chat binding, not a client-provided file path.""" if request.message.strip() not in INIT_COMMANDS: return None - from core.services.chat_service import ChatService + from core.chat.service import ChatService project_id = request.project_id pair = ChatService(db).get_session_with_access(request.chat_id, user_id) @@ -31,7 +31,7 @@ def resolve_project_init(db: Session, request, user_id: str) -> str | None: project_id = chat.project_id if not project_id: raise HTTPException(400, "请先选择具体项目，再使用 /init 或 /初始化指令") - from core.services.project_access import require_visible_project + from core.projects.access import require_visible_project project = require_visible_project(db, user_id, project_id) if resolve_project_permission(db, user_id, project) not in ("admin", "edit"): diff --git a/src/backend/core/chat/projection/__init__.py b/src/backend/core/chat/projection/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/backend/core/chat/projection/codec.py b/src/backend/core/chat/projection/codec.py new file mode 100644 index 00000000..20dee0ae --- /dev/null +++ b/src/backend/core/chat/projection/codec.py @@ -0,0 +1,8 @@ +"""Compose chat projection reducers and checkpoint codecs for the run host.""" +from core.runs.events.projection import ProjectionCodec +from core.chat.projection.reducer import fold_projection, new_projection +from core.chat.projection.storage import affected_tools, decode_checkpoint, encode_checkpoint + + +def chat_projection_codec() -> ProjectionCodec: + return ProjectionCodec(new_projection, fold_projection, affected_tools, encode_checkpoint, decode_checkpoint) diff --git a/src/backend/orchestration/run_projection.py b/src/backend/core/chat/projection/reducer.py similarity index 62% rename from src/backend/orchestration/run_projection.py rename to src/backend/core/chat/projection/reducer.py index b1990bb2..f5cfe367 100644 --- a/src/backend/orchestration/run_projection.py +++ b/src/backend/core/chat/projection/reducer.py @@ -6,16 +6,29 @@ def new_projection(run_id): - return {"run_id": run_id, "message_id": "", "started_at": None, - "event_offset": 0, "blocks": [], "tools": [], "signals": {}, - "terminal": False, "history": [], "citations": []} + return { + "run_id": run_id, + "message_id": "", + "started_at": None, + "event_offset": 0, + "blocks": [], + "tools": [], + "signals": {}, + "terminal": False, + "history": [], + "citations": [], + } def _text(state, kind, content, **fields): if not content: return blocks = state["blocks"] - if blocks and blocks[-1]["kind"] == kind and all(blocks[-1].get(k) == v for k, v in fields.items()): + if ( + blocks + and blocks[-1]["kind"] == kind + and all(blocks[-1].get(k) == v for k, v in fields.items()) + ): blocks[-1]["content"] += content else: blocks.append({"kind": kind, "content": content, **fields}) @@ -27,9 +40,16 @@ def _tool(state, event): index = next((i for i, t in enumerate(tools) if identity and t.get("id") == identity), -1) if index < 0: index = len(tools) - tools.append({"id": identity or None, "name": event.get("tool_name") or "", - "displayName": event.get("tool_display_name"), "input": {}, "status": "running", - "timestamp": event.get("started_at") or event.get("server_ts")}) + tools.append( + { + "id": identity or None, + "name": event.get("tool_name") or "", + "displayName": event.get("tool_display_name"), + "input": {}, + "status": "running", + "timestamp": event.get("started_at") or event.get("server_ts"), + } + ) state["blocks"].append({"kind": "tool", "index": index}) return tools[index] @@ -44,18 +64,32 @@ def _child(tool, event): else: steps.append({"kind": kind, "text": str(event.get("delta") or "")}) elif kind in {"tool_call", "tool_call_start", "tool_call_delta", "tool_result"}: - step = next((s for s in steps if s["kind"] == "tool" and tid and s.get("toolId") == tid), None) + step = next( + (s for s in steps if s["kind"] == "tool" and tid and s.get("toolId") == tid), None + ) if step is None: - step = {"kind": "tool", "toolId": tid, "name": event.get("tool_name") or "", "status": "running"} + step = { + "kind": "tool", + "toolId": tid, + "name": event.get("tool_name") or "", + "status": "running", + } steps.append(step) if kind == "tool_call": step["input"] = event.get("input", event.get("tool_args", {})) if kind == "tool_call_delta": step["inputText"] = step.get("inputText", "") + str(event.get("arguments_delta") or "") if kind == "tool_result": - step.update(output=event.get("output", event.get("result")), status=event.get("status") or "success") + step.update( + output=event.get("output", event.get("result")), + status=event.get("status") or "success", + ) elif kind == "end": - tool["status"] = "interrupted" if event.get("status") == "cancelled" else "success" if event.get("ok") else "error" + tool["status"] = ( + "interrupted" + if event.get("status") == "cancelled" + else "success" if event.get("ok") else "error" + ) for step in steps: if step["kind"] == "tool" and step.get("status") == "running": step["status"] = "interrupted" @@ -63,7 +97,6 @@ def _child(tool, event): tool["subagentName"] = event["agent_name"] - def _signal(state, event): kind = event.get("type", "") identity = event.get("request_id") or event.get("confirm_id") or event.get("tool_id") or "" @@ -82,18 +115,26 @@ def _signal(state, event): if kind in {"file_confirm", "design_pick"} and event.get("expired"): state["signals"].pop(key, None) return - value = {**state["signals"].get(key, {}), **{k: v for k, v in event.items() if not k.startswith("_")}} + value = { + **state["signals"].get(key, {}), + **{k: v for k, v in event.items() if not k.startswith("_")}, + } for field in ("delta", "arguments_delta"): if field in event and key in state["signals"]: value[field] = str(state["signals"][key].get(field) or "") + str(event[field] or "") state["signals"][key] = value + def fold_projection(previous, event): """Copy-on-write: concurrent appenders never mutate a published base.""" - state = {**previous, "blocks": [dict(b) for b in previous["blocks"]], - "tools": [dict(t) for t in previous["tools"]], - "signals": dict(previous["signals"]), "history": list(previous["history"]), - "citations": list(previous.get("citations", []))} + state = { + **previous, + "blocks": [dict(b) for b in previous["blocks"]], + "tools": [dict(t) for t in previous["tools"]], + "signals": dict(previous["signals"]), + "history": list(previous["history"]), + "citations": list(previous.get("citations", [])), + } kind = event.get("type") if kind == "run_started": state = new_projection(state["run_id"]) @@ -105,18 +146,36 @@ def fold_projection(previous, event): previous_message["terminal"] = True history.append({"role": "assistant", "state": previous_message}) if event.get("message"): - history.append({"role": "user", "message_id": event.get("message_id"), - "content": event["message"], "timestamp": event.get("server_ts")}) + history.append( + { + "role": "user", + "message_id": event.get("message_id"), + "content": event["message"], + "timestamp": event.get("server_ts"), + } + ) state = new_projection(state["run_id"]) state["history"] = history - state.update(message_id=event.get("next_assistant_message_id") or "", - started_at=event.get("server_ts")) + state.update( + message_id=event.get("next_assistant_message_id") or "", + started_at=event.get("server_ts"), + ) elif kind in {"content", "ai_message", "text", "delta"}: - _text(state, "text", str(event.get("delta") or event.get("content") or event.get("text") or "")) + _text( + state, + "text", + str(event.get("delta") or event.get("content") or event.get("text") or ""), + ) elif kind == "compaction_progress": block = {"kind": "compaction", "progress": dict(event)} - index = next((i for i, b in enumerate(state["blocks"]) - if b["kind"] == "compaction" and b.get("progress", {}).get("id") == event.get("id")), -1) + index = next( + ( + i + for i, b in enumerate(state["blocks"]) + if b["kind"] == "compaction" and b.get("progress", {}).get("id") == event.get("id") + ), + -1, + ) if index >= 0: state["blocks"][index] = block else: @@ -125,10 +184,16 @@ def fold_projection(previous, event): if event.get("structured_reasoning") is True and not state.get("structured_reasoning"): state["blocks"].append({"kind": "protocol", "structured": True}) state["structured_reasoning"] = True - _text(state, "thinking", str(event.get("delta") or event.get("message") or ""), - structured=event.get("structured_reasoning") is True) + _text( + state, + "thinking", + str(event.get("delta") or event.get("message") or ""), + structured=event.get("structured_reasoning") is True, + ) elif kind == "content_replace": - state["blocks"].append({"kind": "answer", "content": str(event.get("content") or event.get("text") or "")}) + state["blocks"].append( + {"kind": "answer", "content": str(event.get("content") or event.get("text") or "")} + ) state["structured_reasoning"] = True elif event.get("scope") == "ontology_revision": _signal(state, event) @@ -140,15 +205,22 @@ def fold_projection(previous, event): elif kind == "tool_call_delta": tool["inputText"] = tool.get("inputText", "") + str(event.get("arguments_delta") or "") elif kind == "tool_result": - tool.update(output=event.get("result"), status=event.get("status") or "success", - durationMs=event.get("duration_ms")) + tool.update( + output=event.get("result"), + status=event.get("status") or "success", + durationMs=event.get("duration_ms"), + ) state["blocks"].append({"kind": "phase"}) if tool["name"] == "choose_design": - state["signals"] = {k: v for k, v in state["signals"].items() if v.get("type") != "design_pick"} + state["signals"] = { + k: v for k, v in state["signals"].items() if v.get("type") != "design_pick" + } elif kind == "subagent_event" and event.get("sub_type") == "job_progress": _signal(state, event) elif kind == "subagent_event": - parent = next((t for t in state["tools"] if t.get("id") == event.get("parent_tool_id")), None) + parent = next( + (t for t in state["tools"] if t.get("id") == event.get("parent_tool_id")), None + ) if parent is not None: parent["subSteps"] = [dict(step) for step in parent.get("subSteps", [])] _child(parent, event) diff --git a/src/backend/orchestration/run_projection_storage.py b/src/backend/core/chat/projection/storage.py similarity index 89% rename from src/backend/orchestration/run_projection_storage.py rename to src/backend/core/chat/projection/storage.py index 7fceb460..98b3e0d6 100644 --- a/src/backend/orchestration/run_projection_storage.py +++ b/src/backend/core/chat/projection/storage.py @@ -1,4 +1,5 @@ """Checkpoint fields: changing one child card must not rewrite every card.""" + import json @@ -28,5 +29,8 @@ def affected_tools(state, event): if kind in {"run_started", "steer_applied"}: return set(range(len(state["tools"]))) identity = event.get("parent_tool_id") if kind == "subagent_event" else event.get("tool_id") - return {index for index, tool in enumerate(state["tools"]) - if identity and tool.get("id") == str(identity)} + return { + index + for index, tool in enumerate(state["tools"]) + if identity and tool.get("id") == str(identity) + } diff --git a/src/backend/orchestration/run_subscription.py b/src/backend/core/chat/projection/subscription.py similarity index 80% rename from src/backend/orchestration/run_subscription.py rename to src/backend/core/chat/projection/subscription.py index 485dfb26..7d977ceb 100644 --- a/src/backend/orchestration/run_subscription.py +++ b/src/backend/core/chat/projection/subscription.py @@ -1,12 +1,15 @@ """Native snapshot + tail subscription. Transport EOF carries no task verdict.""" + import asyncio import contextlib import json import time -from core.llm import interaction_store -from orchestration import chat_run_executor -from orchestration.run_event_stream import START, _sort_key, get_run_event_stream +import core.runs.clock as clock +import core.runs.runtime as runtime +import core.runs.settings as settings +from core.llm.runtime import interaction_store +from core.runs.events.store import START, sort_key, get_run_event_stream async def subscribe_run(run_id): @@ -20,27 +23,32 @@ async def snapshot_frame(state, run): if signal.get("type") not in {"file_confirm", "design_pick"}: continue record = await interaction_store.read("confirm", run.chat_id, signal.get("confirm_id")) - if (record is None or record.get("decision") - or record["expires_at"] <= time.time() or record["lease_until"] <= time.time()): + if ( + record is None + or record.get("decision") + or record["expires_at"] <= time.time() + or record["lease_until"] <= time.time() + ): signals.pop(key) state = {**state, "signals": signals} return {"type": "run_snapshot", "state": state, "event_offset": state["event_offset"]} async def frames(): state, cursor = await stream.capture(run_id) - run = await asyncio.to_thread(chat_run_executor.get_run, run_id) + run = await asyncio.to_thread(runtime.get_run, run_id) if run is None: raise RuntimeError("run no longer exists") if state is None and cursor == START and not run.last_event_offset: # Admission already owns an empty assistant row, before event 1. # This is the empty event prefix, not a reconstructed history row. - from orchestration.run_projection import new_projection + from core.chat.projection.reducer import new_projection + state = new_projection(run_id) state["message_id"] = run.message_id - state["started_at"] = chat_run_executor._epoch_ms(run.started_at) + state["started_at"] = clock._epoch_ms(run.started_at) if state is None or not state.get("message_id"): raise RuntimeError("run projection unavailable") - if run.status in chat_run_executor._TERMINAL_STATUSES: + if run.status in settings._TERMINAL_STATUSES: state, cursor = await stream.capture(run_id) if state is None: raise RuntimeError("run projection expired") @@ -51,15 +59,15 @@ async def frames(): while True: batch = await stream.wait(run_id, after=cursor, limit=100, timeout_ms=5000) if not batch: - run = await asyncio.to_thread(chat_run_executor.get_run, run_id) - if run and run.status in chat_run_executor._TERMINAL_STATUSES: + run = await asyncio.to_thread(runtime.get_run, run_id) + if run and run.status in settings._TERMINAL_STATUSES: state, _ = await stream.capture(run_id) if state is not None: state["terminal"] = True yield await snapshot_frame(state, run) return first = await stream.read(run_id, limit=1) - if cursor != START and first and _sort_key(first[0][0]) > _sort_key(cursor): + if cursor != START and first and sort_key(first[0][0]) > sort_key(cursor): state, cursor = await stream.capture(run_id) if state is None: raise RuntimeError("run projection expired") diff --git a/src/backend/core/services/public_chat_service.py b/src/backend/core/chat/public.py similarity index 78% rename from src/backend/core/services/public_chat_service.py rename to src/backend/core/chat/public.py index ed25cb94..f429f328 100644 --- a/src/backend/core/services/public_chat_service.py +++ b/src/backend/core/chat/public.py @@ -1,26 +1,23 @@ """Public entry links reuse scoped API credentials; visitor scope never grants account access.""" -import hashlib -import re import secrets +from core.auth.public_identity import digest import uuid from datetime import datetime, timezone from core.db.models import ContentBlock, UserApiKey, UserShadow from core.infra.crypto import decrypt_secret, encrypt_secret -from core.services.api_key_service import ApiKeyService, resolve_api_key_identity +from core.auth.api_keys import ApiKeyService from fastapi import HTTPException from sqlalchemy.exc import IntegrityError -def digest(value): - return hashlib.sha256(value.encode()).hexdigest() def require_manager(db, user_id, kind, resource_id): from core.auth.capabilities import resolve_user_capabilities - from core.services.agent_api_service import require_agent_manager - from core.services.project_api_service import require_project_key_access + from core.auth.agent_api import require_agent_manager + from core.projects.api import require_project_key_access if kind not in ("agents", "projects"): raise HTTPException(404, detail="Unknown resource") @@ -113,30 +110,6 @@ def set_enabled(db, user_id, kind, resource_id, enabled): return get_link(db, user_id, kind, resource_id) -def resolve_public_user(db, token, visitor): - from core.auth.backend import UserContext - - if not re.fullmatch(r"[A-Za-z0-9_-]{43}", token) or not re.fullmatch( - r"[A-Za-z0-9_-]{32,64}", visitor - ): - raise HTTPException(401, detail="Invalid public chat link") - row = db.get(ContentBlock, "publicchat:token:" + digest(token)) - key = db.get(UserApiKey, row.payload["key_id"]) if row else None - identity = ( - resolve_api_key_identity(db, decrypt_secret(key.key_enc)) if key and key.key_enc else None - ) - if not identity: - raise HTTPException(401, detail="Public chat link is unavailable") - owner, key = identity - return UserContext( - user_id=owner.user_id, - user_center_id=owner.user_center_id or "", - username=owner.username, - api_key_id=key.id, - api_key_agent_id=key.agent_id, - api_key_project_id=key.project_id, - public_chat_visitor=digest(visitor), - ) def visitor_preference_id(user): @@ -144,14 +117,14 @@ def visitor_preference_id(user): def visitor_approval_mode(db, user, default=None): - from core.llm.tool_permissions import normalize_approval_mode + from core.sandbox.permissions.approval import normalize_approval_mode row = db.get(ContentBlock, visitor_preference_id(user)) return normalize_approval_mode((row.payload if row else {}).get("tool_approval_mode", default)) def set_visitor_approval_mode(db, user, mode): - from core.llm.tool_permissions import normalize_approval_mode + from core.sandbox.permissions.approval import normalize_approval_mode key = visitor_preference_id(user) row = db.get(ContentBlock, key) @@ -165,8 +138,8 @@ def set_visitor_approval_mode(db, user, mode): def prepare_visitor_upload(db, user, chat_id, folder_id=None): from core.chat.schemas import ChatRequest - from core.services.agent_api_service import prepare_agent_api_request - from core.services.project_api_service import prepare_project_api_request + from core.auth.agent_api import prepare_agent_api_request + from core.projects.api import prepare_project_api_request if not chat_id or folder_id: raise HTTPException(422, detail="A public upload requires its conversation") diff --git a/src/backend/core/services/chat_reference_service.py b/src/backend/core/chat/references.py similarity index 98% rename from src/backend/core/services/chat_reference_service.py rename to src/backend/core/chat/references.py index 4be774e7..f6a6637a 100644 --- a/src/backend/core/services/chat_reference_service.py +++ b/src/backend/core/chat/references.py @@ -17,7 +17,7 @@ from core.db.models import ChatMessage, ChatSession from core.db.reserved_chats import ordinary_chat_filter -from core.services.chat_service import ChatService +from core.chat.service import ChatService from sqlalchemy import func, or_ from sqlalchemy.orm import Session @@ -286,8 +286,8 @@ async def build_session_digest( import asyncio from core.db.engine import SessionLocal - from core.services.compaction_service import _load_history - from core.services.compaction_summary import _summarize + from core.chat.compaction.service import _load_history + from core.chat.compaction.summary import _summarize def _prepare(): with SessionLocal() as db: diff --git a/src/backend/core/chat/runtime/__init__.py b/src/backend/core/chat/runtime/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/backend/core/chat/runtime/admission.py b/src/backend/core/chat/runtime/admission.py new file mode 100644 index 00000000..63a1434c --- /dev/null +++ b/src/backend/core/chat/runtime/admission.py @@ -0,0 +1,98 @@ +"""Admit a normal chat run and start its lease-owning worker.""" + +from __future__ import annotations + +import asyncio +from typing import Any, Dict, List, Optional + +import core.chat.runtime.worker as worker +import core.db.engine as engine +import core.db.models as models +import core.runs.clock as clock +import core.runs.events.writer as writer +import core.runs.ownership as ownership +import core.runs.runtime as runtime +from core.db.models import ChatRun +from core.infra.logging import get_logger + +logger = get_logger(__name__) + + +async def start_run( + *, + accepted_run: Optional[ChatRun] = None, + chat_id: str, + user_id: str, + session_messages: List[Dict[str, Any]], + effective_user_message: str, + raw_user_message: str, + context: Dict[str, Any], + request_payload: Optional[Dict[str, Any]] = None, + model_name: Optional[str] = None, +) -> ChatRun: + """Launch a ChatRun that ChatSequencer has already durably accepted. + + Acceptance (user message + pending run) must commit before this function is + called. Keeping launch separate makes a process crash between the two + recoverable by the Run Journal in #64 instead of losing the request. + """ + + def persist_launch(): + recovery_snapshot = writer._json_safe( + { + "kind": "chat", + "worker_args": { + "session_messages": session_messages, + "effective_user_message": effective_user_message, + "raw_user_message": raw_user_message, + "context": context, + "model_name": model_name, + }, + } + ) + if accepted_run is None: + run = ownership.create_run_record( + chat_id=chat_id, + user_id=user_id, + request_payload=dict(request_payload or {}), + recovery_snapshot=recovery_snapshot, + ) + else: + run = accepted_run + if run.chat_id != chat_id or run.user_id != user_id or run.status != "pending": + raise ValueError("accepted_run does not match the launch request") + with engine.SessionLocal() as db: + durable_run = db.get(models.ChatRun, run.run_id) + if durable_run is None or durable_run.status != "pending": + raise ValueError("accepted_run is no longer pending") + durable_run.recovery_snapshot = recovery_snapshot + durable_run.snapshot_version = max(int(durable_run.snapshot_version or 0), 1) + durable_run.run_phase = durable_run.run_phase or "accepted" + durable_run.last_operation_safety = ( + durable_run.last_operation_safety or "replayable" + ) + durable_run.updated_at = clock._utcnow() + db.commit() + return run + + run = await asyncio.to_thread(persist_launch) + journal_owner = ownership.new_worker_owner(run.run_id) + runtime.register_run_task( + run.run_id, + worker._run_workflow( + run_id=run.run_id, + chat_id=chat_id, + user_id=user_id, + message_id=run.message_id, + assistant_chat_seq=run.assistant_chat_seq, + session_messages=session_messages, + effective_user_message=effective_user_message, + raw_user_message=raw_user_message, + context=context, + model_name=model_name, + journal_owner=journal_owner, + ), + name=f"chat_run:{run.run_id}", + ) + logger.info("chat_run_started", run_id=run.run_id, chat_id=chat_id, user_id=user_id) + return run diff --git a/src/backend/core/chat/runtime/chunks.py b/src/backend/core/chat/runtime/chunks.py new file mode 100644 index 00000000..544b62a5 --- /dev/null +++ b/src/backend/core/chat/runtime/chunks.py @@ -0,0 +1,183 @@ +"""Apply workflow events to chat output and durable projections.""" + +from __future__ import annotations + +import core.llm.runtime.model_steps as model_steps +from core.infra.text_bounds import bound_result_for_display +from core.chat.tool_log import ( + attach_subagent_step, + build_thinking_event, + build_tool_call_delta_event, + build_tool_call_event, + build_tool_call_start_event, + build_tool_result_event, + build_user_question_event, + build_user_question_resolved_event, +) +from core.infra.logging import get_logger + + +from core.chat.runtime import completion, snapshots, steering + +logger = get_logger(__name__) + + +async def consume_chunk(turn, chunk): + chunk_type = chunk.get("type") + if chunk_type == "model_dispatch": + turn.journal.append_operation( + turn.run_id, + owner=turn.owner, + operation_type="model_dispatch", + phase="model_inflight", + safety="replayable", + ) + elif chunk_type == "thinking": + _thinking_evt = build_thinking_event(chunk, turn.chat_id) + if _thinking_evt.get("delta"): + turn.thinking_parts.append(str(_thinking_evt["delta"])) + await turn.emit(_thinking_evt) + elif chunk_type in {"ai_message", "content"}: + delta = chunk.get("delta", "") + if delta: + snapshots._flush_thinking(turn) + turn.full_response += delta + turn.segments.add_text(delta) + await turn.emit( + {"type": "content", "event": "ai_message", "delta": delta, "chat_id": turn.chat_id} + ) + elif chunk_type == "content_replace": + replacement = str(chunk.get("content") or "") + turn.thinking_parts.clear() + turn.full_response = replacement + turn.round_block = None + turn.thinking_log.clear() + model_steps.replace_final_text(turn.model_steps, replacement) + turn.segments.reset(preserve_compaction=True) + turn.segments.add_text(replacement) + await turn.emit( + { + "type": "content_replace", + "content": replacement, + "reason": chunk.get("reason"), + "chat_id": turn.chat_id, + } + ) + await snapshots._flush_inflight(turn) + elif chunk_type == "steer_applied": + await steering.apply_steer(turn, chunk) + elif chunk_type == "tool_call": + snapshots._flush_thinking(turn) + turn.round_block = None + _tc_evt = build_tool_call_event(chunk, turn.chat_id, turn.tool_calls_log) + turn.segments.add_tools(turn.tool_calls_log) + await turn.emit(_tc_evt) + await snapshots._flush_inflight(turn) + elif chunk_type == "tool_call_start": + snapshots._flush_thinking(turn) + turn.round_block = None + await turn.emit(build_tool_call_start_event(chunk, turn.chat_id)) + elif chunk_type == "tool_call_delta": + await turn.emit(build_tool_call_delta_event(chunk, turn.chat_id)) + elif chunk_type == "tool_progress": + # Live status for a running tool card; the result supersedes it, so it is not persisted. + await turn.emit({**chunk, "chat_id": turn.chat_id}) + elif chunk_type == "tool_result": + step = chunk.get("model_step") + if isinstance(step, dict): + turn.model_steps.append(step) + _tr_evt = build_tool_result_event(chunk, turn.chat_id, turn.tool_calls_log) + turn.segments.add_tools(turn.tool_calls_log) + if isinstance(step, dict): + if not await snapshots._flush_inflight(turn) and ( + not await snapshots._flush_inflight(turn) + ): + raise RuntimeError("Could not persist completed tool result") + await turn.emit(_tr_evt) + await snapshots._flush_inflight(turn) + elif chunk_type == "model_step": + step = chunk.get("step") + if isinstance(step, dict): + turn.model_steps.append(step) + if step.get("kind") == "tool_result": + if not await snapshots._flush_inflight(turn) and ( + not await snapshots._flush_inflight(turn) + ): + raise RuntimeError("Could not persist completed tool result") + else: + await snapshots._flush_inflight(turn) + elif chunk_type == "compaction_progress": + turn.segments.add_compaction(chunk) + await turn.emit(chunk) + elif chunk_type == "context_usage": + await turn.emit({**chunk, "chat_id": turn.chat_id}) + elif chunk_type in ("heartbeat", "model_progress"): + return + elif chunk_type == "tool_pending": + pending_event = { + "type": "tool_pending", + "chat_id": turn.chat_id, + "reason": chunk.get("reason", "llm_buffering"), + } + if chunk.get("scope"): + pending_event["scope"] = chunk["scope"] + await turn.emit(pending_event) + elif chunk_type == "subagent_event": + try: + attach_subagent_step( + turn.tool_calls_log, str(chunk.get("parent_tool_id", "") or ""), chunk + ) + except Exception: + logger.debug("attach_subagent_step failed (ignored)", exc_info=True) + chunk["chat_id"] = turn.chat_id + await turn.emit(bound_result_for_display(chunk)[0]) + elif chunk_type == "file_confirm": + await turn.emit( + { + "type": "file_confirm", + "chat_id": turn.chat_id, + "confirm_id": chunk.get("confirm_id"), + "op": chunk.get("op"), + "logical_path": chunk.get("logical_path"), + "message": chunk.get("message"), + "expired": chunk.get("expired", False), + } + ) + elif chunk_type == "design_pick": + await turn.emit( + { + "type": "design_pick", + "chat_id": turn.chat_id, + "confirm_id": chunk.get("confirm_id"), + "question": chunk.get("question", ""), + "options": chunk.get("options", []), + "message": chunk.get("message"), + "expired": chunk.get("expired", False), + } + ) + elif chunk_type == "user_question": + await turn.emit(build_user_question_event(chunk, turn.chat_id)) + elif chunk_type == "user_question_resolved": + await turn.emit(build_user_question_resolved_event(chunk, turn.chat_id)) + elif chunk_type == "plan_update": + await turn.emit( + { + "type": "plan_update", + "chat_id": turn.chat_id, + "title": chunk.get("title", ""), + "steps": chunk.get("steps", []), + } + ) + elif chunk_type in { + "ontology_activation", + "ontology_gate", + "ontology_review", + "ontology_repair", + "ontology_revision", + "ontology_revision_thinking", + }: + ontology_event = dict(chunk) + ontology_event["chat_id"] = turn.chat_id + await turn.emit(bound_result_for_display(ontology_event)[0]) + elif chunk_type == "meta": + await completion.complete_turn(turn, chunk) diff --git a/src/backend/orchestration/post_turn_compaction.py b/src/backend/core/chat/runtime/compaction.py similarity index 84% rename from src/backend/orchestration/post_turn_compaction.py rename to src/backend/core/chat/runtime/compaction.py index ca07912b..4ffa6ae9 100644 --- a/src/backend/orchestration/post_turn_compaction.py +++ b/src/backend/core/chat/runtime/compaction.py @@ -1,8 +1,10 @@ """Schedule background compression using the completed turn's frozen window.""" from __future__ import annotations + import asyncio from typing import Any, Dict, Optional + from core.infra.logging import get_logger logger = get_logger(__name__) @@ -20,13 +22,8 @@ def _should_schedule_compaction( if not _settings.compaction.enabled: return False - from core.llm.context_manager import resolve_model_context_window - from core.services.compaction_service import ( - resolve_active_tokens, - resolve_token_limit, - resolve_trigger_ratio, - should_compact, - ) + from core.models.context_window import resolve_model_context_window + from core.chat.compaction.service import resolve_active_tokens, resolve_token_limit, resolve_trigger_ratio, should_compact active_tokens = resolve_active_tokens(usage) window = int((budget_inputs or {}).get("context_window") or 0) @@ -54,7 +51,7 @@ def _spawn_compaction_task( Fire-and-forget; never affects the main conversation. This is *not* a second compactor — it calls the same - :func:`core.services.compaction_service.run_compaction` the mid-turn hook + :func:`core.chat.compaction.service.run_compaction` the mid-turn hook does, and normally finds nothing to do: a turn that crossed the threshold while it still had steps left has already compacted mid-turn and persisted its checkpoint. It earns its keep only when the threshold is crossed by a @@ -77,12 +74,8 @@ def _spawn_compaction_task( async def _bg() -> None: try: - from core.llm.context_manager import resolve_model_context_window - from core.services.compaction_service import ( - resolve_token_limit, - resolve_trigger_ratio, - run_post_turn_compaction, - ) + from core.models.context_window import resolve_model_context_window + from core.chat.compaction.service import resolve_token_limit, resolve_trigger_ratio, run_post_turn_compaction # Trigger criterion — see resolve_active_tokens: real end-of-turn # context occupancy, not the cumulative billing value. Gating here @@ -99,7 +92,7 @@ async def _bg() -> None: chat_id=chat_id, limit=limit, ) - from core.services.compaction_progress import progress_scope + from core.chat.compaction.progress import progress_scope with progress_scope(None, chat_id, run_id, context={"message_id": message_id}): await run_post_turn_compaction( diff --git a/src/backend/core/chat/runtime/completion.py b/src/backend/core/chat/runtime/completion.py new file mode 100644 index 00000000..cae41a44 --- /dev/null +++ b/src/backend/core/chat/runtime/completion.py @@ -0,0 +1,231 @@ +"""Commit successful chat output within the run journal transaction.""" + +from __future__ import annotations + +import asyncio +import functools +import time +import typing as _typing + +import core.chat.runtime.compaction as compaction +import core.chat.runtime.followups as followups +import core.chat.runtime.handoff as handoff +import core.runs.events.writer as writer +import core.chat.service as services_ChatService_module +from core.chat.context import now_iso +from core.infra.logging import get_logger +from core.llm.runtime import workspace as _workspace_mod +from core.files.artifacts.service import persist_artifacts as _persist_artifacts + + +from core.chat.runtime import snapshots + +logger = get_logger(__name__) + + +async def complete_turn(turn, chunk): + snapshots._flush_thinking(turn) + internal_budget = chunk.get("_compaction_budget_inputs") + if isinstance(internal_budget, dict): + turn.compaction_budget_inputs = dict(internal_budget) + # Strict workspace gate: pinned list is the sole source of + # user-visible artifacts. See chats.py:_stream_sse_response. + _ws_pinned = _workspace_mod.get_pinned() + _ws_files = _workspace_mod.get_pinned_file_ids() + usage_payload = chunk.get("usage") or None + context_usage_payload = chunk.get("context_usage") or None + compaction_pending = not chunk.get( + "manual_compaction" + ) and compaction._should_schedule_compaction( + model_name=turn.model_name, + usage=usage_payload, + budget_inputs=turn.compaction_budget_inputs, + ) + # 本轮总耗时：同一个值既下发给正在看的这一页，也写进 extra_data 供历史 + # 加载使用。以前只写库不下发，前端只能自己按「占位气泡创建到现在」估一个， + # 把网络往返和排队也算了进去 —— 于是同一条回答，实时看到的耗时和刷新后 + # 从历史读出来的对不上。 + _duration_ms = int((time.monotonic() - turn.run_started_monotonic) * 1000) + turn.metadata = { + "type": "meta", + "duration_ms": _duration_ms, + "route": chunk.get("route", "main"), + "sources": chunk.get("sources", []), + "artifacts": _ws_pinned, + "warnings": chunk.get("warnings", []), + "is_markdown": chunk.get("is_markdown", False), + "chat_id": turn.chat_id, + "message_id": turn.current_message_id, + "citations": chunk.get("citations", []), + "workspace_files": _ws_files, + "context_usage": context_usage_payload, + "compaction_pending": compaction_pending, + "ontology_governance": chunk.get("ontology_governance"), + # Skeleton marker: settlement runs after the stream closes. + "evolution_pending": chunk.get("evolution_pending"), + } + await turn.emit(turn.metadata) + + # Persist: assistant message + artifacts + _persist_extra = { + "timestamp": now_iso(), + "route": turn.metadata.get("route"), + "is_markdown": turn.metadata.get("is_markdown", False), + "sources": turn.metadata.get("sources", []), + "artifacts": turn.metadata.get("artifacts", []), + "warnings": turn.metadata.get("warnings", []), + "citations": turn.metadata.get("citations", []), + "message_id": turn.current_message_id, + "workspace_files": _ws_files, + "duration_ms": _duration_ms, + "segments": turn.segments.payload(), + } + if isinstance(context_usage_payload, dict): + _persist_extra["context_usage"] = context_usage_payload + if turn.metadata.get("ontology_governance"): + _persist_extra["ontology_governance"] = turn.metadata["ontology_governance"] + if turn.context.get("model_provider_id"): + _persist_extra["model_provider_id"] = turn.context.get("model_provider_id") + turn.journal.save_snapshot( + turn.run_id, + owner=turn.owner, + phase="model_completed", + safety="replayable", + snapshot=writer._json_safe( + { + "assistant_content": turn.full_response, + "message_id": turn.current_message_id, + "model_name": turn.model_name, + "thinking": snapshots._thinking_payload(turn), + "tool_calls": turn.tool_calls_log if turn.tool_calls_log else None, + "model_steps": snapshots._model_steps_payload(turn), + "usage": usage_payload, + "extra_data": _persist_extra, + "artifacts": _ws_pinned, + "context": turn.context, + } + ), + ) + + queued_handoff: _typing.Dict[str, _typing.Any] = {} + + def _commit_output(db) -> None: + chat_service = services_ChatService_module.ChatService(db) + # The row already exists (created at admission, refreshed + # while streaming); this is the final write that also drops + # the in-flight marker because ``_persist_extra`` never + # carries it. + chat_service.upsert_message( + chat_id=turn.chat_id, + role="assistant", + content=turn.full_response, + model=turn.model_name, + thinking=snapshots._thinking_payload(turn), + tool_calls=turn.tool_calls_log if turn.tool_calls_log else None, + model_steps=snapshots._model_steps_payload(turn), + usage=usage_payload, + message_id=turn.current_message_id, + chat_seq=turn.current_chat_seq, + extra_data=_persist_extra, + invalidate_mid_turn_checkpoint=compaction_pending, + commit=False, + ) + # Build a ProjectScope from the workflow context and pass it + # explicitly so pinned files keep their project ownership. + from core.projects.edition import project_scope_from_context + + _persist_artifacts( + db, + turn.user_id, + turn.chat_id, + _ws_pinned, + scope=project_scope_from_context(turn.context), + commit=False, + ) + prepared = handoff._commit_queued_handoff_in_session( + db, + source_run_id=turn.run_id, + chat_id=turn.chat_id, + user_id=turn.user_id, + context=turn.context, + model_name=turn.model_name, + ) + if prepared is not None: + queued_handoff.update(prepared) + + # Offloaded to a worker thread on purpose. This is the one + # journal call that commits the turn's real side effects (the + # assistant message, pinned artifacts, a queued handoff), so it + # is the one that can sit waiting on a row lock. ``complete`` + # is synchronous SQLAlchemy; calling it inline blocks the event + # loop, and a blocked loop cannot run the coroutine holding the + # conflicting transaction — the process deadlocks and every + # request, stream and health probe stops answering. In a thread + # a lock wait costs this run its latency and nothing else. + # ``_commit_output`` and everything it calls are plain sync + # code, so they are safe to run off the loop. + completed = await asyncio.to_thread( + functools.partial( + turn.journal.complete, + turn.run_id, + owner=turn.owner, + status="completed", + usage=usage_payload, + last_event_offset=turn.offsets.last, + commit_effect=_commit_output, + committed_operation={ + "operation_type": "message_committed", + "phase": "message_committed", + "safety": "side_effect_committed", + "payload": {"message_id": turn.current_message_id}, + }, + ) + ) + if not completed: + logger.warning( + "chat_run_terminal_cas_lost", + run_id=turn.run_id, + owner=turn.owner, + ) + turn.stopped = True + return + + # The DB lease has intentionally ended. Stop the heartbeat + # before scheduling the handoff and projecting terminal events; + # otherwise its next renew would interpret normal completion as + # lease loss and cancel this coroutine mid-handoff. + turn.lease_task.cancel() + + if queued_handoff: + handoff._schedule_queued_handoff( + queued_handoff, + chat_id=turn.chat_id, + user_id=turn.user_id, + ) + await turn.emit( + handoff._queued_run_started_event(turn.run_id, turn.chat_id, queued_handoff), + terminal=True, + ) + + if not chunk.get("manual_compaction"): + followups._spawn_followup_task( + run_id=turn.run_id, + chat_id=turn.chat_id, + user_msg=turn.latest_user_message, + response=turn.full_response, + msg_id=turn.current_message_id, + ) + + # End-of-turn compaction: when real token usage crosses the + # threshold → generate a summary checkpoint in the background; + # doesn't block stream close, and failures don't affect the main + # conversation. + compaction._spawn_compaction_task( + run_id=turn.run_id, + chat_id=turn.chat_id, + model_name=turn.model_name, + usage=usage_payload, + budget_inputs=turn.compaction_budget_inputs, + should_schedule=compaction_pending, + message_id=turn.current_message_id, + ) diff --git a/src/backend/core/chat/runtime/followups.py b/src/backend/core/chat/runtime/followups.py new file mode 100644 index 00000000..1a0335e6 --- /dev/null +++ b/src/backend/core/chat/runtime/followups.py @@ -0,0 +1,63 @@ +"""Schedule post-turn suggestions and drain admitted follow-up work.""" + +from __future__ import annotations + +import asyncio +import typing as _typing +from typing import Any, Dict, List, Optional, Tuple + +import core.db.engine as engine +import core.runs.settings as settings +import core.chat.service as services_ChatService_module +from core.infra.logging import get_logger +from core.runs.events.store import Entry + +logger = get_logger(__name__) + + +def _spawn_followup_task( + *, run_id: str, chat_id: str, user_msg: str, response: str, msg_id: str +) -> None: + """Equivalent to the existing _generate_followups_bg in chats.py.""" + from core.chat.followups import get_followup_generator + + async def _bg() -> None: + try: + questions = await asyncio.wait_for( + get_followup_generator().generate(user_msg, response, run_id=run_id), + timeout=10, + ) + if questions: + with engine.SessionLocal() as bg_db: + services_ChatService_module.ChatService(bg_db).update_message_extra_data( + msg_id, {"follow_up_questions": questions} + ) + except Exception as exc: + logger.warning("background_followup_failed", error=str(exc)) + + asyncio.create_task(_bg()) + + +def _drain( + entries: Optional[List[Entry]], from_offset: int, cursor: str +) -> Tuple[List[Dict[str, Any]], str, bool]: + """Split a batch into deliverable events, the new cursor, and "saw terminal". + + Skips ``model_progress`` markers: nothing writes them anymore (the reaper + trusts in-process task liveness), but entries from older builds may survive + within the log's TTL and must never reach clients — the frontend treats an + unknown event type as "pending ended". + + An empty batch may arrive as ``None`` rather than ``[]``; a read that + returns nothing must never surface to the client as "流式响应中断". + """ + delivered: _typing.List[_typing.Dict[str, _typing.Any]] = [] + for entry_cursor, event in entries or (): + cursor = entry_cursor + if not isinstance(event, dict) or event.get("type") == "model_progress": + continue + if event.get("type") == settings.TERMINAL_TYPE: + return delivered, cursor, True + if event.get("_offset", 0) > from_offset: + delivered.append(event) + return delivered, cursor, False diff --git a/src/backend/core/chat/runtime/handoff.py b/src/backend/core/chat/runtime/handoff.py new file mode 100644 index 00000000..13064e89 --- /dev/null +++ b/src/backend/core/chat/runtime/handoff.py @@ -0,0 +1,272 @@ +"""Commit queued chat instructions and hand off to the next durable turn.""" + +from __future__ import annotations + +import uuid +from typing import Any, Dict, List, Optional + +import core.chat.inflight as inflight +import core.chat.runtime.worker as worker +import core.db.engine as engine +import core.db.models as models +import core.runs.clock as clock +import core.runs.events.writer as writer +import core.runs.ownership as ownership +import core.runs.runtime as runtime +import core.chat.service as services_ChatService_module +import core.runs.journal as run_journal +from core.db.models import ChatRun +from core.infra.logging import get_logger +from core.chat.service import ChatService + +logger = get_logger(__name__) + + +def _persisted_history(chat_service: ChatService, chat_id: str) -> List[Dict[str, Any]]: + """The model-facing history a successor or recovery run starts from. + + Read back from the rows already committed in this transaction — the step + record included — never rebuilt from text the source run held in memory. + """ + from core.chat.compaction.service import _load_history + + return writer._json_safe(_load_history(chat_service, chat_id, repair=False)) + + +def _commit_queued_handoff_in_session( + db, + *, + source_run_id: str, + chat_id: str, + user_id: str, + context: Dict[str, Any], + model_name: Optional[str], +) -> Optional[Dict[str, Any]]: + """Atomically consume one terminal queued item and stage its ChatRun. + + This includes a steer accepted after the source run crossed its final safe + model/tool boundary. Such an instruction can no longer enter the source + ReAct loop, so completion hands it to a successor instead of stranding it. + + The successor's history is read back from the rows this transaction just + committed — the step record included — never rebuilt from the text the + source run happened to hold in memory. + """ + from core.db.models.chat import reserve_chat_sequences + from core.chat.steer_queue import SteerQueue + + source_run = db.get(models.ChatRun, source_run_id) + if source_run is None: + return None + root_run_id = str(context.get("handoff_root_run_id") or source_run_id) + handoff = SteerQueue.consume_next_handoff_in_session( + db, + source_run_id=source_run_id, + root_run_id=root_run_id, + chat_id=chat_id, + applied_operation_seq=int(source_run.operation_seq or 0) + 1, + ) + if handoff is None or not handoff.applied_run_id: + return None + + first_seq = reserve_chat_sequences( + db, + chat_id, + count=2, + owner_user_id=user_id, + ) + next_run_id = handoff.applied_run_id + next_message_id = ( + f"msg_{uuid.uuid5(uuid.NAMESPACE_URL, f'{handoff.queue_id}:assistant').hex[:16]}" + ) + queued_user_message_id = ( + f"msg_{uuid.uuid5(uuid.NAMESPACE_URL, f'{handoff.queue_id}:user').hex[:16]}" + ) + next_context = writer._json_safe(dict(context or {})) + next_context.pop("journal_owner", None) + next_context.update( + { + "run_id": next_run_id, + "message_id": next_message_id, + "chat_id": chat_id, + "user_id": user_id, + "handoff_queue_id": handoff.queue_id, + "handoff_delivery_mode": handoff.delivery_mode, + "handoff_root_run_id": root_run_id, + } + ) + next_session_messages = _persisted_history(services_ChatService_module.ChatService(db), chat_id) + next_session_messages.append({"role": "user", "content": handoff.message}) + worker_args = { + "session_messages": next_session_messages, + "effective_user_message": handoff.message, + "raw_user_message": handoff.message, + "context": next_context, + "model_name": model_name, + } + request_payload = dict(source_run.request_payload or {}) + request_payload.update( + { + "message": handoff.message, + "chat_id": chat_id, + "attachments": [], + "handoff": { + "queue_id": handoff.queue_id, + "steer_id": handoff.steer_id, + "delivery_mode": handoff.delivery_mode, + "source_run_id": source_run_id, + }, + } + ) + run_journal.RunJournal.accept_in_session( + db, + run_id=next_run_id, + message_id=next_message_id, + chat_id=chat_id, + user_id=user_id, + request_payload=writer._json_safe(request_payload), + recovery_snapshot={"kind": "chat", "worker_args": writer._json_safe(worker_args)}, + user_message_id=queued_user_message_id, + user_chat_seq=first_seq, + assistant_chat_seq=first_seq + 1, + writer_slot="main", + ) + services_ChatService_module.ChatService(db).upsert_message( + chat_id=chat_id, + role="user", + content=handoff.message, + model=model_name, + message_id=queued_user_message_id, + chat_seq=first_seq, + extra_data={ + "queued_handoff": True, + "steer_queue_id": handoff.queue_id, + "steer_id": handoff.steer_id, + "steer_seq": handoff.steer_seq, + "delivery_mode": handoff.delivery_mode, + "source_run_id": source_run_id, + "run_id": next_run_id, + }, + commit=False, + ) + db.add( + inflight.new_assistant_row( + message_id=next_message_id, + chat_id=chat_id, + chat_seq=first_seq + 1, + run_id=next_run_id, + model=model_name, + created_at=clock._utcnow(), + ) + ) + db.flush() + return { + "run_id": next_run_id, + "message_id": next_message_id, + "user_message_id": queued_user_message_id, + "assistant_chat_seq": first_seq + 1, + "queue_id": handoff.queue_id, + "steer_id": handoff.steer_id, + "steer_seq": handoff.steer_seq, + "delivery_mode": handoff.delivery_mode, + **worker_args, + } + + +def _schedule_queued_handoff( + handoff: Dict[str, Any], + *, + chat_id: str, + user_id: str, + task_prefix: str = "chat_run_handoff", +) -> str: + """Start a committed handoff; pending DB state remains recovery-safe on failure.""" + next_run_id = str(handoff["run_id"]) + next_owner = ownership.new_worker_owner(next_run_id) + runtime.register_run_task( + next_run_id, + worker._run_workflow( + run_id=next_run_id, + chat_id=chat_id, + user_id=user_id, + message_id=str(handoff["message_id"]), + assistant_chat_seq=( + int(handoff["assistant_chat_seq"]) + if handoff.get("assistant_chat_seq") is not None + else None + ), + session_messages=list(handoff.get("session_messages") or []), + effective_user_message=str(handoff.get("effective_user_message") or ""), + raw_user_message=str(handoff.get("raw_user_message") or ""), + context=dict(handoff.get("context") or {}), + model_name=(str(handoff["model_name"]) if handoff.get("model_name") else None), + journal_owner=next_owner, + ), + name=f"{task_prefix}:{next_run_id}", + ) + return next_run_id + + +def _queued_run_started_event( + source_run_id: str, + chat_id: str, + handoff: Dict[str, Any], +) -> Dict[str, Any]: + return { + "type": "queued_run_started", + "chat_id": chat_id, + "source_run_id": source_run_id, + "run_id": str(handoff["run_id"]), + "message_id": handoff["message_id"], + "user_message_id": handoff["user_message_id"], + "message": handoff["raw_user_message"], + "queue_id": handoff["queue_id"], + "steer_id": handoff["steer_id"], + "steer_seq": handoff["steer_seq"], + "delivery_mode": handoff["delivery_mode"], + } + + +async def _write_queued_run_started_projection( + source_run_id: str, + *, + chat_id: str, + handoff: Dict[str, Any], +) -> None: + """Best-effort event projection; the queue API is the durable backfill seam.""" + try: + offset = ownership.journal().allocate_event_offset(source_run_id, terminal=True) + await writer.xadd_event( + source_run_id, + offset, + _queued_run_started_event(source_run_id, chat_id, handoff), + ) + except Exception as exc: # Redis projection must not undo the committed handoff + logger.warning( + "chat_run_handoff_projection_failed", + run_id=source_run_id, + child_run_id=handoff.get("run_id"), + error=str(exc), + ) + + +def get_successor_run(run_id: str, *, user_id: str) -> Optional[ChatRun]: + """Read the successor committed by the durable steering handoff.""" + from core.db.models import ChatSteerQueueItem + + with engine.SessionLocal() as db: + source = db.get(models.ChatRun, run_id) + if source is None or source.user_id != user_id: + raise runtime.ChatRunPermissionDenied("run does not belong to user") + successor_id = ( + db.query(ChatSteerQueueItem.applied_run_id) + .filter( + ChatSteerQueueItem.applied_source_run_id == run_id, + ChatSteerQueueItem.applied_run_id != run_id, + ChatSteerQueueItem.status == "applied", + ChatSteerQueueItem.chat_id == source.chat_id, + ChatSteerQueueItem.user_id == user_id, + ) + .scalar() + ) + return runtime.get_run(successor_id) if successor_id else None diff --git a/src/backend/orchestration/job_wakeup.py b/src/backend/core/chat/runtime/job_wakeup.py similarity index 94% rename from src/backend/orchestration/job_wakeup.py rename to src/backend/core/chat/runtime/job_wakeup.py index ab09d17e..71294941 100644 --- a/src/backend/orchestration/job_wakeup.py +++ b/src/backend/core/chat/runtime/job_wakeup.py @@ -27,16 +27,12 @@ import logging from typing import Any, Dict, Optional +import core.chat.runtime.admission as admission from core.chat.plan_progress import load_plan_progress, plan_is_unfinished, render_plan_checklist from core.db.engine import SessionLocal from core.db.models import Job -from core.llm.context_ir import ( - KIND_REMINDER, - POLICY_NEVER, - SESSION_CONTEXT_META_KEY, - make_text_context_item, - session_context_metadata, -) +from core.llm.context.ir_constants import KIND_REMINDER, POLICY_NEVER, SESSION_CONTEXT_META_KEY +from core.llm.context.items import make_text_context_item, session_context_metadata from sqlalchemy.orm.attributes import flag_modified logger = logging.getLogger(__name__) @@ -163,13 +159,14 @@ async def wake_on_job_progress( prompt = _progress_prompt(job, stats, budget_left, stalled) try: - from orchestration.chat_run_executor import get_active_run_for_chat + from core.runs.runtime import get_active_run_for_chat if get_active_run_for_chat(chat_id, user_id) is not None: logger.info("[job-wake] skip progress wake, chat busy chat=%s job=%s", chat_id, job_id) return False - except Exception: # noqa: BLE001 —— 探测失败就当它不忙，宁可多叫一次也别哑掉 - pass + except Exception as fallback_error: # noqa: BLE001 —— 探测失败就当它不忙，宁可多叫一次也别哑掉 + from core.infra.failures import record_best_effort_failure + record_best_effort_failure('core.chat.runtime.job_wakeup:wake_on_job_progress', fallback_error) try: await _enqueue_followup_run( @@ -204,7 +201,7 @@ async def wake_on_job_finish(job_id: str) -> bool: user_id = job.user_id model_name = start_params.get("model_name") - from core.services.job_service import JobService + from core.jobs.store import JobService stats = JobService(db).stats(job_id) prompt = _wake_prompt(job, stats, load_plan_progress(chat_id, db)) @@ -241,12 +238,11 @@ async def _enqueue_followup_run( 这样前端不用任何改动：它本来就会在会话里看到新的一轮（断线还能按 run_id 续播）。 """ - from core.services.session_operations.session_context import _load_session_messages from core.chat.context import build_runtime_context - from core.config.catalog_resolver import resolve_all_runtime_enabled - from core.services.chat_sequencer import ChatSequencer - from core.services.chat_service import ChatService - from orchestration import chat_run_executor + from core.capabilities.catalog.resolver import resolve_all_runtime_enabled + from core.chat.sequencer import ChatSequencer + from core.chat.service import ChatService + from core.chat.operations.session_context import load_session_messages as _load_session_messages accepted_run_id: Optional[str] = None try: @@ -303,7 +299,7 @@ async def _enqueue_followup_run( enabled_mcps=mcps, ) context["workflow_chat"] = workflow_chat - await chat_run_executor.start_run( + await admission.start_run( accepted_run=accepted.run, chat_id=chat_id, user_id=user_id, diff --git a/src/backend/core/chat/runtime/pauses.py b/src/backend/core/chat/runtime/pauses.py new file mode 100644 index 00000000..e3b2e628 --- /dev/null +++ b/src/backend/core/chat/runtime/pauses.py @@ -0,0 +1,58 @@ +"""Persist tool-outcome and human-hook pauses without reporting success.""" + +from __future__ import annotations + +import core.runs.runtime as runtime +import core.runs.settings as settings +from core.harness.hooks import HookPaused +from core.infra.logging import get_logger + +logger = get_logger(__name__) + + +async def _pause_for_tool_outcome(turn, exc: BaseException) -> bool: + logger.warning("chat_run_tool_outcome_unknown", run_id=turn.run_id, effect_id=str(exc)) + if not turn.journal.needs_attention( + turn.run_id, + owner=turn.owner, + reason=f"tool outcome pending policy recovery: {exc}", + ): + current = runtime.get_run(turn.run_id) + if current is None or current.status != "needs_attention": + logger.warning("chat_run_tool_unknown_cas_lost", run_id=turn.run_id, owner=turn.owner) + return False + await turn.emit( + { + "type": "error", + "error": "工具调用结果无法安全确定，任务已暂停等待处理", + "delta": "工具调用结果无法安全确定，任务已暂停等待处理", + "chat_id": turn.chat_id, + }, + terminal=True, + ) + await turn.emit({"type": settings.TERMINAL_TYPE, "chat_id": turn.chat_id}, terminal=True) + return True + + +async def _pause_for_hook(turn, exc: HookPaused) -> bool: + logger.info("chat_run_hook_paused", run_id=turn.run_id, reason=str(exc)) + if not turn.journal.needs_attention( + turn.run_id, + owner=turn.owner, + reason=f"hook requested pause: {exc}", + ): + current = runtime.get_run(turn.run_id) + if current is None or current.status != "needs_attention": + logger.warning("chat_run_hook_pause_cas_lost", run_id=turn.run_id, owner=turn.owner) + return False + await turn.emit( + { + "type": "error", + "error": "任务已按执行策略暂停，等待后续处理", + "delta": "任务已按执行策略暂停，等待后续处理", + "chat_id": turn.chat_id, + }, + terminal=True, + ) + await turn.emit({"type": settings.TERMINAL_TYPE, "chat_id": turn.chat_id}, terminal=True) + return True diff --git a/src/backend/core/chat/runtime/recovery.py b/src/backend/core/chat/runtime/recovery.py new file mode 100644 index 00000000..40a1c570 --- /dev/null +++ b/src/backend/core/chat/runtime/recovery.py @@ -0,0 +1,176 @@ +"""Recover chat workers from committed snapshots after process loss.""" + +from __future__ import annotations + +import core.chat.runtime.recovery_snapshot as recovery_snapshot +import core.chat.runtime.worker as worker +import core.db.engine as engine +import core.db.models as models +import core.runs.events.writer as writer +import core.runs.ownership as ownership +import core.runs.runtime as runtime +import core.runs.settings as settings +import core.runs.tasks as tasks +from core.infra.logging import get_logger +from core.runs.journal import RecoveryDecision + +logger = get_logger(__name__) + + +async def recover_orphan_runs() -> int: + """Recover claimable runs from their last committed database safe point.""" + from core.runs.tool_effects.journal import ToolEffectJournal + from core.runs.tool_effects.recovery import recover_incomplete_tool_effects + + # Desktop channel runs require a live device lease and a scoped I/O adapter. + # Neither survives process restart. Fail closed before any tool-effect recovery. + with engine.SessionLocal() as db: + local_channel_runs = [ + (run.run_id, run.chat_id) + for run in db.query(models.ChatRun) + .filter(models.ChatRun.status.in_(settings.LIVE_STATUSES)) + .all() + if (run.request_payload or {}).get("source") == "desktop_channel" + and run.run_id not in tasks.process_state().active_runs + ] + for run_id, chat_id in local_channel_runs: + if ownership.journal().cancel( + run_id, reason="desktop channel lease cannot survive process restart" + ): + await writer._write_terminal_to_stream( + run_id, + chat_id=chat_id, + error_text="本机机器人执行已中断，请检查结果后从渠道重新发起", + cancelled=True, + ) + effect_decisions = await recover_incomplete_tool_effects( + journal=ToolEffectJournal(engine.SessionLocal) + ) + effect_attention_runs = { + item.run_id for item in effect_decisions if item.action == "needs_attention" + } + for run_id in effect_attention_runs: + run = runtime.get_run(run_id) + if run is not None: + await writer._write_terminal_to_stream( + run_id, + chat_id=run.chat_id or "", + error_text="工具调用结果无法安全确定，任务已暂停等待处理", + ) + decisions = ownership.journal().recover() + for decision in decisions: + if decision.action == "resume": + if not _register_recovered_chat(decision): + reason = "run kind or recovery snapshot cannot be resumed automatically" + if ownership.journal().needs_attention(decision.run_id, reason=reason): + await writer._write_terminal_to_stream( + decision.run_id, + chat_id=decision.chat_id or "", + error_text="任务缺少可恢复快照，已暂停等待处理", + ) + elif decision.action == "resume_from_snapshot": + snapshot_result = await recovery_snapshot._commit_recovered_chat_snapshot(decision) + if snapshot_result is False: + await writer._write_terminal_to_stream( + decision.run_id, + chat_id=decision.chat_id or "", + error_text="已保存的模型结果无法安全提交，任务已暂停等待处理", + ) + elif decision.action == "needs_attention": + if decision.phase == "model_inflight": + recovery_error = "模型调用结果不确定，任务已暂停等待恢复决策" + elif decision.phase == "legacy_unrecoverable": + recovery_error = "旧版本任务缺少恢复快照，已暂停等待处理" + else: + recovery_error = "任务在工具结果不确定的安全边界暂停，等待恢复决策" + await writer._write_terminal_to_stream( + decision.run_id, + chat_id=decision.chat_id or "", + error_text=recovery_error, + ) + elif decision.action == "completed": + await recovery_snapshot._write_recovered_terminal_marker( + decision.run_id, + chat_id=decision.chat_id or "", + ) + elif decision.action == "failed": + await writer._write_terminal_to_stream( + decision.run_id, + chat_id=decision.chat_id or "", + error_text="服务重启发生在任务准备完成前，请重新发起", + ) + recovered_count = len( + {item.run_id for item in decisions} + | effect_attention_runs + | {run_id for run_id, _ in local_channel_runs} + ) + if recovered_count: + logger.info("chat_run_orphan_recovered", count=recovered_count) + return recovered_count + + +def _register_recovered_chat(decision: RecoveryDecision) -> bool: + snapshot = dict(decision.snapshot or {}) + if snapshot.get("kind") != "chat": + return False + args = snapshot.get("worker_args") + if not isinstance(args, dict): + return False + required = { + "session_messages", + "effective_user_message", + "raw_user_message", + "context", + } + if not required.issubset(args): + return False + owner = ownership.new_worker_owner(decision.run_id) + recovered_run = runtime.get_run(decision.run_id) + if recovered_run and (recovered_run.request_payload or {}).get("source") == "desktop_channel": + return False + from core.runs.api_recovery import validate_recovery_context + from sqlalchemy.exc import SQLAlchemyError + + try: + with engine.SessionLocal() as db: + restored_context = validate_recovery_context( + db, + recovered_run, + args.get("context"), + user_id=decision.user_id, + chat_id=decision.chat_id, + ) + if "context" in snapshot: + validate_recovery_context( + db, + recovered_run, + snapshot["context"], + user_id=decision.user_id, + chat_id=decision.chat_id, + ) + except (ValueError, SQLAlchemyError): + logger.warning("chat_run_recovery_scope_rejected", run_id=decision.run_id) + return False + runtime.register_run_task( + decision.run_id, + worker._run_workflow( + run_id=decision.run_id, + chat_id=decision.chat_id, + user_id=decision.user_id, + message_id=decision.message_id, + assistant_chat_seq=( + int(recovered_run.assistant_chat_seq) + if recovered_run is not None and recovered_run.assistant_chat_seq is not None + else None + ), + session_messages=list(args.get("session_messages") or []), + effective_user_message=str(args.get("effective_user_message") or ""), + raw_user_message=str(args.get("raw_user_message") or ""), + context=restored_context, + model_name=(str(args["model_name"]) if args.get("model_name") else None), + journal_owner=owner, + recovering=True, + ), + name=f"chat_run_recovery:{decision.run_id}", + ) + return True diff --git a/src/backend/core/chat/runtime/recovery_snapshot.py b/src/backend/core/chat/runtime/recovery_snapshot.py new file mode 100644 index 00000000..c828cc05 --- /dev/null +++ b/src/backend/core/chat/runtime/recovery_snapshot.py @@ -0,0 +1,201 @@ +"""Persist interrupted turn snapshots and recovery terminal markers.""" + +from __future__ import annotations + +import typing as _typing +from typing import Optional + +import core.chat.runtime.handoff as handoff +import core.db.engine as engine +import core.runs.events.writer as writer +import core.runs.ownership as ownership +import core.runs.runtime as runtime +import core.runs.settings as settings +import core.chat.service as services_ChatService_module +from core.infra.logging import get_logger +from core.runs.journal import RecoveryDecision + +logger = get_logger(__name__) + + +async def _commit_recovered_chat_snapshot(decision: RecoveryDecision) -> Optional[bool]: + snapshot = dict(decision.snapshot or {}) + content = snapshot.get("assistant_content") + message_id = str(snapshot.get("message_id") or decision.message_id) + if not isinstance(content, str) or not message_id: + paused = ownership.journal().needs_attention( + decision.run_id, + reason="model-completed snapshot is incomplete", + ) + return False if paused else None + owner = ownership.new_worker_owner(decision.run_id) + recovered_run = runtime.get_run(decision.run_id) + journal = ownership.journal() + from core.runs.api_recovery import validate_recovery_context + from sqlalchemy.exc import SQLAlchemyError + + try: + with engine.SessionLocal() as db: + restored_context = validate_recovery_context( + db, + recovered_run, + snapshot.get("context"), + user_id=decision.user_id, + chat_id=decision.chat_id, + ) + worker_args = snapshot.get("worker_args") + if isinstance(worker_args, dict) and "context" in worker_args: + with engine.SessionLocal() as db: + validate_recovery_context( + db, + recovered_run, + worker_args["context"], + user_id=decision.user_id, + chat_id=decision.chat_id, + ) + except (ValueError, SQLAlchemyError): + paused = journal.needs_attention( + decision.run_id, reason="agent API recovery scope is unavailable or inconsistent" + ) + return False if paused else None + if not journal.claim( + decision.run_id, + owner=owner, + lease_seconds=settings.RUN_LEASE_SECONDS, + ): + return None + try: + from core.files.artifacts.service import persist_artifacts as _persist_artifacts + from core.projects.edition import project_scope_from_context + + queued_handoff: _typing.Dict[str, _typing.Any] = {} + + def _commit_recovered_output(db) -> None: + services_ChatService_module.ChatService(db).upsert_message( + chat_id=decision.chat_id, + role="assistant", + content=content, + message_id=message_id, + chat_seq=( + int(recovered_run.assistant_chat_seq) + if recovered_run is not None and recovered_run.assistant_chat_seq is not None + else None + ), + model=( + str(snapshot["model_name"]) if snapshot.get("model_name") is not None else None + ), + thinking=( + list(snapshot["thinking"]) + if isinstance(snapshot.get("thinking"), list) + else None + ), + tool_calls=( + list(snapshot["tool_calls"]) + if isinstance(snapshot.get("tool_calls"), list) + else None + ), + model_steps=( + list(snapshot["model_steps"]) + if isinstance(snapshot.get("model_steps"), list) + else None + ), + usage=( + dict(snapshot["usage"]) if isinstance(snapshot.get("usage"), dict) else None + ), + extra_data=( + dict(snapshot["extra_data"]) + if isinstance(snapshot.get("extra_data"), dict) + else {} + ), + commit=False, + ) + artifacts = snapshot.get("artifacts") + if isinstance(artifacts, list): + _persist_artifacts( + db, + decision.user_id, + decision.chat_id, + artifacts, + scope=project_scope_from_context(restored_context), + commit=False, + ) + worker_args = snapshot.get("worker_args") + recovery_args = worker_args if isinstance(worker_args, dict) else {} + context_snapshot = snapshot.get("context") + prepared = handoff._commit_queued_handoff_in_session( + db, + source_run_id=decision.run_id, + chat_id=decision.chat_id, + user_id=decision.user_id, + context=( + restored_context + if isinstance(context_snapshot, dict) + else dict(recovery_args.get("context") or {}) + ), + model_name=( + str(snapshot["model_name"]) + if snapshot.get("model_name") is not None + else ( + str(recovery_args["model_name"]) + if recovery_args.get("model_name") is not None + else None + ) + ), + ) + if prepared is not None: + queued_handoff.update(prepared) + + completed = journal.complete( + decision.run_id, + owner=owner, + status="completed", + usage=(dict(snapshot["usage"]) if isinstance(snapshot.get("usage"), dict) else None), + commit_effect=_commit_recovered_output, + committed_operation={ + "operation_type": "message_committed_from_snapshot", + "phase": "message_committed", + "safety": "side_effect_committed", + "payload": {"message_id": message_id}, + }, + ) + if completed: + if queued_handoff: + handoff._schedule_queued_handoff( + queued_handoff, + chat_id=decision.chat_id, + user_id=decision.user_id, + task_prefix="chat_run_handoff_recovery", + ) + await handoff._write_queued_run_started_projection( + decision.run_id, + chat_id=decision.chat_id, + handoff=queued_handoff, + ) + await _write_recovered_terminal_marker( + decision.run_id, + chat_id=decision.chat_id, + ) + return completed + except Exception as exc: # noqa: BLE001 - pause instead of replaying model output + logger.error( + "chat_run_snapshot_commit_failed", + run_id=decision.run_id, + error=str(exc), + exc_info=True, + ) + paused = journal.needs_attention( + decision.run_id, + reason=str(exc)[:1000], + owner=owner, + ) + return False if paused else None + + +async def _write_recovered_terminal_marker(run_id: str, *, chat_id: str) -> None: + event = {"type": settings.TERMINAL_TYPE, "chat_id": chat_id, "_recovered": True} + try: + offset = ownership.journal().allocate_event_offset(run_id, terminal=True) + await writer.xadd_event(run_id, offset, event) + await writer.expire_stream(run_id) + except Exception as exc: # Redis is only a projection + logger.warning("chat_run_recovered_terminal_write_failed", run_id=run_id, error=str(exc)) diff --git a/src/backend/core/chat/runtime/snapshots.py b/src/backend/core/chat/runtime/snapshots.py new file mode 100644 index 00000000..1924dd8b --- /dev/null +++ b/src/backend/core/chat/runtime/snapshots.py @@ -0,0 +1,201 @@ +"""Checkpoint visible turn output and preserve partial output on cancellation.""" + +from __future__ import annotations + +import asyncio +import functools +import time +from typing import Any, Dict, List, Optional + +import core.chat.inflight as inflight +import core.chat.message_parser as message_parser +import core.db.engine as engine +import core.llm.runtime.model_steps as model_steps +import core.runs.events.writer as writer +import core.chat.service as services_ChatService_module +from core.chat.context import now_iso +from core.infra.text_bounds import bound_result_for_history +from core.infra.logging import get_logger +from core.llm.runtime import workspace as _workspace_mod +from core.files.artifacts.service import persist_artifacts as _persist_artifacts + +logger = get_logger(__name__) + + +def _flush_thinking(turn) -> None: + if not turn.thinking_parts: + return + block = "".join(turn.thinking_parts) + turn.thinking_parts.clear() + # 结构化 reasoning 的尾部增量可能在正文已开始后才到达（正文首 token + # 已出、思考收尾的"。"后到）。实时侧把它并回前一个思考块 + # （appendThinkingContentBeforeTrailingText），落库遵循同一规则—— + # 否则思考块会把正文句子从中间切开，刷新后与实时展示不一致。 + if turn.round_block is not None: + turn.round_block["content"] += block + return + turn.round_block = {"content": block} + turn.thinking_log.append(turn.round_block) + turn.segments.add_thinking(len(turn.thinking_log) - 1) + + +def _thinking_payload(turn) -> Optional[List[Dict[str, Any]]]: + blocks = [b for b in turn.thinking_log if b.get("content")] + return blocks or None + + +def _model_steps_payload(turn) -> Optional[List[Dict[str, Any]]]: + return list(turn.model_steps) or None + + +def _turn_extra(turn, **more: Any) -> Dict[str, Any]: + return { + "timestamp": now_iso(), + "message_id": turn.current_message_id, + "run_id": turn.run_id, + "is_markdown": message_parser.looks_markdown(turn.full_response), + **more, + } + + +def _write_inflight_row(turn, snapshot: Dict[str, Any]) -> bool: + with engine.SessionLocal() as db: + owned = services_ChatService_module.ChatService(db).refresh_streaming_message( + run_id=turn.run_id, + owner=turn.owner, + message_id=snapshot["message_id"], + content=snapshot["content"], + thinking=snapshot["thinking"], + tool_calls=snapshot["tool_calls"], + model_steps=snapshot["model_steps"], + extra_data={ + inflight.IN_FLIGHT_KEY: inflight.mark( + turn.run_id, "streaming", event_offset=snapshot["event_offset"] + ), + "message_id": snapshot["message_id"], + "run_id": turn.run_id, + "is_markdown": message_parser.looks_markdown(snapshot["content"]), + "segments": snapshot["segments"], + }, + ) + db.commit() + return owned + + +async def _flush_inflight(turn) -> bool: + _flush_thinking(turn) + if not (turn.full_response or turn.tool_calls_log or turn.model_steps or turn.thinking_log): + return True + async with turn.flush_lock: + # Canonical steps can advance without a user-visible SSE event. + snapshot_key = (turn.current_message_id, turn.offsets.last, len(turn.model_steps)) + if snapshot_key == turn.flushed_snapshot_key: + return True + # Tool results go in at history size; the read path bounds them the + # same way, and the final write carries the full payload. + snapshot = writer._json_safe( + { + "content": turn.full_response, + "thinking": _thinking_payload(turn), + "model_steps": _model_steps_payload(turn), + "tool_calls": [ + ( + {**tc, "result": bound_result_for_history(tc["result"])[0]} + if "result" in tc + else tc + ) + for tc in turn.tool_calls_log + ] + or None, + "segments": turn.segments.snapshot(), + "message_id": turn.current_message_id, + "event_offset": turn.offsets.last, + } + ) + try: + if not await asyncio.to_thread(functools.partial(_write_inflight_row, turn), snapshot): + logger.warning("chat_run_inflight_fenced", run_id=turn.run_id, owner=turn.owner) + return False + except Exception: # noqa: BLE001 - a missed checkpoint must not fail the run + logger.warning("chat_run_inflight_persist_failed", run_id=turn.run_id, exc_info=True) + return False # Keep dirty so the periodic checkpoint retries this snapshot. + turn.flushed_snapshot_key = snapshot_key + return True + + +async def _checkpoint_loop(turn) -> None: + while True: + await asyncio.sleep(turn.inflight_interval_s) + await _flush_inflight(turn) + + +def _commit_cancelled_partial(turn) -> None: + """把用户按下停止那一刻已产出的正文与工具卡片落库。 + + cancel_run 先写终态并清租约，之后 journal.complete 的 CAS 必然失败， + worker 再也借不到 commit_effect 提交本轮输出 —— 所以这里直接写库。 + message_id 固定，用 upsert 保证重复取消 / 恢复不会留下两条。 + """ + _flush_thinking(turn) + if not (turn.full_response or turn.tool_calls_log or turn.model_steps or turn.thinking_log): + return + # 停下来时还没回结果的工具：不标记的话前端按"缺 status 即成功"渲染， + # 刷新后一张没跑完的卡片会显示成执行成功。模型侧的记录同样要配平： + # 悬空的调用补一条"被中断"的结果，下一轮模型看到的是事实而不是半步。 + for _tc in turn.tool_calls_log: + if "result" not in _tc and not _tc.get("status"): + _tc["status"] = "interrupted" + model_steps.close_dangling_calls(turn.model_steps) + _ws_pinned = _workspace_mod.get_pinned() + _extra = { + "timestamp": now_iso(), + "is_markdown": bool( + "\n" in turn.full_response or "```" in turn.full_response or "**" in turn.full_response + ), + "message_id": turn.current_message_id, + "run_id": turn.run_id, + "cancelled": True, + "artifacts": _ws_pinned, + "workspace_files": _workspace_mod.get_pinned_file_ids(), + "duration_ms": int((time.monotonic() - turn.run_started_monotonic) * 1000), + "segments": turn.segments.payload(), + } + if turn.context.get("model_provider_id"): + _extra["model_provider_id"] = turn.context.get("model_provider_id") + from core.projects.edition import project_scope_from_context + + with engine.SessionLocal() as db: + chat_service = services_ChatService_module.ChatService(db) + chat_service.upsert_message( + chat_id=turn.chat_id, + role="assistant", + content=turn.full_response, + model=turn.model_name, + thinking=_thinking_payload(turn), + tool_calls=turn.tool_calls_log if turn.tool_calls_log else None, + model_steps=_model_steps_payload(turn), + message_id=turn.current_message_id, + chat_seq=turn.current_chat_seq, + extra_data=_extra, + commit=False, + ) + _persist_artifacts( + db, + turn.user_id, + turn.chat_id, + _ws_pinned, + scope=project_scope_from_context(turn.context), + commit=False, + ) + db.commit() + + +def _persist_cancelled_partial(turn) -> None: + try: + _commit_cancelled_partial(turn) + except Exception: # noqa: BLE001 - 落库失败不能吞掉取消收尾 + logger.warning( + "chat_run_cancelled_partial_persist_failed", + run_id=turn.run_id, + exc_info=True, + ) diff --git a/src/backend/core/chat/runtime/steering.py b/src/backend/core/chat/runtime/steering.py new file mode 100644 index 00000000..85ac6b62 --- /dev/null +++ b/src/backend/core/chat/runtime/steering.py @@ -0,0 +1,199 @@ +"""Apply queued steering while preserving ordered message and event commits.""" + +from __future__ import annotations + +import time +import uuid + +import core.chat.inflight as inflight +import core.chat.runtime.handoff as handoff +import core.db.models as models +import core.runs.clock as clock +import core.runs.events.writer as writer +import core.chat.service as services_ChatService_module +from core.chat.context import now_iso +from core.infra.logging import get_logger + + +from core.chat.runtime import snapshots + +logger = get_logger(__name__) + + +async def apply_steer(turn, chunk): + snapshots._flush_thinking(turn) + steer_id = str(chunk.get("steer_id") or "")[:64] + steer_queue_id = str(chunk.get("queue_id") or "")[:64] + steer_claim_owner = str(chunk.get("claim_owner") or "")[:160] + steer_seq = int(chunk.get("steer_seq") or 0) + steer_message = str(chunk.get("message") or "").strip() + steer_message_id = ( + f"msg_{uuid.uuid5(uuid.NAMESPACE_URL, f'{turn.run_id}:{steer_id}').hex[:16]}" + if steer_id + else f"msg_{uuid.uuid4().hex[:16]}" + ) + next_assistant_message_id = ( + f"msg_{uuid.uuid5(uuid.NAMESPACE_URL, f'{turn.run_id}:{steer_id}:assistant').hex[:16]}" + if steer_id + else f"msg_{uuid.uuid4().hex[:16]}" + ) + had_assistant_output = bool(turn.full_response or turn.tool_calls_log) + if steer_message: + # Close the visible assistant segment before inserting the + # user's steer. Persisting in this order is what keeps a + # refresh from moving every mid-run user message above the + # whole assistant response. + def _commit_steer(db) -> None: + if steer_queue_id and steer_claim_owner: + from core.chat.steer_queue import SteerQueue + + owned_run = db.get(models.ChatRun, turn.run_id) + SteerQueue.mark_applied_in_session( + db, + queue_id=steer_queue_id, + claim_owner=steer_claim_owner, + applied_run_id=turn.run_id, + applied_operation_seq=int(owned_run.operation_seq or 0) + 1, + ) + else: + owned_run = db.get(models.ChatRun, turn.run_id) + chat_service = services_ChatService_module.ChatService(db) + if had_assistant_output: + chat_service.upsert_message( + chat_id=turn.chat_id, + role="assistant", + content=turn.full_response, + model=turn.model_name, + thinking=snapshots._thinking_payload(turn), + tool_calls=turn.tool_calls_log if turn.tool_calls_log else None, + model_steps=snapshots._model_steps_payload(turn), + message_id=turn.current_message_id, + chat_seq=turn.current_chat_seq, + extra_data={ + "timestamp": now_iso(), + "is_markdown": bool( + "\n" in turn.full_response + or "```" in turn.full_response + or "**" in turn.full_response + ), + "message_id": turn.current_message_id, + "run_id": turn.run_id, + "steer_segment": True, + "duration_ms": int((time.monotonic() - turn.run_started_monotonic) * 1000), + "segments": turn.segments.payload(), + }, + commit=False, + ) + # Persist once the middleware has actually injected the + # instruction. A deterministic id makes replay safe. + chat_service.upsert_message( + chat_id=turn.chat_id, + role="user", + content=steer_message, + message_id=steer_message_id, + extra_data={ + "timestamp": now_iso(), + "steer": True, + "run_id": turn.run_id, + "steer_id": steer_id, + "steer_queue_id": steer_queue_id or None, + "steer_seq": steer_seq or None, + }, + commit=False, + ) + # The next segment's row exists before its first token, + # like every other admitted turn. + db.add( + inflight.new_assistant_row( + message_id=next_assistant_message_id, + chat_id=turn.chat_id, + chat_seq=None, + run_id=turn.run_id, + model=turn.model_name, + created_at=clock._utcnow(), + ) + ) + # The durable queue acknowledgement and the exact + # post-steer prompt must share one transaction. If this + # worker disappears before the next model snapshot, a + # replacement worker resumes from this context instead + # of losing an already-applied instruction. + post_steer_messages = handoff._persisted_history(chat_service, turn.chat_id) + post_steer_context = writer._json_safe(dict(turn.context or {})) + post_steer_context.pop("journal_owner", None) + post_steer_context.update( + { + "run_id": turn.run_id, + "message_id": next_assistant_message_id, + "chat_id": turn.chat_id, + "user_id": turn.user_id, + } + ) + owned_run.recovery_snapshot = { + "kind": "chat", + "worker_args": { + "session_messages": post_steer_messages, + "effective_user_message": steer_message, + "raw_user_message": steer_message, + "context": post_steer_context, + "model_name": turn.model_name, + }, + } + owned_run.snapshot_version = int(owned_run.snapshot_version or 0) + 1 + + turn.journal.append_operation( + turn.run_id, + owner=turn.owner, + operation_type="steer_messages_committed", + phase="steer_committed", + safety="side_effect_committed", + payload={ + "steer_id": steer_id, + "queue_id": steer_queue_id or None, + "steer_seq": steer_seq or None, + "message_id": steer_message_id, + "previous_assistant_message_id": ( + turn.current_message_id if had_assistant_output else None + ), + }, + commit_effect=_commit_steer, + ) + turn.latest_user_message = steer_message + await turn.emit( + { + "type": "steer_applied", + "chat_id": turn.chat_id, + "run_id": turn.run_id, + "steer_id": steer_id, + "queue_id": steer_queue_id or None, + "steer_seq": steer_seq or None, + "message": steer_message, + "message_id": steer_message_id, + "previous_assistant_message_id": ( + turn.current_message_id if had_assistant_output else None + ), + "next_assistant_message_id": next_assistant_message_id, + "had_assistant_output": had_assistant_output, + } + ) + turn.current_message_id = next_assistant_message_id + turn.current_chat_seq = None + # The workflow reads this dict again when it emits the final + # evolution/memory settlement marker, so keep that marker bound + # to the post-steer assistant segment too. + turn.context["message_id"] = turn.current_message_id + turn.full_response = "" + turn.tool_calls_log = [] + turn.model_steps = [] + turn.thinking_parts.clear() + turn.round_block = None + turn.thinking_log.clear() + turn.segments.reset() + turn.metadata = {} + try: + from core.harness.log_context import set_current_message_id + + set_current_message_id(turn.current_message_id) + except Exception as fallback_error: # pragma: no cover - logging must never fail a run + from core.infra.failures import record_best_effort_failure + record_best_effort_failure('core.chat.runtime.steering:apply_steer', fallback_error) diff --git a/src/backend/core/chat/runtime/turn_state.py b/src/backend/core/chat/runtime/turn_state.py new file mode 100644 index 00000000..49eac43a --- /dev/null +++ b/src/backend/core/chat/runtime/turn_state.py @@ -0,0 +1,44 @@ +"""State owned by one chat worker; persistence callbacks share this exact turn.""" + +from __future__ import annotations + +import asyncio +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any, Awaitable, Callable + +if TYPE_CHECKING: + from core.chat.tool_log import SegmentRecorder + from core.runs.events.writer import EventOffsets + from core.runs.journal import RunJournal + + +@dataclass +class TurnState: + chat_id: str + context: dict[str, Any] + message_id: str + model_name: str + run_id: str + user_id: str + stopped: bool = False + inflight_interval_s: float = 2.0 + emit: Callable[[dict[str, Any]], Awaitable[Any]] | None = None + flush_lock: asyncio.Lock | None = None + flushed_snapshot_key: Any = None + model_steps: list[dict[str, Any]] = field(default_factory=list) + round_block: dict[str, Any] | None = None + run_started_monotonic: float = 0.0 + segments: SegmentRecorder | None = None + thinking_log: list[dict[str, Any]] = field(default_factory=list) + thinking_parts: list[str] = field(default_factory=list) + compaction_budget_inputs: dict[str, Any] = field(default_factory=dict) + current_chat_seq: int | None = None + current_message_id: str = "" + full_response: str = "" + journal: RunJournal | None = None + latest_user_message: str = "" + lease_task: asyncio.Task | None = None + metadata: dict[str, Any] = field(default_factory=dict) + offsets: EventOffsets | None = None + owner: str = "" + tool_calls_log: list[dict[str, Any]] = field(default_factory=list) diff --git a/src/backend/core/chat/runtime/worker.py b/src/backend/core/chat/runtime/worker.py new file mode 100644 index 00000000..09317713 --- /dev/null +++ b/src/backend/core/chat/runtime/worker.py @@ -0,0 +1,404 @@ +"""Own one chat worker lifecycle and coordinate its persistence phases.""" + +from __future__ import annotations +from core.runs.tool_effects import errors + +import asyncio +import contextlib +import datetime as _datetime +import time +import typing as _typing +from typing import Any, Dict, List, Optional + +from core.harness.registry.chat_execution import stream_chat_workflow +import core.db.engine as engine +import core.harness.hooks as hooks +import core.llm.runtime.model_steps as model_steps +import core.runs.clock as clock +import core.runs.events.writer as writer +import core.runs.ownership as ownership +import core.runs.runtime as runtime +import core.runs.settings as settings +import core.runs.watchdog as watchdog +import core.chat.service as services_ChatService_module +import core.runs.journal as run_journal +from core.infra.logging import get_logger + + +from core.chat.runtime import chunks, pauses, snapshots +from core.chat.runtime.turn_state import TurnState + +logger = get_logger(__name__) + + +async def _run_workflow( + *, + run_id: str, + chat_id: str, + user_id: str, + message_id: str, + session_messages: List[Dict[str, Any]], + effective_user_message: str, + raw_user_message: str, + context: Dict[str, Any], + model_name: Optional[str], + assistant_chat_seq: Optional[int] = None, + journal_owner: Optional[str] = None, + recovering: bool = False, +) -> None: + """Consume astream_chat_workflow, forward chunks to the Redis Stream, persist at the end. + + Every chunk is written to ``jx:chat:run:{run_id}:events`` via ``_emit``. + The SSE event format stays fully identical to the existing protocol in + chats.py:889-1087; the frontend needs no changes to its chunk parsing — + only the new ``run_started`` event type is added. + """ + turn = TurnState( + chat_id=chat_id, + context=context, + message_id=message_id, + model_name=model_name, + run_id=run_id, + user_id=user_id, + ) + from core.chat.context import resolve_user_facing_error + from core.chat.tool_log import SegmentRecorder + + # 本轮的墙钟起点。认领这一轮的 worker 手里就有它；只有接管别人未完成的 + # 那一轮时才需要回库里取（见 _run_started_at_ms）。 + claimed_at: _typing.Optional[_datetime.datetime] = None + initial_claimed = False + if not recovering: + claimed_at = ownership._claim_run_execution(turn.run_id) + initial_claimed = claimed_at is not None + if not initial_claimed: + ownership._acknowledge_never_started_terminal_writer(turn.run_id) + return + + turn.owner = journal_owner or ownership.new_worker_owner(turn.run_id) + turn.journal = ownership.journal() + if not turn.journal.claim( + turn.run_id, owner=turn.owner, lease_seconds=settings.RUN_LEASE_SECONDS + ): + logger.info("chat_run_claim_skipped", run_id=turn.run_id, owner=turn.owner) + if initial_claimed: + ownership._acknowledge_terminal_writer(turn.run_id) + return + try: + turn.journal.append_operation( + turn.run_id, + owner=turn.owner, + operation_type="worker_started", + phase="pre_model", + safety="replayable", + ) + except run_journal.RunLeaseLost: + logger.warning("chat_run_worker_fenced_before_start", run_id=turn.run_id, owner=turn.owner) + return + worker_task = asyncio.current_task() + if worker_task is None: # pragma: no cover - asyncio always owns this coroutine + raise RuntimeError("chat run worker has no asyncio task") + lease_lost = asyncio.Event() + turn.lease_task = asyncio.create_task( + ownership.lease_heartbeat(turn.run_id, turn.owner, worker_task, lease_lost), + name=f"chat_run_lease:{turn.run_id}", + ) + # 本轮回答总耗时起点 —— 持久化进 extra_data.duration_ms，历史加载后「用时」不再消失 + turn.run_started_monotonic = time.monotonic() + + turn.offsets = writer.EventOffsets( + turn.journal, turn.run_id, turn.owner, lease_lost=lease_lost + ) + + turn.emit = turn.offsets.emit + + from core.llm.runtime import workspace as _workspace_mod + + turn.full_response = "" + # A steer starts a new visible assistant segment in the same backend run. + # Each segment gets its own message id so persisted history stays in the + # same chronological order the user saw while streaming. + turn.current_message_id = turn.message_id + turn.current_chat_seq = assistant_chat_seq + turn.latest_user_message = raw_user_message + turn.metadata: _typing.Dict[str, _typing.Any] = {} + turn.compaction_budget_inputs: _typing.Dict[str, _typing.Any] = {} + turn.tool_calls_log: list = [] + _workspace_mod.init_state() + # 结构化 reasoning 通道（deepseek 系 reasoning_content/reasoning）的思考增量。 + # 落进独立的 thinking 列，不再拼进 full_response —— 拼进正文会把正文顶出 + # content 的 10 万字上限，截断切掉闭合标签后整段思考会漏成正文。 + # 每个思考块记录它出现时的正文偏移，历史重建按此原位还原。 + turn.thinking_parts: _typing.List[str] = [] + turn.thinking_log: _typing.List[_typing.Dict[str, _typing.Any]] = [] + # 模型回放用的规范记录：每个模型响应、每个工具结果按发生顺序各记一条 + # （core/llm/runtime/model_steps.py）。上面的正文 / 思考 / 工具卡是给界面看的投影， + # 下一轮发给模型的历史从这一份恢复。 + turn.model_steps: _typing.List[_typing.Dict[str, _typing.Any]] = [] + # 正文 / 思考 / 工具卡片的先后，在它们产生的那一刻就记下来，落进 + # metadata.segments；刷新后照着渲染，不做任何反推。 + turn.segments = SegmentRecorder() + # 本轮已开的思考块；跨轮（工具调用边界）置 None 强制另起一块。不设边界的话， + # 正文一旦出现，后面每一轮的思考都会并进同一块里无限膨胀。 + turn.round_block: _typing.Optional[_typing.Dict[str, _typing.Any]] = None + next_cancel_poll = 0.0 + + # In-flight persistence. The assistant row was created at admission and is + # refreshed with this turn's display state — right away at tool boundaries, + # and by the checkpoint task every couple of seconds while text streams — + # so the server holds the turn at all times, not only at the end. Each + # refresh is one lease-guarded UPDATE off the event loop; the lock keeps a + # boundary flush from racing the periodic one with an older snapshot. + turn.flush_lock = asyncio.Lock() + turn.flushed_snapshot_key = None + turn.inflight_interval_s = 2.0 + checkpoint_task: _typing.Optional[asyncio.Task] = None + + try: + if recovering: + await writer._reset_stream_projection(turn.run_id) + # First frame: run_started — carries run_id / message_id; the frontend uses these to resume / cancel + await turn.emit( + { + "type": "run_started", + "run_id": turn.run_id, + "message_id": turn.message_id, + "chat_id": turn.chat_id, + "started_at": ( + clock._epoch_ms(claimed_at) + if claimed_at + else clock._run_started_at_ms(turn.run_id) + ), + } + ) + if recovering: + await turn.emit( + { + "type": "content_replace", + "content": "", + "reason": "run_recovered", + "chat_id": turn.chat_id, + } + ) + + # Compaction notice (mirrors Codex's post-compaction Warning): after the + # previous turn's stream closed, a new compaction checkpoint was written + # in the background → notify the user once in this turn's first frame. + # Failures are silent and must never affect the main conversation. + try: + from core.chat.compaction.service import pop_compaction_notice + from core.chat.compaction.usage_projection import get_chat_context_state + + with engine.SessionLocal() as _cn_db: + _cn_service = services_ChatService_module.ChatService(_cn_db) + _notify_compaction = pop_compaction_notice(_cn_service, turn.chat_id) + _compaction_state = ( + get_chat_context_state(_cn_service, turn.chat_id)["context_compaction"] + if _notify_compaction + else None + ) + if _notify_compaction: + await turn.emit( + { + "type": "compaction_notice", + "chat_id": turn.chat_id, + "context_compaction": _compaction_state, + } + ) + except Exception as _cn_exc: # noqa: BLE001 + logger.debug("compaction_notice_failed", chat_id=turn.chat_id, error=str(_cn_exc)) + + # Evidence-plane join keys (GCE ticket 04). Injected here rather than at + # every context construction site: this is the one place that owns both + # the run and its pre-allocated assistant message id. + turn.context["run_id"] = turn.run_id + turn.context["message_id"] = turn.message_id + turn.context["journal_owner"] = turn.owner + + # Stamp the message id onto tool logging for this run. The plumbing + # already existed but had no caller, so every tool call was written with + # an empty message_id — which silently broke the join the evidence plane + # depends on, and left every episode with no tool sequence to learn from. + try: + from core.harness.log_context import set_current_message_id + + set_current_message_id(turn.message_id) + except Exception as fallback_error: # pragma: no cover - logging must never fail a run + from core.infra.failures import record_best_effort_failure + record_best_effort_failure('core.chat.runtime.worker:_run_workflow', fallback_error) + + checkpoint_task = asyncio.create_task(snapshots._checkpoint_loop(turn)) + async for chunk in watchdog._aiter_with_inactivity_timeout( + stream_chat_workflow( + raw_user_message=raw_user_message, + session_messages=session_messages, + user_message=effective_user_message, + context=turn.context, + ), + settings._INACTIVITY_TIMEOUT_SEC, + is_activity=lambda item: watchdog._is_chat_run_activity(item, turn.chat_id), + first_response_timeout=settings._FIRST_RESPONSE_TIMEOUT_SEC, + ): + now = time.monotonic() + if now >= next_cancel_poll: + next_cancel_poll = now + settings._CANCEL_VISIBILITY_SEC + if runtime.is_run_cancelled(turn.run_id): + raise asyncio.CancelledError + await chunks.consume_chunk(turn, chunk) + if turn.stopped: + return + + # AgentScope may absorb a tool adapter exception after ToolEffectMiddleware + # has already durably paused the run. Re-read the DB before projecting a + # normal terminal marker; otherwise allocate_event_offset correctly + # rejects the non-completed run and the outer handler overwrites + # ``needs_attention`` with ``failed``. + try: + current = runtime.get_run(turn.run_id) + except AttributeError: + # Some narrow unit adapters replace SessionLocal with a commit-only + # fake that intentionally has no query surface. + current = None + if current is not None: + if current.status == "needs_attention": + await pauses._pause_for_tool_outcome( + turn, RuntimeError(current.failure_reason or "tool outcome requires recovery") + ) + return + if current.status in settings._TERMINAL_STATUSES and current.status != "completed": + raise asyncio.CancelledError + + # Workflow returned normally (with or without a meta event): write the terminal marker + await turn.emit({"type": settings.TERMINAL_TYPE, "chat_id": turn.chat_id}, terminal=True) + + except asyncio.CancelledError: + if lease_lost.is_set(): + # Another process owns recovery now. The fenced worker must not + # project a user cancellation or mutate the durable row. The one + # exception is fencing caused by the user's own cancel: nobody will + # take that run over, so the partial answer must be persisted here. + logger.warning("chat_run_worker_fenced", run_id=turn.run_id, owner=turn.owner) + if ownership.cancelled_by_user(turn.run_id): + snapshots._persist_cancelled_partial(turn) + else: + current = runtime.get_run(turn.run_id) + if current is None or current.status != "cancelled": + # Reaper/hard-cap/external recovery may cancel the local task + # after publishing its own authoritative terminal verdict. + logger.warning( + "chat_run_worker_stopped_by_external_terminal", + run_id=turn.run_id, + status=(current.status if current is not None else None), + ) + else: + # Triggered by cancel_run: persist what was already produced, + # then write a user-cancelled event + terminal marker so + # followers exit gracefully. + logger.info("chat_run_cancelled", run_id=turn.run_id) + snapshots._persist_cancelled_partial(turn) + await turn.emit( + { + "type": "error", + "error": "任务已被用户取消", + "delta": "任务已被用户取消", + "chat_id": turn.chat_id, + "_cancelled": True, + }, + terminal=True, + ) + await turn.emit( + {"type": settings.TERMINAL_TYPE, "chat_id": turn.chat_id, "_cancelled": True}, + terminal=True, + ) + + except errors.ToolOutcomeUnknown as exc: + await pauses._pause_for_tool_outcome(turn, exc) + + except hooks.HookPaused as exc: + await pauses._pause_for_hook(turn, exc) + + except run_journal.RunLeaseLost: + # A takeover/cancel may race with any journal append. Once fenced, the + # old worker must not persist fallback messages or publish a terminal — + # except when the fencing came from the user's own cancel. + logger.warning("chat_run_worker_fenced", run_id=turn.run_id, owner=turn.owner) + if ownership.cancelled_by_user(turn.run_id): + snapshots._persist_cancelled_partial(turn) + + except Exception as exc: + nested_pause = hooks.find_hook_paused(exc) + if nested_pause is not None: + await pauses._pause_for_hook(turn, nested_pause) + return + nested_unknown = errors.find_tool_outcome_unknown(exc) + if nested_unknown is not None: + await pauses._pause_for_tool_outcome(turn, nested_unknown) + return + logger.error("chat_run_failed", run_id=turn.run_id, error=str(exc), exc_info=True) + try: + user_facing = resolve_user_facing_error(exc) + except Exception: + user_facing = "请求处理失败，请稍后重试" + + failure_text = str(exc) + + def _commit_failed_output(db) -> None: + # Whatever the turn produced before failing stays in the row, with + # the error beside it. Runs in the failed-terminal transaction so a + # fenced worker can never leave a late message behind. + model_steps.close_dangling_calls(turn.model_steps) + services_ChatService_module.ChatService(db).upsert_message( + chat_id=turn.chat_id, + role="assistant", + content=turn.full_response, + model=turn.model_name, + thinking=snapshots._thinking_payload(turn), + tool_calls=turn.tool_calls_log if turn.tool_calls_log else None, + model_steps=snapshots._model_steps_payload(turn), + message_id=turn.current_message_id, + chat_seq=turn.current_chat_seq, + error={"error": failure_text, "timestamp": clock._utcnow().isoformat()}, + extra_data=snapshots._turn_extra( + turn, + duration_ms=int((time.monotonic() - turn.run_started_monotonic) * 1000), + segments=turn.segments.payload(), + ), + commit=False, + ) + + failed = turn.journal.complete( + turn.run_id, + owner=turn.owner, + status="failed", + failure_reason=str(exc)[:1000], + last_event_offset=turn.offsets.last, + commit_effect=_commit_failed_output, + ) + if not failed: + logger.warning( + "chat_run_failure_cas_lost", + run_id=turn.run_id, + owner=turn.owner, + ) + return + await turn.emit( + { + "type": "error", + "error": user_facing, + "delta": user_facing, + "chat_id": turn.chat_id, + }, + terminal=True, + ) + await turn.emit({"type": settings.TERMINAL_TYPE, "chat_id": turn.chat_id}, terminal=True) + + finally: + if checkpoint_task is not None: + checkpoint_task.cancel() + # After the terminal state, keep the event stream for the full human + # wait window plus a recovery grace period so late reconnects can replay. + await writer.expire_stream(turn.run_id) + turn.lease_task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await turn.lease_task + ownership._acknowledge_terminal_writer(turn.run_id) diff --git a/src/backend/core/chat/schemas.py b/src/backend/core/chat/schemas.py index e99bceb1..1968086e 100644 --- a/src/backend/core/chat/schemas.py +++ b/src/backend/core/chat/schemas.py @@ -34,7 +34,7 @@ class ReferencedChatItem(BaseModel): """用户为本轮显式引用的一段历史会话。 只带 ID：标题、时间、概览等名片信息一律由后端按当前权限现查（见 - ``core.services.chat_reference_service``），前端传什么都不作数——否则任何人都能伪造 + ``core.chat.references``），前端传什么都不作数——否则任何人都能伪造 一段"来自另一个会话"的内容塞进模型。 """ diff --git a/src/backend/core/services/chat_sequencer.py b/src/backend/core/chat/sequencer.py similarity index 97% rename from src/backend/core/services/chat_sequencer.py rename to src/backend/core/chat/sequencer.py index 70e27a72..54e8216c 100644 --- a/src/backend/core/services/chat_sequencer.py +++ b/src/backend/core/chat/sequencer.py @@ -21,7 +21,7 @@ from core.chat import inflight from core.db.models import ChatMessage, ChatRun, ChatSession from core.db.models.chat import reserve_chat_sequences -from core.services.chat_service import clamp_message_content +from core.chat.limits import clamp_message_content LIVE_RUN_STATUSES = ("pending", "running") TERMINAL_RUN_STATUSES = ("completed", "failed", "cancelled") @@ -92,6 +92,7 @@ def launching_main_run( request_payload: Dict[str, Any], model: Optional[str] = None, user_extra_data: Optional[Dict[str, Any]] = None, + on_accept=None, ): """Admit a main conversation and release its pending writer if launch fails. @@ -105,6 +106,7 @@ def launching_main_run( request_payload=request_payload, model=model, user_extra_data=user_extra_data, + on_accept=on_accept, ) try: yield accepted @@ -137,7 +139,7 @@ def accept_main_run( assistant_message_id = f"msg_{uuid.uuid4().hex[:16]}" try: - from core.services.api_key_runs import bind_request_key, require_admission_key + from core.auth.api_key_runs import bind_request_key, require_admission_key request_payload = bind_request_key(request_payload, user_id) require_admission_key(self.db, request_payload, user_id, now) @@ -148,7 +150,7 @@ def accept_main_run( chat_id, count=2, ) - from core.services.chat_service import ChatService + from core.chat.service import ChatService access = ChatService(self.db).get_session_with_access(chat_id, user_id) if access is None or access[1] not in ("admin", "edit"): raise PermissionError("Conversation is not writable") @@ -225,7 +227,7 @@ def accept_existing_user_run( run_id = f"run_{uuid.uuid4().hex[:16]}" assistant_message_id = f"msg_{uuid.uuid4().hex[:16]}" try: - from core.services.api_key_runs import bind_request_key, require_admission_key + from core.auth.api_key_runs import bind_request_key, require_admission_key request_payload = bind_request_key(request_payload, user_id) require_admission_key(self.db, request_payload, user_id, now) @@ -331,7 +333,7 @@ def accept_replacement_user_run( user_message_id = f"msg_{uuid.uuid4().hex[:16]}" assistant_message_id = f"msg_{uuid.uuid4().hex[:16]}" try: - from core.services.api_key_runs import bind_request_key, require_admission_key + from core.auth.api_key_runs import bind_request_key, require_admission_key request_payload = bind_request_key(request_payload, user_id) require_admission_key(self.db, request_payload, user_id, now) diff --git a/src/backend/core/chat/service.py b/src/backend/core/chat/service.py new file mode 100644 index 00000000..0d9349ea --- /dev/null +++ b/src/backend/core/chat/service.py @@ -0,0 +1,256 @@ +"""Chat sessions and messages behind one service facade.""" +from __future__ import annotations +from core.chat import session_operations +from core.chat import message_writes +from core.chat import message_queries +from core.chat import compaction_snapshots +from core.chat import compaction_checkpoints +from dataclasses import replace +from typing import Any, Dict, List, Optional, Tuple +from core.auth.edition import ChatAccessLevel +from core.db.models import ChatCompactionState, ChatMessage, ChatSession +from core.db.repository import AuditLogRepository, ChatMessageRepository, ChatSessionRepository +from sqlalchemy.orm import Session +import logging +from core.chat import compaction_types + +logger = logging.getLogger(__name__) + +class ChatService: + """Service for chat-related operations.""" + + def __init__(self, db: Session): + self.db = db + self.session_repo = ChatSessionRepository(db) + self.message_repo = ChatMessageRepository(db) + self.audit_repo = AuditLogRepository(db) + + def create_session( + self, + user_id: str, + title: str = "新对话", + extra_data: Dict = None, + chat_id: Optional[str] = None, + project_id: Optional[str] = None, + ) -> ChatSession: + return session_operations.create_session(self, user_id, title, extra_data, chat_id, project_id) + + def ensure_session( + self, + chat_id: str, + user_id: str, + title: str = "新对话", + extra_data: Optional[Dict] = None, + project_id: Optional[str] = None, + ) -> Optional[ChatSession]: + return session_operations.ensure_session(self, chat_id, user_id, title, extra_data, project_id) + + def list_sessions( + self, + user_id: str, + page: int = 1, + page_size: int = 20, + pinned_only: bool = False, + favorite_only: bool = False, + exclude_automation: bool = False, + ) -> Tuple[List[ChatSession], int, int]: + return session_operations.list_sessions(self, user_id, page, page_size, pinned_only, favorite_only, exclude_automation) + + def get_session(self, chat_id: str, user_id: str) -> Optional[ChatSession]: + return session_operations.get_session(self, chat_id, user_id) + + def get_session_with_access( + self, chat_id: str, user_id: str + ) -> Optional[Tuple[ChatSession, ChatAccessLevel]]: + return session_operations.get_session_with_access(self, chat_id, user_id) + + def update_session_fields( + self, + chat_id: str, + fields: Dict[str, Any], + *, + actor_user_id: Optional[str] = None, + ) -> Optional[ChatSession]: + return session_operations.update_session_fields(self, chat_id, fields, actor_user_id=actor_user_id) + + def delete_session_force(self, chat_id: str, *, actor_user_id: str) -> bool: + return session_operations.delete_session_force(self, chat_id, actor_user_id=actor_user_id) + + def update_session( + self, chat_id: str, user_id: str, update_data: Dict[str, Any] + ) -> Optional[ChatSession]: + return session_operations.update_session(self, chat_id, user_id, update_data) + + def delete_session(self, chat_id: str, user_id: str) -> bool: + return session_operations.delete_session(self, chat_id, user_id) + + # chat_messages.content 有 CHECK 约束 char_length <= 100000。长任务的一轮回复 + # （反复重试 + 大量思考正文）真的会撞上，而撞上的后果是**整条 INSERT 抛 + # CheckViolation → 整个 run failed**，一轮的产出全部丢失（实测踩过）。 + # 与其让约束把一轮工作全毁掉，不如夹逼后落库并明确标注被截断。 + def add_message( + self, + chat_id: str, + role: str, + content: str, + model: Optional[str] = None, + thinking: Optional[List[Dict[str, Any]]] = None, + tool_calls: Optional[List[Dict]] = None, + usage: Optional[Dict] = None, + error: Optional[Dict] = None, + extra_data: Dict = None, + message_id: Optional[str] = None, + commit: bool = True, + chat_seq: Optional[int] = None, + model_steps: Optional[List[Dict[str, Any]]] = None, + ) -> ChatMessage: + return message_writes.add_message(self, chat_id, role, content, model, thinking, tool_calls, usage, error, extra_data, message_id, commit, chat_seq, model_steps) + + def refresh_streaming_message( + self, + *, + run_id: str, + owner: str, + message_id: str, + content: str, + thinking: Optional[List[Dict[str, Any]]], + tool_calls: Optional[List[Dict]], + extra_data: Dict[str, Any], + model_steps: Optional[List[Dict[str, Any]]] = None, + ) -> bool: + return message_writes.refresh_streaming_message(self, run_id=run_id, owner=owner, message_id=message_id, content=content, thinking=thinking, tool_calls=tool_calls, extra_data=extra_data, model_steps=model_steps) + + def upsert_message( + self, + chat_id: str, + role: str, + content: str, + *, + message_id: str, + model: Optional[str] = None, + thinking: Optional[List[Dict[str, Any]]] = None, + tool_calls: Optional[List[Dict]] = None, + usage: Optional[Dict] = None, + error: Optional[Dict] = None, + extra_data: Dict = None, + commit: bool = True, + chat_seq: Optional[int] = None, + model_steps: Optional[List[Dict[str, Any]]] = None, + invalidate_mid_turn_checkpoint: bool = False, + ) -> ChatMessage: + return message_writes.upsert_message(self, chat_id, role, content, message_id=message_id, model=model, thinking=thinking, tool_calls=tool_calls, usage=usage, error=error, extra_data=extra_data, commit=commit, chat_seq=chat_seq, model_steps=model_steps, invalidate_mid_turn_checkpoint=invalidate_mid_turn_checkpoint) + + def list_all_messages( + self, chat_id: str, user_id: str + ) -> Optional[List[ChatMessage]]: + return message_queries.list_all_messages(self, chat_id, user_id) + + def list_messages( + self, chat_id: str, user_id: str, page: int = 1, page_size: int = 50 + ) -> Optional[Tuple[List[ChatMessage], int, int]]: + return message_queries.list_messages(self, chat_id, user_id, page, page_size) + + def delete_messages_from(self, chat_id: str, message_id: str) -> int: + return message_writes.delete_messages_from(self, chat_id, message_id) + + def _invalidate_compaction_lineage_for_rewrite( + self, + chat_id: str, + *, + rewritten_seq: int, + ) -> bool: + return compaction_checkpoints.invalidate_compaction_lineage_for_rewrite(self, chat_id, rewritten_seq=rewritten_seq) + + def add_compaction_checkpoint( + self, + chat_id: str, + *, + summary_text: str, + replacement_history: List[Dict], + covered_seq: Optional[int] = None, + source_hash: Optional[str] = None, + base_checkpoint_version: Optional[int] = None, + replacement_manifest: Optional[Dict[str, Any]] = None, + ) -> ChatMessage: + return compaction_checkpoints.add_compaction_checkpoint(self, chat_id, summary_text=summary_text, replacement_history=replacement_history, covered_seq=covered_seq, source_hash=source_hash, base_checkpoint_version=base_checkpoint_version, replacement_manifest=replacement_manifest) + + def _latest_compaction_checkpoint_fallback( + self, chat_id: str + ) -> Optional[ChatMessage]: + return compaction_checkpoints.latest_compaction_checkpoint_fallback(self, chat_id) + + @staticmethod + def _checkpoint_covered_seq(checkpoint: Optional[ChatMessage]) -> int: + return compaction_checkpoints.checkpoint_covered_seq(checkpoint) + + def _ensure_message_sequences(self, chat_id: str) -> None: + return compaction_snapshots.ensure_message_sequences(self, chat_id) + + def _ensure_compaction_state(self, chat_id: str) -> ChatCompactionState: + return compaction_snapshots.ensure_compaction_state(self, chat_id) + + def acquire_compaction_snapshot( + self, + chat_id: str, + *, + owner: str, + lease_seconds: int, + ) -> Optional[compaction_types.CompactionSnapshot]: + return compaction_snapshots.acquire_compaction_snapshot(self, chat_id, owner=owner, lease_seconds=lease_seconds) + + def _acquire_compaction_snapshot( + self, + chat_id: str, + *, + owner: str, + lease_seconds: int, + ) -> Optional[compaction_types.CompactionSnapshot]: + return compaction_snapshots.acquire_compaction_snapshot_impl(self, chat_id, owner=owner, lease_seconds=lease_seconds) + + def release_compaction_lease(self, chat_id: str, *, owner: str) -> bool: + return compaction_snapshots.release_compaction_lease(self, chat_id, owner=owner) + + def commit_compaction_checkpoint( + self, + snapshot: compaction_types.CompactionSnapshot, + *, + owner: str, + summary_text: str, + replacement_history: List[Dict[str, Any]], + replacement_manifest: Dict[str, Any], + ) -> ChatMessage: + return compaction_checkpoints.commit_compaction_checkpoint(self, snapshot, owner=owner, summary_text=summary_text, replacement_history=replacement_history, replacement_manifest=replacement_manifest) + + def get_latest_compaction_checkpoint(self, chat_id: str) -> Optional[ChatMessage]: + return compaction_checkpoints.get_latest_compaction_checkpoint(self, chat_id) + + def get_message_by_id(self, message_id: str) -> Optional[ChatMessage]: + return message_queries.get_message_by_id(self, message_id) + + def get_message_by_index(self, chat_id: str, index: int) -> Optional[ChatMessage]: + return message_queries.get_message_by_index(self, chat_id, index) + + def get_user_message_before( + self, chat_id: str, message_id: str + ) -> Optional[ChatMessage]: + return message_queries.get_user_message_before(self, chat_id, message_id) + + def update_message_extra_data( + self, + message_id: str, + patch: Dict[str, Any], + ) -> bool: + return message_writes.update_message_extra_data(self, message_id, patch) + + def accept_ontology_revision(self, message_id: str) -> Optional[ChatMessage]: + return message_writes.accept_ontology_revision(self, message_id) + + def search_sessions( + self, + user_id: str, + query: str, + page: int = 1, + page_size: int = 20, + scope: str = "title", + ) -> Tuple[list, int]: + return session_operations.search_sessions(self, user_id, query, page, page_size, scope) diff --git a/src/backend/core/chat/session_operations.py b/src/backend/core/chat/session_operations.py new file mode 100644 index 00000000..c68c6469 --- /dev/null +++ b/src/backend/core/chat/session_operations.py @@ -0,0 +1,284 @@ +"""Create, list, update and delete chat sessions, with ownership and sharing checks.""" +from __future__ import annotations +import uuid +from typing import Any, Dict, List, Optional, Tuple +from core.auth.edition import ChatAccessLevel, resolve_chat_access +from core.db.models import ChatSession +from core.db.reserved_chats import reserved_chat_owner + +def create_session( + self, + user_id: str, + title: str = "新对话", + extra_data: Dict = None, + chat_id: Optional[str] = None, + project_id: Optional[str] = None, +) -> ChatSession: + """Create a new chat session. + + If `chat_id` is provided and belongs to the same user, reuse it. + If it belongs to another user, generate a new chat_id. + """ + if chat_id: + existing = self.session_repo.get_by_id(chat_id) + if existing: + if existing.user_id == user_id: + return existing + chat_id = None + + session_data = { + "chat_id": chat_id or f"chat_{uuid.uuid4().hex[:16]}", + "user_id": user_id, + "title": title, + "extra_data": extra_data or {}, + "project_id": project_id, + } + session = self.session_repo.create(session_data) + + # Audit log + self.audit_repo.create( + { + "user_id": user_id, + "action": "chat.session.created", + "resource_type": "chat_session", + "resource_id": session.chat_id, + "status": "success", + } + ) + + return session + + +def ensure_session( + self, + chat_id: str, + user_id: str, + title: str = "新对话", + extra_data: Optional[Dict] = None, + project_id: Optional[str] = None, +) -> Optional[ChatSession]: + """Ensure a chat session exists for user and chat_id. + + When ``project_id`` is given, attach the session to that project (first write wins; if + already attached to a different project it stays unchanged — in-project chats never drift + across projects). All session-creation entry points share this attachment rule. + """ + existing = self.session_repo.get_by_id(chat_id) + if existing: + if existing.user_id != user_id: + return None + # Merge any missing metadata flags into existing session + if extra_data: + merged = dict(existing.extra_data or {}) + changed = False + for k, v in extra_data.items(): + if k not in merged: + merged[k] = v + changed = True + if changed: + existing.extra_data = merged + self.db.commit() + session = existing + else: + session = self.create_session( + user_id=user_id, + title=title, + extra_data=extra_data or {}, + chat_id=chat_id, + project_id=project_id, + ) + if session is not None and project_id and not session.project_id: + session.project_id = project_id + self.db.commit() + return session + + +def list_sessions( + self, + user_id: str, + page: int = 1, + page_size: int = 20, + pinned_only: bool = False, + favorite_only: bool = False, + exclude_automation: bool = False, +) -> Tuple[List[ChatSession], int, int]: + """List chat sessions with pagination.""" + sessions, total = self.session_repo.list_by_user( + user_id, + page, + page_size, + pinned_only, + favorite_only, + exclude_automation=exclude_automation, + ) + + total_pages = (total + page_size - 1) // page_size + + return sessions, total, total_pages + + +def get_session(self, chat_id: str, user_id: str) -> Optional[ChatSession]: + """Get chat session with ownership check. + + Historical semantics unchanged: only the owner can get the session. For + edition-specific sharing, use :py:meth:`get_session_with_access`. + """ + # Module-owned chats are reachable only through their module, never as ordinary chats. + if reserved_chat_owner(chat_id): + return None + session = self.session_repo.get_by_id(chat_id) + + if session and session.user_id != user_id: + # Access denied - user doesn't own this session + return None + + return session + + +def get_session_with_access( + self, chat_id: str, user_id: str +) -> Optional[Tuple[ChatSession, ChatAccessLevel]]: + """Get a session in a sharing context + compute the access level. + + - Does not exist / soft-deleted → ``None`` + - No access (``resolve_chat_access`` returns ``'none'``) → ``None`` + - Module-owned (reserved namespace) → ``None``: not an ordinary chat + - Otherwise returns ``(session, 'admin'|'edit'|'read')`` + """ + if reserved_chat_owner(chat_id): + return None + session = self.session_repo.get_by_id(chat_id) + if session is None: + return None + level = resolve_chat_access(self.db, user_id, session) + if level == "none": + return None + return session, level + + +def update_session_fields( + self, + chat_id: str, + fields: Dict[str, Any], + *, + actor_user_id: Optional[str] = None, +) -> Optional[ChatSession]: + """Field update without ownership check (for sharing contexts). + + The caller is responsible for permission checks beforehand. ``actor_user_id`` is used + only for the audit log. The ``extra_data`` field is merged rather than overwritten. + """ + session = self.session_repo.get_by_id(chat_id) + if session is None: + return None + normalized = dict(fields) + extra_patch = normalized.get("extra_data") + if isinstance(extra_patch, dict): + merged = dict(session.extra_data or {}) + merged.update(extra_patch) + normalized["extra_data"] = merged + updated = self.session_repo.update(chat_id, normalized) + self.audit_repo.create( + { + "user_id": actor_user_id or session.user_id, + "action": "chat.session.updated", + "resource_type": "chat_session", + "resource_id": chat_id, + "details": normalized, + "status": "success", + } + ) + return updated + + +def delete_session_force(self, chat_id: str, *, actor_user_id: str) -> bool: + """Forced delete in a sharing context (no ownership check). Caller handles permissions.""" + session = self.session_repo.get_by_id(chat_id) + if session is None: + return False + result = self.session_repo.soft_delete(chat_id) + if result: + self.audit_repo.create( + { + "user_id": actor_user_id, + "action": "chat.session.deleted", + "resource_type": "chat_session", + "resource_id": chat_id, + "details": { + "owner_user_id": session.user_id, + "deleted_by_owner": session.user_id == actor_user_id, + }, + "status": "success", + } + ) + return result + + +def update_session( + self, chat_id: str, user_id: str, update_data: Dict[str, Any] +) -> Optional[ChatSession]: + """Update chat session.""" + session = self.get_session(chat_id, user_id) + if not session: + return None + + normalized_update_data = dict(update_data) + extra_data_patch = normalized_update_data.get("extra_data") + if isinstance(extra_data_patch, dict): + merged_extra_data = dict(session.extra_data or {}) + merged_extra_data.update(extra_data_patch) + normalized_update_data["extra_data"] = merged_extra_data + + updated_session = self.session_repo.update(chat_id, normalized_update_data) + + # Audit log + self.audit_repo.create( + { + "user_id": user_id, + "action": "chat.session.updated", + "resource_type": "chat_session", + "resource_id": chat_id, + "details": normalized_update_data, + "status": "success", + } + ) + + return updated_session + + +def delete_session(self, chat_id: str, user_id: str) -> bool: + """Delete chat session (soft delete).""" + session = self.get_session(chat_id, user_id) + if not session: + return False + + result = self.session_repo.soft_delete(chat_id) + + if result: + # Audit log + self.audit_repo.create( + { + "user_id": user_id, + "action": "chat.session.deleted", + "resource_type": "chat_session", + "resource_id": chat_id, + "status": "success", + } + ) + + return result + + +def search_sessions( + self, + user_id: str, + query: str, + page: int = 1, + page_size: int = 20, + scope: str = "title", +) -> Tuple[list, int]: + """Search chat sessions by title (and optionally message content).""" + results, total = self.session_repo.search( + user_id, query, page, page_size, scope=scope + ) + return results, total diff --git a/src/backend/core/services/sidebar_status.py b/src/backend/core/chat/sidebar_status.py similarity index 100% rename from src/backend/core/services/sidebar_status.py rename to src/backend/core/chat/sidebar_status.py diff --git a/src/backend/core/services/chat_steer_service.py b/src/backend/core/chat/steer.py similarity index 95% rename from src/backend/core/services/chat_steer_service.py rename to src/backend/core/chat/steer.py index 92cc2d87..5dc37396 100644 --- a/src/backend/core/services/chat_steer_service.py +++ b/src/backend/core/chat/steer.py @@ -16,7 +16,7 @@ from core.db.engine import SessionLocal from core.db.models import ChatRun from core.infra.ephemeral import get_ephemeral_state -from core.services.steer_queue import SteerQueue, SteerQueueConflict +from core.chat.steer_queue import SteerQueue, SteerQueueConflict _STEER_KEY = "jx:chat:run:{run_id}:steer" _STEER_TTL_SECONDS = 3600 @@ -127,8 +127,9 @@ async def remove_pending_steer(run_id: str, steer_id: str) -> bool: if removed: try: await get_ephemeral_state().drop(_key(run_id)) - except Exception: - pass + except Exception as fallback_error: + from core.infra.failures import record_best_effort_failure + record_best_effort_failure('core.chat.steer:remove_pending_steer', fallback_error) return removed diff --git a/src/backend/core/chat/steer_hook.py b/src/backend/core/chat/steer_hook.py new file mode 100644 index 00000000..3cd83053 --- /dev/null +++ b/src/backend/core/chat/steer_hook.py @@ -0,0 +1,70 @@ +"""Deliver queued user instructions at neutral tool and context boundaries.""" +import asyncio +from core.harness.events import thaw_value +from core.harness.hooks import Decision, DecisionKind, HookSpec, HookStage + + +class SteeringHook: + def __init__(self, take_pending=None, acknowledge=None): + self.take_pending = take_pending + self.acknowledge = acknowledge + self.lock = asyncio.Lock() + self.delivery = None + self.interrupted = False + + async def poll(self, run_id): + if self.delivery is None and run_id: + from core.chat.steer import take_pending_steer + self.delivery = await (self.take_pending or take_pending_steer)(run_id) + return self.delivery + + async def before_tool(self, invocation): + async with self.lock: + if not await self.poll(invocation.run_id): + return Decision() + self.interrupted = True + notice = "用户追加了新指令；本工具调用已在执行前中止，等待模型按新指令重新规划。" + return Decision(DecisionKind.REJECT, patch={"tool_response": { + "state": "interrupted", "content": [{"type": "text", "text": notice}]}}, reason=notice) + + async def transform(self, invocation): + if invocation.operation_name != "agent_reasoning": + return Decision() + async with self.lock: + delivery = await self.poll(invocation.run_id) + message = str((delivery or {}).get("message") or "").strip() + if not message: + self.delivery = None + self.interrupted = False + return Decision() + suffix = ("请立即按这条新指令调整后续计划；不要继续已经被中止的旧工具调用。" + if self.interrupted else + "请立即按这条新指令调整后续计划；上一轮工具结果已经完成，可按需使用。") + item = dict(content="[用户在当前执行中追加的新指令]\n" + message + "\n" + suffix, + kind="steer", origin="user:steer", trust="user", priority=950, + token_budget=4000, truncation_policy="never") + return Decision.modify({"context_items": [*invocation.data.get("context_items", ()), item], + "steer_delivery": delivery}) + + async def applied(self, invocation): + delivery = invocation.data.get("steer_delivery") + if delivery is None: + return Decision() + # Called only after the framework has appended the context successfully. + async with self.lock: + if self.acknowledge is not None: + await self.acknowledge(thaw_value(delivery)) + self.delivery = None + self.interrupted = False + return Decision() + + def specs(self): + return ( + HookSpec("user_steering_admission", HookStage.BEFORE_TOOL, self.before_tool, + priority=100, timeout_ms=30000, mutable_fields=frozenset({"tool_response"})), + HookSpec("user_steering_context", HookStage.TRANSFORM_CONTEXT, self.transform, + priority=100, timeout_ms=30000, + mutable_fields=frozenset({"context_items", "steer_delivery"})), + HookSpec("user_steering_applied", HookStage.AFTER_CONTEXT, self.applied, + priority=100, timeout_ms=30000), + ) diff --git a/src/backend/core/services/steer_queue.py b/src/backend/core/chat/steer_queue.py similarity index 100% rename from src/backend/core/services/steer_queue.py rename to src/backend/core/chat/steer_queue.py diff --git a/src/backend/core/services/subagent_routing_service.py b/src/backend/core/chat/subagent_routing.py similarity index 100% rename from src/backend/core/services/subagent_routing_service.py rename to src/backend/core/chat/subagent_routing.py diff --git a/src/backend/core/services/chat_titles.py b/src/backend/core/chat/titles.py similarity index 92% rename from src/backend/core/services/chat_titles.py rename to src/backend/core/chat/titles.py index f3fe16cd..cb1b2d89 100644 --- a/src/backend/core/services/chat_titles.py +++ b/src/backend/core/chat/titles.py @@ -1,13 +1,14 @@ """One non-blocking first-question title call, with durable claim and rename fencing.""" +from __future__ import annotations import asyncio import logging +from dataclasses import dataclass, field from core.db import engine as db_engine from core.db.models import ChatMessage, ChatSession -from core.llm.summarizer import get_summarizer +from core.models.completions.summarizer import get_summarizer logger = logging.getLogger(__name__) -_tasks: set[asyncio.Task] = set() def _claim(chat_id: str, user_id: str, run_id: str) -> str | None: @@ -99,11 +100,23 @@ def schedule_first_title(chat_id: str, user_id: str, run_id: str) -> asyncio.Tas if question is None: return None task = asyncio.create_task(_generate(chat_id, user_id, run_id, question), name=f"chat-title:{chat_id}") - _tasks.add(task) + process_state().tasks.add(task) def settled(done: asyncio.Task) -> None: - _tasks.discard(done) + process_state().tasks.discard(done) if not done.cancelled() and done.exception() is not None: logger.warning("chat_title_task_failed: %s", type(done.exception()).__name__) task.add_done_callback(settled) return task + + +@dataclass +class TitlesState: + tasks: set[asyncio.Task] = field(default_factory=lambda: set()) + + +_STATE = TitlesState() + + +def process_state() -> TitlesState: + return _STATE diff --git a/src/backend/core/chat/tool_log.py b/src/backend/core/chat/tool_log.py index c6c6beaf..0f368009 100644 --- a/src/backend/core/chat/tool_log.py +++ b/src/backend/core/chat/tool_log.py @@ -10,7 +10,7 @@ import logging import time from typing import Any, Dict, List, Optional -from core.chat.display_bounds import bound_result_for_display +from core.infra.text_bounds import bound_result_for_display logger = logging.getLogger(__name__) diff --git a/src/backend/orchestration/tool_payloads.py b/src/backend/core/chat/tool_payloads.py similarity index 92% rename from src/backend/orchestration/tool_payloads.py rename to src/backend/core/chat/tool_payloads.py index f70b74a1..462efd6d 100644 --- a/src/backend/orchestration/tool_payloads.py +++ b/src/backend/core/chat/tool_payloads.py @@ -10,7 +10,7 @@ from typing import Any, Dict -from core.config.catalog_details import get_skill_curated_detail +from core.capabilities.catalog.details import get_skill_curated_detail _FILE_PREVIEW_MAX_LINES = 8 _FILE_PREVIEW_MAX_CHARS = 400 @@ -114,8 +114,15 @@ def _build_read_tool_payload(tool_args: Any, result: Any) -> Dict[str, Any]: # type == "text" content = result.get("content") if isinstance(result.get("content"), str) else "" extra2: Dict[str, Any] = {} - for k in ("start_line", "end_line", "truncated", "parsed_text", - "recovered_from_artifact", "persistent", "physical_path"): + for k in ( + "start_line", + "end_line", + "truncated", + "parsed_text", + "recovered_from_artifact", + "persistent", + "physical_path", + ): if k in result and result[k] not in (None, "", False): extra2[k] = result[k] return { @@ -146,8 +153,16 @@ def _build_read_artifact_payload(result: Any) -> Dict[str, Any]: # output structure from the read_artifact tool (see core/llm/tool.py::read_artifact) content = result.get("content") extra: Dict[str, Any] = {} - for k in ("file_id", "offset", "next_offset", "read_chars_so_far", - "budget_remaining", "sheet_names", "slide_count", "slide_index"): + for k in ( + "file_id", + "offset", + "next_offset", + "read_chars_so_far", + "budget_remaining", + "sheet_names", + "slide_count", + "slide_index", + ): if k in result and result[k] not in (None, ""): extra[k] = result[k] return { @@ -198,15 +213,17 @@ def _build_skill_load_payload(skill_id: str) -> Dict[str, Any]: # LLM a noticeable amount of time to finish writing (e.g. long JSON params), # otherwise the UI shows only the generic typing placeholder until the tool # result finally arrives. -_FAST_EMIT_TOOLS = frozenset({ - "Bash", - "write_stdin", - "Read", - "Edit", - "Write", - "Glob", - "Grep", -}) +_FAST_EMIT_TOOLS = frozenset( + { + "Bash", + "write_stdin", + "Read", + "Edit", + "Write", + "Glob", + "Grep", + } +) def _tool_args_ready(tool_name: str, tool_args: Any) -> bool: diff --git a/src/backend/core/config/__init__.py b/src/backend/core/config/__init__.py index b1508569..35218ba3 100644 --- a/src/backend/core/config/__init__.py +++ b/src/backend/core/config/__init__.py @@ -1,6 +1,5 @@ -# ``core.config`` holds static config + ``settings`` only (a clean L1 leaf). -# The DB-backed config *services* live in ``core.services`` so config no longer -# imports ``core.db`` at module level: -# from core.services.model_config import ModelConfigService -# from core.services.system_config import SystemConfigService -# from core.services.mcp_service import McpServerConfigService +# ``core.config`` holds static config + ``settings`` only and never imports +# ``core.db`` at module level. DB-backed config services live elsewhere: +# from core.models.config import ModelConfigService +# from core.config.store.system_config import SystemConfigService +# from core.capabilities.mcp.service import McpServerConfigService diff --git a/src/backend/core/config/display_names.py b/src/backend/core/config/display_names.py index f2c595a1..31fa3f74 100644 --- a/src/backend/core/config/display_names.py +++ b/src/backend/core/config/display_names.py @@ -8,7 +8,7 @@ from typing import Dict -from core.config.edition_display_names import ( +from core.config.edition import ( edition_mcp_server_descriptions, edition_mcp_server_display_names, edition_tool_display_names, diff --git a/src/backend/core/config/edition_display_names.py b/src/backend/core/config/edition.py similarity index 78% rename from src/backend/core/config/edition_display_names.py rename to src/backend/core/config/edition.py index 1ae1d377..1716fd29 100644 --- a/src/backend/core/config/edition_display_names.py +++ b/src/backend/core/config/edition.py @@ -28,3 +28,11 @@ def edition_tool_display_names() -> dict[str, str]: "edition_mcp_user_intros", "edition_tool_display_names", ] + +"""Community edition contributes no enterprise service configuration.""" + +SEED_CONFIGS = [] +CONFIG_KEY_TO_ENV = {} + + +__all__ += ["CONFIG_KEY_TO_ENV", "SEED_CONFIGS"] diff --git a/src/backend/core/config/environment.py b/src/backend/core/config/environment.py index c8fbccae..5f981998 100644 --- a/src/backend/core/config/environment.py +++ b/src/backend/core/config/environment.py @@ -8,10 +8,7 @@ from __future__ import annotations import os -import tempfile -from dataclasses import dataclass, field from pathlib import Path -from typing import Optional from dotenv import dotenv_values diff --git a/src/backend/core/config/identity_settings.py b/src/backend/core/config/identity_settings.py index 1e81a1f5..5eafc0d4 100644 --- a/src/backend/core/config/identity_settings.py +++ b/src/backend/core/config/identity_settings.py @@ -3,9 +3,8 @@ import os import tempfile from dataclasses import dataclass, field -from pathlib import Path from typing import Optional -from .environment import _env, _bool, _int, _float, _REPO_ROOT +from .environment import _env, _bool, _int @dataclass(frozen=True) class AuthSettings: diff --git a/src/backend/core/config/mcp_config.py b/src/backend/core/config/mcp_config.py index 25c2472b..a6281c46 100755 --- a/src/backend/core/config/mcp_config.py +++ b/src/backend/core/config/mcp_config.py @@ -19,10 +19,10 @@ from extensions.mcp_servers._ports import PORTS as _PORTS -def _mcp_http_url(server_id: str) -> str: +def mcp_http_url(server_id: str) -> str: from core.config.settings import settings - return f"http://{settings.server.mcp_host}:{_PORTS[server_id]}/mcp/" + return f"http://{settings.server.mcp_host}:{_PORTS[server_id]}/mcp" def builtin_mcp_configs(location=None) -> Dict[str, dict]: @@ -34,7 +34,7 @@ def builtin_mcp_configs(location=None) -> Dict[str, dict]: return { server.server_id: { "transport": "streamable_http", - "url": _mcp_http_url(server.server_id), + "url": mcp_http_url(server.server_id), "env": {}, } for server in process_servers(location) @@ -68,7 +68,7 @@ def builtin_launcher_serves_this_plane() -> bool: if not settings.deploy.is_local: return True - from core.auth.desktop_bridge import bridge_enabled + from core.harness.registry.host_runtime import requires_remote_execution as bridge_enabled return not bridge_enabled() diff --git a/src/backend/core/config/product_settings.py b/src/backend/core/config/product_settings.py index 50eaffb7..217ea86c 100644 --- a/src/backend/core/config/product_settings.py +++ b/src/backend/core/config/product_settings.py @@ -1,11 +1,8 @@ """Environment-backed configuration values.""" from __future__ import annotations import os -import tempfile from dataclasses import dataclass, field -from pathlib import Path -from typing import Optional -from .environment import _env, _bool, _int, _float, _REPO_ROOT +from .environment import _env, _bool, _int @dataclass(frozen=True) class EditionSettings: diff --git a/src/backend/core/config/runtime_env.py b/src/backend/core/config/runtime_env.py index ca82ad9f..c6eb252d 100644 --- a/src/backend/core/config/runtime_env.py +++ b/src/backend/core/config/runtime_env.py @@ -1,19 +1,6 @@ -"""DB-backed env lookup for MCP servers running in the streamable-http container. - -The MCP container's ``os.environ`` is frozen at ``docker-compose up`` time, -so admin-panel changes never reach it. ``get_runtime_value`` re-routes the -existing ``os.getenv`` idiom through ``SystemConfigService`` / -``ModelConfigService`` (both DB-cached for 30s) before falling back to env. -""" - -from __future__ import annotations - -import logging +"""Environment-only paths available before database initialization.""" import os from pathlib import Path -from typing import Optional - -logger = logging.getLogger(__name__) def local_data_dir() -> Path: @@ -24,37 +11,5 @@ def local_data_dir() -> Path: """ return Path(os.getenv("HUGAGENT_HOME", str(Path.home() / ".hugagent"))).expanduser() -# Env vars produced by ``ModelConfigService.get_mcp_env_overlay()``. Looking up -# a non-model env var should not pay the cost of building that dict. -_MODEL_CONFIG_ENV_VARS: frozenset[str] = frozenset({ - "MODEL_URL", "API_KEY", "BASE_MODEL_NAME", - "OPENAI_API_BASE", "OPENAI_BASE_URL", "OPENAI_API_KEY", - "MEM0_EMBED_URL", "MEM0_EMBED_MODEL", "MEM0_EMBED_API_KEY", "MEM0_EMBED_DIMS", - "RERANKER_URL", "RERANKER_MODEL", "RERANKER_API_KEY", -}) - - -def get_runtime_value(env_var: str, default: Optional[str] = None) -> Optional[str]: - """Resolve an env-var name against admin DB configs first, then env.""" - try: - from core.services.system_config import SystemConfigService, get_config_key_for_env - - config_key = get_config_key_for_env(env_var) - if config_key: - val = SystemConfigService.get_instance().get(config_key) - if val: - return val - except Exception as exc: - logger.debug("[runtime_env] system_config lookup failed for %s: %s", env_var, exc) - - if env_var in _MODEL_CONFIG_ENV_VARS: - try: - from core.services.model_config import ModelConfigService - - val = ModelConfigService.get_instance().get_mcp_env_overlay().get(env_var) - if val: - return val - except Exception as exc: - logger.debug("[runtime_env] model_config lookup failed for %s: %s", env_var, exc) - return os.getenv(env_var, default) +CAPS_ROOT_ENV = "HUGAGENT_CAPS_ROOT" diff --git a/src/backend/core/config/runtime_settings.py b/src/backend/core/config/runtime_settings.py index 8bf49dc0..117efe56 100644 --- a/src/backend/core/config/runtime_settings.py +++ b/src/backend/core/config/runtime_settings.py @@ -1,11 +1,9 @@ """Environment-backed configuration values.""" from __future__ import annotations import os -import tempfile from dataclasses import dataclass, field from pathlib import Path -from typing import Optional -from .environment import _env, _bool, _int, _float, _REPO_ROOT, DEFAULT_CHAT_MODEL_ALIAS, _load_env_files +from .environment import _env, _bool, _int, _float @dataclass(frozen=True) class LLMSettings: @@ -187,29 +185,6 @@ class ServerSettings: def is_prod(self) -> bool: return self.env in ("prod", "production") -@dataclass(frozen=True) -class RateLimitSettings: - enabled: bool = field(default_factory=lambda: _bool(_env("RATE_LIMIT_ENABLED", "true"))) - storage: str = field(default_factory=lambda: _env("RATE_LIMIT_STORAGE", "memory://")) - cb_user_center_threshold: int = field( - default_factory=lambda: _int(_env("CB_USER_CENTER_THRESHOLD", "5"), 5) - ) - cb_user_center_timeout: int = field( - default_factory=lambda: _int(_env("CB_USER_CENTER_TIMEOUT", "60"), 60) - ) - cb_model_api_threshold: int = field( - default_factory=lambda: _int(_env("CB_MODEL_API_THRESHOLD", "10"), 10) - ) - cb_model_api_timeout: int = field( - default_factory=lambda: _int(_env("CB_MODEL_API_TIMEOUT", "30"), 30) - ) - cb_storage_threshold: int = field( - default_factory=lambda: _int(_env("CB_STORAGE_THRESHOLD", "5"), 5) - ) - cb_storage_timeout: int = field( - default_factory=lambda: _int(_env("CB_STORAGE_TIMEOUT", "60"), 60) - ) - @dataclass(frozen=True) class RoutingSettings: strategy: str = field( @@ -219,11 +194,11 @@ class RoutingSettings: @dataclass(frozen=True) class CompactionSettings: - """Context compaction (see core/llm/compaction.py). + """Context compaction (see core/llm/context/compaction.py). Compresses history into "recent user messages + summary" and persists the summary as a checkpoint. One implementation serves all three phases - (mid-turn / pre-turn / post-turn, see ``core.llm.compaction.CompactionPhase``), + (mid-turn / pre-turn / post-turn, see ``core.llm.context.compaction.CompactionPhase``), so every knob below governs all of them — there is no separate in-turn ratio any more. """ @@ -260,7 +235,7 @@ class CompactionSettings: # Per-tool-result cap fed to AgentScope's ContextConfig. This is the # deterministic, model-free layer that runs before any summarization (the # counterpart of Codex's output truncation and dsh's toolResultPruner); - # the overflow is spilled to the sandbox by core/llm/offloader.py. + # the overflow is spilled to the sandbox by core/llm/context/offloader.py. tool_result_limit: int = field( default_factory=lambda: _int(_env("CHAT_TOOL_RESULT_LIMIT", "20000"), 20000) ) diff --git a/src/backend/core/config/sandbox_settings.py b/src/backend/core/config/sandbox_settings.py index f881f28c..df4171d8 100644 --- a/src/backend/core/config/sandbox_settings.py +++ b/src/backend/core/config/sandbox_settings.py @@ -1,11 +1,8 @@ """Environment-backed configuration values.""" from __future__ import annotations import os -import tempfile from dataclasses import dataclass, field -from pathlib import Path -from typing import Optional -from .environment import _env, _bool, _int, _float, _REPO_ROOT +from .environment import _env, _bool, _int @dataclass(frozen=True) class SandboxSettings: @@ -123,7 +120,7 @@ class SandboxSettings: # The all-modes code-execution capability switch has moved out of settings: # the sole control source is sandbox.code_capability_enable in the Config # admin panel "System Config" (option B, see - # core.services.system_config.code_capability_enabled). The env is retired. + # core.config.store.system_config.code_capability_enabled). The env is retired. # Hard safety for user confirmation of MySpace write operations (docs §13). # true (default): Write/Edit/Delete/Move on /myspace must pass out-of-band # user confirmation; non-interactive mode (sub-agent) rejects writes @@ -265,7 +262,7 @@ class SandboxSettings: # then hot-swaps _template with the new template id + writes it back to # .env. An empty host means "remote rebuild not configured": the rebuild # endpoint errors out for cube and directs users to the manual docs path. - # See core/services/cube_template_builder.py for details. + # See features/sandbox_ops/cube_template.py for details. cube_node_ssh_host: str = field( default_factory=lambda: _env("CUBE_NODE_SSH_HOST", _env("CUBE_NODE_IP", "")).strip() ) diff --git a/src/backend/core/config/settings.py b/src/backend/core/config/settings.py index fa6b3fc1..cdc8c4b9 100644 --- a/src/backend/core/config/settings.py +++ b/src/backend/core/config/settings.py @@ -1,11 +1,8 @@ """Environment-backed configuration values.""" from __future__ import annotations import os -import tempfile from dataclasses import dataclass, field -from pathlib import Path -from typing import Optional -from .environment import _env, _bool, _int, _float, _REPO_ROOT, DEFAULT_CHAT_MODEL_ALIAS, _load_env_files +from .environment import _env as env_value, _bool as env_bool, _int as env_int, DEFAULT_CHAT_MODEL_ALIAS from .identity_settings import ( AuthSettings, @@ -21,7 +18,6 @@ KnowledgeBaseSettings, RedisSettings, ServerSettings, - RateLimitSettings, RoutingSettings, CompactionSettings, PromptSettings, @@ -52,7 +48,6 @@ class AppSettings: kb: KnowledgeBaseSettings = field(default_factory=KnowledgeBaseSettings) redis: RedisSettings = field(default_factory=RedisSettings) server: ServerSettings = field(default_factory=ServerSettings) - rate_limit: RateLimitSettings = field(default_factory=RateLimitSettings) routing: RoutingSettings = field(default_factory=RoutingSettings) compaction: CompactionSettings = field(default_factory=CompactionSettings) prompt: PromptSettings = field(default_factory=PromptSettings) diff --git a/src/backend/core/config/store/__init__.py b/src/backend/core/config/store/__init__.py new file mode 100644 index 00000000..8e4f6e51 --- /dev/null +++ b/src/backend/core/config/store/__init__.py @@ -0,0 +1 @@ +"""System configuration stored in the database, with environment fallbacks.""" diff --git a/src/backend/core/config/store/config_keys.py b/src/backend/core/config/store/config_keys.py new file mode 100644 index 00000000..695edc32 --- /dev/null +++ b/src/backend/core/config/store/config_keys.py @@ -0,0 +1,12 @@ +"""Map environment variables to the system-config keys that override them.""" +from __future__ import annotations +from typing import Optional +from core.config.store import system_config_constants + +def get_config_key_for_env(env_var: str) -> Optional[str]: + """Return the system-config key that backs ``env_var``, if any. + + Public lookup helper so callers (e.g. ``runtime_env``) don't need to import + the internal reverse map. + """ + return system_config_constants.ENV_TO_CONFIG_KEY.get(env_var) diff --git a/src/backend/core/config/store/execution_options.py b/src/backend/core/config/store/execution_options.py new file mode 100644 index 00000000..df1a4220 --- /dev/null +++ b/src/backend/core/config/store/execution_options.py @@ -0,0 +1,93 @@ +"""Execution switches read from system config: code capability, turbo mode, planning.""" +from __future__ import annotations +from core.config.store import system_config +from core.config.store import system_config_constants + +def code_capability_enabled() -> bool: + """Master switch for the main agent's code execution capability (single source of truth at runtime). + + Sole control source = the DB value of Config console "System config" (plan B, env fallback retired); + seeded explicitly to "true". Saving in the console calls invalidate_cache; takes effect ≤30s without restart. + agent_factory and the admin_prompts preview call the same source, ensuring the console display matches runtime. + """ + try: + val = system_config.SystemConfigService.get_instance().get("sandbox.code_capability_enable", "false") + except Exception: # noqa: BLE001 — conservatively disable on config-layer errors + return False + return str(val).strip().lower() in ("1", "true", "yes", "on") + + +def turbo_mcp_server_ids() -> frozenset[str]: + """极速模式装配的 MCP server 集合（运行时唯一真源）。 + + 控制源 = Config 管理台「系统配置 → 极速模式」的 ``turbo.mcp_server_ids`` + （逗号分隔）。刻意不与能力目录（catalog）的启停求交集——极速模式的工具面 + 是独立的产品契约。配置异常或解析为空时回退到内置检索三件套。 + """ + default = ",".join(sorted(system_config_constants.DEFAULT_TURBO_MCP_SERVER_IDS)) + try: + raw = system_config.SystemConfigService.get_instance().get("turbo.mcp_server_ids", default) or default + except Exception: # noqa: BLE001 — conservatively fall back to the built-in trio + return system_config_constants.DEFAULT_TURBO_MCP_SERVER_IDS + ids = frozenset(part.strip() for part in str(raw).split(",") if part.strip()) + return ids or system_config_constants.DEFAULT_TURBO_MCP_SERVER_IDS + + +def turbo_skill_ids() -> tuple[str, ...]: + """极速模式装配的技能集合（``AdminSkill.skill_id`` / 内置技能 id 列表）。 + + 控制源 = Config 管理台「系统配置 → 极速模式」的 ``turbo.skill_ids``（逗号 + 分隔）。与用户本轮显式呼唤的技能合并去重。默认空 —— 不选就没有技能， + 极速模式的系统提示词里也就不出现技能清单。 + """ + try: + raw = system_config.SystemConfigService.get_instance().get("turbo.skill_ids", "") or "" + except Exception: # noqa: BLE001 — 配置层异常时按「没配技能」处理 + return () + return tuple(dict.fromkeys(part.strip() for part in str(raw).split(",") if part.strip())) + + +def turbo_plugin_ids() -> tuple[str, ...]: + """极速模式装配的插件集合（``InstalledPlugin.install_id`` 列表）。 + + 控制源 = Config 管理台「系统配置 → 极速模式」的 ``turbo.plugin_ids``（逗号 + 分隔）。插件是「技能 + 工具」的能力包，装配时由 agent_factory 展开成其组件 + 技能 / MCP，与 ``turbo.mcp_server_ids`` 合并去重。默认空 —— 不选就没有插件。 + """ + try: + raw = system_config.SystemConfigService.get_instance().get("turbo.plugin_ids", "") or "" + except Exception: # noqa: BLE001 — 配置层异常时按「没配插件」处理 + return () + return tuple(dict.fromkeys(part.strip() for part in str(raw).split(",") if part.strip())) + + +def turbo_manual_invoke_enabled() -> bool: + """极速模式下是否允许显式呼唤（斜杠技能 / @子智能体 / 插件）临时装配。""" + try: + val = system_config.SystemConfigService.get_instance().get("turbo.manual_invoke_enable", "true") + except Exception: # noqa: BLE001 + return True + return str(val).strip().lower() in ("1", "true", "yes", "on") + + +def turbo_max_iters() -> int: + """极速模式的 ReAct 迭代硬上限（最小 2，防止配错把回答轮也掐掉）。""" + try: + val = system_config.SystemConfigService.get_instance().get("turbo.max_iters", "4") + return max(2, int(str(val).strip())) + except Exception: # noqa: BLE001 + return 4 + + +def auto_plan_entry_enabled() -> bool: + """Master switch for the main agent's in-conversation plan tracker (the update_plan tool). + + Control source = the Config console "System config" DB value ``chat.auto_plan_entry_enable``, + defaulting to "true" (enabled by default). Ops can set it to false to disable the capability, ≤30s, no restart. + Conservatively disable on config-layer errors (don't register the tool, fall back to manually clicking "Plan mode"). + """ + try: + val = system_config.SystemConfigService.get_instance().get("chat.auto_plan_entry_enable", "true") + except Exception: # noqa: BLE001 — conservatively disable on config-layer errors + return False + return str(val).strip().lower() in ("1", "true", "yes", "on") diff --git a/src/backend/core/config/store/runtime_env.py b/src/backend/core/config/store/runtime_env.py new file mode 100644 index 00000000..07a56cba --- /dev/null +++ b/src/backend/core/config/store/runtime_env.py @@ -0,0 +1,52 @@ +"""DB-backed env lookup for MCP servers running in the streamable-http container. + +The MCP container's ``os.environ`` is frozen at ``docker-compose up`` time, +so admin-panel changes never reach it. ``get_runtime_value`` re-routes the +existing ``os.getenv`` idiom through ``SystemConfigService`` / +``ModelConfigService`` (both DB-cached for 30s) before falling back to env. +""" + +from __future__ import annotations + +import logging +import os +from typing import Optional + +logger = logging.getLogger(__name__) + + +# Env vars produced by ``ModelConfigService.get_mcp_env_overlay()``. Looking up +# a non-model env var should not pay the cost of building that dict. +_MODEL_CONFIG_ENV_VARS: frozenset[str] = frozenset({ + "MODEL_URL", "API_KEY", "BASE_MODEL_NAME", + "OPENAI_API_BASE", "OPENAI_BASE_URL", "OPENAI_API_KEY", + "MEM0_EMBED_URL", "MEM0_EMBED_MODEL", "MEM0_EMBED_API_KEY", "MEM0_EMBED_DIMS", + "RERANKER_URL", "RERANKER_MODEL", "RERANKER_API_KEY", +}) + + +def get_runtime_value(env_var: str, default: Optional[str] = None) -> Optional[str]: + """Resolve an env-var name against admin DB configs first, then env.""" + try: + from core.config.store.system_config import SystemConfigService + from core.config.store.config_keys import get_config_key_for_env + + config_key = get_config_key_for_env(env_var) + if config_key: + val = SystemConfigService.get_instance().get(config_key) + if val: + return val + except Exception as exc: + logger.debug("[runtime_env] system_config lookup failed for %s: %s", env_var, exc) + + if env_var in _MODEL_CONFIG_ENV_VARS: + try: + from core.harness.registry.model_services import model_environment + + val = model_environment().get(env_var) + if val: + return val + except Exception as exc: + logger.debug("[runtime_env] model_config lookup failed for %s: %s", env_var, exc) + + return os.getenv(env_var, default) diff --git a/src/backend/core/config/store/system_config.py b/src/backend/core/config/store/system_config.py new file mode 100644 index 00000000..0f5fc1e0 --- /dev/null +++ b/src/backend/core/config/store/system_config.py @@ -0,0 +1,120 @@ +"""Thread-safe service that resolves service configs from the database with env fallback.""" +from __future__ import annotations +import os +import threading +import time +from typing import Optional +from core.harness.registry.system_configuration import system_configuration_store +import logging +from core.config.store import system_config_constants + +logger = logging.getLogger(__name__) + +class SystemConfigService: + """Thread-safe singleton that resolves service configs from DB with env fallback.""" + + _instance: Optional["SystemConfigService"] = None + _lock = threading.Lock() + + def __init__(self) -> None: + self._cache: dict[str, Optional[str]] = {} + self._cache_meta: dict[str, dict] = {} # full row metadata + self._cache_ts: float = 0.0 + self._cache_lock = threading.Lock() + + @classmethod + def get_instance(cls) -> "SystemConfigService": + if cls._instance is None: + with cls._lock: + if cls._instance is None: + cls._instance = cls() + return cls._instance + + # ── get / get_group ───────────────────────────────────────────── + + def get(self, key: str, default: str | None = None) -> str | None: + """Get a config value. DB first, then env fallback, then default.""" + self._maybe_refresh() + if key in self._cache and self._cache[key] is not None: + return self._cache[key] + # env fallback + env_key = system_config_constants.CONFIG_KEY_TO_ENV.get(key) + if env_key: + env_val = os.getenv(env_key) + if env_val is not None: + return env_val.strip() + return default + + def get_group(self, group_key: str) -> dict[str, str]: + """Return all config key→value pairs for a group.""" + self._maybe_refresh() + result: dict[str, str] = {} + for key, meta in self._cache_meta.items(): + if meta.get("group_key") == group_key: + val = self.get(key) + if val is not None: + result[key] = val + return result + + def get_all_configs(self) -> list[dict]: + """Return all config rows as dicts (for API responses).""" + self._maybe_refresh() + return list(self._cache_meta.values()) + + def get_group_configs(self, group_key: str) -> list[dict]: + """Return config rows for a specific group.""" + self._maybe_refresh() + return [m for m in self._cache_meta.values() if m.get("group_key") == group_key] + + # ── set ───────────────────────────────────────────────────────── + + def set(self, key: str, value: str | None, updated_by: str = "admin") -> None: + system_configuration_store().set(key, value, updated_by) + self.invalidate_cache() + + def bulk_set(self, items: list[dict], updated_by: str = "admin") -> None: + system_configuration_store().bulk_set(items, updated_by) + self.invalidate_cache() + + # ── cache management ────────────────────────────────────────── + + def invalidate_cache(self) -> None: + with self._cache_lock: + self._cache.clear() + self._cache_meta.clear() + self._cache_ts = 0.0 + + def _maybe_refresh(self) -> None: + now = time.monotonic() + if now - self._cache_ts < system_config_constants.CACHE_TTL_SECONDS and self._cache_meta: + return + with self._cache_lock: + if now - self._cache_ts < system_config_constants.CACHE_TTL_SECONDS and self._cache_meta: + return + self._load_from_db() + self._cache_ts = time.monotonic() + + + def _load_from_db(self) -> None: + try: + values, metadata = system_configuration_store().load() + except Exception as exc: + logger.warning("[SystemConfigService] DB load failed, keeping stale cache: %s", type(exc).__name__) + return + self._cache = values + self._cache_meta = metadata + + # ── env overlay for MCP sub-processes ───────────────────────── + + def get_service_env_overlay(self) -> dict[str, str]: + """Return env-var style dict for injecting into MCP sub-processes. + + Maps DB configs to the env var names that MCP servers already read via os.getenv(). + Only includes keys that have a non-None value (from DB or env fallback). + """ + overlay: dict[str, str] = {} + for config_key, env_key in system_config_constants.CONFIG_KEY_TO_ENV.items(): + val = self.get(config_key) + if val is not None: + overlay[env_key] = val + return overlay diff --git a/src/backend/core/config/store/system_config_constants.py b/src/backend/core/config/store/system_config_constants.py new file mode 100644 index 00000000..698d9458 --- /dev/null +++ b/src/backend/core/config/store/system_config_constants.py @@ -0,0 +1,302 @@ +"""Cache lifetime and defaults used by the system config service.""" +from __future__ import annotations +from core.config.edition import CONFIG_KEY_TO_ENV as EDITION_CONFIG_KEY_TO_ENV +from core.config.edition import SEED_CONFIGS as EDITION_SEED_CONFIGS + +CACHE_TTL_SECONDS = 30.0 + + +PROVIDER_SCOPED_KNOWLEDGE_FIELDS = ("url", "api_key", "allowed_dataset_ids") + + +SEED_CONFIGS: list[tuple[str, str | None, str, str, str, bool]] = [ + # query_database + ( + "query_database.url", + None, + "数据库查询接口 URL", + "NL2SQL 数据库查询服务地址", + "query_database", + False, + ), + ("query_database.timeout", "40", "超时时间(秒)", "请求超时时间", "query_database", False), + ("query_database.retry_times", "1", "重试次数", "请求失败重试次数", "query_database", False), + ( + "query_database.max_output_tokens", + "45000", + "最大输出 Token", + "数据库查询最大输出 Token 数", + "query_database", + False, + ), + ( + "database_query.capability_enabled", + "true", + "数据库查询能力总开关", + "控制统一「数据库查询」能力是否在能力中心可用。由数据库工具页维护，不写入 catalog.json。", + "query_database", + False, + ), + ( + "knowledge_base.detail_max_chars", + "50000", + "详情最大字符数", + "知识库文档详情最大字符数", + "knowledge_base", + False, + ), + # file_parser + ("file_parser.api_url", None, "文件解析 API URL", "PDF/文档解析服务地址", "file_parser", False), + ("file_parser.timeout", "60", "超时时间(秒)", "文件解析请求超时", "file_parser", False), + ("file_parser.lang_list", "ch", "语言列表", "OCR 语言列表", "file_parser", False), + ("file_parser.backend", "pipeline", "解析后端", "解析后端引擎", "file_parser", False), + ("file_parser.parse_method", "auto", "解析方法", "auto / ocr / txt", "file_parser", False), + ( + "file_parser.formula_enable", + "true", + "启用公式识别", + "是否启用公式识别", + "file_parser", + False, + ), + ("file_parser.table_enable", "true", "启用表格识别", "是否启用表格识别", "file_parser", False), + # internet_search + ( + "internet_search.engine", + "tavily", + "搜索引擎", + "互联网搜索引擎 (tavily / baidu / langsearch)", + "internet_search", + False, + ), + ( + "internet_search.tavily_api_key", + None, + "Tavily API Key", + "Tavily 互联网搜索服务密钥", + "internet_search", + True, + ), + ( + "internet_search.baidu_api_key", + None, + "百度搜索 API Key", + "百度千帆 AppBuilder 搜索服务密钥", + "internet_search", + True, + ), + ( + "internet_search.langsearch_api_key", + None, + "LangSearch API Key", + "LangSearch Web Search API 服务密钥", + "internet_search", + True, + ), + # sandbox / code execution. The admin toggle is the **sole authority** (no env fallback): + # explicitly defaults to "true"; admins can turn it off in "System config", which writes "false" and persists. + ( + "sandbox.code_capability_enable", + "true", + "主智能体代码执行能力", + "开启后主/计划/批量/子智能体默认具备文件工具+沙箱+我的空间访问；关闭则仅 Lab " + "入口可用。本开关为唯一控制源，保存后 ≤30s 生效、无需重启。", + "sandbox", + False, + ), + # ontology —— independent enterprise governance gate for plugin imports. + # Unlike the user's ontology preference, this switch cannot be bypassed by + # turning off ontology validation in personal settings. + ( + "ontology.force_plugin_import_build_validation", + "false", + "强制构建时本体校验", + "插件导入或安装时必须通过激活 Domain Pack 的构建校验；个人本体开关不能关闭此门禁。", + "ontology", + False, + ), + # context —— the single compaction trigger ratio. Read per turn by + # compaction_service.resolve_trigger_ratio and applied by every phase (mid-turn at the ReAct step + # boundary, pre-turn before a turn samples, post-turn background pre-warm) — it is no longer an + # "in-turn only" knob competing with CHAT_COMPACT_TRIGGER_RATIO. Admin config takes precedence over + # env; takes effect on new conversations after saving, no restart needed. + # ⚠️ Grouped under context, not chat: the chat group is in service_configs._HIDDEN_GROUPS + # (chat.user_model_switch_enabled is exclusively managed by the model management panel to prevent a double entry point), + # so attaching to the chat group would hide the whole group in the system config UI. + ( + "chat.compress_in_turn_ratio", + "0.90", + "轮内压缩触发比例", + "上下文占用超过「模型窗口 × 该值」时触发压缩（0.5~0.90）。计数优先用模型返回的" + "真实 token 用量，拿不到时才退回字节估算，两者都不含模型输出长度——输出余量已由" + "「窗口 95% 可用」统一预留。调大=更晚压缩（上下文更完整）；调小=更早压缩。" + "该比例对轮内、轮前、轮末三个时机统一生效。", + "context", + False, + ), + # dingtalk —— DingTalk workbench Custom App (enterprise self-built app) credentials. Once AppKey/AppSecret are filled, + # DingTalk login and the dws command inside the sandbox execute as that self-built app, bypassing DingTalk's co-creation-era + # "CLI data access" org toggle and self-controlling scope; leave empty to use the dws default shared app (org admin must + # enable CLI data access first). Sole authority = admin config (no env fallback), takes effect ≤30s after saving, no restart. + ( + "dingtalk.client_id", + None, + "钉钉 AppKey (client-id)", + "企业自建钉钉应用的 AppKey。在 open-dev.dingtalk.com 创建企业内部应用后获取。", + "dingtalk", + True, + ), + ( + "dingtalk.client_secret", + None, + "钉钉 AppSecret (client-secret)", + "企业自建钉钉应用的 AppSecret，与 AppKey 配对。", + "dingtalk", + True, + ), + ( + "dingtalk.trusted_domains", + "*.dingtalk.com", + "受信域名白名单", + "dws bearer token 允许发送的域名，逗号分隔，默认锁死 *.dingtalk.com 防 token 外泄。", + "dingtalk", + False, + ), + # firecrawl —— deployment-level credentials for the firecrawl web scraping/retrieval CLI. The firecrawl CLI is preinstalled + # in the sandbox; at runtime these two items are injected as FIRECRAWL_API_KEY / FIRECRAWL_API_URL env vars into + # every user's sandbox commands (one shared config for all users, mirroring the DingTalk dws Custom App approach). + # - Cloud version: fill only api_key (starts with fc-). + # - Self-hosted: fill api_url pointing to the self-built instance (the CLI skips cloud auth once it sees api_url); if the + # self-built instance also needs a key, fill both. + # Sole authority = admin config (no env fallback), takes effect ≤30s after saving, no restart. + ( + "firecrawl.api_key", + None, + "Firecrawl API Key", + "firecrawl 云版 API Key（fc- 开头），在 firecrawl.dev 注册获取。用自托管实例时可留空。", + "firecrawl", + True, + ), + ( + "firecrawl.api_url", + None, + "Firecrawl 自托管地址", + "自托管 firecrawl 实例地址（如 http://firecrawl:3002）。填了即走自建实例、CLI 跳过云端鉴权；用云版时留空。", + "firecrawl", + False, + ), + # lark —— Feishu workbench app (enterprise self-built app) credentials. Once AppID/AppSecret are filled, the lark-cli inside + # the sandbox uses that self-built app as its OAuth app: admin configures once, and each user completes the device-flow + # auth login to operate Feishu as themselves. The app config is seeded by the backend into each user's lark HOME + # (config init), not injected as a global env var — this avoids lark-cli defaulting to the tenant/bot token flow when + # LARK_APP_* is in the environment, which would break --as user. Sole authority = admin config, ≤30s after saving, no restart. + ( + "lark.app_id", + None, + "飞书 App ID", + "企业自建飞书应用的 App ID。在 open.feishu.cn 创建企业自建应用后获取。", + "lark", + True, + ), + ( + "lark.app_secret", + None, + "飞书 App Secret", + "企业自建飞书应用的 App Secret，与 App ID 配对。", + "lark", + True, + ), + # auth —— login and session. Number of days a session is kept after the user checks "remember login state" (checked by + # default) on the login page. Sole authority = admin config (no env fallback), takes effect ≤30s after saving, no restart; + # if unchecked, the default session duration (SESSION_TTL_HOURS) is used. + ( + "auth.remember_me_days", + "30", + "记住登录状态时长(天)", + "用户在登录页勾选「记住登录状态」后，登录会话保持的天数（默认 30 天）。" + "未勾选则使用默认会话时长。", + "auth", + False, + ), + # turbo —— 极速模式（检索速查）。工具集独立于能力目录（catalog）的启停状态： + # 极速模式装配的 MCP 以本组配置为唯一来源，管理台改动 ≤30s 生效、无需重启。 + ( + "turbo.mcp_server_ids", + "retrieve_dataset_content,internet_search,web_fetch", + "极速模式可用工具", + "极速模式下装配的 MCP 工具集合（逗号分隔的 server id）。" + "该集合不受「能力目录」中 MCP/技能/智能体/插件启停的影响，是极速模式工具面的唯一来源。", + "turbo", + False, + ), + ( + "turbo.skill_ids", + "", + "极速模式可用技能", + "极速模式下装配的技能集合（逗号分隔的 skill id）。" + "极速模式没有 Bash / 沙箱 / 文件写入，选纯文本类技能（写作模板、话术规范等）效果最好；" + "需要代码执行的技能只会被读到说明、无法执行。同样不受「能力目录」启停影响。", + "turbo", + False, + ), + ( + "turbo.plugin_ids", + "", + "极速模式可用插件", + "极速模式下装配的插件集合（逗号分隔的 install_id）。" + "插件是「技能 + 工具」的能力包，选中后其全部工具与技能一并在极速模式生效；" + "与上面的 MCP 工具集合合并去重，同样不受「能力目录」启停影响。", + "turbo", + False, + ), + ( + "turbo.manual_invoke_enable", + "true", + "允许手动呼唤能力", + "开启后，用户在极速模式对话框中通过斜杠命令呼唤技能/插件、@ 呼唤子智能体时，" + "被呼唤的能力将临时装配到本次请求（仅显式呼唤时可用，不会把全量能力清单注入提示词）。" + "关闭后极速模式严格只用上面的工具集合。", + "turbo", + False, + ), + ( + "turbo.max_iters", + "4", + "迭代轮数上限", + "极速模式单次回答的最大 ReAct 迭代轮数（含最终作答轮）。" + "上限越小响应越快；默认 4 对应「最多 1-2 轮并行检索 + 直接作答」的产品契约。", + "turbo", + False, + ), +] + EDITION_SEED_CONFIGS + + +CONFIG_KEY_TO_ENV: dict[str, str] = { + "query_database.url": "QUERY_DATABASE_URL", + "query_database.timeout": "QUERY_DATABASE_TIMEOUT_SECONDS", + "query_database.retry_times": "QUERY_DATABASE_RETRY_TIMES", + "query_database.max_output_tokens": "QUERY_DATABASE_MAX_OUTPUT_TOKENS", + "knowledge_base.detail_max_chars": "KB_DETAIL_CONTENT_MAX_CHARS", + "file_parser.api_url": "FILE_PARSER_API_URL", + "file_parser.timeout": "FILE_PARSER_TIMEOUT", + "file_parser.lang_list": "FILE_PARSER_LANG_LIST", + "file_parser.backend": "FILE_PARSER_BACKEND", + "file_parser.parse_method": "FILE_PARSER_PARSE_METHOD", + "file_parser.formula_enable": "FILE_PARSER_FORMULA_ENABLE", + "file_parser.table_enable": "FILE_PARSER_TABLE_ENABLE", + "internet_search.engine": "INTERNET_SEARCH_ENGINE", + "internet_search.tavily_api_key": "TAVILY_API_KEY", + "internet_search.baidu_api_key": "BAIDU_API_KEY", + "internet_search.langsearch_api_key": "LANGSEARCH_API_KEY", + # Note: sandbox.code_capability_enable deliberately does **not** map to an env var —— the admin toggle is the sole + # authority (plan B), explicitly seeded to "true", and env CODE_CAPABILITY_ENABLED is retired. + **EDITION_CONFIG_KEY_TO_ENV, +} + + +ENV_TO_CONFIG_KEY: dict[str, str] = {v: k for k, v in CONFIG_KEY_TO_ENV.items()} + + +DEFAULT_TURBO_MCP_SERVER_IDS = frozenset( + {"retrieve_dataset_content", "internet_search", "web_fetch"} +) diff --git a/src/backend/core/config/user_intros.py b/src/backend/core/config/user_intros.py index 295dfccf..54e1bb21 100644 --- a/src/backend/core/config/user_intros.py +++ b/src/backend/core/config/user_intros.py @@ -20,7 +20,7 @@ from typing import Dict -from core.config.edition_display_names import edition_mcp_user_intros +from core.config.edition import edition_mcp_user_intros # ── Skills ──────────────────────────────────────────────────────────── diff --git a/src/backend/core/content/content_blocks.py b/src/backend/core/content/content_blocks.py deleted file mode 100644 index 273ff82e..00000000 --- a/src/backend/core/content/content_blocks.py +++ /dev/null @@ -1,701 +0,0 @@ -"""Helpers for exporting and importing editable docs content blocks.""" - -from __future__ import annotations - -import re -from datetime import datetime, timezone -from typing import Any, Mapping - -from core.config.settings import settings -from core.db.models import ContentBlock -from sqlalchemy.orm import Session - -# Branding source of truth (seam C7): in-code defaults stay neutral; the concrete -# brand (e.g. "HugAgentOS") is injected via env BRAND_PRODUCT_NAME / the deploy-time -# content_blocks DB seed. -_BRAND_NAME = settings.branding.product_name - -DOCS_BLOCK_MAP = { - "updates": "docs_updates", - "capabilities": "docs_capabilities", - "prompt_hub": "prompt_hub", - "page_config": "page_config", - "app_config": "app_config", -} - -# Aliases retired from DOCS_BLOCK_MAP; ignored (not rejected) when importing an -# older snapshot that still carries them. -RETIRED_BLOCK_ALIASES = {"homepage_shortcuts"} - -# Blocks whose payload is a dict instead of a list (default is list). -DICT_PAYLOAD_BLOCKS = {"page_config", "prompt_versions", "app_config"} - -# Prompt versions block — managed separately from DOCS_BLOCK_MAP so a routine -# docs export/import never silently clobbers the system prompt pool. It rides -# its own opt-in snapshot (build_prompt_snapshot / import_prompt_snapshot). -PROMPT_VERSIONS_BLOCK_ID = "prompt_versions" - -# Opt-in migration set for prompt content: the system prompt version pool -# (prompt_versions) plus the prompt hub (prompt_hub). Same ContentBlock storage -# and snapshot format as docs blocks, but a separate map keeps the two apart so -# a prompts migration never drags page_config / branding fields along with it. -# Note: prompt_hub also appears in DOCS_BLOCK_MAP — that is intentional and -# harmless (a docs snapshot and a prompts snapshot are imported independently). -PROMPT_BLOCK_MAP = { - "prompt_versions": PROMPT_VERSIONS_BLOCK_ID, - "prompt_hub": "prompt_hub", -} - -SNAPSHOT_SCHEMA_VERSION = 1 - - -class ContentSnapshotError(ValueError): - """Raised when a docs content snapshot is invalid.""" - - -DEFAULT_PAGE_CONFIG: dict[str, Any] = { - "branding": { - "product_name": _BRAND_NAME, - "product_subtitle": "AI 智能助手", - "logo_url": "/home/header.svg", - "favicon_url": "/icon.png", - "page_title": _BRAND_NAME, - "hero_title": f"你好，我是 {_BRAND_NAME}", - "hero_subtitle": "今天想从哪里开始？", - "disclaimer": "本平台生成内容由AI大模型生成，不构成任何建议；涉及业务决策请以权威信息为准。", - }, - "homepage": { - # The homepage uses the transparent standalone brand mark rather than - # the square sidebar logo, mirroring the lighter Qwen Office hero. - "show_logo": True, - "logo_url": "/icon.png", - "show_suggestions": True, - "suggested_questions": [ - "分析新能源汽车产业链的关键环节与代表企业", - "对比两份产业政策，提炼支持方向和申报条件", - "查询一家企业的基本情况、经营风险和产业链位置", - "从这份材料中提炼核心观点和待办事项", - "分析这组经营数据的趋势、异常和可能原因", - "检索相关政策依据，并给出可核验的来源", - ], - }, - "navigation": { - "panel_titles": { - "ability_center": "能力中心", - "skills": "技能", - "agents": "智能体", - "mcp": "连接器", - "kb": "知识库", - "docs": "更新记录", - "app_center": "应用中心", - "automation": "定时任务", - "sites": "站点", - "projects": "项目", - "lab": "实验室", - "settings": "系统设置", - "my_space": "我的空间", - }, - "panel_subtitles": { - "ability_center": "智能体基础能力管理，包含智能体、技能、连接器与插件", - "skills": "启用/停用技能，并查看详细介绍、输入输出与示例。", - "agents": "选择与启用智能体，并查看其职责边界与路由提示。", - "mcp": "管理连接器服务，并查看其作用范围与可靠性影响。", - "kb": "浏览知识库、查看文档列表，并支持文档内检索。", - "docs": "查看功能更新、能力中心与平台说明。", - "app_center": "基于 AI 能力的场景化智能应用", - "automation": "按计划自动执行任务，也可随时手动触发。在任意对话中描述你想定期做的事，即可快速创建", - "sites": "在对话里描述需求，AI 生成完整网站并一键发布，由平台托管、凭链接即可访问", - "projects": "把对话、文件和指令打包成专属工作空间", - "lab": "AI 能力实验性应用", - "settings": "", - "my_space": "", - }, - "admin_header": { - "title": f"{_BRAND_NAME} — 后台管理", - "subtitle": "后台管理", - }, - # Unified branding + per-page labels for the admin platform (content - # management / system config / API docs) — product_name drives the header - # brand and browser tab title of all three admin pages, replacing the old - # admin_header. - "admin_platform": { - "product_name": _BRAND_NAME, - "content_label": "内容管理", - "config_label": "系统配置", - "apidoc_label": "接口文档", - }, - # 与前端 utils/pageConfigDefaults.ts 的 DEFAULT_SIDEBAR_ITEMS / DEFAULT_MENU_ITEMS 保持一致。 - # 这只是新装环境的出厂值——已存在的部署以 DB 现有取值为准，由 backfill_navigation_entries - # 补齐新增条目，管理员随后可在 /config「页面配置」里自由编排。 - "sidebar_items": ["ability_center", "automation", "sites", "my_space"], - "menu_items": ["settings", "app_center", "projects", "lab"], - }, - "texts": { - "input_placeholder": "请输入您的问题…", - "input_placeholder_agent": "请输入您的问题…", - "search_placeholder": "搜索对话", - "btn_new_chat": "新建对话", - "btn_logout": "退出", - "history_label": "历史对话", - "sidebar_empty_state": "暂无对话记录", - "search_no_results": "无匹配结果", - "dialog_logout_confirm_title": "确认退出登录？", - "dialog_logout_confirm_content": "退出登录不会丢失任何数据，重新登录后可继续使用。", - "dialog_logout_confirm_ok": "退出登录", - "recommend_banner_text": "", - }, - "defaults": { - # Initial mode on every user login / new chat: - # - chat_mode: 'turbo' / 'fast' / 'medium' / 'high' / 'max' (recommended) - # - thinking_mode: bool (legacy field, fallback compat; true→medium, false→fast) - "chat_mode": "fast", - "thinking_mode": False, - }, - "auth": { - # Whether the login page shows the "register" entry (tab + register subpage). - # When off, the login page keeps only login, and backend register submissions - # are rejected outright (not just hidden on the frontend). - "allow_register": True, - }, -} - - -# No external sub-apps are built in by default — the app config is entirely managed -# by admins via the "app config" panel. (Historically two presets, "enterprise -# profile / enterprise survey", were seeded here, which caused deleted apps to be -# re-injected on refresh.) -DEFAULT_APP_CONFIG: dict[str, Any] = { - "apps": [], -} - - -def normalize_app_config(payload: Any) -> dict[str, Any]: - """Normalize app_config payload to the canonical `{"apps": [...]}` shape. - - An empty list (admin deleted all apps) is respected as "no apps" and no longer - falls back to any default entries. - """ - if not isinstance(payload, dict): - return {"apps": list(DEFAULT_APP_CONFIG["apps"])} - - apps_raw = payload.get("apps") - if isinstance(apps_raw, list): - apps: list[dict[str, Any]] = [] - for item in apps_raw: - if not isinstance(item, dict): - continue - app_id = str(item.get("id") or "").strip() - if not app_id: - continue - apps.append( - { - "id": app_id, - "enabled": bool(item.get("enabled", True)), - "name": str(item.get("name") or app_id), - "description": str(item.get("description") or ""), - "url": str(item.get("url") or ""), - "icon": str(item.get("icon") or ""), - } - ) - return {"apps": apps} - - return {"apps": list(DEFAULT_APP_CONFIG["apps"])} - - -# 空的初始负载：哪些 kind 存在、默认版本叫什么，由 -# prompt_version_service.KIND_SPECS 单独持有并在 seed 时写入，这里不再抄一份清单。 -DEFAULT_PROMPT_VERSIONS: dict[str, Any] = { - "active": {}, - "versions": [], -} - - -def _default_payload(alias: str) -> Any: - if alias == "page_config": - return DEFAULT_PAGE_CONFIG - if alias == "app_config": - return DEFAULT_APP_CONFIG - if alias == "prompt_versions": - return DEFAULT_PROMPT_VERSIONS - return {} if alias in DICT_PAYLOAD_BLOCKS else [] - - -def seed_page_config_if_missing(db: Session) -> bool: - """Insert default page_config row if it does not yet exist. Returns True if inserted.""" - row = db.query(ContentBlock).filter(ContentBlock.id == "page_config").first() - if row: - return False - db.add( - ContentBlock( - id="page_config", - payload=DEFAULT_PAGE_CONFIG, - updated_at=datetime.now(timezone.utc), - updated_by="system_seed", - ) - ) - db.commit() - return True - - -def enforce_ce_branding(db: Session) -> bool: - """Keep the fixed CE product identity and auth policy consistent. - - ``page_config`` survives image upgrades, so merely changing the code/env - defaults does not repair an already-running CE instance. CE does not expose - the content-management console and therefore has no supported per-instance - branding override or self-service registration; normalize both the small - set of branding fields and the registration switch on startup. EE - deployments are never touched. - """ - if settings.edition.edition != "ce": - return False - - row = db.query(ContentBlock).filter(ContentBlock.id == "page_config").first() - if row is None or not isinstance(row.payload, dict): - return False - - product_name = "HugAgentOS" - payload = dict(row.payload) - branding = dict(payload.get("branding") or {}) - navigation = dict(payload.get("navigation") or {}) - auth = dict(payload.get("auth") or {}) - admin_header = dict(navigation.get("admin_header") or {}) - admin_platform = dict(navigation.get("admin_platform") or {}) - - desired_branding = { - "product_name": product_name, - "product_subtitle": "AI 智能助手", - "page_title": product_name, - "hero_title": f"你好，我是 {product_name}", - } - changed = any(branding.get(key) != value for key, value in desired_branding.items()) - changed = changed or admin_header.get("title") != f"{product_name} — 后台管理" - changed = changed or admin_platform.get("product_name") != product_name - changed = changed or auth.get("allow_register") is not False - if not changed: - return False - - branding.update(desired_branding) - admin_header["title"] = f"{product_name} — 后台管理" - admin_platform["product_name"] = product_name - navigation["admin_header"] = admin_header - navigation["admin_platform"] = admin_platform - auth["allow_register"] = False - payload["branding"] = branding - payload["navigation"] = navigation - payload["auth"] = auth - row.payload = payload - row.updated_at = datetime.now(timezone.utc) - row.updated_by = "system_seed" - from sqlalchemy.orm.attributes import flag_modified - - flag_modified(row, "payload") - db.commit() - return True - - -# ── Idempotent backfill for navigation entries added after initial deploy ─ -# Lesson learned: when the projects MVP shipped, only the frontend defaults and the -# backend DEFAULT_PAGE_CONFIG were changed, but **already existing** page_config rows -# do not upgrade automatically — the frontend's "use DB if present" override -# semantics pinned the old sidebar_items in place. This repeated for every new -# entry. The idempotent seeder is the safety net: on startup, backfill missing nav -# entries from a whitelist. -# -# Design principles: -# - Explicit whitelist; do **not** deep-merge the full defaults — avoids -# "resurrecting" entries operators deleted on purpose (e.g. someone deliberately -# turned off app_center) and avoids overwriting branding fields in reverse. -# - Each entry carries an insert_after anchor; if the anchor is absent, degrade to -# appending at the end. -# - ``bucket`` picks which list the entry is seeded into ("sidebar" / "menu"). -# - **Presence is checked across BOTH lists.** An operator who moved an entry from the -# sidebar into the user menu (or vice versa) has not "lost" it — re-inserting it into -# the bucket we happen to prefer would silently undo their placement on every restart -# and leave the entry duplicated in two buckets. Only a genuinely absent key is seeded. -# (Deliberately hidden entries live in neither list, so they *do* come back — that is -# the documented trade-off of this safety net; use the admin UI to hide them again.) -# - Titles/subtitles are **not** repeated here — they are read from DEFAULT_PAGE_CONFIG above, -# so panel copy has exactly one home and a fresh install can never disagree with a -# backfilled one. -# - On failure, quietly return False; never block startup. -_NAV_BACKFILL_ENTRIES: list[dict[str, Any]] = [ - {"key": "projects", "bucket": "menu", "insert_after": "app_center"}, - {"key": "automation", "bucket": "sidebar", "insert_after": "ability_center"}, - {"key": "sites", "bucket": "sidebar", "insert_after": "automation"}, -] - - -# 能力中心统一表述（技能库→技能 / 子智能体→智能体 / MCP工具库→连接器 / 插件库→插件）之前 -# 建起来的部署，DB 里存的还是旧出厂文案。这里做一次性改写：**仅当**存量取值与旧出厂默认 -# 完全一致时才改，管理员自己改过的标题原样保留。 -_LEGACY_PANEL_TEXT_RENAMES: dict[str, dict[str, tuple[tuple[str, ...], str]]] = { - "panel_titles": { - "skills": (("技能库",), "技能"), - "agents": (("子智能体",), "智能体"), - "plugins": (("插件库",), "插件"), - "mcp": (("MCP工具库", "MCP 工具库", "MCP 工具"), "连接器"), - }, - "panel_subtitles": { - "agents": ( - ("选择与启用子智能体，并查看其职责边界与路由提示。",), - "选择与启用智能体，并查看其职责边界与路由提示。", - ), - "mcp": ( - ( - "管理 MCP 工具服务，并查看其作用范围与可靠性影响。", - "管理 MCP 连接器服务，并查看其作用范围与可靠性影响。", - ), - "管理连接器服务，并查看其作用范围与可靠性影响。", - ), - }, -} - - -def backfill_navigation_entries(db: Session) -> int: - """Patch existing page_config row to add navigation entries listed in - ``_NAV_BACKFILL_ENTRIES`` if they are missing. - - Returns the number of *individual fields* mutated (across all entries - and all three locations: sidebar_items / panel_titles / panel_subtitles). - Zero means nothing changed (idempotent steady state). - """ - row = db.query(ContentBlock).filter(ContentBlock.id == "page_config").first() - if not row: - return 0 # seed_page_config_if_missing handles fresh installs - payload = row.payload if isinstance(row.payload, dict) else {} - nav = payload.get("navigation") - if not isinstance(nav, dict): - return 0 # malformed payload — leave it alone, admin UI will surface - - default_nav = DEFAULT_PAGE_CONFIG["navigation"] - - changed = 0 - for entry in _NAV_BACKFILL_ENTRIES: - key = entry["key"] - bucket_field = "menu_items" if entry["bucket"] == "menu" else "sidebar_items" - items = nav.get(bucket_field) - # Present in *either* list counts as placed: the operator may have moved the entry - # between buckets on purpose, and re-seeding it would duplicate it across both. - already_placed = any( - key in value - for field in ("sidebar_items", "menu_items") - if isinstance(value := nav.get(field), list) - ) - if isinstance(items, list) and not already_placed: - anchor = entry.get("insert_after") - new_items = list(items) - if anchor and anchor in new_items: - new_items.insert(new_items.index(anchor) + 1, key) - else: - new_items.append(key) - nav[bucket_field] = new_items - changed += 1 - - # Copy comes from DEFAULT_PAGE_CONFIG so it is never re-typed here. - for field in ("panel_titles", "panel_subtitles"): - target = nav.get(field) - text = default_nav[field].get(key) - if isinstance(target, dict) and key not in target and text: - target[key] = text - changed += 1 - - # 旧出厂文案改名（能力中心统一表述），与上面的补齐共用一次提交 - for field, renames in _LEGACY_PANEL_TEXT_RENAMES.items(): - target = nav.get(field) - if not isinstance(target, dict): - continue - for key, (legacy_values, new_text) in renames.items(): - if target.get(key) in legacy_values: - target[key] = new_text - changed += 1 - - if changed: - payload["navigation"] = nav - row.payload = payload - row.updated_at = datetime.now(timezone.utc) - # Mark updated_by so audit can distinguish automatic backfill vs manual edits - row.updated_by = "system_seed" - # SQLAlchemy JSONB does not auto-detect mutation — flag explicitly - from sqlalchemy.orm.attributes import flag_modified - - flag_modified(row, "payload") - db.commit() - return changed - - -def get_branding_info() -> dict[str, str]: - """Read the branding fields from page_config (product_name + logo_url), falling back to defaults on failure. - - Reused by server-side rendering paths (e.g. the SSO login page); manages its - own DB session internally. - """ - defaults = { - "product_name": DEFAULT_PAGE_CONFIG["branding"]["product_name"], - "logo_url": DEFAULT_PAGE_CONFIG["branding"]["logo_url"], - } - try: - from core.db.engine import SessionLocal - - db = SessionLocal() - try: - row = db.query(ContentBlock).filter(ContentBlock.id == "page_config").first() - payload = (row.payload if row else {}) or {} - branding = payload.get("branding", {}) if isinstance(payload, dict) else {} - return { - "product_name": branding.get("product_name") or defaults["product_name"], - "logo_url": branding.get("logo_url") or defaults["logo_url"], - } - finally: - db.close() - except Exception: - return defaults - - -def is_register_allowed() -> bool: - """Return whether self-service local-account registration is enabled. - - CE is permanently single-account and therefore always returns ``False``. - Other editions read ``page_config.auth.allow_register`` and keep the legacy - default of ``True`` when the setting is missing or cannot be read. The value - gates both the server-rendered UI and registration submissions. - """ - if settings.edition.edition == "ce": - return False - - try: - from core.db.engine import SessionLocal - - db = SessionLocal() - try: - row = db.query(ContentBlock).filter(ContentBlock.id == "page_config").first() - payload = (row.payload if row else {}) or {} - auth = payload.get("auth", {}) if isinstance(payload, dict) else {} - val = auth.get("allow_register") if isinstance(auth, dict) else None - return True if val is None else bool(val) - finally: - db.close() - except Exception: - return True - - -def get_admin_platform_info() -> dict[str, str]: - """Read page_config.navigation.admin_platform (admin platform brand + per-page labels). - - When admin_platform is missing, derive the brand from the legacy - admin_header.title (shaped like " — 后台管理") as the product_name - fallback, then fall back to code defaults — consistent with the frontend - mergeAdminPlatform normalization. Reused by server-side rendering paths - (Swagger / ReDoc doc page titles etc.); manages its own DB session internally. - """ - ap_def = DEFAULT_PAGE_CONFIG["navigation"]["admin_platform"] - defaults = { - "product_name": ap_def["product_name"], - "content_label": ap_def["content_label"], - "config_label": ap_def["config_label"], - "apidoc_label": ap_def["apidoc_label"], - } - try: - from core.db.engine import SessionLocal - - db = SessionLocal() - try: - row = db.query(ContentBlock).filter(ContentBlock.id == "page_config").first() - payload = (row.payload if row else {}) or {} - nav = payload.get("navigation", {}) if isinstance(payload, dict) else {} - ap = nav.get("admin_platform") if isinstance(nav, dict) else None - ap = ap if isinstance(ap, dict) else {} - header = nav.get("admin_header") if isinstance(nav, dict) else None - derived = "" - if isinstance(header, dict) and isinstance(header.get("title"), str): - derived = re.split(r"\s+[—-]\s+", header["title"])[0].strip() - - def pick(key: str, fallback: str) -> str: - v = ap.get(key) - return v.strip() if isinstance(v, str) and v.strip() else fallback - - return { - "product_name": pick("product_name", derived or defaults["product_name"]), - "content_label": pick("content_label", defaults["content_label"]), - "config_label": pick("config_label", defaults["config_label"]), - "apidoc_label": pick("apidoc_label", defaults["apidoc_label"]), - } - finally: - db.close() - except Exception: - return defaults - - -def _serialize_block(row: ContentBlock | None, alias: str | None = None) -> dict[str, Any]: - return { - "payload": row.payload if row else _default_payload(alias or ""), - "updated_at": row.updated_at.isoformat() if row and row.updated_at else None, - "updated_by": row.updated_by if row else None, - } - - -def build_docs_snapshot( - db: Session, - block_map: Mapping[str, str] = DOCS_BLOCK_MAP, -) -> dict[str, Any]: - """Build a portable JSON snapshot for content blocks. - - Defaults to the docs blocks; pass ``block_map=PROMPT_BLOCK_MAP`` (or use - ``build_prompt_snapshot``) to snapshot the system prompt pool instead. - """ - rows = db.query(ContentBlock).filter(ContentBlock.id.in_(list(block_map.values()))).all() - row_map = {row.id: row for row in rows} - - return { - "schema_version": SNAPSHOT_SCHEMA_VERSION, - "exported_at": datetime.now(timezone.utc).isoformat(), - "blocks": { - alias: _serialize_block(row_map.get(db_id), alias) for alias, db_id in block_map.items() - }, - } - - -def _parse_snapshot_datetime(value: Any) -> datetime | None: - if value in (None, ""): - return None - if not isinstance(value, str): - raise ContentSnapshotError("updated_at must be an ISO datetime string") - try: - return datetime.fromisoformat(value.replace("Z", "+00:00")) - except ValueError as exc: - raise ContentSnapshotError(f"Invalid datetime: {value}") from exc - - -def normalize_docs_snapshot( - snapshot: Mapping[str, Any], - block_map: Mapping[str, str] = DOCS_BLOCK_MAP, -) -> dict[str, Any]: - """Validate and normalize an incoming content snapshot.""" - if not isinstance(snapshot, Mapping): - raise ContentSnapshotError("Snapshot body must be a JSON object") - - schema_version = snapshot.get("schema_version", SNAPSHOT_SCHEMA_VERSION) - if schema_version != SNAPSHOT_SCHEMA_VERSION: - raise ContentSnapshotError( - f"Unsupported schema_version: {schema_version}. Expected {SNAPSHOT_SCHEMA_VERSION}" - ) - - raw_blocks = snapshot.get("blocks") - if not isinstance(raw_blocks, Mapping): - raise ContentSnapshotError("Snapshot.blocks must be an object") - - unknown_blocks = set(raw_blocks.keys()) - set(block_map.keys()) - RETIRED_BLOCK_ALIASES - if unknown_blocks: - raise ContentSnapshotError( - f"Unknown blocks in snapshot: {', '.join(sorted(unknown_blocks))}" - ) - - normalized_blocks: dict[str, dict[str, Any]] = {} - for alias in block_map: - raw_block = raw_blocks.get(alias) - if raw_block is None: - continue - if not isinstance(raw_block, Mapping): - raise ContentSnapshotError(f"Snapshot block '{alias}' must be an object") - - payload = raw_block.get("payload", _default_payload(alias)) - expected_type = dict if alias in DICT_PAYLOAD_BLOCKS else list - if not isinstance(payload, expected_type): - kind = "object" if expected_type is dict else "list" - raise ContentSnapshotError(f"Snapshot block '{alias}'.payload must be a {kind}") - - normalized_blocks[alias] = { - "payload": payload, - "updated_at": _parse_snapshot_datetime(raw_block.get("updated_at")), - "updated_by": raw_block.get("updated_by"), - } - - if not normalized_blocks: - raise ContentSnapshotError("Snapshot does not contain any importable content blocks") - - exported_at = snapshot.get("exported_at") - if exported_at not in (None, "") and not isinstance(exported_at, str): - raise ContentSnapshotError("Snapshot.exported_at must be a string") - - return { - "schema_version": SNAPSHOT_SCHEMA_VERSION, - "exported_at": exported_at, - "blocks": normalized_blocks, - } - - -def import_docs_snapshot( - db: Session, - snapshot: Mapping[str, Any], - *, - overwrite: bool = True, - default_updated_by: str | None = None, - block_map: Mapping[str, str] = DOCS_BLOCK_MAP, -) -> dict[str, Any]: - """Import a content snapshot into the current database.""" - normalized = normalize_docs_snapshot(snapshot, block_map=block_map) - imported: list[str] = [] - skipped: list[str] = [] - - for alias, block in normalized["blocks"].items(): - db_id = block_map[alias] - row = db.query(ContentBlock).filter(ContentBlock.id == db_id).first() - if row and not overwrite: - skipped.append(alias) - continue - - updated_at = block["updated_at"] or datetime.now(timezone.utc) - updated_by = block["updated_by"] or default_updated_by - - if row: - row.payload = block["payload"] - row.updated_at = updated_at - row.updated_by = updated_by - else: - row = ContentBlock( - id=db_id, - payload=block["payload"], - updated_at=updated_at, - updated_by=updated_by, - ) - db.add(row) - imported.append(alias) - - db.commit() - - return { - "schema_version": normalized["schema_version"], - "imported": imported, - "skipped": skipped, - "count": len(imported), - "overwrite": overwrite, - } - - -# ── System prompt pool snapshot (opt-in migration) ────────────────────────── - - -def build_prompt_snapshot(db: Session) -> dict[str, Any]: - """Build a portable snapshot of the system prompt pool (prompt_versions). - - Same format as the docs snapshot, but scoped to PROMPT_BLOCK_MAP so it can - be migrated across environments independently of docs/page config. - """ - return build_docs_snapshot(db, block_map=PROMPT_BLOCK_MAP) - - -def import_prompt_snapshot( - db: Session, - snapshot: Mapping[str, Any], - *, - overwrite: bool = True, - default_updated_by: str | None = None, -) -> dict[str, Any]: - """Import a system prompt pool snapshot (counterpart of build_prompt_snapshot).""" - return import_docs_snapshot( - db, - snapshot, - overwrite=overwrite, - default_updated_by=default_updated_by, - block_map=PROMPT_BLOCK_MAP, - ) diff --git a/src/backend/core/content/file_parser.py b/src/backend/core/content/file_parser.py deleted file mode 100644 index 1e2ebb16..00000000 --- a/src/backend/core/content/file_parser.py +++ /dev/null @@ -1,752 +0,0 @@ -"""File parsing utilities for chat attachments. - -Supported formats: - - PDF : calls external file-parser API service - - DOCX : pandoc (docx → markdown) - - DOC / WPS : LibreOffice headless → DOCX, then pandoc - - TXT : UTF-8 / GBK direct decode - - XLSX : openpyxl → markdown tables - - XLS : xlrd → markdown tables (fallback: LibreOffice → XLSX → openpyxl) - - CSV : csv module → markdown table - -All public functions return a markdown/plain-text string, or raise RuntimeError -with a human-readable Chinese error message on failure. -""" - -from __future__ import annotations - -import base64 -import binascii -import csv -import io -import json -import logging -import os -import subprocess -import tempfile -from dataclasses import dataclass, field -from typing import Optional - -import requests - -from core.content.office import find_libreoffice_binary, find_pandoc_binary -from core.infra.proc import no_window_kwargs - -logger = logging.getLogger(__name__) - - -# ── config helpers (DB-first, env fallback) ────────────────────────────────── - -def _svc_get(key: str, default: str = "") -> str: - """Read from SystemConfigService (DB-first), fall back to env.""" - try: - from core.services.system_config import SystemConfigService - val = SystemConfigService.get_instance().get(key) - if val is not None: - return val.strip() - except Exception: - pass - return (os.getenv(_ENV_MAP.get(key, ""), default) or default).strip() - -# config_key → env var fallback mapping -_ENV_MAP = { - "file_parser.api_url": "FILE_PARSER_API_URL", - "file_parser.timeout": "FILE_PARSER_TIMEOUT", - "file_parser.lang_list": "FILE_PARSER_LANG_LIST", - "file_parser.backend": "FILE_PARSER_BACKEND", - "file_parser.parse_method": "FILE_PARSER_PARSE_METHOD", - "file_parser.formula_enable": "FILE_PARSER_FORMULA_ENABLE", - "file_parser.table_enable": "FILE_PARSER_TABLE_ENABLE", -} - - -def _cfg_api_url() -> str: - return _svc_get("file_parser.api_url") - - -def _cfg_timeout() -> int: - try: - return int(_svc_get("file_parser.timeout", "60")) - except ValueError: - return 60 - - -def _cfg_parse_params() -> dict: - return { - "lang_list": _svc_get("file_parser.lang_list", "ch"), - "backend": _svc_get("file_parser.backend", "pipeline"), - "parse_method": _svc_get("file_parser.parse_method", "auto"), - "formula_enable": _svc_get("file_parser.formula_enable", "true"), - "table_enable": _svc_get("file_parser.table_enable", "true"), - } - - -# ── PDF ─────────────────────────────────────────────────────────────────────── - - -@dataclass -class RichParseResult: - """Layout-aware parse output: markdown plus the media the layout engine cut out. - - ``markdown`` is byte-for-byte what :func:`parse_pdf` returns, so the text path is - unaffected. ``images`` maps the path used inside the markdown (``images/.jpg`` - in a ``![](...)`` link) to the decoded bytes, and ``blocks`` is the parsed - ``content_list`` — one entry per layout block, images carrying ``image_caption`` / - ``bbox`` / ``page_idx``. - """ - - markdown: str - images: dict[str, bytes] = field(default_factory=dict) - blocks: list[dict] = field(default_factory=list) - - -def _post_parse(file_bytes: bytes, filename: str, mime: str, *, rich: bool) -> dict: - """POST to the external parse service and return the single document's result dict. - - ``rich`` additionally requests the media and layout payloads. They are opt-in - because the images come back base64-inlined in the JSON body — several MB for an - image-heavy PDF — which the chat-attachment path has no use for. - """ - api_url = _cfg_api_url() - if not api_url: - raise RuntimeError("FILE_PARSER_API_URL 未配置，无法解析 PDF 文件") - - timeout = _cfg_timeout() - data = _cfg_parse_params() - if rich: - data = {**data, "return_images": "true", "return_content_list": "true"} - - try: - resp = requests.post( - api_url, - files={"files": (filename, file_bytes, mime)}, - data=data, - timeout=timeout, - ) - resp.raise_for_status() - except requests.exceptions.Timeout: - raise RuntimeError(f"PDF 解析服务超时（{timeout}s），请稍后重试") - except requests.exceptions.RequestException as e: - raise RuntimeError(f"PDF 解析服务请求失败: {e}") - - results = (resp.json() or {}).get("results", {}) - if not results: - raise RuntimeError("PDF 解析服务返回结果为空") - return results[next(iter(results))] or {} - - -def _decode_data_uri(value: str) -> Optional[bytes]: - """Decode one ``images`` entry. The service returns ``data:image/jpeg;base64,...``. - - A bare base64 payload (no data-URI prefix) is accepted too, so a service variant - that omits the prefix still works. Undecodable entries are dropped rather than - failing the whole document. - """ - if not isinstance(value, str) or not value: - return None - payload = value.split(",", 1)[1] if value.startswith("data:") else value - try: - return base64.b64decode(payload, validate=False) - except (binascii.Error, ValueError): - return None - - -def _parse_content_list(raw) -> list[dict]: - """Normalise ``content_list`` — the service returns it as a JSON **string**.""" - if isinstance(raw, list): - return [b for b in raw if isinstance(b, dict)] - if isinstance(raw, str) and raw.strip(): - try: - data = json.loads(raw) - except ValueError: - return [] - if isinstance(data, list): - return [b for b in data if isinstance(b, dict)] - return [] - - -def parse_pdf(file_bytes: bytes, filename: str = "file.pdf") -> str: - """Parse PDF via external file-parser API. Returns markdown text.""" - content = _post_parse(file_bytes, filename, "application/pdf", rich=False).get("md_content", "") - if not content: - raise RuntimeError("PDF 解析服务返回内容为空") - return content - - -def parse_pdf_rich(file_bytes: bytes, filename: str = "file.pdf") -> RichParseResult: - """Parse PDF keeping the figures the layout engine cut out. - - Same call as :func:`parse_pdf` plus ``return_images`` / ``return_content_list``. - Used by knowledge-base indexing to build image assets; the chat-attachment path - stays on the text-only call. - """ - result = _post_parse(file_bytes, filename, "application/pdf", rich=True) - markdown = result.get("md_content", "") - if not markdown: - raise RuntimeError("PDF 解析服务返回内容为空") - - images: dict[str, bytes] = {} - for name, value in (result.get("images") or {}).items(): - decoded = _decode_data_uri(value) - if decoded: - # The markdown references the file under an ``images/`` prefix while the - # payload is keyed by bare filename; index both so lookups by either work. - images[str(name)] = decoded - images.setdefault(f"images/{name}", decoded) - - return RichParseResult( - markdown=markdown, - images=images, - blocks=_parse_content_list(result.get("content_list")), - ) - - -# ── DOCX (via pandoc) ───────────────────────────────────────────────────────── - -def _docx_bytes_to_markdown(docx_bytes: bytes) -> str: - """Convert DOCX bytes to markdown string via pandoc.""" - with tempfile.TemporaryDirectory() as tmpdir: - input_path = os.path.join(tmpdir, "input.docx") - with open(input_path, "wb") as f: - f.write(docx_bytes) - - try: - result = subprocess.run( - [find_pandoc_binary() or "pandoc", input_path, "-t", "markdown", "--wrap=none"], - capture_output=True, - text=True, - timeout=60, - **no_window_kwargs(), - ) - except FileNotFoundError: - raise RuntimeError("pandoc 未安装，无法解析 Word 文档") - except subprocess.TimeoutExpired: - raise RuntimeError("pandoc 转换超时") - - if result.returncode != 0: - raise RuntimeError(f"pandoc 转换失败: {result.stderr[:300]}") - - return result.stdout - - -def parse_docx(file_bytes: bytes) -> str: - """Parse DOCX bytes → markdown via pandoc.""" - return _docx_bytes_to_markdown(file_bytes) - - -# ── DOC / WPS (LibreOffice → DOCX → pandoc) ────────────────────────────────── - -def _convert_to_docx_bytes(file_bytes: bytes, suffix: str) -> bytes: - """Use LibreOffice headless to convert DOC/WPS bytes → DOCX bytes.""" - with tempfile.TemporaryDirectory() as tmpdir: - input_path = os.path.join(tmpdir, f"input{suffix}") - with open(input_path, "wb") as f: - f.write(file_bytes) - - try: - result = subprocess.run( - [ - find_libreoffice_binary() or "libreoffice", - "--headless", - "--convert-to", "docx", - "--outdir", tmpdir, - input_path, - ], - capture_output=True, - text=True, - timeout=120, - **no_window_kwargs(), - ) - except FileNotFoundError: - raise RuntimeError("LibreOffice 未安装，无法解析 DOC/WPS 文件") - except subprocess.TimeoutExpired: - raise RuntimeError("LibreOffice 转换超时") - - output_path = os.path.join(tmpdir, "input.docx") - if not os.path.exists(output_path): - raise RuntimeError(f"LibreOffice 转换失败: {result.stderr[:300]}") - - with open(output_path, "rb") as f: - return f.read() - - -def parse_doc_wps(file_bytes: bytes, suffix: str) -> str: - """Parse DOC/WPS via LibreOffice → DOCX → pandoc markdown.""" - docx_bytes = _convert_to_docx_bytes(file_bytes, suffix) - return _docx_bytes_to_markdown(docx_bytes) - - -# ── TXT ─────────────────────────────────────────────────────────────────────── - -def parse_txt(file_bytes: bytes) -> str: - """Decode text file bytes, trying UTF-8 then GBK then latin-1.""" - for enc in ("utf-8", "gbk", "latin-1"): - try: - return file_bytes.decode(enc) - except UnicodeDecodeError: - continue - return file_bytes.decode("utf-8", errors="replace") - - -# ── XLSX ────────────────────────────────────────────────────────────────────── - -def _deduplicate_headers(header_raw: list[str]) -> list[str]: - """Disambiguate empty/duplicate header names so {placeholder} rendering - downstream is unambiguous (e.g., two ``""`` columns become ``col``/``col_1``). - """ - seen: dict[str, int] = {} - cleaned: list[str] = [] - for h in header_raw: - base = h if h else "col" - if base in seen: - seen[base] += 1 - cleaned.append(f"{base}_{seen[base]}") - else: - seen[base] = 0 - cleaned.append(base) - return cleaned - - -def _rows_to_markdown(header: list[str], data_rows: list[list[str]], - sheet_title: str | None = None) -> str: - """Convert header + data rows into a markdown table string.""" - lines: list[str] = [] - if sheet_title: - lines.append(f"## Sheet: {sheet_title}") - lines.append("") - lines.append("| " + " | ".join(header) + " |") - lines.append("|" + "|".join("---" for _ in header) + "|") - for row in data_rows: - lines.append("| " + " | ".join(row) + " |") - return "\n".join(lines) - - -def _open_xlsx(file_bytes: bytes): - """Open a workbook in read-only mode; raise RuntimeError with a Chinese - install hint if openpyxl is missing.""" - try: - import openpyxl - except ImportError: - raise RuntimeError("openpyxl 未安装，无法解析 XLSX 文件（pip install openpyxl）") - return openpyxl.load_workbook(io.BytesIO(file_bytes), read_only=True, data_only=True) - - -def _xlsx_sheet_to_markdown(sheet) -> str | None: - """Render one openpyxl sheet to a markdown table, or None if empty. - Returns None (not raise) so callers can decide whether emptiness is an - error (single-sheet mode) or just a skip (whole-workbook mode).""" - rows = list(sheet.iter_rows(values_only=True)) - if not rows: - return None - header = [str(c or "") for c in rows[0]] - data_rows = [ - [str(c or "") for c in row] - for row in rows[1:] - if not all(c is None for c in row) - ] - return _rows_to_markdown(header, data_rows, sheet.title) - - -def parse_xlsx(file_bytes: bytes) -> str: - """Parse XLSX bytes → markdown tables (one per sheet).""" - wb = _open_xlsx(file_bytes) - try: - sections = [md for sheet in wb.worksheets if (md := _xlsx_sheet_to_markdown(sheet))] - finally: - wb.close() - if not sections: - raise RuntimeError("XLSX 文件无有效数据") - return "\n\n".join(sections) - - -def parse_xlsx_sheet_names(file_bytes: bytes) -> list[str]: - """Return the list of sheet names in workbook order. Cheap probe: does - not iterate any rows, just reads workbook structure.""" - wb = _open_xlsx(file_bytes) - try: - return list(wb.sheetnames) - finally: - wb.close() - - -def parse_xlsx_single_sheet(file_bytes: bytes, sheet_name: str) -> str: - """Parse only the named sheet → markdown table. Raises RuntimeError if - the sheet does not exist or is empty.""" - wb = _open_xlsx(file_bytes) - try: - if sheet_name not in wb.sheetnames: - raise RuntimeError( - f"sheet 不存在: '{sheet_name}'。可选 sheet: {wb.sheetnames}" - ) - md = _xlsx_sheet_to_markdown(wb[sheet_name]) - if md is None: - raise RuntimeError(f"sheet '{sheet_name}' 无数据") - return md - finally: - wb.close() - - -def _default_xlsx_max_cell_chars() -> int: - """Bound spreadsheet preview cells; the default remains 2000 characters.""" - try: - return max(50, int(os.getenv("XLSX_PREVIEW_MAX_CELL_CHARS", "2000"))) - except ValueError: - return 2000 - - -def parse_xlsx_preview( - file_bytes: bytes, - *, - char_budget: int = 4000, - max_columns: int = 40, - max_cell_chars: Optional[int] = None, -) -> dict: - """Parse XLSX into a bounded markdown preview + accurate total counts. - - Produces what the main agent's file_context hook should inject for an - uploaded spreadsheet: a markdown table containing as many rows as fit - within ``char_budget``, plus the *real* total_rows / total_columns so - the agent knows the full size without seeing every row. - - The total_rows count walks the entire sheet (cheap — read_only mode - streams the first column only). The preview stops as soon as the - cumulative markdown length exceeds ``char_budget``. - - Returns: - ``{ - "header": list[str], # column names (post-dedup) - "preview_md": str, # markdown table, ≤ char_budget chars - "total_rows": int, # all data rows in sheet 1 - "total_columns": int, # raw column count (before max_columns) - "preview_rows": int, # rows actually included in preview_md - "truncated_columns": int, # cols dropped to fit max_columns - "truncated_cells": int, # cells truncated to max_cell_chars - }`` - - Reads only the first sheet. - """ - if max_cell_chars is None: - max_cell_chars = _default_xlsx_max_cell_chars() - - try: - import openpyxl - except ImportError: - raise RuntimeError("openpyxl 未安装，无法解析 XLSX 文件（pip install openpyxl）") - - wb = openpyxl.load_workbook(io.BytesIO(file_bytes), read_only=True, data_only=True) - try: - sheet = wb.worksheets[0] if wb.worksheets else None - if sheet is None: - raise RuntimeError("XLSX 文件无 sheet") - - rows_iter = sheet.iter_rows(values_only=True) - try: - header_row = next(rows_iter) - except StopIteration: - raise RuntimeError("XLSX 文件无数据行") from None - - total_columns = len(header_row) - truncated_columns = max(0, total_columns - max_columns) - usable_cols = min(total_columns, max_columns) - - header = _deduplicate_headers([ - str(c).strip() if c is not None else f"col{idx + 1}" - for idx, c in enumerate(header_row[:usable_cols]) - ]) - - # Markdown header + separator (always present, counts toward budget) - md_lines: list[str] = [ - "| " + " | ".join(header) + " |", - "|" + "|".join("---" for _ in header) + "|", - ] - running_chars = sum(len(ln) + 1 for ln in md_lines) # +1 for newline - - preview_rows = 0 - truncated_cells = 0 - total_rows_seen = 0 - budget_reached = False - - for row in rows_iter: - total_rows_seen += 1 - if all(c is None for c in row): - continue - - if budget_reached: - continue # keep counting total_rows accurately - - cells: list[str] = [] - for val in row[:usable_cols]: - if val is None: - cells.append("") - continue - s = str(val).replace("|", "\\|").replace("\n", " ") - if len(s) > max_cell_chars: - s = s[:max_cell_chars] + "…" - truncated_cells += 1 - cells.append(s) - - # Pad short rows so the markdown table stays well-formed - while len(cells) < len(header): - cells.append("") - - line = "| " + " | ".join(cells) + " |" - line_cost = len(line) + 1 - if running_chars + line_cost > char_budget and preview_rows > 0: - # We have at least one row already; respect the budget. - budget_reached = True - continue - md_lines.append(line) - running_chars += line_cost - preview_rows += 1 - - return { - "header": header, - "preview_md": "\n".join(md_lines), - "total_rows": total_rows_seen, - "total_columns": total_columns, - "preview_rows": preview_rows, - "truncated_columns": truncated_columns, - "truncated_cells": truncated_cells, - } - finally: - wb.close() - - -def parse_xls(file_bytes: bytes) -> str: - """Parse legacy XLS bytes → markdown tables. - - Tries xlrd first; falls back to LibreOffice → XLSX → openpyxl. - """ - try: - import xlrd - wb = xlrd.open_workbook(file_contents=file_bytes) - sections: list[str] = [] - for sheet in wb.sheets(): - if sheet.nrows == 0: - continue - header = [str(sheet.cell_value(0, c) or "") for c in range(sheet.ncols)] - data_rows = [] - for r in range(1, sheet.nrows): - row = [str(sheet.cell_value(r, c) or "") for c in range(sheet.ncols)] - if all(v == "" for v in row): - continue - data_rows.append(row) - sections.append(_rows_to_markdown(header, data_rows, sheet.name)) - if not sections: - raise RuntimeError("XLS 文件无有效数据") - return "\n\n".join(sections) - except ImportError: - pass # xlrd not installed, try LibreOffice fallback - - # Fallback: LibreOffice → XLSX → openpyxl - xlsx_bytes = _convert_xls_to_xlsx(file_bytes) - return parse_xlsx(xlsx_bytes) - - -def _convert_xls_to_xlsx(file_bytes: bytes) -> bytes: - """Use LibreOffice headless to convert XLS → XLSX.""" - with tempfile.TemporaryDirectory() as tmpdir: - input_path = os.path.join(tmpdir, "input.xls") - with open(input_path, "wb") as f: - f.write(file_bytes) - - try: - result = subprocess.run( - [ - find_libreoffice_binary() or "libreoffice", - "--headless", - "--convert-to", "xlsx", - "--outdir", tmpdir, - input_path, - ], - capture_output=True, - text=True, - timeout=120, - **no_window_kwargs(), - ) - except FileNotFoundError: - raise RuntimeError( - "xlrd 和 LibreOffice 均不可用，无法解析 XLS 文件" - "（pip install xlrd 或安装 LibreOffice）" - ) - except subprocess.TimeoutExpired: - raise RuntimeError("LibreOffice 转换 XLS → XLSX 超时") - - output_path = os.path.join(tmpdir, "input.xlsx") - if not os.path.exists(output_path): - raise RuntimeError(f"LibreOffice 转换 XLS 失败: {result.stderr[:300]}") - - with open(output_path, "rb") as f: - return f.read() - - -# ── CSV ────────────────────────────────────────────────────────────────────── - -def parse_csv(file_bytes: bytes) -> str: - """Parse CSV bytes → markdown table.""" - # Decode with encoding detection - text = parse_txt(file_bytes) - - # Sniff dialect - try: - sample = text[:8192] - dialect = csv.Sniffer().sniff(sample) - except csv.Error: - dialect = csv.excel - - reader = csv.reader(io.StringIO(text), dialect) - rows = list(reader) - - if not rows: - raise RuntimeError("CSV 文件无有效数据") - - header = [c.strip() for c in rows[0]] - data_rows = [] - for row in rows[1:]: - if all(c.strip() == "" for c in row): - continue - # Pad or trim to match header length - padded = [c.strip() for c in row] - while len(padded) < len(header): - padded.append("") - data_rows.append(padded[:len(header)]) - - return _rows_to_markdown(header, data_rows) - - -# ── PPTX ────────────────────────────────────────────────────────────────────── - -def _open_pptx(file_bytes: bytes): - """Open a Presentation from bytes; raise RuntimeError with a Chinese - install hint if python-pptx is missing.""" - try: - from pptx import Presentation - except ImportError: - raise RuntimeError( - "python-pptx 未安装，无法解析 PPTX 文件（pip install python-pptx）" - ) - return Presentation(io.BytesIO(file_bytes)) - - -def _extract_slide_text(slide) -> str: - # Iterate in author/Z order so the rendered output mirrors what a - # reader sees (title → body → footers), since python-pptx exposes - # shapes in that order on the slide.shapes collection. - parts: list[str] = [] - for shape in slide.shapes: - if not getattr(shape, "has_text_frame", False): - continue - try: - text = shape.text_frame.text - except Exception: - continue - text = (text or "").strip() - if text: - parts.append(text) - return "\n".join(parts) - - -def parse_pptx(file_bytes: bytes) -> str: - """Parse PPTX bytes → markdown with one ``## Slide N`` section per slide.""" - p = _open_pptx(file_bytes) - sections: list[str] = [] - for idx, slide in enumerate(p.slides): - body = _extract_slide_text(slide) or "(此页无文本)" - sections.append(f"## Slide {idx + 1}\n\n{body}") - if not sections: - raise RuntimeError("PPTX 文件无幻灯片") - return "\n\n".join(sections) - - -def parse_pptx_slide_count(file_bytes: bytes) -> int: - """Cheap probe: return the slide count without extracting any text.""" - return len(_open_pptx(file_bytes).slides) - - -def parse_pptx_slide(file_bytes: bytes, slide_index: int) -> str: - """Return only the named slide's text (0-indexed). Raises RuntimeError - on out-of-range index.""" - slides = _open_pptx(file_bytes).slides - total = len(slides) - if total == 0: - raise RuntimeError("PPTX 文件无幻灯片") - if slide_index < 0 or slide_index >= total: - raise RuntimeError( - f"slide_index 越界: {slide_index}（共 {total} 页，可选 0..{total - 1}）" - ) - return _extract_slide_text(slides[slide_index]) or "(此页无文本)" - - -# ── Dispatcher ──────────────────────────────────────────────────────────────── - -_EXT_MAP: dict[str, str] = { - ".pdf": "pdf", - ".docx": "docx", - ".doc": "doc_wps", - ".wps": "doc_wps", - ".txt": "txt", - ".xlsx": "xlsx", - ".xls": "xls", - ".csv": "csv", - ".pptx": "pptx", -} - -# Text-like extensions that don't need a real parser — just utf-8 decode. -# Covers HTML/Markdown/code/config formats the agent commonly produces. -# Mapping value "raw_text" routes to ``parse_raw_text``. -_RAW_TEXT_EXTS = { - ".html", ".htm", - ".md", ".markdown", - ".json", ".jsonl", ".ndjson", - ".yaml", ".yml", - ".toml", ".ini", ".cfg", - ".xml", ".svg", - ".log", - ".py", ".js", ".ts", ".tsx", ".jsx", ".sh", ".bash", - ".css", ".scss", - ".tex", ".rst", - ".sql", -} - -SUPPORTED_EXTENSIONS = list(_EXT_MAP.keys()) + sorted(_RAW_TEXT_EXTS) - - -def parse_raw_text(file_bytes: bytes) -> str: - """UTF-8 decode for plain-text formats (HTML/MD/code/etc.). - - No structural extraction—just the source bytes as a string. Replaces - undecodable bytes so we always return something usable. - """ - return file_bytes.decode("utf-8", errors="replace") - - -def parse_file(file_bytes: bytes, filename: str) -> Optional[str]: - """ - Dispatch to the correct parser by file extension. - - Returns extracted text/markdown, or None if the format is unsupported. - Raises RuntimeError with a Chinese message on parse failure. - """ - suffix = os.path.splitext(filename.lower())[1] - kind = _EXT_MAP.get(suffix) - - if kind == "pdf": - return parse_pdf(file_bytes, filename) - elif kind == "docx": - return parse_docx(file_bytes) - elif kind == "doc_wps": - return parse_doc_wps(file_bytes, suffix) - elif kind == "txt": - return parse_txt(file_bytes) - elif kind == "xlsx": - return parse_xlsx(file_bytes) - elif kind == "xls": - return parse_xls(file_bytes) - elif kind == "csv": - return parse_csv(file_bytes) - elif kind == "pptx": - return parse_pptx(file_bytes) - elif suffix in _RAW_TEXT_EXTS: - return parse_raw_text(file_bytes) - else: - return None diff --git a/src/backend/core/db/__init__.py b/src/backend/core/db/__init__.py index a8ad7c28..d9283f7e 100644 --- a/src/backend/core/db/__init__.py +++ b/src/backend/core/db/__init__.py @@ -1,2 +1,26 @@ -from core.db.engine import get_db, init_db, Base # noqa: F401 -from core.db.models import * # noqa: F401,F403 +"""Database package facade. + +Leaf modules (engine, utc_datetime, ...) must stay importable without loading +every ORM model: edition model extensions import them while ``core.db.models`` +is importing those extensions. Model names resolve lazily on first access. +""" + +import importlib +import importlib.util + +from core.db.engine import Base, get_db, init_db # noqa: F401 + + +def __getattr__(name): + if name.startswith("__"): + raise AttributeError(name) + if importlib.util.find_spec(f"{__name__}.{name}") is not None: + # ``from core.db import `` must not drag in every model. + return importlib.import_module(f"{__name__}.{name}") + models = importlib.import_module("core.db.models") + try: + value = getattr(models, name) + except AttributeError: + raise AttributeError(f"module 'core.db' has no attribute {name!r}") from None + globals()[name] = value + return value diff --git a/src/backend/core/db/artifact_scope.py b/src/backend/core/db/artifact_scope.py new file mode 100644 index 00000000..fb28ee7c --- /dev/null +++ b/src/backend/core/db/artifact_scope.py @@ -0,0 +1,13 @@ +"""Personal artifact predicates use the installed database edition policy.""" +from typing import Any +from core.db.edition import personal_artifact_scope_fields + + +def personal_artifact_predicates(artifact_model: Any) -> list[Any]: + return [getattr(artifact_model, name).is_(value) + for name, value in personal_artifact_scope_fields().items()] + + +def is_personal_artifact(artifact: Any) -> bool: + return all(getattr(artifact, name) == value + for name, value in personal_artifact_scope_fields().items()) diff --git a/src/backend/core/db/chat_search.py b/src/backend/core/db/chat_search.py index b3b258b3..d1fa1ac9 100644 --- a/src/backend/core/db/chat_search.py +++ b/src/backend/core/db/chat_search.py @@ -6,7 +6,7 @@ until after the authorized page has been selected. """ -from sqlalchemy import Text, and_, case, func, or_, select, null +from sqlalchemy import Text, and_, case, or_, select, null from sqlalchemy.ext.compiler import compiles from sqlalchemy.sql.functions import FunctionElement from core.db.reserved_chats import ordinary_chat_filter diff --git a/src/backend/core/db/edition.py b/src/backend/core/db/edition.py new file mode 100644 index 00000000..b258a048 --- /dev/null +++ b/src/backend/core/db/edition.py @@ -0,0 +1,27 @@ +"""Community database schema and personal resource ownership.""" +from core.db.personal_model_fields import ProjectEditionFields, project_edition_table_args, ArtifactEditionFields, artifact_edition_table_args, UserAgentEditionFields, user_agent_edition_table_args, ChatSessionEditionFields, chat_session_edition_table_args, MarketplaceListingEditionFields, EDITION_MODEL_EXPORTS +from core.db.personal_schema import ce_create_all, ce_reconcile_schema + +EDITION_MODEL_NAMES = () + + +def personal_artifact_scope_fields() -> dict[str, None]: + return {} + + +def __getattr__(name): + if name == "UserAgentRepository": + from core.db.repository.personal_agent import UserAgentRepository + return UserAgentRepository + if name == "AuditLogRepository": + from core.db.repository.personal_audit import AuditLogRepository + return AuditLogRepository + raise AttributeError(name) + + +def personal_index_scope(enterprise=False): + return "" + + +def edition_model_exports(): + return {} diff --git a/src/backend/core/db/engine.py b/src/backend/core/db/engine.py index e4cfbcd6..339781e2 100644 --- a/src/backend/core/db/engine.py +++ b/src/backend/core/db/engine.py @@ -159,12 +159,12 @@ def init_db(): reconcile_audio_model_type(engine) reconcile_resource_owners(engine) from core.db.site_storage_retirement import retire_site_storage - from core.services.application_store import initialize_local_store + from core.harness.registry.schema_lifecycle import initialize_local_schema directory = settings.storage.root / "migration-backups" directory = Path(os.getenv("SITE_STORAGE_BACKUP_DIR", str(directory))) retire_site_storage(engine, directory) - initialize_local_store() + initialize_local_schema() if settings.edition.edition == "ce": if settings.deploy.is_local: from core.db.local_schema_upgrade import reconcile_local_chat_sequences @@ -180,7 +180,7 @@ def init_db(): if any(local_report.values()): logger.info("Local database compatibility schema reconciled: %s", local_report) - from core.db.edition_tables import ce_reconcile_schema + from core.db.edition import ce_reconcile_schema report = ce_reconcile_schema(engine) if any(report.values()): diff --git a/src/backend/core/db/history_projection.py b/src/backend/core/db/history_projection.py index c41762b5..1f868b9e 100644 --- a/src/backend/core/db/history_projection.py +++ b/src/backend/core/db/history_projection.py @@ -2,7 +2,7 @@ def tool_calls_for_history(raw): - from core.chat.display_bounds import bound_result_for_history + from core.infra.text_bounds import bound_result_for_history if not isinstance(raw, list): return raw diff --git a/src/backend/core/db/models/__init__.py b/src/backend/core/db/models/__init__.py index bacfc1c7..dc5d9ca0 100644 --- a/src/backend/core/db/models/__init__.py +++ b/src/backend/core/db/models/__init__.py @@ -1,70 +1,29 @@ -"""Community-edition ORM model exports.""" +"""SQLAlchemy ORM models. + +Split into submodules by domain; this __init__ re-exports all model classes verbatim (plus Base/JSONType/INETType), +so `from core.db.models import X` and `from core.db.models import *` both stay unchanged. +""" from core.db.engine import Base -from core.db.models.admin import ( - AdminMcpServer, - AdminPromptPart, - AdminSkill, - InstalledPlugin, - MarketplaceListingState, - MarketplaceSubmission, - McpMarketInstallation, - McpMarketItem, - McpMarketSubmission, - McpMarketVersion, - PluginMarketPackage, - PluginMarketSkillExclusion, - SkillDependencyRequest, -) -from core.db.models.agent import ( - AgentLoop, - AgentMarketSubmission, - LoopIteration, - Plan, - PlanStep, - UserAgent, -) +from core.db.models.column_types import INETType, JSONType + +from core.db.edition import EDITION_MODEL_EXPORTS +from core.db.models.chat import ChatCompactionState, ChatMessage, ChatSession +from core.db.models.chat_runs import ChatRun, ChatRunOperation, ChatSteerQueueItem +from core.db.models.chat_resources import ChatSandboxSnapshot, ToolMediaBlob, MessageFeedback +from core.db.models.identity import LocalUser, UserApiKey, UserFolder, UserShadow +from core.db.models.project import Project, ProjectFavorite + +globals().update(EDITION_MODEL_EXPORTS) +from core.db.models.admin import AdminMcpServer, AdminPromptPart, AdminSkill, InstalledPlugin, SkillDependencyRequest +from core.db.models.agent import AgentLoop, LoopIteration, Plan, PlanStep, UserAgent from core.db.models.agent_api import AgentApiCallLog from core.db.models.artifact import Artifact, ContentBlock -from core.db.models.automation import PersonaDistillJob, ScheduledTask, ScheduledTaskRun -from core.db.models.chat import ( - ChatCompactionState, - ChatMessage, - ChatRun, - ChatRunOperation, - ChatSandboxSnapshot, - ChatSession, - ChatSteerQueueItem, - MessageFeedback, - ToolMediaBlob, -) + from core.db.models.chat_mode import ChatMode from core.db.models.config import ModelProvider, ModelRoleAssignment, SystemConfig -from core.db.models.evolution import ( - EvolutionAgentProfile, - EvolutionCandidate, - EvolutionCreditDecision, - EvolutionEpisode, - EvolutionEvaluation, - EvolutionEvidencePack, - EvolutionMemoryOp, - EvolutionPromotionLink, - EvolutionRelease, - EvolutionTraceEvent, -) -from core.db.models.identity import ( - ChannelConnection, - DingTalkConnection, - EmailConnection, - LarkConnection, - LocalUser, - UserApiKey, - UserFolder, - UserShadow, -) -from core.db.models.job import JOB_LIVE_STATUSES, JOB_TERMINAL_STATUSES, Job, JobCall, JobItem -from core.db.models.kb_wiki import KBWikiFolder, KBWikiJob, KBWikiPage -from core.db.models.knowledge import CatalogOverride, KBAsset, KBChunk, KBDocument, KBSpace + +from core.db.models.knowledge import CatalogOverride from core.db.models.logs import ( HarnessEventCursor, HarnessEventLog, @@ -78,6 +37,18 @@ ToolEffectReceipt, RemoteToolEffect, ) +from core.db.models.evolution import ( + EvolutionAgentProfile, + EvolutionCandidate, + EvolutionCreditDecision, + EvolutionEpisode, + EvolutionEvaluation, + EvolutionEvidencePack, + EvolutionMemoryOp, + EvolutionPromotionLink, + EvolutionRelease, + EvolutionTraceEvent, +) from core.db.models.memory import ( MemoryOutbox, MemoryRefShadow, @@ -91,39 +62,100 @@ OntologyPackVersion, OntologyReviewRun, ) -from core.db.models.project import Project, ProjectFavorite -from core.db.models.site import Site -from sqlalchemy import JSON, String -from sqlalchemy.dialects.postgresql import INET, JSONB - -JSONType = JSON().with_variant(JSONB(), "postgresql") -INETType = String(45).with_variant(INET(), "postgresql") - -# These models use the shared JSONType defined above. -from core.db.models.capability import ( - DeviceCapabilityComponent, - DeviceCapabilityInstallation, - DeviceCapabilityNamePreference, - DeviceCapabilityTransaction, -) +from core.db.models.command import CommandTask +from core.db.models.job import JOB_LIVE_STATUSES, JOB_TERMINAL_STATUSES, Job, JobCall, JobItem +from core.db.edition import EDITION_MODEL_NAMES, edition_model_exports +globals().update(edition_model_exports()) -__all__ = [name for name in globals() if not name.startswith("_")] -from core.db.models.observability import DesktopOutbox, DesktopRecord, DesktopSyncDevice +def __getattr__(name): + # Edition models are absent above only when the edition model package + # itself started this import; it has finished loading by first access. + if name in EDITION_MODEL_NAMES: + value = edition_model_exports()[name] + globals()[name] = value + return value + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") -from core.db.models.channel_relay import DesktopChannelBinding, ChannelRelayDelivery, ChannelRelayOperation -from core.db.models.plugin_resource import PluginResource, PluginResourceTicket, PluginResourceCheckpoint +__all__ = [ + 'ChatMode', + 'Base', + 'JSONType', + 'INETType', + 'UserShadow', + 'UserApiKey', + 'AgentApiCallLog', + 'LocalUser', + 'UserFolder', + 'Project', + 'ProjectFavorite', + 'ChatSession', + *EDITION_MODEL_EXPORTS.keys(), + 'ChatMessage', + 'ChatCompactionState', + 'ChatRun', + 'CommandTask', + 'ChatRunOperation', + 'ChatSteerQueueItem', + 'ToolEffectLedger', + 'ToolEffectLease', + 'ToolEffectReceipt', + 'RemoteToolEffect', + 'HarnessUsageCursor', + 'HarnessUsageAttempt', + 'HarnessEventCursor', + 'HarnessEventLog', + 'MessageFeedback', + 'ChatSandboxSnapshot', + 'ToolMediaBlob', + 'CatalogOverride', + 'Artifact', + 'ContentBlock', + 'ModelProvider', + 'SystemConfig', + 'ModelRoleAssignment', + 'AdminSkill', + 'SkillDependencyRequest', + 'AdminPromptPart', + 'AdminMcpServer', + 'InstalledPlugin', + 'UserAgent', + 'Plan', + 'PlanStep', + 'AgentLoop', + 'LoopIteration', + 'Job', + 'JobItem', + 'JobCall', + 'JOB_TERMINAL_STATUSES', + 'JOB_LIVE_STATUSES', + 'ToolCallLog', + 'SubAgentCallLog', + 'SkillCallLog', + 'ProfileMemory', + 'MemoryOutbox', + 'MemorySanitizerRule', + 'MemoryRefShadow', + 'EvolutionEpisode', + 'EvolutionTraceEvent', + 'EvolutionCandidate', + 'EvolutionRelease', + 'EvolutionEvaluation', + 'EvolutionAgentProfile', + 'EvolutionMemoryOp', + 'EvolutionEvidencePack', + 'EvolutionCreditDecision', + 'EvolutionPromotionLink', + 'OntologyPack', + 'OntologyPackVersion', + 'OntologyEnforcementEvent', + 'OntologyReviewRun', + 'OntologyDraft', + *EDITION_MODEL_NAMES, +] -# Shared feature tables exist even when the runtime module is disabled. -from personal_agent.models import ( - PersonalAgent, PersonalWorkItem, PersonalDevice, PersonalDeviceCommand, - PersonalMessageTask, PersonalMessageInbox, PersonalSessionOperation, -) -__all__ += ["PersonalAgent", "PersonalWorkItem", "PersonalDevice", - "PersonalDeviceCommand", "PersonalMessageTask", "PersonalMessageInbox", - "PersonalSessionOperation"] +from .plugin_resource import PluginResource, PluginResourceTicket, PluginResourceCheckpoint -from core.db.models.command import CommandTask -__all__ += ["CommandTask"] +__all__ = [name for name in __all__ if name in globals() or name in EDITION_MODEL_NAMES] diff --git a/src/backend/core/db/models/admin.py b/src/backend/core/db/models/admin.py index 262f7529..20c4fca9 100644 --- a/src/backend/core/db/models/admin.py +++ b/src/backend/core/db/models/admin.py @@ -2,29 +2,18 @@ from core.db.utc_datetime import UTCDateTime from core.infra.time import utc_now -from datetime import datetime, timezone from core.db.engine import Base -from core.db.model_extensions import MarketplaceListingEditionFields from sqlalchemy import ( - JSON, Boolean, CheckConstraint, Column, - ForeignKey, Index, Integer, - Numeric, String, Text, - UniqueConstraint, - text, ) -from sqlalchemy.dialects.postgresql import INET, JSONB -from sqlalchemy.orm import mapped_column, relationship - -JSONType = JSON().with_variant(JSONB(), "postgresql") -INETType = String(45).with_variant(INET(), "postgresql") +from core.db.models.column_types import JSONType class AdminSkill(Base): @@ -145,7 +134,6 @@ class AdminMcpServer(Base): env_vars = Column(JSONType, default=dict) env_inherit = Column(JSONType, default=list) headers = Column(JSONType, default=dict) - is_stable = Column(Boolean, nullable=False, default=True) is_enabled = Column(Boolean, nullable=False, default=True) sort_order = Column(Integer, nullable=False, default=0) extra_config = Column(JSONType, default=dict) @@ -172,247 +160,6 @@ class AdminMcpServer(Base): ) -class McpMarketItem(Base): - """Installable MCP marketplace entry. - - The marketplace stores a credential-free definition. Concrete credentials - and enablement live on the ``AdminMcpServer`` created for each installation. - ``latest_version_id`` points at an immutable ``McpMarketVersion`` snapshot. - """ - - __tablename__ = "mcp_market_items" - - slug = Column(String(128), primary_key=True) - display_name = Column(String(255), nullable=False) - description = Column(Text, nullable=False, default="") - user_intro = Column(Text) - category = Column(String(64), nullable=False, default="通用工具") - tags = Column(JSONType, default=list) - icon = Column(String(500)) - publisher_id = Column(String(64)) - publisher_name = Column(String(255), nullable=False, default="") - source = Column(String(16), nullable=False, default="community") - latest_version_id = Column(String(64), nullable=False) - # active = installable; changed = remote tool drift awaiting review; - # suspended = security kill switch (derived installations are disabled). - status = Column(String(16), nullable=False, default="active") - status_reason = Column(Text) - last_verified_at = Column(UTCDateTime(timezone=True)) - created_at = Column(UTCDateTime(timezone=True), default=utc_now, nullable=False) - updated_at = Column( - UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now, nullable=False - ) - deleted_at = Column(UTCDateTime(timezone=True)) - - __table_args__ = ( - CheckConstraint( - "source IN ('admin', 'community')", - name="mcp_market_items_source_check", - ), - CheckConstraint( - "status IN ('active', 'changed', 'suspended')", - name="mcp_market_items_status_check", - ), - Index("idx_mcp_market_items_status", "status", "updated_at"), - Index("idx_mcp_market_items_category", "category"), - ) - - -class McpMarketVersion(Base): - """Immutable, reviewed MCP marketplace version snapshot.""" - - __tablename__ = "mcp_market_versions" - - version_id = Column(String(64), primary_key=True) - slug = Column( - String(128), - ForeignKey("mcp_market_items.slug", ondelete="CASCADE"), - nullable=False, - ) - version = Column(String(50), nullable=False, default="1.0.0") - transport = Column(String(20), nullable=False, default="streamable_http") - url = Column(Text, nullable=False) - # [{key, label, target: "header" | "query" | "url", required, secret, ...}]; - # values are never stored here. - auth_schema = Column(JSONType, default=list) - # Provider-neutral authentication contract. ``methods`` may contain - # none/token/oauth2 options; OAuth endpoint discovery follows MCP metadata. - auth_config = Column(JSONType, default=dict) - tools_json = Column(JSONType, default=list) - tool_hash = Column(String(64), nullable=False) - listing_notice = Column(JSONType, default=dict) - source_server_id = Column(String(100)) - approved_by = Column(String(64)) - approved_at = Column(UTCDateTime(timezone=True)) - created_at = Column(UTCDateTime(timezone=True), default=utc_now, nullable=False) - - __table_args__ = ( - CheckConstraint( - "transport IN ('streamable_http', 'sse')", - name="mcp_market_versions_transport_check", - ), - UniqueConstraint("slug", "version", name="uq_mcp_market_versions_slug_version"), - Index("idx_mcp_market_versions_slug", "slug", "created_at"), - ) - - -class McpMarketSubmission(Base): - """Credential-free snapshot submitted by a user for MCP marketplace review.""" - - __tablename__ = "mcp_market_submissions" - - submission_id = Column(String(64), primary_key=True) - slug = Column(String(128), nullable=False) - source_server_id = Column(String(100), nullable=False) - owner_user_id = Column(String(64), nullable=False) - submitter_name = Column(String(255), nullable=False, default="") - display_name = Column(String(255), nullable=False) - description = Column(Text, nullable=False, default="") - user_intro = Column(Text) - category = Column(String(64), nullable=False, default="通用工具") - tags = Column(JSONType, default=list) - icon = Column(String(500)) - version = Column(String(50), nullable=False, default="1.0.0") - transport = Column(String(20), nullable=False) - url = Column(Text, nullable=False) - auth_schema = Column(JSONType, default=list) - auth_config = Column(JSONType, default=dict) - tools_json = Column(JSONType, default=list) - tool_hash = Column(String(64), nullable=False) - listing_notice = Column(JSONType, default=dict) - note = Column(Text, nullable=False, default="") - status = Column(String(16), nullable=False, default="pending") - review_note = Column(Text) - reviewed_by = Column(String(64)) - reviewed_at = Column(UTCDateTime(timezone=True)) - created_at = Column(UTCDateTime(timezone=True), default=utc_now, nullable=False) - updated_at = Column( - UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now, nullable=False - ) - deleted_at = Column(UTCDateTime(timezone=True)) - - __table_args__ = ( - CheckConstraint( - "status IN ('pending', 'approved', 'rejected', 'withdrawn')", - name="mcp_market_submissions_status_check", - ), - CheckConstraint( - "transport IN ('streamable_http', 'sse')", - name="mcp_market_submissions_transport_check", - ), - Index("idx_mcp_market_submissions_status", "status", "created_at"), - Index("idx_mcp_market_submissions_owner", "owner_user_id", "created_at"), - Index("idx_mcp_market_submissions_slug", "slug"), - ) - - -class McpMarketInstallation(Base): - """Links a reviewed market version to the concrete installed MCP server.""" - - __tablename__ = "mcp_market_installations" - - install_id = Column(String(64), primary_key=True) - slug = Column( - String(128), - ForeignKey("mcp_market_items.slug", ondelete="CASCADE"), - nullable=False, - ) - version_id = Column( - String(64), - ForeignKey("mcp_market_versions.version_id", ondelete="RESTRICT"), - nullable=False, - ) - server_id = Column( - String(100), - ForeignKey("admin_mcp_servers.server_id", ondelete="CASCADE"), - nullable=False, - unique=True, - ) - # NULL means the admin installed it globally; otherwise it is private to the user. - owner_user_id = Column(String(64)) - status = Column(String(16), nullable=False, default="active") - installed_by = Column(String(64)) - created_at = Column(UTCDateTime(timezone=True), default=utc_now, nullable=False) - updated_at = Column( - UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now, nullable=False - ) - - __table_args__ = ( - CheckConstraint( - "status IN ('active', 'suspended')", - name="mcp_market_installations_status_check", - ), - Index("idx_mcp_market_installations_owner", "owner_user_id", "created_at"), - Index("idx_mcp_market_installations_slug", "slug", "status"), - Index( - "uq_mcp_market_installations_user_slug", - "slug", - "owner_user_id", - unique=True, - postgresql_where=text("owner_user_id IS NOT NULL"), - sqlite_where=text("owner_user_id IS NOT NULL"), - ), - Index( - "uq_mcp_market_installations_global_slug", - "slug", - unique=True, - postgresql_where=text("owner_user_id IS NULL"), - sqlite_where=text("owner_user_id IS NULL"), - ), - ) - - -class MarketplaceSubmission(Base): - """A user's private skill "apply for listing on the skill marketplace" record (pending admin review). - - At submission time a content snapshot of the source ``AdminSkill`` is - taken (skill_content / extra_files); review and listing are both based - on the snapshot — the user later modifying or deleting the original - skill does not affect listed content. Records with - ``status='approved'`` appear in the skill marketplace list - (source=community) and can be installed by everyone; the admin can - reject at any time (including delisting already-listed items). - """ - - __tablename__ = "marketplace_submissions" - - submission_id = Column(String(64), primary_key=True) - # Marketplace slug / entry_name after listing (derived at submission time, - # globally unique, does not clash with the preset marketplace catalog) - slug = Column(String(128), nullable=False, unique=True) - skill_id = Column(String(100), nullable=False) # source AdminSkill.skill_id - owner_user_id = Column(String(64), nullable=False) # applicant - submitter_name = Column( - String(255), default="" - ) # applicant display name (denormalized, for display) - - display_name = Column(String(255), nullable=False) - summary = Column(Text, default="") - user_intro = Column(Text) # explicit user-facing intro snapshot; NULL falls back to SKILL.md - category = Column(String(64), default="社区共享") - tags = Column(JSONType, default=list) - version = Column(String(50), default="1.0.0") - note = Column(Text, default="") # application note (for the admin) - - skill_content = Column(Text, nullable=False) # SKILL.md snapshot - extra_files = Column(JSONType, default=dict) # attachment snapshot {path: content} - - status = Column(String(16), nullable=False, default="pending") - review_note = Column(Text) # rejection reason / review note - reviewed_at = Column(UTCDateTime(timezone=True)) - created_at = Column(UTCDateTime(timezone=True), default=utc_now) - updated_at = Column(UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now) - - __table_args__ = ( - CheckConstraint( - "status IN ('pending', 'approved', 'rejected')", - name="marketplace_submissions_status_check", - ), - Index("idx_marketplace_submissions_status", "status", "created_at"), - Index("idx_marketplace_submissions_owner", "owner_user_id"), - ) - - class InstalledPlugin(Base): """An installed plugin (bundle) — the single source of truth for uninstall/upgrade. @@ -465,86 +212,3 @@ class InstalledPlugin(Base): Index("idx_installed_plugins_owner", "owner_user_id"), Index("idx_installed_plugins_slug", "slug"), ) - - -class PluginMarketPackage(Base): - """Admin-uploaded plugin marketplace package (stored/listed in the DB). - - Counterpart of the skill marketplace's ``MarketplaceSubmission``: a - plugin zip uploaded by the admin from the /admin console is no longer - installed globally right away, but persisted as a "market package" — - storing the original zip's base64 (``package_b64``) + a set of display - metadata (extracted by a single normalize pass). Once listed it appears - in the plugin marketplace list for admins/users to install explicitly; - installation unzips and re-runs the existing ``import_plugin`` chain. - It coexists with filesystem preset packages - (``extensions/plugins/{default,marketplace}//``); when resolving by - slug, **filesystem takes precedence, DB market package is the - fallback**. Install records still go into ``InstalledPlugin``; this - table is only an "installable source". - """ - - __tablename__ = "plugin_market_packages" - - slug = Column(String(100), primary_key=True) - name = Column(String(255), nullable=False) - version = Column(String(50), nullable=False, default="1.0.0") - description = Column(Text, default="") - category = Column(String(64), default="") - # Library path / URL / inline data-URI (uploaded custom icons, capped at ~200KB by the service layer) - icon = Column(Text) - # Package kind: native / claude / codex (determined by normalize), display only - kind = Column(String(16), nullable=False, default="native") - skills_count = Column(Integer, nullable=False, default=0) - required_secrets = Column(JSONType, default=list) - has_admin_config = Column(Boolean, nullable=False, default=False) - package_b64 = Column(Text, nullable=False) # base64 of the originally uploaded zip - created_at = Column(UTCDateTime(timezone=True), default=utc_now) - updated_at = Column(UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now) - created_by = Column(String(64)) - - __table_args__ = (Index("idx_plugin_market_packages_category", "category"),) - - -class PluginMarketSkillExclusion(Base): - """A single skill "deleted" by the admin from a plugin in the plugin marketplace (excluded by plugin slug + skill name). - - The plugin marketplace originally only supported whole-package deletion; - this table lets the admin precisely remove a specific skill within a - plugin. The market list's ``skills_count``, market detail ``skills``, - and DB persistence at install time are all filtered by it — built-in - filesystem packages (whose source files cannot be modified at runtime) - and DB-listed packages are uniformly covered by this exclusion list. - Installed instances are unaffected (those are install snapshots, managed - separately via enable/disable and uninstall). - """ - - __tablename__ = "plugin_market_skill_exclusions" - - slug = Column(String(100), primary_key=True) - skill_name = Column(String(100), primary_key=True) - created_at = Column(UTCDateTime(timezone=True), default=utc_now) - created_by = Column(String(64)) - - -class MarketplaceListingState(MarketplaceListingEditionFields, Base): - """Marketplace listing switch: controls whether a plugin/skill is shown in the (plugin/skill) marketplace. - - ``kind`` = ``plugin`` | ``skill``; ``item_id`` = plugin slug or skill - marketplace slug. **A missing row counts as enabled** (everything listed - by default); a row with ``enabled=false`` is written only when the admin - explicitly disables — so built-in/existing items are not wiped after an - upgrade. The user-facing marketplace shows only enabled items; the admin - console shows everything with toggles. Physical deletion (only for - uploaded DB items) still goes through the respective delete endpoints; - this table only governs "listing visibility". - - """ - - __tablename__ = "marketplace_listing_states" - - kind = Column(String(16), primary_key=True) # plugin | skill - item_id = Column(String(160), primary_key=True) # plugin slug / skill marketplace slug - enabled = Column(Boolean, nullable=False, default=True) - updated_at = Column(UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now) - updated_by = Column(String(64)) diff --git a/src/backend/core/db/models/agent.py b/src/backend/core/db/models/agent.py index 20a29eb7..b8bbe71d 100644 --- a/src/backend/core/db/models/agent.py +++ b/src/backend/core/db/models/agent.py @@ -2,12 +2,10 @@ from core.db.utc_datetime import UTCDateTime from core.infra.time import utc_now -from datetime import datetime, timezone from core.db.engine import Base -from core.db.model_extensions import UserAgentEditionFields, user_agent_edition_table_args +from core.db.edition import UserAgentEditionFields, user_agent_edition_table_args from sqlalchemy import ( - JSON, BigInteger, Boolean, CheckConstraint, @@ -18,13 +16,9 @@ Numeric, String, Text, - UniqueConstraint, ) -from sqlalchemy.dialects.postgresql import INET, JSONB -from sqlalchemy.orm import mapped_column, relationship - -JSONType = JSON().with_variant(JSONB(), "postgresql") -INETType = String(45).with_variant(INET(), "postgresql") +from sqlalchemy.orm import relationship +from core.db.models.column_types import JSONType class UserAgent(UserAgentEditionFields, Base): @@ -94,67 +88,6 @@ class UserAgent(UserAgentEditionFields, Base): ) -class AgentMarketSubmission(Base): - """Record of a user-created sub-agent's "apply for listing on the sub-agent marketplace" (pending admin review). - - Mirrors the skill marketplace's ``MarketplaceSubmission``, but snapshots a - "sub-agent" rather than skill files: on submission, the source ``UserAgent`` is - content-snapshotted (prompt/welcome message/suggested questions/model config/ - capability bindings); review and listing are both based on the snapshot — later - edits or deletion of the original agent do not affect listed content. - ``status='approved'`` records appear in the sub-agent marketplace list - (source=community) and can be "install-cloned" by everyone as a private sub-agent; - admins can reject at any time (including delisting already-listed items). Direct - admin uploads to the marketplace are distinguished by the sentinel owner - ``__admin_upload__``. - """ - - __tablename__ = "agent_market_submissions" - - submission_id = Column(String(64), primary_key=True) - # Marketplace slug once listed (derived from name at submission, globally unique, never colliding with the preset marketplace catalog) - slug = Column(String(128), nullable=False, unique=True) - agent_id = Column(String(64), nullable=False) # source UserAgent.agent_id - owner_user_id = Column(String(64), nullable=False) # applicant (or __admin_upload__) - submitter_name = Column( - String(255), default="" - ) # applicant display name (denormalized for display) - - # Marketplace display metadata - name = Column(String(255), nullable=False) - avatar = Column(Text) - description = Column(Text, default="") - summary = Column(Text, default="") - category = Column(String(64), default="通用助手") - tags = Column(JSONType, default=list) - version = Column(String(50), default="1.0.0") - note = Column(Text, default="") # application note (for the admin) - - # Sub-agent content snapshot (decoupled from the source UserAgent) - system_prompt = Column(Text, default="") - welcome_message = Column(Text, default="") - suggested_questions = Column(JSONType, default=list) - # {temperature, max_tokens, max_iters, timeout} - model_config_snapshot = Column(JSONType, default=dict) - # {skill_ids, mcp_server_ids, plugin_ids, kb_ids} - bindings_snapshot = Column(JSONType, default=dict) - - status = Column(String(16), nullable=False, default="pending") - review_note = Column(Text) # rejection reason / review note - reviewed_at = Column(UTCDateTime(timezone=True)) - created_at = Column(UTCDateTime(timezone=True), default=utc_now) - updated_at = Column(UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now) - - __table_args__ = ( - CheckConstraint( - "status IN ('pending', 'approved', 'rejected')", - name="agent_market_submissions_status_check", - ), - Index("idx_agent_market_submissions_status", "status", "created_at"), - Index("idx_agent_market_submissions_owner", "owner_user_id"), - ) - - class Plan(Base): """Plan mode - plans table""" diff --git a/src/backend/core/db/models/artifact.py b/src/backend/core/db/models/artifact.py index 0c97e645..2a23c8f2 100644 --- a/src/backend/core/db/models/artifact.py +++ b/src/backend/core/db/models/artifact.py @@ -2,29 +2,20 @@ from core.db.utc_datetime import UTCDateTime from core.infra.time import utc_now -from datetime import datetime, timezone from core.db.engine import Base -from core.db.model_extensions import ArtifactEditionFields, artifact_edition_table_args +from core.db.edition import ArtifactEditionFields, artifact_edition_table_args from sqlalchemy import ( - JSON, BigInteger, - Boolean, CheckConstraint, Column, ForeignKey, Index, - Integer, - Numeric, String, Text, - UniqueConstraint, ) -from sqlalchemy.dialects.postgresql import INET, JSONB -from sqlalchemy.orm import mapped_column, relationship - -JSONType = JSON().with_variant(JSONB(), "postgresql") -INETType = String(45).with_variant(INET(), "postgresql") +from sqlalchemy.orm import relationship +from core.db.models.column_types import JSONType class Artifact(ArtifactEditionFields, Base): @@ -50,7 +41,7 @@ class Artifact(ArtifactEditionFields, Base): storage_key = Column(Text, nullable=False) storage_url = Column(Text) extra_data = Column("metadata", JSONType, default={}) - # Lazy caches for cross-turn file reading (populated by core/content/artifact_reader.py) + # Lazy caches for cross-turn file reading (populated by core/files/artifacts/reader.py) parsed_text = Column(Text, nullable=True) summary = Column(Text, nullable=True) parsed_at = Column(UTCDateTime(timezone=True), nullable=True) diff --git a/src/backend/core/db/models/channel_relay.py b/src/backend/core/db/models/channel_relay.py deleted file mode 100644 index cf92575b..00000000 --- a/src/backend/core/db/models/channel_relay.py +++ /dev/null @@ -1,40 +0,0 @@ -"""Durable, device-scoped channel delivery. Shared by CE and EE.""" - -from core.db.engine import Base -from sqlalchemy import JSON, Boolean, Column, Float, Index, String - - -class DesktopChannelBinding(Base): - __tablename__ = "desktop_channel_bindings" - binding_id = Column(String(64), primary_key=True) - owner_id = Column(String(64), nullable=False, index=True) - device_id = Column(String(128), nullable=False) - device_name = Column(String(100), nullable=False) - consumed = Column(Boolean, nullable=False, default=False) - created_at = Column(Float, nullable=False) - last_seen = Column(Float, nullable=False) - - -class ChannelRelayDelivery(Base): - __tablename__ = "channel_relay_deliveries" - delivery_id = Column(String(64), primary_key=True) - channel_id = Column(String(64), nullable=False) - binding_id = Column(String(64), nullable=False) - owner_id = Column(String(64), nullable=False) - device_id = Column(String(128), nullable=False) - status = Column(String(24), nullable=False) - payload = Column(JSON, nullable=False) - lease = Column(String(64), nullable=True) - created_at = Column(Float, nullable=False) - deadline = Column(Float, nullable=False) - error = Column(String(300), nullable=True) - __table_args__ = ( - Index("ix_channel_relay_queue", "owner_id", "device_id", "status", "created_at"), - ) - - -class ChannelRelayOperation(Base): - __tablename__ = "channel_relay_operations" - operation_id = Column(String(128), primary_key=True) - status = Column(String(24), nullable=False) - result = Column(JSON, nullable=True) diff --git a/src/backend/core/db/models/chat.py b/src/backend/core/db/models/chat.py index 2b092214..36e953d6 100644 --- a/src/backend/core/db/models/chat.py +++ b/src/backend/core/db/models/chat.py @@ -1,45 +1,14 @@ -"""SQLAlchemy ORM models — chat sessions/messages.""" +"""ORM models for chat sessions, messages and compaction state, plus sequence allocation.""" +from __future__ import annotations from core.db.utc_datetime import UTCDateTime from core.infra.time import utc_now - from datetime import datetime, timezone - from core.db.engine import Base -from core.db.model_extensions import ( - ChatSessionEditionFields, - chat_session_edition_table_args, -) -from sqlalchemy import ( - JSON, - BigInteger, - Boolean, - CheckConstraint, - Column, - Computed, - ForeignKey, - LargeBinary, - Index, - Integer, - Numeric, - String, - Text, - UniqueConstraint, - event, - case, - func, - select, - update, -) -from sqlalchemy.dialects.postgresql import INET, JSONB -from sqlalchemy.orm import deferred, mapped_column, relationship - -JSONType = JSON().with_variant(JSONB(), "postgresql") -INETType = String(45).with_variant(INET(), "postgresql") -# BigInteger autoincrement PK: SQLite only auto-increments the INTEGER rowid, not -# BIGINT, so an unqualified BigInteger PK stays NULL on the no-Docker SQLite -# profile. Fall back to Integer on SQLite (still BIGINT on Postgres). -BigIntPK = BigInteger().with_variant(Integer(), "sqlite") - +from core.db.edition import ChatSessionEditionFields, chat_session_edition_table_args +from sqlalchemy import JSON, BigInteger, Boolean, CheckConstraint, Column, Computed, ForeignKey, Index, Integer, String, Text, UniqueConstraint, event, case, func, select, update +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.orm import deferred, relationship +from core.db.models.column_types import JSONType class ChatSession(ChatSessionEditionFields, Base): """Chat session table.""" @@ -158,7 +127,7 @@ class ChatMessage(Base): default=display_default, onupdate=display_default, )) # 模型回放用的规范记录：按发生顺序的每一步（模型响应块 / 工具结果），见 - # ``core/llm/model_steps.py``。上面三列是给界面看的投影，模型上下文从这一列恢复。 + # ``core/llm/runtime/model_steps.py``。上面三列是给界面看的投影，模型上下文从这一列恢复。 # NULL = 该行写入时还没有这份记录，回放退回 segments / 历史摘要。 model_steps = Column(JSONType) usage = Column(JSONType) @@ -296,7 +265,7 @@ def reserve_chat_message_sequences( @event.listens_for(ChatMessage, "before_insert") -def _assign_chat_sequence(_mapper, connection, target: ChatMessage) -> None: +def assign_chat_sequence(_mapper, connection, target: ChatMessage) -> None: """Compatibility guard for legacy/direct message writers. New main-chat admission reserves its user/reply pair explicitly. Existing @@ -306,304 +275,3 @@ def _assign_chat_sequence(_mapper, connection, target: ChatMessage) -> None: if target.chat_seq is None: target.chat_seq = reserve_chat_sequences(connection, target.chat_id) - - -class ChatRun(Base): - """Chat Run — decouples AI tasks from the HTTP connection lifecycle. - - Every message send creates a run; a background asyncio.Task runs the workflow, - chunks are written to a Redis Stream, and the SSE endpoint pulls from the - Stream. After a page refresh, playback resumes via follow_run + offset. - """ - - __tablename__ = "chat_runs" - - run_id = Column(String(64), primary_key=True) - chat_id = Column( - String(64), - ForeignKey("chat_sessions.chat_id", ondelete="CASCADE"), - nullable=False, - ) - user_id = Column(String(64), nullable=False) - message_id = Column( - String(64), nullable=False - ) # pre-allocated assistant message id - user_message_id = Column(String(64)) - user_chat_seq = Column(BigInteger) - assistant_chat_seq = Column(BigInteger) - # Non-null only while this run owns the chat's main-writer slot. A unique - # (chat_id, writer_slot) constraint permits unlimited terminal NULL rows but - # only one live "main" owner, on both PostgreSQL and SQLite. - writer_slot = Column(String(20)) - status = Column(String(20), nullable=False, default="pending") - request_payload = Column( - JSONType - ) # serialized ChatRequest (for the worker to rebuild the context) - last_event_offset = Column(Integer, default=0, nullable=False) - error_message = Column(Text) - usage = Column(JSONType) - # Durable run-journal state. Redis remains an event projection only; these - # columns decide ownership, recovery and terminal CAS after a restart. - run_phase = Column(String(40), nullable=False, default="accepted") - lease_owner = Column(String(160)) - lease_expires_at = Column(UTCDateTime(timezone=True)) - operation_seq = Column(Integer, nullable=False, default=0) - snapshot_version = Column(Integer, nullable=False, default=0) - # Holds the whole in-flight run payload (megabytes on long chats); loading it - # with every ChatRun row decodes that JSON again, so it is fetched on access. - recovery_snapshot = deferred(Column(JSONType)) - last_operation_safety = Column(String(40), nullable=False, default="replayable") - failure_reason = Column(Text) - updated_at = Column( - UTCDateTime(timezone=True), - default=lambda: datetime.now(timezone.utc), - onupdate=lambda: datetime.now(timezone.utc), - nullable=False, - ) - created_at = Column( - UTCDateTime(timezone=True), - default=lambda: datetime.now(timezone.utc), - nullable=False, - ) - started_at = Column(UTCDateTime(timezone=True)) - completed_at = Column(UTCDateTime(timezone=True)) - - __table_args__ = ( - CheckConstraint( - "status IN ('pending', 'running', 'needs_attention', 'completed', 'failed', 'cancelled')", - name="chat_runs_status_check", - ), - CheckConstraint( - "writer_slot IS NULL OR writer_slot = 'main'", - name="chat_runs_writer_slot_check", - ), - UniqueConstraint("chat_id", "writer_slot", name="uq_chat_runs_writer_slot"), - Index("idx_chat_runs_chat_status", "chat_id", "status"), - Index("idx_chat_runs_status_lease", "status", "lease_expires_at"), - Index("idx_chat_runs_user_id", "user_id"), - Index("idx_chat_runs_created_at", "created_at"), - ) - - -class ChatRunOperation(Base): - """Append-only, per-run operation journal used for recovery decisions.""" - - __tablename__ = "chat_run_operations" - - operation_id = Column(BigIntPK, primary_key=True, autoincrement=True) - run_id = Column( - String(64), ForeignKey("chat_runs.run_id", ondelete="CASCADE"), nullable=False - ) - operation_seq = Column(Integer, nullable=False) - operation_type = Column(String(64), nullable=False) - phase = Column(String(40), nullable=False) - safety = Column(String(40), nullable=False) - owner = Column(String(160)) - snapshot_version = Column(Integer, nullable=False, default=0) - payload = Column(JSONType) - created_at = Column( - UTCDateTime(timezone=True), - default=lambda: datetime.now(timezone.utc), - nullable=False, - ) - - __table_args__ = ( - UniqueConstraint("run_id", "operation_seq", name="uq_chat_run_operation_seq"), - CheckConstraint("operation_seq > 0", name="chat_run_operations_seq_positive"), - Index("idx_chat_run_operations_run_seq", "run_id", "operation_seq"), - ) - - -class ChatSteerQueueItem(Base): - """Durable user input queued around a ChatRun safe boundary.""" - - __tablename__ = "chat_steer_queue" - - queue_id = Column(String(64), primary_key=True) - steer_id = Column(String(64), nullable=False) - chat_id = Column( - String(64), - ForeignKey("chat_sessions.chat_id", ondelete="CASCADE"), - nullable=False, - ) - user_id = Column(String(64), nullable=False) - target_run_id = Column( - String(64), ForeignKey("chat_runs.run_id", ondelete="SET NULL"), nullable=True - ) - steer_seq = Column(BigInteger, nullable=False) - target_operation_seq = Column(Integer) - delivery_mode = Column(String(20), nullable=False, default="steer") - status = Column(String(20), nullable=False, default="accepted") - message = Column(Text, nullable=False) - lease_owner = Column(String(160)) - lease_expires_at = Column(UTCDateTime(timezone=True)) - delivery_attempt = Column(Integer, nullable=False, default=0) - superseded_by = Column(String(64)) - applied_run_id = Column(String(64)) - applied_source_run_id = Column( - String(64), ForeignKey("chat_runs.run_id", ondelete="SET NULL"), nullable=True - ) - applied_operation_seq = Column(Integer) - accepted_at = Column( - UTCDateTime(timezone=True), - nullable=False, - default=lambda: datetime.now(timezone.utc), - ) - claimed_at = Column(UTCDateTime(timezone=True)) - applied_at = Column(UTCDateTime(timezone=True)) - cancelled_at = Column(UTCDateTime(timezone=True)) - updated_at = Column( - UTCDateTime(timezone=True), - nullable=False, - default=lambda: datetime.now(timezone.utc), - onupdate=lambda: datetime.now(timezone.utc), - ) - - __table_args__ = ( - CheckConstraint( - "delivery_mode IN ('steer', 'follow_up', 'next_run')", - name="chat_steer_queue_delivery_mode_check", - ), - CheckConstraint( - "status IN ('accepted', 'claimed', 'applied', 'cancelled', 'superseded')", - name="chat_steer_queue_status_check", - ), - CheckConstraint("steer_seq > 0", name="chat_steer_queue_seq_positive"), - CheckConstraint( - "delivery_attempt >= 0", name="chat_steer_queue_attempt_nonnegative" - ), - CheckConstraint( - "length(message) BETWEEN 1 AND 10000", - name="chat_steer_queue_message_length", - ), - UniqueConstraint("chat_id", "steer_id", name="uq_chat_steer_queue_chat_steer"), - UniqueConstraint("chat_id", "steer_seq", name="uq_chat_steer_queue_chat_seq"), - Index( - "idx_chat_steer_queue_run_status_seq", - "target_run_id", - "status", - "steer_seq", - ), - Index( - "idx_chat_steer_queue_chat_status_seq", - "chat_id", - "status", - "steer_seq", - ), - Index( - "idx_chat_steer_queue_applied_source_status_seq", - "applied_source_run_id", - "status", - "steer_seq", - ), - Index("idx_chat_steer_queue_lease", "status", "lease_expires_at"), - ) - - -class MessageFeedback(Base): - """Message feedback table - stores like/dislike ratings and optional comments.""" - - __tablename__ = "message_feedback" - - feedback_id = Column(BigIntPK, primary_key=True, autoincrement=True) - message_id = Column( - String(64), - ForeignKey("chat_messages.message_id", ondelete="CASCADE"), - nullable=False, - ) - chat_id = Column( - String(64), - ForeignKey("chat_sessions.chat_id", ondelete="CASCADE"), - nullable=False, - ) - user_id = Column( - String(64), - ForeignKey("users_shadow.user_id", ondelete="SET NULL"), - nullable=True, - ) - rating = Column(String(10), nullable=False) # 'like' or 'dislike' - comment = Column(Text, nullable=True) - created_at = Column(UTCDateTime(timezone=True), default=utc_now) - updated_at = Column( - UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now - ) - - __table_args__ = ( - CheckConstraint( - "rating IN ('like', 'dislike')", name="message_feedback_rating_check" - ), - Index("idx_message_feedback_message_id", "message_id"), - Index("idx_message_feedback_chat_id", "chat_id"), - Index("idx_message_feedback_user_id", "user_id"), - ) - - -class ChatSandboxSnapshot(Base): - """Per-chat opensandbox snapshot pointer. - - A chat keeps at most 1 snapshot at any time (the latest one). The background - idle worker upserts on snapshot+kill; the next time that chat reconnects, - _get_or_create_session prefers this snapshot when starting a new sandbox, - restoring the filesystem state. The GC worker periodically scans rows with - expires_at < now and deletes remote + DB together. Design in - internal design docs. - """ - - __tablename__ = "chat_sandbox_snapshots" - - chat_id = Column(String(64), primary_key=True) - snapshot_id = Column(String(64), nullable=False, unique=True) - sandbox_id = Column( - String(64), nullable=False - ) # source sandbox id parked at the time, for debugging / reconciliation - created_at = Column( - UTCDateTime(timezone=True), - nullable=False, - default=lambda: datetime.now(timezone.utc), - ) - expires_at = Column( - UTCDateTime(timezone=True), nullable=False - ) # created_at + SNAPSHOT_RETENTION_DAYS - size_bytes = Column(BigInteger) # for metrics, nullable - extra = Column( - "metadata", JSONType, default=dict - ) # reserved: image uri / pool kind / notes - - __table_args__ = ( - Index("idx_chat_sandbox_snapshots_expires", "expires_at"), - Index("idx_chat_sandbox_snapshots_created", "created_at"), - ) - - -class ToolMediaBlob(Base): - """工具返回的媒体字节，按内容寻址，供历史回放取回原件。 - - ``model_steps`` 是「模型自己的历史」的唯一真源，回放它应当得到模型当时真正看过 - 的那串消息。媒体曾经是这条真源上唯一的缺口：落库时图被换成一行 - ``[image omitted from replay history: …]``，于是图只能活一轮——下一轮重建历史时， - 模型看到的是一句「这里原本有张图」，而它不会说自己看不见，会拿手上剩下的信息接 - 着编。 - - 字节单独放一张表、只在 ``model_steps`` 里留 ``sha256`` 指针，是为了不把历史列 - 撑大：拼上下文要把检查点之后每一条消息的 ``model_steps`` 整列拉出来，一页 200dpi - 的图编码后一两兆，内联进去就意味着**每一轮**都要整份读一遍，哪怕预算马上要把那 - 一步挤掉。主键用内容哈希，所以同一张图读多少次都只存一份。 - - 与历史行同库，所以备份、迁移、恢复都把两者一起带走，不存在第二个会和它走散的 - 系统。字节先写、引用后写（引用是随那条消息落库的），所以中途崩溃最坏留下一份没人 - 引用的字节，不会出现「引用还在、字节没了」。 - """ - - __tablename__ = "tool_media_blobs" - - sha256 = Column(String(64), primary_key=True) - media_type = Column(String(128), nullable=False) - data = Column(LargeBinary, nullable=False) - size_bytes = Column(Integer, nullable=False) - created_at = Column( - UTCDateTime(timezone=True), - nullable=False, - default=lambda: datetime.now(timezone.utc), - ) - - __table_args__ = (Index("idx_tool_media_blobs_created", "created_at"),) diff --git a/src/backend/core/db/models/chat_mode.py b/src/backend/core/db/models/chat_mode.py index 996efc06..6dc193dd 100644 --- a/src/backend/core/db/models/chat_mode.py +++ b/src/backend/core/db/models/chat_mode.py @@ -20,11 +20,9 @@ from core.db.utc_datetime import UTCDateTime from core.infra.time import utc_now -from datetime import datetime from core.db.engine import Base from sqlalchemy import ( - JSON, Boolean, CheckConstraint, Column, @@ -35,9 +33,8 @@ Text, UniqueConstraint, ) -from sqlalchemy.dialects.postgresql import JSONB +from core.db.models.column_types import JSONType -JSONType = JSON().with_variant(JSONB(), "postgresql") #: 工具面范围。``all`` = 不额外收窄（标准模式：跟随 catalog 与用户自身授权）； #: ``restricted`` = 只装配本行显式列出的 MCP / 技能 / 插件（极速模式与自定义模式）。 diff --git a/src/backend/core/db/models/chat_resources.py b/src/backend/core/db/models/chat_resources.py new file mode 100644 index 00000000..dff58a8c --- /dev/null +++ b/src/backend/core/db/models/chat_resources.py @@ -0,0 +1,116 @@ +"""ORM models for message feedback, sandbox snapshots and tool media blobs.""" +from __future__ import annotations +from core.db.utc_datetime import UTCDateTime +from core.infra.time import utc_now +from datetime import datetime, timezone +from core.db.engine import Base +from sqlalchemy import BigInteger, CheckConstraint, Column, ForeignKey, LargeBinary, Index, Integer, String, Text +from core.db.models.column_types import BigIntPK, JSONType + +class MessageFeedback(Base): + """Message feedback table - stores like/dislike ratings and optional comments.""" + + __tablename__ = "message_feedback" + + feedback_id = Column(BigIntPK, primary_key=True, autoincrement=True) + message_id = Column( + String(64), + ForeignKey("chat_messages.message_id", ondelete="CASCADE"), + nullable=False, + ) + chat_id = Column( + String(64), + ForeignKey("chat_sessions.chat_id", ondelete="CASCADE"), + nullable=False, + ) + user_id = Column( + String(64), + ForeignKey("users_shadow.user_id", ondelete="SET NULL"), + nullable=True, + ) + rating = Column(String(10), nullable=False) # 'like' or 'dislike' + comment = Column(Text, nullable=True) + created_at = Column(UTCDateTime(timezone=True), default=utc_now) + updated_at = Column( + UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now + ) + + __table_args__ = ( + CheckConstraint( + "rating IN ('like', 'dislike')", name="message_feedback_rating_check" + ), + Index("idx_message_feedback_message_id", "message_id"), + Index("idx_message_feedback_chat_id", "chat_id"), + Index("idx_message_feedback_user_id", "user_id"), + ) + + +class ChatSandboxSnapshot(Base): + """Per-chat opensandbox snapshot pointer. + + A chat keeps at most 1 snapshot at any time (the latest one). The background + idle worker upserts on snapshot+kill; the next time that chat reconnects, + _get_or_create_session prefers this snapshot when starting a new sandbox, + restoring the filesystem state. The GC worker periodically scans rows with + expires_at < now and deletes remote + DB together. Design in + internal design docs. + """ + + __tablename__ = "chat_sandbox_snapshots" + + chat_id = Column(String(64), primary_key=True) + snapshot_id = Column(String(64), nullable=False, unique=True) + sandbox_id = Column( + String(64), nullable=False + ) # source sandbox id parked at the time, for debugging / reconciliation + created_at = Column( + UTCDateTime(timezone=True), + nullable=False, + default=lambda: datetime.now(timezone.utc), + ) + expires_at = Column( + UTCDateTime(timezone=True), nullable=False + ) # created_at + SNAPSHOT_RETENTION_DAYS + size_bytes = Column(BigInteger) # for metrics, nullable + extra = Column( + "metadata", JSONType, default=dict + ) # reserved: image uri / pool kind / notes + + __table_args__ = ( + Index("idx_chat_sandbox_snapshots_expires", "expires_at"), + Index("idx_chat_sandbox_snapshots_created", "created_at"), + ) + + +class ToolMediaBlob(Base): + """工具返回的媒体字节，按内容寻址，供历史回放取回原件。 + + ``model_steps`` 是「模型自己的历史」的唯一真源，回放它应当得到模型当时真正看过 + 的那串消息。媒体曾经是这条真源上唯一的缺口：落库时图被换成一行 + ``[image omitted from replay history: …]``，于是图只能活一轮——下一轮重建历史时， + 模型看到的是一句「这里原本有张图」，而它不会说自己看不见，会拿手上剩下的信息接 + 着编。 + + 字节单独放一张表、只在 ``model_steps`` 里留 ``sha256`` 指针，是为了不把历史列 + 撑大：拼上下文要把检查点之后每一条消息的 ``model_steps`` 整列拉出来，一页 200dpi + 的图编码后一两兆，内联进去就意味着**每一轮**都要整份读一遍，哪怕预算马上要把那 + 一步挤掉。主键用内容哈希，所以同一张图读多少次都只存一份。 + + 与历史行同库，所以备份、迁移、恢复都把两者一起带走，不存在第二个会和它走散的 + 系统。字节先写、引用后写（引用是随那条消息落库的），所以中途崩溃最坏留下一份没人 + 引用的字节，不会出现「引用还在、字节没了」。 + """ + + __tablename__ = "tool_media_blobs" + + sha256 = Column(String(64), primary_key=True) + media_type = Column(String(128), nullable=False) + data = Column(LargeBinary, nullable=False) + size_bytes = Column(Integer, nullable=False) + created_at = Column( + UTCDateTime(timezone=True), + nullable=False, + default=lambda: datetime.now(timezone.utc), + ) + + __table_args__ = (Index("idx_tool_media_blobs_created", "created_at"),) diff --git a/src/backend/core/db/models/chat_runs.py b/src/backend/core/db/models/chat_runs.py new file mode 100644 index 00000000..e0afa2f3 --- /dev/null +++ b/src/backend/core/db/models/chat_runs.py @@ -0,0 +1,199 @@ +"""ORM models for chat runs, their operation journal and the steering queue.""" +from __future__ import annotations +from core.db.utc_datetime import UTCDateTime +from datetime import datetime, timezone +from core.db.engine import Base +from sqlalchemy import BigInteger, CheckConstraint, Column, ForeignKey, Index, Integer, String, Text, UniqueConstraint +from sqlalchemy.orm import deferred +from core.db.models.column_types import BigIntPK, JSONType + +class ChatRun(Base): + """Chat Run — decouples AI tasks from the HTTP connection lifecycle. + + Every message send creates a run; a background asyncio.Task runs the workflow, + chunks are written to a Redis Stream, and the SSE endpoint pulls from the + Stream. After a page refresh, playback resumes via follow_run + offset. + """ + + __tablename__ = "chat_runs" + + run_id = Column(String(64), primary_key=True) + chat_id = Column( + String(64), + ForeignKey("chat_sessions.chat_id", ondelete="CASCADE"), + nullable=False, + ) + user_id = Column(String(64), nullable=False) + message_id = Column( + String(64), nullable=False + ) # pre-allocated assistant message id + user_message_id = Column(String(64)) + user_chat_seq = Column(BigInteger) + assistant_chat_seq = Column(BigInteger) + # Non-null only while this run owns the chat's main-writer slot. A unique + # (chat_id, writer_slot) constraint permits unlimited terminal NULL rows but + # only one live "main" owner, on both PostgreSQL and SQLite. + writer_slot = Column(String(20)) + status = Column(String(20), nullable=False, default="pending") + request_payload = Column( + JSONType + ) # serialized ChatRequest (for the worker to rebuild the context) + last_event_offset = Column(Integer, default=0, nullable=False) + error_message = Column(Text) + usage = Column(JSONType) + # Durable run-journal state. Redis remains an event projection only; these + # columns decide ownership, recovery and terminal CAS after a restart. + run_phase = Column(String(40), nullable=False, default="accepted") + lease_owner = Column(String(160)) + lease_expires_at = Column(UTCDateTime(timezone=True)) + operation_seq = Column(Integer, nullable=False, default=0) + snapshot_version = Column(Integer, nullable=False, default=0) + # Holds the whole in-flight run payload (megabytes on long chats); loading it + # with every ChatRun row decodes that JSON again, so it is fetched on access. + recovery_snapshot = deferred(Column(JSONType)) + last_operation_safety = Column(String(40), nullable=False, default="replayable") + failure_reason = Column(Text) + updated_at = Column( + UTCDateTime(timezone=True), + default=lambda: datetime.now(timezone.utc), + onupdate=lambda: datetime.now(timezone.utc), + nullable=False, + ) + created_at = Column( + UTCDateTime(timezone=True), + default=lambda: datetime.now(timezone.utc), + nullable=False, + ) + started_at = Column(UTCDateTime(timezone=True)) + completed_at = Column(UTCDateTime(timezone=True)) + + __table_args__ = ( + CheckConstraint( + "status IN ('pending', 'running', 'needs_attention', 'completed', 'failed', 'cancelled')", + name="chat_runs_status_check", + ), + CheckConstraint( + "writer_slot IS NULL OR writer_slot = 'main'", + name="chat_runs_writer_slot_check", + ), + UniqueConstraint("chat_id", "writer_slot", name="uq_chat_runs_writer_slot"), + Index("idx_chat_runs_chat_status", "chat_id", "status"), + Index("idx_chat_runs_status_lease", "status", "lease_expires_at"), + Index("idx_chat_runs_user_id", "user_id"), + Index("idx_chat_runs_created_at", "created_at"), + ) + + +class ChatRunOperation(Base): + """Append-only, per-run operation journal used for recovery decisions.""" + + __tablename__ = "chat_run_operations" + + operation_id = Column(BigIntPK, primary_key=True, autoincrement=True) + run_id = Column( + String(64), ForeignKey("chat_runs.run_id", ondelete="CASCADE"), nullable=False + ) + operation_seq = Column(Integer, nullable=False) + operation_type = Column(String(64), nullable=False) + phase = Column(String(40), nullable=False) + safety = Column(String(40), nullable=False) + owner = Column(String(160)) + snapshot_version = Column(Integer, nullable=False, default=0) + payload = Column(JSONType) + created_at = Column( + UTCDateTime(timezone=True), + default=lambda: datetime.now(timezone.utc), + nullable=False, + ) + + __table_args__ = ( + UniqueConstraint("run_id", "operation_seq", name="uq_chat_run_operation_seq"), + CheckConstraint("operation_seq > 0", name="chat_run_operations_seq_positive"), + Index("idx_chat_run_operations_run_seq", "run_id", "operation_seq"), + ) + + +class ChatSteerQueueItem(Base): + """Durable user input queued around a ChatRun safe boundary.""" + + __tablename__ = "chat_steer_queue" + + queue_id = Column(String(64), primary_key=True) + steer_id = Column(String(64), nullable=False) + chat_id = Column( + String(64), + ForeignKey("chat_sessions.chat_id", ondelete="CASCADE"), + nullable=False, + ) + user_id = Column(String(64), nullable=False) + target_run_id = Column( + String(64), ForeignKey("chat_runs.run_id", ondelete="SET NULL"), nullable=True + ) + steer_seq = Column(BigInteger, nullable=False) + target_operation_seq = Column(Integer) + delivery_mode = Column(String(20), nullable=False, default="steer") + status = Column(String(20), nullable=False, default="accepted") + message = Column(Text, nullable=False) + lease_owner = Column(String(160)) + lease_expires_at = Column(UTCDateTime(timezone=True)) + delivery_attempt = Column(Integer, nullable=False, default=0) + superseded_by = Column(String(64)) + applied_run_id = Column(String(64)) + applied_source_run_id = Column( + String(64), ForeignKey("chat_runs.run_id", ondelete="SET NULL"), nullable=True + ) + applied_operation_seq = Column(Integer) + accepted_at = Column( + UTCDateTime(timezone=True), + nullable=False, + default=lambda: datetime.now(timezone.utc), + ) + claimed_at = Column(UTCDateTime(timezone=True)) + applied_at = Column(UTCDateTime(timezone=True)) + cancelled_at = Column(UTCDateTime(timezone=True)) + updated_at = Column( + UTCDateTime(timezone=True), + nullable=False, + default=lambda: datetime.now(timezone.utc), + onupdate=lambda: datetime.now(timezone.utc), + ) + + __table_args__ = ( + CheckConstraint( + "delivery_mode IN ('steer', 'follow_up', 'next_run')", + name="chat_steer_queue_delivery_mode_check", + ), + CheckConstraint( + "status IN ('accepted', 'claimed', 'applied', 'cancelled', 'superseded')", + name="chat_steer_queue_status_check", + ), + CheckConstraint("steer_seq > 0", name="chat_steer_queue_seq_positive"), + CheckConstraint( + "delivery_attempt >= 0", name="chat_steer_queue_attempt_nonnegative" + ), + CheckConstraint( + "length(message) BETWEEN 1 AND 10000", + name="chat_steer_queue_message_length", + ), + UniqueConstraint("chat_id", "steer_id", name="uq_chat_steer_queue_chat_steer"), + UniqueConstraint("chat_id", "steer_seq", name="uq_chat_steer_queue_chat_seq"), + Index( + "idx_chat_steer_queue_run_status_seq", + "target_run_id", + "status", + "steer_seq", + ), + Index( + "idx_chat_steer_queue_chat_status_seq", + "chat_id", + "status", + "steer_seq", + ), + Index( + "idx_chat_steer_queue_applied_source_status_seq", + "applied_source_run_id", + "status", + "steer_seq", + ), + Index("idx_chat_steer_queue_lease", "status", "lease_expires_at"), + ) diff --git a/src/backend/core/db/models/column_types.py b/src/backend/core/db/models/column_types.py new file mode 100644 index 00000000..1306a123 --- /dev/null +++ b/src/backend/core/db/models/column_types.py @@ -0,0 +1,9 @@ +"""Portable column types shared by every ORM model: JSONB/INET on PostgreSQL, plain types elsewhere.""" +from __future__ import annotations +from sqlalchemy import JSON, BigInteger, Integer, String +from sqlalchemy.dialects.postgresql import INET, JSONB + +JSONType = JSON().with_variant(JSONB(), "postgresql") +INETType = String(45).with_variant(INET(), "postgresql") +# SQLite only auto-increments the INTEGER rowid, so a BIGINT key would stay NULL there. +BigIntPK = BigInteger().with_variant(Integer(), "sqlite") diff --git a/src/backend/core/db/models/config.py b/src/backend/core/db/models/config.py index 694beb1b..d16b5346 100644 --- a/src/backend/core/db/models/config.py +++ b/src/backend/core/db/models/config.py @@ -2,28 +2,20 @@ from core.db.utc_datetime import UTCDateTime from core.infra.time import utc_now -from datetime import datetime, timezone from core.db.engine import Base from sqlalchemy import ( - JSON, - BigInteger, Boolean, CheckConstraint, Column, ForeignKey, Index, Integer, - Numeric, String, Text, - UniqueConstraint, ) -from sqlalchemy.dialects.postgresql import INET, JSONB -from sqlalchemy.orm import mapped_column, relationship - -JSONType = JSON().with_variant(JSONB(), "postgresql") -INETType = String(45).with_variant(INET(), "postgresql") +from sqlalchemy.orm import relationship +from core.db.models.column_types import JSONType class ModelProvider(Base): diff --git a/src/backend/core/db/models/evolution.py b/src/backend/core/db/models/evolution.py index a82e3330..f27ade76 100644 --- a/src/backend/core/db/models/evolution.py +++ b/src/backend/core/db/models/evolution.py @@ -12,12 +12,9 @@ from core.db.utc_datetime import UTCDateTime from core.infra.time import utc_now -from datetime import datetime from core.db.engine import Base from sqlalchemy import ( - JSON, - BigInteger, Boolean, CheckConstraint, Column, @@ -28,10 +25,7 @@ Text, UniqueConstraint, ) -from sqlalchemy.dialects.postgresql import JSONB - -JSONType = JSON().with_variant(JSONB(), "postgresql") -BigIntPK = BigInteger().with_variant(Integer(), "sqlite") +from core.db.models.column_types import JSONType class EvolutionEpisode(Base): @@ -282,7 +276,7 @@ class EvolutionEvaluation(Base): class EvolutionAgentProfile(Base): - """A published orchestration profile — see :mod:`core.evolution.agent_profile`. + """A published orchestration profile — see :mod:`core.evolution.profiles.agent_profile`. Stored as one JSON payload rather than a column per knob. The profile's shape is itself under evolution (a new intervention signal, a new routing diff --git a/src/backend/core/db/models/identity.py b/src/backend/core/db/models/identity.py index e02a0a64..865bf8dc 100644 --- a/src/backend/core/db/models/identity.py +++ b/src/backend/core/db/models/identity.py @@ -2,29 +2,21 @@ from core.db.utc_datetime import UTCDateTime from core.infra.time import utc_now -from datetime import datetime, timezone from core.db.engine import Base from sqlalchemy import ( - JSON, - BigInteger, Boolean, CheckConstraint, Column, ForeignKey, Index, - Integer, - Numeric, String, Text, UniqueConstraint, text, ) -from sqlalchemy.dialects.postgresql import INET, JSONB -from sqlalchemy.orm import mapped_column, relationship - -JSONType = JSON().with_variant(JSONB(), "postgresql") -INETType = String(45).with_variant(INET(), "postgresql") +from sqlalchemy.orm import relationship +from core.db.models.column_types import JSONType class UserShadow(Base): @@ -47,7 +39,6 @@ class UserShadow(Base): catalog_overrides = relationship( "CatalogOverride", back_populates="user", cascade="all, delete-orphan" ) - kb_spaces = relationship("KBSpace", back_populates="user", cascade="all, delete-orphan") artifacts = relationship("Artifact", back_populates="user", cascade="all, delete-orphan") user_agents = relationship( "UserAgent", foreign_keys="[UserAgent.user_id]", back_populates="user" @@ -102,7 +93,7 @@ class UserFolder(Base): """Personal folder — tree structure under My Space; NULL parent means the root directory. Constraints such as tree depth and name validation are enforced by UserFolderService - (see core/services/user_folder_service.py). + (see core/projects/user_folders.py). """ __tablename__ = "user_folders" @@ -186,271 +177,7 @@ class UserApiKey(Base): ) -class DingTalkConnection(Base): - """Per-user DingTalk account connection (used by the dingtalk skill / dws CLI). 1:1 with users_shadow. - - The credentials themselves (OAuth token + encrypted keychain) live on the backend - persistent volume at ``$STORAGE/dws_cache/{user_id}/``, bind-mounted into the sandbox - dws's ``$HOME`` — they **never go into the DB**. This table only stores the connection - **status and a DingTalk identity summary**, plus an optional portable credential bundle - (``auth_bundle``, for plan-B fallback/migration, stored after application-layer - encryption). See internal design docs. - """ - - __tablename__ = "dingtalk_connections" - - user_id = Column( - String(64), - ForeignKey("users_shadow.user_id", ondelete="CASCADE"), - primary_key=True, - ) - # disconnected (default / disconnected) | pending (device-flow login in progress) | connected | error - status = Column(String(16), nullable=False, default="disconnected") - dingtalk_user_id = Column( - String(128) - ) # DingTalk userId (backfilled from get-self after a successful connect) - dingtalk_name = Column(String(255)) # DingTalk display name - corp_id = Column(String(128)) # DingTalk corp corpId - granted_scopes = Column( - JSONType, default=list - ) # List of granted scopes (accumulated via PAT grants) - # Device-flow login state (echoed to the frontend while pending; cleared on success) - login_verification_url = Column( - Text - ) # Plain verification URL (must be used together with user_code) - login_verification_url_complete = Column( - Text - ) # Full URL with the code embedded → target of the QR code - login_user_code = Column(String(64)) - login_started_at = Column(UTCDateTime(timezone=True)) - auth_bundle = Column( - Text - ) # Plan B: dws auth export --base64 (application-layer encrypted), nullable - last_verified_at = Column(UTCDateTime(timezone=True)) - last_error = Column(Text) - created_at = Column(UTCDateTime(timezone=True), default=utc_now) - updated_at = Column(UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now) - - __table_args__ = ( - CheckConstraint( - "status IN ('disconnected', 'pending', 'connected', 'error')", - name="dingtalk_connections_status_check", - ), - Index("idx_dingtalk_connections_status", "status"), - ) - - -class LarkConnection(Base): - """Per-user Lark (Feishu) account connection (used by the feishu-cli plugin / lark-cli). 1:1 with users_shadow. - - Isomorphic to [[DingTalkConnection]]: the credentials themselves (user_access_token + - file-based encrypted store, including master.key) live on the backend persistent volume at - ``$STORAGE/lark_cache/{user_id}/``, bind-mounted into the sandbox lark-cli's ``$HOME`` - (~/.lark-cli + ~/.local/share/lark-cli) — they **never go into the DB**. This table only - stores the connection status, a Lark identity summary, and the device-flow login state. - The Lark device flow uses ``auth login --no-wait`` to first obtain device_code/ - verification_url (persisted into login_device_code), then completes with ``--device-code``. - The app's app_id/secret are seeded into the per-user HOME by the backend via config init - (not injected as env vars, to avoid breaking --as user). - See internal design docs. - """ - - __tablename__ = "lark_connections" - - user_id = Column( - String(64), - ForeignKey("users_shadow.user_id", ondelete="CASCADE"), - primary_key=True, - ) - # disconnected (default / disconnected) | pending (device-flow login in progress) | connected | error - status = Column(String(16), nullable=False, default="disconnected") - lark_open_id = Column( - String(128) - ) # Lark open_id / union_id (backfilled after a successful connect) - lark_name = Column(String(255)) # Lark display name - tenant_key = Column(String(128)) # Lark tenant tenant_key (counterpart of DingTalk corp_id) - granted_scopes = Column( - JSONType, default=list - ) # List of granted scopes (accumulated via incremental grants) - # Device-flow login state (echoed to the frontend while pending; cleared on success) - login_verification_url = Column( - Text - ) # Plain verification URL (must be used together with user_code) - login_verification_url_complete = Column( - Text - ) # Full URL with the code embedded → target of the QR code - login_user_code = Column(String(64)) - login_device_code = Column( - Text - ) # device_code obtained via --no-wait, used when completing with --device-code - login_started_at = Column(UTCDateTime(timezone=True)) - auth_bundle = Column( - Text - ) # Portable credential bundle (for cube cross-machine use, application-layer encrypted), nullable - last_verified_at = Column(UTCDateTime(timezone=True)) - last_error = Column(Text) - created_at = Column(UTCDateTime(timezone=True), default=utc_now) - updated_at = Column(UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now) - - __table_args__ = ( - CheckConstraint( - "status IN ('disconnected', 'pending', 'connected', 'error')", - name="lark_connections_status_check", - ), - Index("idx_lark_connections_status", "status"), - ) - - -class EmailConnection(Base): - """Per-user email account connection (used by the email plugin / himalaya CLI). 1:1 with users_shadow. +# Optional modules own their dependent rows; all cleanups share this transaction. +from core.db.user_cleanup import install_user_cleanup - Isomorphic to [[DingTalkConnection]]/[[LarkConnection]], but with **no device flow / OAuth / - QR code**: email binds via per-user IMAP/SMTP + an authorization code (app password), - covering Gmail/Outlook/Exchange/NetEase enterprise mail/Tencent enterprise mail/self-hosted - mailboxes. Binding is **synchronous** ("save the form → write himalaya config.toml → run - ``himalaya folder list`` to verify"), so the status is only - disconnected/connected/error (no pending). - - The credentials themselves (himalaya ``config.toml``, containing plaintext authorization-code - lines) live on the backend persistent volume at - ``$STORAGE/email_cache/{user_id}/home/.config/himalaya/`` (0600), bind-mounted into the - sandbox himalaya's ``~/.config/himalaya``. This table only stores the connection status + - account/server metadata + the encrypted authorization code (``secret_enc``, - application-layer Fernet encryption) + a portable config bundle (``config_bundle``, injected - per-session when cube has no bind-mount, base64(config.toml)). - See internal design docs. - """ - - __tablename__ = "email_connections" - - user_id = Column( - String(64), - ForeignKey("users_shadow.user_id", ondelete="CASCADE"), - primary_key=True, - ) - # disconnected (default / disconnected) | connected | error (no pending — binding is a synchronous check) - status = Column(String(16), nullable=False, default="disconnected") - email_address = Column(String(320)) # Bound email address (= IMAP/SMTP login name) - display_name = Column(String(255)) # Sender display name - provider = Column( - String(32) - ) # Auto-detected provider: gmail/outlook/netease/qq/qiye163/exmail/custom - - # Server settings (auto-detected + user-overridable) - imap_host = Column(String(255)) - imap_port = Column(Integer) - imap_security = Column(String(16)) # tls | starttls | none - smtp_host = Column(String(255)) - smtp_port = Column(Integer) - smtp_security = Column(String(16)) - - secret_enc = Column( - Text - ) # Authorization code (application-layer Fernet encryption, never stored in plaintext) - config_bundle = Column(Text) # base64(config.toml), for cube cross-session injection, nullable - last_verified_at = Column(UTCDateTime(timezone=True)) - last_error = Column(Text) - created_at = Column(UTCDateTime(timezone=True), default=utc_now) - updated_at = Column(UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now) - - __table_args__ = ( - CheckConstraint( - "status IN ('disconnected', 'connected', 'error')", - name="email_connections_status_check", - ), - Index("idx_email_connections_status", "status"), - ) - - -class ChannelConnection(Base): - """Inbound channel bot (owner service-account model). One row = one user-created external IM bot. - - Orthogonal to the **outbound** plugins [[LarkConnection]]/[[DingTalkConnection]]: those are - "the agent operates Lark/DingTalk with the user's OAuth identity"; this table is "an - external IM pushes messages in to trigger the agent" (**inbound**), and the credentials are - an **application identity** (App ID/Secret), not user OAuth. - - Multi-tenancy uses the owner service-account model: the bot is bound to ``owner_user_id``, - and all inbound messages run with the owner's identity + permissions; people in a group are - not tied to platform teams and there is no per-sender identity resolution (open_id is only - recorded for audit). p2p and group share the same orchestration path; the only difference is - session keying (by sender open_id vs by group chat_id). - - Credentials (App Secret / Encrypt Key / Verification Token) are encrypted with - application-layer Fernet via ``core.infra.crypto`` and stored in the ``config`` JSON — - **never stored in plaintext, never unioned into the global environment**. ``app_id`` is - redundantly stored in plaintext in its own column for the unique constraint (token lock: - the same Lark app cannot be bound by two people). - - See internal design docs. - """ - - __tablename__ = "channel_connections" - - channel_id = Column(String(64), primary_key=True) - owner_user_id = Column( - String(64), - ForeignKey("users_shadow.user_id", ondelete="CASCADE"), - nullable=False, - ) - channel_type = Column(String(16), nullable=False) # 'lark' | 'dingtalk' | 'wecom' | ... - display_name = Column(String(100), nullable=False, default="我的机器人") - transport = Column(String(16), nullable=False, default="long_conn") # 'long_conn' | 'webhook' - app_id = Column( - String(128), nullable=False - ) # Plaintext redundancy: for the unique constraint + establishing the long connection - config = Column( - JSONType, nullable=False, default=dict - ) # Encrypted credentials {app_secret_enc, encrypt_key_enc, verification_token_enc} - # Resource allowlist (for when the owner wants precise control over what the bot exposes): - # {"kb_ids": [...], "skill_ids": [...]}; NULL = expose everything the owner has (default). - # Retrieval/skill loading narrows down by this on top of the owner's permissions. - resource_scope = Column(JSONType, nullable=True) - # Binding to a specific sub-agent: non-NULL → inbound messages are pinned to that - # [[UserAgent]] (runs with its own prompt/tools/model); written when binding on the - # "sub-agent page"; NULL → main agent (owner's default capabilities), left empty when - # binding in the settings "My Bots" page. - # ondelete=SET NULL: deleting the sub-agent makes the bot fall back to the main agent - # automatically, without cascading the bot's deletion. - agent_id = Column( - String(64), ForeignKey("user_agents.agent_id", ondelete="SET NULL"), nullable=True - ) - # Group listening: how much of a group conversation the bot may see. - # mention_only (default) — only messages that @ the bot; anything else is dropped. - # observe_all — bystander group messages are recorded as background context - # (never replied to) and folded into the next @bot turn. - # observe_all is inert unless the channel app also holds the platform's "read all group - # messages" permission (Lark ``im:message.group_msg`` / DingTalk group-message read) — - # without it the platform never delivers non-@ messages in the first place. - # Defaults to mention_only because observing a whole group's chatter is privacy-sensitive - # and must be an explicit, per-bot decision by the owner. - group_listen_mode = Column( - String(16), nullable=False, default="mention_only", server_default="mention_only" - ) - # disconnected (default / disconnected) | pending (verifying) | connected | error - status = Column(String(16), nullable=False, default="pending") - enabled = Column(Boolean, nullable=False, default=True) - last_event_at = Column(UTCDateTime(timezone=True)) - last_error = Column(Text) - created_at = Column(UTCDateTime(timezone=True), default=utc_now) - updated_at = Column(UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now) - - __table_args__ = ( - CheckConstraint( - "status IN ('disconnected', 'pending', 'connected', 'error')", - name="channel_connections_status_check", - ), - CheckConstraint( - "transport IN ('long_conn', 'webhook')", - name="channel_connections_transport_check", - ), - CheckConstraint( - "group_listen_mode IN ('mention_only', 'observe_all')", - name="channel_connections_group_listen_check", - ), - # token lock: within the same channel, the same app can only be bound once (prevents multiple people grabbing the same bot credentials) - UniqueConstraint("channel_type", "app_id", name="uq_channel_connections_type_app"), - Index("idx_channel_connections_owner", "owner_user_id"), - Index("idx_channel_connections_enabled", "enabled", "status"), - Index("idx_channel_connections_agent", "agent_id"), - ) +install_user_cleanup(UserShadow) diff --git a/src/backend/core/db/models/job.py b/src/backend/core/db/models/job.py index 2c4da1a4..bd2f533f 100644 --- a/src/backend/core/db/models/job.py +++ b/src/backend/core/db/models/job.py @@ -18,14 +18,18 @@ from datetime import datetime, timezone from sqlalchemy import ( - Column, String, Integer, BigInteger, Text, TIMESTAMP, - ForeignKey, CheckConstraint, Index, JSON + Column, + String, + Integer, + BigInteger, + Text, + ForeignKey, + CheckConstraint, + Index, ) -from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.orm import relationship from core.db.engine import Base - -JSONType = JSON().with_variant(JSONB(), "postgresql") +from core.db.models.column_types import JSONType def _utcnow() -> datetime: diff --git a/src/backend/core/db/models/kb_wiki.py b/src/backend/core/db/models/kb_wiki.py deleted file mode 100644 index aff8e0e5..00000000 --- a/src/backend/core/db/models/kb_wiki.py +++ /dev/null @@ -1,224 +0,0 @@ -"""SQLAlchemy ORM models — 自建知识库的 LLM Wiki 层。 - -Wiki 是知识库的**结构地图**：管线从原文分块里抽出实体页与概念页，为每页写一段带 -出处的 Markdown，页与页之间用 ``[[slug|显示名]]`` 互链成图。它不是第二个检索源—— -答案与出处始终回到 ``kb_chunks`` 的原文，页上的 ``source_refs`` / ``chunk_refs`` -就是回原文的血缘坐标。 - -与外接知识库的 Wiki 透传不同，这三张表与 ``kb_documents`` / ``kb_chunks`` 同库， -血缘可以按 ID 直接 JOIN 取回，不需要「拉整篇文档再本地过滤」。 -""" -from core.db.utc_datetime import UTCDateTime -from core.infra.time import utc_now - -from datetime import datetime - -from core.db.engine import Base -from sqlalchemy import ( - JSON, - CheckConstraint, - Column, - ForeignKey, - Index, - Integer, - String, - Text, - text, -) -from sqlalchemy.dialects.postgresql import JSONB - -JSONType = JSON().with_variant(JSONB(), "postgresql") - -# 目录树最大层级。提示词只要 2 级，这里留 3 级是防御性上限：模型偶尔多给一层、 -# 或后续 remap 叠了一层时，不至于长出无边界的面包屑。 -WIKI_CATEGORY_MAX_DEPTH = 3 - -# 页面类型 -WIKI_PAGE_TYPE_SUMMARY = "summary" # 单篇源文档的摘要页，slug = summary/ -WIKI_PAGE_TYPE_ENTITY = "entity" # 实体页（人/组织/产品/地点/技术/事件） -WIKI_PAGE_TYPE_CONCEPT = "concept" # 概念页（主题/方法/理论） -WIKI_PAGE_TYPE_INDEX = "index" # 全库索引页 -WIKI_PAGE_TYPE_SYNTHESIS = "synthesis" # 综述页——管线不产，仅由智能体写入 -WIKI_PAGE_TYPE_COMPARISON = "comparison" # 对比页——管线不产，仅由智能体写入 - -WIKI_PAGE_TYPES = ( - WIKI_PAGE_TYPE_SUMMARY, - WIKI_PAGE_TYPE_ENTITY, - WIKI_PAGE_TYPE_CONCEPT, - WIKI_PAGE_TYPE_INDEX, - WIKI_PAGE_TYPE_SYNTHESIS, - WIKI_PAGE_TYPE_COMPARISON, -) - -# 「知识」视图合并展示的页面类型：这四类形态相近，索引页与文档摘要页各自独立成组 -WIKI_KNOWLEDGE_PAGE_TYPES = ( - WIKI_PAGE_TYPE_ENTITY, - WIKI_PAGE_TYPE_CONCEPT, - WIKI_PAGE_TYPE_SYNTHESIS, - WIKI_PAGE_TYPE_COMPARISON, -) - -# 页面类型 → 中文展示名 -WIKI_PAGE_TYPE_LABELS = { - WIKI_PAGE_TYPE_CONCEPT: "概念", - WIKI_PAGE_TYPE_ENTITY: "实体", - WIKI_PAGE_TYPE_SYNTHESIS: "综述", - WIKI_PAGE_TYPE_COMPARISON: "对比", - WIKI_PAGE_TYPE_SUMMARY: "文档摘要", - WIKI_PAGE_TYPE_INDEX: "总索引", -} - -# 编辑来源：区分「这段是管线写的还是人改的」 -WIKI_EDIT_SOURCE_PIPELINE = "pipeline" -WIKI_EDIT_SOURCE_AGENT = "agent" -WIKI_EDIT_SOURCE_USER = "user" - - -class KBWikiFolder(Base): - """Wiki 目录节点（邻接表）。 - - ``parent_id`` 为空串表示根层。``path`` 是用 ``/`` 连接的名称链（物化路径）， - 只为展示与筛选服务；归属关系的真源永远是 ``parent_id``。空目录允许独立存在， - 这样用户可以先搭骨架再往里填页。 - """ - - __tablename__ = "kb_wiki_folders" - - folder_id = Column(String(64), primary_key=True) - kb_id = Column(String(64), ForeignKey("kb_spaces.kb_id", ondelete="CASCADE"), nullable=False) - parent_id = Column(String(64), nullable=False, default="") - name = Column(String(255), nullable=False) - path = Column(String(1024), nullable=False, default="") - depth = Column(Integer, nullable=False, default=0) - sort_order = Column(Integer, nullable=False, default=0) - created_at = Column(UTCDateTime(timezone=True), default=utc_now) - updated_at = Column(UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now) - deleted_at = Column(UTCDateTime(timezone=True)) - - __table_args__ = ( - Index("idx_kb_wiki_folders_kb_parent", "kb_id", "parent_id"), - Index( - "idx_kb_wiki_folders_unique_name", - "kb_id", - "parent_id", - "name", - unique=True, - postgresql_where=Column("deleted_at").is_(None), - ), - ) - - -class KBWikiPage(Base): - """一个 Wiki 页面。 - - ``folder_id`` 是页面归属目录的唯一真源；``category_path`` / ``wiki_path`` / - ``depth`` / ``sort_order`` 都是从目录链派生出来的缓存投影，重建目录时会一并刷新。 - - 血缘两列的语义要分清：``source_refs`` 是该页取材自哪些文档 - （``kb_documents.document_id``），``chunk_refs`` 是具体哪几个父块支撑了页面内容 - （``kb_chunks.chunk_id``）。回溯原文按后者直取，前者用于文档删除时定位受影响页。 - """ - - __tablename__ = "kb_wiki_pages" - - page_id = Column(String(64), primary_key=True) - kb_id = Column(String(64), ForeignKey("kb_spaces.kb_id", ondelete="CASCADE"), nullable=False) - slug = Column(String(255), nullable=False) - title = Column(String(512), nullable=False, default="") - page_type = Column(String(32), nullable=False, default=WIKI_PAGE_TYPE_SUMMARY) - status = Column(String(32), nullable=False, default="published") - content = Column(Text, nullable=False, default="") - summary = Column(Text, nullable=False, default="") - - folder_id = Column(String(64), nullable=False, default="") - category_path = Column(JSONType, default=list) - wiki_path = Column(String(1024), nullable=False, default="") - depth = Column(Integer, nullable=False, default=0) - sort_order = Column(Integer, nullable=False, default=0) - - source_refs = Column(JSONType, default=list) # [document_id] - chunk_refs = Column(JSONType, default=list) # [chunk_id] - in_links = Column(JSONType, default=list) # [slug]，反向链接 - out_links = Column(JSONType, default=list) # [slug]，正向链接 - aliases = Column(JSONType, default=list) - page_metadata = Column(JSONType, default=dict) - - version = Column(Integer, nullable=False, default=1) - last_edit_source = Column(String(16), nullable=False, default=WIKI_EDIT_SOURCE_PIPELINE) - last_editor_id = Column(String(64), nullable=False, default="") - - created_at = Column(UTCDateTime(timezone=True), default=utc_now) - updated_at = Column(UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now) - deleted_at = Column(UTCDateTime(timezone=True)) - - __table_args__ = ( - CheckConstraint( - "status IN ('draft', 'published', 'archived')", name="kb_wiki_pages_status_check" - ), - # slug 在单个知识库内唯一，跨库可以重复 - Index( - "idx_kb_wiki_pages_kb_slug", - "kb_id", - "slug", - unique=True, - postgresql_where=Column("deleted_at").is_(None), - ), - Index("idx_kb_wiki_pages_kb_type", "kb_id", "page_type"), - Index("idx_kb_wiki_pages_folder", "kb_id", "folder_id"), - Index("idx_kb_wiki_pages_tree", "kb_id", "page_type", "wiki_path", "sort_order", "title"), - Index("idx_kb_wiki_pages_updated_at", "updated_at"), - ) - - -class KBWikiJob(Base): - """Wiki 生成作业。 - - Wiki 生成是分钟级到小时级的长任务，挂在请求生命周期上（BackgroundTasks）会在 - 容器重启时丢一半，所以状态必须落库：worker 认领后写心跳，重启时按心跳超时回收 - 僵尸作业重跑。``progress`` 供前端直接渲染进度，不用另开接口。 - """ - - __tablename__ = "kb_wiki_jobs" - - job_id = Column(String(64), primary_key=True) - kb_id = Column(String(64), ForeignKey("kb_spaces.kb_id", ondelete="CASCADE"), nullable=False) - job_type = Column(String(20), nullable=False) # ingest | retract | finalize - payload = Column(JSONType, default=dict) # {"document_ids": [...]} - status = Column(String(20), nullable=False, default="pending") - attempt = Column(Integer, nullable=False, default=0) - max_attempts = Column(Integer, nullable=False, default=3) - last_error = Column(Text) - progress = Column(JSONType, default=dict) # {"stage": ..., "done": n, "total": m} - - claimed_by = Column(String(64)) - claimed_at = Column(UTCDateTime(timezone=True)) - heartbeat_at = Column(UTCDateTime(timezone=True)) - - created_at = Column(UTCDateTime(timezone=True), default=utc_now) - updated_at = Column(UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now) - finished_at = Column(UTCDateTime(timezone=True)) - - __table_args__ = ( - CheckConstraint( - "job_type IN ('ingest', 'retract', 'finalize')", name="kb_wiki_jobs_type_check" - ), - CheckConstraint( - "status IN ('pending', 'running', 'succeeded', 'failed', 'cancelled')", - name="kb_wiki_jobs_status_check", - ), - Index("idx_kb_wiki_jobs_status", "status", "created_at"), - Index("idx_kb_wiki_jobs_kb", "kb_id", "status"), - # 同一知识库只保留一个未完成的 finalize：多篇文档摄入完只需要收尾一次。 - # 条件里必须带 status，否则一次成功的 finalize 会永久占位、再也排不进下一次。 - # - # 两个方言的 where 都要给：只写 postgresql_where 时，SQLite 会把谓词整个 - # 丢掉、退化成 UNIQUE(kb_id)——那等于「一个知识库只能有一个作业」，本地 - # 开发和测试库上批量摄入会直接撞唯一约束。 - Index( - "idx_kb_wiki_jobs_pending_finalize", - "kb_id", - unique=True, - postgresql_where=text("job_type = 'finalize' AND status IN ('pending', 'running')"), - sqlite_where=text("job_type = 'finalize' AND status IN ('pending', 'running')"), - ), - ) diff --git a/src/backend/core/db/models/knowledge.py b/src/backend/core/db/models/knowledge.py index 4fbef84d..17fddd7f 100644 --- a/src/backend/core/db/models/knowledge.py +++ b/src/backend/core/db/models/knowledge.py @@ -2,198 +2,20 @@ from core.db.utc_datetime import UTCDateTime from core.infra.time import utc_now -from datetime import datetime, timezone from core.db.engine import Base from sqlalchemy import ( - JSON, - BigInteger, Boolean, CheckConstraint, Column, ForeignKey, Index, Integer, - Numeric, String, - Text, UniqueConstraint, ) -from sqlalchemy.dialects.postgresql import INET, JSONB -from sqlalchemy.orm import mapped_column, relationship - -JSONType = JSON().with_variant(JSONB(), "postgresql") -INETType = String(45).with_variant(INET(), "postgresql") - - -class KBSpace(Base): - """Knowledge base space table.""" - - __tablename__ = "kb_spaces" - - kb_id = Column(String(64), primary_key=True) - user_id = Column( - String(64), ForeignKey("users_shadow.user_id", ondelete="CASCADE"), nullable=False - ) - name = Column(String(255), nullable=False) - description = Column(Text) - document_count = Column(Integer, default=0) - total_size_bytes = Column(BigInteger, default=0) - visibility = Column(String(16), nullable=False, default="private") - chunk_method = Column(String(32), nullable=False, default="semantic") - extra_data = Column("metadata", JSONType, default={}) - created_at = Column(UTCDateTime(timezone=True), default=utc_now) - updated_at = Column(UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now) - deleted_at = Column(UTCDateTime(timezone=True)) - - # Relationships - user = relationship("UserShadow", back_populates="kb_spaces") - documents = relationship("KBDocument", back_populates="kb_space", cascade="all, delete-orphan") - - __table_args__ = ( - CheckConstraint("length(name) >= 1 AND length(name) <= 255", name="kb_spaces_name_length"), - CheckConstraint("document_count >= 0", name="kb_spaces_document_count_check"), - CheckConstraint("total_size_bytes >= 0", name="kb_spaces_total_size_check"), - CheckConstraint( - "visibility IN ('public', 'private', 'scoped')", name="kb_spaces_visibility_check" - ), - Index("idx_kb_spaces_user_id", "user_id"), - Index("idx_kb_spaces_updated_at", "updated_at"), - Index( - "idx_kb_spaces_deleted", "deleted_at", postgresql_where=Column("deleted_at").isnot(None) - ), - Index("idx_kb_spaces_visibility", "visibility"), - ) - - -class KBDocument(Base): - """Knowledge base document table.""" - - __tablename__ = "kb_documents" - - document_id = Column(String(64), primary_key=True) - kb_id = Column(String(64), ForeignKey("kb_spaces.kb_id", ondelete="CASCADE"), nullable=False) - title = Column(String(500), nullable=False) - filename = Column(String(500), nullable=False) - size_bytes = Column(BigInteger, nullable=False) - mime_type = Column(String(100), nullable=False) - storage_key = Column(Text, nullable=False) - storage_url = Column(Text) - checksum = Column(String(64)) - indexing_status = Column( - String(20), nullable=False, default="processing" - ) # processing | completed | failed - extra_data = Column("metadata", JSONType, default={}) - uploaded_at = Column(UTCDateTime(timezone=True), default=utc_now) - deleted_at = Column(UTCDateTime(timezone=True)) - - # Relationships - kb_space = relationship("KBSpace", back_populates="documents") - chunks = relationship("KBChunk", back_populates="document", cascade="all, delete-orphan") - - __table_args__ = ( - CheckConstraint("size_bytes > 0", name="kb_documents_size_check"), - CheckConstraint("length(filename) >= 1", name="kb_documents_filename_length"), - Index("idx_kb_documents_kb_id", "kb_id"), - Index("idx_kb_documents_uploaded_at", "uploaded_at"), - Index("idx_kb_documents_kb_uploaded", "kb_id", "uploaded_at"), - Index( - "idx_kb_documents_deleted", - "deleted_at", - postgresql_where=Column("deleted_at").isnot(None), - ), - Index("idx_kb_documents_metadata_gin", "metadata", postgresql_using="gin"), - ) - - -class KBChunk(Base): - """Knowledge base chunk table - stores parent chunks for context retrieval. - - Each document is split into parent chunks (stored here) and child chunks - (vectorised in Milvus hugagent_kb_private collection). Retrieval finds child - chunks via vector search, then fetches the parent content from this table. - """ - - __tablename__ = "kb_chunks" - - chunk_id = Column(String(64), primary_key=True) - kb_id = Column(String(64), ForeignKey("kb_spaces.kb_id", ondelete="CASCADE"), nullable=False) - document_id = Column( - String(64), ForeignKey("kb_documents.document_id", ondelete="CASCADE"), nullable=False - ) - chunk_index = Column(Integer, nullable=False) - content = Column( - Text, nullable=False - ) # parent chunk original text, returned to the LLM on retrieval hit - tags = Column(JSONType, default=list) # tag list ["数字化转型", "申报条件"] - questions = Column(JSONType, default=list) # associated question list (array of strings) - char_start = Column(Integer) - char_end = Column(Integer) - created_at = Column(UTCDateTime(timezone=True), default=utc_now) - updated_at = Column(UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now) - - # Relationships - document = relationship("KBDocument", back_populates="chunks") - - __table_args__ = ( - Index("idx_kb_chunks_kb_id", "kb_id"), - Index("idx_kb_chunks_document_id", "document_id"), - Index("idx_kb_chunks_kb_doc", "kb_id", "document_id"), - ) - - -class KBAsset(Base): - """Non-text media cut out of a knowledge-base document (图片 / 后续音视频). - - One row per figure the layout engine extracted (or per standalone uploaded image). - The bytes live in object storage under ``storage_key``; this table holds the - linkage (which document / which parent chunk), where it sits in the source - (``locator``), and the two text projections retrieval runs on: - - ``text_content`` literal text carried by the medium — 图注 / OCR 文字；音视频接入后是 ASR 转写。 - ``caption`` 语义描述，由 VLM 生成（未配置 vlm 角色时为空，检索退化为只用 text_content）。 - - Deliberately medium-agnostic so audio/video need no migration: ``kind`` widens, - ``locator`` holds ``{page_idx, bbox}`` for images and would hold ``{start_ms, - end_ms}`` for a transcript segment. - - ``vector_state`` records which vector spaces this asset has been indexed into - (``{"text": "ok"}`` today). Path B — a true multimodal embedding in its own - collection — adds a second key here instead of another column. - """ - - __tablename__ = "kb_assets" - - asset_id = Column(String(64), primary_key=True) - kb_id = Column(String(64), ForeignKey("kb_spaces.kb_id", ondelete="CASCADE"), nullable=False) - document_id = Column( - String(64), ForeignKey("kb_documents.document_id", ondelete="CASCADE"), nullable=False - ) - # Owning parent chunk. Nullable: an asset whose placeholder cannot be located in - # any chunk is still worth keeping (it stays retrievable on its own row). - chunk_id = Column(String(64)) - kind = Column(String(16), nullable=False, default="image") # image | audio | video - mime_type = Column(String(100), nullable=False, default="application/octet-stream") - storage_key = Column(Text, nullable=False) - size_bytes = Column(BigInteger, default=0) - asset_index = Column(Integer, nullable=False, default=0) # order within the document - locator = Column(JSONType, default=dict) # image: {page_idx, bbox}; av: {start_ms, end_ms} - caption = Column(Text) # VLM 语义描述 - text_content = Column(Text) # 图注 / OCR / ASR 转写 - caption_status = Column( - String(20), nullable=False, default="pending" - ) # pending | completed | skipped | failed - vector_state = Column(JSONType, default=dict) # {"text": "ok"} —— 路径 B 在此加键 - extra_data = Column("metadata", JSONType, default={}) - created_at = Column(UTCDateTime(timezone=True), default=utc_now) - updated_at = Column(UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now) - - __table_args__ = ( - CheckConstraint("kind IN ('image', 'audio', 'video')", name="kb_assets_kind_check"), - Index("idx_kb_assets_kb_id", "kb_id"), - Index("idx_kb_assets_document_id", "document_id"), - Index("idx_kb_assets_chunk_id", "chunk_id"), - ) +from sqlalchemy.orm import relationship +from core.db.models.column_types import JSONType class CatalogOverride(Base): diff --git a/src/backend/core/db/models/logs.py b/src/backend/core/db/models/logs.py index ff4b90de..f88dcf87 100644 --- a/src/backend/core/db/models/logs.py +++ b/src/backend/core/db/models/logs.py @@ -6,28 +6,17 @@ from core.db.engine import Base from sqlalchemy import ( - JSON, - BigInteger, Boolean, CheckConstraint, Column, ForeignKey, Index, Integer, - Numeric, String, Text, UniqueConstraint, ) -from sqlalchemy.dialects.postgresql import INET, JSONB -from sqlalchemy.orm import mapped_column, relationship - -JSONType = JSON().with_variant(JSONB(), "postgresql") -INETType = String(45).with_variant(INET(), "postgresql") -# BigInteger autoincrement PK: SQLite only auto-increments the INTEGER rowid, not -# BIGINT, so an unqualified BigInteger PK stays NULL on the no-Docker SQLite -# profile. Fall back to Integer on SQLite (still BIGINT on Postgres). -BigIntPK = BigInteger().with_variant(Integer(), "sqlite") +from core.db.models.column_types import JSONType, BigIntPK class ToolCallLog(Base): diff --git a/src/backend/core/db/models/memory.py b/src/backend/core/db/models/memory.py index 35134acf..89c33245 100644 --- a/src/backend/core/db/models/memory.py +++ b/src/backend/core/db/models/memory.py @@ -2,32 +2,19 @@ from core.db.utc_datetime import UTCDateTime from core.infra.time import utc_now -from datetime import datetime, timezone from core.db.engine import Base from sqlalchemy import ( - JSON, - BigInteger, Boolean, CheckConstraint, Column, - ForeignKey, Index, Integer, - Numeric, String, Text, UniqueConstraint, ) -from sqlalchemy.dialects.postgresql import INET, JSONB -from sqlalchemy.orm import mapped_column, relationship - -JSONType = JSON().with_variant(JSONB(), "postgresql") -INETType = String(45).with_variant(INET(), "postgresql") -# BigInteger autoincrement PK: SQLite only auto-increments the INTEGER rowid, not -# BIGINT, so an unqualified BigInteger PK stays NULL on the no-Docker SQLite -# profile. Fall back to Integer on SQLite (still BIGINT on Postgres). -BigIntPK = BigInteger().with_variant(Integer(), "sqlite") +from core.db.models.column_types import JSONType, BigIntPK class ProfileMemory(Base): diff --git a/src/backend/core/db/models/observability.py b/src/backend/core/db/models/observability.py deleted file mode 100644 index 6032705c..00000000 --- a/src/backend/core/db/models/observability.py +++ /dev/null @@ -1,61 +0,0 @@ -"""Durable desktop delivery references and non-executable cloud management copies.""" - -from datetime import datetime, timezone - -from core.db.engine import Base -from sqlalchemy import JSON, BigInteger, Boolean, Column, Index, Integer, String, Text - - -def utcnow(): - return datetime.now(timezone.utc).isoformat() - - -class DesktopOutbox(Base): - __tablename__ = "desktop_observability_outbox" - id = Column(String(64), primary_key=True) - cloud_base = Column(String(512), nullable=False) - subject = Column(String(64), nullable=False) - device_id = Column(String(128), nullable=False) - local_user_id = Column(String(64), nullable=False) - kind = Column(String(24), nullable=False) - object_id = Column(String(128), nullable=False) - revision = Column(BigInteger, nullable=False, default=1) - pending = Column(Boolean, nullable=False, default=True) - deleted = Column(Boolean, nullable=False, default=False) - digest = Column(String(64), nullable=False, default="") - updated_at = Column(String(40), nullable=False, default=utcnow) - __table_args__ = ( - Index("idx_desktop_outbox_pending", "cloud_base", "subject", "device_id", "pending"), - ) - - -class DesktopRecord(Base): - __tablename__ = "desktop_observability_records" - id = Column(String(64), primary_key=True) - user_id = Column(String(64), nullable=False) - device_id = Column(String(128), nullable=False) - kind = Column(String(24), nullable=False) - object_id = Column(String(128), nullable=False) - revision = Column(BigInteger, nullable=False) - chat_id = Column(String(128), nullable=False, default="") - run_id = Column(String(128), nullable=False, default="") - observation = Column(String(16), nullable=False, default="desktop") - payload = Column(JSON, nullable=False) - received_at = Column(String(40), nullable=False, default=utcnow) - __table_args__ = ( - Index("idx_desktop_records_identity", "kind", "observation", "user_id", "device_id", "object_id", unique=True), - Index("idx_desktop_records_user_kind", "user_id", "kind", "received_at"), - Index("idx_desktop_records_chat", "user_id", "device_id", "chat_id"), - ) - - -class DesktopSyncDevice(Base): - __tablename__ = "desktop_observability_devices" - id = Column(String(64), primary_key=True) - user_id = Column(String(64), nullable=False) - device_id = Column(String(128), nullable=False) - last_seen = Column(String(40), nullable=False, default=utcnow) - pending = Column(Integer, nullable=False, default=0) - capture_errors = Column(Integer, nullable=False, default=0) - reconciling = Column(Boolean, nullable=False, default=False) - __table_args__ = (Index("idx_desktop_devices_user", "user_id"),) diff --git a/src/backend/core/db/models/ontology.py b/src/backend/core/db/models/ontology.py index 435d890b..8f85169b 100644 --- a/src/backend/core/db/models/ontology.py +++ b/src/backend/core/db/models/ontology.py @@ -2,11 +2,9 @@ from core.db.utc_datetime import UTCDateTime from core.infra.time import utc_now -from datetime import datetime from core.db.engine import Base from sqlalchemy import ( - JSON, Boolean, CheckConstraint, Column, @@ -18,9 +16,7 @@ UniqueConstraint, text, ) -from sqlalchemy.dialects.postgresql import JSONB - -JSONType = JSON().with_variant(JSONB(), "postgresql") +from core.db.models.column_types import JSONType class OntologyPack(Base): diff --git a/src/backend/core/db/models/project.py b/src/backend/core/db/models/project.py index 43bbd444..d001185f 100644 --- a/src/backend/core/db/models/project.py +++ b/src/backend/core/db/models/project.py @@ -2,29 +2,19 @@ from core.db.utc_datetime import UTCDateTime from core.infra.time import utc_now -from datetime import datetime, timezone from core.db.engine import Base -from core.db.model_extensions import ProjectEditionFields, project_edition_table_args +from core.db.edition import ProjectEditionFields, project_edition_table_args from sqlalchemy import ( - JSON, - BigInteger, Boolean, CheckConstraint, Column, ForeignKey, Index, - Integer, - Numeric, String, Text, - UniqueConstraint, ) -from sqlalchemy.dialects.postgresql import INET, JSONB -from sqlalchemy.orm import mapped_column, relationship - -JSONType = JSON().with_variant(JSONB(), "postgresql") -INETType = String(45).with_variant(INET(), "postgresql") +from core.db.models.column_types import JSONType class Project(ProjectEditionFields, Base): diff --git a/src/backend/core/db/personal_file_names.py b/src/backend/core/db/personal_file_names.py index 38cc4d3a..3e637248 100644 --- a/src/backend/core/db/personal_file_names.py +++ b/src/backend/core/db/personal_file_names.py @@ -8,7 +8,9 @@ def personal_file_indexes(*, enterprise=False): - scope = "" + from core.db.edition import personal_index_scope + + scope = personal_index_scope(enterprise) return tuple( Index( name, *columns, unique=True, @@ -37,7 +39,7 @@ def validate_filename(filename): def check_name(db, user_id, folder_id, filename, exclude_id=None, exclude_folder=None): from core.db.models import Artifact, UserFolder - from core.services.artifact_edition import personal_artifact_predicates + from core.db.artifact_scope import personal_artifact_predicates validate_filename(filename) with db.no_autoflush: @@ -63,8 +65,8 @@ def check_name(db, user_id, folder_id, filename, exclude_id=None, exclude_folder @event.listens_for(Session, "before_flush") def validate_personal_filenames(db, flush_context, instances): from core.db.models import Artifact, UserFolder, UserShadow - from core.services.artifact_edition import is_personal_artifact - from core.services.folder_batch import lock_folder_owner + from core.db.artifact_scope import is_personal_artifact + from core.db.repository.folder import lock_folder_owner candidates = [] for row in db.new.union(db.dirty): @@ -72,6 +74,9 @@ def validate_personal_filenames(db, flush_context, instances): fields = ("user_id", "parent_folder_id", "name", "deleted_at") elif isinstance(row, Artifact) and is_personal_artifact(row): fields = ("user_id", "user_folder_id", "filename", "deleted_at") + from core.db.edition import personal_artifact_scope_fields + + fields += tuple(name for name in personal_artifact_scope_fields() if name in inspect(row).attrs) else: continue if row.deleted_at is None and (row in db.new or any( diff --git a/src/backend/core/db/model_extensions.py b/src/backend/core/db/personal_model_fields.py similarity index 100% rename from src/backend/core/db/model_extensions.py rename to src/backend/core/db/personal_model_fields.py diff --git a/src/backend/core/db/edition_tables.py b/src/backend/core/db/personal_schema.py similarity index 97% rename from src/backend/core/db/edition_tables.py rename to src/backend/core/db/personal_schema.py index a7bd1abf..53848c11 100644 --- a/src/backend/core/db/edition_tables.py +++ b/src/backend/core/db/personal_schema.py @@ -19,7 +19,7 @@ } -def _ce_metadata(): +def ce_metadata(): """Build metadata with foreign keys to physically omitted tables removed.""" import core.db.models # noqa: F401 from core.db.engine import Base @@ -44,7 +44,7 @@ def _ce_metadata(): def ce_create_all(bind) -> list[str]: """Create the CE metadata and remove foreign keys to omitted edition tables.""" - clone = _ce_metadata() + clone = ce_metadata() clone.create_all(bind=bind) return sorted(clone.tables) @@ -55,6 +55,6 @@ def ce_reconcile_schema(bind) -> dict[str, list[str]]: return reconcile_metadata_schema( bind, - _ce_metadata(), + ce_metadata(), bootstrap_server_defaults=_CE_BOOTSTRAP_SERVER_DEFAULTS, ) diff --git a/src/backend/core/db/repository/__init__.py b/src/backend/core/db/repository/__init__.py index 32654911..4cda10c1 100644 --- a/src/backend/core/db/repository/__init__.py +++ b/src/backend/core/db/repository/__init__.py @@ -1,22 +1,33 @@ -"""Community-edition repository exports.""" +"""Data access layer — Repository pattern. -from core.db.repository.agent import UserAgentRepository +Repositories are organised into domain submodules; this package re-exports +every public class so existing imports (``from core.db.repository import +XxxRepository``) keep working. +""" + +from core.db.edition import UserAgentRepository from core.db.repository.artifact import ROOT_FOLDER_SENTINEL, ArtifactRepository -from core.db.repository.audit import AuditLogRepository +from core.db.edition import AuditLogRepository from core.db.repository.catalog import CatalogRepository -from core.db.repository.channel import ChannelConnectionRepository + from core.db.repository.chat import ChatMessageRepository, ChatSessionRepository -from core.db.repository.kb import KBRepository + from core.db.repository.ontology import OntologyRepository -from core.db.repository.site import SiteRepository -from core.db.repository.user import ( - DingTalkConnectionRepository, - EmailConnectionRepository, - LarkConnectionRepository, - LocalUserRepository, - UserRepository, -) +from core.db.repository.user import LocalUserRepository, UserRepository + + + + __all__ = [ - name for name in globals() if name.endswith("Repository") or name == "ROOT_FOLDER_SENTINEL" + "UserRepository", + "LocalUserRepository", + "ChatSessionRepository", + "ChatMessageRepository", + "CatalogRepository", + "ArtifactRepository", + "ROOT_FOLDER_SENTINEL", + "AuditLogRepository", + "UserAgentRepository", + "OntologyRepository", ] diff --git a/src/backend/core/db/repository/artifact.py b/src/backend/core/db/repository/artifact.py index c462779a..ad0d45b8 100644 --- a/src/backend/core/db/repository/artifact.py +++ b/src/backend/core/db/repository/artifact.py @@ -6,7 +6,6 @@ """ from core.infra.time import utc_now -import logging from datetime import datetime from typing import Any, Dict, List, Optional @@ -120,10 +119,9 @@ def list_by_user_with_chat( ) if personal_only: - # Import lazily: ``core.services`` re-exports repositories, so importing - # this edition seam while the repository package is initializing would - # create a package-level cycle. - from core.services.artifact_edition import personal_artifact_predicates + # Import lazily: this edition seam must not load while the repository + # package is still initializing. + from core.db.artifact_scope import personal_artifact_predicates query = query.filter(*personal_artifact_predicates(Artifact)) if folder_id == ROOT_FOLDER_SENTINEL: diff --git a/src/backend/core/db/repository/catalog.py b/src/backend/core/db/repository/catalog.py index d17f4baf..36d222e4 100644 --- a/src/backend/core/db/repository/catalog.py +++ b/src/backend/core/db/repository/catalog.py @@ -6,13 +6,10 @@ """ from core.infra.time import utc_now -import logging from datetime import datetime -from typing import Any, Dict, List, Optional +from typing import Dict, List, Optional -import sqlalchemy as sa from core.db.models import CatalogOverride -from sqlalchemy import and_, desc, func, or_, select from sqlalchemy.orm import Session diff --git a/src/backend/core/db/repository/chat.py b/src/backend/core/db/repository/chat.py index 088af4e2..67ea5b28 100644 --- a/src/backend/core/db/repository/chat.py +++ b/src/backend/core/db/repository/chat.py @@ -6,7 +6,6 @@ """ from core.infra.time import utc_now -import logging from datetime import datetime from typing import Any, Dict, List, Optional @@ -14,7 +13,7 @@ from core.db.reserved_chats import ordinary_chat_filter, reserved_chat_error, reserved_chat_owner from core.db.models import ChatMessage, ChatSession from core.db.models.chat import reserve_chat_message_sequences -from sqlalchemy import and_, desc, func, or_, select +from sqlalchemy import desc, func, or_ from sqlalchemy.orm import Session @@ -240,18 +239,6 @@ def count_visible_through_seq(self, chat_id: str, covered_seq: int) -> int: stable sequence boundary in the visible history without exposing the checkpoint row or its replacement payload. """ - return int( - self.db.query(ChatMessage) - .filter( - ChatMessage.chat_id == chat_id, - ChatMessage.role != "system", - ChatMessage.chat_seq <= covered_seq, - ) - .count() - ) - - def count_visible_through_seq(self, chat_id: str, covered_seq: int) -> int: - """Count user-facing rows at or below an exact compaction watermark.""" return int( self.db.query(ChatMessage) .filter( diff --git a/src/backend/core/db/repository/folder.py b/src/backend/core/db/repository/folder.py new file mode 100644 index 00000000..34547f29 --- /dev/null +++ b/src/backend/core/db/repository/folder.py @@ -0,0 +1,40 @@ +"""Transaction-scoped owner locks shared by folder and artifact writers.""" +import hashlib +from threading import Lock +from sqlalchemy import event +from sqlalchemy.orm import Session + +_SQLITE_LOCKS = [Lock() for _ in range(64)] + + +def _release_sqlite_locks(session, transaction): + if transaction.parent is None: + for lock in session.info.pop("folder_upload_locks", []): + lock.release() + + +def lock_folder_owner(db: Session, owner_model, owner_column: str, owner_id: str) -> None: + if db.get_bind().dialect.name == "sqlite": + # SQLite ignores FOR UPDATE. Its supported single-process deployment still + # serves requests on multiple threads, so retain a bounded process lock + # until commit/rollback/close, including early-return and error paths. + if not db.in_transaction(): + db.begin() + if not db.info.get("folder_upload_lock_listener"): + event.listen(db, "after_transaction_end", _release_sqlite_locks) + db.info["folder_upload_lock_listener"] = True + digest = hashlib.sha256((owner_model.__tablename__ + ":" + owner_id).encode()).digest() + lock = _SQLITE_LOCKS[int.from_bytes(digest[:2], "big") % len(_SQLITE_LOCKS)] + held = db.info.setdefault("folder_upload_locks", []) + if lock not in held: + lock.acquire() + held.append(lock) + # NO KEY UPDATE does not conflict with FK KEY SHARE taken by artifact inserts. + owner = ( + db.query(owner_model) + .filter(getattr(owner_model, owner_column) == owner_id) + .with_for_update(key_share=True) + .first() + ) + if owner is None: + raise ValueError("文件夹所属空间不存在") diff --git a/src/backend/core/db/repository/agent.py b/src/backend/core/db/repository/personal_agent.py similarity index 100% rename from src/backend/core/db/repository/agent.py rename to src/backend/core/db/repository/personal_agent.py diff --git a/src/backend/core/db/repository/audit.py b/src/backend/core/db/repository/personal_audit.py similarity index 96% rename from src/backend/core/db/repository/audit.py rename to src/backend/core/db/repository/personal_audit.py index 98018da9..a82365d6 100644 --- a/src/backend/core/db/repository/audit.py +++ b/src/backend/core/db/repository/personal_audit.py @@ -1,6 +1,6 @@ """No-storage audit repository for the community edition.""" -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List class AuditLogRepository: diff --git a/src/backend/core/db/repository/user.py b/src/backend/core/db/repository/user.py index 0cedef37..ae2c16ce 100644 --- a/src/backend/core/db/repository/user.py +++ b/src/backend/core/db/repository/user.py @@ -6,19 +6,10 @@ """ from core.infra.time import utc_now -import logging from datetime import datetime -from typing import Any, Dict, List, Optional - -import sqlalchemy as sa -from core.db.models import ( - DingTalkConnection, - EmailConnection, - LarkConnection, - LocalUser, - UserShadow, -) -from sqlalchemy import and_, desc, func, or_, select +from typing import Any, Dict, Optional + +from core.db.models import LocalUser, UserShadow from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import Session @@ -150,103 +141,3 @@ def set_status(self, user_id: str, status: str) -> bool: record.updated_at = utc_now() self.db.commit() return True - - -class DingTalkConnectionRepository: - """Per-user DingTalk connection (dingtalk_connections) CRUD. 1:1 with users_shadow, no soft delete.""" - - def __init__(self, db: Session): - self.db = db - - def get(self, user_id: str) -> Optional[DingTalkConnection]: - return ( - self.db.query(DingTalkConnection).filter(DingTalkConnection.user_id == user_id).first() - ) - - def ensure(self, user_id: str) -> DingTalkConnection: - """Idempotent: return existing record, otherwise create a disconnected record.""" - record = self.get(user_id) - if record: - return record - record = DingTalkConnection(user_id=user_id, status="disconnected") - self.db.add(record) - self.db.commit() - self.db.refresh(record) - return record - - def update(self, user_id: str, data: Dict[str, Any]) -> Optional[DingTalkConnection]: - record = self.get(user_id) - if not record: - return None - for k, v in data.items(): - setattr(record, k, v) - record.updated_at = utc_now() - self.db.commit() - self.db.refresh(record) - return record - - -class LarkConnectionRepository: - """Per-user Lark connection (lark_connections) CRUD. 1:1 with users_shadow, no soft delete. - Structurally identical to [[DingTalkConnectionRepository]].""" - - def __init__(self, db: Session): - self.db = db - - def get(self, user_id: str) -> Optional[LarkConnection]: - return self.db.query(LarkConnection).filter(LarkConnection.user_id == user_id).first() - - def ensure(self, user_id: str) -> LarkConnection: - """Idempotent: return existing record, otherwise create a disconnected record.""" - record = self.get(user_id) - if record: - return record - record = LarkConnection(user_id=user_id, status="disconnected") - self.db.add(record) - self.db.commit() - self.db.refresh(record) - return record - - def update(self, user_id: str, data: Dict[str, Any]) -> Optional[LarkConnection]: - record = self.get(user_id) - if not record: - return None - for k, v in data.items(): - setattr(record, k, v) - record.updated_at = utc_now() - self.db.commit() - self.db.refresh(record) - return record - - -class EmailConnectionRepository: - """Per-user email connection (email_connections) CRUD. 1:1 with users_shadow, no soft delete. - Structurally identical to [[LarkConnectionRepository]].""" - - def __init__(self, db: Session): - self.db = db - - def get(self, user_id: str) -> Optional[EmailConnection]: - return self.db.query(EmailConnection).filter(EmailConnection.user_id == user_id).first() - - def ensure(self, user_id: str) -> EmailConnection: - """Idempotent: return existing record, otherwise create a disconnected record.""" - record = self.get(user_id) - if record: - return record - record = EmailConnection(user_id=user_id, status="disconnected") - self.db.add(record) - self.db.commit() - self.db.refresh(record) - return record - - def update(self, user_id: str, data: Dict[str, Any]) -> Optional[EmailConnection]: - record = self.get(user_id) - if not record: - return None - for k, v in data.items(): - setattr(record, k, v) - record.updated_at = utc_now() - self.db.commit() - self.db.refresh(record) - return record diff --git a/src/backend/core/db/site_storage_retirement.py b/src/backend/core/db/site_storage_retirement.py index 2769e9eb..51d463b3 100644 --- a/src/backend/core/db/site_storage_retirement.py +++ b/src/backend/core/db/site_storage_retirement.py @@ -20,7 +20,7 @@ def retire_site_storage(engine, directory: Path): return retire_connection(connection, directory, strict=False) -def _digest(payload): +def digest(payload): return hashlib.sha256( json.dumps(payload, sort_keys=True, ensure_ascii=False).encode() ).hexdigest() @@ -50,7 +50,7 @@ def retire_connection(connection, directory: Path, *, strict=True): required = {row["site_id"] for rows in payload.values() for row in rows} if ( receipt.get("verified") is True - and receipt.get("legacy_sha256") == _digest(payload) + and receipt.get("legacy_sha256") == digest(payload) and required <= covered ): verified = True @@ -66,7 +66,7 @@ def retire_connection(connection, directory: Path, *, strict=True): archive = directory / f"site-storage-{uuid4().hex}.json" manifest = { "revision": revision, - "sha256": _digest(payload), + "sha256": digest(payload), "tables": payload, "created_at": datetime.now(timezone.utc).isoformat(), } @@ -76,7 +76,7 @@ def retire_connection(connection, directory: Path, *, strict=True): handle.flush() os.fsync(handle.fileno()) verified = json.loads(archive.read_text(encoding="utf-8")) - if verified["sha256"] != _digest(verified["tables"]) or verified["tables"] != payload: + if verified["sha256"] != digest(verified["tables"]) or verified["tables"] != payload: raise RuntimeError("Retired storage backup verification failed") for name in existing: sa.Table(name, sa.MetaData(), autoload_with=connection).drop(connection) diff --git a/src/backend/core/db/system_configuration.py b/src/backend/core/db/system_configuration.py new file mode 100644 index 00000000..1e988a97 --- /dev/null +++ b/src/backend/core/db/system_configuration.py @@ -0,0 +1,146 @@ +"""Database persistence and seed migration for system configuration.""" +from __future__ import annotations +import logging +from typing import Optional +from core.db.engine import SessionLocal +from core.db.models import SystemConfig +from core.config.store import system_config_constants + +logger = logging.getLogger(__name__) + + +class SystemConfigurationStore: + def set(self, key: str, value: str | None, updated_by: str = "admin") -> None: + """Update a config value in DB.""" + try: + db = SessionLocal() + try: + row = db.query(SystemConfig).filter(SystemConfig.config_key == key).first() + if row is None: + logger.warning("[SystemConfigService] key %s not found, skipping set", key) + return + row.config_value = value + row.updated_by = updated_by + db.commit() + finally: + db.close() + except Exception as exc: + logger.error("[SystemConfigService] set(%s) failed: %s", key, exc) + raise + def bulk_set(self, items: list[dict], updated_by: str = "admin") -> None: + """Batch update multiple config values. Each item: {key, value}. + + For secret fields, masked values (containing '****') are skipped to + prevent the frontend from accidentally overwriting real secrets with + their masked representation. + """ + try: + db = SessionLocal() + try: + for item in items: + key = item.get("key", "").strip() + value = item.get("value") + if not key: + continue + row = db.query(SystemConfig).filter(SystemConfig.config_key == key).first() + if row is None: + continue + # Skip masked values for secret fields + if row.is_secret and isinstance(value, str) and "****" in value: + continue + row.config_value = value if value != "" else None + row.updated_by = updated_by + db.commit() + finally: + db.close() + except Exception as exc: + logger.error("[SystemConfigService] bulk_set failed: %s", exc) + raise + def _ensure_seed_rows(self, db) -> None: + """Insert any missing seed config rows. Idempotent.""" + rows_by_key = {row.config_key: row for row in db.query(SystemConfig).all()} + existing_keys = set(rows_by_key) + inserted_keys: set[str] = set() + inserted = 0 + for key, val, display, desc, group, secret in system_config_constants.SEED_CONFIGS: + if key not in existing_keys: + row = SystemConfig( + config_key=key, + config_value=val, + display_name=display, + description=desc, + group_key=group, + is_secret=secret, + ) + db.add(row) + rows_by_key[key] = row + inserted_keys.add(key) + inserted += 1 + migrated = self._migrate_legacy_knowledge_config(rows_by_key, inserted_keys) + if inserted or migrated: + db.commit() + logger.info( + "[SystemConfigService] Inserted %d missing seed row(s), migrated %d legacy KB value(s)", + inserted, + migrated, + ) + @staticmethod + def _migrate_legacy_knowledge_config( + rows_by_key: dict[str, SystemConfig], + inserted_keys: set[str], + ) -> int: + """Move the old shared KB connection into one provider exactly once. + + Older releases stored every external backend in the shared + knowledge_base.url/api_key/allowed_dataset_ids rows. Those rows cannot + prove which dropdown option they belonged to, and copying them into the + currently selected provider can reproduce the original cross-provider + leak. Treat them as Dify compatibility values and leave FastGPT/WeKnora + empty until each provider is configured explicitly. + + inserted_keys is the one-time guard: after scoped rows exist, later + legacy edits can never overwrite provider-specific values. + """ + migrated = 0 + for field in system_config_constants.PROVIDER_SCOPED_KNOWLEDGE_FIELDS: + destination_key = f"knowledge_base.dify.{field}" + if destination_key not in inserted_keys: + continue + destination = rows_by_key.get(destination_key) + legacy = rows_by_key.get(f"knowledge_base.{field}") + legacy_value = getattr(legacy, "config_value", None) + if ( + destination is not None + and not getattr(destination, "config_value", None) + and legacy_value not in (None, "") + ): + destination.config_value = legacy_value + migrated += 1 + return migrated + def load(self): + new_cache: dict[str, Optional[str]] = {} + new_meta: dict[str, dict] = {} + try: + db = SessionLocal() + try: + self._ensure_seed_rows(db) + rows = db.query(SystemConfig).all() + for row in rows: + new_cache[row.config_key] = row.config_value + new_meta[row.config_key] = { + "config_key": row.config_key, + "config_value": row.config_value, + "display_name": row.display_name, + "description": row.description, + "group_key": row.group_key, + "is_secret": row.is_secret, + "updated_at": row.updated_at.isoformat() if row.updated_at else None, + "updated_by": row.updated_by, + } + finally: + db.close() + except Exception as exc: + logger.warning("[SystemConfigService] DB load failed, keeping stale cache: %s", exc) + raise + + return new_cache, new_meta diff --git a/src/backend/core/db/user_cleanup.py b/src/backend/core/db/user_cleanup.py new file mode 100644 index 00000000..867cf070 --- /dev/null +++ b/src/backend/core/db/user_cleanup.py @@ -0,0 +1,13 @@ +"""Run module-owned cleanup within the deleting session's transaction.""" +from sqlalchemy import event +from sqlalchemy.orm import Session + +from core.harness.registry.user_cleanup import cleanup_user + + +def install_user_cleanup(user_model): + @event.listens_for(Session, "before_flush") + def clean_deleted_users(session, flush_context, instances): + for instance in tuple(session.deleted): + if isinstance(instance, user_model): + cleanup_user(session, instance) diff --git a/src/backend/core/evolution/__init__.py b/src/backend/core/evolution/__init__.py index 383ada1c..dbdbc548 100644 --- a/src/backend/core/evolution/__init__.py +++ b/src/backend/core/evolution/__init__.py @@ -18,11 +18,7 @@ AssetKind, AssetRef, ) -from core.evolution.runtime_binding import ( - bind_runtime_assets, - get_bundle, - resolve_bundle_for_run, -) +from core.evolution.cycle.runtime_binding import bind_runtime_assets, get_bundle, resolve_bundle_for_run __all__ = [ "ASSET_KINDS", diff --git a/src/backend/core/evolution/activation.py b/src/backend/core/evolution/activation.py deleted file mode 100644 index 09b70bb6..00000000 --- a/src/backend/core/evolution/activation.py +++ /dev/null @@ -1,1502 +0,0 @@ -"""Materialise an approved candidate into a runtime capability. - -This is the step the loop was missing. Everything upstream — discovery, -attribution, promotion, the IR, replay — produced a *candidate*, and a candidate -is a proposal. Nothing turned it into an asset the agent could actually load, -so measured end to end the system learned nothing: the benchmark's "after" arm -saw exactly zero new skills. - -Materialisation closes that gap, and it is deliberately the **only** way an -evolution artefact reaches the runtime. It sits behind the release controller -so every governance property upstream still holds: - -* the candidate must have cleared the verification its risk tier demands; -* the approver must not be the proposer; -* the previous state is captured **before** the write, so rollback has somewhere - to go rather than having to reconstruct one afterwards. - -Materialised skills are tagged so they can always be told apart from -hand-authored ones. A capability the system wrote about itself should never be -indistinguishable from one a person wrote. -""" - -from __future__ import annotations - -import logging -import uuid -from datetime import datetime, timezone -from typing import Any, Dict, List, Optional, Sequence, Tuple - -from core.db.engine import SessionLocal - -logger = logging.getLogger(__name__) - -# Marks a skill that the evolution loop wrote. Never remove this on update: it -# is what lets an operator see at a glance which capabilities were authored by -# the system rather than by a person. -EVOLUTION_TAG = "evolved" -EVOLUTION_PREFIX = "evo-" - -# Operator stop switch, in ``system_configs`` so an incident does not need a -# redeploy to halt every layer. -FREEZE_CONFIG_KEY = "evolution_frozen" - - -class ActivationError(Exception): - def __init__(self, code: str, message: str): - super().__init__(message) - self.code = code - self.message = message - - -def _skill_label(tools: Sequence[str], *, rules_only: bool = False) -> str: - """A human- and model-readable skill name. - - The name and description are the *only* things the model sees before it - decides whether to open a skill, so they have to describe the job. The - candidate's ``hypothesis`` describes the evidence ("「…」类请求在 N 次独立 - 会话中以同一序列成功完成") — true, and useless for that decision, since it - says nothing about when the skill applies. It belongs in the provenance - section, not the title. - """ - head = " → ".join(tools[:4]) + ("…" if len(tools) > 4 else "") - return ("工具顺序规则：" if rules_only else "固定调用顺序：") + head - - -def _skill_description( - tools: Sequence[str], constraints: Sequence[Dict[str, Any]], *, rules_only: bool -) -> str: - """One line, single-line by necessity — the frontmatter parser is line-based.""" - joined = "、".join(f"`{t}`" for t in tools[:6]) - if rules_only: - return ( - f"当任务用到 {joined} 时，按已验证的先后顺序约束安排调用次序" - f"（{len(constraints)} 条约束，来自历史成功执行）。" - ) - return ( - f"当任务需要 {joined} 且执行顺序会影响结果时使用；" - f"按 {' → '.join(tools)} 的顺序调用。" - ) - - -def _rules_consistent_with( - constraints: Sequence[Dict[str, Any]], tools: Sequence[str] -) -> List[Dict[str, Any]]: - """Drop rules whose direction the skill's own step list violates. - - The loop attaches every rule touching a skill's tools, and scoping makes that - formally sound — a rule valid in ``finance`` and a sequence distilled from - ``trap`` genuinely can point opposite ways. But the rendered document then - tells the model to call A before B in its steps and B before A two sections - later, with nothing saying which wins. Observed on a real materialisation. - - For this skill's own tool set the step list is the governing statement, so a - rule contradicting it is describing some other context and does not belong in - this document. It remains attached to the skills whose sequences agree with - it, and in ``ordering_constraints.json`` for any consumer that reasons about - scope itself. - """ - if not tools: - return list(constraints) - position = {tool: index for index, tool in enumerate(tools)} - kept: List[Dict[str, Any]] = [] - for rule in constraints: - before, after = rule.get("before"), rule.get("after") - if before in position and after in position and position[before] > position[after]: - logger.info( - "[activation] rule %s<%s omitted: contradicts this skill's own order", - before, - after, - ) - continue - kept.append(rule) - return kept - - -def _ordering_rules_section(constraints: Sequence[Dict[str, Any]]) -> str: - """Render the learned ordering constraints into the body the model reads. - - Without this the transferable half of what was learned never reaches the - runtime. ``ordering_constraints.json`` is written alongside, but nothing in - the agent path reads extra files as policy — the model reads SKILL.md, so a - rule that is not written here is a rule the model never receives. - - Each rule carries its own scope. Stating where it was contradicted matters - as much as stating where it held: a rule applied in the context that - disproved it actively breaks that context. - """ - shippable = [r for r in constraints if not (r.get("contradicted_in") or [])] - withheld = [r for r in constraints if (r.get("contradicted_in") or [])] - - lines: List[str] = [] - if shippable: - lines += ["", "## 顺序约束（可迁移）", ""] - lines.append("下列约束在历史成功执行中无反例，适用于本技能覆盖的工具：") - lines.append("") - for rule in shippable: - before, after = rule.get("before"), rule.get("after") - validated = [str(x) for x in (rule.get("validated_in") or [])] - line = f"- 先 `{before}` 再 `{after}`" - if validated: - line += f"——已在 {'、'.join(validated)} 类任务中验证" - line += f"（证据 {rule.get('support', 0)} 条）" - lines.append(line) - - if withheld: - # Scoped rules are deliberately kept out of the model's view. - # - # Measured twice against a real model, and both framings failed: - # - # * exception as a trailing qualifier ("在 X 类任务中不适用") — the model - # applied the rule in X anyway, 5 of 5 times; - # * exception first and imperative ("若属于 X 则整条作废，那里顺序相反") - # — the model then applied the *reversed* order everywhere, dropping the - # family the rule was valid in from 50% to 17% and disturbing an - # unrelated control family. - # - # So a text annotation cannot make a model route on task type: it either - # ignores the exception or over-applies it. Handing it a rule that is - # wrong somewhere makes the system worse where the rule is wrong, which - # is a poor trade for a benefit available only where it is right. - # - # The rule is not discarded — it stays in ``ordering_constraints.json`` - # for a runtime applier that knows the task type and can decide before - # injection, which is where a scope decision belongs. Until that exists, - # withholding is the conservative and measurably better option. - lines += ["", "## 已扣留的上下文相关约束", ""] - lines.append( - f"另有 {len(withheld)} 条约束在部分任务类型中被证据推翻，" - "因此**不在本技能中给出**——它们只在特定任务类型下成立，" - "由运行时按任务类型判定后注入，不交由模型自行取舍。" - ) - for rule in withheld: - lines.append( - f"- （已扣留）`{rule.get('before')}` / `{rule.get('after')}`：" - f"在 {'、'.join(str(x) for x in (rule.get('validated_in') or []))} 成立，" - f"在 {'、'.join(str(x) for x in (rule.get('contradicted_in') or []))} 被推翻" - ) - return "\n".join(lines) - - -def _skill_markdown( - *, - skill_id: str, - label: str, - description: str, - tools: Sequence[str], - evidence: int, - constraints: Sequence[Dict[str, Any]] = (), - rationale: str = "", - rules_only: bool = False, -) -> str: - """Render the SKILL.md body for a distilled skill. - - ``name`` is the skill **id** — the loader treats that field as the id and - rejects anything outside ``[a-z0-9_-]``. Writing the human label there made - every materialised skill unparseable: it still showed up in the capability - list (that metadata comes from table columns) while ``load_skill_full`` - returned ``None``, so the skill was present and inert. - - States its provenance in the document itself, so anyone reading the skill — - not just anyone reading the console — knows where it came from and on how - much evidence it rests. - """ - # Applicability is declared for the **whole skill**, not per rule. Measured: - # withholding a contradicted rule from the text was not enough, because the - # step list generalises on its own — the model applied this skill's order to - # a family where the reverse is correct, 5 of 5 times, with the rule absent. - # So the exclusion has to bind the steps too, which means stating it once, at - # the top, before anything the model could copy. - excluded = sorted( - { - str(name) - for rule in constraints - for name in (rule.get("contradicted_in") or []) - if str(name) - } - ) - covered = sorted( - { - str(name) - for rule in constraints - for name in (rule.get("validated_in") or []) - if str(name) and str(name) not in excluded - } - ) - scope_block = "" - if excluded: - scope_block = ( - "## 不适用范围（先读这一段）\n\n" - f"**本技能不适用于以下任务类型：{'、'.join(excluded)}。**" - "在这些场景中，本技能给出的顺序已被历史证据推翻——" - "遇到它们请完全忽略本技能，按任务本身的依赖关系自行规划。\n\n" - + ( - f"本技能仅适用于：{'、'.join(covered)}。\n\n" - if covered - else "" - ) - ) - - if rules_only: - body_head = ( - scope_block - + "## 适用条件\n\n" - "当任务用到下列工具、且它们的先后顺序会影响结果时，按「顺序约束」小节排序。\n\n" - "## 涉及工具\n\n" - + "\n".join(f"- `{tool}`" for tool in tools) - ) - else: - body_head = ( - scope_block - + "## 适用条件\n\n" - "当任务需要按固定顺序完成下列工具调用时使用本技能。\n\n" - "## 步骤\n\n" - + "\n".join(f"{i}. `{tool}`" for i, tool in enumerate(tools, start=1)) - ) - - # A rules-only skill has no governing step list, so nothing there can be - # contradicted; for a sequence skill the steps win and conflicting rules are - # about other contexts. - shown_rules = constraints if rules_only else _rules_consistent_with(constraints, tools) - - provenance = ( - f"本技能由能力进化流水线从 {evidence} 条历史执行证据中沉淀而成，非人工撰写。" - ) - if rationale: - provenance += f"\n\n沉淀依据：{rationale}" - - return f"""--- -name: {skill_id} -display_name: {label} -description: {description} ---- - -# {label} - -{description} - -{body_head} -{_ordering_rules_section(shown_rules)} - -## 来源 - -{provenance} -""" - - -def _body_of(rendered: str) -> str: - """The body of a rendered SKILL.md, without frontmatter or provenance. - - Re-rendering from the body rather than storing the generator's output - verbatim is what lets the id change between drafting and activation (an - evolution prefix, an owner suffix) without the document's ``name`` field - disagreeing with the row it is stored under — a mismatch that makes the - skill unloadable while still listing it as available. - """ - text = rendered or "" - if text.startswith("---"): - end = text.find("\n---", 3) - if end != -1: - text = text[end + 4 :] - # Drop the rendered title/description pair and the provenance section; both - # are regenerated, and keeping them would double on every re-render. - marker = text.find("## ") - if marker != -1: - text = text[marker:] - source = text.find("\n## 来源") - if source != -1: - text = text[:source] - return text.strip() - - -def _capture_previous(db, skill_id: str) -> Optional[Dict[str, Any]]: - """Snapshot the current state so rollback has a concrete target.""" - from core.db.models import AdminSkill - - row = db.query(AdminSkill).filter(AdminSkill.skill_id == skill_id).first() - if row is None: - return None - return { - "skill_id": row.skill_id, - "skill_content": row.skill_content, - "display_name": row.display_name, - "description": row.description, - "version": row.version, - "tags": list(row.tags or []), - "allowed_tools": list(row.allowed_tools or []), - "extra_files": dict(row.extra_files or {}), - "is_enabled": bool(row.is_enabled), - } - - -def _is_frozen(db) -> bool: - """Global stop switch, readable without a redeploy. - - An incident needs a way to stop every layer activating *now*, so it lives in - ``system_configs`` rather than in an environment variable. - """ - try: - from core.db.models import SystemConfig - - row = ( - db.query(SystemConfig.config_value) - .filter(SystemConfig.config_key == FREEZE_CONFIG_KEY) - .first() - ) - return bool(row) and str(row[0]).strip().lower() in ("1", "true", "yes", "on") - except Exception: # noqa: BLE001 - absence of the row means "not frozen" - return False - - -def _skill_layer_state(db) -> Tuple[Optional[datetime], int]: - """When a skill last went live, and how many releases are mid-flight. - - Read from the release ledger rather than kept in memory: a per-process - limiter resets on every restart and multiplies by the number of replicas, - which is exactly how a cooldown quietly stops existing. - """ - from core.db.models.evolution import EvolutionRelease - from core.evolution.release import ACTIVE, IN_FLIGHT_STAGES - - last_row = ( - db.query(EvolutionRelease.created_at) - .filter( - EvolutionRelease.target_kind == "skill", - EvolutionRelease.stage == ACTIVE, - ) - .order_by(EvolutionRelease.created_at.desc()) - .first() - ) - in_flight = ( - db.query(EvolutionRelease) - .filter(EvolutionRelease.stage.in_(list(IN_FLIGHT_STAGES))) - .count() - ) - last = last_row[0] if last_row else None - if last is not None and last.tzinfo is None: - last = last.replace(tzinfo=timezone.utc) - return last, int(in_flight or 0) - - -def _check_pace(db, *, tenant_id: str, layer: str = "skill") -> None: - """Enforce the multi-timescale gate the design declares. - - Skills sit on an hours-to-days cadence so the system does not become - non-stationary. That was a documented number with no enforcement: nothing - consulted ``timescales`` before writing a live asset, so ten activations - could land in one minute and no metric movement afterwards would be - attributable to any of them. - """ - from core.evolution.timescales import RateLimiter - - last, in_flight = _skill_layer_state(db) - limiter = RateLimiter(frozen=_is_frozen(db)) - if last is not None: - limiter.last_activation[layer] = last - allowed, reason = limiter.may_activate( - layer, tenant=tenant_id or "default", concurrent_activations=in_flight - ) - if not allowed: - raise ActivationError("pace_limited", f"节奏闸门拒绝本次激活：{reason}") - - -def _entry_stage( - candidate, - *, - approver: str, - force: bool, - owner_user_id: Optional[str], -) -> Tuple[str, int]: - """The stage this candidate has actually earned, and its traffic share. - - Previously every activation wrote ``active`` at 100%, whatever the risk tier - said. ``ir.MIN_VERIFICATION`` declares that a high-risk change owes replay, - shadow, canary *and* human approval before going live, and a brand-new - skill is exactly that tier — so the old behaviour skipped two of the four - stages its own IR demanded. - - A personal activation is the one case that legitimately goes straight live: - its blast radius is one consenting user, and the ramp exists to bound blast - radius. It is recorded as scoped rather than as full traffic. - """ - from core.evolution.ir import RISK_LOW, required_verification - from core.evolution.release import ACTIVE, REPLAY_PASSED, SHADOW - - risk = str(candidate.risk_tier or "").strip() or RISK_LOW - replayed = candidate.status in (REPLAY_PASSED, ACTIVE) - - if not replayed: - # ``force`` waives the replay *stage* — and only for the tier whose - # verification requirement is replay alone. Waiving it for a tier that - # also owes shadow and canary is not a shortcut, it is the whole gate. - if not force: - raise ActivationError( - "verification_incomplete", - f"候选尚未通过回放（当前 {candidate.status}）", - ) - if risk != RISK_LOW: - raise ActivationError( - "force_forbidden_for_risk", - f"force 只允许用于 low 风险候选，本候选为 {risk}；" - f"需完成 {list(required_verification(risk))}", - ) - - if owner_user_id: - return ACTIVE, 0 - - if set(required_verification(risk)) <= {"replay"}: - return ACTIVE, 100 - - # Higher tiers enter at shadow: the asset exists and is measurable, and - # reaches nobody until an operator ramps it. - return SHADOW, 0 - - -def materialise_skill_candidate( - candidate_id: str, - *, - approver: str, - force: bool = False, - owner_user_id: Optional[str] = None, - tenant_id: str = "default", -) -> Dict[str, Any]: - """Materialise an approved skill candidate, at the stage it has earned. - - This is the only path by which an evolution artefact reaches the agent, and - it now enters the release pipeline rather than bypassing it: - - * ``low`` risk (verification = replay only) goes live at 100%; - * ``medium`` / ``high`` enter **shadow**, reaching nobody, and must be ramped - through canary by :func:`advance_skill_release`; - * a personal activation (``owner_user_id``) goes live scoped to that user. - - ``force`` waives the replay *stage* and only for ``low`` risk. It never - waives separation of duties — an override is exactly what an attacker or an - over-eager script would reach for. - - ``owner_user_id`` scopes the resulting skill to one person. A personal - activation cannot change what anyone else's agent does, which is what lets a - user accept a capability distilled from their own conversations without an - administrator standing in the way. - """ - from core.db.models import AdminSkill - from core.db.models.evolution import EvolutionCandidate, EvolutionRelease - from core.evolution.release import ( - ACTIVE, - IN_FLIGHT_STAGES, - REPLAY_PASSED, - ROLLBACK_MANUAL, - new_release_id, - ) - - with SessionLocal() as db: - candidate = db.get(EvolutionCandidate, candidate_id) - if candidate is None: - raise ActivationError("candidate_not_found", f"候选不存在: {candidate_id}") - if candidate.target_kind != "skill": - raise ActivationError( - "unsupported_kind", f"暂不支持物化 {candidate.target_kind} 候选" - ) - - # Separation of duties. No force path — see the docstring. - if approver and approver == (candidate.proposer or ""): - raise ActivationError( - "self_approval_forbidden", "生成者不能激活自己提出的候选" - ) - if not approver: - raise ActivationError("approver_required", "激活必须署名") - - stage, traffic = _entry_stage( - candidate, approver=approver, force=force, owner_user_id=owner_user_id - ) - # Pace and freeze apply to anything that becomes live immediately. A - # shadow entry changes nothing users can see, so holding it back would - # only delay measurement. - if stage == ACTIVE and not owner_user_id: - _check_pace(db, tenant_id=tenant_id) - - ir = candidate.ir or {} - changes = ir.get("changes") or [] - tools: List[str] = [] - constraints: List[Dict[str, Any]] = [] - document: Optional[Dict[str, Any]] = None - manifest: Optional[Dict[str, Any]] = None - # ref → owning user. Demotion happens inside one person's store, so a - # skill compiled from several people's memories needs the owner of each. - source_memories: Dict[str, str] = {} - for change in changes: - sequence = change.get("tool_sequence") - if isinstance(sequence, list) and sequence and not tools: - tools = [str(t) for t in sequence] - rules = change.get("ordering_constraints") - if isinstance(rules, list) and rules: - constraints = rules - written = change.get("document") - if isinstance(written, dict) and written.get("content"): - document = written - declared = change.get("manifest") - if isinstance(declared, dict) and declared: - manifest = declared - for item in change.get("source_memories") or []: - if isinstance(item, dict) and item.get("ref"): - source_memories[str(item["ref"])] = str(item.get("user_id") or "") - - # 图谱证据自相矛盾的候选可以存在、可以进 shadow 观测，但不允许自动 - # 走到 ACTIVE：冲突意味着「适用条件」本身不可信，必须先由人裁决。 - if manifest and manifest.get("graph_conflict") and stage == ACTIVE: - raise ActivationError( - "graph_conflict_requires_review", - f"L3 图谱证据存在冲突，禁止直接激活：{manifest.get('graph_conflicts')}", - ) - - skill_id = candidate.target_asset_id - if not skill_id.startswith(EVOLUTION_PREFIX): - skill_id = EVOLUTION_PREFIX + skill_id.replace("auto-", "") - if owner_user_id: - # Distinct id per owner: a personal activation must never overwrite - # the global asset, nor another user's copy of the same candidate. - skill_id = f"{skill_id}-u{owner_user_id[-8:]}" - - if document is not None: - # A written skill: the generator produced the body and the machine - # validator already refused everything it was told to refuse. It is - # re-checked here against this candidate's id, because the id gains - # a prefix (and, for a personal activation, an owner suffix) after - # the draft was validated — and an id the loader rejects yields a - # skill that shows up in the capability list and loads as nothing. - from core.evolution.skill_gen import render_skill_markdown, validate_draft - - tools = [str(t) for t in (document.get("allowed_tools") or [])] - display_name = str(document.get("display_name") or "") - description = str(document.get("description") or "") - body = _body_of(str(document.get("content") or "")) - violations = validate_draft( - skill_id=skill_id, - display_name=display_name, - description=description, - body=body, - allowed_tools=tools, - ) - if violations: - raise ActivationError( - "draft_failed_validation", - f"技能正文未通过机器校验，拒绝物化: {violations}", - ) - rules_only = False - content = render_skill_markdown( - skill_id=skill_id, - display_name=display_name, - description=description, - body=body, - support=len(candidate.evidence_refs or []), - rationale=candidate.hypothesis or "", - ) - else: - # A candidate carrying only ordering rules is still a capability, - # and on real traffic it is the *common* case: five runs repeating - # one identical tool sequence is rare, while the same pairwise - # ordering recurring across different tool sets is not. - rules_only = not tools and bool(constraints) - if rules_only: - tools = _tools_from_constraints(constraints) - if not tools: - raise ActivationError( - "no_tool_sequence", "候选既未声明工具顺序、也未携带顺序约束，无法物化" - ) - display_name = _skill_label(tools, rules_only=rules_only) - description = _skill_description(tools, constraints, rules_only=rules_only) - content = _skill_markdown( - skill_id=skill_id, - label=display_name, - description=description, - tools=tools, - evidence=len(candidate.evidence_refs or []), - constraints=constraints, - rationale=candidate.hypothesis or "", - rules_only=rules_only, - ) - - previous = _capture_previous(db, skill_id) - - # One release in flight per asset. Two concurrent ramps on the same skill - # make any measured effect unattributable and leave rollback without a - # single target. - conflicting = ( - db.query(EvolutionRelease) - .filter( - EvolutionRelease.target_asset_id == skill_id, - EvolutionRelease.stage.in_(list(IN_FLIGHT_STAGES)), - ) - .first() - ) - if conflicting is not None: - raise ActivationError( - "release_in_flight", - f"该资产已有进行中的发布 {conflicting.release_id}（{conflicting.stage}）", - ) - - row = db.query(AdminSkill).filter(AdminSkill.skill_id == skill_id).first() - if row is None: - row = AdminSkill(skill_id=skill_id, created_by=f"evolution:{approver}") - db.add(row) - row.owner_user_id = owner_user_id or None - - row.skill_content = content - row.display_name = display_name - row.description = description - row.version = _next_version(previous) - row.tags = sorted(set((previous or {}).get("tags", [])) | {EVOLUTION_TAG}) - # The distilled ordering, in the shape the runtime reads. - extra_files = { - **(previous or {}).get("extra_files", {}), - "tool_sequence.json": _json(tools), - # The transferable part. A consumer that only understands sequences - # still works; one that understands rules gets the wider benefit. - "ordering_constraints.json": _json(constraints), - } - if manifest: - # 机器读 manifest（边界/依赖/证据引用），Agent 读 SKILL.md —— 两者 - # 职责分离。作用域以本次激活实际发生的为准，而不是候选申报的。 - extra_files["evolution_manifest.json"] = _json( - { - **manifest, - "scope": "personal" if owner_user_id else str( - manifest.get("scope") or "workspace" - ), - } - ) - row.extra_files = extra_files - row.allowed_tools = tools - row.is_enabled = True - - # The scope the IR asked for, now actually enforced at load time by - # core.evolution.exposure. Recording it here rather than deriving it - # later means a scope decision is auditable against the release it - # governs. - scope_filter: Dict[str, Any] = {"level": (ir.get("scope") or {}).get("level") or "workspace"} - if owner_user_id: - scope_filter["owner_user_id"] = owner_user_id - else: - scope_filter["tenant_id"] = tenant_id or "default" - - release = EvolutionRelease( - release_id=new_release_id(), - candidate_id=candidate_id, - target_kind="skill", - target_asset_id=skill_id, - stage=stage, - # A personal activation reaches exactly one user, so recording it as - # full traffic would misreport the blast radius on the release board. - traffic_percent=traffic, - scope_filter=scope_filter, - # Written now, not derived later: rollback needs a target that - # existed before anything changed. - rollback_version_id=(previous or {}).get("version") or "", - guardrails={ - "previous": previous, - # Which verification stages are behind us. Read by - # advance_skill_release before every hop. - "completed_verification": ( - ["replay"] if candidate.status in (REPLAY_PASSED, ACTIVE) else [] - ), - "proposer": candidate.proposer or "", - }, - approved_by=approver, - ) - db.add(release) - - # A personal activation leaves the candidate available for others to - # accept, and for an administrator to consider globally. Marking it - # ACTIVE would end its life for everyone on one person's decision. - # A shadow entry is likewise not the end of the candidate's life. - if not owner_user_id and stage == ACTIVE: - candidate.status = ACTIVE - candidate.approved_by = approver - candidate.approved_at = datetime.now(timezone.utc) - db.commit() - - release_id = release.release_id - new_version = row.version - candidate_owner = str((ir.get("scope") or {}).get("user_id") or "") - logger.info( - "[activation] materialised %s from %s at %s/%d%% (tools=%s written=%s)", - skill_id, - candidate_id, - stage, - traffic, - tools, - document is not None, - ) - - # Close the promotion chain: the memories this skill was compiled from now - # have a cheaper carrier, so they stop consuming the retrieval budget that - # made compiling them worthwhile. Outside the transaction above because it - # writes to its own ledger, and down-weighted rather than removed, so - # rolling the skill back restores them exactly. - demoted: List[str] = [] - if source_memories: - from core.evolution.memory_apply import demote_promoted_memories - - by_owner: Dict[str, List[str]] = {} - for ref, owner in source_memories.items(): - # ``owner_user_id`` for a personal activation, the ref's recorded - # owner otherwise. A ref with neither is skipped rather than guessed - # at: applying a change to the wrong person's memories is worse than - # not applying it. - resolved = owner_user_id or owner or candidate_owner - if resolved: - by_owner.setdefault(resolved, []).append(ref) - - for owner, refs in by_owner.items(): - outcome = demote_promoted_memories( - candidate_id=candidate_id, - user_id=owner, - memory_refs=refs, - skill_id=skill_id, - ) - demoted.extend(op["memory_ref"] for op in outcome.get("applied", [])) - - # 血缘落账：这一版技能由哪些 L2 记忆编译（compiled_from）、被哪些 L3 关系 - # 约束（constrained_by）、由哪些 Episode 验证（validated_by）。尽力而为—— - # 账本失败绝不回滚一次已经成立的激活。 - lineage_written = 0 - try: - from core.evolution import lineage - - with SessionLocal() as db: - refreshed = db.get(EvolutionCandidate, candidate_id) - pack_id = str(getattr(refreshed, "evidence_pack_id", "") or "") - pack = lineage.load_evidence_pack(pack_id) if pack_id else None - if pack is not None: - lineage_written = len( - lineage.record_links( - lineage.links_for_skill( - pack, skill_id=skill_id, skill_version=new_version - ), - candidate_id=candidate_id, - ) - ) - except Exception as exc: # noqa: BLE001 - logger.debug("[activation] lineage recording skipped: %s", exc) - - return { - "skill_id": skill_id, - "release_id": release_id, - "stage": stage, - "traffic_percent": traffic, - "scope": scope_filter, - "demoted_memories": demoted, - "lineage_links": lineage_written, - # Spelled out because "activated" previously implied "live for - # everyone", and for anything above low risk it no longer does. - "exposed_to": ( - "all users in scope" - if stage == ACTIVE and not owner_user_id - else (f"only {owner_user_id}" if owner_user_id else "nobody (shadow)") - ), - "tools": tools, - "version": new_version, - "replaced_previous": previous is not None, - } - - -def advance_skill_release( - release_id: str, - *, - actor: str, - observed_metrics: Optional[Dict[str, float]] = None, - guardrail_thresholds: Optional[Dict[str, float]] = None, -) -> Dict[str, Any]: - """Ramp a release one step: shadow → canary 5 → 25 → 100 → active. - - One step per call, deliberately. A single "go live" button that walked the - whole ladder would reproduce the behaviour this replaces — the point of the - ladder is that somebody looks at the metrics between rungs. - - Guardrails are checked when metrics are supplied, and a breach rolls back - rather than pausing: the recorded threshold already *is* the level at which - the release is unacceptable. - """ - from core.db.models.evolution import EvolutionCandidate, EvolutionRelease - from core.evolution.release import ( - ACTIVE, - CANARY, - CANARY_STEPS, - SHADOW, - can_promote, - evaluate_guardrails, - next_traffic, - should_auto_rollback, - ) - - if not actor: - raise ActivationError("actor_required", "推进发布必须署名") - - with SessionLocal() as db: - release = db.get(EvolutionRelease, release_id) - if release is None: - raise ActivationError("release_not_found", f"发布记录不存在: {release_id}") - if _is_frozen(db): - raise ActivationError("globally_frozen", "全局冻结开关已打开，禁止推进发布") - - candidate = db.get(EvolutionCandidate, release.candidate_id) - risk = str(getattr(candidate, "risk_tier", "") or "low") - guardrails = dict(release.guardrails or {}) - completed: List[str] = list(guardrails.get("completed_verification") or []) - proposer = str(guardrails.get("proposer") or getattr(candidate, "proposer", "") or "") - - if observed_metrics: - thresholds = guardrail_thresholds or guardrails.get("thresholds") or {} - breaches = evaluate_guardrails(observed_metrics, thresholds, window="canary") - if should_auto_rollback(breaches): - detail = ", ".join( - f"{b.metric}={b.observed}>{b.threshold}" for b in breaches - ) - db.commit() - rollback_skill_activation( - release_id, - actor=actor, - reason=f"guardrail breach: {detail}", - kind="guardrail", - ) - return { - "release_id": release_id, - "stage": "rolled_back", - "guardrail_breaches": [b.to_dict() for b in breaches], - } - - stage = str(release.stage or "") - traffic = int(release.traffic_percent or 0) - - if stage == SHADOW: - target_stage, target_traffic = CANARY, CANARY_STEPS[0] - completed = sorted(set(completed) | {"shadow"}) - elif stage == CANARY and traffic < CANARY_STEPS[-1]: - target_stage, target_traffic = CANARY, next_traffic(traffic) - elif stage == CANARY: - target_stage, target_traffic = ACTIVE, 100 - completed = sorted(set(completed) | {"canary"}) - else: - raise ActivationError( - "not_rampable", f"当前阶段 {stage} 无法继续放量" - ) - - allowed, reason = can_promote( - current_stage=stage, - target_stage=target_stage, - risk_tier=risk, - # ``human_approval`` is this call: a named actor who is not the - # proposer. Recording it before the check keeps the requirement - # satisfied by the act itself rather than by a stored flag. - completed_verification=sorted(set(completed) | {"human_approval"}) - if target_stage == ACTIVE - else completed, - proposer=proposer, - approver=actor, - ) - # A same-stage traffic bump is not a transition; can_promote would reject - # canary→canary, which is correct for stages and wrong for ramp steps. - if not allowed and not (stage == target_stage == CANARY): - raise ActivationError("promotion_refused", reason) - - if target_stage == ACTIVE: - # 冲突的 L3 证据在 shadow/canary 里观测没问题，但升到全量必须先由 - # 人裁决冲突本身——适用条件不可信的技能不配自动放大到所有人。 - if _has_graph_conflict(candidate): - raise ActivationError( - "graph_conflict_requires_review", - "候选的 L3 图谱证据存在未裁决的冲突，禁止放量到 active", - ) - _check_pace(db, tenant_id=str((release.scope_filter or {}).get("tenant_id") or "default")) - - release.stage = target_stage - release.traffic_percent = target_traffic - guardrails["completed_verification"] = completed - if target_stage == ACTIVE: - guardrails["approved_by"] = actor - release.guardrails = guardrails - release.approved_by = actor - if candidate is not None and target_stage == ACTIVE: - candidate.status = ACTIVE - db.commit() - - logger.info( - "[activation] release %s → %s/%d%% by %s", - release_id, - target_stage, - target_traffic, - actor, - ) - return { - "release_id": release_id, - "stage": target_stage, - "traffic_percent": target_traffic, - "completed_verification": completed, - "exposed_to": ( - "all users in scope" - if target_stage == ACTIVE - else f"{target_traffic}% of users in scope" - ), - } - - -def _has_graph_conflict(candidate) -> bool: - """Whether the candidate's manifest flags contradictory L3 evidence.""" - if candidate is None: - return False - for change in (getattr(candidate, "ir", None) or {}).get("changes") or []: - manifest = change.get("manifest") if isinstance(change, dict) else None - if isinstance(manifest, dict) and manifest.get("graph_conflict"): - return True - return False - - -def rollback_skill_activation( - release_id: str, *, actor: str, reason: str, kind: str = "manual" -) -> Dict[str, Any]: - """Undo a materialisation, restoring the snapshot taken at activation time. - - With no prior version the skill is disabled rather than deleted: history - referencing it must stay resolvable, and a disabled row is both reversible - and visible in a way a missing row is not. - - ``kind`` separates "a guardrail caught it" from "a person caught it". - Collapsing the two would hide how well the evaluation stage is working, - which is the number that tells you whether the ramp is worth its cost. - """ - from core.db.models import AdminSkill - from core.db.models.evolution import ( - EvolutionAgentProfile, - EvolutionCandidate, - EvolutionRelease, - ) - from core.evolution.release import ROLLBACK_GUARDRAIL, ROLLBACK_MANUAL, ROLLED_BACK - - if not actor or not reason: - raise ActivationError("audit_required", "回滚必须署名并填写理由") - - with SessionLocal() as db: - release = db.get(EvolutionRelease, release_id) - if release is None: - raise ActivationError("release_not_found", f"发布记录不存在: {release_id}") - - previous = (release.guardrails or {}).get("previous") - skill_id = release.target_asset_id - row = ( - db.query(AdminSkill) - .filter(AdminSkill.skill_id == skill_id) - .first() - ) - invalidated: List[str] = [] - if row is not None: - if previous: - row.skill_content = previous["skill_content"] - row.display_name = previous["display_name"] - row.description = previous["description"] - row.version = previous["version"] - row.tags = previous["tags"] - row.allowed_tools = previous["allowed_tools"] - row.extra_files = previous["extra_files"] - row.is_enabled = previous["is_enabled"] - else: - row.is_enabled = False - # No prior version means the skill just stopped being loadable, - # so a profile assembled on top of it is now promising a - # capability that fails silently — same cross-layer rule as - # retirement. With a restored prior version the skill stays - # loadable and the profiles stay valid. - for profile_row in ( - db.query(EvolutionAgentProfile) - .filter(EvolutionAgentProfile.is_active.is_(True)) - .all() - ): - payload = profile_row.payload or {} - if skill_id in (payload.get("skill_ids") or []): - profile_row.is_active = False - invalidated.append(profile_row.profile_id) - - release.stage = ROLLED_BACK - release.rolled_back_at = datetime.now(timezone.utc) - release.rollback_reason = reason - release.rollback_kind = ( - ROLLBACK_GUARDRAIL if kind == "guardrail" else ROLLBACK_MANUAL - ) - - candidate = db.get(EvolutionCandidate, release.candidate_id) - if candidate is not None: - candidate.status = ROLLED_BACK - candidate_id = release.candidate_id - db.commit() - - # The reverse edge of the promotion chain. Materialisation down-weighted the - # memories this skill was compiled from; rolling the skill back must restore - # them in the same motion, or the knowledge stays half-priced with no - # carrier — retrieval skips it because a skill supposedly covers it, and the - # skill is gone. Same ledger the retire path already replays. - from core.evolution.memory_apply import revert_memory_ops - - restored = revert_memory_ops(candidate_id=candidate_id) if candidate_id else {} - - # 血缘：这一版回滚到了哪一版。审计问「技能 v2 后来怎么了」时，答案在链上。 - try: - from core.evolution import lineage - - current_version = str((previous or {}).get("version") or "") - lineage.record_links( - [ - ( - "skill", - f"{skill_id}@rolled_back", - lineage.REL_ROLLED_BACK_TO, - "skill", - f"{skill_id}@{current_version or 'disabled'}", - ) - ], - candidate_id=candidate_id, - ) - except Exception as exc: # noqa: BLE001 - logger.debug("[activation] rollback lineage skipped: %s", exc) - - logger.info( - "[activation] rolled back %s (release=%s memories restored=%d profiles invalidated=%s)", - skill_id, - release_id, - len(restored.get("reverted", [])), - invalidated, - ) - return { - "release_id": release_id, - "restored_previous": bool(previous), - "restored_memory_ops": restored.get("reverted", []), - "invalidated_profiles": invalidated, - "reason": reason, - } - - -def _tools_from_constraints(constraints: Sequence[Dict[str, Any]]) -> List[str]: - """The tools a set of ordering rules talks about, in an order they permit. - - Not a claimed execution plan — a rules-only skill has no fixed sequence. It - is the tool list the skill declares, listed in an order that does not - contradict its own rules so the document does not read as self-refuting. - """ - tools: List[str] = [] - for rule in constraints: - for key in ("before", "after"): - name = str(rule.get(key) or "") - if name and name not in tools: - tools.append(name) - if not tools: - return [] - from core.evolution.promotion import OrderingConstraint, apply_ordering_constraints - - parsed = [ - OrderingConstraint( - before=str(rule.get("before") or ""), - after=str(rule.get("after") or ""), - support=int(rule.get("support") or 0), - contexts=int(rule.get("contexts") or 0), - validated_in=tuple(rule.get("validated_in") or ()), - # Scope is deliberately dropped for this one purpose: we are only - # ordering the *document's* tool list, not deciding applicability. - ) - for rule in constraints - if rule.get("before") and rule.get("after") - ] - return apply_ordering_constraints(tools, parsed) - - -def _next_version(previous: Optional[Dict[str, Any]]) -> str: - if not previous: - return "1.0.0" - parts = str(previous.get("version") or "1.0.0").split(".") - try: - parts[-1] = str(int(parts[-1]) + 1) - return ".".join(parts) - except Exception: - return "1.0.1" - - -def _json(value: Any) -> str: - import json - - return json.dumps(value, ensure_ascii=False) - - -# ── Orchestration assets ───────────────────────────────────────────────────── - - -def materialise_profile_candidate( - candidate_id: str, *, approver: str, tenant_id: str = "default" -) -> Dict[str, Any]: - """Publish an approved agent profile. - - Profiles do not ride the shadow → canary ladder that skills do, and the - reason is not leniency. A profile *is* the assembly, so a canary would mean - two different assemblies running concurrently under one profile id, with - every downstream measurement of either one confounded by the other. Instead - a profile is scoped: it governs one task type, and its blast radius is that - task type's traffic, recorded on the release row like any other change. - - Rolling back is switching ``is_active`` off, which restores the built-in - assembly on the next run — no restart, because profiles are read per run. - """ - from core.db.models.evolution import EvolutionAgentProfile - from core.evolution.agent_profile import AgentProfile, TARGET_KIND, validate_profile - from core.evolution.release import ACTIVE as _ACTIVE, new_release_id - - if not approver: - raise ActivationError("approver_required", "激活必须署名") - - with SessionLocal() as db: - candidate = db.get(EvolutionCandidate, candidate_id) - if candidate is None: - raise ActivationError("candidate_not_found", f"候选不存在: {candidate_id}") - if candidate.target_kind != TARGET_KIND: - raise ActivationError( - "unsupported_kind", f"该入口只处理编排候选，收到 {candidate.target_kind}" - ) - if approver == (candidate.proposer or ""): - raise ActivationError("self_approval_forbidden", "生成者不能激活自己提出的候选") - if _is_frozen(db): - raise ActivationError("layer_frozen", "进化已被全局冻结") - - payload: Dict[str, Any] = {} - for change in candidate.ir.get("changes") or []: - if isinstance(change.get("profile"), dict): - payload = change["profile"] - break - if not payload: - raise ActivationError("no_profile_payload", "候选未携带 profile 定义") - - profile = AgentProfile.from_dict(payload) - profile.scope = {**(profile.scope or {}), "tenant_id": tenant_id or "default"} - - # Re-validated at activation against the *current* tool grant, not the - # one that existed when the candidate was written. A grant narrowed in - # the meantime must not be widened again by approving an older proposal. - from core.config.catalog import get_enabled_ids - - base_tools = [str(t) for t in get_enabled_ids("tools")] + [ - str(t) for t in get_enabled_ids("mcp") - ] - ok, problems = validate_profile(profile, base_tools=base_tools) - if not ok: - raise ActivationError("profile_invalid", f"编排定义未通过校验: {problems}") - - previous = db.get(EvolutionAgentProfile, profile.profile_id) - previous_payload = dict(previous.payload) if previous is not None else None - - # One active profile per (task type, scope). Two would make the - # resolution order the thing that decides behaviour, which is not a - # decision anyone reviewed. - for row in ( - db.query(EvolutionAgentProfile) - .filter(EvolutionAgentProfile.is_active.is_(True)) - .all() - ): - if row.profile_id == profile.profile_id: - continue - same_types = set(row.task_types or []) == set(profile.task_types) - same_scope = dict(row.scope or {}) == dict(profile.scope) - if same_types and same_scope: - row.is_active = False - - row = previous or EvolutionAgentProfile(profile_id=profile.profile_id) - row.version = profile.version - row.task_types = list(profile.task_types) - row.scope = dict(profile.scope) - row.payload = profile.to_dict() - row.candidate_id = candidate_id - row.is_active = True - row.created_by = f"evolution:{approver}" - if previous is None: - db.add(row) - - release = EvolutionRelease( - release_id=new_release_id(), - candidate_id=candidate_id, - target_kind=TARGET_KIND, - target_asset_id=profile.profile_id, - stage=_ACTIVE, - traffic_percent=100, - scope_filter=dict(profile.scope), - guardrails={"previous": previous_payload, "proposer": candidate.proposer or ""}, - rollback_version_id=(previous_payload or {}).get("version") or "builtin", - approved_by=approver, - ) - db.add(release) - - candidate.status = _ACTIVE - candidate.approved_by = approver - candidate.approved_at = datetime.now(timezone.utc) - db.commit() - - # 血缘：哪些技能被装配进了这个 profile。 - try: - from core.evolution import lineage - - lineage.record_links( - [ - ( - "skill", - str(skill_id), - lineage.REL_ASSEMBLED_INTO, - "agent_profile", - f"{profile.profile_id}@{profile.version}", - ) - for skill_id in profile.skill_ids - ], - candidate_id=candidate_id, - ) - except Exception as exc: # noqa: BLE001 - logger.debug("[activation] profile lineage skipped: %s", exc) - - logger.info( - "[activation] profile %s active for %s (tools=%s skills=%s)", - profile.profile_id, - profile.task_types or ["*"], - "narrowed" if profile.tool_allowlist is not None else "unchanged", - len(profile.skill_ids), - ) - return { - "profile_id": profile.profile_id, - "release_id": release.release_id, - "task_types": list(profile.task_types), - "scope": dict(profile.scope), - "tool_allowlist": profile.tool_allowlist, - "skill_ids": list(profile.skill_ids), - "replaced_previous": previous_payload is not None, - } - - -def materialise_subagent_candidate( - candidate_id: str, *, approver: str, owner_user_id: Optional[str] = None -) -> Dict[str, Any]: - """Create an approved sub-agent, and nothing else. - - Deliberately does **not** install the route that sends work to it. The route - is a sentence in the parent's prompt, and a sentence in the parent's prompt - changes every request the parent handles — so it goes through prompt review - as its own candidate. Creating the agent here and routing to it there means - approving the agent cannot silently also approve a behaviour change. - - The agent is created disabled for the same reason: an agent nobody routes to - does nothing, and one that is enabled before its route is reviewed is a - capability sitting in the product that no reviewer has seen used. - """ - from core.db.models import UserAgent - from core.evolution.release import ACTIVE as _ACTIVE, SHADOW, new_release_id - - if not approver: - raise ActivationError("approver_required", "激活必须署名") - - with SessionLocal() as db: - candidate = db.get(EvolutionCandidate, candidate_id) - if candidate is None: - raise ActivationError("candidate_not_found", f"候选不存在: {candidate_id}") - if candidate.target_kind != "subagent": - raise ActivationError( - "unsupported_kind", f"该入口只处理子智能体候选，收到 {candidate.target_kind}" - ) - if approver == (candidate.proposer or ""): - raise ActivationError("self_approval_forbidden", "生成者不能激活自己提出的候选") - if _is_frozen(db): - raise ActivationError("layer_frozen", "进化已被全局冻结") - - spec: Dict[str, Any] = {} - for change in candidate.ir.get("changes") or []: - if change.get("agent_id"): - spec = change - break - if not spec: - raise ActivationError("no_subagent_payload", "候选未携带子智能体定义") - - # The subset check runs again at activation against the current grant. - from core.config.catalog import get_enabled_ids - - base_tools = {str(t) for t in get_enabled_ids("tools")} | { - str(t) for t in get_enabled_ids("mcp") - } - requested = {str(t) for t in (spec.get("tools_allowlist") or [])} - widened = sorted(requested - base_tools) - if widened: - raise ActivationError( - "tool_escalation", f"子智能体工具集超出主体授权: {widened}" - ) - if spec.get("may_delegate"): - raise ActivationError("subagent_nesting", "子智能体不得再派生子智能体") - - agent_id = str(spec.get("agent_id")) - existing = db.get(UserAgent, agent_id) - if existing is not None: - raise ActivationError("agent_exists", f"子智能体已存在: {agent_id}") - - agent = UserAgent( - agent_id=agent_id, - owner_type="user" if owner_user_id else "admin", - user_id=owner_user_id, - name=str(spec.get("name") or agent_id)[:255], - description=str(spec.get("responsibility") or "")[:2000], - system_prompt=str(spec.get("responsibility") or ""), - skill_ids=[str(s) for s in (spec.get("required_skills") or [])], - mcp_server_ids=sorted(requested), - kb_ids=[], - plugin_ids=[], - max_iters=12, - timeout=int(spec.get("timeout") or 180), - # Disabled until its route is approved — see the docstring. - is_enabled=False, - extra_config={ - "origin": "evolution", - "candidate_id": candidate_id, - "route_hint": str(spec.get("route_hint") or ""), - "task_types": [str(t) for t in (spec.get("task_types") or [])], - "tools_allowlist": sorted(requested), - "may_delegate": False, - }, - ) - db.add(agent) - - release = EvolutionRelease( - release_id=new_release_id(), - candidate_id=candidate_id, - target_kind="subagent", - target_asset_id=agent_id, - # Reaches nobody until the route exists, which is what shadow means. - stage=SHADOW, - traffic_percent=0, - scope_filter={"owner_user_id": owner_user_id} if owner_user_id else {}, - guardrails={"proposer": candidate.proposer or "", "route_pending": True}, - rollback_version_id="v0", - approved_by=approver, - ) - db.add(release) - - candidate.approved_by = approver - candidate.approved_at = datetime.now(timezone.utc) - db.commit() - - logger.info( - "[activation] subagent %s created (disabled, route pending) tools=%s", - agent_id, - sorted(requested), - ) - return { - "agent_id": agent_id, - "release_id": release.release_id, - "stage": SHADOW, - "is_enabled": False, - "tools_allowlist": sorted(requested), - "route_hint": str(spec.get("route_hint") or ""), - "next_step": "路由句需作为提示词候选单独审批后，子智能体才会被启用", - } - - -def retire_skill_candidate(candidate_id: str, *, approver: str) -> Dict[str, Any]: - """Take an approved retirement into effect. - - Retirement is disabling plus a release row, never a delete. Three reasons it - is done this way: the evidence behind the decision stays inspectable, the - skill can be brought back in one operation if the decision was wrong, and - anything built on top of it — a profile naming it, a sub-agent requiring it - — can be found and invalidated rather than silently pointing at nothing. - """ - from core.db.models import AdminSkill - from core.db.models.evolution import EvolutionAgentProfile - from core.evolution.release import RETIRED, new_release_id - - if not approver: - raise ActivationError("approver_required", "退役必须署名") - - with SessionLocal() as db: - candidate = db.get(EvolutionCandidate, candidate_id) - if candidate is None: - raise ActivationError("candidate_not_found", f"候选不存在: {candidate_id}") - if candidate.operation != "deprecate": - raise ActivationError("not_a_retirement", "该候选不是退役候选") - if approver == (candidate.proposer or ""): - raise ActivationError("self_approval_forbidden", "生成者不能批准自己提出的退役") - - skill_id = candidate.target_asset_id - row = db.query(AdminSkill).filter(AdminSkill.skill_id == skill_id).first() - if row is None: - raise ActivationError("skill_not_found", f"技能不存在: {skill_id}") - - previous_enabled = bool(row.is_enabled) - row.is_enabled = False - - # Cross-layer: anything assembled on top of this skill stops being - # applied. Leaving them active would leave a profile promising a - # capability that is no longer loadable, which fails silently. - invalidated: List[str] = [] - for profile_row in ( - db.query(EvolutionAgentProfile) - .filter(EvolutionAgentProfile.is_active.is_(True)) - .all() - ): - payload = profile_row.payload or {} - if skill_id in (payload.get("skill_ids") or []): - profile_row.is_active = False - invalidated.append(profile_row.profile_id) - - release = EvolutionRelease( - release_id=new_release_id(), - candidate_id=candidate_id, - target_kind="skill", - target_asset_id=skill_id, - stage=RETIRED, - traffic_percent=0, - scope_filter={}, - guardrails={ - "previous_enabled": previous_enabled, - "invalidated_profiles": invalidated, - "proposer": candidate.proposer or "", - }, - rollback_version_id=row.version or "", - approved_by=approver, - ) - db.add(release) - - candidate.status = "active" - candidate.approved_by = approver - candidate.approved_at = datetime.now(timezone.utc) - db.commit() - - # Restore the memories this skill was compiled from: the knowledge lost - # its cheaper carrier, so it goes back to being paid for by retrieval. - from core.evolution.memory_apply import revert_memory_ops - - source_candidate = _source_candidate_for(db, skill_id) - restored = revert_memory_ops(candidate_id=source_candidate) if source_candidate else {} - - logger.info( - "[activation] retired %s (profiles invalidated=%s memories restored=%d)", - skill_id, - invalidated, - len(restored.get("reverted", [])), - ) - return { - "skill_id": skill_id, - "release_id": release.release_id, - "invalidated_profiles": invalidated, - "restored_memory_ops": restored.get("reverted", []), - } - - -def _source_candidate_for(db, skill_id: str) -> str: - """The candidate that first materialised this skill. - - Memory demotions are recorded against that candidate, so it is the key that - finds them again when the skill is retired. - """ - row = ( - db.query(EvolutionRelease) - .filter( - EvolutionRelease.target_kind == "skill", - EvolutionRelease.target_asset_id == skill_id, - ) - .order_by(EvolutionRelease.created_at.asc()) - .first() - ) - return str(row.candidate_id) if row is not None else "" diff --git a/src/backend/core/evolution/agent_profile.py b/src/backend/core/evolution/agent_profile.py deleted file mode 100644 index 7d73981f..00000000 --- a/src/backend/core/evolution/agent_profile.py +++ /dev/null @@ -1,505 +0,0 @@ -"""The orchestration asset: what one ReAct assembly is made of. - -Orchestration evolution used to target :class:`~core.evolution.policies.WorkflowPolicy`, -a set of retry/budget knobs read by ``autonomous_loop``. That was the wrong -target, for a reason that has nothing to do with how the knobs were written: the -product's main axis is a **ReAct loop**, and a ReAct loop has no DAG to tune. It -re-decides every turn from whatever was assembled in front of it. So the -orchestration of a ReAct system *is* the assembly, and it has six parts: - -1. **prompt fragments** — what standing instructions this task type carries; -2. **tool allowlist** — which tools are even visible (a narrower set is a better - set: every irrelevant tool is a wrong turn the model can take); -3. **skill set** — which capabilities are offered, i.e. "the combination of - skills", which is what a recurring skill co-occurrence should promote *into*; -4. **subagent routes** — what gets handed to a separate context entirely; -5. **retrieval and budget** — memory budget, max turns, reviewer checkpoints; -6. **intervention rules** — what to do when the loop stops making progress, - which applies to a ReAct loop exactly as it applied to the autonomous one. - -A profile is **declarative only**. It selects, orders, budgets and routes; it -never carries executable content. ``OP_WHITELIST["agent_profile"]`` has no -operation resembling ``edit_source``, and the tool allowlist can only ever -narrow: :func:`validate_profile` refuses a profile requesting a tool outside the -base grant, so an evolved profile cannot become a privilege-escalation path. - -Scope resolution is explicit and ordered — user, then tenant, then deployment -default — and a profile belonging to *another* tenant is never a fallback. -Inheriting someone else's orchestration is invisible when it happens and shows -up as "the system got slower last week", which is the hardest class of problem -to attribute. -""" - -from __future__ import annotations - -import logging -from dataclasses import dataclass, field -from typing import Any, Dict, List, Optional, Sequence, Tuple - -logger = logging.getLogger(__name__) - -# ── Intervention vocabulary (signals a ReAct loop can actually observe) ─────── - -SIGNAL_NO_PROGRESS = "no_progress" -SIGNAL_REPEATED_ACTIONS = "repeated_actions" -SIGNAL_NO_DIFF = "no_diff" -SIGNAL_REVIEWER_FLAT = "reviewer_score_flat" -SIGNAL_BUDGET_LOW = "budget_low" -SIGNAL_TOOL_ERROR_STREAK = "tool_error_streak" -ALLOWED_SIGNALS = ( - SIGNAL_NO_PROGRESS, - SIGNAL_REPEATED_ACTIONS, - SIGNAL_NO_DIFF, - SIGNAL_REVIEWER_FLAT, - SIGNAL_BUDGET_LOW, - SIGNAL_TOOL_ERROR_STREAK, -) - -# What a single ReAct turn can actually observe about itself. The rest need a -# file tree and a reviewer verdict — things a long-running loop has and one -# answer does not — so a rule keyed on them is inert on the main axis. Stated -# here rather than discovered later: a rule that can never fire looks identical, -# in the console, to one that simply has not fired yet. -REACT_OBSERVABLE_SIGNALS = frozenset( - {SIGNAL_NO_PROGRESS, SIGNAL_REPEATED_ACTIONS, SIGNAL_TOOL_ERROR_STREAK} -) -LOOP_ONLY_SIGNALS = frozenset( - {SIGNAL_NO_DIFF, SIGNAL_REVIEWER_FLAT, SIGNAL_BUDGET_LOW} -) - -ACTION_RETRY = "retry" -ACTION_CHANGE_STRATEGY = "change_strategy" -ACTION_ROLLBACK_AND_FORK = "rollback_and_fork" -ACTION_NARROW_SCOPE = "narrow_scope" -ACTION_DELEGATE = "delegate" -ACTION_ESCALATE = "escalate" -ACTION_STOP = "stop" -ALLOWED_ACTIONS = ( - ACTION_RETRY, - ACTION_CHANGE_STRATEGY, - ACTION_ROLLBACK_AND_FORK, - ACTION_NARROW_SCOPE, - ACTION_DELEGATE, - ACTION_ESCALATE, - ACTION_STOP, -) - -# Budget multipliers outside this band stop being tuning and start being a -# different product: a 10× budget is a cost incident, a 0.05× budget is an -# outage. The band is validated, not clamped — silently correcting an -# out-of-range value hides the fact that something proposed it. -MIN_BUDGET_MULTIPLIER = 0.25 -MAX_BUDGET_MULTIPLIER = 3.0 -# 0 is the default and means "no turn cap" — the main agent is unbounded unless -# a published profile deliberately fences it. Any other value must land in the -# band below; the band starts at 3 because a 1-2 turn main agent cannot finish a -# single tool round-trip. -MIN_REACT_TURNS = 3 -MAX_REACT_TURNS = 80 -UNBOUNDED_REACT_TURNS = 0 -MIN_MEMORY_BUDGET_MS = 100 -MAX_MEMORY_BUDGET_MS = 5000 -MIN_LOOP_ATTEMPTS = 2 -MAX_LOOP_ATTEMPTS = 20 - -TARGET_KIND = "agent_profile" - - -@dataclass -class InterventionRule: - """signal → threshold → action.""" - - signal: str - threshold: int - action: str - note: str = "" - - def to_dict(self) -> Dict[str, Any]: - return { - "signal": self.signal, - "threshold": self.threshold, - "action": self.action, - "note": self.note, - } - - @classmethod - def from_dict(cls, raw: Dict[str, Any]) -> "InterventionRule": - return cls( - signal=str(raw.get("signal") or ""), - threshold=int(raw.get("threshold") or 0), - action=str(raw.get("action") or ""), - note=str(raw.get("note") or ""), - ) - - -@dataclass -class SubagentRoute: - """Hand this shape of sub-task to a separate context. - - ``agent_id`` points at a real sub-agent; the route is only the *decision to - delegate*. Creating the sub-agent is a separate, separately-approved change - (see :mod:`core.evolution.orchestration_gen`), because a route to an agent - that does not exist is a dead end and an agent nobody routes to is dead - weight — they have to be reviewed as one change but recorded as two. - """ - - agent_id: str - when: str = "" - task_types: List[str] = field(default_factory=list) - - def to_dict(self) -> Dict[str, Any]: - return {"agent_id": self.agent_id, "when": self.when, "task_types": list(self.task_types)} - - @classmethod - def from_dict(cls, raw: Dict[str, Any]) -> "SubagentRoute": - return cls( - agent_id=str(raw.get("agent_id") or ""), - when=str(raw.get("when") or ""), - task_types=[str(t) for t in (raw.get("task_types") or [])], - ) - - -@dataclass -class AgentProfile: - """One assembled ReAct configuration, as data.""" - - profile_id: str = "builtin" - version: str = "v0" - # Which tasks this profile governs. Empty means "every task in scope" — the - # deployment default. A profile with task types is only consulted for those, - # which is how a capability learned in one family stops leaking into others. - task_types: List[str] = field(default_factory=list) - scope: Dict[str, str] = field(default_factory=dict) - - prompt_fragments: List[str] = field(default_factory=list) - # ``None`` means "no restriction beyond the base grant". An empty list means - # "no tools", which is a legitimate profile for a pure-reasoning task type - # and must not be confused with the former. - tool_allowlist: Optional[List[str]] = None - # Which skills this task type may load at all. Empty means "no restriction" - # — skill loading keeps its own logic and this only ever narrows it, and - # only for task types an approved profile actually covers. - skill_ids: List[str] = field(default_factory=list) - subagent_routes: List[SubagentRoute] = field(default_factory=list) - - memory_budget_ms: int = 600 - # 0 = 不限（默认）。See UNBOUNDED_REACT_TURNS: the main agent's round ceiling - # was removed, so a turn budget here is an opt-in narrowing a published - # profile performs deliberately, not a floor every profile inherits. - max_react_turns: int = UNBOUNDED_REACT_TURNS - budget_multiplier: float = 1.0 - reviewer_checkpoints: List[str] = field(default_factory=list) - intervention_rules: List[InterventionRule] = field(default_factory=list) - - # The autonomous loop's own caps. Held as their own fields rather than - # derived from ``max_react_turns``: the two loops bound different things — - # turns inside one answer versus attempts at one requirement across many - # answers — and a ratio between them would be an invented relationship that - # quietly changes one whenever the other is tuned. - loop_max_attempts_per_requirement: int = 6 - loop_strategy_change_after: int = 2 - - def to_dict(self) -> Dict[str, Any]: - return { - "profile_id": self.profile_id, - "version": self.version, - "task_types": list(self.task_types), - "scope": dict(self.scope), - "prompt_fragments": list(self.prompt_fragments), - "tool_allowlist": None if self.tool_allowlist is None else list(self.tool_allowlist), - "skill_ids": list(self.skill_ids), - "subagent_routes": [r.to_dict() for r in self.subagent_routes], - "memory_budget_ms": self.memory_budget_ms, - "max_react_turns": self.max_react_turns, - "budget_multiplier": self.budget_multiplier, - "reviewer_checkpoints": list(self.reviewer_checkpoints), - "intervention_rules": [r.to_dict() for r in self.intervention_rules], - "loop_max_attempts_per_requirement": self.loop_max_attempts_per_requirement, - "loop_strategy_change_after": self.loop_strategy_change_after, - } - - @classmethod - def from_dict(cls, raw: Dict[str, Any]) -> "AgentProfile": - allowlist = raw.get("tool_allowlist", None) - return cls( - profile_id=str(raw.get("profile_id") or "builtin"), - version=str(raw.get("version") or "v0"), - task_types=[str(t) for t in (raw.get("task_types") or [])], - scope=dict(raw.get("scope") or {}), - prompt_fragments=[str(f) for f in (raw.get("prompt_fragments") or [])], - tool_allowlist=None if allowlist is None else [str(t) for t in allowlist], - skill_ids=[str(s) for s in (raw.get("skill_ids") or [])], - subagent_routes=[ - SubagentRoute.from_dict(r) for r in (raw.get("subagent_routes") or []) - ], - memory_budget_ms=int(raw.get("memory_budget_ms") or 600), - # Missing key → unbounded, not an invented ceiling: a profile - # published for unrelated reasons (tool allowlist, memory budget) - # must not silently re-fence the main loop. - max_react_turns=int(raw.get("max_react_turns") or UNBOUNDED_REACT_TURNS), - budget_multiplier=float(raw.get("budget_multiplier") or 1.0), - reviewer_checkpoints=[str(c) for c in (raw.get("reviewer_checkpoints") or [])], - intervention_rules=[ - InterventionRule.from_dict(r) for r in (raw.get("intervention_rules") or []) - ], - loop_max_attempts_per_requirement=int( - raw.get("loop_max_attempts_per_requirement") or 6 - ), - loop_strategy_change_after=int(raw.get("loop_strategy_change_after") or 2), - ) - - def applies_to(self, task_type: str) -> bool: - return not self.task_types or str(task_type or "chat") in self.task_types - - -def builtin_profile() -> AgentProfile: - """The deployment default. - - Reproduces today's behaviour exactly — no tool restriction, no forced skill - set, the retrieval budget and turn cap the runtime already uses — so - installing profile resolution changes nothing until a profile is published. - """ - return AgentProfile( - profile_id="builtin", - version="v0", - tool_allowlist=None, - memory_budget_ms=600, - # The main agent's runtime default, reproduced exactly — which is now - # "no cap". This field only bounds the loop when a published profile - # sets it on purpose. - max_react_turns=UNBOUNDED_REACT_TURNS, - budget_multiplier=1.0, - loop_max_attempts_per_requirement=6, - loop_strategy_change_after=2, - intervention_rules=[ - InterventionRule(SIGNAL_NO_PROGRESS, 2, ACTION_CHANGE_STRATEGY), - InterventionRule(SIGNAL_REPEATED_ACTIONS, 3, ACTION_NARROW_SCOPE), - InterventionRule(SIGNAL_TOOL_ERROR_STREAK, 3, ACTION_CHANGE_STRATEGY), - InterventionRule(SIGNAL_BUDGET_LOW, 1, ACTION_ESCALATE), - ], - ) - - -def validate_profile( - profile: AgentProfile, - *, - base_tools: Optional[Sequence[str]] = None, - known_skill_ids: Optional[Sequence[str]] = None, - known_agent_ids: Optional[Sequence[str]] = None, -) -> Tuple[bool, List[str]]: - """Structural and security validation. - - ``base_tools`` is the authority this profile is allowed to draw from. A - profile naming a tool outside it is refused — orchestration evolution may - narrow authority and may never widen it, and this is the check that makes - "narrow only" a property of the system rather than an intention. - """ - problems: List[str] = [] - - if not profile.profile_id: - problems.append("profile_id 不能为空") - - if profile.tool_allowlist is not None and base_tools is not None: - widened = sorted(set(profile.tool_allowlist) - set(base_tools)) - if widened: - problems.append(f"工具授权越界（只允许收敛，不允许扩权）: {widened}") - - if known_skill_ids is not None: - unknown = sorted(set(profile.skill_ids) - set(known_skill_ids)) - if unknown: - problems.append(f"引用了不存在的技能: {unknown}") - - seen_agents = set() - for route in profile.subagent_routes: - if not route.agent_id: - problems.append("子智能体路由缺少 agent_id") - continue - if route.agent_id in seen_agents: - problems.append(f"重复的子智能体路由: {route.agent_id}") - seen_agents.add(route.agent_id) - if known_agent_ids is not None and route.agent_id not in set(known_agent_ids): - problems.append(f"路由指向不存在的子智能体: {route.agent_id}") - - for rule in profile.intervention_rules: - if rule.signal not in ALLOWED_SIGNALS: - problems.append(f"未知信号 {rule.signal}") - if rule.action not in ALLOWED_ACTIONS: - problems.append(f"未知动作 {rule.action}") - if rule.threshold < 1: - problems.append(f"{rule.signal} 阈值必须 ≥1（0 会让干预立刻触发）") - - if not (MIN_BUDGET_MULTIPLIER <= profile.budget_multiplier <= MAX_BUDGET_MULTIPLIER): - problems.append( - f"预算倍数 {profile.budget_multiplier} 超出 " - f"[{MIN_BUDGET_MULTIPLIER}, {MAX_BUDGET_MULTIPLIER}]" - ) - if profile.max_react_turns != UNBOUNDED_REACT_TURNS and not ( - MIN_REACT_TURNS <= profile.max_react_turns <= MAX_REACT_TURNS - ): - problems.append( - f"最大轮数 {profile.max_react_turns} 超出 " - f"[{MIN_REACT_TURNS}, {MAX_REACT_TURNS}]（{UNBOUNDED_REACT_TURNS} 表示不限）" - ) - if not (MIN_MEMORY_BUDGET_MS <= profile.memory_budget_ms <= MAX_MEMORY_BUDGET_MS): - problems.append( - f"记忆检索预算 {profile.memory_budget_ms}ms 超出 " - f"[{MIN_MEMORY_BUDGET_MS}, {MAX_MEMORY_BUDGET_MS}]" - ) - if not ( - MIN_LOOP_ATTEMPTS - <= profile.loop_max_attempts_per_requirement - <= MAX_LOOP_ATTEMPTS - ): - problems.append( - f"单需求最大尝试次数 {profile.loop_max_attempts_per_requirement} 超出 " - f"[{MIN_LOOP_ATTEMPTS}, {MAX_LOOP_ATTEMPTS}]" - ) - if profile.loop_strategy_change_after < 1: - problems.append("换策略阈值必须 ≥1（0 会让每一轮都换策略）") - - return (not problems), problems - - -# ── Resolution ─────────────────────────────────────────────────────────────── - - -def _scope_rank(scope: Dict[str, str], *, user_id: str, tenant_id: str) -> Optional[int]: - """How specific this profile is to the subject, or ``None`` if it is not. - - Lower is more specific. Returning ``None`` for a profile owned by another - subject is the whole point: it must not act as a fallback. - """ - owner = str((scope or {}).get("user_id") or "") - tenant = str((scope or {}).get("tenant_id") or "") - if owner: - return 0 if owner == user_id else None - if tenant: - return 1 if tenant == (tenant_id or "default") else None - return 2 - - -def pick_profile( - profiles: Sequence[AgentProfile], - *, - task_type: str, - user_id: str = "", - tenant_id: str = "default", -) -> Optional[AgentProfile]: - """The one profile that governs this run, or ``None``. - - A task-type-specific profile beats a general one at the same scope level: - the narrower statement is the more deliberate one. - """ - ranked: List[Tuple[int, int, AgentProfile]] = [] - for profile in profiles: - if not profile.applies_to(task_type): - continue - rank = _scope_rank(profile.scope, user_id=user_id, tenant_id=tenant_id) - if rank is None: - continue - ranked.append((rank, 0 if profile.task_types else 1, profile)) - if not ranked: - return None - ranked.sort(key=lambda item: (item[0], item[1])) - return ranked[0][2] - - -def _profile_from_row(row: Any) -> AgentProfile: - """The profile a stored row describes, with the columns believed over payload. - - ``profile_id`` / ``version`` / ``task_types`` / ``scope`` exist as real - columns *and* inside ``payload`` (:meth:`AgentProfile.to_dict` writes them), - so for anything published through :mod:`core.evolution.activation` the two - agree and this is a no-op. They stop agreeing for a row written by hand or - by a seeding script, and reading only ``payload`` then fails open in the - worst possible direction: a missing ``task_types`` parses as ``[]``, which - :meth:`AgentProfile.applies_to` treats as *every* task type, and a missing - ``scope`` parses as ``{}``, which :func:`_scope_rank` treats as *every* - subject. A profile published for one task type silently governs every run — - reported under the id ``builtin``, because that is what - :meth:`AgentProfile.from_dict` falls back to, so the log line naming the - culprit names the deployment default instead. - - The two narrowing fields resolve to whichever side is non-empty rather than - to the column unconditionally: an empty column against a populated payload - is the same widening failure with the sides swapped. When both are populated - and disagree the column wins — it is what the activation uniqueness check and - the console read — and the disagreement is logged, because it means something - wrote this row outside the activation path. - """ - raw = dict(row.payload or {}) - - # Identity: the column is the primary key and NOT NULL, so it is always the - # more trustworthy of the two, and a wrong id here is what makes a bad - # profile untraceable in the logs. - raw["profile_id"] = row.profile_id - raw["version"] = row.version - - col_types = [str(t) for t in (row.task_types or [])] - payload_types = [str(t) for t in (raw.get("task_types") or [])] - if col_types and payload_types and set(col_types) != set(payload_types): - logger.warning( - "[agent-profile] %s: task_types column %s != payload %s, using column", - row.profile_id, col_types, payload_types, - ) - raw["task_types"] = col_types or payload_types - - col_scope = dict(row.scope or {}) - payload_scope = dict(raw.get("scope") or {}) - if col_scope and payload_scope and col_scope != payload_scope: - logger.warning( - "[agent-profile] %s: scope column %s != payload %s, using column", - row.profile_id, col_scope, payload_scope, - ) - raw["scope"] = col_scope or payload_scope - - return AgentProfile.from_dict(raw) - - -def load_active_profile( - *, - task_type: str = "chat", - user_id: str = "", - tenant_id: str = "default", -) -> AgentProfile: - """The profile in force for this run. - - Read fresh on every call. Resolving once at import — which is what the old - orchestration policy did — means a published change needs a process restart - to take effect and different replicas disagree in the meantime, neither of - which is acceptable for an asset whose entire purpose is to change behaviour - between runs. - """ - from core.db.engine import SessionLocal - from core.db.models.evolution import EvolutionAgentProfile - - try: - with SessionLocal() as db: - rows = ( - db.query(EvolutionAgentProfile) - .filter(EvolutionAgentProfile.is_active.is_(True)) - .order_by(EvolutionAgentProfile.updated_at.desc()) - .all() - ) - profiles = [_profile_from_row(row) for row in rows] - except Exception as exc: # noqa: BLE001 - logger.warning("[agent-profile] load failed, using built-in: %s", exc) - return builtin_profile() - - chosen = pick_profile( - profiles, task_type=task_type, user_id=user_id, tenant_id=tenant_id - ) - if chosen is None: - return builtin_profile() - ok, problems = validate_profile(chosen) - if not ok: - # A stored profile that no longer validates is not applied. The - # alternative — applying it partially — produces a configuration nobody - # reviewed and nobody can reproduce. - logger.warning( - "[agent-profile] stored profile %s rejected at load: %s", - chosen.profile_id, - problems, - ) - return builtin_profile() - return chosen diff --git a/src/backend/core/evolution/assessment/__init__.py b/src/backend/core/evolution/assessment/__init__.py new file mode 100644 index 00000000..c3a8e199 --- /dev/null +++ b/src/backend/core/evolution/assessment/__init__.py @@ -0,0 +1 @@ +"""Evaluate evolution candidates: replay benchmarks, metrics and similarity.""" diff --git a/src/backend/core/evolution/applicability.py b/src/backend/core/evolution/assessment/applicability.py similarity index 98% rename from src/backend/core/evolution/applicability.py rename to src/backend/core/evolution/assessment/applicability.py index 37351eae..754a2e51 100644 --- a/src/backend/core/evolution/applicability.py +++ b/src/backend/core/evolution/assessment/applicability.py @@ -107,7 +107,7 @@ def assess_replay_coverage( task_ids: Sequence[str], ) -> Dict[str, Any]: """Whether a replay set covers what the pack claims. ``pack`` is a - :class:`~core.evolution.evidence_contract.CapabilityEvidencePackV2`. + :class:`~core.evolution.evidence.evidence_contract.CapabilityEvidencePackV2`. Required, each only when the pack actually contains that class of evidence: diff --git a/src/backend/core/evolution/benchmark.py b/src/backend/core/evolution/assessment/benchmark.py similarity index 98% rename from src/backend/core/evolution/benchmark.py rename to src/backend/core/evolution/assessment/benchmark.py index ac65aee5..936ee3ad 100644 --- a/src/backend/core/evolution/benchmark.py +++ b/src/backend/core/evolution/assessment/benchmark.py @@ -41,7 +41,7 @@ import hashlib import logging from dataclasses import dataclass, field -from typing import Any, Dict, List, Optional, Sequence, Set, Tuple +from typing import Any, Dict, List, Optional, Sequence, Tuple logger = logging.getLogger(__name__) @@ -211,7 +211,7 @@ def plan( proposed, replans = self._unaided(task) if ordering_rules: - from core.evolution.promotion import apply_ordering_constraints + from core.evolution.cycle.ordering import apply_ordering_constraints applicable = [ r for r in ordering_rules if r.applies_to(proposed, task.family) @@ -314,7 +314,7 @@ def to_dict(self) -> Dict[str, Any]: def compare(before: ArmSummary, after: ArmSummary) -> Dict[str, Any]: """Compare arms, including the checks that can invalidate the result.""" - from core.evolution.replay import mcnemar_p_value + from core.evolution.evidence.replay import mcnemar_p_value by_before = {r.task_id: r for r in before.results} by_after = {r.task_id: r for r in after.results} diff --git a/src/backend/core/evolution/evaluator.py b/src/backend/core/evolution/assessment/evaluator.py similarity index 100% rename from src/backend/core/evolution/evaluator.py rename to src/backend/core/evolution/assessment/evaluator.py diff --git a/src/backend/core/evolution/experiments.py b/src/backend/core/evolution/assessment/experiments.py similarity index 99% rename from src/backend/core/evolution/experiments.py rename to src/backend/core/evolution/assessment/experiments.py index cc6af77c..21eb841d 100644 --- a/src/backend/core/evolution/experiments.py +++ b/src/backend/core/evolution/assessment/experiments.py @@ -26,7 +26,7 @@ import logging from dataclasses import dataclass, field -from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple +from typing import Any, Dict, List, Optional, Sequence, Tuple logger = logging.getLogger(__name__) diff --git a/src/backend/core/evolution/fault_injection.py b/src/backend/core/evolution/assessment/fault_injection.py similarity index 98% rename from src/backend/core/evolution/fault_injection.py rename to src/backend/core/evolution/assessment/fault_injection.py index 2c37de26..6535060c 100644 --- a/src/backend/core/evolution/fault_injection.py +++ b/src/backend/core/evolution/assessment/fault_injection.py @@ -38,20 +38,7 @@ from dataclasses import dataclass, field from typing import Any, Callable, Dict, List, Optional, Tuple -from core.evolution.credit import ( - ALL_TARGETS, - NON_ASSET_TARGETS, - T_ENVIRONMENT, - T_MEMORY, - T_MODEL, - T_NO_UPDATE, - T_ONTOLOGY, - T_RETRIEVAL, - T_SKILL, - T_ORCHESTRATION, - CreditFeatures, - assign_credit, -) +from core.evolution.evidence.credit import NON_ASSET_TARGETS, T_ENVIRONMENT, T_MEMORY, T_MODEL, T_NO_UPDATE, T_ONTOLOGY, T_RETRIEVAL, T_SKILL, T_ORCHESTRATION, CreditFeatures, assign_credit logger = logging.getLogger(__name__) diff --git a/src/backend/core/evolution/metrics.py b/src/backend/core/evolution/assessment/metrics.py similarity index 100% rename from src/backend/core/evolution/metrics.py rename to src/backend/core/evolution/assessment/metrics.py diff --git a/src/backend/core/evolution/similarity.py b/src/backend/core/evolution/assessment/similarity.py similarity index 98% rename from src/backend/core/evolution/similarity.py rename to src/backend/core/evolution/assessment/similarity.py index e49d34c3..8e033e94 100644 --- a/src/backend/core/evolution/similarity.py +++ b/src/backend/core/evolution/assessment/similarity.py @@ -30,9 +30,8 @@ import logging import math import re -from collections import defaultdict from dataclasses import dataclass, field -from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple +from typing import Any, Dict, List, Optional, Sequence, Tuple logger = logging.getLogger(__name__) @@ -219,7 +218,7 @@ def _normalise(text: str) -> str: return _PUNCT.sub("", str(text or "")).lower() -def _ngrams(text: str, n: int = 3) -> set: +def ngrams(text: str, n: int = 3) -> set: """Character n-grams. Character-level on purpose: word segmentation for Chinese needs a tokeniser @@ -239,7 +238,7 @@ def lexical_similarity(a: str, b: str, n: int = 3) -> float: and concept names rather than whole questions — want ``n=2``: trigrams over a four-character name leave too few grams to overlap on. """ - ga, gb = _ngrams(a, n), _ngrams(b, n) + ga, gb = ngrams(a, n), ngrams(b, n) if not ga or not gb: return 0.0 intersection = len(ga & gb) @@ -267,7 +266,7 @@ def _embed_all(texts: Sequence[str]) -> Optional[List[List[float]]]: artefact of which calls happened to succeed. """ try: - from core.kb.kb_vector import embed_batch + from core.models.embedding import embed_batch vectors = embed_batch(list(texts)) except Exception as exc: # noqa: BLE001 diff --git a/src/backend/core/evolution/sop_judge.py b/src/backend/core/evolution/assessment/sop_judge.py similarity index 98% rename from src/backend/core/evolution/sop_judge.py rename to src/backend/core/evolution/assessment/sop_judge.py index 28268fe0..f83dc8fb 100644 --- a/src/backend/core/evolution/sop_judge.py +++ b/src/backend/core/evolution/assessment/sop_judge.py @@ -10,7 +10,7 @@ * 把这套做法固化下来照做，可能有规则看不见的**明显风险**。 所以在规则幸存者上加一道 LLM 裁决。边界与技能编译的「两段式生成」一致 -（见 :mod:`core.evolution.skill_gen`）：模型只回答「这值不值得沉淀为 SOP」 +（见 :mod:`core.evolution.generators.skill_gen`）：模型只回答「这值不值得沉淀为 SOP」 并给出理由和触发条件，它不能扩权、不能改工具序列、不能凭空造资产——那些 仍由代码与 IR 不变量把守。裁决结论随候选落库，评审员能看到机器放行的理由。 diff --git a/src/backend/core/evolution/backfill.py b/src/backend/core/evolution/backfill.py index 0a51bd02..c831e481 100644 --- a/src/backend/core/evolution/backfill.py +++ b/src/backend/core/evolution/backfill.py @@ -22,7 +22,7 @@ from core.db.engine import SessionLocal from core.db.usage_filters import model_usage_message_filter -from core.evolution.trace_assembler import assemble_episode +from core.evolution.evidence.trace_assembler import assemble_episode logger = logging.getLogger(__name__) @@ -108,7 +108,7 @@ def _outcome_from_logs(db, message_id: str, chat_id: str) -> Optional[Dict[str, apart from an explicit user judgement and weight it accordingly. """ from core.db.models import ChatMessage - from core.evolution.trace_assembler import _tool_rows_for + from core.evolution.evidence.trace_assembler import _tool_rows_for rows = _tool_rows_for(db, message_id, chat_id) if not rows: @@ -154,7 +154,6 @@ def backfill_episodes( start = end - timedelta(days=days) stats = {"scanned": 0, "created": 0, "skipped": 0, "batches": 0} - offset = 0 while True: if max_batches is not None and stats["batches"] >= max_batches: @@ -174,8 +173,8 @@ def backfill_episodes( # Historical runs have tool logs but no trace events; # bridge them so the backfill is actually usable for mining. - from core.evolution.events import TraceSink - from core.evolution.trace_assembler import emit_tool_events + from core.evolution.evidence.events import TraceSink + from core.evolution.evidence.trace_assembler import emit_tool_events sink = TraceSink(message_id=message_id, chat_id=chat_id or "", user_id=user_id) if emit_tool_events(sink, message_id, chat_id or ""): diff --git a/src/backend/core/evolution/contract.py b/src/backend/core/evolution/contract.py index b56518d0..231117c9 100644 --- a/src/backend/core/evolution/contract.py +++ b/src/backend/core/evolution/contract.py @@ -20,10 +20,10 @@ import hashlib import json -from dataclasses import asdict, dataclass, field +from dataclasses import dataclass, field from typing import Any, Dict, List, Mapping, Optional, Tuple -from core.immutable import FrozenDict, freeze_json, thaw_json +from core.infra.immutable import FrozenDict, freeze_json, thaw_json # The four governed asset classes, plus the ones that turned out to matter in a # multi-provider / multi-KB deployment. Prompt is included because this project diff --git a/src/backend/core/evolution/cycle/__init__.py b/src/backend/core/evolution/cycle/__init__.py new file mode 100644 index 00000000..fbdf0bdd --- /dev/null +++ b/src/backend/core/evolution/cycle/__init__.py @@ -0,0 +1 @@ +"""The evolution cycle: mine episodes, propose candidates, evaluate and release them.""" diff --git a/src/backend/core/evolution/cycle/activation.py b/src/backend/core/evolution/cycle/activation.py new file mode 100644 index 00000000..b92c3f66 --- /dev/null +++ b/src/backend/core/evolution/cycle/activation.py @@ -0,0 +1,353 @@ +"""Materialise an approved skill candidate at the release stage it has earned.""" +from __future__ import annotations +from datetime import datetime, timezone +from typing import Any, Dict, List, Optional +from core.db.engine import SessionLocal +import logging +from core.evolution.cycle import activation_policy +from core.evolution.cycle import activation_constants +from core.evolution.cycle import activation_types +from core.evolution.cycle import skill_document + +logger = logging.getLogger(__name__) + +def materialise_skill_candidate( + candidate_id: str, + *, + approver: str, + force: bool = False, + owner_user_id: Optional[str] = None, + tenant_id: str = "default", +) -> Dict[str, Any]: + """Materialise an approved skill candidate, at the stage it has earned. + + This is the only path by which an evolution artefact reaches the agent, and + it now enters the release pipeline rather than bypassing it: + + * ``low`` risk (verification = replay only) goes live at 100%; + * ``medium`` / ``high`` enter **shadow**, reaching nobody, and must be ramped + through canary by :func:`advance_skill_release`; + * a personal activation (``owner_user_id``) goes live scoped to that user. + + ``force`` waives the replay *stage* and only for ``low`` risk. It never + waives separation of duties — an override is exactly what an attacker or an + over-eager script would reach for. + + ``owner_user_id`` scopes the resulting skill to one person. A personal + activation cannot change what anyone else's agent does, which is what lets a + user accept a capability distilled from their own conversations without an + administrator standing in the way. + """ + from core.db.models import AdminSkill + from core.db.models.evolution import EvolutionCandidate, EvolutionRelease + from core.evolution.cycle.release import ACTIVE, IN_FLIGHT_STAGES, REPLAY_PASSED, new_release_id + + with SessionLocal() as db: + candidate = db.get(EvolutionCandidate, candidate_id) + if candidate is None: + raise activation_types.ActivationError("candidate_not_found", f"候选不存在: {candidate_id}") + if candidate.target_kind != "skill": + raise activation_types.ActivationError( + "unsupported_kind", f"暂不支持物化 {candidate.target_kind} 候选" + ) + + # Separation of duties. No force path — see the docstring. + if approver and approver == (candidate.proposer or ""): + raise activation_types.ActivationError( + "self_approval_forbidden", "生成者不能激活自己提出的候选" + ) + if not approver: + raise activation_types.ActivationError("approver_required", "激活必须署名") + + stage, traffic = activation_policy.entry_stage( + candidate, approver=approver, force=force, owner_user_id=owner_user_id + ) + # Pace and freeze apply to anything that becomes live immediately. A + # shadow entry changes nothing users can see, so holding it back would + # only delay measurement. + if stage == ACTIVE and not owner_user_id: + activation_policy.check_pace(db, tenant_id=tenant_id) + + ir = candidate.ir or {} + changes = ir.get("changes") or [] + tools: List[str] = [] + constraints: List[Dict[str, Any]] = [] + document: Optional[Dict[str, Any]] = None + manifest: Optional[Dict[str, Any]] = None + # ref → owning user. Demotion happens inside one person's store, so a + # skill compiled from several people's memories needs the owner of each. + source_memories: Dict[str, str] = {} + for change in changes: + sequence = change.get("tool_sequence") + if isinstance(sequence, list) and sequence and not tools: + tools = [str(t) for t in sequence] + rules = change.get("ordering_constraints") + if isinstance(rules, list) and rules: + constraints = rules + written = change.get("document") + if isinstance(written, dict) and written.get("content"): + document = written + declared = change.get("manifest") + if isinstance(declared, dict) and declared: + manifest = declared + for item in change.get("source_memories") or []: + if isinstance(item, dict) and item.get("ref"): + source_memories[str(item["ref"])] = str(item.get("user_id") or "") + + # 图谱证据自相矛盾的候选可以存在、可以进 shadow 观测，但不允许自动 + # 走到 ACTIVE：冲突意味着「适用条件」本身不可信，必须先由人裁决。 + if manifest and manifest.get("graph_conflict") and stage == ACTIVE: + raise activation_types.ActivationError( + "graph_conflict_requires_review", + f"L3 图谱证据存在冲突，禁止直接激活：{manifest.get('graph_conflicts')}", + ) + + skill_id = candidate.target_asset_id + if not skill_id.startswith(activation_constants.EVOLUTION_PREFIX): + skill_id = activation_constants.EVOLUTION_PREFIX + skill_id.replace("auto-", "") + if owner_user_id: + # Distinct id per owner: a personal activation must never overwrite + # the global asset, nor another user's copy of the same candidate. + skill_id = f"{skill_id}-u{owner_user_id[-8:]}" + + if document is not None: + # A written skill: the generator produced the body and the machine + # validator already refused everything it was told to refuse. It is + # re-checked here against this candidate's id, because the id gains + # a prefix (and, for a personal activation, an owner suffix) after + # the draft was validated — and an id the loader rejects yields a + # skill that shows up in the capability list and loads as nothing. + from core.evolution.generators.skill_drafting import render_skill_markdown, validate_draft + + tools = [str(t) for t in (document.get("allowed_tools") or [])] + display_name = str(document.get("display_name") or "") + description = str(document.get("description") or "") + body = skill_document.body_of(str(document.get("content") or "")) + violations = validate_draft( + skill_id=skill_id, + display_name=display_name, + description=description, + body=body, + allowed_tools=tools, + ) + if violations: + raise activation_types.ActivationError( + "draft_failed_validation", + f"技能正文未通过机器校验，拒绝物化: {violations}", + ) + rules_only = False + content = render_skill_markdown( + skill_id=skill_id, + display_name=display_name, + description=description, + body=body, + support=len(candidate.evidence_refs or []), + rationale=candidate.hypothesis or "", + ) + else: + # A candidate carrying only ordering rules is still a capability, + # and on real traffic it is the *common* case: five runs repeating + # one identical tool sequence is rare, while the same pairwise + # ordering recurring across different tool sets is not. + rules_only = not tools and bool(constraints) + if rules_only: + tools = skill_document.tools_from_constraints(constraints) + if not tools: + raise activation_types.ActivationError( + "no_tool_sequence", "候选既未声明工具顺序、也未携带顺序约束，无法物化" + ) + display_name = skill_document.skill_label(tools, rules_only=rules_only) + description = skill_document.skill_description(tools, constraints, rules_only=rules_only) + content = skill_document.skill_markdown( + skill_id=skill_id, + label=display_name, + description=description, + tools=tools, + evidence=len(candidate.evidence_refs or []), + constraints=constraints, + rationale=candidate.hypothesis or "", + rules_only=rules_only, + ) + + previous = activation_policy.capture_previous(db, skill_id) + + # One release in flight per asset. Two concurrent ramps on the same skill + # make any measured effect unattributable and leave rollback without a + # single target. + conflicting = ( + db.query(EvolutionRelease) + .filter( + EvolutionRelease.target_asset_id == skill_id, + EvolutionRelease.stage.in_(list(IN_FLIGHT_STAGES)), + ) + .first() + ) + if conflicting is not None: + raise activation_types.ActivationError( + "release_in_flight", + f"该资产已有进行中的发布 {conflicting.release_id}（{conflicting.stage}）", + ) + + row = db.query(AdminSkill).filter(AdminSkill.skill_id == skill_id).first() + if row is None: + row = AdminSkill(skill_id=skill_id, created_by=f"evolution:{approver}") + db.add(row) + row.owner_user_id = owner_user_id or None + + row.skill_content = content + row.display_name = display_name + row.description = description + row.version = activation_policy.next_version(previous) + row.tags = sorted(set((previous or {}).get("tags", [])) | {activation_constants.EVOLUTION_TAG}) + # The distilled ordering, in the shape the runtime reads. + extra_files = { + **(previous or {}).get("extra_files", {}), + "tool_sequence.json": activation_policy.json(tools), + # The transferable part. A consumer that only understands sequences + # still works; one that understands rules gets the wider benefit. + "ordering_constraints.json": activation_policy.json(constraints), + } + if manifest: + # 机器读 manifest（边界/依赖/证据引用），Agent 读 SKILL.md —— 两者 + # 职责分离。作用域以本次激活实际发生的为准，而不是候选申报的。 + extra_files["evolution_manifest.json"] = activation_policy.json( + { + **manifest, + "scope": "personal" if owner_user_id else str( + manifest.get("scope") or "workspace" + ), + } + ) + row.extra_files = extra_files + row.allowed_tools = tools + row.is_enabled = True + + # The scope the IR asked for, now actually enforced at load time by + # core.evolution.evidence.exposure. Recording it here rather than deriving it + # later means a scope decision is auditable against the release it + # governs. + scope_filter: Dict[str, Any] = {"level": (ir.get("scope") or {}).get("level") or "workspace"} + if owner_user_id: + scope_filter["owner_user_id"] = owner_user_id + else: + scope_filter["tenant_id"] = tenant_id or "default" + + release = EvolutionRelease( + release_id=new_release_id(), + candidate_id=candidate_id, + target_kind="skill", + target_asset_id=skill_id, + stage=stage, + # A personal activation reaches exactly one user, so recording it as + # full traffic would misreport the blast radius on the release board. + traffic_percent=traffic, + scope_filter=scope_filter, + # Written now, not derived later: rollback needs a target that + # existed before anything changed. + rollback_version_id=(previous or {}).get("version") or "", + guardrails={ + "previous": previous, + # Which verification stages are behind us. Read by + # advance_skill_release before every hop. + "completed_verification": ( + ["replay"] if candidate.status in (REPLAY_PASSED, ACTIVE) else [] + ), + "proposer": candidate.proposer or "", + }, + approved_by=approver, + ) + db.add(release) + + # A personal activation leaves the candidate available for others to + # accept, and for an administrator to consider globally. Marking it + # ACTIVE would end its life for everyone on one person's decision. + # A shadow entry is likewise not the end of the candidate's life. + if not owner_user_id and stage == ACTIVE: + candidate.status = ACTIVE + candidate.approved_by = approver + candidate.approved_at = datetime.now(timezone.utc) + db.commit() + + release_id = release.release_id + new_version = row.version + candidate_owner = str((ir.get("scope") or {}).get("user_id") or "") + logger.info( + "[activation] materialised %s from %s at %s/%d%% (tools=%s written=%s)", + skill_id, + candidate_id, + stage, + traffic, + tools, + document is not None, + ) + + # Close the promotion chain: the memories this skill was compiled from now + # have a cheaper carrier, so they stop consuming the retrieval budget that + # made compiling them worthwhile. Outside the transaction above because it + # writes to its own ledger, and down-weighted rather than removed, so + # rolling the skill back restores them exactly. + demoted: List[str] = [] + if source_memories: + from core.evolution.generators.memory_apply import demote_promoted_memories + + by_owner: Dict[str, List[str]] = {} + for ref, owner in source_memories.items(): + # ``owner_user_id`` for a personal activation, the ref's recorded + # owner otherwise. A ref with neither is skipped rather than guessed + # at: applying a change to the wrong person's memories is worse than + # not applying it. + resolved = owner_user_id or owner or candidate_owner + if resolved: + by_owner.setdefault(resolved, []).append(ref) + + for owner, refs in by_owner.items(): + outcome = demote_promoted_memories( + candidate_id=candidate_id, + user_id=owner, + memory_refs=refs, + skill_id=skill_id, + ) + demoted.extend(op["memory_ref"] for op in outcome.get("applied", [])) + + # 血缘落账：这一版技能由哪些 L2 记忆编译（compiled_from）、被哪些 L3 关系 + # 约束（constrained_by）、由哪些 Episode 验证（validated_by）。尽力而为—— + # 账本失败绝不回滚一次已经成立的激活。 + lineage_written = 0 + try: + from core.evolution.evidence import lineage + + with SessionLocal() as db: + refreshed = db.get(EvolutionCandidate, candidate_id) + pack_id = str(getattr(refreshed, "evidence_pack_id", "") or "") + pack = lineage.load_evidence_pack(pack_id) if pack_id else None + if pack is not None: + lineage_written = len( + lineage.record_links( + lineage.links_for_skill( + pack, skill_id=skill_id, skill_version=new_version + ), + candidate_id=candidate_id, + ) + ) + except Exception as exc: # noqa: BLE001 + logger.debug("[activation] lineage recording skipped: %s", exc) + + return { + "skill_id": skill_id, + "release_id": release_id, + "stage": stage, + "traffic_percent": traffic, + "scope": scope_filter, + "demoted_memories": demoted, + "lineage_links": lineage_written, + # Spelled out because "activated" previously implied "live for + # everyone", and for anything above low risk it no longer does. + "exposed_to": ( + "all users in scope" + if stage == ACTIVE and not owner_user_id + else (f"only {owner_user_id}" if owner_user_id else "nobody (shadow)") + ), + "tools": tools, + "version": new_version, + "replaced_previous": previous is not None, + } diff --git a/src/backend/core/evolution/cycle/activation_constants.py b/src/backend/core/evolution/cycle/activation_constants.py new file mode 100644 index 00000000..1831d53a --- /dev/null +++ b/src/backend/core/evolution/cycle/activation_constants.py @@ -0,0 +1,11 @@ +"""Tags and config keys that mark evolution-generated assets.""" +from __future__ import annotations + + +EVOLUTION_TAG = "evolved" + + +EVOLUTION_PREFIX = "evo-" + + +FREEZE_CONFIG_KEY = "evolution_frozen" diff --git a/src/backend/core/evolution/cycle/activation_policy.py b/src/backend/core/evolution/cycle/activation_policy.py new file mode 100644 index 00000000..c04bec0a --- /dev/null +++ b/src/backend/core/evolution/cycle/activation_policy.py @@ -0,0 +1,196 @@ +"""Gates on activation: freeze switch, pacing, entry stage and rollback snapshots.""" +from __future__ import annotations +from datetime import datetime, timezone +from typing import Any, Dict, Optional, Tuple +from core.evolution.cycle import activation_constants +from core.evolution.cycle import activation_types + +def capture_previous(db, skill_id: str) -> Optional[Dict[str, Any]]: + """Snapshot the current state so rollback has a concrete target.""" + from core.db.models import AdminSkill + + row = db.query(AdminSkill).filter(AdminSkill.skill_id == skill_id).first() + if row is None: + return None + return { + "skill_id": row.skill_id, + "skill_content": row.skill_content, + "display_name": row.display_name, + "description": row.description, + "version": row.version, + "tags": list(row.tags or []), + "allowed_tools": list(row.allowed_tools or []), + "extra_files": dict(row.extra_files or {}), + "is_enabled": bool(row.is_enabled), + } + + +def is_frozen(db) -> bool: + """Global stop switch, readable without a redeploy. + + An incident needs a way to stop every layer activating *now*, so it lives in + ``system_configs`` rather than in an environment variable. + """ + try: + from core.db.models import SystemConfig + + row = ( + db.query(SystemConfig.config_value) + .filter(SystemConfig.config_key == activation_constants.FREEZE_CONFIG_KEY) + .first() + ) + return bool(row) and str(row[0]).strip().lower() in ("1", "true", "yes", "on") + except Exception: # noqa: BLE001 - absence of the row means "not frozen" + return False + + +def skill_layer_state(db) -> Tuple[Optional[datetime], int]: + """When a skill last went live, and how many releases are mid-flight. + + Read from the release ledger rather than kept in memory: a per-process + limiter resets on every restart and multiplies by the number of replicas, + which is exactly how a cooldown quietly stops existing. + """ + from core.db.models.evolution import EvolutionRelease + from core.evolution.cycle.release import ACTIVE, IN_FLIGHT_STAGES + + last_row = ( + db.query(EvolutionRelease.created_at) + .filter( + EvolutionRelease.target_kind == "skill", + EvolutionRelease.stage == ACTIVE, + ) + .order_by(EvolutionRelease.created_at.desc()) + .first() + ) + in_flight = ( + db.query(EvolutionRelease) + .filter(EvolutionRelease.stage.in_(list(IN_FLIGHT_STAGES))) + .count() + ) + last = last_row[0] if last_row else None + if last is not None and last.tzinfo is None: + last = last.replace(tzinfo=timezone.utc) + return last, int(in_flight or 0) + + +def check_pace(db, *, tenant_id: str, layer: str = "skill") -> None: + """Enforce the multi-timescale gate the design declares. + + Skills sit on an hours-to-days cadence so the system does not become + non-stationary. That was a documented number with no enforcement: nothing + consulted ``timescales`` before writing a live asset, so ten activations + could land in one minute and no metric movement afterwards would be + attributable to any of them. + """ + from core.evolution.cycle.timescales import RateLimiter + + last, in_flight = skill_layer_state(db) + limiter = RateLimiter(frozen=is_frozen(db)) + if last is not None: + limiter.last_activation[layer] = last + allowed, reason = limiter.may_activate( + layer, tenant=tenant_id or "default", concurrent_activations=in_flight + ) + if not allowed: + raise activation_types.ActivationError("pace_limited", f"节奏闸门拒绝本次激活：{reason}") + + +def entry_stage( + candidate, + *, + approver: str, + force: bool, + owner_user_id: Optional[str], +) -> Tuple[str, int]: + """The stage this candidate has actually earned, and its traffic share. + + Previously every activation wrote ``active`` at 100%, whatever the risk tier + said. ``ir.MIN_VERIFICATION`` declares that a high-risk change owes replay, + shadow, canary *and* human approval before going live, and a brand-new + skill is exactly that tier — so the old behaviour skipped two of the four + stages its own IR demanded. + + A personal activation is the one case that legitimately goes straight live: + its blast radius is one consenting user, and the ramp exists to bound blast + radius. It is recorded as scoped rather than as full traffic. + """ + from core.evolution.ir import RISK_LOW, required_verification + from core.evolution.cycle.release import ACTIVE, REPLAY_PASSED, SHADOW + + risk = str(candidate.risk_tier or "").strip() or RISK_LOW + replayed = candidate.status in (REPLAY_PASSED, ACTIVE) + + if not replayed: + # ``force`` waives the replay *stage* — and only for the tier whose + # verification requirement is replay alone. Waiving it for a tier that + # also owes shadow and canary is not a shortcut, it is the whole gate. + if not force: + raise activation_types.ActivationError( + "verification_incomplete", + f"候选尚未通过回放（当前 {candidate.status}）", + ) + if risk != RISK_LOW: + raise activation_types.ActivationError( + "force_forbidden_for_risk", + f"force 只允许用于 low 风险候选，本候选为 {risk}；" + f"需完成 {list(required_verification(risk))}", + ) + + if owner_user_id: + return ACTIVE, 0 + + if set(required_verification(risk)) <= {"replay"}: + return ACTIVE, 100 + + # Higher tiers enter at shadow: the asset exists and is measurable, and + # reaches nobody until an operator ramps it. + return SHADOW, 0 + + +def has_graph_conflict(candidate) -> bool: + """Whether the candidate's manifest flags contradictory L3 evidence.""" + if candidate is None: + return False + for change in (getattr(candidate, "ir", None) or {}).get("changes") or []: + manifest = change.get("manifest") if isinstance(change, dict) else None + if isinstance(manifest, dict) and manifest.get("graph_conflict"): + return True + return False + + +def next_version(previous: Optional[Dict[str, Any]]) -> str: + if not previous: + return "1.0.0" + parts = str(previous.get("version") or "1.0.0").split(".") + try: + parts[-1] = str(int(parts[-1]) + 1) + return ".".join(parts) + except Exception: + return "1.0.1" + + +def json(value: Any) -> str: + import json + + return json.dumps(value, ensure_ascii=False) + + +def source_candidate_for(db, skill_id: str) -> str: + """The candidate that first materialised this skill. + + Memory demotions are recorded against that candidate, so it is the key that + finds them again when the skill is retired. + """ + from core.db.models.evolution import EvolutionRelease + + row = ( + db.query(EvolutionRelease) + .filter( + EvolutionRelease.target_kind == "skill", + EvolutionRelease.target_asset_id == skill_id, + ) + .order_by(EvolutionRelease.created_at.asc()) + .first() + ) + return str(row.candidate_id) if row is not None else "" diff --git a/src/backend/core/evolution/cycle/activation_types.py b/src/backend/core/evolution/cycle/activation_types.py new file mode 100644 index 00000000..f9f6f31a --- /dev/null +++ b/src/backend/core/evolution/cycle/activation_types.py @@ -0,0 +1,9 @@ +"""Error raised when a candidate cannot be activated.""" +from __future__ import annotations + + +class ActivationError(Exception): + def __init__(self, code: str, message: str): + super().__init__(message) + self.code = code + self.message = message diff --git a/src/backend/core/evolution/cycle/candidate_records.py b/src/backend/core/evolution/cycle/candidate_records.py new file mode 100644 index 00000000..ee136888 --- /dev/null +++ b/src/backend/core/evolution/cycle/candidate_records.py @@ -0,0 +1,197 @@ +"""Persist candidates and derive their attribution features.""" +from __future__ import annotations +from core.evolution.cycle import patterns +from core.evolution.cycle import promotion_constants +from typing import Any, Dict, List, Optional +from core.db.engine import SessionLocal +from core.db.models.evolution import EvolutionCandidate +from core.evolution.evidence.credit import CreditFeatures +from core.evolution.ir import EvolutionIR, new_candidate_id +from core.evolution.cycle.release import DRAFT +import logging +logger = logging.getLogger(__name__) + +def features_for(pattern: patterns.Pattern) -> CreditFeatures: + """Translate a discovered pattern into attribution features. + + Success-subsequence tool patterns no longer come through here: they are an + aggregate observation over successful runs, and inventing a ``failed`` + verdict to route them through the failure attributor is the exact move + :func:`core.evolution.evidence.credit.aggregate_finding` exists to replace. + """ + if pattern.kind == promotion_constants.PATTERN_REPEATED_FAILURE: + return CreditFeatures( + outcome_verdict="failed", + outcome_confidence=0.8, + loop_stagnation=True, + no_diff_iterations=2, + ) + return CreditFeatures(outcome_verdict="unknown", outcome_confidence=0.3) + + +def features_for_ordering_rules(constraints: List[Any]) -> CreditFeatures: + """Attribution features for "a stable ordering keeps being re-derived". + + The mapping, stated so it can be argued with: + + * ``repeated_tool_subsequence`` — a pairwise ordering recurring across + different tool sets is a repeated successful subsequence that is + re-planned on every occurrence, which is what the feature means. + * ``outcome_verdict="failed"`` — the runs this would fix are the ones that + ordered the calls wrongly; the successful ones are the evidence, not the + target. + + ``step_order_suspect`` is deliberately *not* set even though ordering is the + subject. That feature says "this pipeline's step order looks wrong", a + workflow-level claim, and asserting it ties workflow with skill at 0.65 — + the assigner then correctly reports ``unidentifiable`` and nothing is + produced. The evidence here does separate them: an ordering that holds + across several different tool sets and task types is reusable knowledge, not + one fixed pipeline, and "the workflow is fixed" is the reading it rules out. + + Confidence sits just above the ``MIN_CONFIDENCE`` floor on purpose: with + support and context breadth at their minimum it lands below the floor and + resolves to No-Update, so thin evidence cannot mint a permanent asset. + """ + support = max((int(getattr(c, "support", 0) or 0) for c in constraints), default=0) + contexts = max((int(getattr(c, "contexts", 0) or 0) for c in constraints), default=0) + return CreditFeatures( + outcome_verdict="failed", + outcome_confidence=min(0.9, 0.6 + 0.05 * support + 0.05 * contexts), + repeated_tool_subsequence=True, + ) + + +def uncovered_constraints(constraints: List[Any], covered_tool_sets: List[Any]) -> List[Any]: + """Rules that no sequence candidate would carry into the runtime. + + A rule attached to a materialised sequence already reaches the agent through + that skill. The ones worth a candidate of their own are the rest — and on + real traffic that is most of them, because a rule earns its scope by + appearing in *several* tool-set shapes and no single sequence covers them + all. + """ + out: List[Any] = [] + for constraint in constraints: + pair = {constraint.before, constraint.after} + if any(pair <= set(tools) for tools in covered_tool_sets): + continue + out.append(constraint) + return out + + +def persist_candidate( + ir: EvolutionIR, + *, + credit: Dict[str, Any], + proposer: str, + evidence_refs: List[str], +) -> Optional[str]: + """Store a candidate, collapsing duplicates rather than failing on them.""" + from core.evolution.evidence import lineage + + # Attribution outlives the candidate that happened to carry it: every + # decision that reaches persistence gets its own ledger row, and the stored + # copy inside the candidate becomes a cache keyed by ``decision_id``. + decision_id = lineage.record_credit_decision( + credit, episode_id=(evidence_refs[0] if evidence_refs else "") + ) + credit = {**credit, "decision_id": decision_id} + + checksum = ir.change_checksum() + try: + with SessionLocal() as db: + existing = ( + db.query(EvolutionCandidate) + .filter( + EvolutionCandidate.target_kind == ir.target_kind, + EvolutionCandidate.target_asset_id == ir.target_asset_id, + EvolutionCandidate.base_version_id == (ir.base_version or None), + EvolutionCandidate.change_checksum == checksum, + ) + .first() + ) + if existing is not None: + # Same edit proposed again from fresh evidence: one review item, + # not two. + return existing.candidate_id + + candidate_id = new_candidate_id() + db.add( + EvolutionCandidate( + candidate_id=candidate_id, + target_kind=ir.target_kind, + target_asset_id=ir.target_asset_id, + base_version_id=ir.base_version or None, + operation=ir.operation, + ir=ir.to_dict(), + hypothesis=ir.hypothesis, + change_checksum=checksum, + evidence_refs=evidence_refs, + credit_decision=credit, + risk_tier=ir.risk_tier, + scope=ir.scope, + status=DRAFT, + proposer=proposer, + ) + ) + db.commit() + except Exception as exc: + logger.warning("[evolution-loop] persisting candidate failed: %s", exc) + return None + + # No backfill onto the turn cards. A skill candidate is a batch product + # reviewed in the console; pushing it back into a conversation from days ago + # put capability claims on a card whose only subject is what that turn + # remembered. The candidate's own evidence_refs still record which episodes + # produced it, which is where that link belongs. + return candidate_id + + +def cross_user_mining_available() -> bool: + """Whether fleet-wide mining may run in this edition. + + Personal evolution runs everywhere — it needs no aggregation across people + and its output reaches only its owner. Mining *across* users and releasing + to everyone is a governance surface, so it is enterprise-gated. The + community edition keeps a complete personal evolution path rather than a + degraded one. + """ + try: + # Resolved by name rather than a literal enterprise-tree import, so the + # community build's boundary scan does not read this file as reaching + # across the edition boundary. The behaviour is unchanged: the module is + # physically absent in CE and the resulting ImportError still lands in + # the handler below. + from importlib import import_module + + features = import_module("edition_ee.licensing.features") + manager = import_module("edition_ee.licensing.manager") + + return bool(manager.license_manager.has(features.Feature.EVOLUTION_CONTROL)) + except Exception: + # The community tree physically excludes edition_ee, so an import error + # here is the expected CE signal rather than a fault: personal only. + return False + + +def run_async(coro) -> Any: + """Run one coroutine from the cycle's synchronous body. + + The cycle is a background job and must run in a worker thread. Calling it + from inside a running loop is a programming error and is raised as one: + degrading to "skip the generators" would produce a cycle that silently + stopped writing skills, which is the failure mode this whole rework exists + to eliminate. + """ + import asyncio + + try: + asyncio.get_running_loop() + except RuntimeError: + return asyncio.run(coro) + coro.close() + raise RuntimeError( + "run_evolution_cycle must not be called from a running event loop; " + "dispatch it with run_in_executor" + ) diff --git a/src/backend/core/evolution/cycle/composition_candidates.py b/src/backend/core/evolution/cycle/composition_candidates.py new file mode 100644 index 00000000..c3793835 --- /dev/null +++ b/src/backend/core/evolution/cycle/composition_candidates.py @@ -0,0 +1,221 @@ +"""Retirement, merge, profile and sub-agent candidates.""" +from __future__ import annotations +from typing import Any, Dict, List +from core.evolution.evidence.credit import T_ORCHESTRATION, T_SKILL, aggregate_finding, confidence_from_evidence +from core.evolution.ir import RISK_HIGH, RISK_MEDIUM, EvolutionIR, InvariantViolation, Rollback, validate_ir +import logging +from core.evolution.cycle import candidate_records + +logger = logging.getLogger(__name__) + +def decremental_candidates(extras: Dict[str, Any], *, dry_run: bool) -> List[Dict[str, Any]]: + """Retirement and merge findings, as candidates rather than as a list. + + Retiring goes through the same approval path as creating. Removing a + capability produces no error — only a slow decline — so it is precisely the + direction that must not be automatic. + """ + from core.evolution.cycle.promotion_constants import ACTION_REWRITE + + decremental = extras.get("decremental") or {} + created: List[Dict[str, Any]] = [] + + for item in decremental.get("retirements") or []: + asset_id = str(item.get("asset_id") or "") + if not asset_id: + continue + action = str(item.get("action") or "") + if action == ACTION_REWRITE: + # "Opened and ignored" is a content problem. Proposing retirement + # would remove a capability the model wanted; the fix is a rewrite, + # and that is a different candidate written from failure evidence. + created.append({"kind": "needs_rewrite", "asset_id": asset_id, **item}) + continue + + decision = aggregate_finding( + target=T_SKILL, + explanation=str(item.get("explanation") or item.get("reason") or ""), + confidence=confidence_from_evidence(int(item.get("offers", 0) or 0), saturates_at=20), + evidence_count=int(item.get("offers", 0) or 0), + ) + if not decision.may_produce_candidate: + continue + + ir = EvolutionIR( + target_kind="skill", + target_asset_id=asset_id, + base_version="current", + operation="deprecate", + hypothesis=str(item.get("explanation") or item.get("reason") or ""), + changes=[ + { + "retire_reason": item.get("reason"), + "stats": { + k: v for k, v in item.items() if k not in ("asset_id", "explanation") + }, + } + ], + evidence_refs=[asset_id], + risk_tier=RISK_MEDIUM, + scope={"level": "workspace"}, + rollback=Rollback(version_id="current", triggers=["manual"]), + evaluation_plan={"replay_set": "auto_holdout", "primary_metric": "task_success"}, + ) + try: + validate_ir(ir, credit_decision=decision.to_dict()) + except InvariantViolation as violation: + logger.info("[evolution-loop] retirement rejected: %s", violation.code) + continue + if dry_run: + created.append({"dry_run": True, "kind": "retirement", "asset_id": asset_id}) + continue + candidate_id = candidate_records.persist_candidate( + ir, + credit=decision.to_dict(), + proposer="decremental_engine", + evidence_refs=[asset_id], + ) + if candidate_id: + created.append( + { + "candidate_id": candidate_id, + "kind": "retirement", + "asset_id": asset_id, + "reason": item.get("reason"), + } + ) + return created + + +def orchestration_candidates( + views: List[Dict[str, Any]], *, dry_run: bool +) -> List[Dict[str, Any]]: + """Profile and sub-agent candidates. + + The tool authority every candidate here is measured against is the *current* + grant, read once. A profile may only ever narrow it, and a derived sub-agent + may only ever receive a subset — both enforced by the IR gate rather than by + the generator, so the check holds even for a candidate that arrived by some + other route. + """ + from core.capabilities.catalog.api import get_enabled_ids + from core.evolution.profiles.agent_profile_constants import TARGET_KIND + from core.evolution.profiles.agent_profile import builtin_profile, validate_profile + from core.evolution.generators.orchestration_gen import generate_orchestration_proposals, profile_from_proposals + + base_tools = [str(t) for t in get_enabled_ids("tools")] + [ + str(t) for t in get_enabled_ids("mcp") + ] + offered_tools = {str(view.get("task_type") or "chat"): base_tools for view in views} + proposals, subagents = generate_orchestration_proposals( + views, offered_tools=offered_tools, parent_tools=base_tools + ) + created: List[Dict[str, Any]] = [] + + task_types = sorted({p.task_type for p in proposals}) + for task_type in task_types: + task_proposals = [p for p in proposals if p.task_type == task_type] + profile = profile_from_proposals( + task_proposals, task_type=task_type, base=builtin_profile() + ) + ok, problems = validate_profile(profile, base_tools=base_tools) + if not ok: + logger.info("[evolution-loop] profile %s rejected: %s", profile.profile_id, problems) + continue + + evidence = [ref for p in task_proposals for ref in p.evidence_refs][:20] + # An observation across the corpus, not a failure attributed to one run. + # Inventing failure features to make the attributor return "workflow" + # is what put "过程停滞：重复动作、无产出增量" in front of reviewers for a + # candidate whose actual reason was "eight tools were never used". + decision = aggregate_finding( + target=T_ORCHESTRATION, + explanation="；".join(p.rationale for p in task_proposals), + confidence=confidence_from_evidence(sum(p.support for p in task_proposals)), + evidence_count=len(evidence), + ) + if not decision.may_produce_candidate: + continue + ir = EvolutionIR( + target_kind=TARGET_KIND, + target_asset_id=profile.profile_id, + base_version="builtin", + operation="new", + hypothesis="；".join(p.rationale for p in task_proposals), + changes=[ + { + "profile": profile.to_dict(), + "tool_allowlist": profile.tool_allowlist or [], + } + ], + evidence_refs=evidence, + requested_tools=list(profile.tool_allowlist or []), + risk_tier=RISK_MEDIUM, + scope={"level": "workspace"}, + rollback=Rollback(version_id="builtin", triggers=["success -3pp"]), + evaluation_plan={"replay_set": "auto_holdout", "primary_metric": "task_success"}, + ) + try: + validate_ir(ir, credit_decision=decision.to_dict(), base_tools=base_tools) + except InvariantViolation as violation: + logger.info("[evolution-loop] profile candidate rejected: %s", violation.code) + continue + if dry_run: + created.append({"dry_run": True, "kind": "agent_profile", "task_type": task_type}) + continue + candidate_id = candidate_records.persist_candidate( + ir, + credit=decision.to_dict(), + proposer="orchestration_engine", + evidence_refs=evidence, + ) + if candidate_id: + created.append( + {"candidate_id": candidate_id, "kind": "agent_profile", "task_type": task_type} + ) + + for subagent in subagents: + decision = aggregate_finding( + target=T_ORCHESTRATION, + explanation=subagent.rationale, + confidence=confidence_from_evidence(subagent.support), + evidence_count=len(subagent.evidence_refs), + ) + if not decision.may_produce_candidate: + continue + payload = subagent.to_dict() + ir = EvolutionIR( + target_kind="subagent", + target_asset_id=subagent.agent_id, + base_version="v0", + operation="new", + hypothesis=subagent.rationale, + changes=[payload], + evidence_refs=subagent.evidence_refs, + requested_tools=list(subagent.tools_allowlist), + # Creating an agent is the widest-reaching change this system can + # propose, and the route into it changes the parent's prompt. + risk_tier=RISK_HIGH, + scope={"level": "workspace"}, + rollback=Rollback(version_id="v0", triggers=["manual"]), + evaluation_plan={"replay_set": "auto_holdout", "primary_metric": "task_success"}, + ) + try: + validate_ir(ir, credit_decision=decision.to_dict(), base_tools=base_tools) + except InvariantViolation as violation: + logger.info("[evolution-loop] subagent candidate rejected: %s", violation.code) + continue + if dry_run: + created.append({"dry_run": True, "kind": "subagent", "name": subagent.name}) + continue + candidate_id = candidate_records.persist_candidate( + ir, + credit=decision.to_dict(), + proposer="orchestration_engine", + evidence_refs=subagent.evidence_refs, + ) + if candidate_id: + created.append( + {"candidate_id": candidate_id, "kind": "subagent", "name": subagent.name} + ) + return created diff --git a/src/backend/core/evolution/cycle/cycle_extras.py b/src/backend/core/evolution/cycle/cycle_extras.py new file mode 100644 index 00000000..9852f995 --- /dev/null +++ b/src/backend/core/evolution/cycle/cycle_extras.py @@ -0,0 +1,32 @@ +"""Run the cycle's secondary engines in one call.""" +from __future__ import annotations +from typing import Any, Dict, Optional, Sequence +import logging +from core.evolution.cycle import extra_findings +from core.evolution.cycle import memory_findings + +logger = logging.getLogger(__name__) + +def run_extras( + views: Sequence[Dict[str, Any]], + *, + scan_findings: Optional[Sequence[Any]] = None, +) -> Dict[str, Any]: + """Everything the cycle was missing, in one call. + + Each section is independently guarded: a failure in the decremental pass + must not cost the intent pass its findings, because they answer different + questions and an operator needs whichever ones succeeded. + """ + report: Dict[str, Any] = {} + for name, fn in ( + ("intent", lambda: extra_findings.intent_findings(views)), + ("decremental", lambda: extra_findings.decremental_findings(views)), + ("memory", lambda: memory_findings.memory_findings(views, scan_findings=scan_findings)), + ): + try: + report[name] = fn() + except Exception as exc: # noqa: BLE001 + logger.warning("[cycle-extras] %s pass failed: %s", name, exc) + report[name] = {"error": str(exc)} + return report diff --git a/src/backend/core/evolution/cycle/cycle_extras_constants.py b/src/backend/core/evolution/cycle/cycle_extras_constants.py new file mode 100644 index 00000000..967d9902 --- /dev/null +++ b/src/backend/core/evolution/cycle/cycle_extras_constants.py @@ -0,0 +1,20 @@ +"""Thresholds for the intent, demotion and uplift engines.""" +from __future__ import annotations + + +MIN_INTENT_OCCASIONS = 3 + + +INTENT_MIN_SUCCESS_RATE = 0.6 + + +MIN_PLAN_VARIETY = 2 + + +DEMOTE_MIN_HITS = 4 + + +DEMOTE_SUCCESS_CEILING = 0.34 + + +MIN_UPLIFT_SAMPLES = 10 diff --git a/src/backend/core/evolution/cycle/episode_views.py b/src/backend/core/evolution/cycle/episode_views.py new file mode 100644 index 00000000..25798316 --- /dev/null +++ b/src/backend/core/evolution/cycle/episode_views.py @@ -0,0 +1,178 @@ +"""Flatten episodes and their trace events into the views the engines read.""" +from __future__ import annotations +from typing import Any, Dict, List +from core.db.engine import SessionLocal +from core.db.models.evolution import EvolutionEpisode, EvolutionTraceEvent +from core.evolution.profiles.user_settings_constants import PRIVACY_PRIVATE +import logging +logger = logging.getLogger(__name__) + +def view_for(episode, events) -> Dict[str, Any]: + """One Episode plus its events, flattened into what the engines read. + + Three fields here did not exist and each blocked an engine outright: + + ``memory_refs`` now carries the **stable content ref**, not the raw content + hash. The two look alike and are not: the ref is what the overlay keys on, + so an op proposed against a hash could never be applied to anything. + + ``skills_opened`` is what the model chose to read, as opposed to + ``skill_sequence``, which is what it was offered. Every decrement decision + depends on the difference. + + ``selection_degraded`` marks turns where ranking could not run and + everything loaded. Counting those as selections is what made every skill's + offer count identical, and therefore useless. + """ + from core.evolution.evidence.events import EV_MEMORY_RETRIEVED, EV_SKILL_OPENED, EV_SKILL_SELECTED, EV_TOOL_CALLED + from core.memory.ref_shadow import build_ref_id + from core.memory.retrieval_types import LAYER_FACT, LAYER_GRAPH + + tools: List[str] = [] + skills: List[str] = [] + memory_refs: List[str] = [] + graph_refs: List[str] = [] + graph_relations: List[Dict[str, Any]] = [] + skills_opened: List[Dict[str, Any]] = [] + selection_degraded = False + error_count = 0 + + user_id = episode.user_id or "" + for event in events: + payload = event.payload or {} + if event.event_type == EV_TOOL_CALLED: + name = payload.get("tool_name") + if name: + tools.append(str(name)) + if str(payload.get("status") or "") == "error": + error_count += 1 + elif event.event_type == EV_SKILL_SELECTED: + if payload.get("degraded"): + selection_degraded = True + skills.extend(str(s) for s in (payload.get("selected") or [])) + elif event.event_type == EV_SKILL_OPENED: + skills_opened.append( + { + "skill_id": str(payload.get("skill_id") or ""), + "tools_after_open": [str(t) for t in (payload.get("tools_after_open") or [])], + } + ) + elif event.event_type == EV_MEMORY_RETRIEVED: + for item in payload.get("items") or []: + content_hash = str(item.get("content_hash") or "") + if not content_hash: + continue + memory_refs.append( + build_ref_id( + layer=str(item.get("layer") or LAYER_FACT), + user_id=user_id, + workspace_id=str(item.get("workspace_id") or "default"), + content_hash=content_hash, + ) + ) + # L3 is deliberately tracked separately from L2. A graph relation + # is declarative context, not a procedure, and must never enter the + # memory→skill compiler as though it were an instruction. + for relation in payload.get("graph") or []: + content_hash = str(relation.get("content_hash") or "") + if not content_hash: + continue + graph_refs.append( + build_ref_id( + layer=LAYER_GRAPH, + user_id=user_id, + workspace_id=str(relation.get("workspace_id") or "default"), + content_hash=content_hash, + ) + ) + # The full reference, kept for the evidence resolver: a pack's + # L3 entry must carry the store's stable relation_id so an + # activated skill's graph evidence can be resolved back to the + # exact Neo4j edge — a bare ref id cannot do that. + graph_relations.append( + { + "relation_id": str(relation.get("relation_id") or ""), + "content_hash": content_hash, + "predicate": str(relation.get("predicate") or ""), + "confidence": relation.get("confidence") or 0.0, + "workspace_id": str(relation.get("workspace_id") or "default"), + } + ) + + outcome = episode.outcome or {} + return { + "episode_id": episode.episode_id, + "chat_id": episode.chat_id or "", + "user_id": user_id, + "tenant_id": episode.tenant_id or "default", + "verdict": outcome.get("verdict") + or ("success" if (episode.quality_score or 0) >= 0.65 else "unknown"), + "tool_sequence": tools, + "skill_sequence": skills, + "skills_opened": skills_opened, + "selection_degraded": selection_degraded, + "memory_refs": memory_refs, + "graph_refs": graph_refs, + "graph_relations": graph_relations, + "task_type": episode.task_type or "chat", + "backfilled": bool(episode.backfilled), + # Repeated tool errors are the observable form of "this run was stuck", + # which sub-agent derivation needs and nothing else recorded. + "stalled": error_count >= 3, + # What the user actually asked. Clustering only ever saw tool sequences + # before this, so "the same question keeps coming back" was not a + # detectable event. Sanitised at write time, so this is the preview + # rather than the raw text. + "objective": episode.objective_preview or "", + "objective_hash": episode.objective_hash or "", + "attributed_to": str(outcome.get("attributed_to") or ""), + # Needed to place an episode before or after an activation. Without it + # negative-transfer detection has no way to split the corpus and returns + # nothing on every corpus — wired, running, and structurally incapable + # of finding anything. + "created_at": episode.created_at, + } + + +def load_episode_views(*, limit: int = 500, since_days: int = 30) -> List[Dict[str, Any]]: + """Flatten Episodes plus their trace events into the shape pattern mining wants. + + Backfilled Episodes are included on purpose: they cannot be replayed, but + they are perfectly good for discovering what recurs — which is exactly why + the backfill was worth doing. + """ + views: List[Dict[str, Any]] = [] + try: + with SessionLocal() as db: + # The single enforcement point for per-user opt-out. Episodes from + # users who declined to contribute are still recorded (they power + # that user's own card and history) but never feed cross-user + # pattern mining. Filtering here rather than at each call site is + # what makes the promise checkable. + episodes = ( + db.query(EvolutionEpisode) + .filter(EvolutionEpisode.privacy_class != PRIVACY_PRIVATE) + .order_by(EvolutionEpisode.created_at.desc()) + .limit(limit) + .all() + ) + if not episodes: + return [] + + ids = [e.episode_id for e in episodes] + events = ( + db.query(EvolutionTraceEvent) + .filter(EvolutionTraceEvent.episode_id.in_(ids)) + .order_by(EvolutionTraceEvent.seq.asc()) + .all() + ) + by_episode: Dict[str, List[EvolutionTraceEvent]] = {} + for event in events: + by_episode.setdefault(event.episode_id, []).append(event) + + for episode in episodes: + views.append(view_for(episode, by_episode.get(episode.episode_id, []))) + except Exception as exc: + logger.warning("[evolution-loop] loading episodes failed: %s", exc) + return [] + return views diff --git a/src/backend/core/evolution/cycle/evaluation.py b/src/backend/core/evolution/cycle/evaluation.py new file mode 100644 index 00000000..8bccae99 --- /dev/null +++ b/src/backend/core/evolution/cycle/evaluation.py @@ -0,0 +1,153 @@ +"""Replay-evaluate candidates and record the results.""" +from __future__ import annotations +from typing import Any, Dict, List, Optional +from core.db.engine import SessionLocal +from core.db.models.evolution import EvolutionCandidate +import logging +logger = logging.getLogger(__name__) + +def record_evaluation(candidate_id: str, report: Any, eval_type: str) -> None: + """Persist an evaluation result against its candidate.""" + from core.db.models.evolution import EvolutionEvaluation + + try: + payload = report.to_dict() + with SessionLocal() as db: + db.add( + EvolutionEvaluation( + evaluation_id=payload["evaluation_id"], + candidate_id=candidate_id, + eval_type=eval_type, + # Frozen: a dataset edited after the fact invalidates every + # conclusion computed from it. + dataset_snapshot=payload["dataset_snapshot"], + baseline_metrics=payload["baseline"], + candidate_metrics=payload["candidate"], + guardrails={"breaches": payload["guardrail_breaches"]}, + verdict=payload["verdict"], + effect_size=payload["effect_size"], + p_value=payload["p_value"], + sample_size=payload["sample_size"], + ) + ) + db.commit() + except Exception as exc: + logger.warning("[evolution-loop] recording evaluation failed: %s", exc) + + +def replay_coverage_for(candidate_id: str, task_ids: List[str]) -> Optional[Dict[str, Any]]: + """Coverage assessment for candidates that carry an evidence pack. + + Returns ``None`` for pre-pack candidates — the gate governs only what the + unified engine produced, so legacy candidates keep their old semantics. + """ + try: + from core.evolution.evidence import lineage + from core.evolution.assessment.applicability import assess_replay_coverage + + with SessionLocal() as db: + candidate = db.get(EvolutionCandidate, candidate_id) + pack_id = str(getattr(candidate, "evidence_pack_id", "") or "") + if not pack_id: + return None + pack = lineage.load_evidence_pack(pack_id) + if pack is None: + return None + return assess_replay_coverage(pack, task_ids) + except Exception as exc: # noqa: BLE001 + logger.debug("[evolution-loop] replay coverage check skipped: %s", exc) + return None + + +def evaluate_candidate( + candidate_id: str, + run_arm: Any, + task_ids: List[str], +) -> Optional[Dict[str, Any]]: + """Run tiered replay for a candidate and advance it if it earns the step. + + ``run_arm(task_id, is_candidate)`` is injected: the caller owns isolation + (frozen bundle, cassette, sandbox snapshot). Only ``replay_passed`` is + reachable from here — shadow and canary need real traffic, and activation + needs a human, so this can never quietly ship anything. + """ + from core.evolution.cycle.release import DRAFT, REJECTED, REPLAY_PASSED, is_legal_transition + from core.evolution.evidence.replay import VERDICT_IMPROVED, VERDICT_REGRESSED, run_tiered_replay + + tiered = run_tiered_replay(task_ids, run_arm) + report = tiered.report or tiered.sentinel_report + if report is None: + return None + + record_evaluation(candidate_id, report, "replay" if tiered.passed_sentinel else "sentinel") + + # Replay 覆盖闸门：带证据 pack 的候选，其回放集必须覆盖 pack 声称的行为面 + # （成功 / 失败恢复 / 负样本 / 图谱依赖两类场景）。只用成功案例喂出来的 + # "improved" 是单面证据，不允许换来 replay_passed；回归照常生效——安全 + # 方向的结论不需要覆盖完整也成立。 + coverage: Optional[Dict[str, Any]] = None + if report.verdict == VERDICT_IMPROVED: + coverage = replay_coverage_for(candidate_id, task_ids) + if coverage is not None and not coverage.get("covered", True): + return { + "candidate_id": candidate_id, + "verdict": report.verdict, + "status": DRAFT, + "transition": "held_insufficient_coverage", + "coverage": coverage, + "tasks_run": tiered.tasks_run, + "effect_size": report.effect_size, + "p_value": report.p_value, + } + + try: + with SessionLocal() as db: + candidate = db.get(EvolutionCandidate, candidate_id) + if candidate is None: + return None + current = candidate.status + + # Re-evaluating a candidate that has already moved on is a no-op, + # not a failure. Collapsing the two into None left callers unable to + # tell "already advanced" from "something broke". + target = ( + REJECTED + if report.verdict == VERDICT_REGRESSED + else (REPLAY_PASSED if report.verdict == VERDICT_IMPROVED else current) + ) + if target != current and not is_legal_transition(current, target): + return { + "candidate_id": candidate_id, + "verdict": report.verdict, + "status": current, + "transition": "skipped_already_advanced", + "tasks_run": tiered.tasks_run, + "effect_size": report.effect_size, + "p_value": report.p_value, + } + + if report.verdict == VERDICT_REGRESSED: + candidate.status = REJECTED + candidate.reject_reason = ( + f"replay regression: effect={report.effect_size:.3f} " + f"p={report.p_value:.4f} {report.guardrail_breaches}" + ) + elif report.verdict == VERDICT_IMPROVED: + candidate.status = REPLAY_PASSED + else: + # Neutral or insufficient: stays a draft. Not advancing is the + # correct outcome — "we could not show it helps" is not "it helps". + pass + db.commit() + return { + "candidate_id": candidate_id, + "verdict": report.verdict, + "status": candidate.status, + "transition": "applied" if candidate.status != current else "held", + "tasks_run": tiered.tasks_run, + "effect_size": report.effect_size, + "p_value": report.p_value, + } + except Exception as exc: + logger.warning("[evolution-loop] advancing candidate failed: %s", exc) + return None diff --git a/src/backend/core/evolution/cycle/extra_findings.py b/src/backend/core/evolution/cycle/extra_findings.py new file mode 100644 index 00000000..214feb8c --- /dev/null +++ b/src/backend/core/evolution/cycle/extra_findings.py @@ -0,0 +1,166 @@ +"""Intent, decremental and negative-transfer findings.""" +from __future__ import annotations +from core.evolution.cycle import promotion +from core.evolution.cycle import retirement +import hashlib +from typing import Any, Dict, List, Optional, Sequence, Tuple +from core.evolution.cycle import cycle_extras_constants +from core.evolution.cycle import usage_evidence + +def asset_id(prefix: str, seed: str) -> str: + return f"{prefix}-" + hashlib.sha256(seed.encode("utf-8")).hexdigest()[:12] + + +def intent_findings(views: Sequence[Dict[str, Any]]) -> Dict[str, Any]: + """Recurring requests whose handling never settled. + + The pairing matters. Recurrence alone says the question is common; + recurrence **with plan variety** says the system has not found a stable way + to answer it and is paying the planning cost fresh every time. That second + condition is what distinguishes a capability gap from a popular question. + """ + from core.evolution.assessment.similarity import cluster_intents + + clusters, method = cluster_intents(views, min_support=cycle_extras_constants.MIN_INTENT_OCCASIONS) + findings: List[Dict[str, Any]] = [] + for cluster in clusters: + if cluster.occasions < cycle_extras_constants.MIN_INTENT_OCCASIONS: + # Recurrence inside one conversation is a user rephrasing after a + # bad answer, which is evidence about that answer, not about demand. + continue + if cluster.plan_variety < cycle_extras_constants.MIN_PLAN_VARIETY: + # Same question, same plan, every time. Already effectively a skill; + # writing one down adds an asset to maintain and changes nothing. + continue + if cluster.success_rate < cycle_extras_constants.INTENT_MIN_SUCCESS_RATE: + # Recurring *and* failing. Distilling the current plan would encode + # the failure. This belongs to attribution, not to promotion. + findings.append( + { + "kind": "recurring_failure_intent", + "cluster": cluster.to_dict(), + "action": "no_skill_candidate", + "reason": ( + f"「{cluster.representative[:30]}」类请求出现 {cluster.occasions} 次，" + f"但成功率只有 {cluster.success_rate:.0%}——" + "把当前做法蒸馏出来等于把失败固化" + ), + } + ) + continue + findings.append( + { + "kind": "recurring_intent", + "cluster": cluster.to_dict(), + "action": "skill_candidate", + "asset_id": asset_id("auto-intent", cluster.representative), + "reason": ( + f"同一意图在 {cluster.occasions} 次独立会话中出现，却用了 " + f"{cluster.plan_variety} 种不同的工具序列——每次都在重新规划" + ), + } + ) + return {"method": method, "clusters": len(clusters), "findings": findings} + + +def decremental_findings( + views: Sequence[Dict[str, Any]], + *, + skill_stats: Optional[Dict[str, Dict[str, Any]]] = None, + declared_tools: Optional[Dict[str, List[str]]] = None, +) -> Dict[str, Any]: + """Merge, retirement and negative-transfer findings. + + ``skill_stats`` is derived from the episode views when not supplied. + ``declared_tools`` maps skill id → the tools its own document names; overlap + is computed from that and never from the tools observed in episodes where + the skill happened to be loaded, which would pair every co-enabled skill. + """ + + stats = skill_stats if skill_stats is not None else usage_evidence.skill_usage_from_views(views) + + overlaps: List[Tuple[str, str, int]] = [] + if declared_tools is None: + declared_tools = usage_evidence.declared_tools_for(sorted(stats)) + if declared_tools: + overlaps = promotion.find_overlaps(declared_tools) + + retirements = retirement.find_retirement_candidates(stats) + return { + "overlaps": [ + {"a": a, "b": b, "shared_tools": shared} for a, b, shared in overlaps + ], + "retirements": retirements, + "asset_count": len(stats), + "negative_transfer": negative_transfer_for_activations(views), + } + + +def negative_transfer_for_activations( + views: Sequence[Dict[str, Any]], +) -> List[Dict[str, Any]]: + """Check each recent activation for collateral damage. + + The detector needs a corpus split into before and after *something*. Calling + it on views where nothing carries that marker returns nothing on every + corpus, which is a detector that runs and cannot find anything — the exact + shape of defect this rework exists to remove. So the split is computed here, + from the release ledger, once per activated asset. + """ + activations = usage_evidence.recent_activations() + if not activations: + return [] + + findings: List[Dict[str, Any]] = [] + for asset_id, activated_at, related in activations: + split = [ + {**view, "after": usage_evidence.is_after(view.get("created_at"), activated_at)} + for view in views + if view.get("created_at") is not None + ] + if not any(v["after"] for v in split) or not any(not v["after"] for v in split): + # One-sided evidence cannot show a change. Reporting nothing is the + # honest output; reporting a drop from an empty baseline is not. + continue + findings.extend( + negative_transfer_findings(split, asset_id=asset_id, related_task_types=related) + ) + return findings + + +def negative_transfer_findings( + views: Sequence[Dict[str, Any]], + *, + asset_id: str = "", + related_task_types: Sequence[str] = (), + before_key: str = "before", + after_key: str = "after", +) -> List[Dict[str, Any]]: + """Task types that got worse after some activation, unrelated to it. + + The one failure mode of joint evolution that no single release's guardrail + can catch: each change looks fine on the metric it was measured against, and + the damage shows up somewhere nobody was looking. + """ + + by_type_before: Dict[str, List[bool]] = {} + by_type_after: Dict[str, List[bool]] = {} + for view in views: + target = by_type_after if view.get(after_key) else by_type_before + target.setdefault(str(view.get("task_type") or "chat"), []).append( + view.get("verdict") == "success" + ) + + def rates(source: Dict[str, List[bool]]) -> Dict[str, float]: + return {k: (sum(v) / len(v) if v else 0.0) for k, v in source.items()} + + findings = retirement.detect_negative_transfer( + asset_id=asset_id or "unspecified", + # Only *unrelated* task types are examined. A dip on the task the change + # targeted is an ordinary regression its own guardrail catches; the + # dangerous case is collateral damage where nobody is looking. + related_task_types=list(related_task_types), + before=rates(by_type_before), + after=rates(by_type_after), + ) + return [f.to_dict() for f in findings] diff --git a/src/backend/core/evolution/cycle/graph_candidates.py b/src/backend/core/evolution/cycle/graph_candidates.py new file mode 100644 index 00000000..78c8369b --- /dev/null +++ b/src/backend/core/evolution/cycle/graph_candidates.py @@ -0,0 +1,156 @@ +"""L3 graph evolution candidates and the dispatcher for the other engines.""" +from __future__ import annotations +import hashlib +from typing import Any, Dict, List +from core.evolution.evidence.credit import T_MEMORY, aggregate_finding, confidence_from_evidence +from core.evolution.ir import RISK_MEDIUM, EvolutionIR, InvariantViolation, Rollback, validate_ir +import logging +from core.evolution.cycle import candidate_records +from core.evolution.cycle import composition_candidates +from core.evolution.cycle import memory_candidates +from core.evolution.cycle import prompt_candidates + +logger = logging.getLogger(__name__) + +def graph_evolution_candidates( + views: List[Dict[str, Any]], *, dry_run: bool +) -> List[Dict[str, Any]]: + """L3 的演进：强化/降权自动执行，语义变更走人工候选。 + + Per user, because the graph is per user scope: a contradiction between two + people's graphs is not a contradiction at all. + """ + from core.config.settings import settings + + if not (settings.memory.enabled and settings.memory.graph_enabled): + return [] + + from core.evolution.generators.graph_scan import apply_graph_ops, scan_graph_relations + + observed_by_user: Dict[str, Dict[str, int]] = {} + for view in views: + user_id = str(view.get("user_id") or "") + if not user_id: + continue + bucket = observed_by_user.setdefault(user_id, {}) + for relation in view.get("graph_relations") or []: + relation_id = str(relation.get("relation_id") or "") + if relation_id: + bucket[relation_id] = bucket.get(relation_id, 0) + 1 + + created: List[Dict[str, Any]] = [] + for user_id, observed in observed_by_user.items(): + try: + from core.memory.graph import list_graph_relations + + relations = candidate_records.run_async(list_graph_relations(user_id, limit=200)) + except Exception as exc: # noqa: BLE001 + logger.debug("[evolution-loop] graph listing failed for %s: %s", user_id, exc) + continue + if not relations: + continue + + ops = scan_graph_relations(relations, observed_relation_ids=observed) + if not ops: + continue + if dry_run: + created.append( + {"dry_run": True, "kind": "graph", "user_id": user_id, "ops": len(ops)} + ) + continue + + outcome = apply_graph_ops(ops, scope_user_id=user_id) + manual = outcome.get("manual") or [] + entry: Dict[str, Any] = { + "kind": "graph", + "user_id": user_id, + "auto_applied": len(outcome.get("applied") or []), + "manual_proposed": len(manual), + } + if manual: + decision = aggregate_finding( + target=T_MEMORY, + explanation="；".join(str(op.get("reason") or "") for op in manual[:3]), + confidence=confidence_from_evidence(len(manual), saturates_at=6), + evidence_count=len(manual), + ) + if decision.may_produce_candidate: + asset_id = "graph-" + hashlib.sha256(user_id.encode("utf-8")).hexdigest()[:12] + ir = EvolutionIR( + target_kind="memory", + target_asset_id=asset_id, + base_version="current", + operation="update", + hypothesis=( + f"图谱有 {len(manual)} 处语义变更需要人工裁决：" + + "；".join(str(op.get("reason") or "") for op in manual[:3]) + ), + changes=[{"user_id": user_id, "graph_ops": manual}], + evidence_refs=[ + str(op.get("relation_id") or "") for op in manual if op.get("relation_id") + ][:20], + risk_tier=RISK_MEDIUM, + scope={"level": "user", "user_id": user_id}, + rollback=Rollback(version_id="current", triggers=["manual"]), + evaluation_plan={"replay_set": "personal", "primary_metric": "task_success"}, + ) + try: + validate_ir( + ir, + credit_decision=decision.to_dict(), + allowed_scopes=["user", "workspace"], + ) + except InvariantViolation as violation: + logger.info( + "[evolution-loop] graph candidate rejected: %s", violation.code + ) + else: + candidate_id = candidate_records.persist_candidate( + ir, + credit=decision.to_dict(), + proposer="graph_engine", + evidence_refs=list(ir.evidence_refs), + ) + if candidate_id: + entry["candidate_id"] = candidate_id + created.append(entry) + return created + + +def run_other_engines(views: List[Dict[str, Any]], *, dry_run: bool) -> Dict[str, Any]: + """Memory, decremental, intent, orchestration, prompt and graph — each with an exit. + + Independently guarded: a failure in one engine must not cost the others + their findings, because they answer different questions and an operator + needs whichever ones succeeded. + """ + from core.evolution.cycle.cycle_extras import run_extras + + scan_findings = memory_candidates.memory_scan_findings(views) + extras = run_extras(views, scan_findings=scan_findings) + + produced: List[Dict[str, Any]] = [] + for name, fn in ( + ("memory", lambda: memory_candidates.memory_candidates(extras, views, dry_run=dry_run)), + ("intent", lambda: memory_candidates.intent_skill_candidates(extras, views, dry_run=dry_run)), + ("decremental", lambda: composition_candidates.decremental_candidates(extras, dry_run=dry_run)), + ("orchestration", lambda: composition_candidates.orchestration_candidates(views, dry_run=dry_run)), + ("prompt", lambda: prompt_candidates.prompt_candidates(views, dry_run=dry_run)), + ("graph", lambda: graph_evolution_candidates(views, dry_run=dry_run)), + ): + try: + produced.extend(fn()) + except Exception as exc: # noqa: BLE001 + logger.warning("[evolution-loop] %s engine failed: %s", name, exc) + produced.append({"kind": name, "error": str(exc)}) + + # ``ops`` holds MemoryOp objects for the candidate builder; the report is + # serialised into the console, so it must not carry them. + memory_report = dict(extras.get("memory") or {}) + memory_report.pop("ops", None) + extras["memory"] = memory_report + extras["memory"]["scan_findings"] = [ + f.to_dict() if hasattr(f, "to_dict") else f for f in scan_findings + ][:50] + + return {"extras": extras, "engine_candidates": produced} diff --git a/src/backend/core/evolution/cycle/loop.py b/src/backend/core/evolution/cycle/loop.py new file mode 100644 index 00000000..90cca13b --- /dev/null +++ b/src/backend/core/evolution/cycle/loop.py @@ -0,0 +1,292 @@ +"""One fleet-wide evolution pass: discover, attribute, promote, validate, persist.""" +from __future__ import annotations +from core.evolution.cycle import ordering +from core.evolution.cycle import patterns +from core.evolution.cycle import promotion +from core.evolution.cycle import promotion_constants +import hashlib +from typing import Any, Dict, List +from core.evolution.evidence.credit import T_SKILL, aggregate_finding, assign_credit, confidence_from_evidence +from core.evolution.ir import RISK_HIGH, RISK_MEDIUM +import logging +from core.evolution.cycle import candidate_records +from core.evolution.cycle import episode_views +from core.evolution.cycle import graph_candidates + +logger = logging.getLogger(__name__) + +def run_evolution_cycle(*, limit: int = 500, dry_run: bool = False) -> Dict[str, Any]: + """One fleet-wide pass: discover → attribute → promote → validate → persist. + + Returns a report rather than raising: this runs as a background job and a + bad cycle must leave the product untouched. + """ + if not candidate_records.cross_user_mining_available(): + logger.info("[evolution-loop] cross-user mining unavailable; personal only") + return { + "episodes": 0, + "patterns": 0, + "candidates": [], + "rejected": [], + "no_update": 0, + "skipped": "cross_user_mining_requires_enterprise", + } + # Refuse rather than degrade. Without an embedding service two of the three + # engines lose their primary path and the cycle reports success having found + # nothing — which is indistinguishable from "there was nothing to find", and + # is the state nobody investigates. + from core.evolution.cycle.readiness import check_readiness + + readiness = check_readiness() + if not readiness.ready: + logger.warning("[evolution-loop] not ready, refusing to run: %s", readiness.blocking) + return { + "episodes": 0, + "patterns": 0, + "candidates": [], + "rejected": [], + "no_update": 0, + "skipped": "not_ready", + "readiness": readiness.to_dict(), + } + + report: Dict[str, Any] = { + "episodes": 0, + "patterns": 0, + "proposals": 0, + "candidates": [], + "rejected": [], + "no_update": 0, + "readiness": readiness.to_dict(), + } + + views = episode_views.load_episode_views(limit=limit) + report["episodes"] = len(views) + if not views: + return report + + discovered = patterns.discover_patterns(views) + report["patterns"] = len(discovered) + + # Transferable ordering rules, extracted once across the whole corpus rather + # than per pattern. A rigid sequence only helps tasks whose tool set it + # happens to cover; the ordering rule behind it helps every task making the + # same mistake, which is the difference between memorising and learning. + constraints = ordering.extract_ordering_constraints(views) + report["ordering_constraints"] = [c.to_dict() for c in constraints] + + from core.evolution.generators.skill_candidate_engine import STATUS_CREATED, STATUS_DRY_RUN, STATUS_REJECTED, SkillCandidateSpec, create_skill_candidate + + existing_signatures: Dict[str, str] = {} + covered_tool_sets: List[List[str]] = [] + by_episode_id = {str(v.get("episode_id") or ""): v for v in views} + + for pattern in discovered: + if pattern.kind == promotion_constants.PATTERN_SUCCESS_SUBSEQUENCE and pattern.tool_sequence: + # An observation across many *successful* runs — aggregate decision, + # not failure attribution with invented features. + decision = aggregate_finding( + target=T_SKILL, + explanation=( + f"{pattern.support} 个成功 Episode 复现同一 " + f"{len(pattern.tool_sequence)} 步工具序列" + f"（成功率 {pattern.success_rate:.0%}）" + ), + confidence=confidence_from_evidence(pattern.support, saturates_at=10), + evidence_count=pattern.support, + ) + else: + decision = assign_credit(candidate_records.features_for(pattern)) + if not decision.may_produce_candidate: + # No-update / unidentifiable / a non-asset cause. Refusing here is + # the mechanism that stops a KB gap being "fixed" by editing a skill. + report["no_update"] += 1 + continue + + proposal = promotion.promote_tool_sequence_to_skill( + pattern, + episodes=views, + skill_credit=decision.scores.get("skill", 0.0), + workflow_credit=decision.scores.get("workflow", 0.0), + existing_skill_signatures=existing_signatures, + ) + if proposal is None: + continue + report["proposals"] += 1 + + # Stable, content-derived id. Python's built-in hash() is randomised per + # process, so using it here minted a fresh asset id on every cycle and the + # de-duplication constraint never matched — one candidate per run, forever. + asset_id = proposal.patch_target or ( + "auto-" + hashlib.sha256(proposal.signature.encode("utf-8")).hexdigest()[:12] + ) + sequence = [str(t) for t in proposal.payload.get("tool_sequence", [])] + supporting = [ + by_episode_id[e] + for e in (proposal.payload.get("episode_ids") or []) + if e in by_episode_id + ] + + # 规则门测统计，测不了语义——意图簇可能是巧合归并，序列可能是对任何 + # 任务都成立的通用组合。规则幸存者再过一道 LLM 裁决，拒绝即不产候选。 + from core.evolution.assessment.sop_judge import adjudicate_sequence_sop + + verdict = candidate_records.run_async( + adjudicate_sequence_sop( + representative=str(proposal.payload.get("intent") or ""), + objectives=[str(v.get("objective") or "") for v in supporting], + tool_sequence=sequence, + occasions=int(proposal.payload.get("occasions") or 0), + support=proposal.support, + success_rate=pattern.success_rate, + ) + ) + if not verdict.worthy: + report["rejected"].append( + { + "signature": proposal.signature, + "code": ( + "sop_judge_unavailable" + if verdict.judge_unavailable + else "sop_judge_refused" + ), + "reason": verdict.reason, + } + ) + continue + spec = SkillCandidateSpec( + asset_id=asset_id, + proposer="promotion_chain", + hypothesis=proposal.rationale, + views=supporting, + tool_sequence=sequence, + # Attach every rule whose tools this skill touches, and carry its + # scope along with it. Attaching must not apply the usability + # policy: doing so dropped exactly the context-dependent rules the + # scoping exists to preserve — a rule contradicted in one task type + # is still valid in the ones where it held, and that decision + # belongs at use time, where the context is known. + ordering_constraints=[ + c.to_dict() + for c in constraints + if c.before in sequence and c.after in sequence + ], + operation="patch" if proposal.patch_target else "new", + base_version="current" if proposal.patch_target else "v0", + # A brand-new capability is riskier than tweaking an existing one. + risk_tier=RISK_MEDIUM if proposal.patch_target else RISK_HIGH, + scope={"level": "workspace"}, + tenant_id=str( + (supporting[0].get("tenant_id") if supporting else "") or "default" + ), + write_document=False, + ) + # 裁决结论随候选进 credit_decision 落库，评审员能看到机器放行的理由。 + outcome = create_skill_candidate( + spec, + credit={**decision.to_dict(), "sop_judge": verdict.to_dict()}, + dry_run=dry_run, + ) + status = outcome.get("status") + if status == STATUS_REJECTED: + report["rejected"].append( + {"signature": proposal.signature, "code": outcome.get("code")} + ) + elif status == STATUS_DRY_RUN: + report["candidates"].append({"dry_run": True, "signature": proposal.signature}) + elif status == STATUS_CREATED: + covered_tool_sets.append(sequence) + report["candidates"].append( + { + "candidate_id": outcome["candidate_id"], + "signature": proposal.signature, + "evidence_pack_id": outcome.get("pack_id", ""), + } + ) + + # ── Ordering rules that no sequence candidate carries ──────────────────── + # + # Without this the only shape of learning that can become an asset is "five + # runs repeated one identical tool sequence", which on real traffic is close + # to unobservable — measured on this deployment's history, every episode's + # tool sequence was unique. The rules are the part that generalises, so they + # get to stand on their own rather than only riding along on a sequence that + # happens to qualify. + orphan_rules = candidate_records.uncovered_constraints(constraints, covered_tool_sets) + if orphan_rules: + rule_decision = assign_credit(candidate_records.features_for_ordering_rules(orphan_rules)) + if not rule_decision.may_produce_candidate: + report["no_update"] += 1 + else: + rule_payload = [c.to_dict() for c in orphan_rules] + signature = "|".join(sorted(f"{c.before}<{c.after}" for c in orphan_rules)) + asset_id = "auto-rules-" + hashlib.sha256(signature.encode("utf-8")).hexdigest()[:12] + supporting_views = [ + view + for view in views + if view.get("verdict") == "success" + and any( + c.before in (view.get("tool_sequence") or []) + and c.after in (view.get("tool_sequence") or []) + for c in orphan_rules + ) + ] + rule_spec = SkillCandidateSpec( + asset_id=asset_id, + proposer="promotion_chain", + hypothesis=( + f"{len(orphan_rules)} 条工具顺序约束在 {len(supporting_views)} 条成功执行中" + "稳定成立且跨多种工具组合复现，但规划阶段每次仍可能排错" + ), + views=supporting_views, + # Deliberately no tool sequence: this asset claims no fixed + # order of its own, only constraints. Materialisation renders + # it as rules. + tool_sequence=[], + ordering_constraints=rule_payload, + operation="new", + base_version="v0", + risk_tier=RISK_HIGH, + scope={"level": "workspace"}, + tenant_id=str( + (supporting_views[0].get("tenant_id") if supporting_views else "") + or "default" + ), + write_document=False, + ) + outcome = create_skill_candidate( + rule_spec, credit=rule_decision.to_dict(), dry_run=dry_run + ) + status = outcome.get("status") + if status == STATUS_REJECTED: + report["rejected"].append({"signature": signature, "code": outcome.get("code")}) + else: + report["proposals"] += 1 + if status == STATUS_DRY_RUN: + report["candidates"].append( + {"dry_run": True, "signature": signature, "kind": "ordering_rules"} + ) + elif status == STATUS_CREATED: + report["candidates"].append( + { + "candidate_id": outcome["candidate_id"], + "signature": signature, + "kind": "ordering_rules", + "evidence_pack_id": outcome.get("pack_id", ""), + } + ) + + # The other two engines, plus orchestration. Every finding below becomes a + # candidate in the same table, behind the same security gate and the same + # review queue as the skill candidates above — a finding that stops at a + # report is a log line, not an engine. + report.update(graph_candidates.run_other_engines(views, dry_run=dry_run)) + + logger.info( + "[evolution-loop] cycle done: episodes=%d patterns=%d candidates=%d no_update=%d", + report["episodes"], + report["patterns"], + len(report["candidates"]), + report["no_update"], + ) + return report diff --git a/src/backend/core/evolution/cycle/memory_candidates.py b/src/backend/core/evolution/cycle/memory_candidates.py new file mode 100644 index 00000000..9570e4ad --- /dev/null +++ b/src/backend/core/evolution/cycle/memory_candidates.py @@ -0,0 +1,286 @@ +"""Memory and intent candidates built from per-user memory scans.""" +from __future__ import annotations +import hashlib +from typing import Any, Dict, List +from core.evolution.evidence.credit import T_MEMORY, T_SKILL, aggregate_finding, confidence_from_evidence +from core.evolution.ir import RISK_HIGH, RISK_MEDIUM, EvolutionIR, InvariantViolation, Rollback, validate_ir +import logging +from core.evolution.cycle import candidate_records + +logger = logging.getLogger(__name__) + +def memory_scan_findings(views: List[Dict[str, Any]]) -> List[Any]: + """Scan every contributing user's store. + + Per user, because the store is per user: near-duplicates and contradictions + only mean anything within one person's memories, and a cross-user scan would + both mean nothing and read what it has no business reading. + """ + from core.evolution.generators.memory_scan import scan_user_memories + + retrieved: Dict[str, Dict[str, int]] = {} + for view in views: + user_id = str(view.get("user_id") or "") + if not user_id: + continue + bucket = retrieved.setdefault(user_id, {}) + for ref in view.get("memory_refs") or []: + bucket[str(ref)] = bucket.get(str(ref), 0) + 1 + + findings: List[Any] = [] + for user_id, refs in retrieved.items(): + try: + _, user_findings = candidate_records.run_async(scan_user_memories(user_id, retrieved_refs=refs)) + except Exception as exc: # noqa: BLE001 + logger.warning("[evolution-loop] memory scan failed for %s: %s", user_id, exc) + continue + findings.extend(user_findings) + return findings + + +def memory_candidates( + extras: Dict[str, Any], views: List[Dict[str, Any]], *, dry_run: bool +) -> List[Dict[str, Any]]: + """One memory candidate per user, carrying that user's operations. + + Grouped per user rather than per operation because the approval question is + "may the system reorganise *my* memories", and answering it forty times for + forty entries is not review, it is attrition. + """ + memory = extras.get("memory") or {} + ops = memory.get("ops") or [] + if not ops: + return [] + + owner_by_ref: Dict[str, str] = {} + for view in views: + user_id = str(view.get("user_id") or "") + for ref in view.get("memory_refs") or []: + owner_by_ref.setdefault(str(ref), user_id) + + by_user: Dict[str, List[Any]] = {} + for op in ops: + owner = owner_by_ref.get(op.memory_ref, "") + if not owner: + # An operation whose subject cannot be identified must not be + # proposed: applying it would need a user id, and guessing one is + # how a change lands on the wrong person's memories. + continue + by_user.setdefault(owner, []).append(op) + + created: List[Dict[str, Any]] = [] + for user_id, user_ops in by_user.items(): + # Duplicates, contradictions and stale entries are properties of the + # store, found by scanning it — not a conflict with any one request. + decision = aggregate_finding( + target=T_MEMORY, + explanation="；".join(op.reason for op in user_ops[:3]), + confidence=confidence_from_evidence(len(user_ops), saturates_at=8), + evidence_count=len(user_ops), + ) + if not decision.may_produce_candidate: + continue + + asset_id = "mem-" + hashlib.sha256(user_id.encode("utf-8")).hexdigest()[:12] + ir = EvolutionIR( + target_kind="memory", + target_asset_id=asset_id, + base_version="current", + operation="merge" if any(op.operation == "merge" for op in user_ops) else "reweight", + hypothesis=( + f"记忆库有 {len(user_ops)} 处需要重组：" + + "；".join(op.reason for op in user_ops[:3]) + ), + changes=[ + { + "user_id": user_id, + "operations": [op.to_dict() for op in user_ops], + } + ], + evidence_refs=[op.memory_ref for op in user_ops][:20], + # Changing what a person's agent remembers is never low risk, and + # the scope is that person alone. + risk_tier=RISK_MEDIUM, + scope={"level": "user", "user_id": user_id}, + rollback=Rollback(version_id="current", triggers=["user_disable"]), + evaluation_plan={"replay_set": "personal", "primary_metric": "task_success"}, + ) + try: + validate_ir( + ir, credit_decision=decision.to_dict(), allowed_scopes=["user", "workspace"] + ) + except InvariantViolation as violation: + logger.info("[evolution-loop] memory candidate rejected: %s", violation.code) + continue + if dry_run: + created.append({"dry_run": True, "kind": "memory", "user_id": user_id}) + continue + candidate_id = candidate_records.persist_candidate( + ir, + credit=decision.to_dict(), + proposer="memory_engine", + evidence_refs=[op.memory_ref for op in user_ops][:20], + ) + if candidate_id: + created.append( + { + "candidate_id": candidate_id, + "kind": "memory", + "user_id": user_id, + "operations": len(user_ops), + } + ) + return created + + +def intent_skill_candidates( + extras: Dict[str, Any], views: List[Dict[str, Any]], *, dry_run: bool +) -> List[Dict[str, Any]]: + """Recurring intents, compiled through the unified skill candidate engine. + + The intent finding says a capability is missing; it does not say what the + capability is. The engine resolves the cluster's evidence into an immutable + pack, derives the deterministic structure (tools, boundaries, dependencies), + has the model write only the document, and validates the layered contract — + the same chain the tool-sequence path uses, so identical evidence can no + longer produce structurally different skills depending on which engine + noticed it first. + """ + from core.evolution.generators.skill_candidate_engine import STATUS_CREATED, STATUS_DRY_RUN, STATUS_NOT_WRITTEN, STATUS_REJECTED, SkillCandidateSpec, create_skill_candidate + + findings = [ + f + for f in (extras.get("intent") or {}).get("findings", []) + if f.get("action") == "skill_candidate" + ] + if not findings: + return [] + + by_episode = {str(v.get("episode_id") or ""): v for v in views} + created: List[Dict[str, Any]] = [] + + for finding in findings: + cluster = finding.get("cluster") or {} + episode_ids = [str(e) for e in (cluster.get("episode_ids") or [])] + episodes = [by_episode[e] for e in episode_ids if e in by_episode] + if not episodes: + continue + + # The cluster mostly *succeeded* — that is a precondition for + # distilling it. Declaring the turns failed to satisfy the failure + # attributor would contradict the very check that let this through. + decision = aggregate_finding( + target=T_SKILL, + explanation=str(finding.get("reason") or ""), + confidence=confidence_from_evidence(len(episode_ids), saturates_at=10), + evidence_count=len(episode_ids), + ) + if not decision.may_produce_candidate: + continue + + asset_id = str(finding.get("asset_id") or "") + spec = SkillCandidateSpec( + asset_id=asset_id, + proposer="intent_engine", + hypothesis=str(finding.get("reason") or ""), + views=episodes, + intent_cluster=cluster, + procedures=procedures_for_cluster(cluster, views), + operation="new", + base_version="v0", + risk_tier=RISK_HIGH, + scope={"level": "workspace"}, + tenant_id=str(episodes[0].get("tenant_id") or "default"), + write_document=True, + ) + outcome = create_skill_candidate( + spec, credit=decision.to_dict(), dry_run=dry_run, run_async=candidate_records.run_async + ) + status = outcome.get("status") + if status == STATUS_NOT_WRITTEN: + created.append( + { + "kind": "recurring_intent", + "asset_id": asset_id, + "written": False, + "why_not": outcome.get("why_not") or {}, + } + ) + elif status == STATUS_REJECTED: + logger.info( + "[evolution-loop] intent candidate rejected: %s %s", + outcome.get("code"), + outcome.get("violations") or "", + ) + elif status == STATUS_DRY_RUN: + created.append({"dry_run": True, "kind": "recurring_intent", "asset_id": asset_id}) + elif status == STATUS_CREATED: + created.append( + { + "candidate_id": outcome["candidate_id"], + "kind": "recurring_intent", + "asset_id": asset_id, + "written": True, + "attempts": outcome.get("attempts", 0), + "evidence_pack_id": outcome.get("pack_id", ""), + } + ) + return created + + +def procedures_for_cluster( + cluster: Dict[str, Any], views: List[Dict[str, Any]] +) -> List[Dict[str, str]]: + """The procedural memories retrieved during this intent's episodes. + + These are what makes the resulting document a *compilation* of memory rather + than a restatement of a tool list — and their refs are what gets demoted + once the skill is live. + """ + from core.evolution.generators.memory_scan import procedural_records, to_records + from core.memory.ref_shadow import build_ref_id + from core.memory.retrieval_types import LAYER_FACT + + wanted = {str(ref) for ref in (cluster.get("memory_refs") or [])} + if not wanted: + return [] + + episode_ids = {str(e) for e in (cluster.get("episode_ids") or [])} + users = { + str(v.get("user_id") or "") + for v in views + if str(v.get("episode_id") or "") in episode_ids and v.get("user_id") + } + + procedures: List[Dict[str, str]] = [] + for user_id in users: + try: + from core.memory.service import get_all_memories + + raw = candidate_records.run_async(get_all_memories(user_id, top_k=500)) + except Exception as exc: # noqa: BLE001 + logger.debug("[evolution-loop] procedure lookup failed for %s: %s", user_id, exc) + continue + for record in procedural_records(to_records(raw)): + ref = build_ref_id( + layer=LAYER_FACT, + user_id=user_id, + workspace_id=record.workspace_id, + content_hash=record.content_hash, + ) + if ref in wanted: + procedures.append( + { + "ref": ref, + # Carried because a fleet-wide skill can be compiled from + # several people's memories, and demotion is per-store: a + # bare list of refs has no owner to apply them against, so + # the reverse edge would silently do nothing for exactly + # the candidates that need it most. + "user_id": user_id, + "rule": record.content, + "why": record.why, + "applies_to": record.applies_to, + } + ) + return procedures diff --git a/src/backend/core/evolution/cycle/memory_findings.py b/src/backend/core/evolution/cycle/memory_findings.py new file mode 100644 index 00000000..8ddaf33f --- /dev/null +++ b/src/backend/core/evolution/cycle/memory_findings.py @@ -0,0 +1,109 @@ +"""Reorganisation proposals for the memory layer.""" +from __future__ import annotations +from typing import Any, Dict, List, Optional, Sequence +from core.evolution.cycle import cycle_extras_constants + +def memory_findings( + views: Sequence[Dict[str, Any]], + *, + scan_findings: Optional[Sequence[Any]] = None, +) -> Dict[str, Any]: + """Reorganisation proposals for the memory layer. + + Two independent sources, because they answer different questions and only + one of them is answerable from a run's evidence: + + * **inventory** (``scan_findings``) — duplicates, contradictions and stale + entries, which require reading the store; + * **behaviour** (``views``) — memories that keep being injected into turns + that fail, which requires reading the runs. + + The behavioural signal is deliberately *not* framed as duplication. A + memory injected repeatedly is either helping or not, and the difference is + the outcome of the turns it landed in — never the count on its own. + """ + from core.evolution.generators.memory_ops import OP_MERGE, OP_REWEIGHT, SCOPE_USER, MemoryOp, partition_by_approval + from core.evolution.generators.memory_scan import FINDING_CONTRADICTION, FINDING_DUPLICATE, FINDING_STALE + + retrieved: Dict[str, int] = {} + helped: Dict[str, int] = {} + for view in views: + for ref in view.get("memory_refs") or []: + key = str(ref) + retrieved[key] = retrieved.get(key, 0) + 1 + if view.get("verdict") == "success": + helped[key] = helped.get(key, 0) + 1 + + ops: List[MemoryOp] = [] + contradictions: List[Dict[str, Any]] = [] + + # 1. Inventory findings, from an actual scan of the store. + for finding in scan_findings or []: + payload = finding.to_dict() if hasattr(finding, "to_dict") else dict(finding) + kind = payload.get("kind") + refs = [str(r) for r in (payload.get("refs") or [])] + if kind == FINDING_DUPLICATE and len(refs) >= 2: + # The second entry loses. Content is not rewritten and nothing is + # removed — the survivor already says the same thing. + ops.append( + MemoryOp( + operation=OP_MERGE, + memory_ref=refs[1], + scope=SCOPE_USER, + reason=payload.get("reason", ""), + before={"injected": True, "duplicate_of": refs[0]}, + after={"injected": False, "superseded_by": refs[0]}, + ) + ) + elif kind == FINDING_STALE and refs: + ops.append( + MemoryOp( + operation=OP_REWEIGHT, + memory_ref=refs[0], + scope=SCOPE_USER, + reason=payload.get("reason", ""), + before={"weight": 1.0}, + after={"weight": 0.7}, + ) + ) + elif kind == FINDING_CONTRADICTION: + # Neither side is automatically right, and picking by recency is + # how a corrected mistake becomes the new truth. This is reported + # for a human, with both sides and their dates. + contradictions.append(payload) + + # 2. Behavioural: injected often, associated with failure. Down-weighted, + # never deleted — a wrong reweight is recoverable and a wrong deletion is + # not. + for ref, count in retrieved.items(): + if count < cycle_extras_constants.DEMOTE_MIN_HITS: + continue + rate = helped.get(ref, 0) / count + if rate <= cycle_extras_constants.DEMOTE_SUCCESS_CEILING: + ops.append( + MemoryOp( + operation=OP_REWEIGHT, + memory_ref=ref, + scope=SCOPE_USER, + reason=( + f"被注入 {count} 次，其中仅 {helped.get(ref, 0)} 次所在回答成功" + "——检索到它并没有帮上忙" + ), + before={"weight": 1.0}, + after={"weight": 0.5}, + ) + ) + + auto, needs_review = partition_by_approval(ops) + return { + "distinct_memories_retrieved": len(retrieved), + "merge_proposals": sum(1 for op in ops if op.operation == OP_MERGE), + "reweight_proposals": sum(1 for op in ops if op.operation == OP_REWEIGHT), + "contradictions": contradictions, + # Reported separately because the split *is* the governance boundary: + # reweighting is reversible and may proceed, merging changes what gets + # injected and may not. + "auto_applicable": [op.to_dict() for op in auto], + "needs_review": [op.to_dict() for op in needs_review], + "ops": ops, + } diff --git a/src/backend/core/evolution/cycle/ordering.py b/src/backend/core/evolution/cycle/ordering.py new file mode 100644 index 00000000..2df58c89 --- /dev/null +++ b/src/backend/core/evolution/cycle/ordering.py @@ -0,0 +1,103 @@ +"""Learn tool orderings that hold across successful episodes and apply them.""" +from __future__ import annotations +from collections import defaultdict +from typing import Any, Dict, List, Optional, Sequence, Set, Tuple +from core.evolution.cycle import patterns +from core.evolution.cycle import promotion_constants +from core.evolution.cycle import promotion_types + +def extract_ordering_constraints( + episodes: Sequence[Dict[str, Any]], + *, + min_support: int = promotion_constants.MIN_SUPPORT, + min_contexts: int = promotion_constants.MIN_CONSTRAINT_CONTEXTS, +) -> List[promotion_types.OrderingConstraint]: + """Find orderings that hold *invariantly* across successful episodes. + + Conflicting evidence vetoes a constraint outright rather than letting the + majority win. A pair that appears both ways round is not a rule — it is a + signal that ordering depends on context the system cannot yet see, and + promoting the popular direction would actively break the minority case. + """ + forward: Dict[Tuple[str, str], Set[str]] = defaultdict(set) + by_task_type: Dict[Tuple[str, str], Set[str]] = defaultdict(set) + + for episode in episodes: + if episode.get("verdict") != "success": + continue + tools = [str(t) for t in (episode.get("tool_sequence") or [])] + if len(tools) < 2: + continue + shape = patterns.pattern_signature(sorted(set(tools))) + task_type = str(episode.get("task_type") or "unknown") + for i, earlier in enumerate(tools): + for later in tools[i + 1 :]: + if earlier == later: + continue + forward[(earlier, later)].add(shape) + by_task_type[(earlier, later)].add(task_type) + + constraints: List[promotion_types.OrderingConstraint] = [] + for (a, b), shapes in forward.items(): + validated = by_task_type[(a, b)] + contradicted = by_task_type.get((b, a), set()) + + # A pair that flips *within the same task type* is genuinely ambiguous + # there — no amount of context resolves it, so the rule is dropped for + # that type rather than resolved by majority. + ambiguous = validated & contradicted + validated = validated - ambiguous + if not validated: + continue + + support = sum( + 1 + for e in episodes + if e.get("verdict") == "success" + and a in (e.get("tool_sequence") or []) + and b in (e.get("tool_sequence") or []) + and str(e.get("task_type") or "unknown") in validated + ) + if support < min_support or len(shapes) < min_contexts: + continue + constraints.append( + promotion_types.OrderingConstraint( + before=a, + after=b, + support=support, + contexts=len(shapes), + validated_in=tuple(sorted(validated)), + contradicted_in=tuple(sorted(contradicted | ambiguous)), + ) + ) + + constraints.sort(key=lambda c: (c.contexts, c.support), reverse=True) + return constraints + + +def apply_ordering_constraints( + tools: Sequence[str], + constraints: Sequence[promotion_types.OrderingConstraint], + task_type: Optional[str] = None, +) -> List[str]: + """Reorder a task's tools to satisfy every applicable constraint. + + A stable topological sort: tools with no constraint between them keep their + original relative order, so applying a rule about one pair never silently + reshuffles everything else. + """ + items = list(tools) + applicable = [c for c in constraints if c.applies_to(items, task_type)] + if not applicable: + return items + + for _ in range(len(items)): + changed = False + for constraint in applicable: + i, j = items.index(constraint.before), items.index(constraint.after) + if i > j: + items.insert(j, items.pop(i)) + changed = True + if not changed: + break + return items diff --git a/src/backend/core/evolution/cycle/patterns.py b/src/backend/core/evolution/cycle/patterns.py new file mode 100644 index 00000000..56ffa95f --- /dev/null +++ b/src/backend/core/evolution/cycle/patterns.py @@ -0,0 +1,113 @@ +"""Cluster episodes into recurring patterns.""" +from __future__ import annotations +from collections import defaultdict +from dataclasses import dataclass, field +from typing import Any, Dict, List, Sequence +from core.evolution.cycle import promotion_constants + +@dataclass +class Pattern: + """A recurring shape found across episodes.""" + + kind: str + signature: str + support: int + episode_ids: List[str] = field(default_factory=list) + success_rate: float = 0.0 + tool_sequence: List[str] = field(default_factory=list) + skill_sequence: List[str] = field(default_factory=list) + memory_refs: List[str] = field(default_factory=list) + single_shot: bool = False + + @property + def meets_support(self) -> bool: + return self.support >= promotion_constants.MIN_SUPPORT or self.single_shot + + def to_dict(self) -> Dict[str, Any]: + return { + "kind": self.kind, + "signature": self.signature, + "support": self.support, + "success_rate": round(self.success_rate, 3), + "tool_sequence": self.tool_sequence, + "skill_sequence": self.skill_sequence, + "memory_refs": self.memory_refs, + "single_shot": self.single_shot, + "episode_ids": self.episode_ids[:20], + } + + +def pattern_signature(items: Sequence[str]) -> str: + return "→".join(items) + + +def discover_patterns(episodes: List[Dict[str, Any]]) -> List[Pattern]: + """Cluster episodes into recurring shapes. + + ``episodes`` are plain dicts so this stays testable without a database and + usable over backfilled history (which has no asset bundle and therefore + cannot be replayed, but is perfectly good for finding patterns). + """ + by_tool_seq: Dict[str, List[Dict[str, Any]]] = defaultdict(list) + by_skill_seq: Dict[str, List[Dict[str, Any]]] = defaultdict(list) + + for episode in episodes: + tools = [str(t) for t in (episode.get("tool_sequence") or [])] + skills = [str(s) for s in (episode.get("skill_sequence") or [])] + if tools: + by_tool_seq[pattern_signature(tools)].append(episode) + if len(skills) >= 2: + by_skill_seq[pattern_signature(skills)].append(episode) + + patterns: List[Pattern] = [] + + for signature, group in by_tool_seq.items(): + successes = [e for e in group if e.get("verdict") == "success"] + failures = [e for e in group if e.get("verdict") == "failed"] + rate = len(successes) / len(group) if group else 0.0 + + if len(successes) >= promotion_constants.MIN_SUPPORT and rate >= promotion_constants.SKILL_MIN_SUCCESS_RATE: + patterns.append( + Pattern( + kind=promotion_constants.PATTERN_SUCCESS_SUBSEQUENCE, + signature=signature, + support=len(successes), + episode_ids=[e.get("episode_id", "") for e in successes], + success_rate=rate, + tool_sequence=signature.split("→"), + memory_refs=sorted( + { + ref + for e in successes + for ref in (e.get("memory_refs") or []) + } + ), + ) + ) + if len(failures) >= promotion_constants.MIN_SUPPORT: + patterns.append( + Pattern( + kind=promotion_constants.PATTERN_REPEATED_FAILURE, + signature=signature, + support=len(failures), + episode_ids=[e.get("episode_id", "") for e in failures], + success_rate=rate, + tool_sequence=signature.split("→"), + ) + ) + + for signature, group in by_skill_seq.items(): + if len(group) >= promotion_constants.WORKFLOW_MIN_COMBO_FREQUENCY: + successes = [e for e in group if e.get("verdict") == "success"] + patterns.append( + Pattern( + kind=promotion_constants.PATTERN_SUCCESS_SUBSEQUENCE, + signature=signature, + support=len(group), + episode_ids=[e.get("episode_id", "") for e in group], + success_rate=len(successes) / len(group), + skill_sequence=signature.split("→"), + ) + ) + + return [p for p in patterns if p.meets_support] diff --git a/src/backend/core/evolution/cycle/profile_release.py b/src/backend/core/evolution/cycle/profile_release.py new file mode 100644 index 00000000..795ca5b0 --- /dev/null +++ b/src/backend/core/evolution/cycle/profile_release.py @@ -0,0 +1,281 @@ +"""Publish approved agent profiles and sub-agents.""" +from __future__ import annotations +from datetime import datetime, timezone +from typing import Any, Dict, Optional +from core.db.engine import SessionLocal +import logging +from core.evolution.cycle import activation_policy +from core.evolution.cycle import activation_types + +logger = logging.getLogger(__name__) + +def materialise_profile_candidate( + candidate_id: str, *, approver: str, tenant_id: str = "default" +) -> Dict[str, Any]: + """Publish an approved agent profile. + + Profiles do not ride the shadow → canary ladder that skills do, and the + reason is not leniency. A profile *is* the assembly, so a canary would mean + two different assemblies running concurrently under one profile id, with + every downstream measurement of either one confounded by the other. Instead + a profile is scoped: it governs one task type, and its blast radius is that + task type's traffic, recorded on the release row like any other change. + + Rolling back is switching ``is_active`` off, which restores the built-in + assembly on the next run — no restart, because profiles are read per run. + """ + from core.db.models.evolution import EvolutionCandidate, EvolutionRelease + + from core.db.models.evolution import EvolutionAgentProfile + from core.evolution.profiles.agent_profile import AgentProfile, validate_profile + from core.evolution.profiles.agent_profile_constants import TARGET_KIND + from core.evolution.cycle.release import ACTIVE as _ACTIVE, new_release_id + + if not approver: + raise activation_types.ActivationError("approver_required", "激活必须署名") + + with SessionLocal() as db: + candidate = db.get(EvolutionCandidate, candidate_id) + if candidate is None: + raise activation_types.ActivationError("candidate_not_found", f"候选不存在: {candidate_id}") + if candidate.target_kind != TARGET_KIND: + raise activation_types.ActivationError( + "unsupported_kind", f"该入口只处理编排候选，收到 {candidate.target_kind}" + ) + if approver == (candidate.proposer or ""): + raise activation_types.ActivationError("self_approval_forbidden", "生成者不能激活自己提出的候选") + if activation_policy.is_frozen(db): + raise activation_types.ActivationError("layer_frozen", "进化已被全局冻结") + + payload: Dict[str, Any] = {} + for change in candidate.ir.get("changes") or []: + if isinstance(change.get("profile"), dict): + payload = change["profile"] + break + if not payload: + raise activation_types.ActivationError("no_profile_payload", "候选未携带 profile 定义") + + profile = AgentProfile.from_dict(payload) + profile.scope = {**(profile.scope or {}), "tenant_id": tenant_id or "default"} + + # Re-validated at activation against the *current* tool grant, not the + # one that existed when the candidate was written. A grant narrowed in + # the meantime must not be widened again by approving an older proposal. + from core.capabilities.catalog.api import get_enabled_ids + + base_tools = [str(t) for t in get_enabled_ids("tools")] + [ + str(t) for t in get_enabled_ids("mcp") + ] + ok, problems = validate_profile(profile, base_tools=base_tools) + if not ok: + raise activation_types.ActivationError("profile_invalid", f"编排定义未通过校验: {problems}") + + previous = db.get(EvolutionAgentProfile, profile.profile_id) + previous_payload = dict(previous.payload) if previous is not None else None + + # One active profile per (task type, scope). Two would make the + # resolution order the thing that decides behaviour, which is not a + # decision anyone reviewed. + for row in ( + db.query(EvolutionAgentProfile) + .filter(EvolutionAgentProfile.is_active.is_(True)) + .all() + ): + if row.profile_id == profile.profile_id: + continue + same_types = set(row.task_types or []) == set(profile.task_types) + same_scope = dict(row.scope or {}) == dict(profile.scope) + if same_types and same_scope: + row.is_active = False + + row = previous or EvolutionAgentProfile(profile_id=profile.profile_id) + row.version = profile.version + row.task_types = list(profile.task_types) + row.scope = dict(profile.scope) + row.payload = profile.to_dict() + row.candidate_id = candidate_id + row.is_active = True + row.created_by = f"evolution:{approver}" + if previous is None: + db.add(row) + + release = EvolutionRelease( + release_id=new_release_id(), + candidate_id=candidate_id, + target_kind=TARGET_KIND, + target_asset_id=profile.profile_id, + stage=_ACTIVE, + traffic_percent=100, + scope_filter=dict(profile.scope), + guardrails={"previous": previous_payload, "proposer": candidate.proposer or ""}, + rollback_version_id=(previous_payload or {}).get("version") or "builtin", + approved_by=approver, + ) + db.add(release) + + candidate.status = _ACTIVE + candidate.approved_by = approver + candidate.approved_at = datetime.now(timezone.utc) + db.commit() + + # 血缘：哪些技能被装配进了这个 profile。 + try: + from core.evolution.evidence import lineage + + lineage.record_links( + [ + ( + "skill", + str(skill_id), + lineage.REL_ASSEMBLED_INTO, + "agent_profile", + f"{profile.profile_id}@{profile.version}", + ) + for skill_id in profile.skill_ids + ], + candidate_id=candidate_id, + ) + except Exception as exc: # noqa: BLE001 + logger.debug("[activation] profile lineage skipped: %s", exc) + + logger.info( + "[activation] profile %s active for %s (tools=%s skills=%s)", + profile.profile_id, + profile.task_types or ["*"], + "narrowed" if profile.tool_allowlist is not None else "unchanged", + len(profile.skill_ids), + ) + return { + "profile_id": profile.profile_id, + "release_id": release.release_id, + "task_types": list(profile.task_types), + "scope": dict(profile.scope), + "tool_allowlist": profile.tool_allowlist, + "skill_ids": list(profile.skill_ids), + "replaced_previous": previous_payload is not None, + } + + +def materialise_subagent_candidate( + candidate_id: str, *, approver: str, owner_user_id: Optional[str] = None +) -> Dict[str, Any]: + """Create an approved sub-agent, and nothing else. + + Deliberately does **not** install the route that sends work to it. The route + is a sentence in the parent's prompt, and a sentence in the parent's prompt + changes every request the parent handles — so it goes through prompt review + as its own candidate. Creating the agent here and routing to it there means + approving the agent cannot silently also approve a behaviour change. + + The agent is created disabled for the same reason: an agent nobody routes to + does nothing, and one that is enabled before its route is reviewed is a + capability sitting in the product that no reviewer has seen used. + """ + from core.db.models.evolution import EvolutionCandidate, EvolutionRelease + + from core.db.models import UserAgent + from core.evolution.cycle.release import SHADOW, new_release_id + + if not approver: + raise activation_types.ActivationError("approver_required", "激活必须署名") + + with SessionLocal() as db: + candidate = db.get(EvolutionCandidate, candidate_id) + if candidate is None: + raise activation_types.ActivationError("candidate_not_found", f"候选不存在: {candidate_id}") + if candidate.target_kind != "subagent": + raise activation_types.ActivationError( + "unsupported_kind", f"该入口只处理子智能体候选，收到 {candidate.target_kind}" + ) + if approver == (candidate.proposer or ""): + raise activation_types.ActivationError("self_approval_forbidden", "生成者不能激活自己提出的候选") + if activation_policy.is_frozen(db): + raise activation_types.ActivationError("layer_frozen", "进化已被全局冻结") + + spec: Dict[str, Any] = {} + for change in candidate.ir.get("changes") or []: + if change.get("agent_id"): + spec = change + break + if not spec: + raise activation_types.ActivationError("no_subagent_payload", "候选未携带子智能体定义") + + # The subset check runs again at activation against the current grant. + from core.capabilities.catalog.api import get_enabled_ids + + base_tools = {str(t) for t in get_enabled_ids("tools")} | { + str(t) for t in get_enabled_ids("mcp") + } + requested = {str(t) for t in (spec.get("tools_allowlist") or [])} + widened = sorted(requested - base_tools) + if widened: + raise activation_types.ActivationError( + "tool_escalation", f"子智能体工具集超出主体授权: {widened}" + ) + if spec.get("may_delegate"): + raise activation_types.ActivationError("subagent_nesting", "子智能体不得再派生子智能体") + + agent_id = str(spec.get("agent_id")) + existing = db.get(UserAgent, agent_id) + if existing is not None: + raise activation_types.ActivationError("agent_exists", f"子智能体已存在: {agent_id}") + + agent = UserAgent( + agent_id=agent_id, + owner_type="user" if owner_user_id else "admin", + user_id=owner_user_id, + name=str(spec.get("name") or agent_id)[:255], + description=str(spec.get("responsibility") or "")[:2000], + system_prompt=str(spec.get("responsibility") or ""), + skill_ids=[str(s) for s in (spec.get("required_skills") or [])], + mcp_server_ids=sorted(requested), + kb_ids=[], + plugin_ids=[], + max_iters=12, + timeout=int(spec.get("timeout") or 180), + # Disabled until its route is approved — see the docstring. + is_enabled=False, + extra_config={ + "origin": "evolution", + "candidate_id": candidate_id, + "route_hint": str(spec.get("route_hint") or ""), + "task_types": [str(t) for t in (spec.get("task_types") or [])], + "tools_allowlist": sorted(requested), + "may_delegate": False, + }, + ) + db.add(agent) + + release = EvolutionRelease( + release_id=new_release_id(), + candidate_id=candidate_id, + target_kind="subagent", + target_asset_id=agent_id, + # Reaches nobody until the route exists, which is what shadow means. + stage=SHADOW, + traffic_percent=0, + scope_filter={"owner_user_id": owner_user_id} if owner_user_id else {}, + guardrails={"proposer": candidate.proposer or "", "route_pending": True}, + rollback_version_id="v0", + approved_by=approver, + ) + db.add(release) + + candidate.approved_by = approver + candidate.approved_at = datetime.now(timezone.utc) + db.commit() + + logger.info( + "[activation] subagent %s created (disabled, route pending) tools=%s", + agent_id, + sorted(requested), + ) + return { + "agent_id": agent_id, + "release_id": release.release_id, + "stage": SHADOW, + "is_enabled": False, + "tools_allowlist": sorted(requested), + "route_hint": str(spec.get("route_hint") or ""), + "next_step": "路由句需作为提示词候选单独审批后，子智能体才会被启用", + } diff --git a/src/backend/core/evolution/cycle/promotion.py b/src/backend/core/evolution/cycle/promotion.py new file mode 100644 index 00000000..2a57ef72 --- /dev/null +++ b/src/backend/core/evolution/cycle/promotion.py @@ -0,0 +1,317 @@ +"""Promote recurring sequences, procedures and skill combinations into proposals.""" +from __future__ import annotations +from typing import Any, Dict, List, Optional, Sequence, Tuple +import logging +from core.evolution.cycle import patterns +from core.evolution.cycle import promotion_constants +from core.evolution.cycle import promotion_types + +logger = logging.getLogger(__name__) + +def dominant_shared_intent( + pattern: patterns.Pattern, + episodes: Optional[Sequence[Dict[str, Any]]], + *, + support_floor: int, +): + """The one intent behind a sequence's supporting episodes, or ``None``. + + ``None`` is a refusal, and deliberately covers the honest-uncertainty cases + too: no views supplied, request text too short to cluster, no clustering + method available. "We cannot tell whether these conversations had anything + in common" and "they had nothing in common" both mean no SOP claim can be + made — treating the first as a pass is how single-tool coincidences used to + become skills. + """ + if not episodes: + return None + wanted = {str(e) for e in pattern.episode_ids} + supporting = [e for e in episodes if str(e.get("episode_id") or "") in wanted] + if len(supporting) < support_floor: + return None + + from core.evolution.assessment.similarity import METHOD_NONE, cluster_intents + + clusters, method = cluster_intents(supporting, min_support=2) + if method == METHOD_NONE or not clusters: + return None + top = clusters[0] + if len(top.episode_ids) < support_floor: + return None + if len(top.episode_ids) / len(supporting) < promotion_constants.INTENT_COHESION_RATIO: + return None + # 复现必须跨会话：occasions 按 chat_id 去重，同一会话里反复跑同一套 + # 流程只算一次场合，不构成"这件事总在发生"。 + if top.occasions < support_floor: + return None + return top + + +def promote_tool_sequence_to_skill( + pattern: patterns.Pattern, + *, + episodes: Optional[Sequence[Dict[str, Any]]] = None, + min_support: Optional[int] = None, + skill_credit: float = 0.0, + workflow_credit: float = 0.0, + existing_skill_signatures: Optional[Dict[str, str]] = None, +) -> Optional[promotion_types.PromotionProposal]: + """Compile a recurring successful tool sequence into a skill proposal. + + A recurring sequence alone is *not* evidence of a reusable capability. + Generic tools recur across unrelated conversations by construction, and the + limiting case — a whole conversation whose "sequence" is one call to + ``view_text_file`` — says only that the task was trivial. What justifies an + SOP is commonality of *purpose*: the same kind of request, on separate + occasions, repeatedly completed by the same non-trivial procedure. So the + gates hold together, and each excludes a specific way of being wrong: + + * the sequence is **non-trivial** (≥ :data:`MIN_TOOL_SEQUENCE_LEN` steps) — + wrapping one generic tool adds an asset and no knowledge; + * the supporting episodes **share an intent** — a dominant cluster over + their request text (via :mod:`core.evolution.assessment.similarity`) covers + ≥ :data:`INTENT_COHESION_RATIO` of them; without that the recurrence is a + coincidence of tooling, and no commonality means no promotion; + * the recurrence spans **separate conversations** — one chat re-running the + same procedure is one occasion, not evidence it keeps coming back; + * the recurrences **succeed** at ≥ :data:`SKILL_MIN_SUCCESS_RATE`. + + ``episodes`` are the flattened views the pattern was discovered from; the + intent gate reads their request text. A caller that cannot supply them gets + a refusal, not the benefit of the doubt. + + Named for what it reads. This was called ``promote_memory_to_skill`` and + carried ``from_layer="memory"``, but no memory content ever enters the + result — ``pattern.memory_refs`` are hashes of whatever happened to be + retrieved during the supporting episodes. The name promised a memory→skill + chain the code did not have; :func:`promote_procedural_memory_to_skill` is + that chain. + + Refuses when orchestration is the better explanation: a stable tool ordering + can equally mean "the workflow is fixed" rather than "there is a reusable + skill here". Promoting in that case creates a skill with no independent + content, which then has to be maintained forever. + """ + if pattern.kind != promotion_constants.PATTERN_SUCCESS_SUBSEQUENCE or not pattern.tool_sequence: + return None + support_floor = promotion_constants.MIN_SUPPORT if min_support is None else int(min_support) + if len(pattern.tool_sequence) < promotion_constants.MIN_TOOL_SEQUENCE_LEN: + logger.info( + "[promotion] skipping %s: %d-step sequence is below the SOP floor", + pattern.signature, + len(pattern.tool_sequence), + ) + return None + if pattern.support < support_floor: + return None + if pattern.success_rate < promotion_constants.SKILL_MIN_SUCCESS_RATE: + return None + if workflow_credit > skill_credit: + logger.info( + "[promotion] skipping skill promotion for %s: workflow credit higher", + pattern.signature, + ) + return None + + intent = dominant_shared_intent(pattern, episodes, support_floor=support_floor) + if intent is None: + logger.info( + "[promotion] skipping %s: supporting episodes share no intent — a " + "recurring sequence without a recurring request is not an SOP", + pattern.signature, + ) + return None + covered = [e for e in pattern.episode_ids if e in set(intent.episode_ids)] + + existing = (existing_skill_signatures or {}).get(pattern.signature) + return promotion_types.PromotionProposal( + from_layer="episode", + to_layer="skill", + signature=pattern.signature, + support=len(covered), + rationale=( + f"「{intent.representative[:40]}」类请求在 {intent.occasions} 次独立会话中" + f"反复出现，每次都以同一 {len(pattern.tool_sequence)} 步工具序列成功完成" + f"（{len(covered)} 个 Episode，成功率 {pattern.success_rate:.0%}）——" + "同类意图 + 稳定做法，具备沉淀为 SOP 的共性" + ), + # Co-retrieved memories are *context*, not sources. They are carried so + # a reviewer can see what was in view, and deliberately not demoted — + # nothing of theirs went into this skill, so down-weighting them would + # remove knowledge the skill does not carry. + source_refs=[], + payload={ + "tool_sequence": pattern.tool_sequence, + "episode_ids": covered[:20], + "intent": intent.representative, + "occasions": intent.occasions, + "co_retrieved_memory_refs": list(pattern.memory_refs), + }, + # An equivalent skill already exists → patch it instead of adding a twin. + patch_target=existing, + ) + + +def promote_procedural_memory_to_skill( + *, + memories: Sequence[Dict[str, Any]], + intent_cluster: Dict[str, Any], + occurrences: int, + success_rate: float, + plan_variety: int, + existing_skill_id: Optional[str] = None, +) -> Optional[promotion_types.PromotionProposal]: + """Compile *how work is done here* into a skill proposal. + + This is the chain the design described and the code never had. Four + conditions must hold together, and each excludes a specific way of being + wrong: + + * the memories are **procedural** — a fact frozen into a document stops + being current, and a preference is already covered by one profile line; + * the intent recurs across **separate occasions** — one conversation + mentioning something three times is one occasion; + * the recurrences **succeed** — distilling a failing approach makes it the + house style; + * the plans **varied** — the same request handled identically every time is + already effectively a skill, and writing one down adds an asset to + maintain while changing nothing. + + The proposal carries the memories' **content**, which is what makes this a + compilation rather than a rename, and their refs, which is what lets the + source memories be demoted when the skill goes live and restored when it is + rolled back. + """ + procedural = [ + m + for m in memories + if str(m.get("memory_type") or "") == "procedural" and str(m.get("content") or "").strip() + ] + if not procedural: + return None + if occurrences < promotion_constants.PROCEDURAL_MIN_OCCURRENCES: + return None + if success_rate < promotion_constants.PROCEDURAL_MIN_SUCCESS_RATE: + return None + if plan_variety < 2: + return None + + representative = str(intent_cluster.get("representative") or "") + return promotion_types.PromotionProposal( + from_layer="memory", + to_layer="skill", + signature="proc:" + "|".join(sorted(str(m.get("ref") or "") for m in procedural)), + support=occurrences, + rationale=( + f"「{representative[:40]}」类请求出现 {occurrences} 次、成功率 " + f"{success_rate:.0%}，却用了 {plan_variety} 种不同做法；" + f"其中 {len(procedural)} 条做法/口径已被反复检索，应编译为一次性装配的技能" + ), + source_refs=[str(m.get("ref") or "") for m in procedural if m.get("ref")], + payload={ + "procedures": [ + { + "rule": str(m.get("content") or ""), + "why": str(m.get("why") or ""), + "applies_to": str(m.get("applies_to") or ""), + } + for m in procedural + ], + "episode_ids": [str(e) for e in (intent_cluster.get("episode_ids") or [])][:20], + "intent": representative, + "memory_refs": [str(m.get("ref") or "") for m in procedural if m.get("ref")], + }, + patch_target=existing_skill_id, + ) + + +def promote_skills_to_orchestration( + pattern: patterns.Pattern, + *, + combo_success_rate: float, + individual_success_rate: float, + task_types: Optional[Sequence[str]] = None, + skill_versions: Optional[Dict[str, str]] = None, +) -> Optional[promotion_types.PromotionProposal]: + """Promote a recurring skill combination into an **agent profile** proposal. + + Promoting it into a *workflow* — a fixed DAG — was the wrong target: the + main axis is a ReAct loop, which never executes a DAG, so a workflow asset + could only ever govern the separate autonomous-loop product. What a + recurring skill combination actually says is "for this kind of task, these + capabilities belong in view together", and the thing that decides what is in + view is the profile. + + The combination must *beat its parts*: frequency alone would promote every + common sequence regardless of whether grouping them helps. The proposal is + declarative — a skill set and the task types it applies to — and carries no + executable content. + """ + if not pattern.skill_sequence or len(pattern.skill_sequence) < 2: + return None + if pattern.support < promotion_constants.WORKFLOW_MIN_COMBO_FREQUENCY: + return None + if combo_success_rate - individual_success_rate < promotion_constants.WORKFLOW_MIN_UPLIFT: + return None + + versions = skill_versions or {} + return promotion_types.PromotionProposal( + from_layer="skill", + to_layer="agent_profile", + signature=pattern.signature, + support=pattern.support, + rationale=( + f"技能组合出现 {pattern.support} 次，组合成功率 {combo_success_rate:.0%} " + f"显著高于零散调用 {individual_success_rate:.0%}——" + "应固化为该类任务的技能组合，而不是每次重新挑选" + ), + source_refs=list(pattern.skill_sequence), + payload={ + "skill_ids": list(dict.fromkeys(pattern.skill_sequence)), + "task_types": [str(t) for t in (task_types or [])], + # Declared so the candidate auto-invalidates if a dependency retires + # — a profile pointing at a dead skill version is worse than none. + "depends_on": {s: versions.get(s, "") for s in pattern.skill_sequence}, + "kind": "declarative_policy", + }, + ) + + +def dependencies_satisfied( + proposal: promotion_types.PromotionProposal, live_skill_versions: Dict[str, str] +) -> Tuple[bool, List[str]]: + """Whether every declared dependency still exists at the declared version. + + This is the mechanism that makes cross-layer rollback work in one direction: + retire a skill and every profile built on it stops validating, instead of + quietly pointing at something that is no longer there. + """ + declared = (proposal.payload or {}).get("depends_on") or {} + stale = [ + skill_id + for skill_id, version in declared.items() + if version and live_skill_versions.get(skill_id) != version + ] + return (not stale), stale + + +def find_overlaps( + signatures: Dict[str, List[str]], *, min_shared: int = 2 +) -> List[Tuple[str, str, int]]: + """Assets whose **declared** step sequences overlap enough to be merge candidates. + + ``signatures`` must be each asset's own declared tools — the tools its + document names. Feeding it the tools observed in episodes where the asset + was merely loaded produces a merge proposal for every pair of skills that + happened to be enabled together, which is guaranteed rather than + informative: co-enabled skills share the episode's whole tool set by + construction. + """ + overlaps: List[Tuple[str, str, int]] = [] + asset_ids = sorted(signatures) + for i, a in enumerate(asset_ids): + for b in asset_ids[i + 1 :]: + shared = len(set(signatures[a]) & set(signatures[b])) + if shared >= min_shared: + overlaps.append((a, b, shared)) + return overlaps diff --git a/src/backend/core/evolution/cycle/promotion_constants.py b/src/backend/core/evolution/cycle/promotion_constants.py new file mode 100644 index 00000000..992f0aa1 --- /dev/null +++ b/src/backend/core/evolution/cycle/promotion_constants.py @@ -0,0 +1,80 @@ +"""Pattern kinds and support thresholds for promotion.""" +from __future__ import annotations + + +PATTERN_REPEATED_FAILURE = "repeated_failure" + + +PATTERN_SUCCESS_SUBSEQUENCE = "success_subsequence" + + +PATTERN_DRIFT = "execution_drift" + + +MIN_SUPPORT = 5 + + +SINGLE_SHOT_CONFIDENCE = 0.85 + + +SKILL_MIN_SUCCESS_RATE = 0.7 + + +WORKFLOW_MIN_COMBO_FREQUENCY = 5 + + +WORKFLOW_MIN_UPLIFT = 0.05 + + +MIN_TOOL_SEQUENCE_LEN = 3 + + +INTENT_COHESION_RATIO = 0.6 + + +PROCEDURAL_MIN_OCCURRENCES = 3 + + +PROCEDURAL_MIN_SUCCESS_RATE = 0.6 + + +MIN_EXPOSURE_DAYS = 14 + + +MIN_OFFERS_BEFORE_JUDGING = 10 + + +MIN_OPENS_BEFORE_JUDGING = 3 + + +FOLLOW_THROUGH_FLOOR = 0.4 + + +SUCCESS_FLOOR = 0.4 + + +REASON_NEVER_OPENED = "never_opened" + + +REASON_IGNORED_WHEN_OPENED = "ignored_when_opened" + + +REASON_LOW_SUCCESS = "low_success_when_opened" + + +REASON_HARMFUL = "worse_than_without" + + +ACTION_LOWER_EXPOSURE = "lower_exposure" + + +ACTION_REWRITE = "rewrite" + + +ACTION_RETIRE = "retire" + + +ACTION_ROLLBACK = "rollback" + + +MIN_CONSTRAINT_CONTEXTS = 2 diff --git a/src/backend/core/evolution/cycle/promotion_types.py b/src/backend/core/evolution/cycle/promotion_types.py new file mode 100644 index 00000000..9bde9881 --- /dev/null +++ b/src/backend/core/evolution/cycle/promotion_types.py @@ -0,0 +1,94 @@ +"""Promotion proposals, negative-transfer records and ordering constraints.""" +from __future__ import annotations +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional, Sequence, Tuple + +@dataclass +class PromotionProposal: + """A promotion the chain wants to make, before it becomes an IR candidate.""" + + from_layer: str + to_layer: str + signature: str + support: int + rationale: str + source_refs: List[str] = field(default_factory=list) + payload: Dict[str, Any] = field(default_factory=dict) + # Set when an equivalent asset already exists: promotion becomes a patch + # rather than a new asset, so the library does not fill with near-duplicates. + patch_target: Optional[str] = None + + def to_dict(self) -> Dict[str, Any]: + return { + "from_layer": self.from_layer, + "to_layer": self.to_layer, + "signature": self.signature, + "support": self.support, + "rationale": self.rationale, + "source_refs": list(self.source_refs), + "payload": self.payload, + "patch_target": self.patch_target, + } + + +@dataclass +class NegativeTransfer: + asset_id: str + task_type: str + before: float + after: float + + @property + def delta(self) -> float: + return self.after - self.before + + def to_dict(self) -> Dict[str, Any]: + return { + "asset_id": self.asset_id, + "task_type": self.task_type, + "before": round(self.before, 4), + "after": round(self.after, 4), + "delta": round(self.delta, 4), + } + + +@dataclass +class OrderingConstraint: + """A pairwise ordering that held across the supporting evidence. + + Carries the task types it was *validated in* and any it was *contradicted + in*. That distinction is what lets a genuinely context-dependent ordering be + used where it holds instead of being discarded everywhere. + """ + + before: str + after: str + support: int + contexts: int # distinct tool-set shapes it was observed in + validated_in: Tuple[str, ...] = () + contradicted_in: Tuple[str, ...] = () + + def applies_to(self, tools: Sequence[str], task_type: Optional[str] = None) -> bool: + if self.before not in tools or self.after not in tools: + return False + if task_type is None: + # No context given: only a rule contradicted nowhere may be used. + return not self.contradicted_in + if task_type in self.contradicted_in: + return False + if task_type in self.validated_in: + return True + # An unseen context. Extending there is only safe for a rule that has + # never been contradicted; one known to flip somewhere is precisely the + # kind that should not be guessed at. + return not self.contradicted_in + + def to_dict(self) -> Dict[str, Any]: + return { + "before": self.before, + "after": self.after, + "support": self.support, + "contexts": self.contexts, + "validated_in": list(self.validated_in), + "contradicted_in": list(self.contradicted_in), + } diff --git a/src/backend/core/evolution/cycle/prompt_candidates.py b/src/backend/core/evolution/cycle/prompt_candidates.py new file mode 100644 index 00000000..629dc714 --- /dev/null +++ b/src/backend/core/evolution/cycle/prompt_candidates.py @@ -0,0 +1,95 @@ +"""Prompt-fragment candidates for failures attributed to the prompt.""" +from __future__ import annotations +import hashlib +from typing import Any, Dict, List +from core.evolution.evidence.credit import CreditFeatures, assign_credit +from core.evolution.ir import RISK_HIGH, EvolutionIR, InvariantViolation, Rollback, validate_ir +import logging +from core.evolution.cycle import candidate_records + +logger = logging.getLogger(__name__) + +def prompt_candidates(views: List[Dict[str, Any]], *, dry_run: bool) -> List[Dict[str, Any]]: + """Prompt fragments, for the failures attribution already blames on the prompt.""" + from core.evolution.generators.prompt_gen import generate_prompt_candidate + + payload, evidence, draft = candidate_records.run_async( + generate_prompt_candidate( + views, + cause="归因判定该失败源于提示词未声明相关约束", + existing_fragments=active_prompt_fragments(), + ) + ) + if payload is None or evidence is None: + return ( + [] + if draft.violations == ["insufficient_evidence"] + else [{"kind": "prompt", "written": False, "why_not": draft.to_dict()}] + ) + + decision = assign_credit( + CreditFeatures( + outcome_verdict="failed", + outcome_confidence=0.85, + prompt_missing_priority_rule=True, + ) + ) + if not decision.may_produce_candidate: + return [] + + asset_id = "auto-prompt-" + hashlib.sha256(payload["fragment"].encode("utf-8")).hexdigest()[:12] + ir = EvolutionIR( + target_kind="prompt", + target_asset_id=asset_id, + base_version="active", + operation="patch", + hypothesis=payload["cause"], + changes=[{"fragment": payload["fragment"], "task_types": payload["task_types"]}], + evidence_refs=evidence.episode_ids[:20], + # A prompt fragment applies to every request in scope; nothing this + # system produces has a wider reach. + risk_tier=RISK_HIGH, + scope={"level": "workspace"}, + rollback=Rollback(version_id="active", triggers=["success -3pp"]), + evaluation_plan={"replay_set": "auto_holdout", "primary_metric": "task_success"}, + ) + try: + validate_ir(ir, credit_decision=decision.to_dict()) + except InvariantViolation as violation: + return [{"kind": "prompt", "written": False, "why_not": {"code": violation.code}}] + if dry_run: + return [{"dry_run": True, "kind": "prompt", "asset_id": asset_id}] + candidate_id = candidate_records.persist_candidate( + ir, + credit=decision.to_dict(), + proposer="prompt_engine", + evidence_refs=evidence.episode_ids[:20], + ) + return ( + [{"candidate_id": candidate_id, "kind": "prompt", "asset_id": asset_id}] + if candidate_id + else [] + ) + + +def active_prompt_fragments() -> List[str]: + """Fragments already appended by earlier prompt candidates. + + Passed to the generator so it can decline rather than restate: a prompt that + accumulates three phrasings of the same instruction is measurably worse than + one that states it once. + """ + try: + from core.db.engine import SessionLocal as _Session + from core.db.models.evolution import EvolutionRelease as _Release + + with _Session() as db: + rows = ( + db.query(_Release) + .filter(_Release.target_kind == "prompt", _Release.stage == "active") + .all() + ) + return [str(row.target_asset_id) for row in rows] + except Exception as exc: # noqa: BLE001 + logger.debug("[evolution-loop] active prompt fragments unavailable: %s", exc) + return [] diff --git a/src/backend/core/evolution/readiness.py b/src/backend/core/evolution/cycle/readiness.py similarity index 93% rename from src/backend/core/evolution/readiness.py rename to src/backend/core/evolution/cycle/readiness.py index 9685f4db..23daf06f 100644 --- a/src/backend/core/evolution/readiness.py +++ b/src/backend/core/evolution/cycle/readiness.py @@ -81,7 +81,7 @@ def _check_embedding() -> Requirement: surface, so this makes one real call. """ try: - from core.kb.kb_vector import embed_text + from core.models.embedding import embed_text vector = embed_text("能力进化就绪检查", timeout=8.0) except Exception as exc: # noqa: BLE001 @@ -162,13 +162,11 @@ def _check_generator() -> Requirement: # run often — spends a request to re-learn something that was true 30 seconds # ago. Short enough that fixing a misconfiguration shows up promptly. _CACHE_TTL_S = 300.0 -_cached: Optional[Tuple[float, "Readiness"]] = None def reset_cache() -> None: """Drop the cached probe, so a configuration fix is picked up immediately.""" - global _cached - _cached = None + process_state().cached = None def check_readiness(*, use_cache: bool = True) -> Readiness: @@ -178,15 +176,26 @@ def check_readiness(*, use_cache: bool = True) -> Readiness: are fixed by different people: an embedding endpoint is infrastructure, a generation model is a model-configuration decision. """ - global _cached now = time.monotonic() - if use_cache and _cached is not None and now - _cached[0] < _CACHE_TTL_S: - return _cached[1] + if use_cache and process_state().cached is not None and now - process_state().cached[0] < _CACHE_TTL_S: + return process_state().cached[1] requirements = [_check_embedding(), _check_generator()] readiness = Readiness( ready=all(r.ok for r in requirements), requirements=requirements, ) - _cached = (now, readiness) + process_state().cached = (now, readiness) return readiness + + +@dataclass +class ReadinessState: + cached: Optional[Tuple[float, 'Readiness']] = None + + +_STATE = ReadinessState() + + +def process_state() -> ReadinessState: + return _STATE diff --git a/src/backend/core/evolution/release.py b/src/backend/core/evolution/cycle/release.py similarity index 100% rename from src/backend/core/evolution/release.py rename to src/backend/core/evolution/cycle/release.py diff --git a/src/backend/core/evolution/cycle/retirement.py b/src/backend/core/evolution/cycle/retirement.py new file mode 100644 index 00000000..724d042a --- /dev/null +++ b/src/backend/core/evolution/cycle/retirement.py @@ -0,0 +1,158 @@ +"""Find assets to retire and activations that caused negative transfer.""" +from __future__ import annotations +from typing import Any, Dict, List, Sequence +from core.evolution.cycle import promotion_constants +from core.evolution.cycle import promotion_types + +def find_retirement_candidates( + usage: Dict[str, Dict[str, Any]], +) -> List[Dict[str, Any]]: + """Assets that are not earning their place, and what to do about each. + + ``usage`` per asset: + + ``offers`` + turns where the skill was selected into the prompt (non-degraded + selections only). + ``opens`` + turns where the model actually opened its document. This is the usage + signal; ``offers`` alone cannot distinguish "never chosen" from "chosen + and useless". + ``follow_through`` + of the opens, the fraction where the subsequent tool calls stayed + inside what the skill declared. + ``success_rate`` + of the opens, the fraction of turns that succeeded. + ``uplift`` + success with the skill exposed minus success without it, where a paired + comparison exists. ``None`` when it does not. + ``exposure_days`` + how long it has been exposed at all. + + A library that only grows fills with near-duplicates and degrades selection + precision for everything in it — but every judgement here is a *proposal*, + because removing a capability produces no error, only a slow decline. + """ + out: List[Dict[str, Any]] = [] + for asset_id, stats in (usage or {}).items(): + offers = int(stats.get("offers", 0) or 0) + opens = int(stats.get("opens", 0) or 0) + exposure_days = float(stats.get("exposure_days", 0) or 0) + uplift = stats.get("uplift") + + if uplift is not None and float(uplift) <= -0.05: + out.append( + { + "asset_id": asset_id, + "reason": promotion_constants.REASON_HARMFUL, + "action": promotion_constants.ACTION_ROLLBACK, + "explanation": ( + f"成对比较显示有它反而更差（{float(uplift):+.0%}）——" + "这是唯一需要立即停止曝光的情况" + ), + **stats, + } + ) + continue + + if exposure_days < promotion_constants.MIN_EXPOSURE_DAYS: + # Too new to judge. Saying nothing is the honest output; the old + # rule's answer here was "retire it", which punished every skill for + # having just shipped. + continue + + if opens == 0: + if offers < promotion_constants.MIN_OFFERS_BEFORE_JUDGING: + continue + out.append( + { + "asset_id": asset_id, + "reason": promotion_constants.REASON_NEVER_OPENED, + "action": promotion_constants.ACTION_LOWER_EXPOSURE, + "explanation": ( + f"已被挂载 {offers} 次、" + # A hand-authored skill has no release record and so no + # finite exposure window — it has been there from the + # start. Formatting that as a number printed "曝光 inf 天". + + ( + "一直可用" + if exposure_days == float("inf") + else f"曝光 {exposure_days:.0f} 天" + ) + + "，模型一次都没打开——" + "多半是描述没写清它解决什么问题，而不是能力本身没用" + ), + **stats, + } + ) + continue + + if opens < promotion_constants.MIN_OPENS_BEFORE_JUDGING: + continue + + follow_through = stats.get("follow_through") + if follow_through is not None and float(follow_through) < promotion_constants.FOLLOW_THROUGH_FLOOR: + out.append( + { + "asset_id": asset_id, + "reason": promotion_constants.REASON_IGNORED_WHEN_OPENED, + "action": promotion_constants.ACTION_REWRITE, + "explanation": ( + f"被打开 {opens} 次，但只有 {float(follow_through):.0%} 的情况" + "后续动作真的照它说的做——文档本身需要重写，不是该退役" + ), + **stats, + } + ) + continue + + success_rate = float(stats.get("success_rate", 0.0) or 0.0) + if success_rate < promotion_constants.SUCCESS_FLOOR: + out.append( + { + "asset_id": asset_id, + "reason": promotion_constants.REASON_LOW_SUCCESS, + "action": promotion_constants.ACTION_RETIRE, + "explanation": ( + f"被打开 {opens} 次、照做了，但成功率只有 {success_rate:.0%}" + ), + **stats, + } + ) + return out + + +def detect_negative_transfer( + *, + asset_id: str, + related_task_types: Sequence[str], + before: Dict[str, float], + after: Dict[str, float], + threshold: float = 0.05, +) -> List[promotion_types.NegativeTransfer]: + """Find task types that got worse after an activation *and are unrelated to it*. + + Only unrelated tasks are examined. A dip on the task the change targeted is + a straightforward regression its own guardrail catches; the dangerous case + is collateral damage elsewhere, which no single release's guardrail is + watching and which accumulates silently across many individually-harmless + changes. + """ + related = set(related_task_types) + findings: List[promotion_types.NegativeTransfer] = [] + for task_type, after_value in (after or {}).items(): + if task_type in related: + continue + before_value = (before or {}).get(task_type) + if before_value is None: + continue + if before_value - after_value >= threshold: + findings.append( + promotion_types.NegativeTransfer( + asset_id=asset_id, + task_type=task_type, + before=before_value, + after=after_value, + ) + ) + return findings diff --git a/src/backend/core/evolution/runtime_binding.py b/src/backend/core/evolution/cycle/runtime_binding.py similarity index 87% rename from src/backend/core/evolution/runtime_binding.py rename to src/backend/core/evolution/cycle/runtime_binding.py index 97a953da..dcc87496 100644 --- a/src/backend/core/evolution/runtime_binding.py +++ b/src/backend/core/evolution/cycle/runtime_binding.py @@ -19,6 +19,7 @@ import logging import threading +from dataclasses import dataclass, field from datetime import datetime, timezone from typing import Any, Dict, List, Optional @@ -40,32 +41,29 @@ # Bundles are small and short-lived; a bounded in-process cache keeps the hot # path free of database reads. The Episode assembler persists what it needs. _MAX_CACHED_BUNDLES = 512 -_bundles: Dict[str, AssetBundle] = {} -_run_to_bundle: Dict[tuple[str, str], str] = {} -_lock = threading.Lock() def _remember(bundle: AssetBundle, run_id: Optional[str], capability_scope: str = "") -> None: - with _lock: - _bundles[bundle.bundle_id] = bundle + with process_state().lock: + process_state().bundles[bundle.bundle_id] = bundle if run_id: - _run_to_bundle[(str(run_id), str(capability_scope))] = bundle.bundle_id - if len(_bundles) > _MAX_CACHED_BUNDLES: + process_state().run_to_bundle[(str(run_id), str(capability_scope))] = bundle.bundle_id + if len(process_state().bundles) > _MAX_CACHED_BUNDLES: # Cheap eviction: drop the oldest insertion. Losing a cached bundle # is harmless — it is re-resolvable from the persisted Episode. - for stale_key in list(_bundles.keys())[: len(_bundles) - _MAX_CACHED_BUNDLES]: - _bundles.pop(stale_key, None) + for stale_key in list(process_state().bundles.keys())[: len(process_state().bundles) - _MAX_CACHED_BUNDLES]: + process_state().bundles.pop(stale_key, None) def get_bundle(bundle_id: str) -> Optional[AssetBundle]: - with _lock: - return _bundles.get(bundle_id) + with process_state().lock: + return process_state().bundles.get(bundle_id) def resolve_bundle_for_run(run_id: str, *, capability_scope: str = "") -> Optional[AssetBundle]: - with _lock: - bundle_id = _run_to_bundle.get((str(run_id), str(capability_scope))) - return _bundles.get(bundle_id) if bundle_id else None + with process_state().lock: + bundle_id = process_state().run_to_bundle.get((str(run_id), str(capability_scope))) + return process_state().bundles.get(bundle_id) if bundle_id else None def clear_run_binding(run_id: Optional[str], *, capability_scope: str = "") -> None: @@ -77,8 +75,8 @@ def clear_run_binding(run_id: Optional[str], *, capability_scope: str = "") -> N """ if not run_id: return - with _lock: - _run_to_bundle.pop((str(run_id), str(capability_scope)), None) + with process_state().lock: + process_state().run_to_bundle.pop((str(run_id), str(capability_scope)), None) def _safe(resolver, label: str, degraded: List[str]) -> List[AssetRef]: @@ -92,24 +90,17 @@ def _safe(resolver, label: str, degraded: List[str]) -> List[AssetRef]: def _prompt_refs() -> List[AssetRef]: - from core.services import prompt_version_service + from core.harness.registry.model_services import active_prompt_versions - refs: List[AssetRef] = [] - for kind in list(prompt_version_service.KIND_SPECS): - try: - active = prompt_version_service.get_active_version(kind) - except Exception: - continue - if active: - refs.append( - AssetRef( - kind=ASSET_PROMPT, - asset_id=kind, - version=str(active.get("id") or "default"), - detail={"label": active.get("label") or ""}, - ) - ) - return refs + return [ + AssetRef( + kind=ASSET_PROMPT, + asset_id=kind, + version=str(active.get("id") or "default"), + detail={"label": active.get("label") or ""}, + ) + for kind, active in active_prompt_versions() + ] def _prompt_refs_from_manifest(manifest: Dict[str, Any]) -> List[AssetRef]: @@ -150,7 +141,7 @@ def _skill_refs( ) -> List[AssetRef]: if not skill_ids: return [] - from core.agent_skills.loader import get_skill_loader + from core.capabilities.skills.loader_factory import get_skill_loader metadata: Dict[str, Any] = {} try: @@ -163,10 +154,10 @@ def _skill_refs( # the same view and a later re-resolution cannot silently swap versions. bindings: Dict[str, Dict[str, Any]] = {} try: - from core.capabilities.paths import capabilities_enabled - from core.capabilities.runtime import get as get_prepared_run + from core.harness.registry.run_capabilities import has_run_capabilities + from core.harness.registry.capability_runtime import get as get_prepared_run - if capabilities_enabled() and run_id: + if has_run_capabilities() and run_id: prepared = get_prepared_run(run_id, scope_id=capability_scope) if prepared is not None: bindings = { @@ -405,6 +396,20 @@ def rebind_execution_manifest( def reset_for_tests() -> None: - with _lock: - _bundles.clear() - _run_to_bundle.clear() + with process_state().lock: + process_state().bundles.clear() + process_state().run_to_bundle.clear() + + +@dataclass +class RuntimeBindingState: + bundles: Dict[str, AssetBundle] = field(default_factory=lambda: {}) + run_to_bundle: Dict[tuple[str, str], str] = field(default_factory=lambda: {}) + lock: object = field(default_factory=lambda: threading.Lock()) + + +_STATE = RuntimeBindingState() + + +def process_state() -> RuntimeBindingState: + return _STATE diff --git a/src/backend/core/evolution/cycle/skill_document.py b/src/backend/core/evolution/cycle/skill_document.py new file mode 100644 index 00000000..c2ed5142 --- /dev/null +++ b/src/backend/core/evolution/cycle/skill_document.py @@ -0,0 +1,298 @@ +"""Render distilled skills as SKILL.md documents.""" +from __future__ import annotations +from typing import Any, Dict, List, Sequence +import logging +logger = logging.getLogger(__name__) + +def skill_label(tools: Sequence[str], *, rules_only: bool = False) -> str: + """A human- and model-readable skill name. + + The name and description are the *only* things the model sees before it + decides whether to open a skill, so they have to describe the job. The + candidate's ``hypothesis`` describes the evidence ("「…」类请求在 N 次独立 + 会话中以同一序列成功完成") — true, and useless for that decision, since it + says nothing about when the skill applies. It belongs in the provenance + section, not the title. + """ + head = " → ".join(tools[:4]) + ("…" if len(tools) > 4 else "") + return ("工具顺序规则：" if rules_only else "固定调用顺序：") + head + + +def skill_description( + tools: Sequence[str], constraints: Sequence[Dict[str, Any]], *, rules_only: bool +) -> str: + """One line, single-line by necessity — the frontmatter parser is line-based.""" + joined = "、".join(f"`{t}`" for t in tools[:6]) + if rules_only: + return ( + f"当任务用到 {joined} 时，按已验证的先后顺序约束安排调用次序" + f"（{len(constraints)} 条约束，来自历史成功执行）。" + ) + return ( + f"当任务需要 {joined} 且执行顺序会影响结果时使用；" + f"按 {' → '.join(tools)} 的顺序调用。" + ) + + +def rules_consistent_with( + constraints: Sequence[Dict[str, Any]], tools: Sequence[str] +) -> List[Dict[str, Any]]: + """Drop rules whose direction the skill's own step list violates. + + The loop attaches every rule touching a skill's tools, and scoping makes that + formally sound — a rule valid in ``finance`` and a sequence distilled from + ``trap`` genuinely can point opposite ways. But the rendered document then + tells the model to call A before B in its steps and B before A two sections + later, with nothing saying which wins. Observed on a real materialisation. + + For this skill's own tool set the step list is the governing statement, so a + rule contradicting it is describing some other context and does not belong in + this document. It remains attached to the skills whose sequences agree with + it, and in ``ordering_constraints.json`` for any consumer that reasons about + scope itself. + """ + if not tools: + return list(constraints) + position = {tool: index for index, tool in enumerate(tools)} + kept: List[Dict[str, Any]] = [] + for rule in constraints: + before, after = rule.get("before"), rule.get("after") + if before in position and after in position and position[before] > position[after]: + logger.info( + "[activation] rule %s<%s omitted: contradicts this skill's own order", + before, + after, + ) + continue + kept.append(rule) + return kept + + +def ordering_rules_section(constraints: Sequence[Dict[str, Any]]) -> str: + """Render the learned ordering constraints into the body the model reads. + + Without this the transferable half of what was learned never reaches the + runtime. ``ordering_constraints.json`` is written alongside, but nothing in + the agent path reads extra files as policy — the model reads SKILL.md, so a + rule that is not written here is a rule the model never receives. + + Each rule carries its own scope. Stating where it was contradicted matters + as much as stating where it held: a rule applied in the context that + disproved it actively breaks that context. + """ + shippable = [r for r in constraints if not (r.get("contradicted_in") or [])] + withheld = [r for r in constraints if (r.get("contradicted_in") or [])] + + lines: List[str] = [] + if shippable: + lines += ["", "## 顺序约束（可迁移）", ""] + lines.append("下列约束在历史成功执行中无反例，适用于本技能覆盖的工具：") + lines.append("") + for rule in shippable: + before, after = rule.get("before"), rule.get("after") + validated = [str(x) for x in (rule.get("validated_in") or [])] + line = f"- 先 `{before}` 再 `{after}`" + if validated: + line += f"——已在 {'、'.join(validated)} 类任务中验证" + line += f"（证据 {rule.get('support', 0)} 条）" + lines.append(line) + + if withheld: + # Scoped rules are deliberately kept out of the model's view. + # + # Measured twice against a real model, and both framings failed: + # + # * exception as a trailing qualifier ("在 X 类任务中不适用") — the model + # applied the rule in X anyway, 5 of 5 times; + # * exception first and imperative ("若属于 X 则整条作废，那里顺序相反") + # — the model then applied the *reversed* order everywhere, dropping the + # family the rule was valid in from 50% to 17% and disturbing an + # unrelated control family. + # + # So a text annotation cannot make a model route on task type: it either + # ignores the exception or over-applies it. Handing it a rule that is + # wrong somewhere makes the system worse where the rule is wrong, which + # is a poor trade for a benefit available only where it is right. + # + # The rule is not discarded — it stays in ``ordering_constraints.json`` + # for a runtime applier that knows the task type and can decide before + # injection, which is where a scope decision belongs. Until that exists, + # withholding is the conservative and measurably better option. + lines += ["", "## 已扣留的上下文相关约束", ""] + lines.append( + f"另有 {len(withheld)} 条约束在部分任务类型中被证据推翻，" + "因此**不在本技能中给出**——它们只在特定任务类型下成立，" + "由运行时按任务类型判定后注入，不交由模型自行取舍。" + ) + for rule in withheld: + lines.append( + f"- （已扣留）`{rule.get('before')}` / `{rule.get('after')}`：" + f"在 {'、'.join(str(x) for x in (rule.get('validated_in') or []))} 成立，" + f"在 {'、'.join(str(x) for x in (rule.get('contradicted_in') or []))} 被推翻" + ) + return "\n".join(lines) + + +def skill_markdown( + *, + skill_id: str, + label: str, + description: str, + tools: Sequence[str], + evidence: int, + constraints: Sequence[Dict[str, Any]] = (), + rationale: str = "", + rules_only: bool = False, +) -> str: + """Render the SKILL.md body for a distilled skill. + + ``name`` is the skill **id** — the loader treats that field as the id and + rejects anything outside ``[a-z0-9_-]``. Writing the human label there made + every materialised skill unparseable: it still showed up in the capability + list (that metadata comes from table columns) while ``load_skill_full`` + returned ``None``, so the skill was present and inert. + + States its provenance in the document itself, so anyone reading the skill — + not just anyone reading the console — knows where it came from and on how + much evidence it rests. + """ + # Applicability is declared for the **whole skill**, not per rule. Measured: + # withholding a contradicted rule from the text was not enough, because the + # step list generalises on its own — the model applied this skill's order to + # a family where the reverse is correct, 5 of 5 times, with the rule absent. + # So the exclusion has to bind the steps too, which means stating it once, at + # the top, before anything the model could copy. + excluded = sorted( + { + str(name) + for rule in constraints + for name in (rule.get("contradicted_in") or []) + if str(name) + } + ) + covered = sorted( + { + str(name) + for rule in constraints + for name in (rule.get("validated_in") or []) + if str(name) and str(name) not in excluded + } + ) + scope_block = "" + if excluded: + scope_block = ( + "## 不适用范围（先读这一段）\n\n" + f"**本技能不适用于以下任务类型：{'、'.join(excluded)}。**" + "在这些场景中，本技能给出的顺序已被历史证据推翻——" + "遇到它们请完全忽略本技能，按任务本身的依赖关系自行规划。\n\n" + + ( + f"本技能仅适用于：{'、'.join(covered)}。\n\n" + if covered + else "" + ) + ) + + if rules_only: + body_head = ( + scope_block + + "## 适用条件\n\n" + "当任务用到下列工具、且它们的先后顺序会影响结果时，按「顺序约束」小节排序。\n\n" + "## 涉及工具\n\n" + + "\n".join(f"- `{tool}`" for tool in tools) + ) + else: + body_head = ( + scope_block + + "## 适用条件\n\n" + "当任务需要按固定顺序完成下列工具调用时使用本技能。\n\n" + "## 步骤\n\n" + + "\n".join(f"{i}. `{tool}`" for i, tool in enumerate(tools, start=1)) + ) + + # A rules-only skill has no governing step list, so nothing there can be + # contradicted; for a sequence skill the steps win and conflicting rules are + # about other contexts. + shown_rules = constraints if rules_only else rules_consistent_with(constraints, tools) + + provenance = ( + f"本技能由能力进化流水线从 {evidence} 条历史执行证据中沉淀而成，非人工撰写。" + ) + if rationale: + provenance += f"\n\n沉淀依据：{rationale}" + + return f"""--- +name: {skill_id} +display_name: {label} +description: {description} +--- + +# {label} + +{description} + +{body_head} +{ordering_rules_section(shown_rules)} + +## 来源 + +{provenance} +""" + + +def body_of(rendered: str) -> str: + """The body of a rendered SKILL.md, without frontmatter or provenance. + + Re-rendering from the body rather than storing the generator's output + verbatim is what lets the id change between drafting and activation (an + evolution prefix, an owner suffix) without the document's ``name`` field + disagreeing with the row it is stored under — a mismatch that makes the + skill unloadable while still listing it as available. + """ + text = rendered or "" + if text.startswith("---"): + end = text.find("\n---", 3) + if end != -1: + text = text[end + 4 :] + # Drop the rendered title/description pair and the provenance section; both + # are regenerated, and keeping them would double on every re-render. + marker = text.find("## ") + if marker != -1: + text = text[marker:] + source = text.find("\n## 来源") + if source != -1: + text = text[:source] + return text.strip() + + +def tools_from_constraints(constraints: Sequence[Dict[str, Any]]) -> List[str]: + """The tools a set of ordering rules talks about, in an order they permit. + + Not a claimed execution plan — a rules-only skill has no fixed sequence. It + is the tool list the skill declares, listed in an order that does not + contradict its own rules so the document does not read as self-refuting. + """ + tools: List[str] = [] + for rule in constraints: + for key in ("before", "after"): + name = str(rule.get(key) or "") + if name and name not in tools: + tools.append(name) + if not tools: + return [] + from core.evolution.cycle.promotion_types import OrderingConstraint + from core.evolution.cycle.ordering import apply_ordering_constraints + + parsed = [ + OrderingConstraint( + before=str(rule.get("before") or ""), + after=str(rule.get("after") or ""), + support=int(rule.get("support") or 0), + contexts=int(rule.get("contexts") or 0), + validated_in=tuple(rule.get("validated_in") or ()), + # Scope is deliberately dropped for this one purpose: we are only + # ordering the *document's* tool list, not deciding applicability. + ) + for rule in constraints + if rule.get("before") and rule.get("after") + ] + return apply_ordering_constraints(tools, parsed) diff --git a/src/backend/core/evolution/cycle/skill_release.py b/src/backend/core/evolution/cycle/skill_release.py new file mode 100644 index 00000000..1eba4793 --- /dev/null +++ b/src/backend/core/evolution/cycle/skill_release.py @@ -0,0 +1,355 @@ +"""Ramp, roll back and retire skill releases.""" +from __future__ import annotations +from datetime import datetime, timezone +from typing import Any, Dict, List, Optional +from core.db.engine import SessionLocal +import logging +from core.evolution.cycle import activation_policy +from core.evolution.cycle import activation_types + +logger = logging.getLogger(__name__) + +def advance_skill_release( + release_id: str, + *, + actor: str, + observed_metrics: Optional[Dict[str, float]] = None, + guardrail_thresholds: Optional[Dict[str, float]] = None, +) -> Dict[str, Any]: + """Ramp a release one step: shadow → canary 5 → 25 → 100 → active. + + One step per call, deliberately. A single "go live" button that walked the + whole ladder would reproduce the behaviour this replaces — the point of the + ladder is that somebody looks at the metrics between rungs. + + Guardrails are checked when metrics are supplied, and a breach rolls back + rather than pausing: the recorded threshold already *is* the level at which + the release is unacceptable. + """ + from core.db.models.evolution import EvolutionCandidate, EvolutionRelease + from core.evolution.cycle.release import ACTIVE, CANARY, CANARY_STEPS, SHADOW, can_promote, evaluate_guardrails, next_traffic, should_auto_rollback + + if not actor: + raise activation_types.ActivationError("actor_required", "推进发布必须署名") + + with SessionLocal() as db: + release = db.get(EvolutionRelease, release_id) + if release is None: + raise activation_types.ActivationError("release_not_found", f"发布记录不存在: {release_id}") + if activation_policy.is_frozen(db): + raise activation_types.ActivationError("globally_frozen", "全局冻结开关已打开，禁止推进发布") + + candidate = db.get(EvolutionCandidate, release.candidate_id) + risk = str(getattr(candidate, "risk_tier", "") or "low") + guardrails = dict(release.guardrails or {}) + completed: List[str] = list(guardrails.get("completed_verification") or []) + proposer = str(guardrails.get("proposer") or getattr(candidate, "proposer", "") or "") + + if observed_metrics: + thresholds = guardrail_thresholds or guardrails.get("thresholds") or {} + breaches = evaluate_guardrails(observed_metrics, thresholds, window="canary") + if should_auto_rollback(breaches): + detail = ", ".join( + f"{b.metric}={b.observed}>{b.threshold}" for b in breaches + ) + db.commit() + rollback_skill_activation( + release_id, + actor=actor, + reason=f"guardrail breach: {detail}", + kind="guardrail", + ) + return { + "release_id": release_id, + "stage": "rolled_back", + "guardrail_breaches": [b.to_dict() for b in breaches], + } + + stage = str(release.stage or "") + traffic = int(release.traffic_percent or 0) + + if stage == SHADOW: + target_stage, target_traffic = CANARY, CANARY_STEPS[0] + completed = sorted(set(completed) | {"shadow"}) + elif stage == CANARY and traffic < CANARY_STEPS[-1]: + target_stage, target_traffic = CANARY, next_traffic(traffic) + elif stage == CANARY: + target_stage, target_traffic = ACTIVE, 100 + completed = sorted(set(completed) | {"canary"}) + else: + raise activation_types.ActivationError( + "not_rampable", f"当前阶段 {stage} 无法继续放量" + ) + + allowed, reason = can_promote( + current_stage=stage, + target_stage=target_stage, + risk_tier=risk, + # ``human_approval`` is this call: a named actor who is not the + # proposer. Recording it before the check keeps the requirement + # satisfied by the act itself rather than by a stored flag. + completed_verification=sorted(set(completed) | {"human_approval"}) + if target_stage == ACTIVE + else completed, + proposer=proposer, + approver=actor, + ) + # A same-stage traffic bump is not a transition; can_promote would reject + # canary→canary, which is correct for stages and wrong for ramp steps. + if not allowed and not (stage == target_stage == CANARY): + raise activation_types.ActivationError("promotion_refused", reason) + + if target_stage == ACTIVE: + # 冲突的 L3 证据在 shadow/canary 里观测没问题，但升到全量必须先由 + # 人裁决冲突本身——适用条件不可信的技能不配自动放大到所有人。 + if activation_policy.has_graph_conflict(candidate): + raise activation_types.ActivationError( + "graph_conflict_requires_review", + "候选的 L3 图谱证据存在未裁决的冲突，禁止放量到 active", + ) + activation_policy.check_pace(db, tenant_id=str((release.scope_filter or {}).get("tenant_id") or "default")) + + release.stage = target_stage + release.traffic_percent = target_traffic + guardrails["completed_verification"] = completed + if target_stage == ACTIVE: + guardrails["approved_by"] = actor + release.guardrails = guardrails + release.approved_by = actor + if candidate is not None and target_stage == ACTIVE: + candidate.status = ACTIVE + db.commit() + + logger.info( + "[activation] release %s → %s/%d%% by %s", + release_id, + target_stage, + target_traffic, + actor, + ) + return { + "release_id": release_id, + "stage": target_stage, + "traffic_percent": target_traffic, + "completed_verification": completed, + "exposed_to": ( + "all users in scope" + if target_stage == ACTIVE + else f"{target_traffic}% of users in scope" + ), + } + + +def rollback_skill_activation( + release_id: str, *, actor: str, reason: str, kind: str = "manual" +) -> Dict[str, Any]: + """Undo a materialisation, restoring the snapshot taken at activation time. + + With no prior version the skill is disabled rather than deleted: history + referencing it must stay resolvable, and a disabled row is both reversible + and visible in a way a missing row is not. + + ``kind`` separates "a guardrail caught it" from "a person caught it". + Collapsing the two would hide how well the evaluation stage is working, + which is the number that tells you whether the ramp is worth its cost. + """ + from core.db.models import AdminSkill + from core.db.models.evolution import ( + EvolutionAgentProfile, + EvolutionCandidate, + EvolutionRelease, + ) + from core.evolution.cycle.release import ROLLBACK_GUARDRAIL, ROLLBACK_MANUAL, ROLLED_BACK + + if not actor or not reason: + raise activation_types.ActivationError("audit_required", "回滚必须署名并填写理由") + + with SessionLocal() as db: + release = db.get(EvolutionRelease, release_id) + if release is None: + raise activation_types.ActivationError("release_not_found", f"发布记录不存在: {release_id}") + + previous = (release.guardrails or {}).get("previous") + skill_id = release.target_asset_id + row = ( + db.query(AdminSkill) + .filter(AdminSkill.skill_id == skill_id) + .first() + ) + invalidated: List[str] = [] + if row is not None: + if previous: + row.skill_content = previous["skill_content"] + row.display_name = previous["display_name"] + row.description = previous["description"] + row.version = previous["version"] + row.tags = previous["tags"] + row.allowed_tools = previous["allowed_tools"] + row.extra_files = previous["extra_files"] + row.is_enabled = previous["is_enabled"] + else: + row.is_enabled = False + # No prior version means the skill just stopped being loadable, + # so a profile assembled on top of it is now promising a + # capability that fails silently — same cross-layer rule as + # retirement. With a restored prior version the skill stays + # loadable and the profiles stay valid. + for profile_row in ( + db.query(EvolutionAgentProfile) + .filter(EvolutionAgentProfile.is_active.is_(True)) + .all() + ): + payload = profile_row.payload or {} + if skill_id in (payload.get("skill_ids") or []): + profile_row.is_active = False + invalidated.append(profile_row.profile_id) + + release.stage = ROLLED_BACK + release.rolled_back_at = datetime.now(timezone.utc) + release.rollback_reason = reason + release.rollback_kind = ( + ROLLBACK_GUARDRAIL if kind == "guardrail" else ROLLBACK_MANUAL + ) + + candidate = db.get(EvolutionCandidate, release.candidate_id) + if candidate is not None: + candidate.status = ROLLED_BACK + candidate_id = release.candidate_id + db.commit() + + # The reverse edge of the promotion chain. Materialisation down-weighted the + # memories this skill was compiled from; rolling the skill back must restore + # them in the same motion, or the knowledge stays half-priced with no + # carrier — retrieval skips it because a skill supposedly covers it, and the + # skill is gone. Same ledger the retire path already replays. + from core.evolution.generators.memory_apply import revert_memory_ops + + restored = revert_memory_ops(candidate_id=candidate_id) if candidate_id else {} + + # 血缘：这一版回滚到了哪一版。审计问「技能 v2 后来怎么了」时，答案在链上。 + try: + from core.evolution.evidence import lineage + + current_version = str((previous or {}).get("version") or "") + lineage.record_links( + [ + ( + "skill", + f"{skill_id}@rolled_back", + lineage.REL_ROLLED_BACK_TO, + "skill", + f"{skill_id}@{current_version or 'disabled'}", + ) + ], + candidate_id=candidate_id, + ) + except Exception as exc: # noqa: BLE001 + logger.debug("[activation] rollback lineage skipped: %s", exc) + + logger.info( + "[activation] rolled back %s (release=%s memories restored=%d profiles invalidated=%s)", + skill_id, + release_id, + len(restored.get("reverted", [])), + invalidated, + ) + return { + "release_id": release_id, + "restored_previous": bool(previous), + "restored_memory_ops": restored.get("reverted", []), + "invalidated_profiles": invalidated, + "reason": reason, + } + + +def retire_skill_candidate(candidate_id: str, *, approver: str) -> Dict[str, Any]: + """Take an approved retirement into effect. + + Retirement is disabling plus a release row, never a delete. Three reasons it + is done this way: the evidence behind the decision stays inspectable, the + skill can be brought back in one operation if the decision was wrong, and + anything built on top of it — a profile naming it, a sub-agent requiring it + — can be found and invalidated rather than silently pointing at nothing. + """ + from core.db.models.evolution import EvolutionCandidate, EvolutionRelease + + from core.db.models import AdminSkill + from core.db.models.evolution import EvolutionAgentProfile + from core.evolution.cycle.release import RETIRED, new_release_id + + if not approver: + raise activation_types.ActivationError("approver_required", "退役必须署名") + + with SessionLocal() as db: + candidate = db.get(EvolutionCandidate, candidate_id) + if candidate is None: + raise activation_types.ActivationError("candidate_not_found", f"候选不存在: {candidate_id}") + if candidate.operation != "deprecate": + raise activation_types.ActivationError("not_a_retirement", "该候选不是退役候选") + if approver == (candidate.proposer or ""): + raise activation_types.ActivationError("self_approval_forbidden", "生成者不能批准自己提出的退役") + + skill_id = candidate.target_asset_id + row = db.query(AdminSkill).filter(AdminSkill.skill_id == skill_id).first() + if row is None: + raise activation_types.ActivationError("skill_not_found", f"技能不存在: {skill_id}") + + previous_enabled = bool(row.is_enabled) + row.is_enabled = False + + # Cross-layer: anything assembled on top of this skill stops being + # applied. Leaving them active would leave a profile promising a + # capability that is no longer loadable, which fails silently. + invalidated: List[str] = [] + for profile_row in ( + db.query(EvolutionAgentProfile) + .filter(EvolutionAgentProfile.is_active.is_(True)) + .all() + ): + payload = profile_row.payload or {} + if skill_id in (payload.get("skill_ids") or []): + profile_row.is_active = False + invalidated.append(profile_row.profile_id) + + release = EvolutionRelease( + release_id=new_release_id(), + candidate_id=candidate_id, + target_kind="skill", + target_asset_id=skill_id, + stage=RETIRED, + traffic_percent=0, + scope_filter={}, + guardrails={ + "previous_enabled": previous_enabled, + "invalidated_profiles": invalidated, + "proposer": candidate.proposer or "", + }, + rollback_version_id=row.version or "", + approved_by=approver, + ) + db.add(release) + + candidate.status = "active" + candidate.approved_by = approver + candidate.approved_at = datetime.now(timezone.utc) + db.commit() + + # Restore the memories this skill was compiled from: the knowledge lost + # its cheaper carrier, so it goes back to being paid for by retrieval. + from core.evolution.generators.memory_apply import revert_memory_ops + + source_candidate = activation_policy.source_candidate_for(db, skill_id) + restored = revert_memory_ops(candidate_id=source_candidate) if source_candidate else {} + + logger.info( + "[activation] retired %s (profiles invalidated=%s memories restored=%d)", + skill_id, + invalidated, + len(restored.get("reverted", [])), + ) + return { + "skill_id": skill_id, + "release_id": release.release_id, + "invalidated_profiles": invalidated, + "restored_memory_ops": restored.get("reverted", []), + } diff --git a/src/backend/core/evolution/timescales.py b/src/backend/core/evolution/cycle/timescales.py similarity index 99% rename from src/backend/core/evolution/timescales.py rename to src/backend/core/evolution/cycle/timescales.py index bcd93632..94ef52d6 100644 --- a/src/backend/core/evolution/timescales.py +++ b/src/backend/core/evolution/cycle/timescales.py @@ -17,7 +17,7 @@ import logging from dataclasses import dataclass, field -from datetime import datetime, timedelta, timezone +from datetime import datetime, timezone from typing import Any, Dict, List, Optional, Sequence, Tuple logger = logging.getLogger(__name__) diff --git a/src/backend/core/evolution/trigger.py b/src/backend/core/evolution/cycle/trigger.py similarity index 81% rename from src/backend/core/evolution/trigger.py rename to src/backend/core/evolution/cycle/trigger.py index e90acd9c..1a5339e3 100644 --- a/src/backend/core/evolution/trigger.py +++ b/src/backend/core/evolution/cycle/trigger.py @@ -31,7 +31,8 @@ import logging import os import threading -from datetime import datetime, timedelta, timezone +from dataclasses import dataclass, field +from datetime import datetime, timezone from typing import Any, Dict, Optional, Tuple from core.db.engine import SessionLocal @@ -63,12 +64,8 @@ def _env_bool(name: str, default: bool) -> bool: # Stops a burst of conversations from queueing a cycle per turn. DEBOUNCE_SECONDS = _env_int("EVOLUTION_TRIGGER_DEBOUNCE_S", 600) -_state_lock = threading.Lock() + # Per-user debounce, so a busy person cannot crowd out everyone else's cycle. -_personal_last_run: Dict[str, datetime] = {} -_last_cycle_at: Optional[datetime] = None -_last_cycle_episode_count: int = 0 -_running = False def _now() -> datetime: @@ -78,7 +75,7 @@ def _now() -> datetime: def _episode_count() -> int: """Cheap indexed count of mineable episodes.""" from core.db.models.evolution import EvolutionEpisode - from core.evolution.user_settings import PRIVACY_PRIVATE + from core.evolution.profiles.user_settings_constants import PRIVACY_PRIVATE with SessionLocal() as db: return ( @@ -97,10 +94,10 @@ def should_trigger(*, now: Optional[datetime] = None) -> Tuple[bool, str]: if not AUTO_TRIGGER_ENABLED: return False, "auto_trigger_disabled" - with _state_lock: - if _running: + with process_state().state_lock: + if process_state().running: return False, "cycle_already_running" - last_at, last_count = _last_cycle_at, _last_cycle_episode_count + last_at, last_count = process_state().last_cycle_at, process_state().last_cycle_episode_count moment = now or _now() if last_at is not None: @@ -129,17 +126,17 @@ def _mark_started(total: int, *, now: Optional[datetime] = None) -> None: sleeping — a debounce tested only against the real clock is a debounce nobody tests. """ - global _last_cycle_at, _last_cycle_episode_count, _running - with _state_lock: - _running = True - _last_cycle_at = now or _now() - _last_cycle_episode_count = total + pass + with process_state().state_lock: + process_state().running = True + process_state().last_cycle_at = now or _now() + process_state().last_cycle_episode_count = total def _mark_finished() -> None: - global _running - with _state_lock: - _running = False + pass + with process_state().state_lock: + process_state().running = False def run_cycle_if_due(*, limit: int = 400) -> Optional[Dict[str, Any]]: @@ -157,11 +154,11 @@ def run_cycle_if_due(*, limit: int = 400) -> Optional[Dict[str, Any]]: try: total = _episode_count() except Exception: - total = _last_cycle_episode_count + total = process_state().last_cycle_episode_count _mark_started(total) try: - from core.evolution.loop import run_evolution_cycle + from core.evolution.cycle.loop import run_evolution_cycle report = run_evolution_cycle(limit=limit) logger.info( @@ -189,15 +186,15 @@ def schedule_personal(user_id: str) -> bool: return False moment = _now() - with _state_lock: - last = _personal_last_run.get(user_id) + with process_state().state_lock: + last = process_state().personal_last_run.get(user_id) if last is not None and (moment - last).total_seconds() < DEBOUNCE_SECONDS: return False - _personal_last_run[user_id] = moment + process_state().personal_last_run[user_id] = moment def _run() -> None: try: - from core.evolution.personal import run_personal_cycle + from core.evolution.profiles.personal import run_personal_cycle report = run_personal_cycle(user_id) if report.get("candidates"): @@ -245,8 +242,8 @@ def schedule_if_due(*, limit: int = 400) -> bool: def status() -> Dict[str, Any]: """Current trigger state, for the console and for answering "why not?".""" due, reason = should_trigger() - with _state_lock: - last_at, last_count = _last_cycle_at, _last_cycle_episode_count + with process_state().state_lock: + last_at, last_count = process_state().last_cycle_at, process_state().last_cycle_episode_count return { "enabled": AUTO_TRIGGER_ENABLED, "due": due, @@ -259,9 +256,25 @@ def status() -> Dict[str, Any]: def reset_for_tests() -> None: - global _last_cycle_at, _last_cycle_episode_count, _running - with _state_lock: - _last_cycle_at = None - _last_cycle_episode_count = 0 - _running = False - _personal_last_run.clear() + pass + with process_state().state_lock: + process_state().last_cycle_at = None + process_state().last_cycle_episode_count = 0 + process_state().running = False + process_state().personal_last_run.clear() + + +@dataclass +class TriggerState: + state_lock: object = field(default_factory=lambda: threading.Lock()) + personal_last_run: Dict[str, datetime] = field(default_factory=lambda: {}) + last_cycle_at: Optional[datetime] = None + last_cycle_episode_count: int = 0 + running: object = False + + +_STATE = TriggerState() + + +def process_state() -> TriggerState: + return _STATE diff --git a/src/backend/core/evolution/cycle/usage_evidence.py b/src/backend/core/evolution/cycle/usage_evidence.py new file mode 100644 index 00000000..105693ea --- /dev/null +++ b/src/backend/core/evolution/cycle/usage_evidence.py @@ -0,0 +1,283 @@ +"""Skill usage, canary uplift and exposure measured from episode evidence.""" +from __future__ import annotations +from typing import Any, Dict, List, Optional, Sequence, Tuple +import logging +from core.evolution.cycle import cycle_extras_constants + +logger = logging.getLogger(__name__) + +def is_after(created_at: Any, activated_at: Any) -> bool: + from datetime import timezone + + if created_at is None or activated_at is None: + return False + left = created_at if created_at.tzinfo else created_at.replace(tzinfo=timezone.utc) + right = activated_at if activated_at.tzinfo else activated_at.replace(tzinfo=timezone.utc) + return left >= right + + +def recent_activations() -> List[Any]: + """``(asset_id, activated_at, related_task_types)`` for live releases. + + ``related_task_types`` comes from the release's own scope: a dip on the task + the change targeted is an ordinary regression its guardrail already watches, + and treating it as negative transfer would bury the collateral damage this + is looking for in noise from the intended effect. + """ + try: + from core.db.engine import SessionLocal + from core.db.models.evolution import EvolutionRelease + from core.evolution.cycle.release import ACTIVE, CANARY + + with SessionLocal() as db: + rows = ( + db.query(EvolutionRelease) + .filter(EvolutionRelease.stage.in_([ACTIVE, CANARY])) + .order_by(EvolutionRelease.created_at.desc()) + .limit(20) + .all() + ) + return [ + ( + str(row.target_asset_id), + row.created_at, + [str(t) for t in ((row.scope_filter or {}).get("task_types") or [])], + ) + for row in rows + if row.created_at is not None + ] + except Exception as exc: # noqa: BLE001 + logger.debug("[cycle-extras] activation ledger unavailable: %s", exc) + return [] + + +def declared_tools_for(skill_ids: Sequence[str]) -> Dict[str, List[str]]: + """Each skill's own declared tools, read from the skill itself.""" + tools: Dict[str, List[str]] = {} + try: + from core.capabilities.skills.loader_factory import get_skill_loader + + metadata = get_skill_loader().load_all_metadata() + except Exception as exc: # noqa: BLE001 + logger.info("[cycle-extras] skill metadata unavailable, overlap skipped: %s", exc) + return {} + for skill_id in skill_ids: + meta = metadata.get(skill_id) + declared = list(getattr(meta, "allowed_tools", None) or []) if meta else [] + if declared: + tools[skill_id] = [str(t) for t in declared] + return tools + + +def skill_usage_from_views(views: Sequence[Dict[str, Any]]) -> Dict[str, Dict[str, Any]]: + """How each skill actually fared, from evidence that distinguishes use from exposure. + + ``offers`` counts turns where the skill was **available** to the model — its + name and description were in the prompt. Skill loading offers every enabled + skill, so this is a denominator, not a choice: it answers "how many chances + did this skill have to be picked", which is exactly what makes "never + opened" mean something. + + ``opens`` is the choice. The gap between the two is the whole signal. + """ + stats: Dict[str, Dict[str, Any]] = {} + + def bucket(skill_id: str) -> Dict[str, Any]: + return stats.setdefault( + str(skill_id), + { + "offers": 0, + "opens": 0, + "opens_succeeded": 0, + "opens_followed_through": 0, + "opens_measurable": 0, + "exposure_days": 0.0, + }, + ) + + declared_by_skill = declared_tools_for( + sorted( + { + str(opened.get("skill_id") or "") + for view in views + for opened in (view.get("skills_opened") or []) + if opened.get("skill_id") + } + ) + ) + + for view in views: + for skill_id in view.get("skill_sequence") or []: + bucket(skill_id)["offers"] += 1 + + succeeded = view.get("verdict") == "success" + for opened in view.get("skills_opened") or []: + skill_id = str(opened.get("skill_id") or "") + if not skill_id: + continue + entry = bucket(skill_id) + entry["opens"] += 1 + if succeeded: + entry["opens_succeeded"] += 1 + + # Follow-through: after reading the document, did the model do what + # the document is about? Approximated by whether the tools it went + # on to call are among the ones the skill declares. + # + # Read from the skill itself, not from the event — an event cannot + # carry what the skill declared *now*, and a stale copy would score + # a rewritten skill against its old tool list. + declared = {str(t) for t in (declared_by_skill.get(skill_id) or [])} + after = [str(t) for t in (opened.get("tools_after_open") or [])] + if not declared or not after: + # A skill that declares no tools (a rules-only skill shapes + # ordering, not tool choice) or a turn that called nothing after + # the open cannot be measured this way. Excluded from the + # denominator rather than counted as a pass — counting them as + # passes is what made follow-through 100% for everything and the + # "opened but ignored" verdict unreachable. + continue + entry["opens_measurable"] += 1 + if any(tool in declared for tool in after): + entry["opens_followed_through"] += 1 + + uplift_by_skill = uplift_from_canary(views) + for skill_id, entry in stats.items(): + opens = entry["opens"] + measurable = entry["opens_measurable"] + entry["success_rate"] = entry["opens_succeeded"] / opens if opens else 0.0 + entry["follow_through"] = ( + entry["opens_followed_through"] / measurable if measurable else None + ) + entry["exposure_days"] = exposure_days(skill_id) + entry["uplift"] = uplift_by_skill.get(skill_id) + return stats + + +def canary_releases() -> List[Tuple[str, str, int, Any]]: + """``(asset_id, release_id, traffic_percent, started_at)`` for live canaries.""" + from core.evolution.cycle.release import CANARY + + try: + from core.db.engine import SessionLocal + from core.db.models.evolution import EvolutionRelease + + with SessionLocal() as db: + rows = ( + db.query(EvolutionRelease) + .filter( + EvolutionRelease.target_kind == "skill", + EvolutionRelease.stage == CANARY, + ) + .all() + ) + return [ + ( + str(row.target_asset_id), + str(row.release_id), + int(row.traffic_percent or 0), + row.created_at, + ) + for row in rows + ] + except Exception as exc: # noqa: BLE001 + logger.debug("[cycle-extras] canary ledger unavailable: %s", exc) + return [] + + +def uplift_from_canary( + views: Sequence[Dict[str, Any]], + *, + ramping: Optional[Sequence[Tuple[str, str, int, Any]]] = None, +) -> Dict[str, Optional[float]]: + """"Is the deployment better off with this skill?", answered by comparison. + + A canary release already splits users deterministically into "sees it" and + "does not" (:func:`core.evolution.cycle.release.in_canary_bucket`). That split is + the comparison: same period, same task mix, same everything except exposure + to this one asset. Recomputing bucket membership here rather than storing it + per request means the analysis works on history too. + + Returns ``None`` for a skill with no usable comparison. ``None`` and ``0.0`` + are different answers — "we have not measured" must never read as "measured, + no effect", because the second one licenses a rollback decision and the + first one does not. + + ``ramping`` is injectable so this is testable against the real bucketing + rather than against a re-implementation of it in a test helper. + """ + from core.evolution.cycle.release import in_canary_bucket + + if ramping is None: + ramping = canary_releases() + + uplift: Dict[str, Optional[float]] = {} + for skill_id, release_id, traffic, started_at in ramping: + exposed: List[bool] = [] + withheld: List[bool] = [] + for view in views: + created_at = view.get("created_at") + if created_at is None or started_at is None: + continue + if not is_after(created_at, started_at): + continue + user_id = str(view.get("user_id") or "") + if not user_id: + continue + succeeded = view.get("verdict") == "success" + if in_canary_bucket( + release_id=release_id, subject_id=user_id, traffic_percent=traffic + ): + exposed.append(succeeded) + else: + withheld.append(succeeded) + + # Both arms need enough turns for a rate to mean anything. A one-sided + # or tiny comparison produces a number that looks like evidence and is + # not, and this number can trigger an immediate rollback. + if len(exposed) < cycle_extras_constants.MIN_UPLIFT_SAMPLES or len(withheld) < cycle_extras_constants.MIN_UPLIFT_SAMPLES: + uplift[skill_id] = None + continue + uplift[skill_id] = (sum(exposed) / len(exposed)) - ( + sum(withheld) / len(withheld) + ) + return uplift + + +def exposure_days(skill_id: str) -> float: + """How long this skill has been exposed to anyone. + + Read from the release ledger. Without it, "never opened" cannot be told + apart from "shipped yesterday", and a retirement rule that cannot tell them + apart retires whatever is newest. + """ + from datetime import datetime, timezone + + try: + from core.db.engine import SessionLocal + from core.db.models.evolution import EvolutionRelease + from core.evolution.cycle.release import ACTIVE, CANARY + + with SessionLocal() as db: + row = ( + db.query(EvolutionRelease) + .filter( + EvolutionRelease.target_kind == "skill", + EvolutionRelease.target_asset_id == skill_id, + EvolutionRelease.stage.in_([ACTIVE, CANARY]), + ) + .order_by(EvolutionRelease.created_at.asc()) + .first() + ) + except Exception as exc: # noqa: BLE001 + logger.debug("[cycle-extras] exposure lookup failed for %s: %s", skill_id, exc) + return 0.0 + if row is None or row.created_at is None: + # Not an evolved skill, or never released through the ladder. A + # hand-authored skill has always been exposed, so judging it on its + # observed usage is exactly right. + return float("inf") + created = row.created_at + if created.tzinfo is None: + created = created.replace(tzinfo=timezone.utc) + return (datetime.now(timezone.utc) - created).total_seconds() / 86400.0 diff --git a/src/backend/core/evolution/cycle_extras.py b/src/backend/core/evolution/cycle_extras.py deleted file mode 100644 index 5c2b5fbf..00000000 --- a/src/backend/core/evolution/cycle_extras.py +++ /dev/null @@ -1,653 +0,0 @@ -"""The two engines the cycle was missing, and the direction the third lacked. - -A cycle used to do exactly one thing: grow the skill library from identical tool -sequences. Memory reorganisation and capability retirement were implemented and -unreachable, and recurrence detected from *what the user asked* did not exist at -all. - -Wiring them up is not enough on its own, and the first attempt at it showed why. -Three failures worth naming, because they are the ones this file now exists to -not repeat: - -* **A finding with no exit changes nothing.** The first version returned a - report that no code read. Everything here now produces - :class:`~core.evolution.ir.EvolutionIR` candidates that go through the same - security gate, review queue and release ladder as every other change. A - proposal that cannot reach the queue is a log line. -* **Retrieval frequency is not duplication.** Counting how often one content - hash was retrieved and calling four hits a duplicate inverts the meaning: a - memory retrieved in four separate conversations is the store's most useful - entry, and merging it away is a proposal to delete what works. Duplication is - a property of *inventory*, so it is found by scanning the store - (:mod:`core.evolution.memory_scan`), which the event stream cannot do. -* **Being loaded is not being used.** Retirement read a "selections" count that - every enabled skill shared, so it ranked skills by how recently they shipped. - Usage is ``skill.opened`` — the model deciding for itself to read the - document — and follow-through is whether the plan afterwards matched what the - document said. - -Everything here emits **candidates**. Nothing activates and nothing deletes: the -reverse direction is where over-eager automation does damage that is hard to -notice, because removing a capability produces no error, only a slow decline. -""" - -from __future__ import annotations - -import hashlib -import logging -from typing import Any, Dict, List, Optional, Sequence, Tuple - -logger = logging.getLogger(__name__) - -# An intent must recur on this many separate occasions before "we keep -# re-planning this" is a claim about behaviour rather than about one bad -# afternoon of rephrasing. -MIN_INTENT_OCCASIONS = 3 -# Below this success rate a recurring intent is a *failure* pattern: the right -# response is a fix proposal, not a distilled skill blessing the current plan. -INTENT_MIN_SUCCESS_RATE = 0.6 -# A recurring intent handled the same way every time needs no skill — the plan -# is already stable. The signal is variety. -MIN_PLAN_VARIETY = 2 - -# Injected this often that the association is not two unlucky turns. -DEMOTE_MIN_HITS = 4 -# At or below this success rate, retrieving it is not helping. -DEMOTE_SUCCESS_CEILING = 0.34 - -# Turns required in *each* arm before a canary comparison is reported. Below -# this the difference between two rates is noise, and this particular number can -# trigger an immediate rollback — so it has the strictest sample requirement of -# anything the decremental engine computes. -MIN_UPLIFT_SAMPLES = 10 - - -def _asset_id(prefix: str, seed: str) -> str: - return f"{prefix}-" + hashlib.sha256(seed.encode("utf-8")).hexdigest()[:12] - - -# ── Skills: upward, from recurring intent ──────────────────────────────────── - - -def intent_findings(views: Sequence[Dict[str, Any]]) -> Dict[str, Any]: - """Recurring requests whose handling never settled. - - The pairing matters. Recurrence alone says the question is common; - recurrence **with plan variety** says the system has not found a stable way - to answer it and is paying the planning cost fresh every time. That second - condition is what distinguishes a capability gap from a popular question. - """ - from core.evolution.similarity import cluster_intents - - clusters, method = cluster_intents(views, min_support=MIN_INTENT_OCCASIONS) - findings: List[Dict[str, Any]] = [] - for cluster in clusters: - if cluster.occasions < MIN_INTENT_OCCASIONS: - # Recurrence inside one conversation is a user rephrasing after a - # bad answer, which is evidence about that answer, not about demand. - continue - if cluster.plan_variety < MIN_PLAN_VARIETY: - # Same question, same plan, every time. Already effectively a skill; - # writing one down adds an asset to maintain and changes nothing. - continue - if cluster.success_rate < INTENT_MIN_SUCCESS_RATE: - # Recurring *and* failing. Distilling the current plan would encode - # the failure. This belongs to attribution, not to promotion. - findings.append( - { - "kind": "recurring_failure_intent", - "cluster": cluster.to_dict(), - "action": "no_skill_candidate", - "reason": ( - f"「{cluster.representative[:30]}」类请求出现 {cluster.occasions} 次，" - f"但成功率只有 {cluster.success_rate:.0%}——" - "把当前做法蒸馏出来等于把失败固化" - ), - } - ) - continue - findings.append( - { - "kind": "recurring_intent", - "cluster": cluster.to_dict(), - "action": "skill_candidate", - "asset_id": _asset_id("auto-intent", cluster.representative), - "reason": ( - f"同一意图在 {cluster.occasions} 次独立会话中出现，却用了 " - f"{cluster.plan_variety} 种不同的工具序列——每次都在重新规划" - ), - } - ) - return {"method": method, "clusters": len(clusters), "findings": findings} - - -# ── Skills: downward ───────────────────────────────────────────────────────── - - -def decremental_findings( - views: Sequence[Dict[str, Any]], - *, - skill_stats: Optional[Dict[str, Dict[str, Any]]] = None, - declared_tools: Optional[Dict[str, List[str]]] = None, -) -> Dict[str, Any]: - """Merge, retirement and negative-transfer findings. - - ``skill_stats`` is derived from the episode views when not supplied. - ``declared_tools`` maps skill id → the tools its own document names; overlap - is computed from that and never from the tools observed in episodes where - the skill happened to be loaded, which would pair every co-enabled skill. - """ - from core.evolution import promotion as P - - stats = skill_stats if skill_stats is not None else skill_usage_from_views(views) - - overlaps: List[Tuple[str, str, int]] = [] - if declared_tools is None: - declared_tools = _declared_tools_for(sorted(stats)) - if declared_tools: - overlaps = P.find_overlaps(declared_tools) - - retirements = P.find_retirement_candidates(stats) - return { - "overlaps": [ - {"a": a, "b": b, "shared_tools": shared} for a, b, shared in overlaps - ], - "retirements": retirements, - "asset_count": len(stats), - "negative_transfer": negative_transfer_for_activations(views), - } - - -def negative_transfer_for_activations( - views: Sequence[Dict[str, Any]], -) -> List[Dict[str, Any]]: - """Check each recent activation for collateral damage. - - The detector needs a corpus split into before and after *something*. Calling - it on views where nothing carries that marker returns nothing on every - corpus, which is a detector that runs and cannot find anything — the exact - shape of defect this rework exists to remove. So the split is computed here, - from the release ledger, once per activated asset. - """ - activations = _recent_activations() - if not activations: - return [] - - findings: List[Dict[str, Any]] = [] - for asset_id, activated_at, related in activations: - split = [ - {**view, "after": _is_after(view.get("created_at"), activated_at)} - for view in views - if view.get("created_at") is not None - ] - if not any(v["after"] for v in split) or not any(not v["after"] for v in split): - # One-sided evidence cannot show a change. Reporting nothing is the - # honest output; reporting a drop from an empty baseline is not. - continue - findings.extend( - negative_transfer_findings(split, asset_id=asset_id, related_task_types=related) - ) - return findings - - -def _is_after(created_at: Any, activated_at: Any) -> bool: - from datetime import timezone - - if created_at is None or activated_at is None: - return False - left = created_at if created_at.tzinfo else created_at.replace(tzinfo=timezone.utc) - right = activated_at if activated_at.tzinfo else activated_at.replace(tzinfo=timezone.utc) - return left >= right - - -def _recent_activations() -> List[Any]: - """``(asset_id, activated_at, related_task_types)`` for live releases. - - ``related_task_types`` comes from the release's own scope: a dip on the task - the change targeted is an ordinary regression its guardrail already watches, - and treating it as negative transfer would bury the collateral damage this - is looking for in noise from the intended effect. - """ - try: - from core.db.engine import SessionLocal - from core.db.models.evolution import EvolutionRelease - from core.evolution.release import ACTIVE, CANARY - - with SessionLocal() as db: - rows = ( - db.query(EvolutionRelease) - .filter(EvolutionRelease.stage.in_([ACTIVE, CANARY])) - .order_by(EvolutionRelease.created_at.desc()) - .limit(20) - .all() - ) - return [ - ( - str(row.target_asset_id), - row.created_at, - [str(t) for t in ((row.scope_filter or {}).get("task_types") or [])], - ) - for row in rows - if row.created_at is not None - ] - except Exception as exc: # noqa: BLE001 - logger.debug("[cycle-extras] activation ledger unavailable: %s", exc) - return [] - - -def _declared_tools_for(skill_ids: Sequence[str]) -> Dict[str, List[str]]: - """Each skill's own declared tools, read from the skill itself.""" - tools: Dict[str, List[str]] = {} - try: - from core.agent_skills.loader import get_skill_loader - - metadata = get_skill_loader().load_all_metadata() - except Exception as exc: # noqa: BLE001 - logger.info("[cycle-extras] skill metadata unavailable, overlap skipped: %s", exc) - return {} - for skill_id in skill_ids: - meta = metadata.get(skill_id) - declared = list(getattr(meta, "allowed_tools", None) or []) if meta else [] - if declared: - tools[skill_id] = [str(t) for t in declared] - return tools - - -def skill_usage_from_views(views: Sequence[Dict[str, Any]]) -> Dict[str, Dict[str, Any]]: - """How each skill actually fared, from evidence that distinguishes use from exposure. - - ``offers`` counts turns where the skill was **available** to the model — its - name and description were in the prompt. Skill loading offers every enabled - skill, so this is a denominator, not a choice: it answers "how many chances - did this skill have to be picked", which is exactly what makes "never - opened" mean something. - - ``opens`` is the choice. The gap between the two is the whole signal. - """ - stats: Dict[str, Dict[str, Any]] = {} - - def bucket(skill_id: str) -> Dict[str, Any]: - return stats.setdefault( - str(skill_id), - { - "offers": 0, - "opens": 0, - "opens_succeeded": 0, - "opens_followed_through": 0, - "opens_measurable": 0, - "exposure_days": 0.0, - }, - ) - - declared_by_skill = _declared_tools_for( - sorted( - { - str(opened.get("skill_id") or "") - for view in views - for opened in (view.get("skills_opened") or []) - if opened.get("skill_id") - } - ) - ) - - for view in views: - for skill_id in view.get("skill_sequence") or []: - bucket(skill_id)["offers"] += 1 - - succeeded = view.get("verdict") == "success" - for opened in view.get("skills_opened") or []: - skill_id = str(opened.get("skill_id") or "") - if not skill_id: - continue - entry = bucket(skill_id) - entry["opens"] += 1 - if succeeded: - entry["opens_succeeded"] += 1 - - # Follow-through: after reading the document, did the model do what - # the document is about? Approximated by whether the tools it went - # on to call are among the ones the skill declares. - # - # Read from the skill itself, not from the event — an event cannot - # carry what the skill declared *now*, and a stale copy would score - # a rewritten skill against its old tool list. - declared = {str(t) for t in (declared_by_skill.get(skill_id) or [])} - after = [str(t) for t in (opened.get("tools_after_open") or [])] - if not declared or not after: - # A skill that declares no tools (a rules-only skill shapes - # ordering, not tool choice) or a turn that called nothing after - # the open cannot be measured this way. Excluded from the - # denominator rather than counted as a pass — counting them as - # passes is what made follow-through 100% for everything and the - # "opened but ignored" verdict unreachable. - continue - entry["opens_measurable"] += 1 - if any(tool in declared for tool in after): - entry["opens_followed_through"] += 1 - - uplift_by_skill = _uplift_from_canary(views) - for skill_id, entry in stats.items(): - opens = entry["opens"] - measurable = entry["opens_measurable"] - entry["success_rate"] = entry["opens_succeeded"] / opens if opens else 0.0 - entry["follow_through"] = ( - entry["opens_followed_through"] / measurable if measurable else None - ) - entry["exposure_days"] = _exposure_days(skill_id) - entry["uplift"] = uplift_by_skill.get(skill_id) - return stats - - -def _canary_releases() -> List[Tuple[str, str, int, Any]]: - """``(asset_id, release_id, traffic_percent, started_at)`` for live canaries.""" - from core.evolution.release import CANARY - - try: - from core.db.engine import SessionLocal - from core.db.models.evolution import EvolutionRelease - - with SessionLocal() as db: - rows = ( - db.query(EvolutionRelease) - .filter( - EvolutionRelease.target_kind == "skill", - EvolutionRelease.stage == CANARY, - ) - .all() - ) - return [ - ( - str(row.target_asset_id), - str(row.release_id), - int(row.traffic_percent or 0), - row.created_at, - ) - for row in rows - ] - except Exception as exc: # noqa: BLE001 - logger.debug("[cycle-extras] canary ledger unavailable: %s", exc) - return [] - - -def _uplift_from_canary( - views: Sequence[Dict[str, Any]], - *, - ramping: Optional[Sequence[Tuple[str, str, int, Any]]] = None, -) -> Dict[str, Optional[float]]: - """"Is the deployment better off with this skill?", answered by comparison. - - A canary release already splits users deterministically into "sees it" and - "does not" (:func:`core.evolution.release.in_canary_bucket`). That split is - the comparison: same period, same task mix, same everything except exposure - to this one asset. Recomputing bucket membership here rather than storing it - per request means the analysis works on history too. - - Returns ``None`` for a skill with no usable comparison. ``None`` and ``0.0`` - are different answers — "we have not measured" must never read as "measured, - no effect", because the second one licenses a rollback decision and the - first one does not. - - ``ramping`` is injectable so this is testable against the real bucketing - rather than against a re-implementation of it in a test helper. - """ - from core.evolution.release import in_canary_bucket - - if ramping is None: - ramping = _canary_releases() - - uplift: Dict[str, Optional[float]] = {} - for skill_id, release_id, traffic, started_at in ramping: - exposed: List[bool] = [] - withheld: List[bool] = [] - for view in views: - created_at = view.get("created_at") - if created_at is None or started_at is None: - continue - if not _is_after(created_at, started_at): - continue - user_id = str(view.get("user_id") or "") - if not user_id: - continue - succeeded = view.get("verdict") == "success" - if in_canary_bucket( - release_id=release_id, subject_id=user_id, traffic_percent=traffic - ): - exposed.append(succeeded) - else: - withheld.append(succeeded) - - # Both arms need enough turns for a rate to mean anything. A one-sided - # or tiny comparison produces a number that looks like evidence and is - # not, and this number can trigger an immediate rollback. - if len(exposed) < MIN_UPLIFT_SAMPLES or len(withheld) < MIN_UPLIFT_SAMPLES: - uplift[skill_id] = None - continue - uplift[skill_id] = (sum(exposed) / len(exposed)) - ( - sum(withheld) / len(withheld) - ) - return uplift - - -def _exposure_days(skill_id: str) -> float: - """How long this skill has been exposed to anyone. - - Read from the release ledger. Without it, "never opened" cannot be told - apart from "shipped yesterday", and a retirement rule that cannot tell them - apart retires whatever is newest. - """ - from datetime import datetime, timezone - - try: - from core.db.engine import SessionLocal - from core.db.models.evolution import EvolutionRelease - from core.evolution.release import ACTIVE, CANARY - - with SessionLocal() as db: - row = ( - db.query(EvolutionRelease) - .filter( - EvolutionRelease.target_kind == "skill", - EvolutionRelease.target_asset_id == skill_id, - EvolutionRelease.stage.in_([ACTIVE, CANARY]), - ) - .order_by(EvolutionRelease.created_at.asc()) - .first() - ) - except Exception as exc: # noqa: BLE001 - logger.debug("[cycle-extras] exposure lookup failed for %s: %s", skill_id, exc) - return 0.0 - if row is None or row.created_at is None: - # Not an evolved skill, or never released through the ladder. A - # hand-authored skill has always been exposed, so judging it on its - # observed usage is exactly right. - return float("inf") - created = row.created_at - if created.tzinfo is None: - created = created.replace(tzinfo=timezone.utc) - return (datetime.now(timezone.utc) - created).total_seconds() / 86400.0 - - -def negative_transfer_findings( - views: Sequence[Dict[str, Any]], - *, - asset_id: str = "", - related_task_types: Sequence[str] = (), - before_key: str = "before", - after_key: str = "after", -) -> List[Dict[str, Any]]: - """Task types that got worse after some activation, unrelated to it. - - The one failure mode of joint evolution that no single release's guardrail - can catch: each change looks fine on the metric it was measured against, and - the damage shows up somewhere nobody was looking. - """ - from core.evolution import promotion as P - - by_type_before: Dict[str, List[bool]] = {} - by_type_after: Dict[str, List[bool]] = {} - for view in views: - target = by_type_after if view.get(after_key) else by_type_before - target.setdefault(str(view.get("task_type") or "chat"), []).append( - view.get("verdict") == "success" - ) - - def rates(source: Dict[str, List[bool]]) -> Dict[str, float]: - return {k: (sum(v) / len(v) if v else 0.0) for k, v in source.items()} - - findings = P.detect_negative_transfer( - asset_id=asset_id or "unspecified", - # Only *unrelated* task types are examined. A dip on the task the change - # targeted is an ordinary regression its own guardrail catches; the - # dangerous case is collateral damage where nobody is looking. - related_task_types=list(related_task_types), - before=rates(by_type_before), - after=rates(by_type_after), - ) - return [f.to_dict() for f in findings] - - -# ── Memory: the half that makes it evolution rather than accumulation ──────── - - -def memory_findings( - views: Sequence[Dict[str, Any]], - *, - scan_findings: Optional[Sequence[Any]] = None, -) -> Dict[str, Any]: - """Reorganisation proposals for the memory layer. - - Two independent sources, because they answer different questions and only - one of them is answerable from a run's evidence: - - * **inventory** (``scan_findings``) — duplicates, contradictions and stale - entries, which require reading the store; - * **behaviour** (``views``) — memories that keep being injected into turns - that fail, which requires reading the runs. - - The behavioural signal is deliberately *not* framed as duplication. A - memory injected repeatedly is either helping or not, and the difference is - the outcome of the turns it landed in — never the count on its own. - """ - from core.evolution.memory_ops import ( - OP_MERGE, - OP_REWEIGHT, - SCOPE_USER, - MemoryOp, - partition_by_approval, - ) - from core.evolution.memory_scan import ( - FINDING_CONTRADICTION, - FINDING_DUPLICATE, - FINDING_STALE, - ) - - retrieved: Dict[str, int] = {} - helped: Dict[str, int] = {} - for view in views: - for ref in view.get("memory_refs") or []: - key = str(ref) - retrieved[key] = retrieved.get(key, 0) + 1 - if view.get("verdict") == "success": - helped[key] = helped.get(key, 0) + 1 - - ops: List[MemoryOp] = [] - contradictions: List[Dict[str, Any]] = [] - - # 1. Inventory findings, from an actual scan of the store. - for finding in scan_findings or []: - payload = finding.to_dict() if hasattr(finding, "to_dict") else dict(finding) - kind = payload.get("kind") - refs = [str(r) for r in (payload.get("refs") or [])] - if kind == FINDING_DUPLICATE and len(refs) >= 2: - # The second entry loses. Content is not rewritten and nothing is - # removed — the survivor already says the same thing. - ops.append( - MemoryOp( - operation=OP_MERGE, - memory_ref=refs[1], - scope=SCOPE_USER, - reason=payload.get("reason", ""), - before={"injected": True, "duplicate_of": refs[0]}, - after={"injected": False, "superseded_by": refs[0]}, - ) - ) - elif kind == FINDING_STALE and refs: - ops.append( - MemoryOp( - operation=OP_REWEIGHT, - memory_ref=refs[0], - scope=SCOPE_USER, - reason=payload.get("reason", ""), - before={"weight": 1.0}, - after={"weight": 0.7}, - ) - ) - elif kind == FINDING_CONTRADICTION: - # Neither side is automatically right, and picking by recency is - # how a corrected mistake becomes the new truth. This is reported - # for a human, with both sides and their dates. - contradictions.append(payload) - - # 2. Behavioural: injected often, associated with failure. Down-weighted, - # never deleted — a wrong reweight is recoverable and a wrong deletion is - # not. - for ref, count in retrieved.items(): - if count < DEMOTE_MIN_HITS: - continue - rate = helped.get(ref, 0) / count - if rate <= DEMOTE_SUCCESS_CEILING: - ops.append( - MemoryOp( - operation=OP_REWEIGHT, - memory_ref=ref, - scope=SCOPE_USER, - reason=( - f"被注入 {count} 次，其中仅 {helped.get(ref, 0)} 次所在回答成功" - "——检索到它并没有帮上忙" - ), - before={"weight": 1.0}, - after={"weight": 0.5}, - ) - ) - - auto, needs_review = partition_by_approval(ops) - return { - "distinct_memories_retrieved": len(retrieved), - "merge_proposals": sum(1 for op in ops if op.operation == OP_MERGE), - "reweight_proposals": sum(1 for op in ops if op.operation == OP_REWEIGHT), - "contradictions": contradictions, - # Reported separately because the split *is* the governance boundary: - # reweighting is reversible and may proceed, merging changes what gets - # injected and may not. - "auto_applicable": [op.to_dict() for op in auto], - "needs_review": [op.to_dict() for op in needs_review], - "ops": ops, - } - - -# ── One entry point for the cycle ──────────────────────────────────────────── - - -def run_extras( - views: Sequence[Dict[str, Any]], - *, - scan_findings: Optional[Sequence[Any]] = None, -) -> Dict[str, Any]: - """Everything the cycle was missing, in one call. - - Each section is independently guarded: a failure in the decremental pass - must not cost the intent pass its findings, because they answer different - questions and an operator needs whichever ones succeeded. - """ - report: Dict[str, Any] = {} - for name, fn in ( - ("intent", lambda: intent_findings(views)), - ("decremental", lambda: decremental_findings(views)), - ("memory", lambda: memory_findings(views, scan_findings=scan_findings)), - ): - try: - report[name] = fn() - except Exception as exc: # noqa: BLE001 - logger.warning("[cycle-extras] %s pass failed: %s", name, exc) - report[name] = {"error": str(exc)} - return report diff --git a/src/backend/core/evolution/evidence/__init__.py b/src/backend/core/evolution/evidence/__init__.py new file mode 100644 index 00000000..09947758 --- /dev/null +++ b/src/backend/core/evolution/evidence/__init__.py @@ -0,0 +1 @@ +"""Evidence for evolution: traces, replay cassettes, credit assignment and lineage.""" diff --git a/src/backend/core/evolution/cassette.py b/src/backend/core/evolution/evidence/cassette.py similarity index 94% rename from src/backend/core/evolution/cassette.py rename to src/backend/core/evolution/evidence/cassette.py index 7a007f00..0b606d75 100644 --- a/src/backend/core/evolution/cassette.py +++ b/src/backend/core/evolution/evidence/cassette.py @@ -25,7 +25,7 @@ import json import logging from dataclasses import dataclass, field -from typing import Any, Dict, List, Optional, Sequence, Set, Tuple +from typing import Any, Dict, List, Optional, Sequence, Tuple logger = logging.getLogger(__name__) @@ -180,18 +180,13 @@ def _offload_reference(result: Any) -> Optional[str]: def _load_offloaded(result: Any) -> Optional[Any]: - """Fetch offloaded content. Returns ``None`` when it cannot be restored.""" - reference = _offload_reference(result) - if reference is None: - return None - try: - from core.llm.offloader import load_offloaded_result # type: ignore + """Recorded paths alone do not carry a workspace/user authorization scope. - restored = load_offloaded_result(reference) - return restored if restored is not None else None - except Exception as exc: - logger.debug("[cassette] offload restore failed for %s: %s", reference, exc) - return None + The writer has no synchronous restore API. Keep the established replay miss + instead of attempting an unscoped filesystem read or importing a nonexistent + SDK helper. Replay callers receive MISS_OFFLOADED and must use live execution. + """ + return None def build_cassette(message_ids: Sequence[str]) -> Cassette: diff --git a/src/backend/core/evolution/counterfactual.py b/src/backend/core/evolution/evidence/counterfactual.py similarity index 99% rename from src/backend/core/evolution/counterfactual.py rename to src/backend/core/evolution/evidence/counterfactual.py index 95a2ecda..82c566eb 100644 --- a/src/backend/core/evolution/counterfactual.py +++ b/src/backend/core/evolution/evidence/counterfactual.py @@ -30,7 +30,7 @@ import logging from dataclasses import dataclass, field -from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple +from typing import Any, Callable, Dict, List, Optional, Sequence logger = logging.getLogger(__name__) diff --git a/src/backend/core/evolution/credit.py b/src/backend/core/evolution/evidence/credit.py similarity index 99% rename from src/backend/core/evolution/credit.py rename to src/backend/core/evolution/evidence/credit.py index 2744e713..bdd5e752 100644 --- a/src/backend/core/evolution/credit.py +++ b/src/backend/core/evolution/evidence/credit.py @@ -26,7 +26,7 @@ import logging from dataclasses import dataclass, field -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Dict, List, Tuple logger = logging.getLogger(__name__) diff --git a/src/backend/core/evolution/events.py b/src/backend/core/evolution/evidence/events.py similarity index 99% rename from src/backend/core/evolution/events.py rename to src/backend/core/evolution/evidence/events.py index 217a24d6..31ac691e 100644 --- a/src/backend/core/evolution/events.py +++ b/src/backend/core/evolution/evidence/events.py @@ -202,7 +202,7 @@ def flush(self) -> int: if self._flushed or not self._events: return 0 try: - from core.evolution.trace_store import persist_events + from core.evolution.evidence.trace_store import persist_events written = persist_events(self._events) self._flushed = True diff --git a/src/backend/core/evolution/evidence_contract.py b/src/backend/core/evolution/evidence/evidence_contract.py similarity index 100% rename from src/backend/core/evolution/evidence_contract.py rename to src/backend/core/evolution/evidence/evidence_contract.py diff --git a/src/backend/core/evolution/evidence_resolver.py b/src/backend/core/evolution/evidence/evidence_resolver.py similarity index 98% rename from src/backend/core/evolution/evidence_resolver.py rename to src/backend/core/evolution/evidence/evidence_resolver.py index 61202484..a0946fbd 100644 --- a/src/backend/core/evolution/evidence_resolver.py +++ b/src/backend/core/evolution/evidence/evidence_resolver.py @@ -20,13 +20,7 @@ import logging from typing import Any, Dict, List, Optional, Sequence -from core.evolution.evidence_contract import ( - LAYER_GRAPH, - LAYER_PROCEDURE, - CapabilityEvidencePackV2, - EvidenceRef, - role_for_predicate, -) +from core.evolution.evidence.evidence_contract import LAYER_GRAPH, LAYER_PROCEDURE, CapabilityEvidencePackV2, EvidenceRef, role_for_predicate logger = logging.getLogger(__name__) diff --git a/src/backend/core/evolution/exposure.py b/src/backend/core/evolution/evidence/exposure.py similarity index 95% rename from src/backend/core/evolution/exposure.py rename to src/backend/core/evolution/evidence/exposure.py index e8dc39fe..66f73d52 100644 --- a/src/backend/core/evolution/exposure.py +++ b/src/backend/core/evolution/evidence/exposure.py @@ -30,9 +30,9 @@ from __future__ import annotations import logging -from typing import Dict, Iterable, List, Optional, Sequence, Set +from typing import Dict, Iterable, List, Sequence, Set -from core.evolution.activation import EVOLUTION_PREFIX +from core.evolution.cycle.activation_constants import EVOLUTION_PREFIX logger = logging.getLogger(__name__) @@ -90,7 +90,7 @@ def visible_evolved_skill_ids( tenant_id: str = "default", ) -> Set[str]: """Of ``candidate_ids``, the evolved skill ids this subject may load.""" - from core.evolution.release import ACTIVE, CANARY, in_canary_bucket + from core.evolution.cycle.release import ACTIVE, CANARY, in_canary_bucket evolved = {sid for sid in candidate_ids if is_evolved_skill_id(sid)} if not evolved: @@ -134,7 +134,7 @@ def _applicable_by_manifest(db, skill_ids: Set[str], *, task_type: str) -> Set[s for it. Skills without a manifest (hand-authored, pre-manifest) pass. """ from core.db.models import AdminSkill - from core.evolution.applicability import load_manifest, manifest_permits + from core.evolution.assessment.applicability import load_manifest, manifest_permits rows = ( db.query(AdminSkill.skill_id, AdminSkill.extra_files) @@ -198,7 +198,7 @@ def _resolve(session) -> Set[str]: def exposure_state(db, skill_id: str) -> Dict[str, object]: """Human-readable exposure of one evolved skill, for the console.""" - from core.evolution.release import ACTIVE, CANARY, SHADOW + from core.evolution.cycle.release import ACTIVE, CANARY, SHADOW latest = _latest_releases_by_asset(db, [skill_id]) release = latest.get(skill_id) diff --git a/src/backend/core/evolution/lineage.py b/src/backend/core/evolution/evidence/lineage.py similarity index 99% rename from src/backend/core/evolution/lineage.py rename to src/backend/core/evolution/evidence/lineage.py index 98cedb8c..5612f4f2 100644 --- a/src/backend/core/evolution/lineage.py +++ b/src/backend/core/evolution/evidence/lineage.py @@ -16,7 +16,7 @@ import logging from typing import Any, Dict, List, Optional, Sequence, Tuple -from core.evolution.evidence_contract import CapabilityEvidencePackV2 +from core.evolution.evidence.evidence_contract import CapabilityEvidencePackV2 logger = logging.getLogger(__name__) diff --git a/src/backend/core/evolution/outcomes.py b/src/backend/core/evolution/evidence/outcomes.py similarity index 100% rename from src/backend/core/evolution/outcomes.py rename to src/backend/core/evolution/evidence/outcomes.py diff --git a/src/backend/core/evolution/replay.py b/src/backend/core/evolution/evidence/replay.py similarity index 100% rename from src/backend/core/evolution/replay.py rename to src/backend/core/evolution/evidence/replay.py diff --git a/src/backend/core/evolution/trace_assembler.py b/src/backend/core/evolution/evidence/trace_assembler.py similarity index 98% rename from src/backend/core/evolution/trace_assembler.py rename to src/backend/core/evolution/evidence/trace_assembler.py index 112c4ca6..d161ae69 100644 --- a/src/backend/core/evolution/trace_assembler.py +++ b/src/backend/core/evolution/evidence/trace_assembler.py @@ -24,12 +24,12 @@ import re import uuid from datetime import datetime, timezone -from typing import Any, Dict, List, Optional +from typing import Any, Dict, Optional from core.db.engine import SessionLocal from core.db.models.evolution import EvolutionEpisode, EvolutionTraceEvent from core.evolution.contract import AssetBundle -from core.evolution.trace_store import attach_episode +from core.evolution.evidence.trace_store import attach_episode logger = logging.getLogger(__name__) @@ -177,7 +177,7 @@ def emit_tool_events(sink, message_id: str, chat_id: str = "") -> int: if not message_id: return 0 try: - from core.evolution.events import EV_SKILL_OPENED, EV_TOOL_CALLED + from core.evolution.evidence.events import EV_SKILL_OPENED, EV_TOOL_CALLED with SessionLocal() as db: rows = _tool_rows_for(db, message_id, chat_id) diff --git a/src/backend/core/evolution/trace_store.py b/src/backend/core/evolution/evidence/trace_store.py similarity index 97% rename from src/backend/core/evolution/trace_store.py rename to src/backend/core/evolution/evidence/trace_store.py index 490e4524..944bd1ba 100644 --- a/src/backend/core/evolution/trace_store.py +++ b/src/backend/core/evolution/evidence/trace_store.py @@ -12,11 +12,11 @@ from __future__ import annotations import logging -from typing import Iterable, List, Optional, Sequence +from typing import List, Sequence from core.db.engine import SessionLocal from core.db.models.evolution import EvolutionTraceEvent -from core.evolution.events import TraceEvent +from core.evolution.evidence.events import TraceEvent logger = logging.getLogger(__name__) diff --git a/src/backend/core/evolution/generators/__init__.py b/src/backend/core/evolution/generators/__init__.py new file mode 100644 index 00000000..6b0bafac --- /dev/null +++ b/src/backend/core/evolution/generators/__init__.py @@ -0,0 +1 @@ +"""Generate assets from evolution candidates: skills and orchestration profiles.""" diff --git a/src/backend/core/evolution/graph_scan.py b/src/backend/core/evolution/generators/graph_scan.py similarity index 100% rename from src/backend/core/evolution/graph_scan.py rename to src/backend/core/evolution/generators/graph_scan.py diff --git a/src/backend/core/evolution/memory_apply.py b/src/backend/core/evolution/generators/memory_apply.py similarity index 98% rename from src/backend/core/evolution/memory_apply.py rename to src/backend/core/evolution/generators/memory_apply.py index 3ebe4a61..f879c940 100644 --- a/src/backend/core/evolution/memory_apply.py +++ b/src/backend/core/evolution/generators/memory_apply.py @@ -28,11 +28,7 @@ from datetime import datetime, timezone from typing import Any, Dict, List, Optional, Sequence -from core.evolution.memory_ops import ( - OP_MERGE, - OP_REWEIGHT, - MemoryOp, -) +from core.evolution.generators.memory_ops import OP_MERGE, OP_REWEIGHT, MemoryOp from core.memory.weights import OP_SUPERSEDE, STATUS_APPLIED, STATUS_REVERTED logger = logging.getLogger(__name__) @@ -243,7 +239,7 @@ def apply_memory_candidate( """ from core.db.engine import SessionLocal from core.db.models.evolution import EvolutionCandidate - from core.evolution.activation import ActivationError + from core.evolution.cycle.activation_types import ActivationError with SessionLocal() as db: candidate = db.get(EvolutionCandidate, candidate_id) diff --git a/src/backend/core/evolution/memory_ops.py b/src/backend/core/evolution/generators/memory_ops.py similarity index 99% rename from src/backend/core/evolution/memory_ops.py rename to src/backend/core/evolution/generators/memory_ops.py index 7b749c30..ce55fd39 100644 --- a/src/backend/core/evolution/memory_ops.py +++ b/src/backend/core/evolution/generators/memory_ops.py @@ -25,7 +25,7 @@ import logging from dataclasses import dataclass, field -from typing import Any, Dict, List, Optional, Sequence, Tuple +from typing import Any, Dict, List, Sequence, Tuple logger = logging.getLogger(__name__) diff --git a/src/backend/core/evolution/memory_scan.py b/src/backend/core/evolution/generators/memory_scan.py similarity index 99% rename from src/backend/core/evolution/memory_scan.py rename to src/backend/core/evolution/generators/memory_scan.py index fa739858..776e3725 100644 --- a/src/backend/core/evolution/memory_scan.py +++ b/src/backend/core/evolution/generators/memory_scan.py @@ -157,7 +157,7 @@ def _cosine(a: Sequence[float], b: Sequence[float]) -> float: def _embed(texts: List[str]) -> Optional[List[List[float]]]: try: - from core.kb.kb_vector import embed_batch + from core.models.embedding import embed_batch vectors = embed_batch(texts) except Exception as exc: # noqa: BLE001 diff --git a/src/backend/core/evolution/ontology_bridge.py b/src/backend/core/evolution/generators/ontology_bridge.py similarity index 100% rename from src/backend/core/evolution/ontology_bridge.py rename to src/backend/core/evolution/generators/ontology_bridge.py diff --git a/src/backend/core/evolution/orchestration_gen.py b/src/backend/core/evolution/generators/orchestration_gen.py similarity index 99% rename from src/backend/core/evolution/orchestration_gen.py rename to src/backend/core/evolution/generators/orchestration_gen.py index b070aa04..c28f0bc1 100644 --- a/src/backend/core/evolution/orchestration_gen.py +++ b/src/backend/core/evolution/generators/orchestration_gen.py @@ -434,7 +434,7 @@ def profile_from_proposals( always load these two skills" would hide the fact that the second is only safe given the first. """ - from core.evolution.agent_profile import AgentProfile + from core.evolution.profiles.agent_profile import AgentProfile merged = AgentProfile.from_dict(base.to_dict()) merged.profile_id = f"auto-{task_type}" diff --git a/src/backend/core/evolution/prompt_gen.py b/src/backend/core/evolution/generators/prompt_gen.py similarity index 100% rename from src/backend/core/evolution/prompt_gen.py rename to src/backend/core/evolution/generators/prompt_gen.py diff --git a/src/backend/core/evolution/skill_candidate_engine.py b/src/backend/core/evolution/generators/skill_candidate_engine.py similarity index 96% rename from src/backend/core/evolution/skill_candidate_engine.py rename to src/backend/core/evolution/generators/skill_candidate_engine.py index 2492f929..3b234a90 100644 --- a/src/backend/core/evolution/skill_candidate_engine.py +++ b/src/backend/core/evolution/generators/skill_candidate_engine.py @@ -18,17 +18,14 @@ """ from __future__ import annotations +from core.evolution.generators import skill_gen import logging from dataclasses import dataclass, field from typing import Any, Callable, Dict, List, Optional -from core.evolution.evidence_contract import ( - CapabilityEvidencePackV2, - build_manifest, - validate_candidate_structure, -) -from core.evolution.evidence_resolver import resolve_capability_pack +from core.evolution.evidence.evidence_contract import CapabilityEvidencePackV2, build_manifest, validate_candidate_structure +from core.evolution.evidence.evidence_resolver import resolve_capability_pack from core.evolution.ir import ( RISK_HIGH, EvolutionIR, @@ -105,7 +102,6 @@ def create_skill_candidate( if spec.write_document: if run_async is None: return {"status": STATUS_REJECTED, "code": "no_async_runner"} - import core.evolution.skill_gen as skill_gen document, draft = run_async( skill_gen.generate_skill_document( @@ -240,8 +236,8 @@ def _persist( """ from core.db.engine import SessionLocal from core.db.models.evolution import EvolutionCandidate - from core.evolution import lineage - from core.evolution.release import DRAFT + from core.evolution.evidence import lineage + from core.evolution.cycle.release import DRAFT pack_outcome = lineage.persist_evidence_pack(pack) decision_id = lineage.record_credit_decision( diff --git a/src/backend/core/evolution/generators/skill_drafting.py b/src/backend/core/evolution/generators/skill_drafting.py new file mode 100644 index 00000000..89902db3 --- /dev/null +++ b/src/backend/core/evolution/generators/skill_drafting.py @@ -0,0 +1,185 @@ +"""Prompt, parse and validate a model-drafted skill body.""" +from __future__ import annotations +import re +from typing import Any, List, Sequence, Tuple +from core.evolution.generators import skill_evidence +from core.evolution.generators import skill_gen_constants + +def format_list(items: Sequence[Any], empty: str = "（无）") -> str: + if not items: + return empty + lines = [] + for item in items: + if isinstance(item, (list, tuple)): + lines.append("- " + " → ".join(str(x) for x in item)) + elif isinstance(item, dict): + lines.append("- " + "；".join(f"{k}: {v}" for k, v in item.items() if v)) + else: + lines.append(f"- {item}") + return "\n".join(lines) + + +def build_prompt(pack: skill_evidence.EvidencePack, violations: Sequence[str]) -> str: + exclusion_hint = ( + f"证据显示本技能在以下任务类型中被推翻，必须写进不适用范围：{'、'.join(pack.excluded_task_types)}。" + if pack.excluded_task_types + else "如果证据没有显示明确的排除场景，就写明本技能只适用于「适用条件」描述的场景，其余场景请忽略本技能。" + ) + retry_hint = "" + if violations: + retry_hint = ( + "# 上一版被打回的原因（必须修正）\n" + + "\n".join(f"- {v}" for v in violations) + + "\n" + ) + return skill_gen_constants.DRAFT_PROMPT.format( + objective_count=len(pack.objectives), + objectives=format_list(pack.objectives[:10]), + tools=format_list([f"`{t}`" for t in pack.tools]), + sequences=format_list(pack.successful_sequences[:6]), + failures=format_list(pack.failures[:6]), + constraints=format_list( + [ + f"先 {c.get('before')} 再 {c.get('after')}" + + ( + f"（在 {'、'.join(str(x) for x in (c.get('validated_in') or []))} 中验证）" + if c.get("validated_in") + else "" + ) + for c in pack.ordering_constraints + if not (c.get("contradicted_in") or []) + ] + ), + procedures=format_list( + [ + p.get("rule", "") + (f"（原因：{p['why']}）" if p.get("why") else "") + for p in pack.procedures[:10] + ] + ), + exclusion_hint=exclusion_hint, + max_chars=skill_gen_constants.MAX_BODY_CHARS, + retry_hint=retry_hint, + ) + + +def parse_draft(raw: str) -> Tuple[str, str, str]: + """``(display_name, description, body)`` from the model's reply.""" + display_name = "" + description = "" + body = "" + text = (raw or "").strip() + if text.startswith("```"): + lines = text.split("\n") + text = "\n".join(lines[1:]) + if text.rstrip().endswith("```"): + text = text.rstrip()[:-3] + + body_marker = re.search(r"^BODY:\s*$", text, re.MULTILINE) + head = text[: body_marker.start()] if body_marker else text + if body_marker: + body = text[body_marker.end() :].strip() + + for line in head.splitlines(): + if line.upper().startswith("DISPLAY_NAME:"): + display_name = line.split(":", 1)[1].strip() + elif line.upper().startswith("DESCRIPTION:"): + description = line.split(":", 1)[1].strip() + return display_name, description, body + + +def validate_draft( + *, + skill_id: str, + display_name: str, + description: str, + body: str, + allowed_tools: Sequence[str], +) -> List[str]: + """Every constraint stage B was told about, checked. Returns violations.""" + violations: List[str] = [] + + if not body.strip(): + return [skill_gen_constants.REJECT_EMPTY] + if not display_name.strip() or not description.strip(): + violations.append(skill_gen_constants.REJECT_NO_FRONTMATTER) + if not skill_gen_constants.FRONTMATTER_NAME.match(skill_id or ""): + # The loader treats ``name`` as the id and rejects anything outside + # ``[a-z0-9_-]``. A skill that fails this still appears in the capability + # list — that metadata comes from table columns — while loading returns + # nothing, so it is present and inert, which is the hardest failure of + # all to notice. + violations.append(skill_gen_constants.REJECT_BAD_NAME) + + if skill_gen_constants.CODE_FENCE.search(body): + violations.append(skill_gen_constants.REJECT_CODE_BLOCK) + if skill_gen_constants.URL.search(body): + violations.append(skill_gen_constants.REJECT_EXTERNAL_URL) + if len(body) > skill_gen_constants.MAX_BODY_CHARS: + violations.append(skill_gen_constants.REJECT_TOO_LONG) + if len(body) < skill_gen_constants.MIN_BODY_CHARS: + violations.append(skill_gen_constants.REJECT_TOO_SHORT) + + if "## 不适用范围" not in body: + violations.append(skill_gen_constants.REJECT_NO_SCOPE) + else: + # The exclusion has to precede anything the model could copy: measured, + # withholding a contradicted rule was not enough, because the step list + # generalises on its own. + scope_at = body.index("## 不适用范围") + earlier_section = re.search(r"^## ", body[:scope_at], re.MULTILINE) + if earlier_section: + violations.append(skill_gen_constants.REJECT_NO_SCOPE) + if "## 常见失败与处理" not in body and "## 步骤" not in body: + violations.append(skill_gen_constants.REJECT_NO_CHECKPOINTS) + + allowed = {str(t) for t in allowed_tools} + mentioned = {m for m in skill_gen_constants.TOOL_MENTION.findall(body)} + # Only identifiers that look like tool names are checked: prose in backticks + # (a field name, a value) must not be mistaken for a tool grant, and a real + # tool name always appears verbatim in the evidence. + unknown = sorted( + name + for name in mentioned + if name not in allowed and "_" in name and name.islower() + ) + if unknown: + violations.append(f"{skill_gen_constants.REJECT_UNKNOWN_TOOL}: {unknown}") + + return violations + + +def render_skill_markdown( + *, + skill_id: str, + display_name: str, + description: str, + body: str, + support: int, + rationale: str = "", +) -> str: + """Wrap a validated body in the frontmatter the loader requires. + + ``name`` is the skill **id**, not the label — writing the human label there + made every materialised skill unparseable. + """ + provenance = ( + f"本技能由能力进化流水线从 {support} 条历史执行证据中沉淀而成，经机器校验后由人工批准。" + ) + if rationale: + provenance += f"\n\n沉淀依据：{rationale}" + return f"""--- +name: {skill_id} +display_name: {display_name} +description: {description} +--- + +# {display_name} + +{description} + +{body.strip()} + +## 来源 + +{provenance} +""" diff --git a/src/backend/core/evolution/generators/skill_evidence.py b/src/backend/core/evolution/generators/skill_evidence.py new file mode 100644 index 00000000..74a2c836 --- /dev/null +++ b/src/backend/core/evolution/generators/skill_evidence.py @@ -0,0 +1,124 @@ +"""The evidence pack a generated skill may be written from.""" +from __future__ import annotations +from dataclasses import dataclass, field +from typing import Any, Dict, List, Sequence + +@dataclass +class EvidencePack: + """Everything stage B is allowed to see, and nothing else.""" + + skill_id: str + objectives: List[str] = field(default_factory=list) + tools: List[str] = field(default_factory=list) + successful_sequences: List[List[str]] = field(default_factory=list) + failures: List[Dict[str, Any]] = field(default_factory=list) + ordering_constraints: List[Dict[str, Any]] = field(default_factory=list) + procedures: List[Dict[str, str]] = field(default_factory=list) + task_types: List[str] = field(default_factory=list) + excluded_task_types: List[str] = field(default_factory=list) + support: int = 0 + + def to_dict(self) -> Dict[str, Any]: + return { + "skill_id": self.skill_id, + "objectives": self.objectives[:10], + "tools": list(self.tools), + "successful_sequences": self.successful_sequences[:6], + "failures": self.failures[:6], + "ordering_constraints": self.ordering_constraints, + "procedures": self.procedures[:10], + "task_types": self.task_types, + "excluded_task_types": self.excluded_task_types, + "support": self.support, + } + + +@dataclass +class DraftResult: + ok: bool + body: str = "" + display_name: str = "" + description: str = "" + violations: List[str] = field(default_factory=list) + attempts: int = 0 + # ``True`` when no draft could be produced because the model was unreachable + # — an operational state, not a rejection, and the two must not look alike + # in the console. + generator_unavailable: bool = False + + def to_dict(self) -> Dict[str, Any]: + return { + "ok": self.ok, + "display_name": self.display_name, + "description": self.description, + "violations": list(self.violations), + "attempts": self.attempts, + "generator_unavailable": self.generator_unavailable, + "body_chars": len(self.body), + } + + +def build_evidence_pack( + *, + skill_id: str, + episodes: Sequence[Dict[str, Any]], + ordering_constraints: Sequence[Dict[str, Any]] = (), + procedures: Sequence[Dict[str, str]] = (), +) -> EvidencePack: + """Assemble what a skill may be written from. + + Failures are included deliberately. A document distilled only from successes + describes a happy path and says nothing about the step where things actually + go wrong, which is the part a reader needs most. + """ + objectives: List[str] = [] + tools: List[str] = [] + sequences: List[List[str]] = [] + failures: List[Dict[str, Any]] = [] + task_types: List[str] = [] + + for episode in episodes: + objective = str(episode.get("objective") or "").strip() + if objective and objective not in objectives: + objectives.append(objective) + task_type = str(episode.get("task_type") or "").strip() + if task_type and task_type not in task_types: + task_types.append(task_type) + + sequence = [str(t) for t in (episode.get("tool_sequence") or [])] + for tool in sequence: + if tool not in tools: + tools.append(tool) + if episode.get("verdict") == "success": + if sequence and sequence not in sequences: + sequences.append(sequence) + elif episode.get("verdict") == "failed": + failures.append( + { + "objective": objective[:160], + "tool_sequence": sequence[:12], + "failed_at": str(episode.get("failed_tool") or ""), + "error": str(episode.get("error") or "")[:200], + } + ) + + excluded = sorted( + { + str(name) + for rule in ordering_constraints + for name in (rule.get("contradicted_in") or []) + if str(name) + } + ) + return EvidencePack( + skill_id=skill_id, + objectives=objectives, + tools=tools, + successful_sequences=sequences, + failures=failures, + ordering_constraints=list(ordering_constraints), + procedures=[dict(p) for p in procedures], + task_types=[t for t in task_types if t not in excluded], + excluded_task_types=excluded, + support=len(episodes), + ) diff --git a/src/backend/core/evolution/generators/skill_gen.py b/src/backend/core/evolution/generators/skill_gen.py new file mode 100644 index 00000000..0da410d6 --- /dev/null +++ b/src/backend/core/evolution/generators/skill_gen.py @@ -0,0 +1,95 @@ +"""Generate a skill document from evidence, retrying on validation failures.""" +from __future__ import annotations +from typing import Any, Dict, List, Optional, Sequence, Tuple +import logging +from core.evolution.generators import skill_drafting +from core.evolution.generators import skill_evidence +from core.evolution.generators import skill_gen_constants + +logger = logging.getLogger(__name__) + +async def draft_skill(pack: skill_evidence.EvidencePack) -> skill_evidence.DraftResult: + """Draft and validate, retrying with the specific violations. + + Returns ``ok=False`` rather than raising. A skill that cannot be written to + the required standard must produce no candidate at all: shipping a draft + that failed validation would put exactly the document the constraints exist + to prevent in front of the model. + """ + from core.memory.extractors._base import run_llm_with_prompt + + violations: List[str] = [] + for attempt in range(1, skill_gen_constants.MAX_ATTEMPTS + 1): + prompt = skill_drafting.build_prompt(pack, violations) + raw = await run_llm_with_prompt(prompt, timeout_s=skill_gen_constants.DRAFT_TIMEOUT_S, max_tokens=2400) + if raw is None: + return skill_evidence.DraftResult( + ok=False, + violations=["generator_unavailable"], + attempts=attempt, + generator_unavailable=True, + ) + + display_name, description, body = skill_drafting.parse_draft(raw) + violations = skill_drafting.validate_draft( + skill_id=pack.skill_id, + display_name=display_name, + description=description, + body=body, + allowed_tools=pack.tools, + ) + if not violations: + return skill_evidence.DraftResult( + ok=True, + body=body, + display_name=display_name[:60], + description=description[:200].replace("\n", " "), + attempts=attempt, + ) + logger.info( + "[skill-gen] draft attempt %d rejected for %s: %s", + attempt, pack.skill_id, violations, + ) + + return skill_evidence.DraftResult(ok=False, violations=violations, attempts=skill_gen_constants.MAX_ATTEMPTS) + + +async def generate_skill_document( + *, + skill_id: str, + episodes: Sequence[Dict[str, Any]], + ordering_constraints: Sequence[Dict[str, Any]] = (), + procedures: Sequence[Dict[str, str]] = (), + rationale: str = "", +) -> Tuple[Optional[Dict[str, str]], skill_evidence.DraftResult]: + """All three stages. Returns ``(document, result)``; document is ``None`` on failure.""" + pack = skill_evidence.build_evidence_pack( + skill_id=skill_id, + episodes=episodes, + ordering_constraints=ordering_constraints, + procedures=procedures, + ) + if not pack.tools and not pack.procedures: + return None, skill_evidence.DraftResult(ok=False, violations=["no_evidence_to_write_from"]) + + result = await draft_skill(pack) + if not result.ok: + return None, result + + return ( + { + "skill_id": skill_id, + "display_name": result.display_name, + "description": result.description, + "content": skill_drafting.render_skill_markdown( + skill_id=skill_id, + display_name=result.display_name, + description=result.description, + body=result.body, + support=pack.support, + rationale=rationale, + ), + "allowed_tools": list(pack.tools), + }, + result, + ) diff --git a/src/backend/core/evolution/generators/skill_gen_constants.py b/src/backend/core/evolution/generators/skill_gen_constants.py new file mode 100644 index 00000000..c8bc9b4d --- /dev/null +++ b/src/backend/core/evolution/generators/skill_gen_constants.py @@ -0,0 +1,109 @@ +"""Limits and patterns used to validate generated skills.""" +from __future__ import annotations +import re + +MAX_ATTEMPTS = 2 + + +MAX_BODY_CHARS = 4000 + + +MIN_BODY_CHARS = 120 + + +DRAFT_TIMEOUT_S = 60 + + +CODE_FENCE = re.compile(r"```(?:\w+)?\s*\n", re.MULTILINE) + + +URL = re.compile(r"https?://", re.IGNORECASE) + + +FRONTMATTER_NAME = re.compile(r"^[a-z0-9_-]+$") + + +TOOL_MENTION = re.compile(r"`([A-Za-z_][A-Za-z0-9_.-]*)`") + + +REJECT_NO_FRONTMATTER = "missing_frontmatter" + + +REJECT_BAD_NAME = "invalid_skill_id" + + +REJECT_UNKNOWN_TOOL = "tool_not_in_evidence" + + +REJECT_CODE_BLOCK = "contains_code_block" + + +REJECT_EXTERNAL_URL = "contains_external_url" + + +REJECT_TOO_LONG = "body_too_long" + + +REJECT_TOO_SHORT = "body_too_short" + + +REJECT_NO_SCOPE = "missing_scope_section" + + +REJECT_NO_CHECKPOINTS = "missing_checkpoints" + + +REJECT_EMPTY = "empty_draft" + + +DRAFT_PROMPT = """你在为一个 AI 助手撰写一份「技能文档」(SKILL.md)。这份文档会被模型在遇到同类任务时读取并遵照执行。 + +# 你拿到的证据 + +## 用户实际提出过的请求（{objective_count} 条） +{objectives} + +## 成功执行时用过的工具 +{tools} + +## 成功的调用顺序 +{sequences} + +## 失败的执行（这些是最宝贵的部分，说明坑在哪） +{failures} + +## 已验证的顺序约束 +{constraints} + +## 这些任务里沉淀下来的做法/口径 +{procedures} + +# 硬性要求（违反任何一条，文档会被机器校验打回） + +1. **只能引用上面「成功执行时用过的工具」里出现过的工具名**，一个都不能新增、不能猜。 +2. **开头第一段必须是「## 不适用范围」**，写明本技能不适用于哪些场景。{exclusion_hint} +3. 必须有「## 适用条件」，写清楚**遇到什么样的请求才该用本技能**——要具体到能和别的任务区分开，不能写"当需要用到这些工具时"这种同义反复。 +4. 必须有「## 步骤」，每一步都要写清楚：做什么、**关键参数怎么给**、**怎么判断这一步做对了**（检查点）。 +5. 必须有「## 常见失败与处理」，基于上面的失败证据写。 +6. **禁止**出现任何代码块（``` 包裹的内容）、禁止出现外部链接（http/https）。 +7. 全文不超过 {max_chars} 字符。 +8. 用中文撰写，语气是给同事写操作手册，不要写"本技能由AI生成"之类的话。 + +{retry_hint} + +# 输出格式 + +严格按下面结构输出，不要加任何额外说明，不要用代码块包裹整篇文档： + +DISPLAY_NAME: <一行技能名，≤20字，说明这个技能干什么> +DESCRIPTION: <一行描述，≤80字，说明什么时候该用它——模型只看这一行决定要不要打开本技能> +BODY: +## 不适用范围 +... +## 适用条件 +... +## 步骤 +... +## 常见失败与处理 +... +""" diff --git a/src/backend/core/evolution/loop.py b/src/backend/core/evolution/loop.py deleted file mode 100644 index 07bbaa11..00000000 --- a/src/backend/core/evolution/loop.py +++ /dev/null @@ -1,1576 +0,0 @@ -"""The evolution loop, wired end to end (GCE integration). - -Everything up to here has been a component. This module is the loop itself: - - Episodes → patterns → credit → promotion → IR → invariants → candidate → - release (shadow → canary → active) → the next run binds the new version - -It runs off the response path, on the shared worker base, under the same cost -budget as distillation. It cannot activate anything on its own beyond the low -risk tier: everything else stops at a candidate awaiting review, which is the -whole point of calling this *governed* co-evolution. -""" - -from __future__ import annotations - -import hashlib -import logging -import uuid -from datetime import datetime, timezone -from typing import Any, Dict, List, Optional - -from core.db.engine import SessionLocal -from core.db.models.evolution import ( - EvolutionCandidate, - EvolutionEpisode, - EvolutionRelease, - EvolutionTraceEvent, -) -from core.evolution import promotion as P -from core.evolution.credit import ( - T_MEMORY, - T_ORCHESTRATION, - T_SKILL, - CreditFeatures, - aggregate_finding, - assign_credit, - confidence_from_evidence, -) -from core.evolution.ir import ( - RISK_HIGH, - RISK_LOW, - RISK_MEDIUM, - EvolutionIR, - InvariantViolation, - Rollback, - new_candidate_id, - validate_ir, -) -from core.evolution.release import DRAFT -from core.evolution.user_settings import PRIVACY_PRIVATE - -logger = logging.getLogger(__name__) - - -def _view_for(episode, events) -> Dict[str, Any]: - """One Episode plus its events, flattened into what the engines read. - - Three fields here did not exist and each blocked an engine outright: - - ``memory_refs`` now carries the **stable content ref**, not the raw content - hash. The two look alike and are not: the ref is what the overlay keys on, - so an op proposed against a hash could never be applied to anything. - - ``skills_opened`` is what the model chose to read, as opposed to - ``skill_sequence``, which is what it was offered. Every decrement decision - depends on the difference. - - ``selection_degraded`` marks turns where ranking could not run and - everything loaded. Counting those as selections is what made every skill's - offer count identical, and therefore useless. - """ - from core.evolution.events import ( - EV_MEMORY_RETRIEVED, - EV_SKILL_OPENED, - EV_SKILL_SELECTED, - EV_TOOL_CALLED, - ) - from core.memory.ref_shadow import build_ref_id - from core.memory.retrieval_types import LAYER_FACT, LAYER_GRAPH - - tools: List[str] = [] - skills: List[str] = [] - memory_refs: List[str] = [] - graph_refs: List[str] = [] - graph_relations: List[Dict[str, Any]] = [] - skills_opened: List[Dict[str, Any]] = [] - selection_degraded = False - error_count = 0 - - user_id = episode.user_id or "" - for event in events: - payload = event.payload or {} - if event.event_type == EV_TOOL_CALLED: - name = payload.get("tool_name") - if name: - tools.append(str(name)) - if str(payload.get("status") or "") == "error": - error_count += 1 - elif event.event_type == EV_SKILL_SELECTED: - if payload.get("degraded"): - selection_degraded = True - skills.extend(str(s) for s in (payload.get("selected") or [])) - elif event.event_type == EV_SKILL_OPENED: - skills_opened.append( - { - "skill_id": str(payload.get("skill_id") or ""), - "tools_after_open": [str(t) for t in (payload.get("tools_after_open") or [])], - } - ) - elif event.event_type == EV_MEMORY_RETRIEVED: - for item in payload.get("items") or []: - content_hash = str(item.get("content_hash") or "") - if not content_hash: - continue - memory_refs.append( - build_ref_id( - layer=str(item.get("layer") or LAYER_FACT), - user_id=user_id, - workspace_id=str(item.get("workspace_id") or "default"), - content_hash=content_hash, - ) - ) - # L3 is deliberately tracked separately from L2. A graph relation - # is declarative context, not a procedure, and must never enter the - # memory→skill compiler as though it were an instruction. - for relation in payload.get("graph") or []: - content_hash = str(relation.get("content_hash") or "") - if not content_hash: - continue - graph_refs.append( - build_ref_id( - layer=LAYER_GRAPH, - user_id=user_id, - workspace_id=str(relation.get("workspace_id") or "default"), - content_hash=content_hash, - ) - ) - # The full reference, kept for the evidence resolver: a pack's - # L3 entry must carry the store's stable relation_id so an - # activated skill's graph evidence can be resolved back to the - # exact Neo4j edge — a bare ref id cannot do that. - graph_relations.append( - { - "relation_id": str(relation.get("relation_id") or ""), - "content_hash": content_hash, - "predicate": str(relation.get("predicate") or ""), - "confidence": relation.get("confidence") or 0.0, - "workspace_id": str(relation.get("workspace_id") or "default"), - } - ) - - outcome = episode.outcome or {} - return { - "episode_id": episode.episode_id, - "chat_id": episode.chat_id or "", - "user_id": user_id, - "tenant_id": episode.tenant_id or "default", - "verdict": outcome.get("verdict") - or ("success" if (episode.quality_score or 0) >= 0.65 else "unknown"), - "tool_sequence": tools, - "skill_sequence": skills, - "skills_opened": skills_opened, - "selection_degraded": selection_degraded, - "memory_refs": memory_refs, - "graph_refs": graph_refs, - "graph_relations": graph_relations, - "task_type": episode.task_type or "chat", - "backfilled": bool(episode.backfilled), - # Repeated tool errors are the observable form of "this run was stuck", - # which sub-agent derivation needs and nothing else recorded. - "stalled": error_count >= 3, - # What the user actually asked. Clustering only ever saw tool sequences - # before this, so "the same question keeps coming back" was not a - # detectable event. Sanitised at write time, so this is the preview - # rather than the raw text. - "objective": episode.objective_preview or "", - "objective_hash": episode.objective_hash or "", - "attributed_to": str(outcome.get("attributed_to") or ""), - # Needed to place an episode before or after an activation. Without it - # negative-transfer detection has no way to split the corpus and returns - # nothing on every corpus — wired, running, and structurally incapable - # of finding anything. - "created_at": episode.created_at, - } - - -def load_episode_views(*, limit: int = 500, since_days: int = 30) -> List[Dict[str, Any]]: - """Flatten Episodes plus their trace events into the shape pattern mining wants. - - Backfilled Episodes are included on purpose: they cannot be replayed, but - they are perfectly good for discovering what recurs — which is exactly why - the backfill was worth doing. - """ - views: List[Dict[str, Any]] = [] - try: - with SessionLocal() as db: - # The single enforcement point for per-user opt-out. Episodes from - # users who declined to contribute are still recorded (they power - # that user's own card and history) but never feed cross-user - # pattern mining. Filtering here rather than at each call site is - # what makes the promise checkable. - episodes = ( - db.query(EvolutionEpisode) - .filter(EvolutionEpisode.privacy_class != PRIVACY_PRIVATE) - .order_by(EvolutionEpisode.created_at.desc()) - .limit(limit) - .all() - ) - if not episodes: - return [] - - ids = [e.episode_id for e in episodes] - events = ( - db.query(EvolutionTraceEvent) - .filter(EvolutionTraceEvent.episode_id.in_(ids)) - .order_by(EvolutionTraceEvent.seq.asc()) - .all() - ) - by_episode: Dict[str, List[EvolutionTraceEvent]] = {} - for event in events: - by_episode.setdefault(event.episode_id, []).append(event) - - for episode in episodes: - views.append(_view_for(episode, by_episode.get(episode.episode_id, []))) - except Exception as exc: - logger.warning("[evolution-loop] loading episodes failed: %s", exc) - return [] - return views - - -def _features_for(pattern: P.Pattern) -> CreditFeatures: - """Translate a discovered pattern into attribution features. - - Success-subsequence tool patterns no longer come through here: they are an - aggregate observation over successful runs, and inventing a ``failed`` - verdict to route them through the failure attributor is the exact move - :func:`core.evolution.credit.aggregate_finding` exists to replace. - """ - if pattern.kind == P.PATTERN_REPEATED_FAILURE: - return CreditFeatures( - outcome_verdict="failed", - outcome_confidence=0.8, - loop_stagnation=True, - no_diff_iterations=2, - ) - return CreditFeatures(outcome_verdict="unknown", outcome_confidence=0.3) - - -def _features_for_ordering_rules(constraints: List[Any]) -> CreditFeatures: - """Attribution features for "a stable ordering keeps being re-derived". - - The mapping, stated so it can be argued with: - - * ``repeated_tool_subsequence`` — a pairwise ordering recurring across - different tool sets is a repeated successful subsequence that is - re-planned on every occurrence, which is what the feature means. - * ``outcome_verdict="failed"`` — the runs this would fix are the ones that - ordered the calls wrongly; the successful ones are the evidence, not the - target. - - ``step_order_suspect`` is deliberately *not* set even though ordering is the - subject. That feature says "this pipeline's step order looks wrong", a - workflow-level claim, and asserting it ties workflow with skill at 0.65 — - the assigner then correctly reports ``unidentifiable`` and nothing is - produced. The evidence here does separate them: an ordering that holds - across several different tool sets and task types is reusable knowledge, not - one fixed pipeline, and "the workflow is fixed" is the reading it rules out. - - Confidence sits just above the ``MIN_CONFIDENCE`` floor on purpose: with - support and context breadth at their minimum it lands below the floor and - resolves to No-Update, so thin evidence cannot mint a permanent asset. - """ - support = max((int(getattr(c, "support", 0) or 0) for c in constraints), default=0) - contexts = max((int(getattr(c, "contexts", 0) or 0) for c in constraints), default=0) - return CreditFeatures( - outcome_verdict="failed", - outcome_confidence=min(0.9, 0.6 + 0.05 * support + 0.05 * contexts), - repeated_tool_subsequence=True, - ) - - -def _uncovered_constraints(constraints: List[Any], covered_tool_sets: List[Any]) -> List[Any]: - """Rules that no sequence candidate would carry into the runtime. - - A rule attached to a materialised sequence already reaches the agent through - that skill. The ones worth a candidate of their own are the rest — and on - real traffic that is most of them, because a rule earns its scope by - appearing in *several* tool-set shapes and no single sequence covers them - all. - """ - out: List[Any] = [] - for constraint in constraints: - pair = {constraint.before, constraint.after} - if any(pair <= set(tools) for tools in covered_tool_sets): - continue - out.append(constraint) - return out - - -def _persist_candidate( - ir: EvolutionIR, - *, - credit: Dict[str, Any], - proposer: str, - evidence_refs: List[str], -) -> Optional[str]: - """Store a candidate, collapsing duplicates rather than failing on them.""" - from core.evolution import lineage - - # Attribution outlives the candidate that happened to carry it: every - # decision that reaches persistence gets its own ledger row, and the stored - # copy inside the candidate becomes a cache keyed by ``decision_id``. - decision_id = lineage.record_credit_decision( - credit, episode_id=(evidence_refs[0] if evidence_refs else "") - ) - credit = {**credit, "decision_id": decision_id} - - checksum = ir.change_checksum() - try: - with SessionLocal() as db: - existing = ( - db.query(EvolutionCandidate) - .filter( - EvolutionCandidate.target_kind == ir.target_kind, - EvolutionCandidate.target_asset_id == ir.target_asset_id, - EvolutionCandidate.base_version_id == (ir.base_version or None), - EvolutionCandidate.change_checksum == checksum, - ) - .first() - ) - if existing is not None: - # Same edit proposed again from fresh evidence: one review item, - # not two. - return existing.candidate_id - - candidate_id = new_candidate_id() - db.add( - EvolutionCandidate( - candidate_id=candidate_id, - target_kind=ir.target_kind, - target_asset_id=ir.target_asset_id, - base_version_id=ir.base_version or None, - operation=ir.operation, - ir=ir.to_dict(), - hypothesis=ir.hypothesis, - change_checksum=checksum, - evidence_refs=evidence_refs, - credit_decision=credit, - risk_tier=ir.risk_tier, - scope=ir.scope, - status=DRAFT, - proposer=proposer, - ) - ) - db.commit() - except Exception as exc: - logger.warning("[evolution-loop] persisting candidate failed: %s", exc) - return None - - # No backfill onto the turn cards. A skill candidate is a batch product - # reviewed in the console; pushing it back into a conversation from days ago - # put capability claims on a card whose only subject is what that turn - # remembered. The candidate's own evidence_refs still record which episodes - # produced it, which is where that link belongs. - return candidate_id - - -def cross_user_mining_available() -> bool: - """Whether fleet-wide mining may run in this edition. - - Personal evolution runs everywhere — it needs no aggregation across people - and its output reaches only its owner. Mining *across* users and releasing - to everyone is a governance surface, so it is enterprise-gated. The - community edition keeps a complete personal evolution path rather than a - degraded one. - """ - try: - # Resolved by name rather than a literal enterprise-tree import, so the - # community build's boundary scan does not read this file as reaching - # across the edition boundary. The behaviour is unchanged: the module is - # physically absent in CE and the resulting ImportError still lands in - # the handler below. - from importlib import import_module - - features = import_module("edition_ee.licensing.features") - manager = import_module("edition_ee.licensing.manager") - - return bool(manager.license_manager.has(features.Feature.EVOLUTION_CONTROL)) - except Exception: - # The community tree physically excludes edition_ee, so an import error - # here is the expected CE signal rather than a fault: personal only. - return False - - -def _run_async(coro) -> Any: - """Run one coroutine from the cycle's synchronous body. - - The cycle is a background job and must run in a worker thread. Calling it - from inside a running loop is a programming error and is raised as one: - degrading to "skip the generators" would produce a cycle that silently - stopped writing skills, which is the failure mode this whole rework exists - to eliminate. - """ - import asyncio - - try: - asyncio.get_running_loop() - except RuntimeError: - return asyncio.run(coro) - coro.close() - raise RuntimeError( - "run_evolution_cycle must not be called from a running event loop; " - "dispatch it with run_in_executor" - ) - - -def _memory_scan_findings(views: List[Dict[str, Any]]) -> List[Any]: - """Scan every contributing user's store. - - Per user, because the store is per user: near-duplicates and contradictions - only mean anything within one person's memories, and a cross-user scan would - both mean nothing and read what it has no business reading. - """ - from core.evolution.memory_scan import scan_user_memories - - retrieved: Dict[str, Dict[str, int]] = {} - for view in views: - user_id = str(view.get("user_id") or "") - if not user_id: - continue - bucket = retrieved.setdefault(user_id, {}) - for ref in view.get("memory_refs") or []: - bucket[str(ref)] = bucket.get(str(ref), 0) + 1 - - findings: List[Any] = [] - for user_id, refs in retrieved.items(): - try: - _, user_findings = _run_async(scan_user_memories(user_id, retrieved_refs=refs)) - except Exception as exc: # noqa: BLE001 - logger.warning("[evolution-loop] memory scan failed for %s: %s", user_id, exc) - continue - findings.extend(user_findings) - return findings - - -def _memory_candidates( - extras: Dict[str, Any], views: List[Dict[str, Any]], *, dry_run: bool -) -> List[Dict[str, Any]]: - """One memory candidate per user, carrying that user's operations. - - Grouped per user rather than per operation because the approval question is - "may the system reorganise *my* memories", and answering it forty times for - forty entries is not review, it is attrition. - """ - memory = extras.get("memory") or {} - ops = memory.get("ops") or [] - if not ops: - return [] - - owner_by_ref: Dict[str, str] = {} - for view in views: - user_id = str(view.get("user_id") or "") - for ref in view.get("memory_refs") or []: - owner_by_ref.setdefault(str(ref), user_id) - - by_user: Dict[str, List[Any]] = {} - for op in ops: - owner = owner_by_ref.get(op.memory_ref, "") - if not owner: - # An operation whose subject cannot be identified must not be - # proposed: applying it would need a user id, and guessing one is - # how a change lands on the wrong person's memories. - continue - by_user.setdefault(owner, []).append(op) - - created: List[Dict[str, Any]] = [] - for user_id, user_ops in by_user.items(): - # Duplicates, contradictions and stale entries are properties of the - # store, found by scanning it — not a conflict with any one request. - decision = aggregate_finding( - target=T_MEMORY, - explanation="；".join(op.reason for op in user_ops[:3]), - confidence=confidence_from_evidence(len(user_ops), saturates_at=8), - evidence_count=len(user_ops), - ) - if not decision.may_produce_candidate: - continue - - asset_id = "mem-" + hashlib.sha256(user_id.encode("utf-8")).hexdigest()[:12] - ir = EvolutionIR( - target_kind="memory", - target_asset_id=asset_id, - base_version="current", - operation="merge" if any(op.operation == "merge" for op in user_ops) else "reweight", - hypothesis=( - f"记忆库有 {len(user_ops)} 处需要重组：" - + "；".join(op.reason for op in user_ops[:3]) - ), - changes=[ - { - "user_id": user_id, - "operations": [op.to_dict() for op in user_ops], - } - ], - evidence_refs=[op.memory_ref for op in user_ops][:20], - # Changing what a person's agent remembers is never low risk, and - # the scope is that person alone. - risk_tier=RISK_MEDIUM, - scope={"level": "user", "user_id": user_id}, - rollback=Rollback(version_id="current", triggers=["user_disable"]), - evaluation_plan={"replay_set": "personal", "primary_metric": "task_success"}, - ) - try: - validate_ir( - ir, credit_decision=decision.to_dict(), allowed_scopes=["user", "workspace"] - ) - except InvariantViolation as violation: - logger.info("[evolution-loop] memory candidate rejected: %s", violation.code) - continue - if dry_run: - created.append({"dry_run": True, "kind": "memory", "user_id": user_id}) - continue - candidate_id = _persist_candidate( - ir, - credit=decision.to_dict(), - proposer="memory_engine", - evidence_refs=[op.memory_ref for op in user_ops][:20], - ) - if candidate_id: - created.append( - { - "candidate_id": candidate_id, - "kind": "memory", - "user_id": user_id, - "operations": len(user_ops), - } - ) - return created - - -def _intent_skill_candidates( - extras: Dict[str, Any], views: List[Dict[str, Any]], *, dry_run: bool -) -> List[Dict[str, Any]]: - """Recurring intents, compiled through the unified skill candidate engine. - - The intent finding says a capability is missing; it does not say what the - capability is. The engine resolves the cluster's evidence into an immutable - pack, derives the deterministic structure (tools, boundaries, dependencies), - has the model write only the document, and validates the layered contract — - the same chain the tool-sequence path uses, so identical evidence can no - longer produce structurally different skills depending on which engine - noticed it first. - """ - from core.evolution.skill_candidate_engine import ( - STATUS_CREATED, - STATUS_DRY_RUN, - STATUS_NOT_WRITTEN, - STATUS_REJECTED, - SkillCandidateSpec, - create_skill_candidate, - ) - - findings = [ - f - for f in (extras.get("intent") or {}).get("findings", []) - if f.get("action") == "skill_candidate" - ] - if not findings: - return [] - - by_episode = {str(v.get("episode_id") or ""): v for v in views} - created: List[Dict[str, Any]] = [] - - for finding in findings: - cluster = finding.get("cluster") or {} - episode_ids = [str(e) for e in (cluster.get("episode_ids") or [])] - episodes = [by_episode[e] for e in episode_ids if e in by_episode] - if not episodes: - continue - - # The cluster mostly *succeeded* — that is a precondition for - # distilling it. Declaring the turns failed to satisfy the failure - # attributor would contradict the very check that let this through. - decision = aggregate_finding( - target=T_SKILL, - explanation=str(finding.get("reason") or ""), - confidence=confidence_from_evidence(len(episode_ids), saturates_at=10), - evidence_count=len(episode_ids), - ) - if not decision.may_produce_candidate: - continue - - asset_id = str(finding.get("asset_id") or "") - spec = SkillCandidateSpec( - asset_id=asset_id, - proposer="intent_engine", - hypothesis=str(finding.get("reason") or ""), - views=episodes, - intent_cluster=cluster, - procedures=_procedures_for_cluster(cluster, views), - operation="new", - base_version="v0", - risk_tier=RISK_HIGH, - scope={"level": "workspace"}, - tenant_id=str(episodes[0].get("tenant_id") or "default"), - write_document=True, - ) - outcome = create_skill_candidate( - spec, credit=decision.to_dict(), dry_run=dry_run, run_async=_run_async - ) - status = outcome.get("status") - if status == STATUS_NOT_WRITTEN: - created.append( - { - "kind": "recurring_intent", - "asset_id": asset_id, - "written": False, - "why_not": outcome.get("why_not") or {}, - } - ) - elif status == STATUS_REJECTED: - logger.info( - "[evolution-loop] intent candidate rejected: %s %s", - outcome.get("code"), - outcome.get("violations") or "", - ) - elif status == STATUS_DRY_RUN: - created.append({"dry_run": True, "kind": "recurring_intent", "asset_id": asset_id}) - elif status == STATUS_CREATED: - created.append( - { - "candidate_id": outcome["candidate_id"], - "kind": "recurring_intent", - "asset_id": asset_id, - "written": True, - "attempts": outcome.get("attempts", 0), - "evidence_pack_id": outcome.get("pack_id", ""), - } - ) - return created - - -def _procedures_for_cluster( - cluster: Dict[str, Any], views: List[Dict[str, Any]] -) -> List[Dict[str, str]]: - """The procedural memories retrieved during this intent's episodes. - - These are what makes the resulting document a *compilation* of memory rather - than a restatement of a tool list — and their refs are what gets demoted - once the skill is live. - """ - from core.evolution.memory_scan import procedural_records, to_records - from core.memory.ref_shadow import build_ref_id - from core.memory.retrieval_types import LAYER_FACT - - wanted = {str(ref) for ref in (cluster.get("memory_refs") or [])} - if not wanted: - return [] - - episode_ids = {str(e) for e in (cluster.get("episode_ids") or [])} - users = { - str(v.get("user_id") or "") - for v in views - if str(v.get("episode_id") or "") in episode_ids and v.get("user_id") - } - - procedures: List[Dict[str, str]] = [] - for user_id in users: - try: - from core.memory.service import get_all_memories - - raw = _run_async(get_all_memories(user_id, top_k=500)) - except Exception as exc: # noqa: BLE001 - logger.debug("[evolution-loop] procedure lookup failed for %s: %s", user_id, exc) - continue - for record in procedural_records(to_records(raw)): - ref = build_ref_id( - layer=LAYER_FACT, - user_id=user_id, - workspace_id=record.workspace_id, - content_hash=record.content_hash, - ) - if ref in wanted: - procedures.append( - { - "ref": ref, - # Carried because a fleet-wide skill can be compiled from - # several people's memories, and demotion is per-store: a - # bare list of refs has no owner to apply them against, so - # the reverse edge would silently do nothing for exactly - # the candidates that need it most. - "user_id": user_id, - "rule": record.content, - "why": record.why, - "applies_to": record.applies_to, - } - ) - return procedures - - -def _decremental_candidates(extras: Dict[str, Any], *, dry_run: bool) -> List[Dict[str, Any]]: - """Retirement and merge findings, as candidates rather than as a list. - - Retiring goes through the same approval path as creating. Removing a - capability produces no error — only a slow decline — so it is precisely the - direction that must not be automatic. - """ - from core.evolution.promotion import ACTION_REWRITE - - decremental = extras.get("decremental") or {} - created: List[Dict[str, Any]] = [] - - for item in decremental.get("retirements") or []: - asset_id = str(item.get("asset_id") or "") - if not asset_id: - continue - action = str(item.get("action") or "") - if action == ACTION_REWRITE: - # "Opened and ignored" is a content problem. Proposing retirement - # would remove a capability the model wanted; the fix is a rewrite, - # and that is a different candidate written from failure evidence. - created.append({"kind": "needs_rewrite", "asset_id": asset_id, **item}) - continue - - decision = aggregate_finding( - target=T_SKILL, - explanation=str(item.get("explanation") or item.get("reason") or ""), - confidence=confidence_from_evidence(int(item.get("offers", 0) or 0), saturates_at=20), - evidence_count=int(item.get("offers", 0) or 0), - ) - if not decision.may_produce_candidate: - continue - - ir = EvolutionIR( - target_kind="skill", - target_asset_id=asset_id, - base_version="current", - operation="deprecate", - hypothesis=str(item.get("explanation") or item.get("reason") or ""), - changes=[ - { - "retire_reason": item.get("reason"), - "stats": { - k: v for k, v in item.items() if k not in ("asset_id", "explanation") - }, - } - ], - evidence_refs=[asset_id], - risk_tier=RISK_MEDIUM, - scope={"level": "workspace"}, - rollback=Rollback(version_id="current", triggers=["manual"]), - evaluation_plan={"replay_set": "auto_holdout", "primary_metric": "task_success"}, - ) - try: - validate_ir(ir, credit_decision=decision.to_dict()) - except InvariantViolation as violation: - logger.info("[evolution-loop] retirement rejected: %s", violation.code) - continue - if dry_run: - created.append({"dry_run": True, "kind": "retirement", "asset_id": asset_id}) - continue - candidate_id = _persist_candidate( - ir, - credit=decision.to_dict(), - proposer="decremental_engine", - evidence_refs=[asset_id], - ) - if candidate_id: - created.append( - { - "candidate_id": candidate_id, - "kind": "retirement", - "asset_id": asset_id, - "reason": item.get("reason"), - } - ) - return created - - -def _orchestration_candidates( - views: List[Dict[str, Any]], *, dry_run: bool -) -> List[Dict[str, Any]]: - """Profile and sub-agent candidates. - - The tool authority every candidate here is measured against is the *current* - grant, read once. A profile may only ever narrow it, and a derived sub-agent - may only ever receive a subset — both enforced by the IR gate rather than by - the generator, so the check holds even for a candidate that arrived by some - other route. - """ - from core.config.catalog import get_enabled_ids - from core.evolution.agent_profile import TARGET_KIND, builtin_profile, validate_profile - from core.evolution.orchestration_gen import ( - generate_orchestration_proposals, - profile_from_proposals, - ) - - base_tools = [str(t) for t in get_enabled_ids("tools")] + [ - str(t) for t in get_enabled_ids("mcp") - ] - offered_tools = {str(view.get("task_type") or "chat"): base_tools for view in views} - proposals, subagents = generate_orchestration_proposals( - views, offered_tools=offered_tools, parent_tools=base_tools - ) - created: List[Dict[str, Any]] = [] - - task_types = sorted({p.task_type for p in proposals}) - for task_type in task_types: - task_proposals = [p for p in proposals if p.task_type == task_type] - profile = profile_from_proposals( - task_proposals, task_type=task_type, base=builtin_profile() - ) - ok, problems = validate_profile(profile, base_tools=base_tools) - if not ok: - logger.info("[evolution-loop] profile %s rejected: %s", profile.profile_id, problems) - continue - - evidence = [ref for p in task_proposals for ref in p.evidence_refs][:20] - # An observation across the corpus, not a failure attributed to one run. - # Inventing failure features to make the attributor return "workflow" - # is what put "过程停滞：重复动作、无产出增量" in front of reviewers for a - # candidate whose actual reason was "eight tools were never used". - decision = aggregate_finding( - target=T_ORCHESTRATION, - explanation="；".join(p.rationale for p in task_proposals), - confidence=confidence_from_evidence(sum(p.support for p in task_proposals)), - evidence_count=len(evidence), - ) - if not decision.may_produce_candidate: - continue - ir = EvolutionIR( - target_kind=TARGET_KIND, - target_asset_id=profile.profile_id, - base_version="builtin", - operation="new", - hypothesis="；".join(p.rationale for p in task_proposals), - changes=[ - { - "profile": profile.to_dict(), - "tool_allowlist": profile.tool_allowlist or [], - } - ], - evidence_refs=evidence, - requested_tools=list(profile.tool_allowlist or []), - risk_tier=RISK_MEDIUM, - scope={"level": "workspace"}, - rollback=Rollback(version_id="builtin", triggers=["success -3pp"]), - evaluation_plan={"replay_set": "auto_holdout", "primary_metric": "task_success"}, - ) - try: - validate_ir(ir, credit_decision=decision.to_dict(), base_tools=base_tools) - except InvariantViolation as violation: - logger.info("[evolution-loop] profile candidate rejected: %s", violation.code) - continue - if dry_run: - created.append({"dry_run": True, "kind": "agent_profile", "task_type": task_type}) - continue - candidate_id = _persist_candidate( - ir, - credit=decision.to_dict(), - proposer="orchestration_engine", - evidence_refs=evidence, - ) - if candidate_id: - created.append( - {"candidate_id": candidate_id, "kind": "agent_profile", "task_type": task_type} - ) - - for subagent in subagents: - decision = aggregate_finding( - target=T_ORCHESTRATION, - explanation=subagent.rationale, - confidence=confidence_from_evidence(subagent.support), - evidence_count=len(subagent.evidence_refs), - ) - if not decision.may_produce_candidate: - continue - payload = subagent.to_dict() - ir = EvolutionIR( - target_kind="subagent", - target_asset_id=subagent.agent_id, - base_version="v0", - operation="new", - hypothesis=subagent.rationale, - changes=[payload], - evidence_refs=subagent.evidence_refs, - requested_tools=list(subagent.tools_allowlist), - # Creating an agent is the widest-reaching change this system can - # propose, and the route into it changes the parent's prompt. - risk_tier=RISK_HIGH, - scope={"level": "workspace"}, - rollback=Rollback(version_id="v0", triggers=["manual"]), - evaluation_plan={"replay_set": "auto_holdout", "primary_metric": "task_success"}, - ) - try: - validate_ir(ir, credit_decision=decision.to_dict(), base_tools=base_tools) - except InvariantViolation as violation: - logger.info("[evolution-loop] subagent candidate rejected: %s", violation.code) - continue - if dry_run: - created.append({"dry_run": True, "kind": "subagent", "name": subagent.name}) - continue - candidate_id = _persist_candidate( - ir, - credit=decision.to_dict(), - proposer="orchestration_engine", - evidence_refs=subagent.evidence_refs, - ) - if candidate_id: - created.append( - {"candidate_id": candidate_id, "kind": "subagent", "name": subagent.name} - ) - return created - - -def _prompt_candidates(views: List[Dict[str, Any]], *, dry_run: bool) -> List[Dict[str, Any]]: - """Prompt fragments, for the failures attribution already blames on the prompt.""" - from core.evolution.prompt_gen import generate_prompt_candidate - - payload, evidence, draft = _run_async( - generate_prompt_candidate( - views, - cause="归因判定该失败源于提示词未声明相关约束", - existing_fragments=_active_prompt_fragments(), - ) - ) - if payload is None or evidence is None: - return ( - [] - if draft.violations == ["insufficient_evidence"] - else [{"kind": "prompt", "written": False, "why_not": draft.to_dict()}] - ) - - decision = assign_credit( - CreditFeatures( - outcome_verdict="failed", - outcome_confidence=0.85, - prompt_missing_priority_rule=True, - ) - ) - if not decision.may_produce_candidate: - return [] - - asset_id = "auto-prompt-" + hashlib.sha256(payload["fragment"].encode("utf-8")).hexdigest()[:12] - ir = EvolutionIR( - target_kind="prompt", - target_asset_id=asset_id, - base_version="active", - operation="patch", - hypothesis=payload["cause"], - changes=[{"fragment": payload["fragment"], "task_types": payload["task_types"]}], - evidence_refs=evidence.episode_ids[:20], - # A prompt fragment applies to every request in scope; nothing this - # system produces has a wider reach. - risk_tier=RISK_HIGH, - scope={"level": "workspace"}, - rollback=Rollback(version_id="active", triggers=["success -3pp"]), - evaluation_plan={"replay_set": "auto_holdout", "primary_metric": "task_success"}, - ) - try: - validate_ir(ir, credit_decision=decision.to_dict()) - except InvariantViolation as violation: - return [{"kind": "prompt", "written": False, "why_not": {"code": violation.code}}] - if dry_run: - return [{"dry_run": True, "kind": "prompt", "asset_id": asset_id}] - candidate_id = _persist_candidate( - ir, - credit=decision.to_dict(), - proposer="prompt_engine", - evidence_refs=evidence.episode_ids[:20], - ) - return ( - [{"candidate_id": candidate_id, "kind": "prompt", "asset_id": asset_id}] - if candidate_id - else [] - ) - - -def _active_prompt_fragments() -> List[str]: - """Fragments already appended by earlier prompt candidates. - - Passed to the generator so it can decline rather than restate: a prompt that - accumulates three phrasings of the same instruction is measurably worse than - one that states it once. - """ - try: - from core.db.engine import SessionLocal as _Session - from core.db.models.evolution import EvolutionRelease as _Release - - with _Session() as db: - rows = ( - db.query(_Release) - .filter(_Release.target_kind == "prompt", _Release.stage == "active") - .all() - ) - return [str(row.target_asset_id) for row in rows] - except Exception as exc: # noqa: BLE001 - logger.debug("[evolution-loop] active prompt fragments unavailable: %s", exc) - return [] - - -def _graph_evolution_candidates( - views: List[Dict[str, Any]], *, dry_run: bool -) -> List[Dict[str, Any]]: - """L3 的演进：强化/降权自动执行，语义变更走人工候选。 - - Per user, because the graph is per user scope: a contradiction between two - people's graphs is not a contradiction at all. - """ - from core.config.settings import settings - - if not (settings.memory.enabled and settings.memory.graph_enabled): - return [] - - from core.evolution.graph_scan import apply_graph_ops, scan_graph_relations - - observed_by_user: Dict[str, Dict[str, int]] = {} - for view in views: - user_id = str(view.get("user_id") or "") - if not user_id: - continue - bucket = observed_by_user.setdefault(user_id, {}) - for relation in view.get("graph_relations") or []: - relation_id = str(relation.get("relation_id") or "") - if relation_id: - bucket[relation_id] = bucket.get(relation_id, 0) + 1 - - created: List[Dict[str, Any]] = [] - for user_id, observed in observed_by_user.items(): - try: - from core.memory.graph import list_graph_relations - - relations = _run_async(list_graph_relations(user_id, limit=200)) - except Exception as exc: # noqa: BLE001 - logger.debug("[evolution-loop] graph listing failed for %s: %s", user_id, exc) - continue - if not relations: - continue - - ops = scan_graph_relations(relations, observed_relation_ids=observed) - if not ops: - continue - if dry_run: - created.append( - {"dry_run": True, "kind": "graph", "user_id": user_id, "ops": len(ops)} - ) - continue - - outcome = apply_graph_ops(ops, scope_user_id=user_id) - manual = outcome.get("manual") or [] - entry: Dict[str, Any] = { - "kind": "graph", - "user_id": user_id, - "auto_applied": len(outcome.get("applied") or []), - "manual_proposed": len(manual), - } - if manual: - decision = aggregate_finding( - target=T_MEMORY, - explanation="；".join(str(op.get("reason") or "") for op in manual[:3]), - confidence=confidence_from_evidence(len(manual), saturates_at=6), - evidence_count=len(manual), - ) - if decision.may_produce_candidate: - asset_id = "graph-" + hashlib.sha256(user_id.encode("utf-8")).hexdigest()[:12] - ir = EvolutionIR( - target_kind="memory", - target_asset_id=asset_id, - base_version="current", - operation="update", - hypothesis=( - f"图谱有 {len(manual)} 处语义变更需要人工裁决：" - + "；".join(str(op.get("reason") or "") for op in manual[:3]) - ), - changes=[{"user_id": user_id, "graph_ops": manual}], - evidence_refs=[ - str(op.get("relation_id") or "") for op in manual if op.get("relation_id") - ][:20], - risk_tier=RISK_MEDIUM, - scope={"level": "user", "user_id": user_id}, - rollback=Rollback(version_id="current", triggers=["manual"]), - evaluation_plan={"replay_set": "personal", "primary_metric": "task_success"}, - ) - try: - validate_ir( - ir, - credit_decision=decision.to_dict(), - allowed_scopes=["user", "workspace"], - ) - except InvariantViolation as violation: - logger.info( - "[evolution-loop] graph candidate rejected: %s", violation.code - ) - else: - candidate_id = _persist_candidate( - ir, - credit=decision.to_dict(), - proposer="graph_engine", - evidence_refs=list(ir.evidence_refs), - ) - if candidate_id: - entry["candidate_id"] = candidate_id - created.append(entry) - return created - - -def _run_other_engines(views: List[Dict[str, Any]], *, dry_run: bool) -> Dict[str, Any]: - """Memory, decremental, intent, orchestration, prompt and graph — each with an exit. - - Independently guarded: a failure in one engine must not cost the others - their findings, because they answer different questions and an operator - needs whichever ones succeeded. - """ - from core.evolution.cycle_extras import run_extras - - scan_findings = _memory_scan_findings(views) - extras = run_extras(views, scan_findings=scan_findings) - - produced: List[Dict[str, Any]] = [] - for name, fn in ( - ("memory", lambda: _memory_candidates(extras, views, dry_run=dry_run)), - ("intent", lambda: _intent_skill_candidates(extras, views, dry_run=dry_run)), - ("decremental", lambda: _decremental_candidates(extras, dry_run=dry_run)), - ("orchestration", lambda: _orchestration_candidates(views, dry_run=dry_run)), - ("prompt", lambda: _prompt_candidates(views, dry_run=dry_run)), - ("graph", lambda: _graph_evolution_candidates(views, dry_run=dry_run)), - ): - try: - produced.extend(fn()) - except Exception as exc: # noqa: BLE001 - logger.warning("[evolution-loop] %s engine failed: %s", name, exc) - produced.append({"kind": name, "error": str(exc)}) - - # ``ops`` holds MemoryOp objects for the candidate builder; the report is - # serialised into the console, so it must not carry them. - memory_report = dict(extras.get("memory") or {}) - memory_report.pop("ops", None) - extras["memory"] = memory_report - extras["memory"]["scan_findings"] = [ - f.to_dict() if hasattr(f, "to_dict") else f for f in scan_findings - ][:50] - - return {"extras": extras, "engine_candidates": produced} - - -def run_evolution_cycle(*, limit: int = 500, dry_run: bool = False) -> Dict[str, Any]: - """One fleet-wide pass: discover → attribute → promote → validate → persist. - - Returns a report rather than raising: this runs as a background job and a - bad cycle must leave the product untouched. - """ - if not cross_user_mining_available(): - logger.info("[evolution-loop] cross-user mining unavailable; personal only") - return { - "episodes": 0, - "patterns": 0, - "candidates": [], - "rejected": [], - "no_update": 0, - "skipped": "cross_user_mining_requires_enterprise", - } - # Refuse rather than degrade. Without an embedding service two of the three - # engines lose their primary path and the cycle reports success having found - # nothing — which is indistinguishable from "there was nothing to find", and - # is the state nobody investigates. - from core.evolution.readiness import check_readiness - - readiness = check_readiness() - if not readiness.ready: - logger.warning("[evolution-loop] not ready, refusing to run: %s", readiness.blocking) - return { - "episodes": 0, - "patterns": 0, - "candidates": [], - "rejected": [], - "no_update": 0, - "skipped": "not_ready", - "readiness": readiness.to_dict(), - } - - report: Dict[str, Any] = { - "episodes": 0, - "patterns": 0, - "proposals": 0, - "candidates": [], - "rejected": [], - "no_update": 0, - "readiness": readiness.to_dict(), - } - - views = load_episode_views(limit=limit) - report["episodes"] = len(views) - if not views: - return report - - patterns = P.discover_patterns(views) - report["patterns"] = len(patterns) - - # Transferable ordering rules, extracted once across the whole corpus rather - # than per pattern. A rigid sequence only helps tasks whose tool set it - # happens to cover; the ordering rule behind it helps every task making the - # same mistake, which is the difference between memorising and learning. - constraints = P.extract_ordering_constraints(views) - report["ordering_constraints"] = [c.to_dict() for c in constraints] - - from core.evolution.skill_candidate_engine import ( - STATUS_CREATED, - STATUS_DRY_RUN, - STATUS_REJECTED, - SkillCandidateSpec, - create_skill_candidate, - ) - - existing_signatures: Dict[str, str] = {} - covered_tool_sets: List[List[str]] = [] - by_episode_id = {str(v.get("episode_id") or ""): v for v in views} - - for pattern in patterns: - if pattern.kind == P.PATTERN_SUCCESS_SUBSEQUENCE and pattern.tool_sequence: - # An observation across many *successful* runs — aggregate decision, - # not failure attribution with invented features. - decision = aggregate_finding( - target=T_SKILL, - explanation=( - f"{pattern.support} 个成功 Episode 复现同一 " - f"{len(pattern.tool_sequence)} 步工具序列" - f"（成功率 {pattern.success_rate:.0%}）" - ), - confidence=confidence_from_evidence(pattern.support, saturates_at=10), - evidence_count=pattern.support, - ) - else: - decision = assign_credit(_features_for(pattern)) - if not decision.may_produce_candidate: - # No-update / unidentifiable / a non-asset cause. Refusing here is - # the mechanism that stops a KB gap being "fixed" by editing a skill. - report["no_update"] += 1 - continue - - proposal = P.promote_tool_sequence_to_skill( - pattern, - episodes=views, - skill_credit=decision.scores.get("skill", 0.0), - workflow_credit=decision.scores.get("workflow", 0.0), - existing_skill_signatures=existing_signatures, - ) - if proposal is None: - continue - report["proposals"] += 1 - - # Stable, content-derived id. Python's built-in hash() is randomised per - # process, so using it here minted a fresh asset id on every cycle and the - # de-duplication constraint never matched — one candidate per run, forever. - asset_id = proposal.patch_target or ( - "auto-" + hashlib.sha256(proposal.signature.encode("utf-8")).hexdigest()[:12] - ) - sequence = [str(t) for t in proposal.payload.get("tool_sequence", [])] - supporting = [ - by_episode_id[e] - for e in (proposal.payload.get("episode_ids") or []) - if e in by_episode_id - ] - - # 规则门测统计，测不了语义——意图簇可能是巧合归并，序列可能是对任何 - # 任务都成立的通用组合。规则幸存者再过一道 LLM 裁决，拒绝即不产候选。 - from core.evolution.sop_judge import adjudicate_sequence_sop - - verdict = _run_async( - adjudicate_sequence_sop( - representative=str(proposal.payload.get("intent") or ""), - objectives=[str(v.get("objective") or "") for v in supporting], - tool_sequence=sequence, - occasions=int(proposal.payload.get("occasions") or 0), - support=proposal.support, - success_rate=pattern.success_rate, - ) - ) - if not verdict.worthy: - report["rejected"].append( - { - "signature": proposal.signature, - "code": ( - "sop_judge_unavailable" - if verdict.judge_unavailable - else "sop_judge_refused" - ), - "reason": verdict.reason, - } - ) - continue - spec = SkillCandidateSpec( - asset_id=asset_id, - proposer="promotion_chain", - hypothesis=proposal.rationale, - views=supporting, - tool_sequence=sequence, - # Attach every rule whose tools this skill touches, and carry its - # scope along with it. Attaching must not apply the usability - # policy: doing so dropped exactly the context-dependent rules the - # scoping exists to preserve — a rule contradicted in one task type - # is still valid in the ones where it held, and that decision - # belongs at use time, where the context is known. - ordering_constraints=[ - c.to_dict() - for c in constraints - if c.before in sequence and c.after in sequence - ], - operation="patch" if proposal.patch_target else "new", - base_version="current" if proposal.patch_target else "v0", - # A brand-new capability is riskier than tweaking an existing one. - risk_tier=RISK_MEDIUM if proposal.patch_target else RISK_HIGH, - scope={"level": "workspace"}, - tenant_id=str( - (supporting[0].get("tenant_id") if supporting else "") or "default" - ), - write_document=False, - ) - # 裁决结论随候选进 credit_decision 落库，评审员能看到机器放行的理由。 - outcome = create_skill_candidate( - spec, - credit={**decision.to_dict(), "sop_judge": verdict.to_dict()}, - dry_run=dry_run, - ) - status = outcome.get("status") - if status == STATUS_REJECTED: - report["rejected"].append( - {"signature": proposal.signature, "code": outcome.get("code")} - ) - elif status == STATUS_DRY_RUN: - report["candidates"].append({"dry_run": True, "signature": proposal.signature}) - elif status == STATUS_CREATED: - covered_tool_sets.append(sequence) - report["candidates"].append( - { - "candidate_id": outcome["candidate_id"], - "signature": proposal.signature, - "evidence_pack_id": outcome.get("pack_id", ""), - } - ) - - # ── Ordering rules that no sequence candidate carries ──────────────────── - # - # Without this the only shape of learning that can become an asset is "five - # runs repeated one identical tool sequence", which on real traffic is close - # to unobservable — measured on this deployment's history, every episode's - # tool sequence was unique. The rules are the part that generalises, so they - # get to stand on their own rather than only riding along on a sequence that - # happens to qualify. - orphan_rules = _uncovered_constraints(constraints, covered_tool_sets) - if orphan_rules: - rule_decision = assign_credit(_features_for_ordering_rules(orphan_rules)) - if not rule_decision.may_produce_candidate: - report["no_update"] += 1 - else: - rule_payload = [c.to_dict() for c in orphan_rules] - signature = "|".join(sorted(f"{c.before}<{c.after}" for c in orphan_rules)) - asset_id = "auto-rules-" + hashlib.sha256(signature.encode("utf-8")).hexdigest()[:12] - supporting_views = [ - view - for view in views - if view.get("verdict") == "success" - and any( - c.before in (view.get("tool_sequence") or []) - and c.after in (view.get("tool_sequence") or []) - for c in orphan_rules - ) - ] - rule_spec = SkillCandidateSpec( - asset_id=asset_id, - proposer="promotion_chain", - hypothesis=( - f"{len(orphan_rules)} 条工具顺序约束在 {len(supporting_views)} 条成功执行中" - "稳定成立且跨多种工具组合复现，但规划阶段每次仍可能排错" - ), - views=supporting_views, - # Deliberately no tool sequence: this asset claims no fixed - # order of its own, only constraints. Materialisation renders - # it as rules. - tool_sequence=[], - ordering_constraints=rule_payload, - operation="new", - base_version="v0", - risk_tier=RISK_HIGH, - scope={"level": "workspace"}, - tenant_id=str( - (supporting_views[0].get("tenant_id") if supporting_views else "") - or "default" - ), - write_document=False, - ) - outcome = create_skill_candidate( - rule_spec, credit=rule_decision.to_dict(), dry_run=dry_run - ) - status = outcome.get("status") - if status == STATUS_REJECTED: - report["rejected"].append({"signature": signature, "code": outcome.get("code")}) - else: - report["proposals"] += 1 - if status == STATUS_DRY_RUN: - report["candidates"].append( - {"dry_run": True, "signature": signature, "kind": "ordering_rules"} - ) - elif status == STATUS_CREATED: - report["candidates"].append( - { - "candidate_id": outcome["candidate_id"], - "signature": signature, - "kind": "ordering_rules", - "evidence_pack_id": outcome.get("pack_id", ""), - } - ) - - # The other two engines, plus orchestration. Every finding below becomes a - # candidate in the same table, behind the same security gate and the same - # review queue as the skill candidates above — a finding that stops at a - # report is a log line, not an engine. - report.update(_run_other_engines(views, dry_run=dry_run)) - - logger.info( - "[evolution-loop] cycle done: episodes=%d patterns=%d candidates=%d no_update=%d", - report["episodes"], - report["patterns"], - len(report["candidates"]), - report["no_update"], - ) - return report - - -# ── Advancing candidates through the release pipeline ──────────────────────── - - -def _record_evaluation(candidate_id: str, report: Any, eval_type: str) -> None: - """Persist an evaluation result against its candidate.""" - from core.db.models.evolution import EvolutionEvaluation - - try: - payload = report.to_dict() - with SessionLocal() as db: - db.add( - EvolutionEvaluation( - evaluation_id=payload["evaluation_id"], - candidate_id=candidate_id, - eval_type=eval_type, - # Frozen: a dataset edited after the fact invalidates every - # conclusion computed from it. - dataset_snapshot=payload["dataset_snapshot"], - baseline_metrics=payload["baseline"], - candidate_metrics=payload["candidate"], - guardrails={"breaches": payload["guardrail_breaches"]}, - verdict=payload["verdict"], - effect_size=payload["effect_size"], - p_value=payload["p_value"], - sample_size=payload["sample_size"], - ) - ) - db.commit() - except Exception as exc: - logger.warning("[evolution-loop] recording evaluation failed: %s", exc) - - -def _replay_coverage_for(candidate_id: str, task_ids: List[str]) -> Optional[Dict[str, Any]]: - """Coverage assessment for candidates that carry an evidence pack. - - Returns ``None`` for pre-pack candidates — the gate governs only what the - unified engine produced, so legacy candidates keep their old semantics. - """ - try: - from core.evolution import lineage - from core.evolution.applicability import assess_replay_coverage - - with SessionLocal() as db: - candidate = db.get(EvolutionCandidate, candidate_id) - pack_id = str(getattr(candidate, "evidence_pack_id", "") or "") - if not pack_id: - return None - pack = lineage.load_evidence_pack(pack_id) - if pack is None: - return None - return assess_replay_coverage(pack, task_ids) - except Exception as exc: # noqa: BLE001 - logger.debug("[evolution-loop] replay coverage check skipped: %s", exc) - return None - - -def evaluate_candidate( - candidate_id: str, - run_arm: Any, - task_ids: List[str], -) -> Optional[Dict[str, Any]]: - """Run tiered replay for a candidate and advance it if it earns the step. - - ``run_arm(task_id, is_candidate)`` is injected: the caller owns isolation - (frozen bundle, cassette, sandbox snapshot). Only ``replay_passed`` is - reachable from here — shadow and canary need real traffic, and activation - needs a human, so this can never quietly ship anything. - """ - from core.evolution.release import DRAFT, REJECTED, REPLAY_PASSED, is_legal_transition - from core.evolution.replay import VERDICT_IMPROVED, VERDICT_REGRESSED, run_tiered_replay - - tiered = run_tiered_replay(task_ids, run_arm) - report = tiered.report or tiered.sentinel_report - if report is None: - return None - - _record_evaluation(candidate_id, report, "replay" if tiered.passed_sentinel else "sentinel") - - # Replay 覆盖闸门：带证据 pack 的候选，其回放集必须覆盖 pack 声称的行为面 - # （成功 / 失败恢复 / 负样本 / 图谱依赖两类场景）。只用成功案例喂出来的 - # "improved" 是单面证据，不允许换来 replay_passed；回归照常生效——安全 - # 方向的结论不需要覆盖完整也成立。 - coverage: Optional[Dict[str, Any]] = None - if report.verdict == VERDICT_IMPROVED: - coverage = _replay_coverage_for(candidate_id, task_ids) - if coverage is not None and not coverage.get("covered", True): - return { - "candidate_id": candidate_id, - "verdict": report.verdict, - "status": DRAFT, - "transition": "held_insufficient_coverage", - "coverage": coverage, - "tasks_run": tiered.tasks_run, - "effect_size": report.effect_size, - "p_value": report.p_value, - } - - try: - with SessionLocal() as db: - candidate = db.get(EvolutionCandidate, candidate_id) - if candidate is None: - return None - current = candidate.status - - # Re-evaluating a candidate that has already moved on is a no-op, - # not a failure. Collapsing the two into None left callers unable to - # tell "already advanced" from "something broke". - target = ( - REJECTED - if report.verdict == VERDICT_REGRESSED - else (REPLAY_PASSED if report.verdict == VERDICT_IMPROVED else current) - ) - if target != current and not is_legal_transition(current, target): - return { - "candidate_id": candidate_id, - "verdict": report.verdict, - "status": current, - "transition": "skipped_already_advanced", - "tasks_run": tiered.tasks_run, - "effect_size": report.effect_size, - "p_value": report.p_value, - } - - if report.verdict == VERDICT_REGRESSED: - candidate.status = REJECTED - candidate.reject_reason = ( - f"replay regression: effect={report.effect_size:.3f} " - f"p={report.p_value:.4f} {report.guardrail_breaches}" - ) - elif report.verdict == VERDICT_IMPROVED: - candidate.status = REPLAY_PASSED - else: - # Neutral or insufficient: stays a draft. Not advancing is the - # correct outcome — "we could not show it helps" is not "it helps". - pass - db.commit() - return { - "candidate_id": candidate_id, - "verdict": report.verdict, - "status": candidate.status, - "transition": "applied" if candidate.status != current else "held", - "tasks_run": tiered.tasks_run, - "effect_size": report.effect_size, - "p_value": report.p_value, - } - except Exception as exc: - logger.warning("[evolution-loop] advancing candidate failed: %s", exc) - return None diff --git a/src/backend/core/evolution/profiles/__init__.py b/src/backend/core/evolution/profiles/__init__.py new file mode 100644 index 00000000..b8789016 --- /dev/null +++ b/src/backend/core/evolution/profiles/__init__.py @@ -0,0 +1 @@ +"""Agent profiles and per-user evolution settings and reviews.""" diff --git a/src/backend/core/evolution/profiles/agent_profile.py b/src/backend/core/evolution/profiles/agent_profile.py new file mode 100644 index 00000000..80589412 --- /dev/null +++ b/src/backend/core/evolution/profiles/agent_profile.py @@ -0,0 +1,259 @@ +"""Agent profiles: one ReAct configuration expressed as data, with validation.""" +from __future__ import annotations +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional, Sequence, Tuple +from core.evolution.profiles import agent_profile_constants + +@dataclass +class InterventionRule: + """signal → threshold → action.""" + + signal: str + threshold: int + action: str + note: str = "" + + def to_dict(self) -> Dict[str, Any]: + return { + "signal": self.signal, + "threshold": self.threshold, + "action": self.action, + "note": self.note, + } + + @classmethod + def from_dict(cls, raw: Dict[str, Any]) -> "InterventionRule": + return cls( + signal=str(raw.get("signal") or ""), + threshold=int(raw.get("threshold") or 0), + action=str(raw.get("action") or ""), + note=str(raw.get("note") or ""), + ) + + +@dataclass +class SubagentRoute: + """Hand this shape of sub-task to a separate context. + + ``agent_id`` points at a real sub-agent; the route is only the *decision to + delegate*. Creating the sub-agent is a separate, separately-approved change + (see :mod:`core.evolution.generators.orchestration_gen`), because a route to an agent + that does not exist is a dead end and an agent nobody routes to is dead + weight — they have to be reviewed as one change but recorded as two. + """ + + agent_id: str + when: str = "" + task_types: List[str] = field(default_factory=list) + + def to_dict(self) -> Dict[str, Any]: + return {"agent_id": self.agent_id, "when": self.when, "task_types": list(self.task_types)} + + @classmethod + def from_dict(cls, raw: Dict[str, Any]) -> "SubagentRoute": + return cls( + agent_id=str(raw.get("agent_id") or ""), + when=str(raw.get("when") or ""), + task_types=[str(t) for t in (raw.get("task_types") or [])], + ) + + +@dataclass +class AgentProfile: + """One assembled ReAct configuration, as data.""" + + profile_id: str = "builtin" + version: str = "v0" + # Which tasks this profile governs. Empty means "every task in scope" — the + # deployment default. A profile with task types is only consulted for those, + # which is how a capability learned in one family stops leaking into others. + task_types: List[str] = field(default_factory=list) + scope: Dict[str, str] = field(default_factory=dict) + + prompt_fragments: List[str] = field(default_factory=list) + # ``None`` means "no restriction beyond the base grant". An empty list means + # "no tools", which is a legitimate profile for a pure-reasoning task type + # and must not be confused with the former. + tool_allowlist: Optional[List[str]] = None + # Which skills this task type may load at all. Empty means "no restriction" + # — skill loading keeps its own logic and this only ever narrows it, and + # only for task types an approved profile actually covers. + skill_ids: List[str] = field(default_factory=list) + subagent_routes: List[SubagentRoute] = field(default_factory=list) + + memory_budget_ms: int = 600 + # 0 = 不限（默认）。See UNBOUNDED_REACT_TURNS: the main agent's round ceiling + # was removed, so a turn budget here is an opt-in narrowing a published + # profile performs deliberately, not a floor every profile inherits. + max_react_turns: int = agent_profile_constants.UNBOUNDED_REACT_TURNS + budget_multiplier: float = 1.0 + reviewer_checkpoints: List[str] = field(default_factory=list) + intervention_rules: List[InterventionRule] = field(default_factory=list) + + # The autonomous loop's own caps. Held as their own fields rather than + # derived from ``max_react_turns``: the two loops bound different things — + # turns inside one answer versus attempts at one requirement across many + # answers — and a ratio between them would be an invented relationship that + # quietly changes one whenever the other is tuned. + loop_max_attempts_per_requirement: int = 6 + loop_strategy_change_after: int = 2 + + def to_dict(self) -> Dict[str, Any]: + return { + "profile_id": self.profile_id, + "version": self.version, + "task_types": list(self.task_types), + "scope": dict(self.scope), + "prompt_fragments": list(self.prompt_fragments), + "tool_allowlist": None if self.tool_allowlist is None else list(self.tool_allowlist), + "skill_ids": list(self.skill_ids), + "subagent_routes": [r.to_dict() for r in self.subagent_routes], + "memory_budget_ms": self.memory_budget_ms, + "max_react_turns": self.max_react_turns, + "budget_multiplier": self.budget_multiplier, + "reviewer_checkpoints": list(self.reviewer_checkpoints), + "intervention_rules": [r.to_dict() for r in self.intervention_rules], + "loop_max_attempts_per_requirement": self.loop_max_attempts_per_requirement, + "loop_strategy_change_after": self.loop_strategy_change_after, + } + + @classmethod + def from_dict(cls, raw: Dict[str, Any]) -> "AgentProfile": + allowlist = raw.get("tool_allowlist", None) + return cls( + profile_id=str(raw.get("profile_id") or "builtin"), + version=str(raw.get("version") or "v0"), + task_types=[str(t) for t in (raw.get("task_types") or [])], + scope=dict(raw.get("scope") or {}), + prompt_fragments=[str(f) for f in (raw.get("prompt_fragments") or [])], + tool_allowlist=None if allowlist is None else [str(t) for t in allowlist], + skill_ids=[str(s) for s in (raw.get("skill_ids") or [])], + subagent_routes=[ + SubagentRoute.from_dict(r) for r in (raw.get("subagent_routes") or []) + ], + memory_budget_ms=int(raw.get("memory_budget_ms") or 600), + # Missing key → unbounded, not an invented ceiling: a profile + # published for unrelated reasons (tool allowlist, memory budget) + # must not silently re-fence the main loop. + max_react_turns=int(raw.get("max_react_turns") or agent_profile_constants.UNBOUNDED_REACT_TURNS), + budget_multiplier=float(raw.get("budget_multiplier") or 1.0), + reviewer_checkpoints=[str(c) for c in (raw.get("reviewer_checkpoints") or [])], + intervention_rules=[ + InterventionRule.from_dict(r) for r in (raw.get("intervention_rules") or []) + ], + loop_max_attempts_per_requirement=int( + raw.get("loop_max_attempts_per_requirement") or 6 + ), + loop_strategy_change_after=int(raw.get("loop_strategy_change_after") or 2), + ) + + def applies_to(self, task_type: str) -> bool: + return not self.task_types or str(task_type or "chat") in self.task_types + + +def builtin_profile() -> AgentProfile: + """The deployment default. + + Reproduces today's behaviour exactly — no tool restriction, no forced skill + set, the retrieval budget and turn cap the runtime already uses — so + installing profile resolution changes nothing until a profile is published. + """ + return AgentProfile( + profile_id="builtin", + version="v0", + tool_allowlist=None, + memory_budget_ms=600, + # The main agent's runtime default, reproduced exactly — which is now + # "no cap". This field only bounds the loop when a published profile + # sets it on purpose. + max_react_turns=agent_profile_constants.UNBOUNDED_REACT_TURNS, + budget_multiplier=1.0, + loop_max_attempts_per_requirement=6, + loop_strategy_change_after=2, + intervention_rules=[ + InterventionRule(agent_profile_constants.SIGNAL_NO_PROGRESS, 2, agent_profile_constants.ACTION_CHANGE_STRATEGY), + InterventionRule(agent_profile_constants.SIGNAL_REPEATED_ACTIONS, 3, agent_profile_constants.ACTION_NARROW_SCOPE), + InterventionRule(agent_profile_constants.SIGNAL_TOOL_ERROR_STREAK, 3, agent_profile_constants.ACTION_CHANGE_STRATEGY), + InterventionRule(agent_profile_constants.SIGNAL_BUDGET_LOW, 1, agent_profile_constants.ACTION_ESCALATE), + ], + ) + + +def validate_profile( + profile: AgentProfile, + *, + base_tools: Optional[Sequence[str]] = None, + known_skill_ids: Optional[Sequence[str]] = None, + known_agent_ids: Optional[Sequence[str]] = None, +) -> Tuple[bool, List[str]]: + """Structural and security validation. + + ``base_tools`` is the authority this profile is allowed to draw from. A + profile naming a tool outside it is refused — orchestration evolution may + narrow authority and may never widen it, and this is the check that makes + "narrow only" a property of the system rather than an intention. + """ + problems: List[str] = [] + + if not profile.profile_id: + problems.append("profile_id 不能为空") + + if profile.tool_allowlist is not None and base_tools is not None: + widened = sorted(set(profile.tool_allowlist) - set(base_tools)) + if widened: + problems.append(f"工具授权越界（只允许收敛，不允许扩权）: {widened}") + + if known_skill_ids is not None: + unknown = sorted(set(profile.skill_ids) - set(known_skill_ids)) + if unknown: + problems.append(f"引用了不存在的技能: {unknown}") + + seen_agents = set() + for route in profile.subagent_routes: + if not route.agent_id: + problems.append("子智能体路由缺少 agent_id") + continue + if route.agent_id in seen_agents: + problems.append(f"重复的子智能体路由: {route.agent_id}") + seen_agents.add(route.agent_id) + if known_agent_ids is not None and route.agent_id not in set(known_agent_ids): + problems.append(f"路由指向不存在的子智能体: {route.agent_id}") + + for rule in profile.intervention_rules: + if rule.signal not in agent_profile_constants.ALLOWED_SIGNALS: + problems.append(f"未知信号 {rule.signal}") + if rule.action not in agent_profile_constants.ALLOWED_ACTIONS: + problems.append(f"未知动作 {rule.action}") + if rule.threshold < 1: + problems.append(f"{rule.signal} 阈值必须 ≥1（0 会让干预立刻触发）") + + if not (agent_profile_constants.MIN_BUDGET_MULTIPLIER <= profile.budget_multiplier <= agent_profile_constants.MAX_BUDGET_MULTIPLIER): + problems.append( + f"预算倍数 {profile.budget_multiplier} 超出 " + f"[{agent_profile_constants.MIN_BUDGET_MULTIPLIER}, {agent_profile_constants.MAX_BUDGET_MULTIPLIER}]" + ) + if profile.max_react_turns != agent_profile_constants.UNBOUNDED_REACT_TURNS and not ( + agent_profile_constants.MIN_REACT_TURNS <= profile.max_react_turns <= agent_profile_constants.MAX_REACT_TURNS + ): + problems.append( + f"最大轮数 {profile.max_react_turns} 超出 " + f"[{agent_profile_constants.MIN_REACT_TURNS}, {agent_profile_constants.MAX_REACT_TURNS}]（{agent_profile_constants.UNBOUNDED_REACT_TURNS} 表示不限）" + ) + if not (agent_profile_constants.MIN_MEMORY_BUDGET_MS <= profile.memory_budget_ms <= agent_profile_constants.MAX_MEMORY_BUDGET_MS): + problems.append( + f"记忆检索预算 {profile.memory_budget_ms}ms 超出 " + f"[{agent_profile_constants.MIN_MEMORY_BUDGET_MS}, {agent_profile_constants.MAX_MEMORY_BUDGET_MS}]" + ) + if not ( + agent_profile_constants.MIN_LOOP_ATTEMPTS + <= profile.loop_max_attempts_per_requirement + <= agent_profile_constants.MAX_LOOP_ATTEMPTS + ): + problems.append( + f"单需求最大尝试次数 {profile.loop_max_attempts_per_requirement} 超出 " + f"[{agent_profile_constants.MIN_LOOP_ATTEMPTS}, {agent_profile_constants.MAX_LOOP_ATTEMPTS}]" + ) + if profile.loop_strategy_change_after < 1: + problems.append("换策略阈值必须 ≥1（0 会让每一轮都换策略）") + + return (not problems), problems diff --git a/src/backend/core/evolution/profiles/agent_profile_constants.py b/src/backend/core/evolution/profiles/agent_profile_constants.py new file mode 100644 index 00000000..8f937ff1 --- /dev/null +++ b/src/backend/core/evolution/profiles/agent_profile_constants.py @@ -0,0 +1,102 @@ +"""Signals, actions and limits allowed in agent profiles.""" +from __future__ import annotations + + +SIGNAL_NO_PROGRESS = "no_progress" + + +SIGNAL_REPEATED_ACTIONS = "repeated_actions" + + +SIGNAL_NO_DIFF = "no_diff" + + +SIGNAL_REVIEWER_FLAT = "reviewer_score_flat" + + +SIGNAL_BUDGET_LOW = "budget_low" + + +SIGNAL_TOOL_ERROR_STREAK = "tool_error_streak" + + +ALLOWED_SIGNALS = ( + SIGNAL_NO_PROGRESS, + SIGNAL_REPEATED_ACTIONS, + SIGNAL_NO_DIFF, + SIGNAL_REVIEWER_FLAT, + SIGNAL_BUDGET_LOW, + SIGNAL_TOOL_ERROR_STREAK, +) + + +REACT_OBSERVABLE_SIGNALS = frozenset( + {SIGNAL_NO_PROGRESS, SIGNAL_REPEATED_ACTIONS, SIGNAL_TOOL_ERROR_STREAK} +) + + +LOOP_ONLY_SIGNALS = frozenset( + {SIGNAL_NO_DIFF, SIGNAL_REVIEWER_FLAT, SIGNAL_BUDGET_LOW} +) + + +ACTION_RETRY = "retry" + + +ACTION_CHANGE_STRATEGY = "change_strategy" + + +ACTION_ROLLBACK_AND_FORK = "rollback_and_fork" + + +ACTION_NARROW_SCOPE = "narrow_scope" + + +ACTION_DELEGATE = "delegate" + + +ACTION_ESCALATE = "escalate" + + +ACTION_STOP = "stop" + + +ALLOWED_ACTIONS = ( + ACTION_RETRY, + ACTION_CHANGE_STRATEGY, + ACTION_ROLLBACK_AND_FORK, + ACTION_NARROW_SCOPE, + ACTION_DELEGATE, + ACTION_ESCALATE, + ACTION_STOP, +) + + +MIN_BUDGET_MULTIPLIER = 0.25 + + +MAX_BUDGET_MULTIPLIER = 3.0 + + +MIN_REACT_TURNS = 3 + + +MAX_REACT_TURNS = 80 + + +UNBOUNDED_REACT_TURNS = 0 + + +MIN_MEMORY_BUDGET_MS = 100 + + +MAX_MEMORY_BUDGET_MS = 5000 + + +MIN_LOOP_ATTEMPTS = 2 + + +MAX_LOOP_ATTEMPTS = 20 + + +TARGET_KIND = "agent_profile" diff --git a/src/backend/core/evolution/profiles/change_preview.py b/src/backend/core/evolution/profiles/change_preview.py new file mode 100644 index 00000000..b5d1ae64 --- /dev/null +++ b/src/backend/core/evolution/profiles/change_preview.py @@ -0,0 +1,85 @@ +"""Show the concrete content of a proposed evolution change.""" +from __future__ import annotations +from typing import Any, Dict, List, Optional + +def change_preview(candidate) -> Dict[str, Any]: + """The concrete content of the proposed change. + + A one-line hypothesis explains *why* something is proposed and says nothing + about *what* it is. Approving a distilled skill without being able to read + its body is not a decision, it is a guess — the text was already in the IR, + it was simply never sent to the page that asks the user to accept it. + """ + ir = candidate.ir or {} + kind = str(candidate.target_kind or "") + changes = ir.get("changes") or [] + for change in changes: + if kind == "skill": + document = change.get("document") + if isinstance(document, dict) and document.get("content"): + return { + "type": "skill_document", + "display_name": str(document.get("display_name") or ""), + "description": str(document.get("description") or ""), + "allowed_tools": [str(t) for t in (document.get("allowed_tools") or [])], + "content": str(document.get("content") or ""), + } + if kind == "memory": + operations = change.get("operations") or [] + if operations: + return { + "type": "memory_ops", + "operations": [ + { + "operation": str(op.get("operation") or ""), + "text": str(op.get("text") or op.get("memory_ref") or ""), + "reason": str(op.get("reason") or ""), + "before": op.get("before"), + "after": op.get("after"), + } + for op in operations + ], + } + if kind == "skill": + # Sequence-only candidates are the common output of the personal + # evolution path. They do not carry an authored SKILL.md, but approval + # still materialises a concrete name, description and ordered set of + # steps. Surface that exact structure instead of returning {}, which + # made the frontend silently remove the detail button. + tools: List[str] = [] + constraints: List[Dict[str, Any]] = [] + for change in changes: + sequence = change.get("tool_sequence") + if isinstance(sequence, list) and sequence and not tools: + tools = [str(tool) for tool in sequence] + rules = change.get("ordering_constraints") + if isinstance(rules, list) and rules: + constraints = [dict(rule) for rule in rules if isinstance(rule, dict)] + + rules_only = not tools and bool(constraints) + if rules_only: + from core.evolution.cycle.skill_document import tools_from_constraints as _tools_from_constraints + + tools = _tools_from_constraints(constraints) + if tools: + # Reuse the materialiser's wording so the preview and the installed + # skill never disagree about what the user accepted. + from core.evolution.cycle.skill_document import skill_description as _skill_description, skill_label as _skill_label + + return { + "type": "skill_sequence", + "display_name": _skill_label(tools, rules_only=rules_only), + "description": _skill_description(tools, constraints, rules_only=rules_only), + "allowed_tools": tools, + "steps": tools, + "ordering_constraints": constraints, + } + return {} + + +def tool_sequence_of_ir(ir: Optional[Dict[str, Any]]) -> List[str]: + for change in (ir or {}).get("changes") or []: + sequence = change.get("tool_sequence") + if isinstance(sequence, list) and sequence: + return [str(t) for t in sequence] + return [] diff --git a/src/backend/core/evolution/console_view.py b/src/backend/core/evolution/profiles/console_view.py similarity index 99% rename from src/backend/core/evolution/console_view.py rename to src/backend/core/evolution/profiles/console_view.py index ff40ed23..c5c08580 100644 --- a/src/backend/core/evolution/console_view.py +++ b/src/backend/core/evolution/profiles/console_view.py @@ -105,5 +105,3 @@ def release_to_dict(row: EvolutionRelease) -> Dict[str, Any]: "approved_by": row.approved_by, "updated_at": row.updated_at.isoformat() if row.updated_at else None, } - - diff --git a/src/backend/core/evolution/federation.py b/src/backend/core/evolution/profiles/federation.py similarity index 99% rename from src/backend/core/evolution/federation.py rename to src/backend/core/evolution/profiles/federation.py index f42b10f7..e42a1528 100644 --- a/src/backend/core/evolution/federation.py +++ b/src/backend/core/evolution/profiles/federation.py @@ -26,7 +26,7 @@ import logging import re from dataclasses import dataclass, field -from typing import Any, Dict, List, Optional, Sequence, Tuple +from typing import Any, Dict, List, Sequence logger = logging.getLogger(__name__) diff --git a/src/backend/core/evolution/personal.py b/src/backend/core/evolution/profiles/personal.py similarity index 88% rename from src/backend/core/evolution/personal.py rename to src/backend/core/evolution/profiles/personal.py index 2d63db08..039f8240 100644 --- a/src/backend/core/evolution/personal.py +++ b/src/backend/core/evolution/profiles/personal.py @@ -22,17 +22,19 @@ """ from __future__ import annotations +from core.evolution.cycle import ordering +from core.evolution.cycle import patterns +from core.evolution.cycle import promotion +from core.evolution.cycle import promotion_constants import hashlib import logging from typing import Any, Dict, List, Optional from core.db.engine import SessionLocal -from core.evolution import promotion as P -from core.evolution.credit import T_SKILL, aggregate_finding, confidence_from_evidence +from core.evolution.evidence.credit import T_SKILL, aggregate_finding, confidence_from_evidence from core.evolution.ir import ( RISK_LOW, - RISK_MEDIUM, EvolutionIR, InvariantViolation, Rollback, @@ -86,7 +88,7 @@ def load_personal_episodes(user_id: str, *, limit: int = 300) -> List[Dict[str, # this drifted apart once already: the personal path kept reading # raw content hashes as memory refs, which the overlay cannot key # on, so a personal memory candidate could never have been applied. - from core.evolution.loop import _view_for + from core.evolution.cycle.episode_views import view_for as _view_for for episode in episodes: views.append(_view_for(episode, by_episode.get(episode.episode_id, []))) @@ -96,14 +98,14 @@ def load_personal_episodes(user_id: str, *, limit: int = 300) -> List[Dict[str, return views -def _discover(views: List[Dict[str, Any]]) -> List[P.Pattern]: +def _discover(views: List[Dict[str, Any]]) -> List[patterns.Pattern]: """Pattern discovery at the personal support threshold.""" - original = P.MIN_SUPPORT + original = promotion_constants.MIN_SUPPORT try: - P.MIN_SUPPORT = PERSONAL_MIN_SUPPORT - return P.discover_patterns(views) + promotion_constants.MIN_SUPPORT = PERSONAL_MIN_SUPPORT + return patterns.discover_patterns(views) finally: - P.MIN_SUPPORT = original + promotion_constants.MIN_SUPPORT = original def run_personal_cycle( @@ -117,8 +119,8 @@ def run_personal_cycle( Returns a report rather than raising — this runs in the background and a bad cycle must leave the product untouched. """ - from core.evolution.prefs import resolve_prefs - from core.evolution.loop import _persist_candidate + from core.evolution.profiles.prefs import resolve_prefs + from core.evolution.cycle.candidate_records import persist_candidate as _persist_candidate report: Dict[str, Any] = { "user_id": user_id, @@ -130,7 +132,7 @@ def run_personal_cycle( "scope": "personal", } - from core.evolution.readiness import check_readiness + from core.evolution.cycle.readiness import check_readiness readiness = check_readiness() report["readiness"] = readiness.to_dict() @@ -151,17 +153,17 @@ def run_personal_cycle( if not views: return report - patterns = _discover(views) - report["patterns"] = len(patterns) - constraints = P.extract_ordering_constraints( + discovered = _discover(views) + report["patterns"] = len(discovered) + constraints = ordering.extract_ordering_constraints( views, min_support=PERSONAL_MIN_SUPPORT, min_contexts=1 ) report["ordering_constraints"] = [c.to_dict() for c in constraints] known_concepts = _known_concepts(prefs) - for pattern in patterns: - if pattern.kind != P.PATTERN_SUCCESS_SUBSEQUENCE or not pattern.tool_sequence: + for pattern in discovered: + if pattern.kind != promotion_constants.PATTERN_SUCCESS_SUBSEQUENCE or not pattern.tool_sequence: continue # 这是"多次成功里观察到的共性"，不是某次失败的归因，所以走聚合决策。 # 旧实现在这里伪造 outcome_verdict="failed" 逼归因器选中技能引擎—— @@ -183,7 +185,7 @@ def run_personal_cycle( # min_support 必须显式传入：发现阶段用的个人门槛（3）在 _discover 返回 # 后已恢复为全局值（5），不传的话支持度 3–4 的个人模式会在这里被静默拒绝。 - proposal = P.promote_tool_sequence_to_skill( + proposal = promotion.promote_tool_sequence_to_skill( pattern, episodes=views, min_support=PERSONAL_MIN_SUPPORT, @@ -196,8 +198,8 @@ def run_personal_cycle( # 规则门之后的 LLM 语义裁决：意图簇是否巧合归并、序列是否只是对任何 # 任务都成立的通用组合。拒绝即不产候选；模型不可用同样拒绝，但在 # 报告里可区分。 - from core.evolution.loop import _run_async - from core.evolution.sop_judge import adjudicate_sequence_sop + from core.evolution.cycle.candidate_records import run_async as _run_async + from core.evolution.assessment.sop_judge import adjudicate_sequence_sop covered = {str(e) for e in (proposal.payload.get("episode_ids") or [])} verdict = _run_async( @@ -313,13 +315,9 @@ def _personal_other_engines( *every* request in scope is assembled from, so a single user's evidence is not a basis for them — this is the scope boundary, not a missing feature. """ - from core.evolution.cycle_extras import run_extras - from core.evolution.loop import ( - _decremental_candidates, - _intent_skill_candidates, - _memory_candidates, - _memory_scan_findings, - ) + from core.evolution.cycle.cycle_extras import run_extras + from core.evolution.cycle.composition_candidates import decremental_candidates as _decremental_candidates + from core.evolution.cycle.memory_candidates import intent_skill_candidates as _intent_skill_candidates, memory_candidates as _memory_candidates, memory_scan_findings as _memory_scan_findings scan_findings = _memory_scan_findings(views) extras = run_extras(views, scan_findings=scan_findings) diff --git a/src/backend/core/evolution/user_settings.py b/src/backend/core/evolution/profiles/personal_review.py similarity index 60% rename from src/backend/core/evolution/user_settings.py rename to src/backend/core/evolution/profiles/personal_review.py index 0a4ff203..2727e425 100644 --- a/src/backend/core/evolution/user_settings.py +++ b/src/backend/core/evolution/profiles/personal_review.py @@ -1,109 +1,13 @@ -"""Per-user evolution participation, derived from the single evolution switch. - -There used to be two user-facing switches: "start evolution" (does the loop -distil for me) and "contribute to evolution" (do my conversations feed -cross-user learning). The distinction was real but nobody could tell the two -apart in the settings panel, so the product now has exactly one switch — -``evolution_prefs.enabled`` — and contribution follows it: evolution on means -the loop may distil for this user *and* their conversations may serve as -evidence within the tenant; off means neither. - -Enforcement still lives at one point rather than being scattered: the -Episode's ``privacy_class``. With the switch off, episodes are marked -``private`` and pattern mining filters those out — so the user still gets -their own in-chat card and their own history, while nothing of theirs feeds -cross-user learning. Anything weaker would make the promise unverifiable. -""" - +"""Let users review, approve and activate candidates for themselves.""" from __future__ import annotations - -import logging from typing import Any, Dict, List, Optional - from core.db.engine import SessionLocal +import logging +from core.evolution.profiles import change_preview +from core.evolution.profiles import user_settings_constants logger = logging.getLogger(__name__) -# Response-field name kept from the era of a separate contribution switch, so -# older bundled frontends that read /evolution/settings keep working. -SETTING_KEY = "evolution_contribution_enabled" - -# Episodes from an opted-out user. Recorded for that user's own benefit (their -# card, their history) but excluded from anything cross-user. -PRIVACY_PRIVATE = "private" -PRIVACY_TENANT = "tenant" - - -def is_contribution_enabled(metadata: Optional[Dict[str, Any]]) -> bool: - """Whether this user's episodes may feed the evolution loop. - - Reads the single evolution switch (``evolution_prefs.enabled``), so one - stored value governs both distilling *for* the user and learning *from* - them. Off by default: consent is collected by the first-run wizard, and a - user who never enabled evolution contributes nothing. "On" only ever means - *within the tenant*: cross-tenant sharing is a separate, independently- - gated decision (see ``federation``). - """ - from core.evolution.prefs import PREFS_KEY, normalise - - if not metadata: - return False - return bool(normalise(metadata.get(PREFS_KEY)).get("enabled", False)) - - -def privacy_class_for(metadata: Optional[Dict[str, Any]]) -> str: - """The Episode privacy class implied by the user's setting.""" - return PRIVACY_TENANT if is_contribution_enabled(metadata) else PRIVACY_PRIVATE - - -def resolve_for_user(user_id: str) -> Dict[str, Any]: - """Read the setting for one user. Degrades to the default, never raises.""" - if not user_id: - return {SETTING_KEY: False, "privacy_class": PRIVACY_PRIVATE} - try: - from core.db.models import UserShadow - - with SessionLocal() as db: - row = db.query(UserShadow).filter(UserShadow.user_id == user_id).first() - metadata = (getattr(row, "extra_data", None) or {}) if row else {} - except Exception as exc: - logger.debug("[evo-settings] read failed for %s: %s", user_id, exc) - metadata = {} - - enabled = is_contribution_enabled(metadata) - return {SETTING_KEY: enabled, "privacy_class": privacy_class_for(metadata)} - - -def update_for_user(user_id: str, enabled: bool) -> Dict[str, Any]: - """Persist the setting by writing the single evolution switch. - - Kept as an alias so older bundled frontends PATCHing /evolution/settings - still work; it writes the same ``evolution_prefs.enabled`` the preferences - panel writes, never a separate value that could drift from it. - - Turning it off does **not** retroactively delete already-contributed - evidence — that would silently rewrite history other candidates were derived - from. It stops future episodes from contributing; withdrawing past evidence - is a separate, explicit action so its consequences stay visible. - """ - from core.evolution.prefs import update_prefs - - merged = update_prefs(user_id, {"enabled": bool(enabled)}) - on = bool(merged.get("enabled", False)) - return { - SETTING_KEY: on, - "privacy_class": PRIVACY_TENANT if on else PRIVACY_PRIVATE, - "note": "仅影响此后新产生的证据；已贡献的证据需单独撤回", - } - - -# A candidate is the user's to approve when their own episodes account for at -# least this share of its evidence. Below it the capability was learned mostly -# from other people's work, and one person approving it for everyone is a -# different decision that belongs with an administrator. -OWN_EVIDENCE_THRESHOLD = 0.6 - - def personal_action(candidate) -> Optional[Dict[str, str]]: """What this user can actually do with a candidate, or ``None``. @@ -152,7 +56,7 @@ def personal_action(candidate) -> Optional[Dict[str, str]]: # user — but only where the candidate has already passed replay, or its # tier owes nothing beyond replay. Listing a medium-risk draft here put # a button on screen that answered "force 只允许用于 low 风险候选". - from core.evolution.release import ACTIVE, REPLAY_PASSED + from core.evolution.cycle.release import ACTIVE, REPLAY_PASSED replayed = str(candidate.status or "") in (REPLAY_PASSED, ACTIVE) if not replayed and risk != "low": @@ -167,7 +71,7 @@ def personal_action(candidate) -> Optional[Dict[str, str]]: return None -def _personally_activated_ids(db, user_id: str, candidate_ids: List[str]) -> set: +def personally_activated_ids(db, user_id: str, candidate_ids: List[str]) -> set: """Candidates this user has already turned into a live personal release. A personal activation deliberately leaves the candidate's status untouched @@ -182,7 +86,7 @@ def _personally_activated_ids(db, user_id: str, candidate_ids: List[str]) -> set out = set() try: from core.db.models.evolution import EvolutionRelease - from core.evolution.release import TERMINAL_STAGES + from core.evolution.cycle.release import TERMINAL_STAGES for release in ( db.query(EvolutionRelease) @@ -240,7 +144,7 @@ def pending_for_user(user_id: str, *, limit: int = 20) -> List[Dict[str, Any]]: # by design (its life continues for others), so without this it would # reappear on every refresh — an "enable" button for something already # enabled, whose second press just stamps out a duplicate release. - already_mine = _personally_activated_ids( + already_mine = personally_activated_ids( db, user_id, [c.candidate_id for c in candidates] ) @@ -253,12 +157,12 @@ def pending_for_user(user_id: str, *, limit: int = 20) -> List[Dict[str, Any]]: continue mine = refs & own_ids share = len(mine) / len(refs) - if share < OWN_EVIDENCE_THRESHOLD: + if share < user_settings_constants.OWN_EVIDENCE_THRESHOLD: continue action = personal_action(candidate) if action is None: continue - change_preview = _change_preview(candidate) + preview = change_preview.change_preview(candidate) if str(candidate.target_kind or "") == "skill": # Legacy personal cycles could promote a single repeated # tool call as a "sequence". Such a candidate has neither @@ -266,12 +170,12 @@ def pending_for_user(user_id: str, *, limit: int = 20) -> List[Dict[str, Any]]: # approving it would install a tautological skill. Keep # the evidence intact for audit, but do not present noise as # a decision the user ought to make. - if change_preview.get("type") == "skill_sequence": - steps = change_preview.get("steps") or [] - rules = change_preview.get("ordering_constraints") or [] + if preview.get("type") == "skill_sequence": + steps = preview.get("steps") or [] + rules = preview.get("ordering_constraints") or [] if len(steps) < 2 and not rules: continue - elif change_preview.get("type") != "skill_document": + elif preview.get("type") != "skill_document": # A skill with no inspectable content is not informed # consent and would also fail materialisation later. continue @@ -286,14 +190,14 @@ def pending_for_user(user_id: str, *, limit: int = 20) -> List[Dict[str, Any]]: "your_episodes": len(mine), "total_evidence": len(refs), "own_share": round(share, 3), - "tool_sequence": _tool_sequence_of_ir(candidate.ir), + "tool_sequence": change_preview.tool_sequence_of_ir(candidate.ir), # What the button does, and what it will change. Without # these the row states a diagnosis and offers a verb that # does not follow from it. "action": action["action"], "action_label": action["label"], "action_effect": action["effect"], - "change": change_preview, + "change": preview, "created_at": candidate.created_at.isoformat() if candidate.created_at else None, @@ -307,89 +211,6 @@ def pending_for_user(user_id: str, *, limit: int = 20) -> List[Dict[str, Any]]: return [] -def _change_preview(candidate) -> Dict[str, Any]: - """The concrete content of the proposed change. - - A one-line hypothesis explains *why* something is proposed and says nothing - about *what* it is. Approving a distilled skill without being able to read - its body is not a decision, it is a guess — the text was already in the IR, - it was simply never sent to the page that asks the user to accept it. - """ - ir = candidate.ir or {} - kind = str(candidate.target_kind or "") - changes = ir.get("changes") or [] - for change in changes: - if kind == "skill": - document = change.get("document") - if isinstance(document, dict) and document.get("content"): - return { - "type": "skill_document", - "display_name": str(document.get("display_name") or ""), - "description": str(document.get("description") or ""), - "allowed_tools": [str(t) for t in (document.get("allowed_tools") or [])], - "content": str(document.get("content") or ""), - } - if kind == "memory": - operations = change.get("operations") or [] - if operations: - return { - "type": "memory_ops", - "operations": [ - { - "operation": str(op.get("operation") or ""), - "text": str(op.get("text") or op.get("memory_ref") or ""), - "reason": str(op.get("reason") or ""), - "before": op.get("before"), - "after": op.get("after"), - } - for op in operations - ], - } - if kind == "skill": - # Sequence-only candidates are the common output of the personal - # evolution path. They do not carry an authored SKILL.md, but approval - # still materialises a concrete name, description and ordered set of - # steps. Surface that exact structure instead of returning {}, which - # made the frontend silently remove the detail button. - tools: List[str] = [] - constraints: List[Dict[str, Any]] = [] - for change in changes: - sequence = change.get("tool_sequence") - if isinstance(sequence, list) and sequence and not tools: - tools = [str(tool) for tool in sequence] - rules = change.get("ordering_constraints") - if isinstance(rules, list) and rules: - constraints = [dict(rule) for rule in rules if isinstance(rule, dict)] - - rules_only = not tools and bool(constraints) - if rules_only: - from core.evolution.activation import _tools_from_constraints - - tools = _tools_from_constraints(constraints) - if tools: - # Reuse the materialiser's wording so the preview and the installed - # skill never disagree about what the user accepted. - from core.evolution.activation import _skill_description, _skill_label - - return { - "type": "skill_sequence", - "display_name": _skill_label(tools, rules_only=rules_only), - "description": _skill_description(tools, constraints, rules_only=rules_only), - "allowed_tools": tools, - "steps": tools, - "ordering_constraints": constraints, - } - return {} - - -def _tool_sequence_of_ir(ir: Optional[Dict[str, Any]]) -> List[str]: - for change in (ir or {}).get("changes") or []: - sequence = change.get("tool_sequence") - if isinstance(sequence, list) and sequence: - return [str(t) for t in sequence] - return [] - - def approve_for_user(candidate_id: str, user_id: str) -> Dict[str, Any]: """Approve a candidate and activate it **for this user only**. @@ -397,7 +218,7 @@ def approve_for_user(candidate_id: str, user_id: str) -> Dict[str, Any]: anyone else's agent does. That is what makes personal approval safe enough to sit in a settings panel rather than behind an administrator. """ - from core.evolution.activation import materialise_skill_candidate + from core.evolution.cycle.activation import materialise_skill_candidate allowed = {c["candidate_id"]: c for c in pending_for_user(user_id, limit=200)} row = allowed.get(candidate_id) @@ -406,7 +227,7 @@ def approve_for_user(candidate_id: str, user_id: str) -> Dict[str, Any]: # or double click lands here, and telling that user they lack permission # over a skill they just installed reads as a failure. with SessionLocal() as db: - if candidate_id in _personally_activated_ids(db, user_id, [candidate_id]): + if candidate_id in personally_activated_ids(db, user_id, [candidate_id]): raise PermissionError("该候选你已启用过，对应能力已在你的技能列表中，无需重复启用") raise PermissionError( "该候选无法由你单独批准：可能主要来自他人的对话证据，或属于影响全体成员的编排/退役变更，需管理员审批" @@ -416,7 +237,7 @@ def approve_for_user(candidate_id: str, user_id: str) -> Dict[str, Any]: # skill materialiser is what made memory candidates fail with # "暂不支持物化 memory 候选" on every click. if row["action"] == "apply_memory": - from core.evolution.memory_apply import apply_memory_candidate + from core.evolution.generators.memory_apply import apply_memory_candidate result = apply_memory_candidate( candidate_id, approver=f"user:{user_id}", require_user_id=user_id @@ -468,7 +289,7 @@ def summarise_contributions(user_id: str) -> Dict[str, Any]: return empty episode_ids = {e.episode_id for e in episodes} - contributed = [e for e in episodes if e.privacy_class != PRIVACY_PRIVATE] + contributed = [e for e in episodes if e.privacy_class != user_settings_constants.PRIVACY_PRIVATE] memory_written = 0 for event in ( diff --git a/src/backend/core/evolution/policies.py b/src/backend/core/evolution/profiles/policies.py similarity index 88% rename from src/backend/core/evolution/policies.py rename to src/backend/core/evolution/profiles/policies.py index e9e1dda5..abfe1e10 100644 --- a/src/backend/core/evolution/policies.py +++ b/src/backend/core/evolution/profiles/policies.py @@ -17,7 +17,7 @@ fix for that shipped as a ``tenant_id`` parameter no caller passed. All three are gone. There is one orchestration asset — -:class:`~core.evolution.agent_profile.AgentProfile` — and this module is the +:class:`~core.evolution.profiles.agent_profile.AgentProfile` — and this module is the **projection** of it that the autonomous loop consumes. A projection rather than a copy: there is no second store to publish to, no second thing to keep in sync, and no way for the loop's numbers to disagree with the profile they came from. @@ -29,23 +29,9 @@ from dataclasses import dataclass, field from typing import Any, Dict, List, Optional -from core.evolution.agent_profile import ( - ACTION_CHANGE_STRATEGY, - ACTION_ESCALATE, - ACTION_NARROW_SCOPE, - ACTION_RETRY, - ACTION_ROLLBACK_AND_FORK, - ACTION_STOP, - ALLOWED_ACTIONS, - SIGNAL_BUDGET_LOW, - SIGNAL_NO_DIFF, - SIGNAL_NO_PROGRESS, - SIGNAL_REPEATED_ACTIONS, - SIGNAL_REVIEWER_FLAT, - AgentProfile, - InterventionRule, - load_active_profile, -) +from core.evolution.profiles.agent_profile_constants import ACTION_CHANGE_STRATEGY, ACTION_ESCALATE, ACTION_NARROW_SCOPE, ACTION_RETRY, ACTION_ROLLBACK_AND_FORK, ACTION_STOP, ALLOWED_ACTIONS, SIGNAL_BUDGET_LOW, SIGNAL_NO_DIFF, SIGNAL_NO_PROGRESS, SIGNAL_REPEATED_ACTIONS, SIGNAL_REVIEWER_FLAT +from core.evolution.profiles.agent_profile import AgentProfile, InterventionRule +from core.evolution.profiles.profile_selection import load_active_profile logger = logging.getLogger(__name__) diff --git a/src/backend/core/evolution/prefs.py b/src/backend/core/evolution/profiles/prefs.py similarity index 100% rename from src/backend/core/evolution/prefs.py rename to src/backend/core/evolution/profiles/prefs.py diff --git a/src/backend/core/evolution/profiles/profile_selection.py b/src/backend/core/evolution/profiles/profile_selection.py new file mode 100644 index 00000000..6d0fef54 --- /dev/null +++ b/src/backend/core/evolution/profiles/profile_selection.py @@ -0,0 +1,147 @@ +"""Pick the agent profile that governs a run.""" +from __future__ import annotations +from core.evolution.profiles import agent_profile +from typing import Any, Dict, List, Optional, Sequence, Tuple +import logging +logger = logging.getLogger(__name__) + +def scope_rank(scope: Dict[str, str], *, user_id: str, tenant_id: str) -> Optional[int]: + """How specific this profile is to the subject, or ``None`` if it is not. + + Lower is more specific. Returning ``None`` for a profile owned by another + subject is the whole point: it must not act as a fallback. + """ + owner = str((scope or {}).get("user_id") or "") + tenant = str((scope or {}).get("tenant_id") or "") + if owner: + return 0 if owner == user_id else None + if tenant: + return 1 if tenant == (tenant_id or "default") else None + return 2 + + +def pick_profile( + profiles: Sequence[agent_profile.AgentProfile], + *, + task_type: str, + user_id: str = "", + tenant_id: str = "default", +) -> Optional[agent_profile.AgentProfile]: + """The one profile that governs this run, or ``None``. + + A task-type-specific profile beats a general one at the same scope level: + the narrower statement is the more deliberate one. + """ + ranked: List[Tuple[int, int, agent_profile.AgentProfile]] = [] + for profile in profiles: + if not profile.applies_to(task_type): + continue + rank = scope_rank(profile.scope, user_id=user_id, tenant_id=tenant_id) + if rank is None: + continue + ranked.append((rank, 0 if profile.task_types else 1, profile)) + if not ranked: + return None + ranked.sort(key=lambda item: (item[0], item[1])) + return ranked[0][2] + + +def profile_from_row(row: Any) -> agent_profile.AgentProfile: + """The profile a stored row describes, with the columns believed over payload. + + ``profile_id`` / ``version`` / ``task_types`` / ``scope`` exist as real + columns *and* inside ``payload`` (:meth:`AgentProfile.to_dict` writes them), + so for anything published through :mod:`core.evolution.cycle.activation` the two + agree and this is a no-op. They stop agreeing for a row written by hand or + by a seeding script, and reading only ``payload`` then fails open in the + worst possible direction: a missing ``task_types`` parses as ``[]``, which + :meth:`AgentProfile.applies_to` treats as *every* task type, and a missing + ``scope`` parses as ``{}``, which :func:`_scope_rank` treats as *every* + subject. A profile published for one task type silently governs every run — + reported under the id ``builtin``, because that is what + :meth:`AgentProfile.from_dict` falls back to, so the log line naming the + culprit names the deployment default instead. + + The two narrowing fields resolve to whichever side is non-empty rather than + to the column unconditionally: an empty column against a populated payload + is the same widening failure with the sides swapped. When both are populated + and disagree the column wins — it is what the activation uniqueness check and + the console read — and the disagreement is logged, because it means something + wrote this row outside the activation path. + """ + raw = dict(row.payload or {}) + + # Identity: the column is the primary key and NOT NULL, so it is always the + # more trustworthy of the two, and a wrong id here is what makes a bad + # profile untraceable in the logs. + raw["profile_id"] = row.profile_id + raw["version"] = row.version + + col_types = [str(t) for t in (row.task_types or [])] + payload_types = [str(t) for t in (raw.get("task_types") or [])] + if col_types and payload_types and set(col_types) != set(payload_types): + logger.warning( + "[agent-profile] %s: task_types column %s != payload %s, using column", + row.profile_id, col_types, payload_types, + ) + raw["task_types"] = col_types or payload_types + + col_scope = dict(row.scope or {}) + payload_scope = dict(raw.get("scope") or {}) + if col_scope and payload_scope and col_scope != payload_scope: + logger.warning( + "[agent-profile] %s: scope column %s != payload %s, using column", + row.profile_id, col_scope, payload_scope, + ) + raw["scope"] = col_scope or payload_scope + + return agent_profile.AgentProfile.from_dict(raw) + + +def load_active_profile( + *, + task_type: str = "chat", + user_id: str = "", + tenant_id: str = "default", +) -> agent_profile.AgentProfile: + """The profile in force for this run. + + Read fresh on every call. Resolving once at import — which is what the old + orchestration policy did — means a published change needs a process restart + to take effect and different replicas disagree in the meantime, neither of + which is acceptable for an asset whose entire purpose is to change behaviour + between runs. + """ + from core.db.engine import SessionLocal + from core.db.models.evolution import EvolutionAgentProfile + + try: + with SessionLocal() as db: + rows = ( + db.query(EvolutionAgentProfile) + .filter(EvolutionAgentProfile.is_active.is_(True)) + .order_by(EvolutionAgentProfile.updated_at.desc()) + .all() + ) + profiles = [profile_from_row(row) for row in rows] + except Exception as exc: # noqa: BLE001 + logger.warning("[agent-profile] load failed, using built-in: %s", exc) + return agent_profile.builtin_profile() + + chosen = pick_profile( + profiles, task_type=task_type, user_id=user_id, tenant_id=tenant_id + ) + if chosen is None: + return agent_profile.builtin_profile() + ok, problems = agent_profile.validate_profile(chosen) + if not ok: + # A stored profile that no longer validates is not applied. The + # alternative — applying it partially — produces a configuration nobody + # reviewed and nobody can reproduce. + logger.warning( + "[agent-profile] stored profile %s rejected at load: %s", + chosen.profile_id, + problems, + ) + return agent_profile.builtin_profile() + return chosen diff --git a/src/backend/core/evolution/profiles/stall_hook.py b/src/backend/core/evolution/profiles/stall_hook.py new file mode 100644 index 00000000..97b22316 --- /dev/null +++ b/src/backend/core/evolution/profiles/stall_hook.py @@ -0,0 +1,103 @@ +"""Stall decisions from neutral tool attempts, without framework objects.""" +import json +import logging +from core.harness.context_items import reminder_item +from core.harness.hooks import Decision, FailureMode, HookSpec, HookStage + +logger = logging.getLogger(__name__) + + +class StallHook: + def __init__(self, rules): + self._rules = list(rules or []) + self._signals = {} + self._last_call = None + self._fired = set() + + def __call__(self, invocation): + if not self._rules: + return Decision() + if invocation.stage == HookStage.BEFORE_TOOL: + signature = (invocation.operation_name, + json.dumps(invocation.data.get("raw_input"), sort_keys=True, default=str)) + self._signals["repeated_actions"] = (self._signals.get("repeated_actions", 0) + 1 + if signature == self._last_call else 0) + self._last_call = signature + return Decision() + response = invocation.data.get("tool_result") or {} + failed = response.get("state") in {"error", "denied", "interrupted"} + for signal in ("tool_error_streak", "no_progress"): + self._signals[signal] = self._signals.get(signal, 0) + 1 if failed else 0 + return self.intervene(invocation) + + def intervene(self, invocation): + from core.evolution.profiles.agent_profile_constants import ACTION_CHANGE_STRATEGY, ACTION_DELEGATE, ACTION_ESCALATE, ACTION_NARROW_SCOPE, ACTION_RETRY, ACTION_ROLLBACK_AND_FORK, ACTION_STOP + + applicable = [ + rule + for rule in self._rules + if self._signals.get(getattr(rule, "signal", ""), 0) + >= int(getattr(rule, "threshold", 0) or 0) + and int(getattr(rule, "threshold", 0) or 0) > 0 + ] + if not applicable: + return + # Most-specific first: a long stall escalates rather than repeating the + # mild response that already failed to help. + rule = max(applicable, key=lambda r: int(getattr(r, "threshold", 0) or 0)) + action = str(getattr(rule, "action", "")) + signal = str(getattr(rule, "signal", "")) + + # Fire once per (signal, action). Repeating the same reminder every + # iteration turns it into noise the model learns to skip. + key = f"{signal}:{action}" + if key in self._fired: + return + self._fired.add(key) + + observed = { + "repeated_actions": "你已经用完全相同的参数重复调用同一个工具", + "tool_error_streak": "连续多次工具调用失败", + "no_progress": "连续多轮没有取得任何可用结果", + }.get(signal, "当前执行已停滞") + + guidance = { + ACTION_CHANGE_STRATEGY: "换一条思路：不要重试同样的调用，改用别的工具或别的切入角度。", + ACTION_NARROW_SCOPE: "缩小范围：先只解决其中最小的一个子问题，拿到确定结果再往下走。", + ACTION_RETRY: "可以再试一次，但必须改变参数或前置条件，原样重试不会有不同结果。", + ACTION_DELEGATE: "把这段工作交给合适的子智能体处理，只取回结论。", + ACTION_ESCALATE: "停止尝试，向用户说明卡在哪里、还缺什么信息。", + ACTION_STOP: "停止继续尝试，如实汇报已完成的部分和未完成的原因。", + # Loop-only: a single answer has nothing to roll back to. + ACTION_ROLLBACK_AND_FORK: "放弃当前路径，回到上一个可用状态重新规划。", + }.get(action, "") + if not guidance: + return + + logger.info( + "[stall-intervention] %s >= %s -> %s", + signal, + getattr(rule, "threshold", 0), + action, + ) + message = ( + f"{observed}。{guidance}\n" + f"（该干预来自当前生效的编排配置：{signal} ≥ " + f"{getattr(rule, 'threshold', 0)} → {action}）" + ) + return Decision.modify({ + "context_items": [ + *invocation.data.get("context_items", ()), + reminder_item(message, "harness:stall_intervention"), + ], + }) + + + + +def stall_hooks(rules): + policy = StallHook(rules) + return (HookSpec("stall_attempt", HookStage.BEFORE_TOOL, policy, priority=400, + failure_mode=FailureMode.OPEN), + HookSpec("stall_result", HookStage.AFTER_TOOL, policy, priority=400, + failure_mode=FailureMode.OPEN, mutable_fields=frozenset({"context_items"}))) diff --git a/src/backend/core/evolution/profiles/user_settings.py b/src/backend/core/evolution/profiles/user_settings.py new file mode 100644 index 00000000..bb62a77e --- /dev/null +++ b/src/backend/core/evolution/profiles/user_settings.py @@ -0,0 +1,70 @@ +"""Read and update whether a user's episodes feed the evolution loop.""" +from __future__ import annotations +from typing import Any, Dict, Optional +from core.db.engine import SessionLocal +import logging +from core.evolution.profiles import user_settings_constants + +logger = logging.getLogger(__name__) + +def is_contribution_enabled(metadata: Optional[Dict[str, Any]]) -> bool: + """Whether this user's episodes may feed the evolution loop. + + Reads the single evolution switch (``evolution_prefs.enabled``), so one + stored value governs both distilling *for* the user and learning *from* + them. Off by default: consent is collected by the first-run wizard, and a + user who never enabled evolution contributes nothing. "On" only ever means + *within the tenant*: cross-tenant sharing is a separate, independently- + gated decision (see ``federation``). + """ + from core.evolution.profiles.prefs import PREFS_KEY, normalise + + if not metadata: + return False + return bool(normalise(metadata.get(PREFS_KEY)).get("enabled", False)) + + +def privacy_class_for(metadata: Optional[Dict[str, Any]]) -> str: + """The Episode privacy class implied by the user's setting.""" + return user_settings_constants.PRIVACY_TENANT if is_contribution_enabled(metadata) else user_settings_constants.PRIVACY_PRIVATE + + +def resolve_for_user(user_id: str) -> Dict[str, Any]: + """Read the setting for one user. Degrades to the default, never raises.""" + if not user_id: + return {user_settings_constants.SETTING_KEY: False, "privacy_class": user_settings_constants.PRIVACY_PRIVATE} + try: + from core.db.models import UserShadow + + with SessionLocal() as db: + row = db.query(UserShadow).filter(UserShadow.user_id == user_id).first() + metadata = (getattr(row, "extra_data", None) or {}) if row else {} + except Exception as exc: + logger.debug("[evo-settings] read failed for %s: %s", user_id, exc) + metadata = {} + + enabled = is_contribution_enabled(metadata) + return {user_settings_constants.SETTING_KEY: enabled, "privacy_class": privacy_class_for(metadata)} + + +def update_for_user(user_id: str, enabled: bool) -> Dict[str, Any]: + """Persist the setting by writing the single evolution switch. + + Kept as an alias so older bundled frontends PATCHing /evolution/settings + still work; it writes the same ``evolution_prefs.enabled`` the preferences + panel writes, never a separate value that could drift from it. + + Turning it off does **not** retroactively delete already-contributed + evidence — that would silently rewrite history other candidates were derived + from. It stops future episodes from contributing; withdrawing past evidence + is a separate, explicit action so its consequences stay visible. + """ + from core.evolution.profiles.prefs import update_prefs + + merged = update_prefs(user_id, {"enabled": bool(enabled)}) + on = bool(merged.get("enabled", False)) + return { + user_settings_constants.SETTING_KEY: on, + "privacy_class": user_settings_constants.PRIVACY_TENANT if on else user_settings_constants.PRIVACY_PRIVATE, + "note": "仅影响此后新产生的证据；已贡献的证据需单独撤回", + } diff --git a/src/backend/core/evolution/profiles/user_settings_constants.py b/src/backend/core/evolution/profiles/user_settings_constants.py new file mode 100644 index 00000000..7cceead8 --- /dev/null +++ b/src/backend/core/evolution/profiles/user_settings_constants.py @@ -0,0 +1,14 @@ +"""Setting key and privacy classes for evolution contributions.""" +from __future__ import annotations + + +SETTING_KEY = "evolution_contribution_enabled" + + +PRIVACY_PRIVATE = "private" + + +PRIVACY_TENANT = "tenant" + + +OWN_EVIDENCE_THRESHOLD = 0.6 diff --git a/src/backend/core/evolution/promotion.py b/src/backend/core/evolution/promotion.py deleted file mode 100644 index f71a631c..00000000 --- a/src/backend/core/evolution/promotion.py +++ /dev/null @@ -1,890 +0,0 @@ -"""Cross-engine capability promotion (GCE tickets 32 / 33 / 34 / 36). - -This is the part that makes the system *learn* rather than merely record. -Capability flows up a chain: - - raw traces → episodic memory → skill candidate → workflow candidate → domain - constraint → released asset - -The idea is that experience should not stay in whichever form it first arrived -in. A pattern that keeps recurring as scattered memories is re-retrieved and -re-planned on every occurrence; compiled into a skill, it is assembled once. A -combination of skills that keeps appearing in the same order is re-derived every -time; promoted to a workflow, it becomes reusable — which is the direct answer -to "orchestration never accumulates", the largest gap in the current system. - -Two design commitments: - -* **Promotion, not duplication.** A promoted memory is down-weighted rather - than deleted, and the link is bidirectional, so "why does this skill exist?" - has an answer and a rollback can restore the original weighting. -* **Aggregate evidence by default.** A single high-confidence failure may - trigger a candidate, but the normal entry point is a recurring cluster. - Otherwise the system permanently encodes noise. - -Flow also runs downhill: overlapping assets merge, unused ones retire, and -**negative transfer** — a change that quietly degrades tasks it has nothing to do -with — is detected. That last one is the most dangerous failure mode of joint -evolution and the easiest to miss, because no single release's guardrail ever -trips. -""" - -from __future__ import annotations - -import logging -from collections import Counter, defaultdict -from dataclasses import dataclass, field -from typing import Any, Dict, Iterable, List, Optional, Sequence, Set, Tuple - -logger = logging.getLogger(__name__) - -# Pattern families discovered from episode clusters. -PATTERN_REPEATED_FAILURE = "repeated_failure" -PATTERN_SUCCESS_SUBSEQUENCE = "success_subsequence" -PATTERN_DRIFT = "execution_drift" - -# Minimum cluster size before anything downstream is produced. A lower bar would -# let one-off noise become a permanent asset. -MIN_SUPPORT = 5 -# A single failure may still trigger, but is flagged so reviewers can see that -# it rests on one observation. -SINGLE_SHOT_CONFIDENCE = 0.85 - -# Promotion thresholds. -SKILL_MIN_SUCCESS_RATE = 0.7 -WORKFLOW_MIN_COMBO_FREQUENCY = 5 -# A combination must beat its parts, not merely be frequent — otherwise every -# common sequence becomes a workflow whether or not the grouping helps. -WORKFLOW_MIN_UPLIFT = 0.05 - -# 一段 SOP 至少要有的步骤数。长度 1–2 的"序列"几乎总是通用工具的巧合复现—— -# 整轮对话只调一次 view_text_file 也会形成"相同序列"——包装它们不携带任何 -# 过程知识，只往评审队列里添噪声。 -MIN_TOOL_SEQUENCE_LEN = 3 -# 支撑 Episode 中归入同一意图簇的最低占比。序列相同只说明工具巧合， -# 意图相同才说明存在可复用的做事方法——这是把一段序列当 SOP 的前提。 -INTENT_COHESION_RATIO = 0.6 - - -@dataclass -class Pattern: - """A recurring shape found across episodes.""" - - kind: str - signature: str - support: int - episode_ids: List[str] = field(default_factory=list) - success_rate: float = 0.0 - tool_sequence: List[str] = field(default_factory=list) - skill_sequence: List[str] = field(default_factory=list) - memory_refs: List[str] = field(default_factory=list) - single_shot: bool = False - - @property - def meets_support(self) -> bool: - return self.support >= MIN_SUPPORT or self.single_shot - - def to_dict(self) -> Dict[str, Any]: - return { - "kind": self.kind, - "signature": self.signature, - "support": self.support, - "success_rate": round(self.success_rate, 3), - "tool_sequence": self.tool_sequence, - "skill_sequence": self.skill_sequence, - "memory_refs": self.memory_refs, - "single_shot": self.single_shot, - "episode_ids": self.episode_ids[:20], - } - - -def _signature(items: Sequence[str]) -> str: - return "→".join(items) - - -def discover_patterns(episodes: List[Dict[str, Any]]) -> List[Pattern]: - """Cluster episodes into recurring shapes. - - ``episodes`` are plain dicts so this stays testable without a database and - usable over backfilled history (which has no asset bundle and therefore - cannot be replayed, but is perfectly good for finding patterns). - """ - by_tool_seq: Dict[str, List[Dict[str, Any]]] = defaultdict(list) - by_skill_seq: Dict[str, List[Dict[str, Any]]] = defaultdict(list) - - for episode in episodes: - tools = [str(t) for t in (episode.get("tool_sequence") or [])] - skills = [str(s) for s in (episode.get("skill_sequence") or [])] - if tools: - by_tool_seq[_signature(tools)].append(episode) - if len(skills) >= 2: - by_skill_seq[_signature(skills)].append(episode) - - patterns: List[Pattern] = [] - - for signature, group in by_tool_seq.items(): - successes = [e for e in group if e.get("verdict") == "success"] - failures = [e for e in group if e.get("verdict") == "failed"] - rate = len(successes) / len(group) if group else 0.0 - - if len(successes) >= MIN_SUPPORT and rate >= SKILL_MIN_SUCCESS_RATE: - patterns.append( - Pattern( - kind=PATTERN_SUCCESS_SUBSEQUENCE, - signature=signature, - support=len(successes), - episode_ids=[e.get("episode_id", "") for e in successes], - success_rate=rate, - tool_sequence=signature.split("→"), - memory_refs=sorted( - { - ref - for e in successes - for ref in (e.get("memory_refs") or []) - } - ), - ) - ) - if len(failures) >= MIN_SUPPORT: - patterns.append( - Pattern( - kind=PATTERN_REPEATED_FAILURE, - signature=signature, - support=len(failures), - episode_ids=[e.get("episode_id", "") for e in failures], - success_rate=rate, - tool_sequence=signature.split("→"), - ) - ) - - for signature, group in by_skill_seq.items(): - if len(group) >= WORKFLOW_MIN_COMBO_FREQUENCY: - successes = [e for e in group if e.get("verdict") == "success"] - patterns.append( - Pattern( - kind=PATTERN_SUCCESS_SUBSEQUENCE, - signature=signature, - support=len(group), - episode_ids=[e.get("episode_id", "") for e in group], - success_rate=len(successes) / len(group), - skill_sequence=signature.split("→"), - ) - ) - - return [p for p in patterns if p.meets_support] - - -# ── Memory → Skill ─────────────────────────────────────────────────────────── - - -@dataclass -class PromotionProposal: - """A promotion the chain wants to make, before it becomes an IR candidate.""" - - from_layer: str - to_layer: str - signature: str - support: int - rationale: str - source_refs: List[str] = field(default_factory=list) - payload: Dict[str, Any] = field(default_factory=dict) - # Set when an equivalent asset already exists: promotion becomes a patch - # rather than a new asset, so the library does not fill with near-duplicates. - patch_target: Optional[str] = None - - def to_dict(self) -> Dict[str, Any]: - return { - "from_layer": self.from_layer, - "to_layer": self.to_layer, - "signature": self.signature, - "support": self.support, - "rationale": self.rationale, - "source_refs": list(self.source_refs), - "payload": self.payload, - "patch_target": self.patch_target, - } - - -def _dominant_shared_intent( - pattern: Pattern, - episodes: Optional[Sequence[Dict[str, Any]]], - *, - support_floor: int, -): - """The one intent behind a sequence's supporting episodes, or ``None``. - - ``None`` is a refusal, and deliberately covers the honest-uncertainty cases - too: no views supplied, request text too short to cluster, no clustering - method available. "We cannot tell whether these conversations had anything - in common" and "they had nothing in common" both mean no SOP claim can be - made — treating the first as a pass is how single-tool coincidences used to - become skills. - """ - if not episodes: - return None - wanted = {str(e) for e in pattern.episode_ids} - supporting = [e for e in episodes if str(e.get("episode_id") or "") in wanted] - if len(supporting) < support_floor: - return None - - from core.evolution.similarity import METHOD_NONE, cluster_intents - - clusters, method = cluster_intents(supporting, min_support=2) - if method == METHOD_NONE or not clusters: - return None - top = clusters[0] - if len(top.episode_ids) < support_floor: - return None - if len(top.episode_ids) / len(supporting) < INTENT_COHESION_RATIO: - return None - # 复现必须跨会话：occasions 按 chat_id 去重，同一会话里反复跑同一套 - # 流程只算一次场合，不构成"这件事总在发生"。 - if top.occasions < support_floor: - return None - return top - - -def promote_tool_sequence_to_skill( - pattern: Pattern, - *, - episodes: Optional[Sequence[Dict[str, Any]]] = None, - min_support: Optional[int] = None, - skill_credit: float = 0.0, - workflow_credit: float = 0.0, - existing_skill_signatures: Optional[Dict[str, str]] = None, -) -> Optional[PromotionProposal]: - """Compile a recurring successful tool sequence into a skill proposal. - - A recurring sequence alone is *not* evidence of a reusable capability. - Generic tools recur across unrelated conversations by construction, and the - limiting case — a whole conversation whose "sequence" is one call to - ``view_text_file`` — says only that the task was trivial. What justifies an - SOP is commonality of *purpose*: the same kind of request, on separate - occasions, repeatedly completed by the same non-trivial procedure. So the - gates hold together, and each excludes a specific way of being wrong: - - * the sequence is **non-trivial** (≥ :data:`MIN_TOOL_SEQUENCE_LEN` steps) — - wrapping one generic tool adds an asset and no knowledge; - * the supporting episodes **share an intent** — a dominant cluster over - their request text (via :mod:`core.evolution.similarity`) covers - ≥ :data:`INTENT_COHESION_RATIO` of them; without that the recurrence is a - coincidence of tooling, and no commonality means no promotion; - * the recurrence spans **separate conversations** — one chat re-running the - same procedure is one occasion, not evidence it keeps coming back; - * the recurrences **succeed** at ≥ :data:`SKILL_MIN_SUCCESS_RATE`. - - ``episodes`` are the flattened views the pattern was discovered from; the - intent gate reads their request text. A caller that cannot supply them gets - a refusal, not the benefit of the doubt. - - Named for what it reads. This was called ``promote_memory_to_skill`` and - carried ``from_layer="memory"``, but no memory content ever enters the - result — ``pattern.memory_refs`` are hashes of whatever happened to be - retrieved during the supporting episodes. The name promised a memory→skill - chain the code did not have; :func:`promote_procedural_memory_to_skill` is - that chain. - - Refuses when orchestration is the better explanation: a stable tool ordering - can equally mean "the workflow is fixed" rather than "there is a reusable - skill here". Promoting in that case creates a skill with no independent - content, which then has to be maintained forever. - """ - if pattern.kind != PATTERN_SUCCESS_SUBSEQUENCE or not pattern.tool_sequence: - return None - support_floor = MIN_SUPPORT if min_support is None else int(min_support) - if len(pattern.tool_sequence) < MIN_TOOL_SEQUENCE_LEN: - logger.info( - "[promotion] skipping %s: %d-step sequence is below the SOP floor", - pattern.signature, - len(pattern.tool_sequence), - ) - return None - if pattern.support < support_floor: - return None - if pattern.success_rate < SKILL_MIN_SUCCESS_RATE: - return None - if workflow_credit > skill_credit: - logger.info( - "[promotion] skipping skill promotion for %s: workflow credit higher", - pattern.signature, - ) - return None - - intent = _dominant_shared_intent(pattern, episodes, support_floor=support_floor) - if intent is None: - logger.info( - "[promotion] skipping %s: supporting episodes share no intent — a " - "recurring sequence without a recurring request is not an SOP", - pattern.signature, - ) - return None - covered = [e for e in pattern.episode_ids if e in set(intent.episode_ids)] - - existing = (existing_skill_signatures or {}).get(pattern.signature) - return PromotionProposal( - from_layer="episode", - to_layer="skill", - signature=pattern.signature, - support=len(covered), - rationale=( - f"「{intent.representative[:40]}」类请求在 {intent.occasions} 次独立会话中" - f"反复出现，每次都以同一 {len(pattern.tool_sequence)} 步工具序列成功完成" - f"（{len(covered)} 个 Episode，成功率 {pattern.success_rate:.0%}）——" - "同类意图 + 稳定做法，具备沉淀为 SOP 的共性" - ), - # Co-retrieved memories are *context*, not sources. They are carried so - # a reviewer can see what was in view, and deliberately not demoted — - # nothing of theirs went into this skill, so down-weighting them would - # remove knowledge the skill does not carry. - source_refs=[], - payload={ - "tool_sequence": pattern.tool_sequence, - "episode_ids": covered[:20], - "intent": intent.representative, - "occasions": intent.occasions, - "co_retrieved_memory_refs": list(pattern.memory_refs), - }, - # An equivalent skill already exists → patch it instead of adding a twin. - patch_target=existing, - ) - - -# Enough separate occasions that "this keeps coming up" is a claim about -# behaviour rather than about one busy afternoon. -PROCEDURAL_MIN_OCCURRENCES = 3 -# Below this the procedure is being followed into failures, and compiling it -# would encode the failure as the house style. -PROCEDURAL_MIN_SUCCESS_RATE = 0.6 - - -def promote_procedural_memory_to_skill( - *, - memories: Sequence[Dict[str, Any]], - intent_cluster: Dict[str, Any], - occurrences: int, - success_rate: float, - plan_variety: int, - existing_skill_id: Optional[str] = None, -) -> Optional[PromotionProposal]: - """Compile *how work is done here* into a skill proposal. - - This is the chain the design described and the code never had. Four - conditions must hold together, and each excludes a specific way of being - wrong: - - * the memories are **procedural** — a fact frozen into a document stops - being current, and a preference is already covered by one profile line; - * the intent recurs across **separate occasions** — one conversation - mentioning something three times is one occasion; - * the recurrences **succeed** — distilling a failing approach makes it the - house style; - * the plans **varied** — the same request handled identically every time is - already effectively a skill, and writing one down adds an asset to - maintain while changing nothing. - - The proposal carries the memories' **content**, which is what makes this a - compilation rather than a rename, and their refs, which is what lets the - source memories be demoted when the skill goes live and restored when it is - rolled back. - """ - procedural = [ - m - for m in memories - if str(m.get("memory_type") or "") == "procedural" and str(m.get("content") or "").strip() - ] - if not procedural: - return None - if occurrences < PROCEDURAL_MIN_OCCURRENCES: - return None - if success_rate < PROCEDURAL_MIN_SUCCESS_RATE: - return None - if plan_variety < 2: - return None - - representative = str(intent_cluster.get("representative") or "") - return PromotionProposal( - from_layer="memory", - to_layer="skill", - signature="proc:" + "|".join(sorted(str(m.get("ref") or "") for m in procedural)), - support=occurrences, - rationale=( - f"「{representative[:40]}」类请求出现 {occurrences} 次、成功率 " - f"{success_rate:.0%}，却用了 {plan_variety} 种不同做法；" - f"其中 {len(procedural)} 条做法/口径已被反复检索，应编译为一次性装配的技能" - ), - source_refs=[str(m.get("ref") or "") for m in procedural if m.get("ref")], - payload={ - "procedures": [ - { - "rule": str(m.get("content") or ""), - "why": str(m.get("why") or ""), - "applies_to": str(m.get("applies_to") or ""), - } - for m in procedural - ], - "episode_ids": [str(e) for e in (intent_cluster.get("episode_ids") or [])][:20], - "intent": representative, - "memory_refs": [str(m.get("ref") or "") for m in procedural if m.get("ref")], - }, - patch_target=existing_skill_id, - ) - - -# Down-weighting the memories a skill was compiled from lives in -# :mod:`core.evolution.memory_apply`, because it is an applied change with an -# undo rather than a list of dictionaries. The version that used to sit here -# returned a plan nothing executed, and its ``restore_on_rollback: True`` was a -# promise no code kept. - - -# ── Skill → Orchestration ──────────────────────────────────────────────────── - - -def promote_skills_to_orchestration( - pattern: Pattern, - *, - combo_success_rate: float, - individual_success_rate: float, - task_types: Optional[Sequence[str]] = None, - skill_versions: Optional[Dict[str, str]] = None, -) -> Optional[PromotionProposal]: - """Promote a recurring skill combination into an **agent profile** proposal. - - Promoting it into a *workflow* — a fixed DAG — was the wrong target: the - main axis is a ReAct loop, which never executes a DAG, so a workflow asset - could only ever govern the separate autonomous-loop product. What a - recurring skill combination actually says is "for this kind of task, these - capabilities belong in view together", and the thing that decides what is in - view is the profile. - - The combination must *beat its parts*: frequency alone would promote every - common sequence regardless of whether grouping them helps. The proposal is - declarative — a skill set and the task types it applies to — and carries no - executable content. - """ - if not pattern.skill_sequence or len(pattern.skill_sequence) < 2: - return None - if pattern.support < WORKFLOW_MIN_COMBO_FREQUENCY: - return None - if combo_success_rate - individual_success_rate < WORKFLOW_MIN_UPLIFT: - return None - - versions = skill_versions or {} - return PromotionProposal( - from_layer="skill", - to_layer="agent_profile", - signature=pattern.signature, - support=pattern.support, - rationale=( - f"技能组合出现 {pattern.support} 次，组合成功率 {combo_success_rate:.0%} " - f"显著高于零散调用 {individual_success_rate:.0%}——" - "应固化为该类任务的技能组合，而不是每次重新挑选" - ), - source_refs=list(pattern.skill_sequence), - payload={ - "skill_ids": list(dict.fromkeys(pattern.skill_sequence)), - "task_types": [str(t) for t in (task_types or [])], - # Declared so the candidate auto-invalidates if a dependency retires - # — a profile pointing at a dead skill version is worse than none. - "depends_on": {s: versions.get(s, "") for s in pattern.skill_sequence}, - "kind": "declarative_policy", - }, - ) - - -def dependencies_satisfied( - proposal: PromotionProposal, live_skill_versions: Dict[str, str] -) -> Tuple[bool, List[str]]: - """Whether every declared dependency still exists at the declared version. - - This is the mechanism that makes cross-layer rollback work in one direction: - retire a skill and every profile built on it stops validating, instead of - quietly pointing at something that is no longer there. - """ - declared = (proposal.payload or {}).get("depends_on") or {} - stale = [ - skill_id - for skill_id, version in declared.items() - if version and live_skill_versions.get(skill_id) != version - ] - return (not stale), stale - - -# ── Downhill flow: overlap, retirement, negative transfer ──────────────────── - - -def find_overlaps( - signatures: Dict[str, List[str]], *, min_shared: int = 2 -) -> List[Tuple[str, str, int]]: - """Assets whose **declared** step sequences overlap enough to be merge candidates. - - ``signatures`` must be each asset's own declared tools — the tools its - document names. Feeding it the tools observed in episodes where the asset - was merely loaded produces a merge proposal for every pair of skills that - happened to be enabled together, which is guaranteed rather than - informative: co-enabled skills share the episode's whole tool set by - construction. - """ - overlaps: List[Tuple[str, str, int]] = [] - asset_ids = sorted(signatures) - for i, a in enumerate(asset_ids): - for b in asset_ids[i + 1 :]: - shared = len(set(signatures[a]) & set(signatures[b])) - if shared >= min_shared: - overlaps.append((a, b, shared)) - return overlaps - - -# How long a skill must have been exposed before "never opened" means anything. -# Retiring on a short window measures release recency, not usefulness — which -# is exactly what the old selection-count rule did, since every enabled skill -# recorded the same count and the only real variable was how recently a skill -# had shipped. -MIN_EXPOSURE_DAYS = 14 -# Offered this many times without ever being opened → the model consistently -# decides it is not relevant. -MIN_OFFERS_BEFORE_JUDGING = 10 -# Opened this many times before follow-through is worth judging. -MIN_OPENS_BEFORE_JUDGING = 3 -# Opened, but the plan afterwards ignored what it said. -FOLLOW_THROUGH_FLOOR = 0.4 -SUCCESS_FLOOR = 0.4 - -REASON_NEVER_OPENED = "never_opened" -REASON_IGNORED_WHEN_OPENED = "ignored_when_opened" -REASON_LOW_SUCCESS = "low_success_when_opened" -REASON_HARMFUL = "worse_than_without" - -# What to do about it. Distinct because the fixes are distinct: a skill nobody -# opens has a discovery problem (its description does not match the requests it -# serves), a skill that gets opened and ignored has a *content* problem, and a -# skill that makes things worse has to stop being exposed now. -ACTION_LOWER_EXPOSURE = "lower_exposure" -ACTION_REWRITE = "rewrite" -ACTION_RETIRE = "retire" -ACTION_ROLLBACK = "rollback" - - -def find_retirement_candidates( - usage: Dict[str, Dict[str, Any]], -) -> List[Dict[str, Any]]: - """Assets that are not earning their place, and what to do about each. - - ``usage`` per asset: - - ``offers`` - turns where the skill was selected into the prompt (non-degraded - selections only). - ``opens`` - turns where the model actually opened its document. This is the usage - signal; ``offers`` alone cannot distinguish "never chosen" from "chosen - and useless". - ``follow_through`` - of the opens, the fraction where the subsequent tool calls stayed - inside what the skill declared. - ``success_rate`` - of the opens, the fraction of turns that succeeded. - ``uplift`` - success with the skill exposed minus success without it, where a paired - comparison exists. ``None`` when it does not. - ``exposure_days`` - how long it has been exposed at all. - - A library that only grows fills with near-duplicates and degrades selection - precision for everything in it — but every judgement here is a *proposal*, - because removing a capability produces no error, only a slow decline. - """ - out: List[Dict[str, Any]] = [] - for asset_id, stats in (usage or {}).items(): - offers = int(stats.get("offers", 0) or 0) - opens = int(stats.get("opens", 0) or 0) - exposure_days = float(stats.get("exposure_days", 0) or 0) - uplift = stats.get("uplift") - - if uplift is not None and float(uplift) <= -0.05: - out.append( - { - "asset_id": asset_id, - "reason": REASON_HARMFUL, - "action": ACTION_ROLLBACK, - "explanation": ( - f"成对比较显示有它反而更差（{float(uplift):+.0%}）——" - "这是唯一需要立即停止曝光的情况" - ), - **stats, - } - ) - continue - - if exposure_days < MIN_EXPOSURE_DAYS: - # Too new to judge. Saying nothing is the honest output; the old - # rule's answer here was "retire it", which punished every skill for - # having just shipped. - continue - - if opens == 0: - if offers < MIN_OFFERS_BEFORE_JUDGING: - continue - out.append( - { - "asset_id": asset_id, - "reason": REASON_NEVER_OPENED, - "action": ACTION_LOWER_EXPOSURE, - "explanation": ( - f"已被挂载 {offers} 次、" - # A hand-authored skill has no release record and so no - # finite exposure window — it has been there from the - # start. Formatting that as a number printed "曝光 inf 天". - + ( - "一直可用" - if exposure_days == float("inf") - else f"曝光 {exposure_days:.0f} 天" - ) - + "，模型一次都没打开——" - "多半是描述没写清它解决什么问题，而不是能力本身没用" - ), - **stats, - } - ) - continue - - if opens < MIN_OPENS_BEFORE_JUDGING: - continue - - follow_through = stats.get("follow_through") - if follow_through is not None and float(follow_through) < FOLLOW_THROUGH_FLOOR: - out.append( - { - "asset_id": asset_id, - "reason": REASON_IGNORED_WHEN_OPENED, - "action": ACTION_REWRITE, - "explanation": ( - f"被打开 {opens} 次，但只有 {float(follow_through):.0%} 的情况" - "后续动作真的照它说的做——文档本身需要重写，不是该退役" - ), - **stats, - } - ) - continue - - success_rate = float(stats.get("success_rate", 0.0) or 0.0) - if success_rate < SUCCESS_FLOOR: - out.append( - { - "asset_id": asset_id, - "reason": REASON_LOW_SUCCESS, - "action": ACTION_RETIRE, - "explanation": ( - f"被打开 {opens} 次、照做了，但成功率只有 {success_rate:.0%}" - ), - **stats, - } - ) - return out - - -@dataclass -class NegativeTransfer: - asset_id: str - task_type: str - before: float - after: float - - @property - def delta(self) -> float: - return self.after - self.before - - def to_dict(self) -> Dict[str, Any]: - return { - "asset_id": self.asset_id, - "task_type": self.task_type, - "before": round(self.before, 4), - "after": round(self.after, 4), - "delta": round(self.delta, 4), - } - - -def detect_negative_transfer( - *, - asset_id: str, - related_task_types: Sequence[str], - before: Dict[str, float], - after: Dict[str, float], - threshold: float = 0.05, -) -> List[NegativeTransfer]: - """Find task types that got worse after an activation *and are unrelated to it*. - - Only unrelated tasks are examined. A dip on the task the change targeted is - a straightforward regression its own guardrail catches; the dangerous case - is collateral damage elsewhere, which no single release's guardrail is - watching and which accumulates silently across many individually-harmless - changes. - """ - related = set(related_task_types) - findings: List[NegativeTransfer] = [] - for task_type, after_value in (after or {}).items(): - if task_type in related: - continue - before_value = (before or {}).get(task_type) - if before_value is None: - continue - if before_value - after_value >= threshold: - findings.append( - NegativeTransfer( - asset_id=asset_id, - task_type=task_type, - before=before_value, - after=after_value, - ) - ) - return findings - - -# ── Transferable ordering constraints ──────────────────────────────────────── - - -@dataclass -class OrderingConstraint: - """A pairwise ordering that held across the supporting evidence. - - Carries the task types it was *validated in* and any it was *contradicted - in*. That distinction is what lets a genuinely context-dependent ordering be - used where it holds instead of being discarded everywhere. - """ - - before: str - after: str - support: int - contexts: int # distinct tool-set shapes it was observed in - validated_in: Tuple[str, ...] = () - contradicted_in: Tuple[str, ...] = () - - def applies_to(self, tools: Sequence[str], task_type: Optional[str] = None) -> bool: - if self.before not in tools or self.after not in tools: - return False - if task_type is None: - # No context given: only a rule contradicted nowhere may be used. - return not self.contradicted_in - if task_type in self.contradicted_in: - return False - if task_type in self.validated_in: - return True - # An unseen context. Extending there is only safe for a rule that has - # never been contradicted; one known to flip somewhere is precisely the - # kind that should not be guessed at. - return not self.contradicted_in - - def to_dict(self) -> Dict[str, Any]: - return { - "before": self.before, - "after": self.after, - "support": self.support, - "contexts": self.contexts, - "validated_in": list(self.validated_in), - "contradicted_in": list(self.contradicted_in), - } - - -# A constraint seen in only one tool-set shape is indistinguishable from "that -# is just how this one sequence goes". Requiring two distinct shapes is what -# separates a transferable rule from a memorised sequence — and it is the whole -# difference between a skill that helps one task family and one that helps -# every family making the same mistake. -MIN_CONSTRAINT_CONTEXTS = 2 - - -def extract_ordering_constraints( - episodes: Sequence[Dict[str, Any]], - *, - min_support: int = MIN_SUPPORT, - min_contexts: int = MIN_CONSTRAINT_CONTEXTS, -) -> List[OrderingConstraint]: - """Find orderings that hold *invariantly* across successful episodes. - - Conflicting evidence vetoes a constraint outright rather than letting the - majority win. A pair that appears both ways round is not a rule — it is a - signal that ordering depends on context the system cannot yet see, and - promoting the popular direction would actively break the minority case. - """ - forward: Dict[Tuple[str, str], Set[str]] = defaultdict(set) - by_task_type: Dict[Tuple[str, str], Set[str]] = defaultdict(set) - - for episode in episodes: - if episode.get("verdict") != "success": - continue - tools = [str(t) for t in (episode.get("tool_sequence") or [])] - if len(tools) < 2: - continue - shape = _signature(sorted(set(tools))) - task_type = str(episode.get("task_type") or "unknown") - for i, earlier in enumerate(tools): - for later in tools[i + 1 :]: - if earlier == later: - continue - forward[(earlier, later)].add(shape) - by_task_type[(earlier, later)].add(task_type) - - constraints: List[OrderingConstraint] = [] - for (a, b), shapes in forward.items(): - validated = by_task_type[(a, b)] - contradicted = by_task_type.get((b, a), set()) - - # A pair that flips *within the same task type* is genuinely ambiguous - # there — no amount of context resolves it, so the rule is dropped for - # that type rather than resolved by majority. - ambiguous = validated & contradicted - validated = validated - ambiguous - if not validated: - continue - - support = sum( - 1 - for e in episodes - if e.get("verdict") == "success" - and a in (e.get("tool_sequence") or []) - and b in (e.get("tool_sequence") or []) - and str(e.get("task_type") or "unknown") in validated - ) - if support < min_support or len(shapes) < min_contexts: - continue - constraints.append( - OrderingConstraint( - before=a, - after=b, - support=support, - contexts=len(shapes), - validated_in=tuple(sorted(validated)), - contradicted_in=tuple(sorted(contradicted | ambiguous)), - ) - ) - - constraints.sort(key=lambda c: (c.contexts, c.support), reverse=True) - return constraints - - -def apply_ordering_constraints( - tools: Sequence[str], - constraints: Sequence[OrderingConstraint], - task_type: Optional[str] = None, -) -> List[str]: - """Reorder a task's tools to satisfy every applicable constraint. - - A stable topological sort: tools with no constraint between them keep their - original relative order, so applying a rule about one pair never silently - reshuffles everything else. - """ - items = list(tools) - applicable = [c for c in constraints if c.applies_to(items, task_type)] - if not applicable: - return items - - for _ in range(len(items)): - changed = False - for constraint in applicable: - i, j = items.index(constraint.before), items.index(constraint.after) - if i > j: - items.insert(j, items.pop(i)) - changed = True - if not changed: - break - return items diff --git a/src/backend/core/evolution/settlement/__init__.py b/src/backend/core/evolution/settlement/__init__.py new file mode 100644 index 00000000..fc4d4e7c --- /dev/null +++ b/src/backend/core/evolution/settlement/__init__.py @@ -0,0 +1 @@ +"""Settle evolution outcomes back into memory.""" diff --git a/src/backend/core/evolution/settlement/memory_service.py b/src/backend/core/evolution/settlement/memory_service.py new file mode 100644 index 00000000..47bd8198 --- /dev/null +++ b/src/backend/core/evolution/settlement/memory_service.py @@ -0,0 +1,39 @@ +"""Evolution implementation of the memory settlement boundary.""" +from core.db.models import ChatMessage, EvolutionEpisode +from core.evolution.settlement import settlement, settlement_runner, settlement_store +from core.memory.settlement_port import register_memory_settlement + + +class EvolutionMemorySettlement: + def report_writes(self, message_id, *, items, failed): + settlement_runner.report_memory_writes(message_id, items=items, failed=failed) + + def update_entry_text(self, message_id, handle, text): + return settlement_store.update_entry_text(message_id, handle, text) + + def build_summary(self, message_id, *, session_factory, items, failed): + settlement_runner.acknowledge_durable_settlement(message_id) + with session_factory() as db: + episode = db.query(EvolutionEpisode.episode_id).filter( + EvolutionEpisode.message_id == message_id + ).first() + return settlement.settle_turn( + message_id=message_id, + episode_id=str(episode[0]) if episode is not None else "", + memory_entries=items, memory_failed=failed, memory_enabled=True, + ).to_dict() + + def write_summary(self, db, message_id, summary): + # Called inside the outbox acknowledgement transaction. Never commit here. + message = db.query(ChatMessage).filter(ChatMessage.message_id == message_id).first() + if message is None: + raise RuntimeError("assistant message is not durable yet; settlement will retry") + if not isinstance(summary, dict): + raise RuntimeError("settlement result is missing its durable summary") + extra = dict(message.extra_data or {}) + extra[settlement_store.METADATA_KEY] = summary + message.extra_data = extra + + +def register() -> None: + register_memory_settlement(EvolutionMemorySettlement()) diff --git a/src/backend/core/evolution/settlement.py b/src/backend/core/evolution/settlement/settlement.py similarity index 100% rename from src/backend/core/evolution/settlement.py rename to src/backend/core/evolution/settlement/settlement.py diff --git a/src/backend/core/evolution/settlement_runner.py b/src/backend/core/evolution/settlement/settlement_runner.py similarity index 88% rename from src/backend/core/evolution/settlement_runner.py rename to src/backend/core/evolution/settlement/settlement_runner.py index 5dbaec71..fb990a70 100644 --- a/src/backend/core/evolution/settlement_runner.py +++ b/src/backend/core/evolution/settlement/settlement_runner.py @@ -36,8 +36,6 @@ # early would report zero writes for a turn that was about to write. MEMORY_WAIT_SECONDS = 45.0 -_lock = threading.Lock() - @dataclass class _PendingTurn: @@ -53,8 +51,6 @@ class _PendingTurn: timer: Optional[threading.Timer] = None -_pending: Dict[str, _PendingTurn] = {} - # Turns already settled. A late duplicate — the watchdog fired and the pipeline # reported afterwards — must not settle a second time: the second pass would # start from an empty entry and overwrite a good summary with an empty one. @@ -70,10 +66,10 @@ def _remember_settled(message_id: str) -> None: def _entry(message_id: str) -> _PendingTurn: - entry = _pending.get(message_id) + entry = process_state().pending.get(message_id) if entry is None: entry = _PendingTurn(message_id=message_id) - _pending[message_id] = entry + process_state().pending[message_id] = entry return entry @@ -87,7 +83,7 @@ def register_turn( if not message_id: return - with _lock: + with process_state().lock: if message_id in _settled: return entry = _entry(message_id) @@ -120,7 +116,7 @@ def report_memory_writes( if not message_id: return - with _lock: + with process_state().lock: if message_id in _settled: return entry = _entry(message_id) @@ -154,8 +150,8 @@ def acknowledge_durable_settlement(message_id: str) -> None: """ if not message_id: return - with _lock: - entry = _pending.pop(message_id, None) + with process_state().lock: + entry = process_state().pending.pop(message_id, None) if entry is not None and entry.timer is not None: entry.timer.cancel() _remember_settled(message_id) @@ -163,8 +159,8 @@ def acknowledge_durable_settlement(message_id: str) -> None: def _settle(message_id: str, *, timed_out: bool = False) -> None: """Build the summary and persist it. Never raises — this runs post-response.""" - with _lock: - entry = _pending.pop(message_id, None) + with process_state().lock: + entry = process_state().pending.pop(message_id, None) if entry is None or message_id in _settled: return # already settled if entry.timer is not None: @@ -182,8 +178,8 @@ def _settle(message_id: str, *, timed_out: bool = False) -> None: ) try: - from core.evolution.settlement import settle_turn - from core.evolution.settlement_store import persist_summary + from core.evolution.settlement.settlement import settle_turn + from core.evolution.settlement.settlement_store import persist_summary summary = settle_turn( message_id=message_id, @@ -198,9 +194,22 @@ def _settle(message_id: str, *, timed_out: bool = False) -> None: def reset_for_tests() -> None: - with _lock: - for entry in _pending.values(): + with process_state().lock: + for entry in process_state().pending.values(): if entry.timer is not None: entry.timer.cancel() - _pending.clear() + process_state().pending.clear() _settled.clear() + + +@dataclass +class SettlementRunnerState: + lock: object = field(default_factory=lambda: threading.Lock()) + pending: Dict[str, _PendingTurn] = field(default_factory=lambda: {}) + + +_STATE = SettlementRunnerState() + + +def process_state() -> SettlementRunnerState: + return _STATE diff --git a/src/backend/core/evolution/settlement_store.py b/src/backend/core/evolution/settlement/settlement_store.py similarity index 98% rename from src/backend/core/evolution/settlement_store.py rename to src/backend/core/evolution/settlement/settlement_store.py index 1b6f26b8..869fbfe4 100644 --- a/src/backend/core/evolution/settlement_store.py +++ b/src/backend/core/evolution/settlement/settlement_store.py @@ -21,7 +21,7 @@ from typing import Any, Dict, Optional from core.db.engine import SessionLocal -from core.evolution.settlement import EvolutionSummary +from core.evolution.settlement.settlement import EvolutionSummary logger = logging.getLogger(__name__) diff --git a/src/backend/core/evolution/skill_gen.py b/src/backend/core/evolution/skill_gen.py deleted file mode 100644 index 7b29e75d..00000000 --- a/src/backend/core/evolution/skill_gen.py +++ /dev/null @@ -1,521 +0,0 @@ -"""Writing a skill from evidence: evidence pack → LLM draft → machine validation. - -What a materialised skill used to be: - - ## 步骤 - 1. `search_web` - 2. `fetch_page` - -A list of tool names, rendered by string concatenation. It states no applicability -condition worth the name, no parameters, no checkpoint for "did this step work", -no failure mode. Measured against a real model, the only part of that document -that carried any benefit was the ordering-constraint section; the step list was a -*harm* source, generalising this skill's order into a task family where the -reverse is correct, 5 times out of 5. - -So the model writes the document. But it writes only the document — the three -stages are separated precisely so that generation never becomes an authority: - -* **Stage A (code)** assembles the evidence: what was asked, what worked, what - failed and where, the ordering rules with their scopes, and the procedural - memories behind the pattern. Deciding *what a skill may be written from* stays - in code. -* **Stage B (model)** drafts SKILL.md under hard constraints stated in the - prompt. -* **Stage C (code)** validates every one of those constraints and rejects the - draft if any fails, with up to :data:`MAX_ATTEMPTS` regeneration attempts - carrying the specific violations back. - -If stage B could widen a tool grant, self-evolution would be a prompt-injection -away from privilege escalation. It cannot: stage C intersects the draft's tools -with the evidence's tools and refuses anything else, and the same document then -goes through the IR's own escalation gate before it can be persisted. -""" - -from __future__ import annotations - -import logging -import re -from dataclasses import dataclass, field -from typing import Any, Dict, List, Optional, Sequence, Tuple - -logger = logging.getLogger(__name__) - -MAX_ATTEMPTS = 2 -# Long enough for steps, checkpoints and failure modes; short enough that a -# skill cannot start competing with the user's own request for the model's -# attention once it is opened. -MAX_BODY_CHARS = 4000 -MIN_BODY_CHARS = 120 -DRAFT_TIMEOUT_S = 60 - -# A skill body must not contain executable content. Self-modifying code is the -# one capability this design refuses outright, and a fenced shell block inside a -# document the agent reads and follows is executable content with extra steps. -_CODE_FENCE = re.compile(r"```(?:\w+)?\s*\n", re.MULTILINE) -_URL = re.compile(r"https?://", re.IGNORECASE) -_FRONTMATTER_NAME = re.compile(r"^[a-z0-9_-]+$") -_TOOL_MENTION = re.compile(r"`([A-Za-z_][A-Za-z0-9_.-]*)`") - -REJECT_NO_FRONTMATTER = "missing_frontmatter" -REJECT_BAD_NAME = "invalid_skill_id" -REJECT_UNKNOWN_TOOL = "tool_not_in_evidence" -REJECT_CODE_BLOCK = "contains_code_block" -REJECT_EXTERNAL_URL = "contains_external_url" -REJECT_TOO_LONG = "body_too_long" -REJECT_TOO_SHORT = "body_too_short" -REJECT_NO_SCOPE = "missing_scope_section" -REJECT_NO_CHECKPOINTS = "missing_checkpoints" -REJECT_EMPTY = "empty_draft" - - -@dataclass -class EvidencePack: - """Everything stage B is allowed to see, and nothing else.""" - - skill_id: str - objectives: List[str] = field(default_factory=list) - tools: List[str] = field(default_factory=list) - successful_sequences: List[List[str]] = field(default_factory=list) - failures: List[Dict[str, Any]] = field(default_factory=list) - ordering_constraints: List[Dict[str, Any]] = field(default_factory=list) - procedures: List[Dict[str, str]] = field(default_factory=list) - task_types: List[str] = field(default_factory=list) - excluded_task_types: List[str] = field(default_factory=list) - support: int = 0 - - def to_dict(self) -> Dict[str, Any]: - return { - "skill_id": self.skill_id, - "objectives": self.objectives[:10], - "tools": list(self.tools), - "successful_sequences": self.successful_sequences[:6], - "failures": self.failures[:6], - "ordering_constraints": self.ordering_constraints, - "procedures": self.procedures[:10], - "task_types": self.task_types, - "excluded_task_types": self.excluded_task_types, - "support": self.support, - } - - -@dataclass -class DraftResult: - ok: bool - body: str = "" - display_name: str = "" - description: str = "" - violations: List[str] = field(default_factory=list) - attempts: int = 0 - # ``True`` when no draft could be produced because the model was unreachable - # — an operational state, not a rejection, and the two must not look alike - # in the console. - generator_unavailable: bool = False - - def to_dict(self) -> Dict[str, Any]: - return { - "ok": self.ok, - "display_name": self.display_name, - "description": self.description, - "violations": list(self.violations), - "attempts": self.attempts, - "generator_unavailable": self.generator_unavailable, - "body_chars": len(self.body), - } - - -# ── Stage A: the evidence pack ─────────────────────────────────────────────── - - -def build_evidence_pack( - *, - skill_id: str, - episodes: Sequence[Dict[str, Any]], - ordering_constraints: Sequence[Dict[str, Any]] = (), - procedures: Sequence[Dict[str, str]] = (), -) -> EvidencePack: - """Assemble what a skill may be written from. - - Failures are included deliberately. A document distilled only from successes - describes a happy path and says nothing about the step where things actually - go wrong, which is the part a reader needs most. - """ - objectives: List[str] = [] - tools: List[str] = [] - sequences: List[List[str]] = [] - failures: List[Dict[str, Any]] = [] - task_types: List[str] = [] - - for episode in episodes: - objective = str(episode.get("objective") or "").strip() - if objective and objective not in objectives: - objectives.append(objective) - task_type = str(episode.get("task_type") or "").strip() - if task_type and task_type not in task_types: - task_types.append(task_type) - - sequence = [str(t) for t in (episode.get("tool_sequence") or [])] - for tool in sequence: - if tool not in tools: - tools.append(tool) - if episode.get("verdict") == "success": - if sequence and sequence not in sequences: - sequences.append(sequence) - elif episode.get("verdict") == "failed": - failures.append( - { - "objective": objective[:160], - "tool_sequence": sequence[:12], - "failed_at": str(episode.get("failed_tool") or ""), - "error": str(episode.get("error") or "")[:200], - } - ) - - excluded = sorted( - { - str(name) - for rule in ordering_constraints - for name in (rule.get("contradicted_in") or []) - if str(name) - } - ) - return EvidencePack( - skill_id=skill_id, - objectives=objectives, - tools=tools, - successful_sequences=sequences, - failures=failures, - ordering_constraints=list(ordering_constraints), - procedures=[dict(p) for p in procedures], - task_types=[t for t in task_types if t not in excluded], - excluded_task_types=excluded, - support=len(episodes), - ) - - -# ── Stage B: the draft ─────────────────────────────────────────────────────── - -_DRAFT_PROMPT = """你在为一个 AI 助手撰写一份「技能文档」(SKILL.md)。这份文档会被模型在遇到同类任务时读取并遵照执行。 - -# 你拿到的证据 - -## 用户实际提出过的请求（{objective_count} 条） -{objectives} - -## 成功执行时用过的工具 -{tools} - -## 成功的调用顺序 -{sequences} - -## 失败的执行（这些是最宝贵的部分，说明坑在哪） -{failures} - -## 已验证的顺序约束 -{constraints} - -## 这些任务里沉淀下来的做法/口径 -{procedures} - -# 硬性要求（违反任何一条，文档会被机器校验打回） - -1. **只能引用上面「成功执行时用过的工具」里出现过的工具名**，一个都不能新增、不能猜。 -2. **开头第一段必须是「## 不适用范围」**，写明本技能不适用于哪些场景。{exclusion_hint} -3. 必须有「## 适用条件」，写清楚**遇到什么样的请求才该用本技能**——要具体到能和别的任务区分开，不能写"当需要用到这些工具时"这种同义反复。 -4. 必须有「## 步骤」，每一步都要写清楚：做什么、**关键参数怎么给**、**怎么判断这一步做对了**（检查点）。 -5. 必须有「## 常见失败与处理」，基于上面的失败证据写。 -6. **禁止**出现任何代码块（``` 包裹的内容）、禁止出现外部链接（http/https）。 -7. 全文不超过 {max_chars} 字符。 -8. 用中文撰写，语气是给同事写操作手册，不要写"本技能由AI生成"之类的话。 - -{retry_hint} - -# 输出格式 - -严格按下面结构输出，不要加任何额外说明，不要用代码块包裹整篇文档： - -DISPLAY_NAME: <一行技能名，≤20字，说明这个技能干什么> -DESCRIPTION: <一行描述，≤80字，说明什么时候该用它——模型只看这一行决定要不要打开本技能> -BODY: -## 不适用范围 -... -## 适用条件 -... -## 步骤 -... -## 常见失败与处理 -... -""" - - -def _format_list(items: Sequence[Any], empty: str = "（无）") -> str: - if not items: - return empty - lines = [] - for item in items: - if isinstance(item, (list, tuple)): - lines.append("- " + " → ".join(str(x) for x in item)) - elif isinstance(item, dict): - lines.append("- " + "；".join(f"{k}: {v}" for k, v in item.items() if v)) - else: - lines.append(f"- {item}") - return "\n".join(lines) - - -def _build_prompt(pack: EvidencePack, violations: Sequence[str]) -> str: - exclusion_hint = ( - f"证据显示本技能在以下任务类型中被推翻，必须写进不适用范围：{'、'.join(pack.excluded_task_types)}。" - if pack.excluded_task_types - else "如果证据没有显示明确的排除场景，就写明本技能只适用于「适用条件」描述的场景，其余场景请忽略本技能。" - ) - retry_hint = "" - if violations: - retry_hint = ( - "# 上一版被打回的原因（必须修正）\n" - + "\n".join(f"- {v}" for v in violations) - + "\n" - ) - return _DRAFT_PROMPT.format( - objective_count=len(pack.objectives), - objectives=_format_list(pack.objectives[:10]), - tools=_format_list([f"`{t}`" for t in pack.tools]), - sequences=_format_list(pack.successful_sequences[:6]), - failures=_format_list(pack.failures[:6]), - constraints=_format_list( - [ - f"先 {c.get('before')} 再 {c.get('after')}" - + ( - f"（在 {'、'.join(str(x) for x in (c.get('validated_in') or []))} 中验证）" - if c.get("validated_in") - else "" - ) - for c in pack.ordering_constraints - if not (c.get("contradicted_in") or []) - ] - ), - procedures=_format_list( - [ - p.get("rule", "") + (f"（原因：{p['why']}）" if p.get("why") else "") - for p in pack.procedures[:10] - ] - ), - exclusion_hint=exclusion_hint, - max_chars=MAX_BODY_CHARS, - retry_hint=retry_hint, - ) - - -def _parse_draft(raw: str) -> Tuple[str, str, str]: - """``(display_name, description, body)`` from the model's reply.""" - display_name = "" - description = "" - body = "" - text = (raw or "").strip() - if text.startswith("```"): - lines = text.split("\n") - text = "\n".join(lines[1:]) - if text.rstrip().endswith("```"): - text = text.rstrip()[:-3] - - body_marker = re.search(r"^BODY:\s*$", text, re.MULTILINE) - head = text[: body_marker.start()] if body_marker else text - if body_marker: - body = text[body_marker.end() :].strip() - - for line in head.splitlines(): - if line.upper().startswith("DISPLAY_NAME:"): - display_name = line.split(":", 1)[1].strip() - elif line.upper().startswith("DESCRIPTION:"): - description = line.split(":", 1)[1].strip() - return display_name, description, body - - -# ── Stage C: validation ────────────────────────────────────────────────────── - - -def validate_draft( - *, - skill_id: str, - display_name: str, - description: str, - body: str, - allowed_tools: Sequence[str], -) -> List[str]: - """Every constraint stage B was told about, checked. Returns violations.""" - violations: List[str] = [] - - if not body.strip(): - return [REJECT_EMPTY] - if not display_name.strip() or not description.strip(): - violations.append(REJECT_NO_FRONTMATTER) - if not _FRONTMATTER_NAME.match(skill_id or ""): - # The loader treats ``name`` as the id and rejects anything outside - # ``[a-z0-9_-]``. A skill that fails this still appears in the capability - # list — that metadata comes from table columns — while loading returns - # nothing, so it is present and inert, which is the hardest failure of - # all to notice. - violations.append(REJECT_BAD_NAME) - - if _CODE_FENCE.search(body): - violations.append(REJECT_CODE_BLOCK) - if _URL.search(body): - violations.append(REJECT_EXTERNAL_URL) - if len(body) > MAX_BODY_CHARS: - violations.append(REJECT_TOO_LONG) - if len(body) < MIN_BODY_CHARS: - violations.append(REJECT_TOO_SHORT) - - if "## 不适用范围" not in body: - violations.append(REJECT_NO_SCOPE) - else: - # The exclusion has to precede anything the model could copy: measured, - # withholding a contradicted rule was not enough, because the step list - # generalises on its own. - scope_at = body.index("## 不适用范围") - earlier_section = re.search(r"^## ", body[:scope_at], re.MULTILINE) - if earlier_section: - violations.append(REJECT_NO_SCOPE) - if "## 常见失败与处理" not in body and "## 步骤" not in body: - violations.append(REJECT_NO_CHECKPOINTS) - - allowed = {str(t) for t in allowed_tools} - mentioned = {m for m in _TOOL_MENTION.findall(body)} - # Only identifiers that look like tool names are checked: prose in backticks - # (a field name, a value) must not be mistaken for a tool grant, and a real - # tool name always appears verbatim in the evidence. - unknown = sorted( - name - for name in mentioned - if name not in allowed and "_" in name and name.islower() - ) - if unknown: - violations.append(f"{REJECT_UNKNOWN_TOOL}: {unknown}") - - return violations - - -def render_skill_markdown( - *, - skill_id: str, - display_name: str, - description: str, - body: str, - support: int, - rationale: str = "", -) -> str: - """Wrap a validated body in the frontmatter the loader requires. - - ``name`` is the skill **id**, not the label — writing the human label there - made every materialised skill unparseable. - """ - provenance = ( - f"本技能由能力进化流水线从 {support} 条历史执行证据中沉淀而成，经机器校验后由人工批准。" - ) - if rationale: - provenance += f"\n\n沉淀依据：{rationale}" - return f"""--- -name: {skill_id} -display_name: {display_name} -description: {description} ---- - -# {display_name} - -{description} - -{body.strip()} - -## 来源 - -{provenance} -""" - - -# ── The three stages, in order ─────────────────────────────────────────────── - - -async def draft_skill(pack: EvidencePack) -> DraftResult: - """Draft and validate, retrying with the specific violations. - - Returns ``ok=False`` rather than raising. A skill that cannot be written to - the required standard must produce no candidate at all: shipping a draft - that failed validation would put exactly the document the constraints exist - to prevent in front of the model. - """ - from core.memory.extractors._base import run_llm_with_prompt - - violations: List[str] = [] - for attempt in range(1, MAX_ATTEMPTS + 1): - prompt = _build_prompt(pack, violations) - raw = await run_llm_with_prompt(prompt, timeout_s=DRAFT_TIMEOUT_S, max_tokens=2400) - if raw is None: - return DraftResult( - ok=False, - violations=["generator_unavailable"], - attempts=attempt, - generator_unavailable=True, - ) - - display_name, description, body = _parse_draft(raw) - violations = validate_draft( - skill_id=pack.skill_id, - display_name=display_name, - description=description, - body=body, - allowed_tools=pack.tools, - ) - if not violations: - return DraftResult( - ok=True, - body=body, - display_name=display_name[:60], - description=description[:200].replace("\n", " "), - attempts=attempt, - ) - logger.info( - "[skill-gen] draft attempt %d rejected for %s: %s", - attempt, pack.skill_id, violations, - ) - - return DraftResult(ok=False, violations=violations, attempts=MAX_ATTEMPTS) - - -async def generate_skill_document( - *, - skill_id: str, - episodes: Sequence[Dict[str, Any]], - ordering_constraints: Sequence[Dict[str, Any]] = (), - procedures: Sequence[Dict[str, str]] = (), - rationale: str = "", -) -> Tuple[Optional[Dict[str, str]], DraftResult]: - """All three stages. Returns ``(document, result)``; document is ``None`` on failure.""" - pack = build_evidence_pack( - skill_id=skill_id, - episodes=episodes, - ordering_constraints=ordering_constraints, - procedures=procedures, - ) - if not pack.tools and not pack.procedures: - return None, DraftResult(ok=False, violations=["no_evidence_to_write_from"]) - - result = await draft_skill(pack) - if not result.ok: - return None, result - - return ( - { - "skill_id": skill_id, - "display_name": result.display_name, - "description": result.description, - "content": render_skill_markdown( - skill_id=skill_id, - display_name=result.display_name, - description=result.description, - body=result.body, - support=pack.support, - rationale=rationale, - ), - "allowed_tools": list(pack.tools), - }, - result, - ) diff --git a/src/backend/core/extensions/__init__.py b/src/backend/core/extensions/__init__.py deleted file mode 100644 index 6d56959a..00000000 --- a/src/backend/core/extensions/__init__.py +++ /dev/null @@ -1,69 +0,0 @@ -"""Optional feature modules that plug into core through registered hooks. - -Core never imports a module's implementation. A module contributes behaviour by -registering hooks from its ``setup()`` at startup; when the module is disabled -it registers nothing and core behaves as if the module were not installed. - -Shared modules are loaded in every edition. Additional modules are an edition concern (``core.extensions.edition``); whether -an installed module is switched on is the module's own decision, usually an env -flag read inside its ``setup()``. -""" - -from __future__ import annotations - -from importlib import import_module - -from core.extensions.registry import ( - AgentProfile, - agent_profile, - channel_routes, - is_enabled, - lifecycle_hooks, - mark_enabled, - register_agent_profile, - register_channel_route, - register_lifecycle, - register_router, - register_sandbox_route, - routers, - sandbox_route, -) -from core.infra.logging import get_logger - -logger = get_logger(__name__) - -_loaded = False - - -def load_extensions() -> None: - """Import shared and edition modules and let it register itself. Idempotent.""" - global _loaded - if _loaded: - return - _loaded = True - from core.extensions.edition import EDITION_EXTENSIONS - - for module_name in ("personal_agent.extension", *EDITION_EXTENSIONS): - module = import_module(module_name) - if module.setup(): - mark_enabled(module.NAME) - logger.info("extension_enabled", extension=module.NAME) - else: - logger.info("extension_disabled", extension=module.NAME) - - -__all__ = [ - "AgentProfile", - "agent_profile", - "channel_routes", - "is_enabled", - "lifecycle_hooks", - "load_extensions", - "register_agent_profile", - "register_channel_route", - "register_lifecycle", - "register_router", - "register_sandbox_route", - "routers", - "sandbox_route", -] diff --git a/src/backend/core/extensions/registry.py b/src/backend/core/extensions/registry.py deleted file mode 100644 index 64669d69..00000000 --- a/src/backend/core/extensions/registry.py +++ /dev/null @@ -1,107 +0,0 @@ -"""Hook tables that optional modules fill in at startup. - -Every table starts empty. Core consults them at its extension points (agent -assembly, sandbox routing, inbound channel messages, app lifecycle, routers), -so an unregistered module leaves those code paths exactly as they were. -""" - -from __future__ import annotations - -from dataclasses import dataclass, field -from typing import Any, Awaitable, Callable, Optional - - -@dataclass(frozen=True) -class AgentProfile: - """A dedicated agent built by a module instead of the ordinary chat assembly. - - Selected by ``AgentRequest.profile``; ``AgentRequest.profile_ref`` carries the - module's own identifier (for example the task the agent works on). - """ - - # (request) -> system prompt; replaces the whole chat prompt assembly. - system_prompt: Callable[[Any], Awaitable[str]] - # (toolkit, request, loader=...) -> None; replaces the native chat tools. - register_tools: Callable[..., None] - # (request) -> extra policy middlewares appended to the stack. - middlewares: Callable[[Any], list] = field(default=lambda request: []) - # Only function tools whose name passes are exposed; None keeps all. - tool_filter: Optional[Callable[[str], bool]] = None - # False: MCP servers come only from the request's explicit list, never catalog defaults. - # False also prevents history and skill dependencies from widening explicit bindings. - inherit_default_mcps: bool = True - # False: the profile's own system prompt lists its skills and how to read them; the - # chat skill section (which points to view_text_file) is left out. - native_skill_prompt: bool = True - - -_enabled: set[str] = set() -_agent_profiles: dict[str, AgentProfile] = {} -_sandbox_routes: list[tuple[str, Callable[[], Any]]] = [] -_channel_routes: list[tuple[Callable[[Any, Any], bool], Callable[..., Awaitable[None]]]] = [] -_lifecycle: list[tuple[Callable[[], Awaitable[None]], Callable[[], Awaitable[None]]]] = [] -_routers: list[tuple[Any, Optional[str]]] = [] - - -def mark_enabled(name: str) -> None: - _enabled.add(name) - - -def is_enabled(name: str) -> bool: - return name in _enabled - - -def register_agent_profile(name: str, profile: AgentProfile) -> None: - _agent_profiles[name] = profile - - -def agent_profile(name: str) -> AgentProfile: - profile = _agent_profiles.get(name) - if profile is None: - raise ValueError(f"Agent profile {name!r} is not enabled") - return profile - - -def register_sandbox_route(session_prefix: str, provider: Callable[[], Any]) -> None: - """Sandbox sessions whose id starts with ``session_prefix`` use ``provider()``.""" - _sandbox_routes.append((session_prefix, provider)) - - -def sandbox_route(session_id: str) -> Optional[Callable[[], Any]]: - for prefix, provider in _sandbox_routes: - if session_id.startswith(prefix): - return provider - return None - - -def register_channel_route( - handles: Callable[[Any, Any], bool], - admit: Callable[..., Awaitable[None]], -) -> None: - """``handles(db, conn)`` claims an inbound bot message; ``admit(conn, msg, adapter)`` takes it.""" - _channel_routes.append((handles, admit)) - - -def channel_routes() -> tuple: - return tuple(_channel_routes) - - -def register_lifecycle( - start: Callable[[], Awaitable[None]], - stop: Callable[[], Awaitable[None]], -) -> None: - """Background work started on each service worker and stopped on shutdown.""" - _lifecycle.append((start, stop)) - - -def lifecycle_hooks() -> tuple: - return tuple(_lifecycle) - - -def register_router(router: Any, policy_key: Optional[str] = None) -> None: - """An API router mounted with the same edition policy as ``EE_ROUTERS`` entries.""" - _routers.append((router, policy_key)) - - -def routers() -> tuple: - return tuple(_routers) diff --git a/src/backend/core/files/__init__.py b/src/backend/core/files/__init__.py new file mode 100644 index 00000000..20c0b545 --- /dev/null +++ b/src/backend/core/files/__init__.py @@ -0,0 +1 @@ +"""File parsing, artifact access, spaces and storage contracts.""" diff --git a/src/backend/core/files/artifact_bytes.py b/src/backend/core/files/artifact_bytes.py new file mode 100644 index 00000000..f2bb4479 --- /dev/null +++ b/src/backend/core/files/artifact_bytes.py @@ -0,0 +1,75 @@ +"""Shared model/file contracts without execution-layer dependencies.""" +from __future__ import annotations +import logging +from typing import Any, Dict, Optional +logger = logging.getLogger(__name__) +def is_image(f: Dict[str, Any]) -> bool: + mime = (f.get("mime_type") or "").lower() + return mime.startswith("image/") + +def download_artifact_bytes( + file_id: str, + fallback_name: str, + log_prefix: str, + user_id: Optional[str] = None, +) -> Optional[bytes]: + """Resolve an artifact's storage key and download its raw bytes. + + Returns None on any failure (missing artifact, storage error, etc.) and + logs at WARNING. Used by the image and xlsx-preview hook branches that + both need raw bytes (not parsed text). + + When ``user_id`` is supplied, the artifact's owner is verified via + :func:`load_artifact_meta` before any storage round-trip. This is the + defense-in-depth gate for hook-level reads: the agent context is built + from ``request.attachments[].file_id`` (client-supplied), and the + backend boundary doesn't re-verify ownership before reaching the + hook. Without this check, a forged ``file_id`` belonging to another + user would have its raw bytes downloaded and silently materialized + into the agent's memory (image base64 or xlsx preview block). The + matching ``fetch_parsed_text`` already does the same check for parsed + text — this brings the binary paths to parity. + + ``user_id=None`` preserves the legacy "trust the caller" behavior + used by ``core.llm.tools.read_artifact_tool._probe_xlsx_sheet_names`` / + ``_probe_pptx_slide_count`` where ``fetch_parsed_text(user_id=...)`` + is the authoritative gate downstream. + """ + from core.files.artifacts.reader import load_artifact_meta, resolve_artifact_storage + from core.infra.exceptions import StorageError + from core.files.storage import get_storage + + if user_id: + meta = load_artifact_meta(file_id, user_id=user_id) + if meta is None: + logger.warning( + "%s: artifact %s denied for user %s (not owner / missing / deleted)", + log_prefix, + file_id, + user_id, + ) + return None + + from core.files.artifacts.local_reference import authorized_reference, is_local_file_ref + if is_local_file_ref(file_id): + item = authorized_reference(file_id, user_id) + if not item: + return None + try: + from pathlib import Path + return Path(item["path"]).read_bytes() + except OSError: + return None + + storage_key, _ = resolve_artifact_storage(file_id, fallback_name) + if not storage_key: + return None + + try: + return get_storage().download_bytes(storage_key) + except StorageError as e: + logger.warning(f"{log_prefix}: storage download failed for {file_id}: {e}") + return None + except Exception as e: + logger.warning(f"{log_prefix}: unexpected download error for {file_id}: {e}") + return None diff --git a/src/backend/core/artifacts/__init__.py b/src/backend/core/files/artifacts/__init__.py similarity index 96% rename from src/backend/core/artifacts/__init__.py rename to src/backend/core/files/artifacts/__init__.py index aa617bbd..26f96740 100644 --- a/src/backend/core/artifacts/__init__.py +++ b/src/backend/core/files/artifacts/__init__.py @@ -1,2 +1 @@ """Artifact storage helpers.""" - diff --git a/src/backend/core/services/artifact_backfill.py b/src/backend/core/files/artifacts/backfill.py similarity index 95% rename from src/backend/core/services/artifact_backfill.py rename to src/backend/core/files/artifacts/backfill.py index ec94cfcc..5e3d12fb 100644 --- a/src/backend/core/services/artifact_backfill.py +++ b/src/backend/core/files/artifacts/backfill.py @@ -3,13 +3,13 @@ from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import Session from core.db.models import Artifact, ChatMessage, ChatSession -from core.content.artifact_refs import extract_file_refs, infer_artifact_type, require_artifact_storage_key -from core.services.artifact_edition import artifact_scope_fields +from core.files.artifacts.refs import extract_file_refs, infer_artifact_type, require_artifact_storage_key +from core.files.edition import artifact_scope_fields logger = logging.getLogger(__name__) _BACKFILL_BATCH = 500 -def _backfill_artifacts_from_messages(user_id: str, db: Session) -> int: +def backfill_artifacts_from_messages(user_id: str, db: Session) -> int: """Scan historical messages for file references not yet in the Artifact table. Source priority depends on whether the message was written under the @@ -99,9 +99,7 @@ def _backfill_artifacts_from_messages(user_id: str, db: Session) -> int: # Pass B — insert what is missing. A chat with no project can only ever # resolve to the empty scope, so skip the lookup entirely for those; it used # to cost one ChatSession round-trip per referenced message. - from core.services.project_scope import ( # local: avoid top-level cycle - project_scope_from_chat_id, - ) + from core.projects.edition import project_scope_from_chat_id rootless_fields = artifact_scope_fields(None) scope_cache: Dict[str, Dict[str, Optional[str]]] = {} diff --git a/src/backend/core/files/artifacts/generated.py b/src/backend/core/files/artifacts/generated.py new file mode 100644 index 00000000..474a9381 --- /dev/null +++ b/src/backend/core/files/artifacts/generated.py @@ -0,0 +1,210 @@ +"""Persist generated artifacts and resolve authorized artifact inputs.""" +import base64 +import logging +from pathlib import Path +from typing import Any, Optional +from core.config.settings import settings + +logger = logging.getLogger(__name__) +MAX_ARTIFACT_FILE_SIZE = settings.sandbox.artifact_max_bytes + +def store_generated_files( + files_data: list[dict[str, Any]], + *, + user_id: Optional[str], + source: str, + extra_metadata: Optional[dict[str, Any]] = None, +) -> list[dict[str, Any]]: + """Persist sidecar-generated files and return normalized artifact refs.""" + if not files_data: + return [] + + try: + from core.files.artifacts.store import save_artifact_bytes + except Exception as exc: + logger.warning("artifact store unavailable for %s: %s", source, exc) + return [] + + refs: list[dict[str, Any]] = [] + for fd in files_data: + content_b64 = fd.get("content_b64", "") + if not content_b64: + continue + + name = str(fd.get("name", "output")).strip() or "output" + mime_type = ( + str(fd.get("mime_type", "application/octet-stream")).strip() + or "application/octet-stream" + ) + try: + content = base64.b64decode(content_b64) + except Exception: + logger.warning("skip invalid base64 artifact payload: %s", name) + continue + + metadata: dict[str, Any] = {"source": source} + if user_id: + metadata["user_id"] = user_id + if extra_metadata: + metadata.update(extra_metadata) + + try: + item = save_artifact_bytes( + content=content, + name=name, + mime_type=mime_type, + extension=Path(name).suffix.lstrip("."), + metadata=metadata, + ) + except Exception as exc: + logger.warning("failed to persist generated artifact %s: %s", name, exc) + continue + + refs.append( + { + "file_id": item["file_id"], + "name": item.get("name", name), + "url": f"/files/{item['file_id']}", + "mime_type": item.get("mime_type", mime_type), + "size": item.get("size", len(content)), + "storage_key": item.get("storage_key"), + } + ) + + return refs + + +def store_generated_file_path( + file_path: str | Path, + *, + name: str, + mime_type: str, + user_id: Optional[str], + source: str, + extra_metadata: Optional[dict[str, Any]] = None, +) -> dict[str, Any] | None: + """Persist a generated file from disk and return a normalized artifact ref.""" + try: + from core.files.artifacts.store import save_artifact_file + except Exception as exc: + logger.warning("artifact store unavailable for %s: %s", source, exc) + return None + + metadata: dict[str, Any] = {"source": source} + if user_id: + metadata["user_id"] = user_id + if extra_metadata: + metadata.update(extra_metadata) + + try: + item = save_artifact_file( + file_path=file_path, + name=name, + mime_type=mime_type, + extension=Path(name).suffix.lstrip("."), + metadata=metadata, + ) + except Exception as exc: + logger.warning("failed to persist generated artifact %s: %s", name, exc) + return None + + return { + "file_id": item["file_id"], + "name": item.get("name", name), + "url": f"/files/{item['file_id']}", + "mime_type": item.get("mime_type", mime_type), + "size": item.get("size", 0), + "storage_key": item.get("storage_key"), + } + + +def resolve_artifact_files( + artifact_refs: dict[str, str], + user_id: str | None, +) -> tuple[dict[str, str] | None, str | None]: + """Resolve artifact: references to base64-encoded binary content. + + Tries DB lookup first, falls back to local artifact store index. + Returns (files_b64_dict, error_message). + """ + if not artifact_refs: + return None, None + + try: + from core.files.artifacts.refs import resolve_artifact_storage_key + from core.files.storage.factory import get_storage + except Exception as exc: + return None, f"artifact 解析依赖不可用: {exc}" + + result: dict[str, str] = {} + storage = get_storage() + + for filename, artifact_id in artifact_refs.items(): + from core.files.artifacts.local_reference import is_local_file_ref + if is_local_file_ref(artifact_id): + from core.files.artifact_bytes import download_artifact_bytes as _download_artifact_bytes + file_bytes = _download_artifact_bytes(artifact_id, filename, "artifact input", user_id=user_id) + if file_bytes is None: + return None, f"artifact '{artifact_id}' 不存在或无权访问" + if len(file_bytes) > MAX_ARTIFACT_FILE_SIZE: + return None, f"artifact 文件 '{filename}' 过大" + result[filename] = base64.b64encode(file_bytes).decode("ascii") + continue + storage_key = None + owner_ok = True + + # Try 1: DB lookup + try: + from core.db.engine import SessionLocal + from core.db.repository import ArtifactRepository + + db = SessionLocal() + try: + repo = ArtifactRepository(db) + art = repo.get_by_id(artifact_id) + if art: + if user_id and art.user_id != user_id: + owner_ok = False + else: + storage_key = resolve_artifact_storage_key(art.artifact_id, art.storage_key) + finally: + db.close() + except Exception as fallback_error: + from core.infra.failures import record_best_effort_failure + record_best_effort_failure('core.llm.tools._tool_helpers:_resolve_artifact_files', fallback_error) + + if not owner_ok: + return None, f"无权访问 artifact '{artifact_id}'" + + # Try 2: local artifact store fallback + if not storage_key: + try: + from core.files.artifacts.store import get_artifact + + item = get_artifact(artifact_id) + if item: + item_user = (item.get("metadata") or {}).get("user_id") + if user_id and item_user and item_user != user_id: + return None, f"无权访问 artifact '{artifact_id}'" + storage_key = item.get("storage_key") + except Exception as fallback_error: + from core.infra.failures import record_best_effort_failure + record_best_effort_failure('core.llm.tools._tool_helpers:_resolve_artifact_files', fallback_error) + + if not storage_key: + return None, f"artifact '{artifact_id}' 不存在或已删除" + + try: + file_bytes = storage.download_bytes(storage_key) + except Exception as exc: + return None, f"artifact 文件 '{filename}' 读取失败: {exc}" + + if len(file_bytes) > MAX_ARTIFACT_FILE_SIZE: + return None, ( + f"artifact 文件 '{filename}' 过大: " + f"{len(file_bytes)} bytes > {MAX_ARTIFACT_FILE_SIZE} bytes" + ) + + result[filename] = base64.b64encode(file_bytes).decode("ascii") + + return result or None, None diff --git a/src/backend/core/content/artifact_key_repair.py b/src/backend/core/files/artifacts/key_repair.py similarity index 100% rename from src/backend/core/content/artifact_key_repair.py rename to src/backend/core/files/artifacts/key_repair.py diff --git a/src/backend/core/artifacts/local_reference.py b/src/backend/core/files/artifacts/local_reference.py similarity index 89% rename from src/backend/core/artifacts/local_reference.py rename to src/backend/core/files/artifacts/local_reference.py index c7dbc103..15dc2ae6 100644 --- a/src/backend/core/artifacts/local_reference.py +++ b/src/backend/core/files/artifacts/local_reference.py @@ -26,16 +26,17 @@ def _project_root(project) -> Path: def project_file_path(path: str, scope, user_id: str | None, session_id: str | None = None) -> str: - from core.llm.tools._paths import to_physical_path + from core.sandbox.workspace.resolution import to_physical_path return to_physical_path(path, user_id, session_id=session_id, scope=scope) def _allows_live_access(path: str, user_id: str, action: str = "read") -> bool: - from core.llm.tool_permissions import CURRENT_PERMISSION_TICKET, resolve_approval_mode + from core.sandbox.permissions.dispatch_models import CURRENT_PERMISSION_TICKET + from core.sandbox.permissions.approval import resolve_approval_mode from core.sandbox.workspace.paths import WORKSPACE from core.sandbox.permissions.evaluator import evaluate_local_path - from core.services.local_grant_service import grants_for_gate, policy_for_gate + from core.harness.registry.local_files import grants_for_gate, policy_for_gate import os verdict = evaluate_local_path( @@ -47,14 +48,14 @@ def _allows_live_access(path: str, user_id: str, action: str = "read") -> bool: platform="windows" if os.name == "nt" else "posix", ) ticket = CURRENT_PERMISSION_TICKET.get() - return verdict.decision == "allow" or bool(ticket and ticket.authorizes_path(path, action)) + return verdict.decision == "allow" or bool(ticket and ticket.active and ticket.authorizes_path(path, action)) def reference_project_file(path: str, *, scope, user_id: str, name: str = "") -> dict: from core.config.local_mode import local_mode_enabled from core.db.engine import SessionLocal - from core.services.project_source import ProjectSourceService - from core.artifacts.store import _record_artifact, _now_iso + from core.projects.source import ProjectSourceService + from core.files.artifacts.store import _record_artifact, _now_iso if not local_mode_enabled() or not scope or not scope.is_local or not user_id: raise HTTPException(400, "仅本机项目支持按文件路径交付") @@ -73,7 +74,7 @@ def reference_project_file(path: str, *, scope, user_id: str, name: str = "") -> raise HTTPException(403, "文件必须位于当前本机项目目录内") if not source.is_file(): raise HTTPException(404, "文件已移动或删除") - from core.llm.tool_permissions import require_local_path_permission + from core.sandbox.permissions.dispatch_models import require_local_path_permission require_local_path_permission(str(source), "read") relative = source.relative_to(root).as_posix() @@ -104,7 +105,7 @@ def resolve_project_reference(item: dict) -> dict | None: """Revalidate the live binding and symlink boundary on every read/open/preview.""" from core.config.local_mode import local_mode_enabled from core.db.engine import SessionLocal - from core.services.project_source import ProjectSourceService + from core.projects.source import ProjectSourceService if not local_mode_enabled(): return None @@ -139,7 +140,7 @@ def is_project_file_path(path: str, scope) -> bool: def authorized_reference(file_id: str, user_id: str | None) -> dict | None: - from core.artifacts.store import get_artifact + from core.files.artifacts.store import get_artifact if not user_id or not is_local_file_ref(file_id): return None @@ -152,8 +153,8 @@ def authorized_reference(file_id: str, user_id: str | None) -> dict | None: def overwrite_reference(file_id: str, user_id: str, content: bytes, db) -> dict: import os import tempfile - from core.services.project_source import ProjectSourceService - from core.services.local_snapshot_service import snapshot + from core.projects.source import ProjectSourceService + from core.harness.registry.local_files import snapshot item = authorized_reference(file_id, user_id) if not item: @@ -193,9 +194,9 @@ def overwrite_reference(file_id: str, user_id: str, content: bytes, db) -> dict: def reference_local_file(path: str, *, scope, user_id: str, session_id: str | None, name: str = "") -> dict: from core.config.local_mode import local_mode_enabled from core.config.settings import settings - from core.llm.tools._paths import workspace_directory - from core.llm.tool_permissions import require_local_path_permission - from core.artifacts.store import _write_record_local, _now_iso + from core.sandbox.workspace.resolution import workspace_directory + from core.sandbox.permissions.dispatch_models import require_local_path_permission + from core.files.artifacts.store import _write_record_local, _now_iso if not local_mode_enabled(): raise HTTPException(400, "仅本机模式支持按文件路径交付") @@ -236,7 +237,7 @@ def reference_local_file(path: str, *, scope, user_id: str, session_id: str | No def resolve_session_reference(item: dict) -> dict | None: - from core.llm.tools._paths import workspace_directory + from core.sandbox.workspace.resolution import workspace_directory meta = item.get("metadata") or {} try: diff --git a/src/backend/core/services/artifact_location.py b/src/backend/core/files/artifacts/location.py similarity index 52% rename from src/backend/core/services/artifact_location.py rename to src/backend/core/files/artifacts/location.py index 4edde2c2..decad0d0 100644 --- a/src/backend/core/services/artifact_location.py +++ b/src/backend/core/files/artifacts/location.py @@ -1,10 +1,8 @@ -"""Logical personal-file locations for already-authorized CE artifacts.""" +"""Logical UI locations for already-authorized artifacts; never expose storage keys.""" from pathlib import PurePosixPath - from fastapi import HTTPException - -from core.db import models +from core.files.edition import artifact_location_scope, artifact_location_root def artifact_location(db, artifact): @@ -12,20 +10,26 @@ def artifact_location(db, artifact): return {"kind": "unfiled", "root_name": None, "relative_path": None} if artifact.deleted_at is not None: raise HTTPException(404, "文件已移动或删除") - folder_id = artifact.user_folder_id + kind, folder_id, folder_model, owner_field, owner_id = artifact_location_scope(artifact) parts, visited = [], set() while folder_id: - if folder_id in visited or len(visited) >= 128: + if folder_id in visited or len(visited) >= 128 or folder_model is None: raise HTTPException(409, "文件位置暂时无法解析") visited.add(folder_id) - folder = db.get(models.UserFolder, folder_id) - if folder is None or folder.deleted_at is not None or folder.user_id != artifact.user_id: + folder = db.get(folder_model, folder_id) + if ( + folder is None + or folder.deleted_at is not None + or getattr(folder, owner_field) != owner_id + ): raise HTTPException(404, "文件位置不可用") parts.append(folder.name) folder_id = folder.parent_folder_id + root_name = artifact_location_root(db, artifact) + # A filename is a display label, never interpreted as an absolute path. name = str(artifact.filename or artifact.title).replace(chr(92), "/").rsplit("/", 1)[-1] return { - "kind": "personal", - "root_name": None, + "kind": kind, + "root_name": root_name, "relative_path": str(PurePosixPath(*reversed(parts), name)), } diff --git a/src/backend/core/content/artifact_reader.py b/src/backend/core/files/artifacts/reader.py similarity index 91% rename from src/backend/core/content/artifact_reader.py rename to src/backend/core/files/artifacts/reader.py index 62923d9b..7c04def2 100644 --- a/src/backend/core/content/artifact_reader.py +++ b/src/backend/core/files/artifacts/reader.py @@ -57,13 +57,14 @@ def resolve_artifact_storage( filename = fallback_filename try: - from core.artifacts.store import get_artifact + from core.files.artifacts.store import get_artifact item = get_artifact(file_id) if item and item.get("storage_key"): storage_key = item["storage_key"] filename = item.get("name") or filename - except Exception: - pass + except Exception as fallback_error: + from core.infra.failures import record_best_effort_failure + record_best_effort_failure('core.files.artifacts.reader:resolve_artifact_storage', fallback_error) if not storage_key: try: @@ -100,14 +101,14 @@ def fetch_parsed_text( if not file_id: return "" - from core.artifacts.local_reference import authorized_reference, is_local_file_ref + from core.files.artifacts.local_reference import authorized_reference, is_local_file_ref if is_local_file_ref(file_id): item = authorized_reference(file_id, user_id) if not item: return "" try: from pathlib import Path - from core.content.file_parser import parse_file + from core.files.content.file_parser import parse_file # Live files must never use the immutable-artifact parsed-text cache. return parse_file(Path(item["path"]).read_bytes(), item["name"]) or "" except (OSError, RuntimeError, ValueError): @@ -139,9 +140,9 @@ def fetch_parsed_text( filename = art.filename or "file" try: - from core.content.file_parser import parse_file + from core.files.content.file_parser import parse_file from core.infra.exceptions import StorageError - from core.storage import get_storage + from core.files.storage import get_storage file_bytes = ( prefetched_bytes @@ -164,7 +165,7 @@ def fetch_parsed_text( return "" if parsed: - from core.content.artifact_summary import build_summary_from_text + from core.files.artifacts.summary import build_summary_from_text art.parsed_text = parsed art.parsed_at = utc_now() @@ -190,7 +191,7 @@ def load_artifact_meta(file_id: str, user_id: Optional[str] = None) -> Optional[ """Return lightweight artifact metadata (no parsed_text), or None if inaccessible.""" if not file_id: return None - from core.artifacts.local_reference import authorized_reference, is_local_file_ref + from core.files.artifacts.local_reference import authorized_reference, is_local_file_ref if is_local_file_ref(file_id): item = authorized_reference(file_id, user_id) if not item: diff --git a/src/backend/core/content/artifact_refs.py b/src/backend/core/files/artifacts/refs.py similarity index 93% rename from src/backend/core/content/artifact_refs.py rename to src/backend/core/files/artifacts/refs.py index 078f7b34..6a4de499 100644 --- a/src/backend/core/content/artifact_refs.py +++ b/src/backend/core/files/artifacts/refs.py @@ -1,7 +1,7 @@ """Artifact reference helpers — pure data utilities over tool-result payloads. Relocated out of ``api/routes/v1/artifacts.py`` so that lower layers -(``core.llm.tools``, ``core.services.artifact_service``, ``routing.*``) can use +(``core.llm.tools``, ``core.files.artifacts.service``, ``routing.*``) can use them without importing an API route module (which previously forced a ``core/routing → api`` upward dependency). The API route now re-exports these. """ @@ -33,7 +33,7 @@ def resolve_artifact_storage_key(file_id: str, storage_key: Optional[str] = None return storage_key try: - from core.artifacts.store import get_artifact + from core.files.artifacts.store import get_artifact item = get_artifact(file_id) if item and item.get("storage_key"): @@ -51,8 +51,8 @@ def require_artifact_storage_key( key = resolve_artifact_storage_key(file_id, storage_key) if (not key or key == f"artifacts/{file_id}") and filename and size: from types import SimpleNamespace - from core.content.artifact_key_repair import verified_legacy_key - from core.storage import get_storage + from core.files.artifacts.key_repair import verified_legacy_key + from core.files.storage import get_storage repaired = verified_legacy_key(SimpleNamespace( artifact_id=file_id, storage_key=f"artifacts/{file_id}", @@ -62,7 +62,7 @@ def require_artifact_storage_key( if not key: raise ValueError(f"Artifact {file_id} has no authoritative storage key") if key == f"artifacts/{file_id}": - from core.storage import get_storage + from core.files.storage import get_storage if not get_storage().exists(key): raise ValueError(f"Artifact {file_id} has an unverified placeholder storage key") diff --git a/src/backend/core/services/artifact_service.py b/src/backend/core/files/artifacts/service.py similarity index 93% rename from src/backend/core/services/artifact_service.py rename to src/backend/core/files/artifacts/service.py index 234753fe..42732f6c 100644 --- a/src/backend/core/services/artifact_service.py +++ b/src/backend/core/files/artifacts/service.py @@ -5,14 +5,14 @@ import uuid from typing import TYPE_CHECKING, Any, Dict, List, Optional -from core.content.artifact_refs import infer_artifact_type, require_artifact_storage_key +from core.files.artifacts.refs import infer_artifact_type, require_artifact_storage_key from core.db.models import Artifact as ArtifactModel from core.db.repository import ArtifactRepository -from core.services.artifact_edition import artifact_scope_fields +from core.files.edition import artifact_scope_fields from sqlalchemy.orm import Session if TYPE_CHECKING: - from core.services.project_scope import ProjectScope + from core.projects.edition import ProjectScope logger = logging.getLogger(__name__) @@ -50,7 +50,7 @@ def store_bytes_as_artifact( check_name(db, user_id, user_folder_id, filename) artifact_id = artifact_id or f"ua_{uuid.uuid4().hex[:16]}" storage_key = artifact_upload_key(user_id, artifact_id, filename) - from core.storage import get_storage + from core.files.storage import get_storage storage_url = get_storage().upload_bytes(content, storage_key) meta: Dict[str, Any] = {"source": source} @@ -143,8 +143,7 @@ def get_artifact(self, artifact_id: str, user_id: str) -> Optional[Dict[str, Any # ── AI-generated artifact persistence (used by chat route + routing) ────────── -# Relocated from ``api/routes/v1/chats/__init__.py`` so lower layers (``core.llm.tool``, -# ``routing.*``) can persist artifacts without importing an API route module. +# Lives in core so tools and routing can persist artifacts without importing an API route. def persist_artifacts( @@ -161,7 +160,7 @@ def persist_artifacts( Strict mode: callers MUST pass only the list of files the agent explicitly pinned via ``pin_to_workspace`` (e.g. the result of - ``core.llm.workspace.get_pinned()``). Intermediate tool outputs that + ``core.llm.runtime.workspace.get_pinned()``). Intermediate tool outputs that the agent did not pin are deliberately NOT persisted — the user's "我的空间" should only show deliverables, not every transient docx produced by `word_replace_text` etc. @@ -179,7 +178,7 @@ def persist_artifacts( workflow.py's finally had already reset the ContextVar, causing project outputs to land in the wrong scope. """ - from core.artifacts.local_reference import is_local_file_ref + from core.files.artifacts.local_reference import is_local_file_ref # Local project cards are live file references, not MySpace storage objects. collected = [a for a in collected if not is_local_file_ref(a.get("file_id", ""))] diff --git a/src/backend/core/artifacts/store.py b/src/backend/core/files/artifacts/store.py similarity index 96% rename from src/backend/core/artifacts/store.py rename to src/backend/core/files/artifacts/store.py index eaec62fd..3e245fb2 100644 --- a/src/backend/core/artifacts/store.py +++ b/src/backend/core/files/artifacts/store.py @@ -20,6 +20,7 @@ import logging import os import shutil +from dataclasses import dataclass, field from datetime import datetime from pathlib import Path from typing import Any, Dict, Optional @@ -30,7 +31,7 @@ # ── Local paths ────────────────────────────────────────────────────────────── # Prefer STORAGE_PATH (usually /app/storage inside the container) to avoid creating # a no-permission directory under /app; fall back to the project root result/ when unset. -_PROJECT_ROOT = Path(__file__).resolve().parents[3] +_PROJECT_ROOT = Path(__file__).resolve().parents[4] _STORAGE_BASE = os.getenv("STORAGE_PATH", "").strip() _BASE_DIR = Path(_STORAGE_BASE).expanduser() if _STORAGE_BASE else (_PROJECT_ROOT / "result") _STORE_DIR = (_BASE_DIR / "artifacts").resolve() @@ -61,7 +62,6 @@ _OSS_LEGACY_INDEX_KEY = "artifacts/_index.json" # Bucket directories already created in this process (see _ensure_bucket). -_known_buckets: set = set() # ── Helper: get the current STORAGE_TYPE ────────────────────────────────── @@ -71,7 +71,7 @@ def _storage_type() -> str: # ── Helper: get the OSS backend (lazy import to avoid circular deps) ──────────────── def _get_oss_storage(): - from core.storage import get_storage + from core.files.storage import get_storage return get_storage() @@ -125,10 +125,10 @@ def _ensure_bucket(path: Path) -> None: syscall repeated thousands of times during a migration. Keyed on the real path so redirecting _STORE_DIR (tests, a moved STORAGE_PATH) still creates it. """ - if path in _known_buckets: + if path in process_state().known_buckets: return path.mkdir(parents=True, exist_ok=True) - _known_buckets.add(path) + process_state().known_buckets.add(path) def _write_record_local(item: Dict[str, Any]) -> str: @@ -271,7 +271,7 @@ def save_artifact_bytes( # fail-safe — returns the input untouched for non-SVG or on any parse error. if "svg" in mime_type.lower() or ext.lower() == "svg": try: - from core.content.svg_fit import fit_svg_viewbox + from core.files.content.svg_fit import fit_svg_viewbox content = fit_svg_viewbox(content) except Exception as exc: # never block a save on the normaliser @@ -410,7 +410,7 @@ def get_artifact(file_id: str) -> Optional[Dict[str, Any]]: return None if (item.get("metadata") or {}).get("source") == "local_project_reference": - from .local_reference import resolve_project_reference + from core.files.artifacts.local_reference import resolve_project_reference return resolve_project_reference(item) @@ -443,3 +443,15 @@ def prepare_records() -> int: if written or _has_records(): return written return restore_records_from_oss() + + +@dataclass +class StoreState: + known_buckets: set = field(default_factory=lambda: set()) + + +_STATE = StoreState() + + +def process_state() -> StoreState: + return _STATE diff --git a/src/backend/core/content/artifact_summary.py b/src/backend/core/files/artifacts/summary.py similarity index 97% rename from src/backend/core/content/artifact_summary.py rename to src/backend/core/files/artifacts/summary.py index 2cadd220..8849a393 100644 --- a/src/backend/core/content/artifact_summary.py +++ b/src/backend/core/files/artifacts/summary.py @@ -15,7 +15,6 @@ import io import logging import os -from typing import Optional logger = logging.getLogger(__name__) @@ -130,7 +129,7 @@ def build_summary(file_bytes: bytes, filename: str, mime_type: str = "") -> str: def _summary_text_document(file_bytes: bytes, filename: str) -> str: """Parse and truncate first 500 chars.""" - from core.content.file_parser import parse_file + from core.files.content.file_parser import parse_file text = parse_file(file_bytes, filename) or "" if not text: @@ -278,8 +277,9 @@ def _summary_pptx(file_bytes: bytes, filename: str) -> str: try: if slide.shapes.title is not None: title = (slide.shapes.title.text or "").strip() - except Exception: - pass + except Exception as fallback_error: + from core.infra.failures import record_best_effort_failure + record_best_effort_failure('core.files.artifacts.summary:_summary_pptx', fallback_error) if not title: title = "（无标题）" titles.append(f" 第 {i+1} 页: {title[:60]}") diff --git a/src/backend/core/services/artifact_text_service.py b/src/backend/core/files/artifacts/text.py similarity index 92% rename from src/backend/core/services/artifact_text_service.py rename to src/backend/core/files/artifacts/text.py index d99240b3..40efa571 100644 --- a/src/backend/core/services/artifact_text_service.py +++ b/src/backend/core/files/artifacts/text.py @@ -9,11 +9,11 @@ from fastapi import HTTPException from sqlalchemy import update -from core.auth.permissions_iface import resolve_artifact_access +from core.auth.edition import resolve_artifact_access from core.db.models import Artifact from core.infra.time import utc_now -from core.services.artifact_edition import artifact_access_metadata -from core.services.project_source import reserve_sqlite_writer +from core.files.edition import artifact_access_metadata +from core.projects.source import reserve_sqlite_writer MAX_TEXT_BYTES = 2 * 1024 * 1024 TEXT_EXTENSIONS = frozenset( @@ -91,7 +91,7 @@ def _document(self, artifact, level, data): } def read(self, file_id, actor): - from core.storage import get_storage + from core.files.storage import get_storage artifact, level = self._authorized(file_id, actor) try: @@ -101,7 +101,7 @@ def read(self, file_id, actor): return self._document(artifact, level, data) def save(self, file_id, actor, content, revision): - from core.storage import get_storage + from core.files.storage import get_storage try: data = content.encode("utf-8") @@ -148,8 +148,9 @@ def save(self, file_id, actor, content, revision): self.db.rollback() try: storage.delete(key) - except Exception: - pass # Immutable orphan can be collected; never delete the last good bytes. + except Exception as fallback_error: + from core.infra.failures import record_best_effort_failure + record_best_effort_failure('core.files.artifacts.text:save', fallback_error) raise HTTPException(409, "文件已被修改，请保留草稿并重新打开文件后合并") self.db.commit() self.db.refresh(artifact) diff --git a/src/backend/core/files/content/__init__.py b/src/backend/core/files/content/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/src/backend/core/files/content/__init__.py @@ -0,0 +1 @@ + diff --git a/src/backend/core/content/archive_browser.py b/src/backend/core/files/content/archive_browser.py similarity index 99% rename from src/backend/core/content/archive_browser.py rename to src/backend/core/files/content/archive_browser.py index 17bc1f18..801b1109 100644 --- a/src/backend/core/content/archive_browser.py +++ b/src/backend/core/files/content/archive_browser.py @@ -10,7 +10,7 @@ import zipfile import zlib -from core.content.archive_limits import check_zip_directory, open_bounded_xz +from core.files.content.archive_limits import check_zip_directory, open_bounded_xz from collections import Counter from contextlib import ExitStack from pathlib import Path, PurePosixPath diff --git a/src/backend/core/content/archive_limits.py b/src/backend/core/files/content/archive_limits.py similarity index 100% rename from src/backend/core/content/archive_limits.py rename to src/backend/core/files/content/archive_limits.py diff --git a/src/backend/core/files/content/branding.py b/src/backend/core/files/content/branding.py new file mode 100644 index 00000000..d4e6e33f --- /dev/null +++ b/src/backend/core/files/content/branding.py @@ -0,0 +1,299 @@ +"""Product branding and page configuration stored as content.""" +from __future__ import annotations +import re +from datetime import datetime, timezone +from typing import Any +from core.config.settings import settings +from core.db.models import ContentBlock +from sqlalchemy.orm import Session +from core.files.content import content_blocks_constants + +def normalize_app_config(payload: Any) -> dict[str, Any]: + """Normalize app_config payload to the canonical `{"apps": [...]}` shape. + + An empty list (admin deleted all apps) is respected as "no apps" and no longer + falls back to any default entries. + """ + if not isinstance(payload, dict): + return {"apps": list(content_blocks_constants.DEFAULT_APP_CONFIG["apps"])} + + apps_raw = payload.get("apps") + if isinstance(apps_raw, list): + apps: list[dict[str, Any]] = [] + for item in apps_raw: + if not isinstance(item, dict): + continue + app_id = str(item.get("id") or "").strip() + if not app_id: + continue + apps.append( + { + "id": app_id, + "enabled": bool(item.get("enabled", True)), + "name": str(item.get("name") or app_id), + "description": str(item.get("description") or ""), + "url": str(item.get("url") or ""), + "icon": str(item.get("icon") or ""), + } + ) + return {"apps": apps} + + return {"apps": list(content_blocks_constants.DEFAULT_APP_CONFIG["apps"])} + + +def default_payload(alias: str) -> Any: + if alias == "page_config": + return content_blocks_constants.DEFAULT_PAGE_CONFIG + if alias == "app_config": + return content_blocks_constants.DEFAULT_APP_CONFIG + if alias == "prompt_versions": + return content_blocks_constants.DEFAULT_PROMPT_VERSIONS + return {} if alias in content_blocks_constants.DICT_PAYLOAD_BLOCKS else [] + + +def seed_page_config_if_missing(db: Session) -> bool: + """Insert default page_config row if it does not yet exist. Returns True if inserted.""" + row = db.query(ContentBlock).filter(ContentBlock.id == "page_config").first() + if row: + return False + db.add( + ContentBlock( + id="page_config", + payload=content_blocks_constants.DEFAULT_PAGE_CONFIG, + updated_at=datetime.now(timezone.utc), + updated_by="system_seed", + ) + ) + db.commit() + return True + + +def enforce_ce_branding(db: Session) -> bool: + """Keep the fixed CE product identity and auth policy consistent. + + ``page_config`` survives image upgrades, so merely changing the code/env + defaults does not repair an already-running CE instance. CE does not expose + the content-management console and therefore has no supported per-instance + branding override or self-service registration; normalize both the small + set of branding fields and the registration switch on startup. EE + deployments are never touched. + """ + if settings.edition.edition != "ce": + return False + + row = db.query(ContentBlock).filter(ContentBlock.id == "page_config").first() + if row is None or not isinstance(row.payload, dict): + return False + + product_name = "HugAgentOS" + payload = dict(row.payload) + branding = dict(payload.get("branding") or {}) + navigation = dict(payload.get("navigation") or {}) + auth = dict(payload.get("auth") or {}) + admin_header = dict(navigation.get("admin_header") or {}) + admin_platform = dict(navigation.get("admin_platform") or {}) + + desired_branding = { + "product_name": product_name, + "product_subtitle": "AI 智能助手", + "page_title": product_name, + "hero_title": f"你好，我是 {product_name}", + } + changed = any(branding.get(key) != value for key, value in desired_branding.items()) + changed = changed or admin_header.get("title") != f"{product_name} — 后台管理" + changed = changed or admin_platform.get("product_name") != product_name + changed = changed or auth.get("allow_register") is not False + if not changed: + return False + + branding.update(desired_branding) + admin_header["title"] = f"{product_name} — 后台管理" + admin_platform["product_name"] = product_name + navigation["admin_header"] = admin_header + navigation["admin_platform"] = admin_platform + auth["allow_register"] = False + payload["branding"] = branding + payload["navigation"] = navigation + payload["auth"] = auth + row.payload = payload + row.updated_at = datetime.now(timezone.utc) + row.updated_by = "system_seed" + from sqlalchemy.orm.attributes import flag_modified + + flag_modified(row, "payload") + db.commit() + return True + + +def backfill_navigation_entries(db: Session) -> int: + """Patch existing page_config row to add navigation entries listed in + ``_NAV_BACKFILL_ENTRIES`` if they are missing. + + Returns the number of *individual fields* mutated (across all entries + and all three locations: sidebar_items / panel_titles / panel_subtitles). + Zero means nothing changed (idempotent steady state). + """ + row = db.query(ContentBlock).filter(ContentBlock.id == "page_config").first() + if not row: + return 0 # seed_page_config_if_missing handles fresh installs + payload = row.payload if isinstance(row.payload, dict) else {} + nav = payload.get("navigation") + if not isinstance(nav, dict): + return 0 # malformed payload — leave it alone, admin UI will surface + + default_nav = content_blocks_constants.DEFAULT_PAGE_CONFIG["navigation"] + + changed = 0 + for entry in content_blocks_constants.NAV_BACKFILL_ENTRIES: + key = entry["key"] + bucket_field = "menu_items" if entry["bucket"] == "menu" else "sidebar_items" + items = nav.get(bucket_field) + # Present in *either* list counts as placed: the operator may have moved the entry + # between buckets on purpose, and re-seeding it would duplicate it across both. + already_placed = any( + key in value + for field in ("sidebar_items", "menu_items") + if isinstance(value := nav.get(field), list) + ) + if isinstance(items, list) and not already_placed: + anchor = entry.get("insert_after") + new_items = list(items) + if anchor and anchor in new_items: + new_items.insert(new_items.index(anchor) + 1, key) + else: + new_items.append(key) + nav[bucket_field] = new_items + changed += 1 + + # Copy comes from DEFAULT_PAGE_CONFIG so it is never re-typed here. + for field in ("panel_titles", "panel_subtitles"): + target = nav.get(field) + text = default_nav[field].get(key) + if isinstance(target, dict) and key not in target and text: + target[key] = text + changed += 1 + + # 旧出厂文案改名（能力中心统一表述），与上面的补齐共用一次提交 + for field, renames in content_blocks_constants.LEGACY_PANEL_TEXT_RENAMES.items(): + target = nav.get(field) + if not isinstance(target, dict): + continue + for key, (legacy_values, new_text) in renames.items(): + if target.get(key) in legacy_values: + target[key] = new_text + changed += 1 + + if changed: + payload["navigation"] = nav + row.payload = payload + row.updated_at = datetime.now(timezone.utc) + # Mark updated_by so audit can distinguish automatic backfill vs manual edits + row.updated_by = "system_seed" + # SQLAlchemy JSONB does not auto-detect mutation — flag explicitly + from sqlalchemy.orm.attributes import flag_modified + + flag_modified(row, "payload") + db.commit() + return changed + + +def get_branding_info() -> dict[str, str]: + """Read the branding fields from page_config (product_name + logo_url), falling back to defaults on failure. + + Reused by server-side rendering paths (e.g. the SSO login page); manages its + own DB session internally. + """ + defaults = { + "product_name": content_blocks_constants.DEFAULT_PAGE_CONFIG["branding"]["product_name"], + "logo_url": content_blocks_constants.DEFAULT_PAGE_CONFIG["branding"]["logo_url"], + } + try: + from core.db.engine import SessionLocal + + db = SessionLocal() + try: + row = db.query(ContentBlock).filter(ContentBlock.id == "page_config").first() + payload = (row.payload if row else {}) or {} + branding = payload.get("branding", {}) if isinstance(payload, dict) else {} + return { + "product_name": branding.get("product_name") or defaults["product_name"], + "logo_url": branding.get("logo_url") or defaults["logo_url"], + } + finally: + db.close() + except Exception: + return defaults + + +def is_register_allowed() -> bool: + """Return whether self-service local-account registration is enabled. + + CE is permanently single-account and therefore always returns ``False``. + Other editions read ``page_config.auth.allow_register`` and keep the legacy + default of ``True`` when the setting is missing or cannot be read. The value + gates both the server-rendered UI and registration submissions. + """ + if settings.edition.edition == "ce": + return False + + try: + from core.db.engine import SessionLocal + + db = SessionLocal() + try: + row = db.query(ContentBlock).filter(ContentBlock.id == "page_config").first() + payload = (row.payload if row else {}) or {} + auth = payload.get("auth", {}) if isinstance(payload, dict) else {} + val = auth.get("allow_register") if isinstance(auth, dict) else None + return True if val is None else bool(val) + finally: + db.close() + except Exception: + return True + + +def get_admin_platform_info() -> dict[str, str]: + """Read page_config.navigation.admin_platform (admin platform brand + per-page labels). + + When admin_platform is missing, derive the brand from the legacy + admin_header.title (shaped like " — 后台管理") as the product_name + fallback, then fall back to code defaults — consistent with the frontend + mergeAdminPlatform normalization. Reused by server-side rendering paths + (Swagger / ReDoc doc page titles etc.); manages its own DB session internally. + """ + ap_def = content_blocks_constants.DEFAULT_PAGE_CONFIG["navigation"]["admin_platform"] + defaults = { + "product_name": ap_def["product_name"], + "content_label": ap_def["content_label"], + "config_label": ap_def["config_label"], + "apidoc_label": ap_def["apidoc_label"], + } + try: + from core.db.engine import SessionLocal + + db = SessionLocal() + try: + row = db.query(ContentBlock).filter(ContentBlock.id == "page_config").first() + payload = (row.payload if row else {}) or {} + nav = payload.get("navigation", {}) if isinstance(payload, dict) else {} + ap = nav.get("admin_platform") if isinstance(nav, dict) else None + ap = ap if isinstance(ap, dict) else {} + header = nav.get("admin_header") if isinstance(nav, dict) else None + derived = "" + if isinstance(header, dict) and isinstance(header.get("title"), str): + derived = re.split(r"\s+[—-]\s+", header["title"])[0].strip() + + def pick(key: str, fallback: str) -> str: + v = ap.get(key) + return v.strip() if isinstance(v, str) and v.strip() else fallback + + return { + "product_name": pick("product_name", derived or defaults["product_name"]), + "content_label": pick("content_label", defaults["content_label"]), + "config_label": pick("config_label", defaults["config_label"]), + "apidoc_label": pick("apidoc_label", defaults["apidoc_label"]), + } + finally: + db.close() + except Exception: + return defaults diff --git a/src/backend/core/files/content/content_blocks.py b/src/backend/core/files/content/content_blocks.py new file mode 100644 index 00000000..d7d2e5d5 --- /dev/null +++ b/src/backend/core/files/content/content_blocks.py @@ -0,0 +1,183 @@ +"""Export and import content blocks and the prompt pool as snapshots.""" +from __future__ import annotations +from datetime import datetime, timezone +from typing import Any, Mapping +from core.db.models import ContentBlock +from sqlalchemy.orm import Session +from core.files.content import branding +from core.files.content import content_blocks_constants + +class ContentSnapshotError(ValueError): + """Raised when a docs content snapshot is invalid.""" + + +def serialize_block(row: ContentBlock | None, alias: str | None = None) -> dict[str, Any]: + return { + "payload": row.payload if row else branding.default_payload(alias or ""), + "updated_at": row.updated_at.isoformat() if row and row.updated_at else None, + "updated_by": row.updated_by if row else None, + } + + +def build_docs_snapshot( + db: Session, + block_map: Mapping[str, str] = content_blocks_constants.DOCS_BLOCK_MAP, +) -> dict[str, Any]: + """Build a portable JSON snapshot for content blocks. + + Defaults to the docs blocks; pass ``block_map=PROMPT_BLOCK_MAP`` (or use + ``build_prompt_snapshot``) to snapshot the system prompt pool instead. + """ + rows = db.query(ContentBlock).filter(ContentBlock.id.in_(list(block_map.values()))).all() + row_map = {row.id: row for row in rows} + + return { + "schema_version": content_blocks_constants.SNAPSHOT_SCHEMA_VERSION, + "exported_at": datetime.now(timezone.utc).isoformat(), + "blocks": { + alias: serialize_block(row_map.get(db_id), alias) for alias, db_id in block_map.items() + }, + } + + +def parse_snapshot_datetime(value: Any) -> datetime | None: + if value in (None, ""): + return None + if not isinstance(value, str): + raise ContentSnapshotError("updated_at must be an ISO datetime string") + try: + return datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError as exc: + raise ContentSnapshotError(f"Invalid datetime: {value}") from exc + + +def normalize_docs_snapshot( + snapshot: Mapping[str, Any], + block_map: Mapping[str, str] = content_blocks_constants.DOCS_BLOCK_MAP, +) -> dict[str, Any]: + """Validate and normalize an incoming content snapshot.""" + if not isinstance(snapshot, Mapping): + raise ContentSnapshotError("Snapshot body must be a JSON object") + + schema_version = snapshot.get("schema_version", content_blocks_constants.SNAPSHOT_SCHEMA_VERSION) + if schema_version != content_blocks_constants.SNAPSHOT_SCHEMA_VERSION: + raise ContentSnapshotError( + f"Unsupported schema_version: {schema_version}. Expected {content_blocks_constants.SNAPSHOT_SCHEMA_VERSION}" + ) + + raw_blocks = snapshot.get("blocks") + if not isinstance(raw_blocks, Mapping): + raise ContentSnapshotError("Snapshot.blocks must be an object") + + unknown_blocks = set(raw_blocks.keys()) - set(block_map.keys()) - content_blocks_constants.RETIRED_BLOCK_ALIASES + if unknown_blocks: + raise ContentSnapshotError( + f"Unknown blocks in snapshot: {', '.join(sorted(unknown_blocks))}" + ) + + normalized_blocks: dict[str, dict[str, Any]] = {} + for alias in block_map: + raw_block = raw_blocks.get(alias) + if raw_block is None: + continue + if not isinstance(raw_block, Mapping): + raise ContentSnapshotError(f"Snapshot block '{alias}' must be an object") + + payload = raw_block.get("payload", branding.default_payload(alias)) + expected_type = dict if alias in content_blocks_constants.DICT_PAYLOAD_BLOCKS else list + if not isinstance(payload, expected_type): + kind = "object" if expected_type is dict else "list" + raise ContentSnapshotError(f"Snapshot block '{alias}'.payload must be a {kind}") + + normalized_blocks[alias] = { + "payload": payload, + "updated_at": parse_snapshot_datetime(raw_block.get("updated_at")), + "updated_by": raw_block.get("updated_by"), + } + + if not normalized_blocks: + raise ContentSnapshotError("Snapshot does not contain any importable content blocks") + + exported_at = snapshot.get("exported_at") + if exported_at not in (None, "") and not isinstance(exported_at, str): + raise ContentSnapshotError("Snapshot.exported_at must be a string") + + return { + "schema_version": content_blocks_constants.SNAPSHOT_SCHEMA_VERSION, + "exported_at": exported_at, + "blocks": normalized_blocks, + } + + +def import_docs_snapshot( + db: Session, + snapshot: Mapping[str, Any], + *, + overwrite: bool = True, + default_updated_by: str | None = None, + block_map: Mapping[str, str] = content_blocks_constants.DOCS_BLOCK_MAP, +) -> dict[str, Any]: + """Import a content snapshot into the current database.""" + normalized = normalize_docs_snapshot(snapshot, block_map=block_map) + imported: list[str] = [] + skipped: list[str] = [] + + for alias, block in normalized["blocks"].items(): + db_id = block_map[alias] + row = db.query(ContentBlock).filter(ContentBlock.id == db_id).first() + if row and not overwrite: + skipped.append(alias) + continue + + updated_at = block["updated_at"] or datetime.now(timezone.utc) + updated_by = block["updated_by"] or default_updated_by + + if row: + row.payload = block["payload"] + row.updated_at = updated_at + row.updated_by = updated_by + else: + row = ContentBlock( + id=db_id, + payload=block["payload"], + updated_at=updated_at, + updated_by=updated_by, + ) + db.add(row) + imported.append(alias) + + db.commit() + + return { + "schema_version": normalized["schema_version"], + "imported": imported, + "skipped": skipped, + "count": len(imported), + "overwrite": overwrite, + } + + +def build_prompt_snapshot(db: Session) -> dict[str, Any]: + """Build a portable snapshot of the system prompt pool (prompt_versions). + + Same format as the docs snapshot, but scoped to PROMPT_BLOCK_MAP so it can + be migrated across environments independently of docs/page config. + """ + return build_docs_snapshot(db, block_map=content_blocks_constants.PROMPT_BLOCK_MAP) + + +def import_prompt_snapshot( + db: Session, + snapshot: Mapping[str, Any], + *, + overwrite: bool = True, + default_updated_by: str | None = None, +) -> dict[str, Any]: + """Import a system prompt pool snapshot (counterpart of build_prompt_snapshot).""" + return import_docs_snapshot( + db, + snapshot, + overwrite=overwrite, + default_updated_by=default_updated_by, + block_map=content_blocks_constants.PROMPT_BLOCK_MAP, + ) diff --git a/src/backend/core/files/content/content_blocks_constants.py b/src/backend/core/files/content/content_blocks_constants.py new file mode 100644 index 00000000..c46a3a41 --- /dev/null +++ b/src/backend/core/files/content/content_blocks_constants.py @@ -0,0 +1,181 @@ +"""Block maps and schema version for content snapshots.""" +from __future__ import annotations +from typing import Any +from core.config.settings import settings + +BRAND_NAME = settings.branding.product_name + + +DOCS_BLOCK_MAP = { + "updates": "docs_updates", + "capabilities": "docs_capabilities", + "prompt_hub": "prompt_hub", + "page_config": "page_config", + "app_config": "app_config", +} + + +RETIRED_BLOCK_ALIASES = {"homepage_shortcuts"} + + +DICT_PAYLOAD_BLOCKS = {"page_config", "prompt_versions", "app_config"} + + +PROMPT_VERSIONS_BLOCK_ID = "prompt_versions" + + +PROMPT_BLOCK_MAP = { + "prompt_versions": PROMPT_VERSIONS_BLOCK_ID, + "prompt_hub": "prompt_hub", +} + + +SNAPSHOT_SCHEMA_VERSION = 1 + + +DEFAULT_PAGE_CONFIG: dict[str, Any] = { + "branding": { + "product_name": BRAND_NAME, + "product_subtitle": "AI 智能助手", + "logo_url": "/home/header.svg", + "favicon_url": "/icon.png", + "page_title": BRAND_NAME, + "hero_title": f"你好，我是 {BRAND_NAME}", + "hero_subtitle": "今天想从哪里开始？", + "disclaimer": "本平台生成内容由AI大模型生成，不构成任何建议；涉及业务决策请以权威信息为准。", + }, + "homepage": { + # The homepage uses the transparent standalone brand mark rather than + # the square sidebar logo, mirroring the lighter Qwen Office hero. + "show_logo": True, + "logo_url": "/icon.png", + "show_suggestions": True, + "suggested_questions": [ + "分析新能源汽车产业链的关键环节与代表企业", + "对比两份产业政策，提炼支持方向和申报条件", + "查询一家企业的基本情况、经营风险和产业链位置", + "从这份材料中提炼核心观点和待办事项", + "分析这组经营数据的趋势、异常和可能原因", + "检索相关政策依据，并给出可核验的来源", + ], + }, + "navigation": { + "panel_titles": { + "ability_center": "能力中心", + "skills": "技能", + "agents": "智能体", + "mcp": "连接器", + "kb": "知识库", + "docs": "更新记录", + "app_center": "应用中心", + "automation": "定时任务", + "sites": "站点", + "projects": "项目", + "lab": "实验室", + "settings": "系统设置", + "my_space": "我的空间", + }, + "panel_subtitles": { + "ability_center": "智能体基础能力管理，包含智能体、技能、连接器与插件", + "skills": "启用/停用技能，并查看详细介绍、输入输出与示例。", + "agents": "选择与启用智能体，并查看其职责边界与路由提示。", + "mcp": "管理连接器服务，并查看其作用范围与可靠性影响。", + "kb": "浏览知识库、查看文档列表，并支持文档内检索。", + "docs": "查看功能更新、能力中心与平台说明。", + "app_center": "基于 AI 能力的场景化智能应用", + "automation": "按计划自动执行任务，也可随时手动触发。在任意对话中描述你想定期做的事，即可快速创建", + "sites": "在对话里描述需求，AI 生成完整网站并一键发布，由平台托管、凭链接即可访问", + "projects": "把对话、文件和指令打包成专属工作空间", + "lab": "AI 能力实验性应用", + "settings": "", + "my_space": "", + }, + "admin_header": { + "title": f"{BRAND_NAME} — 后台管理", + "subtitle": "后台管理", + }, + # Unified branding + per-page labels for the admin platform (content + # management / system config / API docs) — product_name drives the header + # brand and browser tab title of all three admin pages, replacing the old + # admin_header. + "admin_platform": { + "product_name": BRAND_NAME, + "content_label": "内容管理", + "config_label": "系统配置", + "apidoc_label": "接口文档", + }, + # 与前端 utils/pageConfigDefaults.ts 的 DEFAULT_SIDEBAR_ITEMS / DEFAULT_MENU_ITEMS 保持一致。 + # 这只是新装环境的出厂值——已存在的部署以 DB 现有取值为准，由 backfill_navigation_entries + # 补齐新增条目，管理员随后可在 /config「页面配置」里自由编排。 + "sidebar_items": ["ability_center", "automation", "sites", "my_space"], + "menu_items": ["settings", "app_center", "projects", "lab"], + }, + "texts": { + "input_placeholder": "请输入您的问题…", + "input_placeholder_agent": "请输入您的问题…", + "search_placeholder": "搜索对话", + "btn_new_chat": "新建对话", + "btn_logout": "退出", + "history_label": "历史对话", + "sidebar_empty_state": "暂无对话记录", + "search_no_results": "无匹配结果", + "dialog_logout_confirm_title": "确认退出登录？", + "dialog_logout_confirm_content": "退出登录不会丢失任何数据，重新登录后可继续使用。", + "dialog_logout_confirm_ok": "退出登录", + "recommend_banner_text": "", + }, + "defaults": { + # Initial mode on every user login / new chat: + # - chat_mode: 'turbo' / 'fast' / 'medium' / 'high' / 'max' (recommended) + # - thinking_mode: bool (legacy field, fallback compat; true→medium, false→fast) + "chat_mode": "fast", + "thinking_mode": False, + }, + "auth": { + # Whether the login page shows the "register" entry (tab + register subpage). + # When off, the login page keeps only login, and backend register submissions + # are rejected outright (not just hidden on the frontend). + "allow_register": True, + }, +} + + +DEFAULT_APP_CONFIG: dict[str, Any] = { + "apps": [], +} + + +DEFAULT_PROMPT_VERSIONS: dict[str, Any] = { + "active": {}, + "versions": [], +} + + +NAV_BACKFILL_ENTRIES: list[dict[str, Any]] = [ + {"key": "projects", "bucket": "menu", "insert_after": "app_center"}, + {"key": "automation", "bucket": "sidebar", "insert_after": "ability_center"}, + {"key": "sites", "bucket": "sidebar", "insert_after": "automation"}, +] + + +LEGACY_PANEL_TEXT_RENAMES: dict[str, dict[str, tuple[tuple[str, ...], str]]] = { + "panel_titles": { + "skills": (("技能库",), "技能"), + "agents": (("子智能体",), "智能体"), + "plugins": (("插件库",), "插件"), + "mcp": (("MCP工具库", "MCP 工具库", "MCP 工具"), "连接器"), + }, + "panel_subtitles": { + "agents": ( + ("选择与启用子智能体，并查看其职责边界与路由提示。",), + "选择与启用智能体，并查看其职责边界与路由提示。", + ), + "mcp": ( + ( + "管理 MCP 工具服务，并查看其作用范围与可靠性影响。", + "管理 MCP 连接器服务，并查看其作用范围与可靠性影响。", + ), + "管理连接器服务，并查看其作用范围与可靠性影响。", + ), + }, +} diff --git a/src/backend/core/files/content/file_formats.py b/src/backend/core/files/content/file_formats.py new file mode 100644 index 00000000..b5d6748d --- /dev/null +++ b/src/backend/core/files/content/file_formats.py @@ -0,0 +1,33 @@ +"""File extensions the parsers accept.""" +from __future__ import annotations + + +EXT_MAP: dict[str, str] = { + ".pdf": "pdf", + ".docx": "docx", + ".doc": "doc_wps", + ".wps": "doc_wps", + ".txt": "txt", + ".xlsx": "xlsx", + ".xls": "xls", + ".csv": "csv", + ".pptx": "pptx", +} + + +RAW_TEXT_EXTS = { + ".html", ".htm", + ".md", ".markdown", + ".json", ".jsonl", ".ndjson", + ".yaml", ".yml", + ".toml", ".ini", ".cfg", + ".xml", ".svg", + ".log", + ".py", ".js", ".ts", ".tsx", ".jsx", ".sh", ".bash", + ".css", ".scss", + ".tex", ".rst", + ".sql", +} + + +SUPPORTED_EXTENSIONS = list(EXT_MAP.keys()) + sorted(RAW_TEXT_EXTS) diff --git a/src/backend/core/files/content/file_parser.py b/src/backend/core/files/content/file_parser.py new file mode 100644 index 00000000..333fbc57 --- /dev/null +++ b/src/backend/core/files/content/file_parser.py @@ -0,0 +1,41 @@ +"""Dispatch an uploaded file to the parser for its extension.""" +from __future__ import annotations +import os +from typing import Optional +from core.files.content import file_formats +from core.files.content import pdf_parser +from core.files.content import presentation_parser +from core.files.content import spreadsheet_parser +from core.files.content import text_parser +from core.files.content import word_parser + +def parse_file(file_bytes: bytes, filename: str) -> Optional[str]: + """ + Dispatch to the correct parser by file extension. + + Returns extracted text/markdown, or None if the format is unsupported. + Raises RuntimeError with a Chinese message on parse failure. + """ + suffix = os.path.splitext(filename.lower())[1] + kind = file_formats.EXT_MAP.get(suffix) + + if kind == "pdf": + return pdf_parser.parse_pdf(file_bytes, filename) + elif kind == "docx": + return word_parser.parse_docx(file_bytes) + elif kind == "doc_wps": + return word_parser.parse_doc_wps(file_bytes, suffix) + elif kind == "txt": + return text_parser.parse_txt(file_bytes) + elif kind == "xlsx": + return spreadsheet_parser.parse_xlsx(file_bytes) + elif kind == "xls": + return spreadsheet_parser.parse_xls(file_bytes) + elif kind == "csv": + return spreadsheet_parser.parse_csv(file_bytes) + elif kind == "pptx": + return presentation_parser.parse_pptx(file_bytes) + elif suffix in file_formats.RAW_TEXT_EXTS: + return text_parser.parse_raw_text(file_bytes) + else: + return None diff --git a/src/backend/core/content/file_validation.py b/src/backend/core/files/content/file_validation.py similarity index 100% rename from src/backend/core/content/file_validation.py rename to src/backend/core/files/content/file_validation.py diff --git a/src/backend/core/content/office.py b/src/backend/core/files/content/office.py similarity index 100% rename from src/backend/core/content/office.py rename to src/backend/core/files/content/office.py diff --git a/src/backend/core/content/office_preview_cache.py b/src/backend/core/files/content/office_preview_cache.py similarity index 98% rename from src/backend/core/content/office_preview_cache.py rename to src/backend/core/files/content/office_preview_cache.py index fc763d90..f05c7d41 100644 --- a/src/backend/core/content/office_preview_cache.py +++ b/src/backend/core/files/content/office_preview_cache.py @@ -14,7 +14,7 @@ import tempfile import time -from core.content.office import find_libreoffice_binary +from core.files.content.office import find_libreoffice_binary CACHE_BYTES = 256 * 1024 * 1024 CACHE_ENTRIES = 128 diff --git a/src/backend/core/files/content/parser_config.py b/src/backend/core/files/content/parser_config.py new file mode 100644 index 00000000..2d5f4434 --- /dev/null +++ b/src/backend/core/files/content/parser_config.py @@ -0,0 +1,47 @@ +"""Settings for the external file-parsing service.""" +from __future__ import annotations +import os + +def svc_get(key: str, default: str = "") -> str: + """Read from SystemConfigService (DB-first), fall back to env.""" + try: + from core.config.store.system_config import SystemConfigService + val = SystemConfigService.get_instance().get(key) + if val is not None: + return val.strip() + except Exception as fallback_error: + from core.infra.failures import record_best_effort_failure + record_best_effort_failure('core.files.content.parser_config:svc_get', fallback_error) + return (os.getenv(ENV_MAP.get(key, ""), default) or default).strip() + + +def cfg_api_url() -> str: + return svc_get("file_parser.api_url") + + +def cfg_timeout() -> int: + try: + return int(svc_get("file_parser.timeout", "60")) + except ValueError: + return 60 + + +def cfg_parse_params() -> dict: + return { + "lang_list": svc_get("file_parser.lang_list", "ch"), + "backend": svc_get("file_parser.backend", "pipeline"), + "parse_method": svc_get("file_parser.parse_method", "auto"), + "formula_enable": svc_get("file_parser.formula_enable", "true"), + "table_enable": svc_get("file_parser.table_enable", "true"), + } + + +ENV_MAP = { + "file_parser.api_url": "FILE_PARSER_API_URL", + "file_parser.timeout": "FILE_PARSER_TIMEOUT", + "file_parser.lang_list": "FILE_PARSER_LANG_LIST", + "file_parser.backend": "FILE_PARSER_BACKEND", + "file_parser.parse_method": "FILE_PARSER_PARSE_METHOD", + "file_parser.formula_enable": "FILE_PARSER_FORMULA_ENABLE", + "file_parser.table_enable": "FILE_PARSER_TABLE_ENABLE", +} diff --git a/src/backend/core/files/content/pdf_parser.py b/src/backend/core/files/content/pdf_parser.py new file mode 100644 index 00000000..073eabd0 --- /dev/null +++ b/src/backend/core/files/content/pdf_parser.py @@ -0,0 +1,125 @@ +"""Parse PDFs through the external layout service.""" +from __future__ import annotations +import base64 +import binascii +import json +from dataclasses import dataclass, field +from typing import Optional +import requests +from core.files.content import parser_config + +@dataclass +class RichParseResult: + """Layout-aware parse output: markdown plus the media the layout engine cut out. + + ``markdown`` is byte-for-byte what :func:`parse_pdf` returns, so the text path is + unaffected. ``images`` maps the path used inside the markdown (``images/.jpg`` + in a ``![](...)`` link) to the decoded bytes, and ``blocks`` is the parsed + ``content_list`` — one entry per layout block, images carrying ``image_caption`` / + ``bbox`` / ``page_idx``. + """ + + markdown: str + images: dict[str, bytes] = field(default_factory=dict) + blocks: list[dict] = field(default_factory=list) + + +def post_parse(file_bytes: bytes, filename: str, mime: str, *, rich: bool) -> dict: + """POST to the external parse service and return the single document's result dict. + + ``rich`` additionally requests the media and layout payloads. They are opt-in + because the images come back base64-inlined in the JSON body — several MB for an + image-heavy PDF — which the chat-attachment path has no use for. + """ + api_url = parser_config.cfg_api_url() + if not api_url: + raise RuntimeError("FILE_PARSER_API_URL 未配置，无法解析 PDF 文件") + + timeout = parser_config.cfg_timeout() + data = parser_config.cfg_parse_params() + if rich: + data = {**data, "return_images": "true", "return_content_list": "true"} + + try: + resp = requests.post( + api_url, + files={"files": (filename, file_bytes, mime)}, + data=data, + timeout=timeout, + ) + resp.raise_for_status() + except requests.exceptions.Timeout: + raise RuntimeError(f"PDF 解析服务超时（{timeout}s），请稍后重试") + except requests.exceptions.RequestException as e: + raise RuntimeError(f"PDF 解析服务请求失败: {e}") + + results = (resp.json() or {}).get("results", {}) + if not results: + raise RuntimeError("PDF 解析服务返回结果为空") + return results[next(iter(results))] or {} + + +def decode_data_uri(value: str) -> Optional[bytes]: + """Decode one ``images`` entry. The service returns ``data:image/jpeg;base64,...``. + + A bare base64 payload (no data-URI prefix) is accepted too, so a service variant + that omits the prefix still works. Undecodable entries are dropped rather than + failing the whole document. + """ + if not isinstance(value, str) or not value: + return None + payload = value.split(",", 1)[1] if value.startswith("data:") else value + try: + return base64.b64decode(payload, validate=False) + except (binascii.Error, ValueError): + return None + + +def parse_content_list(raw) -> list[dict]: + """Normalise ``content_list`` — the service returns it as a JSON **string**.""" + if isinstance(raw, list): + return [b for b in raw if isinstance(b, dict)] + if isinstance(raw, str) and raw.strip(): + try: + data = json.loads(raw) + except ValueError: + return [] + if isinstance(data, list): + return [b for b in data if isinstance(b, dict)] + return [] + + +def parse_pdf(file_bytes: bytes, filename: str = "file.pdf") -> str: + """Parse PDF via external file-parser API. Returns markdown text.""" + content = post_parse(file_bytes, filename, "application/pdf", rich=False).get("md_content", "") + if not content: + raise RuntimeError("PDF 解析服务返回内容为空") + return content + + +def parse_pdf_rich(file_bytes: bytes, filename: str = "file.pdf") -> RichParseResult: + """Parse PDF keeping the figures the layout engine cut out. + + Same call as :func:`parse_pdf` plus ``return_images`` / ``return_content_list``. + Used by knowledge-base indexing to build image assets; the chat-attachment path + stays on the text-only call. + """ + result = post_parse(file_bytes, filename, "application/pdf", rich=True) + markdown = result.get("md_content", "") + if not markdown: + raise RuntimeError("PDF 解析服务返回内容为空") + + images: dict[str, bytes] = {} + for name, value in (result.get("images") or {}).items(): + decoded = decode_data_uri(value) + if decoded: + # The markdown references the file under an ``images/`` prefix while the + # payload is keyed by bare filename; index both so lookups by either work. + images[str(name)] = decoded + images.setdefault(f"images/{name}", decoded) + + return RichParseResult( + markdown=markdown, + images=images, + blocks=parse_content_list(result.get("content_list")), + ) diff --git a/src/backend/core/files/content/presentation_parser.py b/src/backend/core/files/content/presentation_parser.py new file mode 100644 index 00000000..76a56824 --- /dev/null +++ b/src/backend/core/files/content/presentation_parser.py @@ -0,0 +1,63 @@ +"""Parse PPTX files into markdown.""" +from __future__ import annotations +import io + +def open_pptx(file_bytes: bytes): + """Open a Presentation from bytes; raise RuntimeError with a Chinese + install hint if python-pptx is missing.""" + try: + from pptx import Presentation + except ImportError: + raise RuntimeError( + "python-pptx 未安装，无法解析 PPTX 文件（pip install python-pptx）" + ) + return Presentation(io.BytesIO(file_bytes)) + + +def extract_slide_text(slide) -> str: + # Iterate in author/Z order so the rendered output mirrors what a + # reader sees (title → body → footers), since python-pptx exposes + # shapes in that order on the slide.shapes collection. + parts: list[str] = [] + for shape in slide.shapes: + if not getattr(shape, "has_text_frame", False): + continue + try: + text = shape.text_frame.text + except Exception: + continue + text = (text or "").strip() + if text: + parts.append(text) + return "\n".join(parts) + + +def parse_pptx(file_bytes: bytes) -> str: + """Parse PPTX bytes → markdown with one ``## Slide N`` section per slide.""" + p = open_pptx(file_bytes) + sections: list[str] = [] + for idx, slide in enumerate(p.slides): + body = extract_slide_text(slide) or "(此页无文本)" + sections.append(f"## Slide {idx + 1}\n\n{body}") + if not sections: + raise RuntimeError("PPTX 文件无幻灯片") + return "\n\n".join(sections) + + +def parse_pptx_slide_count(file_bytes: bytes) -> int: + """Cheap probe: return the slide count without extracting any text.""" + return len(open_pptx(file_bytes).slides) + + +def parse_pptx_slide(file_bytes: bytes, slide_index: int) -> str: + """Return only the named slide's text (0-indexed). Raises RuntimeError + on out-of-range index.""" + slides = open_pptx(file_bytes).slides + total = len(slides) + if total == 0: + raise RuntimeError("PPTX 文件无幻灯片") + if slide_index < 0 or slide_index >= total: + raise RuntimeError( + f"slide_index 越界: {slide_index}（共 {total} 页，可选 0..{total - 1}）" + ) + return extract_slide_text(slides[slide_index]) or "(此页无文本)" diff --git a/src/backend/core/files/content/spreadsheet_parser.py b/src/backend/core/files/content/spreadsheet_parser.py new file mode 100644 index 00000000..f263805f --- /dev/null +++ b/src/backend/core/files/content/spreadsheet_parser.py @@ -0,0 +1,332 @@ +"""Parse XLSX files into markdown tables and bounded previews.""" +from __future__ import annotations +import csv +import io +import os +import subprocess +import tempfile +from typing import Optional +from core.files.content.office import find_libreoffice_binary +from core.infra.proc import no_window_kwargs +from core.files.content import text_parser + +def deduplicate_headers(header_raw: list[str]) -> list[str]: + """Disambiguate empty/duplicate header names so {placeholder} rendering + downstream is unambiguous (e.g., two ``""`` columns become ``col``/``col_1``). + """ + seen: dict[str, int] = {} + cleaned: list[str] = [] + for h in header_raw: + base = h if h else "col" + if base in seen: + seen[base] += 1 + cleaned.append(f"{base}_{seen[base]}") + else: + seen[base] = 0 + cleaned.append(base) + return cleaned + + +def rows_to_markdown(header: list[str], data_rows: list[list[str]], + sheet_title: str | None = None) -> str: + """Convert header + data rows into a markdown table string.""" + lines: list[str] = [] + if sheet_title: + lines.append(f"## Sheet: {sheet_title}") + lines.append("") + lines.append("| " + " | ".join(header) + " |") + lines.append("|" + "|".join("---" for _ in header) + "|") + for row in data_rows: + lines.append("| " + " | ".join(row) + " |") + return "\n".join(lines) + + +def open_xlsx(file_bytes: bytes): + """Open a workbook in read-only mode; raise RuntimeError with a Chinese + install hint if openpyxl is missing.""" + try: + import openpyxl + except ImportError: + raise RuntimeError("openpyxl 未安装，无法解析 XLSX 文件（pip install openpyxl）") + return openpyxl.load_workbook(io.BytesIO(file_bytes), read_only=True, data_only=True) + + +def xlsx_sheet_to_markdown(sheet) -> str | None: + """Render one openpyxl sheet to a markdown table, or None if empty. + Returns None (not raise) so callers can decide whether emptiness is an + error (single-sheet mode) or just a skip (whole-workbook mode).""" + rows = list(sheet.iter_rows(values_only=True)) + if not rows: + return None + header = [str(c or "") for c in rows[0]] + data_rows = [ + [str(c or "") for c in row] + for row in rows[1:] + if not all(c is None for c in row) + ] + return rows_to_markdown(header, data_rows, sheet.title) + + +def parse_xlsx(file_bytes: bytes) -> str: + """Parse XLSX bytes → markdown tables (one per sheet).""" + wb = open_xlsx(file_bytes) + try: + sections = [md for sheet in wb.worksheets if (md := xlsx_sheet_to_markdown(sheet))] + finally: + wb.close() + if not sections: + raise RuntimeError("XLSX 文件无有效数据") + return "\n\n".join(sections) + + +def parse_xlsx_sheet_names(file_bytes: bytes) -> list[str]: + """Return the list of sheet names in workbook order. Cheap probe: does + not iterate any rows, just reads workbook structure.""" + wb = open_xlsx(file_bytes) + try: + return list(wb.sheetnames) + finally: + wb.close() + + +def parse_xlsx_single_sheet(file_bytes: bytes, sheet_name: str) -> str: + """Parse only the named sheet → markdown table. Raises RuntimeError if + the sheet does not exist or is empty.""" + wb = open_xlsx(file_bytes) + try: + if sheet_name not in wb.sheetnames: + raise RuntimeError( + f"sheet 不存在: '{sheet_name}'。可选 sheet: {wb.sheetnames}" + ) + md = xlsx_sheet_to_markdown(wb[sheet_name]) + if md is None: + raise RuntimeError(f"sheet '{sheet_name}' 无数据") + return md + finally: + wb.close() + + +def default_xlsx_max_cell_chars() -> int: + """Bound spreadsheet preview cells; the default remains 2000 characters.""" + try: + return max(50, int(os.getenv("XLSX_PREVIEW_MAX_CELL_CHARS", "2000"))) + except ValueError: + return 2000 + + +def parse_xlsx_preview( + file_bytes: bytes, + *, + char_budget: int = 4000, + max_columns: int = 40, + max_cell_chars: Optional[int] = None, +) -> dict: + """Parse XLSX into a bounded markdown preview + accurate total counts. + + Produces what the main agent's file_context hook should inject for an + uploaded spreadsheet: a markdown table containing as many rows as fit + within ``char_budget``, plus the *real* total_rows / total_columns so + the agent knows the full size without seeing every row. + + The total_rows count walks the entire sheet (cheap — read_only mode + streams the first column only). The preview stops as soon as the + cumulative markdown length exceeds ``char_budget``. + + Returns: + ``{ + "header": list[str], # column names (post-dedup) + "preview_md": str, # markdown table, ≤ char_budget chars + "total_rows": int, # all data rows in sheet 1 + "total_columns": int, # raw column count (before max_columns) + "preview_rows": int, # rows actually included in preview_md + "truncated_columns": int, # cols dropped to fit max_columns + "truncated_cells": int, # cells truncated to max_cell_chars + }`` + + Reads only the first sheet. + """ + if max_cell_chars is None: + max_cell_chars = default_xlsx_max_cell_chars() + + try: + import openpyxl + except ImportError: + raise RuntimeError("openpyxl 未安装，无法解析 XLSX 文件（pip install openpyxl）") + + wb = openpyxl.load_workbook(io.BytesIO(file_bytes), read_only=True, data_only=True) + try: + sheet = wb.worksheets[0] if wb.worksheets else None + if sheet is None: + raise RuntimeError("XLSX 文件无 sheet") + + rows_iter = sheet.iter_rows(values_only=True) + try: + header_row = next(rows_iter) + except StopIteration: + raise RuntimeError("XLSX 文件无数据行") from None + + total_columns = len(header_row) + truncated_columns = max(0, total_columns - max_columns) + usable_cols = min(total_columns, max_columns) + + header = deduplicate_headers([ + str(c).strip() if c is not None else f"col{idx + 1}" + for idx, c in enumerate(header_row[:usable_cols]) + ]) + + # Markdown header + separator (always present, counts toward budget) + md_lines: list[str] = [ + "| " + " | ".join(header) + " |", + "|" + "|".join("---" for _ in header) + "|", + ] + running_chars = sum(len(ln) + 1 for ln in md_lines) # +1 for newline + + preview_rows = 0 + truncated_cells = 0 + total_rows_seen = 0 + budget_reached = False + + for row in rows_iter: + total_rows_seen += 1 + if all(c is None for c in row): + continue + + if budget_reached: + continue # keep counting total_rows accurately + + cells: list[str] = [] + for val in row[:usable_cols]: + if val is None: + cells.append("") + continue + s = str(val).replace("|", "\\|").replace("\n", " ") + if len(s) > max_cell_chars: + s = s[:max_cell_chars] + "…" + truncated_cells += 1 + cells.append(s) + + # Pad short rows so the markdown table stays well-formed + while len(cells) < len(header): + cells.append("") + + line = "| " + " | ".join(cells) + " |" + line_cost = len(line) + 1 + if running_chars + line_cost > char_budget and preview_rows > 0: + # We have at least one row already; respect the budget. + budget_reached = True + continue + md_lines.append(line) + running_chars += line_cost + preview_rows += 1 + + return { + "header": header, + "preview_md": "\n".join(md_lines), + "total_rows": total_rows_seen, + "total_columns": total_columns, + "preview_rows": preview_rows, + "truncated_columns": truncated_columns, + "truncated_cells": truncated_cells, + } + finally: + wb.close() + + +def parse_xls(file_bytes: bytes) -> str: + """Parse legacy XLS bytes → markdown tables. + + Tries xlrd first; falls back to LibreOffice → XLSX → openpyxl. + """ + try: + import xlrd + wb = xlrd.open_workbook(file_contents=file_bytes) + sections: list[str] = [] + for sheet in wb.sheets(): + if sheet.nrows == 0: + continue + header = [str(sheet.cell_value(0, c) or "") for c in range(sheet.ncols)] + data_rows = [] + for r in range(1, sheet.nrows): + row = [str(sheet.cell_value(r, c) or "") for c in range(sheet.ncols)] + if all(v == "" for v in row): + continue + data_rows.append(row) + sections.append(rows_to_markdown(header, data_rows, sheet.name)) + if not sections: + raise RuntimeError("XLS 文件无有效数据") + return "\n\n".join(sections) + except ImportError: + pass # xlrd not installed, try LibreOffice fallback + + # Fallback: LibreOffice → XLSX → openpyxl + xlsx_bytes = convert_xls_to_xlsx(file_bytes) + return parse_xlsx(xlsx_bytes) + + +def convert_xls_to_xlsx(file_bytes: bytes) -> bytes: + """Use LibreOffice headless to convert XLS → XLSX.""" + with tempfile.TemporaryDirectory() as tmpdir: + input_path = os.path.join(tmpdir, "input.xls") + with open(input_path, "wb") as f: + f.write(file_bytes) + + try: + result = subprocess.run( + [ + find_libreoffice_binary() or "libreoffice", + "--headless", + "--convert-to", "xlsx", + "--outdir", tmpdir, + input_path, + ], + capture_output=True, + text=True, + timeout=120, + **no_window_kwargs(), + ) + except FileNotFoundError: + raise RuntimeError( + "xlrd 和 LibreOffice 均不可用，无法解析 XLS 文件" + "（pip install xlrd 或安装 LibreOffice）" + ) + except subprocess.TimeoutExpired: + raise RuntimeError("LibreOffice 转换 XLS → XLSX 超时") + + output_path = os.path.join(tmpdir, "input.xlsx") + if not os.path.exists(output_path): + raise RuntimeError(f"LibreOffice 转换 XLS 失败: {result.stderr[:300]}") + + with open(output_path, "rb") as f: + return f.read() + + +def parse_csv(file_bytes: bytes) -> str: + """Parse CSV bytes → markdown table.""" + # Decode with encoding detection + text = text_parser.parse_txt(file_bytes) + + # Sniff dialect + try: + sample = text[:8192] + dialect = csv.Sniffer().sniff(sample) + except csv.Error: + dialect = csv.excel + + reader = csv.reader(io.StringIO(text), dialect) + rows = list(reader) + + if not rows: + raise RuntimeError("CSV 文件无有效数据") + + header = [c.strip() for c in rows[0]] + data_rows = [] + for row in rows[1:]: + if all(c.strip() == "" for c in row): + continue + # Pad or trim to match header length + padded = [c.strip() for c in row] + while len(padded) < len(header): + padded.append("") + data_rows.append(padded[:len(header)]) + + return rows_to_markdown(header, data_rows) diff --git a/src/backend/core/content/svg_fit.py b/src/backend/core/files/content/svg_fit.py similarity index 97% rename from src/backend/core/content/svg_fit.py rename to src/backend/core/files/content/svg_fit.py index 86ca9c1e..b4f436d5 100644 --- a/src/backend/core/content/svg_fit.py +++ b/src/backend/core/files/content/svg_fit.py @@ -142,8 +142,9 @@ def _walk(el, tx: float, ty: float, font_size: float, acc: _Bounds) -> None: w = _f(el, "width"); h = _f(el, "height") if w or h: acc.add(tx + x, ty + y, tx + x + w, ty + y + h) - except Exception: # never let one weird node abort the whole fit - pass + except Exception as fallback_error: # never let one weird node abort the whole fit + from core.infra.failures import record_best_effort_failure + record_best_effort_failure('core.files.content.svg_fit:_walk', fallback_error) for child in el: _walk(child, tx, ty, cur_fs, acc) diff --git a/src/backend/core/files/content/text_parser.py b/src/backend/core/files/content/text_parser.py new file mode 100644 index 00000000..a98766ef --- /dev/null +++ b/src/backend/core/files/content/text_parser.py @@ -0,0 +1,21 @@ +"""Decode plain-text files.""" +from __future__ import annotations + + +def parse_txt(file_bytes: bytes) -> str: + """Decode text file bytes, trying UTF-8 then GBK then latin-1.""" + for enc in ("utf-8", "gbk", "latin-1"): + try: + return file_bytes.decode(enc) + except UnicodeDecodeError: + continue + return file_bytes.decode("utf-8", errors="replace") + + +def parse_raw_text(file_bytes: bytes) -> str: + """UTF-8 decode for plain-text formats (HTML/MD/code/etc.). + + No structural extraction—just the source bytes as a string. Replaces + undecodable bytes so we always return something usable. + """ + return file_bytes.decode("utf-8", errors="replace") diff --git a/src/backend/core/files/content/word_parser.py b/src/backend/core/files/content/word_parser.py new file mode 100644 index 00000000..6b0e3917 --- /dev/null +++ b/src/backend/core/files/content/word_parser.py @@ -0,0 +1,77 @@ +"""Parse DOCX, DOC and WPS files into markdown.""" +from __future__ import annotations +import os +import subprocess +import tempfile +from core.files.content.office import find_libreoffice_binary, find_pandoc_binary +from core.infra.proc import no_window_kwargs + +def docx_bytes_to_markdown(docx_bytes: bytes) -> str: + """Convert DOCX bytes to markdown string via pandoc.""" + with tempfile.TemporaryDirectory() as tmpdir: + input_path = os.path.join(tmpdir, "input.docx") + with open(input_path, "wb") as f: + f.write(docx_bytes) + + try: + result = subprocess.run( + [find_pandoc_binary() or "pandoc", input_path, "-t", "markdown", "--wrap=none"], + capture_output=True, + text=True, + timeout=60, + **no_window_kwargs(), + ) + except FileNotFoundError: + raise RuntimeError("pandoc 未安装，无法解析 Word 文档") + except subprocess.TimeoutExpired: + raise RuntimeError("pandoc 转换超时") + + if result.returncode != 0: + raise RuntimeError(f"pandoc 转换失败: {result.stderr[:300]}") + + return result.stdout + + +def parse_docx(file_bytes: bytes) -> str: + """Parse DOCX bytes → markdown via pandoc.""" + return docx_bytes_to_markdown(file_bytes) + + +def convert_to_docx_bytes(file_bytes: bytes, suffix: str) -> bytes: + """Use LibreOffice headless to convert DOC/WPS bytes → DOCX bytes.""" + with tempfile.TemporaryDirectory() as tmpdir: + input_path = os.path.join(tmpdir, f"input{suffix}") + with open(input_path, "wb") as f: + f.write(file_bytes) + + try: + result = subprocess.run( + [ + find_libreoffice_binary() or "libreoffice", + "--headless", + "--convert-to", "docx", + "--outdir", tmpdir, + input_path, + ], + capture_output=True, + text=True, + timeout=120, + **no_window_kwargs(), + ) + except FileNotFoundError: + raise RuntimeError("LibreOffice 未安装，无法解析 DOC/WPS 文件") + except subprocess.TimeoutExpired: + raise RuntimeError("LibreOffice 转换超时") + + output_path = os.path.join(tmpdir, "input.docx") + if not os.path.exists(output_path): + raise RuntimeError(f"LibreOffice 转换失败: {result.stderr[:300]}") + + with open(output_path, "rb") as f: + return f.read() + + +def parse_doc_wps(file_bytes: bytes, suffix: str) -> str: + """Parse DOC/WPS via LibreOffice → DOCX → pandoc markdown.""" + docx_bytes = convert_to_docx_bytes(file_bytes, suffix) + return docx_bytes_to_markdown(docx_bytes) diff --git a/src/backend/core/files/edition.py b/src/backend/core/files/edition.py new file mode 100644 index 00000000..7f51dc7b --- /dev/null +++ b/src/backend/core/files/edition.py @@ -0,0 +1,98 @@ +"""Personal ownership, mounts and artifact policy for Community Edition.""" +from __future__ import annotations + +def project_directory(scope): + return "" + +def project_directory_by_id(project_id): + return "" + +def workspace_instructions(): + return "" + +def is_organization_path(scope, actor, path): + return False + +def resolve_alias(path): + return None + +def resolve_project_path(path, user_id, root, scope): + return None + +def is_shared_scope(scope): + return False + +def context_directory(context): + return "" + +def source_directory(project): + return "" + +def folder_scope_label(kind): + return "我的空间" + +def command_instructions(scope): + return "" + +from typing import Any, Optional + +def artifact_list_scope() -> str: + return "personal" + +def extend_artifact_item(artifact, item: dict) -> dict: + return item + +def personal_artifact_create_fields() -> dict[str, None]: + return {} + +def artifact_scope_fields(scope: Any) -> dict[str, Optional[str]]: + return { + "user_folder_id": ( + scope.root_folder_id or None if scope is not None and scope.is_personal else None + ) + } + +def artifact_scope_folder_id(scope: Any, artifact: Any) -> Optional[str]: + return artifact.user_folder_id + +def can_access_artifact(db: Any, user_id: str, artifact: Any) -> bool: + return str(artifact.user_id) == str(user_id) + +def artifact_access_metadata(artifact: Any) -> dict[str, Optional[str]]: + return {"owner_id": artifact.user_id} + +def can_access_artifact_metadata(db: Any, user_id: str, metadata: dict[str, Any]) -> bool: + owner_id = metadata.get("owner_id") or metadata.get("user_id") + return owner_id is None or str(owner_id) == str(user_id) + +def upload_target_matches(artifact: Any, scope: str, folder_id: Optional[str]) -> bool: + return scope == "personal" and artifact.user_folder_id == folder_id + +workspace_mount_names = ("myspace", "skills") + +def team_grants(db, actor): + return () + +def session_entrypoint(entrypoint, user_id): + return list(entrypoint) + +def project_location(db, project): + return None + +def matches_scope(project, scope): + return scope.kind == project.kind and scope.root_folder_id == project.linked_folder_id + +def owner_space(column, owner): + raise ValueError("Unsupported directory owner") + +def revoke_computers(actor): + return None + + +def artifact_location_scope(artifact): + from core.db.models import UserFolder + return "personal", artifact.user_folder_id, UserFolder, "user_id", artifact.user_id + + +def artifact_location_root(db, artifact): + return None diff --git a/src/backend/core/services/folder_batch.py b/src/backend/core/files/folder_batch.py similarity index 69% rename from src/backend/core/services/folder_batch.py rename to src/backend/core/files/folder_batch.py index 7196e419..81d0c382 100644 --- a/src/backend/core/services/folder_batch.py +++ b/src/backend/core/files/folder_batch.py @@ -1,47 +1,12 @@ """Create a bounded manifest in one transaction; edition models are supplied by callers.""" -import hashlib import uuid -from threading import Lock from core.db.repository import AuditLogRepository -from sqlalchemy import event, or_ +from sqlalchemy import or_ from sqlalchemy.orm import Session -_SQLITE_LOCKS = [Lock() for _ in range(64)] - - -def _release_sqlite_locks(session, transaction): - if transaction.parent is None: - for lock in session.info.pop("folder_upload_locks", []): - lock.release() - - -def lock_folder_owner(db: Session, owner_model, owner_column: str, owner_id: str) -> None: - if db.get_bind().dialect.name == "sqlite": - # SQLite ignores FOR UPDATE. Its supported single-process deployment still - # serves requests on multiple threads, so retain a bounded process lock - # until commit/rollback/close, including early-return and error paths. - if not db.in_transaction(): - db.begin() - if not db.info.get("folder_upload_lock_listener"): - event.listen(db, "after_transaction_end", _release_sqlite_locks) - db.info["folder_upload_lock_listener"] = True - digest = hashlib.sha256((owner_model.__tablename__ + ":" + owner_id).encode()).digest() - lock = _SQLITE_LOCKS[int.from_bytes(digest[:2], "big") % len(_SQLITE_LOCKS)] - held = db.info.setdefault("folder_upload_locks", []) - if lock not in held: - lock.acquire() - held.append(lock) - # NO KEY UPDATE does not conflict with FK KEY SHARE taken by artifact inserts. - owner = ( - db.query(owner_model) - .filter(getattr(owner_model, owner_column) == owner_id) - .with_for_update(key_share=True) - .first() - ) - if owner is None: - raise ValueError("文件夹所属空间不存在") +from core.db.repository.folder import lock_folder_owner def create_folder_batch( @@ -104,10 +69,10 @@ def create_folder_batch( siblings[(parent_id, name)] = row mapping[path] = row.folder_id db.flush() - from core.spaces.files import SpaceFiles - from core.spaces.projects import folder_path - from core.spaces.edition_access import owner_space - from core.storage import get_storage + from core.files.spaces.files import SpaceFiles + from core.files.spaces.projects import folder_path + from core.files.edition import owner_space + from core.files.storage import get_storage if owner_column == "user_id": if owner_id != actor: raise ValueError("Directory owner does not match the account") diff --git a/src/backend/core/services/personal_file_operations.py b/src/backend/core/files/personal_operations.py similarity index 86% rename from src/backend/core/services/personal_file_operations.py rename to src/backend/core/files/personal_operations.py index dd356c3e..6cfaaff8 100644 --- a/src/backend/core/services/personal_file_operations.py +++ b/src/backend/core/files/personal_operations.py @@ -1,14 +1,14 @@ """Save personal conversation attachments into mounted directories.""" from core.db.models import Artifact from core.infra.time import utc_now -from core.services.artifact_edition import is_personal_artifact -from core.spaces.files import SpaceFiles -from core.storage import get_storage +from core.db.artifact_scope import is_personal_artifact +from core.files.spaces.files import SpaceFiles +from core.files.storage import get_storage class PersonalFileOperations: def _save_artifact(self, artifact_id, target_folder_id, actor, *, move): - from core.services.user_folder_service import FolderResult + from core.projects.user_folders import FolderResult artifact = self.db.query(Artifact).filter( Artifact.artifact_id == artifact_id, Artifact.deleted_at.is_(None), ).first() @@ -18,7 +18,7 @@ def _save_artifact(self, artifact_id, target_folder_id, actor, *, move): if target_folder_id and target is None: return FolderResult(False, "目标文件夹不存在") name = artifact.filename - from core.storage.entries import validate_key + from core.files.storage.entries import validate_key try: validate_key(name) if "/" in name: diff --git a/src/backend/core/spaces/__init__.py b/src/backend/core/files/spaces/__init__.py similarity index 100% rename from src/backend/core/spaces/__init__.py rename to src/backend/core/files/spaces/__init__.py diff --git a/src/backend/core/spaces/access.py b/src/backend/core/files/spaces/access.py similarity index 95% rename from src/backend/core/spaces/access.py rename to src/backend/core/files/spaces/access.py index 7b3dacb6..0c44f5cb 100644 --- a/src/backend/core/spaces/access.py +++ b/src/backend/core/files/spaces/access.py @@ -6,7 +6,7 @@ import re from fastapi import HTTPException -from core.storage.entries import validate_key +from core.files.storage.entries import validate_key def identifier(value: str) -> str: if not isinstance(value, str) or not re.fullmatch(r"[A-Za-z0-9_-]{1,128}", value): @@ -49,7 +49,7 @@ def resolve_grants(db, actor: str) -> tuple[SpaceGrant, ...]: actor = identifier(actor) if actor.startswith("api_ak_"): return () - from core.spaces.edition_access import team_grants + from core.files.edition import team_grants return (personal_grant(actor), *team_grants(db, actor)) def require_grant(db, actor: str, space_id: str, *, write=False) -> SpaceGrant: diff --git a/src/backend/core/spaces/files.py b/src/backend/core/files/spaces/files.py similarity index 96% rename from src/backend/core/spaces/files.py rename to src/backend/core/files/spaces/files.py index 122dc31d..07a7a937 100644 --- a/src/backend/core/spaces/files.py +++ b/src/backend/core/files/spaces/files.py @@ -1,9 +1,9 @@ """File operations within an authenticated space grant.""" from fastapi import HTTPException -from core.spaces.access import require_grant -from core.spaces.leases import authorized_operation -from core.spaces.search import search_directory -from core.spaces.tree import ensure_parents, tree_entries, copy_tree, remove_tree +from core.files.spaces.access import require_grant +from core.files.spaces.leases import authorized_operation +from core.files.spaces.search import search_directory +from core.files.spaces.tree import ensure_parents, tree_entries, copy_tree, remove_tree MAX_TRANSFER_BYTES = 50 * 1024 * 1024 diff --git a/src/backend/core/spaces/leases.py b/src/backend/core/files/spaces/leases.py similarity index 86% rename from src/backend/core/spaces/leases.py rename to src/backend/core/files/spaces/leases.py index ced7dde5..a4d98850 100644 --- a/src/backend/core/spaces/leases.py +++ b/src/backend/core/files/spaces/leases.py @@ -1,6 +1,8 @@ """Serialize file access, sandbox creation and permission changes per account.""" +from __future__ import annotations from contextlib import contextmanager, asynccontextmanager import asyncio +from dataclasses import dataclass, field from datetime import timedelta from functools import wraps import hashlib @@ -9,9 +11,6 @@ from fastapi import HTTPException from sqlalchemy import text -_local_guard = threading.Lock() -_local_locks = {} - @contextmanager def space_lock(identity, *, engine=None): @@ -25,8 +24,8 @@ def space_lock(identity, *, engine=None): connection.execute(text("SELECT pg_advisory_xact_lock(:key)"), {"key": key}) yield else: - with _local_guard: - lock = _local_locks.setdefault((str(engine.url), key), threading.Lock()) + with process_state().local_guard: + lock = process_state().local_locks.setdefault((str(engine.url), key), threading.Lock()) if not lock.acquire(timeout=30): raise HTTPException(503, "文件权限正在更新，请重试") try: @@ -37,7 +36,7 @@ def space_lock(identity, *, engine=None): def revoke_sandboxes(actor): from core.config.settings import settings - from core.spaces.edition_revocation import revoke_computers + from core.files.edition import revoke_computers revoke_computers(actor) if settings.sandbox.provider != "opensandbox": return @@ -100,3 +99,16 @@ async def async_space_lock(identity): yield finally: await asyncio.to_thread(guard.__exit__, None, None, None) + + +@dataclass +class LeasesState: + local_guard: object = field(default_factory=lambda: threading.Lock()) + local_locks: object = field(default_factory=lambda: {}) + + +_STATE = LeasesState() + + +def process_state() -> LeasesState: + return _STATE diff --git a/src/backend/core/spaces/mounts.py b/src/backend/core/files/spaces/mounts.py similarity index 78% rename from src/backend/core/spaces/mounts.py rename to src/backend/core/files/spaces/mounts.py index e6d836ba..dc7c0c5d 100644 --- a/src/backend/core/spaces/mounts.py +++ b/src/backend/core/files/spaces/mounts.py @@ -3,8 +3,8 @@ import base64 from pathlib import Path from fastapi import HTTPException -from core.spaces.access import fingerprint, resolve_grants -from core.storage.entries import validate_key +from core.files.spaces.access import fingerprint, resolve_grants +from core.files.storage.entries import validate_key MOUNT_REVISION_TAG = "hugagent-space-revision" @@ -31,14 +31,12 @@ def mount_revision(grants): def provider_volume(grant): from opensandbox.models.sandboxes import Host, OSSFS, Volume from core.config.settings import settings - from core.storage import get_storage backend = os.getenv("STORAGE_TYPE", "local").lower() name = "space-" + fingerprint((grant,))[:24] if backend == "local": root = settings.storage.root.resolve() path = root / grant.prefix - get_storage().make_directory(grant.prefix) if path.is_symlink() or not path.resolve().is_relative_to(root): raise HTTPException(503, "Invalid local space root") host_root = settings.sandbox.opensandbox_host_storage_path @@ -58,8 +56,6 @@ def provider_volume(grant): key_prefix = os.getenv("OSS_KEY_PREFIX", "").strip("/") if backend == "oss" else "" path = "/".join(part for part in (key_prefix, grant.prefix.rstrip("/")) if part) validate_key(path) - if grant.writable: - get_storage().make_directory(grant.prefix) options = ["hugagent_driver=s3fs", "allow_other", "uid=1000", "gid=1000", "umask=0022", "stat_cache_expire=0", "attr_timeout=0", "entry_timeout=0", "negative_timeout=0"] @@ -79,9 +75,21 @@ def prepare_volumes(actor): from core.db.engine import SessionLocal with SessionLocal() as db: grants = resolve_grants(db, actor) - revision = mount_revision(grants) - volumes = [provider_volume(grant) for grant in grants] - db.expire_all() - if mount_revision(resolve_grants(db, actor)) != revision: - raise HTTPException(409, "Space permissions changed while preparing volumes") - return volumes, {MOUNT_REVISION_TAG: revision} + return [provider_volume(grant) for grant in grants], {MOUNT_REVISION_TAG: mount_revision(grants)} + + +def provision_volumes(actor): + """Initialize directory roots once, at sandbox creation.""" + from core.files.storage import get_storage + volumes, metadata = prepare_volumes(actor) + # Grants are represented by the authorized volume specifications. Resolve + # writable prefixes once for initialization, then verify the same revision. + from core.db.engine import SessionLocal + with SessionLocal() as db: + grants = resolve_grants(db, actor) + if mount_revision(grants) != metadata[MOUNT_REVISION_TAG]: + raise HTTPException(409, "Space permissions changed before provisioning") + for grant in grants: + if grant.writable or os.getenv("STORAGE_TYPE", "local").lower() == "local": + get_storage().make_directory(grant.prefix) + return volumes, metadata diff --git a/src/backend/core/spaces/projects.py b/src/backend/core/files/spaces/projects.py similarity index 94% rename from src/backend/core/spaces/projects.py rename to src/backend/core/files/spaces/projects.py index a191fa79..0f5b3fca 100644 --- a/src/backend/core/spaces/projects.py +++ b/src/backend/core/files/spaces/projects.py @@ -3,8 +3,8 @@ import mimetypes from fastapi import HTTPException from core.db.models import UserFolder -from core.spaces.access import namespace -from core.storage.entries import validate_key +from core.files.spaces.access import namespace +from core.files.storage.entries import validate_key def folder_path(db, model, owner_filter, folder_id): @@ -28,7 +28,7 @@ def project_location(db, project): if project.kind == "personal" and project.linked_folder_id: path = folder_path(db, UserFolder, {"user_id": project.owner_user_id}, project.linked_folder_id) return "personal", path, namespace("users", project.owner_user_id) + path + "/" - from core.spaces.edition_access import project_location as edition_location + from core.files.edition import project_location as edition_location location = edition_location(db, project) if location is not None: return location diff --git a/src/backend/core/spaces/recovery.py b/src/backend/core/files/spaces/recovery.py similarity index 94% rename from src/backend/core/spaces/recovery.py rename to src/backend/core/files/spaces/recovery.py index 027753e4..b7604d6f 100644 --- a/src/backend/core/spaces/recovery.py +++ b/src/backend/core/files/spaces/recovery.py @@ -5,8 +5,8 @@ import httpx from fastapi import HTTPException -from core.spaces.leases import async_space_lock -from core.spaces.mounts import MOUNT_REVISION_TAG, prepare_volumes +from core.files.spaces.leases import async_space_lock +from core.files.spaces.mounts import MOUNT_REVISION_TAG, prepare_volumes EPOCH_TAG = "hugagent-mount-epoch" diff --git a/src/backend/core/spaces/sandbox.py b/src/backend/core/files/spaces/sandbox.py similarity index 73% rename from src/backend/core/spaces/sandbox.py rename to src/backend/core/files/spaces/sandbox.py index 652eb804..7940d6c4 100644 --- a/src/backend/core/spaces/sandbox.py +++ b/src/backend/core/files/spaces/sandbox.py @@ -3,8 +3,8 @@ from fastapi import HTTPException from core.sandbox.processes.dispatch import _on_service_loop -from core.spaces.access import resolve_grants -from core.spaces.mounts import MOUNT_REVISION_TAG, mount_revision +from core.files.spaces.access import resolve_grants +from core.files.spaces.mounts import MOUNT_REVISION_TAG, mount_revision def current_revision(actor): @@ -14,14 +14,14 @@ def current_revision(actor): class SpaceSandboxMixin: - from core.spaces.edition_access import workspace_mount_names + from core.files.edition import workspace_mount_names def _session_entrypoint(self, user_id): - from core.spaces.edition_access import session_entrypoint + from core.files.edition import session_entrypoint return session_entrypoint(super()._session_entrypoint(user_id), user_id) async def _create_jupyter_sandbox(self, *, from_snapshot_id=None, user_id=None): - from core.spaces.leases import async_space_lock + from core.files.spaces.leases import async_space_lock if not user_id: return await super()._create_jupyter_sandbox(from_snapshot_id=from_snapshot_id) async with async_space_lock(user_id): @@ -35,7 +35,7 @@ async def _grants_match(self, session): if (info.metadata or {}).get(MOUNT_REVISION_TAG) != revision: return False from core.config.settings import settings - from core.spaces.recovery import ensure_mounts + from core.files.spaces.recovery import ensure_mounts await ensure_mounts(session.sandbox, session.user_id, info, domain=settings.sandbox.opensandbox_domain, api_key=settings.sandbox.opensandbox_api_key) @@ -55,18 +55,10 @@ async def _resolve_bound_session(self, session_id, user_id=None): async def ensure_user_workspace(self, session_id, user_id): await self._get_or_create_session(session_id, user_id) - @_on_service_loop - async def write_stdin( - self, session_id, *, sandbox_session_id, user_id=None, chars="", yield_time_ms=60000 - ): - await self._get_or_create_session(sandbox_session_id, user_id) - return await super().write_stdin( - session_id, sandbox_session_id=sandbox_session_id, user_id=user_id, - chars=chars, yield_time_ms=yield_time_ms, - ) - async def _renew_in_background(self, session_id, session): - if session.user_id and not await self._grants_match(session): + if session.user_id and (await asyncio.to_thread(current_revision, session.user_id)) != ( + (await session.sandbox.get_info()).metadata or {} + ).get(MOUNT_REVISION_TAG): await self._destroy_session(session_id, session, reuse_sandbox=False) return await super()._renew_in_background(session_id, session) diff --git a/src/backend/core/spaces/search.py b/src/backend/core/files/spaces/search.py similarity index 97% rename from src/backend/core/spaces/search.py rename to src/backend/core/files/spaces/search.py index f3a36346..02784067 100644 --- a/src/backend/core/spaces/search.py +++ b/src/backend/core/files/spaces/search.py @@ -5,7 +5,7 @@ import time from fastapi import HTTPException -from core.storage.entries import DirectoryPage, page_limit +from core.files.storage.entries import DirectoryPage, page_limit BATCH_SIZE = 1000 SCAN_SECONDS = 20 diff --git a/src/backend/core/spaces/text.py b/src/backend/core/files/spaces/text.py similarity index 92% rename from src/backend/core/spaces/text.py rename to src/backend/core/files/spaces/text.py index f31337f8..f3cae529 100644 --- a/src/backend/core/spaces/text.py +++ b/src/backend/core/files/spaces/text.py @@ -2,8 +2,8 @@ import hashlib from fastapi import HTTPException -from core.services.artifact_text_service import decode_text, MAX_TEXT_BYTES, TEXT_EXTENSIONS -from core.spaces.leases import authorized_operation +from core.files.artifacts.text import decode_text, MAX_TEXT_BYTES, TEXT_EXTENSIONS +from core.files.spaces.leases import authorized_operation class SpaceText: diff --git a/src/backend/core/spaces/tree.py b/src/backend/core/files/spaces/tree.py similarity index 100% rename from src/backend/core/spaces/tree.py rename to src/backend/core/files/spaces/tree.py diff --git a/src/backend/core/storage/__init__.py b/src/backend/core/files/storage/__init__.py similarity index 56% rename from src/backend/core/storage/__init__.py rename to src/backend/core/files/storage/__init__.py index dc8b02b4..9e5d14a4 100644 --- a/src/backend/core/storage/__init__.py +++ b/src/backend/core/files/storage/__init__.py @@ -1,17 +1,12 @@ """Object storage backend abstraction layer. Re-exports all public symbols from the split modules for backwards compatibility -with ``from core.storage import get_storage, generate_storage_key`` etc. +with ``from core.files.storage import get_storage, generate_storage_key`` etc. """ -from core.storage.protocol import StorageBackend -from core.storage.local import LocalStorageBackend -from core.storage.factory import ( - get_storage_backend, - get_storage, - generate_storage_key, - get_storage_category_for_resource, -) +from core.files.storage.protocol import StorageBackend +from core.files.storage.local import LocalStorageBackend +from core.files.storage.factory import get_storage_backend, get_storage, generate_storage_key, get_storage_category_for_resource __all__ = [ "StorageBackend", @@ -24,12 +19,12 @@ # Cloud backends re-exported on demand (seam C4): the CE tree physically lacks s3/oss, so the import must not crash. try: - from core.storage.s3 import S3StorageBackend # noqa: F401 + from core.files.storage.s3 import S3StorageBackend # noqa: F401 __all__.append("S3StorageBackend") except ModuleNotFoundError: pass try: - from core.storage.oss import OSSStorageBackend # noqa: F401 + from core.files.storage.oss import OSSStorageBackend # noqa: F401 __all__.append("OSSStorageBackend") except ModuleNotFoundError: pass diff --git a/src/backend/core/storage/entries.py b/src/backend/core/files/storage/entries.py similarity index 100% rename from src/backend/core/storage/entries.py rename to src/backend/core/files/storage/entries.py diff --git a/src/backend/core/storage/factory.py b/src/backend/core/files/storage/factory.py similarity index 81% rename from src/backend/core/storage/factory.py rename to src/backend/core/files/storage/factory.py index 4268d0f2..ccd914cb 100644 --- a/src/backend/core/storage/factory.py +++ b/src/backend/core/files/storage/factory.py @@ -1,8 +1,10 @@ """Storage backend factory and utility functions.""" +from __future__ import annotations from core.infra.time import utc_now import logging import os +from dataclasses import dataclass from datetime import datetime from typing import Optional @@ -18,8 +20,8 @@ def secure_filename(filename: str) -> str: sanitized = sanitized.strip("._") return sanitized or "file" -from core.storage.protocol import StorageBackend -from core.storage.local import LocalStorageBackend +from core.files.storage.protocol import StorageBackend +from core.files.storage.local import LocalStorageBackend logger = logging.getLogger(__name__) @@ -40,13 +42,13 @@ def get_storage_backend() -> StorageBackend: storage_type = os.getenv("STORAGE_TYPE", "local").lower() if storage_type == "oss": - from core.storage.oss import OSSStorageBackend + from core.files.storage.oss import OSSStorageBackend return OSSStorageBackend() elif storage_type == "s3": - from core.storage.s3 import S3StorageBackend + from core.files.storage.s3 import S3StorageBackend return S3StorageBackend() elif storage_type == "obs": - from core.storage.obs import OBSStorageBackend + from core.files.storage.obs import OBSStorageBackend return OBSStorageBackend() elif storage_type == "local": return LocalStorageBackend() @@ -105,14 +107,24 @@ def get_storage_category_for_resource(resource_type: str) -> str: # Module-level storage instance (lazy-loaded) -_storage_instance: Optional[StorageBackend] = None def get_storage() -> StorageBackend: """ Get the global storage backend instance (lazy-loaded singleton). """ - global _storage_instance - if _storage_instance is None: - _storage_instance = get_storage_backend() - return _storage_instance + if process_state().storage_instance is None: + process_state().storage_instance = get_storage_backend() + return process_state().storage_instance + + +@dataclass +class FactoryState: + storage_instance: Optional[StorageBackend] = None + + +_STATE = FactoryState() + + +def process_state() -> FactoryState: + return _STATE diff --git a/src/backend/core/storage/local.py b/src/backend/core/files/storage/local.py similarity index 96% rename from src/backend/core/storage/local.py rename to src/backend/core/files/storage/local.py index 3f9b551c..ae7f9f75 100644 --- a/src/backend/core/storage/local.py +++ b/src/backend/core/files/storage/local.py @@ -7,10 +7,10 @@ from pathlib import Path from core.infra.exceptions import StorageError -from core.storage.protocol import StorageBackend -from core.storage.local_directory import LocalDirectoryOperations -from core.storage.entries import validate_key -from core.storage import local_paths +from core.files.storage.protocol import StorageBackend +from core.files.storage.local_directory import LocalDirectoryOperations +from core.files.storage.entries import validate_key +from core.files.storage import local_paths logger = logging.getLogger(__name__) diff --git a/src/backend/core/storage/local_directory.py b/src/backend/core/files/storage/local_directory.py similarity index 94% rename from src/backend/core/storage/local_directory.py rename to src/backend/core/files/storage/local_directory.py index 49684fbc..ce60b674 100644 --- a/src/backend/core/storage/local_directory.py +++ b/src/backend/core/files/storage/local_directory.py @@ -2,8 +2,8 @@ import os import stat from datetime import datetime, timezone -from core.storage.entries import DirectoryPage, StorageEntry, page_limit, validate_key -from core.storage.local_paths import directory_fd, file_stat, parent_fd +from core.files.storage.entries import DirectoryPage, StorageEntry, page_limit, validate_key +from core.files.storage.local_paths import directory_fd, file_stat, parent_fd def entry(name, key, info): diff --git a/src/backend/core/storage/local_paths.py b/src/backend/core/files/storage/local_paths.py similarity index 98% rename from src/backend/core/storage/local_paths.py rename to src/backend/core/files/storage/local_paths.py index 45630a12..d1a1a3c1 100644 --- a/src/backend/core/storage/local_paths.py +++ b/src/backend/core/files/storage/local_paths.py @@ -5,7 +5,7 @@ import uuid from pathlib import Path -from core.storage.entries import validate_key +from core.files.storage.entries import validate_key @contextmanager diff --git a/src/backend/core/storage/protocol.py b/src/backend/core/files/storage/protocol.py similarity index 100% rename from src/backend/core/storage/protocol.py rename to src/backend/core/files/storage/protocol.py diff --git a/src/backend/core/storage/s3.py b/src/backend/core/files/storage/s3.py similarity index 98% rename from src/backend/core/storage/s3.py rename to src/backend/core/files/storage/s3.py index ff3231a9..b697a66d 100644 --- a/src/backend/core/storage/s3.py +++ b/src/backend/core/files/storage/s3.py @@ -5,8 +5,8 @@ from pathlib import Path from core.infra.exceptions import StorageError -from core.storage.protocol import StorageBackend -from core.storage.s3_directory import S3DirectoryOperations +from core.files.storage.protocol import StorageBackend +from core.files.storage.s3_directory import S3DirectoryOperations logger = logging.getLogger(__name__) diff --git a/src/backend/core/storage/s3_directory.py b/src/backend/core/files/storage/s3_directory.py similarity index 96% rename from src/backend/core/storage/s3_directory.py rename to src/backend/core/files/storage/s3_directory.py index 2531530c..f61b80bf 100644 --- a/src/backend/core/storage/s3_directory.py +++ b/src/backend/core/files/storage/s3_directory.py @@ -1,5 +1,5 @@ """S3 directory operations shared by S3-compatible services.""" -from core.storage.entries import DirectoryPage, StorageEntry, page_limit, validate_key +from core.files.storage.entries import DirectoryPage, StorageEntry, page_limit, validate_key class S3DirectoryOperations: def list_directory(self, prefix, *, cursor=None, limit=200): diff --git a/src/backend/core/services/upload_retry.py b/src/backend/core/files/upload_retry.py similarity index 92% rename from src/backend/core/services/upload_retry.py rename to src/backend/core/files/upload_retry.py index d4feb439..270004a1 100644 --- a/src/backend/core/services/upload_retry.py +++ b/src/backend/core/files/upload_retry.py @@ -5,8 +5,8 @@ from uuid import UUID from core.db.models import Artifact, UserShadow -from core.services.artifact_edition import upload_target_matches -from core.services.folder_batch import lock_folder_owner +from core.files.edition import upload_target_matches +from core.db.repository.folder import lock_folder_owner from fastapi import HTTPException diff --git a/src/backend/core/capabilities/errors.py b/src/backend/core/harness/capability_errors.py similarity index 100% rename from src/backend/core/capabilities/errors.py rename to src/backend/core/harness/capability_errors.py diff --git a/src/backend/core/harness/context_items.py b/src/backend/core/harness/context_items.py new file mode 100644 index 00000000..8654de64 --- /dev/null +++ b/src/backend/core/harness/context_items.py @@ -0,0 +1,6 @@ +"""Framework-neutral context additions emitted by execution policies.""" + +def reminder_item(text, origin, *, priority=850, token_budget=4000): + return dict(content=f"\n{text}\n", + kind="reminder", origin=origin, trust="system", + priority=priority, token_budget=token_budget) diff --git a/src/backend/core/harness/events.py b/src/backend/core/harness/events.py index 9a9a117c..f54932fb 100644 --- a/src/backend/core/harness/events.py +++ b/src/backend/core/harness/events.py @@ -5,12 +5,15 @@ import asyncio import copy import inspect -from collections.abc import Awaitable, Callable, Mapping +from collections import deque +from collections.abc import Awaitable, Callable, Mapping, Sequence from dataclasses import dataclass from datetime import UTC, datetime from types import MappingProxyType from typing import Any +from core.harness.observation_writer import observation_writer + def freeze_value(value: Any) -> Any: """Recursively detach and freeze a value before exposing it to consumers.""" @@ -67,6 +70,28 @@ def create( EventStore = Callable[[Event], Event] +def _detach(event: Event) -> Event: + return Event( + run_id=event.run_id, + event_type=event.event_type, + phase=event.phase, + payload=freeze_value(event.payload), + created_at=event.created_at, + event_seq=event.event_seq, + ) + + +def store_events(store: EventStore, events: Sequence[Event]) -> tuple[Event, ...]: + """Write a batch through ``store.store_many`` when the store supports it.""" + store_many = getattr(store, "store_many", None) + if store_many is not None: + return tuple(store_many(events)) + return tuple(store(event) for event in events) + + +MEMORY_EVENT_LIMIT = 256 + + class EventSink: """Append first, then notify observers with immutable detached events. @@ -85,20 +110,15 @@ def __init__( self._store_timeout_ms = max(1, int(store_timeout_ms)) self._subscriber_timeout_ms = max(1, int(subscriber_timeout_ms)) self._handlers: list[EventHandler] = [] - self._memory: list[Event] = [] + # Recent events for in-process inspection; durable history lives in the store. + self._memory: deque[Event] = deque(maxlen=MEMORY_EVENT_LIMIT) def subscribe(self, handler: EventHandler) -> None: self._handlers.append(handler) async def append(self, event: Event) -> Event: - detached = Event( - run_id=event.run_id, - event_type=event.event_type, - phase=event.phase, - payload=freeze_value(event.payload), - created_at=event.created_at, - event_seq=event.event_seq, - ) + """Store, then notify: observers see the durable sequence number.""" + detached = _detach(event) recorded = detached if self._store is not None: try: @@ -111,15 +131,25 @@ async def append(self, event: Event) -> Event: # channel. Durable stores log their own failures. recorded = detached self._memory.append(recorded) + await self._notify(recorded) + return recorded + + async def publish(self, event: Event) -> Event: + """Record without waiting for the store. + + The durable write is queued for the background observation writer, so + the returned and observed event carries no ``event_seq``. + """ + detached = _detach(event) + if self._store is not None: + observation_writer().submit(self._store, store_events, detached.run_id, (detached,)) + self._memory.append(detached) + await self._notify(detached) + return detached + + async def _notify(self, recorded: Event) -> None: for handler in tuple(self._handlers): - consumer_copy = Event( - run_id=recorded.run_id, - event_type=recorded.event_type, - phase=recorded.phase, - payload=freeze_value(recorded.payload), - created_at=recorded.created_at, - event_seq=recorded.event_seq, - ) + consumer_copy = _detach(recorded) try: async def notify( @@ -144,7 +174,6 @@ def call_handler() -> Any: ) except Exception: # noqa: BLE001, S112 continue - return recorded def events(self) -> tuple[Event, ...]: return tuple(self._memory) diff --git a/src/backend/core/llm/execution_manifest.py b/src/backend/core/harness/execution_manifest.py similarity index 99% rename from src/backend/core/llm/execution_manifest.py rename to src/backend/core/harness/execution_manifest.py index 98c14c97..e49e15b8 100644 --- a/src/backend/core/llm/execution_manifest.py +++ b/src/backend/core/harness/execution_manifest.py @@ -17,7 +17,7 @@ from pathlib import Path from typing import Any, Dict, Iterable, Mapping, Optional, Tuple -from core.immutable import FrozenDict, freeze_json, thaw_json +from core.infra.immutable import FrozenDict, freeze_json, thaw_json MANIFEST_SCHEMA_VERSION = "harness.execution-manifest.v1" _PUBLIC_CONTEXT_REFS = { diff --git a/src/backend/core/harness/hooks.py b/src/backend/core/harness/hooks.py index 83ec2555..94ab6256 100644 --- a/src/backend/core/harness/hooks.py +++ b/src/backend/core/harness/hooks.py @@ -12,12 +12,14 @@ from typing import Any from core.harness.events import Event, EventSink, freeze_value, thaw_value -from core.harness.usage import AttemptUsage, UsageAttempt, UsageRecorder +from core.harness.observation_writer import observation_writer +from core.harness.usage import AttemptUsage, UsageAttempt, UsageRecorder, record_attempt_batch class HookStage(str, Enum): BEFORE_RUN = "before_run" TRANSFORM_CONTEXT = "transform_context" + AFTER_CONTEXT = "after_context" BEFORE_MODEL = "before_model" AFTER_MODEL = "after_model" BEFORE_TOOL = "before_tool" @@ -41,24 +43,25 @@ class FailureMode(str, Enum): STAGE_MUTABLE_FIELDS: Mapping[HookStage, frozenset[str]] = MappingProxyType( { - HookStage.BEFORE_RUN: frozenset(), - HookStage.TRANSFORM_CONTEXT: frozenset({"messages", "context"}), + HookStage.BEFORE_RUN: frozenset({"model_selection"}), + HookStage.TRANSFORM_CONTEXT: frozenset({"messages", "context", "context_items", "tool_choice", "steer_delivery"}), # AgentScope reaches on_model_call only after HugAgentOS has assembled and # bound the execution manifest. Rewriting provider input here would make # the evidence bundle describe different inputs than the real request. + HookStage.AFTER_CONTEXT: frozenset(), HookStage.BEFORE_MODEL: frozenset(), HookStage.AFTER_MODEL: frozenset(), # AgentScope reaches on_acting after schema and permission checks. A # rewrite here would bypass both, so the call is observation-only. - HookStage.BEFORE_TOOL: frozenset(), - # Tool chunks may already be visible by this point, so results are - # read-only here; policies that rewrite calls must act BEFORE_TOOL. - HookStage.AFTER_TOOL: frozenset(), + HookStage.BEFORE_TOOL: frozenset({"ontology_runtime", "context_items", "tool_response"}), + # Tool results may change only while buffered; run() rejects a rewrite + # when the adapter has already exposed any result chunks. + HookStage.AFTER_TOOL: frozenset({"context_items", "tool_result"}), # Source history is read-only because the checkpoint watermark/hash # must continue to describe exactly the rows being covered. HookStage.BEFORE_COMPACTION: frozenset({"budget"}), HookStage.AFTER_COMPACTION: frozenset({"replacement"}), - HookStage.BEFORE_FINISH: frozenset(), + HookStage.BEFORE_FINISH: frozenset({"pin_file_ids", "continue_run"}), } ) @@ -131,7 +134,7 @@ class HookSpec: stage: HookStage handler: HookHandler priority: int = 100 - timeout_ms: int = 1_000 + timeout_ms: int | None = 1_000 failure_mode: FailureMode = FailureMode.CLOSED mutable_fields: frozenset[str] = field(default_factory=frozenset) @@ -174,11 +177,9 @@ def __init__( *, event_sink: EventSink | None = None, usage_recorder: UsageRecorder | None = None, - usage_timeout_ms: int = 1_000, ) -> None: self.event_sink = event_sink or EventSink() self.usage_recorder = usage_recorder - self.usage_timeout_ms = max(1, int(usage_timeout_ms)) self._hooks: dict[HookStage, list[tuple[int, int, HookSpec]]] = {} self._registration_seq = 0 @@ -195,52 +196,55 @@ def register(self, spec: HookSpec) -> None: bucket.append((spec.priority, self._registration_seq, spec)) bucket.sort(key=lambda item: (item[0], item[1])) + def registered_specs(self) -> tuple[HookSpec, ...]: + """Expose the ordered policy inventory without mutable registry internals.""" + return tuple(spec for stage in HookStage + for _, _, spec in self._hooks.get(stage, ())) + async def _call(self, spec: HookSpec, invocation: Invocation) -> Decision: - async def invoke() -> Decision: - if inspect.iscoroutinefunction(spec.handler): - result = await spec.handler(invocation) + # Synchronous handlers are in-memory policies and run inline: a thread + # hop plus timeout would let executor saturation reject or skip them. + # Policies that block on I/O must be async and own their offloading. + result = spec.handler(invocation) + if inspect.isawaitable(result): + # Interactive policies own their timeout and must retain caller cancellation. + if spec.timeout_ms is None: + result = await result else: - result = await asyncio.to_thread(spec.handler, invocation) - if inspect.isawaitable(result): - result = await result - if result is None: - return Decision() - if not isinstance(result, Decision): - raise HookContractError( - f"hook {spec.name} returned {type(result).__name__}" - ) - return result - - return await asyncio.wait_for(invoke(), timeout=max(1, spec.timeout_ms) / 1_000) + result = await asyncio.wait_for(result, timeout=max(1, spec.timeout_ms) / 1_000) + if result is None: + return Decision() + if not isinstance(result, Decision): + raise HookContractError(f"hook {spec.name} returned {type(result).__name__}") + return result async def run(self, invocation: Invocation) -> HookOutcome: current = invocation executed: list[str] = [] terminal = Decision() - await self.event_sink.append( - Event.create( - run_id=current.run_id, - event_type="hook_stage_started", - phase=current.stage.value, - payload={"operation_name": current.operation_name}, - ) - ) - for _, _, spec in tuple(self._hooks.get(current.stage, ())): + hooks = tuple(self._hooks.get(current.stage, ())) + if not hooks: + return HookOutcome(current, terminal, ()) + attempts: list[dict[str, Any]] = [] + usage: list[UsageAttempt] = [] + cancellation: asyncio.CancelledError | None = None + for _, _, spec in hooks: started = time.monotonic() status = "success" decision = Decision() - cancellation: asyncio.CancelledError | None = None try: decision = await self._call(spec, current) - if decision.kind == DecisionKind.MODIFY: + if decision.patch: invalid = set(decision.patch) - set(spec.mutable_fields) if invalid: raise HookContractError( f"hook {spec.name} modified undeclared fields: " f"{sorted(invalid)}" ) + if "tool_result" in decision.patch and current.data.get("result_exposed"): + raise HookContractError("cannot rewrite an exposed tool result") current = current.patched(decision.patch) - elif decision.kind in {DecisionKind.REJECT, DecisionKind.PAUSE}: + if decision.kind in {DecisionKind.REJECT, DecisionKind.PAUSE}: terminal = decision except asyncio.CancelledError as exc: status = "cancelled" @@ -255,57 +259,56 @@ async def run(self, invocation: Invocation) -> HookOutcome: terminal = Decision.reject(f"hook {spec.name} failed: {exc}") latency_ms = int((time.monotonic() - started) * 1_000) executed.append(spec.name) - if self.usage_recorder is not None: - try: - await asyncio.wait_for( - asyncio.to_thread( - self.usage_recorder.record_attempt, - UsageAttempt( - run_id=current.run_id, - kind="hook", - operation_name=spec.name, - status=status, - latency_ms=latency_ms, - usage=decision.usage, - metadata={"stage": current.stage.value}, - ), - ), - timeout=self.usage_timeout_ms / 1_000, - ) - except Exception: # noqa: BLE001, S110 - pass - await self.event_sink.append( - Event.create( + usage.append( + UsageAttempt( run_id=current.run_id, - event_type="hook_attempt_finished", - phase=current.stage.value, - payload={ - "hook": spec.name, - "status": status, - "decision": terminal.kind.value - if terminal.kind != DecisionKind.CONTINUE - else decision.kind.value, - "latency_ms": latency_ms, - }, + kind="hook", + operation_name=spec.name, + status=status, + latency_ms=latency_ms, + usage=decision.usage, + metadata={"stage": current.stage.value}, ) ) - if terminal.kind in {DecisionKind.REJECT, DecisionKind.PAUSE}: + attempts.append( + { + "hook": spec.name, + "status": status, + "decision": terminal.kind.value + if terminal.kind != DecisionKind.CONTINUE + else decision.kind.value, + "latency_ms": latency_ms, + } + ) + if terminal.kind in {DecisionKind.REJECT, DecisionKind.PAUSE} or cancellation is not None: break - if cancellation is not None: - raise cancellation - await self.event_sink.append( + await self._record_stage(current, terminal, executed, attempts, usage) + if cancellation is not None: + raise cancellation + return HookOutcome(current, terminal, tuple(executed)) + + async def _record_stage(self, current, terminal, executed, attempts, usage) -> None: + """Queue one usage batch and one event per stage, not one per hook. + + Both are written by the background observation writer; the stage never + waits for the database and a failed write cannot change its outcome. + """ + if self.usage_recorder is not None and current.run_id: + observation_writer().submit(self.usage_recorder, record_attempt_batch, current.run_id, usage) + await self.event_sink.publish( Event.create( run_id=current.run_id, event_type="hook_stage_finished", phase=current.stage.value, payload={ + "operation_name": current.operation_name, "decision": terminal.kind.value, "executed_hooks": executed, + "attempts": attempts, }, ) ) - return HookOutcome(current, terminal, tuple(executed)) async def enforce(self, invocation: Invocation) -> Invocation: outcome = await self.run(invocation) diff --git a/src/backend/core/services/harness_ledger.py b/src/backend/core/harness/ledger.py similarity index 53% rename from src/backend/core/services/harness_ledger.py rename to src/backend/core/harness/ledger.py index bb5cb2d7..d346f0cb 100644 --- a/src/backend/core/services/harness_ledger.py +++ b/src/backend/core/harness/ledger.py @@ -39,18 +39,59 @@ def _insert_cursor_if_missing(db, model: type[Any], values: dict[str, Any]) -> N db.execute(statement) -def _reserve(db, model: type[Any], run_id: str, column) -> int: +def _reserve(db, model: type[Any], run_id: str, column, count: int = 1) -> int: + """Reserve ``count`` consecutive sequence numbers and return the last one.""" _insert_cursor_if_missing(db, model, {"run_id": run_id, column.key: 0}) value = db.execute( update(model) .where(model.run_id == run_id) - .values({column.key: column + 1}) + .values({column.key: column + count}) .returning(column) .execution_options(synchronize_session=False) ).scalar_one() return int(value) +def _previous_failed_attempt(db, attempt: UsageAttempt) -> int | None: + """A tool effect's retry links to its latest failed physical attempt.""" + if attempt.kind != "tool" or not attempt.effect_id: + return None + previous = ( + db.query(HarnessUsageAttempt) + .filter( + HarnessUsageAttempt.run_id == attempt.run_id, + HarnessUsageAttempt.kind == attempt.kind, + HarnessUsageAttempt.operation_name == attempt.operation_name, + HarnessUsageAttempt.status.in_(("failed", "timeout")), + HarnessUsageAttempt.effect_id == attempt.effect_id, + ) + .order_by(HarnessUsageAttempt.attempt_seq.desc()) + .first() + ) + return int(previous.attempt_seq) if previous is not None else None + + +def _attempt_row(attempt: UsageAttempt, seq: int, retry_of: int | None) -> HarnessUsageAttempt: + return HarnessUsageAttempt( + attempt_id=f"hat_{uuid.uuid4().hex}", + run_id=attempt.run_id, + attempt_seq=seq, + kind=attempt.kind, + operation_name=attempt.operation_name, + provider=attempt.provider, + model=attempt.model, + effect_id=attempt.effect_id or None, + prompt_tokens=max(0, int(attempt.usage.prompt_tokens)), + completion_tokens=max(0, int(attempt.usage.completion_tokens)), + cache_read_tokens=max(0, int(attempt.usage.cache_read_tokens)), + cache_write_tokens=max(0, int(attempt.usage.cache_write_tokens)), + latency_ms=max(0, int(attempt.latency_ms)), + status=attempt.status, + retry_of=retry_of, + attempt_metadata=thaw_value(attempt.metadata), + ) + + class HarnessUsageLedger: """Append attempts and derive aggregates without mutable summary rows.""" @@ -67,48 +108,35 @@ def _validate(attempt: UsageAttempt) -> None: raise ValueError(f"invalid attempt status: {attempt.status}") def record_attempt(self, attempt: UsageAttempt) -> UsageAttempt: - self._validate(attempt) + return self.record_attempts((attempt,))[0] + + def record_attempts(self, attempts) -> tuple[UsageAttempt, ...]: + """Append attempts of one run in a single transaction and cursor update.""" + attempts = tuple(attempts) + if not attempts: + return () + for attempt in attempts: + self._validate(attempt) + run_id = attempts[0].run_id + if any(attempt.run_id != run_id for attempt in attempts): + raise ValueError("a usage batch must belong to one run") with self._session_factory() as db: - seq = _reserve( + last = _reserve( db, HarnessUsageCursor, - attempt.run_id, + run_id, HarnessUsageCursor.next_attempt_seq, + len(attempts), ) - retry_of = attempt.retry_of - if retry_of is None and attempt.kind == "tool" and attempt.effect_id: - query = db.query(HarnessUsageAttempt).filter( - HarnessUsageAttempt.run_id == attempt.run_id, - HarnessUsageAttempt.kind == attempt.kind, - HarnessUsageAttempt.operation_name == attempt.operation_name, - HarnessUsageAttempt.status.in_(("failed", "timeout")), - ) - query = query.filter(HarnessUsageAttempt.effect_id == attempt.effect_id) - previous = query.order_by( - HarnessUsageAttempt.attempt_seq.desc() - ).first() - retry_of = int(previous.attempt_seq) if previous is not None else None - row = HarnessUsageAttempt( - attempt_id=f"hat_{uuid.uuid4().hex}", - run_id=attempt.run_id, - attempt_seq=seq, - kind=attempt.kind, - operation_name=attempt.operation_name, - provider=attempt.provider, - model=attempt.model, - effect_id=attempt.effect_id or None, - prompt_tokens=max(0, int(attempt.usage.prompt_tokens)), - completion_tokens=max(0, int(attempt.usage.completion_tokens)), - cache_read_tokens=max(0, int(attempt.usage.cache_read_tokens)), - cache_write_tokens=max(0, int(attempt.usage.cache_write_tokens)), - latency_ms=max(0, int(attempt.latency_ms)), - status=attempt.status, - retry_of=retry_of, - attempt_metadata=thaw_value(attempt.metadata), - ) - db.add(row) + recorded = [] + for seq, attempt in enumerate(attempts, start=last - len(attempts) + 1): + retry_of = attempt.retry_of + if retry_of is None: + retry_of = _previous_failed_attempt(db, attempt) + db.add(_attempt_row(attempt, seq, retry_of)) + recorded.append(replace(attempt, attempt_seq=seq, retry_of=retry_of)) db.commit() - return replace(attempt, attempt_seq=seq, retry_of=retry_of) + return tuple(recorded) def attempts(self, run_id: str) -> tuple[UsageAttempt, ...]: with self._session_factory() as db: @@ -167,32 +195,39 @@ def __init__(self, session_factory=SessionLocal) -> None: self._session_factory = session_factory def __call__(self, event: Event) -> Event: - if not event.run_id: - return event + return self.store_many((event,))[0] + + def store_many(self, events) -> tuple[Event, ...]: + """Append events in one transaction, one cursor update per run.""" + recorded = list(events) + by_run: dict[str, list[int]] = {} + for index, event in enumerate(recorded): + if event.run_id: + by_run.setdefault(event.run_id, []).append(index) + if not by_run: + return tuple(recorded) with self._session_factory() as db: - seq = _reserve( - db, - HarnessEventCursor, - event.run_id, - HarnessEventCursor.next_event_seq, - ) - db.add( - HarnessEventLog( - event_id=f"hev_{uuid.uuid4().hex}", - run_id=event.run_id, - event_seq=seq, - event_type=event.event_type, - phase=event.phase, - payload=thaw_value(event.payload), - created_at=event.created_at, + for run_id, indexes in by_run.items(): + last = _reserve( + db, + HarnessEventCursor, + run_id, + HarnessEventCursor.next_event_seq, + len(indexes), ) - ) + for seq, index in enumerate(indexes, start=last - len(indexes) + 1): + event = recorded[index] + db.add( + HarnessEventLog( + event_id=f"hev_{uuid.uuid4().hex}", + run_id=run_id, + event_seq=seq, + event_type=event.event_type, + phase=event.phase, + payload=thaw_value(event.payload), + created_at=event.created_at, + ) + ) + recorded[index] = replace(event, payload=freeze_value(event.payload), event_seq=seq) db.commit() - return Event( - run_id=event.run_id, - event_type=event.event_type, - phase=event.phase, - payload=freeze_value(event.payload), - created_at=event.created_at, - event_seq=seq, - ) + return tuple(recorded) diff --git a/src/backend/core/harness/log_audit.py b/src/backend/core/harness/log_audit.py new file mode 100644 index 00000000..7340a7cd --- /dev/null +++ b/src/backend/core/harness/log_audit.py @@ -0,0 +1,144 @@ +"""Write sandbox tool calls as security audit events.""" +from __future__ import annotations +from datetime import datetime +from typing import Any, Dict, Optional +from sqlalchemy.orm import Session +from core.db.engine import SessionLocal +from core.infra.logging import get_logger +from core.harness import log_context + +logger = get_logger(__name__) + +async def resolve_sandbox_id(session_id: Optional[str]) -> Optional[str]: + """Look up the current sandbox instance id from the sandbox provider by session key (default = chat_id). Pure query; None on failure.""" + if not session_id: + return None + try: + from core.sandbox import get_sandbox_provider + + return await get_sandbox_provider().current_sandbox_id(session_id) + except ( + Exception + ): # noqa: BLE001 — audit enrichment is best-effort, must never affect the main path + return None + + +def audit_status(raw: Optional[str]) -> str: + return AUDIT_STATUS_MAP.get(raw or "success", "failure") + + +def insert_sandbox_audit( + *, + action: str, + sandbox_id: Any, + user_id: Optional[str], + trace_id: Optional[str], + details: Dict[str, Any], + status: str, + created_at: Optional[datetime], +) -> None: + """Write one resource=sandbox security audit row. Best-effort: failure only degrades to a log entry, never affects the main path.""" + from core.db.repository import AuditLogRepository + + db: Session = SessionLocal() + try: + # Reuse the audit repository's write (has its own add/commit/rollback; audit failure doesn't block the main path). + AuditLogRepository(db).create( + { + "user_id": user_id + or None, # empty string would trip the FK constraint; normalize to NULL + "action": action, + "resource_type": "sandbox", + "resource_id": str(sandbox_id)[:64], + "sandbox_id": str(sandbox_id)[:128], + "details": details, + "trace_id": trace_id, + "status": status, + "created_at": created_at or log_context.now_utc(), + } + ) + except Exception as exc: # noqa: BLE001 + logger.warning("sandbox_audit_write_failed", error=str(exc), sandbox=str(sandbox_id)) + finally: + db.close() + + +def write_sandbox_audit_sync(record: Dict[str, Any]) -> None: + """Write one sandbox-type tool call as a security audit event (resource=sandbox / sandbox instance id). + + Separate from tool-call logging: audits let "Security Management → Audit + Logs" distinguish and filter by sandbox instance. details only summarizes + the input args (redacted + truncated), never dumps full results. + """ + sandbox_id = record.get("sandbox_id") + if not sandbox_id: + return + tool_name = record.get("tool_name") + action = SANDBOX_AUDIT_ACTIONS.get(tool_name, "sandbox.tool.call") + args_safe = log_context.sanitize_tool_args(record.get("tool_args")) + details: Dict[str, Any] = {"tool": tool_name} + if isinstance(args_safe, dict): + cmd = args_safe.get("command") + if cmd is not None: + preview, _ = log_context.truncate_text(str(cmd), 2000) + details["command"] = preview + # File/search tools: record path and pattern (not large fields like content) to trace what was changed/read. + for k in ("file_path", "filename", "path", "pattern", "artifact_id", "name"): + if args_safe.get(k) is not None: + details[k] = args_safe[k] + if record.get("error_message"): + details["error"], _ = log_context.truncate_text(str(record["error_message"]), 500) + + insert_sandbox_audit( + action=action, + sandbox_id=sandbox_id, + user_id=record.get("user_id"), + trace_id=record.get("trace_id"), + details=details, + status=audit_status(record.get("status")), + created_at=record.get("created_at"), + ) + + +def write_skill_sandbox_audit_sync(record: Dict[str, Any]) -> None: + """Skill script execution (run_script) = running code in the sandbox; write one sandbox.skill.run audit. + + Triggered only by run_script (view / auto_load do not execute code in the + sandbox); skipped when there is no sandbox instance. + """ + sandbox_id = record.get("sandbox_id") + if not sandbox_id: + return + details: Dict[str, Any] = { + "skill_id": record.get("skill_id"), + "skill_name": record.get("skill_name"), + "script_name": record.get("script_name"), + "script_language": record.get("script_language"), + } + if record.get("error_message"): + details["error"], _ = log_context.truncate_text(str(record["error_message"]), 500) + + insert_sandbox_audit( + action="sandbox.skill.run", + sandbox_id=sandbox_id, + user_id=record.get("user_id"), + trace_id=record.get("trace_id"), + details=details, + status=audit_status(record.get("status")), + created_at=record.get("created_at"), + ) + + +SANDBOX_AUDIT_ACTIONS = { + "Bash": "sandbox.bash.exec", + "sandbox_put_artifact": "sandbox.artifact.put", + "sandbox_get_artifact": "sandbox.artifact.get", + "Write": "sandbox.file.write", + "Edit": "sandbox.file.edit", + "Read": "sandbox.file.read", + "Grep": "sandbox.file.grep", + "Glob": "sandbox.file.glob", +} + + +AUDIT_STATUS_MAP = {"success": "success", "error": "error"} diff --git a/src/backend/core/harness/log_context.py b/src/backend/core/harness/log_context.py new file mode 100644 index 00000000..d8474190 --- /dev/null +++ b/src/backend/core/harness/log_context.py @@ -0,0 +1,161 @@ +"""Shared helpers for call logs: ids, truncation and the current log scope.""" +from __future__ import annotations +import json +import os +import uuid +from contextlib import contextmanager +from contextvars import ContextVar +from datetime import datetime, timezone +from typing import Any, Dict, Iterator, Optional +from sqlalchemy.orm import Session +from core.db.models import LocalUser, UserShadow +from core.infra.data_masking import mask_sensitive_data +from core.infra.logging import chat_id_var, trace_id_var, user_id_var +from core.infra.logging import get_logger +logger = get_logger(__name__) + +def new_id() -> str: + return uuid.uuid4().hex + + +def now_utc() -> datetime: + return datetime.now(timezone.utc) + + +def truncate_text(text: Optional[str], limit: int) -> tuple[Optional[str], bool]: + if text is None: + return None, False + if not isinstance(text, str): + text = str(text) + raw = text.encode("utf-8", errors="ignore") + if len(raw) <= limit: + return text, False + return raw[:limit].decode("utf-8", errors="ignore") + "\n…[truncated]", True + + +def truncate_json(payload: Any, limit: int) -> tuple[Any, bool]: + if payload is None: + return None, False + try: + encoded = json.dumps(payload, ensure_ascii=False, default=str).encode( + "utf-8", errors="ignore" + ) + except Exception: + return {"_repr": str(payload)[:limit]}, True + if len(encoded) <= limit: + return payload, False + preview = encoded[: limit // 2].decode("utf-8", errors="ignore") + return {"_truncated": True, "_preview": preview}, True + + +def context_ids() -> Dict[str, str]: + return { + "trace_id": trace_id_var.get() or "", + "user_id": user_id_var.get() or "", + "chat_id": chat_id_var.get() or "", + } + + +def fetch_username(db: Session, user_id: str) -> Optional[str]: + """The "user name" used in logs: prefer the local account display name + (nickname → real_name), fall back to the login account username. + + When the audit log panel shows this column, operators should see a + recognizable person name (e.g. "张三"), not the login account (e.g. + "zhangsan01") — the latter would not match the display names shown + elsewhere in the UI. + """ + if not user_id: + return None + try: + row = ( + db.query(UserShadow.username, LocalUser.nickname, LocalUser.real_name) + .outerjoin(LocalUser, LocalUser.user_id == UserShadow.user_id) + .filter(UserShadow.user_id == user_id) + .one_or_none() + ) + if not row: + return None + return row.nickname or row.real_name or row.username + except Exception as exc: # noqa: BLE001 + logger.debug("username_lookup_failed", user_id=user_id, error=str(exc)) + return None + + +@contextmanager +def subagent_scope(subagent_log_id: Optional[str], source: str = "subagent") -> Iterator[None]: + tok_id = subagent_log_id_var.set(subagent_log_id) + tok_src = source_var.set(source) + try: + yield + finally: + subagent_log_id_var.reset(tok_id) + source_var.reset(tok_src) + + +@contextmanager +def skill_scope(source: str = "skill") -> Iterator[None]: + tok = source_var.set(source) + try: + yield + finally: + source_var.reset(tok) + + +def set_current_message_id(message_id: str) -> None: + current_message_id.set(message_id or "") + + +def current_subagent_log_id() -> Optional[str]: + return subagent_log_id_var.get() + + +def current_source() -> str: + return source_var.get() or "main_agent" + + +def effective_status(declared: Optional[str], result: Any) -> str: + """声明的状态优先，但"成功"要经得起载荷检验。 + + 只认**顶层** error，且只认 dict / 能解析成 dict 的 JSON 串——正文里出现 error 字样的 + 正常内容（比如搜到一篇讲错误码的网页）不该被误判成故障。 + """ + status = declared or "success" + if status != "success": + return status + from core.chat.tool_log import _payload_carries_error + + return "error" if _payload_carries_error(result) else "success" + + +def sanitize_tool_args(value: Any) -> Any: + from core.runs.tool_effects.arguments import canonical_tool_args + + if isinstance(value, dict): + return canonical_tool_args(value)[2] + return mask_sensitive_data(value, field_patterns=REDACT_FIELDS) + + +source_var: ContextVar[str] = ContextVar("log_current_source", default="main_agent") + + +subagent_log_id_var: ContextVar[Optional[str]] = ContextVar( + "log_current_subagent", default=None +) + + +current_message_id: ContextVar[str] = ContextVar("log_current_message", default="") + + +REDACT_FIELDS = [ + s.strip() + for s in os.getenv( + "LOG_REDACT_FIELDS", + "password,token,api_key,apikey,authorization,secret,access_key", + ).split(",") + if s.strip() +] + + +# Shared request correlation survives model gateway token rotation. +model_call_id = ContextVar("desktop_model_call_id", default="") diff --git a/src/backend/core/harness/log_records.py b/src/backend/core/harness/log_records.py new file mode 100644 index 00000000..66dc0eda --- /dev/null +++ b/src/backend/core/harness/log_records.py @@ -0,0 +1,212 @@ +"""Synchronous writers for tool, sub-agent and skill call logs.""" +from __future__ import annotations +from typing import Any, Dict +from sqlalchemy.orm import Session +from core.db.engine import SessionLocal +from core.db.models import SkillCallLog, SubAgentCallLog, ToolCallLog, ToolEffectLedger +from core.infra.data_masking import mask_sensitive_data +from core.infra.logging import get_logger +from core.harness import log_context +from core.harness import log_writer_constants + +logger = get_logger(__name__) + +def write_tool_call_sync(record: Dict[str, Any]) -> None: + if not log_writer_constants.TOOL_LOG_ENABLED: + return + db: Session = SessionLocal() + try: + args_safe = log_context.sanitize_tool_args(record.get("tool_args")) + args_safe, _ = log_context.truncate_json(args_safe, log_writer_constants.MAX_RESULT_BYTES) + result_safe, result_trunc = log_context.truncate_json(record.get("tool_result"), log_writer_constants.MAX_RESULT_BYTES) + username = log_context.fetch_username(db, record.get("user_id", "")) + effect = None + effect_id = str(record.get("effect_id") or "") + run_id = str(record.get("run_id") or "") + tool_call_id = str(record.get("tool_call_id") or "") + if effect_id: + effect = ( + db.query(ToolEffectLedger) + .filter( + ToolEffectLedger.effect_id == effect_id, + ToolEffectLedger.event_type == "intent", + ) + .one_or_none() + ) + if effect is None and run_id and tool_call_id: + effect = ( + db.query(ToolEffectLedger) + .filter( + ToolEffectLedger.run_id == run_id, + ToolEffectLedger.tool_call_id == tool_call_id, + ToolEffectLedger.event_type == "intent", + ) + .order_by(ToolEffectLedger.event_id.desc()) + .first() + ) + # ToolCallLog stays a replaceable query projection. The pre-allocated + # result id makes retries update the same row, while ToolEffectLedger + # remains the append-only recovery authority. + row_id = ( + (effect.result_id if effect is not None else None) + or record.get("result_id") + or record.get("id") + or log_context.new_id() + ) + row = db.get(ToolCallLog, row_id) or ToolCallLog(id=row_id) + row.effect_id = effect.effect_id if effect is not None else record.get("effect_id") + row.trace_id = record.get("trace_id") + row.chat_id = record.get("chat_id") + row.message_id = record.get("message_id") + row.user_id = record.get("user_id") + row.user_name = username + row.tool_name = record["tool_name"] + row.tool_display_name = record.get("tool_display_name") + row.tool_call_id = record.get("tool_call_id") + row.mcp_server = record.get("mcp_server") + row.tool_args = args_safe + row.tool_result = result_safe + row.result_truncated = bool(result_trunc) + # A top-level error payload is a failed projection even when an MCP + # adapter declared the transport call successful. + row.status = log_context.effective_status(record.get("status"), record.get("tool_result")) + row.error_message = record.get("error_message") + row.duration_ms = record.get("duration_ms") + row.source = record.get("source") or "main_agent" + row.subagent_log_id = record.get("subagent_log_id") + row.skill_log_id = record.get("skill_log_id") + row.sandbox_id = record.get("sandbox_id") + row.started_at = record.get("started_at") + row.created_at = record.get("created_at") or log_context.now_utc() + db.add(row) + db.commit() + except Exception as exc: # noqa: BLE001 + db.rollback() + logger.warning("tool_call_log_write_failed", error=str(exc), tool=record.get("tool_name")) + finally: + db.close() + + +def write_subagent_create_sync(record: Dict[str, Any]) -> None: + if not log_writer_constants.SUBAGENT_LOG_ENABLED: + return + db: Session = SessionLocal() + try: + input_safe, _ = log_context.truncate_json(record.get("input_messages"), log_writer_constants.MAX_RESULT_BYTES) + username = log_context.fetch_username(db, record.get("user_id", "")) + row = SubAgentCallLog( + id=record["id"], + trace_id=record.get("trace_id"), + chat_id=record.get("chat_id"), + message_id=record.get("message_id"), + user_id=record.get("user_id"), + user_name=username, + subagent_id=record.get("subagent_id"), + subagent_name=record["subagent_name"], + subagent_type=record.get("subagent_type"), + plan_id=record.get("plan_id"), + step_id=record.get("step_id"), + step_index=record.get("step_index"), + step_title=record.get("step_title"), + model=record.get("model"), + input_messages=input_safe, + status="running", + parent_subagent_log_id=record.get("parent_subagent_log_id"), + started_at=record.get("started_at") or log_context.now_utc(), + created_at=log_context.now_utc(), + ) + db.add(row) + db.commit() + except Exception as exc: # noqa: BLE001 + db.rollback() + logger.warning( + "subagent_log_create_failed", error=str(exc), name=record.get("subagent_name") + ) + finally: + db.close() + + +def write_subagent_finish_sync( + subagent_log_id: str, + status: str, + updates: Dict[str, Any], +) -> None: + if not log_writer_constants.SUBAGENT_LOG_ENABLED: + return + db: Session = SessionLocal() + try: + row = db.query(SubAgentCallLog).filter(SubAgentCallLog.id == subagent_log_id).one_or_none() + if row is None: + return + row.status = status + if "output_content" in updates: + output, _ = log_context.truncate_text(updates.get("output_content"), log_writer_constants.MAX_RESULT_BYTES) + row.output_content = output + if "intermediate_steps" in updates: + steps_safe, _ = log_context.truncate_json(updates.get("intermediate_steps"), log_writer_constants.MAX_RESULT_BYTES) + row.intermediate_steps = steps_safe + if "token_usage" in updates: + row.token_usage = updates.get("token_usage") + if "tool_calls_count" in updates: + row.tool_calls_count = updates.get("tool_calls_count") + if "skill_calls_count" in updates: + row.skill_calls_count = updates.get("skill_calls_count") + if "error_message" in updates: + row.error_message = updates.get("error_message") + if "duration_ms" in updates: + row.duration_ms = updates.get("duration_ms") + row.completed_at = log_context.now_utc() + db.commit() + except Exception as exc: # noqa: BLE001 + db.rollback() + logger.warning("subagent_log_finish_failed", error=str(exc), id=subagent_log_id) + finally: + db.close() + + +def write_skill_call_sync(record: Dict[str, Any]) -> None: + if not log_writer_constants.SKILL_LOG_ENABLED: + return + db: Session = SessionLocal() + try: + args_safe = mask_sensitive_data(record.get("script_args"), field_patterns=log_context.REDACT_FIELDS) + args_safe, _ = log_context.truncate_json(args_safe, log_writer_constants.MAX_RESULT_BYTES) + stdout_safe, trunc_out = log_context.truncate_text(record.get("script_stdout"), log_writer_constants.MAX_STDOUT_BYTES) + stderr_safe, trunc_err = log_context.truncate_text(record.get("script_stderr"), log_writer_constants.MAX_STDOUT_BYTES) + stdin_safe, _ = log_context.truncate_text(record.get("script_stdin"), log_writer_constants.MAX_STDOUT_BYTES) + username = log_context.fetch_username(db, record.get("user_id", "")) + row = SkillCallLog( + id=record.get("id") or log_context.new_id(), + trace_id=record.get("trace_id"), + chat_id=record.get("chat_id"), + message_id=record.get("message_id"), + user_id=record.get("user_id"), + user_name=username, + skill_id=record["skill_id"], + skill_name=record.get("skill_name"), + skill_version=record.get("skill_version"), + skill_source=record.get("skill_source"), + invocation_type=record.get("invocation_type") or "auto_load", + script_name=record.get("script_name"), + script_language=record.get("script_language"), + script_args=args_safe, + script_stdin=stdin_safe, + script_stdout=stdout_safe, + script_stderr=stderr_safe, + output_truncated=bool(trunc_out or trunc_err), + exit_code=record.get("exit_code"), + status=record.get("status") or "success", + error_message=record.get("error_message"), + duration_ms=record.get("duration_ms"), + source=record.get("source") or log_context.current_source(), + subagent_log_id=record.get("subagent_log_id") or log_context.current_subagent_log_id(), + started_at=record.get("started_at"), + created_at=record.get("created_at") or log_context.now_utc(), + ) + db.add(row) + db.commit() + except Exception as exc: # noqa: BLE001 + db.rollback() + logger.warning("skill_call_log_write_failed", error=str(exc), skill=record.get("skill_id")) + finally: + db.close() diff --git a/src/backend/core/harness/log_writer.py b/src/backend/core/harness/log_writer.py new file mode 100644 index 00000000..a4ccb852 --- /dev/null +++ b/src/backend/core/harness/log_writer.py @@ -0,0 +1,121 @@ +"""Async entry points that schedule call-log writes off the event loop.""" +from __future__ import annotations +import asyncio +from typing import Any, Dict, List, Optional +from core.infra.background import spawn +from core.infra.logging import get_logger +from core.harness import log_audit +from core.harness import log_context +from core.harness import log_records +from core.harness import log_writer_constants + +logger = get_logger(__name__) + +async def write_tool_call(record: Dict[str, Any]) -> None: + record.setdefault("source", log_context.current_source()) + record.setdefault("subagent_log_id", log_context.current_subagent_log_id()) + ctx = log_context.context_ids() + record.setdefault("trace_id", ctx["trace_id"]) + record.setdefault("user_id", ctx["user_id"]) + record.setdefault("chat_id", ctx["chat_id"]) + record.setdefault("message_id", log_context.current_message_id.get()) + # Sandbox-type tools: resolve the sandbox instance id they run in (session + # key default = chat_id). Awaiting in place on this async path — provider + # reads its active-session table (synchronous in-memory read, no side + # effects) — is more reliable than passing a ContextVar across tasks. + if record.get("tool_name") in log_writer_constants.SANDBOX_TOOL_NAMES: + if not record.get("sandbox_id"): + record["sandbox_id"] = await log_audit.resolve_sandbox_id(record.get("chat_id")) + # Security audit is independent of the tool-call log switch: write the audit first, then the (optional) tool log. + if log_writer_constants.SANDBOX_AUDIT_ENABLED: + try: + await asyncio.to_thread(log_audit.write_sandbox_audit_sync, record) + except Exception as exc: # noqa: BLE001 + logger.warning("sandbox_audit_task_failed", error=str(exc)) + if not log_writer_constants.TOOL_LOG_ENABLED: + return + try: + await asyncio.to_thread(log_records.write_tool_call_sync, record) + except Exception as exc: # noqa: BLE001 + logger.warning("tool_call_log_task_failed", error=str(exc)) + + +async def start_subagent_log(record: Dict[str, Any]) -> str: + log_id = record.get("id") or log_context.new_id() + record["id"] = log_id + if not log_writer_constants.SUBAGENT_LOG_ENABLED: + return log_id + ctx = log_context.context_ids() + record.setdefault("trace_id", ctx["trace_id"]) + record.setdefault("user_id", ctx["user_id"]) + record.setdefault("chat_id", ctx["chat_id"]) + record.setdefault("message_id", log_context.current_message_id.get()) + record.setdefault("parent_subagent_log_id", log_context.current_subagent_log_id()) + record.setdefault("started_at", log_context.now_utc()) + try: + await asyncio.to_thread(log_records.write_subagent_create_sync, record) + except Exception as exc: # noqa: BLE001 + logger.warning("subagent_log_start_failed", error=str(exc)) + return log_id + + +async def finish_subagent_log( + subagent_log_id: str, + *, + status: str = "success", + output_content: Optional[str] = None, + intermediate_steps: Optional[List[Dict[str, Any]]] = None, + token_usage: Optional[Dict[str, int]] = None, + tool_calls_count: int = 0, + skill_calls_count: int = 0, + error_message: Optional[str] = None, + duration_ms: Optional[int] = None, +) -> None: + if not log_writer_constants.SUBAGENT_LOG_ENABLED or not subagent_log_id: + return + updates = { + "output_content": output_content, + "intermediate_steps": intermediate_steps, + "token_usage": token_usage, + "tool_calls_count": tool_calls_count, + "skill_calls_count": skill_calls_count, + "error_message": error_message, + "duration_ms": duration_ms, + } + try: + await asyncio.to_thread(log_records.write_subagent_finish_sync, subagent_log_id, status, updates) + except Exception as exc: # noqa: BLE001 + logger.warning("subagent_log_finish_failed", error=str(exc)) + + +async def write_skill_call(record: Dict[str, Any]) -> str: + log_id = record.get("id") or log_context.new_id() + record["id"] = log_id + ctx = log_context.context_ids() + record.setdefault("trace_id", ctx["trace_id"]) + record.setdefault("user_id", ctx["user_id"]) + record.setdefault("chat_id", ctx["chat_id"]) + record.setdefault("message_id", log_context.current_message_id.get()) + # Skill script execution (run_script) = running code in the sandbox → write a security audit (independent of the skill log switch). + if log_writer_constants.SANDBOX_AUDIT_ENABLED and record.get("invocation_type") == "run_script": + if not record.get("sandbox_id"): + record["sandbox_id"] = await log_audit.resolve_sandbox_id(record.get("chat_id")) + try: + await asyncio.to_thread(log_audit.write_skill_sandbox_audit_sync, record) + except Exception as exc: # noqa: BLE001 + logger.warning("skill_sandbox_audit_task_failed", error=str(exc)) + if not log_writer_constants.SKILL_LOG_ENABLED: + return log_id + try: + await asyncio.to_thread(log_records.write_skill_call_sync, record) + except Exception as exc: # noqa: BLE001 + logger.warning("skill_call_log_task_failed", error=str(exc)) + return log_id + + +def schedule_tool_call_write(record: Dict[str, Any]) -> None: + spawn(write_tool_call(record), name="log_writer.tool_call") + + +def schedule_skill_call_write(record: Dict[str, Any]) -> None: + spawn(write_skill_call(record), name="log_writer.skill_call") diff --git a/src/backend/core/harness/log_writer_constants.py b/src/backend/core/harness/log_writer_constants.py new file mode 100644 index 00000000..ecaf3e65 --- /dev/null +++ b/src/backend/core/harness/log_writer_constants.py @@ -0,0 +1,32 @@ +"""Switches and size limits for call logging.""" +from __future__ import annotations +import os + +TOOL_LOG_ENABLED = os.getenv("TOOL_CALL_LOG_ENABLED", "true").lower() == "true" + + +SUBAGENT_LOG_ENABLED = os.getenv("SUBAGENT_LOG_ENABLED", "true").lower() == "true" + + +SKILL_LOG_ENABLED = os.getenv("SKILL_LOG_ENABLED", "true").lower() == "true" + + +SANDBOX_AUDIT_ENABLED = os.getenv("SANDBOX_AUDIT_ENABLED", "true").lower() == "true" + + +MAX_RESULT_BYTES = int(os.getenv("LOG_MAX_RESULT_BYTES", 64 * 1024)) + + +MAX_STDOUT_BYTES = int(os.getenv("LOG_MAX_STDOUT_BYTES", 64 * 1024)) + + +SANDBOX_TOOL_NAMES = { + "Bash", + "sandbox_put_artifact", + "sandbox_get_artifact", + "Write", + "Edit", + "Read", + "Grep", + "Glob", +} diff --git a/src/backend/core/harness/observation_writer.py b/src/backend/core/harness/observation_writer.py new file mode 100644 index 00000000..ad630370 --- /dev/null +++ b/src/backend/core/harness/observation_writer.py @@ -0,0 +1,134 @@ +"""Bounded background persistence for hook-path observability records. + +Hook stages and framework events are recorded on every model and tool step. +Waiting for their database writes would put observability on the execution +critical path, so they are queued here and written by one daemon thread. +``submit`` never blocks: a full queue drops the record and logs the loss. +""" + +from __future__ import annotations + +import atexit +import logging +import os +import queue +import threading +import time +from collections.abc import Callable, Sequence +from typing import Any + +logger = logging.getLogger(__name__) + +MAX_PENDING = 10_000 +MAX_BATCH = 500 +DROP_LOG_EVERY = 1_000 + +BatchWrite = Callable[[Any, Sequence[Any]], Any] + + +class ObservationWriter: + """Drain ``(sink, write, run_id, records)`` items in per-sink, per-run batches.""" + + def __init__(self, *, max_pending: int = MAX_PENDING, max_batch: int = MAX_BATCH) -> None: + self._max_pending = max(1, int(max_pending)) + self._max_batch = max(1, int(max_batch)) + self._lock = threading.Lock() + self._queue: queue.Queue | None = None + self._pid: int | None = None + self.dropped = 0 + + def _ensure_started(self) -> queue.Queue: + with self._lock: + # A forked worker inherits the queue but not the thread. + if self._queue is None or self._pid != os.getpid(): + self._queue = queue.Queue(self._max_pending) + self._pid = os.getpid() + threading.Thread( + target=self._run, + args=(self._queue,), + name="harness-observation-writer", + daemon=True, + ).start() + return self._queue + + def submit(self, sink: Any, write: BatchWrite, run_id: str, records: Sequence[Any]) -> bool: + """Queue ``write(sink, records)``; return False when the record was dropped.""" + if not records: + return True + try: + self._ensure_started().put_nowait((sink, write, run_id, tuple(records))) + return True + except queue.Full: + with self._lock: + self.dropped += 1 + dropped = self.dropped + if dropped == 1 or dropped % DROP_LOG_EVERY == 0: + logger.warning( + "Observation queue full (%d pending); dropped %d record batch(es) so far", + self._max_pending, + dropped, + ) + return False + + def _run(self, pending: queue.Queue) -> None: + while True: + items = [pending.get()] + while len(items) < self._max_batch: + try: + items.append(pending.get_nowait()) + except queue.Empty: + break + try: + self._write(items) + finally: + for _ in items: + pending.task_done() + + @staticmethod + def _write(items) -> None: + # Order is preserved within a sink and run, which is all sequence + # allocation depends on; one call per group lets stores batch a run. + groups: dict[tuple[int, BatchWrite, str], tuple[Any, list]] = {} + for sink, write, run_id, records in items: + groups.setdefault((id(sink), write, run_id), (sink, []))[1].extend(records) + for (_, write, run_id), (sink, records) in groups.items(): + try: + write(sink, records) + except Exception as exc: # noqa: BLE001 + logger.warning( + "Observation write failed for run %s (%d record(s)): %s", + run_id, + len(records), + exc, + ) + + def flush(self, timeout: float = 5.0) -> bool: + """Wait until every queued record is written; False on timeout.""" + pending = self._queue + if pending is None or self._pid != os.getpid(): + return True + deadline = time.monotonic() + max(0.0, timeout) + with pending.all_tasks_done: + while pending.unfinished_tasks: + remaining = deadline - time.monotonic() + if remaining <= 0: + return False + pending.all_tasks_done.wait(remaining) + return True + + +_WRITER = ObservationWriter() + + +def observation_writer() -> ObservationWriter: + return _WRITER + + +def flush_observations(timeout: float = 5.0) -> bool: + flushed = _WRITER.flush(timeout) + if not flushed: + logger.warning("Observation writes still pending after %.1fs flush", timeout) + return flushed + + +atexit.register(flush_observations) diff --git a/src/backend/core/harness/registry/__init__.py b/src/backend/core/harness/registry/__init__.py new file mode 100644 index 00000000..486ec361 --- /dev/null +++ b/src/backend/core/harness/registry/__init__.py @@ -0,0 +1,79 @@ +"""Optional feature modules that plug into core through registered hooks. + +Core never imports a module's implementation. A module contributes behaviour by +registering hooks from its ``setup()`` at startup; when the module is disabled +it registers nothing and core behaves as if the module were not installed. + +Shared modules are loaded in every edition. Additional modules are an edition concern (``core.harness.registry.edition``); whether +an installed module is switched on is the module's own decision, usually an env +flag read inside its ``setup()``. +""" + +from __future__ import annotations + +from collections.abc import Iterable +from importlib import import_module + +from core.harness.registry.user_cleanup import register_user_cleanup +from core.harness.registry.hook_factories import hook_specs, register_hook_factory + +from core.harness.registry.registry import AgentProfile, NativeToolContext, register_tool, install_tools, SandboxCredentials, register_sandbox_credentials, sandbox_credentials, agent_profile, channel_routes, is_enabled, lifecycle_hooks, mark_enabled, register_agent_profile, register_channel_route, register_lifecycle, register_router, register_sandbox_route, routers, sandbox_route +from core.infra.logging import get_logger + +logger = get_logger(__name__) + +_loaded = False + + +def load_models(installed: Iterable[str]) -> None: + """Register installed schemas independently of runtime enablement.""" + for package in installed: + import_module(f"{package}.models") + + +def load_extensions(installed: Iterable[str]) -> None: + """Import shared and edition modules and let it register itself. Idempotent.""" + global _loaded + if _loaded: + return + _loaded = True + from core.harness.registry.edition import EDITION_EXTENSIONS + + installed = tuple(installed) + load_models(installed) + from core.harness.registry.schema_lifecycle import load_schema_lifecycle + load_schema_lifecycle(installed) + for module_name in (*(f"{name}.extension" for name in installed), *EDITION_EXTENSIONS): + module = import_module(module_name) + if module.setup(): + mark_enabled(module.NAME) + logger.info("extension_enabled", extension=module.NAME) + else: + logger.info("extension_disabled", extension=module.NAME) + + +__all__ = [ + "register_user_cleanup", + "hook_specs", + "register_hook_factory", + "AgentProfile", + "NativeToolContext", + "register_tool", + "install_tools", + "SandboxCredentials", + "register_sandbox_credentials", + "sandbox_credentials", + "agent_profile", + "channel_routes", + "is_enabled", + "lifecycle_hooks", + "load_extensions", + "load_models", + "register_agent_profile", + "register_channel_route", + "register_lifecycle", + "register_router", + "register_sandbox_route", + "routers", + "sandbox_route", +] diff --git a/src/backend/core/harness/registry/authentication.py b/src/backend/core/harness/registry/authentication.py new file mode 100644 index 00000000..ccd7abf4 --- /dev/null +++ b/src/backend/core/harness/registry/authentication.py @@ -0,0 +1,26 @@ +"""Optional host authentication before the ordinary cookie/session resolver.""" +from __future__ import annotations +from collections.abc import Callable +from dataclasses import dataclass +from typing import Any + + +def register_host_authentication(resolver: Callable[[Any, Any], Any]) -> None: + pass + registration_state().resolver = resolver + + +def resolve_host_user(request, db): + return registration_state().resolver(request, db) if registration_state().resolver is not None else None + + +@dataclass +class RegistrationState: + resolver: Callable[[Any, Any], Any] | None = None + + +_REGISTRATIONS = RegistrationState() + + +def registration_state() -> RegistrationState: + return _REGISTRATIONS diff --git a/src/backend/core/harness/registry/build_governance.py b/src/backend/core/harness/registry/build_governance.py new file mode 100644 index 00000000..1fd4907b --- /dev/null +++ b/src/backend/core/harness/registry/build_governance.py @@ -0,0 +1,51 @@ +"""Governance policy applied before capability definitions are persisted.""" +from dataclasses import dataclass +from functools import lru_cache +from typing import Callable + + +@dataclass +class BuildGovernance: + ensure: Callable | None = None + validate: Callable | None = None + tag_options: Callable | None = None + import_policy: Callable | None = None + + +@lru_cache(maxsize=1) +def build_governance() -> BuildGovernance: + return BuildGovernance() + + +def register_build_governance(*, ensure, validate, tag_options, import_policy): + services = build_governance() + services.ensure, services.validate = ensure, validate + services.tag_options, services.import_policy = tag_options, import_policy + + +def ensure_ontology_build_valid(*args, **kwargs): + ensure = build_governance().ensure + if ensure is None: + raise RuntimeError("Capability build governance is not registered") + return ensure(*args, **kwargs) + + +def validate_build(db, **kwargs): + validate = build_governance().validate + if validate is None: + raise RuntimeError("Capability build validator is not registered") + return validate(db, **kwargs) + + +def list_asset_tag_options(db, asset_type): + options = build_governance().tag_options + if options is None: + raise RuntimeError("Capability tag governance is not registered") + return options(db, asset_type) + + +def resolve_plugin_import_ontology_validation(*args, **kwargs): + policy = build_governance().import_policy + if policy is None: + raise RuntimeError("Capability import governance is not registered") + return policy(*args, **kwargs) diff --git a/src/backend/core/harness/registry/cache_invalidation.py b/src/backend/core/harness/registry/cache_invalidation.py new file mode 100644 index 00000000..02c7286b --- /dev/null +++ b/src/backend/core/harness/registry/cache_invalidation.py @@ -0,0 +1,27 @@ +"""Named host cache invalidations invoked by domain mutations.""" +from collections.abc import Callable +from dataclasses import dataclass, field +from functools import lru_cache +from threading import RLock + + +@dataclass +class CacheInvalidators: + callbacks: dict[tuple[str, str], Callable[[], None]] = field(default_factory=dict) + lock: RLock = field(default_factory=RLock) + + def register(self, stage: str, name: str, callback: Callable[[], None]) -> None: + with self.lock: + self.callbacks[stage, name] = callback + + def run(self, stage: str) -> None: + with self.lock: + callbacks = tuple(callback for (phase, _), callback in self.callbacks.items() if phase == stage) + for callback in callbacks: + callback() + + +@lru_cache(maxsize=1) +def cache_invalidators() -> CacheInvalidators: + """Process-owned registration state; callbacks are installed by the host.""" + return CacheInvalidators() diff --git a/src/backend/core/harness/registry/capability_management.py b/src/backend/core/harness/registry/capability_management.py new file mode 100644 index 00000000..564fb6c0 --- /dev/null +++ b/src/backend/core/harness/registry/capability_management.py @@ -0,0 +1,27 @@ +"""Installed host implementation of local capability management.""" +from dataclasses import dataclass +from functools import lru_cache +from typing import Callable + + +@dataclass +class ManagerRegistration: + execute: Callable | None = None + + +@lru_cache(maxsize=1) +def manager_registration() -> ManagerRegistration: + return ManagerRegistration() + + +def register_manager_executor(execute: Callable) -> None: + manager_registration().execute = execute + + +def execute_manager(*, manager, server_id, headers, cloud_backed, tool_name, input_schema, arguments): + execute = manager_registration().execute + if execute is None: + raise RuntimeError("Local capability manager is not registered") + return execute(manager=manager, server_id=server_id, headers=headers, + cloud_backed=cloud_backed, tool_name=tool_name, + input_schema=input_schema, arguments=arguments) diff --git a/src/backend/core/harness/registry/capability_revisions.py b/src/backend/core/harness/registry/capability_revisions.py new file mode 100644 index 00000000..ede0293a --- /dev/null +++ b/src/backend/core/harness/registry/capability_revisions.py @@ -0,0 +1,34 @@ +"""Authorized capability revision snapshots supplied by the installed host.""" +from dataclasses import dataclass +from functools import lru_cache +from typing import Protocol + + +class CapabilityRevisions(Protocol): + def model(self, db, kind, key): ... + def snapshot_in_session(self, db, user, kind, key): ... + def snapshot(self, user, kind, key): ... + def commit(self, user, kind, key, body): ... + def revision(self, files): ... + def record_id(self, prefix, user, kind, key): ... + + +@dataclass +class RevisionRegistration: + provider: CapabilityRevisions | None = None + + +@lru_cache(maxsize=1) +def revision_registration() -> RevisionRegistration: + return RevisionRegistration() + + +def register_capability_revisions(provider: CapabilityRevisions) -> None: + revision_registration().provider = provider + + +def capability_revisions() -> CapabilityRevisions: + provider = revision_registration().provider + if provider is None: + raise RuntimeError("Capability revision service is not registered") + return provider diff --git a/src/backend/core/harness/registry/capability_runtime.py b/src/backend/core/harness/registry/capability_runtime.py new file mode 100644 index 00000000..95326b57 --- /dev/null +++ b/src/backend/core/harness/registry/capability_runtime.py @@ -0,0 +1,74 @@ +"""Frozen capability authorization supplied by the execution host.""" +from __future__ import annotations +from dataclasses import dataclass +from typing import Any, Protocol +from core.harness.capability_errors import ViewUnavailable + + +class CapabilityRuntime(Protocol): + def pin_agent_definition(self, run_id, user_id, agent, *, scope_id=""): ... + def view_for_execution(self, run_id, user_id, *, scope_id=""): ... + def get(self, run_id: str, scope_id: str = "") -> Any: ... + def validate(self, run: Any, *, user_id: str | None = None, + execution_plane: str = "local", only_skill: str | None = None) -> None: ... + def validate_activation(self, run: Any, nodes: Any, *, available_mcp=()) -> None: ... + def require_root_tool_scope(self, run_id: str, user_id: str, tool_call_id: str, tool_name: str) -> None: ... + def child_scope(self, parent_scope: str, kind: str, *identities: str) -> str: ... + + +def register_capability_runtime(provider: CapabilityRuntime) -> None: + pass + registration_state().provider = provider + + +def require_runtime() -> CapabilityRuntime: + if registration_state().provider is None: + raise ViewUnavailable("Execution host capability runtime is not registered") + return registration_state().provider + + +def get(run_id: str, scope_id: str = ""): + return require_runtime().get(run_id, scope_id=scope_id) + + +def validate(run, *, user_id=None, execution_plane="local", only_skill=None): + return require_runtime().validate(run, user_id=user_id, execution_plane=execution_plane, only_skill=only_skill) + + +def validate_activation(run, nodes, *, available_mcp=()): + return require_runtime().validate_activation(run, nodes, available_mcp=available_mcp) + + +def require_root_tool_scope(run_id, user_id, tool_call_id, tool_name): + return require_runtime().require_root_tool_scope(run_id, user_id, tool_call_id, tool_name) + + +def child_scope(parent_scope, kind, *identities): + # Scope identities are used even on cloud hosts without device capabilities. + import hashlib + import json + from core.harness.capability_errors import IntegrityFailed + if not kind or not identities or any(not str(value or "").strip() for value in identities): + raise IntegrityFailed("capability scope requires durable invocation identities") + material = json.dumps([str(parent_scope or ""), str(kind), *map(str, identities)], separators=(",", ":")) + return "s_" + hashlib.sha256(material.encode()).hexdigest() + + +def pin_agent_definition(run_id, user_id, agent, *, scope_id=""): + return require_runtime().pin_agent_definition(run_id, user_id, agent, scope_id=scope_id) + + +def view_for_execution(run_id, user_id, *, scope_id=""): + return require_runtime().view_for_execution(run_id, user_id, scope_id=scope_id) + + +@dataclass +class RegistrationState: + provider: CapabilityRuntime | None = None + + +_REGISTRATIONS = RegistrationState() + + +def registration_state() -> RegistrationState: + return _REGISTRATIONS diff --git a/src/backend/core/harness/registry/catalog_options.py b/src/backend/core/harness/registry/catalog_options.py new file mode 100644 index 00000000..e7a70220 --- /dev/null +++ b/src/backend/core/harness/registry/catalog_options.py @@ -0,0 +1,26 @@ +"""Optional domain-owned option lists used by agent administration.""" +from __future__ import annotations +from collections.abc import Callable +from dataclasses import dataclass, field +from typing import Any + + +def register_catalog_options(kind: str, provider: Callable[[Any], list[dict]]) -> None: + registration_state().providers[kind] = provider + + +def catalog_options(kind: str, db: Any) -> list[dict]: + provider = registration_state().providers.get(kind) + return provider(db) if provider is not None else [] + + +@dataclass +class RegistrationState: + providers: dict[str, Callable[[Any], list[dict]]] = field(default_factory=dict) + + +_REGISTRATIONS = RegistrationState() + + +def registration_state() -> RegistrationState: + return _REGISTRATIONS diff --git a/src/backend/core/harness/registry/chat_execution.py b/src/backend/core/harness/registry/chat_execution.py new file mode 100644 index 00000000..3c8f558a --- /dev/null +++ b/src/backend/core/harness/registry/chat_execution.py @@ -0,0 +1,27 @@ +"""Top-level chat execution supplied to durable chat workers by the host.""" +from collections.abc import AsyncIterator, Callable +from dataclasses import dataclass +from functools import lru_cache +from typing import Any + + +@dataclass +class ChatExecution: + stream: Callable[..., AsyncIterator[Any]] | None = None + + +@lru_cache(maxsize=1) +def chat_execution() -> ChatExecution: + return ChatExecution() + + +def register_chat_execution(stream: Callable[..., AsyncIterator[Any]]) -> None: + chat_execution().stream = stream + + +def stream_chat_workflow(*, raw_user_message, session_messages, user_message, context): + stream = chat_execution().stream + if stream is None: + raise RuntimeError("Chat execution workflow is not registered") + return stream(raw_user_message=raw_user_message, session_messages=session_messages, + user_message=user_message, context=context) diff --git a/src/backend/core/harness/registry/chat_history.py b/src/backend/core/harness/registry/chat_history.py new file mode 100644 index 00000000..5dc083ba --- /dev/null +++ b/src/backend/core/harness/registry/chat_history.py @@ -0,0 +1,21 @@ +"""Composition contract for user-scoped history tools.""" +from dataclasses import dataclass +from functools import lru_cache +from typing import Callable + +@dataclass +class HistoryRegistration: + factory: Callable | None = None + +@lru_cache(maxsize=1) +def history_registration(): + return HistoryRegistration() + +def register_chat_history(factory: Callable): + history_registration().factory = factory + +def history_operations(**scope): + factory = history_registration().factory + if factory is None: + raise RuntimeError("Chat history services are not registered") + return factory(**scope) diff --git a/src/backend/core/harness/registry/compaction.py b/src/backend/core/harness/registry/compaction.py new file mode 100644 index 00000000..355237be --- /dev/null +++ b/src/backend/core/harness/registry/compaction.py @@ -0,0 +1,62 @@ +"""Conversation-owned compaction and checkpoint services for the model runtime.""" +from dataclasses import dataclass +from functools import lru_cache +from typing import Callable + + +@dataclass +class CompactionServices: + run: Callable | None = None + mid_turn: Callable | None = None + token_limit: Callable | None = None + trigger_ratio: Callable | None = None + checkpoint: Callable | None = None + + +@lru_cache(maxsize=1) +def compaction_services() -> CompactionServices: + return CompactionServices() + + +def register_compaction_services(*, run, mid_turn, token_limit, trigger_ratio, checkpoint): + services = compaction_services() + services.run = run + services.mid_turn = mid_turn + services.token_limit = token_limit + services.trigger_ratio = trigger_ratio + services.checkpoint = checkpoint + + +async def run_compaction(*args, **kwargs): + run = compaction_services().run + if run is None: + raise RuntimeError("Conversation compaction is not registered") + return await run(*args, **kwargs) + + +async def run_mid_turn_compaction(*args, **kwargs): + run = compaction_services().mid_turn + if run is None: + raise RuntimeError("Mid-turn compaction is not registered") + return await run(*args, **kwargs) + + +def resolve_token_limit(context_window, *, ratio=None): + resolve = compaction_services().token_limit + if resolve is None: + raise RuntimeError("Compaction token policy is not registered") + return resolve(context_window, ratio=ratio) + + +def resolve_trigger_ratio(): + resolve = compaction_services().trigger_ratio + if resolve is None: + raise RuntimeError("Compaction trigger policy is not registered") + return resolve() + + +def create_context_checkpoint(**kwargs): + factory = compaction_services().checkpoint + if factory is None: + raise RuntimeError("Context checkpoint service is not registered") + return factory(**kwargs) diff --git a/src/backend/core/harness/registry/device_catalog.py b/src/backend/core/harness/registry/device_catalog.py new file mode 100644 index 00000000..ee18da46 --- /dev/null +++ b/src/backend/core/harness/registry/device_catalog.py @@ -0,0 +1,79 @@ +"""Optional execution-device catalog projected into domain selection.""" +from dataclasses import dataclass +from functools import lru_cache +from typing import Protocol, Any + + +class DeviceCatalog(Protocol): + def account_plugin_installations(self, user_id): ... + def owned_identities(self, user_id): ... + def resolve_plugin(self, slug, user_id, *, install_id=None): ... + def active(self) -> bool: ... + def agent_entries(self) -> list[dict]: ... + def plugin_entries(self, user_id=None) -> list[dict]: ... + def catalog_overlay(self, user_id=None) -> dict: ... + def installations(self, kind: str, user_id=None) -> list[Any]: ... + def merge_items(self, base, projected, *, key): ... + + +@dataclass +class CatalogRegistration: + provider: DeviceCatalog | None = None + + +@lru_cache(maxsize=1) +def catalog_registration() -> CatalogRegistration: + return CatalogRegistration() + + +def register_device_catalog(provider: DeviceCatalog) -> None: + catalog_registration().provider = provider + + +def active() -> bool: + provider = catalog_registration().provider + return bool(provider and provider.active()) + + +def agent_entries(): + provider = catalog_registration().provider + return provider.agent_entries() if provider else [] + + +def plugin_entries(user_id=None): + provider = catalog_registration().provider + return provider.plugin_entries(user_id) if provider else [] + + +def catalog_overlay(user_id=None): + provider = catalog_registration().provider + return provider.catalog_overlay(user_id) if provider else {} + + +def installations(kind, user_id=None): + provider = catalog_registration().provider + return provider.installations(kind, user_id=user_id) if provider else [] + + +def merge_items(base, projected, *, key): + provider = catalog_registration().provider + if provider is None: + if projected: + raise RuntimeError("Device catalog is not registered") + return base + return provider.merge_items(base, projected, key=key) + + +def resolve_plugin(slug, user_id, *, install_id=None): + provider = catalog_registration().provider + return provider.resolve_plugin(slug, user_id, install_id=install_id) if provider else None + + +def account_plugin_installations(user_id): + provider = catalog_registration().provider + return provider.account_plugin_installations(user_id) if provider else [] + + +def owned_identities(user_id): + provider = catalog_registration().provider + return provider.owned_identities(user_id) if provider else set() diff --git a/src/backend/core/extensions/edition.py b/src/backend/core/harness/registry/edition.py similarity index 100% rename from src/backend/core/extensions/edition.py rename to src/backend/core/harness/registry/edition.py diff --git a/src/backend/core/harness/registry/evaluation.py b/src/backend/core/harness/registry/evaluation.py new file mode 100644 index 00000000..4c82baaa --- /dev/null +++ b/src/backend/core/harness/registry/evaluation.py @@ -0,0 +1,72 @@ +"""Reserved execution policy installed by the evaluation host.""" +from __future__ import annotations +from dataclasses import dataclass +from typing import Any, Protocol + + +class EvaluationPolicy(Protocol): + def apply_context(self, context: dict) -> dict: ... + def parse_scope(self, **identity: Any) -> Any: ... + def project_agent(self, agent: Any) -> Any: ... + def current_paths(self) -> Any: ... + def register_tools(self, toolkit: Any, scope: Any, **options: Any) -> None: ... + async def authorize(self, request: Any, user_id: str, *, agent_scoped: bool = False) -> None: ... + + +def register_evaluation_policy(policy: EvaluationPolicy) -> None: + pass + registration_state().policy = policy + + +def is_evaluation_session(value): + return isinstance(value, str) and value.startswith("eval_") + + +def require_policy(): + if registration_state().policy is None: + raise ValueError("Reserved evaluation execution is not installed") + return registration_state().policy + + +def apply_evaluation_scope(context): + if registration_state().policy is None: + if any(is_evaluation_session(context.get(k)) for k in ("chat_id", "sandbox_session_id")): + require_policy() + return context + return registration_state().policy.apply_context(context) + + +def parse_evaluation_scope(**identity): + if registration_state().policy is None and not any(is_evaluation_session(identity.get(k)) for k in ("chat_id", "session_id")): + return None + return require_policy().parse_scope(**identity) + + +def evaluation_user_agent(agent): + return require_policy().project_agent(agent) + + +def current_path_rules(): + return registration_state().policy.current_paths() if registration_state().policy is not None else None + + +def register_evaluation_tools(toolkit, scope, **options): + require_policy().register_tools(toolkit, scope, **options) + + +async def authorize(request, user_id, *, agent_scoped=False): + if not is_evaluation_session(request.chat_id): + return + await require_policy().authorize(request, user_id, agent_scoped=agent_scoped) + + +@dataclass +class RegistrationState: + policy: EvaluationPolicy | None = None + + +_REGISTRATIONS = RegistrationState() + + +def registration_state() -> RegistrationState: + return _REGISTRATIONS diff --git a/src/backend/core/harness/registry/execution_account.py b/src/backend/core/harness/registry/execution_account.py new file mode 100644 index 00000000..8c9cd4ed --- /dev/null +++ b/src/backend/core/harness/registry/execution_account.py @@ -0,0 +1,35 @@ +"""Host account authorization for account-scoped capability selection.""" +from __future__ import annotations +from dataclasses import dataclass +from typing import Protocol + +class ExecutionAccount(Protocol): + def current_account_profile(self) -> str | None: ... + def current_local_user_id(self) -> str | None: ... + def account_authorized_for(self, user_id: str | None) -> bool: ... + + +def register_execution_account(provider: ExecutionAccount | None) -> None: + pass + registration_state().provider = provider + +def current_account_profile() -> str | None: + return registration_state().provider.current_account_profile() if registration_state().provider is not None else None + +def current_local_user_id() -> str | None: + return registration_state().provider.current_local_user_id() if registration_state().provider is not None else None + +def account_authorized_for(user_id: str | None) -> bool: + return bool(registration_state().provider is not None and registration_state().provider.account_authorized_for(user_id)) + + +@dataclass +class RegistrationState: + provider: ExecutionAccount | None = None + + +_REGISTRATIONS = RegistrationState() + + +def registration_state() -> RegistrationState: + return _REGISTRATIONS diff --git a/src/backend/core/harness/registry/execution_catalog.py b/src/backend/core/harness/registry/execution_catalog.py new file mode 100644 index 00000000..fd90b656 --- /dev/null +++ b/src/backend/core/harness/registry/execution_catalog.py @@ -0,0 +1,51 @@ +"""Host-owned execution catalog selection, optional on cloud-only hosts.""" +from __future__ import annotations +from dataclasses import dataclass +from typing import Any, Protocol + +class ExecutionCatalog(Protocol): + def local_plugin_configs(self, user_id, enabled_only=True) -> dict: ... + def bridge_context(self) -> dict[str, Any] | None: ... + def apply_to_enabled_mcp_ids(self, ids: list[str] | None) -> list[str] | None: ... + def apply_to_enabled_skill_ids(self, ids: list[str] | None) -> list[str] | None: ... + def cloud_gateway_mcp_configs(self, enabled_mcp_ids=None, *, resolution_out=None, unavailable_out=None) -> dict: ... + + +def register_execution_catalog(provider: ExecutionCatalog | None) -> None: + pass + registration_state().provider = provider + +def bridge_context(): + return registration_state().provider.bridge_context() if registration_state().provider is not None else None + +def apply_to_enabled_mcp_ids(ids): + return registration_state().provider.apply_to_enabled_mcp_ids(ids) if registration_state().provider is not None else ids + +def apply_to_enabled_skill_ids(ids): + return registration_state().provider.apply_to_enabled_skill_ids(ids) if registration_state().provider is not None else ids + +def cloud_gateway_mcp_configs(enabled_mcp_ids=None, *, resolution_out=None, unavailable_out=None): + if registration_state().provider is None: + return {} + return registration_state().provider.cloud_gateway_mcp_configs(enabled_mcp_ids, resolution_out=resolution_out, unavailable_out=unavailable_out) + + +def local_plugin_configs(user_id, enabled_only=True): + return registration_state().provider.local_plugin_configs(user_id, enabled_only=enabled_only) if registration_state().provider else {} + + +def authorizes_manager(user_id, server_id, manager): + config = local_plugin_configs(user_id).get(server_id) + return bool(config and config.get("gateway_plugin") == manager) + + +@dataclass +class RegistrationState: + provider: ExecutionCatalog | None = None + + +_REGISTRATIONS = RegistrationState() + + +def registration_state() -> RegistrationState: + return _REGISTRATIONS diff --git a/src/backend/core/harness/registry/execution_identity.py b/src/backend/core/harness/registry/execution_identity.py new file mode 100644 index 00000000..dc0308d2 --- /dev/null +++ b/src/backend/core/harness/registry/execution_identity.py @@ -0,0 +1,78 @@ +"""Host-owned execution identity contract for core model and tool operations.""" +from __future__ import annotations +from contextlib import AbstractContextManager +from dataclasses import dataclass +from typing import Any, Protocol + + +class ExecutionIdentity(Protocol): + def context_headers(self, *, model=False) -> dict[str, str]: ... + def notify_cloud_changed(self) -> None: ... + def get_state(self) -> dict[str, Any] | None: ... + def state_fingerprint(self, state: dict[str, Any] | None) -> str: ... + def require_current_account(self, state: dict[str, Any]) -> None: ... + def ensure_current_authorization(self, state: dict[str, Any] | None = None) -> None: ... + def cloud_headers(self, state: dict[str, Any]) -> dict[str, str]: ... + def account_scope(self, state: dict[str, Any]) -> AbstractContextManager: ... + def clear_state(self) -> None: ... + + +def register_execution_identity(provider: ExecutionIdentity | None) -> None: + pass + registration_state().provider = provider + + +def require_provider() -> ExecutionIdentity: + if registration_state().provider is None: + raise RuntimeError("Execution identity provider is not registered") + return registration_state().provider + + +def get_state() -> dict[str, Any] | None: + return registration_state().provider.get_state() if registration_state().provider is not None else None + + +def state_fingerprint(state: dict[str, Any] | None) -> str: + if state is None and registration_state().provider is None: + return "" + return require_provider().state_fingerprint(state) + + +def require_current_account(state: dict[str, Any]) -> None: + require_provider().require_current_account(state) + + +def ensure_current_authorization(state: dict[str, Any] | None = None) -> None: + require_provider().ensure_current_authorization(state) + + +def cloud_headers(state: dict[str, Any]) -> dict[str, str]: + return require_provider().cloud_headers(state) + + +def account_scope(state: dict[str, Any]) -> AbstractContextManager: + return require_provider().account_scope(state) + + +def clear_state() -> None: + require_provider().clear_state() + + +def context_headers(*, model=False): + return registration_state().provider.context_headers(model=model) if registration_state().provider else {} + + +def notify_cloud_changed(): + return require_provider().notify_cloud_changed() + + +@dataclass +class RegistrationState: + provider: ExecutionIdentity | None = None + + +_REGISTRATIONS = RegistrationState() + + +def registration_state() -> RegistrationState: + return _REGISTRATIONS diff --git a/src/backend/core/harness/registry/hook_factories.py b/src/backend/core/harness/registry/hook_factories.py new file mode 100644 index 00000000..11b41778 --- /dev/null +++ b/src/backend/core/harness/registry/hook_factories.py @@ -0,0 +1,69 @@ +"""Per-run hooks contributed by installed modules.""" +from __future__ import annotations + +from collections.abc import Callable, Iterable, Iterator +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from core.harness.hooks import HookSpec + +HookFactory = Callable[[Any], Iterable["HookSpec"]] + + +def register_hook_factory(factory: HookFactory) -> None: + """Register once; invoke afresh for each agent assembly.""" + if factory not in registration_state().factories: + registration_state().factories.append(factory) + + +def hook_specs(request: Any) -> Iterator[HookSpec]: + for factory in tuple(registration_state().factories): + yield from factory(request) + + +# Run-domain policies are created at assembly time, never shared between agents. +from dataclasses import dataclass, field +from functools import lru_cache + + +@dataclass +class RunHookFactories: + steering: Callable[[], Any] | None = None + command_completion: Callable[[], Iterable[HookSpec]] | None = None + + +@lru_cache(maxsize=1) +def run_hook_factories() -> RunHookFactories: + return RunHookFactories() + + +def register_run_hook_factories(*, steering, command_completion) -> None: + factories = run_hook_factories() + factories.steering = steering + factories.command_completion = command_completion + + +def create_steering_policy(): + factory = run_hook_factories().steering + if factory is None: + raise RuntimeError("Run steering policy is not registered") + return factory() + + +def command_completion_hooks(): + factory = run_hook_factories().command_completion + if factory is None: + raise RuntimeError("Command completion hooks are not registered") + return factory() + + +@dataclass +class RegistrationState: + factories: list[HookFactory] = field(default_factory=list) + + +_REGISTRATIONS = RegistrationState() + + +def registration_state() -> RegistrationState: + return _REGISTRATIONS diff --git a/src/backend/core/harness/registry/host_runtime.py b/src/backend/core/harness/registry/host_runtime.py new file mode 100644 index 00000000..199a56a6 --- /dev/null +++ b/src/backend/core/harness/registry/host_runtime.py @@ -0,0 +1,13 @@ +"""Host policy for operations that must be forwarded to a remote execution plane.""" +from collections.abc import Callable + +_remote_execution_required: Callable[[], bool] = lambda: False + + +def register_remote_execution_policy(predicate: Callable[[], bool]) -> None: + global _remote_execution_required + _remote_execution_required = predicate + + +def requires_remote_execution() -> bool: + return bool(_remote_execution_required()) diff --git a/src/backend/core/harness/registry/job_operations.py b/src/backend/core/harness/registry/job_operations.py new file mode 100644 index 00000000..8348949b --- /dev/null +++ b/src/backend/core/harness/registry/job_operations.py @@ -0,0 +1,21 @@ +"""Host binding for scoped job action operations.""" +from dataclasses import dataclass +from functools import lru_cache +from typing import Callable + +@dataclass +class JobRegistration: + factory: Callable | None = None + +@lru_cache(maxsize=1) +def job_registration(): + return JobRegistration() + +def register_job_operations(factory: Callable): + job_registration().factory = factory + +def job_operations(**scope): + factory = job_registration().factory + if factory is None: + raise RuntimeError("Job operations are not registered") + return factory(**scope) diff --git a/src/backend/core/harness/registry/local_files.py b/src/backend/core/harness/registry/local_files.py new file mode 100644 index 00000000..fa6f13ca --- /dev/null +++ b/src/backend/core/harness/registry/local_files.py @@ -0,0 +1,65 @@ +"""Host-owned file grants and recovery snapshots used by core operations.""" +from __future__ import annotations + +from dataclasses import dataclass +from functools import lru_cache +from typing import Protocol, Any + + +class LocalSecurityProvider(Protocol): + def grants_for_gate(self) -> list[Any]: ... + def policy_for_gate(self, approval_mode: str) -> Any: ... + + +class LocalSnapshotProvider(Protocol): + def snapshot(self, real_path: str) -> str | None: ... + def maybe_snapshot_local(self, real_path: str) -> str | None: ... + + +@dataclass +class LocalFileServices: + security: LocalSecurityProvider | None = None + snapshots: LocalSnapshotProvider | None = None + + +@lru_cache(maxsize=1) +def local_file_services() -> LocalFileServices: + return LocalFileServices() + + +def register_local_file_services(security: LocalSecurityProvider, + snapshots: LocalSnapshotProvider) -> None: + services = local_file_services() + services.security = security + services.snapshots = snapshots + + +def grants_for_gate() -> list[Any]: + provider = local_file_services().security + if provider is None: + raise RuntimeError("Local file security provider is not registered") + return provider.grants_for_gate() + + +def policy_for_gate(approval_mode: str) -> Any: + provider = local_file_services().security + if provider is None: + raise RuntimeError("Local file security provider is not registered") + return provider.policy_for_gate(approval_mode) + + +def snapshot(real_path: str) -> str | None: + provider = local_file_services().snapshots + if provider is None: + raise RuntimeError("Local file snapshot provider is not registered") + return provider.snapshot(real_path) + + +def maybe_snapshot_local(real_path: str) -> str | None: + from core.config.local_mode import local_mode_enabled + if not local_mode_enabled(): + return None + provider = local_file_services().snapshots + if provider is None: + raise RuntimeError("Local file snapshot provider is not registered") + return provider.maybe_snapshot_local(real_path) diff --git a/src/backend/core/harness/registry/local_publication.py b/src/backend/core/harness/registry/local_publication.py new file mode 100644 index 00000000..f5a35e5e --- /dev/null +++ b/src/backend/core/harness/registry/local_publication.py @@ -0,0 +1,69 @@ +"""Host-owned publication of local skill views and installation revisions.""" +from __future__ import annotations +from dataclasses import dataclass +from typing import Any, Protocol + +class LocalPublication(Protocol): + def management_lock_path(self): ... + def provision_plugin_assets(self, slug): ... + def migrate_local_plugins(self): ... + def migrate_local_skills(self): ... + async def ensure_native_servers(self, configs): ... + def publish_local_plugin(self, definition, *, owner_user_id): ... + def remove_local_plugin(self, slug): ... + def skill_backend(self, *, priority=150, local=False): ... + def merge_skill_infos(self, groups, *, backend_hash): ... + def view_generation(self): ... + def skill_root(self): ... + def merge_visible_agents(self, user_id, rows): ... + def account_agent_definition(self, agent_id): ... + def publish_local_agent(self, definition): ... + def remove_local_agent(self, agent_id): ... + def rebuild_device_view(self) -> Any: ... + def rebuild_user_view(self, user_id: str) -> Any: ... + def rebuild_views(self, user_id: str | None) -> Any: ... + def remove_local_skill(self, skill_id: str) -> bool: ... + def prune_local(self, owners: dict) -> int: ... + def publish_local_skill(self, *args: Any, **kwargs: Any) -> Any: ... + def device_view_dir(self) -> Any: ... + def user_view_dir(self, user_id: str | None) -> Any: ... + def stamp_installation(self, db: Any, install_id: str, stamp: Any) -> None: ... + def claim_revision(self, db: Any, install_id: str, expected_revision: str) -> int: ... + + +def register_local_publication(provider: LocalPublication) -> None: + pass + registration_state().provider = provider + +def local_publication() -> LocalPublication: + if registration_state().provider is None: + raise RuntimeError("Host must register local publication before changing local capabilities") + return registration_state().provider + + +def provision_plugin_assets(slug): + return local_publication().provision_plugin_assets(slug) + + +def migrate_local_plugins(): + return local_publication().migrate_local_plugins() + + +def migrate_local_skills(): + return local_publication().migrate_local_skills() + + +async def ensure_native_servers(configs): + return await local_publication().ensure_native_servers(configs) + + +@dataclass +class RegistrationState: + provider: LocalPublication | None = None + + +_REGISTRATIONS = RegistrationState() + + +def registration_state() -> RegistrationState: + return _REGISTRATIONS diff --git a/src/backend/core/harness/registry/marketplace_catalog.py b/src/backend/core/harness/registry/marketplace_catalog.py new file mode 100644 index 00000000..c13b7ad4 --- /dev/null +++ b/src/backend/core/harness/registry/marketplace_catalog.py @@ -0,0 +1,44 @@ +"""Marketplace-owned listings, grants and installation operations.""" +from __future__ import annotations +from dataclasses import dataclass +from typing import Any, Protocol + +class MarketplaceCatalog(Protocol): + def plugin_metadata(self, db: Any, slug: str) -> dict: ... + def mcp_version(self, db: Any, slug: str) -> tuple[Any, Any]: ... + def normalize_auth_config(self, config: Any, schema: list) -> dict: ... + def installation_secrets(self, version: Any, credentials: dict, existing: dict, *, auth_method: str) -> dict: ... + async def install_market_item(self, db: Any, slug: str, **kwargs: Any) -> dict: ... + def market_package(self, db: Any, slug: str) -> Any: ... + def publish_package(self, db: Any, slug: str, fields: dict, *, created_by: str | None, now: Any) -> str: ... + def delete_package(self, db: Any, slug: str) -> None: ... + def excluded_skills(self, db: Any, slug: str) -> set[str]: ... + def exclude_skill(self, db: Any, slug: str, skill_name: str, *, created_by: str | None) -> bool: ... + def suspended_mcp_ids(self, db: Any, ids: list[str]) -> set[str]: ... + def list_marketplace_skills(self, db: Any) -> list[dict]: ... + def list_marketplace_agents(self, db: Any) -> list[dict]: ... + def list_plugins(self, db: Any, user_id: str | None) -> list[dict]: ... + def install_marketplace_skill(self, db: Any, slug: str, *, owner_user_id: str | None) -> dict: ... + def install_marketplace_agent(self, db: Any, slug: str, *, owner_user_id: str | None) -> dict: ... + + +def register_marketplace_catalog(provider: MarketplaceCatalog) -> None: + pass + registration_state().provider = provider + +def marketplace_catalog() -> MarketplaceCatalog: + if registration_state().provider is None: + raise RuntimeError("Host must register the marketplace catalog") + return registration_state().provider + + +@dataclass +class RegistrationState: + provider: MarketplaceCatalog | None = None + + +_REGISTRATIONS = RegistrationState() + + +def registration_state() -> RegistrationState: + return _REGISTRATIONS diff --git a/src/backend/core/harness/registry/mcp_clients.py b/src/backend/core/harness/registry/mcp_clients.py new file mode 100644 index 00000000..65ca4de3 --- /dev/null +++ b/src/backend/core/harness/registry/mcp_clients.py @@ -0,0 +1,21 @@ +"""Host-owned SDK client factory used by capability connectivity probes.""" +from dataclasses import dataclass +from functools import lru_cache +from typing import Callable, Any + +@dataclass +class ClientFactory: + create: Callable[..., Any] | None = None + +@lru_cache(maxsize=1) +def client_factory() -> ClientFactory: + return ClientFactory() + +def register_mcp_client_factory(factory: Callable[..., Any]) -> None: + client_factory().create = factory + +def discover_tools(*args, **kwargs): + factory = client_factory().create + if factory is None: + raise RuntimeError("MCP client factory is not registered") + return factory(*args, **kwargs) diff --git a/src/backend/core/harness/registry/model_credentials.py b/src/backend/core/harness/registry/model_credentials.py new file mode 100644 index 00000000..85410553 --- /dev/null +++ b/src/backend/core/harness/registry/model_credentials.py @@ -0,0 +1,70 @@ +"""Host-selected request authorization; no execution-environment imports.""" +from __future__ import annotations +from collections.abc import Callable +from dataclasses import dataclass +from typing import Protocol + + +def register_request_header_resolver(resolver: Callable | None) -> None: + pass + registration_state().resolver = resolver + + +def prepare_request_headers(api_key, base_url, captured_state=None): + if registration_state().resolver is not None: + return registration_state().resolver(api_key, base_url, captured_state) + return {"Content-Type": "application/json", "Authorization": "Bearer " + api_key} + + +class ModelCredentialProvider(Protocol): + """Host contract for account-bound references and request authorization.""" + def is_reference(self, value): ... + def bind(self, reference, base_url): ... + def request_hook(self, reference, base_url): ... + def sync_client_kwargs(self, reference, base_url, *, timeout=600): ... + + +def register_model_credential_provider(provider): + pass + registration_state().provider = provider + + +def require_provider(): + if registration_state().provider is None: + from core.harness.capability_errors import CloudUnavailable + raise CloudUnavailable("Model credential provider is not registered") + return registration_state().provider + + +def is_reference(value): + if registration_state().provider is not None: + return registration_state().provider.is_reference(value) + # A persisted reference must never be sent as a literal bearer credential. + return isinstance(value, str) and value.startswith("desktop-capability:") + + +def bind(reference, base_url): + return require_provider().bind(reference, base_url) + + +def request_hook(reference, base_url): + return require_provider().request_hook(reference, base_url) + + +def sync_client_kwargs(reference, base_url, *, timeout=600): + if not is_reference(reference): + return {} + return require_provider().sync_client_kwargs(reference, base_url, timeout=timeout) + + +@dataclass +class RegistrationState: + resolver: Callable | None = None + provider: ModelCredentialProvider | None = None + + +_REGISTRATIONS = RegistrationState() + + +def registration_state() -> RegistrationState: + return _REGISTRATIONS diff --git a/src/backend/core/harness/registry/model_services.py b/src/backend/core/harness/registry/model_services.py new file mode 100644 index 00000000..590e2069 --- /dev/null +++ b/src/backend/core/harness/registry/model_services.py @@ -0,0 +1,59 @@ +"""Host-provided model operations used by capability and evolution domains.""" +from __future__ import annotations +from dataclasses import dataclass +from typing import Any, Callable + + +def register_model_services(*, ranker: Callable[..., list[str]], prompt_reader: Callable[[], list[tuple[str, dict[str, Any]]]], prompt_renderer: Callable, text_agent: Callable, environment: Callable) -> None: + pass + registration_state().ranker = ranker + registration_state().prompt_reader = prompt_reader + registration_state().prompt_renderer = prompt_renderer + registration_state().text_agent = text_agent + registration_state().environment = environment + + +def rank_skills(**kwargs: Any) -> list[str]: + if registration_state().ranker is None: + raise RuntimeError("Model services are not registered") + return registration_state().ranker(**kwargs) + + +def active_prompt_versions() -> list[tuple[str, dict[str, Any]]]: + if registration_state().prompt_reader is None: + raise RuntimeError("Model services are not registered") + return registration_state().prompt_reader() + + +def render_prompt_part(kind, part): + if registration_state().prompt_renderer is None: + raise RuntimeError("Prompt rendering service is not registered") + return registration_state().prompt_renderer(kind, part) + + +async def run_text_agent(prompt, **context): + if registration_state().text_agent is None: + raise RuntimeError("Text review agent is not registered") + return await registration_state().text_agent(prompt, **context) + + +def model_environment(): + if registration_state().environment is None: + raise RuntimeError("Model configuration environment is not registered") + return registration_state().environment() + + +@dataclass +class RegistrationState: + ranker: Callable[..., list[str]] | None = None + prompt_reader: Callable[[], list[tuple[str, dict[str, Any]]]] | None = None + prompt_renderer: Callable | None = None + text_agent: Callable | None = None + environment: Callable | None = None + + +_REGISTRATIONS = RegistrationState() + + +def registration_state() -> RegistrationState: + return _REGISTRATIONS diff --git a/src/backend/core/harness/registry/package_sources.py b/src/backend/core/harness/registry/package_sources.py new file mode 100644 index 00000000..ce2f485c --- /dev/null +++ b/src/backend/core/harness/registry/package_sources.py @@ -0,0 +1,48 @@ +"""Source and upgrade contracts for plugin packages.""" +from __future__ import annotations +from dataclasses import dataclass, field +from typing import Any, Protocol + + +class PackageSource(Protocol): + def normalize(self, db: Any, slug: str) -> Any: ... + def archive(self, db: Any, slug: str) -> bytes | None: ... + + +class PackageUpgradePolicy(Protocol): + def lock_configuration(self, db: Any) -> None: ... + def prepare(self, db: Any, existing: Any, package: Any, source: str, approved_revision: str | None) -> Any: ... + def reserve_local(self, existing: Any, revision: str, files: Any) -> None: ... + def prepare_local(self, db: Any, current: Any, prepared: Any) -> Any: ... + + +def register_package_source(name: str, source: PackageSource) -> None: + registration_state().sources[name] = source + + +def package_source(name: str) -> PackageSource: + return registration_state().sources[name] + + +def register_package_upgrade_policy(policy: PackageUpgradePolicy) -> None: + pass + registration_state().upgrade_policy = policy + + +def package_upgrade_policy() -> PackageUpgradePolicy: + if registration_state().upgrade_policy is None: + raise RuntimeError("Host must register the package upgrade policy before installing plugins") + return registration_state().upgrade_policy + + +@dataclass +class RegistrationState: + sources: dict[str, PackageSource] = field(default_factory=dict) + upgrade_policy: PackageUpgradePolicy | None = None + + +_REGISTRATIONS = RegistrationState() + + +def registration_state() -> RegistrationState: + return _REGISTRATIONS diff --git a/src/backend/core/harness/registry/package_views.py b/src/backend/core/harness/registry/package_views.py new file mode 100644 index 00000000..e4692e5d --- /dev/null +++ b/src/backend/core/harness/registry/package_views.py @@ -0,0 +1,66 @@ +"""Host-owned package interface validation, presentation and enablement.""" +from collections.abc import Callable +from dataclasses import dataclass +from functools import lru_cache +from typing import Any + + +@dataclass +class PackageViews: + project: Callable | None = None + normalize: Callable | None = None + find: Callable | None = None + enabled: Callable | None = None + workspace_only: Callable | None = None + set_enabled: Callable | None = None + + +@lru_cache(maxsize=1) +def package_views() -> PackageViews: + return PackageViews() + + +def register_package_view(project, *, normalize, find, enabled, workspace_only, set_enabled) -> None: + services = package_views() + services.project = project + services.normalize = normalize + services.find = find + services.enabled = enabled + services.workspace_only = workspace_only + services.set_enabled = set_enabled + + +def public_package_view(declaration: Any, *, slug: str) -> dict[str, Any]: + project = package_views().project + return project(declaration, slug=slug) if project is not None else {} + + +def normalize_ui(declaration): + normalize = package_views().normalize + if normalize is None: + raise RuntimeError("Package interface validator is not registered") + return normalize(declaration) + + +def find_module(stored, module_id): + find = package_views().find + if find is None: + raise RuntimeError("Package interface resolver is not registered") + return find(stored, module_id) + + +def enabled(db, row, user_id): + predicate = package_views().enabled + return bool(predicate and predicate(db, row, user_id)) + + +def workspace_only(row): + predicate = package_views().workspace_only + return bool(predicate and predicate(row)) + + +def set_enabled(db, row, value, user_id=None): + update = package_views().set_enabled + if update is None: + raise RuntimeError("Package interface enablement policy is not registered") + return update(db, row, value, user_id=user_id) diff --git a/src/backend/core/harness/registry/registry.py b/src/backend/core/harness/registry/registry.py new file mode 100644 index 00000000..9b7798ca --- /dev/null +++ b/src/backend/core/harness/registry/registry.py @@ -0,0 +1,169 @@ +"""Hook tables that optional modules fill in at startup. + +Every table starts empty. Core consults them at its extension points (agent +assembly, sandbox routing, inbound channel messages, app lifecycle, routers), +so an unregistered module leaves those code paths exactly as they were. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Awaitable, Callable, Optional + + +@dataclass(frozen=True) +class AgentProfile: + """A dedicated agent built by a module instead of the ordinary chat assembly. + + Selected by ``AgentRequest.profile``; ``AgentRequest.profile_ref`` carries the + module's own identifier (for example the task the agent works on). + """ + + # (request) -> system prompt; replaces the whole chat prompt assembly. + system_prompt: Callable[[Any], Awaitable[str]] + # (toolkit, request, loader=...) -> None; replaces the native chat tools. + register_tools: Callable[..., None] + # Framework-neutral per-agent policies and optional steering source. + hooks: Callable[[Any], list] = field(default=lambda request: []) + steering: Callable[[Any], Any] = field(default=lambda request: None) + # Only function tools whose name passes are exposed; None keeps all. + tool_filter: Optional[Callable[[str], bool]] = None + # False: MCP servers come only from the request's explicit list, never catalog defaults. + # False also prevents history and skill dependencies from widening explicit bindings. + inherit_default_mcps: bool = True + # False: the profile's own system prompt lists its skills and how to read them; the + # chat skill section (which points to view_text_file) is left out. + native_skill_prompt: bool = True + + +def mark_enabled(name: str) -> None: + registration_state().enabled.add(name) + + +def is_enabled(name: str) -> bool: + return name in registration_state().enabled + + +def register_agent_profile(name: str, profile: AgentProfile) -> None: + registration_state().agent_profiles[name] = profile + + +def agent_profile(name: str) -> AgentProfile: + profile = registration_state().agent_profiles.get(name) + if profile is None: + raise ValueError(f"Agent profile {name!r} is not enabled") + return profile + + +def register_sandbox_route(session_prefix: str, provider: Callable[[], Any]) -> None: + """Sandbox sessions whose id starts with ``session_prefix`` use ``provider()``.""" + registration_state().sandbox_routes.append((session_prefix, provider)) + + +def sandbox_route(session_id: str) -> Optional[Callable[[], Any]]: + for prefix, provider in registration_state().sandbox_routes: + if session_id.startswith(prefix): + return provider + return None + + +def register_channel_route( + handles: Callable[[Any, Any], bool], + admit: Callable[..., Awaitable[None]], +) -> None: + """``handles(db, conn)`` claims an inbound bot message; ``admit(conn, msg, adapter)`` takes it.""" + registration_state().channel_routes.append((handles, admit)) + + +def channel_routes() -> tuple: + return tuple(registration_state().channel_routes) + + +def register_lifecycle( + start: Callable[[], Awaitable[None]], + stop: Callable[[], Awaitable[None]], +) -> None: + """Background work started on each service worker and stopped on shutdown.""" + registration_state().lifecycle.append((start, stop)) + + +def lifecycle_hooks() -> tuple: + return tuple(registration_state().lifecycle) + + +def register_router(router: Any, policy_key: Optional[str] = None) -> None: + """An API router mounted with the same edition policy as ``EE_ROUTERS`` entries.""" + registration_state().routers.append((router, policy_key)) + + +def routers() -> tuple: + return tuple(registration_state().routers) + + +@dataclass(frozen=True) +class SandboxCredentials: + """Contribution to cloud credentials; callbacks retain provider-specific SDK context.""" + name: str + environment: Optional[Callable[[], dict[str, str]]] = None + mounts: Optional[Callable[[Optional[str]], list[Any]]] = None + before_command: Optional[Callable[[Any, Any, str, str], Awaitable[None]]] = None + after_command: Optional[Callable[[Any, Any, str, str], Awaitable[None]]] = None + prepare_order: int = 0 + + +def register_sandbox_credentials(provider: SandboxCredentials) -> None: + registration_state().sandbox_credentials[provider.name] = provider + + +def sandbox_credentials() -> tuple[SandboxCredentials, ...]: + return tuple(registration_state().sandbox_credentials.values()) + + +@dataclass(frozen=True) +class NativeToolContext: + request: Any + interactive: bool + is_channel_run: bool + sandbox_session_id: Any + project_scope: Any + skill_ids: Any + enabled_mcp_keys: Any + logger: Any + + +@dataclass(frozen=True) +class NativeToolContribution: + name: str + factory: Callable[[Any, NativeToolContext], Awaitable[None]] + enabled: Callable[[NativeToolContext], bool] + stage: str + + +def register_tool(name, factory, enabled, *, stage): + """Register a conditional tool at an ordered native assembly stage.""" + registration_state().tools[name] = NativeToolContribution(name, factory, enabled, stage) + + +async def install_tools(stage, toolkit, context): + for contribution in tuple(registration_state().tools.values()): + if contribution.stage == stage and contribution.enabled(context): + await contribution.factory(toolkit, context) + + +@dataclass +class RegistrationState: + enabled: set[str] = field(default_factory=set) + agent_profiles: dict[str, AgentProfile] = field(default_factory=dict) + sandbox_routes: list[tuple[str, Callable[[], Any]]] = field(default_factory=list) + channel_routes: list[tuple[Callable[[Any, Any], bool], Callable[..., Awaitable[None]]]] = field(default_factory=list) + lifecycle: list[tuple[Callable[[], Awaitable[None]], Callable[[], Awaitable[None]]]] = field(default_factory=list) + routers: list[tuple[Any, Optional[str]]] = field(default_factory=list) + sandbox_credentials: dict[str, SandboxCredentials] = field(default_factory=dict) + tools: dict[str, NativeToolContribution] = field(default_factory=dict) + + +_REGISTRATIONS = RegistrationState() + + +def registration_state() -> RegistrationState: + return _REGISTRATIONS diff --git a/src/backend/core/harness/registry/resource_owners.py b/src/backend/core/harness/registry/resource_owners.py new file mode 100644 index 00000000..9edce5ac --- /dev/null +++ b/src/backend/core/harness/registry/resource_owners.py @@ -0,0 +1,48 @@ +"""Installed application owners supply policy for generic plugin resources.""" +from __future__ import annotations +from dataclasses import dataclass +from typing import Any, Callable + + +@dataclass(frozen=True) +class ResourceOwner: + binding: Callable[..., Any] + resolve_installation: Callable[..., Any] + exact_installation: Callable[..., Any] + authorize: Callable[..., Any] + observe: Callable[..., Any] + record: Callable[..., Any] + renew: Callable[..., Any] + arm: Callable[..., Any] + invalidate: Callable[..., Any] + sweep: Callable[..., Any] + collect: Callable[..., Any] + create: Callable[..., Any] + owned: Callable[..., Any] + lease: Callable[..., Any] + reserve: Callable[..., Any] + close: Callable[..., Any] + revoke_session: Callable[..., Any] + + +def register_resource_owner(owner: ResourceOwner) -> None: + pass + registration_state().owner = owner + + +def resource_owner() -> ResourceOwner: + if registration_state().owner is None: + raise RuntimeError("Host must register the application resource owner") + return registration_state().owner + + +@dataclass +class RegistrationState: + owner: ResourceOwner | None = None + + +_REGISTRATIONS = RegistrationState() + + +def registration_state() -> RegistrationState: + return _REGISTRATIONS diff --git a/src/backend/core/harness/registry/retrieval_context.py b/src/backend/core/harness/registry/retrieval_context.py new file mode 100644 index 00000000..ba24ff18 --- /dev/null +++ b/src/backend/core/harness/registry/retrieval_context.py @@ -0,0 +1,37 @@ +"""External retrieval context supplied by an installed domain.""" +from __future__ import annotations +from collections.abc import Callable +from dataclasses import dataclass + + +def register_retrieval_context(provider: Callable[[list[str]], tuple[dict, dict]] | None) -> None: + pass + registration_state().provider = provider + + +def runtime_request_context(selected_ids: list[str]) -> tuple[dict, dict]: + return registration_state().provider(selected_ids) if registration_state().provider is not None else ({}, {}) + + +def register_retrieval_access_filter(provider: Callable) -> None: + pass + registration_state().access_filter = provider + + +def filter_accessible_kb_ids(db, user_id: str, selected_ids: list[str]) -> list[str]: + if registration_state().access_filter is None: + raise RuntimeError("Retrieval access policy is not registered") + return registration_state().access_filter(db, user_id, selected_ids) + + +@dataclass +class RegistrationState: + provider: Callable[[list[str]], tuple[dict, dict]] | None = None + access_filter: Callable | None = None + + +_REGISTRATIONS = RegistrationState() + + +def registration_state() -> RegistrationState: + return _REGISTRATIONS diff --git a/src/backend/core/harness/registry/run_cancellation.py b/src/backend/core/harness/registry/run_cancellation.py new file mode 100644 index 00000000..49e51e6e --- /dev/null +++ b/src/backend/core/harness/registry/run_cancellation.py @@ -0,0 +1,37 @@ +"""Run cancellation ports preserve the caller's transaction boundary.""" +from collections.abc import Awaitable, Callable +from dataclasses import dataclass +from functools import lru_cache +from typing import Any + + +@dataclass +class RunCancellationServices: + cancel: Callable[..., Awaitable[bool]] | None = None + cancel_in_session: Callable[..., Any] | None = None + + +@lru_cache(maxsize=1) +def cancellation_services() -> RunCancellationServices: + return RunCancellationServices() + + +def register_run_cancellation(cancel: Callable[..., Awaitable[bool]], *, + cancel_in_session: Callable[..., Any]) -> None: + services = cancellation_services() + services.cancel = cancel + services.cancel_in_session = cancel_in_session + + +async def cancel_run(run_id: str, *, user_id: str, stop_terminal_worker: bool = False) -> bool: + cancel = cancellation_services().cancel + if cancel is None: + raise RuntimeError("Run cancellation service is not registered") + return await cancel(run_id, user_id=user_id, stop_terminal_worker=stop_terminal_worker) + + +def cancel_run_in_session(db, run, *, reason: str, now) -> None: + cancel = cancellation_services().cancel_in_session + if cancel is None: + raise RuntimeError("Transactional run cancellation service is not registered") + cancel(db, run, reason=reason, now=now) diff --git a/src/backend/core/harness/registry/run_capabilities.py b/src/backend/core/harness/registry/run_capabilities.py new file mode 100644 index 00000000..d241094c --- /dev/null +++ b/src/backend/core/harness/registry/run_capabilities.py @@ -0,0 +1,114 @@ +"""Per-run capability contracts supplied by an installed execution environment.""" +from __future__ import annotations +from collections.abc import Callable, Mapping +from dataclasses import dataclass, field +from typing import Any, Protocol + + +class PreparedCapabilities(Protocol): + run_id: str + bindings: Mapping[str, Any] + dependency_report: dict + unavailable: dict + + def validate_tool_call(self, server_id: str) -> bool: ... + def release(self) -> None: ... + def validate_selected_skills(self, skill_ids: list[str] | None) -> None: ... + def validate(self, *, user_id: str) -> None: ... + def record_tool_scope(self, tool_call_id: str, tool_name: str) -> None: ... + def child_scope(self, kind: str, *identities: str) -> str: ... + def skill_loader(self) -> Any: ... + def tool_guard(self, *, scope_id: str): ... + def bind_mcp(self, servers: dict, resolution: Any) -> dict: ... + def preflight(self, **context: Any) -> PreparedCapabilities: ... + + +class RunCapabilityProvider(Protocol): + def cloud_plugin_selection(self, plugin_id, user_id): ... + def resolve_explicit_ids(self, user_id, skill_ids, mcp_ids): ... + def plugin_skill_defaults(self, user_id, skill_ids): ... + def ensure_ready(self, user_id, *, skill_keys=None, plugin_keys=None): ... + def available_skill_names(self, names, *, user_id): ... + def plugin_binding_ids(self, plugin_ids, *, user_id): ... + async def prepare(self, request: Any, **selection: Any) -> tuple[Any, Any, PreparedCapabilities | None]: ... + + +class CloudCapabilities: + def cloud_plugin_selection(self, plugin_id, user_id): + return None + + def resolve_explicit_ids(self, user_id, skill_ids, mcp_ids): + return skill_ids, mcp_ids + + def plugin_skill_defaults(self, user_id, skill_ids): + return skill_ids + + def ensure_ready(self, user_id, *, skill_keys=None, plugin_keys=None): + return None + + def available_skill_names(self, names, *, user_id): + return list(names) + + def plugin_binding_ids(self, plugin_ids, *, user_id): + return [], [] + + async def prepare(self, request, **selection): + return selection["_caps_skill_ids"], None, None + + +_default = CloudCapabilities() + + +def register_run_capability_provider(name, provider, *, enabled): + registration_state().providers[name] = (enabled, provider) + + +def run_capability_provider() -> RunCapabilityProvider: + for enabled, provider in tuple(registration_state().providers.values()): + if enabled(): + return provider + return _default + + +def has_run_capabilities() -> bool: + return run_capability_provider() is not _default + + +async def prepare_run_capabilities(request, **selection): + return await run_capability_provider().prepare(request, **selection) + + +def ensure_cloud_ready(user_id, *, skill_keys=None, plugin_keys=None): + return run_capability_provider().ensure_ready(user_id, skill_keys=skill_keys, plugin_keys=plugin_keys) + + +def filter_available_names(names, *, user_id): + return run_capability_provider().available_skill_names(names, user_id=user_id) + + +def cloud_binding_ids(plugin_ids, *, user_id=None): + return run_capability_provider().plugin_binding_ids(plugin_ids, user_id=user_id) + + +def plugin_skill_defaults(user_id, skill_ids): + return run_capability_provider().plugin_skill_defaults(user_id, skill_ids) + + +def resolve_explicit_ids(user_id, skill_ids, mcp_ids): + return run_capability_provider().resolve_explicit_ids(user_id, skill_ids, mcp_ids) + + +def cloud_plugin_selection(plugin_id, user_id): + return run_capability_provider().cloud_plugin_selection(plugin_id, user_id) + + +@dataclass +class RegistrationState: + providers: dict[str, tuple[Callable[[], bool], RunCapabilityProvider]] = field(default_factory=dict) + + +_REGISTRATIONS = RegistrationState() + + +def registration_state() -> RegistrationState: + return _REGISTRATIONS diff --git a/src/backend/core/harness/registry/schema_lifecycle.py b/src/backend/core/harness/registry/schema_lifecycle.py new file mode 100644 index 00000000..3e1a30d5 --- /dev/null +++ b/src/backend/core/harness/registry/schema_lifecycle.py @@ -0,0 +1,36 @@ +"""Synchronous installed-module lifecycle callbacks during local schema startup.""" +from __future__ import annotations +from collections.abc import Callable, Iterable +from dataclasses import dataclass, field +from importlib import import_module + + +def register_schema_lifecycle(name: str, callback: Callable[[], None]) -> None: + """Register an idempotent local-SQLite initializer by stable owner name.""" + registration_state().callbacks[name] = callback + + +def load_schema_lifecycle(installed: Iterable[str]) -> None: + """Compose schema hooks without starting runtime workers or enabling features.""" + for package in installed: + register = getattr(import_module(f"{package}.extension"), "register_schema", None) + if register is not None: + register() + + +def initialize_local_schema() -> None: + """Run in startup order; initialization failures must block startup.""" + for callback in tuple(registration_state().callbacks.values()): + callback() + + +@dataclass +class RegistrationState: + callbacks: dict[str, Callable[[], None]] = field(default_factory=dict) + + +_REGISTRATIONS = RegistrationState() + + +def registration_state() -> RegistrationState: + return _REGISTRATIONS diff --git a/src/backend/core/harness/registry/skill_content.py b/src/backend/core/harness/registry/skill_content.py new file mode 100644 index 00000000..e9bb7495 --- /dev/null +++ b/src/backend/core/harness/registry/skill_content.py @@ -0,0 +1,24 @@ +"""Installed integrations adapt their own skill text for the execution host.""" +from collections.abc import Callable +from dataclasses import dataclass, field +from functools import lru_cache + + +@dataclass +class SkillContentAdapters: + adapters: dict[str, Callable[[str, str | None], str]] = field(default_factory=dict) + + +@lru_cache(maxsize=1) +def skill_content_adapters() -> SkillContentAdapters: + return SkillContentAdapters() + + +def register_skill_content_adapter(name, adapter): + skill_content_adapters().adapters[name] = adapter + + +def adapt_skill_content(content: str, user_id: str | None) -> str: + for adapter in tuple(skill_content_adapters().adapters.values()): + content = adapter(content, user_id) + return content diff --git a/src/backend/core/harness/registry/system_configuration.py b/src/backend/core/harness/registry/system_configuration.py new file mode 100644 index 00000000..8df46ff5 --- /dev/null +++ b/src/backend/core/harness/registry/system_configuration.py @@ -0,0 +1,31 @@ +"""Persistence contract for the configuration cache.""" +from dataclasses import dataclass +from functools import lru_cache +from typing import Protocol + + +class SystemConfigurationStore(Protocol): + def set(self, key: str, value: str | None, updated_by: str): ... + def bulk_set(self, items: list[dict], updated_by: str): ... + def load(self) -> tuple[dict, dict]: ... + + +@dataclass +class ConfigurationRegistration: + store: SystemConfigurationStore | None = None + + +@lru_cache(maxsize=1) +def configuration_registration() -> ConfigurationRegistration: + return ConfigurationRegistration() + + +def register_system_configuration_store(store: SystemConfigurationStore) -> None: + configuration_registration().store = store + + +def system_configuration_store() -> SystemConfigurationStore: + store = configuration_registration().store + if store is None: + raise RuntimeError("System configuration persistence is not registered") + return store diff --git a/src/backend/core/harness/registry/tool_operations.py b/src/backend/core/harness/registry/tool_operations.py new file mode 100644 index 00000000..916bda30 --- /dev/null +++ b/src/backend/core/harness/registry/tool_operations.py @@ -0,0 +1,59 @@ +"""Host registrations for component-owned local dispatch and durable receipts.""" +from __future__ import annotations +from collections.abc import Callable +from dataclasses import dataclass, field +from typing import Any + + +class RemoteOperationConflict(ValueError): + """The authenticated operation key is already bound to another request.""" + + +@dataclass(frozen=True) +class ToolOperations: + receipt_tools: frozenset[str] + local_tools: frozenset[str] + bind: Callable[..., str] + lookup: Callable[..., Any] + remember: Callable[..., Any] + dispatch_local: Callable[..., Any] + prepare_arguments: Callable[..., dict] | None = None + + +def register_tool_operations(component: str, operations: ToolOperations) -> None: + registration_state().components[component] = operations + + +def receipt_operations(tool_name: str) -> ToolOperations | None: + return next((value for value in registration_state().components.values() if tool_name in value.receipt_tools), None) + + +def local_tool_allowed(component: str, tool_name: str) -> bool: + provider = registration_state().components.get(component) + return provider is not None and tool_name in provider.local_tools + + +async def local_tool_result(component, tool_name, arguments, headers, *, authorize=None): + provider = registration_state().components.get(component) + if provider is None or tool_name not in provider.local_tools: + return None + return await provider.dispatch_local(component, tool_name, arguments, headers, authorize=authorize) + + +def prepare_tool_arguments(tool_name, arguments, user_id, chat_id): + provider = receipt_operations(tool_name) + if provider is None or provider.prepare_arguments is None: + return arguments + return provider.prepare_arguments(tool_name, arguments, user_id, chat_id) + + +@dataclass +class RegistrationState: + components: dict[str, ToolOperations] = field(default_factory=dict) + + +_REGISTRATIONS = RegistrationState() + + +def registration_state() -> RegistrationState: + return _REGISTRATIONS diff --git a/src/backend/core/services/desktop_gateway_uploads.py b/src/backend/core/harness/registry/transfer_channels.py similarity index 53% rename from src/backend/core/services/desktop_gateway_uploads.py rename to src/backend/core/harness/registry/transfer_channels.py index 51faaf1e..fd736fd4 100644 --- a/src/backend/core/services/desktop_gateway_uploads.py +++ b/src/backend/core/harness/registry/transfer_channels.py @@ -8,7 +8,7 @@ from __future__ import annotations -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import Any, Awaitable, Callable, Dict, Optional, Tuple # Wire contract shared by the uploading client and the receiving gateway route. @@ -27,35 +27,39 @@ class UploadChannel: package: Callable[[Dict[str, Any], Dict[str, str]], Awaitable[Tuple[bytes, str]]] -def _site_publish_channel() -> UploadChannel: - from core.services.desktop_site_publish import package_local_site +@dataclass(frozen=True) +class UploadReceiver: + options_model: Any + max_bytes: int + publish: Callable[..., Any] + + +def register_upload_channel(source_plugin, tool_name, factory, *, receiver=None): + registration_state().CHANNELS.setdefault(source_plugin, {})[tool_name] = factory + if receiver is not None: + registration_state().RECEIVERS[factory().endpoint] = receiver - return UploadChannel( - endpoint="site-publish", - content_type="application/gzip", - package=package_local_site, - ) +def register_result_localizer(source_plugin, tool_name, localize): + registration_state().LOCALIZERS.setdefault(source_plugin, {})[tool_name] = localize -# Providing plugin slug → its uploading tools. The slug is the fact the platform -# registered for the server (``AdminMcpServer.source_plugin``); keying on it means -# a shared and a per-user install of the same plugin resolve identically, and no -# id string ever has to be taken apart. Values are factories so a channel's module -# is imported only when that plugin's tool is actually invoked. -_CHANNELS: Dict[str, Dict[str, Callable[[], UploadChannel]]] = { - "sites": {"publish_site": _site_publish_channel}, -} + +def upload_receiver(endpoint): + receiver = registration_state().RECEIVERS.get(endpoint) + if receiver is None: + raise RuntimeError("gateway upload component is not registered") + return receiver def upload_channel(source_plugin: str, tool_name: str) -> Optional[UploadChannel]: """The declared upload channel for this plugin's tool, if it has one.""" - factory = _CHANNELS.get(str(source_plugin or ""), {}).get(tool_name) + factory = registration_state().CHANNELS.get(str(source_plugin or ""), {}).get(tool_name) return factory() if factory is not None else None def endpoint_plugin(endpoint: str) -> Optional[str]: """Which providing plugin a gateway upload endpoint belongs to.""" - for slug, tools in _CHANNELS.items(): + for slug, tools in registration_state().CHANNELS.items(): for factory in tools.values(): if factory().endpoint == endpoint: return slug @@ -64,12 +68,18 @@ def endpoint_plugin(endpoint: str) -> Optional[str]: def result_localizer(source_plugin: str, tool_name: str): """Resolve component-owned receipt handling independently of upload transport.""" - if source_plugin != "sites": - return None - from core.services.desktop_site_publish import localize_application_result, localize_site_result - - return { - "publish_site": localize_site_result, - "manage_application": localize_application_result, - "publish_mcp": localize_application_result, - }.get(tool_name) + return registration_state().LOCALIZERS.get(source_plugin, {}).get(tool_name) + + +@dataclass +class RegistrationState: + CHANNELS: Dict[str, Dict[str, Callable[[], UploadChannel]]] = field(default_factory=dict) + LOCALIZERS: Dict[str, Dict[str, Callable[..., Any]]] = field(default_factory=dict) + RECEIVERS: Dict[str, UploadReceiver] = field(default_factory=dict) + + +_REGISTRATIONS = RegistrationState() + + +def registration_state() -> RegistrationState: + return _REGISTRATIONS diff --git a/src/backend/core/harness/registry/user_cleanup.py b/src/backend/core/harness/registry/user_cleanup.py new file mode 100644 index 00000000..862a4d14 --- /dev/null +++ b/src/backend/core/harness/registry/user_cleanup.py @@ -0,0 +1,27 @@ +"""Transactional cleanup contributions for ORM user deletion.""" +from __future__ import annotations +from collections.abc import Callable +from dataclasses import dataclass, field +from typing import Any + + +def register_user_cleanup(handler: Callable[[Any, Any], None]) -> None: + if handler not in registration_state().handlers: + registration_state().handlers.append(handler) + + +def cleanup_user(session: Any, user: Any) -> None: + for handler in tuple(registration_state().handlers): + handler(session, user) + + +@dataclass +class RegistrationState: + handlers: list[Callable[[Any, Any], None]] = field(default_factory=list) + + +_REGISTRATIONS = RegistrationState() + + +def registration_state() -> RegistrationState: + return _REGISTRATIONS diff --git a/src/backend/core/harness/tool_call_context.py b/src/backend/core/harness/tool_call_context.py new file mode 100644 index 00000000..c0906a5d --- /dev/null +++ b/src/backend/core/harness/tool_call_context.py @@ -0,0 +1,5 @@ +"""Execution identity shared by nested tool calls and run entry points.""" +from contextvars import ContextVar + +CURRENT_TOOL_CALL_ID: ContextVar[str] = ContextVar("jx_current_tool_call_id", default="") +CURRENT_RUN_BINDING: ContextVar[tuple[str, str] | None] = ContextVar("jx_current_run_binding", default=None) diff --git a/src/backend/core/harness/tool_effect_context.py b/src/backend/core/harness/tool_effect_context.py new file mode 100644 index 00000000..78d3f461 --- /dev/null +++ b/src/backend/core/harness/tool_effect_context.py @@ -0,0 +1,18 @@ +"""Invocation identity shared by tool execution and transport adapters.""" +from __future__ import annotations +from contextvars import ContextVar +from dataclasses import dataclass +from typing import Optional + +@dataclass(frozen=True) +class ToolInvocationContext: + effect_id: str + result_id: str + idempotency_key: str + run_id: str + fencing_owner: str + + +CURRENT_TOOL_EFFECT: ContextVar[Optional[ToolInvocationContext]] = ContextVar( + "jx_current_tool_effect", default=None +) diff --git a/src/backend/core/harness/usage.py b/src/backend/core/harness/usage.py index 230a0c9f..d6ad31fb 100644 --- a/src/backend/core/harness/usage.py +++ b/src/backend/core/harness/usage.py @@ -4,7 +4,7 @@ import asyncio import threading -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from dataclasses import dataclass, field from typing import Any, Protocol @@ -50,33 +50,56 @@ def record_attempt(self, attempt: UsageAttempt) -> UsageAttempt: """Append one attempt and return it with its allocated sequence.""" +def record_attempt_batch(recorder: UsageRecorder, attempts: Sequence[UsageAttempt]) -> tuple[UsageAttempt, ...]: + batch = getattr(recorder, "record_attempts", None) + if batch is not None: + return tuple(batch(attempts)) + return tuple(recorder.record_attempt(attempt) for attempt in attempts) + + def attempt_status_for_exception(exc: BaseException) -> str: """Classify provider/library timeout types without importing each SDK.""" name = type(exc).__name__.lower() return "timeout" if isinstance(exc, TimeoutError) or "timeout" in name else "failed" -async def record_usage_safely( +async def record_attempts_safely( recorder: UsageRecorder, - attempt: UsageAttempt, + attempts: Sequence[UsageAttempt], *, timeout_ms: int = 1_000, -) -> UsageAttempt | None: - """Bound observability I/O so it cannot become an execution dependency.""" +) -> tuple[UsageAttempt, ...] | None: + """Bound observability I/O so it cannot become an execution dependency. + + All attempts go to the recorder in one call, so batching recorders write + them in one transaction. + """ + if not attempts: + return () try: return await asyncio.wait_for( - asyncio.to_thread(recorder.record_attempt, attempt), + asyncio.to_thread(record_attempt_batch, recorder, tuple(attempts)), timeout=max(1, int(timeout_ms)) / 1_000, ) except Exception: # noqa: BLE001 return None +async def record_usage_safely( + recorder: UsageRecorder, + attempt: UsageAttempt, + *, + timeout_ms: int = 1_000, +) -> UsageAttempt | None: + recorded = await record_attempts_safely(recorder, (attempt,), timeout_ms=timeout_ms) + return recorded[0] if recorded else None + + class MemoryUsageRecorder: """Thread-safe append-only recorder for unit tests and embedded runtimes.""" def __init__(self) -> None: - self._lock = threading.Lock() + self._lock = threading.RLock() self._rows: dict[str, list[UsageAttempt]] = {} def record_attempt(self, attempt: UsageAttempt) -> UsageAttempt: @@ -91,6 +114,10 @@ def record_attempt(self, attempt: UsageAttempt) -> UsageAttempt: rows.append(recorded) return recorded + def record_attempts(self, attempts: Sequence[UsageAttempt]) -> tuple[UsageAttempt, ...]: + with self._lock: + return tuple(self.record_attempt(attempt) for attempt in attempts) + def attempts(self, run_id: str) -> tuple[UsageAttempt, ...]: with self._lock: return tuple(self._rows.get(run_id, ())) diff --git a/src/backend/core/infra/background.py b/src/backend/core/infra/background.py index fdb2628f..2083485f 100644 --- a/src/backend/core/infra/background.py +++ b/src/backend/core/infra/background.py @@ -18,14 +18,13 @@ from __future__ import annotations import asyncio +from dataclasses import dataclass, field from typing import Any, Coroutine, Set from core.infra.logging import get_logger logger = get_logger(__name__) -_tasks: Set[asyncio.Task] = set() - def spawn(coro: Coroutine[Any, Any, Any], *, name: str) -> asyncio.Task: """把 *coro* 挂到当前事件循环后台执行，返回值不必接收。 @@ -42,16 +41,27 @@ def spawn(coro: Coroutine[Any, Any, Any], *, name: str) -> asyncio.Task: ) from None task = loop.create_task(coro, name=name) - _tasks.add(task) + process_state().tasks.add(task) task.add_done_callback(_forget) return task def _forget(task: asyncio.Task) -> None: - _tasks.discard(task) + process_state().tasks.discard(task) if task.cancelled(): return exc = task.exception() if exc is not None: logger.warning("background_task_failed", task=task.get_name(), error=str(exc)) + +@dataclass +class BackgroundState: + tasks: Set[asyncio.Task] = field(default_factory=lambda: set()) + + +_STATE = BackgroundState() + + +def process_state() -> BackgroundState: + return _STATE diff --git a/src/backend/core/infra/crypto.py b/src/backend/core/infra/crypto.py index 77b9aefc..72eff767 100644 --- a/src/backend/core/infra/crypto.py +++ b/src/backend/core/infra/crypto.py @@ -20,6 +20,7 @@ import hashlib import logging import os +from dataclasses import dataclass from typing import Optional logger = logging.getLogger(__name__) @@ -27,38 +28,34 @@ # Local-dev fallback key: used only when neither EMAIL_SECRET_KEY nor ADMIN_TOKEN is configured; emits a warning. # In production always configure EMAIL_SECRET_KEY (or at least ADMIN_TOKEN), otherwise moving/reinstalling machines makes old ciphertext undecryptable. _DEV_FALLBACK = "hugagent-email-dev-secret-please-override" -_warned = False def _deployment_secret() -> str: - global _warned sec = (os.environ.get("EMAIL_SECRET_KEY") or "").strip() if sec: return sec sec = (os.environ.get("ADMIN_TOKEN") or "").strip() if sec: return sec - if not _warned: + if not process_state().warned: logger.warning( "[crypto] 未配置 EMAIL_SECRET_KEY / ADMIN_TOKEN，邮箱授权码加密使用 dev 兜底密钥；" "生产请务必配置 EMAIL_SECRET_KEY，否则重装/换机后旧凭据无法解密。" ) - _warned = True + process_state().warned = True return _DEV_FALLBACK # The derived key comes from the deployment-level secret and is constant at runtime → lazily construct it once and reuse, avoiding the SHA256+Fernet init on every encrypt/decrypt. -_fernet_cache = None def _fernet(): - global _fernet_cache - if _fernet_cache is None: + if process_state().fernet_cache is None: from cryptography.fernet import Fernet key = base64.urlsafe_b64encode(hashlib.sha256(_deployment_secret().encode("utf-8")).digest()) - _fernet_cache = Fernet(key) - return _fernet_cache + process_state().fernet_cache = Fernet(key) + return process_state().fernet_cache def encrypt_secret(plaintext: str) -> str: @@ -75,3 +72,16 @@ def decrypt_secret(token: Optional[str]) -> Optional[str]: except Exception as exc: # noqa: BLE001 (InvalidToken etc. are all downgraded to None) logger.warning("[crypto] decrypt_secret failed: %s", type(exc).__name__) return None + + +@dataclass +class CryptoState: + warned: object = False + fernet_cache: object = None + + +_STATE = CryptoState() + + +def process_state() -> CryptoState: + return _STATE diff --git a/src/backend/core/capabilities/junction.py b/src/backend/core/infra/directory_links.py similarity index 92% rename from src/backend/core/capabilities/junction.py rename to src/backend/core/infra/directory_links.py index 2ebfc25d..b7482ac0 100644 --- a/src/backend/core/capabilities/junction.py +++ b/src/backend/core/infra/directory_links.py @@ -46,7 +46,7 @@ def wrapped(*args, **kwargs): return decorate -def _strip_windows_prefix(raw: str) -> str: +def strip_windows_prefix(raw: str) -> str: if raw.startswith("\\\\?\\UNC\\"): return "\\\\" + raw[8:] if raw.startswith(_WIN_PREFIX): @@ -54,7 +54,7 @@ def _strip_windows_prefix(raw: str) -> str: return raw -def _extended_windows_path(raw: str) -> str: +def extended_windows_path(raw: str) -> str: """Extended Win32 paths work without changing machine-wide path policy.""" raw = ntpath.abspath(raw) if raw.startswith(_WIN_PREFIX): @@ -64,18 +64,18 @@ def _extended_windows_path(raw: str) -> str: return _WIN_PREFIX + raw -def _native(path: Path) -> Path: - return Path(_extended_windows_path(str(path))) if _IS_WINDOWS else path +def native_path(path: Path) -> Path: + return Path(extended_windows_path(str(path))) if _IS_WINDOWS else path def _resolved(path: Path) -> Path: - raw = os.path.realpath(_native(path)) - return Path(_strip_windows_prefix(raw) if _IS_WINDOWS else raw) + raw = os.path.realpath(native_path(path)) + return Path(strip_windows_prefix(raw) if _IS_WINDOWS else raw) def _lstat(path: Path) -> Optional[os.stat_result]: try: - return os.lstat(_native(path)) + return os.lstat(native_path(path)) except FileNotFoundError: return None @@ -98,9 +98,9 @@ def read_directory_link(path: Path) -> Optional[Path]: """Absolute target of a link, or None when ``path`` is not a link.""" if not is_directory_link(path): return None - raw = os.readlink(_native(path)) + raw = os.readlink(native_path(path)) if _IS_WINDOWS: - raw = _strip_windows_prefix(raw) + raw = strip_windows_prefix(raw) target = Path(raw) if not target.is_absolute(): target = path.parent / target @@ -128,12 +128,12 @@ def assert_target_allowed(target: Path, allowed_roots: Iterable[Path]) -> None: def remove_directory_link(path: Path) -> None: """Remove the link itself; never follows it, never deletes a real directory.""" if not is_directory_link(path): - if _native(path).exists(): + if native_path(path).exists(): raise LinkError(f"{path} is a real directory, not a link; refusing to remove") return if _IS_WINDOWS: # Junctions and directory symlinks are removed with rmdir on NTFS. - os.rmdir(_native(path)) + os.rmdir(native_path(path)) else: os.unlink(path) @@ -181,7 +181,7 @@ def _create_windows_junction(link: Path, target: Path) -> None: ] kernel.DeviceIoControl.restype = wintypes.BOOL kernel.CloseHandle.argtypes = [wintypes.HANDLE] - native_link = _native(link) + native_link = native_path(link) native_link.mkdir() handle = kernel.CreateFileW(str(native_link), 0x40000000, 7, None, 3, 0x02200000, None) invalid = ctypes.c_void_p(-1).value @@ -213,13 +213,13 @@ def _create_windows_junction(link: Path, target: Path) -> None: @_link_errors("create") def create_directory_link(link: Path, target: Path, *, allowed_roots: Iterable[Path]) -> None: """Create ``link`` → ``target``. ``target`` must exist and be under an allowed root.""" - if not _native(target).is_dir(): + if not native_path(target).is_dir(): raise LinkError(f"link target does not exist or is not a directory: {target}") assert_target_allowed(target, allowed_roots) if _lstat(link) is not None: raise LinkError(f"link path already exists: {link}") - if not _native(link.parent).is_dir(): - _native(link.parent).mkdir(parents=True, exist_ok=True) + if not native_path(link.parent).is_dir(): + native_path(link.parent).mkdir(parents=True, exist_ok=True) if _IS_WINDOWS: try: _create_windows_junction(link, _resolved(target)) @@ -248,7 +248,7 @@ def ensure_directory_link(link: Path, target: Path, *, allowed_roots: Iterable[P return True wanted = Path(os.path.normpath(_resolved(target))) assert_target_allowed(wanted, allowed_roots) - if not _native(wanted).is_dir(): + if not native_path(wanted).is_dir(): raise LinkError(f"link target does not exist or is not a directory: {wanted}") current = read_directory_link(link) if current is not None: diff --git a/src/backend/core/infra/dns_cache.py b/src/backend/core/infra/dns_cache.py index 90e75509..64b3a90c 100644 --- a/src/backend/core/infra/dns_cache.py +++ b/src/backend/core/infra/dns_cache.py @@ -29,67 +29,79 @@ import socket import threading import time +from dataclasses import dataclass, field from typing import Any, Dict, Optional, Tuple DEFAULT_TTL_S = 300.0 MAX_ENTRIES = 256 -_lock = threading.Lock() -_cache: Dict[Tuple[Any, ...], Tuple[float, Any]] = {} -_original = None + _ttl_s = DEFAULT_TTL_S def _store(key: Tuple[Any, ...], answer: Any, now: float) -> None: - with _lock: - if len(_cache) >= MAX_ENTRIES: - for stale in [k for k, (expires, _) in _cache.items() if expires <= now]: - del _cache[stale] - if len(_cache) >= MAX_ENTRIES: - _cache.clear() - _cache[key] = (now + _ttl_s, answer) + with process_state().lock: + if len(process_state().cache) >= MAX_ENTRIES: + for stale in [k for k, (expires, _) in process_state().cache.items() if expires <= now]: + del process_state().cache[stale] + if len(process_state().cache) >= MAX_ENTRIES: + process_state().cache.clear() + process_state().cache[key] = (now + _ttl_s, answer) def _cached_getaddrinfo(host, port, family=0, type=0, proto=0, flags=0): key = (host, port, family, type, proto, flags) now = time.monotonic() - with _lock: - hit = _cache.get(key) + with process_state().lock: + hit = process_state().cache.get(key) if hit is not None and hit[0] > now: return hit[1] # A failure is never cached: the exception propagates from here, so a name # that starts resolving is picked up on the very next attempt. - answer = _original(host, port, family, type, proto, flags) + answer = process_state().original(host, port, family, type, proto, flags) _store(key, answer, now) return answer def install(ttl_s: Optional[float] = None) -> bool: """Install the cache process-wide. Idempotent; True the first time it takes.""" - global _original, _ttl_s + global _ttl_s configured = os.getenv("HUGAGENT_DNS_CACHE_TTL_S") _ttl_s = float(configured) if configured else float(DEFAULT_TTL_S if ttl_s is None else ttl_s) - if _ttl_s <= 0 or _original is not None: + if _ttl_s <= 0 or process_state().original is not None: return False - _original = socket.getaddrinfo + process_state().original = socket.getaddrinfo socket.getaddrinfo = _cached_getaddrinfo return True def uninstall() -> None: - global _original clear() - if _original is not None: - socket.getaddrinfo = _original - _original = None + if process_state().original is not None: + socket.getaddrinfo = process_state().original + process_state().original = None def clear() -> None: - with _lock: - _cache.clear() + with process_state().lock: + process_state().cache.clear() def installed() -> bool: - return _original is not None + return process_state().original is not None + + +@dataclass +class DnsCacheState: + lock: object = field(default_factory=lambda: threading.Lock()) + cache: Dict[Tuple[Any, ...], Tuple[float, Any]] = field(default_factory=lambda: {}) + original: object = None + + +_STATE = DnsCacheState() + + +def process_state() -> DnsCacheState: + return _STATE diff --git a/src/backend/core/infra/failures.py b/src/backend/core/infra/failures.py new file mode 100644 index 00000000..50c2e4fa --- /dev/null +++ b/src/backend/core/infra/failures.py @@ -0,0 +1,13 @@ +"""Observable best-effort failures without logging payloads or credentials.""" +import logging + +logger = logging.getLogger(__name__) + + +def record_best_effort_failure(operation: str, error: Exception) -> None: + """Preserve a caller's fallback while identifying the failed operation. + + Exception messages and tracebacks can contain tokens or user content. Record + only the operation and exception class at this generic boundary. + """ + logger.warning("best_effort_failure operation=%s error_type=%s", operation, type(error).__name__) diff --git a/src/backend/core/capabilities/lockfile.py b/src/backend/core/infra/file_lock.py similarity index 100% rename from src/backend/core/capabilities/lockfile.py rename to src/backend/core/infra/file_lock.py diff --git a/src/backend/core/immutable.py b/src/backend/core/infra/immutable.py similarity index 100% rename from src/backend/core/immutable.py rename to src/backend/core/infra/immutable.py diff --git a/src/backend/core/infra/json_output.py b/src/backend/core/infra/json_output.py new file mode 100644 index 00000000..7eabceaa --- /dev/null +++ b/src/backend/core/infra/json_output.py @@ -0,0 +1,49 @@ +"""Parse structured model output with the existing fenced/prose fallbacks.""" + +import json +import re +from typing import Any, Dict, List, Optional + + +def parse_json_array_lenient(text: str) -> Optional[List[Any]]: + if not text: + return None + candidates = [text.strip()] + m = re.search(r"```(?:json)?\s*\n?(.*?)\n?```", text, re.DOTALL) + if m: + candidates.append(m.group(1).strip()) + s, e = text.find("["), text.rfind("]") + if s != -1 and e != -1 and e > s: + candidates.append(text[s : e + 1]) + for c in candidates: + try: + obj = json.loads(c) + if isinstance(obj, list): + return obj + except (json.JSONDecodeError, ValueError): + continue + return None + + +def parse_json_lenient(text: str) -> Optional[Dict[str, Any]]: + """Three-level lenient parsing (object), following plan_mode._parse_plan_json.""" + for candidate in _json_candidates(text): + try: + obj = json.loads(candidate) + if isinstance(obj, dict): + return obj + except (json.JSONDecodeError, ValueError): + continue + return None + + +def _json_candidates(text: str): + if not text: + return + yield text.strip() + m = re.search(r"```(?:json)?\s*\n?(.*?)\n?```", text, re.DOTALL) + if m: + yield m.group(1).strip() + s, e = text.find("{"), text.rfind("}") + if s != -1 and e != -1 and e > s: + yield text[s : e + 1] diff --git a/src/backend/core/infra/leader.py b/src/backend/core/infra/leader.py index c91b284d..d3b28370 100644 --- a/src/backend/core/infra/leader.py +++ b/src/backend/core/infra/leader.py @@ -1,207 +1,138 @@ -"""Work that must happen in exactly one backend process. - -The backend serves requests from several uvicorn workers, but a large part of -what it starts at boot is not per-worker work: reapers, schedulers, recovery -passes, outbound channel connections and the seeding of default rows. Running -those once per worker is not merely wasteful — a scheduler tick fires a task N -times, a recovery pass fights itself over the same orphan rows, and two seeders -race on the same unique key. - -Two shapes cover every such step, and both are expressed against the -``hold``/``claim`` mutex in :mod:`core.infra.ephemeral`, so a deployment -without Redis (the desktop's single-process backend) gets the same semantics -from the in-process store it already uses for every other short-lived key. - -``Leadership`` - A renewed lease for the long-lived loops. One worker wins it and starts - them; it keeps the lease alive for as long as it is healthy. If that worker - dies, the lease lapses and the next worker to look takes over and starts - them there — which is the property a bare "decide once at boot" check does - not have, and the reason this is a loop rather than a single check. - -``run_once`` - A barrier for the one-shot boot steps. Exactly one worker runs the step; - the others wait for it to finish before continuing, so no worker begins - serving against half-seeded state. -""" - +"""Lease-owned background work and fail-closed worker startup barriers.""" from __future__ import annotations import asyncio import contextlib +import math import os import socket import uuid -from typing import Awaitable, Callable, Optional +from typing import Awaitable, Callable from core.infra.ephemeral import get_ephemeral_state from core.infra.logging import get_logger from core.infra.worker_count import resolve as resolved_workers logger = get_logger(__name__) - -_LEADER_PREFIX = "jx:leader:" -_ONCE_PREFIX = "jx:boot:" - -# The lease has to outlive a renewal that is merely slow — a worker pausing for -# a GC or a busy disk must not hand its loops to a second process — while still -# expiring soon enough that a *dead* worker's loops restart promptly. Renewal -# runs at a third of it, so two consecutive renewals can be lost before the -# lease actually lapses. _LEASE_SECONDS = 45 -_RENEW_SECONDS = _LEASE_SECONDS / 3 - -# How often a worker waiting on a boot barrier looks to see whether the worker -# running the step has finished. _POLL_SECONDS = 0.2 +Callback = Callable[[], Awaitable[None]] def _token() -> str: - """An identity no other process can accidentally present.""" - return f"{socket.gethostname()}:{os.getpid()}:{uuid.uuid4().hex[:8]}" + return f"{socket.gethostname()}:{os.getpid()}:{uuid.uuid4().hex}" def coordination_required() -> bool: - """Whether this deployment actually runs more than one backend process. - - A single-process backend has nobody to coordinate with, and saying so is - not an optimisation — it is the difference between booting and not: the - desktop profile runs one process with no Redis at all, and neither it nor a - deployment whose Redis is briefly down should acquire a dependency on a - mutex that can only ever be uncontended. - - It asks :func:`~core.infra.worker_count.resolve` rather than reading - ``WEB_CONCURRENCY``, and that distinction is load-bearing twice over. The - resolved count already refuses multi-worker without Redis, which is exactly - when :func:`get_ephemeral_state` would hand back the per-process store and - this lease would quietly stop being a lease. And a launch path that never - ran the entrypoint — ``cli.py``, ``api.app.main`` — would otherwise read the - *requested* value and believe a single process is a cluster. - """ return resolved_workers() > 1 class Leadership: - """A renewed lease naming one process the owner of a role's singleton work.""" + """Stop owned work before releasing the token; uncertainty revokes ownership.""" def __init__(self, role: str, *, lease_seconds: int = _LEASE_SECONDS) -> None: - self._key = f"{_LEADER_PREFIX}{role}" + self._key = f"jx:leader:{role}" self._role = role self._token = _token() self._lease = int(lease_seconds) self._elected = False - self._task: Optional[asyncio.Task] = None - self._work: Optional[asyncio.Task] = None + self._task: asyncio.Task | None = None + self._work: asyncio.Task | None = None + self._on_deposed: Callback | None = None async def _hold(self) -> bool: if not coordination_required(): return True try: - return await get_ephemeral_state().hold(self._key, self._token, ttl=self._lease) - except Exception as exc: # noqa: BLE001 - a store blip must not depose a healthy leader + state = get_ephemeral_state() + # Renewal must never recreate an expired lease. + operation = ( + state.compare_exchange(self._key, self._token, self._token, ttl=self._lease) + if self._elected + else state.hold(self._key, self._token, ttl=self._lease) + ) + return await asyncio.wait_for(operation, timeout=self._lease / 3) + except Exception as exc: logger.warning("leader_hold_failed", role=self._role, error=str(exc)) - return self._elected + return False + + async def _depose(self) -> None: + if not self._elected: + return + if self._work is not None: + if not self._work.done(): + self._work.cancel() + with contextlib.suppress(asyncio.CancelledError, Exception): + await self._work + self._work = None + # A failed stop must not release ownership or permit a new local start. + if self._on_deposed is not None: + await self._on_deposed() + self._elected = False + if coordination_required(): + try: + await get_ephemeral_state().release(self._key, self._token) + except Exception as exc: + logger.warning("leader_release_failed", role=self._role, error=str(exc)) - async def _supervise( - self, - on_elected: Callable[[], Awaitable[None]], - on_deposed: Callable[[], Awaitable[None]], - ) -> None: + async def _supervise(self, on_elected: Callback) -> None: while True: holding = await self._hold() - if holding and not self._elected: + failed = self._work is not None and self._work.done() and ( + self._work.cancelled() or self._work.exception() is not None + ) + if self._elected and (not holding or failed): + logger.warning("leader_deposed", role=self._role, startup_failed=failed) + await self._depose() + elif holding and not self._elected: self._elected = True - logger.info("leader_elected", role=self._role, token=self._token) - # Started, never awaited: the callback brings up every singleton - # loop in turn and can easily outlast a lease. Awaiting it here - # would stop renewing for that whole stretch, the lease would - # lapse mid-startup, and a second worker would elect itself and - # bring up a duplicate set — the exact outcome this exists to - # prevent, now paid for twice. self._work = asyncio.create_task(on_elected(), name=f"leader_work:{self._role}") - if not coordination_required(): - # Uncontended by construction: nothing can take this role, - # so there is nothing left to supervise. Returning keeps - # single-process deployments — the default, and the - # battery-powered desktop one — free of a forever timer. - return - elif not holding and self._elected: - # Losing a lease we were renewing means this process stalled - # past the whole lease; another worker owns the loops now and - # ours have to stop, or both run. - self._elected = False - logger.warning("leader_deposed", role=self._role, token=self._token) - await self._cancel_work() - await on_deposed() - await asyncio.sleep(_RENEW_SECONDS) - - async def _cancel_work(self) -> None: - if self._work is None or self._work.done(): - return - self._work.cancel() - with contextlib.suppress(asyncio.CancelledError): - await self._work - - def start( - self, - *, - on_elected: Callable[[], Awaitable[None]], - on_deposed: Callable[[], Awaitable[None]], - ) -> None: - self._task = asyncio.create_task( - self._supervise(on_elected, on_deposed), name=f"leader:{self._role}" - ) + await asyncio.sleep(self._lease / 3) + + def start(self, *, on_elected: Callback, on_deposed: Callback) -> None: + if self._task is not None: + raise RuntimeError("Leadership already started") + self._on_deposed = on_deposed + self._task = asyncio.create_task(self._supervise(on_elected), name=f"leader:{self._role}") async def stop(self) -> None: - if self._task is not None and not self._task.done(): - self._task.cancel() + if self._task is not None: + if not self._task.done(): + self._task.cancel() with contextlib.suppress(asyncio.CancelledError): await self._task - # The elected work used to live in the deferred-startup task, which the - # shutdown path cancels; now it is ours, so cancelling it is too. - await self._cancel_work() - if self._elected: - self._elected = False - try: - await get_ephemeral_state().drop(self._key) - except Exception as exc: # noqa: BLE001 - shutdown must not raise - logger.warning("leader_release_failed", role=self._role, error=str(exc)) + self._task = None + await self._depose() async def run_once( - name: str, step: Callable[[], Awaitable[None]], *, timeout: float = 300.0 + name: str, step: Callback, *, timeout: float = 300.0 ) -> None: - """Run *step* in one worker of this boot; the rest wait for it to finish. - - The workers of a single boot share a parent — uvicorn's master — so its pid - names this boot without any coordination between them. The done marker is - deliberately short-lived: a later restart is *supposed* to run the boot - steps again, and only workers of the same boot, which all start within a - few seconds of one another, ever need to see it. - """ + """All workers observe success, failure or timeout of required initialization.""" if not coordination_required(): await step() return - state = get_ephemeral_state() - boot = f"{_ONCE_PREFIX}{socket.gethostname()}:{os.getppid()}:{name}" + boot = f"jx:boot:{socket.gethostname()}:{os.getppid()}:{name}" lock, done = f"{boot}:lock", f"{boot}:done" - - if await state.claim(lock, ttl=int(timeout)): - try: - await step() - finally: - # The marker is written even when the step raised: a failed boot - # step is this worker's problem to log, not a reason to hang every - # other worker until the timeout. - await state.put(done, "1", ttl=60) - return - - for _ in range(int(timeout / _POLL_SECONDS)): - if await state.get(done) is not None: + deadline = asyncio.get_running_loop().time() + timeout + while True: + outcome = await state.get(done) + if outcome == "ok": + return + if outcome is not None: + raise RuntimeError(f"Required startup step failed: {name}") + if await state.claim(lock, ttl=max(1, math.ceil(timeout) + 1)): + try: + await asyncio.wait_for(step(), timeout=timeout) + except BaseException: + await state.put(done, "failed", ttl=max(60, math.ceil(timeout) * 2)) + raise + await state.put(done, "ok", ttl=max(60, math.ceil(timeout) * 2)) return + if asyncio.get_running_loop().time() >= deadline: + raise TimeoutError(f"Required startup step timed out: {name}") await asyncio.sleep(_POLL_SECONDS) - logger.warning("boot_step_wait_timeout", name=name, after_seconds=timeout) __all__ = ["Leadership", "coordination_required", "run_once"] diff --git a/src/backend/core/infra/local_runtime/__init__.py b/src/backend/core/infra/local_runtime/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/src/backend/core/infra/local_runtime/__init__.py @@ -0,0 +1 @@ + diff --git a/src/backend/core/infra/local_runtime/dynamic_servers.py b/src/backend/core/infra/local_runtime/dynamic_servers.py new file mode 100644 index 00000000..0754ef8b --- /dev/null +++ b/src/backend/core/infra/local_runtime/dynamic_servers.py @@ -0,0 +1,64 @@ +"""Own lifecycle and loop affinity for dynamically selected bundled sidecars.""" +import asyncio +import sys + +_lock = asyncio.Lock() + +async def ensure_dynamic_servers(bindings): + import core.infra.local_runtime.startup as host + + if not host._cloud_serves_tools(): + return # The local-only launcher already owns every bundled MCP. + if not bindings: + return + # Isolated agents execute on worker loops. Process handles, the watchdog + # and the serialization lock remain on the backend's lifecycle loop. + owner_loop = ( + host._WATCHDOG.get_loop() if host._WATCHDOG is not None else asyncio.get_running_loop() + ) + if owner_loop is not asyncio.get_running_loop(): + future = asyncio.run_coroutine_threadsafe(_start(bindings), owner_loop) + await asyncio.wrap_future(future) + return + await _start(bindings) + + +async def _start(bindings): + import core.infra.local_runtime.startup as host + from extensions.mcp_servers._ports import package_name + + async with _lock: + for sid, port in sorted(bindings): + label = "plugin_mcp:" + sid + running = next( + (p for name, p in host._PROCS if name == label and p.returncode is None), None + ) + if running is not None: + continue + if await host._tcp_port_ready("127.0.0.1", port): + raise RuntimeError("local plugin MCP port is already owned by another process") + proc = await host._spawn( + label, + [ + sys.executable, + "-m", + f"extensions.mcp_servers.{package_name(sid)}.server", + "--transport", + "streamable-http", + "--port", + str(port), + ], + ) + if proc is None: + raise RuntimeError("local plugin MCP could not start") + try: + await host._wait_for_mcp_ports( + proc, {sid: port}, timeout=host._ready_timeout_seconds() + ) + await host._list_mcp_tool_names(sid, port) + except BaseException: + from core.infra.local_runtime.lifecycle import discard + + await discard(label, proc) + raise + host._start_watchdog() diff --git a/src/backend/orchestration/local_sidecar_lifecycle.py b/src/backend/core/infra/local_runtime/lifecycle.py similarity index 89% rename from src/backend/orchestration/local_sidecar_lifecycle.py rename to src/backend/core/infra/local_runtime/lifecycle.py index 57c5a033..3221b0e5 100644 --- a/src/backend/orchestration/local_sidecar_lifecycle.py +++ b/src/backend/core/infra/local_runtime/lifecycle.py @@ -1,19 +1,28 @@ """Lifecycle ownership shared by startup, lazy plugin activation and restart.""" + from __future__ import annotations + import asyncio import contextlib import os import signal from typing import List, Optional -from orchestration import local_subprocess as host + +import core.infra.local_runtime.startup as host +import core.infra.local_runtime.runner_auth as runner_auth _spawn_lock = asyncio.Lock() -async def _spawn(label: str, argv: List[str], *, require_registered=False) -> Optional["asyncio.subprocess.Process"]: + +async def _spawn( + label: str, argv: List[str], *, require_registered=False +) -> Optional["asyncio.subprocess.Process"]: async with _spawn_lock: if host._SHUTTING_DOWN or (require_registered and host._SPECS.get(label) != argv): return None - running = next((p for name, p in host._PROCS if name == label and p.returncode is None), None) + running = next( + (p for name, p in host._PROCS if name == label and p.returncode is None), None + ) if running is not None: return running return await _create(label, argv) @@ -54,7 +63,7 @@ async def _start_script_runner(py: str) -> None: # 关起来的那条命令（回环在沙箱里依然可达）。没有这个密钥，它再调一次 /processes/start # 就能拿到一个不带任何约束的子进程，等于从自己所在的沙箱里走出来。密钥经环境变量 # 交给 sidecar，用户命令那一侧用的是白名单 env，拿不到它。 - os.environ[host.runner_auth.ENV_VAR] = host.runner_auth.ensure_token() + os.environ[runner_auth.ENV_VAR] = runner_auth.ensure_token() runner_port = host._script_runner_port() # 清理后目标端口仍有监听者 → 是外部程序在占用。不同品牌的桌面壳 # 应通过独立端口命名空间避免走到这里。 @@ -100,8 +109,7 @@ async def _start_script_runner(py: str) -> None: if asyncio.get_event_loop().time() > runner_deadline: await stop_local_sidecars() raise RuntimeError( - f"本机代码执行服务未就绪（127.0.0.1:{runner_port}）——" - "端口可能被其它程序占用" + f"本机代码执行服务未就绪（127.0.0.1:{runner_port}）——" "端口可能被其它程序占用" ) await asyncio.sleep(0.5) @@ -114,9 +122,7 @@ async def _supervise_sidecars() -> None: for index, (label, proc) in enumerate(list(host._PROCS)): if proc.returncode is None: continue - host.logger.warning( - "local_sidecar_exited", sidecar=label, returncode=proc.returncode - ) + host.logger.warning("local_sidecar_exited", sidecar=label, returncode=proc.returncode) try: host._PROCS.remove((label, proc)) except ValueError: @@ -160,8 +166,9 @@ async def stop_local_sidecars() -> None: except asyncio.TimeoutError: try: proc.kill() - except Exception: # noqa: BLE001 - pass + except Exception as fallback_error: # noqa: BLE001 + from core.infra.failures import record_best_effort_failure + record_best_effort_failure('core.infra.local_runtime.lifecycle:stop_local_sidecars', fallback_error) host.logger.info("local_sidecar_stopped", sidecar=label) host._PROCS.clear() @@ -191,9 +198,7 @@ def _child_env() -> dict: # single-machine profile. env["MCP_HOST"] = "127.0.0.1" env["MCP_BIND_HOST"] = "127.0.0.1" - inherited_pythonpath = [ - entry for entry in env.get("PYTHONPATH", "").split(os.pathsep) if entry - ] + inherited_pythonpath = [entry for entry in env.get("PYTHONPATH", "").split(os.pathsep) if entry] pythonpath = [host._BACKEND_DIR] pythonpath.extend(entry for entry in inherited_pythonpath if entry != host._BACKEND_DIR) env["PYTHONPATH"] = os.pathsep.join(pythonpath) diff --git a/src/backend/core/sandbox/providers/script_runner/auth.py b/src/backend/core/infra/local_runtime/runner_auth.py similarity index 100% rename from src/backend/core/sandbox/providers/script_runner/auth.py rename to src/backend/core/infra/local_runtime/runner_auth.py diff --git a/src/backend/orchestration/local_subprocess.py b/src/backend/core/infra/local_runtime/startup.py similarity index 93% rename from src/backend/orchestration/local_subprocess.py rename to src/backend/core/infra/local_runtime/startup.py index fa780c0a..11fdb3fd 100644 --- a/src/backend/orchestration/local_subprocess.py +++ b/src/backend/core/infra/local_runtime/startup.py @@ -35,7 +35,6 @@ from core.config.settings import settings from core.infra.logging import get_logger from core.infra.proc import no_window_kwargs -import core.sandbox.providers.script_runner.auth as runner_auth logger = get_logger(__name__) @@ -51,7 +50,7 @@ # of automatically adding ``src/backend`` (the CLI script's directory) to # ``sys.path``. Keep the packaged source root explicit so the desktop runtime # can import ``mcp_servers`` and ``services`` on every platform. -_BACKEND_DIR = str(Path(__file__).resolve().parents[1]) +_BACKEND_DIR = str(Path(__file__).resolve().parents[3]) # These are not optional conveniences in the local/desktop product: they back # the three plugins installed on the first zero-state boot. Keeping this @@ -75,7 +74,6 @@ "services.script_runner_service.server:app", ) - def _packaged_install_root() -> Optional[str]: """打包桌面形态的安装根指纹；开发形态返回 None（跳过清理）。 @@ -91,7 +89,6 @@ def _packaged_install_root() -> Optional[str]: return None return exe[: idx + len(seg) - 1] - def _find_stale_sidecar_pids( ps_output: str, *, root_marker: str, my_pid: int, fold_case: bool = False ) -> List[int]: @@ -121,7 +118,6 @@ def _find_stale_sidecar_pids( pids.append(pid) return pids - async def _process_table() -> str: """`` `` 一行一个进程。Windows 没有 ps，走 CIM。""" if os.name == "nt": @@ -143,7 +139,6 @@ async def _process_table() -> str: out, _ = await proc.communicate() return out.decode("utf-8", "replace") - async def _describe_port_owner(port: int) -> str: """占用 loopback 端口的进程描述（``pid name``），查不到时返回空串。""" try: @@ -183,7 +178,6 @@ async def _describe_port_owner(port: int) -> str: except Exception: # noqa: BLE001 — 诊断信息缺失不改变失败本身 return "" - def _listener_pid_from_netstat(output: str, port: int) -> Optional[int]: """从 ``netstat -ano`` 输出里取 loopback/任意地址上该端口的监听进程。""" suffix = f":{port}" @@ -199,7 +193,6 @@ def _listener_pid_from_netstat(output: str, port: int) -> Optional[int]: continue return None - async def _terminate_process_trees(pids: List[int]) -> None: """Windows 没有 SIGTERM；结束进程时连子进程树一起结束，MCP launcher 的 子服务才不会变成下一次启动要清理的孤儿。""" @@ -216,7 +209,6 @@ async def _terminate_process_trees(pids: List[int]) -> None: ) await proc.wait() - def _pid_alive(pid: int) -> bool: try: os.kill(pid, 0) @@ -226,7 +218,6 @@ def _pid_alive(pid: int) -> bool: return True return True - async def _reap_stale_sidecars() -> None: """启动期一次性清掉上次会话遗留的孤儿 sidecar，再拉本会话自己的。 @@ -272,13 +263,10 @@ async def _reap_stale_sidecars() -> None: if reaped and settings.sandbox.provider == "script_runner": runner_port = _script_runner_port() deadline = time.monotonic() + 3.0 - while time.monotonic() < deadline and await _tcp_port_ready( - "127.0.0.1", runner_port - ): + while time.monotonic() < deadline and await _tcp_port_ready("127.0.0.1", runner_port): await asyncio.sleep(0.2) logger.info("stale_sidecars_reaped", count=reaped) - def _script_runner_port() -> int: """Return the local runner port from the already-resolved settings URL.""" @@ -293,14 +281,11 @@ def _script_runner_port() -> int: "127.0.0.1", "localhost", }: - raise RuntimeError( - "本机模式 SANDBOX_RUNNER_URL 必须指向 loopback：http://127.0.0.1:" - ) + raise RuntimeError("本机模式 SANDBOX_RUNNER_URL 必须指向 loopback：http://127.0.0.1:") if port is None or port == 0: raise RuntimeError(f"SANDBOX_RUNNER_URL 缺少端口：{raw!r}") return port - def _ready_timeout_seconds() -> float: raw = os.getenv("LOCAL_SIDECAR_READY_TIMEOUT_SECONDS", "30") try: @@ -308,17 +293,15 @@ def _ready_timeout_seconds() -> float: except (TypeError, ValueError): return 30.0 - def _child_env(): - from orchestration.local_sidecar_lifecycle import _child_env as child_env + from core.infra.local_runtime.lifecycle import _child_env as child_env return child_env() - async def _spawn(label, argv): - from orchestration import local_sidecar_lifecycle as lifecycle - return await lifecycle._spawn(label, argv) + import core.infra.local_runtime.lifecycle as lifecycle + return await lifecycle._spawn(label, argv) async def _tcp_port_ready(host: str, port: int) -> bool: try: @@ -334,7 +317,6 @@ async def _tcp_port_ready(host: str, port: int) -> bool: except (OSError, asyncio.TimeoutError): return False - async def _wait_for_mcp_ports( launcher: "asyncio.subprocess.Process", ports: dict[str, int], @@ -361,22 +343,20 @@ async def _wait_for_mcp_ports( details = ", ".join(f"{server_id}:{port}" for server_id, port in pending.items()) raise RuntimeError(f"MCP 服务未在 {timeout:.0f} 秒内就绪：{details}") - async def _list_mcp_tool_names(server_id: str, port: int) -> set[str]: """Use the production MCP client to verify a server's actual tool list.""" - from core.llm.mcp_pool import make_client + from core.harness.registry.mcp_clients import discover_tools - client = make_client( + client = await discover_tools( server_id, { "transport": "streamable_http", "url": f"http://127.0.0.1:{port}/mcp/", "transport_timeout": 5, }, - is_stateful=False, ) try: - tools = await client.list_tools() + tools = client.tools return {str(name) for tool in tools if (name := getattr(tool, "name", None))} finally: try: @@ -388,9 +368,9 @@ async def _list_mcp_tool_names(server_id: str, port: int) -> set[str]: task = asyncio.current_task() if task is not None and task.cancelling(): raise - except Exception: # noqa: BLE001 — readiness result is already known - pass - + except Exception as fallback_error: # noqa: BLE001 — readiness result is already known + from core.infra.failures import record_best_effort_failure + record_best_effort_failure('core.infra.local_runtime.startup:_list_mcp_tool_names', fallback_error) async def _verify_required_plugin_tools(ports: dict[str, int]) -> None: """Assert that every default plugin MCP exposes its contract tool.""" @@ -410,24 +390,22 @@ async def _verify(server_id: str, expected_tool: str) -> None: ) ) - def _cloud_serves_tools() -> bool: """混合模式（桌面壳注入了桥接密钥）下，工具全部来自云端 capability manifest： 本机执行面只负责本地项目的代码执行，不运行任何内置 MCP。""" - from core.auth.desktop_bridge import bridge_enabled + from core.harness.registry.host_runtime import requires_remote_execution as bridge_enabled return bridge_enabled() - def _start_watchdog(): - from orchestration import local_sidecar_lifecycle as lifecycle - return lifecycle._start_watchdog() + import core.infra.local_runtime.lifecycle as lifecycle + return lifecycle._start_watchdog() async def _start_script_runner(py: str): - from orchestration import local_sidecar_lifecycle as lifecycle - return await lifecycle._start_script_runner(py) + import core.infra.local_runtime.lifecycle as lifecycle + return await lifecycle._start_script_runner(py) async def start_local_sidecars() -> None: """Spawn the script_runner sidecar, plus the MCP launcher for local-only installs.""" @@ -443,7 +421,9 @@ async def start_local_sidecars() -> None: if _cloud_serves_tools(): await _start_script_runner(py) _start_watchdog() - logger.info("local_sidecars_ready", mode="cloud_tools", sidecars=[label for label, _ in _PROCS]) + logger.info( + "local_sidecars_ready", mode="cloud_tools", sidecars=[label for label, _ in _PROCS] + ) return from extensions.mcp_servers._launcher import PORTS as launcher_ports @@ -489,12 +469,12 @@ async def start_local_sidecars() -> None: _start_watchdog() - async def _supervise_sidecars(): - from orchestration import local_sidecar_lifecycle as lifecycle - return await lifecycle._supervise_sidecars() + import core.infra.local_runtime.lifecycle as lifecycle + return await lifecycle._supervise_sidecars() async def stop_local_sidecars(): - from orchestration import local_sidecar_lifecycle as lifecycle + import core.infra.local_runtime.lifecycle as lifecycle + return await lifecycle.stop_local_sidecars() diff --git a/src/backend/core/infra/model_output.py b/src/backend/core/infra/model_output.py new file mode 100644 index 00000000..157af94e --- /dev/null +++ b/src/backend/core/infra/model_output.py @@ -0,0 +1,95 @@ +"""Shared reasoning-block cleanup and tolerant JSON extraction for model output.""" +from __future__ import annotations + +import json +import re +from typing import Any, Dict, Optional + + +THINK_BLOCK_RE = re.compile(r".*?", re.DOTALL | re.IGNORECASE) + + +_THINK_CLOSE_RE = re.compile(r"", re.IGNORECASE) + + +_THINK_OPEN_RE = re.compile(r"", re.IGNORECASE) + + +def strip_think_blocks(text: str) -> str: + """Strip a reasoning model's think blocks and return the body. + + Covers three shapes: + - closed ``...`` blocks (possibly multiple); + - only ```` (some serving stacks swallow the opening tag into the chat template, with the body following the closing tag) + → take the content after the last closing tag; + - only ```` (output was truncated mid-thought by max_tokens, with no body after) + → discard everything from the opening tag onward. + """ + t = THINK_BLOCK_RE.sub("", text) + m = None + for m in _THINK_CLOSE_RE.finditer(t): + pass + if m is not None: + t = t[m.end():] + m = _THINK_OPEN_RE.search(t) + if m is not None: + t = t[: m.start()] + return t.strip() + + +def extract_first_json_object(text: str) -> Optional[str]: + start = text.find("{") + if start < 0: + return None + depth = 0 + in_str = False + escape = False + for i, ch in enumerate(text[start:], start=start): + if in_str: + if escape: + escape = False + elif ch == "\\": + escape = True + elif ch == '"': + in_str = False + continue + if ch == '"': + in_str = True + elif ch == "{": + depth += 1 + elif ch == "}": + depth -= 1 + if depth == 0: + return text[start:i + 1] + return None + + +def parse_strict_json(text: str) -> Optional[Dict[str, Any]]: + """Parse JSON tolerantly: strips blocks, ``` fences, and surrounding prose.""" + if not text: + return None + # If the body is already valid JSON, return it directly to avoid corruption from stripping a think tag that happens to appear inside a string value + try: + parsed = json.loads(text.strip()) + if isinstance(parsed, dict): + return parsed + except json.JSONDecodeError: + parsed = None + t = strip_think_blocks(text.strip()) + + if t.startswith("```"): + lines = [l for l in t.splitlines() if not l.strip().startswith("```")] + t = "\n".join(lines).strip() + + try: + return json.loads(t) + except json.JSONDecodeError: + parsed = None + + candidate = extract_first_json_object(t) + if candidate: + try: + return json.loads(candidate) + except json.JSONDecodeError: + return None + return None diff --git a/src/backend/core/infra/probes/__init__.py b/src/backend/core/infra/probes/__init__.py new file mode 100644 index 00000000..4d450022 --- /dev/null +++ b/src/backend/core/infra/probes/__init__.py @@ -0,0 +1 @@ +"""Health probes for the services the backend depends on.""" diff --git a/src/backend/core/services/edition_service_probe.py b/src/backend/core/infra/probes/edition.py similarity index 100% rename from src/backend/core/services/edition_service_probe.py rename to src/backend/core/infra/probes/edition.py diff --git a/src/backend/core/services/service_probes.py b/src/backend/core/infra/probes/service_probes.py similarity index 89% rename from src/backend/core/services/service_probes.py rename to src/backend/core/infra/probes/service_probes.py index 59b685c6..371428c5 100644 --- a/src/backend/core/services/service_probes.py +++ b/src/backend/core/infra/probes/service_probes.py @@ -38,27 +38,6 @@ def config_row_to_dict(cfg: dict, mask: bool = True) -> dict: # ── After config changes: rebuild the MCP connection pool ──────────────────── -async def reinitialize_mcp_pool(source: str = "service_configs") -> None: - """Invalidate MCP config cache and reinitialize stable connections. - - Called after system config changes so that MCP sub-processes pick up - new env vars (e.g. updated API URLs/keys). - """ - try: - from core.services.mcp_service import McpServerConfigService - - mcp_svc = McpServerConfigService.get_instance() - mcp_svc.invalidate_cache() - - from core.llm.mcp_pool import MCPConnectionPool - - pool = MCPConnectionPool.get_instance() - if pool.is_initialized: - new_configs = mcp_svc.get_all_servers() - await pool.reinitialize_if_config_changed(new_configs) - logger.info("[%s] MCP pool reinitialize triggered after config update", source) - except Exception as exc: - logger.warning("[%s] MCP pool reinitialize failed: %s", source, exc) # ── Connectivity probes ────────────────────────────────────────────────────── @@ -190,12 +169,12 @@ async def test_service_group(group_key: str) -> dict: Supports knowledge_base / file_parser / internet_search; other groups return "unsupported". Shared by service_configs (EE) and me_system (CE). """ - from core.services.system_config import SystemConfigService + from core.config.store.system_config import SystemConfigService svc = SystemConfigService.get_instance() if group_key == "knowledge_base": - from core.services.edition_service_probe import test_external_knowledge + from core.infra.probes.edition import test_external_knowledge provider = (svc.get("knowledge_base.provider") or "").strip().lower() if provider in {"custom", "local"}: diff --git a/src/backend/core/infra/rate_limit.py b/src/backend/core/infra/rate_limit.py deleted file mode 100644 index eb106ba3..00000000 --- a/src/backend/core/infra/rate_limit.py +++ /dev/null @@ -1,21 +0,0 @@ -"""HTTP rate limiter configuration. Circuit state lives in circuit_breaker.""" - -from core.config.settings import settings -from slowapi import Limiter -from slowapi.util import get_remote_address - - -def get_rate_limit_enabled() -> bool: - return settings.rate_limit.enabled - - -def get_rate_limit_storage() -> str: - return settings.rate_limit.storage - - -limiter = Limiter( - key_func=get_remote_address, - storage_uri=get_rate_limit_storage(), - enabled=get_rate_limit_enabled(), - headers_enabled=True, -) diff --git a/src/backend/core/infra/redis.py b/src/backend/core/infra/redis.py index 05514d8b..f361e7a1 100644 --- a/src/backend/core/infra/redis.py +++ b/src/backend/core/infra/redis.py @@ -1,11 +1,12 @@ """Event-loop-local Redis clients for session storage.""" +from __future__ import annotations import asyncio import threading +from dataclasses import dataclass, field from weakref import WeakKeyDictionary import redis.asyncio as aioredis - from core.config.settings import settings from core.infra.logging import get_logger @@ -30,14 +31,13 @@ _stream_pools: WeakKeyDictionary[asyncio.AbstractEventLoop, aioredis.Redis] = ( WeakKeyDictionary() ) -_redis_pools_lock = threading.Lock() def redis_configured() -> bool: """Whether this deployment has a Redis to talk to. The single predicate every store-agnostic seam branches on - (:mod:`core.infra.ephemeral`, :mod:`orchestration.run_event_stream`). A + (:mod:`core.infra.ephemeral`, :mod:`core.runs.events.store`). A deployment without Redis — the desktop's single-process backend — leaves ``REDIS_URL`` empty and those seams use their in-process backend instead. """ @@ -52,8 +52,7 @@ def _create_client(max_connections: int) -> tuple[aioredis.Redis, str]: # in-process backend from redis_configured() long before reaching here, # so arriving with no URL means a caller bypassed one. raise RuntimeError( - "REDIS_URL is not configured; use core.infra.ephemeral / " - "orchestration.run_event_stream instead of get_redis()" + 'REDIS_URL is not configured; use core.infra.ephemeral / core.runs.events.store instead of get_redis()' ) # NOTE: redis-py 8.0 changed the default socket_timeout from None -> 5s. # The chat-stream follower blocks on `XREAD BLOCK 5000`; if the socket @@ -79,7 +78,7 @@ def _get_pooled( role: str, ) -> aioredis.Redis: loop = asyncio.get_running_loop() - with _redis_pools_lock: + with process_state().redis_pools_lock: existing = pools.get(loop) if existing is not None: return existing @@ -114,7 +113,7 @@ def get_redis(*, blocking: bool = False) -> aioredis.Redis: Only call this where Redis is known to be configured. Deployments without one (the desktop backend) leave ``REDIS_URL`` empty and reach the same capabilities through :mod:`core.infra.ephemeral` and - :mod:`orchestration.run_event_stream`, which pick an in-process backend. + :mod:`core.runs.events.store`, which pick an in-process backend. There is deliberately no imitation Redis to fall back on: an imitation has to track redis-py's wire protocol, and a mismatch shows up as a silent stall rather than a clear failure. @@ -131,7 +130,7 @@ def get_redis(*, blocking: bool = False) -> aioredis.Redis: async def close_redis() -> None: """Close every Redis client owned by the current event loop.""" loop = asyncio.get_running_loop() - with _redis_pools_lock: + with process_state().redis_pools_lock: clients = [ pools.pop(loop, None) for pools in (_redis_pools, _stream_pools) ] @@ -139,3 +138,15 @@ async def close_redis() -> None: if client is not None: await client.aclose() logger.info("redis_pool_closed", event_loop=id(loop)) + + +@dataclass +class RedisState: + redis_pools_lock: object = field(default_factory=lambda: threading.Lock()) + + +_STATE = RedisState() + + +def process_state() -> RedisState: + return _STATE diff --git a/src/backend/core/infra/runtime_state.py b/src/backend/core/infra/runtime_state.py index af97dc56..d00de8e9 100644 --- a/src/backend/core/infra/runtime_state.py +++ b/src/backend/core/infra/runtime_state.py @@ -2,27 +2,39 @@ The FastAPI app starts several background workers (idle-session reaper, automation scheduler, distillation scheduler, …) and stores their handles as -module globals. Monitoring code (``core.services.security_service``) needs to +module globals. Monitoring code (``features.security_console.console``) needs to report whether they are running, but it must not import the API app module (that would be a ``core → api`` upward dependency). The app registers handles here on startup; monitoring reads them back through this neutral registry. """ +from __future__ import annotations +from dataclasses import dataclass, field from typing import Any, Dict -_REGISTRY: Dict[str, Any] = {} - def register(name: str, handle: Any) -> None: """Register (or replace) a named background task/scheduler handle.""" - _REGISTRY[name] = handle + process_state().REGISTRY[name] = handle def get(name: str) -> Any: """Return the handle registered under ``name`` (or None).""" - return _REGISTRY.get(name) + return process_state().REGISTRY.get(name) def clear() -> None: """Drop all registered handles (used on shutdown).""" - _REGISTRY.clear() + process_state().REGISTRY.clear() + + +@dataclass +class RuntimeState: + REGISTRY: Dict[str, Any] = field(default_factory=lambda: {}) + + +_STATE = RuntimeState() + + +def process_state() -> RuntimeState: + return _STATE diff --git a/src/backend/core/chat/display_bounds.py b/src/backend/core/infra/text_bounds.py similarity index 100% rename from src/backend/core/chat/display_bounds.py rename to src/backend/core/infra/text_bounds.py diff --git a/src/backend/core/infra/time.py b/src/backend/core/infra/time.py index 118562f9..a646ef02 100644 --- a/src/backend/core/infra/time.py +++ b/src/backend/core/infra/time.py @@ -1,17 +1,29 @@ """Application instants use aware UTC; calendar schedules keep their explicit zone.""" from datetime import datetime, timezone +from typing import Optional, overload def utc_now() -> datetime: return datetime.now(timezone.utc) -def as_utc(value: datetime) -> datetime: +@overload +def as_utc(value: datetime) -> datetime: ... + + +@overload +def as_utc(value: None) -> None: ... + + +def as_utc(value: Optional[datetime]) -> Optional[datetime]: """Normalize an instant; legacy timezone-less application values mean UTC. + ``None`` passes through so nullable columns can be normalized in place. User-entered local calendar values must first be localized with their declared business timezone, not passed here as naive timestamps. """ + if value is None: + return None if value.tzinfo is None: return value.replace(tzinfo=timezone.utc) return value.astimezone(timezone.utc) diff --git a/src/backend/core/infra/worker_count.py b/src/backend/core/infra/worker_count.py index f0d73ee7..adee4233 100644 --- a/src/backend/core/infra/worker_count.py +++ b/src/backend/core/infra/worker_count.py @@ -11,7 +11,7 @@ * ``SESSION_STORE=memory`` — a login handled by worker A is invisible to worker B, so a browser is randomly signed out on roughly ``1 - 1/N`` of its requests. * No Redis — the per-run SSE event log falls back to an in-process buffer - (:mod:`orchestration.run_event_stream`), and so does the mutex behind + (:mod:`core.runs.events.store`), and so does the mutex behind :mod:`core.infra.leader`, so neither the stream nor the leader lease would actually cross a process boundary. * ``DEPLOY_PROFILE=local`` — the no-Docker desktop profile hosts MCP and sandbox diff --git a/src/backend/orchestration/jobs/__init__.py b/src/backend/core/jobs/__init__.py similarity index 100% rename from src/backend/orchestration/jobs/__init__.py rename to src/backend/core/jobs/__init__.py diff --git a/src/backend/orchestration/jobs/agent.py b/src/backend/core/jobs/agent.py similarity index 94% rename from src/backend/orchestration/jobs/agent.py rename to src/backend/core/jobs/agent.py index c2b8a322..f0a9bc17 100644 --- a/src/backend/orchestration/jobs/agent.py +++ b/src/backend/core/jobs/agent.py @@ -58,9 +58,9 @@ def snapshot_parent_runtime(request, *, skill_ids, mcp_ids, sandbox_session_id, def resolve_subagents(runtime: dict, user_id: str) -> list[dict]: """Recheck current accessibility; never add agents absent from the parent.""" from core.db.engine import SessionLocal - from core.llm.builtin_subagents import merge_builtin_subagents - from core.services.user_agent_service import UserAgentService - from core.services.user_service import UserService + from core.capabilities.agents.builtin import merge_builtin_subagents + from core.capabilities.agents.edition import UserAgentService + from core.auth.users import UserService ids = set(runtime.get("subagent_ids") or []) if not ids: @@ -86,11 +86,11 @@ async def run_derived_agent( tools: list[str] | None = None, ): """Use the main factory and streaming loop with fresh task-local context.""" - from core.llm import workspace + from core.llm.runtime import workspace from core.llm.factory import create_agent_executor - from core.llm.mcp_manager import close_clients - from core.services.chat_mode_service import ChatModeSpec - from orchestration.streaming import StreamingAgent + from core.llm.mcp.manager import close_clients + from core.llm.runtime.streaming import StreamingAgent + from core.chat.mode_types import ChatModeSpec if not isinstance(parent_runtime, dict): raise ValueError("作业缺少主智能体运行配置，请重新提交工作流") diff --git a/src/backend/orchestration/jobs/callback.py b/src/backend/core/jobs/callback.py similarity index 100% rename from src/backend/orchestration/jobs/callback.py rename to src/backend/core/jobs/callback.py diff --git a/src/backend/orchestration/jobs/files.py b/src/backend/core/jobs/files.py similarity index 95% rename from src/backend/orchestration/jobs/files.py rename to src/backend/core/jobs/files.py index 3712a2f9..4465df26 100644 --- a/src/backend/orchestration/jobs/files.py +++ b/src/backend/core/jobs/files.py @@ -3,17 +3,15 @@ import base64 from typing import Tuple -from core.sandbox import ProcessRequest, get_sandbox_provider - def _job_directory(session_id: str, job_row_id: str) -> str: - from core.llm.tools._paths import workspace_directory + from core.sandbox.workspace.resolution import workspace_directory return f"{workspace_directory(session_id)}/.job/{job_row_id}" def _quote_path(path: str) -> str: - from core.llm.tools._paths import path_rules + from core.sandbox.workspace.resolution import path_rules return path_rules().quote_shell_path(path) diff --git a/src/backend/orchestration/jobs/notifications.py b/src/backend/core/jobs/notifications.py similarity index 81% rename from src/backend/orchestration/jobs/notifications.py rename to src/backend/core/jobs/notifications.py index 78a681af..54a03057 100644 --- a/src/backend/orchestration/jobs/notifications.py +++ b/src/backend/core/jobs/notifications.py @@ -6,7 +6,7 @@ logger = logging.getLogger(__name__) -def _emit_progress(chat_id: Optional[str], note: str) -> None: +def emit_progress(chat_id: Optional[str], note: str) -> None: """活性信号：转译成 model_progress 喂主 run 的无活动看门狗。 长作业期间主对话没有任何可渲染输出，缺了这条会被 600s 看门狗当成卡死强杀。 @@ -17,7 +17,7 @@ def _emit_progress(chat_id: Optional[str], note: str) -> None: if not chat_id: return try: - from core.llm import _subagent_stream + from core.llm.subagents import stream as _subagent_stream if not _subagent_stream.is_active(chat_id): return @@ -25,8 +25,9 @@ def _emit_progress(chat_id: Optional[str], note: str) -> None: chat_id, {"sub_type": "progress", "agent_id": "job", "agent_name": "批量作业", "note": note}, ) - except Exception: # noqa: BLE001 —— 活性信号是尽力而为，永远不该拖垮作业 - pass + except Exception as fallback_error: # noqa: BLE001 —— 活性信号是尽力而为，永远不该拖垮作业 + from core.infra.failures import record_best_effort_failure + record_best_effort_failure('core.jobs.notifications:_emit_progress', fallback_error) def _emit_ui_progress( @@ -51,12 +52,12 @@ def _emit_ui_progress( if not chat_id: return try: - from core.llm import _subagent_stream + from core.llm.subagents import stream as _subagent_stream if not _subagent_stream.is_active(chat_id): return try: - from core.llm.middlewares import CURRENT_TOOL_CALL_ID + from core.harness.tool_call_context import CURRENT_TOOL_CALL_ID parent_tool_id = CURRENT_TOOL_CALL_ID.get("") or "" except Exception: # noqa: BLE001 @@ -82,14 +83,15 @@ def _emit_ui_progress( "pending": int(stats.get("pending", 0) or 0), }, ) - except Exception: # noqa: BLE001 —— 进度是尽力而为，永远不该拖垮作业 - pass + except Exception as fallback_error: # noqa: BLE001 —— 进度是尽力而为，永远不该拖垮作业 + from core.infra.failures import record_best_effort_failure + record_best_effort_failure('core.jobs.notifications:_emit_ui_progress', fallback_error) async def _maybe_wake(job_row_id: str) -> None: """作业终态后叫醒会话（幂等；失败只记日志，不影响作业结果）。""" try: - from orchestration.job_wakeup import wake_on_job_finish + from core.chat.runtime.job_wakeup import wake_on_job_finish await wake_on_job_finish(job_row_id) except Exception as exc: # noqa: BLE001 @@ -101,7 +103,7 @@ async def _maybe_wake_progress( ) -> None: """中途播报进度（失败只记日志，绝不影响作业本身）。""" try: - from orchestration.job_wakeup import wake_on_job_progress + from core.chat.runtime.job_wakeup import wake_on_job_progress await wake_on_job_progress( job_row_id, stats=stats, budget_left=budget_left, stalled=stalled diff --git a/src/backend/orchestration/jobs/owner.py b/src/backend/core/jobs/owner.py similarity index 62% rename from src/backend/orchestration/jobs/owner.py rename to src/backend/core/jobs/owner.py index a500bc3b..8db024bb 100644 --- a/src/backend/orchestration/jobs/owner.py +++ b/src/backend/core/jobs/owner.py @@ -1,46 +1,43 @@ """Route job control onto the long-lived service loop, never a model worker loop.""" +from __future__ import annotations import asyncio import functools import logging import threading +from dataclasses import dataclass, field logger = logging.getLogger(__name__) -_loop = None -_guard = threading.Lock() -_tasks = set() def bind(): - global _loop current = asyncio.get_running_loop() - with _guard: - if _loop is not None and _loop.is_running() and _loop is not current: + with process_state().guard: + if process_state().loop is not None and process_state().loop.is_running() and process_state().loop is not current: raise RuntimeError("Job service is already bound to another live loop") - _loop = current + process_state().loop = current def service_loop(): - global _loop from core.sandbox import get_sandbox_provider - with _guard: - if _loop is None or _loop.is_closed(): + with process_state().guard: + if process_state().loop is None or process_state().loop.is_closed(): provider_loop = getattr(get_sandbox_provider(), "_service_loop", None) - _loop = provider_loop - if _loop is None: + process_state().loop = provider_loop + if process_state().loop is None: raise RuntimeError("Job service is not initialized on the application loop") - if not _loop.is_running(): + if not process_state().loop.is_running(): raise RuntimeError("Job service loop is not running") - return _loop + return process_state().loop def _retain(coro): task = asyncio.create_task(coro) - _tasks.add(task) + process_state().tasks.add(task) def done(finished): - _tasks.discard(finished) + process_state().tasks.discard(finished) if not finished.cancelled() and finished.exception(): logger.warning("Job control operation failed: %s", type(finished.exception()).__name__) @@ -64,3 +61,17 @@ async def operation(): return await asyncio.shield(asyncio.wrap_future(future)) return wrapper + + +@dataclass +class OwnerState: + loop: object = None + guard: object = field(default_factory=lambda: threading.Lock()) + tasks: object = field(default_factory=lambda: set()) + + +_STATE = OwnerState() + + +def process_state() -> OwnerState: + return _STATE diff --git a/src/backend/orchestration/jobs/process.py b/src/backend/core/jobs/process.py similarity index 95% rename from src/backend/orchestration/jobs/process.py rename to src/backend/core/jobs/process.py index d1985fa7..b8dc7d30 100644 --- a/src/backend/orchestration/jobs/process.py +++ b/src/backend/core/jobs/process.py @@ -7,9 +7,11 @@ from typing import Any from core.sandbox import ProcessRequest, get_sandbox_provider -from core.sandbox.providers.opensandbox.recovery import execution_reference -from core.sandbox.providers.opensandbox.recovery import recover_execution -from core.sandbox.providers.opensandbox.recovery import release_execution +from core.sandbox.providers.opensandbox.recovery import ( + execution_reference, + recover_execution, + release_execution, +) from . import state from .callback import resolve_callback_base @@ -82,8 +84,9 @@ async def receipt(self): "failed", ): return data - except Exception: - pass + except Exception as fallback_error: + from core.infra.failures import record_best_effort_failure + record_best_effort_failure('core.jobs.process:receipt', fallback_error) return None async def log(self): diff --git a/src/backend/core/jobs/reminder_hook.py b/src/backend/core/jobs/reminder_hook.py new file mode 100644 index 00000000..26375f33 --- /dev/null +++ b/src/backend/core/jobs/reminder_hook.py @@ -0,0 +1,80 @@ +"""Job-ledger reminder contributed by the execution host.""" +import asyncio +from typing import Optional +from core.harness.hooks import Decision, FailureMode, HookSpec, HookStage + + +class JobLedgerHook: + def __init__(self, *, chat_id: Optional[str], user_id: Optional[str]) -> None: + self._chat_id = chat_id or "" + self._user_id = user_id or "" + self._last_key: tuple | None = None + + def _ledger_lines(self) -> list[str]: + from core.db.engine import SessionLocal + from core.db.models import Job + from core.jobs.store import JobService + + with SessionLocal() as db: + jobs = ( + db.query(Job) + .filter(Job.chat_id == self._chat_id, Job.user_id == self._user_id) + .order_by(Job.created_at.desc()) + .limit(3) + .all() + ) + svc = JobService(db) + lines = [] + for job in jobs: + stats = svc.stats(job.job_id) + unsettled = int(stats.get("pending", 0)) + int(stats.get("failed", 0)) + if unsettled <= 0 and job.status in ("completed", "cancelled"): + continue + lines.append( + f"- 作业「{job.name or job.job_id}」（{job.job_id}，状态 {job.status}）：" + f"总计 {stats.get('total', 0)}，已完成 {stats.get('done', 0)}，" + f"查无 {stats.get('not_found', 0)}，待办 {stats.get('pending', 0)}，" + f"失败 {stats.get('failed', 0)}" + ) + return lines + + async def __call__(self, invocation): + if invocation.operation_name != "agent_reasoning": + return Decision() + runtime = invocation.data["runtime"] + if not self._chat_id: + return + key = ( + runtime.get("reply_id", "") or "", + int(runtime.get("cur_iter", 0) or 0), + ) + if key == self._last_key: + return + # HookBus runs sync handlers inline on the event loop; keep the ledger + # query off it. + lines = await asyncio.to_thread(self._ledger_lines) + if not lines: + return + + self._last_key = key + reminder = ( + "本会话有尚未结算完的批量作业（数字取自台账，是可查证的事实，不是估计）：\n" + + "\n".join(lines) + + "\n\n还有待办或失败项时：用 run_job(action='resume', job_id=...) 续跑" + "（已完成的项不会重做），或先查 run_job(action='status', job_id=...) 看明细。" + "**不得**把未完成当作完成来交付；确实要收尾，也必须如实报出分母、完成数与未覆盖清单。" + ) + return Decision.modify({"context_items": [*invocation.data.get("context_items", ()), { + "content": f"\n{reminder}\n", + "kind": "reminder", "origin": "harness:job_ledger", "trust": "system", + "priority": 850, "token_budget": 4000, + }]}) + + + +def job_ledger_hooks(request): + if request.workflow_mode and not request.isolated and request.chat_id: + yield HookSpec("job_ledger", HookStage.TRANSFORM_CONTEXT, + JobLedgerHook(chat_id=request.chat_id, user_id=request.current_user_id), + priority=500, failure_mode=FailureMode.OPEN, + mutable_fields=frozenset({"context_items"})) diff --git a/src/backend/orchestration/jobs/runner.py b/src/backend/core/jobs/runner.py similarity index 100% rename from src/backend/orchestration/jobs/runner.py rename to src/backend/core/jobs/runner.py diff --git a/src/backend/orchestration/job_runtime.py b/src/backend/core/jobs/runtime.py similarity index 88% rename from src/backend/orchestration/job_runtime.py rename to src/backend/core/jobs/runtime.py index d0334eab..4283425d 100644 --- a/src/backend/orchestration/job_runtime.py +++ b/src/backend/core/jobs/runtime.py @@ -1,19 +1,23 @@ """Public job lifecycle API backed by service-owned managed sandbox processes.""" +from __future__ import annotations import asyncio import logging +from dataclasses import dataclass, field +import core.jobs.owner as owner +import core.jobs.process as process +import core.jobs.state as state +import core.jobs.supervisor as supervisor from core.db.engine import SessionLocal -from core.services.job_service import JobService -from orchestration.jobs import owner, process, state, supervisor +from core.jobs.store import JobService logger = logging.getLogger(__name__) -_launches = {} async def _launch(jid, user_id, *, resume=False): attempt = state.begin_attempt(jid, user_id, resume=resume) - _launches[jid] = asyncio.current_task() + process_state().launches[jid] = asyncio.current_task() try: execution = await asyncio.wait_for(process.launch(state.snapshot(jid)), timeout=90) entry = supervisor.attach(execution) @@ -34,7 +38,7 @@ async def _launch(jid, user_id, *, resume=False): state.finish(jid, attempt, "failed", f"作业启动失败：{type(exc).__name__}: {exc}") raise finally: - _launches.pop(jid, None) + process_state().launches.pop(jid, None) @owner.owned @@ -90,7 +94,7 @@ async def cancel_job(job_row_id, *, user_id): raise RuntimeError("旧作业已撤销回调权限，但无持久进程引用，无法确认停止；请勿自动重跑") state.finish(job_row_id, attempt, "cancelled", "用户取消", require_owner=False) state.take_control(job_row_id, attempt) - launching = _launches.get(job_row_id) + launching = process_state().launches.get(job_row_id) if launching: await asyncio.gather(asyncio.shield(launching), return_exceptions=True) row = state.snapshot(job_row_id) @@ -122,7 +126,7 @@ async def resume_job(job_row_id, *, user_id, chat_id=None): async def reap_orphan_jobs(): count = 0 for jid in state.claim_expired(): - if jid in supervisor._entries: + if jid in supervisor.process_state().entries: continue row = state.snapshot(jid) attempt = row["meta"]["execution"]["attempt_id"] @@ -141,7 +145,7 @@ async def reap_orphan_jobs(): "interrupted", "执行实例失联，已撤销旧回调权限；确认原进程退出后才能 resume：" + str(exc), ) - from orchestration.jobs.notifications import _maybe_wake + from core.jobs.notifications import _maybe_wake await _maybe_wake(jid) count += 1 @@ -164,3 +168,15 @@ async def run_job_reaper_loop(): await asyncio.sleep(15) finally: await supervisor.shutdown() + + +@dataclass +class RuntimeState: + launches: object = field(default_factory=lambda: {}) + + +_STATE = RuntimeState() + + +def process_state() -> RuntimeState: + return _STATE diff --git a/src/backend/orchestration/jobs/sdk.py b/src/backend/core/jobs/sdk.py similarity index 100% rename from src/backend/orchestration/jobs/sdk.py rename to src/backend/core/jobs/sdk.py diff --git a/src/backend/orchestration/jobs/state.py b/src/backend/core/jobs/state.py similarity index 98% rename from src/backend/orchestration/jobs/state.py rename to src/backend/core/jobs/state.py index 852fedce..513025d1 100644 --- a/src/backend/orchestration/jobs/state.py +++ b/src/backend/core/jobs/state.py @@ -5,7 +5,7 @@ from core.db.engine import SessionLocal from core.db.models import Job -from core.services.job_service import JobService, mint_token +from core.jobs.store import JobService, mint_token from sqlalchemy.orm.attributes import flag_modified OWNER = uuid.uuid4().hex diff --git a/src/backend/core/services/job_service.py b/src/backend/core/jobs/store.py similarity index 100% rename from src/backend/core/services/job_service.py rename to src/backend/core/jobs/store.py diff --git a/src/backend/orchestration/jobs/supervisor.py b/src/backend/core/jobs/supervisor.py similarity index 88% rename from src/backend/orchestration/jobs/supervisor.py rename to src/backend/core/jobs/supervisor.py index 4c29c8e4..0913658e 100644 --- a/src/backend/orchestration/jobs/supervisor.py +++ b/src/backend/core/jobs/supervisor.py @@ -1,4 +1,5 @@ """Service-owned execution monitors: handshake, budgets, receipts and cancellation.""" +from __future__ import annotations import asyncio import logging @@ -7,13 +8,11 @@ from datetime import datetime from . import state -from .notifications import _emit_progress, _emit_ui_progress, _maybe_wake, _maybe_wake_progress +from core.jobs.notifications import emit_progress as _emit_progress, _emit_ui_progress, _maybe_wake, _maybe_wake_progress logger = logging.getLogger(__name__) STARTUP_SECONDS = 30.0 PROGRESS_WAKE_SECONDS = 300.0 -_entries = {} -_notifications = set() def _notify(coro): @@ -24,8 +23,8 @@ async def deliver(): logger.warning("Job notification failed: %s", type(exc).__name__) task = asyncio.create_task(deliver()) - _notifications.add(task) - task.add_done_callback(_notifications.discard) + process_state().notifications.add(task) + task.add_done_callback(process_state().notifications.discard) @dataclass @@ -37,17 +36,17 @@ class Entry: def attach(execution): jid = execution.job_id - if jid in _entries: + if jid in process_state().entries: raise RuntimeError("Job already has an execution monitor") entry = Entry(execution) - _entries[jid] = entry + process_state().entries[jid] = entry entry.task = asyncio.create_task(_monitor(entry)) entry.task.add_done_callback(lambda task: task.exception() if not task.cancelled() else None) return entry async def interrupt(jid): - entry = _entries.get(jid) + entry = process_state().entries.get(jid) if entry: if not await entry.execution.stop(): raise RuntimeError("作业执行权已转移，无法确认停止") @@ -56,7 +55,7 @@ async def interrupt(jid): async def wait(jid, timeout=0): - entry = _entries.get(jid) + entry = process_state().entries.get(jid) if entry: try: if timeout > 0: @@ -181,19 +180,32 @@ async def _monitor(entry): state.finish(jid, attempt, "interrupted", "作业监控失联，原执行状态待确认") finally: entry.ready.set() - if _entries.get(jid) is entry: - _entries.pop(jid) + if process_state().entries.get(jid) is entry: + process_state().entries.pop(jid) row = state.snapshot(jid) _notify(_maybe_wake(jid)) return {"status": row["status"], "stats": row["stats"], "error": row["error"]} async def shutdown(): - tasks = [entry.task for entry in list(_entries.values())] + tasks = [entry.task for entry in list(process_state().entries.values())] for task in tasks: task.cancel() await asyncio.gather(*tasks, return_exceptions=True) - notices = list(_notifications) + notices = list(process_state().notifications) for notice in notices: notice.cancel() await asyncio.gather(*notices, return_exceptions=True) + + +@dataclass +class SupervisorState: + entries: object = field(default_factory=lambda: {}) + notifications: object = field(default_factory=lambda: set()) + + +_STATE = SupervisorState() + + +def process_state() -> SupervisorState: + return _STATE diff --git a/src/backend/core/jobs/tool_operations.py b/src/backend/core/jobs/tool_operations.py new file mode 100644 index 00000000..94359cb4 --- /dev/null +++ b/src/backend/core/jobs/tool_operations.py @@ -0,0 +1,387 @@ +"""run_job 工具 —— 主对话智能体提交批量作业的唯一入口。 + +定位：这不是「又一个固定形状的批处理工具」，而是一个**可编程的编排面**。智能体先用 +``write`` 把一段作业脚本写进沙箱，再用 ``run_job`` 提交；脚本里的控制流（循环、条件、 +分层取数、动态收敛、嵌套 map）都是普通 Python，平台不需要知道任务在做什么。 + +与 Bash / read / write / glob / grep 同级原生注册，不走 catalog 开关——「对话内默认 +可用」就落在这里。 +""" + +from __future__ import annotations + +import json as _json +import logging +from typing import Any, Dict, List, Optional + + +from core.sandbox.workspace.resolution import quote_shell_path as _quote_path + +logger = logging.getLogger(__name__) + +# 前台等待（wait=True）的硬上限。超过就把作业转后台，不再阻塞这一轮对话。 +# 该等多久是模型自己判断的，而它对"这批要跑多久"的估计经常偏乐观——实测把一个 265 项、 +# 每项都要过模型的分类作业当成"小作业"，前台一口气阻塞了 5 分钟，用户看不到进度、 +# 插不上话、也没法中途取消。提示词已经写过"几分钟以上就走后台"仍然拦不住，所以在 +# 机制上给前台等待封顶：等够 90 秒还没完，作业照跑，对话先还给用户。 +FOREGROUND_WAIT_CAP_S = 90.0 + + +def mark_wake_on_finish(job_id: str) -> None: + """转后台后要有人叫醒会话播报终态，否则作业跑完没人吭声。""" + try: + from core.db.engine import SessionLocal + from core.jobs.store import JobService + + with SessionLocal() as db: + JobService(db).set_wake_on_finish(job_id, True) + except Exception: # noqa: BLE001 —— 叫醒是尽力而为，不该反过来弄挂工具调用 + logger.warning("[run_job] mark wake_on_finish failed job=%s", job_id, exc_info=True) + + +def detached_payload(job_id: str, *, resumed: bool = False) -> Dict[str, Any]: + """前台等超时 → 按后台语义返回（作业没停，只是不占着这一轮对话了）。""" + mark_wake_on_finish(job_id) + return { + "ok": True, + "job_id": job_id, + "status": "started", + "waited": False, + "detached": True, + "next": ( + f"作业比预期久（前台已等满 {int(FOREGROUND_WAIT_CAP_S)} 秒），" + "**已自动转入后台继续跑，没有中断**。" + "现在就结束这轮回复：告诉用户作业仍在进行、" + "输入框上方的状态条会实时显示进度，期间可以继续聊别的、也可以随时取消。" + "作业跑完会自动叫醒你播报，不要在这里轮询 action='status' 干等，" + "也不要重新提交一遍（它还在跑）。" + ), + } + + +def job_operations( + *, + user_id: str, + chat_id: Optional[str], + sandbox_session_id: Optional[str], + allowed_tools: Optional[List[str]] = None, + model_name: Optional[str] = None, + model_provider_id: Optional[str] = None, + project_ctx: Optional[Dict[str, Any]] = None, + parent_runtime: Optional[Dict[str, Any]] = None, +): + """Build scoped job operations. + + ``parent_runtime`` 保存主智能体的能力配置；每个 agent() 使用独立上下文派生实例。 + ``allowed_tools`` 保留脚本显式选择 MCP 的权限检查。 + """ + + async def run_job( + action: str = "start", + script_path: str = "", + name: str = "", + job_id: str = "", + wait: bool = False, + # 默认必须是**带三方库**的那个解释器。沙箱里裸 python3 是干净的系统解释器， + # 连 openpyxl/pandas 都没有——默认值写成 python3 的直接后果是作业脚本第一行 + # import 就 ModuleNotFoundError 当场失败（实测踩过）。$PY_BIN 指向预装全套数据 + # 依赖的解释器，取不到时才退回 python3。 + interpreter: str = "${PY_BIN:-python3}", + max_calls: int = 0, + max_seconds: int = 0, + concurrency: int = 8, + dest_path: str = "", + status: str = "", + progress_wake_sec: int = 300, + on_conflict: str = "block", + ) -> Dict[str, Any]: + import core.jobs.files as job_files + import core.jobs.runtime as job_runtime + from core.db.engine import SessionLocal + from core.db.models import Job + from core.jobs.store import JobService + + act = (action or "start").strip().lower() + + if act == "start": + if not script_path: + return {"ok": False, "error": "script_path 必填：先用 write 把作业脚本写进沙箱"} + if not sandbox_session_id: + return {"ok": False, "error": "当前会话没有可用沙箱，无法提交作业"} + + # 同一会话不许叠加在跑的作业。被进度唤醒后"改个脚本再交一份"是很自然的动作， + # 但旧作业不会自己消失：两份并存会同时烧预算、同时叫醒会话，用户在状态条上 + # 也分不清哪份才算数。要么先 cancel，要么 resume（已完成的项不会重做）。 + # 本会话已有在跑的作业时**默认**拦一道：默默叠加两份会双倍烧预算、双份叫醒会话， + # 用户在状态条上也分不清哪份算数。但这只是默认，不是禁令——确实要换新版本时 + # 用 on_conflict 明说：'replace' 停掉旧的再跑新的，'parallel' 两份并行。 + # 判断权留给调用方，工具只负责让"叠加"变成一个显式选择而不是意外。 + conflict = (on_conflict or "block").strip().lower() + if chat_id and conflict != "parallel": + # 字段必须在 session 内取出：出了 with 块 ORM 实例就 detached，再读属性会炸 + with SessionLocal() as db: + live = ( + db.query(Job) + .filter( + Job.chat_id == chat_id, + Job.status.in_(("pending", "running")), + ) + .order_by(Job.created_at.desc()) + .first() + ) + live_id = live.job_id if live is not None else "" + live_name = (live.name or "未命名") if live is not None else "" + if live_id and conflict == "replace": + await job_runtime.cancel_job(live_id, user_id=user_id) + logger.info("[run_job] replaced live job %s in chat %s", live_id, chat_id) + elif live_id: + return { + "ok": False, + "error": ( + f"本会话已有在跑的作业 {live_id}（{live_name}）。三条路选一条：" + "① 想用新脚本取代它 → 再调一次本工具并带 on_conflict='replace'" + "（自动停掉旧作业再提交新的）；" + f"② 旧作业只是中断、脚本没问题 → run_job(action='resume', job_id='{live_id}')" + "断点续跑，已完成的项不会重做；" + "③ 确实需要两份同时跑 → on_conflict='parallel'。" + "默认拦下只是为了让叠加成为显式选择，不是不让跑。" + ), + "job_id": live_id, + "hint": "on_conflict=replace|parallel", + } + + # 读脚本正文：既做存在性校验，也作为 job 的审计快照与 resume 依据。 + # ⚠️ 必须走 base64，不能直接 `cat`：沙箱 execute 回传的 stdout 会丢换行， + # 脚本会被压成一行落地 → SyntaxError（实测踩过）。base64 -w0 保字节不变。 + try: + code, out, err = await job_files._sbx_bash( + f"base64 -w0 {_quote_path(script_path)} 2>/dev/null || base64 -i {_quote_path(script_path)}", + session_id=sandbox_session_id, + user_id=user_id, + timeout=30, + ) + except Exception as exc: # noqa: BLE001 + return {"ok": False, "error": f"读取脚本失败: {exc}"} + if code != 0 or not (out or "").strip(): + return {"ok": False, "error": f"脚本不存在或为空: {script_path} {(err or '')[:200]}"} + import base64 as _b64dec + + try: + out = _b64dec.b64decode("".join((out or "").split())).decode("utf-8") + except Exception as exc: # noqa: BLE001 + return {"ok": False, "error": f"脚本解码失败: {exc}"} + if not out.strip(): + return {"ok": False, "error": f"脚本为空: {script_path}"} + + budget: Dict[str, Any] = {"concurrency": concurrency} + if max_calls: + budget["max_calls"] = max_calls + if max_seconds: + budget["max_seconds"] = max_seconds + + try: + jid = await job_runtime.start_job( + user_id=user_id, + chat_id=chat_id, + name=name or "批量作业", + script_path=script_path, + script_text=out, + session_id=sandbox_session_id, + budget=budget, + start_params={ + "allowed_tools": list(allowed_tools or []), + "model_name": model_name, + "model_provider_id": model_provider_id, + "project_ctx": project_ctx, + "parent_runtime": parent_runtime, + # wait=False 时作业跑完没人叫醒本会话 → 打标，让驱动在终态入队一轮续跑 + "wake_on_finish": not wait, + # 中途进度播报间隔（秒）；0 = 只在终态叫一次 + "progress_wake_sec": max(0, int(progress_wake_sec)), + }, + interpreter=interpreter or "${PY_BIN:-python3}", + ) + except Exception as exc: # noqa: BLE001 + logger.warning("[run_job] start failed: %s", exc) + return {"ok": False, "error": f"作业启动失败: {exc}"} + + if not wait: + return { + "ok": True, + "job_id": jid, + "status": "started", + "waited": False, + "next": ( + "作业已在后台开始。**现在就结束这轮回复**：告诉用户作业已提交、" + "输入框上方的状态条会实时显示进度（几分之几、失败数、已运行多久），" + "期间可以继续聊别的、也可以点状态条上的按钮取消。" + "作业跑完和中途都会自动叫醒你播报，不要在这里轮询 action='status' 干等。" + ), + } + + res = await job_runtime.run_and_wait( + jid, chat_id=chat_id, detach_after=FOREGROUND_WAIT_CAP_S + ) + if res.get("status") == "detached": + return detached_payload(jid) + with SessionLocal() as db: + svc = JobService(db) + stats = svc.stats(jid) + job = svc.get(jid) + usage = dict(job.usage or {}) if job else {} + payload = { + "ok": res.get("status") == "completed", + "job_id": jid, + "status": res.get("status"), + "stats": stats, + "usage": usage, + } + if res.get("error"): + payload["error"] = res["error"] + if stats.get("remaining", 0) > 0: + payload["note"] = ( + f"仍有 {stats['remaining']} 项未完成——必须如实告知用户并给出未覆盖清单与" + f"续跑方式（run_job action=resume job_id={jid}），不得当作已完成。" + ) + # 脚本崩溃时把 runner 日志尾巴带回来，省一轮排查 + if res.get("status") == "failed" and sandbox_session_id: + try: + payload["runner_log"] = ( + await job_files.read_runner_log( + jid, user_id=user_id, session_id=sandbox_session_id + ) + )[-1500:] + except Exception as fallback_error: # noqa: BLE001 + from core.infra.failures import record_best_effort_failure + record_best_effort_failure("jobs.tool_operations:runner_log", fallback_error) + return payload + + if not job_id: + return {"ok": False, "error": f"action={act} 需要 job_id"} + + # job_id 容错 + 找不到时把本会话的作业列出来。 + # 实测踩过：模型凭记忆抄 id 时漏了 "job_" 前缀，只回一句「job 不存在」它无从自救， + # 结果决定把 568 次子调用整个重跑一遍。错误信息必须自带下一步。 + with SessionLocal() as db: + svc = JobService(db) + resolved = svc.get(job_id) + if (resolved is None or resolved.user_id != user_id) and not job_id.startswith("job_"): + cand = svc.get(f"job_{job_id}") + if cand is not None and cand.user_id == user_id: + resolved, job_id = cand, f"job_{job_id}" + if resolved is None or resolved.user_id != user_id: + from core.db.models import Job + + rows = ( + db.query(Job) + .filter(Job.user_id == user_id, Job.chat_id == chat_id) + .order_by(Job.created_at.desc()) + .limit(5) + .all() + ) + return { + "ok": False, + "error": f"job 不存在或无权访问: {job_id}", + "jobs_in_this_chat": [ + { + "job_id": r.job_id, + "name": r.name, + "status": r.status, + "stats": svc.stats(r.job_id), + } + for r in rows + ], + "hint": "用上面列表里的完整 job_id 重试（注意 job_ 前缀）；" + "已经跑完的作业**不要重跑**，直接 export 取结果。", + } + + if act == "status": + with SessionLocal() as db: + svc = JobService(db) + job = svc.get(job_id) + if job is None or job.user_id != user_id: + return {"ok": False, "error": "job 不存在或无权访问"} + return { + "ok": True, + "job_id": job_id, + "status": job.status, + "stats": svc.stats(job_id), + "usage": dict(job.usage or {}), + "budget_left": svc.budget_left(job_id), + "error": job.error_message, + } + + if act == "export": + # 台账在 DB，逐项结果**不进对话上下文**——所以导出的方式是写成沙箱里的 + # JSONL 文件，之后用 Bash/python 处理（写 Excel、跑校验、喂评审）。 + if not sandbox_session_id: + return {"ok": False, "error": "当前会话没有可用沙箱"} + with SessionLocal() as db: + svc = JobService(db) + job = svc.get(job_id) + if job is None or job.user_id != user_id: + return {"ok": False, "error": "job 不存在或无权访问"} + wanted = [s.strip() for s in (status or "").split(",") if s.strip()] or [ + "done", + "not_found", + "failed", + "needs_review", + "pending", + ] + rows = [] + for st in wanted: + rows.extend(svc.pending(job_id, status=st)) + stats = svc.stats(job_id) + + dest = dest_path or f"jobs/{job_id}_ledger.jsonl" + body = "\n".join(_json.dumps(r, ensure_ascii=False) for r in rows) + # 分块写 + 读回校验：单条 Bash 携带大 base64 到约 170KB 会**静默失败** + # （exit=0、stderr 空、文件不存在）。568 行台账正好落在这个区间——历史上 + # 因此出现过「导出报成功 rows=568，文件却从来没出现」。成功与否只认沙箱 + # 里读回的真实字节数，绝不用 Python 侧的行数报平安。 + try: + ok_write, detail = await job_files.write_sandbox_file( + dest, body, session_id=sandbox_session_id, user_id=user_id + ) + except Exception as exc: # noqa: BLE001 + return {"ok": False, "error": f"导出失败: {exc}"} + if not ok_write: + return {"ok": False, "error": f"导出失败（落盘未通过校验）: {detail}"} + return { + "ok": True, + "job_id": job_id, + "path": dest, + "rows": len(rows), + "bytes": detail, + "stats": stats, + "hint": "每行一个 JSON：{key, status, payload, result, review, attempts}。" + "用 Bash/python 读这个文件写产物，不要把逐项内容读回对话。", + } + + if act == "cancel": + ok = await job_runtime.cancel_job(job_id, user_id=user_id) + return {"ok": ok, "job_id": job_id, "status": "cancelled" if ok else "unknown"} + + if act == "resume": + res = await job_runtime.resume_job(job_id, user_id=user_id, chat_id=chat_id) + if not res.get("ok"): + return {"ok": False, "error": res.get("error")} + if not wait: + return {"ok": True, "job_id": job_id, "status": "running", "waited": False} + done = await job_runtime.run_and_wait( + job_id, chat_id=chat_id, detach_after=FOREGROUND_WAIT_CAP_S + ) + if done.get("status") == "detached": + return detached_payload(job_id, resumed=True) + with SessionLocal() as db: + stats = JobService(db).stats(job_id) + return { + "ok": done.get("status") == "completed", + "job_id": job_id, + "status": done.get("status"), + "stats": stats, + } + + return {"ok": False, "error": f"未知 action: {act}"} + + return run_job diff --git a/src/backend/core/kb/kb_assets.py b/src/backend/core/kb/kb_assets.py deleted file mode 100644 index 3f186ceb..00000000 --- a/src/backend/core/kb/kb_assets.py +++ /dev/null @@ -1,567 +0,0 @@ -"""Knowledge-base media assets: persistence, 图生文, and their retrieval rows. - -The text pipeline is unchanged — this module is the "路径 A" layer on top of it: - - parse → parents + RawAsset[] (core/kb/kb_parser.py) - → persist_assets() bytes to object storage, rows to kb_assets, - placeholders in the parent text rewritten to - real URLs, each asset linked to its parent chunk - → generate_captions() 图像理解走 core/vision 视觉桥（与对话同一条通路） - → index_assets() 一条 ``row_type="image"`` - row per asset in the existing Milvus collection - -Because the image rows carry ``parent_chunk_id`` of the chunk that contains them, every -downstream behaviour — dedup by parent, parent content fetched from PostgreSQL, RRF, -reranking, per-document deletion — keeps working with no changes. - -Two extension seams are deliberate: - -* **路径 B（真多模态向量）** — a CLIP-family embedding cannot share this collection: the - dimension differs and ``get_or_create_collection`` drops the collection on a dimension - mismatch. It belongs in its own collection, recorded per asset under - ``KBAsset.vector_state["visual"]``, and fused with these results in Python. -* **音视频** — ``KBAsset.kind`` and ``RawAsset.kind`` already widen to audio/video. The - medium-specific work is producing ``text_content`` (ASR transcript) and ``caption``; - everything from :func:`index_assets` onwards is medium-agnostic. -""" - -from __future__ import annotations - -import logging -import os -import uuid -from typing import Iterable, Optional, Sequence - -from core.kb.kb_parser import RawAsset, image_extension_for_mime - -logger = logging.getLogger(__name__) - -# Milvus ``row_type`` for a media asset row (alongside "chunk" / "question"). -ROW_TYPE_IMAGE = "image" - -# Milvus primary keys must not collide with chunk ids; asset rows are namespaced. -_ASSET_ROW_PREFIX = "a_" - -# Images below this many bytes are almost always icons / rules / bullet glyphs that the -# layout engine cut out. Describing them wastes a vision call per item and pollutes -# retrieval with near-empty descriptions. -_MIN_ASSET_BYTES = int(os.getenv("KB_ASSET_MIN_BYTES", "4096")) -# 单文档图像理解张数上限，防一篇图海文档独占视觉桥。体积上限、单次调用超时都归视觉桥 -# 自己管（``VISION_MAX_IMAGE_BYTES`` / ``VISION_CALL_TIMEOUT_SECONDS``），这里不再重复设。 -_MAX_CAPTIONS_PER_DOC = int(os.getenv("KB_ASSET_CAPTION_MAX_PER_DOC", "200")) -# Minimum pixel edge; only enforced when Pillow is importable (it arrives transitively -# via python-pptx, so it is treated as optional). -_MIN_ASSET_EDGE_PX = int(os.getenv("KB_ASSET_MIN_EDGE_PX", "64")) - - - -def multimodal_indexing_enabled(indexing_config: Optional[dict]) -> bool: - """Whether to extract and index media for this document. - - Per-KB opt-out via ``indexing_config.multimodal_indexing``; the global default - lives in ``KB_MULTIMODAL_INDEXING`` so an operator can turn the extra parse cost - off fleet-wide without touching each knowledge base. - """ - if isinstance(indexing_config, dict) and indexing_config.get("multimodal_indexing") is not None: - return bool(indexing_config.get("multimodal_indexing")) - return (os.getenv("KB_MULTIMODAL_INDEXING", "true") or "").strip().lower() not in ( - "false", - "0", - "no", - ) - - -def asset_public_url(asset_id: str) -> str: - """Frontend-resolvable URL for an asset. Matches the route in ``api/routes/v1/kb.py``. - - The ``/api`` prefix mirrors the frontend's ``VITE_API_BASE_URL`` default (nginx - strips it before the backend sees the path). It is written into stored chunk text, - so a deployment that serves the API elsewhere overrides ``KB_ASSET_URL_PREFIX`` - before indexing rather than rewriting rows afterwards. - """ - prefix = (os.getenv("KB_ASSET_URL_PREFIX", "/api") or "").rstrip("/") - return f"{prefix}/v1/catalog/kb/assets/{asset_id}" - - -# ── Persistence ──────────────────────────────────────────────────────────────── - - -def _is_worth_indexing(asset: RawAsset) -> bool: - """Filter out layout noise (rules, bullets, logos) before it costs a vision call.""" - if len(asset.data) < _MIN_ASSET_BYTES: - return False - try: - import io - - from PIL import Image # optional dependency (transitive via python-pptx) - - with Image.open(io.BytesIO(asset.data)) as im: - width, height = im.size - if min(width, height) < _MIN_ASSET_EDGE_PX: - return False - except Exception: - # Pillow missing or the bytes are not a decodable image — the byte-size floor - # above is the fallback filter; never drop an asset because probing failed. - pass - return True - - -def _locate_parent_chunk(parents: Sequence, placeholder: str) -> Optional[object]: - """Find the parent chunk whose text references this asset.""" - for parent in parents: - if placeholder and placeholder in (parent.content or ""): - return parent - return None - - -def _rewrite_placeholder(parents: Sequence, placeholder: str, url: str) -> None: - """Point the markdown image link at the stored asset. - - Before this, ``![](images/.jpg)`` in a chunk was a dead link — the layout - service's temp path, with the bytes discarded. Rewriting in both the parent text - and its children keeps the stored chunk, the embedded text, and what the UI renders - consistent. - """ - needle = f"]({placeholder})" - replacement = f"]({url})" - for parent in parents: - if needle in (parent.content or ""): - parent.content = parent.content.replace(needle, replacement) - for child in getattr(parent, "children", []) or []: - if needle in (child.content or ""): - child.content = child.content.replace(needle, replacement) - - -def persist_assets( - db, - *, - kb_id: str, - document_id: str, - user_id: str, - assets: Sequence[RawAsset], - parents: Sequence, -) -> list: - """Store asset bytes, create ``kb_assets`` rows, and link them to parent chunks. - - Must run **before** the parent chunks are embedded and written: it rewrites the - placeholder links inside ``parents`` in place, and the rewritten text is what gets - stored in PostgreSQL and vectorised. - - Failures are per-asset — a storage error on one figure must not fail the document, - whose text is already parsed and perfectly usable. - """ - from core.db.models import KBAsset - from core.storage import generate_storage_key, get_storage - - if not assets: - return [] - - env = os.getenv("ENVIRONMENT", "dev") - storage = get_storage() - created: list = [] - - for asset in assets: - if not _is_worth_indexing(asset): - continue - - asset_id = uuid.uuid4().hex[:32] - suffix = image_extension_for_mime(asset.mime_type) if asset.kind == "image" else ".bin" - storage_key = generate_storage_key( - env=env, - user_id=user_id, - category="kb_assets", - filename=f"{asset_id}{suffix}", - ) - try: - storage.upload_bytes(asset.data, storage_key) - except Exception as exc: - logger.warning("资产 %s 落盘失败，跳过: %s", asset.placeholder, exc) - continue - - parent = _locate_parent_chunk(parents, asset.placeholder) - row = KBAsset( - asset_id=asset_id, - kb_id=kb_id, - document_id=document_id, - chunk_id=getattr(parent, "parent_id", None), - kind=asset.kind, - mime_type=asset.mime_type, - storage_key=storage_key, - size_bytes=len(asset.data), - asset_index=asset.index, - locator=asset.locator or {}, - text_content=asset.text_content or "", - caption_status="pending", - vector_state={}, - extra_data={"placeholder": asset.placeholder}, - ) - db.add(row) - created.append(row) - _rewrite_placeholder(parents, asset.placeholder, asset_public_url(asset_id)) - - if created: - db.commit() - for row in created: - db.refresh(row) - logger.info("文档 %s 抽出并落盘 %d 个资产", document_id, len(created)) - return created - - -def purge_document_assets(db, document_id: str) -> int: - """Delete a document's assets (storage objects + rows). Used on re-index.""" - from core.db.models import KBAsset - - rows = db.query(KBAsset).filter(KBAsset.document_id == document_id).all() - if not rows: - return 0 - - try: - from core.storage import get_storage - - storage = get_storage() - for row in rows: - try: - storage.delete(row.storage_key) - except Exception as exc: - logger.warning("资产 %s 存储清理失败: %s", row.asset_id, exc) - except Exception as exc: - logger.warning("存储后端不可用，仅清理资产表: %s", exc) - - count = db.query(KBAsset).filter(KBAsset.document_id == document_id).delete() - db.commit() - return count - - -# ── 图生文：走视觉桥 ─────────────────────────────────────────────────────────── - -# 单批提交给视觉桥的图片数。视觉桥的并发信号量是**进程级**的 -# （``VISION_MAX_CONCURRENCY``，默认 4），一篇上百张图的文档如果一次性 gather 过去， -# 会把信号量占满，对话里的看图请求全被排在后面——批量离线任务不该抢交互路径的道。 -# 按小批提交保证 KB 任何时刻最多占用其中几格，其余留给对话。 -_CAPTION_BATCH = max(1, int(os.getenv("KB_ASSET_CAPTION_BATCH", "2"))) -# 落进 ``text_content`` 的转写上限：它要参与向量化，一张满是文字的扫描件能顶出几万字。 -_MAX_TEXT_CONTENT_CHARS = max(200, int(os.getenv("KB_ASSET_TEXT_MAX_CHARS", "4000"))) - -# 给视觉桥的 focus：同一套证据契约，但把注意力挪到"文档插图"这个场景上。 -_CAPTION_FOCUS = ( - "这是文档中的插图，识别结果会被向量化用于知识库检索。请重点覆盖：" - "图的类型（照片/流程图/架构图/折线图/柱状图/表格截图等）、" - "图中出现的全部文字与数字（坐标轴、图例、标签、单位）、" - "以及图所反映的关键结论或趋势。读不出来的一律写进 uncertainty，不要猜。" -) - - -def vision_available() -> bool: - """当前是否有可用的图像理解模型（视觉桥）。""" - try: - from core.vision import is_available - - return is_available() - except Exception as exc: # noqa: BLE001 — 视觉桥不可用不该让索引失败 - logger.debug("视觉桥可用性判定失败，按不可用处理: %s", exc) - return False - - -def _describe_batch(images: list, loop=None) -> list: - """把一批图交给视觉桥识别。返回与输入等长的结果列表（失败位为 None）。 - - 视觉桥是异步的，而索引是同步函数、跑在没有运行中事件循环的工作线程里。按可靠性 - 从高到低三级取宿主循环——只有跑在**宿主循环**上，视觉桥的证据缓存与并发信号量才 - 真的生效： - - 1. ``loop``：常驻索引 worker 显式传进来的调度循环。这是当前的生产路径——索引跑在 - worker 自己的 ``ThreadPoolExecutor`` 里（见 ``core/kb/index_worker.py``），那不是 - anyio 的工作线程，第 2 级在这里拿不到东西。 - 2. ``anyio.from_thread``：仍经 ``BackgroundTasks`` 进来的调用方（如我的空间同步产物 - 入库）跑在 Starlette 的 anyio 线程池里，天然带着回宿主循环的通路。 - 3. 临时循环 + **关掉证据缓存**：CLI / 脚本 / 测试。``core/infra/redis.py`` 的连接池是 - 模块级全局、绑定首次使用它的循环，让一个转瞬即逝的临时循环抢先创建它，会把整个 - 进程的 Redis 绑死在已销毁的循环上——那比"这次没吃到缓存"严重得多。 - """ - import asyncio - import functools - - from core.vision import get_vision_bridge - - bridge = get_vision_bridge() - - if loop is not None and not loop.is_closed(): - future = asyncio.run_coroutine_threadsafe( - bridge.describe_many(images, focus=_CAPTION_FOCUS), loop - ) - return future.result() - - try: - import anyio.from_thread - - return anyio.from_thread.run( - functools.partial(bridge.describe_many, images, focus=_CAPTION_FOCUS) - ) - except RuntimeError as exc: - logger.debug("视觉桥回宿主循环不可用，改用临时循环且不走缓存: %s", exc) - - async def _uncached() -> list: - tasks = [ - bridge.describe(data, mime, focus=_CAPTION_FOCUS, use_cache=False) - for data, mime in images - ] - return list(await asyncio.gather(*tasks)) - - return asyncio.run(_uncached()) - - -def _merge_text_content(existing: str, ocr_text: str) -> str: - """图注 + OCR 转写合成检索用文本，去重后截断。 - - 两者都留：图注是文档作者写的、往往点明了图的用途；OCR 是图里实际的字。 - """ - parts: list[str] = [] - for part in ((existing or "").strip(), (ocr_text or "").strip()): - if part and part not in parts: - parts.append(part) - merged = "\n".join(parts) - return merged[:_MAX_TEXT_CONTENT_CHARS] - - -def _evidence_digest(result) -> dict: - """留档进 ``extra_data`` 的证据摘要。 - - 刻意不存 ``layout.regions`` 与 ``ocr.lines``：整段文字已经进了 ``text_content``， - 再存一份逐行结构，一篇几百张图的文档会把 JSONB 撑得很难看。留下的是之后真会用到 - 的部分——实体/关系（可喂本体与 Wiki 层）、不确定项（人工复核时看）、模型元数据 - （换模型后判断哪些资产该重跑）。 - """ - evidence = result.evidence - return { - "summary": evidence.summary, - "scene": evidence.semantics.scene, - "intent": evidence.semantics.intent, - "entities": [e.model_dump(exclude_none=True) for e in evidence.semantics.entities], - "relations": [r.model_dump(exclude_none=True) for r in evidence.semantics.relations], - "uncertainty": list(evidence.uncertainty), - "meta": result.to_meta(), - } - - -def generate_captions(db, assets: Sequence, *, loop=None) -> int: - """给资产补上图像理解结果：``caption`` 写概述，``text_content`` 并入 OCR 转写。 - - 走的是 ``core/vision`` 视觉桥——和对话里"纯文本主模型读图"完全同一条通路、同一个 - ``vision`` 模型角色，因此知识库这边白得三样东西：按图片内容哈希的证据缓存（重新 - 索引同一篇文档不用再付一次钱）、三协议与结构化输出降级、以及"该角色只能指派多模态 - 模型"的硬约束。 - - 没有可用的视觉模型是正常状态，不是错误：资产保留文档自带的图注并照常入索引，之后 - 配好 ``vision`` 角色再回填即可，**不用重新解析文档**。 - """ - if not assets: - return 0 - - if not vision_available(): - logger.info("视觉桥不可用（未指派 vision 角色且主模型不支持读图），跳过图生文；图注仍会入索引") - for row in assets: - row.caption_status = "skipped" - db.commit() - return 0 - - from core.storage import get_storage - - storage = get_storage() - images = [a for a in assets if a.kind == "image"] - budget, overflow = images[:_MAX_CAPTIONS_PER_DOC], images[_MAX_CAPTIONS_PER_DOC:] - # 没轮到的显式标注，让"上限生效"在数据里看得见，而不是像视觉模型什么都没返回。 - for row in overflow + [a for a in assets if a.kind != "image"]: - row.caption_status = "skipped" - if overflow: - logger.warning( - "文档图片数 %d 超过单文档图生文上限 %d，超出部分仅用图注索引", - len(images), - _MAX_CAPTIONS_PER_DOC, - ) - - done = 0 - for offset in range(0, len(budget), _CAPTION_BATCH): - chunk = budget[offset : offset + _CAPTION_BATCH] - payloads: list = [] - pending: list = [] - for row in chunk: - try: - data = storage.download_bytes(row.storage_key) - except Exception as exc: # noqa: BLE001 — 单张读不出来不该拖垮整篇 - logger.warning("资产 %s 读取失败，跳过图生文: %s", row.asset_id, exc) - row.caption_status = "failed" - continue - payloads.append((data, row.mime_type)) - pending.append(row) - - if not payloads: - continue - - try: - results = _describe_batch(payloads, loop) - except Exception as exc: # noqa: BLE001 — 整批失败也只降级，正文索引已经好了 - logger.warning("视觉桥调用失败，该批 %d 张仅用图注索引: %s", len(pending), exc) - for row in pending: - row.caption_status = "failed" - continue - - for row, result in zip(pending, results): - if result is None: - row.caption_status = "failed" - continue - evidence = result.evidence - summary = (evidence.summary or "").strip() - row.caption = summary - row.text_content = _merge_text_content(row.text_content, evidence.ocr.full_text) - meta = dict(row.extra_data or {}) - meta["vision"] = _evidence_digest(result) - row.extra_data = meta - row.caption_status = "completed" if summary else "failed" - if summary: - done += 1 - - # 这里给 ``extra_data`` 赋的是**新字典**（不是原地改键），SQLAlchemy 的属性事件能 - # 直接捕获，不需要 flag_modified —— 那个只有原地改 JSON 才必须。 - db.commit() - logger.info("图生文完成 %d/%d", done, len(budget)) - return done - - -# ── Vector rows ──────────────────────────────────────────────────────────────── - - -def asset_index_text(asset, *, title: str = "") -> str: - """The text an asset is retrieved by: 视觉桥概述 + 图注/OCR 转写 + 文档标题。 - - Audio/video reuse this unchanged — ``text_content`` becomes the ASR transcript. - """ - parts = [ - (asset.caption or "").strip(), - (asset.text_content or "").strip(), - (title or "").strip(), - ] - return "\n".join(p for p in parts if p) - - -def index_assets( - assets: Sequence, *, user_id: str, kb_id: str, document_id: str, title: str -) -> int: - """Write one ``row_type="image"`` row per asset into the existing collection. - - ``parent_chunk_id`` points at the chunk that contains the asset, so a hit here - resolves to that chunk's full text through the existing parent-fetch path. Assets - with no owning chunk (a standalone upload whose placeholder was lost) point at - themselves and fall back to the caption as their content. - """ - if not assets: - return 0 - - from core.kb.kb_vector import ( - TAGS_FIELD_MAX_BYTES, - TITLE_FIELD_MAX_BYTES, - embed_batch, - text_to_sparse, - truncate_utf8, - upsert_rows, - ) - - indexable = [(a, asset_index_text(a, title=title)) for a in assets] - indexable = [(a, t) for a, t in indexable if t.strip()] - if not indexable: - return 0 - - try: - vectors = embed_batch([t for _, t in indexable]) - except Exception as exc: - logger.error("资产向量化失败，文档 %s 的图片不可检索: %s", document_id, exc) - return 0 - - rows = [] - for (asset, text), vector in zip(indexable, vectors): - rows.append( - { - "chunk_id": f"{_ASSET_ROW_PREFIX}{asset.asset_id}", - "parent_chunk_id": asset.chunk_id or f"{_ASSET_ROW_PREFIX}{asset.asset_id}", - "row_type": ROW_TYPE_IMAGE, - "user_id": user_id, - "kb_id": kb_id, - "document_id": document_id, - "title": truncate_utf8(title, TITLE_FIELD_MAX_BYTES), - "content": truncate_utf8(text), - "tags_text": truncate_utf8(asset.text_content or "", TAGS_FIELD_MAX_BYTES), - "chunk_index": asset.asset_index or 0, - "dense_embedding": vector, - "sparse_embedding": text_to_sparse(text), - } - ) - - BATCH = 100 - for i in range(0, len(rows), BATCH): - upsert_rows(rows[i : i + BATCH]) - logger.info("文档 %s 写入 %d 条资产检索行", document_id, len(rows)) - return len(rows) - - -def mark_vector_state(db, assets: Sequence, space: str, state: str) -> None: - """Record that these assets are indexed into a vector space. - - ``space`` is ``"text"`` for the caption/OCR rows written by :func:`index_assets`. - 路径 B 落地时用 ``"visual"``，与文本行互不覆盖。 - """ - for row in assets: - current = dict(row.vector_state or {}) - current[space] = state - # 同上：赋新字典而不是原地改键，属性事件自然捕获得到。 - row.vector_state = current - db.commit() - - -# ── Retrieval helper ─────────────────────────────────────────────────────────── - - -def fetch_assets_for_chunks(chunk_ids: Iterable[str], *, max_per_chunk: int = 4) -> dict: - """Map parent-chunk id → its assets, for attaching to retrieval results. - - Also keyed by the synthetic ``a_`` parent used when an asset has no - owning chunk, so a hit on such a row still resolves to its own image. - """ - ids = [c for c in dict.fromkeys(chunk_ids) if c] - if not ids: - return {} - - direct = [c for c in ids if not c.startswith(_ASSET_ROW_PREFIX)] - orphan_asset_ids = [c[len(_ASSET_ROW_PREFIX) :] for c in ids if c.startswith(_ASSET_ROW_PREFIX)] - - try: - from core.db.engine import SessionLocal - from core.db.models import KBAsset - from sqlalchemy import or_ - - with SessionLocal() as db: - clauses = [] - if direct: - clauses.append(KBAsset.chunk_id.in_(direct)) - if orphan_asset_ids: - clauses.append(KBAsset.asset_id.in_(orphan_asset_ids)) - if not clauses: - return {} - rows = db.query(KBAsset).filter(or_(*clauses)).order_by(KBAsset.asset_index).all() - grouped: dict[str, list[dict]] = {} - for row in rows: - key = row.chunk_id or f"{_ASSET_ROW_PREFIX}{row.asset_id}" - bucket = grouped.setdefault(key, []) - if len(bucket) >= max_per_chunk: - continue - bucket.append( - { - "asset_id": row.asset_id, - "kind": row.kind, - "url": asset_public_url(row.asset_id), - "caption": row.caption or row.text_content or "", - "locator": row.locator or {}, - } - ) - return grouped - except Exception as exc: - logger.warning("检索结果附图失败（不影响文本结果）: %s", exc) - return {} diff --git a/src/backend/core/kb/kb_parser.py b/src/backend/core/kb/kb_parser.py deleted file mode 100644 index 60525109..00000000 --- a/src/backend/core/kb/kb_parser.py +++ /dev/null @@ -1,933 +0,0 @@ -"""Document parser and parent-child chunker for private knowledge base. - -Supported formats: PDF, DOCX, TXT, MD, XLSX, 图片 -Output: list of ParentChunk, each containing child chunks for vector indexing, -plus the non-text assets (图片；后续音视频) cut out of the document. -""" - -from __future__ import annotations - -import json -import logging -import math -import re -import uuid -from dataclasses import dataclass, field -from typing import Callable, Optional - -logger = logging.getLogger(__name__) - -# ── Data structures ──────────────────────────────────────────────────────────── - -@dataclass -class ChildChunk: - child_id: str - content: str - index: int # position within parent - - -@dataclass -class ParentChunk: - parent_id: str - content: str # full parent text, returned to LLM on retrieval - children: list[ChildChunk] - char_start: Optional[int] = None - char_end: Optional[int] = None - - -@dataclass -class RawAsset: - """A non-text medium extracted from the source document, before persistence. - - ``placeholder`` is the exact token this medium is referenced by inside the - paragraph text (``images/.jpg`` for a layout-extracted figure, a synthetic - ``asset://`` for a standalone upload). It is what links the medium back to - the parent chunk that contains it, and what gets rewritten to a real URL once the - asset has an id. - - Medium-agnostic on purpose: ``kind`` widens to audio/video, and ``locator`` holds - ``{page_idx, bbox}`` for an image where a transcript segment would hold - ``{start_ms, end_ms}``. - """ - - data: bytes - placeholder: str - kind: str = "image" - mime_type: str = "application/octet-stream" - text_content: str = "" # 图注 / OCR（音视频接入后：ASR 转写） - locator: dict = field(default_factory=dict) - index: int = 0 - - -# ── Token counting ───────────────────────────────────────────────────────────── - -_tiktoken_enc = None - - -def _get_tiktoken_enc(): - """Return cached tiktoken cl100k_base encoder.""" - global _tiktoken_enc - if _tiktoken_enc is None: - try: - import tiktoken - _tiktoken_enc = tiktoken.get_encoding("cl100k_base") - except Exception: - pass - return _tiktoken_enc - - -def _count_tokens(text: str) -> int: - """Approximate token count using tiktoken cl100k_base.""" - enc = _get_tiktoken_enc() - if enc is not None: - return len(enc.encode(text)) - # Fallback: rough Chinese/English estimate - return max(1, len(text) // 2) - - -# ── MIME → filename suffix mapping ───────────────────────────────────────────── - -_MIME_TO_SUFFIX: dict[str, str] = { - "application/pdf": ".pdf", - "application/vnd.openxmlformats-officedocument.wordprocessingml.document": ".docx", - "application/msword": ".doc", - "text/plain": ".txt", - "text/markdown": ".txt", - "text/x-markdown": ".txt", - "text/csv": ".csv", - "application/json": ".txt", -} - - -# ── Text extraction ──────────────────────────────────────────────────────────── - -def extract_text(file_bytes: bytes, mime_type: str) -> list[dict]: - """Extract structured paragraphs from a document. - - Delegates to core/file_parser.py for PDF, DOCX, DOC, TXT (uses the - external file-parser API for PDF, pandoc for DOCX, etc.). - Falls back to built-in xlsx/plain-text extraction for unsupported types. - - Returns a list of paragraph dicts: - {"text": str, "heading_level": int|None, "element_type": str} - """ - paragraphs, _ = extract_text_with_assets(file_bytes, mime_type, with_assets=False) - return paragraphs - - -def extract_text_with_assets( - file_bytes: bytes, - mime_type: str, - *, - with_assets: bool = True, -) -> tuple[list[dict], list[RawAsset]]: - """Extract paragraphs and, when ``with_assets``, the media the document carries. - - Asset extraction is opt-in because it costs a heavier parse call: the layout - service inlines every figure as base64 in the JSON response, which the text-only - callers have no use for. When the rich call fails for any reason we fall back to - the plain text path — a parse service that doesn't support the media params must - degrade to today's behaviour, never fail the document. - """ - mt = (mime_type or "").lower() - - # Standalone image upload. Without this branch an image falls through to - # ``_extract_plain_text``, which latin-1 decodes the binary into garbage - # paragraphs and indexes them — the upload allowlist accepts png/jpg/webp/gif - # (``core/content/file_validation.py``) so this path is reachable in production. - if mt.startswith("image/"): - return _extract_standalone_image(file_bytes, mt, with_assets=with_assets) - - # XLSX — keep built-in handler (file_parser doesn't support spreadsheets) - if mt in ( - "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", - "application/vnd.ms-excel", - ): - return _extract_xlsx(file_bytes), [] - - if with_assets and mt == "application/pdf": - rich = _extract_pdf_rich(file_bytes) - if rich is not None: - return rich - - # Try core/file_parser for all other known types - suffix = _MIME_TO_SUFFIX.get(mt) - if suffix: - try: - from core.content.file_parser import parse_file - filename = f"upload{suffix}" - content = parse_file(file_bytes, filename) - if content: - return _markdown_to_paragraphs(content), [] - except RuntimeError: - # file_parser failed (e.g. service down) — fall through to built-in - pass - - # Fallback: treat as plain text - return _extract_plain_text(file_bytes), [] - - -# Media placeholders the layout service emits: ``![alt](images/.jpg)``. -_MD_IMAGE_RE = re.compile(r"!\[[^\]]*\]\(([^)\s]+)\)") - -_IMAGE_MIME_TO_EXT = { - "image/png": ".png", - "image/jpeg": ".jpg", - "image/webp": ".webp", - "image/gif": ".gif", -} - - -def _extract_standalone_image( - file_bytes: bytes, - mime_type: str, - *, - with_assets: bool, -) -> tuple[list[dict], list[RawAsset]]: - """An uploaded image is one asset with one synthetic paragraph holding its ref. - - The paragraph exists so the normal chunker still produces a parent chunk: retrieval - fetches parent content from PostgreSQL, and an image-only document with no chunk - would be unreachable. The caption/description is attached to the asset, not - inlined here — it is not known until the VLM pass has run. - """ - placeholder = f"asset://{uuid.uuid4().hex[:16]}" - paragraphs = [{"text": f"![]({placeholder})", "heading_level": None, "element_type": "image"}] - if not with_assets: - return paragraphs, [] - asset = RawAsset( - data=file_bytes, - placeholder=placeholder, - kind="image", - mime_type=mime_type, - index=0, - ) - return paragraphs, [asset] - - -def _extract_pdf_rich(file_bytes: bytes) -> Optional[tuple[list[dict], list[RawAsset]]]: - """Parse a PDF keeping its figures. Returns None to signal "fall back to text-only".""" - try: - from core.content.file_parser import parse_pdf_rich - - result = parse_pdf_rich(file_bytes, "upload.pdf") - except Exception as exc: - logger.warning("PDF 富解析失败，退回纯文本解析: %s", exc) - return None - - paragraphs = _markdown_to_paragraphs(result.markdown) - if not result.images: - return paragraphs, [] - - # Layout metadata (caption / bbox / page) keyed by the same path the markdown - # references, so each figure's asset carries where it came from. - meta_by_path: dict[str, dict] = {} - for block in result.blocks: - if block.get("type") != "image": - continue - path = str(block.get("img_path") or "") - if path: - meta_by_path[path] = block - - assets: list[RawAsset] = [] - seen: set[str] = set() - # Iterate in document order: the markdown's image links are the source of truth for - # ordering, and only images actually referenced by the text get an asset. - for idx, path in enumerate(_MD_IMAGE_RE.findall(result.markdown)): - if path in seen: - continue - data = result.images.get(path) - if not data: - continue - seen.add(path) - meta = meta_by_path.get(path, {}) - caption_parts = [ - str(x).strip() - for x in (meta.get("image_caption") or []) + (meta.get("image_footnote") or []) - if str(x).strip() - ] - locator = {} - if meta.get("page_idx") is not None: - locator["page_idx"] = meta.get("page_idx") - if meta.get("bbox"): - locator["bbox"] = meta.get("bbox") - assets.append( - RawAsset( - data=data, - placeholder=path, - kind="image", - mime_type=_guess_image_mime(path), - text_content=" ".join(caption_parts), - locator=locator, - index=idx, - ) - ) - - return paragraphs, assets - - -def _guess_image_mime(path: str) -> str: - lowered = path.lower() - for mime, ext in _IMAGE_MIME_TO_EXT.items(): - if lowered.endswith(ext): - return mime - if lowered.endswith(".jpeg"): - return "image/jpeg" - return "image/jpeg" - - -def image_extension_for_mime(mime_type: str) -> str: - """Filename suffix for an image mime, used when building the storage key.""" - return _IMAGE_MIME_TO_EXT.get((mime_type or "").lower(), ".jpg") - - -def _markdown_to_paragraphs(text: str) -> list[dict]: - """Parse markdown text (from file_parser) into structured paragraph dicts.""" - paragraphs: list[dict] = [] - table_buffer: list[str] = [] - - def _flush_table(): - if table_buffer: - paragraphs.append({ - "text": "\n".join(table_buffer), - "heading_level": None, - "element_type": "table", - }) - table_buffer.clear() - - for line in text.splitlines(): - stripped = line.strip() - - # Empty line — flush table buffer, skip - if not stripped: - _flush_table() - continue - - # Markdown table row - if stripped.startswith("|") and stripped.endswith("|"): - table_buffer.append(stripped) - continue - - # Not a table row — flush any pending table - _flush_table() - - # Markdown heading - md_match = re.match(r'^(#{1,6})\s+(.*)', stripped) - if md_match: - level = len(md_match.group(1)) - paragraphs.append({ - "text": stripped, - "heading_level": level, - "element_type": "heading", - }) - continue - - # Chinese legal/report heading patterns - heading_level = _detect_heading_level(stripped) - if heading_level: - paragraphs.append({ - "text": stripped, - "heading_level": heading_level, - "element_type": "heading", - }) - continue - - # Regular paragraph - paragraphs.append({ - "text": stripped, - "heading_level": None, - "element_type": "paragraph", - }) - - _flush_table() - return paragraphs - - -def _extract_xlsx(file_bytes: bytes) -> list[dict]: - import io - try: - import openpyxl - except ImportError: - raise RuntimeError("openpyxl is required: pip install openpyxl") - - wb = openpyxl.load_workbook(io.BytesIO(file_bytes), read_only=True, data_only=True) - paragraphs = [] - for sheet in wb.worksheets: - rows = list(sheet.iter_rows(values_only=True)) - if not rows: - continue - header = [str(c or "") for c in rows[0]] - paragraphs.append({ - "text": f"## Sheet: {sheet.title}", - "heading_level": 2, - "element_type": "heading", - }) - md_lines = ["| " + " | ".join(header) + " |"] - md_lines.append("|" + "|".join("---" for _ in header) + "|") - for row in rows[1:]: - if all(c is None for c in row): - continue - md_lines.append("| " + " | ".join(str(c or "") for c in row) + " |") - paragraphs.append({ - "text": "\n".join(md_lines), - "heading_level": None, - "element_type": "table", - }) - return paragraphs - - -def _extract_plain_text(file_bytes: bytes) -> list[dict]: - """Fallback: decode as text and split into paragraphs.""" - # Try common encodings - for enc in ("utf-8", "gbk", "latin-1"): - try: - text = file_bytes.decode(enc) - break - except UnicodeDecodeError: - continue - else: - text = file_bytes.decode("utf-8", errors="replace") - - return _markdown_to_paragraphs(text) - - -def _detect_heading_level(line: str) -> Optional[int]: - """Detect Chinese legal/report heading patterns.""" - patterns = [ - (r'^第[一二三四五六七八九十百]+章', 1), - (r'^第[一二三四五六七八九十百]+节', 2), - (r'^第[一二三四五六七八九十百\d]+条', 3), - (r'^[一二三四五六七八九十]+[、．.]', 2), - (r'^\d+[、．.]\s*\S', 3), - ] - for pattern, level in patterns: - if re.match(pattern, line): - return level - return None - - -# ── Chunking ─────────────────────────────────────────────────────────────────── - -def build_parent_child_chunks( - paragraphs: list[dict], - parent_size: int = 1024, - child_size: int = 128, - overlap: int = 20, - chunk_method: str = "semantic", - embed_fn: Optional[Callable[[list[str]], list[list[float]]]] = None, - separators: Optional[list[str]] = None, - child_separators: Optional[list[str]] = None, -) -> list[ParentChunk]: - """Build parent-child chunk pairs from extracted paragraphs. - - Parent chunks: large semantic units (~parent_size tokens), stored in PostgreSQL, - returned to LLM as full context. - Child chunks: small slices (~child_size tokens), vectorised in Milvus for retrieval. - - ``separators`` (parent-chunk separators): custom separator hierarchy that decides - parent-chunk boundaries; only effective on paths that go through recursive - splitting (the recursive method itself, and the recursive fallback of semantic - chunking for oversized parents); when empty, falls back to the default hierarchy. - ``child_separators`` (child-chunk separators): when a parent is split into - children, split on these separators (then pack by child_size); when empty, - children use the default fixed-length sliding-window split. The two are - independent and do not affect each other. - """ - seps = normalize_separators(separators) - csep = normalize_separators(child_separators) - if chunk_method == "qa": - parents = _build_qa_chunks(paragraphs) - elif chunk_method == "laws": - parents = _build_law_chunks(paragraphs, parent_size, child_size, overlap) - elif chunk_method == "recursive": - parents = _build_recursive_chunks(paragraphs, parent_size, child_size, overlap, separators=seps) - elif chunk_method == "embedding_semantic": - parents = _build_embedding_semantic_chunks( - paragraphs, parent_size, child_size, overlap, embed_fn=embed_fn, separators=seps, - ) - else: - # "semantic" / "structured" / default - parents = _build_structured_chunks(paragraphs, parent_size, child_size, overlap) - - # Child-chunk separators: re-split children uniformly after parents are built - # (overriding each builder's default fixed-length children), so we don't have to - # thread the same parameter into every builder — one post-processing step covers - # all chunking methods. - if csep: - _apply_child_separators(parents, child_size, csep) - return parents - - -def _apply_child_separators(parents: list[ParentChunk], child_size: int, separators: list[str]) -> None: - """Re-split each parent's children by child_separators (replaces ``pc.children`` in place). - - Splits the parent body with the recursive separators and packs by child_size, - aligning child boundaries to the user-specified separators. - """ - for pc in parents: - texts = [s.strip() for s in _recursive_split(pc.content, separators, child_size) if s.strip()] - pc.children = [ - ChildChunk(child_id=f"{pc.parent_id}_{i}", content=c, index=i) - for i, c in enumerate(texts or [pc.content]) - ] - - -def _new_id() -> str: - return uuid.uuid4().hex[:16] - - -def _split_into_children(parent_text: str, child_size: int, overlap: int) -> list[str]: - """Split parent text into overlapping child chunks by token count.""" - words = parent_text.split() - if not words: - return [parent_text] - - # Use character-based approximation when tiktoken unavailable - chars_per_token = 2 # conservative for Chinese - child_chars = child_size * chars_per_token - overlap_chars = overlap * chars_per_token - - children = [] - start = 0 - while start < len(parent_text): - end = start + child_chars - chunk = parent_text[start:end].strip() - if chunk: - children.append(chunk) - if end >= len(parent_text): - break - start = end - overlap_chars - return children if children else [parent_text] - - -def _make_parent_chunk(text: str, children_texts: list[str]) -> ParentChunk: - pid = _new_id() - children = [ - ChildChunk(child_id=f"{pid}_{i}", content=c, index=i) - for i, c in enumerate(children_texts) - ] - return ParentChunk(parent_id=pid, content=text, children=children) - - -# In structure-aware chunking, a heading is a "preferred break point" that only takes -# effect once the buffer has accumulated to this fraction of parent_size. -# 0.5 was chosen: subsections keep merging into the parent chunk until about half a -# parent_size, then we break before the next heading — preserving structure awareness -# while letting parent_size truly determine parent-chunk size (avoids heading-dense -# documents being shredded). -_HEADING_SOFT_RATIO = 0.5 - - -def _build_structured_chunks( - paragraphs: list[dict], - parent_size: int, - child_size: int, - overlap: int, -) -> list[ParentChunk]: - """Structure-aware chunking. - - A heading is a "preferred break point", not a "forced break point": we only break - before the next heading once the buffer has accumulated to a certain fraction of - parent_size (``_HEADING_SOFT_RATIO``); otherwise consecutive subsections are - merged into the same parent chunk until it approaches parent_size. This keeps - the heading/paragraph-aligned structure awareness while making ``parent_size`` - actually effective — heading-dense documents are no longer cut into lots of tiny - pieces, and parents can yield enough children. - """ - result = [] - buffer: list[str] = [] - buffer_tokens = 0 - soft_threshold = parent_size * _HEADING_SOFT_RATIO - - def _flush() -> None: - nonlocal buffer, buffer_tokens - parent_text = "\n".join(buffer) - result.append(_make_parent_chunk(parent_text, _split_into_children(parent_text, child_size, overlap))) - buffer = [] - buffer_tokens = 0 - - for para in paragraphs: - text = para["text"] - tokens = _count_tokens(text) - - # Hard cap: adding this paragraph would exceed parent_size, so break off the - # accumulated content first - if buffer and buffer_tokens + tokens > parent_size: - _flush() - # Soft boundary: break before a heading only when enough content has - # accumulated (>= a fraction of parent_size); otherwise merge this - # subsection into the current parent so parent_size is the real determinant - # of parent-chunk size - elif buffer and para.get("heading_level") and buffer_tokens >= soft_threshold: - _flush() - - buffer.append(text) - buffer_tokens += tokens - - if buffer: - _flush() - - return result - - -def _build_law_chunks( - paragraphs: list[dict], - parent_size: int, - child_size: int, - overlap: int, -) -> list[ParentChunk]: - """Split at article/chapter/section boundaries for legal documents.""" - result = [] - buffer: list[str] = [] - - article_pattern = re.compile(r'^第[一二三四五六七八九十百\d]+[条章节]') - - for para in paragraphs: - text = para["text"] - is_boundary = bool(article_pattern.match(text)) or para.get("heading_level") in (1, 2) - - if is_boundary and buffer: - parent_text = "\n".join(buffer) - result.append(_make_parent_chunk(parent_text, _split_into_children(parent_text, child_size, overlap))) - buffer = [] - - buffer.append(text) - - if buffer: - parent_text = "\n".join(buffer) - result.append(_make_parent_chunk(parent_text, _split_into_children(parent_text, child_size, overlap))) - - return result - - -def _build_qa_chunks(paragraphs: list[dict]) -> list[ParentChunk]: - """Each Q&A pair becomes one parent chunk with one child.""" - result = [] - for para in paragraphs: - text = para["text"] - pid = _new_id() - child = ChildChunk(child_id=f"{pid}_0", content=text, index=0) - result.append(ParentChunk(parent_id=pid, content=text, children=[child])) - return result - - -# ── Recursive chunking ───────────────────────────────────────────────────────── - -_RECURSIVE_SEPARATORS = ["\n\n", "\n", "。", ".", "；", ";", "，", ",", " "] - -# Users may write escape sequences (e.g. \n, \t) in custom separators; restore them -# to real characters here in one place -_SEPARATOR_ESCAPES = {"\\n": "\n", "\\t": "\t", "\\r": "\r", "\\\\": "\\"} - - -def normalize_separators(seps: Optional[list[str]]) -> Optional[list[str]]: - """Normalize user-supplied custom separators into a usable hierarchy list. - - - Restore escapes like ``\\n`` / ``\\t``; - - Drop empty strings, de-duplicate while preserving order; - - Return None when all empty (callers fall back to the ``_RECURSIVE_SEPARATORS`` default hierarchy). - """ - if not seps: - return None - out: list[str] = [] - seen: set[str] = set() - for raw in seps: - if raw is None: - continue - s = str(raw) - for token, real in _SEPARATOR_ESCAPES.items(): - s = s.replace(token, real) - if s == "" or s in seen: - continue - seen.add(s) - out.append(s) - return out or None - - -def parse_separators_form(raw: Optional[str]) -> Optional[list[str]]: - """Parse custom separators passed as a JSON-array string in a multipart form into a list. - - The preview endpoint takes Form params and cannot receive a list directly, so by - convention a JSON string is passed (e.g. ``["\\n\\n", "。"]``). On parse failure - or non-list input, silently return None (fall back to the default hierarchy) - instead of 500-ing over tuning input. - """ - if not raw or not raw.strip(): - return None - try: - data = json.loads(raw) - except (ValueError, TypeError): - return None - if not isinstance(data, list): - return None - return normalize_separators([str(x) for x in data]) - - -def _recursive_split(text: str, separators: list[str], max_tokens: int) -> list[str]: - """Recursively split text using a hierarchy of separators. - - Tries to split on the first separator that produces segments. - Merges adjacent small segments up to max_tokens. - Over-sized segments recurse with the next separator level. - Final fallback: character-based truncation. - """ - if _count_tokens(text) <= max_tokens: - return [text] - - for i, sep in enumerate(separators): - parts = text.split(sep) - if len(parts) <= 1: - continue - - # Merge adjacent small parts up to max_tokens - result: list[str] = [] - current = "" - current_tokens = 0 - - for part in parts: - candidate = (current + sep + part) if current else part - candidate_tokens = _count_tokens(candidate) - - if candidate_tokens <= max_tokens: - current = candidate - current_tokens = candidate_tokens - else: - if current: - result.append(current) - # If this single part exceeds max_tokens, recurse deeper - part_tokens = _count_tokens(part) - if part_tokens > max_tokens: - deeper = _recursive_split(part, separators[i + 1:], max_tokens) - result.extend(deeper) - current = "" - current_tokens = 0 - else: - current = part - current_tokens = part_tokens - - if current: - result.append(current) - - if result: - return result - - # Final fallback: character-based truncation - chars_per_token = 2 - chunk_chars = max_tokens * chars_per_token - result = [] - start = 0 - while start < len(text): - end = start + chunk_chars - chunk = text[start:end].strip() - if chunk: - result.append(chunk) - start = end - return result if result else [text] - - -def _build_recursive_chunks( - paragraphs: list[dict], - parent_size: int, - child_size: int, - overlap: int, - separators: Optional[list[str]] = None, -) -> list[ParentChunk]: - """Recursive multi-level separator chunking.""" - # Join all paragraph text - full_text = "\n".join(p["text"] for p in paragraphs) - segments = _recursive_split(full_text, separators or _RECURSIVE_SEPARATORS, parent_size) - - result = [] - for seg in segments: - seg = seg.strip() - if not seg: - continue - result.append(_make_parent_chunk(seg, _split_into_children(seg, child_size, overlap))) - return result - - -# ── Embedding-based semantic chunking ────────────────────────────────────────── - -def _cosine_similarity(a: list[float], b: list[float]) -> float: - """Compute cosine similarity between two vectors (pure Python).""" - dot = sum(x * y for x, y in zip(a, b)) - norm_a = math.sqrt(sum(x * x for x in a)) - norm_b = math.sqrt(sum(x * x for x in b)) - if norm_a == 0 or norm_b == 0: - return 0.0 - return dot / (norm_a * norm_b) - - -def _build_embedding_semantic_chunks( - paragraphs: list[dict], - parent_size: int, - child_size: int, - overlap: int, - embed_fn: Optional[Callable[[list[str]], list[list[float]]]] = None, - separators: Optional[list[str]] = None, -) -> list[ParentChunk]: - """Semantic chunking based on embedding similarity between adjacent blocks. - - Algorithm: - 1. Merge paragraphs into minimum ~50-token blocks - 2. Batch-embed all blocks - 3. Compute cosine similarity between adjacent blocks - 4. Set breakpoints where similarity drops below P25 or is a local minimum - 5. Merge blocks between breakpoints into parent chunks - 6. Fallback to recursive splitting for oversized parents - """ - if embed_fn is None: - # Fallback to structured chunking when no embed function available - return _build_structured_chunks(paragraphs, parent_size, child_size, overlap) - - # Step 1: Merge paragraphs into minimum-size blocks (~50 tokens) - MIN_BLOCK_TOKENS = 50 - blocks: list[str] = [] - current_block = "" - current_tokens = 0 - - for para in paragraphs: - text = para["text"] - tokens = _count_tokens(text) - if current_block and current_tokens + tokens > MIN_BLOCK_TOKENS: - blocks.append(current_block) - current_block = text - current_tokens = tokens - else: - current_block = (current_block + "\n" + text) if current_block else text - current_tokens += tokens - - if current_block: - blocks.append(current_block) - - if len(blocks) <= 1: - # Single block — just use structured - return _build_structured_chunks(paragraphs, parent_size, child_size, overlap) - - # Step 2: Batch embed - embeddings = embed_fn(blocks) - - # Step 3: Compute adjacent cosine similarities - similarities: list[float] = [] - for i in range(len(embeddings) - 1): - sim = _cosine_similarity(embeddings[i], embeddings[i + 1]) - similarities.append(sim) - - if not similarities: - return _build_structured_chunks(paragraphs, parent_size, child_size, overlap) - - # Step 4: Find breakpoints - sorted_sims = sorted(similarities) - p25_idx = max(0, len(sorted_sims) // 4 - 1) - p25_threshold = sorted_sims[p25_idx] - - breakpoints: set[int] = set() - for i, sim in enumerate(similarities): - # Below P25 threshold - if sim <= p25_threshold: - breakpoints.add(i) - continue - # Local minimum: lower than both neighbors - is_local_min = True - if i > 0 and sim >= similarities[i - 1]: - is_local_min = False - if i < len(similarities) - 1 and sim >= similarities[i + 1]: - is_local_min = False - if is_local_min and sim < p25_threshold * 1.2: - breakpoints.add(i) - - # Step 5: Merge blocks between breakpoints into parent chunks - result: list[ParentChunk] = [] - segment_start = 0 - - sorted_breaks = sorted(breakpoints) - # Add an artificial breakpoint at the end - sorted_breaks.append(len(blocks) - 1) - - for bp in sorted_breaks: - segment_blocks = blocks[segment_start:bp + 1] - parent_text = "\n".join(segment_blocks).strip() - segment_start = bp + 1 - - if not parent_text: - continue - - # Step 6: If parent is too large, use recursive split as fallback - if _count_tokens(parent_text) > parent_size * 1.5: - sub_segments = _recursive_split(parent_text, separators or _RECURSIVE_SEPARATORS, parent_size) - for sub in sub_segments: - sub = sub.strip() - if sub: - result.append(_make_parent_chunk(sub, _split_into_children(sub, child_size, overlap))) - else: - result.append(_make_parent_chunk(parent_text, _split_into_children(parent_text, child_size, overlap))) - - # Handle remaining blocks - if segment_start < len(blocks): - remaining = "\n".join(blocks[segment_start:]).strip() - if remaining: - result.append(_make_parent_chunk(remaining, _split_into_children(remaining, child_size, overlap))) - - return result - - -# ── Pipeline entry point ─────────────────────────────────────────────────────── - -def parse_and_chunk( - file_bytes: bytes, - mime_type: str, - chunk_method: str = "semantic", - parent_size: int = 1024, - child_size: int = 128, - overlap: int = 20, - embed_fn: Optional[Callable[[list[str]], list[list[float]]]] = None, - separators: Optional[list[str]] = None, - child_separators: Optional[list[str]] = None, -) -> list[ParentChunk]: - """Full pipeline: extract text then build parent-child chunks.""" - parents, _ = parse_and_chunk_rich( - file_bytes, - mime_type, - chunk_method=chunk_method, - parent_size=parent_size, - child_size=child_size, - overlap=overlap, - embed_fn=embed_fn, - separators=separators, - child_separators=child_separators, - with_assets=False, - ) - return parents - - -def parse_and_chunk_rich( - file_bytes: bytes, - mime_type: str, - chunk_method: str = "semantic", - parent_size: int = 1024, - child_size: int = 128, - overlap: int = 20, - embed_fn: Optional[Callable[[list[str]], list[list[float]]]] = None, - separators: Optional[list[str]] = None, - child_separators: Optional[list[str]] = None, - with_assets: bool = True, -) -> tuple[list[ParentChunk], list[RawAsset]]: - """Same pipeline as :func:`parse_and_chunk`, also returning the document's media. - - The assets are returned unpersisted; the caller links them to parent chunks and - stores the bytes (see ``core/kb/kb_assets.py``). - """ - paragraphs, assets = extract_text_with_assets( - file_bytes, mime_type, with_assets=with_assets - ) - parents = build_parent_child_chunks( - paragraphs, - parent_size=parent_size, - child_size=child_size, - overlap=overlap, - chunk_method=chunk_method, - embed_fn=embed_fn, - separators=separators, - child_separators=child_separators, - ) - return parents, assets diff --git a/src/backend/core/kb/kb_vector.py b/src/backend/core/kb/kb_vector.py deleted file mode 100644 index 0582a462..00000000 --- a/src/backend/core/kb/kb_vector.py +++ /dev/null @@ -1,755 +0,0 @@ -"""Milvus vector store for private knowledge base. - -Collection: hugagent_kb_private -- Stores both chunk rows (row_type='chunk') and question rows (row_type='question') -- Child chunk vectors used for retrieval; parent chunk content fetched from PostgreSQL -- User isolation enforced via user_id field in every query expression -""" - -from __future__ import annotations - -import json -import logging -import os -from typing import Any, Optional - -import requests - -logger = logging.getLogger(__name__) - -COLLECTION_NAME = "hugagent_kb_private" - -# Milvus VARCHAR field capacity (**bytes**, not characters) —— the schema definition and the write truncation share the same -# set of constants, to avoid changing max_length but forgetting the truncation and hitting "length exceeds max length" again. -# The content column is only a display fallback for retrieval results (dense/sparse vectors are computed from the full text, -# and returned content is fetched full from PostgreSQL), so safe byte-wise truncation does not affect retrieval quality. -TITLE_FIELD_MAX_BYTES = 500 -CONTENT_FIELD_MAX_BYTES = 4096 -TAGS_FIELD_MAX_BYTES = 1000 - - -def truncate_utf8(text: str, max_bytes: int = CONTENT_FIELD_MAX_BYTES) -> str: - """Safe truncation by UTF-8 bytes (Chinese is 3 bytes/char); never cuts in the middle of a multi-byte character. - - Historical bug: using ``text[:4096]`` truncated by "character"; 4096 Chinese chars ≈ 12KB bytes, still exceeding Milvus's - 4096 **byte** limit → ``length exceeds max length`` error. - """ - if not text: - return "" - encoded = text.encode("utf-8") - if len(encoded) <= max_bytes: - return text - return encoded[:max_bytes].decode("utf-8", "ignore") - - -# Fallback dimension: used only when the real dimension can't be probed from the online embedding endpoint (not configured/unreachable). -# In normal operation the dimension is auto-determined by detect_embed_dim() from the actual output of the chosen embedding model, no manual filling needed. -_EMBED_DIMS = int(os.getenv("MEM0_EMBED_DIMS", "1024")) - -# Cache of probed dimensions: key=(embed_url, embed_model) → dim, to avoid hitting /embeddings on every index. -_DETECTED_DIM_CACHE: dict[tuple[str, str], int] = {} -# Collection dimension already confirmed in this process (avoids a describe on every get_or_create). -_VERIFIED_COLLECTION_DIM: Optional[int] = None - -# Sparse vector dimension space (hash modulo) -_SPARSE_DIM_SPACE = 100_000 - - -def _resolve_embed_config() -> tuple[str, str, str]: - """Resolve embedding config from DB, with env fallback.""" - try: - from core.services.model_config import ModelConfigService - - cfg = ModelConfigService.get_instance().resolve("embedding") - if cfg: - return cfg.base_url.rstrip("/"), cfg.model_name, cfg.api_key - except Exception: - pass - return ( - os.getenv("MEM0_EMBED_URL", "").rstrip("/"), - os.getenv("MEM0_EMBED_MODEL", ""), - os.getenv("MEM0_EMBED_API_KEY", ""), - ) - - -def _resolve_reranker_config() -> tuple[str, str, str]: - """Resolve reranker config from DB, with env fallback.""" - try: - from core.services.model_config import ModelConfigService - - cfg = ModelConfigService.get_instance().resolve("reranker") - if cfg: - return cfg.base_url.rstrip("/"), cfg.model_name, cfg.api_key - except Exception: - pass - return ( - os.getenv("RERANKER_URL", "").rstrip("/"), - os.getenv("RERANKER_MODEL", ""), - os.getenv("RERANKER_API_KEY", ""), - ) - - -# ── Embedding ────────────────────────────────────────────────────────────────── - -# Max number of texts packed into a single embedding request. A large document is split into hundreds or thousands of chunks; -# sending the whole batch at once to a slow embedding model (e.g. 8B, ~0.45s/item serially) will inevitably hit the single-request -# read timeout; splitting into sub-batches at this granularity keeps each request's latency under the timeout line. -# Overridable via KB_EMBED_BATCH_SIZE. -_EMBED_BATCH_SIZE = max(1, int(os.getenv("KB_EMBED_BATCH_SIZE", "32"))) -# Read timeout (seconds) for a single embedding request. Raise it to tolerate a slow model's per-batch latency; the real timeout -# prevention is sub-batching, the two work together —— a sub-batch always has an upper bound, and a larger timeout is only a fallback. -# Overridable via KB_EMBED_TIMEOUT. -_EMBED_TIMEOUT = max(1, int(os.getenv("KB_EMBED_TIMEOUT", "120"))) - - -def _embed_request( - url: str, - headers: dict, - inputs: list[str], - embed_model: str, - *, - timeout: float | None = None, -) -> list[list[float]]: - """Post a single embedding request and return vectors ordered by input index.""" - payload = {"input": inputs, "model": embed_model} - resp = requests.post( - url, - headers=headers, - json=payload, - timeout=timeout or _EMBED_TIMEOUT, - ) - resp.raise_for_status() - data = resp.json() - return [item["embedding"] for item in sorted(data["data"], key=lambda x: x["index"])] - - -def embed_text(text: str, *, timeout: float | None = None) -> list[float]: - """Call the configured embedding service and return a dense vector.""" - embed_url, embed_model, api_key = _resolve_embed_config() - - if not embed_url: - raise RuntimeError("Embedding model is not configured") - - url = f"{embed_url}/embeddings" - from core.services.desktop_model_credentials import prepare_request_headers - headers = prepare_request_headers(api_key, embed_url) - vector = _embed_request(url, headers, [text], embed_model, timeout=timeout)[0] - if vector: - _DETECTED_DIM_CACHE[(embed_url, embed_model)] = len(vector) - return vector - - -def embed_batch(texts: list[str]) -> list[list[float]]: - """Batch embed multiple texts, splitting into sub-batches with one-by-one fallback. - - A large document is split into hundreds or thousands of chunks. Sending the whole batch at once to a slow embedding - model (e.g. 8B, ~0.45s/item serially) would time out the single request and fail the whole document's vectorization. So we - split into sub-batches of _EMBED_BATCH_SIZE and request them one sub-batch at a time; if a sub-batch fails, we downgrade to - embedding one item at a time within that sub-batch, preserving as many other chunks as possible so one timeout doesn't drag - down the whole document. Only if the one-by-one attempt still fails do we re-raise (treated as a true failure). - """ - embed_url, embed_model, api_key = _resolve_embed_config() - - if not embed_url: - raise RuntimeError("Embedding model is not configured") - - url = f"{embed_url}/embeddings" - from core.services.desktop_model_credentials import prepare_request_headers - headers = prepare_request_headers(api_key, embed_url) - - results: list[list[float]] = [] - for start in range(0, len(texts), _EMBED_BATCH_SIZE): - sub = texts[start : start + _EMBED_BATCH_SIZE] - try: - results.extend(_embed_request(url, headers, sub, embed_model)) - except Exception as exc: - logger.warning( - "Sub-batch embed failed for [%d:%d] (%s); falling back to one-by-one", - start, - start + len(sub), - exc, - ) - for one in sub: - results.extend(_embed_request(url, headers, [one], embed_model)) - return results - - -def detect_embed_dim() -> int: - """Probe the real output dimension of the currently chosen embedding model (one /embeddings call, take the vector length). - - The result is cached by (embed_url, embed_model) to avoid repeated probing on every index; on probe failure (endpoint not - configured/unreachable) it falls back to MEM0_EMBED_DIMS. This way the table dimension auto-follows after switching the - embedding model, no manual filling needed. - """ - embed_url, embed_model, _ = _resolve_embed_config() - key = (embed_url, embed_model) - cached = _DETECTED_DIM_CACHE.get(key) - if cached: - return cached - try: - dim = len(embed_text("dimension probe")) - if dim > 0: - _DETECTED_DIM_CACHE[key] = dim - logger.info("Auto-detected embedding dim=%d (model=%s)", dim, embed_model or "?") - return dim - except Exception as exc: - logger.warning( - "Embedding dim auto-detect failed (%s); falling back to MEM0_EMBED_DIMS=%d", - exc, - _EMBED_DIMS, - ) - return _EMBED_DIMS - - -# ── Reranker ────────────────────────────────────────────────────────────────── - - -def is_reranker_configured() -> bool: - """Check if a reranker model endpoint is configured.""" - reranker_url, reranker_model, _ = _resolve_reranker_config() - return bool(reranker_url and reranker_model) - - -def rerank( - query: str, - documents: list[str], - top_n: int | None = None, - *, - timeout: float = 30, -) -> list[dict]: - """Call the configured reranker endpoint (OpenAI-compatible /rerank). - - Returns a list of {"index": int, "relevance_score": float} sorted by score descending. - """ - reranker_url, reranker_model, reranker_key = _resolve_reranker_config() - - if not reranker_url or not reranker_model: - raise RuntimeError("Reranker is not configured") - - url = f"{reranker_url}/rerank" - from core.services.desktop_model_credentials import prepare_request_headers - headers = prepare_request_headers(reranker_key, reranker_url) - payload: dict = { - "model": reranker_model, - "query": query, - "documents": documents, - } - if top_n is not None: - payload["top_n"] = top_n - - resp = requests.post(url, headers=headers, json=payload, timeout=timeout) - resp.raise_for_status() - data = resp.json() - - # Response format: {"results": [{"index": 0, "relevance_score": 0.95}, ...]} - results = data.get("results", []) - return sorted(results, key=lambda x: x.get("relevance_score", 0), reverse=True) - - -# ── Milvus helpers ───────────────────────────────────────────────────────────── - - -def _milvus_uri() -> str: - from core.config.settings import settings - - return os.getenv("MILVUS_URL") or settings.memory.milvus_url - - -def _is_lite() -> bool: - """True when the vector backend is embedded Milvus Lite (a local file uri). - - Milvus Lite (no-Docker local profile) does **not** support SPARSE_FLOAT_VECTOR - / SPARSE_INVERTED_INDEX / hybrid search / IVF / scalar INVERTED indexes, so the - KB degrades to **dense-only** on it. A server uri (``http(s)://`` / ``tcp://``) - is a full Milvus and keeps the dense+sparse hybrid path byte-for-byte unchanged. - """ - return not _milvus_uri().startswith(("http://", "https://", "tcp://", "unix:")) - - -def _get_client(): - """Return a MilvusClient connected to the configured Milvus instance.""" - from pymilvus import MilvusClient - - url = _milvus_uri() - token = os.getenv("MILVUS_TOKEN", "") - if token: - return MilvusClient(uri=url, token=token) - return MilvusClient(uri=url) - - -def _upsert(client, data: list[dict[str, Any]]) -> None: - """Upsert helper that drops the ``sparse_embedding`` field on Milvus Lite. - - On Lite the collection has no sparse field (see ``get_or_create_collection``), - and dynamic fields are disabled, so an extra ``sparse_embedding`` key would be - rejected. Stripping it here keeps every caller's row-building code identical - across both backends. - """ - if _is_lite(): - data = [{k: v for k, v in row.items() if k != "sparse_embedding"} for row in data] - client.upsert(collection_name=COLLECTION_NAME, data=data) - - -def _collection_dense_dim(client, *, timeout: float | None = None) -> Optional[int]: - """Read the dense_embedding dimension of the existing collection. - - Connection and schema-read errors deliberately propagate to the caller. Treating - an unavailable schema as a dimension mismatch used to trigger a destructive - collection rebuild. - """ - desc = client.describe_collection(COLLECTION_NAME, timeout=timeout) - for f in desc.get("fields", []): - if f.get("name") == "dense_embedding": - dim = f.get("params", {}).get("dim") - return int(dim) if dim is not None else None - return None - - -def get_or_create_collection(*, timeout: float | None = None) -> None: - """Idempotently create hugagent_kb_private (hybrid retrieval schema) at the current embedding model's real dimension. - - New collections use the current embedding model's detected dimension. Existing - collections are never dropped implicitly: an unreadable schema or dimension - mismatch preserves all vectors and aborts the operation until an operator can - verify the embedding configuration and explicitly re-index the collection. - """ - global _VERIFIED_COLLECTION_DIM - from pymilvus import DataType - - client = _get_client() - target_dim = detect_embed_dim() - - if client.has_collection(COLLECTION_NAME, timeout=timeout): - if _VERIFIED_COLLECTION_DIM == target_dim: - return - try: - existing_dim = _collection_dense_dim(client, timeout=timeout) - except Exception as exc: - message = ( - "无法确认现有知识库集合的向量维度，已保留集合并中止本次操作；" - "请检查 Milvus 连接后重试。" - ) - logger.error("%s 原因：%s", message, exc, exc_info=True) - raise RuntimeError(message) from exc - if existing_dim is None: - message = ( - "无法确认现有知识库集合的向量维度，已保留集合并中止本次操作；" - "dense_embedding 字段或 dim 参数缺失。" - ) - logger.error(message) - raise RuntimeError(message) - if existing_dim == target_dim: - _VERIFIED_COLLECTION_DIM = target_dim - return - message = ( - f"KB collection 维度不匹配（已存在={existing_dim}，当前模型={target_dim}），" - "已保留集合并中止本次操作；请核实 embedding 配置并通过显式重建流程重新索引。" - ) - logger.error(message) - raise RuntimeError(message) - - lite = _is_lite() - - schema = client.create_schema(auto_id=False, enable_dynamic_field=False) - schema.add_field("chunk_id", DataType.VARCHAR, max_length=64, is_primary=True) - schema.add_field("parent_chunk_id", DataType.VARCHAR, max_length=64) - schema.add_field("row_type", DataType.VARCHAR, max_length=16) # "chunk" | "question" - schema.add_field("user_id", DataType.VARCHAR, max_length=64) - schema.add_field("kb_id", DataType.VARCHAR, max_length=64) - schema.add_field("document_id", DataType.VARCHAR, max_length=64) - schema.add_field("title", DataType.VARCHAR, max_length=TITLE_FIELD_MAX_BYTES) - schema.add_field("content", DataType.VARCHAR, max_length=CONTENT_FIELD_MAX_BYTES) - schema.add_field( - "tags_text", DataType.VARCHAR, max_length=TAGS_FIELD_MAX_BYTES - ) # BM25 augmentation - schema.add_field("chunk_index", DataType.INT64) - schema.add_field("dense_embedding", DataType.FLOAT_VECTOR, dim=target_dim) - if not lite: - schema.add_field("sparse_embedding", DataType.SPARSE_FLOAT_VECTOR) - - index_params = client.prepare_index_params() - if lite: - # Milvus Lite: only a dense AUTOINDEX (FLAT under the hood). No sparse, no - # scalar INVERTED indexes — filtering falls back to brute force, fine at - # single-machine scale. - index_params.add_index( - field_name="dense_embedding", - index_type="AUTOINDEX", - metric_type="IP", - ) - else: - index_params.add_index( - field_name="dense_embedding", - index_type="IVF_FLAT", - metric_type="IP", - params={"nlist": 128}, - ) - index_params.add_index( - field_name="sparse_embedding", - index_type="SPARSE_INVERTED_INDEX", - metric_type="IP", - ) - # Scalar indexes for filtering acceleration - for field in ("user_id", "kb_id", "document_id", "row_type"): - index_params.add_index(field_name=field, index_type="INVERTED") - - client.create_collection( - collection_name=COLLECTION_NAME, - schema=schema, - index_params=index_params, - timeout=timeout, - ) - _VERIFIED_COLLECTION_DIM = target_dim - logger.info( - "Created Milvus collection: %s (dim=%d, mode=%s)", - COLLECTION_NAME, - target_dim, - "lite/dense-only" if lite else "hybrid", - ) - - -# ── Write ────────────────────────────────────────────────────────────────────── - - -def upsert_rows(rows: list[dict[str, Any]]) -> None: - """Upsert a batch of rows into hugagent_kb_private. - - Each row must include all schema fields. dense_embedding and sparse_embedding - must be pre-computed by the caller (use embed_text / build_sparse_text). - """ - if not rows: - return - get_or_create_collection() - client = _get_client() - _upsert(client, rows) - logger.debug("Upserted %d rows into %s", len(rows), COLLECTION_NAME) - - -def delete_by_document(document_id: str, user_id: str) -> None: - """Delete all Milvus rows (chunk + question) for a document.""" - get_or_create_collection() - client = _get_client() - expr = f'document_id == "{document_id}" and user_id == "{user_id}"' - client.delete(collection_name=COLLECTION_NAME, filter=expr) - logger.info("Deleted Milvus rows for document_id=%s user_id=%s", document_id, user_id) - - -def delete_by_kb(kb_id: str, user_id: str) -> None: - """Delete all Milvus rows for an entire KB space.""" - get_or_create_collection() - client = _get_client() - expr = f'kb_id == "{kb_id}" and user_id == "{user_id}"' - client.delete(collection_name=COLLECTION_NAME, filter=expr) - logger.info("Deleted Milvus rows for kb_id=%s user_id=%s", kb_id, user_id) - - -def get_children_for_parent(parent_chunk_id: str) -> list[dict]: - """Return the child-chunk rows of a given parent chunk in Milvus (for the "view chunks" parent→child display). - - Child rows have ``chunk_id`` shaped like ``{pid}_{i}`` with ``parent_chunk_id == pid``; in flat mode a parent has only one - representative row (``chunk_id == parent_chunk_id``), which is filtered out —— i.e. flat mode returns an empty list. - Returns an empty list when Milvus is unavailable / the query fails, without affecting the parent-chunk list display. - """ - try: - get_or_create_collection() - client = _get_client() - rows = client.query( - collection_name=COLLECTION_NAME, - # In flat mode a parent has only one representative row (chunk_id == parent_chunk_id); exclude it right in the query - filter=( - f'parent_chunk_id == "{parent_chunk_id}" and row_type == "chunk" ' - f'and chunk_id != "{parent_chunk_id}"' - ), - output_fields=["chunk_id", "parent_chunk_id", "chunk_index", "content"], - limit=4096, - ) - except Exception as exc: - logger.warning("get_children_for_parent(%s) failed: %s", parent_chunk_id, exc) - return [] - children = [ - { - "chunk_id": r.get("chunk_id"), - "chunk_index": r.get("chunk_index", 0), - "content": r.get("content", ""), - } - for r in rows - ] - children.sort(key=lambda x: x["chunk_index"]) - return children - - -def delete_by_chunk(parent_chunk_id: str) -> None: - """Delete all Milvus rows (chunk + question) for a single parent chunk.""" - get_or_create_collection() - client = _get_client() - expr = f'parent_chunk_id == "{parent_chunk_id}"' - client.delete(collection_name=COLLECTION_NAME, filter=expr) - logger.info("Deleted Milvus rows for parent_chunk_id=%s", parent_chunk_id) - - -# ── Search ───────────────────────────────────────────────────────────────────── - - -def _hit_to_dict(ent, score: float) -> dict: - """Build the shared KB search-result row. ``ent`` is any entity accessor with - ``.get`` (a dict from Milvus Lite, or a Hit entity from hybrid search).""" - return { - "chunk_id": ent.get("chunk_id"), - "parent_chunk_id": ent.get("parent_chunk_id") or ent.get("chunk_id"), - "row_type": ent.get("row_type"), - "kb_id": ent.get("kb_id"), - "document_id": ent.get("document_id"), - "title": ent.get("title"), - "content": ent.get("content"), - "chunk_index": ent.get("chunk_index"), - "score": score, - } - - -def hybrid_search( - user_id: str, - kb_ids: list[str], - query: str, - query_vec: list[float], - top_k: int = 10, - *, - public_kb_ids: list[str] | None = None, - timeout: float | None = None, -) -> list[dict[str, Any]]: - """Hybrid search over dense + sparse vectors with RRF fusion. - - Searches both chunk rows and question rows simultaneously. - Results are sorted by fused score; dedup by parent_chunk_id is done by the caller. - - ``kb_ids`` are private spaces and are filtered by ``user_id`` (owner isolation). - ``public_kb_ids`` are admin-managed public spaces visible to everyone — they are - filtered by ``kb_id`` only (no owner check), since public KBs are global and - ``kb_id`` is globally unique. - """ - from pymilvus import AnnSearchRequest, MilvusClient, RRFRanker - - get_or_create_collection(timeout=timeout) - client = _get_client() - - clauses: list[str] = [] - if kb_ids: - clauses.append(f'(user_id == "{user_id}" and kb_id in {json.dumps(kb_ids)})') - if public_kb_ids: - clauses.append(f"kb_id in {json.dumps(public_kb_ids)}") - if not clauses: - return [] - expr = " or ".join(clauses) - - output_fields = [ - "chunk_id", - "parent_chunk_id", - "row_type", - "kb_id", - "document_id", - "title", - "content", - "chunk_index", - ] - - # Milvus Lite (no-Docker local): no sparse field / hybrid search — fall back to - # a plain dense ANN search over the same collection. Same result shape. - if _is_lite(): - lite_res = client.search( - collection_name=COLLECTION_NAME, - data=[query_vec], - anns_field="dense_embedding", - search_params={"metric_type": "IP"}, - limit=top_k * 2, - filter=expr, - output_fields=output_fields, - timeout=timeout, - ) - return [ - _hit_to_dict(hit.get("entity", hit), hit.get("distance", 0.0)) for hit in lite_res[0] - ] - - # Dense vector search request - dense_req = AnnSearchRequest( - data=[query_vec], - anns_field="dense_embedding", - param={"metric_type": "IP", "params": {"nprobe": 10}}, - limit=top_k * 3, - expr=expr, - ) - - # Sparse search request — bag-of-words vector - sparse_vec = text_to_sparse(query) - sparse_req = AnnSearchRequest( - data=[sparse_vec], - anns_field="sparse_embedding", - param={"metric_type": "IP"}, - limit=top_k * 3, - expr=expr, - ) - - results = client.hybrid_search( - collection_name=COLLECTION_NAME, - reqs=[dense_req, sparse_req], - ranker=RRFRanker(k=60), - limit=top_k * 2, - output_fields=output_fields, - timeout=timeout, - ) - - return [_hit_to_dict(hit.entity, hit.score) for hit in results[0]] - - -# ── Tag/question re-index ────────────────────────────────────────────────────── - - -def build_sparse_text(content: str, tags: list[str]) -> str: - """Build combined text for sparse vectorisation.""" - if not tags: - return content - tag_str = " ".join(f"[{t}]" for t in tags) - return f"{content} {tag_str}" - - -def text_to_sparse(text: str) -> dict[int, float]: - """Convert text to a bag-of-words sparse vector using term-frequency + hashing. - - Compatible with Milvus SPARSE_FLOAT_VECTOR field (v2.4+). - Each unique token is hashed to a dimension index; value = TF. - """ - import hashlib - import re - from collections import Counter - - # Simple tokenisation: split on non-alphanumeric (works for CJK + Latin) - tokens = re.findall(r"[\w\u4e00-\u9fff]+", text.lower()) - if not tokens: - return {0: 1.0} - - counts = Counter(tokens) - total = sum(counts.values()) - - sparse: dict[int, float] = {} - for token, cnt in counts.items(): - dim = int(hashlib.md5(token.encode()).hexdigest()[:8], 16) % _SPARSE_DIM_SPACE - tf = cnt / total - sparse[dim] = sparse.get(dim, 0.0) + tf - - return sparse if sparse else {0: 1.0} - - -def reindex_chunk_tags(chunk_id: str, content: str, tags: list[str]) -> None: - """Re-compute and upsert sparse_embedding for a chunk row after tag changes.""" - # Fetch the existing row to preserve all other fields - get_or_create_collection() - client = _get_client() - rows = client.query( - collection_name=COLLECTION_NAME, - filter=f'chunk_id == "{chunk_id}"', - output_fields=[ - "chunk_id", - "parent_chunk_id", - "row_type", - "user_id", - "kb_id", - "document_id", - "title", - "chunk_index", - "dense_embedding", - ], - ) - if not rows: - logger.warning("reindex_chunk_tags: chunk_id %s not found in Milvus", chunk_id) - return - - row = rows[0] - sparse_text = build_sparse_text(content, tags) - row["content"] = truncate_utf8(content) - row["tags_text"] = truncate_utf8(" ".join(tags), TAGS_FIELD_MAX_BYTES) - row["sparse_embedding"] = text_to_sparse(sparse_text) - _upsert(client, [row]) - - -def reindex_chunk_content( - chunk_id: str, - content: str, - tags: list[str], - user_id: str, - kb_id: str, - document_id: str, - title: str, - chunk_index: int, -) -> None: - """Re-embed a parent chunk after a manual content edit. - - Removes the parent's existing chunk rows (child or flat) and upserts a single - fresh chunk row with recomputed dense + sparse embeddings. Question rows are - managed separately by :func:`upsert_question_rows`. Retrieval returns the parent - content from PostgreSQL, so this single representative row is sufficient for the - edited chunk to be matched and surfaced. - """ - get_or_create_collection() - client = _get_client() - # Drop existing chunk rows for this parent (keep questions; they are re-upserted separately) - client.delete( - collection_name=COLLECTION_NAME, - filter=f'parent_chunk_id == "{chunk_id}" and row_type == "chunk"', - ) - sparse_text = build_sparse_text(content, tags or []) - row = { - "chunk_id": chunk_id, - "parent_chunk_id": chunk_id, - "row_type": "chunk", - "user_id": user_id, - "kb_id": kb_id, - "document_id": document_id, - "title": truncate_utf8(title, TITLE_FIELD_MAX_BYTES), - "content": truncate_utf8(content), - "tags_text": truncate_utf8(" ".join(tags or []), TAGS_FIELD_MAX_BYTES), - "chunk_index": chunk_index, - "dense_embedding": embed_text(content), - "sparse_embedding": text_to_sparse(sparse_text), - } - _upsert(client, [row]) - - -def upsert_question_rows( - parent_chunk_id: str, - questions: list[str], - user_id: str, - kb_id: str, - document_id: str, - title: str, - chunk_index: int, -) -> None: - """Delete existing question rows for a chunk and insert fresh ones.""" - get_or_create_collection() - client = _get_client() - - # Delete old question rows - expr = f'parent_chunk_id == "{parent_chunk_id}" and row_type == "question"' - client.delete(collection_name=COLLECTION_NAME, filter=expr) - - if not questions: - return - - # Embed all questions in one batch call - vecs = embed_batch(questions) - rows = [] - for i, (q, vec) in enumerate(zip(questions, vecs)): - rows.append( - { - "chunk_id": f"q_{parent_chunk_id}_{i}", - "parent_chunk_id": parent_chunk_id, - "row_type": "question", - "user_id": user_id, - "kb_id": kb_id, - "document_id": document_id, - "title": truncate_utf8(title, TITLE_FIELD_MAX_BYTES), - "content": truncate_utf8(q), - "tags_text": "", - "chunk_index": chunk_index, - "dense_embedding": vec, - "sparse_embedding": text_to_sparse(q), - } - ) - _upsert(client, rows) diff --git a/src/backend/core/kb/wiki/local_provider.py b/src/backend/core/kb/wiki/local_provider.py deleted file mode 100644 index 70a11701..00000000 --- a/src/backend/core/kb/wiki/local_provider.py +++ /dev/null @@ -1,523 +0,0 @@ -"""自建知识库 Wiki 的读取面。 - -本模块的函数签名与返回结构，与外接知识库的 Wiki 模块**逐字段同构**——这是有意 -为之：路由层与 MCP 工具层只需要按 kb_id 选到不同的模块，调用代码一行都不用改， -前端组件更是完全无感。 - -相比外接透传，这里有一处实质优势：页面与 ``kb_chunks`` 同库，回溯原文是按 -chunk_id 直接查，而不是"拉整篇文档的全部分块再本地过滤"。 -""" - -from __future__ import annotations - -import logging -import re -from typing import Any, Dict, List, Optional, Sequence - -from core.db.engine import SessionLocal -from core.db.models import KBChunk, KBDocument, KBWikiPage -from core.db.models.kb_wiki import ( - WIKI_KNOWLEDGE_PAGE_TYPES, - WIKI_PAGE_TYPE_INDEX, - WIKI_PAGE_TYPE_LABELS, - WIKI_PAGE_TYPE_SUMMARY, -) -from core.kb.wiki.config import MAX_GRAPH_NODES -from core.kb.wiki.finalize import INDEX_SLUG -from core.kb.wiki.slug import readable_from_slug -from core.kb.wiki.store import ( - children_index, - clean_category_path, - descendant_folder_ids, - get_page, - load_folder_tree, - page_counts_by_folder, - subtree_counts, - walk_folder_path, -) -from sqlalchemy import func, or_ -from sqlalchemy.orm import Session - -logger = logging.getLogger(__name__) - -PAGE_TYPE_LABELS = WIKI_PAGE_TYPE_LABELS -KNOWLEDGE_PAGE_TYPES = WIKI_KNOWLEDGE_PAGE_TYPES - -_MAX_GRAPH_NODES = MAX_GRAPH_NODES -_MAX_SEARCH_LIMIT = 50 - - -def _split_types(raw: str) -> List[str]: - return [t.strip() for t in str(raw or "").split(",") if t.strip()] - - -def _slim_page(page: KBWikiPage, *, with_content: bool = False) -> Dict[str, Any]: - """页面 → 精简结构。 - - 定位阶段只要标题/摘要/链接/血缘，正文按需带：一页动辄数千字，无差别塞进模型 - 上下文会把「先定位再取回」的成本优势整个吃掉。 - """ - item: Dict[str, Any] = { - "slug": page.slug, - "title": page.title, - "page_type": page.page_type, - "type_label": PAGE_TYPE_LABELS.get(page.page_type or "", ""), - "summary": page.summary or "", - "aliases": list(page.aliases or []), - "category_path": list(page.category_path or []), - "wiki_path": page.wiki_path or "", - "out_links": list(page.out_links or []), - "in_links": list(page.in_links or []), - "source_refs": list(page.source_refs or []), - "chunk_refs": list(page.chunk_refs or []), - "updated_at": page.updated_at.isoformat() if page.updated_at else None, - } - if with_content: - item["content"] = page.content or "" - return item - - -def _base_query(db: Session, kb_id: str): - return db.query(KBWikiPage).filter(KBWikiPage.kb_id == kb_id, KBWikiPage.deleted_at.is_(None)) - - -# ── 统计 ──────────────────────────────────────────────────────────────────── - - -def wiki_stats(kb_id: str) -> Dict[str, Any]: - """规模统计：总页数、分类型页数、链接数、孤儿页数。 - - 只投影统计需要的三列——生成期间前端每 10 秒轮一次，把每页几 KB 的 ``content`` - 拉出来纯属浪费。 - """ - with SessionLocal() as db: - rows = ( - db.query(KBWikiPage.page_type, KBWikiPage.in_links, KBWikiPage.out_links) - .filter(KBWikiPage.kb_id == kb_id, KBWikiPage.deleted_at.is_(None)) - .all() - ) - by_type: Dict[str, int] = {} - total_links = 0 - orphan = 0 - for page_type, in_links, out_links in rows: - by_type[page_type] = by_type.get(page_type, 0) + 1 - total_links += len(out_links or []) - if not (in_links or []) and not (out_links or []): - orphan += 1 - return { - "total_pages": len(rows), - "pages_by_type": by_type, - "total_links": total_links, - "orphan_count": orphan, - } - - -# ── 页面列表 ──────────────────────────────────────────────────────────────── - - -def wiki_list_pages( - kb_id: str, - page: int = 1, - page_size: int = 50, - page_type: str = "", - category_path: str = "", - category_depth: int = 0, -) -> Dict[str, Any]: - """分页列出页面。 - - ``page_type`` 支持逗号分隔多选（「知识」视图把实体/概念/综述/对比合成一次 - 请求，避免按类型扇出）；``category_path`` 把范围收窄到某个目录子树。 - """ - page = max(1, int(page or 1)) - page_size = max(1, int(page_size or 50)) - with SessionLocal() as db: - query = _base_query(db, kb_id) - types = _split_types(page_type) - if types: - query = query.filter(KBWikiPage.page_type.in_(types)) - - labels = clean_category_path([category_path] if category_path else []) - if labels: - folder_id, _, _ = walk_folder_path(db, kb_id, labels, create=False) - if folder_id is None: - return {"pages": [], "total": 0, "page": page, "page_size": page_size} - query = query.filter( - KBWikiPage.folder_id.in_(descendant_folder_ids(db, kb_id, folder_id)) - ) - - total = query.count() - rows = ( - query.order_by(KBWikiPage.sort_order, KBWikiPage.title) - .offset((page - 1) * page_size) - .limit(page_size) - .all() - ) - return { - "pages": [_slim_page(row) for row in rows], - "total": total, - "page": page, - "page_size": page_size, - } - - -# ── 目录 ──────────────────────────────────────────────────────────────────── - - -def wiki_folders(kb_id: str, parent_id: str = "", page_types: str = "") -> Dict[str, Any]: - """某一层的子目录，带**递归**页面计数，供前端逐层懒加载。 - - 两次查询搞定：整棵目录树 + 一次按目录分组的页数聚合，其余在内存里算。原来是 - 每个子目录一次子树查询加一次计数，用户逐层展开时会反复付这个代价。 - """ - types = _split_types(page_types) - with SessionLocal() as db: - folders = load_folder_tree(db, kb_id) - if not folders: - return {"parent_id": parent_id or "", "folders": []} - totals = subtree_counts(folders, page_counts_by_folder(db, kb_id, types)) - children = children_index(folders) - items = [ - { - "id": folder.folder_id, - "name": folder.name, - "parent_id": folder.parent_id or "", - "page_count": totals.get(folder.folder_id, 0), - "has_children": bool(children.get(folder.folder_id)), - } - for folder in children.get(parent_id or "", []) - ] - return {"parent_id": parent_id or "", "folders": items} - - -# ── 索引总览 ──────────────────────────────────────────────────────────────── - - -def wiki_index(kb_id: str, limit: int = 20, types: str = "") -> Dict[str, Any]: - """索引总览：一段总述 + 按类型分组的条目清单。""" - limit = max(1, int(limit or 20)) - wanted = _split_types(types) - with SessionLocal() as db: - index_page = get_page(db, kb_id, INDEX_SLUG) - intro = "" - if index_page: - intro = str((index_page.page_metadata or {}).get("intro") or "") - - groups = [] - for page_type in [*KNOWLEDGE_PAGE_TYPES, WIKI_PAGE_TYPE_SUMMARY]: - if wanted and page_type not in wanted: - continue - query = _base_query(db, kb_id).filter(KBWikiPage.page_type == page_type) - total = query.count() - if not total: - continue - rows = query.order_by(KBWikiPage.title).limit(limit).all() - groups.append( - { - "type": page_type, - "type_label": PAGE_TYPE_LABELS.get(page_type, ""), - "total": total, - "items": [ - { - "slug": row.slug, - "title": row.title, - "summary": row.summary or "", - "wiki_path": row.wiki_path or "", - } - for row in rows - ], - } - ) - return { - "intro": intro, - "version": index_page.version if index_page else None, - "groups": groups, - } - - -# ── 检索 ──────────────────────────────────────────────────────────────────── - - -def wiki_search(kb_id: str, query: str, limit: int = 10) -> Dict[str, Any]: - """按标题/别名/摘要/正文命中页面。 - - 支持 ``资质|牌照`` 这样的交替写法——口语说法常常匹配不上书面术语，一次给多个 - 说法能显著提高命中率。交替在 Python 侧拆成多个 LIKE，避免依赖数据库正则。 - """ - limit = max(1, min(int(limit or 10), _MAX_SEARCH_LIMIT)) - terms = [t.strip() for t in re.split(r"[|｜]", str(query or "")) if t.strip()] - if not terms: - return {"pages": [], "total": 0} - - with SessionLocal() as db: - conditions = [] - for term in terms[:8]: - pattern = f"%{term.lower()}%" - conditions.append(func.lower(KBWikiPage.title).like(pattern)) - conditions.append(func.lower(KBWikiPage.summary).like(pattern)) - conditions.append(func.lower(KBWikiPage.content).like(pattern)) - rows = ( - _base_query(db, kb_id) - .filter(KBWikiPage.page_type != WIKI_PAGE_TYPE_INDEX) - .filter(or_(*conditions)) - .order_by(KBWikiPage.page_type, KBWikiPage.title) - .limit(limit * 3) - .all() - ) - - # 别名命中在 SQL 里跨方言不好写，回到 Python 侧补一遍；同时按"命中位置" - # 排序——标题命中远比正文里出现一次更相关 - lowered_terms = [t.lower() for t in terms] - - def _rank(page: KBWikiPage) -> int: - title = (page.title or "").lower() - aliases = " ".join(page.aliases or []).lower() - summary = (page.summary or "").lower() - if any(t == title for t in lowered_terms): - return 0 - if any(t in title for t in lowered_terms): - return 1 - if any(t in aliases for t in lowered_terms): - return 2 - if any(t in summary for t in lowered_terms): - return 3 - return 4 - - ranked = sorted(rows, key=lambda p: (_rank(p), p.title or ""))[:limit] - return {"pages": [_slim_page(p) for p in ranked], "total": len(ranked)} - - -# ── 单页 ──────────────────────────────────────────────────────────────────── - - -def wiki_read_page(kb_id: str, slug: str, *, resolve_links: bool = True) -> Dict[str, Any]: - """读取整页（含正文、双链、血缘坐标）。 - - ``out_links`` 只有 slug，直接展示就是一串拼音，人和模型都读不出是什么。这里 - 顺手把邻居的标题查回来贴上——同库一次 IN 查询就够。 - """ - with SessionLocal() as db: - page = get_page(db, kb_id, slug) - if page is None: - return {} - item = _slim_page(page, with_content=True) - item["related_pages"] = _resolve_link_titles( - db, kb_id, item["out_links"], resolve=resolve_links - ) - return item - - -def _resolve_link_titles( - db: Session, kb_id: str, slugs: Sequence[str], *, resolve: bool -) -> List[Dict[str, str]]: - if not slugs: - return [] - titles: Dict[str, KBWikiPage] = {} - if resolve: - rows = _base_query(db, kb_id).filter(KBWikiPage.slug.in_(list(slugs))).all() - titles = {row.slug: row for row in rows} - resolved = [] - for slug in slugs: - row = titles.get(slug) - resolved.append( - { - "slug": slug, - "title": (row.title if row else "") or readable_from_slug(slug), - "page_type": (row.page_type if row else "") or slug.split("/", 1)[0], - } - ) - return resolved - - -# ── 图谱 ──────────────────────────────────────────────────────────────────── - - -def wiki_graph( - kb_id: str, - mode: str = "overview", - center: str = "", - depth: int = 1, - limit: int = 60, - types: str = "", -) -> Dict[str, Any]: - """概念图谱。 - - ``overview`` 取全局连接度最高的节点，``ego`` 以 center 为中心展开 depth 层。 - 大库有数千节点上万条边，前端必须靠这两种模式分批取，不能一次拉全量。 - """ - limit = max(1, min(int(limit or 60), _MAX_GRAPH_NODES)) - depth = max(1, min(int(depth or 1), 3)) - wanted = _split_types(types) - - with SessionLocal() as db: - # 图谱只用到这四列；页面正文对邻接表毫无用处，却是最大的一列 - query = db.query( - KBWikiPage.slug, - KBWikiPage.title, - KBWikiPage.page_type, - KBWikiPage.out_links, - ).filter( - KBWikiPage.kb_id == kb_id, - KBWikiPage.deleted_at.is_(None), - KBWikiPage.page_type != WIKI_PAGE_TYPE_INDEX, - ) - if wanted: - query = query.filter(KBWikiPage.page_type.in_(wanted)) - rows = query.all() - - by_slug = {row.slug: row for row in rows} - adjacency: Dict[str, List[str]] = { - row.slug: [s for s in (row.out_links or []) if s in by_slug] for row in rows - } - # 边是无向展开用的，两个方向都要能走 - undirected: Dict[str, set] = {slug: set(links) for slug, links in adjacency.items()} - for slug, links in adjacency.items(): - for target in links: - undirected.setdefault(target, set()).add(slug) - - if mode == "ego": - selected = _ego_nodes(undirected, center, depth, limit) - if not selected: - return { - "nodes": [], - "edges": [], - "meta": { - "mode": "ego", - "center": center, - "depth": depth, - "truncated": False, - "total": len(rows), - "returned": 0, - "total_pages": len(rows), - }, - } - else: - ranked = sorted(rows, key=lambda r: len(undirected.get(r.slug, ())), reverse=True) - selected = {r.slug for r in ranked[:limit]} - - nodes = [ - { - "slug": slug, - "title": by_slug[slug].title or readable_from_slug(slug), - "page_type": by_slug[slug].page_type, - "link_count": len(undirected.get(slug, ())), - } - for slug in selected - if slug in by_slug - ] - edges = [] - seen_edges = set() - for slug in selected: - for target in adjacency.get(slug, []): - if target not in selected: - continue - key = tuple(sorted((slug, target))) - if key in seen_edges: - continue - seen_edges.add(key) - edges.append({"source": slug, "target": target}) - - return { - "nodes": nodes, - "edges": edges, - "meta": { - "mode": mode, - "center": center, - "depth": depth if mode == "ego" else None, - "truncated": len(rows) > len(selected), - # 前端类型声明读 total/returned；total_pages 是旧键，保留兼容 - "total": len(rows), - "returned": len(nodes), - "total_pages": len(rows), - }, - } - - -def _ego_nodes(adjacency: Dict[str, set], center: str, depth: int, limit: int) -> set: - """从 center 出发做 BFS，取 depth 层内的节点。""" - if not center or center not in adjacency: - return set() - selected = {center} - frontier = {center} - for _ in range(depth): - nxt = set() - for slug in frontier: - for neighbour in adjacency.get(slug, ()): - if neighbour not in selected: - nxt.add(neighbour) - if not nxt: - break - for slug in sorted(nxt, key=lambda s: len(adjacency.get(s, ())), reverse=True): - if len(selected) >= limit: - break - selected.add(slug) - frontier = nxt - if len(selected) >= limit: - break - return selected - - -# ── 顺血缘取回原文 ────────────────────────────────────────────────────────── - - -def wiki_source_chunks(kb_id: str, slug: str, max_chunks: int = 6) -> Dict[str, Any]: - """按页面的 ``chunk_refs`` 直取原文分块。 - - 这是「地图 → 定位 → 顺坐标取回原文」这条链的最后一跳：**按 ID 直取**，不是 - 再检索一次。页面与分块同库，一次 IN 查询就够。 - """ - max_chunks = max(1, min(int(max_chunks or 6), 20)) - with SessionLocal() as db: - page = get_page(db, kb_id, slug) - if page is None: - return {"page": {}, "chunks": []} - - item = _slim_page(page, with_content=True) - item["related_pages"] = _resolve_link_titles(db, kb_id, item["out_links"], resolve=True) - - chunk_ids = list(page.chunk_refs or []) - rows: List[KBChunk] = [] - if chunk_ids: - rows = ( - db.query(KBChunk) - .filter(KBChunk.kb_id == kb_id, KBChunk.chunk_id.in_(chunk_ids)) - .all() - ) - # 保持页面记录的引用顺序：那是证据的呈现次序，不是数据库的返回次序 - order = {cid: i for i, cid in enumerate(chunk_ids)} - rows.sort(key=lambda r: order.get(r.chunk_id, 10**6)) - - # chunk_refs 对不上（页面被改写过、或文档重新索引换了分块）时，退回该页 - # 主要来源文档的前几块正文——给不出精确证据，至少不要空手而归 - if not rows and (page.source_refs or []): - rows = ( - db.query(KBChunk) - .filter(KBChunk.kb_id == kb_id, KBChunk.document_id == page.source_refs[0]) - .order_by(KBChunk.chunk_index) - .limit(max_chunks) - .all() - ) - - titles = _document_titles(db, kb_id, [r.document_id for r in rows]) - chunks = [ - { - "chunk_id": row.chunk_id, - "document_id": row.document_id, - "document_title": titles.get(row.document_id, ""), - "content": row.content or "", - } - for row in rows[:max_chunks] - if (row.content or "").strip() - ] - return {"page": item, "chunks": chunks} - - -def _document_titles(db: Session, kb_id: str, document_ids: Sequence[str]) -> Dict[str, str]: - ids = [d for d in dict.fromkeys(document_ids) if d] - if not ids: - return {} - rows = ( - db.query(KBDocument.document_id, KBDocument.title, KBDocument.filename) - .filter(KBDocument.kb_id == kb_id, KBDocument.document_id.in_(ids)) - .all() - ) - return {row.document_id: (row.title or row.filename or row.document_id) for row in rows} diff --git a/src/backend/core/llm/__init__.py b/src/backend/core/llm/__init__.py index 4d9fbb7d..0546c2bc 100644 --- a/src/backend/core/llm/__init__.py +++ b/src/backend/core/llm/__init__.py @@ -1,17 +1,5 @@ """LLM package. Keep package import framework-neutral: pure context/manifest modules must be -usable in tests and offline tooling without importing AgentScope. Legacy -attribute exports remain available lazily. +usable in tests and offline tooling without importing AgentScope. """ - - -def __getattr__(name): # noqa: ANN001, ANN202 - if name in {"make_chat_model", "get_default_model"}: - from core.llm import chat_models - - return getattr(chat_models, name) - raise AttributeError(name) - - -__all__ = ["get_default_model", "make_chat_model"] diff --git a/src/backend/core/llm/_distill_shared.py b/src/backend/core/llm/_distill_shared.py deleted file mode 100644 index 96569922..00000000 --- a/src/backend/core/llm/_distill_shared.py +++ /dev/null @@ -1,156 +0,0 @@ -"""Shared helpers for distillation LLM callers (skill_distiller / persona_distiller). - -Extracted from skill_distiller.py so the per-chat auto distiller and the -persona-level (colleague / personal) distiller share one implementation of: -- model resolution (role 'skill_distiller' → fallback 'main_agent') -- tolerant strict-JSON parsing ( blocks, ``` fences, surrounding prose) -- rough cost estimation -- skill dict → SKILL.md rendering -""" - -from __future__ import annotations - -import json -import logging -import re -from typing import Any, Dict, Optional - -from core.services.model_config import ModelConfigService, ResolvedModelConfig - -logger = logging.getLogger(__name__) - - -# OpenAI-style rough price table per model. Override via env / admin panel later. -# USD per 1M tokens. Default to a conservative flat rate when model unknown. -PRICE_PER_MTOK: Dict[str, Dict[str, float]] = { - "default": {"input": 3.0, "output": 15.0}, - "deepseek": {"input": 0.27, "output": 1.1}, - "qwen": {"input": 0.5, "output": 1.5}, -} - - -def estimate_cost_usd(model: str, prompt_tokens: int, completion_tokens: int) -> float: - price = PRICE_PER_MTOK["default"] - for key, v in PRICE_PER_MTOK.items(): - if key in (model or "").lower(): - price = v - break - return (prompt_tokens * price["input"] + completion_tokens * price["output"]) / 1_000_000.0 - - -def resolve_distiller_model() -> ResolvedModelConfig: - svc = ModelConfigService.get_instance() - cfg = svc.resolve("skill_distiller") or svc.resolve("main_agent") - if cfg is None: - raise RuntimeError("skill_distiller: no model configured for 'skill_distiller' or 'main_agent'") - return cfg - - -_THINK_BLOCK_RE = re.compile(r".*?", re.DOTALL | re.IGNORECASE) -_THINK_CLOSE_RE = re.compile(r"", re.IGNORECASE) -_THINK_OPEN_RE = re.compile(r"", re.IGNORECASE) - - -def strip_think_blocks(text: str) -> str: - """Strip a reasoning model's think blocks and return the body. - - Covers three shapes: - - closed ``...`` blocks (possibly multiple); - - only ```` (some serving stacks swallow the opening tag into the chat template, with the body following the closing tag) - → take the content after the last closing tag; - - only ```` (output was truncated mid-thought by max_tokens, with no body after) - → discard everything from the opening tag onward. - """ - t = _THINK_BLOCK_RE.sub("", text) - m = None - for m in _THINK_CLOSE_RE.finditer(t): - pass - if m is not None: - t = t[m.end():] - m = _THINK_OPEN_RE.search(t) - if m is not None: - t = t[: m.start()] - return t.strip() - - -def extract_first_json_object(text: str) -> Optional[str]: - start = text.find("{") - if start < 0: - return None - depth = 0 - in_str = False - escape = False - for i, ch in enumerate(text[start:], start=start): - if in_str: - if escape: - escape = False - elif ch == "\\": - escape = True - elif ch == '"': - in_str = False - continue - if ch == '"': - in_str = True - elif ch == "{": - depth += 1 - elif ch == "}": - depth -= 1 - if depth == 0: - return text[start:i + 1] - return None - - -def parse_strict_json(text: str) -> Optional[Dict[str, Any]]: - """Parse JSON tolerantly: strips blocks, ``` fences, and surrounding prose.""" - if not text: - return None - # If the body is already valid JSON, return it directly to avoid corruption from stripping a think tag that happens to appear inside a string value - try: - parsed = json.loads(text.strip()) - if isinstance(parsed, dict): - return parsed - except Exception: - pass - t = strip_think_blocks(text.strip()) - - if t.startswith("```"): - lines = [l for l in t.splitlines() if not l.strip().startswith("```")] - t = "\n".join(lines).strip() - - try: - return json.loads(t) - except Exception: - pass - - candidate = extract_first_json_object(t) - if candidate: - try: - return json.loads(candidate) - except Exception: - return None - return None - - -def skill_to_markdown(skill: Dict[str, Any]) -> str: - """Render a skill dict back to SKILL.md text with YAML frontmatter.""" - try: - import yaml # type: ignore - except ImportError: - yaml = None # type: ignore - - fm = skill.get("frontmatter") or {} - body = skill.get("instructions_md") or "" - - if yaml is not None: - fm_text = yaml.safe_dump(fm, allow_unicode=True, sort_keys=False).strip() - else: - # Fallback: basic key: value formatting (only primitive types) - parts = [] - for k, v in fm.items(): - if isinstance(v, (list, dict)): - parts.append(f"{k}: {json.dumps(v, ensure_ascii=False)}") - else: - parts.append(f"{k}: {v}") - fm_text = "\n".join(parts) - - return f"---\n{fm_text}\n---\n\n{body}\n" diff --git a/src/backend/core/llm/api_runtime/__init__.py b/src/backend/core/llm/api_runtime/__init__.py new file mode 100644 index 00000000..14f23927 --- /dev/null +++ b/src/backend/core/llm/api_runtime/__init__.py @@ -0,0 +1 @@ +"""Agent runtime for authenticated API-key requests.""" diff --git a/src/backend/core/llm/agent_api_runtime.py b/src/backend/core/llm/api_runtime/agent.py similarity index 97% rename from src/backend/core/llm/agent_api_runtime.py rename to src/backend/core/llm/api_runtime/agent.py index 5d2b0f5f..46afdd5e 100644 --- a/src/backend/core/llm/agent_api_runtime.py +++ b/src/backend/core/llm/api_runtime/agent.py @@ -106,7 +106,7 @@ def scope_mcp_servers(servers: dict, scope: AgentApiExecutionScope) -> dict: existing = next((key for key in headers if key.lower() == header.lower()), header) if not str(headers.get(existing) or "").strip(): headers[existing] = "__agent_api_no_kb__" - from core.llm.mcp_invocation import for_url + from core.llm.mcp.invocation import for_url headers.update(for_url(item["url"], scope.sandbox_user_id, scope.chat_id)) item["headers"] = headers out[name] = item @@ -119,7 +119,7 @@ def artifact_in_scope(file_id: str, scope: AgentApiExecutionScope) -> bool: return False from core.db.engine import SessionLocal from core.db.models import Artifact, ChatSession - from core.services.agent_api_service import session_matches_scope + from core.auth.agent_api import session_matches_scope with SessionLocal() as db: session = db.get(ChatSession, scope.chat_id) if not session or not session_matches_scope(session, {"version": 1, **{key: value for key, value in asdict(scope).items() if value is not None}}): @@ -133,7 +133,7 @@ def artifact_in_scope(file_id: str, scope: AgentApiExecutionScope) -> bool: ) # An output is in the file index before pinning/finalization persists it. # Read only index metadata here; get_artifact resolves host project paths. - from core.artifacts.store import _read_record + from core.files.artifacts.store import _read_record item = _read_record(file_id) if file_id else None metadata = (item or {}).get("metadata") or {} return ( diff --git a/src/backend/core/llm/agent_api_skills.py b/src/backend/core/llm/api_runtime/agent_skills.py similarity index 96% rename from src/backend/core/llm/agent_api_skills.py rename to src/backend/core/llm/api_runtime/agent_skills.py index 110a38a1..e6b8beb1 100644 --- a/src/backend/core/llm/agent_api_skills.py +++ b/src/backend/core/llm/api_runtime/agent_skills.py @@ -5,7 +5,7 @@ import re from pathlib import Path -from core.llm.agent_api_runtime import AgentApiExecutionScope +from core.llm.api_runtime.agent import AgentApiExecutionScope def _skill_files(skill_dirs: dict[str, str]): diff --git a/src/backend/core/llm/agent_api_tools.py b/src/backend/core/llm/api_runtime/agent_tools.py similarity index 90% rename from src/backend/core/llm/agent_api_tools.py rename to src/backend/core/llm/api_runtime/agent_tools.py index 96c41212..7149bf29 100644 --- a/src/backend/core/llm/agent_api_tools.py +++ b/src/backend/core/llm/api_runtime/agent_tools.py @@ -1,5 +1,6 @@ """Native tools for an API agent: private account data is never mounted.""" from __future__ import annotations +from core.llm.factory.tools.definitions import function_tool import asyncio import base64 @@ -9,8 +10,8 @@ import tempfile from pathlib import Path -from core.llm.agent_api_runtime import AgentApiExecutionScope, artifact_in_scope -from core.llm.tools._tool_helpers import _resp_json +from core.llm.api_runtime.agent import AgentApiExecutionScope, artifact_in_scope +from core.llm.tools._tool_helpers import resp_json as _resp_json def _provider(): @@ -38,7 +39,7 @@ def _workspace_path(value: str) -> str: def register_agent_api_tools(toolkit, scope: AgentApiExecutionScope, *, read_only=False, skill_dirs=None): """Register a fixed scoped surface; do not wrap account-private tools.""" from core.sandbox import ProcessRequest, SandboxError - from core.llm.agent_api_skills import ApiSkillStager + from core.llm.api_runtime.agent_skills import ApiSkillStager skills = ApiSkillStager(skill_dirs) async def Bash(command: str, timeout: int | None = None, yield_time_ms: int = 60000): @@ -77,11 +78,11 @@ async def read_artifact( """读取当前 API 会话中生成的文件，不能读取账号其他文件。""" if not await asyncio.to_thread(artifact_in_scope, file_id, scope): return _resp_json({"error": "文件不属于当前 API 会话或已删除"}) - from core.llm.tool_collector import ToolCollector + from core.llm.factory.tools.definitions import ToolAssembly from core.llm.tools.read_artifact_tool import register_read_artifact - collector = ToolCollector() + collector = ToolAssembly() register_read_artifact(collector, user_id=scope.owner_user_id) - return await collector.get_tool("read_artifact")._func( + return await next(t for t in collector if t.name == "read_artifact")._func( file_id=file_id, offset=offset, limit=limit, sheet_name=sheet_name, slide_index=slide_index, ) @@ -93,11 +94,11 @@ async def pin_to_workspace(file_ids: list[str]): for file_id in file_ids: if not isinstance(file_id, str) or not await asyncio.to_thread(artifact_in_scope, file_id, scope): return _resp_json({"error": "文件不属于当前 API 会话或已删除"}) - from core.artifacts.store import get_artifact + from core.files.artifacts.store import get_artifact from core.db.engine import SessionLocal from core.db.models import Artifact - from core.llm import workspace - from core.services.artifact_service import persist_artifacts + from core.llm.runtime import workspace + from core.files.artifacts.service import persist_artifacts def persist(): refs = [] @@ -142,7 +143,7 @@ async def sandbox_put_artifact(artifact_id: str, dest_path: str): path = _workspace_path(dest_path) if not await asyncio.to_thread(artifact_in_scope, artifact_id, scope): raise ValueError("文件不属于当前 API 会话或已删除") - from core.llm.tools._tool_helpers import _resolve_artifact_files + from core.files.artifacts.generated import resolve_artifact_files as _resolve_artifact_files files, error = await asyncio.to_thread( _resolve_artifact_files, {path: artifact_id}, scope.owner_user_id, ) @@ -159,7 +160,7 @@ async def sandbox_put_artifact(artifact_id: str, dest_path: str): async def sandbox_get_artifact(src_path: str, name: str = ""): """保存当前 API 沙箱中的产物并返回文件 ID；不读取个人空间。""" from core.config.settings import settings - from core.llm.tools._tool_helpers import _store_generated_file_path + from core.files.artifacts.generated import store_generated_file_path as _store_generated_file_path target = None try: provider = _provider() @@ -192,7 +193,7 @@ async def sandbox_get_artifact(src_path: str, name: str = ""): target.unlink(missing_ok=True) for tool in (read_artifact, sandbox_get_artifact, pin_to_workspace): - toolkit.register_tool_function(tool, namesake_strategy="override") + toolkit.append(function_tool(tool)) if not read_only: for tool in (Bash, write_stdin, sandbox_put_artifact): - toolkit.register_tool_function(tool, namesake_strategy="override") + toolkit.append(function_tool(tool)) diff --git a/src/backend/core/llm/project_api_runtime.py b/src/backend/core/llm/api_runtime/project.py similarity index 94% rename from src/backend/core/llm/project_api_runtime.py rename to src/backend/core/llm/api_runtime/project.py index 1a7c638b..2bb36c2e 100644 --- a/src/backend/core/llm/project_api_runtime.py +++ b/src/backend/core/llm/api_runtime/project.py @@ -23,7 +23,7 @@ async def bind_project_api_runtime(request) -> None: def authorize(): from core.db.engine import SessionLocal - from core.services.api_key_runs import require_admission_key + from core.auth.api_key_runs import require_admission_key with SessionLocal() as db: require_admission_key( diff --git a/src/backend/core/llm/capability_tools.py b/src/backend/core/llm/capability_tools.py deleted file mode 100644 index 5e1ed5ff..00000000 --- a/src/backend/core/llm/capability_tools.py +++ /dev/null @@ -1,106 +0,0 @@ -"""Per-request connector guards; pooled clients remain account-neutral.""" - -import asyncio -from agentscope.tool import ToolBase, ToolChunk -from agentscope.message import TextBlock, ToolResultState -from core.capabilities import runtime -from core.capabilities.errors import CapabilityError -from core.capabilities.paths import LOCAL_PROFILE - - -def connector_available(run, server_id): - runtime.validate(run) - latest = runtime.get(run.run_id, scope_id=run.scope_id) or run - if "mcp:" + server_id in latest.unavailable: - return False - binding = run.mcp_bindings.get(server_id) - if binding is not None: - from core.capabilities import registry - - relationships = latest.dependency_report.get("connector_parents", {}) - parents = { - *relationships.get(server_id, []), - *relationships.get(binding["install_id"].split(":", 2)[2], []), - } - for parent_id in parents: - parent = registry.get(parent_id) - if parent is None or not parent.enabled or parent.state == "removed": - return False - owner = parent.payload.get("owner_user_id") - if owner and str(owner) != run.user_id: - return False - if binding is None or not binding.get("authorization_checked"): - return True - _, profile, key = binding["install_id"].split(":", 2) - if profile == "local-json": - from core.capabilities.mcp_json import local_server_configs - - return key in local_server_configs() - if profile == LOCAL_PROFILE: - from core.services.mcp_service import McpServerConfigService - - service = McpServerConfigService.get_instance() - return key in { - **service.get_all_servers(enabled_only=True), - **service.get_owned_servers(run.user_id, enabled_only=True), - } - from core.services.desktop_cloud_bridge import cloud_gateway_mcp_configs - - return server_id in cloud_gateway_mcp_configs([server_id]) - - -class AvailableMCPTool(ToolBase): - def __init__(self, tool, run, server_id): - self.tool, self.run, self.server_id = tool, run, server_id - for name in ( - "name", - "description", - "input_schema", - "is_concurrency_safe", - "is_read_only", - "is_external_tool", - "is_state_injected", - "is_mcp", - "mcp_name", - ): - setattr(self, name, getattr(tool, name)) - - def __getattr__(self, name): - return getattr(self.tool, name) - - async def check_permissions(self, *args, **kwargs): - return await self.tool.check_permissions(*args, **kwargs) - - async def __call__(self, **kwargs): - try: - available = await asyncio.to_thread(connector_available, self.run, self.server_id) - except (CapabilityError, OSError): - available = False - if not available: - return ToolChunk( - state=ToolResultState.ERROR, - content=[ - TextBlock( - text="该连接器已停用或当前不可用，本次调用未执行。请说明情况并继续处理可完成的部分。" - ) - ], - ) - # Do not catch an uncertain write outcome or retry a side effect. - return await self.tool(**kwargs) - - -class AvailableMCPClient: - def __init__(self, client, run): - self.client, self.run = client, run - self.name = client.name - - def __getattr__(self, name): - return getattr(self.client, name) - - async def list_tools(self): - return [ - AvailableMCPTool(tool, self.run, self.name) for tool in await self.client.list_tools() - ] - - async def get_tool(self, name): - return AvailableMCPTool(await self.client.get_tool(name), self.run, self.name) diff --git a/src/backend/core/llm/citations/__init__.py b/src/backend/core/llm/citations/__init__.py new file mode 100644 index 00000000..242360b5 --- /dev/null +++ b/src/backend/core/llm/citations/__init__.py @@ -0,0 +1 @@ +"""Number tool-result sources as citation anchors the model can cite.""" diff --git a/src/backend/core/llm/citations/allocator.py b/src/backend/core/llm/citations/allocator.py new file mode 100644 index 00000000..b481a98d --- /dev/null +++ b/src/backend/core/llm/citations/allocator.py @@ -0,0 +1,115 @@ +"""Per-chat citation anchor allocator and registry.""" +from __future__ import annotations +from contextvars import ContextVar +from typing import Any, Dict, List, Optional, Sequence +import logging +from core.llm.citations import anchor_state +from core.llm.citations import types + +logger = logging.getLogger(__name__) + +class AnchorAllocator: + """会话内单调锚点计数器 + 按 tool_id 的引用注册表。 + + 并发安全性：分配是同步代码段（无 await 点），asyncio 并行工具任务 + 通过 ContextVar 共享同一实例，不会重号。 + """ + + def __init__(self, start: int = 1) -> None: + self._next = max(1, int(start or 1)) + self._by_tool_id: Dict[str, List[types.AnchorCitation]] = {} + self.all: List[types.AnchorCitation] = [] + + def next_id(self) -> str: + aid = f"e{self._next}" + self._next += 1 + return aid + + def register(self, tool_id: Optional[str], items: Sequence[types.AnchorCitation]) -> None: + if not items: + return + self.all.extend(items) + if tool_id: + self._by_tool_id.setdefault(str(tool_id), []).extend(items) + + def citations_for(self, tool_id: Optional[str]) -> List[types.AnchorCitation]: + if not tool_id: + return [] + return list(self._by_tool_id.get(str(tool_id), [])) + + +def attach_allocator(agent: Any, allocator: AnchorAllocator) -> AnchorAllocator: + """run 入口调用：把发号器绑定到 agent（并写 ContextVar 供子链路兜底）。""" + try: + setattr(agent, ALLOC_ATTR, allocator) + except Exception as exc: # noqa: BLE001 - pydantic 模型等禁止动态属性时降级到 ContextVar + logger.debug("[citation-anchor] attach to agent failed: %s", exc) + CITATION_ALLOCATOR.set(allocator) + return allocator + + +def resolve_allocator(agent: Any) -> AnchorAllocator: + """中间件调用：取本 run 的发号器；缺失时就地建一个并绑定，保证始终可用。""" + allocator = getattr(agent, ALLOC_ATTR, None) + if isinstance(allocator, AnchorAllocator): + return allocator + allocator = CITATION_ALLOCATOR.get() + if not isinstance(allocator, AnchorAllocator): + allocator = AnchorAllocator() + return attach_allocator(agent, allocator) + + +def collect_citation_dicts( + tool_id: Optional[str], + allocator: Optional[AnchorAllocator] = None, +) -> List[Dict[str, Any]]: + """编排层统一取引用入口（dict 形态，直接可进 SSE / trace / 落库）。 + + 发号器是**唯一**编号方：中间件在结果回给模型前已完成注号并按 tool_id 登记， + 这里只做精确取回，不做二次提取。``allocator`` 由调用方显式传入（run 入口绑定 + 在 agent 上的那一个）；不传时退到 ContextVar（子智能体链路）。 + """ + if allocator is None: + allocator = CITATION_ALLOCATOR.get() + if allocator is None: + return [] + return [item.to_dict() for item in allocator.citations_for(tool_id)] + + +def anchor_start_for_chat(chat_id: Optional[str]) -> int: + """跨轮续号：扫描该会话已落库消息的 citations，取最大锚点 + 1。 + + 任何异常都回退 1（引用功能降级，不影响对话）。 + """ + if not chat_id: + return 1 + try: + from core.db.engine import SessionLocal + from core.db.models import ChatMessage + + max_seen = 0 + with SessionLocal() as db: + rows = ( + db.query(ChatMessage.extra_data) + .filter(ChatMessage.chat_id == chat_id) + .order_by(ChatMessage.chat_seq.desc()) + .limit(200) + .all() + ) + for (extra,) in rows: + for cit in (extra or {}).get("citations") or []: + m = anchor_state.ANCHOR_RE.match(str(cit.get("id", "")) if isinstance(cit, dict) else "") + if m: + max_seen = max(max_seen, int(m.group(1))) + return max_seen + 1 + except Exception as exc: # noqa: BLE001 + logger.debug("[citation-anchor] start scan failed for chat %s: %s", chat_id, exc) + return 1 + + +CITATION_ALLOCATOR: ContextVar[Optional[AnchorAllocator]] = ContextVar( + "jx_citation_allocator", default=None +) + + +ALLOC_ATTR = "_jx_citation_allocator" diff --git a/src/backend/core/llm/citations/anchor.py b/src/backend/core/llm/citations/anchor.py new file mode 100644 index 00000000..2843cafd --- /dev/null +++ b/src/backend/core/llm/citations/anchor.py @@ -0,0 +1,63 @@ +"""Annotate tool results with citation anchors.""" +from __future__ import annotations +import json +from typing import List, Optional, Tuple +import logging +from core.llm.citations.allocator import AnchorAllocator +from core.llm.citations import anchor_state +from core.llm.citations import extraction +from core.llm.citations import types + +logger = logging.getLogger(__name__) + +def annotate_tool_result( + tool_name: str, + tool_id: Optional[str], + text: str, + allocator: AnchorAllocator, +) -> Tuple[str, List[types.AnchorCitation]]: + """给一份工具结果文本注号。 + + 返回 (注号后的文本, 登记的引用条目)。不可注号（跳过名单 / 空结果 / + 解析失败且判定无引用价值）时原样返回 (text, [])。**绝不抛错**。 + """ + if not text or tool_name in anchor_state.SKIP_TOOLS: + return text, [] + try: + parsed = json.loads(text) + except (json.JSONDecodeError, TypeError): + parsed = None + + try: + if isinstance(parsed, dict): + items = extraction.extract_and_inject(tool_name, tool_id, parsed, allocator) + if not items: + return text, [] + return json.dumps(parsed, ensure_ascii=False), items + if isinstance(parsed, list): + # 裸数组结果：包壳后按启发式处理，回注仍输出裸数组语义（保持兼容 → 整份 1 条） + parsed = {"items": parsed} + items = extraction.extract_and_inject(tool_name, tool_id, parsed, allocator) + if not items: + return text, [] + return json.dumps(parsed["items"], ensure_ascii=False), items + # 纯文本结果：整份 1 条，文末追加锚点行 + spec = anchor_state.TOOL_SPECS.get(tool_name) + if spec and spec.mode == "list": + # 声明为列表型但拿到纯文本 → 形态异常，放行不注号 + return text, [] + aid = allocator.next_id() + item = types.AnchorCitation( + id=aid, + tool_name=tool_name, + tool_id=tool_id, + title=extraction.display_title(tool_name, spec), + url="", + snippet=str(text)[:anchor_state.SNIPPET_MAX], + source_type=(spec.source_type if spec else "unknown"), + item_index=-1, + ) + return f"{text}\n\n[cite_id: {aid}]", [item] + except Exception as exc: # noqa: BLE001 + logger.warning("[citation-anchor] annotate failed tool=%s: %s", tool_name, exc) + return text, [] diff --git a/src/backend/core/llm/citations/anchor_state.py b/src/backend/core/llm/citations/anchor_state.py new file mode 100644 index 00000000..a5e99dd3 --- /dev/null +++ b/src/backend/core/llm/citations/anchor_state.py @@ -0,0 +1,101 @@ +"""Anchor patterns, length limits and tool lists for citations.""" +from __future__ import annotations +import re +from typing import Dict, Tuple +from core.llm.citations import types + +ANCHOR_RE = re.compile(r"^e(\d+)$") + + +LIST_FIELD_ALIASES: Tuple[str, ...] = ( + "items", + "results", + "events", + "pages", + "records", + "entries", + "data", + "list", + "rows", + "docs", +) + + +TITLE_MAX = 120 + + +SNIPPET_MAX = 3000 + + +WHOLE_SNIPPET_MAX = 500 + + +SKIP_TOOLS = frozenset( + { + # 自研文件/沙盒操作 + "Write", + "Edit", + "Bash", + "write_stdin", + "sandbox_put_artifact", + "sandbox_get_artifact", + "pin_to_workspace", + "channel_read_attachment", + "choose_design", + "ask_user_question", + "view_text_file", + "get_data_context", + "load_skill", + "Glob", + "Grep", + # 产物/发布/批量 + "publish_site", + # 定时任务写操作 + "create_scheduled_task", + "update_scheduled_task", + "pause_scheduled_task", + "resume_scheduled_task", + "delete_scheduled_task", + "list_scheduled_tasks", + "list_channel_conversations", + "get_scheduled_task", + # 技能管理 + "search_marketplace", + "list_my_skills", + "install_from_marketplace", + "register_skill", + "submit_to_marketplace", + "delete_skill", + "edit_skill", + # 能力/资产发现 + "list_datasets", + # 子智能体的引用由其内部工具的嵌套事件承载 + "call_subagent", + } +) + + +TOOL_SPECS: Dict[str, types.CitationSpec] = { + # 检索/知识类（逐条） + "internet_search": types.CitationSpec( + mode="list", + items_paths=(("result", "results"), ("results",)), + source_type="internet", + ), + "retrieve_dataset_content": types.CitationSpec( + mode="list", items_paths=(("items",),), source_type="knowledge_base" + ), + "retrieve_local_kb": types.CitationSpec( + mode="list", items_paths=(("items",),), source_type="knowledge_base" + ), + "wiki_fetch_source": types.CitationSpec( + mode="list", items_paths=(("items",),), source_type="knowledge_base" + ), + "wiki_locate": types.CitationSpec( + mode="list", items_paths=(("pages",),), source_type="knowledge_base" + ), + "wiki_read_page": types.CitationSpec(mode="whole", title="Wiki 页面", source_type="knowledge_base"), + "web_fetch": types.CitationSpec(mode="whole", title="网页内容", source_type="internet"), + # 数据/报告类（整份） + "query_database": types.CitationSpec(mode="whole", title="数据库查询结果", source_type="database"), +} diff --git a/src/backend/core/llm/citations/extraction.py b/src/backend/core/llm/citations/extraction.py new file mode 100644 index 00000000..f0abbc17 --- /dev/null +++ b/src/backend/core/llm/citations/extraction.py @@ -0,0 +1,208 @@ +"""Extract citation items from tool results and inject their ids.""" +from __future__ import annotations +import json +import re +from typing import Any, Dict, List, Optional, Sequence, Tuple +from core.llm.citations.allocator import AnchorAllocator +from core.llm.citations import anchor_state +from core.llm.citations import types + +def first_str(item: Dict[str, Any], keys: Sequence[str]) -> str: + for key in keys: + val = item.get(key) + if isinstance(val, (str, int, float)) and str(val).strip(): + return str(val) + return "" + + +def normalize_citation_url(raw: str) -> str: + """把工具结果里的来源链接规整成「浏览器点了不会跑偏」的绝对地址。 + + 检索工具返回的 url 千奇百怪，直接透传会有两类坏结果： + + - **缺协议**（``www.example.gov.cn/a.html``）：前端 ```` 会把它当相对 + 路径，按当前站点 origin 解析，于是点开变成 ``http://<本平台>/www.example…``， + 落到我们自己的 nginx 上返回 404 —— 看起来像「引用地址错误」。 + - **纯相对路径**（``/module/download/x.doc``）：没有站点根就无从还原，同样会 + 指到本平台。这种宁可不给链接，也好过给一个必然打不开的。 + + 另外只放行 http/https：javascript:、data: 之类既没有意义也不该出现在来源里。 + """ + url = (raw or "").strip() + if not url: + return "" + lowered = url.lower() + if lowered.startswith(("http://", "https://")): + return url + # 协议相对：//example.com/a → https://example.com/a + if url.startswith("//"): + return f"https:{url}" + # 明确的其它协议（javascript:/data:/ftp: …）一律丢弃 + if re.match(r"^[a-zA-Z][a-zA-Z0-9+.\-]*:", url): + return "" + # 相对路径：没有站点根，还原不出来 + if url.startswith("/"): + return "" + # 剩下的形如 example.gov.cn/path —— 看着像域名就补上协议 + host = url.split("/", 1)[0] + if "." in host and " " not in host: + return f"https://{url}" + return "" + + +def dig(data: Dict[str, Any], path: Sequence[str]) -> Any: + cur: Any = data + for key in path: + if not isinstance(cur, dict): + return None + cur = cur.get(key) + return cur + + +def display_title(tool_name: str, spec: Optional[types.CitationSpec]) -> str: + if spec and spec.title: + return spec.title + return f"{tool_name} 结果" + + +def item_citation( + aid: str, + tool_name: str, + tool_id: Optional[str], + spec: types.CitationSpec, + item: Dict[str, Any], + index: int, +) -> types.AnchorCitation: + title = first_str(item, spec.title_keys)[:anchor_state.TITLE_MAX] or display_title(tool_name, spec) + if spec.snippet_join: + parts = [first_str(item, (k,)) for k in spec.snippet_join] + snippet = " · ".join(p for p in parts if p) + else: + snippet = first_str(item, spec.snippet_keys) + return types.AnchorCitation( + id=aid, + tool_name=tool_name, + tool_id=tool_id, + title=title, + url=normalize_citation_url(first_str(item, spec.url_keys)), + snippet=snippet[:anchor_state.SNIPPET_MAX], + source_type=spec.source_type, + item_index=index, + ) + + +def looks_like_error(data: Dict[str, Any]) -> bool: + if not data.get("error"): + return False + # 列表字段仍有内容时不算整体失败（部分工具 error+items 并存） + for key in anchor_state.LIST_FIELD_ALIASES: + val = data.get(key) + if isinstance(val, list) and val: + return False + return True + + +def heuristic_list_field(data: Dict[str, Any]) -> Optional[Tuple[Tuple[str, ...], List[Any]]]: + """启发式找列表字段：别名优先；否则要求「唯一」的字典数组字段。""" + scopes: List[Tuple[Tuple[str, ...], Dict[str, Any]]] = [((), data)] + inner = data.get("result") + if isinstance(inner, dict): + scopes.append((("result",), inner)) + for prefix, scope in scopes: + for alias in anchor_state.LIST_FIELD_ALIASES: + val = scope.get(alias) + if isinstance(val, list) and val and all(isinstance(x, dict) for x in val): + return prefix + (alias,), val + for prefix, scope in scopes: + candidates = [ + (key, val) + for key, val in scope.items() + if isinstance(val, list) and val and all(isinstance(x, dict) for x in val) + ] + if len(candidates) == 1: + key, val = candidates[0] + return prefix + (key,), val + return None + + +def extract_and_inject( + tool_name: str, + tool_id: Optional[str], + data: Dict[str, Any], + allocator: AnchorAllocator, +) -> List[types.AnchorCitation]: + """在 data 上就地注入 cite_id，返回登记的引用条目。""" + spec = anchor_state.TOOL_SPECS.get(tool_name) + + # L1：工具自声明 __citations__（优先级最高，条目顺序即 item_index） + declared = data.get("__citations__") + if isinstance(declared, list) and declared and all(isinstance(x, dict) for x in declared): + base = spec or types.CitationSpec(source_type="unknown") + out: List[types.AnchorCitation] = [] + for idx, entry in enumerate(declared): + aid = allocator.next_id() + entry["cite_id"] = aid + out.append( + types.AnchorCitation( + id=aid, + tool_name=tool_name, + tool_id=tool_id, + title=str(entry.get("title") or display_title(tool_name, spec))[:anchor_state.TITLE_MAX], + url=normalize_citation_url(str(entry.get("url") or "")), + snippet=str(entry.get("snippet") or "")[:anchor_state.SNIPPET_MAX], + source_type=str(entry.get("source_type") or base.source_type), + item_index=idx, + ) + ) + return out + + if looks_like_error(data): + return [] + + # L2：配置映射（列表型） + if spec and spec.mode == "list": + out = [] + for path in spec.items_paths: + items = dig(data, path) + if not isinstance(items, list): + continue + for item in items: + if not isinstance(item, dict) or item.get("error"): + continue + aid = allocator.next_id() + item["cite_id"] = aid + out.append(item_citation(aid, tool_name, tool_id, spec, item, len(out))) + return out + + # L3：无 spec → 启发式找唯一字典数组 + if spec is None: + found = heuristic_list_field(data) + if found is not None: + _path, items = found + generic = types.CitationSpec(mode="list", source_type="unknown") + out = [] + for item in items: + if item.get("error"): + continue + aid = allocator.next_id() + item["cite_id"] = aid + out.append(item_citation(aid, tool_name, tool_id, generic, item, len(out))) + if out: + return out + + # L4：整份 1 条（显式 whole spec 或彻底认不出） + aid = allocator.next_id() + data["cite_id"] = aid + snippet = json.dumps(data, ensure_ascii=False)[:anchor_state.WHOLE_SNIPPET_MAX] + return [ + types.AnchorCitation( + id=aid, + tool_name=tool_name, + tool_id=tool_id, + title=display_title(tool_name, spec), + url=normalize_citation_url(first_str(data, (spec or types.CitationSpec()).url_keys)), + snippet=snippet, + source_type=(spec.source_type if spec else "unknown"), + item_index=-1, + ) + ] diff --git a/src/backend/core/llm/citations/hook.py b/src/backend/core/llm/citations/hook.py new file mode 100644 index 00000000..2dc9133e --- /dev/null +++ b/src/backend/core/llm/citations/hook.py @@ -0,0 +1,38 @@ +"""Annotate a buffered neutral tool result before model and SSE exposure.""" +from collections.abc import Mapping +from core.harness.events import thaw_value +from core.harness.hooks import Decision, FailureMode, HookSpec, HookStage +from core.llm.citations.allocator import CITATION_ALLOCATOR +from core.llm.citations.anchor import annotate_tool_result +from core.llm.citations.anchor_state import SKIP_TOOLS + + +def citation_candidate(name): + return bool(name and name not in SKIP_TOOLS) + + +def annotate_result(invocation): + if not citation_candidate(invocation.operation_name): + return Decision() + result = thaw_value(invocation.data.get("tool_result")) + if not result or result.get("state") != "success": + return Decision() + blocks = result.get("content") or [] + if not blocks or any(not isinstance(block, Mapping) or block.get("type") != "text" for block in blocks): + return Decision() + allocator = CITATION_ALLOCATOR.get() + if allocator is None: + return Decision() + tool_id = str(invocation.data.get("metadata", {}).get("tool_call_id") or "") + text = "".join(block.get("text") or "" for block in blocks) + annotated, items = annotate_tool_result(invocation.operation_name, tool_id, text, allocator) + if not items: + return Decision() + allocator.register(tool_id, items) + result["content"] = [{"type": "text", "text": annotated}] + return Decision.modify({"tool_result": result}) + + +def citation_hook(): + return HookSpec("citation_anchor", HookStage.AFTER_TOOL, annotate_result, priority=200, + failure_mode=FailureMode.OPEN, mutable_fields=frozenset({"tool_result"})) diff --git a/src/backend/core/llm/citations/types.py b/src/backend/core/llm/citations/types.py new file mode 100644 index 00000000..b1809f0d --- /dev/null +++ b/src/backend/core/llm/citations/types.py @@ -0,0 +1,63 @@ +"""Citation item and per-tool extraction spec types.""" +from __future__ import annotations +from dataclasses import asdict, dataclass +from typing import Any, Dict, Optional, Tuple + +@dataclass +class AnchorCitation: + """带全局锚点的引用条目（CitationItem 的超集，dict 形态向后兼容）。""" + + id: str # 全局锚点，如 "e7" + tool_name: str + tool_id: Optional[str] + title: str + url: str + snippet: str + source_type: str + item_index: int = -1 # 条目在该次工具结果列表中的 0-based 下标；整体型为 -1 + + def to_dict(self) -> Dict[str, Any]: + return asdict(self) + + +DEFAULT_TITLE_KEYS: Tuple[str, ...] = ( + "title", + "标题", + "文件名称", + "企业名称", + "产品名称", + "name", + "名称", + "document_name", +) + + +DEFAULT_URL_KEYS: Tuple[str, ...] = ("url", "链接", "link", "href") + + +DEFAULT_SNIPPET_KEYS: Tuple[str, ...] = ( + "content", + "文件内容", + "snippet", + "摘要", + "summary", + "description", + "abstract", + "text", +) + + +@dataclass(frozen=True) +class CitationSpec: + """单个工具的引用提取规格。mode: list（逐条）/ whole（整份 1 条）。""" + + mode: str = "whole" + # 列表字段路径（支持多层如 ("result", "results")；支持并列多列表） + items_paths: Tuple[Tuple[str, ...], ...] = () + title_keys: Tuple[str, ...] = DEFAULT_TITLE_KEYS + url_keys: Tuple[str, ...] = DEFAULT_URL_KEYS + snippet_keys: Tuple[str, ...] = DEFAULT_SNIPPET_KEYS + # 摘要拼接键（如企业搜索的 法定代表人·注册资金·企业状态）；命中则优先于 snippet_keys + snippet_join: Tuple[str, ...] = () + source_type: str = "unknown" + title: str = "" # whole 模式的显示标题；空则用工具名 diff --git a/src/backend/core/llm/command_middleware.py b/src/backend/core/llm/command_middleware.py deleted file mode 100644 index 48b55717..00000000 --- a/src/backend/core/llm/command_middleware.py +++ /dev/null @@ -1,39 +0,0 @@ -"""Insert terminal command facts only between complete ReAct steps.""" -import asyncio -from agentscope.middleware import MiddlewareBase -from core.llm.context_adapter import append_context_text -from core.llm.context_ir import KIND_REMINDER, POLICY_NEVER -from core.services.commands import delivery -from core.services.commands import store - -class CommandCompletionMiddleware(MiddlewareBase): - async def on_reasoning(self, agent, input_kwargs, next_handler): - state = agent.state - run_id = str(state.run_id or "") - rows = [] - if (run_id and state.chat_id and state.user_id - and await asyncio.to_thread(store.has_pending, state.chat_id, state.user_id)): - rows = await asyncio.to_thread(delivery.reserve_boundary, - state.chat_id, state.user_id, run_id) - if rows: - append_context_text(agent, delivery.notice_text(rows), kind=KIND_REMINDER, - origin="harness:command_completion", trust="system", priority=950, - token_budget=4000, truncation_policy=POLICY_NEVER) - async for item in next_handler(**input_kwargs): - yield item - - async def on_reply(self, agent, input_kwargs, next_handler): - if not (agent.state.run_id and agent.state.chat_id and agent.state.user_id): - async for item in next_handler(**input_kwargs): - yield item - return - current = input_kwargs - while True: - async for item in next_handler(**current): - yield item - # Close the final-response race: completions arriving during the last - # output receive a new reasoning step in the same active run. - rows = await asyncio.to_thread(store.pending_notices, agent.state.chat_id) - if not any(r["notice_status"] == "pending" and r["user_id"] == agent.state.user_id for r in rows): - return - current = {**input_kwargs, "inputs": None} diff --git a/src/backend/core/llm/context/__init__.py b/src/backend/core/llm/context/__init__.py new file mode 100644 index 00000000..454203f7 --- /dev/null +++ b/src/backend/core/llm/context/__init__.py @@ -0,0 +1 @@ +"""Framework-neutral context IR: items, assembly and conversion to provider messages.""" diff --git a/src/backend/core/llm/context/adapter.py b/src/backend/core/llm/context/adapter.py new file mode 100644 index 00000000..a5da94ad --- /dev/null +++ b/src/backend/core/llm/context/adapter.py @@ -0,0 +1,362 @@ +"""Translate between the context IR and AgentScope messages.""" +from __future__ import annotations +from core.llm.context import provider_conversion +from core.llm.context import reference_conversion +from core.llm.context import message_defaults +from dataclasses import replace +from typing import Any, Iterable, Mapping, Optional, Sequence +from agentscope.message import Msg, TextBlock +from core.infra.immutable import thaw_json +from core.llm.context.ir_constants import CONTEXT_SEQUENCE_STRIDE, KIND_COMPACTION, KIND_USER_INPUT, POLICY_DROP, POLICY_HEAD_TAIL, POLICY_NEVER, SESSION_CONTEXT_META_KEY, STEP_KINDS, VISIBILITY_MODEL +from core.llm.context.items import ContextItem + + +def _block_payload(block: Any) -> Any: + if isinstance(block, Mapping): + return dict(block) + if hasattr(block, "model_dump"): + return block.model_dump(mode="json") + return block + + +def _block_type(block: Any) -> str: + if isinstance(block, Mapping): + return str(block.get("type") or "") + return str(getattr(block, "type", "") or "") + + +def _block_text(block: Any) -> str: + if isinstance(block, Mapping): + return str(block.get("text") or "") + return str(getattr(block, "text", "") or "") + + +CONTEXT_META_KEY = "harness_context_items" + + +class AgentScopeContextAdapter: + """Translate the IR while preserving current model-facing roles/blocks.""" + + @staticmethod + def wrap_content(content: Any) -> list[Any]: + """Construct AgentScope-compatible blocks inside the sole SDK seam.""" + if isinstance(content, str): + return [TextBlock(type="text", text=content)] if content else [] + if isinstance(content, list): + return content + return [content] if content else [] + + @staticmethod + def _defaults(role: str, block_type: str) -> dict[str, Any]: + return message_defaults.defaults(role, block_type) + + @staticmethod + def _metadata_entries(message: Msg) -> list[Mapping[str, Any]]: + metadata = getattr(message, "metadata", None) or {} + raw = metadata.get(CONTEXT_META_KEY) if isinstance(metadata, Mapping) else None + if not isinstance(raw, list): + return [] + return [entry for entry in raw if isinstance(entry, Mapping)] + + def items_from_messages( + self, + messages: Sequence[Msg], + *, + summary_text: Any = None, + promote_latest_user: bool = True, + ) -> list[ContextItem]: + items: list[ContextItem] = [] + summary = str(summary_text or "") + tool_occurrences: dict[str, int] = {} + pending_tool_pairs: dict[str, list[str]] = {} + orphan_results: dict[str, int] = {} + # One ReAct step = what the model produced in one response plus the + # results of the tool calls in it. A step opens on the first thinking, + # text or tool_call block after a tool result (or after any other + # message) and its results join it through the pending call ids. This + # is the same boundary the reasoning-echo formatter flushes on. + step_unit: Optional[str] = None + step_closed = False + pair_units: dict[str, str] = {} + for message_index, message in enumerate(messages): + role = str(getattr(message, "role", "user") or "user") + name = str(getattr(message, "name", role) or role) + blocks = list(getattr(message, "content", None) or []) + entries = self._metadata_entries(message) + for block_index, block in enumerate(blocks): + block_type = _block_type(block) + payload = _block_text(block) if block_type == "text" else _block_payload(block) + defaults = self._defaults(role, block_type) + explicit = entries[block_index] if block_index < len(entries) else {} + if ( + not explicit + and summary + and role == "user" + and isinstance(payload, str) + and payload == summary + ): + defaults.update( + kind=KIND_COMPACTION, + origin="harness:compaction", + trust="system", + priority=850, + truncation_policy=POLICY_HEAD_TAIL, + render_role="user", + ) + raw_pair_id = str( + payload.get("id") + if block_type in {"tool_call", "tool_result"} and isinstance(payload, Mapping) + else "" + ) + pair_id = str(explicit.get("pair_id") or "") + if block_type == "tool_call" and raw_pair_id: + occurrence = tool_occurrences.get(raw_pair_id, 0) + 1 + tool_occurrences[raw_pair_id] = occurrence + pair_id = f"{raw_pair_id}#{occurrence}" + pending_tool_pairs.setdefault(raw_pair_id, []).append(pair_id) + elif block_type == "tool_result" and raw_pair_id: + pending = pending_tool_pairs.get(raw_pair_id) or [] + if pending: + pair_id = pending.pop(0) + else: + occurrence = orphan_results.get(raw_pair_id, 0) + 1 + orphan_results[raw_pair_id] = occurrence + pair_id = f"{raw_pair_id}#orphan-result-{occurrence}" + item_id = str( + ( + f"message:{message_index}:block:{block_index}:{block_type}:{pair_id}" + if block_type in {"tool_call", "tool_result"} + else explicit.get("item_id") + or f"message:{message_index}:block:{block_index}" + ) + ) + kind = str(explicit.get("kind") or defaults["kind"]) + if kind in STEP_KINDS and block_type != "tool_result": + if step_unit is None or step_closed: + step_unit = f"step:{message_index}:{block_index}" + step_closed = False + unit_id = step_unit + if block_type == "tool_call": + pair_units[pair_id] = unit_id + elif block_type == "tool_result": + unit_id = pair_units.pop(pair_id, f"orphan:{item_id}") + step_closed = True + else: + unit_id = f"item:{item_id}" + step_unit = None + step_closed = False + pair_units.clear() + created_seq = int( + explicit.get("created_seq") + if explicit.get("created_seq") is not None + else message_index * CONTEXT_SEQUENCE_STRIDE + block_index + ) + item = ContextItem.create( + item_id=item_id, + kind=kind, + origin=str(explicit.get("origin") or defaults["origin"]), + trust=str(explicit.get("trust") or defaults["trust"]), + visibility=str(explicit.get("visibility") or VISIBILITY_MODEL), + priority=int( + explicit["priority"] + if explicit.get("priority") is not None + else defaults["priority"] + ), + token_budget=int( + explicit["token_budget"] + if explicit.get("token_budget") is not None + else defaults["token_budget"] + ), + truncation_policy=str( + explicit.get("truncation_policy") or defaults["truncation_policy"] + ), + content=payload, + cache_class=str(explicit.get("cache_class") or "dynamic"), + created_seq=created_seq, + render_role=str(explicit.get("render_role") or defaults["render_role"]), + # The outer message carries the actual transport name. + # Persisted manifests contain only its hash. + render_name=name, + pair_id=pair_id, + message_group=str(explicit.get("message_group") or f"message:{message_index}"), + unit_id=unit_id, + content_ref=( + str(explicit["content_ref"]) if explicit.get("content_ref") else None + ), + content_hash=( + str(explicit["content_hash"]) if explicit.get("content_hash") else None + ), + token_estimate=( + int(explicit["token_estimate"]) + if explicit.get("token_estimate") is not None + else None + ), + metadata={}, + ) + items.append(item) + + # The latest real user instruction is a hard inclusion. Memory and + # reminders may render as user for compatibility but keep distinct kind. + user_indices = [i for i, item in enumerate(items) if item.kind == KIND_USER_INPUT] + if promote_latest_user and user_indices: + idx = user_indices[-1] + items[idx] = replace( + items[idx], + priority=max(900, items[idx].priority), + truncation_policy=POLICY_NEVER, + ) + return items + + def items_from_session_dict( + self, + message: Mapping[str, Any], + *, + created_seq: int, + ) -> list[ContextItem]: + role = str(message.get("role") or "user") + role = {"human": "user", "ai": "assistant", "tool": "assistant"}.get(role, role) + content = message.get("content") + blocks = ( + content + if isinstance(content, list) + else ([{"type": "text", "text": content}] if content else []) + ) + explicit = message.get(SESSION_CONTEXT_META_KEY) + metadata_entries = [] + if isinstance(explicit, Mapping): + metadata_entries = [dict(explicit) for _ in blocks] + msg = Msg( + name=str(message.get("name") or role), + role=role if role in {"system", "assistant", "user"} else "user", + content=blocks, + metadata={CONTEXT_META_KEY: metadata_entries} if metadata_entries else {}, + ) + # A persisted row is converted in isolation, so it cannot know whether + # it is the current user instruction. Promotion happens once, later, + # when ManifestBoundAgent sees the complete final request. + items = self.items_from_messages([msg], promote_latest_user=False) + has_explicit_seq = isinstance(explicit, Mapping) and explicit.get("created_seq") is not None + return [ + replace( + item, + item_id=( + item.item_id + if isinstance(explicit, Mapping) and explicit.get("item_id") + else f"session:{created_seq}:block:{index}" + ), + created_seq=( + item.created_seq + if has_explicit_seq + else created_seq * CONTEXT_SEQUENCE_STRIDE + index + ), + message_group=f"session:{created_seq}", + unit_id=f"session:{created_seq}:{item.unit_id}", + ) + for index, item in enumerate(items) + ] + + def message_from_session_dict( + self, + message: Mapping[str, Any], + *, + created_seq: int, + ) -> Msg: + """Render one persisted history record through the canonical adapter.""" + items = self.items_from_session_dict(message, created_seq=created_seq) + rendered = self.messages_from_items(items) + if rendered: + return rendered[0] + role = str(message.get("role") or "user") + role = {"human": "user", "ai": "assistant", "tool": "assistant"}.get(role, role) + if role not in {"system", "assistant", "user"}: + role = "user" + return Msg( + name=str(message.get("name") or role), + role=role, + content=[], + metadata={CONTEXT_META_KEY: []}, + ) + + def messages_from_items(self, items: Iterable[ContextItem]) -> list[Msg]: + visible = [item for item in items if item.visibility == VISIBILITY_MODEL] + messages: list[Msg] = [] + current_group: Optional[tuple[str, str]] = None + current_items: list[ContextItem] = [] + + def flush() -> None: + nonlocal current_group, current_items + if not current_items or current_group is None: + return + blocks = [] + manifests = [] + for item in current_items: + content = thaw_json(item.content) + if isinstance(content, str): + blocks.append(TextBlock(type="text", text=content)) + manifests.append(item.to_manifest()) + elif isinstance(content, Mapping): + blocks.append(dict(content)) + manifests.append(item.to_manifest()) + elif isinstance(content, list): + for block_index, block in enumerate(content): + blocks.append(block) + normalized = ( + _block_text(block) + if _block_type(block) == "text" + else _block_payload(block) + ) + derived = replace( + item, + item_id=f"{item.item_id}:block:{block_index}", + truncation_policy=( + POLICY_DROP + if _block_type(block) == "data" + else item.truncation_policy + ), + ).with_content(normalized) + manifests.append(derived.to_manifest()) + elif content is not None: + blocks.append(TextBlock(type="text", text=str(content))) + manifests.append(item.with_content(str(content)).to_manifest()) + if blocks: + name = str(current_items[0].render_name or current_group[1]) + messages.append( + Msg( + name=name, + role=current_group[1], + content=blocks, + metadata={CONTEXT_META_KEY: manifests}, + ) + ) + current_group = None + current_items = [] + + for item in visible: + group = (item.message_group, item.render_role) + if current_group is not None and group != current_group: + flush() + current_group = group + current_items.append(item) + flush() + return messages + + def annotate_provider_messages( + self, + source_messages: Sequence[Msg], + formatted_messages: Sequence[Mapping[str, Any]], + row_counts: Sequence[int], + ) -> list[dict[str, Any]]: + return provider_conversion.annotate_provider_messages(self, source_messages, formatted_messages, row_counts) + + def items_from_provider_messages( + self, + messages: Sequence[Mapping[str, Any]], + ) -> list[ContextItem]: + return provider_conversion.items_from_provider_messages(self, messages) + + @staticmethod + def provider_messages_from_items(items: Iterable[ContextItem]) -> list[dict[str, Any]]: + return provider_conversion.provider_messages_from_items(items) + + def reference_items_from_execution_manifest(self, manifest: Any) -> list[ContextItem]: + return reference_conversion.reference_items_from_execution_manifest(self, manifest) diff --git a/src/backend/core/llm/context/attachment_hook.py b/src/backend/core/llm/context/attachment_hook.py new file mode 100644 index 00000000..b9c71d19 --- /dev/null +++ b/src/backend/core/llm/context/attachment_hook.py @@ -0,0 +1,81 @@ +"""Attachment context policy using immutable hook inputs and neutral content.""" +import asyncio +from core.files import artifact_bytes +from core.harness.hooks import Decision, FailureMode, HookSpec, HookStage, Invocation +from core.llm.runtime import file_context +from core.models.vision import resolve_vision_mode + + +def supports_native_images(provider_id: str) -> bool: + return resolve_vision_mode(provider_id or "") == "native" + + +def native_image_content(image_files: list, user_id): + blocks, names = [], [] + for file in image_files: + result = file_context.fetch_image_base64(file, user_id=user_id) + if result: + encoded, mime = result + blocks.append({"type": "data", "source": { + "type": "base64", "media_type": mime, "data": encoded, + }}) + names.append(file.get("name", "图片")) + if not blocks: + return None + prefix = {"type": "text", "text": f"[用户上传了 {len(blocks)} 张图片：{'、'.join(names)}]"} + return [prefix, *blocks] + + +async def vision_evidence(image_files: list, user_id, precomputed: str = "") -> str: + if precomputed.strip(): + return precomputed.strip() + from core.models.vision.attachments import transcribe_attachments + result = await transcribe_attachments(image_files, user_id=user_id, + download_bytes=artifact_bytes.download_artifact_bytes) + return result.text if result is not None and result.text else "" + + +def payload(content, *, kind, origin, trust, token_budget=8000): + return dict(content=content, kind=kind, origin=origin, trust=trust, + priority=650, token_budget=token_budget) + + +class AttachmentContextHook: + def __init__(self): + self.injected = False + + async def __call__(self, invocation: Invocation) -> Decision: + if invocation.operation_name != "agent_reply": + return Decision() + runtime = invocation.data.get("runtime", {}) + uploaded = list(runtime.get("uploaded_files") or []) + historical = list(runtime.get("historical_files") or []) + if self.injected or not (uploaded or historical): + return Decision() + self.injected = True + user_id = runtime.get("user_id") or None + items = list(invocation.data.get("context_items") or []) + if historical: + text = file_context.build_historical_files_context(historical) + if text: + items.append(payload(text, kind="project_material", origin="workspace:historical_files", trust="workspace")) + text = await asyncio.to_thread(file_context.build_file_context, uploaded, user_id=user_id) if uploaded else "" + if text: + items.append(payload(text, kind="attachment", origin="user:uploaded_files", trust="user")) + images = [file for file in uploaded if artifact_bytes.is_image(file)] + if images: + if supports_native_images(str(runtime.get("model_provider_id") or "")): + content = await asyncio.to_thread(native_image_content, images, user_id) + if content: + items.append(payload(content, kind="attachment", origin="user:uploaded_images", trust="user", token_budget=16000)) + else: + text = await vision_evidence(images, user_id, str(runtime.get("vision_evidence_text") or "")) + if text: + items.append(payload(text, kind="attachment", origin="vision:transcription", trust="tool")) + return Decision.modify({"context_items": items}) + + +def attachment_context_hook() -> HookSpec: + return HookSpec("attachment_context", HookStage.TRANSFORM_CONTEXT, AttachmentContextHook(), + priority=100, timeout_ms=180000, failure_mode=FailureMode.OPEN, + mutable_fields=frozenset({"context_items"})) diff --git a/src/backend/core/services/compaction_budget.py b/src/backend/core/llm/context/budget.py similarity index 98% rename from src/backend/core/services/compaction_budget.py rename to src/backend/core/llm/context/budget.py index 9fb16d3e..4ddf2687 100644 --- a/src/backend/core/services/compaction_budget.py +++ b/src/backend/core/llm/context/budget.py @@ -5,7 +5,7 @@ import json from typing import Any, Dict, List, Optional -from core.llm import compaction as C +from core.llm.context import compaction as C def estimate_history_tokens(history: List[Dict[str, Any]]) -> int: diff --git a/src/backend/core/llm/compacting_agent.py b/src/backend/core/llm/context/compacting_agent.py similarity index 96% rename from src/backend/core/llm/compacting_agent.py rename to src/backend/core/llm/context/compacting_agent.py index 0822defb..679e2748 100644 --- a/src/backend/core/llm/compacting_agent.py +++ b/src/backend/core/llm/context/compacting_agent.py @@ -15,9 +15,9 @@ from agentscope.message import DataBlock, TextBlock, ToolResultBlock from agentscope.model import ChatUsage from agentscope.workspace import Offloader -from core.llm.context_safety import enforce_context_budget -from core.llm.execution_manifest import stable_hash -from core.llm.manifest_agent import ManifestBoundAgent +from core.llm.context.safety import enforce_context_budget +from core.harness.execution_manifest import stable_hash +from core.llm.runtime.manifest_agent import ManifestBoundAgent logger = logging.getLogger(__name__) @@ -247,7 +247,7 @@ async def _measure_context_tokens(self) -> int: trailing = None self._jx_observation = None if trailing is not None: - from core.services.compaction_budget import estimate_history_tokens + from core.llm.context.budget import estimate_history_tokens return obs.tokens + (estimate_history_tokens(trailing) if trailing else 0) return self._estimate_prompt_tokens() @@ -260,7 +260,7 @@ def _estimate_prompt_tokens(self) -> int: units as the observed one above — both reserve media instead of billing its transport bytes. """ - from core.services.compaction_budget import estimate_context_budget + from core.llm.context.budget import estimate_context_budget system_prompt, tool_schema = frozen_prompt_surface(self) return estimate_context_budget( @@ -271,7 +271,7 @@ def _estimate_prompt_tokens(self) -> int: # ── MidTurn compaction ─────────────────────────────────────────────────── - async def compress_context(self, context_config: ContextConfig | None = None) -> None: + async def compress_context(self, context_config: ContextConfig | None = None, instructions=None) -> None: """Compact at a ReAct step boundary without replacing failed work with a summary.""" del context_config await enforce_context_budget(self) @@ -317,8 +317,8 @@ async def compress_context(self, context_config: ContextConfig | None = None) -> history = self._history_for_summary() try: - from core.llm.compaction import CompactionPhase - from core.services.compaction_service import run_compaction, run_mid_turn_compaction + from core.llm.context.compaction import CompactionPhase + from core.harness.registry.compaction import run_compaction, run_mid_turn_compaction if chat_id: system_prompt, tool_schema = frozen_prompt_surface(self) @@ -365,7 +365,7 @@ async def _context_offload_notice(self, dropped: list[Any]) -> str: async def _should_compact(self) -> tuple[Optional[int], int]: """Return ``(limit, measured_tokens)``; ``limit`` is None when undeterminable.""" - from core.services.compaction_service import resolve_token_limit + from core.harness.registry.compaction import resolve_token_limit window = int(getattr(self.model, "context_size", 0) or 0) limit = resolve_token_limit(window, ratio=self._jx_trigger_ratio) @@ -389,7 +389,7 @@ async def _apply_replacement(self, replacement: List[Dict[str, Any]]) -> None: session that owns that path need not outlive this turn, and a checkpoint that points at a vanished file is worse than one that never mentions it. """ - from core.llm.message_compat import session_to_msgs + from core.llm.runtime.message_compat import session_to_msgs dropped = list(self.state.context) replacement = [dict(m) for m in replacement] diff --git a/src/backend/core/llm/compaction.py b/src/backend/core/llm/context/compaction.py similarity index 98% rename from src/backend/core/llm/compaction.py rename to src/backend/core/llm/context/compaction.py index 8f3dd6c5..7fb835b2 100644 --- a/src/backend/core/llm/compaction.py +++ b/src/backend/core/llm/context/compaction.py @@ -6,13 +6,10 @@ from enum import Enum from typing import Any, Dict, List, Optional, Tuple -from core.llm.context_ir import ( - KIND_COMPACTION, - SESSION_CONTEXT_META_KEY, - make_text_context_item, - image_token_reserve, -) -from core.llm.model_steps import explode_history_rows +from core.llm.context.ir_constants import KIND_COMPACTION, SESSION_CONTEXT_META_KEY +from core.llm.context.items import make_text_context_item +from core.llm.context.content import image_token_reserve +from core.llm.runtime.model_steps import explode_history_rows # ── Constants ──────────────────────────────────────────────────────────────── diff --git a/src/backend/core/llm/context/content.py b/src/backend/core/llm/context/content.py new file mode 100644 index 00000000..e3ea04a3 --- /dev/null +++ b/src/backend/core/llm/context/content.py @@ -0,0 +1,212 @@ +"""Normalise, measure and truncate context content.""" +from __future__ import annotations +from core.models.constants import IMAGE_TOKEN_RESERVE +from typing import Any, Mapping +from core.infra.immutable import thaw_json +from core.harness.execution_manifest import canonical_json +from core.llm.context import ir_constants + +def normalize_context_content(content: Any) -> Any: + """Convert SDK block objects into stable, framework-neutral JSON values.""" + if hasattr(content, "model_dump"): + return normalize_context_content(content.model_dump(mode="json")) + if isinstance(content, Mapping): + return { + str(key): normalize_context_content(value) + for key, value in content.items() + } + if isinstance(content, (list, tuple)): + return [normalize_context_content(value) for value in content] + return content + + +def tool_result_images(content: Any) -> list[Any]: + """Image payloads are atomic media, not part of a tool's text allowance.""" + if not isinstance(content, Mapping) or content.get("type") != "tool_result": + return [] + output = content.get("output") + if not isinstance(output, (list, tuple)): + return [] + return [ + block + for block in output + if isinstance(block, Mapping) + and block.get("type") == "data" + and isinstance(block.get("source"), Mapping) + and str(block["source"].get("media_type") or "").startswith("image/") + ] + + +def image_token_reserve(content: Any) -> int: + """Reserve media occupancy without measuring image transport bytes.""" + if isinstance(content, (list, tuple)): + return sum(image_token_reserve(block) for block in content) + if isinstance(content, Mapping): + source = content.get("source") + if ( + content.get("type") == "data" + and isinstance(source, Mapping) + and str(source.get("media_type") or "").startswith("image/") + ): + return IMAGE_TOKEN_RESERVE + if content.get("type") == "tool_result": + return image_token_reserve(content.get("output", content.get("content"))) + return 0 + + +def estimate_context_tokens(content: Any) -> int: + """Deterministic conservative estimate without provider-specific imports.""" + content = normalize_context_content(content) + if content is None: + return 0 + if isinstance(content, str): + return max(1, (len(content.encode("utf-8")) + 3) // 4) if content else 0 + if isinstance(content, Mapping) and content.get("type") == "data": + # Provider image accounting is not proportional to base64 bytes. Keep a + # conservative fixed reserve without letting transport encoding evict + # the actual attachment before the provider can count it precisely. + return IMAGE_TOKEN_RESERVE + if tool_result_images(content): + # Count image reserves recursively instead of tokenizing their base64 + # inside the enclosing tool-result JSON. + return estimate_context_tokens({**content, "output": []}) + sum( + estimate_context_tokens(block) for block in content["output"] + ) + if isinstance(content, list): + return sum(estimate_context_tokens(item) for item in content) + return max(1, (len(canonical_json(content).encode("utf-8")) + 3) // 4) + + +def truncate_text(text: str, max_tokens: int, *, tail_only: bool = False) -> str: + if not text or max_tokens <= 0: + return "" + max_bytes = max(4, max_tokens * 4) + if len(text.encode("utf-8")) <= max_bytes: + return text + # Character slicing is deterministic and safe for Unicode. Iterate down if + # multi-byte text still exceeds the byte reserve. + max_chars = max(1, max_bytes) + head = 0 + tail = 0 + marker = "" + if tail_only: + tail = max_chars + result = text[-tail:] + elif max_chars < 40: + head = max_chars + result = text[:head] + else: + marker = "\n[… omitted …]\n" + body = max(2, max_chars - len(marker)) + head = max(1, int(body * 0.6)) + tail = max(1, body - head) + result = text[:head] + marker + text[-tail:] + while len(result.encode("utf-8")) > max_bytes and len(result) > 1: + if tail_only: + tail = max(1, tail - 1) + result = text[-tail:] + elif marker: + # Preserve both semantic ends; shrink the larger slice first. + if head >= tail and head > 1: + head -= 1 + elif tail > 1: + tail -= 1 + elif head > 1: + head -= 1 + else: + break + result = text[:head] + marker + text[-tail:] + else: + head = max(1, head - 1) + result = text[:head] + return result + + +def truncate_content(content: Any, max_tokens: int, policy: str) -> Any: + """Shorten prunable text only; every structured block keeps its shape. + + Plain text and a tool result's output are the only things a budget may + shorten. Any other block — thinking, tool call, data, or a shape this + module does not know — is returned untouched so the caller keeps or drops + it whole. Rewriting such a block into a JSON string would hand the model + its own reasoning back as an answer. + """ + if max_tokens <= 0: + return "" + if isinstance(content, str): + return truncate_text(content, max_tokens, tail_only=policy == ir_constants.POLICY_TAIL) + if isinstance(content, Mapping): + mutable = thaw_json(content) + block_type = str(mutable.get("type") or "") + if block_type not in ir_constants.TRUNCATABLE_BLOCK_TYPES: + return mutable + for key in ("output", "content", "text"): + value = mutable.get(key) + if isinstance(value, str): + overhead = estimate_context_tokens({**mutable, key: ""}) + mutable[key] = truncate_text( + value, + max(1, max_tokens - overhead), + tail_only=policy == ir_constants.POLICY_TAIL, + ) + return mutable + if key == "output" and isinstance(value, (list, Mapping)): + images = tool_result_images(mutable) + if images: + text = "\n".join( + ( + str(block["text"]) + if isinstance(block, Mapping) and block.get("text") is not None + else block if isinstance(block, str) else canonical_json(block) + ) + for block in value + if block not in images + ) + output = [{"type": "text", "text": ""}, *images] + overhead = estimate_context_tokens({**mutable, key: output}) + output[0]["text"] = truncate_text( + text, max(1, max_tokens - overhead), tail_only=policy == ir_constants.POLICY_TAIL + ) + mutable[key] = output + return mutable + if isinstance(value, list): + parts = [] + for block in value: + if isinstance(block, Mapping) and block.get("text") is not None: + parts.append(str(block["text"])) + elif isinstance(block, str): + parts.append(block) + else: + parts.append(canonical_json(block)) + flattened = "\n".join(parts) + else: + flattened = canonical_json(value) + overhead = estimate_context_tokens({**mutable, key: ""}) + mutable[key] = truncate_text( + flattened, + max(1, max_tokens - overhead), + tail_only=policy == ir_constants.POLICY_TAIL, + ) + return mutable + return mutable + return thaw_json(content) + + +def prune_tool_output(content: Any, original_tokens: int) -> Any: + """Replace a tool result's output with an explicit placeholder.""" + mutable = thaw_json(content) + if not isinstance(mutable, dict) or str(mutable.get("type") or "") != "tool_result": + raise ValueError("only tool_result content can be pruned") + images = tool_result_images(mutable) + if images: + mutable["output"] = [ + {"type": "text", "text": ir_constants.PRUNED_OUTPUT_TEMPLATE.format(tokens=original_tokens)}, + *images, + ] + return mutable + for key in ("output", "content", "text"): + if key in mutable: + mutable[key] = ir_constants.PRUNED_OUTPUT_TEMPLATE.format(tokens=original_tokens) + return mutable + mutable["output"] = ir_constants.PRUNED_OUTPUT_TEMPLATE.format(tokens=original_tokens) + return mutable diff --git a/src/backend/core/llm/context/ir.py b/src/backend/core/llm/context/ir.py new file mode 100644 index 00000000..bb1f8a84 --- /dev/null +++ b/src/backend/core/llm/context/ir.py @@ -0,0 +1,327 @@ +"""Assemble context items into the window sent to the model.""" +from __future__ import annotations +from dataclasses import replace +from typing import Any, Iterable, Mapping, Optional, Sequence +from core.harness.execution_manifest import stable_hash +from core.llm.context import content +from core.llm.context import ir_constants +from core.llm.context.items import ContextAssembly, ContextItem + +class ContextAssembler: + """Explicit, unit-level selection in context order. + + The unit of selection is a whole ReAct step — the reasoning, text and tool + calls the model produced together with the results those calls returned — + never a single block. A step is kept or dropped as one; the only thing a + budget may shorten inside it is a tool result's output. + + Ordering is **not** the assembler's job: items arrive in the order the agent + context already holds them, and they leave in that same order. The live + user instruction carries a sequence 64 strides above the context tail (see + ``next_request_sequence``), so sorting by sequence would move it behind the + turn it had just started. + + Budget pressure is resolved in a fixed order: per-item caps first, then + tool outputs of older steps are pruned (oldest first), and only then are + whole steps cut from the oldest end of the conversation. The steps of the + turn in progress — everything after the current user instruction — are + protected: they are never pruned, cut or split. When they alone exceed the + budget the assembly reports ``over_budget`` instead of quietly removing + anything; compaction owns that case. + """ + + def __init__( + self, + *, + total_budget: int, + budget_details: Optional[Mapping[str, int]] = None, + ) -> None: + self.total_budget = max(0, int(total_budget)) + self.budget_details = { + str(key): max(0, int(value)) + for key, value in sorted((budget_details or {}).items()) + } + + @staticmethod + def _mandatory(item: ContextItem) -> bool: + if item.visibility == ir_constants.VISIBILITY_MANIFEST_ONLY: + return True + if item.kind in ir_constants.STEP_KINDS: + return False + return item.truncation_policy == ir_constants.POLICY_NEVER + + @staticmethod + def _unit_key(items: Sequence[ContextItem]) -> tuple[Any, ...]: + return ( + -max(item.priority for item in items), + -max(item.created_seq for item in items), + tuple(sorted(item.item_id for item in items)), + ) + + @staticmethod + def _tokens(item: ContextItem) -> int: + return 0 if item.visibility == ir_constants.VISIBILITY_MANIFEST_ONLY else item.token_estimate + + def _cap_item( + self, + item: ContextItem, + records: dict[str, dict[str, Any]], + ) -> Optional[ContextItem]: + original = item.token_estimate + # Media still contributes to whole-context occupancy, but cannot spend + # or be cut by the per-item text allowance. + budget = item.token_budget + sum( + content.estimate_context_tokens(block) for block in content.tool_result_images(item.content) + ) + if ( + item.visibility == ir_constants.VISIBILITY_MANIFEST_ONLY + or item.truncation_policy == ir_constants.POLICY_NEVER + or original <= budget + ): + records[item.item_id] = { + "action": "included", + "original_tokens": original, + "final_tokens": original, + } + return item + if item.truncation_policy in {ir_constants.POLICY_HEAD_TAIL, ir_constants.POLICY_TAIL} and item.token_budget > 0: + target = budget + capped = item.with_content( + content.truncate_content(item.content, target, item.truncation_policy) + ) + # JSON escaping of newlines/quotes can exceed the initial text + # allowance. Refit using the measured cost, never discard an image + # result just because its accompanying text needs a smaller slice. + while capped.token_estimate > budget and target > 1: + target = max(1, target - (capped.token_estimate - budget)) + capped = item.with_content( + content.truncate_content(item.content, target, item.truncation_policy) + ) + if capped.token_estimate <= budget: + records[item.item_id] = { + "action": "truncated", + "original_tokens": original, + "final_tokens": capped.token_estimate, + } + return capped + records[item.item_id] = { + "action": "excluded", + "reason": "item_budget", + "original_tokens": original, + "final_tokens": 0, + } + return None + + @staticmethod + def _tool_pairs_balanced(unit: Sequence[ContextItem]) -> bool: + pairs: dict[str, list[ContextItem]] = {} + for item in unit: + if item.kind in ir_constants.TOOL_KINDS: + if not item.pair_id: + return False + pairs.setdefault(item.pair_id, []).append(item) + for pair_items in pairs.values(): + calls = [item for item in pair_items if item.kind == ir_constants.KIND_TOOL_CALL] + results = [item for item in pair_items if item.kind == ir_constants.KIND_TOOL_RESULT] + if not calls or len(calls) != len(results): + return False + return True + + def assemble(self, items: Iterable[ContextItem]) -> ContextAssembly: + raw_items = tuple(items) + item_ids = [item.item_id for item in raw_items] + duplicates = sorted( + item_id for item_id in set(item_ids) if item_ids.count(item_id) > 1 + ) + if duplicates: + raise ValueError(f"context item_id values must be unique: {duplicates}") + original_items = list(raw_items) + arrival = {item.item_id: index for index, item in enumerate(original_items)} + records: dict[str, dict[str, Any]] = {} + excluded: list[ContextItem] = [] + + capped: list[ContextItem] = [] + for item in original_items: + candidate = self._cap_item(item, records) + if candidate is None: + excluded.append(item) + continue + capped.append(candidate) + + # A tool result belongs to the step that issued its call, whatever unit + # the caller stamped on it: the pair id is the structural link. + call_units: dict[str, str] = {} + ambiguous_pairs: set[str] = set() + for item in capped: + if item.kind == ir_constants.KIND_TOOL_CALL and item.pair_id: + # One pair id may name a provider parallel batch (several calls + # in one unit); the same id across two units is unresolvable. + if call_units.get(item.pair_id, item.unit_id) != item.unit_id: + ambiguous_pairs.add(item.pair_id) + call_units[item.pair_id] = item.unit_id + units: dict[str, list[ContextItem]] = {} + for candidate in capped: + if candidate.kind in ir_constants.TOOL_KINDS and candidate.pair_id in ambiguous_pairs: + records[candidate.item_id].update( + action="excluded", reason="malformed_tool_pair", final_tokens=0 + ) + excluded.append(candidate) + continue + if candidate.kind == ir_constants.KIND_TOOL_RESULT and candidate.pair_id in call_units: + candidate = replace(candidate, unit_id=call_units[candidate.pair_id]) + units.setdefault(candidate.unit_id, []).append(candidate) + + # The request that opened the turn in progress. Everything after it is + # the model's own live work and is protected as a whole. + request_index = -1 + for item in original_items: + if ( + item.kind == ir_constants.KIND_USER_INPUT + and item.truncation_policy == ir_constants.POLICY_NEVER + and item.visibility == ir_constants.VISIBILITY_MODEL + ): + request_index = arrival[item.item_id] + + def unit_start(unit: Sequence[ContextItem]) -> int: + return min(arrival[item.item_id] for item in unit) + + mandatory: list[list[ContextItem]] = [] + protected: list[list[ContextItem]] = [] + aside: list[list[ContextItem]] = [] + conversation: list[list[ContextItem]] = [] + for unit in units.values(): + is_step = any(item.kind in ir_constants.STEP_KINDS for item in unit) + if any(self._mandatory(item) for item in unit): + mandatory.append(unit) + elif is_step and request_index >= 0 and unit_start(unit) > request_index: + protected.append(unit) + elif not self._tool_pairs_balanced(unit): + for item in unit: + records[item.item_id].update( + action="excluded", reason="malformed_tool_pair", final_tokens=0 + ) + excluded.append(item) + elif all(item.kind in ir_constants.CONVERSATION_KINDS for item in unit): + conversation.append(unit) + else: + aside.append(unit) + + mandatory.sort(key=unit_start) + protected.sort(key=unit_start) + conversation.sort(key=unit_start) + # Selection order only for non-conversation material; it never reaches + # the output, which is re-sorted into context order. + aside.sort(key=self._unit_key) + + def unit_tokens(unit: Sequence[ContextItem]) -> int: + return sum(self._tokens(item) for item in unit) + + included: list[ContextItem] = [] + used = 0 + for unit in mandatory + protected: + included.extend(unit) + used += unit_tokens(unit) + over_budget = used > self.total_budget + + for unit in aside: + tokens = unit_tokens(unit) + remaining = max(0, self.total_budget - used) + if tokens <= remaining: + included.extend(unit) + used += tokens + continue + item = unit[0] + if ( + len(unit) == 1 + and remaining > 0 + and item.truncation_policy in {ir_constants.POLICY_HEAD_TAIL, ir_constants.POLICY_TAIL} + ): + shrunk = item.with_content( + content.truncate_content(item.content, remaining, item.truncation_policy) + ) + if self._tokens(shrunk) < self._tokens(item) and self._tokens(shrunk) <= remaining: + records[item.item_id].update( + action="truncated", final_tokens=shrunk.token_estimate + ) + included.append(shrunk) + used += self._tokens(shrunk) + continue + for item in unit: + records[item.item_id].update(action="excluded", reason="budget", final_tokens=0) + excluded.append(item) + + # Conversation history: prune old tool outputs first, then cut whole + # steps from the oldest end. Never a hole in the middle. + pruned_items = 0 + conversation_tokens = sum(unit_tokens(unit) for unit in conversation) + if used + conversation_tokens > self.total_budget: + for unit in conversation: + if used + conversation_tokens <= self.total_budget: + break + for index, item in enumerate(unit): + if used + conversation_tokens <= self.total_budget: + break + if item.kind != ir_constants.KIND_TOOL_RESULT or item.truncation_policy == ir_constants.POLICY_NEVER: + continue + pruned = item.with_content(content.prune_tool_output(item.content, item.token_estimate)) + if self._tokens(pruned) >= self._tokens(item): + continue + conversation_tokens -= self._tokens(item) - self._tokens(pruned) + unit[index] = pruned + records[item.item_id].update(action="pruned", final_tokens=pruned.token_estimate) + pruned_items += 1 + cut_units = 0 + while conversation and used + conversation_tokens > self.total_budget: + dropped = conversation.pop(0) + conversation_tokens -= unit_tokens(dropped) + cut_units += 1 + for item in dropped: + records[item.item_id].update(action="excluded", reason="budget_cut", final_tokens=0) + excluded.append(item) + for unit in conversation: + included.extend(unit) + used += unit_tokens(unit) + + included.sort(key=lambda item: arrival[item.item_id]) + excluded_by_id = {item.item_id: item for item in excluded} + excluded = sorted(excluded_by_id.values(), key=lambda item: arrival[item.item_id]) + + included_manifest = [] + excluded_manifest = [] + included_ids = {item.item_id for item in included} + final_by_id = {item.item_id: item for item in included} + for original in original_items: + record = dict(records.get(original.item_id) or {}) + final = final_by_id.get(original.item_id, original) + entry = final.to_manifest() + entry.update(record) + if original.item_id in included_ids: + included_manifest.append(entry) + else: + excluded_manifest.append(entry) + + payload = { + "schema_version": ir_constants.CONTEXT_SCHEMA_VERSION, + "total_budget": self.total_budget, + "used_tokens": used, + "over_budget": over_budget, + "protected_units": len(protected), + "cut_units": cut_units, + "pruned_items": pruned_items, + "included": included_manifest, + "excluded": excluded_manifest, + } + if self.budget_details: + payload["budget_details"] = dict(self.budget_details) + manifest_hash = stable_hash(payload) + return ContextAssembly( + included=tuple(included), + excluded=tuple(excluded), + _manifest=payload, + manifest_hash=manifest_hash, + used_tokens=used, + total_budget=self.total_budget, + over_budget=over_budget, + cut_units=cut_units, + pruned_items=pruned_items, + ) diff --git a/src/backend/core/llm/context/ir_constants.py b/src/backend/core/llm/context/ir_constants.py new file mode 100644 index 00000000..931faf2b --- /dev/null +++ b/src/backend/core/llm/context/ir_constants.py @@ -0,0 +1,86 @@ +"""Context item kinds, policies and metadata keys.""" +from __future__ import annotations + + +CONTEXT_SCHEMA_VERSION = "harness.context.v2" + + +SESSION_CONTEXT_META_KEY = "_context_item" + + +CONTEXT_SEQUENCE_STRIDE = 1_000 + + +KIND_SYSTEM_RULE = "system_rule" + + +KIND_USER_INPUT = "user_input" + + +KIND_ASSISTANT = "assistant_history" + + +KIND_MEMORY = "memory" + + +KIND_IDENTITY = "identity" + + +KIND_PROJECT = "project_material" + + +KIND_THINKING = "thinking" + + +KIND_TOOL_CALL = "tool_call" + + +KIND_TOOL_RESULT = "tool_result" + + +KIND_REMINDER = "reminder" + + +KIND_COMPACTION = "compaction_summary" + + +KIND_ATTACHMENT = "attachment" + + +KIND_STEER = "steer" + + +KIND_REFERENCE = "reference" + + +POLICY_NEVER = "never" + + +POLICY_DROP = "drop" + + +POLICY_HEAD_TAIL = "head_tail" + + +POLICY_TAIL = "tail" + + +VISIBILITY_MODEL = "model" + + +VISIBILITY_MANIFEST_ONLY = "manifest_only" + + +TOOL_KINDS = {KIND_TOOL_CALL, KIND_TOOL_RESULT} + + +STEP_KINDS = frozenset({KIND_ASSISTANT, KIND_THINKING, KIND_TOOL_CALL, KIND_TOOL_RESULT}) + + +CONVERSATION_KINDS = STEP_KINDS | {KIND_USER_INPUT} + + +TRUNCATABLE_BLOCK_TYPES = {"tool_result"} + + +PRUNED_OUTPUT_TEMPLATE = "[tool output pruned to fit the context budget: {tokens} tokens omitted]" diff --git a/src/backend/core/llm/context/items.py b/src/backend/core/llm/context/items.py new file mode 100644 index 00000000..22618c10 --- /dev/null +++ b/src/backend/core/llm/context/items.py @@ -0,0 +1,215 @@ +"""Context items and their provenance metadata.""" +from __future__ import annotations +from dataclasses import dataclass, field, replace +from typing import Any, Mapping, Optional +from core.infra.immutable import FrozenDict, freeze_json, thaw_json +from core.harness.execution_manifest import stable_hash +from core.llm.context.content import estimate_context_tokens, normalize_context_content +from core.llm.context import ir_constants + +@dataclass(frozen=True) +class ContextItem: + item_id: str + kind: str + origin: str + trust: str + visibility: str + priority: int + token_budget: int + truncation_policy: str + content_ref: str + content_hash: str + cache_class: str + created_seq: int + token_estimate: int + render_role: str = "user" + render_name: str = "" + pair_id: str = "" + message_group: str = "" + unit_id: str = "" + content: Any = field(default=None, repr=False, compare=False) + metadata: Mapping[str, Any] = field( + default_factory=FrozenDict, repr=False, compare=False + ) + + def __post_init__(self) -> None: + object.__setattr__(self, "content", freeze_json(self.content)) + object.__setattr__(self, "metadata", freeze_json(self.metadata or {})) + + @classmethod + def create( + cls, + *, + item_id: str, + kind: str, + origin: str, + trust: str, + visibility: str, + priority: int, + token_budget: int, + truncation_policy: str, + content: Any, + cache_class: str, + created_seq: int, + render_role: str = "user", + pair_id: str = "", + message_group: str = "", + unit_id: str = "", + content_ref: Optional[str] = None, + content_hash: Optional[str] = None, + token_estimate: Optional[int] = None, + render_name: str = "", + metadata: Optional[Mapping[str, Any]] = None, + ) -> "ContextItem": + normalized_content = normalize_context_content(content) + if ( + isinstance(normalized_content, Mapping) + and str(normalized_content.get("type") or "") == "data" + ): + truncation_policy = ir_constants.POLICY_DROP + computed_hash = stable_hash(normalized_content) + supplied_hash = str(content_hash or "") + external_reference = normalized_content is None and bool(supplied_hash) + digest = supplied_hash if external_reference else computed_hash + estimate = ( + max(0, int(token_estimate or 0)) + if external_reference + else estimate_context_tokens(normalized_content) + ) + reference = ( + str(content_ref) + if content_ref is not None + and (external_reference or supplied_hash == computed_hash) + else f"sha256:{digest}" + ) + return cls( + item_id=str(item_id), + kind=str(kind), + origin=str(origin), + trust=str(trust), + visibility=str(visibility), + priority=int(priority), + token_budget=max(0, int(token_budget)), + truncation_policy=str(truncation_policy), + content_ref=reference, + content_hash=digest, + cache_class=str(cache_class), + created_seq=int(created_seq), + token_estimate=estimate, + render_role=str(render_role), + render_name=str(render_name or ""), + pair_id=str(pair_id or ""), + message_group=str(message_group or f"item:{item_id}"), + unit_id=str(unit_id or f"item:{item_id}"), + content=normalized_content, + metadata=metadata or {}, + ) + + def with_content(self, content: Any) -> "ContextItem": + normalized_content = normalize_context_content(content) + digest = stable_hash(normalized_content) + return replace( + self, + content=freeze_json(normalized_content), + content_hash=digest, + content_ref=f"sha256:{digest}", + token_estimate=estimate_context_tokens(normalized_content), + ) + + def to_manifest(self) -> dict[str, Any]: + return { + "item_id": self.item_id, + "kind": self.kind, + "origin": self.origin, + "trust": self.trust, + "visibility": self.visibility, + "priority": self.priority, + "token_budget": self.token_budget, + "truncation_policy": self.truncation_policy, + "content_ref": self.content_ref, + "content_hash": self.content_hash, + "cache_class": self.cache_class, + "created_seq": self.created_seq, + "token_estimate": self.token_estimate, + "render_role": self.render_role, + "render_name_hash": stable_hash(self.render_name), + "pair_id": self.pair_id or None, + "message_group": self.message_group, + "unit_id": self.unit_id, + } + + +def make_text_context_item( + text: str, + *, + item_id: str, + kind: str, + origin: str, + trust: str, + created_seq: int, + priority: int = 700, + token_budget: int = 4_000, + truncation_policy: str = ir_constants.POLICY_HEAD_TAIL, + render_role: str = "user", + cache_class: str = "dynamic", +) -> ContextItem: + """Create a framework-neutral, explicitly-provenanced text item.""" + return ContextItem.create( + item_id=item_id, + kind=kind, + origin=origin, + trust=trust, + visibility=ir_constants.VISIBILITY_MODEL, + priority=priority, + token_budget=token_budget, + truncation_policy=truncation_policy, + content=str(text or ""), + cache_class=cache_class, + created_seq=created_seq, + render_role=render_role, + render_name=render_role, + message_group=item_id, + ) + + +def session_context_metadata(item: ContextItem) -> dict[str, Any]: + """Carry provenance on a positional session row without stale identity/sequence.""" + payload = item.to_manifest() + for key in ( + "item_id", + "created_seq", + "message_group", + "unit_id", + "content_ref", + "content_hash", + "token_estimate", + "render_name_hash", + ): + payload.pop(key, None) + return payload + + +@dataclass(frozen=True) +class ContextAssembly: + included: tuple[ContextItem, ...] + excluded: tuple[ContextItem, ...] + _manifest: Mapping[str, Any] = field(repr=False) + manifest_hash: str + used_tokens: int + total_budget: int + over_budget: bool = False + cut_units: int = 0 + pruned_items: int = 0 + + def __post_init__(self) -> None: + object.__setattr__(self, "included", tuple(self.included)) + object.__setattr__(self, "excluded", tuple(self.excluded)) + object.__setattr__(self, "_manifest", freeze_json(self._manifest)) + + @property + def manifest(self) -> dict[str, Any]: + """Return a detached JSON-serializable copy of the audit manifest.""" + manifest = thaw_json(self._manifest) + if not isinstance(manifest, dict): # pragma: no cover - frozen in __post_init__ + raise TypeError("context manifest must thaw to a dictionary") + return manifest diff --git a/src/backend/core/llm/context/message_defaults.py b/src/backend/core/llm/context/message_defaults.py new file mode 100644 index 00000000..ef5ce116 --- /dev/null +++ b/src/backend/core/llm/context/message_defaults.py @@ -0,0 +1,78 @@ +"""Default kind and truncation policy for each message role and block type.""" +from __future__ import annotations +from typing import Any +from core.llm.context.ir_constants import KIND_ASSISTANT, KIND_ATTACHMENT, KIND_SYSTEM_RULE, KIND_THINKING, KIND_TOOL_CALL, KIND_TOOL_RESULT, KIND_USER_INPUT, POLICY_DROP, POLICY_HEAD_TAIL, POLICY_NEVER + +def defaults(role: str, block_type: str) -> dict[str, Any]: + if block_type == "thinking": + # Reasoning is handed back to the model verbatim. It is counted on its + # own so the usage panel can show what it costs, but it is never + # shortened or dropped apart from the step that produced it. + return { + "kind": KIND_THINKING, + "origin": "assistant:reasoning", + "trust": "assistant", + "priority": 400, + "token_budget": 0, + "truncation_policy": POLICY_NEVER, + "render_role": "assistant", + } + if block_type == "tool_call": + return { + "kind": KIND_TOOL_CALL, + "origin": "agent:tool_call", + "trust": "assistant", + "priority": 700, + "token_budget": 0, + "truncation_policy": POLICY_NEVER, + "render_role": "assistant", + } + if block_type == "tool_result": + return { + "kind": KIND_TOOL_RESULT, + "origin": "tool:result", + "trust": "tool", + "priority": 700, + "token_budget": 20_000, + "truncation_policy": POLICY_HEAD_TAIL, + "render_role": "assistant", + } + if block_type == "data": + return { + "kind": KIND_ATTACHMENT, + "origin": "user:attachment", + "trust": "user", + "priority": 650, + "token_budget": 4_000, + "truncation_policy": POLICY_DROP, + "render_role": "user", + } + if role == "system": + return { + "kind": KIND_SYSTEM_RULE, + "origin": "platform:system_prompt", + "trust": "platform", + "priority": 1_000, + "token_budget": 100_000, + "truncation_policy": POLICY_NEVER, + "render_role": "system", + } + if role == "assistant": + return { + "kind": KIND_ASSISTANT, + "origin": "assistant:history", + "trust": "assistant", + "priority": 400, + "token_budget": 0, + "truncation_policy": POLICY_NEVER, + "render_role": "assistant", + } + return { + "kind": KIND_USER_INPUT, + "origin": "user:chat", + "trust": "user", + "priority": 500, + "token_budget": 100_000, + "truncation_policy": POLICY_HEAD_TAIL, + "render_role": "user", + } diff --git a/src/backend/core/llm/offloader.py b/src/backend/core/llm/context/offloader.py similarity index 96% rename from src/backend/core/llm/offloader.py rename to src/backend/core/llm/context/offloader.py index 75e5d1cb..886e086e 100644 --- a/src/backend/core/llm/offloader.py +++ b/src/backend/core/llm/context/offloader.py @@ -14,7 +14,7 @@ import uuid from typing import Any, Optional -from core.llm.message_compat import flatten_tool_output +from core.llm.runtime.message_compat import flatten_tool_output logger = logging.getLogger(__name__) _WRITE_TIMEOUT_SEC = 30 @@ -43,7 +43,7 @@ def __init__( self._user_id = user_id def _path(self, prefix: str) -> str: - from core.llm.tools._paths import workspace_directory + from core.sandbox.workspace.resolution import workspace_directory if not self._sess: raise OffloadError("A persistent tool workspace is required") diff --git a/src/backend/core/llm/context/provider_conversion.py b/src/backend/core/llm/context/provider_conversion.py new file mode 100644 index 00000000..901d19a6 --- /dev/null +++ b/src/backend/core/llm/context/provider_conversion.py @@ -0,0 +1,239 @@ +"""Carry context provenance through provider-formatted messages.""" +from __future__ import annotations +from core.models.constants import PROVIDER_CONTEXT_META_KEY +from typing import Any, Iterable, Mapping, Sequence +from agentscope.message import Msg +from core.infra.immutable import thaw_json +from core.harness.execution_manifest import stable_hash +from core.llm.context.ir_constants import CONTEXT_SEQUENCE_STRIDE, KIND_ASSISTANT, KIND_ATTACHMENT, KIND_SYSTEM_RULE, KIND_TOOL_CALL, KIND_TOOL_RESULT, KIND_USER_INPUT, POLICY_NEVER, VISIBILITY_MODEL +from core.llm.context.items import ContextItem +from core.llm.context.content import estimate_context_tokens + +def annotate_provider_messages( + self, + source_messages: Sequence[Msg], + formatted_messages: Sequence[Mapping[str, Any]], + row_counts: Sequence[int], +) -> list[dict[str, Any]]: + """Carry canonical provenance across an SDK formatter boundary. + + AgentScope's OpenAI formatter intentionally drops arbitrary message + metadata and may expand one tool-result message into a tool row plus a + synthetic media row. ``row_counts`` is obtained by formatting each + source message in isolation after a provider has rejected media. This + lets the retry inherit the original IR provenance without guessing it + from compatibility roles or XML-like text. + """ + annotated = [dict(message) for message in formatted_messages] + cursor = 0 + for source, count in zip(source_messages, row_counts): + entries = self._metadata_entries(source) + source_rows = annotated[cursor : cursor + max(0, int(count))] + cursor += max(0, int(count)) + if not entries: + continue + + def pick(kind: str) -> Mapping[str, Any]: + match = next((entry for entry in entries if entry.get("kind") == kind), None) + if match is not None: + return match + user = next( + (entry for entry in entries if entry.get("kind") == KIND_USER_INPUT), + None, + ) + return user or entries[0] + + for row in source_rows: + role = str(row.get("role") or "user") + name = str(row.get("name") or role) + if name == "system-reminder": + base = pick(KIND_TOOL_RESULT) + metadata = { + "kind": KIND_ATTACHMENT, + "origin": f"{base.get('origin') or 'tool:result'}:media", + "trust": str(base.get("trust") or "tool"), + "priority": int(base.get("priority") or 700), + } + elif role == "tool": + base = pick(KIND_TOOL_RESULT) + metadata = { + "kind": KIND_TOOL_RESULT, + "origin": str(base.get("origin") or "tool:result"), + "trust": str(base.get("trust") or "tool"), + "priority": int(base.get("priority") or 700), + } + elif role == "assistant" and row.get("tool_calls"): + base = pick(KIND_TOOL_CALL) + metadata = { + "kind": KIND_TOOL_CALL, + "origin": str(base.get("origin") or "agent:tool_call"), + "trust": str(base.get("trust") or "assistant"), + "priority": int(base.get("priority") or 700), + } + else: + base = pick("") + metadata = { + key: base[key] + for key in ("kind", "origin", "trust", "priority") + if base.get(key) is not None + } + row[PROVIDER_CONTEXT_META_KEY] = metadata + if cursor != len(annotated): + # A formatter shape change must fail visibly rather than silently + # assigning later rows the wrong trust boundary. + raise ValueError( + "provider formatter row mapping mismatch: " + f"mapped={cursor} formatted={len(annotated)}" + ) + return annotated + + +def items_from_provider_messages( + self, + messages: Sequence[Mapping[str, Any]], +) -> list[ContextItem]: + """Reify an already-formatted provider retry as exact canonical items. + + A provider may reject media after the normal AgentScope request has + already been assembled. The retry is therefore a new execution + surface. Each complete wire message is kept atomically so its hash + covers role, name, tool identifiers and content together; transient + provenance metadata is removed before hashing and before transport. + """ + items: list[ContextItem] = [] + pending_tool_pairs: dict[str, list[str]] = {} + orphan_results: dict[str, int] = {} + for index, raw_message in enumerate(messages): + message = dict(raw_message) + explicit = message.pop(PROVIDER_CONTEXT_META_KEY, None) + explicit = explicit if isinstance(explicit, Mapping) else {} + role = str(message.get("role") or "user") + name = str(message.get("name") or role) + tool_calls = message.get("tool_calls") + if role == "assistant" and isinstance(tool_calls, list) and tool_calls: + pair_id = f"provider-tool-batch:{index}" + message_hash = stable_hash(message) + message_tokens = estimate_context_tokens(message) + for call_index, call in enumerate(tool_calls): + call = call if isinstance(call, Mapping) else {} + raw_pair_id = str(call.get("id") or "") + pending_tool_pairs.setdefault(raw_pair_id, []).append(pair_id) + items.append( + ContextItem.create( + item_id=f"provider-retry:{index}:tool-call:{call_index}", + kind=str(explicit.get("kind") or KIND_TOOL_CALL), + origin=str(explicit.get("origin") or "assistant:provider_tool_call"), + trust=str(explicit.get("trust") or "assistant"), + visibility=VISIBILITY_MODEL, + priority=int(explicit.get("priority") or 900), + token_budget=message_tokens if call_index == 0 else 0, + truncation_policy=POLICY_NEVER, + content=message if call_index == 0 else None, + content_ref=f"provider-message:{index}", + content_hash=message_hash, + token_estimate=message_tokens if call_index == 0 else 0, + cache_class="dynamic", + created_seq=index * CONTEXT_SEQUENCE_STRIDE + call_index, + render_role=role, + render_name=name, + pair_id=pair_id, + message_group=f"provider-retry:{index}", + unit_id=pair_id, + ) + ) + continue + if role == "tool": + raw_pair_id = str(message.get("tool_call_id") or "") + pending = pending_tool_pairs.get(raw_pair_id) or [] + if pending: + pair_id = pending.pop(0) + else: + occurrence = orphan_results.get(raw_pair_id, 0) + 1 + orphan_results[raw_pair_id] = occurrence + pair_id = f"{raw_pair_id}#orphan-result-{occurrence}" + defaults = { + "kind": KIND_TOOL_RESULT, + "origin": "tool:provider_result", + "trust": "tool", + "priority": 900, + "pair_id": pair_id, + } + elif role == "system": + defaults = { + "kind": KIND_SYSTEM_RULE, + "origin": "platform:provider_request", + "trust": "platform", + "priority": 1_000, + } + elif role == "assistant": + defaults = { + "kind": KIND_ASSISTANT, + "origin": "assistant:provider_history", + "trust": "assistant", + "priority": 600, + } + else: + defaults = { + "kind": KIND_USER_INPUT, + "origin": "user:provider_request", + "trust": "user", + "priority": 900, + } + items.append( + ContextItem.create( + item_id=str(explicit.get("item_id") or f"provider-retry:{index}"), + kind=str(explicit.get("kind") or defaults["kind"]), + origin=str(explicit.get("origin") or defaults["origin"]), + trust=str(explicit.get("trust") or defaults["trust"]), + visibility=VISIBILITY_MODEL, + priority=int(explicit.get("priority") or defaults["priority"]), + # These rows have already survived the canonical budget + # pass. Media recovery only removes data or replaces it + # with a smaller textual description, so the retry must + # preserve every resulting row exactly. + token_budget=max(1, estimate_context_tokens(message)), + truncation_policy=POLICY_NEVER, + content=message, + cache_class="dynamic", + created_seq=int( + explicit.get("created_seq") + if explicit.get("created_seq") is not None + else index * CONTEXT_SEQUENCE_STRIDE + ), + render_role=role, + render_name=name, + pair_id=str(defaults.get("pair_id") or ""), + message_group=f"provider-retry:{index}", + unit_id=str(defaults.get("pair_id") or f"provider-retry:{index}"), + ) + ) + return items + + +def provider_messages_from_items(items: Iterable[ContextItem]) -> list[dict[str, Any]]: + """Render exact provider rows while stripping harness-only metadata.""" + rendered: list[dict[str, Any]] = [] + by_group: dict[str, dict[str, Any]] = {} + deferred_groups: set[str] = set() + for item in items: + if item.visibility != VISIBILITY_MODEL: + continue + value = thaw_json(item.content) + if value is None: + deferred_groups.add(item.message_group) + continue + if not isinstance(value, Mapping): + raise ValueError("provider retry context item must contain one message mapping") + row = dict(value) + row.pop(PROVIDER_CONTEXT_META_KEY, None) + previous = by_group.get(item.message_group) + if previous is not None: + if previous != row: + raise ValueError("provider retry group contains conflicting wire messages") + continue + by_group[item.message_group] = row + rendered.append(row) + missing = deferred_groups - set(by_group) + if missing: + raise ValueError(f"provider retry groups lack a wire row: {sorted(missing)}") + return rendered diff --git a/src/backend/core/llm/context/reasoning_hooks.py b/src/backend/core/llm/context/reasoning_hooks.py new file mode 100644 index 00000000..68cbf7c6 --- /dev/null +++ b/src/backend/core/llm/context/reasoning_hooks.py @@ -0,0 +1,49 @@ +"""Reasoning policies consume plain snapshots and emit bounded context additions.""" +from core.harness.hooks import Decision, FailureMode, HookSpec, HookStage + + +def reminder(text, origin): + return dict(content=f"\n{text}\n", + kind="reminder", origin=origin, trust="system", + priority=850, token_budget=4000) + + +class IterationBudgetHook: + def __init__(self, threshold=2): + from core.config.settings import env_bool as _bool, env_value as _env + self.threshold = threshold + self.force_text = _bool(_env("CHAT_FINAL_ITER_FORCE_TEXT", "true")) + self.last_key = None + + def __call__(self, invocation): + if invocation.operation_name != "agent_reasoning": + return Decision() + runtime = invocation.data["runtime"] + maximum = int(runtime.get("max_iters") or 0) + current = int(runtime.get("cur_iter") or 0) + remaining = maximum - current + if maximum <= self.threshold + 1: + return Decision() + patch = {} + if remaining <= 1 and self.force_text and invocation.data.get("tool_choice") is None: + patch["tool_choice"] = {"mode": "none"} + key = (runtime.get("reply_id") or "", current) + if remaining <= self.threshold and key != self.last_key: + self.last_key = key + if remaining <= 1: + text = ("这是本次回复的最后一轮推理：不要再调用任何工具（再调用会被系统强制" + "中断，已完成的工作将无法交付）。请直接基于已获得的信息输出最终回复；" + "若任务未全部完成，如实汇报已产出的成果、当前进展和未完成的部分。") + else: + text = (f"注意：推理-工具调用轮次即将用尽，包含本轮在内最多还剩 {remaining} 轮。" + "请停止新的探索，不要再重试已反复失败的操作，立即整合已获得的信息进行" + "收尾；若任务无法在剩余轮次内全部完成，优先输出已有成果与进展说明。") + patch["context_items"] = [*invocation.data.get("context_items", ()), + reminder(text, "harness:iteration_budget")] + return Decision.modify(patch) if patch else Decision() + + +def iteration_budget_hook(): + return HookSpec("iteration_budget", HookStage.TRANSFORM_CONTEXT, IterationBudgetHook(), + priority=300, failure_mode=FailureMode.OPEN, + mutable_fields=frozenset({"context_items", "tool_choice"})) diff --git a/src/backend/core/llm/context/reference_conversion.py b/src/backend/core/llm/context/reference_conversion.py new file mode 100644 index 00000000..426832eb --- /dev/null +++ b/src/backend/core/llm/context/reference_conversion.py @@ -0,0 +1,79 @@ +"""Context items for the references listed in an execution manifest.""" +from __future__ import annotations +from typing import Any, Mapping +from core.llm.context.ir_constants import KIND_PROJECT, KIND_SYSTEM_RULE, POLICY_HEAD_TAIL, POLICY_NEVER, VISIBILITY_MANIFEST_ONLY, VISIBILITY_MODEL +from core.llm.context.items import ContextItem + +def reference_items_from_execution_manifest(self, manifest: Any) -> list[ContextItem]: + if hasattr(manifest, "to_dict"): + payload = manifest.to_dict() + elif isinstance(manifest, Mapping): + payload = dict(manifest) + else: + return [] + sections = (payload.get("prompt_manifest") or {}).get("sections") or [] + items: list[ContextItem] = [] + for index, section in enumerate(sections): + if not isinstance(section, Mapping): + continue + section_id = str(section.get("id") or "") + content_hash = str(section.get("content_hash") or "") + if not section_id or not content_hash: + continue + is_project = section_id == "runtime/project" + live_content = "" + content_reader = getattr(manifest, "prompt_section_content", None) + if callable(content_reader): + live_content = str(content_reader(index) or "") + if is_project and live_content: + items.append( + ContextItem.create( + item_id=f"prompt:{index}:{section_id}", + kind=KIND_PROJECT, + origin=str(section.get("origin") or "workspace:project"), + trust=str(section.get("trust") or "workspace"), + visibility=VISIBILITY_MODEL, + priority=int(section.get("priority") or 900), + token_budget=max( + 1, + int(section.get("budget") or section.get("token_estimate") or 1), + ), + truncation_policy=POLICY_HEAD_TAIL, + content=live_content, + content_ref=( + str(section["reference"]) if section.get("reference") else None + ), + content_hash=content_hash, + token_estimate=int(section.get("token_estimate") or 0), + cache_class=str(section.get("cache_class") or "workspace"), + created_seq=1, + render_role="system", + render_name="system", + message_group=f"prompt:{index}", + ) + ) + continue + items.append( + ContextItem.create( + item_id=f"prompt:{index}:{section_id}", + kind=KIND_PROJECT if is_project else KIND_SYSTEM_RULE, + origin=str(section.get("origin") or "prompt:unknown"), + trust=str(section.get("trust") or "platform"), + visibility=VISIBILITY_MANIFEST_ONLY, + priority=int(section.get("priority") or 0), + token_budget=int(section.get("budget") or 0), + truncation_policy=POLICY_NEVER, + content=None, + content_ref=str( + section.get("reference") + or f"prompt:{section_id}@{section.get('version') or '1'}" + ), + content_hash=content_hash, + token_estimate=int(section.get("token_estimate") or 0), + cache_class=str(section.get("cache_class") or "stable"), + created_seq=-10_000 + index, + render_role="system", + message_group=f"prompt:{index}", + ) + ) + return items diff --git a/src/backend/core/llm/context/rendering.py b/src/backend/core/llm/context/rendering.py new file mode 100644 index 00000000..9eff0361 --- /dev/null +++ b/src/backend/core/llm/context/rendering.py @@ -0,0 +1,114 @@ +"""Render context items into AgentScope state and allocate their sequence numbers.""" +from __future__ import annotations +from core.llm.context import adapter +from dataclasses import replace +from typing import Any, Mapping, Sequence +from agentscope.message import Msg, TextBlock +from core.llm.context.ir_constants import CONTEXT_SEQUENCE_STRIDE, POLICY_HEAD_TAIL +from core.llm.context.items import ContextItem, make_text_context_item + +def append_state_context_item(state: Any, item: ContextItem) -> Msg: + """Render and append one explicitly-provenanced item to AgentScope state.""" + messages = adapter.AgentScopeContextAdapter().messages_from_items([item]) + if len(messages) != 1: + raise ValueError("context item did not render to exactly one message") + message = messages[0] + state.context.append(message) + return message + + +def render_text_block(text: str) -> TextBlock: + """Construct an AgentScope text block inside the SDK adapter boundary.""" + return TextBlock(type="text", text=str(text or "")) + + +def next_context_sequence(messages: Sequence[Any]) -> int: + """Allocate one monotonic sequence across replay and live injections.""" + max_sequence = -CONTEXT_SEQUENCE_STRIDE + for message_index, message in enumerate(messages): + entries = adapter.AgentScopeContextAdapter._metadata_entries(message) + blocks = list(getattr(message, "content", None) or []) + for block_index, _block in enumerate(blocks or [None]): + explicit = entries[block_index] if block_index < len(entries) else {} + candidate = explicit.get("created_seq") + if candidate is None: + candidate = message_index * CONTEXT_SEQUENCE_STRIDE + block_index + try: + max_sequence = max(max_sequence, int(candidate)) + except (TypeError, ValueError): + continue + return ((max_sequence // CONTEXT_SEQUENCE_STRIDE) + 1) * CONTEXT_SEQUENCE_STRIDE + + +def next_request_sequence(messages: Sequence[Any]) -> int: + """Reserve a pre-reply lane above the context tail for the user instruction. + + AgentScope runs ``on_reply`` middleware before it appends ``inputs`` to + state. File/project attachment middleware can therefore add a few explicit + context items first without colliding with the request's sequence. The + sequence no longer decides message order — the assembler emits items in + context order — it only keeps identifiers distinct and marks recency for + budget eviction. + """ + pre_reply_slots = 64 + return next_context_sequence(messages) + pre_reply_slots * CONTEXT_SEQUENCE_STRIDE + + +def render_context_item(item: ContextItem) -> Msg: + """Render one IR item for ``Agent.reply`` without mutating agent state.""" + messages = adapter.AgentScopeContextAdapter().messages_from_items([item]) + if len(messages) != 1: + raise ValueError("context item did not render to exactly one message") + return messages[0] + + +def render_session_input(message: Mapping[str, Any], *, created_seq: int) -> Msg: + """Render one pending positional row at an explicit live request sequence.""" + context_adapter = adapter.AgentScopeContextAdapter() + items = context_adapter.items_from_session_dict(message, created_seq=0) + positioned = [ + replace( + item, + created_seq=created_seq + index, + message_group=f"request:{created_seq}", + ) + for index, item in enumerate(items) + ] + rendered = context_adapter.messages_from_items(positioned) + if len(rendered) != 1: + raise ValueError("session input did not render to exactly one message") + return rendered[0] + + +def append_context_item(agent: Any, item: ContextItem) -> Msg: + """Append one explicitly-provenanced item through the adapter seam.""" + return append_state_context_item(agent.state, item) + + +def append_context_text( + agent: Any, + text: str, + *, + kind: str, + origin: str, + trust: str, + priority: int = 700, + token_budget: int = 4_000, + truncation_policy: str = POLICY_HEAD_TAIL, + render_role: str = "user", +) -> Msg: + """Append provenanced text without creating AgentScope messages elsewhere.""" + created_seq = next_context_sequence(agent.state.context) + item = make_text_context_item( + text, + item_id=f"{origin}:{created_seq}", + kind=kind, + origin=origin, + trust=trust, + created_seq=created_seq, + priority=priority, + token_budget=token_budget, + truncation_policy=truncation_policy, + render_role=render_role, + ) + return append_context_item(agent, item) diff --git a/src/backend/core/llm/context_safety.py b/src/backend/core/llm/context/safety.py similarity index 87% rename from src/backend/core/llm/context_safety.py rename to src/backend/core/llm/context/safety.py index a1836da8..94f0ea76 100644 --- a/src/backend/core/llm/context_safety.py +++ b/src/backend/core/llm/context/safety.py @@ -8,7 +8,7 @@ from typing import Any from core.harness.hooks import HookPaused -from core.llm.context_manager import usable_context_window +from core.models.context_window import usable_context_window class ContextBudgetExceeded(HookPaused): @@ -40,9 +40,11 @@ def check(self) -> None: @contextmanager def request_admission_scope(*, context=None, checkpoint=None): admission = RequestAdmission(context=context, checkpoint=checkpoint) + from core.models.admission import request_admission_callback token = _admission.set(admission) try: - yield admission + with request_admission_callback(admission.check): + yield admission finally: _admission.reset(token) @@ -77,16 +79,10 @@ async def enforce_context_budget(agent: Any, assembly: Any = None) -> None: def remember_context_output(*, message_id: str, steps: list, content: str) -> None: admission = _admission.get() if admission is not None: - from core.services.context_budget_checkpoint import ContextStopCheckpoint + from core.harness.registry.compaction import create_context_checkpoint - admission.checkpoint = ContextStopCheckpoint( + admission.checkpoint = create_context_checkpoint( message_id=message_id, steps=list(steps), content=content, ) - - -def check_request_admission() -> None: - admission = _admission.get() - if admission is not None: - admission.check() diff --git a/src/backend/core/llm/system_reminder.py b/src/backend/core/llm/context/system_reminder.py similarity index 94% rename from src/backend/core/llm/system_reminder.py rename to src/backend/core/llm/context/system_reminder.py index a4c10113..543dd68a 100644 --- a/src/backend/core/llm/system_reminder.py +++ b/src/backend/core/llm/context/system_reminder.py @@ -16,8 +16,8 @@ import logging from typing import Any -from core.llm.context_adapter import append_context_text -from core.llm.context_ir import KIND_REMINDER +from core.llm.context.rendering import append_context_text +from core.llm.context.ir_constants import KIND_REMINDER logger = logging.getLogger(__name__) diff --git a/src/backend/core/llm/context_usage.py b/src/backend/core/llm/context/usage.py similarity index 99% rename from src/backend/core/llm/context_usage.py rename to src/backend/core/llm/context/usage.py index 6d596cb4..c32537c3 100644 --- a/src/backend/core/llm/context_usage.py +++ b/src/backend/core/llm/context/usage.py @@ -9,7 +9,7 @@ import math import time -from typing import Any, Dict, Mapping, Optional +from typing import Any, Dict, Mapping SCHEMA_VERSION = "context-usage.v2" BREAKDOWN_KEYS = ("messages", "tools", "thinking", "files", "system", "input") diff --git a/src/backend/core/llm/context/workspace_hooks.py b/src/backend/core/llm/context/workspace_hooks.py new file mode 100644 index 00000000..dc1d604c --- /dev/null +++ b/src/backend/core/llm/context/workspace_hooks.py @@ -0,0 +1,125 @@ +"""Workspace and plan reminders operate on plain context snapshots.""" +import json +from collections.abc import Mapping +from core.harness.events import thaw_value +from core.harness.hooks import Decision, FailureMode, HookSpec, HookStage +from core.llm.context.reasoning_hooks import reminder +from core.llm.runtime.hooks_constants import FILE_ID_RE, PIN_HINT_SKIP_TOOLS +from core.llm.runtime.message_compat import flatten_tool_output +from core.llm.tools.plan_update import parse_plan_update_args + + +def content_blocks(context, kind): + for message in context: + if not isinstance(message, Mapping): + continue + for block in message.get("content") or (): + if isinstance(block, Mapping) and block.get("type") == kind: + yield block + + +def append_reminder(invocation, text, origin): + return Decision.modify({"context_items": [*invocation.data.get("context_items", ()), + reminder(text, origin)]}) + + +class WorkspacePinHook: + def __init__(self): + self.seen = set() + self.last_signature = "" + + def reset(self, invocation): + self.seen.clear() + self.last_signature = "" + return Decision() + + def finish(self, invocation): + runtime = invocation.data.get("runtime", {}) + if runtime.get("finish_pins_completed"): + return Decision() + remaining = sorted(self.seen - set(runtime.get("pinned_file_ids") or ())) + return Decision.modify({"pin_file_ids": remaining}) if remaining else Decision() + + def __call__(self, invocation): + if invocation.operation_name != "agent_reasoning": + return Decision() + context = list(invocation.data.get("context") or ()) + for block in content_blocks(context[-12:], "tool_result"): + if block.get("name", "") in PIN_HINT_SKIP_TOOLS: + continue + text = flatten_tool_output(block.get("output")) + if '"file_id"' in text: + self.seen.update(FILE_ID_RE.findall(text)) + unpinned = self.seen - set(invocation.data["runtime"].get("pinned_file_ids") or ()) + signature = ",".join(sorted(unpinned)) + if not unpinned or signature == self.last_signature: + return Decision() + preview = sorted(unpinned)[:6] + rendered = ", ".join(preview) + if len(unpinned) > len(preview): + rendered += f", …(+{len(unpinned) - len(preview)} 个)" + self.last_signature = signature + return append_reminder(invocation, + f"沙盒里有 {len(unpinned)} 个 file_id 还没 pin：[{rendered}]。" + "若是给用户的最终产物，必须调 `pin_to_workspace(file_ids=[...])` 才能交付。", + "harness:workspace_pin") + + +class PlanStaleHook: + def __init__(self, interval=10): + self.interval = max(1, int(interval)) + self.since_last = 0 + self.fingerprint = None + + def __call__(self, invocation): + if invocation.operation_name != "agent_reasoning": + return Decision() + blocks = list(content_blocks(invocation.data.get("context") or (), "tool_call")) + plan = None + for block in reversed(blocks): + if block.get("name") != "update_plan": + continue + raw = block.get("input") + if isinstance(raw, str): + try: + raw = json.loads(raw) + except (ValueError, TypeError): + raw = None + plan = parse_plan_update_args(thaw_value(raw)) + fingerprint = str(block.get("id") or "") or json.dumps(plan or {}, ensure_ascii=False, sort_keys=True) + break + if not plan: + return Decision() + if fingerprint != self.fingerprint: + self.fingerprint, self.since_last = fingerprint, 0 + return Decision() + steps = plan.get("steps") or [] + if not any(step.get("status") in ("pending", "in_progress") for step in steps): + return Decision() + self.since_last += 1 + if self.since_last < self.interval: + return Decision() + done = sum(step.get("status") == "completed" for step in steps) + checklist = "\n".join(f" {i}. [{step.get('status', 'pending')}] {step.get('title', '')}" + for i, step in enumerate(steps, 1)) + text = PLAN_REMINDER.format(done=done, total=len(steps), rounds=self.since_last, checklist=checklist) + self.since_last = 0 + return append_reminder(invocation, text, "harness:plan_stale") + + +def workspace_pin_hooks(*, allow_finish=False): + policy = WorkspacePinHook() + specs = (HookSpec("workspace_pin_reset", HookStage.BEFORE_RUN, policy.reset, failure_mode=FailureMode.OPEN), + HookSpec("workspace_pin", HookStage.TRANSFORM_CONTEXT, policy, priority=200, + failure_mode=FailureMode.OPEN, mutable_fields=frozenset({"context_items"}))) + if allow_finish: + specs += (HookSpec("finish_pin_guard", HookStage.BEFORE_FINISH, policy.finish, + failure_mode=FailureMode.OPEN, mutable_fields=frozenset({"pin_file_ids"})),) + return specs + + +def plan_stale_hook(): + return HookSpec("plan_stale", HookStage.TRANSFORM_CONTEXT, PlanStaleHook(), priority=600, + failure_mode=FailureMode.OPEN, mutable_fields=frozenset({"context_items"})) + +PLAN_REMINDER = '`update_plan` 已经 {rounds} 轮没有用过了。当前计划栏停在 {done}/{total}：\n{checklist}\n\n如果其中有步骤其实已经做完，可以调用一次 `update_plan` 把状态补上（传全量列表）；如果这份\n清单已经不符合你正在做的事，重写它比让它停在原地更好。仅在确实相关时才使用——这只是一条\n温和的提醒，不适用就忽略，不要为了回应它而打断手上的工作。不要向用户提起这条提醒。' diff --git a/src/backend/core/llm/context_adapter.py b/src/backend/core/llm/context_adapter.py deleted file mode 100644 index 297f5a10..00000000 --- a/src/backend/core/llm/context_adapter.py +++ /dev/null @@ -1,867 +0,0 @@ -"""The sole Canonical Context IR ↔ AgentScope message adapter. - -Compatibility roles are a rendering detail here. Provenance and trust travel -in structured message metadata and never have to be inferred from XML labels. -""" - -from __future__ import annotations - -from dataclasses import replace -from typing import Any, Iterable, Mapping, Optional, Sequence - -from agentscope.message import Msg, TextBlock - -from core.immutable import thaw_json -from core.llm.execution_manifest import stable_hash -from core.llm.context_ir import ( - CONTEXT_SEQUENCE_STRIDE, - ContextItem, - KIND_ASSISTANT, - KIND_ATTACHMENT, - KIND_COMPACTION, - KIND_MEMORY, - KIND_PROJECT, - KIND_REFERENCE, - KIND_REMINDER, - KIND_SYSTEM_RULE, - KIND_THINKING, - KIND_TOOL_CALL, - KIND_TOOL_RESULT, - KIND_USER_INPUT, - POLICY_DROP, - POLICY_HEAD_TAIL, - POLICY_NEVER, - SESSION_CONTEXT_META_KEY, - STEP_KINDS, - VISIBILITY_MANIFEST_ONLY, - VISIBILITY_MODEL, - estimate_context_tokens, - make_text_context_item, -) - -CONTEXT_META_KEY = "harness_context_items" -PROVIDER_CONTEXT_META_KEY = "_harness_context_item" - - -def _block_payload(block: Any) -> Any: - if isinstance(block, Mapping): - return dict(block) - if hasattr(block, "model_dump"): - return block.model_dump(mode="json") - return block - - -def _block_type(block: Any) -> str: - if isinstance(block, Mapping): - return str(block.get("type") or "") - return str(getattr(block, "type", "") or "") - - -def _block_text(block: Any) -> str: - if isinstance(block, Mapping): - return str(block.get("text") or "") - return str(getattr(block, "text", "") or "") - - -class AgentScopeContextAdapter: - """Translate the IR while preserving current model-facing roles/blocks.""" - - @staticmethod - def wrap_content(content: Any) -> list[Any]: - """Construct AgentScope-compatible blocks inside the sole SDK seam.""" - if isinstance(content, str): - return [TextBlock(type="text", text=content)] if content else [] - if isinstance(content, list): - return content - return [content] if content else [] - - @staticmethod - def _defaults(role: str, block_type: str) -> dict[str, Any]: - if block_type == "thinking": - # Reasoning is handed back to the model verbatim. It is counted on its - # own so the usage panel can show what it costs, but it is never - # shortened or dropped apart from the step that produced it. - return { - "kind": KIND_THINKING, - "origin": "assistant:reasoning", - "trust": "assistant", - "priority": 400, - "token_budget": 0, - "truncation_policy": POLICY_NEVER, - "render_role": "assistant", - } - if block_type == "tool_call": - return { - "kind": KIND_TOOL_CALL, - "origin": "agent:tool_call", - "trust": "assistant", - "priority": 700, - "token_budget": 0, - "truncation_policy": POLICY_NEVER, - "render_role": "assistant", - } - if block_type == "tool_result": - return { - "kind": KIND_TOOL_RESULT, - "origin": "tool:result", - "trust": "tool", - "priority": 700, - "token_budget": 20_000, - "truncation_policy": POLICY_HEAD_TAIL, - "render_role": "assistant", - } - if block_type == "data": - return { - "kind": KIND_ATTACHMENT, - "origin": "user:attachment", - "trust": "user", - "priority": 650, - "token_budget": 4_000, - "truncation_policy": POLICY_DROP, - "render_role": "user", - } - if role == "system": - return { - "kind": KIND_SYSTEM_RULE, - "origin": "platform:system_prompt", - "trust": "platform", - "priority": 1_000, - "token_budget": 100_000, - "truncation_policy": POLICY_NEVER, - "render_role": "system", - } - if role == "assistant": - return { - "kind": KIND_ASSISTANT, - "origin": "assistant:history", - "trust": "assistant", - "priority": 400, - "token_budget": 0, - "truncation_policy": POLICY_NEVER, - "render_role": "assistant", - } - return { - "kind": KIND_USER_INPUT, - "origin": "user:chat", - "trust": "user", - "priority": 500, - "token_budget": 100_000, - "truncation_policy": POLICY_HEAD_TAIL, - "render_role": "user", - } - - @staticmethod - def _metadata_entries(message: Msg) -> list[Mapping[str, Any]]: - metadata = getattr(message, "metadata", None) or {} - raw = metadata.get(CONTEXT_META_KEY) if isinstance(metadata, Mapping) else None - if not isinstance(raw, list): - return [] - return [entry for entry in raw if isinstance(entry, Mapping)] - - def items_from_messages( - self, - messages: Sequence[Msg], - *, - summary_text: Any = None, - promote_latest_user: bool = True, - ) -> list[ContextItem]: - items: list[ContextItem] = [] - summary = str(summary_text or "") - tool_occurrences: dict[str, int] = {} - pending_tool_pairs: dict[str, list[str]] = {} - orphan_results: dict[str, int] = {} - # One ReAct step = what the model produced in one response plus the - # results of the tool calls in it. A step opens on the first thinking, - # text or tool_call block after a tool result (or after any other - # message) and its results join it through the pending call ids. This - # is the same boundary the reasoning-echo formatter flushes on. - step_unit: Optional[str] = None - step_closed = False - pair_units: dict[str, str] = {} - for message_index, message in enumerate(messages): - role = str(getattr(message, "role", "user") or "user") - name = str(getattr(message, "name", role) or role) - blocks = list(getattr(message, "content", None) or []) - entries = self._metadata_entries(message) - for block_index, block in enumerate(blocks): - block_type = _block_type(block) - payload = _block_text(block) if block_type == "text" else _block_payload(block) - defaults = self._defaults(role, block_type) - explicit = entries[block_index] if block_index < len(entries) else {} - if ( - not explicit - and summary - and role == "user" - and isinstance(payload, str) - and payload == summary - ): - defaults.update( - kind=KIND_COMPACTION, - origin="harness:compaction", - trust="system", - priority=850, - truncation_policy=POLICY_HEAD_TAIL, - render_role="user", - ) - raw_pair_id = str( - payload.get("id") - if block_type in {"tool_call", "tool_result"} and isinstance(payload, Mapping) - else "" - ) - pair_id = str(explicit.get("pair_id") or "") - if block_type == "tool_call" and raw_pair_id: - occurrence = tool_occurrences.get(raw_pair_id, 0) + 1 - tool_occurrences[raw_pair_id] = occurrence - pair_id = f"{raw_pair_id}#{occurrence}" - pending_tool_pairs.setdefault(raw_pair_id, []).append(pair_id) - elif block_type == "tool_result" and raw_pair_id: - pending = pending_tool_pairs.get(raw_pair_id) or [] - if pending: - pair_id = pending.pop(0) - else: - occurrence = orphan_results.get(raw_pair_id, 0) + 1 - orphan_results[raw_pair_id] = occurrence - pair_id = f"{raw_pair_id}#orphan-result-{occurrence}" - item_id = str( - ( - f"message:{message_index}:block:{block_index}:{block_type}:{pair_id}" - if block_type in {"tool_call", "tool_result"} - else explicit.get("item_id") - or f"message:{message_index}:block:{block_index}" - ) - ) - kind = str(explicit.get("kind") or defaults["kind"]) - if kind in STEP_KINDS and block_type != "tool_result": - if step_unit is None or step_closed: - step_unit = f"step:{message_index}:{block_index}" - step_closed = False - unit_id = step_unit - if block_type == "tool_call": - pair_units[pair_id] = unit_id - elif block_type == "tool_result": - unit_id = pair_units.pop(pair_id, f"orphan:{item_id}") - step_closed = True - else: - unit_id = f"item:{item_id}" - step_unit = None - step_closed = False - pair_units.clear() - created_seq = int( - explicit.get("created_seq") - if explicit.get("created_seq") is not None - else message_index * CONTEXT_SEQUENCE_STRIDE + block_index - ) - item = ContextItem.create( - item_id=item_id, - kind=kind, - origin=str(explicit.get("origin") or defaults["origin"]), - trust=str(explicit.get("trust") or defaults["trust"]), - visibility=str(explicit.get("visibility") or VISIBILITY_MODEL), - priority=int( - explicit["priority"] - if explicit.get("priority") is not None - else defaults["priority"] - ), - token_budget=int( - explicit["token_budget"] - if explicit.get("token_budget") is not None - else defaults["token_budget"] - ), - truncation_policy=str( - explicit.get("truncation_policy") or defaults["truncation_policy"] - ), - content=payload, - cache_class=str(explicit.get("cache_class") or "dynamic"), - created_seq=created_seq, - render_role=str(explicit.get("render_role") or defaults["render_role"]), - # The outer message carries the actual transport name. - # Persisted manifests contain only its hash. - render_name=name, - pair_id=pair_id, - message_group=str(explicit.get("message_group") or f"message:{message_index}"), - unit_id=unit_id, - content_ref=( - str(explicit["content_ref"]) if explicit.get("content_ref") else None - ), - content_hash=( - str(explicit["content_hash"]) if explicit.get("content_hash") else None - ), - token_estimate=( - int(explicit["token_estimate"]) - if explicit.get("token_estimate") is not None - else None - ), - metadata={}, - ) - items.append(item) - - # The latest real user instruction is a hard inclusion. Memory and - # reminders may render as user for compatibility but keep distinct kind. - user_indices = [i for i, item in enumerate(items) if item.kind == KIND_USER_INPUT] - if promote_latest_user and user_indices: - idx = user_indices[-1] - items[idx] = replace( - items[idx], - priority=max(900, items[idx].priority), - truncation_policy=POLICY_NEVER, - ) - return items - - def items_from_session_dict( - self, - message: Mapping[str, Any], - *, - created_seq: int, - ) -> list[ContextItem]: - role = str(message.get("role") or "user") - role = {"human": "user", "ai": "assistant", "tool": "assistant"}.get(role, role) - content = message.get("content") - blocks = ( - content - if isinstance(content, list) - else ([{"type": "text", "text": content}] if content else []) - ) - explicit = message.get(SESSION_CONTEXT_META_KEY) - metadata_entries = [] - if isinstance(explicit, Mapping): - metadata_entries = [dict(explicit) for _ in blocks] - msg = Msg( - name=str(message.get("name") or role), - role=role if role in {"system", "assistant", "user"} else "user", - content=blocks, - metadata={CONTEXT_META_KEY: metadata_entries} if metadata_entries else {}, - ) - # A persisted row is converted in isolation, so it cannot know whether - # it is the current user instruction. Promotion happens once, later, - # when ManifestBoundAgent sees the complete final request. - items = self.items_from_messages([msg], promote_latest_user=False) - has_explicit_seq = isinstance(explicit, Mapping) and explicit.get("created_seq") is not None - return [ - replace( - item, - item_id=( - item.item_id - if isinstance(explicit, Mapping) and explicit.get("item_id") - else f"session:{created_seq}:block:{index}" - ), - created_seq=( - item.created_seq - if has_explicit_seq - else created_seq * CONTEXT_SEQUENCE_STRIDE + index - ), - message_group=f"session:{created_seq}", - unit_id=f"session:{created_seq}:{item.unit_id}", - ) - for index, item in enumerate(items) - ] - - def message_from_session_dict( - self, - message: Mapping[str, Any], - *, - created_seq: int, - ) -> Msg: - """Render one persisted history record through the canonical adapter.""" - items = self.items_from_session_dict(message, created_seq=created_seq) - rendered = self.messages_from_items(items) - if rendered: - return rendered[0] - role = str(message.get("role") or "user") - role = {"human": "user", "ai": "assistant", "tool": "assistant"}.get(role, role) - if role not in {"system", "assistant", "user"}: - role = "user" - return Msg( - name=str(message.get("name") or role), - role=role, - content=[], - metadata={CONTEXT_META_KEY: []}, - ) - - def messages_from_items(self, items: Iterable[ContextItem]) -> list[Msg]: - visible = [item for item in items if item.visibility == VISIBILITY_MODEL] - messages: list[Msg] = [] - current_group: Optional[tuple[str, str]] = None - current_items: list[ContextItem] = [] - - def flush() -> None: - nonlocal current_group, current_items - if not current_items or current_group is None: - return - blocks = [] - manifests = [] - for item in current_items: - content = thaw_json(item.content) - if isinstance(content, str): - blocks.append(TextBlock(type="text", text=content)) - manifests.append(item.to_manifest()) - elif isinstance(content, Mapping): - blocks.append(dict(content)) - manifests.append(item.to_manifest()) - elif isinstance(content, list): - for block_index, block in enumerate(content): - blocks.append(block) - normalized = ( - _block_text(block) - if _block_type(block) == "text" - else _block_payload(block) - ) - derived = replace( - item, - item_id=f"{item.item_id}:block:{block_index}", - truncation_policy=( - POLICY_DROP - if _block_type(block) == "data" - else item.truncation_policy - ), - ).with_content(normalized) - manifests.append(derived.to_manifest()) - elif content is not None: - blocks.append(TextBlock(type="text", text=str(content))) - manifests.append(item.with_content(str(content)).to_manifest()) - if blocks: - name = str(current_items[0].render_name or current_group[1]) - messages.append( - Msg( - name=name, - role=current_group[1], - content=blocks, - metadata={CONTEXT_META_KEY: manifests}, - ) - ) - current_group = None - current_items = [] - - for item in visible: - group = (item.message_group, item.render_role) - if current_group is not None and group != current_group: - flush() - current_group = group - current_items.append(item) - flush() - return messages - - def annotate_provider_messages( - self, - source_messages: Sequence[Msg], - formatted_messages: Sequence[Mapping[str, Any]], - row_counts: Sequence[int], - ) -> list[dict[str, Any]]: - """Carry canonical provenance across an SDK formatter boundary. - - AgentScope's OpenAI formatter intentionally drops arbitrary message - metadata and may expand one tool-result message into a tool row plus a - synthetic media row. ``row_counts`` is obtained by formatting each - source message in isolation after a provider has rejected media. This - lets the retry inherit the original IR provenance without guessing it - from compatibility roles or XML-like text. - """ - annotated = [dict(message) for message in formatted_messages] - cursor = 0 - for source, count in zip(source_messages, row_counts): - entries = self._metadata_entries(source) - source_rows = annotated[cursor : cursor + max(0, int(count))] - cursor += max(0, int(count)) - if not entries: - continue - - def pick(kind: str) -> Mapping[str, Any]: - match = next((entry for entry in entries if entry.get("kind") == kind), None) - if match is not None: - return match - user = next( - (entry for entry in entries if entry.get("kind") == KIND_USER_INPUT), - None, - ) - return user or entries[0] - - for row in source_rows: - role = str(row.get("role") or "user") - name = str(row.get("name") or role) - if name == "system-reminder": - base = pick(KIND_TOOL_RESULT) - metadata = { - "kind": KIND_ATTACHMENT, - "origin": f"{base.get('origin') or 'tool:result'}:media", - "trust": str(base.get("trust") or "tool"), - "priority": int(base.get("priority") or 700), - } - elif role == "tool": - base = pick(KIND_TOOL_RESULT) - metadata = { - "kind": KIND_TOOL_RESULT, - "origin": str(base.get("origin") or "tool:result"), - "trust": str(base.get("trust") or "tool"), - "priority": int(base.get("priority") or 700), - } - elif role == "assistant" and row.get("tool_calls"): - base = pick(KIND_TOOL_CALL) - metadata = { - "kind": KIND_TOOL_CALL, - "origin": str(base.get("origin") or "agent:tool_call"), - "trust": str(base.get("trust") or "assistant"), - "priority": int(base.get("priority") or 700), - } - else: - base = pick("") - metadata = { - key: base[key] - for key in ("kind", "origin", "trust", "priority") - if base.get(key) is not None - } - row[PROVIDER_CONTEXT_META_KEY] = metadata - if cursor != len(annotated): - # A formatter shape change must fail visibly rather than silently - # assigning later rows the wrong trust boundary. - raise ValueError( - "provider formatter row mapping mismatch: " - f"mapped={cursor} formatted={len(annotated)}" - ) - return annotated - - def items_from_provider_messages( - self, - messages: Sequence[Mapping[str, Any]], - ) -> list[ContextItem]: - """Reify an already-formatted provider retry as exact canonical items. - - A provider may reject media after the normal AgentScope request has - already been assembled. The retry is therefore a new execution - surface. Each complete wire message is kept atomically so its hash - covers role, name, tool identifiers and content together; transient - provenance metadata is removed before hashing and before transport. - """ - items: list[ContextItem] = [] - pending_tool_pairs: dict[str, list[str]] = {} - orphan_results: dict[str, int] = {} - for index, raw_message in enumerate(messages): - message = dict(raw_message) - explicit = message.pop(PROVIDER_CONTEXT_META_KEY, None) - explicit = explicit if isinstance(explicit, Mapping) else {} - role = str(message.get("role") or "user") - name = str(message.get("name") or role) - tool_calls = message.get("tool_calls") - if role == "assistant" and isinstance(tool_calls, list) and tool_calls: - pair_id = f"provider-tool-batch:{index}" - message_hash = stable_hash(message) - message_tokens = estimate_context_tokens(message) - for call_index, call in enumerate(tool_calls): - call = call if isinstance(call, Mapping) else {} - raw_pair_id = str(call.get("id") or "") - pending_tool_pairs.setdefault(raw_pair_id, []).append(pair_id) - items.append( - ContextItem.create( - item_id=f"provider-retry:{index}:tool-call:{call_index}", - kind=str(explicit.get("kind") or KIND_TOOL_CALL), - origin=str(explicit.get("origin") or "assistant:provider_tool_call"), - trust=str(explicit.get("trust") or "assistant"), - visibility=VISIBILITY_MODEL, - priority=int(explicit.get("priority") or 900), - token_budget=message_tokens if call_index == 0 else 0, - truncation_policy=POLICY_NEVER, - content=message if call_index == 0 else None, - content_ref=f"provider-message:{index}", - content_hash=message_hash, - token_estimate=message_tokens if call_index == 0 else 0, - cache_class="dynamic", - created_seq=index * CONTEXT_SEQUENCE_STRIDE + call_index, - render_role=role, - render_name=name, - pair_id=pair_id, - message_group=f"provider-retry:{index}", - unit_id=pair_id, - ) - ) - continue - if role == "tool": - raw_pair_id = str(message.get("tool_call_id") or "") - pending = pending_tool_pairs.get(raw_pair_id) or [] - if pending: - pair_id = pending.pop(0) - else: - occurrence = orphan_results.get(raw_pair_id, 0) + 1 - orphan_results[raw_pair_id] = occurrence - pair_id = f"{raw_pair_id}#orphan-result-{occurrence}" - defaults = { - "kind": KIND_TOOL_RESULT, - "origin": "tool:provider_result", - "trust": "tool", - "priority": 900, - "pair_id": pair_id, - } - elif role == "system": - defaults = { - "kind": KIND_SYSTEM_RULE, - "origin": "platform:provider_request", - "trust": "platform", - "priority": 1_000, - } - elif role == "assistant": - defaults = { - "kind": KIND_ASSISTANT, - "origin": "assistant:provider_history", - "trust": "assistant", - "priority": 600, - } - else: - defaults = { - "kind": KIND_USER_INPUT, - "origin": "user:provider_request", - "trust": "user", - "priority": 900, - } - items.append( - ContextItem.create( - item_id=str(explicit.get("item_id") or f"provider-retry:{index}"), - kind=str(explicit.get("kind") or defaults["kind"]), - origin=str(explicit.get("origin") or defaults["origin"]), - trust=str(explicit.get("trust") or defaults["trust"]), - visibility=VISIBILITY_MODEL, - priority=int(explicit.get("priority") or defaults["priority"]), - # These rows have already survived the canonical budget - # pass. Media recovery only removes data or replaces it - # with a smaller textual description, so the retry must - # preserve every resulting row exactly. - token_budget=max(1, estimate_context_tokens(message)), - truncation_policy=POLICY_NEVER, - content=message, - cache_class="dynamic", - created_seq=int( - explicit.get("created_seq") - if explicit.get("created_seq") is not None - else index * CONTEXT_SEQUENCE_STRIDE - ), - render_role=role, - render_name=name, - pair_id=str(defaults.get("pair_id") or ""), - message_group=f"provider-retry:{index}", - unit_id=str(defaults.get("pair_id") or f"provider-retry:{index}"), - ) - ) - return items - - @staticmethod - def provider_messages_from_items(items: Iterable[ContextItem]) -> list[dict[str, Any]]: - """Render exact provider rows while stripping harness-only metadata.""" - rendered: list[dict[str, Any]] = [] - by_group: dict[str, dict[str, Any]] = {} - deferred_groups: set[str] = set() - for item in items: - if item.visibility != VISIBILITY_MODEL: - continue - value = thaw_json(item.content) - if value is None: - deferred_groups.add(item.message_group) - continue - if not isinstance(value, Mapping): - raise ValueError("provider retry context item must contain one message mapping") - row = dict(value) - row.pop(PROVIDER_CONTEXT_META_KEY, None) - previous = by_group.get(item.message_group) - if previous is not None: - if previous != row: - raise ValueError("provider retry group contains conflicting wire messages") - continue - by_group[item.message_group] = row - rendered.append(row) - missing = deferred_groups - set(by_group) - if missing: - raise ValueError(f"provider retry groups lack a wire row: {sorted(missing)}") - return rendered - - def reference_items_from_execution_manifest(self, manifest: Any) -> list[ContextItem]: - if hasattr(manifest, "to_dict"): - payload = manifest.to_dict() - elif isinstance(manifest, Mapping): - payload = dict(manifest) - else: - return [] - sections = (payload.get("prompt_manifest") or {}).get("sections") or [] - items: list[ContextItem] = [] - for index, section in enumerate(sections): - if not isinstance(section, Mapping): - continue - section_id = str(section.get("id") or "") - content_hash = str(section.get("content_hash") or "") - if not section_id or not content_hash: - continue - is_project = section_id == "runtime/project" - live_content = "" - content_reader = getattr(manifest, "prompt_section_content", None) - if callable(content_reader): - live_content = str(content_reader(index) or "") - if is_project and live_content: - items.append( - ContextItem.create( - item_id=f"prompt:{index}:{section_id}", - kind=KIND_PROJECT, - origin=str(section.get("origin") or "workspace:project"), - trust=str(section.get("trust") or "workspace"), - visibility=VISIBILITY_MODEL, - priority=int(section.get("priority") or 900), - token_budget=max( - 1, - int(section.get("budget") or section.get("token_estimate") or 1), - ), - truncation_policy=POLICY_HEAD_TAIL, - content=live_content, - content_ref=( - str(section["reference"]) if section.get("reference") else None - ), - content_hash=content_hash, - token_estimate=int(section.get("token_estimate") or 0), - cache_class=str(section.get("cache_class") or "workspace"), - created_seq=1, - render_role="system", - render_name="system", - message_group=f"prompt:{index}", - ) - ) - continue - items.append( - ContextItem.create( - item_id=f"prompt:{index}:{section_id}", - kind=KIND_PROJECT if is_project else KIND_SYSTEM_RULE, - origin=str(section.get("origin") or "prompt:unknown"), - trust=str(section.get("trust") or "platform"), - visibility=VISIBILITY_MANIFEST_ONLY, - priority=int(section.get("priority") or 0), - token_budget=int(section.get("budget") or 0), - truncation_policy=POLICY_NEVER, - content=None, - content_ref=str( - section.get("reference") - or f"prompt:{section_id}@{section.get('version') or '1'}" - ), - content_hash=content_hash, - token_estimate=int(section.get("token_estimate") or 0), - cache_class=str(section.get("cache_class") or "stable"), - created_seq=-10_000 + index, - render_role="system", - message_group=f"prompt:{index}", - ) - ) - return items - - -def append_state_context_item(state: Any, item: ContextItem) -> Msg: - """Render and append one explicitly-provenanced item to AgentScope state.""" - messages = AgentScopeContextAdapter().messages_from_items([item]) - if len(messages) != 1: - raise ValueError("context item did not render to exactly one message") - message = messages[0] - state.context.append(message) - return message - - -def render_text_block(text: str) -> TextBlock: - """Construct an AgentScope text block inside the SDK adapter boundary.""" - return TextBlock(type="text", text=str(text or "")) - - -def next_context_sequence(messages: Sequence[Any]) -> int: - """Allocate one monotonic sequence across replay and live injections.""" - max_sequence = -CONTEXT_SEQUENCE_STRIDE - for message_index, message in enumerate(messages): - entries = AgentScopeContextAdapter._metadata_entries(message) - blocks = list(getattr(message, "content", None) or []) - for block_index, _block in enumerate(blocks or [None]): - explicit = entries[block_index] if block_index < len(entries) else {} - candidate = explicit.get("created_seq") - if candidate is None: - candidate = message_index * CONTEXT_SEQUENCE_STRIDE + block_index - try: - max_sequence = max(max_sequence, int(candidate)) - except (TypeError, ValueError): - continue - return ((max_sequence // CONTEXT_SEQUENCE_STRIDE) + 1) * CONTEXT_SEQUENCE_STRIDE - - -def next_request_sequence(messages: Sequence[Any]) -> int: - """Reserve a pre-reply lane above the context tail for the user instruction. - - AgentScope runs ``on_reply`` middleware before it appends ``inputs`` to - state. File/project attachment middleware can therefore add a few explicit - context items first without colliding with the request's sequence. The - sequence no longer decides message order — the assembler emits items in - context order — it only keeps identifiers distinct and marks recency for - budget eviction. - """ - pre_reply_slots = 64 - return next_context_sequence(messages) + pre_reply_slots * CONTEXT_SEQUENCE_STRIDE - - -def render_context_item(item: ContextItem) -> Msg: - """Render one IR item for ``Agent.reply`` without mutating agent state.""" - messages = AgentScopeContextAdapter().messages_from_items([item]) - if len(messages) != 1: - raise ValueError("context item did not render to exactly one message") - return messages[0] - - -def render_session_input(message: Mapping[str, Any], *, created_seq: int) -> Msg: - """Render one pending positional row at an explicit live request sequence.""" - adapter = AgentScopeContextAdapter() - items = adapter.items_from_session_dict(message, created_seq=0) - positioned = [ - replace( - item, - created_seq=created_seq + index, - message_group=f"request:{created_seq}", - ) - for index, item in enumerate(items) - ] - rendered = adapter.messages_from_items(positioned) - if len(rendered) != 1: - raise ValueError("session input did not render to exactly one message") - return rendered[0] - - -def append_context_item(agent: Any, item: ContextItem) -> Msg: - """Append one explicitly-provenanced item through the adapter seam.""" - return append_state_context_item(agent.state, item) - - -def append_context_text( - agent: Any, - text: str, - *, - kind: str, - origin: str, - trust: str, - priority: int = 700, - token_budget: int = 4_000, - truncation_policy: str = POLICY_HEAD_TAIL, - render_role: str = "user", -) -> Msg: - """Append provenanced text without creating AgentScope messages elsewhere.""" - created_seq = next_context_sequence(agent.state.context) - item = make_text_context_item( - text, - item_id=f"{origin}:{created_seq}", - kind=kind, - origin=origin, - trust=trust, - created_seq=created_seq, - priority=priority, - token_budget=token_budget, - truncation_policy=truncation_policy, - render_role=render_role, - ) - return append_context_item(agent, item) - - -__all__ = [ - "AgentScopeContextAdapter", - "CONTEXT_META_KEY", - "PROVIDER_CONTEXT_META_KEY", - "SESSION_CONTEXT_META_KEY", - "append_context_item", - "append_state_context_item", - "append_context_text", - "make_text_context_item", - "next_context_sequence", - "next_request_sequence", - "render_context_item", - "render_session_input", - "render_text_block", -] diff --git a/src/backend/core/llm/context_ir.py b/src/backend/core/llm/context_ir.py deleted file mode 100644 index 5f68c755..00000000 --- a/src/backend/core/llm/context_ir.py +++ /dev/null @@ -1,855 +0,0 @@ -"""Framework-neutral canonical context IR and deterministic budget assembler. - -This module deliberately has no AgentScope dependency. It owns provenance, -trust, selection, truncation and the sanitized inclusion/exclusion manifest; -``context_adapter`` is the only layer that translates these items to model SDK -messages. -""" - -from __future__ import annotations - -from dataclasses import dataclass, field, replace -from typing import Any, Iterable, Mapping, Optional, Protocol, Sequence - -from core.immutable import FrozenDict, freeze_json, thaw_json -from core.llm.execution_manifest import canonical_json, stable_hash - -CONTEXT_SCHEMA_VERSION = "harness.context.v2" -SESSION_CONTEXT_META_KEY = "_context_item" -CONTEXT_SEQUENCE_STRIDE = 1_000 -# Matches openai/codex's per-image estimate (RESIZED_IMAGE_BYTES_ESTIMATE = 7373 -# bytes over its 4-bytes/token heuristic). One price per image, never its -# transport bytes; the provider's reported usage supersedes it once a reply lands. -IMAGE_TOKEN_RESERVE = 1_844 - -KIND_SYSTEM_RULE = "system_rule" -KIND_USER_INPUT = "user_input" -KIND_ASSISTANT = "assistant_history" -KIND_MEMORY = "memory" -KIND_IDENTITY = "identity" -KIND_PROJECT = "project_material" -KIND_THINKING = "thinking" -KIND_TOOL_CALL = "tool_call" -KIND_TOOL_RESULT = "tool_result" -KIND_REMINDER = "reminder" -KIND_COMPACTION = "compaction_summary" -KIND_ATTACHMENT = "attachment" -KIND_STEER = "steer" -KIND_REFERENCE = "reference" - -POLICY_NEVER = "never" -POLICY_DROP = "drop" -POLICY_HEAD_TAIL = "head_tail" -POLICY_TAIL = "tail" - -VISIBILITY_MODEL = "model" -VISIBILITY_MANIFEST_ONLY = "manifest_only" - -_TOOL_KINDS = {KIND_TOOL_CALL, KIND_TOOL_RESULT} -# One ReAct step is the atomic unit of history: what the model said (thinking, -# text, tool calls) together with the results those calls produced. These -# kinds are never budgeted block by block. -STEP_KINDS = frozenset({KIND_ASSISTANT, KIND_THINKING, KIND_TOOL_CALL, KIND_TOOL_RESULT}) -# Conversation history is selected as a contiguous tail; everything else -# (memory, project material, reminders, attachments, the compaction summary) -# keeps priority-based selection because it is not ordered by time. -CONVERSATION_KINDS = STEP_KINDS | {KIND_USER_INPUT} -_TRUNCATABLE_BLOCK_TYPES = {"tool_result"} -_PRUNED_OUTPUT_TEMPLATE = "[tool output pruned to fit the context budget: {tokens} tokens omitted]" - - -class ContextAdapterProtocol(Protocol): - """Framework-neutral seam for turning transport rows into canonical IR.""" - - def items_from_messages( - self, - messages: Sequence[Any], - *, - summary_text: Any = None, - promote_latest_user: bool = True, - ) -> list["ContextItem"]: ... - - def messages_from_items(self, items: Iterable["ContextItem"]) -> list[Any]: ... - - def reference_items_from_execution_manifest( - self, manifest: Any - ) -> list["ContextItem"]: ... - - def items_from_provider_messages( - self, - messages: Sequence[Mapping[str, Any]], - ) -> list["ContextItem"]: ... - - def provider_messages_from_items( - self, - items: Iterable["ContextItem"], - ) -> list[Mapping[str, Any]]: ... - - -def _normalize_context_content(content: Any) -> Any: - """Convert SDK block objects into stable, framework-neutral JSON values.""" - if hasattr(content, "model_dump"): - return _normalize_context_content(content.model_dump(mode="json")) - if isinstance(content, Mapping): - return { - str(key): _normalize_context_content(value) - for key, value in content.items() - } - if isinstance(content, (list, tuple)): - return [_normalize_context_content(value) for value in content] - return content - - -def _tool_result_images(content: Any) -> list[Any]: - """Image payloads are atomic media, not part of a tool's text allowance.""" - if not isinstance(content, Mapping) or content.get("type") != "tool_result": - return [] - output = content.get("output") - if not isinstance(output, (list, tuple)): - return [] - return [ - block - for block in output - if isinstance(block, Mapping) - and block.get("type") == "data" - and isinstance(block.get("source"), Mapping) - and str(block["source"].get("media_type") or "").startswith("image/") - ] - - -def image_token_reserve(content: Any) -> int: - """Reserve media occupancy without measuring image transport bytes.""" - if isinstance(content, (list, tuple)): - return sum(image_token_reserve(block) for block in content) - if isinstance(content, Mapping): - source = content.get("source") - if ( - content.get("type") == "data" - and isinstance(source, Mapping) - and str(source.get("media_type") or "").startswith("image/") - ): - return IMAGE_TOKEN_RESERVE - if content.get("type") == "tool_result": - return image_token_reserve(content.get("output", content.get("content"))) - return 0 - - -def estimate_context_tokens(content: Any) -> int: - """Deterministic conservative estimate without provider-specific imports.""" - content = _normalize_context_content(content) - if content is None: - return 0 - if isinstance(content, str): - return max(1, (len(content.encode("utf-8")) + 3) // 4) if content else 0 - if isinstance(content, Mapping) and content.get("type") == "data": - # Provider image accounting is not proportional to base64 bytes. Keep a - # conservative fixed reserve without letting transport encoding evict - # the actual attachment before the provider can count it precisely. - return IMAGE_TOKEN_RESERVE - if _tool_result_images(content): - # Count image reserves recursively instead of tokenizing their base64 - # inside the enclosing tool-result JSON. - return estimate_context_tokens({**content, "output": []}) + sum( - estimate_context_tokens(block) for block in content["output"] - ) - if isinstance(content, list): - return sum(estimate_context_tokens(item) for item in content) - return max(1, (len(canonical_json(content).encode("utf-8")) + 3) // 4) - - -def _truncate_text(text: str, max_tokens: int, *, tail_only: bool = False) -> str: - if not text or max_tokens <= 0: - return "" - max_bytes = max(4, max_tokens * 4) - if len(text.encode("utf-8")) <= max_bytes: - return text - # Character slicing is deterministic and safe for Unicode. Iterate down if - # multi-byte text still exceeds the byte reserve. - max_chars = max(1, max_bytes) - head = 0 - tail = 0 - marker = "" - if tail_only: - tail = max_chars - result = text[-tail:] - elif max_chars < 40: - head = max_chars - result = text[:head] - else: - marker = "\n[… omitted …]\n" - body = max(2, max_chars - len(marker)) - head = max(1, int(body * 0.6)) - tail = max(1, body - head) - result = text[:head] + marker + text[-tail:] - while len(result.encode("utf-8")) > max_bytes and len(result) > 1: - if tail_only: - tail = max(1, tail - 1) - result = text[-tail:] - elif marker: - # Preserve both semantic ends; shrink the larger slice first. - if head >= tail and head > 1: - head -= 1 - elif tail > 1: - tail -= 1 - elif head > 1: - head -= 1 - else: - break - result = text[:head] + marker + text[-tail:] - else: - head = max(1, head - 1) - result = text[:head] - return result - - -def _truncate_content(content: Any, max_tokens: int, policy: str) -> Any: - """Shorten prunable text only; every structured block keeps its shape. - - Plain text and a tool result's output are the only things a budget may - shorten. Any other block — thinking, tool call, data, or a shape this - module does not know — is returned untouched so the caller keeps or drops - it whole. Rewriting such a block into a JSON string would hand the model - its own reasoning back as an answer. - """ - if max_tokens <= 0: - return "" - if isinstance(content, str): - return _truncate_text(content, max_tokens, tail_only=policy == POLICY_TAIL) - if isinstance(content, Mapping): - mutable = thaw_json(content) - block_type = str(mutable.get("type") or "") - if block_type not in _TRUNCATABLE_BLOCK_TYPES: - return mutable - for key in ("output", "content", "text"): - value = mutable.get(key) - if isinstance(value, str): - overhead = estimate_context_tokens({**mutable, key: ""}) - mutable[key] = _truncate_text( - value, - max(1, max_tokens - overhead), - tail_only=policy == POLICY_TAIL, - ) - return mutable - if key == "output" and isinstance(value, (list, Mapping)): - images = _tool_result_images(mutable) - if images: - text = "\n".join( - ( - str(block["text"]) - if isinstance(block, Mapping) and block.get("text") is not None - else block if isinstance(block, str) else canonical_json(block) - ) - for block in value - if block not in images - ) - output = [{"type": "text", "text": ""}, *images] - overhead = estimate_context_tokens({**mutable, key: output}) - output[0]["text"] = _truncate_text( - text, max(1, max_tokens - overhead), tail_only=policy == POLICY_TAIL - ) - mutable[key] = output - return mutable - if isinstance(value, list): - parts = [] - for block in value: - if isinstance(block, Mapping) and block.get("text") is not None: - parts.append(str(block["text"])) - elif isinstance(block, str): - parts.append(block) - else: - parts.append(canonical_json(block)) - flattened = "\n".join(parts) - else: - flattened = canonical_json(value) - overhead = estimate_context_tokens({**mutable, key: ""}) - mutable[key] = _truncate_text( - flattened, - max(1, max_tokens - overhead), - tail_only=policy == POLICY_TAIL, - ) - return mutable - return mutable - return thaw_json(content) - - -def _prune_tool_output(content: Any, original_tokens: int) -> Any: - """Replace a tool result's output with an explicit placeholder.""" - mutable = thaw_json(content) - if not isinstance(mutable, dict) or str(mutable.get("type") or "") != "tool_result": - raise ValueError("only tool_result content can be pruned") - images = _tool_result_images(mutable) - if images: - mutable["output"] = [ - {"type": "text", "text": _PRUNED_OUTPUT_TEMPLATE.format(tokens=original_tokens)}, - *images, - ] - return mutable - for key in ("output", "content", "text"): - if key in mutable: - mutable[key] = _PRUNED_OUTPUT_TEMPLATE.format(tokens=original_tokens) - return mutable - mutable["output"] = _PRUNED_OUTPUT_TEMPLATE.format(tokens=original_tokens) - return mutable - - -@dataclass(frozen=True) -class ContextItem: - item_id: str - kind: str - origin: str - trust: str - visibility: str - priority: int - token_budget: int - truncation_policy: str - content_ref: str - content_hash: str - cache_class: str - created_seq: int - token_estimate: int - render_role: str = "user" - render_name: str = "" - pair_id: str = "" - message_group: str = "" - unit_id: str = "" - content: Any = field(default=None, repr=False, compare=False) - metadata: Mapping[str, Any] = field( - default_factory=FrozenDict, repr=False, compare=False - ) - - def __post_init__(self) -> None: - object.__setattr__(self, "content", freeze_json(self.content)) - object.__setattr__(self, "metadata", freeze_json(self.metadata or {})) - - @classmethod - def create( - cls, - *, - item_id: str, - kind: str, - origin: str, - trust: str, - visibility: str, - priority: int, - token_budget: int, - truncation_policy: str, - content: Any, - cache_class: str, - created_seq: int, - render_role: str = "user", - pair_id: str = "", - message_group: str = "", - unit_id: str = "", - content_ref: Optional[str] = None, - content_hash: Optional[str] = None, - token_estimate: Optional[int] = None, - render_name: str = "", - metadata: Optional[Mapping[str, Any]] = None, - ) -> "ContextItem": - normalized_content = _normalize_context_content(content) - if ( - isinstance(normalized_content, Mapping) - and str(normalized_content.get("type") or "") == "data" - ): - truncation_policy = POLICY_DROP - computed_hash = stable_hash(normalized_content) - supplied_hash = str(content_hash or "") - external_reference = normalized_content is None and bool(supplied_hash) - digest = supplied_hash if external_reference else computed_hash - estimate = ( - max(0, int(token_estimate or 0)) - if external_reference - else estimate_context_tokens(normalized_content) - ) - reference = ( - str(content_ref) - if content_ref is not None - and (external_reference or supplied_hash == computed_hash) - else f"sha256:{digest}" - ) - return cls( - item_id=str(item_id), - kind=str(kind), - origin=str(origin), - trust=str(trust), - visibility=str(visibility), - priority=int(priority), - token_budget=max(0, int(token_budget)), - truncation_policy=str(truncation_policy), - content_ref=reference, - content_hash=digest, - cache_class=str(cache_class), - created_seq=int(created_seq), - token_estimate=estimate, - render_role=str(render_role), - render_name=str(render_name or ""), - pair_id=str(pair_id or ""), - message_group=str(message_group or f"item:{item_id}"), - unit_id=str(unit_id or f"item:{item_id}"), - content=normalized_content, - metadata=metadata or {}, - ) - - def with_content(self, content: Any) -> "ContextItem": - normalized_content = _normalize_context_content(content) - digest = stable_hash(normalized_content) - return replace( - self, - content=freeze_json(normalized_content), - content_hash=digest, - content_ref=f"sha256:{digest}", - token_estimate=estimate_context_tokens(normalized_content), - ) - - def to_manifest(self) -> dict[str, Any]: - return { - "item_id": self.item_id, - "kind": self.kind, - "origin": self.origin, - "trust": self.trust, - "visibility": self.visibility, - "priority": self.priority, - "token_budget": self.token_budget, - "truncation_policy": self.truncation_policy, - "content_ref": self.content_ref, - "content_hash": self.content_hash, - "cache_class": self.cache_class, - "created_seq": self.created_seq, - "token_estimate": self.token_estimate, - "render_role": self.render_role, - "render_name_hash": stable_hash(self.render_name), - "pair_id": self.pair_id or None, - "message_group": self.message_group, - "unit_id": self.unit_id, - } - - -def make_text_context_item( - text: str, - *, - item_id: str, - kind: str, - origin: str, - trust: str, - created_seq: int, - priority: int = 700, - token_budget: int = 4_000, - truncation_policy: str = POLICY_HEAD_TAIL, - render_role: str = "user", - cache_class: str = "dynamic", -) -> ContextItem: - """Create a framework-neutral, explicitly-provenanced text item.""" - return ContextItem.create( - item_id=item_id, - kind=kind, - origin=origin, - trust=trust, - visibility=VISIBILITY_MODEL, - priority=priority, - token_budget=token_budget, - truncation_policy=truncation_policy, - content=str(text or ""), - cache_class=cache_class, - created_seq=created_seq, - render_role=render_role, - render_name=render_role, - message_group=item_id, - ) - - -def session_context_metadata(item: ContextItem) -> dict[str, Any]: - """Carry provenance on a positional session row without stale identity/sequence.""" - payload = item.to_manifest() - for key in ( - "item_id", - "created_seq", - "message_group", - "unit_id", - "content_ref", - "content_hash", - "token_estimate", - "render_name_hash", - ): - payload.pop(key, None) - return payload - - -@dataclass(frozen=True) -class ContextAssembly: - included: tuple[ContextItem, ...] - excluded: tuple[ContextItem, ...] - _manifest: Mapping[str, Any] = field(repr=False) - manifest_hash: str - used_tokens: int - total_budget: int - over_budget: bool = False - cut_units: int = 0 - pruned_items: int = 0 - - def __post_init__(self) -> None: - object.__setattr__(self, "included", tuple(self.included)) - object.__setattr__(self, "excluded", tuple(self.excluded)) - object.__setattr__(self, "_manifest", freeze_json(self._manifest)) - - @property - def manifest(self) -> dict[str, Any]: - """Return a detached JSON-serializable copy of the audit manifest.""" - manifest = thaw_json(self._manifest) - if not isinstance(manifest, dict): # pragma: no cover - frozen in __post_init__ - raise TypeError("context manifest must thaw to a dictionary") - return manifest - - -class ContextAssembler: - """Explicit, unit-level selection in context order. - - The unit of selection is a whole ReAct step — the reasoning, text and tool - calls the model produced together with the results those calls returned — - never a single block. A step is kept or dropped as one; the only thing a - budget may shorten inside it is a tool result's output. - - Ordering is **not** the assembler's job: items arrive in the order the agent - context already holds them, and they leave in that same order. The live - user instruction carries a sequence 64 strides above the context tail (see - ``next_request_sequence``), so sorting by sequence would move it behind the - turn it had just started. - - Budget pressure is resolved in a fixed order: per-item caps first, then - tool outputs of older steps are pruned (oldest first), and only then are - whole steps cut from the oldest end of the conversation. The steps of the - turn in progress — everything after the current user instruction — are - protected: they are never pruned, cut or split. When they alone exceed the - budget the assembly reports ``over_budget`` instead of quietly removing - anything; compaction owns that case. - """ - - def __init__( - self, - *, - total_budget: int, - budget_details: Optional[Mapping[str, int]] = None, - ) -> None: - self.total_budget = max(0, int(total_budget)) - self.budget_details = { - str(key): max(0, int(value)) - for key, value in sorted((budget_details or {}).items()) - } - - @staticmethod - def _mandatory(item: ContextItem) -> bool: - if item.visibility == VISIBILITY_MANIFEST_ONLY: - return True - if item.kind in STEP_KINDS: - return False - return item.truncation_policy == POLICY_NEVER - - @staticmethod - def _unit_key(items: Sequence[ContextItem]) -> tuple[Any, ...]: - return ( - -max(item.priority for item in items), - -max(item.created_seq for item in items), - tuple(sorted(item.item_id for item in items)), - ) - - @staticmethod - def _tokens(item: ContextItem) -> int: - return 0 if item.visibility == VISIBILITY_MANIFEST_ONLY else item.token_estimate - - def _cap_item( - self, - item: ContextItem, - records: dict[str, dict[str, Any]], - ) -> Optional[ContextItem]: - original = item.token_estimate - # Media still contributes to whole-context occupancy, but cannot spend - # or be cut by the per-item text allowance. - budget = item.token_budget + sum( - estimate_context_tokens(block) for block in _tool_result_images(item.content) - ) - if ( - item.visibility == VISIBILITY_MANIFEST_ONLY - or item.truncation_policy == POLICY_NEVER - or original <= budget - ): - records[item.item_id] = { - "action": "included", - "original_tokens": original, - "final_tokens": original, - } - return item - if item.truncation_policy in {POLICY_HEAD_TAIL, POLICY_TAIL} and item.token_budget > 0: - target = budget - capped = item.with_content( - _truncate_content(item.content, target, item.truncation_policy) - ) - # JSON escaping of newlines/quotes can exceed the initial text - # allowance. Refit using the measured cost, never discard an image - # result just because its accompanying text needs a smaller slice. - while capped.token_estimate > budget and target > 1: - target = max(1, target - (capped.token_estimate - budget)) - capped = item.with_content( - _truncate_content(item.content, target, item.truncation_policy) - ) - if capped.token_estimate <= budget: - records[item.item_id] = { - "action": "truncated", - "original_tokens": original, - "final_tokens": capped.token_estimate, - } - return capped - records[item.item_id] = { - "action": "excluded", - "reason": "item_budget", - "original_tokens": original, - "final_tokens": 0, - } - return None - - @staticmethod - def _tool_pairs_balanced(unit: Sequence[ContextItem]) -> bool: - pairs: dict[str, list[ContextItem]] = {} - for item in unit: - if item.kind in _TOOL_KINDS: - if not item.pair_id: - return False - pairs.setdefault(item.pair_id, []).append(item) - for pair_items in pairs.values(): - calls = [item for item in pair_items if item.kind == KIND_TOOL_CALL] - results = [item for item in pair_items if item.kind == KIND_TOOL_RESULT] - if not calls or len(calls) != len(results): - return False - return True - - def assemble(self, items: Iterable[ContextItem]) -> ContextAssembly: - raw_items = tuple(items) - item_ids = [item.item_id for item in raw_items] - duplicates = sorted( - item_id for item_id in set(item_ids) if item_ids.count(item_id) > 1 - ) - if duplicates: - raise ValueError(f"context item_id values must be unique: {duplicates}") - original_items = list(raw_items) - arrival = {item.item_id: index for index, item in enumerate(original_items)} - records: dict[str, dict[str, Any]] = {} - excluded: list[ContextItem] = [] - - capped: list[ContextItem] = [] - for item in original_items: - candidate = self._cap_item(item, records) - if candidate is None: - excluded.append(item) - continue - capped.append(candidate) - - # A tool result belongs to the step that issued its call, whatever unit - # the caller stamped on it: the pair id is the structural link. - call_units: dict[str, str] = {} - ambiguous_pairs: set[str] = set() - for item in capped: - if item.kind == KIND_TOOL_CALL and item.pair_id: - # One pair id may name a provider parallel batch (several calls - # in one unit); the same id across two units is unresolvable. - if call_units.get(item.pair_id, item.unit_id) != item.unit_id: - ambiguous_pairs.add(item.pair_id) - call_units[item.pair_id] = item.unit_id - units: dict[str, list[ContextItem]] = {} - for candidate in capped: - if candidate.kind in _TOOL_KINDS and candidate.pair_id in ambiguous_pairs: - records[candidate.item_id].update( - action="excluded", reason="malformed_tool_pair", final_tokens=0 - ) - excluded.append(candidate) - continue - if candidate.kind == KIND_TOOL_RESULT and candidate.pair_id in call_units: - candidate = replace(candidate, unit_id=call_units[candidate.pair_id]) - units.setdefault(candidate.unit_id, []).append(candidate) - - # The request that opened the turn in progress. Everything after it is - # the model's own live work and is protected as a whole. - request_index = -1 - for item in original_items: - if ( - item.kind == KIND_USER_INPUT - and item.truncation_policy == POLICY_NEVER - and item.visibility == VISIBILITY_MODEL - ): - request_index = arrival[item.item_id] - - def unit_start(unit: Sequence[ContextItem]) -> int: - return min(arrival[item.item_id] for item in unit) - - mandatory: list[list[ContextItem]] = [] - protected: list[list[ContextItem]] = [] - aside: list[list[ContextItem]] = [] - conversation: list[list[ContextItem]] = [] - for unit in units.values(): - is_step = any(item.kind in STEP_KINDS for item in unit) - if any(self._mandatory(item) for item in unit): - mandatory.append(unit) - elif is_step and request_index >= 0 and unit_start(unit) > request_index: - protected.append(unit) - elif not self._tool_pairs_balanced(unit): - for item in unit: - records[item.item_id].update( - action="excluded", reason="malformed_tool_pair", final_tokens=0 - ) - excluded.append(item) - elif all(item.kind in CONVERSATION_KINDS for item in unit): - conversation.append(unit) - else: - aside.append(unit) - - mandatory.sort(key=unit_start) - protected.sort(key=unit_start) - conversation.sort(key=unit_start) - # Selection order only for non-conversation material; it never reaches - # the output, which is re-sorted into context order. - aside.sort(key=self._unit_key) - - def unit_tokens(unit: Sequence[ContextItem]) -> int: - return sum(self._tokens(item) for item in unit) - - included: list[ContextItem] = [] - used = 0 - for unit in mandatory + protected: - included.extend(unit) - used += unit_tokens(unit) - over_budget = used > self.total_budget - - for unit in aside: - tokens = unit_tokens(unit) - remaining = max(0, self.total_budget - used) - if tokens <= remaining: - included.extend(unit) - used += tokens - continue - item = unit[0] - if ( - len(unit) == 1 - and remaining > 0 - and item.truncation_policy in {POLICY_HEAD_TAIL, POLICY_TAIL} - ): - shrunk = item.with_content( - _truncate_content(item.content, remaining, item.truncation_policy) - ) - if self._tokens(shrunk) < self._tokens(item) and self._tokens(shrunk) <= remaining: - records[item.item_id].update( - action="truncated", final_tokens=shrunk.token_estimate - ) - included.append(shrunk) - used += self._tokens(shrunk) - continue - for item in unit: - records[item.item_id].update(action="excluded", reason="budget", final_tokens=0) - excluded.append(item) - - # Conversation history: prune old tool outputs first, then cut whole - # steps from the oldest end. Never a hole in the middle. - pruned_items = 0 - conversation_tokens = sum(unit_tokens(unit) for unit in conversation) - if used + conversation_tokens > self.total_budget: - for unit in conversation: - if used + conversation_tokens <= self.total_budget: - break - for index, item in enumerate(unit): - if used + conversation_tokens <= self.total_budget: - break - if item.kind != KIND_TOOL_RESULT or item.truncation_policy == POLICY_NEVER: - continue - pruned = item.with_content(_prune_tool_output(item.content, item.token_estimate)) - if self._tokens(pruned) >= self._tokens(item): - continue - conversation_tokens -= self._tokens(item) - self._tokens(pruned) - unit[index] = pruned - records[item.item_id].update(action="pruned", final_tokens=pruned.token_estimate) - pruned_items += 1 - cut_units = 0 - while conversation and used + conversation_tokens > self.total_budget: - dropped = conversation.pop(0) - conversation_tokens -= unit_tokens(dropped) - cut_units += 1 - for item in dropped: - records[item.item_id].update(action="excluded", reason="budget_cut", final_tokens=0) - excluded.append(item) - for unit in conversation: - included.extend(unit) - used += unit_tokens(unit) - - included.sort(key=lambda item: arrival[item.item_id]) - excluded_by_id = {item.item_id: item for item in excluded} - excluded = sorted(excluded_by_id.values(), key=lambda item: arrival[item.item_id]) - - included_manifest = [] - excluded_manifest = [] - included_ids = {item.item_id for item in included} - final_by_id = {item.item_id: item for item in included} - for original in original_items: - record = dict(records.get(original.item_id) or {}) - final = final_by_id.get(original.item_id, original) - entry = final.to_manifest() - entry.update(record) - if original.item_id in included_ids: - included_manifest.append(entry) - else: - excluded_manifest.append(entry) - - payload = { - "schema_version": CONTEXT_SCHEMA_VERSION, - "total_budget": self.total_budget, - "used_tokens": used, - "over_budget": over_budget, - "protected_units": len(protected), - "cut_units": cut_units, - "pruned_items": pruned_items, - "included": included_manifest, - "excluded": excluded_manifest, - } - if self.budget_details: - payload["budget_details"] = dict(self.budget_details) - manifest_hash = stable_hash(payload) - return ContextAssembly( - included=tuple(included), - excluded=tuple(excluded), - _manifest=payload, - manifest_hash=manifest_hash, - used_tokens=used, - total_budget=self.total_budget, - over_budget=over_budget, - cut_units=cut_units, - pruned_items=pruned_items, - ) - - -__all__ = [ - "CONTEXT_SCHEMA_VERSION", - "CONTEXT_SEQUENCE_STRIDE", - "CONVERSATION_KINDS", - "STEP_KINDS", - "SESSION_CONTEXT_META_KEY", - "ContextAssembler", - "ContextAdapterProtocol", - "ContextAssembly", - "ContextItem", - "KIND_ASSISTANT", - "KIND_ATTACHMENT", - "KIND_COMPACTION", - "KIND_MEMORY", - "KIND_IDENTITY", - "KIND_PROJECT", - "KIND_REFERENCE", - "KIND_REMINDER", - "KIND_STEER", - "KIND_SYSTEM_RULE", - "KIND_TOOL_CALL", - "KIND_TOOL_RESULT", - "KIND_USER_INPUT", - "POLICY_DROP", - "POLICY_HEAD_TAIL", - "POLICY_NEVER", - "POLICY_TAIL", - "VISIBILITY_MANIFEST_ONLY", - "VISIBILITY_MODEL", - "estimate_context_tokens", - "make_text_context_item", - "session_context_metadata", -] diff --git a/src/backend/core/llm/factory/README.md b/src/backend/core/llm/factory/README.md index 1cd4e5c4..e9839f53 100644 --- a/src/backend/core/llm/factory/README.md +++ b/src/backend/core/llm/factory/README.md @@ -1,7 +1,7 @@ # Agent factory Public entry: `from core.llm.factory import create_agent_executor`. -`warmup_mcp_tools` is the startup entry for shared MCP discovery. +`mcp_connections` discovers native MCP capabilities within the authorized run. The initializer loads these entries lazily so importing selection helpers does not initialize the model/tool stack. No legacy `agent_factory` module remains. @@ -17,8 +17,8 @@ not initialize the model/tool stack. No legacy `agent_factory` module remains. | `runtime/` | Select the model and budgets; attach middleware, manifests and listeners | Agent execution policy changes | | `models.py`, `defaults.py` | Assembly-owned value types and defaults | Multiple phases share a domain value | -The factory consumes `core.plugins.runtime`; it does not own plugin lifecycle -or manifest ingestion. Plugin behavior belongs in `core/plugins/`. +The factory consumes `core.capabilities.plugins.runtime`; it does not own plugin lifecycle +or manifest ingestion. Plugin behavior belongs in `core/capabilities/plugins/`. The orchestration layer calls the public entry; it should not coordinate the private phases itself. Keep phase inputs explicit rather than introducing a mutable catch-all context or arbitrary pipeline registry. diff --git a/src/backend/core/llm/factory/__init__.py b/src/backend/core/llm/factory/__init__.py index 2e2b49c5..a5f0a66a 100644 --- a/src/backend/core/llm/factory/__init__.py +++ b/src/backend/core/llm/factory/__init__.py @@ -6,11 +6,7 @@ def __getattr__(name): from .build import create_agent_executor return create_agent_executor - if name == "warmup_mcp_tools": - from .tools.mcp_config import warmup_mcp_tools - - return warmup_mcp_tools raise AttributeError(name) -__all__ = ["create_agent_executor", "warmup_mcp_tools"] +__all__ = ["create_agent_executor"] diff --git a/src/backend/core/llm/factory/assembly.py b/src/backend/core/llm/factory/assembly.py index 6e0380e1..eb682a07 100644 --- a/src/backend/core/llm/factory/assembly.py +++ b/src/backend/core/llm/factory/assembly.py @@ -11,8 +11,8 @@ from core.llm.factory.runtime.manifest_binding import manifest_binding from core.llm.factory.runtime.middleware_stack import middleware_stack from core.llm.factory.runtime.model_selection import model_selection -from core.llm.factory.selection.device_bindings import device_bindings -from core.llm.factory.selection.device_preparation import device_preparation +from core.llm.factory.runtime.capability_bindings import device_bindings +from core.harness.registry.run_capabilities import prepare_run_capabilities from core.llm.factory.selection.entry_policy import entry_policy from core.llm.factory.selection.mode_bindings import mode_bindings from core.llm.factory.selection.profile_selection import profile_selection @@ -28,7 +28,7 @@ async def assemble_agent(request: AgentRequest): - from core.llm.project_environment import bind_project_environment + from core.llm.runtime.project_environment import bind_project_environment _caps_skill_ids = None ( @@ -124,7 +124,7 @@ async def assemble_agent(request: AgentRequest): _caps_skill_ids, _desktop_progressive, _prepared_capabilities, - ) = await device_preparation( + ) = await prepare_run_capabilities( request, _api_scope=_api_scope, _capability_run_key=_capability_run_key, @@ -177,11 +177,9 @@ async def assemble_agent(request: AgentRequest): _required_connector_tool_names, _required_plugin_mcp_tool_names, _tool_approval_available, - http_clients, + mcp_resources, loader, - mcp_clients, toolkit, - transient_mcp_clients, ) = await mcp_connections( request, _api_scope=_api_scope, @@ -261,9 +259,8 @@ async def assemble_agent(request: AgentRequest): required=required, _required_connector_tool_names=_required_connector_tool_names, _required_plugin_mcp_tool_names=_required_plugin_mcp_tool_names, - http_clients=http_clients, + mcp_resources=mcp_resources, loader=loader, - mcp_clients=mcp_clients, skill_ids_to_register=skill_ids_to_register, toolkit=toolkit, ) @@ -400,10 +397,9 @@ async def assemble_agent(request: AgentRequest): _manifest_for_surface=_manifest_for_surface, _plugin_runtime=_plugin_runtime, agent=agent, - http_clients=http_clients, + mcp_resources=mcp_resources, profile=profile, skill_selection=skill_selection, system_prompt=system_prompt, toolkit=toolkit, - transient_mcp_clients=transient_mcp_clients, ) diff --git a/src/backend/core/llm/factory/build.py b/src/backend/core/llm/factory/build.py index 2b68d9df..39c40de5 100644 --- a/src/backend/core/llm/factory/build.py +++ b/src/backend/core/llm/factory/build.py @@ -8,8 +8,8 @@ from agentscope.mcp import MCPClient from core.llm.factory.assembly import assemble_agent from core.llm.factory.request import AgentRequest -from core.llm.mcp_manager import close_factory_clients_on_error -from orchestration.registry import AgentSpec +from core.llm.factory.specs import AgentSpec +from core.llm.mcp.manager import close_factory_clients_on_error @close_factory_clients_on_error @@ -115,7 +115,7 @@ async def create_agent_executor( required_plugin_name: Optional[str] = None, required_plugin_skill_ids: Optional[List[str]] = None, required_plugin_mcp_ids: Optional[List[str]] = None, - # mode_spec: 对话模式的装配契约（core/services/chat_mode_service.ChatModeSpec）。 + # mode_spec: 对话模式的装配契约（core.chat.mode_types.ChatModeSpec）。 # 「模式」把原来写死的极速模式泛化成一张表：工具面 / 技能 / 插件 / 提示词 kind / # 迭代上限都由它给。turbo_mode 现在的含义是"这个模式要收窄工具面"，收窄成什么 # 由 mode_spec 说了算。缺省（老调用方没传）时退回历史的 turbo.* 配置键读取， diff --git a/src/backend/core/llm/factory/defaults.py b/src/backend/core/llm/factory/defaults.py index 2aeabf55..d6705a88 100644 --- a/src/backend/core/llm/factory/defaults.py +++ b/src/backend/core/llm/factory/defaults.py @@ -2,6 +2,7 @@ from __future__ import annotations +from dataclasses import dataclass, field from typing import Dict from core.config.settings import settings as _settings @@ -90,9 +91,6 @@ "一旦写入，「哪些还没做」就不再可判定。\n" ) -_HTTP_MCP_FAIL_AT: Dict[str, float] = {} - -_HTTP_MCP_FAIL_COOLDOWN_S = 60.0 KB_MCP_HTTP_URL = ( f"http://{_settings.server.mcp_host}:{_MCP_PORTS['retrieve_dataset_content']}/mcp/" @@ -109,3 +107,15 @@ "# 可用技能（技能名、说明文件、适用场景）：{% for skill in skills %}\n" "- `{{ skill.name }}`：`{{ skill.dir }}/SKILL.md` — {{ skill.description }}{% endfor %}" ) + + +@dataclass +class DefaultsState: + HTTP_MCP_FAIL_AT: Dict[str, float] = field(default_factory=lambda: {}) + + +_STATE = DefaultsState() + + +def process_state() -> DefaultsState: + return _STATE diff --git a/src/backend/core/llm/factory/prompts/policy.py b/src/backend/core/llm/factory/prompts/policy.py index eabe1708..cfeff1b8 100644 --- a/src/backend/core/llm/factory/prompts/policy.py +++ b/src/backend/core/llm/factory/prompts/policy.py @@ -4,9 +4,10 @@ import logging -logger = logging.getLogger(__name__) from core.llm.factory.defaults import DYNAMIC_BLOCK_HEADER +logger = logging.getLogger(__name__) + def _render_turn_budget_hint(max_iters: int) -> str: """Tell an agent that *does* have a turn budget how to spend it. diff --git a/src/backend/core/llm/factory/prompts/system.py b/src/backend/core/llm/factory/prompts/system.py index aa76009d..7aec1737 100644 --- a/src/backend/core/llm/factory/prompts/system.py +++ b/src/backend/core/llm/factory/prompts/system.py @@ -1,15 +1,17 @@ """Agent assembly phase: system prompt. """ from __future__ import annotations +from core.llm.tools.plugin_activation import register_load_plugin +from core.llm import prompt_rendering import os from typing import Any, Dict from core.llm.factory.defaults import _WORKFLOW_MODE_HINT from core.llm.factory.request import AgentRequest -from core.llm.project_environment import validate_project_environment -from core.ontology.validator import render_runtime_prompt -from core.services.system_config import code_capability_enabled +from core.llm.runtime.project_environment import validate_project_environment +from core.ontology.runtime_pack import render_runtime_prompt +from core.config.store.execution_options import code_capability_enabled from prompts.prompt_runtime import build_subagent_system_prompt, build_system_prompt @@ -37,7 +39,7 @@ async def system_prompt( toolkit, ): if request.profile: - from core.extensions import agent_profile + from core.harness.registry import agent_profile system_prompt = await agent_profile(request.profile).system_prompt(request) return _agent_ref, _plan_tool_enabled, _plugin_runtime, system_prompt @@ -93,7 +95,7 @@ async def system_prompt( # Bound plugins deferred in the override block above; in-run activation # only (persist=False — see the note there). if _subagent_progressive is not None: - from core.plugins import runtime as _plug + from core.capabilities.plugins import runtime as _plug _plugin_dir_section = _plug.build_plugin_directory_section( _subagent_progressive.directory @@ -128,7 +130,7 @@ async def system_prompt( "approval_available": _tool_approval_available, "ontology_runtime": _ontology_runtime, } - _plug.register_load_plugin( + register_load_plugin( toolkit, _subagent_progressive.deferred_by_slug(), _plugin_runtime ) _log.info( @@ -146,14 +148,13 @@ async def system_prompt( # 能配自己的提示词。之前把它写在 turbo 分支里，非收窄模式配了也不生效。 _mode_prompt = "" if request.mode_spec is not None: - from core.services import prompt_version_service as _pvs_mode _mode_prompt_text = getattr(request.mode_spec, "prompt_text", None) _mode_prompt_kind = getattr(request.mode_spec, "prompt_kind", None) if _mode_prompt_text: _mode_prompt = str(_mode_prompt_text) elif _mode_prompt_kind and _mode_prompt_kind != "turbo": - _mode_prompt = _pvs_mode.render_kind_segment(_mode_prompt_kind, fs_fallback=False) + _mode_prompt = prompt_rendering.render_kind_segment(_mode_prompt_kind, fs_fallback=False) if request.turbo_mode and not _turbo_code_exec: # Turbo swaps in the standalone prompt (DB "turbo" active version → @@ -162,10 +163,9 @@ async def system_prompt( # （开了代码执行位的收窄模式不进这条：装配确实带沙箱/文件工具， # turbo 正文"秒级检索、不执行代码"的叙事反而是错的——没配专属 # 提示词时走默认装配，工具段按实际 toolkit 动态生成。） - from core.services import prompt_version_service as _pvs_turbo # 收窄模式没配专属提示词时退回历史的 turbo 正文——极速模式绑的就是它。 - system_prompt = _mode_prompt or _pvs_turbo.render_turbo_system_prompt() + system_prompt = _mode_prompt or prompt_rendering.render_turbo_system_prompt() from core.config.local_mode import local_mode_enabled if local_mode_enabled(): @@ -275,9 +275,9 @@ async def system_prompt( # Single source of truth render_kind_segment (same source as the Config console preview). if code_capability_enabled() and (not request.turbo_mode or _turbo_code_exec): try: - from core.services import prompt_version_service as _pvs + from core.llm import prompt_versions as _pvs - _code_exec_text = _pvs.render_kind_segment("code_exec") + _code_exec_text = prompt_rendering.render_kind_segment("code_exec") except Exception: _code_exec_text = "" if _code_exec_text: @@ -302,7 +302,7 @@ async def system_prompt( # independent of activation state) so an activation never perturbs this # part of the prefix; only the tools/skills it adds do, once. if _progressive is not None: - from core.plugins import runtime as _plug + from core.capabilities.plugins import runtime as _plug _plugin_dir_section = _plug.build_plugin_directory_section(_progressive.directory) if _plugin_dir_section: @@ -334,7 +334,7 @@ async def system_prompt( "approval_available": _tool_approval_available, "ontology_runtime": _ontology_runtime, } - _plug.register_load_plugin(toolkit, _progressive.deferred_by_slug(), _plugin_runtime) + register_load_plugin(toolkit, _progressive.deferred_by_slug(), _plugin_runtime) _log.info( "[factory] +%s progressive plugins: %d in directory, %d deferred " "(skills=%d, mcp=%d)", @@ -361,8 +361,8 @@ async def system_prompt( # ── Register call_subagent tool for main agent ── if request.visible_subagents: - from core.llm.builtin_subagents import refresh_builtin_subagents - from core.llm.subagent_tool import build_subagent_prompt_section, register_subagent_tool + from core.capabilities.agents.builtin import refresh_builtin_subagents + from core.llm.subagents.tool import build_subagent_prompt_section, register_subagent_tool # Refresh platform-default rows only after the parent toolset has # completed catalog defaults, permission filtering, skill-bound MCP @@ -443,13 +443,10 @@ async def system_prompt( # derived/non-interactive constructions get nothing by default. The DB # switch auto_plan_entry_enabled (which itself returns False on # config-layer errors) can turn this off entirely. - from core.services.system_config import auto_plan_entry_enabled + from core.config.store.execution_options import auto_plan_entry_enabled if request.top_level_chat and auto_plan_entry_enabled(): - from core.llm.plan_update_tool import ( - build_plan_update_prompt_section, - register_plan_update_tool, - ) + from core.llm.tools.plan_update import build_plan_update_prompt_section, register_plan_update_tool # 计划栏的催更中间件挂在同一个正向开关上：工具没注册就绝不该有人催更新。 _plan_tool_enabled = True diff --git a/src/backend/core/llm/factory/request.py b/src/backend/core/llm/factory/request.py index 09ef6c5a..51af1a0c 100644 --- a/src/backend/core/llm/factory/request.py +++ b/src/backend/core/llm/factory/request.py @@ -5,12 +5,12 @@ from dataclasses import dataclass from typing import Any, Dict, List, Optional -from orchestration.registry import AgentSpec +from core.llm.factory.specs import AgentSpec @dataclass class AgentRequest: - # A registered module agent profile (core.extensions.AgentProfile) and that + # A registered module agent profile (core.harness.registry.AgentProfile) and that # module's own reference, e.g. the task the dedicated agent works on. profile: Optional[str] = None profile_ref: Optional[str] = None diff --git a/src/backend/core/llm/factory/runtime/agent_construction.py b/src/backend/core/llm/factory/runtime/agent_construction.py index 0b545539..8ca8118e 100644 --- a/src/backend/core/llm/factory/runtime/agent_construction.py +++ b/src/backend/core/llm/factory/runtime/agent_construction.py @@ -4,14 +4,15 @@ import logging -logger = logging.getLogger(__name__) import os from agentscope.agent import ReActConfig -from agentscope.agent._config import ModelConfig -from core.llm.compacting_agent import CompactingAgent +from agentscope.agent import ModelConfig, InjectionConfig +from core.llm.context.compacting_agent import CompactingAgent from core.llm.factory.request import AgentRequest +logger = logging.getLogger(__name__) + async def agent_construction( request: AgentRequest, @@ -63,11 +64,11 @@ async def agent_construction( _offloader = None if not request.disable_tools and os.getenv("SANDBOX_TOOLS_ENABLED", "true").lower() == "true": try: - from core.llm.offloader import SandboxOffloader + from core.llm.context.offloader import SandboxOffloader from core.sandbox.factory import get_sandbox_provider if _api_scope is not None: - from core.llm.agent_api_tools import _provider as api_sandbox_provider + from core.llm.api_runtime.agent_tools import _provider as api_sandbox_provider _offloader = SandboxOffloader( api_sandbox_provider(), @@ -83,7 +84,7 @@ async def agent_construction( # CompactingAgent, not the bare framework Agent: its compress_context # override is what routes the ReAct step boundary into the one compaction - # engine and persists the checkpoint (see core/llm/compacting_agent.py). + # engine and persists the checkpoint (see core/llm/context/compacting_agent.py). agent = CompactingAgent( name=_agent_name, system_prompt=system_prompt, @@ -93,7 +94,8 @@ async def agent_construction( state=_state, context_config=context_config, model_config=ModelConfig(max_retries=3, fallback_model=None), - react_config=ReActConfig(max_iters=_max_iters), + injection_config=InjectionConfig(inject_runtime_state=False), + react_config=ReActConfig(max_iters=_max_iters, interruption_raise_cancelled_error=True), offloader=_offloader, ) # Compaction trigger accounting reuses the exact already-frozen execution @@ -107,8 +109,9 @@ async def agent_construction( # and a seed write on a cold config cache, inside the ReAct loop). try: agent._jx_trigger_ratio = _trigger_ratio - except Exception: # pragma: no cover - agent may reject attributes - pass + except Exception as fallback_error: # pragma: no cover - agent may reject attributes + from core.infra.failures import record_best_effort_failure + record_best_effort_failure('core.llm.factory.runtime.agent_construction:agent_construction', fallback_error) # Set the agent reference so the call_subagent closure can extract shared context if _agent_ref is not None: diff --git a/src/backend/core/llm/factory/selection/device_bindings.py b/src/backend/core/llm/factory/runtime/capability_bindings.py similarity index 93% rename from src/backend/core/llm/factory/selection/device_bindings.py rename to src/backend/core/llm/factory/runtime/capability_bindings.py index c6006bf7..768d7398 100644 --- a/src/backend/core/llm/factory/selection/device_bindings.py +++ b/src/backend/core/llm/factory/runtime/capability_bindings.py @@ -4,8 +4,7 @@ import asyncio -from core.capabilities import runtime as capability_runtime -from core.llm.agent_api_runtime import scope_mcp_servers +from core.llm.api_runtime.agent import scope_mcp_servers from core.llm.factory.models import RequiredCapabilities, StickyCapabilities from core.llm.factory.request import AgentRequest from core.llm.factory.tools import mcp_config as factory_mcp_config @@ -58,8 +57,7 @@ async def device_bindings( ) if _prepared_capabilities is not None: enabled_servers = await asyncio.to_thread( - capability_runtime.bind_mcp, - _prepared_capabilities, + _prepared_capabilities.bind_mcp, enabled_servers, _capability_mcp_resolution[0] if _capability_mcp_resolution else None, ) @@ -73,8 +71,7 @@ async def device_bindings( if required.plugin_id: _dependency_plugins.append(required.plugin_id) _prepared_capabilities = await asyncio.to_thread( - capability_runtime.preflight, - _prepared_capabilities, + _prepared_capabilities.preflight, plugin_ids=list(dict.fromkeys(_dependency_plugins)), available_mcp=set(enabled_servers), available_kb=set(request.enabled_kb_ids or []), @@ -115,7 +112,7 @@ async def device_bindings( else: _progressive = _desktop_progressive - enabled_servers = factory_mcp_config._inject_runtime_headers( + enabled_servers = factory_mcp_config.inject_runtime_headers( enabled_servers, current_user_id=request.current_user_id, chat_id=request.chat_id, diff --git a/src/backend/core/llm/factory/runtime/execution_budget.py b/src/backend/core/llm/factory/runtime/execution_budget.py index 9d5f7d0f..9558944e 100644 --- a/src/backend/core/llm/factory/runtime/execution_budget.py +++ b/src/backend/core/llm/factory/runtime/execution_budget.py @@ -4,8 +4,8 @@ from agentscope.agent import ContextConfig from core.config.settings import settings as _settings -from core.llm.compaction import SUMMARIZATION_PROMPT -from core.llm.context_manager import AUTO_COMPACT_MAX_RATIO +from core.llm.context.compaction import SUMMARIZATION_PROMPT +from core.models.context_window import AUTO_COMPACT_MAX_RATIO from core.llm.factory.prompts import policy as factory_prompt_policy from core.llm.factory.request import AgentRequest @@ -24,7 +24,7 @@ async def execution_budget( ): _ctx_window = int(getattr(default_model, "context_size", 0) or 0) # Resolve the shared ratio once instead of reading config at each ReAct step. - from core.services.compaction_service import resolve_token_limit, resolve_trigger_ratio + from core.harness.registry.compaction import resolve_token_limit, resolve_trigger_ratio _trigger_ratio = resolve_trigger_ratio() _log.info( @@ -128,7 +128,7 @@ async def execution_budget( # like all env config). if profile.max_react_turns: _max_iters = profile.max_react_turns - from core.config.settings import _env, _int + from core.config.settings import env_value as _env, env_int as _int _env_iters = _int(_env("CHAT_MAIN_MAX_ITERS"), 0) if _env_iters > 0: @@ -138,7 +138,7 @@ async def execution_budget( # Hard cap for the quick-lookup contract: 1-2 retrieval rounds (each may # fan out parallel calls) plus the final answer. Admin-tunable via # 系统配置「极速模式」turbo.max_iters; wins over profile/env. - from core.services.system_config import turbo_max_iters + from core.config.store.execution_options import turbo_max_iters _mode_iters = getattr(request.mode_spec, "max_iters", None) if request.mode_spec else None if _mode_iters: diff --git a/src/backend/core/llm/factory/runtime/listeners.py b/src/backend/core/llm/factory/runtime/listeners.py index 2b4738c0..5ace4861 100644 --- a/src/backend/core/llm/factory/runtime/listeners.py +++ b/src/backend/core/llm/factory/runtime/listeners.py @@ -15,17 +15,16 @@ async def runtime_listeners( _manifest_for_surface, _plugin_runtime, agent, - http_clients, + mcp_resources, profile, skill_selection, system_prompt, toolkit, - transient_mcp_clients, ): async def _publish_context_manifest(request_manifest): # noqa: ANN001, ANN202 """Bind the exact post-budget context manifest used for this request.""" try: - from core.evolution.runtime_binding import rebind_execution_manifest + from core.evolution.cycle.runtime_binding import rebind_execution_manifest request_bundle = rebind_execution_manifest( run_id=request.run_id, @@ -46,7 +45,7 @@ async def _publish_context_manifest(request_manifest): # noqa: ANN001, ANN202 # treats a missing bundle as partial instead of falsely persisting # the pre-context surface as complete request evidence. agent.clear_request_evidence(request_manifest) - from core.evolution.runtime_binding import clear_run_binding + from core.evolution.cycle.runtime_binding import clear_run_binding clear_run_binding(request.run_id, capability_scope=request.capability_scope) @@ -71,15 +70,17 @@ async def _publish_surface_generation(surface): # noqa: ANN001, ANN202 _log.warning("[manifest] surface generation refresh unavailable: %s", exc) try: agent.bind_execution_surface(None, bundle=_bind_manifest(None)) - except Exception: # pragma: no cover - last-resort availability path - pass + except Exception as fallback_error: # pragma: no cover - last-resort availability path + from core.infra.failures import record_best_effort_failure + record_best_effort_failure('core.llm.factory.runtime.listeners:_publish_surface_generation', fallback_error) toolkit.set_execution_surface_listener(_publish_surface_generation) if skill_selection is not None: try: setattr(agent, "_jx_skill_selection", skill_selection) - except Exception: # pragma: no cover - pass + except Exception as fallback_error: # pragma: no cover + from core.infra.failures import record_best_effort_failure + record_best_effort_failure('core.llm.factory.runtime.listeners:runtime_listeners', fallback_error) # Which assembly governed this run, for the turn card. Carried rather than # re-resolved after the response: by then a profile may have been published # or switched off, and the card would name one the run never used. @@ -94,17 +95,21 @@ async def _publish_surface_generation(surface): # noqa: ANN001, ANN202 "narrowed_tools": profile.tool_allowlist is not None, }, ) - except Exception: # pragma: no cover - pass + except Exception as fallback_error: # pragma: no cover + from core.infra.failures import record_best_effort_failure + record_best_effort_failure('core.llm.factory.runtime.listeners:runtime_listeners', fallback_error) _log.info("[factory] +%s agent created, TOTAL setup done", _elapsed()) - # Only transient (per-request) stdio clients + HTTP clients get closed; - # pooled stable clients stay open for reuse (closing them would defeat the - # pool and hand dead clients to the next request). - all_transient = [*transient_mcp_clients, *http_clients] + for resource in mcp_resources: + if resource.interactions is not None: + resource.interactions.agent = agent + # Release each run's leases; chat-owned STDIO sessions expire independently. + all_transient = list(mcp_resources) # Clients connected by a mid-run load_plugin activation are appended to this # same list object, so the caller's close_clients() teardown covers them. if _plugin_runtime is not None: _plugin_runtime["close_list"] = all_transient + _plugin_runtime["agent"] = agent + _plugin_runtime["request"] = request return agent, all_transient diff --git a/src/backend/core/llm/factory/runtime/manifest_binding.py b/src/backend/core/llm/factory/runtime/manifest_binding.py index 5cc32d85..6daa142e 100644 --- a/src/backend/core/llm/factory/runtime/manifest_binding.py +++ b/src/backend/core/llm/factory/runtime/manifest_binding.py @@ -2,7 +2,7 @@ from __future__ import annotations -from core.llm.execution_manifest import tool_manifest_from_schemas +from core.harness.execution_manifest import tool_manifest_from_schemas from core.llm.factory.request import AgentRequest @@ -46,7 +46,7 @@ def _manifest_for_surface(surface): # noqa: ANN001, ANN202 ) def _bind_manifest(manifest): # noqa: ANN001, ANN202 - from core.evolution.runtime_binding import bind_runtime_assets + from core.evolution.cycle.runtime_binding import bind_runtime_assets return bind_runtime_assets( run_id=request.run_id, diff --git a/src/backend/core/llm/factory/runtime/middleware_stack.py b/src/backend/core/llm/factory/runtime/middleware_stack.py index cc4d4edd..493ff855 100644 --- a/src/backend/core/llm/factory/runtime/middleware_stack.py +++ b/src/backend/core/llm/factory/runtime/middleware_stack.py @@ -2,27 +2,11 @@ from __future__ import annotations -from core.llm.steer_middleware import SteerMiddleware -from core.llm.command_middleware import CommandCompletionMiddleware -from core.llm.agentscope_hook_adapter import AgentScopeHookAdapter +from core.llm.runtime.agentscope_hook_adapter import AgentScopeHookAdapter from core.llm.factory.request import AgentRequest from core.llm.factory.runtime import evidence as factory_evidence_helpers -from core.llm.middlewares import ( - ActingToolCallIdMiddleware, - AgentRuntimeState, - CitationAnchorMiddleware, - DynamicModelMiddleware, - FileContextMiddleware, - FinishPinGuardMiddleware, - IterBudgetReminderMiddleware, - JobLedgerReminderMiddleware, - OntologyGateMiddleware, - PlanStaleReminderMiddleware, - StallInterventionMiddleware, - ToolEffectMiddleware, - WorkspacePinHintMiddleware, -) +from core.llm.runtime.state import AgentRuntimeState async def middleware_stack( @@ -43,14 +27,11 @@ async def middleware_stack( tool_schemas, toolkit, ): - from core.llm.tool_permissions import ( - PermissionRuntime, - ToolPermissionMiddleware, - ToolPermissionRegistry, - ToolPermissionService, - ) + from core.sandbox.permissions.dispatch_models import PermissionRuntime + from core.sandbox.permissions.tool_specs import ToolPermissionRegistry + from core.llm.tool_permissions import ToolPermissionService - _builtin_tool_names = {ft.name for ft in toolkit.function_tools} + _builtin_tool_names = {ft.name for ft in list(toolkit)} _all_tool_names = { str(s.get("function", {}).get("name") or "") for s in tool_schemas @@ -84,7 +65,7 @@ async def middleware_stack( ) # AgentScope's native permission engine remains a coarse first gate. Names - # present in the registry additionally pass ToolPermissionMiddleware and + # present in the registry additionally pass the permission hook and # the final Toolkit ticket guard below. from agentscope.permission import PermissionContext @@ -104,62 +85,51 @@ async def middleware_stack( permission_context=PermissionContext(), ) - _policy_middlewares: list = [ - DynamicModelMiddleware(), # on_reply: switch models by chat_mode - FileContextMiddleware(), # on_reply: inject file context - *([CommandCompletionMiddleware()] if _eval_scope is None and _api_scope is None else []), - SteerMiddleware(), # on_acting/on_reasoning: inject queued user steer before tool I/O - WorkspacePinHintMiddleware(), # on_reasoning: remind to pin - IterBudgetReminderMiddleware(), # on_reasoning: inject a wrap-up reminder near max_iters - # on_acting: the active profile's intervention rules, applied to *this* - # loop. Previously they only reached the autonomous loop, which left the - # orchestration profile with one field that governed nothing on the axis - # almost all traffic takes. - StallInterventionMiddleware(profile.intervention_rules), - OntologyGateMiddleware(_ontology_runtime), # on_acting: zero-LLM L-a contract gate - ToolPermissionMiddleware(_permission_service), - CitationAnchorMiddleware(), # on_acting: 证据锚点——工具结果回给模型前发号回注 cite_id - ActingToolCallIdMiddleware(), # on_acting: expose call_subagent's tool_call.id to tools (parent-child linkage) - ToolEffectMiddleware(), # on_acting: durable Intent before every actual tool invocation - ] - if request.project_environment is not None: - from core.llm.project_environment_middleware import ProjectEnvironmentMiddleware - - _policy_middlewares.insert(0, ProjectEnvironmentMiddleware(request)) + from core.harness.registry import hook_specs - # on_reasoning: 会话里有未收敛的批量作业时，每轮把台账数字回灌进上下文。 - # 进度是外部事实（job_items 表），不是模型的记忆——不主动回灌，隔十几轮之后就会 - # 退化成"边际收益递减，先交付吧"（568 行只补 66 行正是这么停的）。 - # 子作业内部不挂（isolated），避免嵌套噪声。 - if request.workflow_mode and not request.isolated and request.chat_id: - _policy_middlewares.append( - JobLedgerReminderMiddleware(chat_id=request.chat_id, user_id=request.current_user_id) - ) - # on_reasoning: 计划栏停在半路时把当前清单回灌回去，催模型调 update_plan。 - # 只在真的注册了 update_plan 工具的那条路径上挂（同一个正向开关），派生/非交互 - # 的构造一律拿不到——催一个不存在的工具只会让模型编造调用。 - if _plan_tool_enabled: - _policy_middlewares.append(PlanStaleReminderMiddleware()) - # The legacy guard directly pins ids found in arbitrary tool output. API - # delivery must go through the scoped tool's ownership check and persistence. - if _api_scope is None and _eval_scope is None: - _policy_middlewares.append(FinishPinGuardMiddleware()) + from core.llm.tool_permissions.hook import ToolPermissionHook + permission_policy = ToolPermissionHook(_permission_service) + adapter = AgentScopeHookAdapter(permission_policy=permission_policy, durable_tools=True) + from core.harness.registry.hook_factories import create_steering_policy + steering = None if request.profile: - from core.extensions import agent_profile - - for middleware in agent_profile(request.profile).middlewares(request): - if isinstance(middleware, SteerMiddleware): - # Profiles change the source, not its position before approvals - # and durable tool effects. - index = next(i for i, item in enumerate(_policy_middlewares) - if isinstance(item, SteerMiddleware)) - _policy_middlewares[index] = middleware - else: - _policy_middlewares.append(middleware) - # The Agent sees one framework adapter. Transitional AgentScope policies - # execute inside its compatibility chain and can be deleted one by one as - # their neutral HookSpec replacements reach parity. - _middlewares: list = [AgentScopeHookAdapter(legacy_middlewares=tuple(_policy_middlewares))] + from core.harness.registry import agent_profile + selected_profile = agent_profile(request.profile) + steering = selected_profile.steering(request) + for spec in selected_profile.hooks(request): + adapter.bus.register(spec) + for spec in (steering or create_steering_policy()).specs(): + adapter.bus.register(spec) + if request.project_environment is not None: + from core.llm.runtime.project_environment_hook import project_environment_hook + adapter.bus.register(project_environment_hook(request)) + if _eval_scope is None and _api_scope is None: + from core.harness.registry.hook_factories import command_completion_hooks + for spec in command_completion_hooks(): + adapter.bus.register(spec) + adapter.bus.register(permission_policy.spec()) + from core.models.selection_hook import model_selection_hook + adapter.bus.register(model_selection_hook()) + from core.llm.citations.hook import citation_hook + adapter.bus.register(citation_hook()) + from core.ontology.tool_hook import ontology_tool_hooks + for spec in ontology_tool_hooks(_ontology_runtime): + adapter.bus.register(spec) + from core.evolution.profiles.stall_hook import stall_hooks + for spec in stall_hooks(profile.intervention_rules): + adapter.bus.register(spec) + from core.llm.context.attachment_hook import attachment_context_hook + adapter.bus.register(attachment_context_hook()) + from core.llm.context.reasoning_hooks import iteration_budget_hook + adapter.bus.register(iteration_budget_hook()) + from core.llm.context.workspace_hooks import workspace_pin_hooks, plan_stale_hook + for spec in workspace_pin_hooks(allow_finish=_api_scope is None and _eval_scope is None): + adapter.bus.register(spec) + if _plan_tool_enabled: + adapter.bus.register(plan_stale_hook()) + for spec in hook_specs(request): + adapter.bus.register(spec) + _middlewares: list = [adapter] # At this point the collector has gathered all tools (including any subagent tools) → construct the final Toolkit. toolkit = _build_toolkit() diff --git a/src/backend/core/llm/factory/runtime/model_selection.py b/src/backend/core/llm/factory/runtime/model_selection.py index f5bcb2de..e92800ca 100644 --- a/src/backend/core/llm/factory/runtime/model_selection.py +++ b/src/backend/core/llm/factory/runtime/model_selection.py @@ -4,7 +4,7 @@ from dataclasses import replace -from core.llm.chat_models import get_default_model, make_chat_model +from core.models.chat_models import get_default_model, make_chat_model from core.llm.factory.request import AgentRequest @@ -15,15 +15,15 @@ async def model_selection(request: AgentRequest, *, _elapsed, _log, _shared_suba _selected_provider_id = (request.model_provider_id or "").strip() if _selected_provider_id: try: - from core.services.model_config import ModelConfigService + from core.models.config import ModelConfigService _selected_provider_cfg = ( _shared_subagent_cfg or ModelConfigService.get_instance().resolve_provider(_selected_provider_id) ) if _selected_provider_cfg: - from core.llm.chat_models import build_model_for_mode - from core.llm.failover import with_failover + from core.models.chat_models import build_model_for_mode + from core.models.failover import with_failover _mode = (request.chat_mode or "medium").lower() default_model = with_failover( @@ -41,11 +41,11 @@ async def model_selection(request: AgentRequest, *, _elapsed, _log, _shared_suba _mode_role = request.model_role or ("plan_agent" if request.plan_mode else None) if default_model is None and _mode_role: try: - from core.services.model_config import ModelConfigService + from core.models.config import ModelConfigService _mode_cfg = ModelConfigService.get_instance().resolve(_mode_role) if _mode_cfg: - from core.llm.failover import with_failover + from core.models.failover import with_failover default_model = with_failover( make_chat_model( @@ -90,8 +90,8 @@ async def model_selection(request: AgentRequest, *, _elapsed, _log, _shared_suba if _user_provider_id or _user_temp is not None or _user_max_tokens or _user_timeout: try: - from core.llm.chat_models import build_model_for_mode - from core.services.model_config import ModelConfigService + from core.models.chat_models import build_model_for_mode + from core.models.config import ModelConfigService _model_service = ModelConfigService.get_instance() _provider_cfg = ( @@ -111,7 +111,7 @@ async def model_selection(request: AgentRequest, *, _elapsed, _log, _shared_suba max_tokens=_user_max_tokens or _base_cfg.max_tokens, timeout=_user_timeout or _base_cfg.timeout, ) - from core.llm.failover import with_failover + from core.models.failover import with_failover _overrides = { key: value @@ -143,7 +143,7 @@ async def model_selection(request: AgentRequest, *, _elapsed, _log, _shared_suba _log.warning("[factory] subagent config override failed: %s, using default", exc) if request.run_id: - from core.llm.model_usage import instrument_model_usage + from core.models.usage import instrument_model_usage instrument_model_usage(default_model) diff --git a/src/backend/core/llm/factory/selection/capabilities.py b/src/backend/core/llm/factory/selection/capabilities.py index 390144da..f490a25d 100644 --- a/src/backend/core/llm/factory/selection/capabilities.py +++ b/src/backend/core/llm/factory/selection/capabilities.py @@ -1,11 +1,13 @@ """Agent assembly: capabilities. """ from __future__ import annotations +from core.capabilities.skills import loader_factory +from core.harness.registry import execution_account as skill_account +from core.harness.registry import run_capabilities as skills from typing import Optional -import core.agent_skills.loader as skill_loader -import core.config.catalog as catalog +import core.capabilities.catalog.api as catalog def _device_available_skill_ids( @@ -23,18 +25,17 @@ def _device_available_skill_ids( import logging as _logging _log = _logging.getLogger(__name__) - from core.capabilities.paths import capabilities_enabled + from core.harness.registry.run_capabilities import has_run_capabilities - if not capabilities_enabled(): + if not has_run_capabilities(): return skill_ids - from core.capabilities import skills as _caps_skills - from core.capabilities.preparation import ensure_cloud_ready + from core.harness.registry.run_capabilities import ensure_cloud_ready try: ensure_cloud_ready(user_id, skill_keys=skill_ids, plugin_keys=plugin_ids) available = set( - _caps_skills.filter_available_names( - skill_ids, user_id=_caps_skills.current_local_user_id() + skills.filter_available_names( + skill_ids, user_id=skill_account.current_local_user_id() ) ) except Exception as exc: # noqa: BLE001 — 解析失败时不放大成整轮失败，退回原始集合 @@ -46,7 +47,7 @@ def _device_available_skill_ids( return [sid for sid in skill_ids if sid in available] -def _filter_skill_ids_for_user(skill_ids: list[str], user_id: Optional[str]) -> list[str]: +def filter_skill_ids_for_user(skill_ids: list[str], user_id: Optional[str]) -> list[str]: """Strip out skill ids this user must not load. Two independent rules, applied at the one choke point every agent (main, @@ -67,7 +68,7 @@ def _filter_skill_ids_for_user(skill_ids: list[str], user_id: Optional[str]) -> try: from core.db.engine import SessionLocal from core.db.models import AdminSkill - from core.evolution.exposure import filter_skill_ids as filter_evolved + from core.evolution.evidence.exposure import filter_skill_ids as filter_evolved with SessionLocal() as db: owned = dict( @@ -86,7 +87,7 @@ def _filter_skill_ids_for_user(skill_ids: list[str], user_id: Optional[str]) -> # read is withheld, because the unsafe direction here is loading an # unvetted self-authored capability. try: - from core.evolution.exposure import is_evolved_skill_id + from core.evolution.evidence.exposure import is_evolved_skill_id return [sid for sid in skill_ids if not is_evolved_skill_id(sid)] except Exception: @@ -96,7 +97,7 @@ def _filter_skill_ids_for_user(skill_ids: list[str], user_id: Optional[str]) -> def _filter_kb_ids_for_user(kb_ids: list[str], user_id: Optional[str]) -> list[str]: """Strip out KB ids the current user has no access to (local + external collections), preventing unauthorized ids passed in from the frontend. - Single source of truth ``core.auth.kb_permissions``: public KBs are visible + Single source of truth ``features.knowledge.edition``: public KBs are visible to everyone, private KBs to their owner, and scoped-visibility KBs per grant. On failure, fall back to returning the input unchanged (this doesn't escalate permissions — it only avoids hurting availability, and the @@ -105,7 +106,7 @@ def _filter_kb_ids_for_user(kb_ids: list[str], user_id: Optional[str]) -> list[s if not kb_ids or not user_id: return kb_ids try: - from core.auth.kb_permissions import filter_accessible_kb_ids + from core.harness.registry.retrieval_context import filter_accessible_kb_ids from core.db.engine import SessionLocal with SessionLocal() as db: @@ -137,16 +138,16 @@ def _expand_plugin_bindings( ) for r in rows: cids = r.component_ids or {} - from core.plugins.packaging.sources import _component_keys + from core.capabilities.plugins.packaging.sources import component_keys as _component_keys skills.extend(_component_keys(cids, "skills")) mcp.extend(_component_keys(cids, "mcp")) except Exception: # noqa: BLE001 skills, mcp = [], [] - from core.capabilities.paths import capabilities_enabled + from core.harness.registry.run_capabilities import has_run_capabilities - if capabilities_enabled(): - from core.capabilities.plugins import cloud_binding_ids + if has_run_capabilities(): + from core.harness.registry.run_capabilities import cloud_binding_ids cloud_skills, cloud_mcp = cloud_binding_ids(plugin_ids, user_id=user_id) skills.extend(cloud_skills) @@ -154,16 +155,16 @@ def _expand_plugin_bindings( return list(dict.fromkeys(skills)), list(dict.fromkeys(mcp)) -def _effective_main_available_skills() -> list[str]: +def effective_main_available_skills() -> list[str]: """Resolve main-agent skills from currently enabled catalog skills.""" def available(names): - from core.capabilities.paths import capabilities_enabled + from core.harness.registry.run_capabilities import has_run_capabilities - if capabilities_enabled(): - from core.capabilities import skills + if has_run_capabilities(): + from core.harness.registry import run_capabilities as skills - return skills.filter_available_names(names, user_id=skills.current_local_user_id()) + return skills.filter_available_names(names, user_id=skill_account.current_local_user_id()) return names enabled_ids = [ @@ -174,18 +175,19 @@ def available(names): # On the server, an empty default catalog means no ambient skills. The # loader also knows installed-but-disabled skills for explicit selection. - from core.capabilities.paths import capabilities_enabled + from core.harness.registry.run_capabilities import has_run_capabilities - if not capabilities_enabled(): + if not has_run_capabilities(): return [] try: - loader = skill_loader.get_skill_loader() + loader = loader_factory.get_skill_loader() discovered = sorted(loader.load_all_metadata().keys()) if discovered: return available(discovered) - except Exception: - pass + except Exception as fallback_error: + from core.infra.failures import record_best_effort_failure + record_best_effort_failure('core.llm.factory.selection.capabilities:_effective_main_available_skills', fallback_error) return [] @@ -195,7 +197,7 @@ def _mcp_ids_bound_to_skills(skill_ids: list[str]) -> list[str]: if not skill_ids: return [] try: - metadata = skill_loader.get_skill_loader().load_all_metadata() + metadata = loader_factory.get_skill_loader().load_all_metadata() except Exception: return [] result: list[str] = [] diff --git a/src/backend/core/llm/factory/selection/entry_policy.py b/src/backend/core/llm/factory/selection/entry_policy.py index 11d84ae0..9d608ad2 100644 --- a/src/backend/core/llm/factory/selection/entry_policy.py +++ b/src/backend/core/llm/factory/selection/entry_policy.py @@ -10,16 +10,16 @@ async def entry_policy(request: AgentRequest): """Normalize execution scopes and initialize per-construction capability state.""" - from core.llm.project_api_runtime import bind_project_api_runtime + from core.llm.api_runtime.project import bind_project_api_runtime await bind_project_api_runtime(request) required = RequiredCapabilities() sticky = StickyCapabilities() import asyncio - from core.llm.agent_api_runtime import parse_api_scope - from core.llm.evaluation_runtime import evaluation_user_agent, parse_evaluation_scope - from core.llm.middlewares import CURRENT_RUN_BINDING + from core.llm.api_runtime.agent import parse_api_scope + from core.harness.registry.evaluation import evaluation_user_agent, parse_evaluation_scope + from core.harness.tool_call_context import CURRENT_RUN_BINDING _eval_scope = parse_evaluation_scope( owner_user_id=request.current_user_id, @@ -43,7 +43,7 @@ async def entry_policy(request: AgentRequest): request.ontology_runtime = {} request.workspace_id = request.sandbox_session_id = _eval_scope.session_id request.user_agent = evaluation_user_agent(request.user_agent) - from core.llm.builtin_subagents import get_builtin_subagent + from core.capabilities.agents.builtin import get_builtin_subagent request.visible_subagents = [ item @@ -73,7 +73,7 @@ async def entry_policy(request: AgentRequest): if request.user_agent is not None and not ( _api_scope and _api_scope.public_chat_visitor and request.model_provider_id ): - from core.services.model_config import ModelConfigService + from core.models.config import ModelConfigService _model_service = ModelConfigService.get_instance() _role_cfg = await asyncio.to_thread(_model_service.resolve, "subagent") @@ -90,13 +90,13 @@ async def entry_policy(request: AgentRequest): request.run_id = request.run_id or inherited_run_binding[0] request.journal_owner = request.journal_owner or inherited_run_binding[1] - from core.capabilities.paths import capabilities_enabled + from core.harness.registry.run_capabilities import has_run_capabilities _capability_run_key = request.run_id - if capabilities_enabled(): + if has_run_capabilities(): import uuid as _cap_uuid - from core.capabilities import runtime as capability_runtime + from core.harness.registry import capability_runtime _capability_run_key = request.run_id or "ephemeral:" + _cap_uuid.uuid4().hex if request.user_agent is not None: @@ -111,7 +111,7 @@ async def entry_policy(request: AgentRequest): import logging import time - from core.llm.tool_permissions import resolve_approval_mode + from core.sandbox.permissions.approval import resolve_approval_mode request.approval_mode = resolve_approval_mode( request.approval_mode, user_id=request.current_user_id diff --git a/src/backend/core/llm/factory/selection/mode_bindings.py b/src/backend/core/llm/factory/selection/mode_bindings.py index 52d258df..f27f57d0 100644 --- a/src/backend/core/llm/factory/selection/mode_bindings.py +++ b/src/backend/core/llm/factory/selection/mode_bindings.py @@ -22,12 +22,7 @@ async def mode_bindings(request: AgentRequest, *, required: RequiredCapabilities _mode_plugin_ids = list(getattr(request.mode_spec, "plugin_ids", ()) or ()) _turbo_code_exec = bool(getattr(request.mode_spec, "code_exec_enabled", False)) else: - from core.services.system_config import ( - turbo_manual_invoke_enabled, - turbo_mcp_server_ids, - turbo_plugin_ids, - turbo_skill_ids, - ) + from core.config.store.execution_options import turbo_manual_invoke_enabled, turbo_mcp_server_ids, turbo_plugin_ids, turbo_skill_ids _mode_manual_invoke = turbo_manual_invoke_enabled() _mode_mcp_ids = sorted(turbo_mcp_server_ids()) diff --git a/src/backend/core/llm/factory/selection/profile_selection.py b/src/backend/core/llm/factory/selection/profile_selection.py index aa07f8c9..cb57c964 100644 --- a/src/backend/core/llm/factory/selection/profile_selection.py +++ b/src/backend/core/llm/factory/selection/profile_selection.py @@ -6,15 +6,17 @@ from core.llm.factory.models import RequiredCapabilities, StickyCapabilities -logger = logging.getLogger(__name__) import asyncio -import core.config.catalog as catalog +import core.capabilities.catalog.api as catalog from core.auth.tenancy import tenant_of -from core.evolution.agent_profile import builtin_profile, load_active_profile -from core.llm.execution_manifest import PromptManifestBuilder +from core.evolution.profiles.agent_profile import builtin_profile +from core.evolution.profiles.profile_selection import load_active_profile +from core.harness.execution_manifest import PromptManifestBuilder from core.llm.factory.request import AgentRequest +logger = logging.getLogger(__name__) + async def profile_selection( request: AgentRequest, @@ -88,7 +90,7 @@ async def profile_selection( # what the decremental engine needs; narrowing what gets loaded is not. skill_selection = None try: - from core.agent_skills.selection_record import build_selection + from core.capabilities.skills.selection_record import build_selection skill_selection = build_selection( all_candidate_ids=list(skill_ids_for_bindings or []), diff --git a/src/backend/core/llm/factory/selection/progressive_bindings.py b/src/backend/core/llm/factory/selection/progressive_bindings.py index 8f637fd9..87bbec9a 100644 --- a/src/backend/core/llm/factory/selection/progressive_bindings.py +++ b/src/backend/core/llm/factory/selection/progressive_bindings.py @@ -4,15 +4,15 @@ import asyncio -import core.config.catalog as catalog -from core.capabilities.paths import capabilities_enabled +import core.capabilities.catalog.api as catalog +from core.harness.registry.run_capabilities import has_run_capabilities from core.llm.factory.request import AgentRequest from core.llm.factory.selection import capabilities as factory_capabilities async def progressive_bindings(request: AgentRequest, *, _eval_scope, _log): - if capabilities_enabled() and not request.disable_tools and request.enabled_skill_ids: - from core.capabilities.preparation import ensure_cloud_ready + if has_run_capabilities() and not request.disable_tools and request.enabled_skill_ids: + from core.harness.registry.run_capabilities import ensure_cloud_ready await asyncio.to_thread( ensure_cloud_ready, request.current_user_id, skill_keys=request.enabled_skill_ids @@ -20,7 +20,7 @@ async def progressive_bindings(request: AgentRequest, *, _eval_scope, _log): # Security: strip out other users' private skills, preventing unauthorized skill_ids passed in from the frontend if request.enabled_skill_ids: - request.enabled_skill_ids = factory_capabilities._filter_skill_ids_for_user( + request.enabled_skill_ids = factory_capabilities.filter_skill_ids_for_user( request.enabled_skill_ids, request.current_user_id ) @@ -31,7 +31,7 @@ async def progressive_bindings(request: AgentRequest, *, _eval_scope, _log): # its explicit lists are its whole surface and its tool filter would hide # `load_plugin`, leaving deferred plugins unreachable. Here, installed plugins' components are # removed from this run's enabled sets and replaced by a one-line directory - # entry + a `load_plugin` activation tool (see core/plugins/runtime/__init__.py). + # entry + a `load_plugin` activation tool (see core/capabilities/plugins/runtime/__init__.py). # Deferral happens BEFORE the skill-bound-MCP merge below so a deferred # skill doesn't pull its bound MCP servers into the assembly either. _progressive = None @@ -43,16 +43,16 @@ async def progressive_bindings(request: AgentRequest, *, _eval_scope, _log): and request.current_user_id and _eval_scope is None ): - from core.plugins import runtime as _plug + from core.capabilities.plugins import runtime as _plug - if not capabilities_enabled() and _plug.progressive_plugin_loading_enabled(): + if not has_run_capabilities() and _plug.progressive_plugin_loading_enabled(): try: # Normalize the None fallbacks to their concrete resolutions # (identical sources to the later phases) so the subtraction # below has explicit lists to operate on. if request.enabled_skill_ids is None: - request.enabled_skill_ids = factory_capabilities._filter_skill_ids_for_user( - factory_capabilities._effective_main_available_skills(), + request.enabled_skill_ids = factory_capabilities.filter_skill_ids_for_user( + factory_capabilities.effective_main_available_skills(), request.current_user_id, ) if not isinstance(request.enabled_mcp_ids, list): @@ -95,7 +95,7 @@ async def progressive_bindings(request: AgentRequest, *, _eval_scope, _log): skill_ids_for_bindings = ( request.enabled_skill_ids if request.enabled_skill_ids is not None - else factory_capabilities._effective_main_available_skills() + else factory_capabilities.effective_main_available_skills() ) skill_bound_mcp_ids = ( factory_capabilities._mcp_ids_bound_to_skills(skill_ids_for_bindings) @@ -103,7 +103,7 @@ async def progressive_bindings(request: AgentRequest, *, _eval_scope, _log): else [] ) if request.profile: - from core.extensions import agent_profile + from core.harness.registry import agent_profile if not agent_profile(request.profile).inherit_default_mcps: skill_bound_mcp_ids = [m for m in skill_bound_mcp_ids if m in (request.enabled_mcp_ids or [])] if skill_bound_mcp_ids: diff --git a/src/backend/core/llm/factory/selection/server_selection.py b/src/backend/core/llm/factory/selection/server_selection.py index 542feab1..f61ed6c5 100644 --- a/src/backend/core/llm/factory/selection/server_selection.py +++ b/src/backend/core/llm/factory/selection/server_selection.py @@ -2,11 +2,11 @@ from __future__ import annotations -from core.capabilities.paths import capabilities_enabled +from core.harness.registry.run_capabilities import has_run_capabilities from core.llm.factory.models import RequiredCapabilities, StickyCapabilities from core.llm.factory.request import AgentRequest from core.llm.factory.tools import mcp_config as factory_mcp_config -from core.services.mcp_service import McpServerConfigService +from core.capabilities.mcp.service import McpServerConfigService async def server_selection( @@ -54,7 +54,7 @@ async def server_selection( _capability_mcp_resolution = [] _unavailable_connectors = {} try: - from core.services.desktop_cloud_bridge import cloud_gateway_mcp_configs + from core.harness.registry.execution_catalog import cloud_gateway_mcp_configs bridge_mcp_servers = cloud_gateway_mcp_configs( request.enabled_mcp_ids, @@ -62,7 +62,7 @@ async def server_selection( unavailable_out=_unavailable_connectors, ) except Exception: # noqa: BLE001 - if capabilities_enabled(): + if has_run_capabilities(): raise # a desktop binding failure must not fall back to another source bridge_mcp_servers = {} @@ -77,7 +77,7 @@ async def server_selection( ] # Determine which MCP servers to connect - enabled_mcp_keys = factory_mcp_config._effective_mcp_server_keys( + enabled_mcp_keys = factory_mcp_config.effective_mcp_server_keys( cfg, request.agent_spec, enabled_mcp_ids=request.enabled_mcp_ids, diff --git a/src/backend/core/llm/factory/selection/sticky_capabilities.py b/src/backend/core/llm/factory/selection/sticky_capabilities.py index a0eed260..04520eb0 100644 --- a/src/backend/core/llm/factory/selection/sticky_capabilities.py +++ b/src/backend/core/llm/factory/selection/sticky_capabilities.py @@ -4,8 +4,8 @@ import asyncio -import core.config.catalog as catalog -from core.capabilities.paths import capabilities_enabled +import core.capabilities.catalog.api as catalog +from core.harness.registry.run_capabilities import has_run_capabilities from core.llm.factory.models import RequiredCapabilities, StickyCapabilities from core.llm.factory.request import AgentRequest from core.llm.factory.selection import capabilities as factory_capabilities @@ -23,7 +23,7 @@ async def sticky_capabilities( sticky: StickyCapabilities, ): # Dedicated profiles own their full binding list. Chat history must not widen it. - from core.extensions import agent_profile + from core.harness.registry import agent_profile explicit_profile = bool(request.profile and not agent_profile(request.profile).inherit_default_mcps) inherited_skills = set(request.enabled_skill_ids or []) inherited_mcps = set(request.enabled_mcp_ids or []) @@ -31,18 +31,18 @@ async def sticky_capabilities( request.capability_source_chat_id if request.derived_main else None ) or request.chat_id if ( - capabilities_enabled() + has_run_capabilities() and not request.disable_tools and not request.turbo_mode and request.user_agent is None and request.enabled_skill_ids is None ): - from core.plugins.runtime import prepare_desktop_plugin_skill_defaults + from core.harness.registry.run_capabilities import plugin_skill_defaults request.enabled_skill_ids = await asyncio.to_thread( - prepare_desktop_plugin_skill_defaults, + plugin_skill_defaults, request.current_user_id, - factory_capabilities._effective_main_available_skills(), + factory_capabilities.effective_main_available_skills(), ) _log.info("[factory] +%s default capabilities prepared", _elapsed()) @@ -56,7 +56,7 @@ async def sticky_capabilities( base_skill_ids = ( list(request.enabled_skill_ids) if isinstance(request.enabled_skill_ids, list) - else factory_capabilities._effective_main_available_skills() + else factory_capabilities.effective_main_available_skills() ) request.enabled_skill_ids = list( dict.fromkeys([*base_skill_ids, *required.plugin_skill_ids]) @@ -72,7 +72,7 @@ async def sticky_capabilities( base_skill_ids = ( list(request.enabled_skill_ids) if isinstance(request.enabled_skill_ids, list) - else factory_capabilities._effective_main_available_skills() + else factory_capabilities.effective_main_available_skills() ) request.enabled_skill_ids = list(dict.fromkeys([*base_skill_ids, required.skill_id])) @@ -101,8 +101,8 @@ async def sticky_capabilities( and _eval_scope is None and (not request.turbo_mode or _mode_manual_invoke) ): - from core.llm import session_capabilities as _sticky_direct - from core.plugins import runtime as _sticky_plugins + from core.llm.runtime import session_capabilities as _sticky_direct + from core.capabilities.plugins import runtime as _sticky_plugins try: _sticky, _direct = await asyncio.gather( @@ -132,7 +132,7 @@ async def sticky_capabilities( base_skill_ids = ( list(request.enabled_skill_ids) if isinstance(request.enabled_skill_ids, list) - else factory_capabilities._effective_main_available_skills() + else factory_capabilities.effective_main_available_skills() ) request.enabled_skill_ids = list( dict.fromkeys([*base_skill_ids, *sticky_skill_ids]) @@ -154,7 +154,7 @@ async def sticky_capabilities( len(sticky_mcp_ids), ) except Exception as exc: # noqa: BLE001 - if capabilities_enabled(): + if has_run_capabilities(): raise # a saved source must not silently disappear or change account _log.warning("[factory] sticky capability restore failed: %s", exc) diff --git a/src/backend/core/llm/factory/selection/subagent_bindings.py b/src/backend/core/llm/factory/selection/subagent_bindings.py index 81d25453..d9a6bfaf 100644 --- a/src/backend/core/llm/factory/selection/subagent_bindings.py +++ b/src/backend/core/llm/factory/selection/subagent_bindings.py @@ -6,7 +6,7 @@ from dataclasses import replace import prompts.prompt_config as prompt_config -from core.capabilities.paths import capabilities_enabled +from core.harness.registry.run_capabilities import has_run_capabilities from core.llm.factory.request import AgentRequest from core.llm.factory.selection import capabilities as factory_capabilities @@ -49,11 +49,11 @@ async def subagent_bindings(request: AgentRequest, *, _api_scope, _elapsed, _log # Expand bound plugins into their component skills + MCPs (a plugin = a detachable capability bundle). Merge with the loose bindings, deduplicated. plugin_ids = request.user_agent.plugin_ids or [] if plugin_ids: - from core.plugins import runtime as _plug_sub + from core.capabilities.plugins import runtime as _plug_sub if ( not request.disable_tools - and not capabilities_enabled() + and not has_run_capabilities() and _api_scope is None and _plug_sub.progressive_plugin_loading_enabled() ): @@ -64,7 +64,7 @@ def _resolve_bound(): # would have received via factory_capabilities._filter_skill_ids_for_user. return _plug_sub.resolve_bound_progressive_plugins( list(plugin_ids), - skill_filter=lambda sids: factory_capabilities._filter_skill_ids_for_user( + skill_filter=lambda sids: factory_capabilities.filter_skill_ids_for_user( sids, request.current_user_id ), ) diff --git a/src/backend/orchestration/registry.py b/src/backend/core/llm/factory/specs.py similarity index 99% rename from src/backend/orchestration/registry.py rename to src/backend/core/llm/factory/specs.py index 3e8a7e58..e68c4f56 100755 --- a/src/backend/orchestration/registry.py +++ b/src/backend/core/llm/factory/specs.py @@ -9,7 +9,6 @@ from dataclasses import dataclass, field from typing import Any, Callable, List, Optional - AgentFactory = Callable[[], Any] @@ -48,4 +47,3 @@ class AgentSpec: model_name: Optional[str] = None temperature: Optional[float] = None max_tokens: Optional[int] = None - diff --git a/src/backend/core/llm/factory/tools/definitions.py b/src/backend/core/llm/factory/tools/definitions.py new file mode 100644 index 00000000..fddc89e0 --- /dev/null +++ b/src/backend/core/llm/factory/tools/definitions.py @@ -0,0 +1,35 @@ +"""Native tool values and their domain policy declarations during assembly.""" +from agentscope.tool import FunctionTool, ToolBase +from agentscope.permission import PermissionBehavior, PermissionDecision +from core.sandbox.permissions.tool_specs import builtin_tool_permission + + +class ToolAssembly(list[ToolBase]): + def __init__(self): + super().__init__() + self.skill_loaders = [] + + @property + def permission_specs(self): + from extensions.mcp_servers.registry import SERVERS + specs = {} + for server in SERVERS.values(): + specs.update(server.permission_specs()) + for tool in self: + spec = getattr(tool, "dispatch_spec", None) or builtin_tool_permission(tool.name) + if spec is not None: + previous = specs.get(tool.name) + if previous is not None and previous.key != spec.key: + raise ValueError(f"Conflicting permission declaration for {tool.name!r}") + specs[tool.name] = spec + return specs + + +def function_tool(func, *, description=None, dispatch_spec=None): + tool = FunctionTool( + func, description=description, is_concurrency_safe=False, + permission=PermissionDecision(behavior=PermissionBehavior.ALLOW, + message="Subject to the run's execution policy"), + ) + tool.dispatch_spec = dispatch_spec + return tool diff --git a/src/backend/core/llm/factory/tools/interactive.py b/src/backend/core/llm/factory/tools/interactive.py index bb4d8b4c..d30fbe9a 100644 --- a/src/backend/core/llm/factory/tools/interactive.py +++ b/src/backend/core/llm/factory/tools/interactive.py @@ -24,7 +24,7 @@ async def interactive_tools( ): _log.info("[factory] +%s skills registered", _elapsed()) - from core.services.project_scope import project_scope_from_context + from core.projects.edition import project_scope_from_context _proj_scope = project_scope_from_context(request.project_ctx or {}) diff --git a/src/backend/core/llm/factory/tools/mcp_config.py b/src/backend/core/llm/factory/tools/mcp_config.py index 7ca5bfd2..e5b2956d 100644 --- a/src/backend/core/llm/factory/tools/mcp_config.py +++ b/src/backend/core/llm/factory/tools/mcp_config.py @@ -4,14 +4,13 @@ from typing import Any, Dict, List, Optional, Set -import core.config.catalog as catalog -from core.config.catalog_loader import DB_HIDDEN_SERVERS, DB_UMBRELLA_ID -from core.llm.mcp_pool import MCPConnectionPool -from core.services.mcp_service import McpServerConfigService -from orchestration.registry import AgentSpec +import core.capabilities.catalog.api as catalog +from core.capabilities.catalog.loader import DB_HIDDEN_SERVERS, DB_UMBRELLA_ID +from core.llm.factory.specs import AgentSpec +from core.capabilities.mcp.service import McpServerConfigService -def _effective_mcp_server_keys( +def effective_mcp_server_keys( cfg, agent_spec: Optional[AgentSpec], enabled_mcp_ids: Optional[list[str]] = None, @@ -103,7 +102,7 @@ def _required_mcp_server_keys( return resolved -def _inject_runtime_headers( +def inject_runtime_headers( enabled_servers: dict, *, current_user_id: Optional[str] = None, @@ -125,7 +124,7 @@ def _inject_runtime_headers( if not enabled_servers: return enabled_servers - from core.kb.external_provider import runtime_request_context + from core.harness.registry.retrieval_context import runtime_request_context normalized = [str(x).strip() for x in (enabled_kb_ids or []) if str(x).strip()] local_ids = [x for x in normalized if x.startswith("kb_")] @@ -163,7 +162,7 @@ def _inject_runtime_headers( if is_http: headers = dict(c.get("headers") or {}) headers.update(ctx_headers) - from core.llm.mcp_invocation import PLUGIN_HEADER, for_url + from core.llm.mcp.invocation import PLUGIN_HEADER, for_url plugin_id = "" if c.get("origin") == "local_plugin" and c.get("source_plugin"): plugin_id = c.get("plugin_install_id") or "plugin:local:" + c["source_plugin"] @@ -176,41 +175,3 @@ def _inject_runtime_headers( c["env"] = env_cfg out[key] = c return out - - -async def warmup_mcp_tools() -> None: - """Initialize the MCP connection pool at startup. - - Reads MCP server configs from DB (via McpServerConfigService) and - connects to all stable servers. Per-request servers (e.g. - retrieve_dataset_content) are spawned on demand. - """ - import logging - import time - - log = logging.getLogger(__name__) - - # DB overlays (model config, system config) are already applied inside - # McpServerConfigService._build_env(), so no manual overlay needed here. - svc = McpServerConfigService.get_instance() - servers = svc.get_all_servers(enabled_only=True) - - if not servers: - log.info("[warmup] No MCP servers configured – skipping warmup") - return - - log.info("[warmup] Initializing MCP connection pool for %d server(s)…", len(servers)) - start = time.monotonic() - - try: - pool = MCPConnectionPool.get_instance() - await pool.initialize(servers) - elapsed = time.monotonic() - start - log.info( - "[warmup] MCP pool initialized: %d stable connections in %.2fs", - pool.stable_client_count, - elapsed, - ) - except Exception as exc: - elapsed = time.monotonic() - start - log.warning("[warmup] MCP pool initialization failed after %.2fs: %s", elapsed, exc) diff --git a/src/backend/core/llm/factory/tools/mcp_connections.py b/src/backend/core/llm/factory/tools/mcp_connections.py index 84f171f9..b09dc69c 100644 --- a/src/backend/core/llm/factory/tools/mcp_connections.py +++ b/src/backend/core/llm/factory/tools/mcp_connections.py @@ -1,24 +1,25 @@ """Agent assembly phase: mcp connections. """ from __future__ import annotations +from core.capabilities.skills import loader_factory import asyncio -import time from typing import List -import core.agent_skills.loader as skill_loader -from agentscope.mcp import MCPClient -from core.llm.factory.defaults import ( - _HTTP_MCP_FAIL_AT, - _HTTP_MCP_FAIL_COOLDOWN_S, - _SKILL_INSTRUCTION_TEMPLATE, - KB_MCP_HTTP_URL, -) +from core.llm.factory.defaults import _SKILL_INSTRUCTION_TEMPLATE from core.llm.factory.models import RequiredCapabilities from core.llm.factory.request import AgentRequest -from core.llm.mcp_manager import own_factory_client -from core.llm.mcp_pool import MCPConnectionPool -from core.llm.tool_collector import ToolCollector +from core.llm.mcp.manager import own_factory_client +from core.llm.factory.tools.definitions import ToolAssembly + + +def _leaf_errors(exc): + """Task groups wrap the transport's real error; report the leaves, not the wrapper.""" + if isinstance(exc, BaseExceptionGroup): + for inner in exc.exceptions: + yield from _leaf_errors(inner) + else: + yield exc async def mcp_connections( @@ -36,7 +37,7 @@ async def mcp_connections( ): def _preload_skill_metadata(): """Pre-warm skill metadata cache so registration is fast.""" - loader = skill_loader.get_skill_loader() + loader = loader_factory.get_skill_loader() loader.load_all_metadata() return loader @@ -48,24 +49,8 @@ def _preload_skill_metadata(): loader = await asyncio.to_thread(_preload_skill_metadata) _log.info("[factory] +%s skill metadata pre-loaded", _elapsed()) - # ── Phase 2: MCP toolkit (async, may spawn per-request subprocesses) ── - mcp_clients: List[MCPClient] = [] - # Stable (pooled) clients must be reused and must never be closed at the end - # of a request; only transient (per-request spawned) stdio clients go on the - # close list. mcp_clients contains stable+transient (for Toolkit - # construction); transient_mcp_clients is only for closing. - transient_mcp_clients: List[MCPClient] = [] - http_clients: List[MCPClient] = [] - # AgentScope 2.0: the Toolkit is constructed once — there are no incremental - # register_* calls. Use ToolCollector to duck-type-compatibly collect our - # in-house tools/skills (the register_* functions barely change), then - # construct the real Toolkit at the end. - toolkit = ToolCollector() - # Policies must exist before deferred plugins become visible mid-run. - from extensions.mcp_servers.registry import SERVERS - for server in SERVERS.values(): - for name, spec in server.permission_specs().items(): - toolkit.register_tool_permission(name, spec) + mcp_resources = [] + toolkit = ToolAssembly() # ⚠️ This is a Jinja2 template, rendered by toolkit.get_skill_instructions() # with the ``skills`` variable. It MUST contain the # ``{% for skill in skills %}`` loop to actually list the skills — otherwise @@ -89,160 +74,41 @@ def _preload_skill_metadata(): ) if not request.disable_tools and enabled_servers: - from core.llm.mcp_pool import HTTP_TRANSPORTS, make_client, uses_manifest_schema - - http_server_cfgs = { - k: v for k, v in enabled_servers.items() if v.get("transport") in HTTP_TRANSPORTS - } - stdio_servers = { - k: v for k, v in enabled_servers.items() if v.get("transport") not in HTTP_TRANSPORTS - } - - # ``isolated`` callers run in their own event loop (subagent_tool - # worker threads), so they MUST NOT touch the shared MCP pool — pool - # clients are bound to the main loop's task scope and would crash - # anyio on cross-loop teardown. Spawn fresh per-request stdio + HTTP - # instead, and rely on close_clients() in the caller's loop. - if request.isolated or _api_scope is not None: - from core.llm.mcp_manager import connect_mcp_clients - - mcp_clients = await connect_mcp_clients(stdio_servers) - transient_mcp_clients = mcp_clients # all freshly spawned → all closable - per_request_http = http_server_cfgs - else: - pool = MCPConnectionPool.get_instance() - pool_managed = pool.stable_server_ids if pool.is_initialized else frozenset() - per_request_http = {k: v for k, v in http_server_cfgs.items() if k not in pool_managed} - if pool.is_initialized: - per_request_stdio = { - k: v for k, v in stdio_servers.items() if k not in pool_managed - } - # 2.0: the pool returns a list of connected MCPClients - # (stable+transient); the Toolkit(mcps=...) is constructed - # uniformly below. - mcp_clients, transient_mcp_clients = await pool.get_request_clients( - enabled_keys=enabled_mcp_keys, - per_request_servers_cfg=per_request_stdio, - ) - else: - from core.llm.mcp_manager import connect_mcp_clients - - mcp_clients = await connect_mcp_clients(stdio_servers) - transient_mcp_clients = mcp_clients # pool off → all transient + from core.llm.mcp.lifetime import discover + from core.harness.registry.local_publication import ensure_native_servers - # Per-request HTTP — pool can't carry per-request headers that some - # servers (e.g. retrieve_dataset_content) require. BaseException is - # caught because the mcp HTTP client's SSE task can propagate - # CancelledError on transient failures. - async def _connect_http(key: str, cfg: dict): - start = time.monotonic() - last_fail = _HTTP_MCP_FAIL_AT.get(key, 0.0) - if last_fail and start - last_fail < _HTTP_MCP_FAIL_COOLDOWN_S: - return None - # ⚠️ 2.0 key point: HTTP MCP uses is_stateful=False (a new connection - # per call), avoiding the stateful client's task-binding problem - # (the connect task differs from the request task → cancel-scope - # crash, so tool_result is never received). Stateless clients need - # not and must not connect(); a single list_tools serves as the - # liveness probe (lazy connect + enumerate, verifying reachability), - # after which the Toolkit opens a fresh connection on every call. - _http_cfg = {**cfg, "url": cfg.get("url", KB_MCP_HTTP_URL), - "read_only": request.read_only} - if _http_cfg.get("schema_source") == "cloud_manifest" and not uses_manifest_schema( - _http_cfg - ): - _log.warning( - "[factory] HTTP MCP '%s' ignored because its cloud manifest is invalid", - key, - ) - return None - client = own_factory_client(make_client(key, _http_cfg, is_stateful=False)) - if uses_manifest_schema(_http_cfg): - # The complete, authorized schema arrived in the dynamic cloud - # manifest. Agent construction is network-free; only a model- - # selected tool call reaches the JSON gateway. - _HTTP_MCP_FAIL_AT.pop(key, None) - _log.info( - "[factory] HTTP MCP '%s' loaded from cloud manifest (%d tools)", - key, - len(_http_cfg.get("manifest_tools") or []), - ) - return client + async def load(key, cfg): + cfg = {**cfg, "read_only": request.read_only} try: - from orchestration.local_plugin_sidecars import ensure_native_servers - - await ensure_native_servers({key: _http_cfg}) - await client.list_tools() - _HTTP_MCP_FAIL_AT.pop(key, None) - _log.info( - "[factory] HTTP MCP '%s' (stateless) probed in %.0fms", - key, - (time.monotonic() - start) * 1000, - ) - return client - except BaseException as exc: - _HTTP_MCP_FAIL_AT[key] = time.monotonic() - _log.warning( - "[factory] HTTP MCP '%s' connect failed (%s, cooldown %.0fs): %s", - key, - type(exc).__name__, - _HTTP_MCP_FAIL_COOLDOWN_S, - exc, - ) - # Only propagate CancelledError when the *outer* task is itself - # being cancelled (real user/system cancel). anyio's SSE-client - # cleanup raises CancelledError as a scope-exit signal even when - # nobody cancelled us — re-raising those was killing the whole - # chat run whenever any single HTTP MCP (e.g. a freshly-removed - # word_mcp / ppt_mcp / excel_mcp / pdf_mcp whose admin_mcp_servers row was still - # ``is_enabled=true``) was unreachable. - if isinstance(exc, asyncio.CancelledError): - current = asyncio.current_task() - if current is not None and getattr(current, "cancelling", lambda: 0)() > 0: - raise + if cfg.get("schema_source") not in {"cloud_manifest", "plugin_manifest"}: + await ensure_native_servers({key: cfg}) + from core.llm.mcp.callbacks import Interactions + interactions = Interactions(request, key, _tool_approval_available) + scope = ((request.current_user_id, request.chat_id, request.capability_scope) + if request.current_user_id and request.chat_id and not request.isolated else None) + resource = await discover(key, cfg, interactions=interactions, scope=scope) + return own_factory_client(resource) + except asyncio.CancelledError: + raise + except Exception as exc: + target = str(cfg.get("url") or cfg.get("command") or "").split("?", 1)[0] + _log.warning("MCP %s discovery failed (%s %s): %s", key, cfg.get("transport") or "stdio", target, + "; ".join(f"{type(e).__name__}: {e}"[:300] for e in _leaf_errors(exc))) return None - if per_request_http: - results = await asyncio.gather( - *(_connect_http(k, v) for k, v in per_request_http.items()), - return_exceptions=False, - ) - http_clients.extend(c for c in results if c is not None) - - _log.info( - "[factory] +%s MCP tools loaded (transient_stdio=%d, http=%d)", - _elapsed(), - len(mcp_clients), - len(http_clients), - ) + results = await asyncio.gather(*(load(k, c) for k, c in enabled_servers.items())) + mcp_resources = [resource for resource in results if resource is not None] + failed = [key for key, resource in zip(enabled_servers, results) if resource is None] + if failed: + _note_unavailable("以下 MCP 服务连接失败，本轮无法使用其工具：" + "、".join(failed)) - connected_clients = { - str(getattr(client, "name", "") or ""): client for client in [*mcp_clients, *http_clients] - } + connected_clients = {resource.name: resource for resource in mcp_resources} async def _mcp_tool_names_for_servers(server_keys: List[str], *, log_prefix: str) -> List[str]: - connected_keys = [key for key in server_keys if key in connected_clients] - if not connected_keys: - return [] - listed_tools = await asyncio.gather( - *(connected_clients[key].list_tools() for key in connected_keys), - return_exceptions=True, - ) - names: List[str] = [] - for server_key, result in zip(connected_keys, listed_tools): - if isinstance(result, BaseException): - _log.warning( - "[%s] list_tools failed for %s: %s", - log_prefix, - server_key, - result, - ) - continue - for tool in result: - tool_name = str(getattr(tool, "name", "") or "") - if tool_name and tool_name not in names: - names.append(tool_name) - return names + return list(dict.fromkeys( + tool.name for key in server_keys if key in connected_clients + for tool in connected_clients[key].tools + )) _required_connector_tool_names = await _mcp_tool_names_for_servers( _required_connector_server_keys, @@ -267,9 +133,7 @@ async def _mcp_tool_names_for_servers(server_keys: List[str], *, log_prefix: str _required_connector_tool_names, _required_plugin_mcp_tool_names, _tool_approval_available, - http_clients, + mcp_resources, loader, - mcp_clients, toolkit, - transient_mcp_clients, ) diff --git a/src/backend/core/llm/factory/tools/native.py b/src/backend/core/llm/factory/tools/native.py index 4675bb22..830e7185 100644 --- a/src/backend/core/llm/factory/tools/native.py +++ b/src/backend/core/llm/factory/tools/native.py @@ -2,15 +2,13 @@ from __future__ import annotations -import asyncio +from core.harness.registry import NativeToolContext, install_tools from core.llm.factory.request import AgentRequest from core.llm.tools import ( ReadStateTracker, register_bash, - register_channel_attachment, register_edit, - register_get_data_context, register_glob, register_grep, register_pin_to_workspace, @@ -22,7 +20,7 @@ register_sandboxed_view_text_file, register_write, ) -from core.services.system_config import code_capability_enabled +from core.config.store.execution_options import code_capability_enabled async def native_tools( @@ -46,7 +44,7 @@ async def native_tools( _progressive=None, ): if request.profile: - from core.extensions import agent_profile + from core.harness.registry import agent_profile agent_profile(request.profile).register_tools(toolkit, request, loader=loader) return @@ -68,12 +66,11 @@ async def native_tools( # Skill files reach the sandbox via the unified /workspace/skills bind # mount (built-in synced at startup, DB skills materialized on demand — # see agent_skills.config.get_sandbox_skills_dir), so Bash needs no - # per-call sync. loader/loaded_skill_ids kept for backward compat. + # per-call sync. if request.allow_bash: register_bash( toolkit, loader=loader, - loaded_skill_ids=loaded_skill_ids, source_run_id=request.run_id, chat_id=request.chat_id, sandbox_session_id=_sbx_sess, @@ -102,29 +99,11 @@ async def native_tools( # longer registered here (see extensions/mcp_servers/site_publish_mcp + # extensions/plugins/marketplace/sites). - # Site-builder design pick (choose one of three): registered only for - # sessions with the site-builder skill enabled (per-run conditional - # registration, not in the catalog). Interactivity uses the second-tier - # judgment _ui_reachable (≠ _interactive): write confirmation has the - # allow_session out-of-band pre-authorization path on IM channels and - # the automation confirmation panel in automation sessions, but the - # suspended picker has no clickable UI in either place — so degrade to - # non-interactive (the tool just lets the model pick its own design - # instead of suspending for 2h). - from core.llm.tools import design_picker_tool - - _ui_reachable = _interactive and not _is_channel_run and not request.automation_run - if skill_ids_to_register and any( - design_picker_tool.skill_uses_choose_design(str(sid)) for sid in skill_ids_to_register - ): - design_picker_tool.register_choose_design( - toolkit, - chat_id=request.chat_id, - interactive=_ui_reachable, - sandbox_session_id=_sbx_sess, - user_id=request.current_user_id, - scope=_proj_scope, - ) + contribution_context = NativeToolContext( + request, _interactive, _is_channel_run, _sbx_sess, _proj_scope, + skill_ids_to_register, enabled_mcp_keys, _log, + ) + await install_tools("after_sandbox", toolkit, contribution_context) # ── Phase 3.6: Register file-operation tools (Read/Edit/Write/Glob/ # Grep). These tools share a single @@ -143,7 +122,7 @@ async def native_tools( # Decided once per build from the model this run will actually use: a # pinned subagent model wins over the request's provider (mirrors the # sub-agent override below and DynamicModelMiddleware). - from core.vision import resolve_vision_mode + from core.models.vision import resolve_vision_mode _vision_mode = resolve_vision_mode( (_shared_subagent_cfg.provider_id if _shared_subagent_cfg else None) @@ -222,18 +201,11 @@ async def native_tools( vision_mode=_vision_mode, ) - # ── Phase 3.7b: channel_read_attachment (channel runs only) ── - # Group listening records bystander attachments by key without downloading them; - # this is how the agent pulls one in on demand. Gated on channel runs so the tool - # never clutters the toolkit of web conversations, where it could never resolve. - if _is_channel_run: - register_channel_attachment( - toolkit, user_id=request.current_user_id, chat_id=request.chat_id - ) + await install_tools("after_image", toolkit, contribution_context) # ── Phase 3.8: Register pin_to_workspace ── # Lets the agent gate which generated files reach the user-visible - # assistant message. See core/llm/workspace.py for the per-run state. + # assistant message. See core/llm/runtime/workspace.py for the per-run state. register_pin_to_workspace(toolkit, scope=_proj_scope, sandbox_session_id=_sbx_sess) # ── Phase 3.85: run_job（工作流模式的作业编排面） ── @@ -245,8 +217,8 @@ async def native_tools( # 就自行收工）。run_job 把循环体交给沙箱脚本，模型调用由后端代持凭据派出—— # 脚本因此拿不到任何 key。isolated=True 的子作业不注册，杜绝 job 套 job。 if request.workflow_mode and not request.isolated and _sbx_sess: + from core.llm.ports import snapshot_parent_runtime from core.llm.tools.job_tool import register_run_job - from orchestration.jobs.agent import snapshot_parent_runtime register_run_job( toolkit, @@ -266,31 +238,11 @@ async def native_tools( ), ) - # ── Phase 3.9: get_data_context (the "data dictionary" tool for direct-DB data retrieval) ── - # Three gates combined: (1) a direct DB server is enabled this run - # (db_query / es_query); (2) the external NL2SQL black box is excluded - # (query_database isn't in the set, so it naturally doesn't trigger); - # (3) the corresponding data source has annotation content. If any is - # unmet, don't attach — avoid adding a useless tool that misleads the - # model. The metadata only ever appears as a tool return value, never in - # the system prompt. See db_metadata_service. - _db_servers_on = {"db_query", "es_query"} & set(enabled_mcp_keys) - if _db_servers_on: - try: - from core.services import db_metadata_service as _dbmeta - - _eligible_ds = await asyncio.to_thread( - _dbmeta.eligible_datasource_ids, _db_servers_on - ) - except Exception as _e: # noqa: BLE001 - _eligible_ds = [] - _log.warning("[factory] eligible_datasource_ids failed: %s", _e) - if _eligible_ds: - register_get_data_context(toolkit, _eligible_ds) + await install_tools("after_workflow", toolkit, contribution_context) if not request.disable_tools and _eval_scope is not None: - from core.llm.evaluation_tools import register_evaluation_tools - from core.vision import resolve_vision_mode + from core.harness.registry.evaluation import register_evaluation_tools + from core.models.vision import resolve_vision_mode register_evaluation_tools( toolkit, @@ -301,7 +253,7 @@ async def native_tools( ) if not request.disable_tools and _api_scope is not None: - from core.llm.agent_api_tools import register_agent_api_tools + from core.llm.api_runtime.agent_tools import register_agent_api_tools register_sandboxed_view_text_file( toolkit, diff --git a/src/backend/core/llm/factory/tools/skill_registration.py b/src/backend/core/llm/factory/tools/skill_registration.py index bc0f9ef1..eaf9cffe 100644 --- a/src/backend/core/llm/factory/tools/skill_registration.py +++ b/src/backend/core/llm/factory/tools/skill_registration.py @@ -5,8 +5,7 @@ import asyncio import os -from core.capabilities import runtime as capability_runtime -from core.capabilities.paths import capabilities_enabled +from core.harness.registry.run_capabilities import has_run_capabilities from core.llm.factory.request import AgentRequest from core.llm.factory.selection import capabilities as factory_capabilities @@ -16,12 +15,12 @@ async def skill_registration( ): skill_ids_to_register = request.enabled_skill_ids if skill_ids_to_register is None: - skill_ids_to_register = factory_capabilities._effective_main_available_skills() + skill_ids_to_register = factory_capabilities.effective_main_available_skills() # The fallback resolves from the static catalog, which today carries no # evolution-authored ids — but the exposure gate is applied here anyway # rather than relying on that. A gate that only covers the paths we # happened to think of is not a gate. - skill_ids_to_register = factory_capabilities._filter_skill_ids_for_user( + skill_ids_to_register = factory_capabilities.filter_skill_ids_for_user( skill_ids_to_register, request.current_user_id ) # Note: a subagent's (user_agent) enabled_skill_ids is always a list ([] @@ -30,15 +29,8 @@ async def skill_registration( # config, no inheriting the full catalog set. if _prepared_capabilities is not None: - missing = set(skill_ids_to_register or []) - set(_prepared_capabilities.bindings) - if missing: - from core.capabilities.errors import PackageMissing - - raise PackageMissing( - "selected skills are absent from this prepared run", - details={"skills": sorted(missing)}, - ) - loader = await asyncio.to_thread(capability_runtime.frozen_loader, _prepared_capabilities) + _prepared_capabilities.validate_selected_skills(skill_ids_to_register) + loader = await asyncio.to_thread(_prepared_capabilities.skill_loader) allowed_skill_dirs: list[str] = [] if not request.disable_tools and skill_ids_to_register: @@ -52,16 +44,12 @@ async def skill_registration( if ( not request.disable_tools - and not capabilities_enabled() + and not has_run_capabilities() and _api_scope is None and _eval_scope is None and request.project_environment is None ): - from core.agent_skills.config import ( - get_enabled_skill_sources, - get_sandbox_skills_dir, - get_user_skills_dir, - ) + from core.capabilities.skills.config import get_enabled_skill_sources, get_sandbox_skills_dir, get_user_skills_dir for src in get_enabled_skill_sources(): root = str(src.root_dir) diff --git a/src/backend/core/llm/factory/tools/tool_surface.py b/src/backend/core/llm/factory/tools/tool_surface.py index fdbb4249..18a7b697 100644 --- a/src/backend/core/llm/factory/tools/tool_surface.py +++ b/src/backend/core/llm/factory/tools/tool_surface.py @@ -6,11 +6,11 @@ from typing import Any, Dict, Optional from agentscope.tool import Toolkit -from core.capabilities import runtime as capability_runtime +from core.capabilities.skills.runtime_loader import ParallelRuntimeSkillLoader from core.llm.factory.models import RequiredCapabilities from core.llm.factory.request import AgentRequest -from core.ontology.toolkit import OntologyFilteredToolkit -from core.ontology.validator import register_runtime_asset_tags +from core.llm.tools.ontology_toolkit import OntologyFilteredToolkit +from core.ontology.runtime_pack import register_runtime_asset_tags async def tool_surface( @@ -24,9 +24,8 @@ async def tool_surface( required: RequiredCapabilities, _required_connector_tool_names, _required_plugin_mcp_tool_names, - http_clients, + mcp_resources, loader, - mcp_clients, skill_ids_to_register, toolkit, ): @@ -72,7 +71,7 @@ async def tool_surface( } def _build_toolkit(*, include_skills: bool = True) -> Toolkit: - # ``toolkit`` here is still the ToolCollector; construct the real Toolkit from the current collection state. + # Construct the native Toolkit from the assembled native tool values. # # "Skill" (AgentScope's builtin SkillViewer) is hidden unconditionally — # distinct from the ontology forbidden_tools above. The Toolkit injects @@ -82,20 +81,10 @@ def _build_toolkit(*, include_skills: bool = True) -> Toolkit: # observability). A model that called Skill instead would bypass all of # that and receive backend-path content unusable in the sandbox — so the # schema is pure per-round prefill waste plus a wrong door. - _visible_mcps = [*mcp_clients, *http_clients] - if _prepared_capabilities is not None: - from core.llm.capability_tools import AvailableMCPClient - - _live_run = ( - capability_runtime.get( - _prepared_capabilities.run_id, scope_id=request.capability_scope - ) - or _prepared_capabilities - ) - _visible_mcps = [AvailableMCPClient(client, _live_run) for client in _visible_mcps] - selected_tools = toolkit.function_tools + mcp_tools = [tool for resource in mcp_resources for tool in resource.tools] + selected_tools = list(toolkit) if request.profile: - from core.extensions import agent_profile + from core.harness.registry import agent_profile _profile = agent_profile(request.profile) if _profile.tool_filter is not None: @@ -103,16 +92,21 @@ def _build_toolkit(*, include_skills: bool = True) -> Toolkit: # The existing MCP resolver already intersects the explicit per-agent list. # A profile that does not inherit defaults gets none when the list is empty. if not _profile.inherit_default_mcps and not request.enabled_mcp_ids: - _visible_mcps = [] + mcp_tools = [] if not _profile.native_skill_prompt: include_skills = False - return OntologyFilteredToolkit( - tools=selected_tools, - mcps=_visible_mcps, - skills_or_loaders=(toolkit.skill_loaders or None) if include_skills else None, + names = [tool.name for tool in [*selected_tools, *mcp_tools]] + if len(names) != len(set(names)): + raise ValueError("Duplicate model-facing tool names") + built = OntologyFilteredToolkit( + tools=[*selected_tools, *mcp_tools], + skills_or_loaders=[ParallelRuntimeSkillLoader(toolkit.skill_loaders)] if include_skills and toolkit.skill_loaders else None, skill_instruction_template=_SKILL_INSTRUCTION_TEMPLATE, hidden_tools={*_ontology_hidden_tools, "Skill"}, ) + if _prepared_capabilities is not None: + built.capability_guard = _prepared_capabilities.tool_guard(scope_id=request.capability_scope) + return built # Compute schemas first in the "subagent tools not yet registered" state # (consistent with 1.x: subagent tools are registered after @@ -162,7 +156,7 @@ def _build_toolkit(*, include_skills: bool = True) -> Toolkit: # Progressive loading normally persists this during its resolution. # Persist here as well so explicit activation is sticky in restricted # modes, dedicated-agent chats and when progressive loading is disabled. - from core.plugins import runtime as _plugin_activation + from core.capabilities.plugins import runtime as _plugin_activation await asyncio.to_thread( _plugin_activation.record_plugin_activation, @@ -171,7 +165,7 @@ def _build_toolkit(*, include_skills: bool = True) -> Toolkit: user_id=str(request.current_user_id or ""), ) if request.chat_id and (_required_skill_registered or _required_connector_tool_names): - from core.llm.session_capabilities import record_session_capability_activation + from core.llm.runtime.session_capabilities import record_session_capability_activation await asyncio.to_thread( record_session_capability_activation, diff --git a/src/backend/core/llm/finish_guard.py b/src/backend/core/llm/finish_guard.py deleted file mode 100644 index 46a1feb3..00000000 --- a/src/backend/core/llm/finish_guard.py +++ /dev/null @@ -1,85 +0,0 @@ -"""finish_pin_guard helpers: auto-pin the files a model forgot when it "dumbly exits". - -Models often finish generating files and then exit directly, forgetting the -``pin_to_workspace`` delivery. ``FinishPinGuardMiddleware`` (core.llm.middlewares), -when a reasoning turn makes no tool calls at all (i.e. the model is about to -finish), calls this module's ``_collect_unpinned`` + ``_direct_pin`` and -**writes to the ContextVar by calling ``workspace.pin()`` directly**, bypassing -the ReAct main loop's _acting mechanism. ``chat_run_executor`` calls -``workspace.get_pinned()`` when wrapping up the stream to pick up this line and -deliver it as an attachment over SSE. - -Design trade-offs: -- **Do not inject a reminder** to the model — it has already decided to exit, - and an injection into context would not survive this turn; the next turn's new - agent would not see it. The auto-pin side effect itself is the only remedy the - user perceives. -- **At most one auto-pin per reply turn** (the middleware uses a ``_fired`` flag - to prevent an infinite loop when all file_ids are garbage); when every pin - fails, ``_fired`` is not set, leaving it for retry. -""" -from __future__ import annotations - -import logging - -logger = logging.getLogger(__name__) - - -def _collect_unpinned() -> list[str]: - """From the file_id set already collected by pin_hint, pick those not yet pinned and return a sorted list.""" - try: - from core.llm import workspace as _ws - from core.llm.hooks import _get_pin_hint_state - state = _get_pin_hint_state() - seen = state.get("seen") or set() - if not seen: - return [] - pinned = set(_ws.get_pinned_file_ids()) - return sorted(seen - pinned) - except Exception as exc: # noqa: BLE001 - logger.warning("[finish_guard] _collect_unpinned failed: %s", exc) - return [] - - -def _direct_pin(file_ids: list[str]) -> int: - """Write file_ids directly to the workspace ContextVar, bypassing the - pin_to_workspace tool + _acting path. Returns the number of pins that actually succeeded. - - Metadata comes from ``core.artifacts.store.get_artifact`` — on failure that id is skipped. - """ - try: - from core.artifacts.store import get_artifact - from core.llm import workspace as _ws - except Exception as exc: # noqa: BLE001 - logger.warning("[finish_guard] _direct_pin import failed: %s", exc) - return 0 - - count = 0 - for fid in file_ids: - try: - art = get_artifact(fid) - if not art: - logger.warning("[finish_guard] artifact not found: %s", fid) - continue - ok = _ws.pin( - fid, - name=art.get("name"), - mime_type=art.get("mime_type"), - size=art.get("size") or art.get("size_bytes"), - url=f"/files/{fid}", - ) - if ok: - count += 1 - except Exception as exc: # noqa: BLE001 - logger.warning("[finish_guard] _direct_pin failed for %s: %s", fid, exc) - # Only mark_active when at least one pin succeeded — avoid an empty set still - # calling mark_active, which would leave the workspace in an "active but pinned=[]" - # state (although downstream currently does not rely on the active flag for - # gating, this is a more robust invariant). - if count > 0: - try: - from core.llm import workspace as _ws - _ws.mark_active() - except Exception: - pass - return count diff --git a/src/backend/core/llm/hooks.py b/src/backend/core/llm/hooks.py deleted file mode 100644 index 684ce668..00000000 --- a/src/backend/core/llm/hooks.py +++ /dev/null @@ -1,588 +0,0 @@ -"""Pure-logic helpers: file-context building, per-turn state, model resolution. - -AgentScope 2.0 replaced the old hook factories with the Middleware classes in -``core.llm.middlewares``; this module has been reduced to the pure-function -helpers those middlewares reuse, independent of the agentscope version -(``_build_file_context`` / ``_build_historical_files_context`` / -``_get_main_model`` / ``_resolve_chat_mode`` / pin-hint & read-budget state, etc.). -The runtime context moved from 1.x's ``agent._jx_context`` to ``agent.state`` -(AgentRuntimeState). -""" - -from __future__ import annotations - -import base64 -import logging -import re -import threading -from functools import wraps -from contextvars import ContextVar -from typing import Any, Dict, List, Optional - -logger = logging.getLogger(__name__) - -MAX_FILE_CONTENT_CHARS = 50_000 # per-turn read_artifact budget per file - -# Per-turn cumulative read budget for the read_artifact tool. -# Mapping ``file_id → chars already returned this turn``. Reset by the -# file_context pre_reply hook on every turn boundary so a single user turn -# can read up to ``MAX_FILE_CONTENT_CHARS`` per file via paginated calls. -# -# The contextvar pattern works because AgentScope dispatches tool calls in -# the same async task as the agent's reply(), so the value set in the hook -# propagates to read_artifact. -_artifact_read_chars_per_turn: ContextVar[Optional[Dict[str, int]]] = ContextVar( - "_artifact_read_chars_per_turn", default=None -) - -# Per-turn cache for the workspace-pin reminder hook: -# seen: file_ids the agent has observed in tool results this turn -# last_reminded_sig: signature of the unpinned set the last reminder covered -# (used for in-hook dedup to avoid spamming when the same unpinned set -# persists across multiple tool calls). -_pin_hint_state: ContextVar[Optional[Dict[str, Any]]] = ContextVar("_pin_hint_state", default=None) - - -def get_artifact_read_state() -> Dict[str, int]: - """Return the current per-turn read tracker, lazily initializing it. - - Used by ``read_artifact`` to enforce the cumulative cap. - """ - state = _artifact_read_chars_per_turn.get() - if state is None: - state = {} - _artifact_read_chars_per_turn.set(state) - return state - - -def reset_artifact_read_state() -> None: - """Clear the per-turn read tracker. Called from file_context_pre_reply.""" - _artifact_read_chars_per_turn.set({}) - - -def reset_pin_hint_state() -> None: - """Clear the per-turn pin-reminder cache. Called at turn boundary.""" - _pin_hint_state.set({"seen": set(), "last_reminded_sig": ""}) - - -def _get_pin_hint_state() -> Dict[str, Any]: - state = _pin_hint_state.get() - if state is None: - state = {"seen": set(), "last_reminded_sig": ""} - _pin_hint_state.set(state) - return state - - -# ── DynamicModel helper (used by DynamicModelMiddleware) ───────────────── - -# Desktop models also capture a cloud session in their request hook. Cached -# instances must never cross that identity boundary, even when DB config is unchanged. -_model_cache: dict[tuple[Any, ...], Any] = {} -_cached_version: int = -1 - - -_model_cache_lock = threading.RLock() - - -def _serialized_model_cache(function): - @wraps(function) - def locked(*args, **kwargs): - with _model_cache_lock: - return function(*args, **kwargs) - - return locked - - -def _check_version(): - """Invalidate cached model instances when ModelConfigService version changes.""" - global _model_cache, _cached_version - from core.services.model_config import ModelConfigService - - current = ModelConfigService.get_instance().version - if current != _cached_version: - _model_cache = {} - _cached_version = current - - -def _model_cache_key(name: str) -> tuple[Any, ...]: - from core.auth.desktop_bridge import bridge_enabled - - import asyncio - - try: - loop = asyncio.get_running_loop() - except RuntimeError: - loop = None - if not bridge_enabled(): - return (name, loop, None) - from core.services.desktop_cloud_bridge import _state_fingerprint, get_state - - identity = _state_fingerprint(get_state()) - # Eviction only releases this cache's reference. In-flight runs keep their - # captured model and its existing request hook, which rejects identity changes. - for key in list(_model_cache): - if isinstance(key, tuple) and key[0] == name and key[2] != identity: - _model_cache.pop(key, None) - return name, loop, identity - - -@_serialized_model_cache -def release_loop_models() -> None: - import asyncio - - loop = asyncio.get_running_loop() - for key in list(_model_cache): - if key[1] is loop: - _model_cache.pop(key, None) - - -@_serialized_model_cache -def _get_main_model(mode: str = "medium"): - """Get the main agent model for the given chat mode (fast/medium/high/max).""" - from core.llm.chat_models import get_default_model - from core.llm.failover import with_failover - from core.services.model_config import ModelConfigService - - _check_version() - key = _model_cache_key(f"main:{mode}") - cached = _model_cache.get(key) - if cached is not None: - return cached - if mode in ("fast", "turbo"): - instance = get_default_model(disable_thinking=True, stream=True) - elif mode in ("low", "high", "xhigh", "max"): - instance = get_default_model(reasoning_effort=mode, stream=True) - else: # medium or unknown - # medium uses the "thinking model", but picks chat_template_kwargs based on - # supports_reasoning_effort (chat_models internally distinguishes - # effort=medium vs None) - instance = get_default_model(reasoning_effort="medium", stream=True) - instance = with_failover( - instance, ModelConfigService.get_instance().resolve("main_agent"), mode=mode - ) - _model_cache[key] = instance - return instance - - -@_serialized_model_cache -def _get_provider_model(provider_id: str, mode: str = "medium"): - """Get a user-selected active chat provider model for the given chat mode.""" - from core.llm.chat_models import build_model_for_mode - from core.llm.failover import with_failover - from core.services.model_config import ModelConfigService - - pid = (provider_id or "").strip() - if not pid: - return _get_main_model(mode) - _check_version() - key = _model_cache_key(f"provider:{pid}:{mode}") - cached = _model_cache.get(key) - if cached is not None: - return cached - resolved = ModelConfigService.get_instance().resolve_provider(pid) - if resolved is None: - return _get_main_model(mode) - - instance = with_failover( - build_model_for_mode(resolved, mode=mode, stream=True), resolved, mode=mode - ) - _model_cache[key] = instance - return instance - - -def _resolve_chat_mode(ctx) -> str: - """Resolve the final chat_mode (turbo/fast/medium/high/max) from agent.state (AgentRuntimeState).""" - raw = getattr(ctx, "chat_mode", None) - if raw in ("turbo", "fast", "low", "medium", "high", "xhigh", "max"): - return raw - # Fallback for legacy clients - return "medium" if getattr(ctx, "enable_thinking", True) else "fast" - - -# ── FileContext hook (replaces FileContextMiddleware) ───────────────────── - - -def _is_image(f: Dict[str, Any]) -> bool: - mime = (f.get("mime_type") or "").lower() - return mime.startswith("image/") - - -def _download_artifact_bytes( - file_id: str, - fallback_name: str, - log_prefix: str, - user_id: Optional[str] = None, -) -> Optional[bytes]: - """Resolve an artifact's storage key and download its raw bytes. - - Returns None on any failure (missing artifact, storage error, etc.) and - logs at WARNING. Used by the image and xlsx-preview hook branches that - both need raw bytes (not parsed text). - - When ``user_id`` is supplied, the artifact's owner is verified via - :func:`load_artifact_meta` before any storage round-trip. This is the - defense-in-depth gate for hook-level reads: the agent context is built - from ``request.attachments[].file_id`` (client-supplied), and the - backend boundary doesn't re-verify ownership before reaching the - hook. Without this check, a forged ``file_id`` belonging to another - user would have its raw bytes downloaded and silently materialized - into the agent's memory (image base64 or xlsx preview block). The - matching ``fetch_parsed_text`` already does the same check for parsed - text — this brings the binary paths to parity. - - ``user_id=None`` preserves the legacy "trust the caller" behavior - used by ``core.llm.tools.read_artifact_tool._probe_xlsx_sheet_names`` / - ``_probe_pptx_slide_count`` where ``fetch_parsed_text(user_id=...)`` - is the authoritative gate downstream. - """ - from core.content.artifact_reader import ( - load_artifact_meta, - resolve_artifact_storage, - ) - from core.infra.exceptions import StorageError - from core.storage import get_storage - - if user_id: - meta = load_artifact_meta(file_id, user_id=user_id) - if meta is None: - logger.warning( - "%s: artifact %s denied for user %s (not owner / missing / deleted)", - log_prefix, - file_id, - user_id, - ) - return None - - from core.artifacts.local_reference import authorized_reference, is_local_file_ref - if is_local_file_ref(file_id): - item = authorized_reference(file_id, user_id) - if not item: - return None - try: - from pathlib import Path - return Path(item["path"]).read_bytes() - except OSError: - return None - - storage_key, _ = resolve_artifact_storage(file_id, fallback_name) - if not storage_key: - return None - - try: - return get_storage().download_bytes(storage_key) - except StorageError as e: - logger.warning(f"{log_prefix}: storage download failed for {file_id}: {e}") - return None - except Exception as e: - logger.warning(f"{log_prefix}: unexpected download error for {file_id}: {e}") - return None - - -def _fetch_image_base64( - f: Dict[str, Any], - user_id: Optional[str] = None, -) -> Optional[tuple[str, str]]: - """Fetch image data for a file attachment and return (base64_data, mime_type).""" - file_id = f.get("file_id") or "" - if not file_id: - return None - mime_type = (f.get("mime_type") or "image/png").lower() - raw = _download_artifact_bytes( - file_id, - f.get("name") or "image", - "Image hook", - user_id=user_id, - ) - if raw is None: - return None - return base64.b64encode(raw).decode("utf-8"), mime_type - - -def _is_xlsx(f: Dict[str, Any]) -> bool: - """Detect xlsx by mime_type or filename extension. - - We only special-case xlsx (not xls/csv) because: - - xlsx is the dominant batch-execution upload format - - parse_xlsx_preview uses openpyxl, same engine as the batch path - - xls would need a separate xlrd-based preview helper (out of scope) - """ - mime = (f.get("mime_type") or "").lower() - if "spreadsheetml.sheet" in mime: - return True - name = (f.get("name") or f.get("filename") or "").lower() - return name.endswith(".xlsx") - - -def _build_xlsx_preview_block( - f: Dict[str, Any], - user_id: Optional[str] = None, -) -> Optional[str]: - """Build the bounded preview + metadata + directive block for a single xlsx. - - Returns None if the file can't be read (caller falls back to the regular - parsed_text path, so the agent at least sees *something*). - """ - from core.content.file_parser import parse_xlsx_preview - - file_id = f.get("file_id") or "" - name = f.get("name") or f.get("filename") or "未命名表格" - if not file_id: - return None - - file_bytes = _download_artifact_bytes( - file_id, - name, - "xlsx preview", - user_id=user_id, - ) - if file_bytes is None: - return None - - try: - from core.content.artifact_reader import ( - ATTACHMENT_PREVIEW_MAX_CHARS, - fetch_parsed_text, - ) - - # Reuse the bytes already downloaded for the structural preview to - # populate parsed_text + summary without a second storage round-trip. - fetch_parsed_text(file_id, user_id=user_id, prefetched_bytes=file_bytes) - info = parse_xlsx_preview(file_bytes, char_budget=ATTACHMENT_PREVIEW_MAX_CHARS) - except RuntimeError as e: - logger.warning(f"xlsx preview: parse failed for {file_id}: {e}") - return None - except Exception as e: - logger.warning(f"xlsx preview: unexpected parse error for {file_id}: {e}") - return None - - total_rows = info["total_rows"] - total_cols = info["total_columns"] - preview_rows = info["preview_rows"] - omitted = max(0, total_rows - preview_rows) - cols_str = ", ".join(info["header"]) if info["header"] else "(无列名)" - - lines: List[str] = [ - f"[文件: {name}]", - f" file_id: {file_id}", - f" 总规模: {total_rows} 行 × {total_cols} 列", - f" 列名: [{cols_str}]", - f" 已展示: 前 {preview_rows} 行（约 {len(info['preview_md'])} 字符，达到预览预算）", - ] - if info.get("truncated_columns"): - lines.append(f" 注意: 共 {total_cols} 列，预览仅含前 {len(info['header'])} 列") - if info.get("truncated_cells"): - lines.append(f" 注意: {info['truncated_cells']} 个超长单元格已截断到 500 字以内") - lines.append("") - lines.append(info["preview_md"]) - if omitted: - lines.append(f"... (剩余 {omitted} 行未展示) ...") - lines.append("") - lines.append("[操作指引]") - lines.append( - f"- 需要预览之外的正文时，调用 read_artifact(file_id='{file_id}', " - "offset=N, limit=4000) 按需读取" - ) - lines.append("- 以上只是有界预览，不要把未展示的行当作不存在，也不要推断未读取内容") - return "\n".join(lines) - - -def _build_file_context( - uploaded_files: List[Dict[str, Any]], - user_id: Optional[str] = None, -) -> str: - """Assemble the text-attachment list into the context text injected into the model (non-image files only). - - Files are resolved exclusively by ``file_id``. The backend lazily parses and - caches the artifact, then injects only metadata plus a bounded preview. The - request body never supplies parsed content. - - ``user_id`` is used as the ownership check when fetching ``fetch_parsed_text`` / - the xlsx preview, preventing a forged request-body ``attachments[].file_id`` - from reading across users — see the :func:`_download_artifact_bytes` docstring - for the core defense point. - """ - text_files = [f for f in uploaded_files if not _is_image(f)] - if not text_files: - return "" - - file_descriptions = [] - for f in text_files: - name = f.get("name", "未知文件") - fid = f.get("file_id", "") - desc = f"- {name}" - if fid: - desc += f" (file_id: {fid})" - file_descriptions.append(desc) - from core.content.artifact_reader import ( - ATTACHMENT_PREVIEW_MAX_CHARS, - fetch_parsed_text, - ) - - file_content_parts: List[str] = [] - for f in text_files: - # ── xlsx: bounded preview + metadata + directive (no full content) ── - if _is_xlsx(f): - block = _build_xlsx_preview_block(f, user_id=user_id) - if block is not None: - file_content_parts.append(block) - continue - # Fall through to regular path on preview failure so the agent - # at least sees the parsed markdown (degraded but non-fatal). - - fid = (f.get("file_id") or "").strip() - mime_type = f.get("mime_type") or "" - if not fid: - file_content_parts.append( - f"[文件: {f.get('name', '未知文件')}]\n" - " 状态: 缺少 file_id，无法读取正文" - ) - continue - - logger.info("Document hook: loading preview for %s (%s)", f.get("name"), fid) - content = fetch_parsed_text(fid, user_id=user_id) - if not content: - file_content_parts.append( - f"[文件: {f.get('name', '未知文件')}]\n" - f" file_id: {fid}\n" - f" MIME: {mime_type or '(未知)'}\n" - " 状态: 文件不可访问或解析失败" - ) - continue - - total_chars = len(content) - preview = content[:ATTACHMENT_PREVIEW_MAX_CHARS] - truncated = total_chars > len(preview) - block = [ - f"[文件: {f.get('name', '未知文件')}]", - f" file_id: {fid}", - f" MIME: {mime_type or '(未知)'}", - f" 解析文本: 共 {total_chars} 字符，本轮自动展示 {len(preview)} 字符", - "", - preview, - ] - if truncated: - block.append(f"... (其余 {total_chars - len(preview)} 字符未展示) ...") - block.extend( - [ - "", - "[读取指引]", - f"- 需要未展示的正文时，调用 read_artifact(file_id='{fid}', offset=N, limit=4000)", - "- 不要推断尚未读取的内容", - ] - ) - file_content_parts.append("\n".join(block)) - file_content = "\n\n---\n\n".join(file_content_parts) - - return ( - f"[file name]: {chr(10).join(file_descriptions)}\n" - f"[file content begin]\n" - f"{file_content}\n" - f"[file content end]\n" - "以上是用户附件的元数据和有界预览；需要预览之外的正文时必须按需调用 read_artifact。" - ) - - -def _source_label(source: str) -> str: - from core.content.artifact_reader import SOURCE_AI_GENERATED, SOURCE_USER_UPLOAD - - return {SOURCE_USER_UPLOAD: "用户上传", SOURCE_AI_GENERATED: "AI 生成"}.get(source, "") - - -def _build_historical_files_context(historical_files: List[Dict[str, Any]]) -> str: - """Build a compact summary block for files from previous turns. - - Includes both user-uploaded files and AI-generated files (reports, - charts, code output, etc.) produced by tools in earlier turns. - Each entry is labeled by provenance so the agent knows what kind of - file it's referencing. - - Agent gets only `{filename, file_id, source, summary}`; full content - is fetched on-demand via the `read_artifact` tool. This keeps prompt - size bounded as the conversation grows. - """ - if not historical_files: - return "" - - lines: List[str] = [] - for f in historical_files: - file_id = f.get("file_id") or "" - name = f.get("name") or f.get("filename") or "未命名文件" - mime = f.get("mime_type") or "" - summary = (f.get("summary") or "").strip() - source = f.get("source") or "" - deleted = bool(f.get("deleted")) - - source_label = _source_label(source) - header_parts = [f"file_id: {file_id}"] - if source_label: - header_parts.append(source_label) - if mime: - header_parts.append(mime) - header = f"- {name} ({', '.join(header_parts)})" - - if deleted: - lines.append(header + " [文件已删除，无法读取]") - continue - - lines.append(header) - # Key point: tell the model the stable /myspace/ path so it can operate - # directly with Read/Edit/Write — better suited than read_artifact for - # "modify HTML/CSS/MD/source code" text-editing scenarios, and changes sync - # back to "My Space" immediately. - if name and source == _ai_source(): - lines.append(f" 沙盒路径: /myspace/{name} ← 用 Read/Edit/Write 直接读改写") - if summary: - indented = "\n".join(f" {ln}" for ln in summary.splitlines()) - lines.append(f" 摘要：\n{indented}") - else: - lines.append(" 摘要：（尚未生成）") - - header = ( - "[历史文件清单]（本会话中之前轮次涉及的文件，包含用户上传和 AI 生成的两类，以下仅为摘要）\n" - "访问方式：\n" - '- **要修改文本/HTML/MD/代码** → 用 `Read("/myspace/")` 读完整内容，' - "再用 `Edit` / `Write` 改；改完会自动同步回「我的空间」（同一 file_id）。\n" - "- **要看 docx/pdf/xlsx 等需要解析的二进制文件** → 用 " - "`read_artifact(file_id, offset, limit)` 拿解析后的文本。" - ) - return header + "\n\n" + "\n".join(lines) - - -def _ai_source() -> str: - """Lazy resolve to avoid circular import at module load.""" - from core.content.artifact_reader import SOURCE_AI_GENERATED - - return SOURCE_AI_GENERATED - - -# ── WorkspacePinHint hook ───────────────────────────────────────────────── -# Strict-mode `pin_to_workspace` gating means generated files are hidden -# from users until the agent explicitly pins them. Multiple input paths -# (MCP tools that return file_id, `sandbox_get_artifact` for bash-produced -# files, skill scripts) all share this responsibility, and the model -# (especially smaller / flash variants under long contexts) often forgets -# the pin step — see trace aeb643c6-5a8f-4874-a4b4-3d472c581fa6 where 6 -# excel_create_workbook + pdf_create + excel_export_pdf -# produced 8 file_ids, all delivered to the user as exactly zero files. -# -# This `post_acting` hook scans every tool result for `"file_id"` mentions, -# diffs them against the workspace state's pinned set, and appends a -# concise system reminder to memory listing the unpinned ids. The reminder -# is internal (added directly via `memory.add`, not via `print`) so it -# doesn't surface to the SSE stream. - -# Pattern picks up top-level `"file_id": ""` keys in tool result JSON. -# We deliberately match the JSON key form so we ignore `artifact_id`-style -# echoes from `Bash` / `sandbox_put_artifact` etc., which -# operate on already-existing artifacts and don't need pinning. -_FILE_ID_RE = re.compile(r'"file_id"\s*:\s*"([A-Za-z0-9_\-]+)"') - -# Tools whose results legitimately mention `file_id` but never produce a -# new deliverable artifact — skipping them avoids spurious reminders. -_PIN_HINT_SKIP_TOOLS: frozenset[str] = frozenset( - { - "pin_to_workspace", # the pin tool itself - "read_artifact", # echoes the input file_id; reading, not producing - "generate_response", # the ReActAgent finish function - } -) diff --git a/src/backend/core/llm/host_services.py b/src/backend/core/llm/host_services.py new file mode 100644 index 00000000..95796e18 --- /dev/null +++ b/src/backend/core/llm/host_services.py @@ -0,0 +1,31 @@ +"""Compose model-owned services at the application host boundary.""" +from core.harness.registry.model_services import register_model_services +from core.llm import prompt_kinds, prompt_versions, skill_selection + + +def active_prompt_versions(): + versions = [] + for kind in list(prompt_kinds.KIND_SPECS): + try: + active = prompt_versions.get_active_version(kind) + except Exception: + continue + if active: + versions.append((kind, active)) + return versions + + +def register(): + from core.harness.registry.mcp_clients import register_mcp_client_factory + from core.llm.mcp.lifetime import discover + register_mcp_client_factory(discover) + from core.models.config import ModelConfigService + from core.llm.prompt_rendering import render_active_prompt_part + from core.llm.runtime.review_text_agent import run_text_agent + register_model_services( + ranker=lambda **kwargs: skill_selection.select_skills_for_query(**kwargs), + prompt_reader=active_prompt_versions, + prompt_renderer=render_active_prompt_part, + text_agent=run_text_agent, + environment=lambda: ModelConfigService.get_instance().get_mcp_env_overlay(), + ) diff --git a/src/backend/core/llm/management_tool.py b/src/backend/core/llm/management_tool.py deleted file mode 100644 index 617bbc70..00000000 --- a/src/backend/core/llm/management_tool.py +++ /dev/null @@ -1,84 +0,0 @@ -"""Local execution of an explicitly declared, installed manager plugin contract.""" -import asyncio -import json -from agentscope.tool import ToolBase, ToolChunk -from agentscope.message import TextBlock, ToolResultState -from agentscope.permission import PermissionBehavior, PermissionDecision -from core.services.management_contract import declared, MANAGERS -from core.services.management_errors import management_error - - -class ManagementTool(ToolBase): - is_mcp = True - is_state_injected = False - is_external_tool = False - is_concurrency_safe = True - - def __init__(self, server_id, tool, manager, headers, *, cloud_backed): - self.mcp_name, self.name = server_id, tool.name - self.description, self.input_schema = tool.description, tool.inputSchema - self.manager, self.headers = manager, dict(headers) - self.cloud_backed = cloud_backed - self.is_read_only = self.name.startswith(("get_", "list_")) - - async def check_permissions(self, *_args, **_kwargs): - return PermissionDecision(behavior=PermissionBehavior.ALLOW if self.is_read_only else PermissionBehavior.ASK, - message="管理当前用户的本机技能或插件") - - def _execute(self, arguments): - from core.services import desktop_cloud_bridge as bridge - from core.capabilities import skills - from core.auth.desktop_bridge import bridge_enabled - state = None - if self.cloud_backed and bridge_enabled(): - state = bridge.get_state() - bridge.require_current_account(state) - user_id = skills.current_local_user_id() - context = bridge._bridge_context() or {} - server = next((s for s in context.get("servers", []) if s["server_id"] == self.mcp_name), None) - if not server or server.get("source_plugin") != self.manager: - raise PermissionError("management plugin is no longer installed or authorized") - # upload is a local contribution authorized by the same declared install capability. - op = "install_skill" if self.name == "upload_skill_to_cloud" else self.name - contract = next((t for t in server.get("tools", []) if t["name"] == op), None) - if contract is None or declared(contract, self.manager) != self.manager: - raise PermissionError("management plugin contract changed; synchronize capabilities") - else: - user_id = next((v for k, v in self.headers.items() if k.lower() == "x-current-user-id"), None) - from core.capabilities.local_plugin_runtime import authorizes_manager - if not authorizes_manager(user_id, self.mcp_name, self.manager): - raise PermissionError("local management plugin is not installed") - if not user_id: - raise PermissionError("authentication required") - from jsonschema import validate - validate(arguments, self.input_schema) - if state: - bridge.require_current_account(state) - if self.name == "upload_skill_to_cloud": - from core.services.local_skill_upload import upload - return upload(user_id, **arguments) - kind = MANAGERS[self.manager] - from core.services import local_skill_service - from core.plugins.local import service as local_plugin_service - service = local_skill_service if kind == "skill" else local_plugin_service - action = self.name.removesuffix("_" + kind) - if self.name == "list_" + kind + "s": - return {"ok": True, "items": getattr(service, "list_" + kind + "s")(user_id)} - args = dict(arguments) - if "source" in args: - source = args.pop("source") - if source.get("kind") != "local_path": - raise ValueError("local installation requires source.kind=local_path") - args["source_path"] = source["path"] - return getattr(service, action)(user_id, **args) - - async def __call__(self, **kwargs): - try: - result = await asyncio.to_thread(self._execute, kwargs) - return ToolChunk(content=[TextBlock(text=json.dumps(result, ensure_ascii=False))], - state=ToolResultState.SUCCESS, - metadata={"origin": "local", "capabilities_changed": not self.is_read_only}) - except Exception as exc: - message = management_error(exc) - return ToolChunk(content=[TextBlock(text=json.dumps({"ok": False, "error": message}, ensure_ascii=False))], - state=ToolResultState.ERROR, metadata={"origin": "local"}) diff --git a/src/backend/core/llm/mcp/__init__.py b/src/backend/core/llm/mcp/__init__.py new file mode 100644 index 00000000..62e5be1f --- /dev/null +++ b/src/backend/core/llm/mcp/__init__.py @@ -0,0 +1 @@ +"""MCP client pool, tool invocation and the gateway tool for agents.""" diff --git a/src/backend/core/llm/mcp/callbacks.py b/src/backend/core/llm/mcp/callbacks.py new file mode 100644 index 00000000..2499287a --- /dev/null +++ b/src/backend/core/llm/mcp/callbacks.py @@ -0,0 +1,168 @@ +"""Run-bound MCP interactions. Remote services never receive the main conversation. + +A server request (sampling, elicitation) is answered only on behalf of the outer +tool call that caused it. On 2026-07-28 connections the SDK runs those callbacks +inside the calling task, so `CURRENT_CALL` names the call exactly; handshake-era +servers deliver them from the session task, where only a sole in-flight call can +be credited. +""" +import asyncio +import json +from contextlib import aclosing, nullcontext +from contextvars import ContextVar +from dataclasses import dataclass +from urllib.parse import urlsplit +import jsonschema +from agentscope.message import Msg, TextBlock +from mcp import types +from core.llm.tools import user_questions + +CURRENT_CALL: ContextVar["CallScope | None"] = ContextVar("mcp_current_call", default=None) + + +def route(session): + call = CURRENT_CALL.get() + if call is not None and call.session is session: + return call + return next(iter(session.in_flight)) if len(session.in_flight) == 1 else None + + +@dataclass +class Interactions: + """What one run allows a server to ask of it; shared by that run's calls.""" + request: object + server: str + interactive: bool + agent: object = None + samples: int = 0 + + def open_call(self, session=None): + return CallScope(self, session) + + +class CallScope: + """One outer tool call; server requests made on its behalf end with it.""" + + def __init__(self, run, session=None): + self.run, self.session = run, session + self.active = True + self.pending = set() + self.budget = None + + async def dispatch(self, method, context, params): + task = asyncio.create_task(getattr(self, method)(context, params)) + self.pending.add(task) + try: + return await task + finally: + self.pending.discard(task) + + async def finish(self): + self.active = False + tasks = list(self.pending) + for task in tasks: + task.cancel() + if tasks: + await asyncio.gather(*tasks, return_exceptions=True) + + async def validate(self): + if not self.active or self.run.agent is None: + raise RuntimeError("MCP interaction is outside an active run") + prepared = getattr(self.run.agent, "_jx_prepared_capabilities", None) + if prepared is not None: + await asyncio.to_thread(prepared.validate, user_id=str(self.run.request.current_user_id or "")) + if not await asyncio.to_thread(prepared.validate_tool_call, self.run.server): + raise PermissionError("MCP capability was revoked") + guard = getattr(getattr(self.run.agent, "toolkit", None), "capability_guard", None) + if guard is not None and not await asyncio.to_thread(guard, self.run.server): + raise PermissionError("MCP capability was revoked") + + async def sampling(self, _context, params): + if not self.active or self.run.agent is None or self.run.samples >= 4: + return types.ErrorData(code=-32600, message="Sampling unavailable or budget exhausted") + if params.tools or params.include_context not in (None, "none"): + return types.ErrorData(code=-32602, message="Sampling accepts isolated text without tools or shared context") + messages = [] + for message in params.messages: + content = message.content if isinstance(message.content, list) else [message.content] + if any(not isinstance(item, types.TextContent) for item in content): + return types.ErrorData(code=-32602, message="Only text sampling is enabled") + messages.append(Msg(name=self.run.server, role=message.role, + content=[TextBlock(text=item.text) for item in content])) + if params.system_prompt: + messages.insert(0, Msg(name=self.run.server, role="system", content=params.system_prompt)) + if len(params.model_dump_json()) > 32000 or not 0 < params.max_tokens <= 4096: + return types.ErrorData(code=-32602, message="Sampling request exceeds the run budget") + self.run.samples += 1 + await self.validate() + from core.models.usage import model_usage_scope + from core.harness.ledger import HarnessUsageLedger + with model_usage_scope(self.run.request.run_id or "", HarnessUsageLedger()): + async with asyncio.timeout(60): + response = await self.run.agent.model(messages, tools=None, max_tokens=params.max_tokens) + response_value = None + if hasattr(response, "__aiter__"): + async with aclosing(response): + async for chunk in response: + response_value = chunk + else: + response_value = response + if response_value is None: + return types.ErrorData(code=-32603, message="Sampling model returned no response") + if not self.active: + return types.ErrorData(code=-32600, message="Outer MCP invocation ended") + await self.validate() + text = "".join(block.text for block in response_value.content if isinstance(block, TextBlock)) + return types.CreateMessageResult(role="assistant", content=types.TextContent(text=text), + model=str(self.run.agent.model.model), stop_reason="endTurn") + + async def elicitation(self, _context, params): + if not self.active or not self.run.interactive: + return types.ElicitResult(action="decline") + await self.validate() + if params.mode == "url": + parsed = urlsplit(params.url) + if parsed.scheme != "https" or not parsed.hostname or parsed.username or parsed.password: + return types.ElicitResult(action="decline") + questions = [{"id": "external", "question": f"{self.run.server}: {params.message}\n{params.url}", + "options": [{"label": "已在外部页面完成"}, {"label": "拒绝"}]}] + else: + schema = params.requested_schema + props = schema.get("properties", {}) + if not props or len(props) > user_questions.MAX_QUESTIONS: + return types.ElicitResult(action="decline") + questions = [] + for key, spec in props.items(): + if spec.get("format") in {"password", "secret"} or spec.get("writeOnly"): + return types.ElicitResult(action="decline") + values = spec.get("enum", [True, False] if spec.get("type") == "boolean" else []) + questions.append({"id": key, "question": f"{self.run.server}: {params.message} — {spec.get('title', key)}", + "options": [{"label": json.dumps(v, ensure_ascii=False)} for v in values]}) + # Only the human wait is excluded: validation, sampling and server + # work before/after this block still consume the original budget. + with self.budget.waiting_for_user() if self.budget is not None else nullcontext(): + outcome = await user_questions.ask(chat_id=self.run.request.chat_id, questions=questions, interactive=True) + if not self.active or outcome["status"] != "answered": + return types.ElicitResult(action="cancel") + await self.validate() + answers = outcome["answers"] + if params.mode == "url": + accepted = answers[0].get("selected_labels") == ["已在外部页面完成"] + return types.ElicitResult(action="accept" if accepted else "decline") + content = {} + try: + for answer in answers: + if answer.get("skipped"): + continue + key = answer["id"] + selected = answer.get("selected_labels") or [] + value = answer.get("custom") + if value is None and selected: + value = json.loads(selected[0]) + elif props[key].get("type") != "string": + value = json.loads(value) + content[key] = value + jsonschema.validate(content, schema) + except (ValueError, TypeError, jsonschema.ValidationError, jsonschema.SchemaError): + return types.ElicitResult(action="decline") + return types.ElicitResult(action="accept", content=content) diff --git a/src/backend/core/llm/mcp/chat_sessions.py b/src/backend/core/llm/mcp/chat_sessions.py new file mode 100644 index 00000000..69527178 --- /dev/null +++ b/src/backend/core/llm/mcp/chat_sessions.py @@ -0,0 +1,181 @@ +"""Bounded pool of chat-scoped STDIO sessions shared by that chat's concurrent runs. + +A session is keyed by the trusted executing user, chat and capability scope plus the +server configuration. Runs hold counted leases; the call gate inside the session +orders writes, so a run never waits for another run to finish. Idle sessions +expire, and are reclaimed oldest-first when capacity is needed. +""" +import asyncio +import hashlib +import json +import time +from dataclasses import dataclass + +IDLE_SECONDS = 60.0 +MAX_SESSIONS = 32 +MAX_USER_SESSIONS = 4 +_entries = {} + + +@dataclass +class Entry: + key: tuple + loop: object + session: object = None + opening: asyncio.Task | None = None + expiry: asyncio.Task | None = None + holders: int = 0 + idle_at: float = 0.0 + retired: bool = False + + @property + def idle(self): + return self.holders == 0 and self.opening is None + + def detach(self): + """Stop handing this session out; its holders keep it until they release.""" + if _entries.get(self.key) is self: + del _entries[self.key] + self.retired = True + if self.expiry is not None: + self.expiry.cancel() + self.expiry = None + + +class Lease: + def __init__(self, entry): + self.entry, self.session = entry, entry.session + self._released = False + + async def release(self, *, abort=False): + if self._released: + return + self._released = True + entry = self.entry + entry.holders -= 1 + if abort or self.session.stopped: + entry.detach() + if entry.holders: + return + if entry.retired: + await self.session.close() + else: + _park(entry) + + +async def _expire(entry): + await asyncio.sleep(IDLE_SECONDS) + if entry.idle and _entries.get(entry.key) is entry: + entry.detach() + await entry.session.close() + + +async def _reclaim(user): + """Make room for one more session, closing idle sessions oldest-first.""" + while True: + user_entries = [e for k, e in _entries.items() if k[0] == user] + if len(user_entries) >= MAX_USER_SESSIONS: + candidates = user_entries + elif len(_entries) >= MAX_SESSIONS: + candidates = list(_entries.values()) + else: + return + idle = [e for e in candidates if e.idle] + if not idle: + raise RuntimeError("MCP STDIO session capacity reached") + victim = min(idle, key=lambda e: e.idle_at) + victim.detach() + await victim.session.close() + + +def _key(scope, name, cfg): + from mcp.client.stdio import get_default_environment + cfg = {**cfg, "env": get_default_environment() | (cfg.get("env") or {})} + cfg.pop("oauth_provider", None) + fingerprint = hashlib.sha256(json.dumps(cfg, sort_keys=True, separators=(",", ":"), default=str).encode()).hexdigest() + return (*scope, name, fingerprint) + + +def _park(entry): + """An unheld session waits out its idle period, or closes now if it was retired.""" + if entry.retired: + return asyncio.create_task(entry.session.close()) + entry.idle_at = time.monotonic() + entry.expiry = asyncio.create_task(_expire(entry)) + + +async def _open(entry, name, cfg): + from core.llm.mcp.session import Session + try: + session = await Session.open(name, cfg) + except BaseException: + if entry.opening is asyncio.current_task(): + entry.opening = None + entry.detach() + raise + entry.opening = None + entry.session = session + if entry.retired: + await session.close() + raise RuntimeError("MCP chat session was revoked") + if entry.holders == 0: + _park(entry) + return session + + +async def acquire(scope, name, cfg): + # Scope contains trusted executing user, chat and capability scope, not remote metadata. + key = _key(scope, name, cfg) + for old in [e for k, e in _entries.items() if k[:-1] == key[:-1] and k != key]: + old.detach() + if old.idle and old.session is not None: + await old.session.close() + entry = _entries.get(key) + if entry is not None and (entry.retired or entry.session is not None and entry.session.stopped): + entry.detach() + entry = None + if entry is None: + await _reclaim(scope[0]) + entry = _entries.setdefault(key, Entry(key, asyncio.get_running_loop())) + if entry.expiry is not None: + entry.expiry.cancel() + entry.expiry = None + entry.holders += 1 + try: + if entry.session is None: + if entry.opening is None: + entry.opening = asyncio.create_task(_open(entry, name, cfg), name=f"mcp-open:{name}") + await asyncio.shield(entry.opening) + if entry.retired or entry.session.stopped: + raise RuntimeError("MCP chat session was revoked") + return Lease(entry) + except BaseException: + entry.holders -= 1 + if entry.holders == 0 and entry.session is not None and entry.expiry is None: + _park(entry) + raise + + +async def close_chat(chat_id=None): + """Revoke every session of a chat (or all); holders' next calls fail as closed.""" + for entry in [e for k, e in list(_entries.items()) if chat_id is None or k[1] == chat_id]: + entry.detach() + if entry.opening is not None: + opening, entry.opening = entry.opening, None + opening.cancel() + await asyncio.gather(opening, return_exceptions=True) + if entry.session is not None: + entry.session.failed = True + await entry.session.close() + + +def invalidate_chat(chat_id): + """Deletion can run on a sync API worker; dispatch teardown to the owning loop.""" + entries = [entry for key, entry in list(_entries.items()) if key[1] == chat_id] + for entry in entries: + # Plain flag writes take effect at once; dict and task work stays on the owning loop. + entry.retired = True + if entry.session is not None: + entry.session.failed = True + for loop in {entry.loop for entry in entries}: + loop.call_soon_threadsafe(lambda: asyncio.create_task(close_chat(chat_id))) diff --git a/src/backend/core/llm/mcp/client.py b/src/backend/core/llm/mcp/client.py new file mode 100644 index 00000000..9106c556 --- /dev/null +++ b/src/backend/core/llm/mcp/client.py @@ -0,0 +1,149 @@ +"""SDK-native connections; project authorization is attached at the transport boundary.""" +import asyncio +import hashlib +import json +import math +import os +import weakref +from contextlib import asynccontextmanager +from typing import Any +import httpx2 +from mcp import Client, StdioServerParameters +from mcp.client.caching import CacheConfig, InMemoryResponseCacheStore +from mcp.client.sse import sse_client +from mcp.client.streamable_http import streamable_http_client +from mcp.shared._httpx_utils import create_mcp_http_client + + +HTTP_TRANSPORTS = frozenset({"streamable_http", "sse"}) +_KB_SERVER_ID = "retrieve_dataset_content" + + +def _positive_timeout(value: Any, default: float) -> float: + try: + parsed = float(value) + return parsed if parsed > 0 else default + except (TypeError, ValueError): + return default + + +def _execution_timeout(name: str, cfg: dict) -> float: + """Resolve a hard tool-call deadline, with a tighter KB-specific default.""" + if cfg.get("execution_timeout") is not None: + return _positive_timeout(cfg["execution_timeout"], 120.0) + if name == _KB_SERVER_ID: + return _positive_timeout(os.getenv("KB_MCP_EXECUTION_TIMEOUT_SECONDS"), 75.0) + return _positive_timeout(os.getenv("MCP_TOOL_EXECUTION_TIMEOUT_SECONDS"), 120.0) + + +def _http_transport_timeout(name: str, cfg: dict, execution_timeout: float) -> float: + if cfg.get("transport_timeout") is not None: + return _positive_timeout(cfg["transport_timeout"], 30.0) + if name == _KB_SERVER_ID: + # Leave a small margin beyond the MCP server's own structured timeout + # so callers receive its tool_result instead of a transport exception. + return execution_timeout + 5.0 + if name == "internet_search": + # Search returns per-query timeouts after its 45-second batch deadline. + return 50.0 + return 30.0 + + +def is_http_cfg(cfg: dict) -> bool: + return cfg.get("transport") in HTTP_TRANSPORTS + + +@asynccontextmanager +async def authorized_transport(name, cfg): + from core.llm.mcp.invocation import HEADER, PLUGIN_HEADER, for_url + provider = cfg.get("oauth_provider") + headers = cfg.get("headers") or {} + signed = any(key.lower() == HEADER.lower() for key in headers) + + async def authorize(request): + from core.llm.mcp.callbacks import CURRENT_CALL + call = CURRENT_CALL.get() + if call is not None and call.budget is not None: + # This request is bounded by the invocation execution budget. HTTP + # read time includes human waits; retain connect/write/pool limits. + request.extensions["timeout"] = {**request.extensions.get("timeout", {}), "read": None} + if provider is not None: + await provider.mcp_request_hook(request) + if signed: + request.headers.update(for_url( + cfg["url"], request.headers.get("x-current-user-id"), + request.headers.get("x-chat-id"), request.headers.get(PLUGIN_HEADER, ""), + )) + + timeout = _http_transport_timeout(name, cfg, _execution_timeout(name, cfg)) + if cfg["transport"] == "sse": + def factory(headers=None, timeout=None, auth=None): + return httpx2.AsyncClient(headers=headers, timeout=timeout, auth=auth, + event_hooks={"request": [authorize]}) + # One SSE stream serves all invocations; a shared idle deadline cannot + # distinguish human waits. Discovery and invocation budgets bound work. + async with sse_client(cfg["url"], headers=headers, timeout=timeout, + sse_read_timeout=math.inf, auth=provider, httpx_client_factory=factory) as streams: + yield streams + else: + async with create_mcp_http_client(headers=headers, timeout=httpx2.Timeout(timeout), + auth=provider) as http: + http.event_hooks["request"].append(authorize) + async with streamable_http_client(cfg["url"], http_client=http) as streams: + yield streams + + +# Listings outlive a run: each run opens its own HTTP client, so discovery results +# are shared through one store per event loop, partitioned by authorization context. +FIRST_PARTY_LISTING_TTL_MS = 60_000 +_VOLATILE_HEADERS = frozenset({"x-chat-id", "x-conversation-id", "x-hugagent-invocation"}) +_listing_stores = weakref.WeakKeyDictionary() + + +def listing_cache(cfg: dict): + """Cache config for an HTTP server, or None to keep the SDK's per-client default.""" + if not is_http_cfg(cfg) or cfg.get("oauth_provider") is not None: + return None + try: + loop = asyncio.get_running_loop() + except RuntimeError: + return None + from core.llm.mcp.invocation import audience_for + headers = {str(k).lower(): str(v) for k, v in (cfg.get("headers") or {}).items() + if str(k).lower() not in _VOLATILE_HEADERS} + partition = hashlib.sha256(json.dumps(headers, sort_keys=True).encode()).hexdigest() + store = _listing_stores.setdefault(loop, InMemoryResponseCacheStore()) + # Built-in servers send no freshness hints; their listings change only on deploy. + ttl = FIRST_PARTY_LISTING_TTL_MS if audience_for(cfg["url"]) else 0 + return CacheConfig(store=store, partition=partition, target_id=cfg["url"], default_ttl_ms=ttl) + + +class AuthorizedClient(Client): + """Reject schema changes within a frozen run instead of replaying an old permission.""" + def freeze(self, descriptors): + self.frozen_tools = {d.name: d.model_dump(mode="json") for d in descriptors} + + async def list_tools(self, **kwargs): + page = await super().list_tools(**kwargs) + frozen = getattr(self, "frozen_tools", None) + if frozen is not None: + for tool in page.tools: + if frozen.get(tool.name) != tool.model_dump(mode="json"): + raise ValueError("MCP tool schema changed; a new authorized run is required") + return page + + +def make_client(name: str, cfg: dict, *, sampling_callback=None, elicitation_callback=None) -> Client: + """One native client per authorized lifetime; negotiation and caches belong to the SDK.""" + transport = (authorized_transport(name, cfg) if is_http_cfg(cfg) else + StdioServerParameters(command=cfg.get("command", "python"), + args=cfg.get("args", []), env=cfg.get("env") or None, + cwd=cfg.get("cwd"))) + cache = listing_cache(cfg) + # Interactive sessions also serve resources/prompts, whose SDK methods expose + # no per-request timeout. Session.open bounds discovery, and SessionTool + # bounds each invocation; never mutate a shared SDK timer during a callback. + sdk_timeout = math.inf if elicitation_callback is not None else _execution_timeout(name, cfg) + return AuthorizedClient(transport, mode="auto", read_timeout_seconds=sdk_timeout, + sampling_callback=sampling_callback, elicitation_callback=elicitation_callback, + **({"cache": cache} if cache is not None else {})) diff --git a/src/backend/core/llm/mcp/context_tools.py b/src/backend/core/llm/mcp/context_tools.py new file mode 100644 index 00000000..75776f26 --- /dev/null +++ b/src/backend/core/llm/mcp/context_tools.py @@ -0,0 +1,69 @@ +"""Expose advertised MCP resources and prompts as permission-governed tools.""" +import hashlib +import json +import jsonschema +from mcp.shared.uri_template import UriTemplate +from agentscope.message import TextBlock +from agentscope.tool import ToolChunk +from agentscope.permission import PermissionBehavior, PermissionDecision +from core.llm.mcp.results import content_blocks +from core.llm.mcp.tools import SessionTool + + +class ContextTool(SessionTool): + is_read_only = True + is_concurrency_safe = True + + def __init__(self, resource, kind, descriptor, timeout): + super().__init__(resource, timeout) + server = resource.name + self.kind, self.descriptor = kind, descriptor + identity = (str(descriptor.uri) if kind == "resource" else descriptor.uri_template + if kind == "resource_template" else descriptor.name) + self.name = "mcp_" + kind + "_" + hashlib.sha256(f"{server}:{identity}".encode()).hexdigest()[:16] + self.description = f"{kind} from {server}: {descriptor.description or identity}" + self.input_schema = {"type": "object", "properties": {}, "additionalProperties": False} + if kind == "resource_template": + self.template = UriTemplate.parse(descriptor.uri_template) + self.input_schema["properties"] = {v.name: ({"type": "array", "items": {"type": "string"}} + if v.explode else {"type": "string"}) for v in self.template.variables} + self.input_schema["required"] = [v for v in self.template.variable_names + if v not in self.template.query_variable_names] + if kind == "prompt": + self.input_schema["properties"] = {a.name: {"type": "string", "description": a.description or a.name} + for a in descriptor.arguments or []} + self.input_schema["required"] = [a.name for a in descriptor.arguments or [] if a.required] + + async def check_permissions(self, *_args, **_kwargs): + return PermissionDecision(behavior=PermissionBehavior.ASK, + message="Read remote MCP context from the selected server") + + async def call(self, **kwargs): + return await self._invoke(lambda: self._read(**kwargs), exclusive=False) + + async def _read(self, **kwargs): + jsonschema.validate(kwargs, self.input_schema) + client = self.resource.session.client + if self.kind in {"resource", "resource_template"}: + uri = str(self.descriptor.uri) if self.kind == "resource" else self.template.expand(kwargs) + # An explicit read wants current content; only listings are cached. + result = await client.read_resource(uri, cache_mode="bypass") + blocks = [TextBlock(text=item.model_dump_json(by_alias=True, exclude_none=True)) for item in result.contents] + else: + result = await client.get_prompt(self.descriptor.name, arguments=kwargs) + blocks = [] + for message in result.messages: + blocks.append(TextBlock(text=json.dumps({"role": message.role}, ensure_ascii=False))) + blocks.extend(content_blocks([message.content])) + return ToolChunk(content=blocks, metadata={"mcp": result.model_dump(mode="json", by_alias=True, exclude_none=True)}) + + +def context_tools(resource, timeout): + tools, names = [], set() + for kind, descriptor in resource.session.context: + tool = ContextTool(resource, kind, descriptor, timeout) + if tool.name in names: + raise ValueError("Duplicate MCP context identity") + names.add(tool.name) + tools.append(tool) + return tools diff --git a/src/backend/core/llm/mcp/deadline.py b/src/backend/core/llm/mcp/deadline.py new file mode 100644 index 00000000..95d119d7 --- /dev/null +++ b/src/backend/core/llm/mcp/deadline.py @@ -0,0 +1,40 @@ +"""Per-invocation execution time, excluding time spent waiting for a person.""" +import asyncio +from contextlib import contextmanager + + +class InvocationBudget: + """One task deadline; overlapping human waits pause the same clock once.""" + + def __init__(self, seconds: float): + self._timeout = asyncio.timeout(seconds) + self._remaining = seconds + self._waiters = 0 + self._active = False + + async def __aenter__(self): + await self._timeout.__aenter__() + self._active = True + return self + + async def __aexit__(self, *exc): + self._active = False + return await self._timeout.__aexit__(*exc) + + @contextmanager + def waiting_for_user(self): + # A late callback must not resurrect a completed or already expired call. + if not self._active or self._timeout.expired(): + raise TimeoutError("MCP invocation is no longer active") + if self._waiters == 0: + self._remaining = self._timeout.when() - asyncio.get_running_loop().time() + if self._remaining <= 0: + raise TimeoutError("MCP execution budget exhausted") + self._timeout.reschedule(None) + self._waiters += 1 + try: + yield + finally: + self._waiters -= 1 + if self._waiters == 0 and self._active and not self._timeout.expired(): + self._timeout.reschedule(asyncio.get_running_loop().time() + self._remaining) diff --git a/src/backend/core/llm/mcp_invocation.py b/src/backend/core/llm/mcp/invocation.py similarity index 87% rename from src/backend/core/llm/mcp_invocation.py rename to src/backend/core/llm/mcp/invocation.py index faa66c06..85778848 100644 --- a/src/backend/core/llm/mcp_invocation.py +++ b/src/backend/core/llm/mcp/invocation.py @@ -26,10 +26,14 @@ def issue(audience, user_id, chat_id, plugin_id=""): signature = hmac.new(key(), body.encode(), hashlib.sha256).hexdigest() return {HEADER: body + "." + signature} -def for_url(url, user_id, chat_id, plugin_id=""): - from core.config.mcp_config import _mcp_http_url +def audience_for(url): + """The built-in server a URL addresses, or None for any other server.""" + from core.config.mcp_config import mcp_http_url as _mcp_http_url from extensions.mcp_servers._ports import PORTS - audience = next((name for name in PORTS if _mcp_http_url(name).rstrip("/") == str(url).rstrip("/")), None) + return next((name for name in PORTS if _mcp_http_url(name).rstrip("/") == str(url).rstrip("/")), None) + +def for_url(url, user_id, chat_id, plugin_id=""): + audience = audience_for(url) if not audience: return {} try: diff --git a/src/backend/core/llm/mcp/lifetime.py b/src/backend/core/llm/mcp/lifetime.py new file mode 100644 index 00000000..c08240c0 --- /dev/null +++ b/src/backend/core/llm/mcp/lifetime.py @@ -0,0 +1,68 @@ +"""A run's handle on an MCP session: its tools, its interaction authority and its release.""" +import asyncio +from dataclasses import dataclass, field +from core.llm.mcp.client import _execution_timeout, is_http_cfg + +MANIFEST_SOURCES = frozenset({"cloud_manifest", "plugin_manifest"}) + + +@dataclass +class MCPResource: + name: str + tools: list = field(default_factory=list) + session: object = None + interactions: object = None + # Write calls whose outcome is unknown in this run; exact replays are refused. + unknown_outcomes: set = field(default_factory=set) + closed: bool = False + _lease: object = None + _closing: asyncio.Task | None = None + + @property + def descriptors(self): + return self.session.descriptors if self.session is not None else [] + + async def close(self, *, abort=False): + """Close a run-owned session, or return a pooled one; abort also retires the pooled session.""" + # Tools reject calls at once; teardown finishes even if the caller is cancelled again. + self.closed = True + if self._closing is None: + self._closing = asyncio.create_task(self._close(abort)) + await asyncio.shield(self._closing) + + async def _close(self, abort): + if self._lease is not None: + await self._lease.release(abort=abort) + elif self.session is not None: + await self.session.close() + + +def _bind_tools(resource, cfg): + from core.llm.mcp.context_tools import context_tools + from core.llm.mcp.tools import RemoteTool + timeout = _execution_timeout(resource.name, cfg) + enabled, disabled = cfg.get("enable_tools"), set(cfg.get("disable_tools") or []) + tools = [RemoteTool(resource, descriptor, timeout) for descriptor in resource.session.descriptors + if (enabled is None or descriptor.name in enabled) and descriptor.name not in disabled] + return tools + context_tools(resource, timeout) + + +async def discover(name, cfg, *, interactions=None, scope=None): + """Bind a run to a session. Chat-scoped STDIO servers share a pooled process; + HTTP and unscoped servers get a run-owned session.""" + if cfg.get("schema_source") in MANIFEST_SOURCES: + from core.llm.tools.cloud_manifest import manifest_tools + return MCPResource(name, tools=manifest_tools(name, cfg)) + if scope and not is_http_cfg(cfg): + from core.llm.mcp.chat_sessions import acquire + lease = await acquire(scope, name, cfg) + resource = MCPResource(name, session=lease.session, interactions=interactions, _lease=lease) + else: + from core.llm.mcp.session import Session + resource = MCPResource(name, session=await Session.open(name, cfg), interactions=interactions) + try: + resource.tools = _bind_tools(resource, cfg) + except BaseException: + await resource.close(abort=True) + raise + return resource diff --git a/src/backend/core/llm/mcp/manager.py b/src/backend/core/llm/mcp/manager.py new file mode 100644 index 00000000..f7492d00 --- /dev/null +++ b/src/backend/core/llm/mcp/manager.py @@ -0,0 +1,62 @@ +"""Factory failure cleanup and run resource teardown.""" +from __future__ import annotations +import asyncio +import logging +from contextvars import ContextVar +from functools import wraps +logger = logging.getLogger(__name__) + +# The factory owns transient clients until it returns them to its caller. +# ContextVar values propagate into parallel HTTP probes, so partial construction +# has one cleanup owner even when cancellation interrupts asyncio.gather. +_factory_clients: ContextVar[list | None] = ContextVar("factory_mcp_clients", default=None) + + +def own_factory_client(client): + clients = _factory_clients.get() + if clients is not None: + clients.append(client) + return client + + +def close_factory_clients_on_error(function): + @wraps(function) + async def construct(*args, **kwargs): + clients = [] + token = _factory_clients.set(clients) + try: + return await function(*args, **kwargs) + except BaseException: + await close_clients(list(reversed(clients)), abort=True) + raise + finally: + _factory_clients.reset(token) + + return construct + + +async def close_clients(clients: list, *, abort=False) -> None: + """Close every client in its owning task, deferring repeated cancellation.""" + cancelled = None + task = asyncio.current_task() + for client in clients: + while True: + before = task.cancelling() if task else 0 + try: + await client.close(abort=True) if abort else await client.close() + break + except asyncio.CancelledError as exc: + cancelled = exc + if task is not None and task.cancelling() > before: + # An external repeat-cancel interrupted this close. Finish + # this same client's cleanup on the same task, then the rest. + continue + # A client scope itself signalled cancellation. It must not + # prevent independent clients from releasing their resources. + logger.debug("MCP client close signalled cancellation") + break + except Exception as exc: + logger.debug("Error closing MCP client: %s", exc) + break + if cancelled is not None: + raise cancelled diff --git a/src/backend/core/llm/mcp/results.py b/src/backend/core/llm/mcp/results.py new file mode 100644 index 00000000..5f5f39f2 --- /dev/null +++ b/src/backend/core/llm/mcp/results.py @@ -0,0 +1,67 @@ +"""Lossless MCP result metadata with explicit model-facing content conversion.""" +import json +import jsonschema +from agentscope.message import TextBlock, DataBlock, Base64Source +from agentscope.tool import ToolChunk +from agentscope.tool._response import ToolResultState +from mcp import MCPError, types +from mcp.types import CONNECTION_CLOSED, REQUEST_TIMEOUT + + +def content_blocks(contents): + blocks = [] + for item in contents: + if isinstance(item, types.TextContent): + blocks.append(TextBlock(text=item.text)) + elif isinstance(item, (types.ImageContent, types.AudioContent)): + blocks.append(DataBlock(source=Base64Source(media_type=item.mime_type, data=item.data))) + else: + # Resource URIs are references, never automatically fetched as network URLs. + # Preserve links, embedded resources and their provenance for the model. + blocks.append(TextBlock(text=item.model_dump_json(by_alias=True, exclude_none=True))) + return blocks + + +def _carried_by_text(structured, blocks): + """Whether a text block already holds the structured value (SDKs wrap non-object returns as {"result": ...}).""" + for block in blocks: + if not isinstance(block, TextBlock): + continue + if structured == {"result": block.text}: + return True + try: + value = json.loads(block.text) + except ValueError: + continue + if value == structured or structured == {"result": value}: + return True + return False + + +def tool_result(result, descriptor): + content = content_blocks(result.content) + metadata = {"mcp": result.model_dump(mode="json", by_alias=True, exclude_none=True), + "output_schema": descriptor.output_schema} + # The model needs the structured value once; skip it only when text already carries it. + if result.structured_content is not None and not _carried_by_text(result.structured_content, content): + content.append(TextBlock(text=json.dumps(result.structured_content, ensure_ascii=False, + separators=(",", ":")))) + return ToolChunk(content=content, metadata=metadata, + state=ToolResultState.ERROR if result.is_error else ToolResultState.RUNNING) + + +def error_result(text, **metadata): + return ToolChunk(content=[TextBlock(text=text)], state=ToolResultState.ERROR, + metadata={"origin": "mcp", **metadata}) + + +def is_timeout(exc): + return isinstance(exc, TimeoutError) or (isinstance(exc, MCPError) and exc.code == REQUEST_TIMEOUT) + + +def poisons_session(exc): + """A request-level failure leaves the session usable; a lost peer, a changed + schema or a cancellation does not.""" + if isinstance(exc, MCPError): + return exc.code == CONNECTION_CLOSED + return not isinstance(exc, (TimeoutError, jsonschema.ValidationError)) diff --git a/src/backend/core/llm/mcp/session.py b/src/backend/core/llm/mcp/session.py new file mode 100644 index 00000000..a2309797 --- /dev/null +++ b/src/backend/core/llm/mcp/session.py @@ -0,0 +1,147 @@ +"""One live SDK session: its owner task, discovered surface and call admission.""" +import asyncio +import contextvars +from contextlib import AsyncExitStack, asynccontextmanager +from mcp import types +from core.llm.mcp.callbacks import route +from core.llm.mcp.client import make_client + +MAX_PAGES = 100 + + +class CallGate: + """Read-only calls share the session; writes run alone.""" + + def __init__(self): + self._readers = 0 + self._writer = False + self._changed = asyncio.Condition() + + @asynccontextmanager + async def _hold(self, exclusive): + async with self._changed: + if exclusive: + await self._changed.wait_for(lambda: not self._writer and not self._readers) + self._writer = True + else: + await self._changed.wait_for(lambda: not self._writer) + self._readers += 1 + try: + yield + finally: + async with self._changed: + if exclusive: + self._writer = False + else: + self._readers -= 1 + self._changed.notify_all() + + def shared(self): + return self._hold(False) + + def exclusive(self): + return self._hold(True) + + +async def _pages(method, field): + items, cursor, seen = [], None, set() + for _ in range(MAX_PAGES): + page = await method(cursor=cursor) + items.extend(getattr(page, field)) + cursor = page.next_cursor + if cursor is None: + return items + if cursor in seen: + raise ValueError("MCP pagination cursor repeated") + seen.add(cursor) + raise ValueError(f"MCP {field} listing exceeded {MAX_PAGES} pages") + + +class Session: + """Owns one SDK client from a dedicated task, so any run's task may call through it.""" + + def __init__(self, name, cfg): + self.name, self.cfg = name, cfg + self.client = None + self.descriptors = [] + self.context = [] # (kind, descriptor) for advertised resources, templates and prompts + self.calls = CallGate() + self.in_flight = set() + self.failed = False + self._stop = asyncio.Event() + self._owner = None + + @property + def stopped(self): + return self._stop.is_set() or self.failed or (self._owner is not None and self._owner.done()) + + async def _sampling(self, context, params): + call = route(self) + if call is None: + return types.ErrorData(code=-32600, message="No active sampling scope") + return await call.dispatch("sampling", context, params) + + async def _elicitation(self, context, params): + call = route(self) + if call is None: + return types.ElicitResult(action="decline") + return await call.dispatch("elicitation", context, params) + + async def _discover(self, client): + capabilities = client.server_capabilities + if capabilities.tools is not None: + descriptors = await _pages(client.list_tools, "tools") + names = [d.name for d in descriptors] + if len(names) != len(set(names)): + raise ValueError("Duplicate MCP tool name") + self.descriptors = descriptors + client.freeze(descriptors) + listings = [] + if capabilities.resources is not None: + listings += [("resource", client.list_resources, "resources"), + ("resource_template", client.list_resource_templates, "resource_templates")] + if capabilities.prompts is not None: + listings.append(("prompt", client.list_prompts, "prompts")) + found = await asyncio.gather(*(_pages(method, field) for _, method, field in listings)) + self.context = [(kind, item) for (kind, _, _), items in zip(listings, found) for item in items] + + async def _own(self, client, ready): + try: + async with AsyncExitStack() as stack: + await stack.enter_async_context(client) + self.client = client + await self._discover(client) + ready.set_result(None) + await self._stop.wait() + except BaseException as exc: + if not ready.done(): + ready.set_exception(exc) + else: + raise + + @classmethod + async def open(cls, name, cfg): + session = cls(name, cfg) + client = make_client(name, cfg, sampling_callback=session._sampling, + elicitation_callback=session._elicitation) + ready = asyncio.get_running_loop().create_future() + # A fresh context keeps run-scoped ContextVars out of the long-lived session task. + session._owner = asyncio.create_task(session._own(client, ready), name=f"mcp-session:{name}", + context=contextvars.Context()) + try: + await asyncio.wait_for(asyncio.shield(ready), timeout=float(cfg.get("discovery_timeout") or 30)) + except BaseException: + session._stop.set() + from core.infra.task_lifecycle import cancel_and_join + try: + await cancel_and_join(session._owner) + finally: + if ready.done() and not ready.cancelled(): + ready.exception() + raise + return session + + async def close(self): + self._stop.set() + if self._owner is not None: + await asyncio.shield(self._owner) diff --git a/src/backend/core/llm/mcp/tools.py b/src/backend/core/llm/mcp/tools.py new file mode 100644 index 00000000..1f42a7e6 --- /dev/null +++ b/src/backend/core/llm/mcp/tools.py @@ -0,0 +1,113 @@ +"""AgentScope tool boundary over a live native MCP session.""" +import hashlib +import json +import math +from agentscope.tool import ToolBase +from agentscope.permission import PermissionBehavior, PermissionDecision +from core.llm.mcp.deadline import InvocationBudget +from core.llm.mcp.callbacks import CURRENT_CALL +from core.llm.mcp.results import error_result, is_timeout, poisons_session, tool_result + +MAX_QUEUED_PROGRESS = 32 + + +def _call_key(name, arguments): + body = json.dumps(arguments, sort_keys=True, ensure_ascii=False, default=str) + return hashlib.sha256(f"{name}\0{body}".encode()).hexdigest() + + +def queue_progress(queue, event): + """Keep only the newest event per call; the stream loop drains the queue between events.""" + for index, queued in enumerate(queue): + if queued["tool_id"] == event["tool_id"]: + queue[index] = event + return + if len(queue) < MAX_QUEUED_PROGRESS: + queue.append(event) + + +class SessionTool(ToolBase): + """A run's tool over a session; the run handle decides whether it may still call.""" + is_mcp = True + + def __init__(self, resource, timeout): + super().__init__() + self.resource, self.mcp_name, self.timeout = resource, resource.name, timeout + + def _require_open(self): + if self.resource.closed or self.resource.session.stopped: + raise RuntimeError("MCP run lease is closed") + + async def _invoke(self, operation, *, exclusive, write_key=None): + """Run one request; only a lost session is fatal, other failures become results.""" + self._require_open() + resource, session = self.resource, self.resource.session + if write_key is not None and write_key in resource.unknown_outcomes: + return error_result("该写操作上次以相同参数调用时结果未知，已拒绝重复执行。请先核对服务端状态。", + server=self.mcp_name, outcome_unknown=True) + async with session.calls.exclusive() if exclusive else session.calls.shared(): + self._require_open() + call = resource.interactions.open_call(session) if resource.interactions else None + token = CURRENT_CALL.set(call) + if call is not None: + session.in_flight.add(call) + try: + async with InvocationBudget(self.timeout) as budget: + if call is not None: + call.budget = budget + return await operation() + except BaseException as exc: + if poisons_session(exc): + session.failed = True + raise + if write_key is not None and is_timeout(exc): + resource.unknown_outcomes.add(write_key) + return error_result( + f"MCP 写操作未在预期内完成（{type(exc).__name__}），结果未知。请核对服务端状态，不要直接重试。", + server=self.mcp_name, outcome_unknown=True) + return error_result(f"MCP 调用失败：{type(exc).__name__}: {exc}"[:2000], server=self.mcp_name) + finally: + CURRENT_CALL.reset(token) + if call is not None: + session.in_flight.discard(call) + await call.finish() + + +class RemoteTool(SessionTool): + def __init__(self, resource, descriptor, timeout): + super().__init__(resource, timeout) + self.name = descriptor.name + self.description = descriptor.description or descriptor.name + self.input_schema = descriptor.input_schema + self.is_read_only = bool(descriptor.annotations and descriptor.annotations.read_only_hint) + # Read-only calls have no effects to order, so the agent may run them together. + self.is_concurrency_safe = self.is_read_only + self.descriptor = descriptor + + async def check_permissions(self, *_args, **_kwargs): + return PermissionDecision(behavior=PermissionBehavior.ALLOW if self.is_read_only else PermissionBehavior.ASK, + message="MCP operation is subject to the run execution policy") + + async def call(self, **kwargs): + from core.harness.tool_call_context import CURRENT_TOOL_CALL_ID + tool_call_id = CURRENT_TOOL_CALL_ID.get() + interactions = self.resource.interactions + + async def progress(value, total, message): + if interactions is None or interactions.agent is None: + return + queue_progress(interactions.agent.state.pending_mcp_progress, { + "tool_id": tool_call_id, "name": self.name, "server_id": self.mcp_name, + "progress": value, "total": total, "message": str(message or "")[:500]}) + + async def operation(): + result = await self.resource.session.client.call_tool( + self.descriptor.name, kwargs, + # The outer pausable budget owns execution time. A second SDK + # wall clock would expire during legacy server-to-client elicitation. + read_timeout_seconds=math.inf if interactions is not None else self.timeout, + progress_callback=progress) + return tool_result(result, self.descriptor) + + return await self._invoke(operation, exclusive=not self.is_read_only, + write_key=None if self.is_read_only else _call_key(self.name, kwargs)) diff --git a/src/backend/core/llm/mcp_manager.py b/src/backend/core/llm/mcp_manager.py deleted file mode 100644 index 879f0a35..00000000 --- a/src/backend/core/llm/mcp_manager.py +++ /dev/null @@ -1,336 +0,0 @@ -"""MCP client pool manager for AgentScope 2.0. - -Manages MCPClient instances with TTL caching. - -Migration notes (1.x → 2.0) ---------------------------- -- ``StdIOStatefulClient(name, command, args, env)`` → - ``MCPClient(name=, is_stateful=True, mcp_config=StdioMCPConfig(command, args, env))``. -- In 2.0 ``Toolkit`` is constructed **once** (``Toolkit(tools=, mcps=, ...)``); there are - no incremental ``register_mcp_client`` / ``register_tool_function`` methods and no - ``namesake_strategy``. Stateful clients must ``connect()`` **before** being passed - into ``Toolkit``. -- Therefore ``connect_mcp_clients`` only connects and returns the client list; the - Toolkit is built once in agent_factory via - ``Toolkit(tools=[...FunctionTool...], mcps=clients)``. -""" - -from __future__ import annotations - -import asyncio -import logging -import time -from contextvars import ContextVar -from functools import wraps -from typing import Any, Dict, List - -import httpx -import mcp.types -from mcp.client.sse import sse_client -from mcp.client.streamable_http import streamable_http_client -from pydantic import ConfigDict, Field - -from agentscope.mcp import MCPClient, StdioMCPConfig -from agentscope.permission import PermissionBehavior, PermissionDecision -from agentscope.tool import MCPTool, ToolBase, ToolChunk - -logger = logging.getLogger(__name__) - - -def has_usable_schema(tool: Any) -> bool: - """Whether a manifest tool entry carries the server's real ``inputSchema``. - - A tool the platform only knows by name and description (a plugin manifest - lists its tools for display, without schemas) must not be handed to a model: - the parameter object would be empty, so the model has nowhere to put its - arguments and every call arrives blank. - - The marker is whether a schema was captured at all, not whether it declares - parameters: a server is free to describe an argument-less tool as - ``{"type": "object"}``, while an entry that never reached the server carries - no ``inputSchema`` key, which normalizes to an empty dict. - """ - if not isinstance(tool, dict): - return False - schema = tool.get("inputSchema") or tool.get("input_schema") - return isinstance(schema, dict) and bool(schema) - - -def _cyfunc_probe() -> None: # after Cython compilation its type is cython_function_or_method - pass - - -# Cython compiles methods into cython_function_or_method; pydantic v2 does not recognize -# it as a method and treats it as an "un-annotated field", raising PydanticUserError, -# which makes the whole module fail to import after hardened compilation and fall back to -# plaintext. Registering that type in ignored_types makes pydantic ignore compiled -# methods; under pure Python it is just FunctionType, so there is no side effect. -_CYFUNCTION_TYPE = type(_cyfunc_probe) - -# TTL for the instance-level memoization of ``list_tools``. Within a single turn, -# agent_factory's HTTP MCP liveness probe (``_connect_http``, parallel list_tools) and -# the subsequent ``Toolkit.get_tool_schemas()`` (AgentScope ``_get_available_tools`` -# does a **serial** ``await list_tools()`` per client) each enumerate tools once per -# server — stateless HTTP creates a new connection every time, and the serial -# re-enumeration is the main cause of the ~2s agent build time (11 servers, 5 slow -# search MCPs at ~450ms each, chained serially). Memoization cuts the second -# enumeration to ~0ms (the probe pass stays parallel at ~465ms). Per-request HTTP -# clients get a fresh instance every turn → cache lifetime = one turn, no header -# leakage across turns/users; pooled stdio clients survive across turns, so the TTL -# is the safety net (tool definitions are static anyway; config changes rebuild the -# pool → new instances). -_LIST_TOOLS_TTL_S = 300.0 - - -class BareNameMCPClient(MCPClient): - """Restore the server-side bare name of MCP tools (``internet_search`` rather than ``mcp__internet_search__internet_search``). - - AgentScope 2.0's ``MCPTool`` adapter rewrites the outward-facing name to - ``mcp____``, but this project's display-name mapping - (core/config/display_names), citation extraction (orchestration/citation_anchor - dispatches on bare names like ``internet_search``), catalog gating, tool - references in system prompts and SKILL.md, and frontend icons/panels/renderers - are all built on the 1.x bare names. ``MCPTool.__call__`` actually calls the - server via ``self._tool.name`` (the bare name), so rewriting the adapter's - ``.name`` only affects the LLM-visible name and the SSE event stream; the call - path is unaffected. - """ - - model_config = ConfigDict(ignored_types=(_CYFUNCTION_TYPE,), arbitrary_types_allowed=True) - oauth_provider: Any = Field(default=None, exclude=True) - - def _create_http_client(self): - """Attach the SDK OAuth provider without forking AgentScope's client.""" - from core.llm.mcp_invocation import HEADER, PLUGIN_HEADER, for_url - config = self.mcp_config - signed = any(key.lower() == HEADER.lower() for key in (config.headers or {})) - if self.oauth_provider is None and not signed: - return super()._create_http_client() - oauth_hook = getattr(self.oauth_provider, "mcp_request_hook", None) - async def request_hook(request): - if oauth_hook: - await oauth_hook(request) - if signed: - request.headers.update(for_url(config.url, request.headers.get("x-current-user-id"), request.headers.get("x-chat-id"), request.headers.get(PLUGIN_HEADER, ""))) - event_hooks = {"request": [request_hook]} - if config.url.endswith("/sse") or config.url.endswith("/messages/"): - def _http_client_factory(headers=None, timeout=None, auth=None): - return httpx.AsyncClient( - headers=headers, - timeout=timeout, - auth=auth, - event_hooks=event_hooks, - ) - - return sse_client( - url=config.url, - headers=config.headers, - timeout=config.timeout, - auth=self.oauth_provider, - httpx_client_factory=_http_client_factory, - ) - http_client = httpx.AsyncClient( - headers=config.headers, - timeout=config.timeout, - auth=self.oauth_provider, - event_hooks=event_hooks, - ) - return streamable_http_client(url=config.url, http_client=http_client) - - async def get_tool(self, name: str) -> MCPTool: - tool = await super().get_tool(name) - tool.name = tool._tool.name - # Use AgentScope's concurrent batch executor for every MCP call. - # Permission checks remain independent of the execution policy. - tool.is_concurrency_safe = True - return tool - - async def list_tools(self) -> List[MCPTool]: - """Instance-level TTL memoization to eliminate duplicate list_tools for the same server within a turn. - - See the ``_LIST_TOOLS_TTL_S`` comment at the top of the module: the liveness - probe and ``get_tool_schemas`` each call once, and the second serial - re-enumeration is the main cause of the ~2s agent build time. The cache hangs - off the instance (``__slots__`` includes ``__dict__``, so arbitrary attributes - can be set); a new client per turn → naturally scoped to a single turn. - """ - cached = getattr(self, "_lt_cache", None) - if cached is not None: - expires_at, tools = cached - if time.monotonic() < expires_at: - return tools - tools = await super().list_tools() - self._lt_cache = (time.monotonic() + _LIST_TOOLS_TTL_S, tools) - return tools - - -from core.llm.gateway_mcp_tool import GatewayMCPTool - - -class ManifestMCPClient(BareNameMCPClient): - """Network-free MCP discovery backed only by the current cloud manifest.""" - - manifest_tools: List[Dict[str, Any]] = Field(default_factory=list, exclude=True) - gateway_invoke_url: str = Field(exclude=True) - schema_hash: str = Field(exclude=True) - gateway_transport: Any = Field(default=None, exclude=True) - gateway_plugin: str = Field(default="", exclude=True) - manager_cloud_available: bool = Field(default=False, exclude=True) - read_only: bool = Field(default=False, exclude=True) - - def _raw_manifest_tools(self) -> List[mcp.types.Tool]: - tools: List[mcp.types.Tool] = [] - from core.capabilities.paths import capabilities_enabled - from core.services.management_contract import localize, declared, tools as manager_tools - items = list(self.manifest_tools) - from extensions.mcp_servers.registry import gateway_server - if self.read_only and gateway_server(self.name) is not None: - items = [item for item in items if item.get("annotations", {}).get("readOnlyHint") is True] - if capabilities_enabled() and self.gateway_plugin == "skill-manager": - if any(t.get("name") == "install_skill" and declared(t, self.gateway_plugin) for t in items): - items += [t for t in manager_tools("skill-manager", "local") if t["name"] == "upload_skill_to_cloud" and not any(x.get("name") == t["name"] for x in items)] - for item in items: - if capabilities_enabled(): - item = localize(item, self.gateway_plugin, cloud_available=self.manager_cloud_available or bool(self.gateway_invoke_url)) - if item is None: - continue - try: - tool = mcp.types.Tool.model_validate(item) - except Exception as exc: # noqa: BLE001 - isolate one malformed tool - logger.warning("Invalid manifest tool in MCP '%s': %s", self.name, exc) - continue - if self.enable_tools is not None and tool.name not in self.enable_tools: - continue - if self.disable_tools is not None and tool.name in self.disable_tools: - continue - if not has_usable_schema(item): - # A manifest entry that never captured the server's inputSchema - # would become a zero-parameter tool: the model then has nowhere - # to put its arguments and retries the same empty call forever. - # Withhold it instead of offering a tool that can never succeed. - logger.warning( - "Manifest tool '%s' in MCP '%s' has no captured inputSchema; withheld", - tool.name, - self.name, - ) - continue - tools.append(tool) - return tools - - async def list_raw_tools(self) -> List[mcp.types.Tool]: - return self._raw_manifest_tools() - - async def get_tool(self, name: str) -> GatewayMCPTool: - for tool in self._raw_manifest_tools(): - if tool.name == name: - from core.capabilities.paths import capabilities_enabled - from core.services.management_contract import declared - if capabilities_enabled() and declared(tool.model_dump(by_alias=True), self.gateway_plugin): - from core.llm.management_tool import ManagementTool - return ManagementTool(self.name, tool, self.gateway_plugin, self.mcp_config.headers or {}, cloud_backed=bool(self.gateway_invoke_url)) - return GatewayMCPTool( - mcp_name=self.name, - tool=tool, - invoke_url=self.gateway_invoke_url, - schema_hash=self.schema_hash, - headers=dict(self.mcp_config.headers or {}), - timeout=float(self.execution_timeout or 120.0), - transport=self.gateway_transport, - source_plugin=self.gateway_plugin, - ) - raise ValueError(f"Tool '{name}' not found in cloud manifest MCP '{self.name}'") - - async def list_tools(self) -> List[GatewayMCPTool]: - return [await self.get_tool(tool.name) for tool in self._raw_manifest_tools()] - - -def make_stdio_client(server_name: str, server_cfg: dict) -> MCPClient: - """Build a stdio MCPClient (not yet connected) from a server config in configs/mcp_config.py.""" - return BareNameMCPClient( - name=server_name, - is_stateful=True, - mcp_config=StdioMCPConfig( - command=server_cfg.get("command", "python"), - args=server_cfg.get("args", []), - env=server_cfg.get("env") or None, - ), - ) - - -# The factory owns transient clients until it returns them to its caller. -# ContextVar values propagate into parallel HTTP probes, so partial construction -# has one cleanup owner even when cancellation interrupts asyncio.gather. -_factory_clients: ContextVar[list | None] = ContextVar("factory_mcp_clients", default=None) - - -def own_factory_client(client): - clients = _factory_clients.get() - if clients is not None: - clients.append(client) - return client - - -def close_factory_clients_on_error(function): - @wraps(function) - async def construct(*args, **kwargs): - clients = [] - token = _factory_clients.set(clients) - try: - return await function(*args, **kwargs) - except BaseException: - await close_clients(list(reversed(clients))) - raise - finally: - _factory_clients.reset(token) - - return construct - - -async def connect_mcp_clients( - mcp_servers: Dict[str, dict], -) -> List[MCPClient]: - """Connect to MCP servers and return the connected MCPClient list.""" - clients: List[MCPClient] = [] - for server_name, server_cfg in mcp_servers.items(): - client = make_stdio_client(server_name, server_cfg) - try: - await client.connect() - clients.append(own_factory_client(client)) - logger.debug("MCP client '%s' connected", server_name) - except asyncio.CancelledError: - await client.close() - raise - except Exception as exc: - logger.warning("Failed to connect MCP server '%s': %s", server_name, exc) - try: - await client.close() - except Exception: - pass - return clients - - -async def close_clients(clients: List[MCPClient]) -> None: - """Close every client in its owning task, deferring repeated cancellation.""" - cancelled = None - task = asyncio.current_task() - for client in clients: - while True: - before = task.cancelling() if task else 0 - try: - await client.close() - break - except asyncio.CancelledError as exc: - cancelled = exc - if task is not None and task.cancelling() > before: - # An external repeat-cancel interrupted this close. Finish - # this same client's cleanup on the same task, then the rest. - continue - # A client scope itself signalled cancellation. It must not - # prevent independent clients from releasing their resources. - logger.debug("MCP client close signalled cancellation") - break - except Exception as exc: - logger.debug("Error closing MCP client: %s", exc) - break - if cancelled is not None: - raise cancelled diff --git a/src/backend/core/llm/mcp_pool.py b/src/backend/core/llm/mcp_pool.py deleted file mode 100644 index dc98f03e..00000000 --- a/src/backend/core/llm/mcp_pool.py +++ /dev/null @@ -1,387 +0,0 @@ -"""Global MCP connection pool (AgentScope 2.0). - -Keeps stable MCP servers connected across requests to eliminate the 1-7s -subprocess spawn overhead. Per-request servers (e.g. retrieve_dataset_content -with runtime KB env vars) are spawned fresh each time. - -Migration notes (1.x → 2.0) ---------------------------- -- ``StdIOStatefulClient`` / ``HttpStatefulClient`` → ``MCPClient(is_stateful=, - mcp_config=StdioMCPConfig|HttpMCPConfig)``. -- The 2.0 ``Toolkit`` is constructed in one shot (``Toolkit(tools=, mcps=, ...)``); - there is **no** ``register_mcp_client`` / ``register_tool_function`` and **no** - ``MCPClient.get_callable_function`` — so the 1.x "cache MCPToolFunction, - register on demand" optimization cannot be ported directly. This pool now only - handles **connection reuse** (stable clients keep their connection across - requests, saving the subprocess spawn), returning a list of connected - ``MCPClient`` per request, and ``agent_factory`` builds - ``Toolkit(tools=[...FunctionTool...], mcps=clients)`` in one go. -- HTTP per-request (KB switches headers per user): use an MCPClient with - ``is_stateful=False``; each call opens a fresh underlying connection, no - pooling needed (see agent_factory). - -Usage: - pool = MCPConnectionPool.get_instance() - await pool.initialize(server_configs) # once at startup - - # Per request (stable reuses connections, per-request spawns fresh): - clients, transient = await pool.get_request_clients(enabled_keys, per_request_cfg) - toolkit = Toolkit(tools=fn_tools, mcps=clients) # in agent_factory - ... - await pool.close_transient(transient) - - await pool.shutdown() # on shutdown -""" - -from __future__ import annotations - -import asyncio -import logging -import os -from threading import Lock -from typing import Any, Dict, List, Optional, Set, Tuple - -from agentscope.mcp import HttpMCPConfig, MCPClient, StdioMCPConfig -from core.llm.mcp_manager import BareNameMCPClient, ManifestMCPClient, has_usable_schema - -logger = logging.getLogger(__name__) - -HTTP_TRANSPORTS = frozenset({"streamable_http", "sse"}) -_KB_SERVER_ID = "retrieve_dataset_content" - - -def _positive_timeout(value: Any, default: float) -> float: - try: - parsed = float(value) - return parsed if parsed > 0 else default - except (TypeError, ValueError): - return default - - -def _execution_timeout(name: str, cfg: dict) -> float: - """Resolve a hard tool-call deadline, with a tighter KB-specific default.""" - if cfg.get("execution_timeout") is not None: - return _positive_timeout(cfg["execution_timeout"], 120.0) - if name == _KB_SERVER_ID: - return _positive_timeout(os.getenv("KB_MCP_EXECUTION_TIMEOUT_SECONDS"), 75.0) - return _positive_timeout(os.getenv("MCP_TOOL_EXECUTION_TIMEOUT_SECONDS"), 120.0) - - -def _http_transport_timeout(name: str, cfg: dict, execution_timeout: float) -> float: - if cfg.get("transport_timeout") is not None: - return _positive_timeout(cfg["transport_timeout"], 30.0) - if name == _KB_SERVER_ID: - # Leave a small margin beyond the MCP server's own structured timeout - # so callers receive its tool_result instead of a transport exception. - return execution_timeout + 5.0 - if name == "internet_search": - # Search returns per-query timeouts after its 45-second batch deadline. - return 50.0 - return 30.0 - - -def is_http_cfg(cfg: dict) -> bool: - return cfg.get("transport") in HTTP_TRANSPORTS - - -def uses_manifest_schema(cfg: dict) -> bool: - """Return whether a config contains a usable dynamic cloud manifest. - - At least one tool must carry the server's real ``inputSchema``; entries known - only by name and description are withheld when the client enumerates tools - (see ``has_usable_schema``), so a manifest holding nothing else describes no - callable tool at all. - """ - tools = cfg.get("manifest_tools") - if cfg.get("schema_source") == "plugin_manifest": - from core.services.management_contract import declared - return bool(tools and cfg.get("origin") == "local_plugin" and all( - declared(tool, cfg.get("gateway_plugin")) for tool in tools)) - return bool( - cfg.get("schema_source") == "cloud_manifest" - and str(cfg.get("gateway_invoke_url") or "").strip() - and str(cfg.get("schema_hash") or "").strip() - and isinstance(tools, list) - and any( - isinstance(tool, dict) - and bool(str(tool.get("name") or "").strip()) - and has_usable_schema(tool) - for tool in tools - ) - ) - - -def make_client( - name: str, - cfg: dict, - *, - is_stateful: bool = True, - client_cls: type[BareNameMCPClient] = BareNameMCPClient, -) -> MCPClient: - """Build an MCPClient (HTTP or stdio) from a config dict. - - HTTP servers may set ``is_stateful=False`` for a fresh per-request connection - (multi-user KB). stdio must be stateful (enforced by 2.0). ``client_cls`` lets - callers substitute a ``BareNameMCPClient`` subclass (e.g. the confirmation-gated - client) while reusing the same URL/config construction. - """ - execution_timeout = _execution_timeout(name, cfg) - if is_http_cfg(cfg): - # ⚠️ The 2.0 streamable_http MCP client does not follow 307 redirects. Many MCP - # servers 307-redirect ``/mcp/`` (trailing slash) to ``/mcp`` → 2.0 throws - # HTTPStatusError outright and triggers an anyio cancel-scope crash. Strip the - # trailing slash to match the server's canonical form and avoid the redirect. - _url = (cfg.get("url") or "").rstrip("/") - client_kwargs = { - "name": name, - "is_stateful": is_stateful, - "oauth_provider": cfg.get("oauth_provider"), - "mcp_config": HttpMCPConfig( - url=_url, - headers=cfg.get("headers") or None, - timeout=_http_transport_timeout(name, cfg, execution_timeout), - ), - "execution_timeout": execution_timeout, - "enable_tools": cfg.get("enable_tools"), - } - if cfg.get("schema_source") in ("cloud_manifest", "plugin_manifest"): - if not uses_manifest_schema(cfg): - raise ValueError(f"MCP '{name}' has an invalid cloud capability manifest") - return ManifestMCPClient( - **client_kwargs, - manifest_tools=list(cfg["manifest_tools"]), - gateway_invoke_url=str(cfg["gateway_invoke_url"]), - schema_hash=str(cfg["schema_hash"]), - gateway_transport=cfg.get("gateway_transport"), - gateway_plugin=str(cfg.get("gateway_plugin") or ""), - manager_cloud_available=bool(cfg.get("manager_cloud_available")), - read_only=bool(cfg.get("read_only")), - ) - return client_cls( - **client_kwargs, - ) - return client_cls( - name=name, - is_stateful=True, - mcp_config=StdioMCPConfig( - command=cfg.get("command", "python"), - args=cfg.get("args", []), - env=cfg.get("env") or None, - ), - execution_timeout=execution_timeout, - ) - - -class MCPConnectionPool: - """Singleton MCP connection pool (connection reuse only, 2.0).""" - - _instance: Optional[MCPConnectionPool] = None - _instance_lock = Lock() - - def __init__(self) -> None: - self._stable_clients: Dict[str, MCPClient] = {} - self._stable_configs: Dict[str, dict] = {} - self._stable_server_ids: Set[str] = set() - self._initialized = False - self._config_version: int = 0 - self._lock = asyncio.Lock() - - @classmethod - def get_instance(cls) -> MCPConnectionPool: - if cls._instance is None: - with cls._instance_lock: - if cls._instance is None: - cls._instance = cls() - return cls._instance - - async def initialize(self, server_configs: Optional[Dict[str, dict]] = None) -> None: - """Connect to all stable MCP servers.""" - async with self._lock: - if server_configs is None: - from core.services.mcp_service import McpServerConfigService - - server_configs = McpServerConfigService.get_instance().get_all_servers() - - # ⚠️ 2.0: only stdio stable servers are pooled. HTTP MCP is **not pooled** even - # when marked is_stable — the 2.0 stateful HTTP client is task-bound, and reusing - # it across request tasks crashes the cancel scope; HTTP always goes per-request - # with is_stateful=False (see agent_factory._connect_http). - self._stable_server_ids = { - name - for name, cfg in server_configs.items() - if cfg.get("is_stable", False) and not is_http_cfg(cfg) - } - await self._close_all_stable() - - stable_items = [ - (name, cfg) - for name, cfg in server_configs.items() - if name in self._stable_server_ids - ] - - async def _connect_one(name: str, cfg: dict) -> bool: - try: - client = make_client(name, cfg) - await client.connect() - self._stable_clients[name] = client - self._stable_configs[name] = cfg - logger.info( - "[mcp_pool] Connected stable server: %s (%s)", - name, - cfg.get("transport", "stdio"), - ) - return True - except Exception as exc: - logger.warning("[mcp_pool] Failed to connect stable server '%s': %s", name, exc) - return False - - results = await asyncio.gather(*(_connect_one(n, c) for n, c in stable_items)) - connected = sum(1 for ok in results if ok) - self._initialized = True - self._config_version += 1 - logger.info( - "[mcp_pool] Initialized with %d/%d stable servers", - connected, - len(self._stable_server_ids), - ) - - async def reinitialize_if_config_changed(self, new_server_configs: Dict[str, dict]) -> None: - new_stable_ids = { - name - for name, cfg in new_server_configs.items() - if cfg.get("is_stable", False) and not is_http_cfg(cfg) - } - changed = new_stable_ids != self._stable_server_ids - if not changed: - for name in new_stable_ids: - if self._stable_configs.get(name) != new_server_configs.get(name): - changed = True - break - if changed: - logger.info("[mcp_pool] Config change detected, reinitializing stable connections") - await self.initialize(new_server_configs) - - @property - def has_cached_tools(self) -> bool: - # No tool-func cache in 2.0; property kept for caller compatibility (means stable servers are connected). - return bool(self._stable_clients) - - async def get_request_clients( - self, - enabled_keys: List[str], - per_request_servers_cfg: Optional[Dict[str, dict]] = None, - ) -> Tuple[List[MCPClient], List[MCPClient]]: - """Return the connected MCPClient list needed for this request + the transient list (for closing). - - - stable servers: reuse the already-connected client (no re-spawn). - - per-request stdio servers: spawn fresh (concurrently). - The caller (agent_factory) uses the returned clients in a single - ``Toolkit(tools=, mcps=clients)``. - """ - request_clients: List[MCPClient] = [] - transient_clients: List[MCPClient] = [] - - # Phase 1: spawn per-request servers concurrently - spawn_tasks: Dict[str, asyncio.Task] = {} - for key in enabled_keys: - if key not in self._stable_server_ids: - cfg = (per_request_servers_cfg or {}).get(key) - if cfg is not None: - spawn_tasks[key] = asyncio.create_task(self._spawn_transient(key, cfg)) - - # Phase 2: collect stable clients (already connected, reconnect if needed) - for key in enabled_keys: - if key not in self._stable_server_ids: - continue - client = self._stable_clients.get(key) - if client is None: - client = await self._reconnect_stable(key) - if client is not None: - request_clients.append(client) - - # Phase 3: await per-request spawns - for key, task in spawn_tasks.items(): - try: - client = await task - if client is not None: - request_clients.append(client) - transient_clients.append(client) - except Exception as exc: - logger.warning("[mcp_pool] Failed per-request server '%s': %s", key, exc) - - return request_clients, transient_clients - - # Backward-compat alias: old callers took (toolkit, transient); now returns (clients, transient). - async def build_toolkit_from_cache( - self, - enabled_keys: List[str], - per_request_servers_cfg: Optional[Dict[str, dict]] = None, - ) -> Tuple[List[MCPClient], List[MCPClient]]: - return await self.get_request_clients(enabled_keys, per_request_servers_cfg) - - async def close_transient(self, transient_clients: List[MCPClient]) -> None: - for client in transient_clients: - try: - await client.close() - except Exception as exc: - logger.debug("[mcp_pool] close transient failed: %s", exc) - - async def _spawn_transient(self, key: str, cfg: dict) -> Optional[MCPClient]: - client = make_client(key, cfg) - try: - await client.connect() - return client - except Exception as exc: - logger.warning("[mcp_pool] spawn_transient '%s' connect failed: %s", key, exc) - try: - await client.close() - except Exception: - pass - raise - - async def _reconnect_stable(self, name: str) -> Optional[MCPClient]: - cfg = self._stable_configs.get(name) - if cfg is None: - return None - old = self._stable_clients.pop(name, None) - if old is not None: - try: - await old.close() - except Exception: - pass - try: - client = make_client(name, cfg) - await client.connect() - self._stable_clients[name] = client - logger.info("[mcp_pool] Reconnected stable server: %s", name) - return client - except Exception as exc: - logger.warning("[mcp_pool] Reconnection failed for '%s': %s", name, exc) - return None - - async def _close_all_stable(self) -> None: - for name, client in list(self._stable_clients.items()): - try: - await client.close() - except Exception as exc: - logger.warning("[mcp_pool] close failed for '%s': %s", name, exc) - self._stable_clients.clear() - self._stable_configs.clear() - - async def shutdown(self) -> None: - async with self._lock: - await self._close_all_stable() - self._initialized = False - logger.info("[mcp_pool] Shut down") - - @property - def is_initialized(self) -> bool: - return self._initialized - - @property - def stable_client_count(self) -> int: - return len(self._stable_clients) - - @property - def stable_server_ids(self) -> frozenset[str]: - return frozenset(self._stable_server_ids) diff --git a/src/backend/core/llm/middleware/__init__.py b/src/backend/core/llm/middleware/__init__.py new file mode 100644 index 00000000..a22094fc --- /dev/null +++ b/src/backend/core/llm/middleware/__init__.py @@ -0,0 +1 @@ +"""Record of the AgentScope middleware chain replaced by HookBus hooks.""" diff --git a/src/backend/core/llm/middleware_migration.py b/src/backend/core/llm/middleware/migration_map.py similarity index 51% rename from src/backend/core/llm/middleware_migration.py rename to src/backend/core/llm/middleware/migration_map.py index 2c1c228d..929b5011 100644 --- a/src/backend/core/llm/middleware_migration.py +++ b/src/backend/core/llm/middleware/migration_map.py @@ -18,82 +18,91 @@ class MiddlewareMigration: MIDDLEWARE_MIGRATION: tuple[MiddlewareMigration, ...] = ( MiddlewareMigration( "DynamicModelMiddleware", - (HookStage.BEFORE_RUN, HookStage.BEFORE_MODEL), - "adapter-covered", - "neutral model-selection policy reaches parity", + (HookStage.BEFORE_RUN,), + "neutral-hook", + "removed; model selection runs through HookSpec", ), MiddlewareMigration( "FileContextMiddleware", (HookStage.TRANSFORM_CONTEXT,), - "adapter-covered", - "neutral attachment transformer reaches parity", + "neutral-hook", + "removed; attachments emit neutral context items", ), MiddlewareMigration( "SteerMiddleware", - (HookStage.BEFORE_MODEL, HookStage.BEFORE_TOOL), - "adapter-covered", - "durable steer policy consumes Invocation only", + (HookStage.TRANSFORM_CONTEXT, HookStage.AFTER_CONTEXT, HookStage.BEFORE_TOOL), + "neutral-hook", + "removed; acknowledge only after context application", ), MiddlewareMigration( "WorkspacePinHintMiddleware", - (HookStage.AFTER_TOOL, HookStage.BEFORE_MODEL), - "adapter-covered", - "pin reminder becomes a neutral context patch", + (HookStage.BEFORE_RUN, HookStage.TRANSFORM_CONTEXT), + "neutral-hook", + "removed; pin reminder emits neutral context items", ), MiddlewareMigration( "IterBudgetReminderMiddleware", - (HookStage.BEFORE_MODEL,), - "adapter-covered", - "turn-budget reminder becomes a neutral context patch", + (HookStage.TRANSFORM_CONTEXT,), + "neutral-hook", + "removed; budget context and tool choice are patched before binding", ), MiddlewareMigration( "StallInterventionMiddleware", - (HookStage.BEFORE_MODEL,), - "adapter-covered", - "stall policy consumes neutral attempt history", + (HookStage.BEFORE_TOOL, HookStage.AFTER_TOOL), + "neutral-hook", + "removed; evolution policy consumes neutral attempt history", ), MiddlewareMigration( "OntologyGateMiddleware", (HookStage.BEFORE_TOOL, HookStage.AFTER_TOOL), - "adapter-covered", - "ontology decision returns reject/modify Decision", + "neutral-hook", + "removed; ontology admission returns reject/modify Decision", ), MiddlewareMigration( "ToolPermissionMiddleware", - (HookStage.BEFORE_TOOL, HookStage.AFTER_TOOL), - "adapter-covered", - "neutral tool runner owns permission tickets and approval routing", + (HookStage.BEFORE_TOOL,), + "neutral-hook", + "removed; neutral admission policy owns tickets and execution scope", ), MiddlewareMigration( "CitationAnchorMiddleware", (HookStage.AFTER_TOOL,), - "adapter-covered", + "neutral-hook", "citation annotation returns tool_result patch", ), MiddlewareMigration( "ActingToolCallIdMiddleware", (HookStage.BEFORE_TOOL,), - "adapter-covered", + "neutral-hook", "tool Invocation carries call id", ), MiddlewareMigration( "ToolEffectMiddleware", (HookStage.BEFORE_TOOL, HookStage.AFTER_TOOL), - "adapter-covered", + "neutral-hook", "effect gateway is invoked by neutral tool runner", ), MiddlewareMigration( "JobLedgerReminderMiddleware", - (HookStage.BEFORE_MODEL,), - "adapter-covered", - "job projection becomes a neutral context patch", + (HookStage.TRANSFORM_CONTEXT,), + "neutral-hook", + "removed; execution host contributes the ledger hook", ), MiddlewareMigration( "FinishPinGuardMiddleware", (HookStage.BEFORE_FINISH,), - "adapter-covered", + "neutral-hook", "finish guard returns an explicit Decision", ), + MiddlewareMigration("CommandCompletionMiddleware", + (HookStage.TRANSFORM_CONTEXT, HookStage.BEFORE_FINISH), + "neutral-hook", "removed; completion decisions continue the active run"), + MiddlewareMigration("ProjectEnvironmentMiddleware", (HookStage.BEFORE_TOOL,), + "neutral-hook", "removed; admission returns an explicit denial"), + MiddlewareMigration("ScreenshotPruningMiddleware", (HookStage.TRANSFORM_CONTEXT,), + "neutral-hook", "removed; profile contributes a plain context transform"), + MiddlewareMigration("PlanStaleReminderMiddleware", (HookStage.TRANSFORM_CONTEXT,), + "neutral-hook", "removed; plan reminder consumes a context snapshot"), ) diff --git a/src/backend/core/llm/middlewares.py b/src/backend/core/llm/middlewares.py deleted file mode 100644 index fbdd63c5..00000000 --- a/src/backend/core/llm/middlewares.py +++ /dev/null @@ -1,1450 +0,0 @@ -"""AgentScope 2.0 middlewares — replace the 1.x hooks (pre_reply / post_acting / post_reasoning). - -1.x → 2.0 migration mapping: - - dynamic_model (pre_reply) → DynamicModelMiddleware.on_reply - - file_context (pre_reply) → FileContextMiddleware.on_reply - - workspace_pin_hint (post_acting) → WorkspacePinHintMiddleware.on_reasoning - (⚠️ on_acting cannot write to context; must use on_reasoning to inject the reminder before the next reasoning round) - - finish_pin_guard (post_reasoning)→ FinishPinGuardMiddleware.on_reasoning - - iter_budget (new in 2.0, no 1.x predecessor) → IterBudgetReminderMiddleware.on_reasoning - -The runtime context moved from ``agent._jx_context`` to ``agent.state`` (fields on an AgentRuntimeState subclass). -Pure-logic helpers still reuse hooks.py / finish_guard.py (they are agentscope-version independent). -""" - -from __future__ import annotations - -import asyncio -import base64 -import json -import logging -import time -from contextlib import suppress -from contextvars import ContextVar -from pathlib import Path -from typing import Any, List, Optional - -from agentscope.agent import Agent -from agentscope.event import ToolCallEndEvent -from agentscope.message import ToolResultState -from agentscope.middleware import MiddlewareBase -from agentscope.state import AgentState -from agentscope.tool._response import ToolChunk, ToolResponse -from core.llm.context_adapter import ( - append_context_text, - append_state_context_item, - next_context_sequence, - render_text_block, -) -from core.llm.context_ir import ( - KIND_ATTACHMENT, - KIND_PROJECT, - KIND_REMINDER, - POLICY_HEAD_TAIL, - POLICY_NEVER, - ContextItem, -) -from core.llm.hooks import ( - _FILE_ID_RE, - _PIN_HINT_SKIP_TOOLS, - _build_file_context, - _build_historical_files_context, - _fetch_image_base64, - _get_main_model, - _get_pin_hint_state, - _get_provider_model, - _is_image, - _resolve_chat_mode, - reset_artifact_read_state, - reset_pin_hint_state, -) -from core.llm.plan_update_tool import parse_plan_update_args -from pydantic import ConfigDict, Field - -logger = logging.getLogger(__name__) - - -def _append_reminder(agent: Agent, text: str, *, origin: str) -> None: - append_context_text( - agent, - f"\n{text}\n", - kind=KIND_REMINDER, - origin=origin, - trust="system", - priority=850, - token_budget=4_000, - truncation_policy=POLICY_HEAD_TAIL, - ) - - -def _append_state_payload( - state: AgentState, - content: Any, - *, - kind: str, - origin: str, - trust: str, - priority: int = 650, - token_budget: int = 8_000, -) -> None: - created_seq = next_context_sequence(state.context) - append_state_context_item( - state, - ContextItem.create( - item_id=f"{origin}:{created_seq}", - kind=kind, - origin=origin, - trust=trust, - visibility="model", - priority=priority, - token_budget=token_budget, - truncation_policy=POLICY_HEAD_TAIL, - content=content, - cache_class="dynamic", - created_seq=created_seq, - render_role="user", - message_group=f"{origin}:{created_seq}", - ), - ) - - -def _cyfunc_probe() -> None: # once compiled by Cython, its type is cython_function_or_method - pass - - -# Cython compiles methods into cython_function_or_method; pydantic v2 doesn't recognize it as a -# method and treats it as an "unannotated field", raising PydanticUserError — so after hardened -# compilation the whole module fails to import and falls back to plaintext. Registering that type -# in ignored_types makes pydantic ignore the compiled methods. Under pure Python it is just a -# regular FunctionType (pydantic already ignores methods), so there is no side effect. -_CYFUNCTION_TYPE = type(_cyfunc_probe) - - -# ── Current tool-call id seam ─────────────────────────────────────────────── -# When a tool function needs to know its own tool_call_id (e.g. call_subagent attaches the -# subagent's streaming events to its own card via parent_tool_id), AgentScope does not pass it -# down (toolkit.call_tool does not inject the id into tool kwargs). Use an on_acting middleware -# to write the current tool_call.id into a ContextVar before each tool executes — the tool runs -# in the **same task call chain**, so it can read it. -# Concurrent tool calls each run in separate tasks spawned by asyncio.gather (each with its own -# copy of the context), so ContextVars don't cross-contaminate — naturally aligned with parallel subagents. -CURRENT_TOOL_CALL_ID: ContextVar[str] = ContextVar("jx_current_tool_call_id", default="") -CURRENT_RUN_BINDING: ContextVar[tuple[str, str] | None] = ContextVar( - "jx_current_run_binding", default=None -) - - -class ActingToolCallIdMiddleware(MiddlewareBase): - """on_acting: expose the current tool_call.id to tool functions (see above). Pure pass-through, does not change tool behavior.""" - - async def on_acting(self, agent: Agent, input_kwargs: dict, next_handler): # noqa: ANN001 - prepared = getattr(agent, "_jx_prepared_capabilities", None) - if prepared is not None: - from core.capabilities.runtime import validate - - await asyncio.to_thread( - validate, prepared, user_id=str(getattr(agent.state, "user_id", "") or "") - ) - tc = input_kwargs.get("tool_call") - tcid = getattr(tc, "id", "") or "" if tc is not None else "" - if prepared is not None: - from core.capabilities.runtime import record_tool_scope - - await asyncio.to_thread( - record_tool_scope, prepared, tcid, str(getattr(tc, "name", "") or "") - ) - token = CURRENT_TOOL_CALL_ID.set(tcid) if tcid else None - try: - async for item in next_handler(**input_kwargs): - yield item - finally: - if token is not None: - try: - CURRENT_TOOL_CALL_ID.reset(token) - except Exception: # noqa: BLE001 - pass - - -class ToolEffectMiddleware(MiddlewareBase): - """Commit durable Intent before every harness-owned tool invocation. - - The final ``ToolResponse`` is committed before it leaves this middleware, - so the existing AgentScope event/SSE mapping remains unchanged while the - database becomes authoritative for replay and reconciliation. - """ - - def __init__(self, session_factory=None, registry=None) -> None: # noqa: ANN001 - from core.db.engine import SessionLocal - from core.services.tool_effect_ledger import ( - DEFAULT_TOOL_RECOVERY_REGISTRY, - ToolEffectGateway, - ToolEffectJournal, - ) - - self._ledger = ToolEffectJournal(session_factory or SessionLocal) - self._registry = registry or DEFAULT_TOOL_RECOVERY_REGISTRY - self._gateway = ToolEffectGateway(self._ledger, self._registry) - from core.services.harness_ledger import HarnessUsageLedger - - self._usage_ledger = HarnessUsageLedger(session_factory or SessionLocal) - - @staticmethod - def _args(tool_call) -> dict: # noqa: ANN001 - raw = getattr(tool_call, "input", "") or "{}" - try: - parsed = json.loads(raw) if isinstance(raw, str) else raw - except (TypeError, json.JSONDecodeError): - parsed = {"_raw": str(raw)} - return parsed if isinstance(parsed, dict) else {"_value": parsed} - - @staticmethod - def _response_payload(response: ToolResponse) -> dict: - return {"tool_response": response.model_dump(mode="json")} - - @staticmethod - def _response_from_payload(payload: Any) -> ToolResponse: - from agentscope.message import DataBlock, TextBlock - - raw = payload.get("tool_response") if isinstance(payload, dict) else None - if not isinstance(raw, dict): - return ToolResponse( - content=[render_text_block(json.dumps(payload, ensure_ascii=False, default=str))], - state=ToolResultState.SUCCESS, - ) - blocks = [] - for item in raw.get("content") or []: - if not isinstance(item, dict): - blocks.append(render_text_block(str(item))) - elif item.get("type") == "data": - blocks.append(DataBlock.model_validate(item)) - else: - blocks.append(TextBlock.model_validate(item)) - state = ToolResultState(str(raw.get("state") or "success")) - return ToolResponse( - id=str(raw.get("id") or ""), - content=blocks, - state=state, - metadata=dict(raw.get("metadata") or {}), - ) - - async def on_acting(self, agent: Agent, input_kwargs: dict, next_handler): # noqa: ANN001 - from core.services.tool_effect_ledger import ToolIntentCommitError - - run_id = str(getattr(agent.state, "run_id", "") or "") - run_owner = str(getattr(agent.state, "journal_owner", "") or "") - tool_call = input_kwargs.get("tool_call") - tool_name = str(getattr(tool_call, "name", "") or "") - tool_call_id = str(getattr(tool_call, "id", "") or "") - if not run_id or not run_owner or not tool_name: - raise ToolIntentCommitError( - "tool execution requires a durable run_id and journal_owner binding" - ) - - args = self._args(tool_call) - if tool_name == "create_scheduled_task": - from core.services.automation_tool_routing import creation_arguments - args = creation_arguments(args, str(getattr(agent.state, "user_id", "") or ""), - str(getattr(agent.state, "chat_id", "") or "")) - live_chunks: asyncio.Queue = asyncio.Queue() - - async def _invoke(): - from core.services.tool_effect_ledger import CURRENT_TOOL_EFFECT - from core.harness.usage import ( - UsageAttempt, - attempt_status_for_exception, - record_usage_safely, - ) - - final = None - started = time.monotonic() - usage_status = "failed" - binding_token = CURRENT_RUN_BINDING.set((run_id, run_owner)) - invoke_kwargs = input_kwargs - effect = CURRENT_TOOL_EFFECT.get() - if effect is not None and tool_name in { - "create_scheduled_task", - "update_scheduled_task", - "delete_scheduled_task", - }: - adapter_args = dict(args) - adapter_args["tool_effect_id"] = effect.effect_id - adapter_call = tool_call.model_copy( - update={"input": json.dumps(adapter_args, ensure_ascii=False)} - ) - invoke_kwargs = {**input_kwargs, "tool_call": adapter_call} - try: - outcome_unknown = False - async for item in next_handler(**invoke_kwargs): - # Toolkit normally converts adapter exceptions into ordinary - # error responses. An ambiguous remote write must retain its - # ledger meaning across that conversion and stop this run. - ambiguous = bool( - (getattr(item, "metadata", None) or {}).get("gateway_outcome_unknown") - ) - outcome_unknown = outcome_unknown or ambiguous - if isinstance(item, ToolResponse): - final = item - elif not ambiguous: - await live_chunks.put(item) - if outcome_unknown: - raise RuntimeError("cloud write outcome unknown; reconciliation required") - usage_status = ( - "success" - if final is not None and final.state == ToolResultState.SUCCESS - else "failed" - ) - except asyncio.CancelledError: - usage_status = "cancelled" - raise - except Exception as exc: - usage_status = attempt_status_for_exception(exc) - raise - finally: - CURRENT_RUN_BINDING.reset(binding_token) - if effect is not None: - try: - await record_usage_safely( - self._usage_ledger, - UsageAttempt( - run_id=run_id, - kind="tool", - operation_name=tool_name, - effect_id=effect.effect_id, - status=usage_status, - latency_ms=int((time.monotonic() - started) * 1_000), - metadata={"tool_call_id": tool_call_id}, - ), - ) - except Exception: # usage cannot change tool execution - logger.debug("tool usage attempt persistence failed", exc_info=True) - if final is None: - from core.services.tool_effect_ledger import ToolEffectError - - raise ToolEffectError(f"tool {tool_name} ended without ToolResponse") - return self._response_payload(final) - - def _response_failed(payload: Any) -> bool: - raw = payload.get("tool_response") if isinstance(payload, dict) else None - return bool(isinstance(raw, dict) and str(raw.get("state") or "success") != "success") - - task = asyncio.create_task( - self._gateway.execute_outcome( - run_id=run_id, - owner=run_owner, - tool_call_id=tool_call_id, - tool_name=tool_name, - args=args, - invoke=_invoke, - classify_failure=_response_failed, - ) - ) - try: - while not task.done() or not live_chunks.empty(): - try: - item = await asyncio.wait_for(live_chunks.get(), timeout=0.02) - except asyncio.TimeoutError: - continue - yield item - outcome = await task - finally: - if not task.done(): - task.cancel() - with suppress(BaseException): - await task - elif not task.cancelled(): - task.exception() - response = self._response_from_payload(outcome.result) - if outcome.result_state == "error" and response.state == ToolResultState.SUCCESS: - response.state = ToolResultState.ERROR - linkage = { - "effect_id": outcome.effect_id, - "result_id": outcome.result_id, - } - response.metadata = {**dict(response.metadata or {}), **linkage} - links = getattr(agent.state, "tool_effect_links", None) - if isinstance(links, dict): - links[tool_call_id] = linkage - if not outcome.invoked: - yield ToolChunk( - content=list(response.content), - state=response.state, - metadata=dict(response.metadata), - ) - yield response - - -class CitationAnchorMiddleware(MiddlewareBase): - """on_acting: 统一证据锚点——工具结果回给模型前完成 提取 → 发号 → cite_id 回注。 - - 协议依据（agentscope 2.0 toolkit.call_tool）：中间 ToolChunk 是增量、最后一个 - yield 是**累积完整**的 ToolResponse；SSE 侧（orchestration/streaming.py）只在 - ToolResultEndEvent 时把 delta 累积成一条 tool_result，不实时消费中间增量。 - 因此这里缓冲中间块，在最终 ToolResponse 上注号后： - 1) 合成一个携带完整注号文本的 ToolChunk（SSE 累积到的就是注号后文本）； - 2) 用注号文本替换 ToolResponse.content（模型上下文 / 落库拿到同一份）。 - 两侧看到的 cite_id 因此严格一致。跳过名单、非纯文本内容、非 SUCCESS 状态、 - 任何异常 → 原样放行（引用功能降级，绝不影响工具本身）。 - """ - - async def on_acting(self, agent: Agent, input_kwargs: dict, next_handler): # noqa: ANN001 - from orchestration.citation_anchor import ( - SKIP_TOOLS, - annotate_tool_result, - resolve_allocator, - ) - - tool_call = input_kwargs.get("tool_call") - tool_name = str(getattr(tool_call, "name", "") or "") - tool_id = str(getattr(tool_call, "id", "") or "") - if not tool_name or tool_name in SKIP_TOOLS: - async for item in next_handler(**input_kwargs): - yield item - return - - buffered: list = [] - final: ToolResponse | None = None - async for item in next_handler(**input_kwargs): - if isinstance(item, ToolResponse): - final = item - break # per call_tool protocol the ToolResponse is the last yield - buffered.append(item) - - if final is None: - for item in buffered: - yield item - return - - annotated = False - if final.state == ToolResultState.SUCCESS: - try: - blocks = list(final.content or []) - text_blocks = [b for b in blocks if getattr(b, "type", "") == "text"] - if text_blocks and len(text_blocks) == len(blocks): - # 发号器绑在 agent 上（run 入口注入）；缺失时就地建并绑定， - # 保证编号在该 agent 的整条流里唯一 - allocator = resolve_allocator(agent) - full_text = "".join((b.text or "") for b in text_blocks) - new_text, items = annotate_tool_result(tool_name, tool_id, full_text, allocator) - if items: - allocator.register(tool_id, items) - final.content = [render_text_block(new_text)] - yield ToolChunk( - content=[render_text_block(new_text)], - state=final.state, - metadata=dict(final.metadata or {}), - ) - annotated = True - except Exception: # noqa: BLE001 - logger.warning( - "[citation-anchor] middleware annotate failed tool=%s", - tool_name, - exc_info=True, - ) - - if not annotated: - for item in buffered: - yield item - yield final - - -class AgentRuntimeState(AgentState): - """Extends AgentState to carry the runtime fields of the former ModelContext (replaces agent._jx_context).""" - - model_config = ConfigDict(ignored_types=(_CYFUNCTION_TYPE,)) - - model_name: str = "" - model_provider_id: str = "" - # When a subagent explicitly configures a model (model_provider_id) → the factory sets this - # True, and DynamicModel must not override it with the main model based on chat_mode - # (otherwise the subagent's own model is effectively useless — neither channels nor web get it). - model_pinned: bool = False - user_id: str | None = None - chat_id: str | None = None - run_id: str | None = None - capability_scope: str = "" - journal_owner: str | None = None - tool_effect_links: dict = Field(default_factory=dict) - enable_thinking: bool = True - chat_mode: str | None = None - uploaded_files: List[dict] = Field(default_factory=list) - historical_files: List[dict] = Field(default_factory=list) - user_message_text: str = "" - # Vision evidence transcribed ahead of the model call by the streaming entry - # point; FileContextMiddleware injects it verbatim instead of redoing the - # network round-trip. Empty = transcribe inline (non-streaming callers). - vision_evidence_text: str = "" - ontology_enabled: bool = False - ontology_runtime: dict = Field(default_factory=dict) - # Set by SteerMiddleware immediately before the next reasoning round. The - # streaming adapter turns it into one ``steer_applied`` event, then clears - # it. Keeping this on the typed runtime state avoids process-global queues. - steer_delivery: dict | None = None - # Model responses recorded by the hook adapter the moment they complete, - # drained by the streaming adapter into ``model_step`` events. Each entry - # is one ``core.llm.model_steps`` assistant record. - pending_model_steps: List[dict] = Field(default_factory=list) - - def apply_request_context(self, context: dict, user_message_text: str) -> None: - """Populate per-request runtime fields from the request ``context`` dict (replaces the 1.x agent._jx_context). - - Shared by both entry points — streaming and workflow (non-streaming) — to keep two - hand-copied field mappings from silently drifting (streaming once didn't lowercase - chat_mode while workflow did). - """ - # A child may use the shared role or its own provider. Request context - # still describes the parent selection; it must not relabel a pinned - # child (vision routing and audit read these fields). - if not self.model_pinned: - self.model_name = str(context.get("model_name", "") or "") - self.model_provider_id = str(context.get("model_provider_id", "") or "") - self.user_id = str(context.get("user_id", "") or "") or None - self.chat_id = str(context.get("chat_id", "") or "") or None - self.run_id = str(context.get("run_id", "") or "") or None - self.journal_owner = str(context.get("journal_owner", "") or "") or None - self.enable_thinking = bool(context.get("enable_thinking", True)) - cm = str(context.get("chat_mode") or "").lower() or None - if cm: - self.chat_mode = cm - self.uploaded_files = list(context.get("uploaded_files", []) or []) - self.historical_files = list(context.get("historical_files", []) or []) - self.user_message_text = user_message_text or "" - self.ontology_enabled = bool(context.get("ontology_enabled", False)) - runtime = context.get("ontology_runtime") - self.ontology_runtime = runtime if isinstance(runtime, dict) else {} - - -class OntologyGateMiddleware(MiddlewareBase): - """L-a deterministic gate: validate every visible tool call without an LLM.""" - - def __init__(self, runtime: dict | None = None) -> None: - self.runtime = runtime if isinstance(runtime, dict) else {} - self.completed_tools: set[str] = set() - self.denial_counts: dict[str, int] = {} - - async def on_acting(self, agent: Agent, input_kwargs: dict, next_handler): # noqa: ANN001 - state_runtime = getattr(agent.state, "ontology_runtime", None) - runtime = state_runtime if isinstance(state_runtime, dict) else self.runtime - if not runtime.get("enabled"): - async for item in next_handler(**input_kwargs): - yield item - return - - from core.ontology.validator import ( - activate_runtime_for_asset, - evaluate_tool_call, - render_runtime_prompt, - ) - from core.services.ontology_service import resolve_runtime_asset_tags - - started = time.monotonic() - tool_call = input_kwargs.get("tool_call") - tool_name = str(getattr(tool_call, "name", "") or "") - raw_input = getattr(tool_call, "input", "{}") - try: - tool_input = raw_input if isinstance(raw_input, dict) else json.loads(raw_input or "{}") - except (TypeError, json.JSONDecodeError): - tool_input = {} - asset_kind, asset_id = self._resolve_invoked_asset(tool_name, tool_input) - await asyncio.to_thread( - resolve_runtime_asset_tags, - runtime=runtime, - kind=asset_kind, - asset_id=asset_id, - user_id=str(getattr(agent.state, "user_id", "") or ""), - ) - activations = activate_runtime_for_asset( - runtime, - kind=asset_kind, - asset_id=asset_id, - ) - if activations: - agent.state.ontology_enabled = True - agent.state.ontology_runtime = runtime - contract = render_runtime_prompt(runtime) - if contract: - _append_reminder( - agent, - "检测到受领域本体治理的运行时资产，策略已升级且本轮不可降级。\n" f"{contract}", - origin="harness:ontology_gate", - ) - await self._audit_activations(agent, runtime, activations) - decision = evaluate_tool_call( - runtime, - tool_name=tool_name, - tool_input=tool_input, - completed_tools=self.completed_tools, - ) - if not decision.allowed: - key = ",".join(decision.matched_rule_ids) or tool_name - self.denial_counts[key] = self.denial_counts.get(key, 0) + 1 - denial_count = self.denial_counts[key] - thresholds = [pack.get("config", {}) for pack in runtime.get("packs", [])] - strategy_threshold = min( - (int(item.get("repeated_denial_threshold", 2)) for item in thresholds), - default=2, - ) - breaker_threshold = min( - (int(item.get("circuit_breaker_threshold", 5)) for item in thresholds), - default=5, - ) - guidance = list(decision.suggestions) - if denial_count >= strategy_threshold: - guidance.append("同一规则已重复拦截，请改变执行策略，不要原样重试。") - if denial_count >= breaker_threshold: - guidance.append("本体门禁已触发熔断；停止调用该工具，并向用户说明缺失条件。") - await self._audit( - agent, - runtime, - tool_name, - decision, - started, - denial_count=denial_count, - circuit_breaker=denial_count >= breaker_threshold, - ) - if denial_count == strategy_threshold and getattr(agent.state, "chat_id", None): - try: - from core.services.ontology_evolution_service import ( - schedule_ontology_evolution, - ) - - schedule_ontology_evolution( - user_id=str(getattr(agent.state, "user_id", "") or "system") - ) - except Exception as exc: # noqa: BLE001 - logger.warning("[ontology] evolution scheduling failed: %s", exc) - payload = { - "code": "ONTOLOGY_GATE_DENIED", - "tool": tool_name, - "violations": decision.violations, - "suggestions": guidance, - "denial_count": denial_count, - "circuit_breaker": denial_count >= breaker_threshold, - } - yield ToolChunk( - content=[render_text_block(json.dumps(payload, ensure_ascii=False))], - state=ToolResultState.DENIED, - metadata={"ontology_gate": payload}, - ) - return - - await self._audit(agent, runtime, tool_name, decision, started) - - last_state = None - async for item in next_handler(**input_kwargs): - last_state = getattr(item, "state", last_state) - yield item - if last_state not in { - ToolResultState.ERROR, - ToolResultState.DENIED, - ToolResultState.INTERRUPTED, - }: - self.completed_tools.add(tool_name) - - async def _audit( - self, - agent: Agent, - runtime: dict[str, Any], - tool_name: str, - decision, - started: float, - *, - denial_count: int = 0, - circuit_breaker: bool = False, - ) -> None: # noqa: ANN001 - if not decision.violations and not decision.matched_rule_ids: - return - runtime.setdefault("runtime_events", []).append( - { - "type": "ontology_gate", - "status": "completed", - "governance_run_id": runtime.get("governance_run_id"), - "stage": "tool", - "decision": decision.decision, - "tool_name": tool_name, - "matched_rule_ids": list(decision.matched_rule_ids), - "violations": list(decision.violations), - "suggestions": list(decision.suggestions), - "denial_count": denial_count, - "circuit_breaker": circuit_breaker, - } - ) - try: - from core.services.ontology_service import record_enforcement_event - - first = decision.violations[0] if decision.violations else {} - pack_id = first.get("pack_id") - version_id = None - for pack in runtime.get("packs", []): - if pack.get("pack_id") == pack_id: - version_id = pack.get("version_id") - break - payload = { - "user_id": getattr(agent.state, "user_id", None), - "chat_id": getattr(agent.state, "chat_id", None), - "pack_id": pack_id, - "version_id": version_id, - "rule_id": first.get("rule_id") - or (decision.matched_rule_ids[0] if decision.matched_rule_ids else None), - "stage": "tool", - "event_type": "tool_call_gate", - "decision": decision.decision, - "mode": first.get("mode", "enforce"), - "target": tool_name, - "latency_ms": round((time.monotonic() - started) * 1000), - "details": { - "governance_run_id": runtime.get("governance_run_id"), - "violations": decision.violations, - "matched_rule_ids": decision.matched_rule_ids, - "suggestions": decision.suggestions, - "denial_count": denial_count, - "circuit_breaker": circuit_breaker, - }, - } - await asyncio.to_thread(record_enforcement_event, payload) - except Exception as exc: # noqa: BLE001 - logger.warning("[ontology] audit event persistence failed: %s", exc) - - @staticmethod - def _resolve_invoked_asset(tool_name: str, tool_input: dict[str, Any]) -> tuple[str, str]: - if tool_name == "call_subagent" and tool_input.get("agent_id"): - return "subagent", str(tool_input["agent_id"]) - if tool_name == "view_text_file": - file_path = str(tool_input.get("file_path") or "") - path = Path(file_path) - if path.name == "SKILL.md" and path.parent.name: - return "skill", path.parent.name - return "tool", tool_name - - @staticmethod - async def _audit_activations( - agent: Agent, - runtime: dict[str, Any], - activations: list[dict[str, Any]], - ) -> None: - try: - from core.services.ontology_service import record_runtime_activation - - for event in activations: - await asyncio.to_thread( - record_runtime_activation, - event, - runtime, - user_id=getattr(agent.state, "user_id", None), - chat_id=getattr(agent.state, "chat_id", None), - ) - except Exception as exc: # noqa: BLE001 - logger.warning("[ontology] activation audit persistence failed: %s", exc) - - -# ── DynamicModel ────────────────────────────────────────────────────────── -class DynamicModelMiddleware(MiddlewareBase): - async def on_reply(self, agent: Agent, input_kwargs: dict, next_handler): - try: - # The subagent pinned its own model → skip the chat_mode-based main-model override - # and respect its configuration. Otherwise the subagent model built by the factory - # from model_provider_id would be unconditionally replaced with the main model here — - # symptom: "subagent channel binding doesn't take effect / web and channel behave differently". - if not getattr(agent.state, "model_pinned", False): - mode = _resolve_chat_mode(agent.state) - provider_id = getattr(agent.state, "model_provider_id", "") or "" - if provider_id: - agent.model = _get_provider_model(provider_id, mode=mode) - else: - agent.model = _get_main_model(mode=mode) - run_id = str(getattr(agent.state, "run_id", "") or "") - if run_id: - from core.llm.model_usage import instrument_model_usage - - instrument_model_usage(agent.model) - except Exception as exc: # noqa: BLE001 - logger.warning("[dynamic_model] failed: %s", exc) - async for evt in next_handler(**input_kwargs): - yield evt - - -# ── FileContext ─────────────────────────────────────────────────────────── -class FileContextMiddleware(MiddlewareBase): - def __init__(self) -> None: - self._injected = False - - async def on_reply(self, agent: Agent, input_kwargs: dict, next_handler): - try: - await self._inject(agent) - except Exception as exc: # noqa: BLE001 - logger.warning("[file_context] failed: %s", exc, exc_info=True) - async for evt in next_handler(**input_kwargs): - yield evt - - async def _inject(self, agent: Agent) -> None: - # Reset per-turn state at each reply-turn boundary (read_artifact budget / pin-reminder bookkeeping) - reset_artifact_read_state() - reset_pin_hint_state() - - st = agent.state - uploaded_files = list(getattr(st, "uploaded_files", None) or []) - historical_files = list(getattr(st, "historical_files", None) or []) - if not uploaded_files and not historical_files: - return - # user_id serves as the ownership-verification gate for attachment reads (prevents forged - # file_id cross-user reads; see the _download_artifact_bytes / _build_file_context docstrings in hooks). - user_id = getattr(st, "user_id", None) or None - if self._injected: - return - self._injected = True - - # 0. Historical files digest - if historical_files: - hist_context = _build_historical_files_context(historical_files) - if hist_context: - _append_state_payload( - st, - hist_context, - kind=KIND_PROJECT, - origin="workspace:historical_files", - trust="workspace", - ) - - # 1. Current-turn text files. - # Off the event loop: on a parse-cache miss _build_file_context downloads the - # artifact and POSTs it to the external file-parser service with blocking - # ``requests`` — measured at 79s for an 8MB PDF, and the service timeout allows - # 120s. Called inline it pinned the whole loop for that long: every SSE stream - # (all chats, all users) emitted nothing, queued requests all landed in the same - # millisecond once it returned, and even /health stalled. The visible symptom is - # "streaming stopped working" on the very turn a document is attached. - text_context = ( - await asyncio.to_thread(_build_file_context, uploaded_files, user_id=user_id) - if uploaded_files - else "" - ) - if text_context: - _append_state_payload( - st, - text_context, - kind=KIND_ATTACHMENT, - origin="user:uploaded_files", - trust="user", - ) - - # 2. Images. Two paths, decided by whether the *effective* model can see: - # - natively multimodal → pass the bytes through as DataBlocks (best fidelity) - # - text-only → vision bridge transcribes them into text evidence - # (core/vision). This is the only place the choice is made: downstream - # never degrades media, so a text-only model that received raw bytes - # would fail the request outright rather than silently go blind. - image_files = [f for f in uploaded_files if _is_image(f)] - if not image_files: - return - if _effective_model_supports_vision(st): - # Same reason: each image is downloaded from storage (S3/OSS in deployments - # that use them) and base64-encoded — blocking I/O plus CPU on the loop. - await asyncio.to_thread(self._inject_native_images, st, image_files, user_id) - else: - await self._inject_vision_evidence(st, image_files, user_id) - - # ── image injection paths ────────────────────────────────────────────── - - @staticmethod - def _inject_native_images(st: AgentState, image_files: list, user_id) -> None: - """Natively multimodal model: append framework-neutral image blocks.""" - image_blocks: list = [] - image_names: list[str] = [] - for f in image_files: - result = _fetch_image_base64(f, user_id=user_id) - if result: - b64_data, mime_type = result - image_blocks.append( - { - "type": "data", - "source": { - "type": "base64", - "media_type": mime_type, - "data": b64_data, - }, - } - ) - image_names.append(f.get("name", "图片")) - if not image_blocks: - return - names_str = "、".join(image_names) - prefix_block = { - "type": "text", - "text": f"[用户上传了 {len(image_blocks)} 张图片：{names_str}]", - } - _append_state_payload( - st, - [prefix_block, *image_blocks], - kind=KIND_ATTACHMENT, - origin="user:uploaded_images", - trust="user", - token_budget=16_000, - ) - - @staticmethod - async def _inject_vision_evidence(st: AgentState, image_files: list, user_id) -> None: - """Text-only model: inject the images as text evidence from the vision bridge. - - The streaming entry point usually transcribes ahead of time (so it can report - "reading image" progress while the network call is in flight) and leaves the - result on ``st.vision_evidence_text``. When that is absent — the non-streaming - workflow, channel bots — this transcribes inline as before. - """ - precomputed = (getattr(st, "vision_evidence_text", "") or "").strip() - if precomputed: - _append_state_payload( - st, - precomputed, - kind=KIND_ATTACHMENT, - origin="vision:transcription", - trust="tool", - ) - return - - from core.vision.attachments import transcribe_attachments - - result = await transcribe_attachments(image_files, user_id=user_id) - if result is None or not result.text: - return - _append_state_payload( - st, - result.text, - kind=KIND_ATTACHMENT, - origin="vision:transcription", - trust="tool", - ) - - -def _effective_model_supports_vision(st: AgentState) -> bool: - """Whether the model this turn will actually run on can read images natively.""" - from core.vision import resolve_vision_mode - - return resolve_vision_mode(getattr(st, "model_provider_id", "") or "") == "native" - - -# ── WorkspacePinHint ─────────────────────────────────────────────────────── -def _recent_tool_results(context: list, scan: int = 12) -> list: - """Return the most recent tool_result blocks at the tail of context (2.0: pydantic blocks on assistant messages).""" - blocks = [] - for msg in context[-scan:]: - try: - if msg.has_content_blocks("tool_result"): - blocks.extend(msg.get_content_blocks("tool_result")) - except Exception: # noqa: BLE001 - continue - return blocks - - -def _tool_result_text(block: Any) -> str: - from core.llm.message_compat import flatten_tool_output - - return flatten_tool_output(getattr(block, "output", None)) - - -class WorkspacePinHintMiddleware(MiddlewareBase): - async def on_reasoning(self, agent: Agent, input_kwargs: dict, next_handler): - try: - self._scan_and_remind(agent) - except Exception as exc: # noqa: BLE001 - logger.warning("[pin_hint] failed: %s", exc, exc_info=True) - async for evt in next_handler(**input_kwargs): - yield evt - - def _scan_and_remind(self, agent: Agent) -> None: - from core.llm import workspace as _ws - - results = _recent_tool_results(agent.state.context) - if not results: - return - state = _get_pin_hint_state() - seen: set = state["seen"] - for block in results: - name = getattr(block, "name", "") or "" - if name in _PIN_HINT_SKIP_TOOLS: - continue - text_blob = _tool_result_text(block) - if '"file_id"' in text_blob: - seen.update(_FILE_ID_RE.findall(text_blob)) - if not seen: - return - pinned = set(_ws.get_pinned_file_ids()) - unpinned = seen - pinned - if not unpinned: - return - sig = ",".join(sorted(unpinned)) - if sig == state.get("last_reminded_sig"): - return - preview = sorted(unpinned)[:6] - preview_str = ", ".join(preview) - if len(unpinned) > len(preview): - preview_str += f", …(+{len(unpinned) - len(preview)} 个)" - reminder = ( - f"沙盒里有 {len(unpinned)} 个 file_id 还没 pin：[{preview_str}]。" - f"若是给用户的最终产物，必须调 `pin_to_workspace(file_ids=[...])` 才能交付。" - ) - _append_reminder(agent, reminder, origin="harness:workspace_pin") - state["last_reminded_sig"] = sig - - -# ── PlanStaleReminder ────────────────────────────────────────────────────── -# 停滞满 N 轮催一次，催完重新计时——等价于 Claude Code 那两个都取 10 的阈值。 -_PLAN_REMINDER_INTERVAL = 10 - -_PLAN_STALE_REMINDER_TEMPLATE = """`update_plan` 已经 {rounds} 轮没有用过了。当前计划栏停在 {done}/{total}： -{checklist} - -如果其中有步骤其实已经做完，可以调用一次 `update_plan` 把状态补上（传全量列表）；如果这份 -清单已经不符合你正在做的事，重写它比让它停在原地更好。仅在确实相关时才使用——这只是一条 -温和的提醒，不适用就忽略，不要为了回应它而打断手上的工作。不要向用户提起这条提醒。""" - - -def _block_attr(block: Any, name: str, default: Any = None) -> Any: - """内容块在 AS2 里是 pydantic 对象，但历史路径上也出现过 dict —— 两种都读。""" - if isinstance(block, dict): - return block.get(name, default) - return getattr(block, name, default) - - -def _latest_plan_call(context: list) -> tuple[Optional[dict], str]: - """倒着找最近一次 ``update_plan`` 调用，返回（解析后的计划, 这次调用的指纹）。 - - 指纹用 tool_call 的 id：id 变了就说明模型刚更新过清单，计时要清零。拿不到 id 时 - 退化成清单内容本身——同样能区分"又调了一次且内容有变"，只在"重复提交完全相同的 - 清单"时无法分辨，而那种情况本来也不该重置计时。 - """ - for msg in reversed(list(context or [])): - try: - if hasattr(msg, "has_content_blocks") and not msg.has_content_blocks("tool_call"): - continue - blocks = list(msg.get_content_blocks("tool_call") or []) - except Exception: # noqa: BLE001 - continue - for b in reversed(blocks): - if _block_attr(b, "name", "") != "update_plan": - continue - raw = _block_attr(b, "input", None) - if isinstance(raw, str): - try: - raw = json.loads(raw) - except (json.JSONDecodeError, ValueError): - raw = None - plan = parse_plan_update_args(raw) - fingerprint = str(_block_attr(b, "id", "") or "") or json.dumps( - plan or {}, ensure_ascii=False, sort_keys=True - ) - return plan, fingerprint - return None, "" - - -class PlanStaleReminderMiddleware(MiddlewareBase): - """计划栏停滞满 ``_PLAN_REMINDER_INTERVAL`` 轮时，把当前清单软提醒给模型一次。 - - 模型没建过清单、或清单已全部结算，都不打扰；模型自己更新了清单则重新计时。 - """ - - def __init__(self, *, interval: int = _PLAN_REMINDER_INTERVAL) -> None: - self._interval = max(1, int(interval)) - self._since_last = 0 - self._last_fingerprint: Optional[str] = None - - async def on_reasoning(self, agent: Agent, input_kwargs: dict, next_handler): - try: - self._maybe_remind(agent) - except Exception as exc: # noqa: BLE001 - logger.warning("[plan_stale] failed: %s", exc) - async for evt in next_handler(**input_kwargs): - yield evt - - def _maybe_remind(self, agent: Agent) -> None: - plan, fingerprint = _latest_plan_call(agent.state.context) - if not plan: - return - if fingerprint != self._last_fingerprint: - self._last_fingerprint = fingerprint - self._since_last = 0 - return - steps = plan.get("steps") or [] - if not any(s.get("status") in ("pending", "in_progress") for s in steps): - return - self._since_last += 1 - if self._since_last < self._interval: - return - done = sum(1 for s in steps if s.get("status") == "completed") - checklist = "\n".join( - f" {i}. [{s.get('status', 'pending')}] {s.get('title', '')}" - for i, s in enumerate(steps, start=1) - ) - _append_reminder( - agent, - _PLAN_STALE_REMINDER_TEMPLATE.format( - done=done, - total=len(steps), - rounds=self._since_last, - checklist=checklist, - ), - origin="harness:plan_stale", - ) - logger.info( - "[plan_stale] reminded at %d/%d after %d idle rounds", - done, - len(steps), - self._since_last, - ) - self._since_last = 0 - - -# ── IterBudgetReminder ───────────────────────────────────────────────────── -class IterBudgetReminderMiddleware(MiddlewareBase): - """Inject a wrap-up reminder as ReAct approaches max_iters, so the model soft-lands instead of erroring on the hard limit. - - When AgentScope 2.0 runs max_iters dry it only yields ExceedMaxItersEvent + one fixed - English error string, and everything produced in the turn is discarded (in the subagent - scenario the error string is returned to the main agent as the tool result). This - middleware appends a system-reminder to context when remaining rounds <= threshold, - with two levels of wording: - - N (>1) rounds left: stop exploring/retrying, consolidate the information gathered so far and wrap up; - - final round: no further tool calls allowed — the framework loop gives no more reasoning - chances before exiting, so a tool call in the final round is guaranteed to trip the hard - limit (_agent.py main-loop semantics). - - Dedup key is (reply_id, cur_iter): remind only once per round; across rounds the wording - escalates in urgency as remaining rounds decrease. When max_iters is very small - (<= threshold+1, e.g. plan wrap-up style micro budgets), don't remind — avoid nagging to - wrap up right after starting. - """ - - def __init__(self, *, threshold: int = 2) -> None: - self._threshold = threshold - self._last_key: tuple | None = None - # Env is immutable for the process lifetime — parse the kill-switch once - # per middleware (one per agent build), not on every reasoning round. - from core.config.settings import _bool, _env - - self._force_text = _bool(_env("CHAT_FINAL_ITER_FORCE_TEXT", "true")) - - async def on_reasoning(self, agent: Agent, input_kwargs: dict, next_handler): - max_iters, cur_iter, remaining = self._budget(agent) - try: - self._maybe_remind(agent, max_iters, cur_iter, remaining) - except Exception as exc: # noqa: BLE001 - logger.warning("[iter_budget] failed: %s", exc) - try: - input_kwargs = self._maybe_force_text(input_kwargs, max_iters, remaining) - except Exception as exc: # noqa: BLE001 - logger.warning("[iter_budget] force-text failed: %s", exc) - async for evt in next_handler(**input_kwargs): - yield evt - - @staticmethod - def _budget(agent: Agent) -> tuple[int, int, int]: - """Return (max_iters, cur_iter, remaining-including-current-round).""" - max_iters = int(getattr(agent.react_config, "max_iters", 0) or 0) - cur_iter = int(getattr(agent.state, "cur_iter", 0) or 0) - return max_iters, cur_iter, max_iters - cur_iter - - def _maybe_force_text(self, input_kwargs: dict, max_iters: int, remaining: int) -> dict: - """Final round: force tool_choice="none" so the model can only produce text. - - The reminder alone is advisory — a model that still emits a tool call in - the final round trips AgentScope's hard limit and the whole turn's output - is discarded. Forcing text turns the hard error into a guaranteed clean - exit (the loop sees no tool calls and yields the reply Msg). Skipped for - micro budgets (same guard as the reminder) and when a caller already - pinned an explicit tool_choice. Kill-switch: CHAT_FINAL_ITER_FORCE_TEXT=false. - """ - if remaining > 1 or max_iters <= self._threshold + 1 or not self._force_text: - return input_kwargs - if input_kwargs.get("tool_choice") is not None: - return input_kwargs - from agentscope.tool import ToolChoice - - logger.info( - "[iter_budget] final round (max_iters=%d): forcing tool_choice=none for clean delivery", - max_iters, - ) - return {**input_kwargs, "tool_choice": ToolChoice(mode="none")} - - def _maybe_remind(self, agent: Agent, max_iters: int, cur_iter: int, remaining: int) -> None: - if max_iters <= self._threshold + 1: - return - if remaining > self._threshold: - return - key = (getattr(agent.state, "reply_id", "") or "", cur_iter) - if key == self._last_key: - return - self._last_key = key - if remaining <= 1: - reminder = ( - "这是本次回复的最后一轮推理：不要再调用任何工具（再调用会被系统强制" - "中断，已完成的工作将无法交付）。请直接基于已获得的信息输出最终回复；" - "若任务未全部完成，如实汇报已产出的成果、当前进展和未完成的部分。" - ) - else: - reminder = ( - f"注意：推理-工具调用轮次即将用尽，包含本轮在内最多还剩 {remaining} 轮。" - "请停止新的探索，不要再重试已反复失败的操作，立即整合已获得的信息进行" - "收尾；若任务无法在剩余轮次内全部完成，优先输出已有成果与进展说明。" - ) - _append_reminder(agent, reminder, origin="harness:iteration_budget") - - -# ── JobLedgerReminder ────────────────────────────────────────────────────── -class JobLedgerReminderMiddleware(MiddlewareBase): - """本会话存在未收敛的批量作业时，每轮把台账状态回灌进上下文。 - - 为什么要 harness 主动回灌，而不是指望模型记着：进度是**外部事实**，写在 - ``job_items`` 表里，不在模型的记忆里。模型隔了十几轮之后对"还剩多少没做"的印象 - 只会越来越糊，最后就演变成"边际收益递减，先交付吧"——上一轮 568 行只补 66 行 - 就是这么停的。 - - 每轮注入一条 ````，内容是可查证的数字（总计/已完成/待办/失败） - 和明确的下一步。同一轮只注入一次；台账已收敛（待办与失败都为 0）就不再打扰。 - """ - - def __init__(self, *, chat_id: Optional[str], user_id: Optional[str]) -> None: - self._chat_id = chat_id or "" - self._user_id = user_id or "" - self._last_key: tuple | None = None - - async def on_reasoning(self, agent: Agent, input_kwargs: dict, next_handler): - try: - self._maybe_remind(agent) - except Exception as exc: # noqa: BLE001 —— 提醒失败绝不该拖垮主链路 - logger.warning("[job_ledger] reminder failed: %s", exc) - async for evt in next_handler(**input_kwargs): - yield evt - - def _maybe_remind(self, agent: Agent) -> None: - if not self._chat_id: - return - key = ( - getattr(agent.state, "reply_id", "") or "", - int(getattr(agent.state, "cur_iter", 0) or 0), - ) - if key == self._last_key: - return - - from core.db.engine import SessionLocal - from core.db.models import Job - from core.services.job_service import JobService - - with SessionLocal() as db: - jobs = ( - db.query(Job) - .filter(Job.chat_id == self._chat_id, Job.user_id == self._user_id) - .order_by(Job.created_at.desc()) - .limit(3) - .all() - ) - if not jobs: - return - svc = JobService(db) - lines = [] - for job in jobs: - stats = svc.stats(job.job_id) - unsettled = int(stats.get("pending", 0)) + int(stats.get("failed", 0)) - if unsettled <= 0 and job.status in ("completed", "cancelled"): - continue - lines.append( - f"- 作业「{job.name or job.job_id}」（{job.job_id}，状态 {job.status}）：" - f"总计 {stats.get('total', 0)}，已完成 {stats.get('done', 0)}，" - f"查无 {stats.get('not_found', 0)}，待办 {stats.get('pending', 0)}，" - f"失败 {stats.get('failed', 0)}" - ) - if not lines: - return - - self._last_key = key - reminder = ( - "本会话有尚未结算完的批量作业（数字取自台账，是可查证的事实，不是估计）：\n" - + "\n".join(lines) - + "\n\n还有待办或失败项时：用 run_job(action='resume', job_id=...) 续跑" - "（已完成的项不会重做），或先查 run_job(action='status', job_id=...) 看明细。" - "**不得**把未完成当作完成来交付；确实要收尾，也必须如实报出分母、完成数与未覆盖清单。" - ) - _append_reminder(agent, reminder, origin="harness:job_ledger") - - -# ── StallIntervention ────────────────────────────────────────────────────── -class StallInterventionMiddleware(MiddlewareBase): - """Apply the active profile's intervention rules to the **ReAct loop**. - - The rules existed before this and only ever reached ``autonomous_loop`` — the - long-running self-driving product — through - :mod:`core.evolution.policies`. That left the orchestration profile with one - field that governed nothing on the axis nearly all traffic takes, and it is a - fair reading of "you are still tuning a workflow": every other part of the - profile shapes the ReAct assembly, this one did not. - - What a ReAct turn can actually observe is narrower than what a multi-run loop - can, and pretending otherwise is how a rule ends up never firing: - - * ``repeated_actions`` — the same tool called with the same arguments again; - * ``tool_error_streak`` — consecutive failing tool calls; - * ``no_progress`` — consecutive calls that returned nothing usable. - - ``no_diff`` and ``reviewer_score_flat`` need a file tree and a reviewer - verdict, which a single answer has neither of; those stay loop-only and are - declared as such in :mod:`core.evolution.agent_profile`. - - The intervention itself is a ```` appended to the context — - the same mechanism the iteration-budget reminder uses. A middleware cannot - reach into the model's plan, and it should not: the model is told what has - been observed and what to do about it, and stays responsible for the how. - """ - - def __init__(self, rules: List[Any] | None = None) -> None: - self._rules = list(rules or []) - self._signals: dict[str, int] = {} - self._last_call: tuple | None = None - self._fired: set[str] = set() - - async def on_acting(self, agent: Agent, input_kwargs: dict, next_handler): # noqa: ANN001 - if not self._rules: - async for item in next_handler(**input_kwargs): - yield item - return - - tool_call = input_kwargs.get("tool_call") - tool_name = str(getattr(tool_call, "name", "") or "") - try: - signature = ( - tool_name, - json.dumps(getattr(tool_call, "input", None), sort_keys=True, default=str), - ) - except Exception: # noqa: BLE001 - signature = (tool_name, "") - - if signature == self._last_call: - self._signals["repeated_actions"] = self._signals.get("repeated_actions", 0) + 1 - else: - # A streak is consecutive by definition; a different call breaks it. - self._signals["repeated_actions"] = 0 - self._last_call = signature - - last_state = None - async for item in next_handler(**input_kwargs): - last_state = getattr(item, "state", last_state) - yield item - - failed = last_state in { - ToolResultState.ERROR, - ToolResultState.DENIED, - ToolResultState.INTERRUPTED, - } - if failed: - self._signals["tool_error_streak"] = self._signals.get("tool_error_streak", 0) + 1 - self._signals["no_progress"] = self._signals.get("no_progress", 0) + 1 - else: - self._signals["tool_error_streak"] = 0 - self._signals["no_progress"] = 0 - - try: - self._maybe_intervene(agent) - except Exception as exc: # noqa: BLE001 - logger.warning("[stall-intervention] failed: %s", exc) - - def _maybe_intervene(self, agent: Agent) -> None: - from core.evolution.agent_profile import ( - ACTION_CHANGE_STRATEGY, - ACTION_DELEGATE, - ACTION_ESCALATE, - ACTION_NARROW_SCOPE, - ACTION_RETRY, - ACTION_ROLLBACK_AND_FORK, - ACTION_STOP, - ) - - applicable = [ - rule - for rule in self._rules - if self._signals.get(getattr(rule, "signal", ""), 0) - >= int(getattr(rule, "threshold", 0) or 0) - and int(getattr(rule, "threshold", 0) or 0) > 0 - ] - if not applicable: - return - # Most-specific first: a long stall escalates rather than repeating the - # mild response that already failed to help. - rule = max(applicable, key=lambda r: int(getattr(r, "threshold", 0) or 0)) - action = str(getattr(rule, "action", "")) - signal = str(getattr(rule, "signal", "")) - - # Fire once per (signal, action). Repeating the same reminder every - # iteration turns it into noise the model learns to skip. - key = f"{signal}:{action}" - if key in self._fired: - return - self._fired.add(key) - - observed = { - "repeated_actions": "你已经用完全相同的参数重复调用同一个工具", - "tool_error_streak": "连续多次工具调用失败", - "no_progress": "连续多轮没有取得任何可用结果", - }.get(signal, "当前执行已停滞") - - guidance = { - ACTION_CHANGE_STRATEGY: "换一条思路：不要重试同样的调用，改用别的工具或别的切入角度。", - ACTION_NARROW_SCOPE: "缩小范围：先只解决其中最小的一个子问题，拿到确定结果再往下走。", - ACTION_RETRY: "可以再试一次，但必须改变参数或前置条件，原样重试不会有不同结果。", - ACTION_DELEGATE: "把这段工作交给合适的子智能体处理，只取回结论。", - ACTION_ESCALATE: "停止尝试，向用户说明卡在哪里、还缺什么信息。", - ACTION_STOP: "停止继续尝试，如实汇报已完成的部分和未完成的原因。", - # Loop-only: a single answer has nothing to roll back to. - ACTION_ROLLBACK_AND_FORK: "放弃当前路径，回到上一个可用状态重新规划。", - }.get(action, "") - if not guidance: - return - - _append_reminder( - agent, - f"{observed}。{guidance}\n" - f"（该干预来自当前生效的编排配置：{signal} ≥ " - f"{getattr(rule, 'threshold', 0)} → {action}）", - origin="harness:stall_intervention", - ) - logger.info( - "[stall-intervention] %s >= %s -> %s", - signal, - getattr(rule, "threshold", 0), - action, - ) - - -# ── FinishPinGuard ───────────────────────────────────────────────────────── -class FinishPinGuardMiddleware(MiddlewareBase): - def __init__(self) -> None: - self._fired = False - - async def on_reasoning(self, agent: Agent, input_kwargs: dict, next_handler): - had_tool_call = False - async for evt in next_handler(**input_kwargs): - if isinstance(evt, ToolCallEndEvent): - had_tool_call = True - yield evt - if had_tool_call or self._fired: - return - try: - from core.llm.finish_guard import _collect_unpinned, _direct_pin - - unpinned = _collect_unpinned() - if unpinned: - pinned_now = _direct_pin(unpinned) - if pinned_now > 0: - self._fired = True - logger.info( - "[finish_guard] auto-pinned %d/%d file_id(s)", - pinned_now, - len(unpinned), - ) - except Exception as exc: # noqa: BLE001 - logger.warning("[finish_guard] failed: %s", exc) - - -__all__ = [ - "AgentRuntimeState", - "DynamicModelMiddleware", - "FileContextMiddleware", - "WorkspacePinHintMiddleware", - "IterBudgetReminderMiddleware", - "FinishPinGuardMiddleware", -] diff --git a/src/backend/core/llm/persona_distiller.py b/src/backend/core/llm/persona_distiller.py deleted file mode 100644 index ff85c3ea..00000000 --- a/src/backend/core/llm/persona_distiller.py +++ /dev/null @@ -1,226 +0,0 @@ -"""Persona distiller — map/reduce LLM caller for colleague / personal skill jobs. - -Two stages (driven by core/services/persona_distillation_service.py): -- map: summarize_session(trajectory) → per-session digest JSON (lightweight call) -- reduce: distill_persona(digests, ...) → final SKILL.md (single synthesis call) - -Prompts live as independent parts of the 'distillation' kind: - session_digest / colleague_distiller / personal_distiller -DB active version (per part) → filesystem fallback. -""" - -from __future__ import annotations - -import json -import logging -from dataclasses import dataclass -from pathlib import Path -from typing import Any, Dict, List, Optional - -from openai import AsyncOpenAI - -from core.config.distillation import DistillationConfig, get_config -from core.llm._distill_shared import ( - estimate_cost_usd, - parse_strict_json, - resolve_distiller_model, -) -from core.services.distillation_service import Trajectory, trajectory_to_dict - -logger = logging.getLogger(__name__) - -_PROMPT_DIR = Path(__file__).resolve().parents[2] / "prompts" / "prompt_text" / "distillation" - -_DIGEST_KEYS = ("topics", "demonstrated_skills", "decision_points", "style_samples", "tool_sequences") - - -def _load_prompt(part_id: str) -> str: - try: - from core.services import prompt_version_service as pvs - rendered = pvs.render_active_prompt_part("distillation", part_id) - if rendered: - return rendered - except Exception as exc: - logger.debug("persona_distiller: prompt_version_service miss for %s (%s)", part_id, exc) - - fp = _PROMPT_DIR / f"{part_id}.system.md" - try: - return fp.read_text(encoding="utf-8") - except Exception as exc: - logger.error("persona_distiller: failed to load prompt %s (%s)", part_id, exc) - raise RuntimeError(f"persona_distiller: prompt '{part_id}' unavailable") from exc - - -# client is cached by (api_key, base_url, timeout) to reuse the connection pool; auto-swapped on hot model config change -_CLIENT_CACHE: dict[tuple, AsyncOpenAI] = {} - - -def _get_client(api_key: str, base_url: str, timeout: float) -> AsyncOpenAI: - key = (api_key, base_url, timeout) - client = _CLIENT_CACHE.get(key) - if client is None: - client = AsyncOpenAI(api_key=api_key, base_url=base_url, timeout=timeout) - _CLIENT_CACHE.clear() # after a config change the old client is no longer needed, avoid accumulation - _CLIENT_CACHE[key] = client - return client - - -async def _call_llm(system_prompt: str, max_tokens: int, cfg: DistillationConfig) -> tuple[str, int, int, str]: - """One chat call. Returns (content, prompt_tokens, completion_tokens, model_name).""" - model_cfg = resolve_distiller_model() - client = _get_client( - model_cfg.api_key or "DUMMY", - model_cfg.base_url, - float(cfg.llm_timeout_s), - ) - resp = await client.chat.completions.create( - model=model_cfg.model_name, - messages=[ - {"role": "system", "content": system_prompt}, - {"role": "user", "content": "请按规则输出严格 JSON。"}, - ], - temperature=cfg.llm_temperature, - max_tokens=max_tokens, - ) - content = resp.choices[0].message.content or "" - usage = resp.usage - ptok = getattr(usage, "prompt_tokens", 0) if usage else 0 - ctok = getattr(usage, "completion_tokens", 0) if usage else 0 - return content, ptok, ctok, model_cfg.model_name - - -# ─────────────────────── map: per-session digest ──────────────────────── - - -async def summarize_session( - trajectory: Trajectory, - cfg: Optional[DistillationConfig] = None, - hint: str = "", -) -> tuple[Optional[Dict[str, Any]], float]: - """Summarize one session. Returns (digest_dict | None, cost_usd). - - None means the LLM output couldn't be parsed — caller skips the session - but still accounts the cost. Retries once on parse failure (reasoning models - occasionally truncate / drift in format, and a single failure would - silently drop the entire session out of the distillation scope). - """ - cfg = cfg or get_config() - system_prompt = ( - _load_prompt("session_digest") - .replace( - "{trajectory_json}", - json.dumps(trajectory_to_dict(trajectory), ensure_ascii=False, indent=2), - ) - .replace("{hint}", hint or "(无)") - ) - cost = 0.0 - content = "" - parsed: Optional[Dict[str, Any]] = None - for attempt in range(2): - content, ptok, ctok, model_name = await _call_llm( - system_prompt, cfg.persona_digest_max_tokens, cfg - ) - cost += estimate_cost_usd(model_name, ptok, ctok) - parsed = parse_strict_json(content) - if parsed is not None: - break - logger.warning( - "persona_distiller: digest parse failed (attempt %d) chat=%s head=%r", - attempt + 1, trajectory.chat_id, (content or "")[:120], - ) - if parsed is None: - return None, cost - - digest: Dict[str, Any] = {"chat_id": trajectory.chat_id} - for key in _DIGEST_KEYS: - val = parsed.get(key) - digest[key] = [str(x)[:300] for x in val][:5] if isinstance(val, list) else [] - digest["low_value"] = bool(parsed.get("low_value", False)) - return digest, cost - - -# ─────────────────────── reduce: persona synthesis ──────────────────────── - - -@dataclass -class PersonaDistillOutput: - skill: Optional[Dict[str, Any]] # {id, frontmatter, instructions_md} - digest_text: str # persona_digest (colleague) / notes (personal) - confidence: float - raw_response: str - cost_usd: float - error: Optional[str] = None # reason for parse failure - - -async def distill_persona( - kind: str, - digests: List[Dict[str, Any]], - memories_grouped: Optional[Dict[str, Any]], - assigned_identity: str, - hint: str, - sampled_ratio: float, - cfg: Optional[DistillationConfig] = None, -) -> PersonaDistillOutput: - """Reduce stage: digests (+ grouped memories) → SKILL.md. - - kind: 'colleague' | 'personal' — picks the prompt template. - Retries once on JSON-parse failure (costs accumulate). - """ - cfg = cfg or get_config() - part_id = "colleague_distiller" if kind == "colleague" else "personal_distiller" - useful = [d for d in digests if not d.get("low_value")] - - system_prompt = ( - _load_prompt(part_id) - .replace("{assigned_identity}", assigned_identity or "(由你决定)") - .replace("{digest_count}", str(len(useful))) - .replace("{sampled_ratio}", f"{sampled_ratio:.2f}") - .replace("{digests_json}", json.dumps(useful, ensure_ascii=False, indent=1)) - .replace( - "{memories_json}", - json.dumps(memories_grouped or {}, ensure_ascii=False, indent=1) - if memories_grouped else "(无)", - ) - .replace("{hint}", hint or "(无)") - ) - - total_cost = 0.0 - content = "" - parsed: Optional[Dict[str, Any]] = None - for attempt in range(2): - content, ptok, ctok, model_name = await _call_llm( - system_prompt, cfg.persona_reduce_max_tokens, cfg - ) - total_cost += estimate_cost_usd(model_name, ptok, ctok) - parsed = parse_strict_json(content) - if parsed is not None: - break - logger.warning( - "persona_distiller: reduce parse failed (attempt %d) head=%r", - attempt + 1, (content or "")[:200], - ) - - if parsed is None: - return PersonaDistillOutput( - skill=None, - digest_text="", - confidence=0.0, - raw_response=content, - cost_usd=total_cost, - error="failed_parse", - ) - - skill = parsed.get("skill") if isinstance(parsed.get("skill"), dict) else None - try: - confidence = max(0.0, min(1.0, float(parsed.get("confidence") or 0.0))) - except (TypeError, ValueError): - confidence = 0.0 - digest_text = str(parsed.get("persona_digest") or parsed.get("notes") or "")[:1000] - return PersonaDistillOutput( - skill=skill, - digest_text=digest_text, - confidence=confidence, - raw_response=content, - cost_usd=total_cost, - error=None if skill else "missing_skill", - ) diff --git a/src/backend/core/llm/ports.py b/src/backend/core/llm/ports.py new file mode 100644 index 00000000..644f6364 --- /dev/null +++ b/src/backend/core/llm/ports.py @@ -0,0 +1,79 @@ +"""Execution-layer services injected into the model runtime by its host.""" +from __future__ import annotations +from dataclasses import dataclass +from typing import Any, Callable, Protocol + + +class ToolEffectOutcome(Protocol): + result: Any + effect_id: str + result_id: str + result_state: str + invoked: bool + + +class ToolEffectRuntime(Protocol): + async def execute_outcome(self, **kwargs: Any) -> ToolEffectOutcome: ... + def binding_error(self, message: str) -> Exception: ... + def response_error(self, message: str) -> Exception: ... + + +def register_tool_effect_runtime(factory: Callable[..., ToolEffectRuntime]) -> None: + process_state().tool_effect_factory = factory + + +def tool_effect_runtime(session_factory=None, registry=None) -> ToolEffectRuntime: + if process_state().tool_effect_factory is None: + raise RuntimeError("Tool effect execution service is not registered") + return process_state().tool_effect_factory(session_factory, registry) + + +class CommandExecution(Protocol): + async def start(self, provider: Any, request: Any, **kwargs: Any) -> dict: ... + async def cancel(self, task_id: str, user_id: str, session_id: str) -> Any: ... + async def wait(self, task_id: str, user_id: str, session_id: str, timeout_ms: int) -> Any: ... + async def adopt(self, provider: Any, reference: str, **kwargs: Any) -> dict: ... + + +class CommandRecords(Protocol): + def resolve_id(self, reference: str, user_id: str, session_id: str) -> str: ... + def read(self, task_id: str, user_id: str, session_id: str) -> dict: ... + def list_owned(self, user_id: str, chat_id: str) -> list[dict]: ... + + +def register_execution_services(*, commands: CommandExecution, records: CommandRecords, + parent_snapshot: Callable[..., dict]) -> None: + process_state().commands, process_state().command_records, process_state().parent_snapshot = commands, records, parent_snapshot + + +def command_execution() -> CommandExecution: + if process_state().commands is None: + raise RuntimeError("Command execution service is not registered") + return process_state().commands + + +def command_records() -> CommandRecords: + if process_state().command_records is None: + raise RuntimeError("Command record service is not registered") + return process_state().command_records + + +def snapshot_parent_runtime(request: Any, **kwargs: Any) -> dict: + if process_state().parent_snapshot is None: + raise RuntimeError("Parent runtime snapshot service is not registered") + return process_state().parent_snapshot(request, **kwargs) + + +@dataclass +class PortsState: + tool_effect_factory: Callable[..., ToolEffectRuntime] | None = None + commands: CommandExecution | None = None + command_records: CommandRecords | None = None + parent_snapshot: Callable[..., dict] | None = None + + +_STATE = PortsState() + + +def process_state() -> PortsState: + return _STATE diff --git a/src/backend/core/llm/project_environment_middleware.py b/src/backend/core/llm/project_environment_middleware.py deleted file mode 100644 index 5aa2f685..00000000 --- a/src/backend/core/llm/project_environment_middleware.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Recheck project authorization immediately before tool I/O.""" - -import asyncio - -from agentscope.message import TextBlock, ToolResultState -from agentscope.middleware import MiddlewareBase -from agentscope.tool._response import ToolChunk -from core.llm.project_environment import validate_project_environment -from fastapi import HTTPException - - -class ProjectEnvironmentMiddleware(MiddlewareBase): - def __init__(self, request): - self.request = request - - async def on_acting(self, agent, input_kwargs, next_handler): - try: - await asyncio.to_thread(validate_project_environment, self.request) - except HTTPException as error: - yield ToolChunk( - content=[TextBlock(type="text", text=str(error.detail))], - state=ToolResultState.DENIED, - metadata={"project_environment_denied": True}, - ) - return - async for item in next_handler(**input_kwargs): - yield item diff --git a/src/backend/core/llm/prompt_files.py b/src/backend/core/llm/prompt_files.py new file mode 100644 index 00000000..98a79ab5 --- /dev/null +++ b/src/backend/core/llm/prompt_files.py @@ -0,0 +1,138 @@ +"""Read prompt parts from markdown files on disk.""" +from __future__ import annotations +import os +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Dict, List, Optional +from core.llm import prompt_kinds +from core.llm import prompt_storage +from core.llm import prompt_versions_state + +def backend_root() -> Path: + # src/backend + return Path(__file__).resolve().parents[2] + + +def fs_dir(kind: str) -> Path: + spec = prompt_kinds.KIND_SPECS.get(kind) + if spec is None: + raise ValueError(f"unknown kind: {kind}") + if spec.root is not None: + return spec.root + return backend_root().joinpath("prompts", "prompt_text", *spec.subdir) + + +def fs_signature(dirp: Path) -> tuple: + """目录里每个 md 的 (文件名, mtime, 大小)。用它判断要不要重读磁盘。""" + try: + names = sorted( + [p.name for p in dirp.glob("*.system.md")] + + [p.relative_to(dirp).as_posix() for p in dirp.glob("*/system.md")] + ) + except OSError: + return () + sig: List[tuple] = [] + for name in names: + try: + st = os.stat(dirp / name) + except OSError: + continue + sig.append((name, st.st_mtime_ns, st.st_size)) + return tuple(sig) + + +def read_fs_parts(kind: str) -> List[Dict[str, Any]]: + """Read on-disk markdown into a parts[] list. + + 布局由 :data:`KIND_SPECS` 的 ``layout`` 决定（concat / independent / single）。 + concat 版的 sort_order 取文件名的数字前缀（``05_x`` → 5），新增文件落到它本该 + 在的位置，不挤动相邻段。 + + 结果按目录内容签名缓存：这些 md 是只读的部署产物，而本函数落在每轮对话的 + 提示词兜底路径和管理台的每次 GET 上，不做缓存就是反复读同一批文件。 + """ + dirp = fs_dir(kind) + sig = fs_signature(dirp) + with prompt_versions_state.process_state().fs_parts_cache_lock: + cached = prompt_versions_state.process_state().fs_parts_cache.get(kind) + if cached is not None and cached[0] == sig: + return prompt_storage.clone(cached[1]) + + parts = build_fs_parts(kind, dirp, [name for name, _, _ in sig]) + with prompt_versions_state.process_state().fs_parts_cache_lock: + prompt_versions_state.process_state().fs_parts_cache[kind] = (sig, parts) + return prompt_storage.clone(parts) + + +def prompt_part_id(filename: str) -> str: + """Map a flat prompt file or an agent package prompt to its stable part ID.""" + if filename.endswith("/system.md"): + return Path(filename).parent.name + return filename.removesuffix(".system.md") + + +def build_fs_parts(kind: str, dirp: Path, files: List[str]) -> List[Dict[str, Any]]: + layout = prompt_kinds.KIND_SPECS[kind].layout + + if layout == "single": + fname = f"{kind}.system.md" + if fname not in files: + return [] + return [ + { + "part_id": kind, + "display_name": kind, + "content": (dirp / fname).read_text(encoding="utf-8"), + "sort_order": 0, + "is_enabled": True, + } + ] + + if layout == "independent": + # 每个文件是一份独立提示词（按 part_id 单取，见 render_active_prompt_part）， + # 不像 concat 那样拼成一份。skill_distiller 固定排头，兼容旧数据。 + if kind == "distillation": + files = sorted(files, key=lambda f: (f != "skill_distiller.system.md", f)) + return [ + { + "part_id": prompt_part_id(fname), + "display_name": prompt_part_id(fname), + "content": (dirp / fname).read_text(encoding="utf-8"), + "sort_order": idx * 10, + "is_enabled": True, + } + for idx, fname in enumerate(files) + ] + + parts: List[Dict[str, Any]] = [] + for idx, fname in enumerate(files): + name = prompt_part_id(fname) + prefix = name.split("_", 1)[0] + parts.append( + { + "part_id": f"system/{name}", + "display_name": name, + "content": (dirp / fname).read_text(encoding="utf-8"), + "sort_order": int(prefix) if prefix.isdigit() else idx * 10, + "is_enabled": True, + } + ) + return parts + + +def default_version_for_kind(kind: str, version_id: Optional[str] = None) -> Dict[str, Any]: + """Build a new version dict from filesystem for a given kind.""" + spec = prompt_kinds.KIND_SPECS.get(kind) + if spec is None: + raise ValueError(f"unknown kind: {kind}") + + now = datetime.now(timezone.utc).isoformat() + return { + "id": version_id or prompt_versions_state.DEFAULT_VERSION_ID, + "kind": kind, + "name": spec.name, + "description": spec.desc, + "parts": read_fs_parts(kind), + "created_at": now, + "updated_at": now, + } diff --git a/src/backend/core/llm/prompt_kinds.py b/src/backend/core/llm/prompt_kinds.py new file mode 100644 index 00000000..96782bc6 --- /dev/null +++ b/src/backend/core/llm/prompt_kinds.py @@ -0,0 +1,193 @@ +"""Prompt kinds: built-in specs plus admin-defined kinds.""" +from __future__ import annotations +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Dict, List, Optional +from extensions.paths import DEFAULT_AGENTS_DIR +from sqlalchemy.orm import Session +from core.llm import prompt_storage + +@dataclass(frozen=True) +class KindSpec: + """一个内置 kind 的全部差异都集中在这里，别处不再按 kind 分支。 + + ``layout`` 决定目录里的 md 文件怎么变成 parts： + + - ``concat``：目录下每个 ``*.system.md`` 是一段，拼起来构成一份提示词 + - ``independent``：每个文件是一份互不相干的独立提示词，按 part_id 单取 + - ``single``：整个 kind 只有 ``.system.md`` 一个文件 + """ + + subdir: tuple[str, ...] + layout: str + label: str + name: str + desc: str + visible: bool = True + preview: str | None = None + #: 兜底 md 的绝对目录。可选模块的提示词随模块代码放（不进 CE），不在 prompt_text/ 下。 + root: Path | None = None + + +def register_kind(key: str, spec: KindSpec) -> None: + """可选模块注册自己的内置 kind（模块关闭时不注册，Config 台就不出现这个 tab）。 + + 须在启动播种（``seed_from_filesystem``）之前调用，即放在模块的 ``setup()`` 里。 + """ + KIND_SPECS[key] = spec + + +def list_custom_kinds(db: Optional[Session] = None) -> List[Dict[str, str]]: + """管理员自建的 kind（``[{"key","label"}]``）。 + + 存在版本池同一个 ContentBlock 的 ``custom_kinds`` 下——它们和内置 kind 共用 + versions/active 两张表，差别只在于没有文件系统兜底、可以删。 + 「对话模式」就是靠这个扩展的：管理员在提示词管理新开一个 tab，模式那边就能绑它。 + """ + payload = prompt_storage.load_payload(db) + out: List[Dict[str, str]] = [] + for item in payload.get("custom_kinds") or []: + if not isinstance(item, dict): + continue + key = str(item.get("key") or "").strip() + if key and key not in KIND_SPECS: + out.append({"key": key, "label": str(item.get("label") or key)}) + return out + + +def all_kinds(db: Optional[Session] = None) -> List[Dict[str, str]]: + """内置 + 自定义的完整 kind 清单，供管理端渲染 tab 与模式绑定下拉。""" + items = [{"key": k, "label": s.label, "builtin": True, "visible": s.visible, "preview": s.preview, "description": s.desc} for k, s in KIND_SPECS.items()] + items += [{**c, "builtin": False} for c in list_custom_kinds(db)] + return items + + +def is_valid_kind(kind: str, db: Optional[Session] = None) -> bool: + if kind in KIND_SPECS: + return True + return any(c["key"] == kind for c in list_custom_kinds(db)) + + +def create_custom_kind( + key: str, label: str, db: Optional[Session] = None, updated_by: str = "admin" +) -> Dict[str, str]: + """新开一个 kind（tab），并给它建一个空的 ``default`` 版本并激活。 + + 没有初始版本的 kind 在管理端会是个点进去什么都没有的空 tab，所以这里顺手建上， + 管理员进去直接就能写正文。 + """ + cleaned = "".join( + ch if (ch.isalnum() and ch.isascii()) or ch in "-_" else "_" for ch in (key or "").strip().lower() + ).strip("-_") + if not cleaned: + raise ValueError("kind key required") + if cleaned in KIND_SPECS: + raise ValueError(f"「{cleaned}」是内置 kind，不能重复创建") + payload = prompt_storage.clone(prompt_storage.load_payload(db)) + customs = payload.setdefault("custom_kinds", []) + if any(str((c or {}).get("key")) == cleaned for c in customs): + raise ValueError(f"kind「{cleaned}」已存在") + customs.append({"key": cleaned, "label": (label or cleaned).strip()[:60]}) + + # 空的初始版本：一个可编辑的 part，内容留空由管理员填。 + versions = payload.setdefault("versions", []) + now = datetime.now(timezone.utc).isoformat() + versions.append( + { + "kind": cleaned, + "id": "default", + "name": (label or cleaned).strip()[:60], + "description": "", + "parts": [{"part_id": cleaned, "display_name": label or cleaned, "content": ""}], + "created_at": now, + "updated_at": now, + "updated_by": updated_by, + } + ) + payload.setdefault("active", {})[cleaned] = "default" + prompt_storage.save_payload(payload, db=db, updated_by=updated_by) + return {"key": cleaned, "label": (label or cleaned).strip()[:60]} + + +def delete_custom_kind(key: str, db: Optional[Session] = None) -> bool: + """删掉一个自定义 kind 及其全部版本。内置 kind 拒绝删除。 + + 注意：绑了这个 kind 的对话模式会退回默认提示词装配（``render_kind_segment(fs_fallback=False)`` + 取不到就返回空串，装配侧按"没配提示词"处理），不会让对话起不来。 + """ + if key in KIND_SPECS: + raise ValueError("内置 kind 不可删除") + payload = prompt_storage.clone(prompt_storage.load_payload(db)) + customs = payload.get("custom_kinds") or [] + remaining = [c for c in customs if str((c or {}).get("key")) != key] + if len(remaining) == len(customs): + return False + payload["custom_kinds"] = remaining + payload["versions"] = [v for v in (payload.get("versions") or []) if v.get("kind") != key] + (payload.get("active") or {}).pop(key, None) + prompt_storage.save_payload(payload, db=db) + return True + + +KIND_SPECS: Dict[str, KindSpec] = { + "desktop": KindSpec(("desktop",), "independent", "桌面端", "default - 桌面端提示词", "本机环境、文件、项目与工具规则；仅桌面端装配", preview="desktop-preview"), + "system": KindSpec( + ("default", "system"), + "concat", + "系统提示词", + "default - 标准系统提示词", + "当前默认系统提示词，包含 role / constraints / tools / workflow / format", + ), + "turbo": KindSpec( + ("turbo",), + "single", + "极速模式", + "default - 极速模式", + "极速模式（快速查询）系统提示词：仅联网搜索/网页抓取/知识库检索三类工具，" + "1-2 轮并行调用后直接作答", + ), + "plan_mode": KindSpec( + ("plan_mode",), + "single", + "计划模式", + "default - 计划模式", + "Plan 模式下用于拆解用户任务为可执行步骤的 sub-agent 系统提示词", + ), + "plan_tool": KindSpec( + ("plan_tool",), + "single", + "任务计划清单", + "default - 任务计划清单", + "顶层对话注册 update_plan 工具时追加的计划清单说明", + ), + "code_exec": KindSpec( + ("code_exec", "system"), + "concat", + "代码执行", + "default - 代码执行 (沙盒)", + "Lab 代码执行模式的系统提示词（沙盒环境、工具能力、执行规范等）", + ), + "distillation": KindSpec( + ("distillation",), + "independent", + "蒸馏", + "default - 技能蒸馏", + "从对话轨迹蒸馏出可复用技能的系统提示词", + ), + "subagents": KindSpec( + ("subagents",), + "independent", + "平台默认子智能体", + "default - 平台默认子智能体", + "探索员、执行员和审查员三个平台内置角色的独立系统提示词", + visible=False, + root=DEFAULT_AGENTS_DIR, + ), +} + + +BUILTIN_KINDS = tuple(KIND_SPECS) + + +VALID_KINDS = BUILTIN_KINDS diff --git a/src/backend/core/llm/prompt_rendering.py b/src/backend/core/llm/prompt_rendering.py new file mode 100644 index 00000000..61c00d0d --- /dev/null +++ b/src/backend/core/llm/prompt_rendering.py @@ -0,0 +1,137 @@ +"""Render the active prompt version, or the file defaults, into text.""" +from __future__ import annotations +from core.llm import prompt_versions +from typing import Any, Dict, List, Optional +from sqlalchemy.orm import Session +import logging +from core.llm import prompt_files +from core.llm import prompt_kinds +from core.llm import prompt_versions_state + +logger = logging.getLogger(__name__) + +def render_active_prompt(kind: str, db: Optional[Session] = None) -> Optional[str]: + """Concatenate enabled parts of the active version into a single string. + + Returns None if no active version exists in DB (caller should fall back). + """ + v = prompt_versions.get_active_version(kind, db=db) + if not v: + return None + parts = v.get("parts") or [] + chunks: List[str] = [] + for p in parts: + if not p.get("is_enabled", True): + continue + content = (p.get("content") or "").strip() + if content: + chunks.append(content) + return "\n\n".join(chunks) if chunks else None + + +def render_active_prompt_part( + kind: str, part_id: str, db: Optional[Session] = None +) -> Optional[str]: + """Return a single named part of the active version (independent prompts). + + Used by distillation and platform-subagent prompts where each part is a standalone + system prompt (skill_distiller / session_digest / colleague_distiller / + personal_distiller) rather than a concatenation segment. + + Back-compat: an active version that predates the multi-part layout holds + exactly one part (the legacy skill_distiller prompt) — only a request for + ``skill_distiller`` may fall back to it; other part_ids must not silently + receive the wrong prompt. + Returns None if no active version / no match (caller falls back to fs). + """ + v = prompt_versions.get_active_version(kind, db=db) + if not v: + return None + parts = v.get("parts") or [] + for p in parts: + if p.get("part_id") == part_id: + content = (p.get("content") or "").strip() + return content or None + if len(parts) == 1 and part_id == "skill_distiller": + content = (parts[0].get("content") or "").strip() + return content or None + return None + + +def skipped_parts(kind: str) -> set: + try: + from core.config.local_mode import local_mode_enabled + + if local_mode_enabled(): + from prompts.desktop_workspace import SKIP_PARTS + return SKIP_PARTS.get(kind) or set() + except Exception: + logger.debug("local-mode check failed while assembling %s", kind, exc_info=True) + return set() + + +def render_kind_segment( + kind: str, db: Optional[Session] = None, *, fs_fallback: bool = True +) -> str: + """某个 kind 装配好的正文：DB 激活版本 → 文件系统默认，都没有就返回空串。 + + 这是「一个 kind 的正文怎么取」的唯一实现——代码执行段、极速模式、任务计划清单 + 以及 Config 管理台 ``/v1/admin/prompts/preview`` 的预览都走它，预览与智能体 + 实际看到的因此不会漂移。 + + 文件系统这一步读的就是播种用的那批 md，所以尚未播种的部署渲染出来的，与它之后 + 存进库里的是同一份。``fs_fallback=False`` 用于「对话模式」绑定的 kind：绑了个 + 空 kind 就该退回默认装配，而不是套用别的 kind 的话术。 + """ + skip = skipped_parts(kind) + try: + rendered = render_active_parts(kind, db=db, skip=skip) + if rendered: + return rendered + except Exception: + logger.debug("render %s active prompt failed", kind, exc_info=True) + if not fs_fallback or kind not in prompt_kinds.KIND_SPECS: + return "" + chunks = [ + (p.get("content") or "").strip() + for p in prompt_files.read_fs_parts(kind) + if p.get("is_enabled", True) + and (p.get("content") or "").strip() + and (p.get("part_id") or "") not in skip + ] + return "\n\n".join(chunks) + + +def render_active_parts(kind: str, *, db: Optional[Session], skip: set) -> Optional[str]: + """激活版本的正文，按 ``skip`` 剔除若干段。``skip`` 为空时等同 render_active_prompt。""" + if not skip: + return render_active_prompt(kind, db=db) + version = prompt_versions.get_active_version(kind, db=db) + if not version: + return None + chunks = [ + (p.get("content") or "").strip() + for p in (version.get("parts") or []) + if p.get("is_enabled", True) + and (p.get("content") or "").strip() + and (p.get("part_id") or "").strip() not in skip + ] + return "\n\n".join(chunks) or None + + +def render_turbo_system_prompt(db: Optional[Session] = None) -> str: + """极速模式（快速查询）的整份系统提示词。 + + 极速模式不做常规装配：智能体只带检索类工具，默认提示词的工具/流程段都不适用。 + """ + return render_kind_segment("turbo", db=db) or prompt_versions_state.TURBO_LAST_RESORT + + +def effective_parts(kind: str, db: Optional[Session] = None) -> List[Dict[str, Any]]: + """Read the active version, with whole-version filesystem fallback only.""" + try: + version = prompt_versions.get_active_version(kind, db=db) + except Exception: + logger.warning("Prompt version unavailable for %s; using filesystem", kind, exc_info=True) + version = None + return version.get("parts", []) if version is not None else prompt_files.read_fs_parts(kind) diff --git a/src/backend/core/llm/prompt_storage.py b/src/backend/core/llm/prompt_storage.py new file mode 100644 index 00000000..c646de50 --- /dev/null +++ b/src/backend/core/llm/prompt_storage.py @@ -0,0 +1,99 @@ +"""Load and cache the stored prompt pool for this worker.""" +from __future__ import annotations +from datetime import datetime, timezone +from typing import Any, Dict, Optional +from core.files.content.content_blocks_constants import DEFAULT_PROMPT_VERSIONS, PROMPT_VERSIONS_BLOCK_ID +from core.db.engine import SessionLocal +from core.db.models import ContentBlock +from sqlalchemy.orm import Session +from core.llm import prompt_versions_state + +def load_payload(db: Optional[Session] = None) -> Dict[str, Any]: + """Read the committed pool revision before reusing a worker-local payload. + + Workers do not share Python memory. Local invalidation alone leaves other + workers serving old versions indefinitely, including on read-modify-write. + Query columns directly so an existing ORM identity-map entry cannot hide a + newer payload. The payload and its revision are loaded in the same query. + """ + pass # State is owned by its responsibility module. + own_session = db is None + if own_session: + db = SessionLocal() + try: + revision = ( + db.query(ContentBlock.updated_at) + .filter(ContentBlock.id == PROMPT_VERSIONS_BLOCK_ID) + .first() + ) + cache_key = (db.get_bind(), tuple(revision) if revision is not None else None) + with prompt_versions_state.process_state().payload_cache_lock: + if prompt_versions_state.process_state().payload_cache is not None and prompt_versions_state.process_state().payload_cache_key == cache_key: + return clone(prompt_versions_state.process_state().payload_cache) + + row = ( + db.query(ContentBlock.updated_at, ContentBlock.payload) + .filter(ContentBlock.id == PROMPT_VERSIONS_BLOCK_ID) + .first() + ) + payload = ( + row.payload + if row is not None and isinstance(row.payload, dict) + else clone(DEFAULT_PROMPT_VERSIONS) + ) + # A write may have committed between the two SELECTs. Cache the revision + # returned with the payload, never the earlier revision probe. + cache_key = (db.get_bind(), (row.updated_at,) if row is not None else None) + with prompt_versions_state.process_state().payload_cache_lock: + prompt_versions_state.process_state().payload_cache = payload + prompt_versions_state.process_state().payload_cache_key = cache_key + return clone(payload) + finally: + if own_session: + db.close() + + +def save_payload( + payload: Dict[str, Any], + *, + db: Optional[Session] = None, + updated_by: str = "system", +) -> None: + own_session = db is None + if own_session: + db = SessionLocal() + try: + row = db.query(ContentBlock).filter(ContentBlock.id == PROMPT_VERSIONS_BLOCK_ID).first() + now = datetime.now(timezone.utc) + if row: + row.payload = payload + row.updated_at = now + row.updated_by = updated_by + else: + row = ContentBlock( + id=PROMPT_VERSIONS_BLOCK_ID, + payload=payload, + updated_at=now, + updated_by=updated_by, + ) + db.add(row) + db.commit() + finally: + if own_session: + db.close() + + invalidate_cache() + + +def invalidate_cache() -> None: + """Drop this worker's cache; other workers detect the DB revision on read.""" + pass # State is owned by its responsibility module. + with prompt_versions_state.process_state().payload_cache_lock: + prompt_versions_state.process_state().payload_cache = None + prompt_versions_state.process_state().payload_cache_key = None + + +def clone(obj: Any) -> Any: + import copy + + return copy.deepcopy(obj) diff --git a/src/backend/core/llm/prompt_versions.py b/src/backend/core/llm/prompt_versions.py new file mode 100644 index 00000000..86242df3 --- /dev/null +++ b/src/backend/core/llm/prompt_versions.py @@ -0,0 +1,296 @@ +"""Create, activate and seed prompt versions.""" +from __future__ import annotations +from datetime import datetime, timezone +from typing import Any, Dict, List, Optional +from sqlalchemy.orm import Session +import logging +from core.llm import prompt_files +from core.llm import prompt_kinds +from core.llm import prompt_storage +from core.llm import prompt_versions_state + +logger = logging.getLogger(__name__) + +def list_versions(kind: Optional[str] = None, db: Optional[Session] = None) -> List[Dict[str, Any]]: + payload = prompt_storage.load_payload(db) + versions = payload.get("versions") or [] + active = payload.get("active") or {} + items = [] + for v in versions: + if kind and v.get("kind") != kind: + continue + items.append( + { + "id": v.get("id"), + "kind": v.get("kind"), + "name": v.get("name") or v.get("id"), + "description": v.get("description") or "", + "parts_count": len(v.get("parts") or []), + "is_active": active.get(v.get("kind")) == v.get("id"), + "created_at": v.get("created_at"), + "updated_at": v.get("updated_at"), + } + ) + return items + + +def get_version( + kind: str, version_id: str, db: Optional[Session] = None +) -> Optional[Dict[str, Any]]: + if not prompt_kinds.is_valid_kind(kind, db): + return None + payload = prompt_storage.load_payload(db) + active = payload.get("active") or {} + for v in payload.get("versions") or []: + if v.get("kind") == kind and v.get("id") == version_id: + data = prompt_storage.clone(v) + data["is_active"] = active.get(kind) == version_id + return data + return None + + +def get_active_version(kind: str, db: Optional[Session] = None) -> Optional[Dict[str, Any]]: + """Return the active version for a kind, or None if not found / DB empty.""" + if not prompt_kinds.is_valid_kind(kind, db): + return None + payload = prompt_storage.load_payload(db) + active_id = (payload.get("active") or {}).get(kind) + if not active_id: + return None + for v in payload.get("versions") or []: + if v.get("kind") == kind and v.get("id") == active_id: + return prompt_storage.clone(v) + return None + + +def upsert_version( + kind: str, + version_id: str, + *, + name: Optional[str] = None, + description: Optional[str] = None, + parts: Optional[List[Dict[str, Any]]] = None, + from_id: Optional[str] = None, + db: Optional[Session] = None, + updated_by: str = "admin", +) -> Dict[str, Any]: + if not prompt_kinds.is_valid_kind(kind, db): + raise ValueError(f"invalid kind: {kind}") + if not version_id: + raise ValueError("version_id required") + + payload = prompt_storage.clone(prompt_storage.load_payload(db)) + versions: List[Dict[str, Any]] = payload.setdefault("versions", []) + now = datetime.now(timezone.utc).isoformat() + + existing = next( + (v for v in versions if v.get("kind") == kind and v.get("id") == version_id), None + ) + + # If cloning, look up source (optional) + source_parts: List[Dict[str, Any]] = [] + if from_id: + src = next((v for v in versions if v.get("kind") == kind and v.get("id") == from_id), None) + if src: + source_parts = prompt_storage.clone(src.get("parts") or []) + if name is None: + name = f"{src.get('name') or from_id} 副本" + if description is None: + description = src.get("description") or "" + + if existing: + if name is not None: + existing["name"] = name + if description is not None: + existing["description"] = description + if parts is not None: + existing["parts"] = normalize_parts(parts) + existing["updated_at"] = now + saved = existing + else: + # On create: if from_id is passed and parts is empty (empty list / not passed), + # always treat it as a clone. + # Only explicitly passing non-empty parts overrides the clone's source content. + if parts and len(parts) > 0: + final_parts = normalize_parts(parts) + elif from_id and source_parts: + final_parts = source_parts + else: + final_parts = normalize_parts(parts) if parts is not None else [] + saved = { + "id": version_id, + "kind": kind, + "name": name or version_id, + "description": description or "", + "parts": final_parts, + "created_at": now, + "updated_at": now, + } + versions.append(saved) + + prompt_storage.save_payload(payload, db=db, updated_by=updated_by) + return saved + + +def delete_version(kind: str, version_id: str, db: Optional[Session] = None) -> None: + if not prompt_kinds.is_valid_kind(kind, db): + raise ValueError(f"invalid kind: {kind}") + payload = prompt_storage.clone(prompt_storage.load_payload(db)) + active = payload.get("active") or {} + if active.get(kind) == version_id: + raise ValueError("cannot delete the currently active version; activate another first") + + versions: List[Dict[str, Any]] = payload.get("versions") or [] + new_list = [v for v in versions if not (v.get("kind") == kind and v.get("id") == version_id)] + if len(new_list) == len(versions): + raise KeyError(f"version not found: {kind}/{version_id}") + payload["versions"] = new_list + prompt_storage.save_payload(payload, db=db) + + +def activate_version(kind: str, version_id: str, db: Optional[Session] = None) -> None: + if not prompt_kinds.is_valid_kind(kind, db): + raise ValueError(f"invalid kind: {kind}") + payload = prompt_storage.clone(prompt_storage.load_payload(db)) + versions: List[Dict[str, Any]] = payload.get("versions") or [] + if not any(v.get("kind") == kind and v.get("id") == version_id for v in versions): + raise KeyError(f"version not found: {kind}/{version_id}") + payload.setdefault("active", {})[kind] = version_id + prompt_storage.save_payload(payload, db=db) + # Downstream prompt builders cache by (active_id, updated_at); poke their cache too. + try: + from prompts.prompt_runtime import invalidate_prompt_cache + + invalidate_prompt_cache() + except Exception as fallback_error: + from core.infra.failures import record_best_effort_failure + record_best_effort_failure('core.llm.prompt_versions:activate_version', fallback_error) + + +def normalize_parts(parts: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + result: List[Dict[str, Any]] = [] + for idx, p in enumerate(parts or []): + pid = (p.get("part_id") or "").strip() + if not pid: + continue + result.append( + { + "part_id": pid, + "display_name": p.get("display_name") or pid.split("/")[-1], + "content": p.get("content") or "", + "sort_order": int( + p.get("sort_order") if p.get("sort_order") is not None else idx * 10 + ), + "is_enabled": bool(p.get("is_enabled", True)), + } + ) + # Stable sort by sort_order + result.sort(key=lambda x: x["sort_order"]) + return result + + +def seed_from_filesystem( + *, + db: Optional[Session] = None, +) -> Dict[str, Any]: + """Create default versions from filesystem if not already present.""" + payload = prompt_storage.clone(prompt_storage.load_payload(db)) + versions: List[Dict[str, Any]] = payload.setdefault("versions", []) + active: Dict[str, str] = payload.setdefault("active", {}) + changed = False + + def exists(kind: str, vid: str) -> bool: + return any(v.get("kind") == kind and v.get("id") == vid for v in versions) + + added: List[str] = [] + + # ── One-time migration: rename legacy system/v4 → system/default ── + # Keeps the user's existing edits; the name change aligns the in-code + # default version id with other kinds (code_exec/default, distillation/default, …). + v4_row = next((v for v in versions if v.get("kind") == "system" and v.get("id") == "v4"), None) + default_row = next( + (v for v in versions if v.get("kind") == "system" and v.get("id") == "default"), None + ) + if v4_row and not default_row: + v4_row["id"] = "default" + # Refresh display name only if it still looks like the factory label + if v4_row.get("name", "").startswith("v4 - 标准版"): + v4_row["name"] = "default - 标准系统提示词" + if active.get("system") == "v4": + active["system"] = "default" + added.append("system/v4 → system/default (renamed)") + changed = True + + # ── One-time migration: extract system/90_plan_mode from every system + # version into a new plan_mode/default version (if not already seeded). + # After extraction, the system/90_plan_mode part is REMOVED from system + # versions so the main agent prompt no longer duplicates it. + if not exists("plan_mode", "default"): + extracted_content: Optional[str] = None + for v in versions: + if v.get("kind") != "system": + continue + new_parts: List[Dict[str, Any]] = [] + for p in v.get("parts") or []: + if (p.get("part_id") or "").strip() == "system/90_plan_mode": + if extracted_content is None and (p.get("content") or "").strip(): + extracted_content = p["content"] + changed = True + continue + new_parts.append(p) + v["parts"] = new_parts + if extracted_content: + now = datetime.now(timezone.utc).isoformat() + versions.append( + { + "id": "default", + "kind": "plan_mode", + "name": "default - 计划模式（从 v4 迁移）", + "description": "由历史 v4 system/90_plan_mode 片段迁移而来的 plan_mode 默认版本", + "parts": [ + { + "part_id": "plan_mode", + "display_name": "plan_mode", + "content": extracted_content, + "sort_order": 0, + "is_enabled": True, + } + ], + "created_at": now, + "updated_at": now, + } + ) + active["plan_mode"] = "default" + added.append("plan_mode/default (migrated)") + changed = True + + # 每个内置 kind 的默认版本：没有就按文件系统建；已有就把**新增**的 md 文件补进去。 + # 补这一步是因为默认版本只在首次冷启动时从文件系统建一次，之后新加的 *.system.md + # 再也进不了已建库的部署。已有 part 的正文原样保留，管理员改过的不会被覆盖。 + for kind in list(prompt_kinds.KIND_SPECS): + if not exists(kind, prompt_versions_state.DEFAULT_VERSION_ID): + default_row = prompt_files.default_version_for_kind(kind, prompt_versions_state.DEFAULT_VERSION_ID) + versions.append(default_row) + added.append(f"{kind}/{prompt_versions_state.DEFAULT_VERSION_ID}") + changed = True + else: + default_row = next( + v for v in versions if v.get("kind") == kind and v.get("id") == prompt_versions_state.DEFAULT_VERSION_ID + ) + existing_pids = { + (p.get("part_id") or "").strip() for p in default_row.get("parts") or [] + } + for fs_part in prompt_files.read_fs_parts(kind): + if fs_part["part_id"] not in existing_pids: + default_row.setdefault("parts", []).append(fs_part) + added.append(f"{kind}/{prompt_versions_state.DEFAULT_VERSION_ID}:{fs_part['part_id']}") + changed = True + if kind not in active: + active[kind] = prompt_versions_state.DEFAULT_VERSION_ID + changed = True + + if added or changed: + prompt_storage.save_payload(payload, db=db, updated_by="system_seed") + if added: + logger.info("[prompt_version_service] seeded: %s", ", ".join(added)) + return {"added": added, "active": active} diff --git a/src/backend/core/llm/prompt_versions_state.py b/src/backend/core/llm/prompt_versions_state.py new file mode 100644 index 00000000..3a529ffa --- /dev/null +++ b/src/backend/core/llm/prompt_versions_state.py @@ -0,0 +1,30 @@ +"""Default version id and caches for prompt versions.""" +from __future__ import annotations +from dataclasses import dataclass, field +from threading import Lock +from typing import Any, Dict, Optional + +DEFAULT_VERSION_ID = "default" + + +TURBO_LAST_RESORT = ( + "你是极速查询助手。收到问题后，最多发起 1-2 轮工具调用（同一轮内可并行" + "调用多个检索工具），拿到结果立即用中文简洁作答，并标注信息来源。" + "不要使用超出检索范围的工具，不要反复迭代。" +) + + +@dataclass +class PromptVersionsState: + payload_cache: Optional[Dict[str, Any]] = None + payload_cache_key: Optional[tuple] = None + payload_cache_lock: object = field(default_factory=lambda: Lock()) + fs_parts_cache: Dict[str, tuple] = field(default_factory=lambda: {}) + fs_parts_cache_lock: object = field(default_factory=lambda: Lock()) + + +_STATE = PromptVersionsState() + + +def process_state() -> PromptVersionsState: + return _STATE diff --git a/src/backend/core/llm/providers/__init__.py b/src/backend/core/llm/providers/__init__.py deleted file mode 100644 index be52242c..00000000 --- a/src/backend/core/llm/providers/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Multi-vendor model provider support: registry + per-engine model implementations.""" - -from .registry import ( # noqa: F401 - PROVIDER_SPECS, - ProviderField, - ProviderSpec, - get_spec, - is_known, - list_specs, - split_provider_extra, - to_frontend_schema, - validate_payload, -) diff --git a/src/backend/core/llm/runtime/__init__.py b/src/backend/core/llm/runtime/__init__.py new file mode 100644 index 00000000..f2f8736a --- /dev/null +++ b/src/backend/core/llm/runtime/__init__.py @@ -0,0 +1 @@ +"""The AgentScope execution boundary: streaming, hooks, state and tool adapters.""" diff --git a/src/backend/core/llm/agentscope_hook_adapter.py b/src/backend/core/llm/runtime/agentscope_hook_adapter.py similarity index 63% rename from src/backend/core/llm/agentscope_hook_adapter.py rename to src/backend/core/llm/runtime/agentscope_hook_adapter.py index 014413cf..25576bac 100644 --- a/src/backend/core/llm/agentscope_hook_adapter.py +++ b/src/backend/core/llm/runtime/agentscope_hook_adapter.py @@ -2,21 +2,18 @@ from __future__ import annotations -import asyncio import inspect -import json -import time from typing import Any from agentscope.message import Msg from agentscope.middleware import MiddlewareBase -from agentscope.tool._response import ToolResponse from core.harness.events import Event, EventSink, thaw_value from core.harness.hooks import HookBus, HookStage, Invocation -from core.harness.usage import UsageAttempt, attempt_status_for_exception, record_usage_safely -from core.llm.model_steps import record_assistant_step -from core.llm.reasoning_replay import stamp_reasoning_origin -from core.services.harness_ledger import DurableEventStore, HarnessUsageLedger +from core.llm.runtime.model_steps import record_assistant_step +from core.models.reasoning_replay import stamp_reasoning_origin +from core.llm.runtime.context_hook_adapter import attachment_input, begin_context_turn, append_hook_context, apply_reasoning_hooks +from core.llm.runtime.model_selection_adapter import apply_model_selection, model_selection_input +from core.harness.ledger import DurableEventStore, HarnessUsageLedger def _plain(value: Any) -> Any: @@ -159,123 +156,13 @@ def __init__( self, bus: HookBus | None = None, *, - legacy_middlewares: tuple[MiddlewareBase, ...] = (), + permission_policy=None, + durable_tools=False, ) -> None: self.bus = bus or build_hook_bus() - # Transitional policies execute only through this compatibility - # boundary. They are no longer mounted beside the adapter in Agent. - self.legacy_middlewares = tuple(legacy_middlewares) - - def _legacy_for(self, hook_name: str) -> tuple[MiddlewareBase, ...]: - return tuple( - middleware - for middleware in self.legacy_middlewares - if middleware.is_implemented(hook_name) - ) - - async def _record_legacy_hook( - self, - *, - run_id: str, - middleware: MiddlewareBase, - hook_name: str, - status: str, - started: float, - ) -> None: - if not run_id or self.bus.usage_recorder is None: - return - await record_usage_safely( - self.bus.usage_recorder, - UsageAttempt( - run_id=run_id, - kind="hook", - operation_name=f"{type(middleware).__name__}.{hook_name}", - status=status, - latency_ms=int((time.monotonic() - started) * 1_000), - metadata={"adapter": "agentscope", "legacy": True}, - ), - ) - - async def _legacy_stream( - self, - hook_name: str, - agent: Any, - input_kwargs: dict, - next_handler, - ): - middlewares = self._legacy_for(hook_name) - - async def execute(index: int, **kwargs): - if index >= len(middlewares): - async for item in next_handler(**kwargs): - yield item - return - middleware = middlewares[index] - - async def following(**next_kwargs): - async for item in execute(index + 1, **next_kwargs): - yield item - - callback = getattr(middleware, hook_name) - started = time.monotonic() - status = "success" - try: - async for item in callback(agent, kwargs, following): - yield item - except asyncio.CancelledError: - status = "cancelled" - raise - except Exception as exc: - status = attempt_status_for_exception(exc) - raise - finally: - await self._record_legacy_hook( - run_id=self._run_id(agent), - middleware=middleware, - hook_name=hook_name, - status=status, - started=started, - ) - - async for item in execute(0, **input_kwargs): - yield item - - async def _legacy_model_call( - self, - agent: Any, - input_kwargs: dict, - next_handler, - ): - middlewares = self._legacy_for("on_model_call") - - async def execute(index: int, **kwargs): - if index >= len(middlewares): - return await next_handler(**kwargs) - middleware = middlewares[index] - - async def following(**next_kwargs): - return await execute(index + 1, **next_kwargs) - - started = time.monotonic() - status = "success" - try: - return await middleware.on_model_call(agent, kwargs, following) - except asyncio.CancelledError: - status = "cancelled" - raise - except Exception as exc: - status = attempt_status_for_exception(exc) - raise - finally: - await self._record_legacy_hook( - run_id=self._run_id(agent), - middleware=middleware, - hook_name="on_model_call", - status=status, - started=started, - ) - - return await execute(0, **input_kwargs) + self.permission_policy = permission_policy + from core.llm.runtime.tool_execution_adapter import DurableToolDispatch + self.tool_dispatch = DurableToolDispatch() if durable_tools else None @staticmethod def _run_id(agent: Any) -> str: @@ -302,7 +189,7 @@ async def _observe_event(self, event: Any, run_id: str) -> None: invocation = adapt_agentscope_event(event, run_id=run_id) if invocation is None: return - await self.bus.event_sink.append( + await self.bus.event_sink.publish( Event.create( run_id=run_id, event_type="framework_invocation", @@ -316,21 +203,25 @@ async def _observe_event(self, event: Any, run_id: str) -> None: ) async def on_reply(self, agent: Any, input_kwargs: dict, next_handler): - from core.llm.model_usage import model_usage_scope + from core.models.usage import model_usage_scope run_id = self._run_id(agent) with model_usage_scope(run_id, self.bus.usage_recorder): - await self._stage( + selected = await self._stage( HookStage.BEFORE_RUN, agent, operation_name="agent_reply", - data={"metadata": {"framework": "agentscope"}}, + data={"metadata": {"framework": "agentscope"}, "runtime": model_selection_input(agent)}, ) + apply_model_selection(agent, selected) + begin_context_turn() transformed = await self._stage( HookStage.TRANSFORM_CONTEXT, agent, operation_name="agent_reply", data={ + "runtime": _plain(attachment_input(agent)), + "context_items": [], "messages": _plain(input_kwargs.get("inputs")), "context": _plain( getattr(getattr(agent, "state", None), "context", None) @@ -350,33 +241,31 @@ async def on_reply(self, agent: Any, input_kwargs: dict, next_handler): state.context = _restore_context( getattr(state, "context", None), transformed.data["context"] ) - final_output = None - async for event in self._legacy_stream( - "on_reply", agent, invoke_kwargs, next_handler - ): - await self._observe_event(event, run_id) - if isinstance(event, Msg): - if final_output is not None: - yield final_output - final_output = event - else: - yield event - await self._stage( - HookStage.BEFORE_FINISH, - agent, - operation_name="agent_reply", - data={"output": _plain(final_output), "metadata": {}}, - ) - if final_output is not None: - yield final_output + append_hook_context(agent, transformed) + from core.llm.runtime.context_hook_adapter import finish_context_hooks + while True: + final_output = None + async for event in next_handler(**invoke_kwargs): + await self._observe_event(event, run_id) + if isinstance(event, Msg): + if final_output is not None: + yield final_output + final_output = event + else: + yield event + continue_run = await finish_context_hooks(self, agent, final_output, _plain) + if final_output is not None: + yield final_output + if not continue_run: + break + invoke_kwargs = {**invoke_kwargs, "inputs": None} async def on_reasoning(self, agent: Any, input_kwargs: dict, next_handler): - from core.llm.model_usage import model_usage_scope + from core.models.usage import model_usage_scope with model_usage_scope(self._run_id(agent), self.bus.usage_recorder): - async for event in self._legacy_stream( - "on_reasoning", agent, input_kwargs, next_handler - ): + input_kwargs = await apply_reasoning_hooks(self, agent, input_kwargs, _plain, _restore_context) + async for event in next_handler(**input_kwargs): yield event @staticmethod @@ -423,7 +312,7 @@ async def on_model_call(self, agent: Any, input_kwargs: dict, next_handler): "parameters": parameters, }, ) - response = await self._legacy_model_call(agent, input_kwargs, next_handler) + response = await next_handler(**input_kwargs) if not inspect.isasyncgen(response): await self._stage( HookStage.AFTER_MODEL, @@ -453,32 +342,8 @@ async def patched_stream(): return patched_stream() async def on_acting(self, agent: Any, input_kwargs: dict, next_handler): - tool_call = input_kwargs.get("tool_call") - raw_args = getattr(tool_call, "input", "") or "{}" - try: - args = json.loads(raw_args) if isinstance(raw_args, str) else raw_args - except (TypeError, json.JSONDecodeError): - args = {"_raw": str(raw_args)} - name = str(getattr(tool_call, "name", "") or "tool") - await self._stage( - HookStage.BEFORE_TOOL, - agent, - operation_name=name, - data={"tool_args": args, "metadata": {}}, - ) - final = None - async for event in self._legacy_stream( - "on_acting", agent, input_kwargs, next_handler - ): - if isinstance(event, ToolResponse): - final = event - else: + from contextlib import aclosing + from core.llm.runtime.tool_result_adapter import run_tool + async with aclosing(run_tool(self, agent, input_kwargs, next_handler, _plain)) as stream: + async for event in stream: yield event - await self._stage( - HookStage.AFTER_TOOL, - agent, - operation_name=name, - data={"tool_result": _plain(final), "metadata": {}}, - ) - if final is not None: - yield final diff --git a/src/backend/core/llm/runtime/context_hook_adapter.py b/src/backend/core/llm/runtime/context_hook_adapter.py new file mode 100644 index 00000000..9989a268 --- /dev/null +++ b/src/backend/core/llm/runtime/context_hook_adapter.py @@ -0,0 +1,80 @@ +"""Translate neutral context additions at the framework boundary.""" +from core.harness.events import thaw_value +from core.llm.context.items import ContextItem +from core.llm.context.ir_constants import POLICY_HEAD_TAIL +from core.llm.context.rendering import append_state_context_item, next_context_sequence +from core.llm.runtime.turn_state import reset_artifact_read_state, reset_pin_hint_state + + +def attachment_input(agent): + state = getattr(agent, "state", None) + return {name: getattr(state, name, default) for name, default in ( + ("uploaded_files", []), ("historical_files", []), ("user_id", ""), + ("model_provider_id", ""), ("vision_evidence_text", ""), + )} + + +def begin_context_turn(): + reset_artifact_read_state() + reset_pin_hint_state() + + +def append_hook_context(agent, invocation): + for item in invocation.data.get("context_items", ()): + item = thaw_value(item) + sequence = next_context_sequence(agent.state.context) + origin = item["origin"] + append_state_context_item(agent.state, ContextItem.create( + item_id=f"{origin}:{sequence}", kind=item["kind"], origin=origin, + trust=item["trust"], visibility="model", priority=item["priority"], + token_budget=item["token_budget"], truncation_policy=item.get("truncation_policy", POLICY_HEAD_TAIL), + content=item["content"], cache_class="dynamic", created_seq=sequence, + render_role="user", message_group=f"{origin}:{sequence}", + )) + + +async def apply_reasoning_hooks(adapter, agent, input_kwargs, plain, restore_context): + from agentscope.tool import ToolChoice + from core.harness.hooks import HookStage + from core.llm.runtime.workspace import get_pinned_file_ids + state = agent.state + transformed = await adapter._stage( + HookStage.TRANSFORM_CONTEXT, agent, operation_name="agent_reasoning", + data={"identity": {"chat_id": getattr(state, "chat_id", None), "user_id": getattr(state, "user_id", None)}, + "context": plain(state.context), "context_items": [], + "tool_choice": plain(input_kwargs.get("tool_choice")), + "runtime": {"pinned_file_ids": list(get_pinned_file_ids()), "cur_iter": getattr(state, "cur_iter", 0), + "reply_id": getattr(state, "reply_id", ""), + "max_iters": getattr(getattr(agent, "react_config", None), "max_iters", 0)}}, + ) + if transformed.data.get("context") is not None: + state.context = restore_context(state.context, transformed.data["context"]) + append_hook_context(agent, transformed) + await adapter._stage(HookStage.AFTER_CONTEXT, agent, operation_name="agent_reasoning", + data={"steer_delivery": transformed.data.get("steer_delivery")}) + if transformed.data.get("steer_delivery") is not None: + from core.harness.events import thaw_value + state.steer_delivery = thaw_value(transformed.data["steer_delivery"]) + choice = transformed.data.get("tool_choice") + if choice is not None: + from core.harness.events import thaw_value + return {**input_kwargs, "tool_choice": ToolChoice.model_validate(thaw_value(choice))} + return input_kwargs + + +async def finish_context_hooks(adapter, agent, output, plain): + from core.harness.hooks import HookStage + from core.llm.runtime.workspace import get_pinned_file_ids + result = await adapter._stage(HookStage.BEFORE_FINISH, agent, operation_name="agent_reply", + data={"output": plain(output), "metadata": {}, + "identity": {"chat_id": getattr(agent.state, "chat_id", None), "user_id": getattr(agent.state, "user_id", None)}, + "runtime": {"pinned_file_ids": list(get_pinned_file_ids()), + "finish_pins_completed": getattr(adapter, "finish_pins_completed", False)}}) + if result.data.get("continue_run"): + return True + pins = result.data.get("pin_file_ids") or () + if pins: + from core.llm.runtime.finish_guard import pin_files + if pin_files(list(pins)) > 0: + adapter.finish_pins_completed = True + return False diff --git a/src/backend/core/llm/runtime/file_context.py b/src/backend/core/llm/runtime/file_context.py new file mode 100644 index 00000000..8f0c26fe --- /dev/null +++ b/src/backend/core/llm/runtime/file_context.py @@ -0,0 +1,277 @@ +"""Turn file attachments into context: text, images and spreadsheet previews.""" +from __future__ import annotations +from core.files import artifact_bytes +import base64 +from typing import Any, Dict, List, Optional +import logging +logger = logging.getLogger(__name__) + +def fetch_image_base64( + f: Dict[str, Any], + user_id: Optional[str] = None, +) -> Optional[tuple[str, str]]: + """Fetch image data for a file attachment and return (base64_data, mime_type).""" + file_id = f.get("file_id") or "" + if not file_id: + return None + mime_type = (f.get("mime_type") or "image/png").lower() + raw = artifact_bytes.download_artifact_bytes( + file_id, + f.get("name") or "image", + "Image hook", + user_id=user_id, + ) + if raw is None: + return None + return base64.b64encode(raw).decode("utf-8"), mime_type + + +def is_xlsx(f: Dict[str, Any]) -> bool: + """Detect xlsx by mime_type or filename extension. + + We only special-case xlsx (not xls/csv) because: + - xlsx is the dominant batch-execution upload format + - parse_xlsx_preview uses openpyxl, same engine as the batch path + - xls would need a separate xlrd-based preview helper (out of scope) + """ + mime = (f.get("mime_type") or "").lower() + if "spreadsheetml.sheet" in mime: + return True + name = (f.get("name") or f.get("filename") or "").lower() + return name.endswith(".xlsx") + + +def build_xlsx_preview_block( + f: Dict[str, Any], + user_id: Optional[str] = None, +) -> Optional[str]: + """Build the bounded preview + metadata + directive block for a single xlsx. + + Returns None if the file can't be read (caller falls back to the regular + parsed_text path, so the agent at least sees *something*). + """ + from core.files.content.spreadsheet_parser import parse_xlsx_preview + + file_id = f.get("file_id") or "" + name = f.get("name") or f.get("filename") or "未命名表格" + if not file_id: + return None + + file_bytes = artifact_bytes.download_artifact_bytes( + file_id, + name, + "xlsx preview", + user_id=user_id, + ) + if file_bytes is None: + return None + + try: + from core.files.artifacts.reader import ATTACHMENT_PREVIEW_MAX_CHARS, fetch_parsed_text + + # Reuse the bytes already downloaded for the structural preview to + # populate parsed_text + summary without a second storage round-trip. + fetch_parsed_text(file_id, user_id=user_id, prefetched_bytes=file_bytes) + info = parse_xlsx_preview(file_bytes, char_budget=ATTACHMENT_PREVIEW_MAX_CHARS) + except RuntimeError as e: + logger.warning(f"xlsx preview: parse failed for {file_id}: {e}") + return None + except Exception as e: + logger.warning(f"xlsx preview: unexpected parse error for {file_id}: {e}") + return None + + total_rows = info["total_rows"] + total_cols = info["total_columns"] + preview_rows = info["preview_rows"] + omitted = max(0, total_rows - preview_rows) + cols_str = ", ".join(info["header"]) if info["header"] else "(无列名)" + + lines: List[str] = [ + f"[文件: {name}]", + f" file_id: {file_id}", + f" 总规模: {total_rows} 行 × {total_cols} 列", + f" 列名: [{cols_str}]", + f" 已展示: 前 {preview_rows} 行（约 {len(info['preview_md'])} 字符，达到预览预算）", + ] + if info.get("truncated_columns"): + lines.append(f" 注意: 共 {total_cols} 列，预览仅含前 {len(info['header'])} 列") + if info.get("truncated_cells"): + lines.append(f" 注意: {info['truncated_cells']} 个超长单元格已截断到 500 字以内") + lines.append("") + lines.append(info["preview_md"]) + if omitted: + lines.append(f"... (剩余 {omitted} 行未展示) ...") + lines.append("") + lines.append("[操作指引]") + lines.append( + f"- 需要预览之外的正文时，调用 read_artifact(file_id='{file_id}', " + "offset=N, limit=4000) 按需读取" + ) + lines.append("- 以上只是有界预览，不要把未展示的行当作不存在，也不要推断未读取内容") + return "\n".join(lines) + + +def build_file_context( + uploaded_files: List[Dict[str, Any]], + user_id: Optional[str] = None, +) -> str: + """Assemble the text-attachment list into the context text injected into the model (non-image files only). + + Files are resolved exclusively by ``file_id``. The backend lazily parses and + caches the artifact, then injects only metadata plus a bounded preview. The + request body never supplies parsed content. + + ``user_id`` is used as the ownership check when fetching ``fetch_parsed_text`` / + the xlsx preview, preventing a forged request-body ``attachments[].file_id`` + from reading across users — see the :func:`artifact_bytes.download_artifact_bytes` docstring + for the core defense point. + """ + text_files = [f for f in uploaded_files if not artifact_bytes.is_image(f)] + if not text_files: + return "" + + file_descriptions = [] + for f in text_files: + name = f.get("name", "未知文件") + fid = f.get("file_id", "") + desc = f"- {name}" + if fid: + desc += f" (file_id: {fid})" + file_descriptions.append(desc) + from core.files.artifacts.reader import ATTACHMENT_PREVIEW_MAX_CHARS, fetch_parsed_text + + file_content_parts: List[str] = [] + for f in text_files: + # ── xlsx: bounded preview + metadata + directive (no full content) ── + if is_xlsx(f): + block = build_xlsx_preview_block(f, user_id=user_id) + if block is not None: + file_content_parts.append(block) + continue + # Fall through to regular path on preview failure so the agent + # at least sees the parsed markdown (degraded but non-fatal). + + fid = (f.get("file_id") or "").strip() + mime_type = f.get("mime_type") or "" + if not fid: + file_content_parts.append( + f"[文件: {f.get('name', '未知文件')}]\n" + " 状态: 缺少 file_id，无法读取正文" + ) + continue + + logger.info("Document hook: loading preview for %s (%s)", f.get("name"), fid) + content = fetch_parsed_text(fid, user_id=user_id) + if not content: + file_content_parts.append( + f"[文件: {f.get('name', '未知文件')}]\n" + f" file_id: {fid}\n" + f" MIME: {mime_type or '(未知)'}\n" + " 状态: 文件不可访问或解析失败" + ) + continue + + total_chars = len(content) + preview = content[:ATTACHMENT_PREVIEW_MAX_CHARS] + truncated = total_chars > len(preview) + block = [ + f"[文件: {f.get('name', '未知文件')}]", + f" file_id: {fid}", + f" MIME: {mime_type or '(未知)'}", + f" 解析文本: 共 {total_chars} 字符，本轮自动展示 {len(preview)} 字符", + "", + preview, + ] + if truncated: + block.append(f"... (其余 {total_chars - len(preview)} 字符未展示) ...") + block.extend( + [ + "", + "[读取指引]", + f"- 需要未展示的正文时，调用 read_artifact(file_id='{fid}', offset=N, limit=4000)", + "- 不要推断尚未读取的内容", + ] + ) + file_content_parts.append("\n".join(block)) + file_content = "\n\n---\n\n".join(file_content_parts) + + return ( + f"[file name]: {chr(10).join(file_descriptions)}\n" + f"[file content begin]\n" + f"{file_content}\n" + f"[file content end]\n" + "以上是用户附件的元数据和有界预览；需要预览之外的正文时必须按需调用 read_artifact。" + ) + + +def describe_source(source: str) -> str: + from core.files.artifacts.reader import SOURCE_AI_GENERATED, SOURCE_USER_UPLOAD + + return {SOURCE_USER_UPLOAD: "用户上传", SOURCE_AI_GENERATED: "AI 生成"}.get(source, "") + + +def build_historical_files_context(historical_files: List[Dict[str, Any]]) -> str: + """Build a compact summary block for files from previous turns. + + Includes both user-uploaded files and AI-generated files (reports, + charts, code output, etc.) produced by tools in earlier turns. + Each entry is labeled by provenance so the agent knows what kind of + file it's referencing. + + Agent gets only `{filename, file_id, source, summary}`; full content + is fetched on-demand via the `read_artifact` tool. This keeps prompt + size bounded as the conversation grows. + """ + if not historical_files: + return "" + + lines: List[str] = [] + for f in historical_files: + file_id = f.get("file_id") or "" + name = f.get("name") or f.get("filename") or "未命名文件" + mime = f.get("mime_type") or "" + summary = (f.get("summary") or "").strip() + source = f.get("source") or "" + deleted = bool(f.get("deleted")) + + source_label = describe_source(source) + header_parts = [f"file_id: {file_id}"] + if source_label: + header_parts.append(source_label) + if mime: + header_parts.append(mime) + header = f"- {name} ({', '.join(header_parts)})" + + if deleted: + lines.append(header + " [文件已删除，无法读取]") + continue + + lines.append(header) + # Key point: tell the model the stable /myspace/ path so it can operate + # directly with Read/Edit/Write — better suited than read_artifact for + # "modify HTML/CSS/MD/source code" text-editing scenarios, and changes sync + # back to "My Space" immediately. + if name and source == ai_source(): + lines.append(f" 沙盒路径: /myspace/{name} ← 用 Read/Edit/Write 直接读改写") + if summary: + indented = "\n".join(f" {ln}" for ln in summary.splitlines()) + lines.append(f" 摘要：\n{indented}") + else: + lines.append(" 摘要：（尚未生成）") + + header = ( + "[历史文件清单]（本会话中之前轮次涉及的文件，包含用户上传和 AI 生成的两类，以下仅为摘要）\n" + "访问方式：\n" + '- **要修改文本/HTML/MD/代码** → 用 `Read("/myspace/")` 读完整内容，' + "再用 `Edit` / `Write` 改；改完会自动同步回「我的空间」（同一 file_id）。\n" + "- **要看 docx/pdf/xlsx 等需要解析的二进制文件** → 用 " + "`read_artifact(file_id, offset, limit)` 拿解析后的文本。" + ) + return header + "\n\n" + "\n".join(lines) + + +def ai_source() -> str: + """Lazy resolve to avoid circular import at module load.""" + from core.files.artifacts.reader import SOURCE_AI_GENERATED + + return SOURCE_AI_GENERATED diff --git a/src/backend/core/llm/runtime/finish_guard.py b/src/backend/core/llm/runtime/finish_guard.py new file mode 100644 index 00000000..9c56f9f6 --- /dev/null +++ b/src/backend/core/llm/runtime/finish_guard.py @@ -0,0 +1,54 @@ +"""Apply neutral finish-pin decisions in the parent execution context. + +Workspace pins use ContextVars and must be written in the framework task, after +all finish hooks return. A failed pin leaves the guard eligible for retry. +""" +from __future__ import annotations + +import logging + +logger = logging.getLogger(__name__) + + +def pin_files(file_ids: list[str]) -> int: + """Write file_ids directly to the workspace ContextVar, bypassing the + pin_to_workspace tool + _acting path. Returns the number of pins that actually succeeded. + + Metadata comes from ``core.files.artifacts.store.get_artifact`` — on failure that id is skipped. + """ + try: + from core.files.artifacts.store import get_artifact + from core.llm.runtime import workspace as _ws + except Exception as exc: # noqa: BLE001 + logger.warning("[finish_guard] _direct_pin import failed: %s", exc) + return 0 + + count = 0 + for fid in file_ids: + try: + art = get_artifact(fid) + if not art: + logger.warning("[finish_guard] artifact not found: %s", fid) + continue + ok = _ws.pin( + fid, + name=art.get("name"), + mime_type=art.get("mime_type"), + size=art.get("size") or art.get("size_bytes"), + url=f"/files/{fid}", + ) + if ok: + count += 1 + except Exception as exc: # noqa: BLE001 + logger.warning("[finish_guard] _direct_pin failed for %s: %s", fid, exc) + # Only mark_active when at least one pin succeeded — avoid an empty set still + # calling mark_active, which would leave the workspace in an "active but pinned=[]" + # state (although downstream currently does not rely on the active flag for + # gating, this is a more robust invariant). + if count > 0: + try: + from core.llm.runtime import workspace as _ws + _ws.mark_active() + except Exception: + logger.warning("[finish_guard] failed to mark workspace active", exc_info=True) + return count diff --git a/src/backend/core/llm/runtime/hooks_constants.py b/src/backend/core/llm/runtime/hooks_constants.py new file mode 100644 index 00000000..3b78f011 --- /dev/null +++ b/src/backend/core/llm/runtime/hooks_constants.py @@ -0,0 +1,17 @@ +"""Limits for file content injected by context hooks.""" +from __future__ import annotations +import re + +MAX_FILE_CONTENT_CHARS = 50_000 # per-turn read_artifact budget per file + + +FILE_ID_RE = re.compile(r'"file_id"\s*:\s*"([A-Za-z0-9_\-]+)"') + + +PIN_HINT_SKIP_TOOLS: frozenset[str] = frozenset( + { + "pin_to_workspace", # the pin tool itself + "read_artifact", # echoes the input file_id; reading, not producing + "generate_response", # the ReActAgent finish function + } +) diff --git a/src/backend/core/llm/human_interaction.py b/src/backend/core/llm/runtime/human_interaction.py similarity index 86% rename from src/backend/core/llm/human_interaction.py rename to src/backend/core/llm/runtime/human_interaction.py index fbb81bd5..6a8d5b03 100644 --- a/src/backend/core/llm/human_interaction.py +++ b/src/backend/core/llm/runtime/human_interaction.py @@ -28,9 +28,9 @@ def has_pending(chat_id: Optional[str]) -> bool: if not chat_id: return False - from core.llm.tools import _tool_confirmation, user_questions + from core.llm.tools import confirmation, user_questions - return _tool_confirmation.has_pending(chat_id) or user_questions.has_pending(chat_id) + return confirmation.has_pending(chat_id) or user_questions.has_pending(chat_id) __all__ = [ diff --git a/src/backend/core/llm/interaction_store.py b/src/backend/core/llm/runtime/interaction_store.py similarity index 100% rename from src/backend/core/llm/interaction_store.py rename to src/backend/core/llm/runtime/interaction_store.py diff --git a/src/backend/core/llm/manifest_agent.py b/src/backend/core/llm/runtime/manifest_agent.py similarity index 93% rename from src/backend/core/llm/manifest_agent.py rename to src/backend/core/llm/runtime/manifest_agent.py index 140d3af9..aea1c7bb 100644 --- a/src/backend/core/llm/manifest_agent.py +++ b/src/backend/core/llm/runtime/manifest_agent.py @@ -6,11 +6,13 @@ import logging from typing import Any, Callable, Optional -from agentscope.agent import Agent -from core.llm.context_adapter import AgentScopeContextAdapter -from core.llm.context_ir import VISIBILITY_MANIFEST_ONLY, ContextAssembler, estimate_context_tokens -from core.llm.context_manager import FALLBACK_CONTEXT_WINDOW, usable_context_window -from core.llm.context_safety import enforce_context_budget +from agentscope.agent import Agent, InjectionConfig +from core.llm.context.adapter import AgentScopeContextAdapter +from core.llm.context.ir_constants import VISIBILITY_MANIFEST_ONLY +from core.llm.context.ir import ContextAssembler +from core.llm.context.content import estimate_context_tokens +from core.models.context_window import FALLBACK_CONTEXT_WINDOW, usable_context_window +from core.llm.context.safety import enforce_context_budget logger = logging.getLogger(__name__) @@ -21,6 +23,7 @@ class ManifestBoundAgent(Agent): def __init__( self, *args, context_adapter_factory=None, **kwargs ) -> None: # noqa: ANN002,ANN003 + kwargs.setdefault("injection_config", InjectionConfig(inject_runtime_state=False)) super().__init__(*args, **kwargs) self._context_adapter_factory = context_adapter_factory or AgentScopeContextAdapter self._tool_reserve_memo: Optional[tuple[int, int]] = None @@ -29,7 +32,7 @@ def __init__( def context_adapter_factory(self): # noqa: ANN201 return self._context_adapter_factory - async def compress_context(self, context_config=None) -> None: # noqa: ANN001 + async def compress_context(self, context_config=None, instructions=None) -> None: # noqa: ANN001 """Disable AgentScope's second, opaque context selector. Durable pre-turn compaction may replace history with an explicit @@ -38,7 +41,7 @@ async def compress_context(self, context_config=None) -> None: # noqa: ANN001 candidate decision. Running AgentScope compression here would delete candidates before the final manifest and make exclusions unauditable. """ - del context_config + del context_config, instructions @property def request_context_manifest(self) -> dict[str, Any]: diff --git a/src/backend/core/llm/message_compat.py b/src/backend/core/llm/runtime/message_compat.py similarity index 93% rename from src/backend/core/llm/message_compat.py rename to src/backend/core/llm/runtime/message_compat.py index 7fe33ecd..f3bcfa4a 100644 --- a/src/backend/core/llm/message_compat.py +++ b/src/backend/core/llm/runtime/message_compat.py @@ -3,9 +3,11 @@ from __future__ import annotations import json +from collections.abc import Mapping from typing import Any, Dict, List +from agentscope.message import Msg -from core.llm.context_adapter import AgentScopeContextAdapter +from core.llm.context.adapter import AgentScopeContextAdapter @@ -28,7 +30,7 @@ def dict_to_msg(d: Dict[str, Any], *, created_seq: int = 0) -> Any: # 1.x practice of putting tool_result on "tool"→"user" is rejected by Msg # validation in 2.0, so "tool" now maps to "assistant". # (The dict layer still keeps the "tool" marker so compaction can tell a - # result carrier from a step boundary — see core/llm/model_steps.py.) + # result carrier from a step boundary — see core/llm/runtime/model_steps.py.) role_map = {"human": "user", "ai": "assistant", "tool": "assistant"} role = role_map.get(role, role) @@ -56,12 +58,12 @@ def session_to_msgs(session_messages: List[Dict[str, Any]]) -> List[Any]: ``await load_session_into_memory(history, agent.memory)``. This is also where tool-returned media stops being a reference and becomes - bytes again (:func:`core.llm.tool_media_store.hydrate_rows`). Everything + bytes again (:func:`core.llm.runtime.tool_media_store.hydrate_rows`). Everything upstream — replay, the compaction checkpoint, the run's recovery snapshot — carries the sha256 only, because none of them needs the pixels and all of them get JSON-serialized; this is the first point where something does. """ - from core.llm.tool_media_store import hydrate_rows + from core.llm.runtime.tool_media_store import hydrate_rows return [ dict_to_msg(message, created_seq=index) @@ -80,10 +82,10 @@ def flatten_tool_output(output: Any) -> str: """ if isinstance(output, str): return output - if isinstance(output, list): + if isinstance(output, (list, tuple)): parts: list[str] = [] for item in output: - if isinstance(item, dict): + if isinstance(item, Mapping): text_val = item.get("text") parts.append(str(text_val) if text_val is not None else str(item)) elif getattr(item, "type", None) == "text": @@ -101,7 +103,7 @@ def _format_tool_output(output: Any) -> str: # In 2.0, output is often list[TextBlock|dict] (pydantic blocks); json.dumps # would fail and degrade to repr. Extract text via flatten_tool_output # first, then truncate. - if isinstance(output, list): + if isinstance(output, (list, tuple)): text = flatten_tool_output(output) return text[:2000] if len(text) > 2000 else text try: diff --git a/src/backend/core/llm/runtime/model_cache.py b/src/backend/core/llm/runtime/model_cache.py new file mode 100644 index 00000000..aa66cbd3 --- /dev/null +++ b/src/backend/core/llm/runtime/model_cache.py @@ -0,0 +1,133 @@ +"""Cache model instances per chat mode and invalidate them on config changes.""" +from __future__ import annotations +import threading +from dataclasses import dataclass, field +from functools import wraps +from typing import Any + +def serialized_model_cache(function): + @wraps(function) + def locked(*args, **kwargs): + with process_state().model_cache_lock: + return function(*args, **kwargs) + + return locked + + +def check_version(): + """Invalidate cached model instances when ModelConfigService version changes.""" + from core.models.config import ModelConfigService + + current = ModelConfigService.get_instance().version + if current != process_state().cached_version: + process_state().model_cache = {} + process_state().cached_version = current + + +def model_cache_key(name: str) -> tuple[Any, ...]: + from core.harness.registry.host_runtime import requires_remote_execution + + import asyncio + + try: + loop = asyncio.get_running_loop() + except RuntimeError: + loop = None + if not requires_remote_execution(): + return (name, loop, None) + from core.harness.registry.execution_identity import state_fingerprint as _state_fingerprint, get_state + + identity = _state_fingerprint(get_state()) + # Eviction only releases this cache's reference. In-flight runs keep their + # captured model and its existing request hook, which rejects identity changes. + for key in list(process_state().model_cache): + if isinstance(key, tuple) and key[0] == name and key[2] != identity: + process_state().model_cache.pop(key, None) + return name, loop, identity + + +@serialized_model_cache +def release_loop_models() -> None: + import asyncio + + loop = asyncio.get_running_loop() + for key in list(process_state().model_cache): + if key[1] is loop: + process_state().model_cache.pop(key, None) + + +@serialized_model_cache +def get_main_model(mode: str = "medium"): + """Get the main agent model for the given chat mode (fast/medium/high/max).""" + from core.models.chat_models import get_default_model + from core.models.failover import with_failover + from core.models.config import ModelConfigService + + check_version() + key = model_cache_key(f"main:{mode}") + cached = process_state().model_cache.get(key) + if cached is not None: + return cached + if mode in ("fast", "turbo"): + instance = get_default_model(disable_thinking=True, stream=True) + elif mode in ("low", "high", "xhigh", "max"): + instance = get_default_model(reasoning_effort=mode, stream=True) + else: # medium or unknown + # medium uses the "thinking model", but picks chat_template_kwargs based on + # supports_reasoning_effort (chat_models internally distinguishes + # effort=medium vs None) + instance = get_default_model(reasoning_effort="medium", stream=True) + instance = with_failover( + instance, ModelConfigService.get_instance().resolve("main_agent"), mode=mode + ) + process_state().model_cache[key] = instance + return instance + + +@serialized_model_cache +def get_provider_model(provider_id: str, mode: str = "medium"): + """Get a user-selected active chat provider model for the given chat mode.""" + from core.models.chat_models import build_model_for_mode + from core.models.failover import with_failover + from core.models.config import ModelConfigService + + pid = (provider_id or "").strip() + if not pid: + return get_main_model(mode) + check_version() + key = model_cache_key(f"provider:{pid}:{mode}") + cached = process_state().model_cache.get(key) + if cached is not None: + return cached + resolved = ModelConfigService.get_instance().resolve_provider(pid) + if resolved is None: + return get_main_model(mode) + + instance = with_failover( + build_model_for_mode(resolved, mode=mode, stream=True), resolved, mode=mode + ) + process_state().model_cache[key] = instance + return instance + + +def resolve_chat_mode(ctx) -> str: + """Resolve the final chat_mode (turbo/fast/medium/high/max) from agent.state (AgentRuntimeState).""" + raw = getattr(ctx, "chat_mode", None) + if raw in ("turbo", "fast", "low", "medium", "high", "xhigh", "max"): + return raw + # Fallback for legacy clients + return "medium" if getattr(ctx, "enable_thinking", True) else "fast" + + +@dataclass +class ModelCacheState: + model_cache: dict[tuple[Any, ...], Any] = field(default_factory=lambda: {}) + cached_version: int = field(default_factory=lambda: -1) + model_cache_lock: object = field(default_factory=lambda: threading.RLock()) + + +_STATE = ModelCacheState() + + +def process_state() -> ModelCacheState: + return _STATE diff --git a/src/backend/core/llm/runtime/model_selection_adapter.py b/src/backend/core/llm/runtime/model_selection_adapter.py new file mode 100644 index 00000000..214ad220 --- /dev/null +++ b/src/backend/core/llm/runtime/model_selection_adapter.py @@ -0,0 +1,28 @@ +"""Translate model-selection decisions into the framework's model reference.""" +import logging +from core.llm.runtime import model_cache +from core.models.usage import instrument_model_usage + +logger = logging.getLogger(__name__) + + +def model_selection_input(agent): + state = getattr(agent, "state", None) + return {name: getattr(state, name, default) for name, default in ( + ("model_pinned", False), ("model_provider_id", ""), ("chat_mode", None), + )} + + +def apply_model_selection(agent, invocation): + selection = invocation.data.get("model_selection") + if selection is None: + return + try: + provider = selection["provider_id"] + agent.model = (model_cache.get_provider_model(provider, mode=selection["mode"]) + if provider else model_cache.get_main_model(mode=selection["mode"])) + if invocation.run_id: + instrument_model_usage(agent.model) + except Exception as exc: + # Model refresh is optional; retain the factory's existing model on error. + logger.warning("[dynamic_model] failed: %s", exc) diff --git a/src/backend/core/llm/model_steps.py b/src/backend/core/llm/runtime/model_steps.py similarity index 98% rename from src/backend/core/llm/model_steps.py rename to src/backend/core/llm/runtime/model_steps.py index ef9ac27b..3d0d8a9b 100644 --- a/src/backend/core/llm/model_steps.py +++ b/src/backend/core/llm/runtime/model_steps.py @@ -67,7 +67,7 @@ def _persistable_output(output: Any) -> Any: """Tool output as the model saw it, media included. Media is the one part too big to inline: the bytes go to the blob table - keyed by their own sha256 (:mod:`core.llm.tool_media_store`) and the record + keyed by their own sha256 (:mod:`core.llm.runtime.tool_media_store`) and the record keeps a reference, so a later turn replays the same pixels the model was shown rather than a note saying a picture used to be here. The reference travels all the way to context assembly, which resolves it; replay itself @@ -92,7 +92,7 @@ def _persistable_output(output: Any) -> Any: if block_type == "text": blocks.append({"type": "text", "text": str(data.get("text") or "")}) elif block_type == "data": - from core.llm import tool_media_store + from core.llm.runtime import tool_media_store stored = tool_media_store.store(data) if stored is not None: @@ -142,6 +142,7 @@ def record_tool_result_step(block: Any) -> Dict[str, Any]: "id": str(data.get("id") or ""), "name": str(data.get("name") or ""), "output": _persistable_output(data.get("output")), + **({"metadata": _plain(data["metadata"])} if "mcp" in (data.get("metadata") or {}) else {}), "state": str(data.get("state") or "success"), } ], diff --git a/src/backend/core/llm/project_environment.py b/src/backend/core/llm/runtime/project_environment.py similarity index 88% rename from src/backend/core/llm/project_environment.py rename to src/backend/core/llm/runtime/project_environment.py index e81b7de2..30c9d24a 100644 --- a/src/backend/core/llm/project_environment.py +++ b/src/backend/core/llm/runtime/project_environment.py @@ -3,19 +3,18 @@ from __future__ import annotations import asyncio -from dataclasses import dataclass +from dataclasses import dataclass, field from threading import Lock -from core.project_environment.config import FIELDS -from core.services.project_environment import ProjectEnvironmentService +from core.projects.environment.config import FIELDS +from core.projects.environment.service import ProjectEnvironmentService from fastapi import HTTPException # Compare-and-store only; resource lookups never run under this lock. -_snapshot_lock = Lock() def _snapshot(context, key, value): - with _snapshot_lock: + with process_state().snapshot_lock: if context.get(key) is None: context[key] = value return context[key] @@ -41,8 +40,8 @@ def check_invocation(self, request): raise HTTPException(403, "所选能力未绑定到此项目，或当前用户没有使用权限") def restrict(self, request, *, resolve_defaults=True): - from core.config.catalog import get_enabled_ids - from core.llm.factory.selection.capabilities import _effective_main_available_skills + from core.capabilities.catalog.api import get_enabled_ids + from core.llm.factory.selection.capabilities import effective_main_available_skills as _effective_main_available_skills if resolve_defaults and request.enabled_skill_ids is None: request.enabled_skill_ids = _effective_main_available_skills() @@ -81,7 +80,7 @@ def restrict_plugins(self, resolution, request): plugin.mcp_ids = [m for m in plugin.mcp_ids if m in self.mcp_ids] plugin.bound_mcp_ids = [m for m in plugin.bound_mcp_ids if m in self.mcp_ids] setattr(resolution, field, entries) - from core.plugins.runtime.resolution import _finalize_resolution + from core.capabilities.plugins.runtime.resolution import finalize_resolution as _finalize_resolution _finalize_resolution(resolution, resolution.directory) # A disallowed plugin sharing an allowed loose skill must not consume @@ -101,11 +100,11 @@ def restrict_plugins(self, resolution, request): def _resolve(request): - from core.auth.permissions_iface import resolve_project_permission + from core.auth.edition import resolve_project_permission from core.db.engine import SessionLocal from core.db.models import ChatSession, Project - from core.project_environment.dependencies import expand_environment - from core.services.project_environment_resources import environment_resources + from core.projects.environment.dependencies import expand_environment + from core.projects.environment_resources import environment_resources context = request.project_ctx if request.project_ctx is not None else {} project_id = context.get("project_id") @@ -169,3 +168,15 @@ def validate_project_environment(request): """Current authorization may narrow a run; project setting edits cannot widen it.""" if request.project_environment is not None and _resolve(request) != request.project_environment: raise HTTPException(403, "项目资源权限已变更，本轮已暂停，请重新发起对话") + + +@dataclass +class ProjectEnvironmentState: + snapshot_lock: object = field(default_factory=lambda: Lock()) + + +_STATE = ProjectEnvironmentState() + + +def process_state() -> ProjectEnvironmentState: + return _STATE diff --git a/src/backend/core/llm/runtime/project_environment_hook.py b/src/backend/core/llm/runtime/project_environment_hook.py new file mode 100644 index 00000000..9b422762 --- /dev/null +++ b/src/backend/core/llm/runtime/project_environment_hook.py @@ -0,0 +1,19 @@ +"""Recheck project authorization before the framework starts tool I/O.""" +import asyncio +from fastapi import HTTPException +from core.harness.hooks import Decision, DecisionKind, HookSpec, HookStage +from core.llm.runtime.project_environment import validate_project_environment + + +def project_environment_hook(request): + async def authorize(invocation): + try: + await asyncio.to_thread(validate_project_environment, request) + except HTTPException as error: + return Decision(DecisionKind.REJECT, reason=str(error.detail), patch={ + "tool_response": {"content": [{"type": "text", "text": str(error.detail)}], + "state": "denied", "metadata": {"project_environment_denied": True}}}) + return Decision() + return HookSpec("project_environment_authorization", HookStage.BEFORE_TOOL, + authorize, priority=50, timeout_ms=30000, + mutable_fields=frozenset({"tool_response"})) diff --git a/src/backend/core/llm/runtime/review_text_agent.py b/src/backend/core/llm/runtime/review_text_agent.py new file mode 100644 index 00000000..d8044b2e --- /dev/null +++ b/src/backend/core/llm/runtime/review_text_agent.py @@ -0,0 +1,51 @@ +"""SDK execution adapter for a tool-free independent ontology review.""" +from typing import Any +from core.harness.registry.capability_runtime import child_scope + +async def run_text_agent( + prompt: str, + *, + user_id: str, + model_name: str | None, + model_provider_id: str | None, + runtime: dict[str, Any], + capability_scope: str = "", +) -> str: + from core.llm.factory import create_agent_executor + from core.llm.mcp.manager import close_clients + from core.llm.runtime.streaming import StreamingAgent + + agent, clients = await create_agent_executor( + disable_tools=True, + enabled_skill_ids=[], + current_user_id=user_id, + model_name=model_name, + model_provider_id=model_provider_id, + chat_mode="fast", + max_iters=2, + isolated=True, + ontology_runtime=runtime, + capability_scope=child_scope(capability_scope, "ontology-model", "text"), + ) + stream = StreamingAgent(agent, clients) + text = "" + try: + async for event_type, payload in stream.stream( + [{"role": "user", "content": prompt}], + { + "user_id": user_id, + "model_name": model_name or "", + "model_provider_id": model_provider_id or "", + "enable_thinking": False, + "chat_mode": "fast", + "ontology_enabled": True, + "ontology_runtime": runtime, + }, + ): + if event_type == "text_delta": + text += payload + elif event_type == "error": + raise RuntimeError(str(payload)) + finally: + await close_clients(clients) + return text diff --git a/src/backend/core/llm/session_capabilities.py b/src/backend/core/llm/runtime/session_capabilities.py similarity index 97% rename from src/backend/core/llm/session_capabilities.py rename to src/backend/core/llm/runtime/session_capabilities.py index 1d19007c..f3422643 100644 --- a/src/backend/core/llm/session_capabilities.py +++ b/src/backend/core/llm/runtime/session_capabilities.py @@ -81,7 +81,7 @@ def resolve_session_activated_capabilities( if not user_id or not chat_id: return result try: - from core.config.catalog_resolver import resolve_explicit_runtime_capabilities + from core.capabilities.catalog.resolver import resolve_explicit_runtime_capabilities from core.db.engine import SessionLocal from core.db.models import ChatSession diff --git a/src/backend/core/llm/runtime/state.py b/src/backend/core/llm/runtime/state.py new file mode 100644 index 00000000..bb26cf26 --- /dev/null +++ b/src/backend/core/llm/runtime/state.py @@ -0,0 +1,75 @@ +"""Typed AgentScope state at the framework boundary.""" +from typing import List +from agentscope.state import AgentState +from pydantic import ConfigDict, Field + +def _cyfunc_probe(): + return None + +_CYFUNCTION_TYPE = type(_cyfunc_probe) + +class AgentRuntimeState(AgentState): + """Extends AgentState to carry the runtime fields of the former ModelContext (replaces agent._jx_context).""" + + model_config = ConfigDict(ignored_types=(_CYFUNCTION_TYPE,)) + + model_name: str = "" + model_provider_id: str = "" + # When a subagent explicitly configures a model (model_provider_id) → the factory sets this + # True, and The model selection hook must not override it with the main model based on chat_mode + # (otherwise the subagent's own model is effectively useless — neither channels nor web get it). + model_pinned: bool = False + user_id: str | None = None + chat_id: str | None = None + run_id: str | None = None + capability_scope: str = "" + journal_owner: str | None = None + tool_effect_links: dict = Field(default_factory=dict) + enable_thinking: bool = True + chat_mode: str | None = None + uploaded_files: List[dict] = Field(default_factory=list) + historical_files: List[dict] = Field(default_factory=list) + user_message_text: str = "" + # Vision evidence transcribed ahead of the model call by the streaming entry + # point; The attachment hook injects it verbatim instead of redoing the + # network round-trip. Empty = transcribe inline (non-streaming callers). + vision_evidence_text: str = "" + ontology_enabled: bool = False + ontology_runtime: dict = Field(default_factory=dict) + # Set by The steering hook immediately before the next reasoning round. The + # streaming adapter turns it into one ``steer_applied`` event, then clears + # it. Keeping this on the typed runtime state avoids process-global queues. + steer_delivery: dict | None = None + # Model responses recorded by the hook adapter the moment they complete, + # drained by the streaming adapter into ``model_step`` events. Each entry + # is one ``core.llm.runtime.model_steps`` assistant record. + pending_model_steps: List[dict] = Field(default_factory=list) + pending_mcp_progress: List[dict] = Field(default_factory=list) + + def apply_request_context(self, context: dict, user_message_text: str) -> None: + """Populate per-request runtime fields from the request ``context`` dict (replaces the 1.x agent._jx_context). + + Shared by both entry points — streaming and workflow (non-streaming) — to keep two + hand-copied field mappings from silently drifting (streaming once didn't lowercase + chat_mode while workflow did). + """ + # A child may use the shared role or its own provider. Request context + # still describes the parent selection; it must not relabel a pinned + # child (vision routing and audit read these fields). + if not self.model_pinned: + self.model_name = str(context.get("model_name", "") or "") + self.model_provider_id = str(context.get("model_provider_id", "") or "") + self.user_id = str(context.get("user_id", "") or "") or None + self.chat_id = str(context.get("chat_id", "") or "") or None + self.run_id = str(context.get("run_id", "") or "") or None + self.journal_owner = str(context.get("journal_owner", "") or "") or None + self.enable_thinking = bool(context.get("enable_thinking", True)) + cm = str(context.get("chat_mode") or "").lower() or None + if cm: + self.chat_mode = cm + self.uploaded_files = list(context.get("uploaded_files", []) or []) + self.historical_files = list(context.get("historical_files", []) or []) + self.user_message_text = user_message_text or "" + self.ontology_enabled = bool(context.get("ontology_enabled", False)) + runtime = context.get("ontology_runtime") + self.ontology_runtime = runtime if isinstance(runtime, dict) else {} diff --git a/src/backend/core/llm/runtime/stream_events.py b/src/backend/core/llm/runtime/stream_events.py new file mode 100644 index 00000000..3f0a48a6 --- /dev/null +++ b/src/backend/core/llm/runtime/stream_events.py @@ -0,0 +1,285 @@ +"""Map AgentScope reply events to SSE events.""" +from __future__ import annotations +from core.llm.runtime import streaming_constants +from core.llm.runtime.stream_text import looks_like_tool_error as _looks_like_tool_error, strip_thinking_answer as _strip_thinking_answer +from core.harness import log_writer +import time +from datetime import datetime, timezone +from typing import Any, AsyncIterator, Dict, Optional, Tuple +import logging +logger = logging.getLogger(__name__) + +async def map_event( + self, ev: Any, *, result_step: Optional[Dict[str, Any]] = None +) -> AsyncIterator[Tuple[str, Any]]: + """Map a single reply_stream event into 0..N SSE events.""" + nm = type(ev).__name__ + + if nm == "TextBlockDeltaEvent": + delta = getattr(ev, "delta", "") or "" + if not delta: + return + # Normal state (enable_thinking): 2.0 events are already incremental — forward + # directly, no accumulate+recompute needed (the frontend parses itself), + # avoiding an O(n) scan of the full answer on every delta. + if self._enable_thinking: + if self._round_index == 0: + self._first_round_text += delta + yield ("text_delta", delta) + return + # Suppression state: may span multiple deltas — accumulate, then strip out the answer after the closing tag. + self._raw_text += delta + answer, self._in_thinking = _strip_thinking_answer( + self._raw_text, self._enable_thinking, self._in_thinking + ) + if answer and answer != self._emitted_answer: + out = ( + answer[len(self._emitted_answer) :] + if answer.startswith(self._emitted_answer) + else answer + ) + if out: + yield ("text_delta", out) + self._emitted_answer = answer + return + + if nm == "ThinkingBlockDeltaEvent": + if self._enable_thinking: + d = getattr(ev, "delta", "") or "" + if d: + yield ("thinking_delta", d) + return + + if nm == "ToolCallStartEvent": + tid = getattr(ev, "tool_call_id", "") or "" + name = getattr(ev, "tool_call_name", "") or "unknown" + self._tool_name_buf[tid] = name + self._tool_args_buf[tid] = "" + self._tool_delta_emit_buf[tid] = "" + self._tool_delta_last_emit[tid] = time.monotonic() + self._pending_tool_calls[tid] = { + "tool_name": name, + "tool_args": None, + "started_monotonic": time.monotonic(), + "started_at": datetime.now(timezone.utc), + } + yield ("tool_call_start", {"name": name, "id": tid}) + return + + if nm == "ToolCallDeltaEvent": + tid = getattr(ev, "tool_call_id", "") or "" + delta = getattr(ev, "delta", "") or "" + if not delta: + return + self._tool_args_buf[tid] = self._tool_args_buf.get(tid, "") + delta + pending_delta = self._tool_delta_emit_buf.get(tid, "") + delta + self._tool_delta_emit_buf[tid] = pending_delta + now = time.monotonic() + last_emit = self._tool_delta_last_emit.get(tid, now) + if ( + len(pending_delta) >= streaming_constants.TOOL_CALL_DELTA_FLUSH_CHARS + or now - last_emit >= streaming_constants.TOOL_CALL_DELTA_FLUSH_INTERVAL_S + ): + self._tool_delta_emit_buf[tid] = "" + self._tool_delta_last_emit[tid] = now + yield ( + "tool_call_delta", + { + "name": self._tool_name_buf.get(tid, "unknown"), + "id": tid, + "delta": pending_delta, + }, + ) + return + + if nm == "ToolCallEndEvent": + tid = getattr(ev, "tool_call_id", "") or "" + name = self._tool_name_buf.pop(tid, "unknown") + args_str = self._tool_args_buf.pop(tid, "") + pending_delta = self._tool_delta_emit_buf.pop(tid, "") + self._tool_delta_last_emit.pop(tid, None) + if pending_delta: + yield ( + "tool_call_delta", + {"name": name, "id": tid, "delta": pending_delta}, + ) + import json + + try: + args = json.loads(args_str) if args_str else {} + except json.JSONDecodeError: + args = {"_raw": args_str} + rec = self._pending_tool_calls.get(tid) + if rec is not None: + rec["tool_args"] = args + try: + from core.llm.runtime.tool_callbacks import note_tool_call + + note_tool_call(self.__dict__.setdefault("_tool_warn_state", {}), name, args) + except Exception as fallback_error: # noqa: BLE001 + from core.infra.failures import record_best_effort_failure + record_best_effort_failure('core.llm.runtime.stream_events:map_event', fallback_error) + yield ("tool_call", {"name": name, "args": args, "id": tid}) + return + + if nm == "ToolResultTextDeltaEvent": + tid = getattr(ev, "tool_call_id", "") or "" + self._tool_result_buf[tid] = self._tool_result_buf.get(tid, "") + ( + getattr(ev, "delta", "") or "" + ) + return + + if nm == "ToolResultEndEvent": + tid = getattr(ev, "tool_call_id", "") or "" + content = self._tool_result_buf.pop(tid, "") + if result_step is None: + result_step = await self._tool_result_step(tid) + if result_step is None: + raise RuntimeError(f"Tool result {tid} is missing from model context") + raw_state = getattr(ev, "state", "") or "" + state = str(getattr(raw_state, "value", raw_state) or "") + pending = self._pending_tool_calls.pop(tid, None) + name = ( + getattr(ev, "tool_call_name", "") + or (pending or {}).get("tool_name") + or self._tool_name_buf.get(tid) + or "unknown" + ) + try: + is_error = state in { + "error", + "denied", + "interrupted", + } or _looks_like_tool_error(content) + effect_link = dict( + (getattr(self.agent.state, "tool_effect_links", None) or {}).pop(tid, {}) + ) + started_mono = pending.get("started_monotonic") if pending else None + duration_ms = ( + int((time.monotonic() - started_mono) * 1000) if started_mono else None + ) + log_writer.schedule_tool_call_write( + { + # See the matching comment in _produce: carry user_id/chat_id explicitly, never rely on contextvars. + "user_id": self.agent.state.user_id or None, + "chat_id": self.agent.state.chat_id or None, + "run_id": self.agent.state.run_id or None, + "effect_id": effect_link.get("effect_id"), + "result_id": effect_link.get("result_id"), + "tool_name": (pending or {}).get("tool_name") or name, + "tool_call_id": tid, + "tool_args": (pending or {}).get("tool_args"), + "tool_result": content, + "status": "failed" if is_error else "success", + "error_message": content if is_error else None, + "duration_ms": duration_ms, + "started_at": (pending or {}).get("started_at"), + } + ) + except Exception: # noqa: BLE001 + logger.debug("tool_call log persist failed", exc_info=True) + # Persist the canonical result before any UI filtering (notably + # update_plan). The consumer must finish this event before the + # display-only result can announce completion. + yield ("model_step", result_step) + yield ( + "tool_result", + { + "name": name, + "id": tid, + "content": content, + "status": state, + "mcp": (result_step["blocks"][0].get("metadata") or {}).get("mcp"), + }, + ) + return + + if nm == "ModelCallStartEvent": + # Marks the wire as "waiting on the model" — see _model_call_inflight. + self._model_call_inflight = True + # Agent construction and manifest-backed tool assembly happen before + # this event. Surface the provider-dispatch boundary accurately. + yield ("model_call_start", None) + return + + if nm == "ModelCallEndEvent": + self._model_call_inflight = False + # The hook adapter recorded this response before the event was + # queued; hand the record downstream in the same order. + for step in self._drain_model_steps(): + yield ("model_step", step) + _prompt_tokens = int(getattr(ev, "input_tokens", 0) or 0) + _completion_tokens = int(getattr(ev, "output_tokens", 0) or 0) + self._usage_records.append( + { + "prompt_tokens": _prompt_tokens, + "completion_tokens": _completion_tokens, + } + ) + from core.llm.context.usage import build_context_usage_snapshot + + self._context_usage = build_context_usage_snapshot( + self.agent, + prompt_tokens=_prompt_tokens, + completion_tokens=_completion_tokens, + model_call_index=len(self._usage_records), + ) + # Compaction usage is anchored by CompactingAgent._save_to_context + # after the response is saved. AgentScope emits this event first; + # observing here would count that response again as new content. + # 首轮权威判定：思考模式下整轮正文没有出现 → 该模型不内联思考 + # （结构化 reasoning 通道，或本轮确实没思考）。补发协议标记，前端据此 + # 把误当思考缓冲/展示的正文重归正文区（bug：无思考时正文进思考块）。 + # 只看首轮——混合形态模型工具后省略 属既有启发式管辖，不在此误判。 + if ( + self._enable_thinking + and self._round_index == 0 + and not self._structured_marker_sent + and self._first_round_text + and "" not in self._first_round_text + ): + self._structured_marker_sent = True + yield ("reasoning_protocol", {"structured_reasoning": True}) + yield ("context_usage", dict(self._context_usage)) + self._round_index += 1 + self._first_round_text = "" + # New model call round → reset answer accumulation (the next text segment computes deltas from scratch) + self._raw_text = "" + self._emitted_answer = "" + self._in_thinking = False + return + + if nm == "RequireUserConfirmEvent": + # 2.0 native HITL (distinct from the myspace gate); our tools default to ALLOW, so this rarely triggers. + try: + yield ( + "file_confirm", + { + "reply_id": getattr(ev, "reply_id", ""), + "tool_calls": [ + { + "id": getattr(tc, "id", ""), + "name": getattr(tc, "name", ""), + "input": getattr(tc, "input", ""), + } + for tc in (getattr(ev, "tool_calls", []) or []) + ], + }, + ) + except Exception as fallback_error: # noqa: BLE001 + from core.infra.failures import record_best_effort_failure + record_best_effort_failure('core.llm.runtime.stream_events:map_event', fallback_error) + return + + if nm == "ExceedMaxItersEvent": + yield ( + "error", + {"kind": "exceed_max_iters", "name": getattr(ev, "name", "")}, + ) + return + + # Everything else is never forwarded, internal only: lifecycle (ReplyStart/End, + # the various *Start/*End) + DataBlockStart/Delta (model-produced + # image_chunk — currently no downstream consumer and the configured models only take + # images as input, so intentionally dropped; add a branch when support is needed). + return diff --git a/src/backend/core/llm/runtime/stream_loop.py b/src/backend/core/llm/runtime/stream_loop.py new file mode 100644 index 00000000..1f554340 --- /dev/null +++ b/src/backend/core/llm/runtime/stream_loop.py @@ -0,0 +1,309 @@ +"""Drive one agent reply and yield its SSE events.""" +from __future__ import annotations +from core.llm.runtime import streaming_constants +from core.harness import log_writer +import asyncio +import time +from typing import Any, AsyncIterator, Dict, List, Optional, Tuple +from agentscope.message import Msg +from core.infra.logging import LogContext +from core.llm.context.rendering import next_request_sequence, render_context_item, render_session_input +from core.llm.context.ir_constants import KIND_USER_INPUT, POLICY_NEVER, SESSION_CONTEXT_META_KEY +from core.llm.context.items import ContextItem +from core.llm.runtime.message_compat import session_to_msgs +import logging +logger = logging.getLogger(__name__) + +async def stream( + self, + session_messages: List[Dict[str, Any]], + context: Dict[str, Any], + *, + effective_user_message: Optional[str] = None, +) -> AsyncIterator[Tuple[str, Any]]: + agent = self.agent + + # Commands keep their short spelling in persisted history, while this + # request must execute the expanded instructions. Copy the current row + # so neither stored history nor earlier user turns are rewritten. + if effective_user_message is not None: + session_messages = list(session_messages) + if session_messages and session_messages[-1].get("role") in ("user", "human"): + session_messages[-1] = { + **session_messages[-1], + "content": effective_user_message, + } + else: + session_messages.append({"role": "user", "content": effective_user_message}) + + _last_user_text = "" + if session_messages: + for _m in reversed(session_messages): + if _m.get("role") in ("user", "human"): + _last_user_text = str(_m.get("content") or "") + break + + # ctx → agent.state (AgentRuntimeState, replaces the 1.x agent._jx_context = ModelContext) + st = agent.state + try: + st.apply_request_context(context, _last_user_text) + except Exception as exc: # noqa: BLE001 + logger.warning("[stream] set agent.state failed: %s", exc) + self._enable_thinking = bool(context.get("enable_thinking", True)) + + # Vision bridge: transcribe uploaded images *before* handing off to the model, + # so the UI can say "读图中" while it happens. Reading an image is a plain + # network wait of several seconds; folded inside the agent turn it is + # indistinguishable from the model thinking, and the user just watches a + # generic spinner count up. FileContextMiddleware injects the result. + async for _vision_event in self._prepare_vision_evidence(st): + yield _vision_event + + _log_ctx = LogContext(user_id=st.user_id or None, chat_id=st.chat_id or None) + _log_ctx.__enter__() + + # Load history into context (excluding the last user message — reply_stream's inputs carries it, avoiding duplication) + history = list(session_messages) + last_user_content = "" + last_user_row = None + if history and history[-1].get("role") in ("user", "human"): + last_user_row = history.pop() + last_user_content = last_user_row.get("content", "") + if history: + try: + agent.state.context.extend(session_to_msgs(history)) + except Exception as exc: # noqa: BLE001 + logger.warning("[stream] load history failed: %s", exc) + + user_msg: Optional[Msg] = None + if last_user_content: + request_seq = next_request_sequence(agent.state.context) + if isinstance((last_user_row or {}).get(SESSION_CONTEXT_META_KEY), dict): + user_msg = render_session_input(last_user_row, created_seq=request_seq) + else: + user_msg = render_context_item( + ContextItem.create( + item_id=f"request:user_input:{request_seq}", + kind=KIND_USER_INPUT, + origin="user:chat", + trust="user", + visibility="model", + priority=1_000, + token_budget=100_000, + truncation_policy=POLICY_NEVER, + content=last_user_content, + cache_class="dynamic", + created_seq=request_seq, + render_role="user", + render_name="user", + message_group=f"request:user_input:{request_seq}", + ) + ) + + # myspace write-confirmation gate (in-house ContextVar, distinct from 2.0 native HITL) + from core.llm.tools import confirmation as _mc + + _confirm_chat_id = st.chat_id or None + + def _drain_confirm_signals() -> list: + out: list = [] + q = _mc.get_ui_queue(_confirm_chat_id) + if q is None: + return out + while True: + try: + out.append(q.get_nowait()) + except asyncio.QueueEmpty: + break + return out + + from core.llm.tools import user_questions as _user_questions + + def _drain_user_question_signals() -> list: + out: list = [] + q = _user_questions.get_ui_queue(_confirm_chat_id) + if q is None: + return out + while True: + try: + out.append(q.get_nowait()) + except asyncio.QueueEmpty: + break + return out + + def _drain_human_interaction_events() -> list: + """Map every queued human-interaction signal to a stream event.""" + + events: list = [] + for signal in _drain_confirm_signals(): + event_type = ( + "design_pick" + if (signal or {}).get("kind") == _mc.KIND_DESIGN_PICK + else "file_confirm" + ) + events.append((event_type, signal)) + for signal in _drain_user_question_signals(): + signal_event = signal.get("event") + if signal_event == "requested": + events.append(("user_question", signal)) + elif signal_event == "resolved": + events.append(("user_question_resolved", signal)) + return events + + # reply_stream runs as a background task feeding the queue; the main loop drains confirm signals concurrently + event_q: asyncio.Queue = asyncio.Queue() + _DONE = object() + + # Subagent streaming bypass: register this run's event_q so call_subagent (separate thread) + # can deliver the subagent's thinking/tool_call/... events back into this main queue in real time. + from core.llm.subagents import stream as _subagent_stream + + _subagent_stream.attach(st.chat_id, asyncio.get_running_loop(), event_q) + + async def _produce(): + try: + async for ev in agent.reply_stream(inputs=user_msg): + result_step = None + if type(ev).__name__ == "ToolResultEndEvent": + tid = getattr(ev, "tool_call_id", "") or "" + # Snapshot while the producer is still at the tool + # boundary, before another model step/steer/compaction + # can replace the live context. + result_step = await self._tool_result_step(tid) + if result_step is None: + raise RuntimeError(f"Tool result {tid} is missing from model context") + await event_q.put(("ev", (ev, result_step))) + except asyncio.CancelledError: + raise + except BaseException as e: # noqa: BLE001 + import traceback + + logger.error("Agent reply_stream failed: %r\n%s", e, traceback.format_exc()) + await event_q.put(("err", e)) + finally: + await event_q.put(("done", _DONE)) + + prod_task = asyncio.create_task(_produce()) + self._producer_task = prod_task + + _stream_start = time.monotonic() + _first_event_logged = False + _poll_interval = streaming_constants.QUEUE_POLL_INTERVAL_S + # Last model_progress emission — throttles the liveness signal emitted + # when upstream events map to nothing. Only the swallowed-event branch + # reads/updates the clock, keeping the mapped hot path free of it. + _last_progress_ts = _stream_start + + try: + while True: + for progress in st.pending_mcp_progress: + if progress.get("tool_id") in self._pending_tool_calls: + yield ("tool_progress", progress) + st.pending_mcp_progress = [] + for _human_event in _drain_human_interaction_events(): + yield _human_event + try: + kind, payload = await asyncio.wait_for(event_q.get(), timeout=_poll_interval) + except asyncio.TimeoutError: + if self._model_call_inflight: + # Model call in flight with a silent wire — count as + # liveness (throttled) so the inactivity watchdog waits + # for the model client's own timeout/retry outcome + # instead of misjudging the run as hung. + _now = time.monotonic() + if _now - _last_progress_ts >= streaming_constants.MODEL_PROGRESS_MIN_INTERVAL_S: + _last_progress_ts = _now + yield ("model_progress", None) + continue + yield ("heartbeat", None) + continue + if kind in {"done", "err"}: + # A human answer wakes the tool before AgentScope queues its + # terminal event. Drain once more here so an immediately + # finishing reply cannot strand the frontend in submitted + # state waiting for user_question_resolved. + for _human_event in _drain_human_interaction_events(): + yield _human_event + if kind == "err": + yield ("error", payload) + break + if kind == _subagent_stream.QUEUE_KIND: + # Subagent bypass events — passed straight through, never into _map_event. + if not _first_event_logged: + _first_event_logged = True + yield ("subagent_event", payload) + continue + # kind == "ev" + payload, result_step = payload + steer_delivery = getattr(agent.state, "steer_delivery", None) + if isinstance(steer_delivery, dict): + # SteerMiddleware appends the user instruction immediately + # before this next reasoning round. Emit one acknowledgement + # before mapping ModelCallStart so the queued-card UI can + # switch from "waiting" to "applied" deterministically. + agent.state.steer_delivery = None + yield ("steer_applied", dict(steer_delivery)) + reasoning_protocol = self._take_reasoning_protocol() + if reasoning_protocol is not None: + yield ("reasoning_protocol", reasoning_protocol) + _mapped_any = False + async for out in self._map_event(payload, result_step=result_step): + if not _first_event_logged: + _ttfe = (time.monotonic() - _stream_start) * 1000 + logger.info("[stream] TTFE: %.0fms, type=%s", _ttfe, out[0]) + _first_event_logged = True + _mapped_any = True + yield out + if not _mapped_any: + # The model is alive (an upstream event just arrived) but + # nothing was forwarded — typical of long tool-call-arg / + # suppressed-thinking streaming. Emit a throttled liveness + # signal so the inactivity watchdog doesn't misjudge a + # healthy run as hung. + _now = time.monotonic() + if _now - _last_progress_ts >= streaming_constants.MODEL_PROGRESS_MIN_INTERVAL_S: + _last_progress_ts = _now + yield ("model_progress", None) + except Exception as e: # noqa: BLE001 + yield ("error", e) + finally: + await self.stop_producer() + # Deregister the subagent bypass — prevents late events from being written into a finished run's event_q. + try: + _subagent_stream.detach(st.chat_id) + except Exception as fallback_error: # noqa: BLE001 + from core.infra.failures import record_best_effort_failure + record_best_effort_failure('core.llm.runtime.stream_loop:stream', fallback_error) + for _tid, _rec in list(self._pending_tool_calls.items()): + try: + _started_mono = _rec.get("started_monotonic") + _dur = int((time.monotonic() - _started_mono) * 1000) if _started_mono else None + log_writer.schedule_tool_call_write( + { + # user_id/chat_id are taken explicitly from agent.state — contextvars are + # unreliable in the stream() generator frame (the agent runs in the context + # snapshot of _produce's create_task, while tool results are written in the + # generator's consumer frame; the two contexts don't sync), so _context_ids + # cannot be relied on. + "user_id": st.user_id or None, + "chat_id": st.chat_id or None, + "tool_name": _rec.get("tool_name", "unknown"), + "tool_call_id": _tid, + "tool_args": _rec.get("tool_args"), + "tool_result": None, + "status": "failed", + "error_message": "no tool_result received (stream ended)", + "duration_ms": _dur, + "started_at": _rec.get("started_at"), + } + ) + except Exception: + logger.debug("pending tool_call flush failed", exc_info=True) + self._pending_tool_calls.clear() + self._tool_delta_emit_buf.clear() + self._tool_delta_last_emit.clear() + try: + _log_ctx.__exit__(None, None, None) + except Exception as fallback_error: + from core.infra.failures import record_best_effort_failure + record_best_effort_failure('core.llm.runtime.stream_loop:stream', fallback_error) diff --git a/src/backend/core/llm/runtime/stream_text.py b/src/backend/core/llm/runtime/stream_text.py new file mode 100644 index 00000000..bebd8ece --- /dev/null +++ b/src/backend/core/llm/runtime/stream_text.py @@ -0,0 +1,63 @@ +"""Split visible answer text from thinking and detect tool error text.""" +from __future__ import annotations +from typing import Tuple +from core.infra.model_output import THINK_BLOCK_RE +from core.llm.runtime import streaming_constants + +def looks_like_tool_error(content: str) -> bool: + if not content: + return False + s = content.strip() + if not s: + return False + head = s[:64] + if s.startswith(('{"error"', "{'error'", '{"ok": false', '{"ok":false')): + return True + if '"ok": false' in head or '"ok":false' in head: + return True + if s.startswith(("Error executing tool", "Error: ", "Traceback (most recent call last)")): + return True + if "validation error for" in head: + return True + if "不存在或已删除" in s or "无权访问" in head: + return True + if streaming_constants.HTTP_ERR_RE.search(s): + return True + return False + + +def strip_thinking_answer(raw: str, enable_thinking: bool, in_thinking: bool) -> Tuple[str, bool]: + """Extract the user-visible "answer" portion from the accumulated raw text. + + Returns (answer, new_in_thinking). When enable_thinking=True, returns as-is (the frontend + parses itself); otherwise suppresses every … span and returns only + the visible text. + + 一次模型调用里可以出现**多段**思考链（ReAct 模型常在工具调用前再想一轮）。 + 早先的实现只取"最后一个 之后的内容"：当第二段思考刚开了 + 还没闭合时，rfind 命中的是第一段的闭标签，于是 ``vis1正在生成的思考…`` + 被整段当成正文吐出去 —— 网页端 enable_thinking=False 只在快速模式出现（模型 + 本来就不思考），而定时任务这条路径恰恰是"模型照常思考、正文里不该有思考"， + 于是思考过程就跟着落进了自动化任务的输出正文。 + + 改成"先删掉所有闭合块，再处理孤儿标签"，未闭合的那段思考不会再漏出来。 + """ + if enable_thinking: + return raw, False + # 1) 闭合的思考块整段丢掉（可能有多段） + text = THINK_BLOCK_RE.sub("", raw) + # 2) 孤儿闭标签：有些服务端把开标签吃进 chat template，正文以裸思考开头， + # 最后一个 之前的都是思考。 + last_end = text.rfind("") + if last_end != -1: + text = text[last_end + len("") :] + return text, False + # 3) 还没闭合的思考块：它之后的内容都还在思考里，不出正文 + open_idx = text.find("") + if open_idx != -1: + return text[:open_idx], True + # 4) 整段没有任何标签：沿用上一次的判定（服务端预填开标签的形态下， + # 在 到来之前只能靠这一位把裸思考压住） + if in_thinking: + return "", True + return text, False diff --git a/src/backend/core/llm/runtime/streaming.py b/src/backend/core/llm/runtime/streaming.py new file mode 100644 index 00000000..9eb969d3 --- /dev/null +++ b/src/backend/core/llm/runtime/streaming.py @@ -0,0 +1,280 @@ +"""Wrap an AgentScope agent so it streams SSE events.""" +from __future__ import annotations +from contextlib import aclosing +from core.llm.runtime import stream_loop +from core.llm.runtime import stream_events +import asyncio +from typing import Any, AsyncIterator, Dict, List, Optional, Tuple +from agentscope.agent import Agent +from core.llm.mcp.lifetime import MCPResource +import logging +logger = logging.getLogger(__name__) + +class StreamingAgent: + """Wraps an AgentScope 2.0 ``Agent`` to produce streaming SSE events via reply_stream.""" + + def __init__( + self, + agent: Agent, + mcp_clients: List[MCPResource], + ): + self._producer_task = None + self.agent = agent + self.mcp_clients = mcp_clients + self._enable_thinking = False + # usage: accumulated from ModelCallEndEvent + self._usage_records: List[Dict[str, int]] = [] + self._context_usage: Optional[Dict[str, Any]] = None + # tool_id → {tool_name, tool_args(str), started_monotonic, started_at} + self._pending_tool_calls: Dict[str, Dict[str, Any]] = {} + # tool_id → name (recorded at ToolCallStart), tool_id → accumulated args string (ToolCallDelta) + self._tool_name_buf: Dict[str, str] = {} + self._tool_args_buf: Dict[str, str] = {} + # tool_id → not-yet-emitted argument delta / last flush timestamp + self._tool_delta_emit_buf: Dict[str, str] = {} + self._tool_delta_last_emit: Dict[str, float] = {} + # tool_id → accumulated result text (ToolResultTextDelta) + self._tool_result_buf: Dict[str, str] = {} + # Accumulated answer text (for dedup in the -suppression scenario) + self._raw_text = "" + self._emitted_answer = "" + # 首轮正文累计（enable_thinking 下）：轮次结束时若整轮没有 ， + # 说明该模型不是"内联思考"形态（结构化 reasoning 通道、或本轮无思考）—— + # 补发 structured_reasoning 标记，让前端把误缓冲成"思考"的正文流回正文区。 + self._first_round_text = "" + self._round_index = 0 + self._structured_marker_sent = False + self._in_thinking = False + self._reasoning_protocol_emitted = False + # True between ModelCallStartEvent and ModelCallEndEvent. While a call is + # in flight the LLM endpoint may legitimately produce nothing for minutes + # (long prefill on a huge prompt / hung gateway being retried); the queue + # poll then emits model_progress instead of heartbeat so the run + # inactivity watchdog doesn't kill a run that is merely waiting on the + # model — the model client's own httpx timeouts remain the backstop. + self._model_call_inflight = False + + def _drain_model_steps(self) -> List[Dict[str, Any]]: + state = getattr(self.agent, "state", None) + pending = getattr(state, "pending_model_steps", None) + if not pending: + return [] + drained = list(pending) + state.pending_model_steps = [] + return drained + + async def _tool_result_step(self, tool_call_id: str) -> Optional[Dict[str, Any]]: + """Record the tool result exactly as AgentScope saved it into context. + + ``_execute_tool_call`` appends the (already bounded) ToolResultBlock to + the last assistant message before it yields ``ToolResultEndEvent``, so + by the time the event reaches this consumer the block is there. + + Recording a result that carries media writes those bytes to the blob + table (see :mod:`core.llm.runtime.tool_media_store`), so the work goes to a + thread instead of blocking the event loop. + """ + from core.llm.runtime.model_steps import record_tool_result_step + + context = getattr(getattr(self.agent, "state", None), "context", None) or [] + last = context[-1] if context else None + if last is None or getattr(last, "role", "") != "assistant": + logger.error("tool result %s ended with no assistant message in context", tool_call_id) + return None + match = None + for block in last.get_content_blocks("tool_result"): + if getattr(block, "id", "") == tool_call_id: + match = block + if match is None: + logger.error("tool result %s ended but its block is not in context", tool_call_id) + return None + return await asyncio.to_thread(record_tool_result_step, match) + + def _take_reasoning_protocol(self) -> Optional[Dict[str, bool]]: + """Return the structured-reasoning marker once the active model is known. + + DynamicModelMiddleware may replace ``agent.model`` at reply start, so this is + evaluated immediately before mapping the first AgentScope event rather than in + ``__init__``. + """ + if self._reasoning_protocol_emitted: + return None + model = getattr(self.agent, "model", None) + if not bool(getattr(model, "structured_reasoning", False)): + return None + self._reasoning_protocol_emitted = True + return {"structured_reasoning": True} + + def get_usage(self) -> Dict[str, int]: + """Return the in-memory fallback; async production paths use aget_usage.""" + total_prompt = sum(r.get("prompt_tokens", 0) for r in self._usage_records) + total_completion = sum(r.get("completion_tokens", 0) for r in self._usage_records) + last = self._usage_records[-1] if self._usage_records else {} + return { + "prompt_tokens": total_prompt, + "completion_tokens": total_completion, + "total_tokens": total_prompt + total_completion, + "llm_call_count": len(self._usage_records), + "context_tokens": int(last.get("prompt_tokens", 0) or 0) + + int(last.get("completion_tokens", 0) or 0), + "last_prompt_tokens": int(last.get("prompt_tokens", 0) or 0), + "last_completion_tokens": int(last.get("completion_tokens", 0) or 0), + } + + async def aget_usage(self) -> Dict[str, int]: + """Derive usage from append-only attempts without blocking the event loop.""" + run_id = str(getattr(getattr(self.agent, "state", None), "run_id", "") or "") + if run_id: + try: + from core.harness.ledger import HarnessUsageLedger + + attempts = await asyncio.wait_for( + asyncio.to_thread(HarnessUsageLedger().attempts, run_id), + timeout=1.0, + ) + model_attempts = tuple(row for row in attempts if row.kind == "model") + if model_attempts: + primary_attempts = tuple( + row + for row in model_attempts + if row.metadata.get("source") not in {"vision", "compaction", "followup"} + ) + last = (primary_attempts or model_attempts)[-1] + return { + "prompt_tokens": sum(row.usage.prompt_tokens for row in model_attempts), + "completion_tokens": sum( + row.usage.completion_tokens for row in model_attempts + ), + "cache_read_tokens": sum( + row.usage.cache_read_tokens for row in model_attempts + ), + "cache_write_tokens": sum( + row.usage.cache_write_tokens for row in model_attempts + ), + "total_tokens": sum(row.usage.total_tokens for row in model_attempts), + "llm_call_count": len(model_attempts), + "context_tokens": last.usage.prompt_tokens + last.usage.completion_tokens, + "last_prompt_tokens": last.usage.prompt_tokens, + "last_completion_tokens": last.usage.completion_tokens, + } + except Exception: + logger.debug("usage ledger aggregate unavailable", exc_info=True) + return self.get_usage() + + def get_context_usage(self, usage: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: + """Return the latest provider measurement or canonical fallback.""" + if self._context_usage and self._context_usage.get("exact"): + return dict(self._context_usage) + last = self._usage_records[-1] if self._usage_records else {} + if usage: + prompt = int(usage.get("last_prompt_tokens", 0) or 0) + completion = int(usage.get("last_completion_tokens", 0) or 0) + else: + prompt = int(last.get("prompt_tokens", 0) or 0) + completion = int(last.get("completion_tokens", 0) or 0) + from core.llm.context.usage import build_context_usage_snapshot + + snapshot = build_context_usage_snapshot( + self.agent, + prompt_tokens=prompt, + completion_tokens=completion, + model_call_index=len(self._usage_records), + ) + self._context_usage = snapshot + return dict(snapshot) + + def restore_context_usage(self, snapshot: Optional[Dict[str, Any]]) -> None: + """Restore a snapshot after a transient call that is not replayed next turn.""" + self._context_usage = dict(snapshot) if snapshot else None + + async def _prepare_vision_evidence(self, st: Any) -> AsyncIterator[Tuple[str, Any]]: + """Transcribe this turn's image attachments, emitting progress around the wait. + + Yields ``("vision_progress", {...})`` when there is actually something to + read, so the frontend can label the wait "图像理解中" instead of the generic + turn spinner. Silent (no events, no delay) when the turn has no images or the + running model reads images natively — in the native case the picture goes to + the model as-is and no transcription happens at all. + + Never raises: a failure here degrades to the middleware's inline path. + """ + try: + from core.llm.context.attachment_hook import supports_native_images + from core.models.vision.attachments import image_attachments, transcribe_attachments + + from core.files import artifact_bytes + images = image_attachments(list(getattr(st, "uploaded_files", None) or []), is_image=artifact_bytes.is_image) + if not images or supports_native_images(getattr(st, "model_provider_id", "") or ""): + return + + yield ( + "vision_progress", + { + "status": "running", + "count": len(images), + "names": [f.get("name") or "图片" for f in images], + }, + ) + from core.models.usage import model_usage_scope + from core.harness.ledger import HarnessUsageLedger + + run_id = str(getattr(st, "run_id", "") or "") + with model_usage_scope(run_id, HarnessUsageLedger()): + result = await transcribe_attachments( + images, download_bytes=artifact_bytes.download_artifact_bytes, + user_id=getattr(st, "user_id", None), + ) + if result is not None and result.text: + st.vision_evidence_text = result.text + yield ( + "vision_progress", + { + "status": "done", + "count": result.count if result else 0, + "ok": bool(result and result.ok), + "duration_seconds": round(result.duration_seconds, 2) if result else 0.0, + }, + ) + except Exception as exc: # noqa: BLE001 — fall back to inline transcription + logger.warning("[stream] vision pre-pass failed, deferring to middleware: %s", exc) + yield ("vision_progress", {"status": "done", "count": 0, "ok": False}) + + async def stream( + self, + session_messages: List[Dict[str, Any]], + context: Dict[str, Any], + *, + effective_user_message: Optional[str] = None, + ) -> AsyncIterator[Tuple[str, Any]]: + async with aclosing(stream_loop.stream(self, session_messages, context, effective_user_message=effective_user_message)) as events: + async for event in events: + yield event + + async def _map_event( + self, ev: Any, *, result_step: Optional[Dict[str, Any]] = None + ) -> AsyncIterator[Tuple[str, Any]]: + async with aclosing(stream_events.map_event(self, ev, result_step=result_step)) as events: + async for event in events: + yield event + + async def stop_producer(self): + """Stop actual model/tool work before closing the stream or its clients.""" + from core.infra.task_lifecycle import cancel_and_join + + task = self._producer_task + if task is not None: + await cancel_and_join(task) + if self._producer_task is task: + self._producer_task = None + + async def shutdown(self): + """Close transient (per-request) MCP clients.""" + from core.llm.mcp.manager import close_clients + + try: + await self.stop_producer() + finally: + try: + await close_clients(self.mcp_clients) + finally: + self.mcp_clients = [] diff --git a/src/backend/core/llm/runtime/streaming_constants.py b/src/backend/core/llm/runtime/streaming_constants.py new file mode 100644 index 00000000..b7acd6ec --- /dev/null +++ b/src/backend/core/llm/runtime/streaming_constants.py @@ -0,0 +1,17 @@ +"""Intervals and patterns used while streaming.""" +from __future__ import annotations +import re + +HTTP_ERR_RE = re.compile(r"HTTP\s*[45]\d\d") + + +MODEL_PROGRESS_MIN_INTERVAL_S = 5.0 + + +QUEUE_POLL_INTERVAL_S = 3.0 + + +TOOL_CALL_DELTA_FLUSH_INTERVAL_S = 0.05 + + +TOOL_CALL_DELTA_FLUSH_CHARS = 256 diff --git a/src/backend/orchestration/tool_callbacks.py b/src/backend/core/llm/runtime/tool_callbacks.py similarity index 70% rename from src/backend/orchestration/tool_callbacks.py rename to src/backend/core/llm/runtime/tool_callbacks.py index 73c909e9..2f89c355 100644 --- a/src/backend/orchestration/tool_callbacks.py +++ b/src/backend/core/llm/runtime/tool_callbacks.py @@ -1,20 +1,4 @@ -"""Tool-call soft warnings (observation only — never blocks). - -Historical background: after the AgentScope migration, tool progress goes through -routing/streaming.py's msg_queue polling, and the old callback mechanism was deprecated. -This module is revived as **observation only**: once the code-execution capability is -enabled in all modes (CODE_CAPABILITY_ENABLED), the default tool set grows larger, and the -model may waver between semantically overlapping tools (docs/code-execution-merge-proposal -§3.4 / §5.5). Here, during the gradual rollout, we record suspicious co-occurrences to help -diagnose whether the model goes off track. **Does not block any call**, and never raises. - -Usage (called once per tool_call by the streaming layer): - - from orchestration.tool_callbacks import note_tool_call - note_tool_call(state, tool_name, tool_args) - -``state`` is an arbitrary per-run mutable dict (held by StreamingAgent); this module manages its own keys. -""" +"Tool-call soft warnings (observation only — never blocks).\n\nHistorical background: after the AgentScope migration, tool progress goes through\nrouting/streaming.py's msg_queue polling, and the old callback mechanism was deprecated.\nThis module is revived as **observation only**: once the code-execution capability is\nenabled in all modes (CODE_CAPABILITY_ENABLED), the default tool set grows larger, and the\nmodel may waver between semantically overlapping tools (docs/code-execution-merge-proposal\n§3.4 / §5.5). Here, during the gradual rollout, we record suspicious co-occurrences to help\ndiagnose whether the model goes off track. **Does not block any call**, and never raises.\n\nUsage (called once per tool_call by the streaming layer):\n\n from core.llm.runtime.tool_callbacks import note_tool_call\n note_tool_call(state, tool_name, tool_args)\n\n``state`` is an arbitrary per-run mutable dict (held by StreamingAgent); this module manages its own keys.\n" from __future__ import annotations @@ -69,8 +53,7 @@ def note_tool_call( sorted(seen), ) - if "Read" in seen and "view_text_file" in seen \ - and "read_vs_view" not in warned: + if "Read" in seen and "view_text_file" in seen and "read_vs_view" not in warned: warned.add("read_vs_view") logger.warning( "[tool-soft-warn] Read 与 view_text_file 同一 run 共现 " @@ -83,7 +66,8 @@ def note_tool_call( logger.warning( "[tool-soft-warn] %s 写 /myspace 路径 %s " "—— 合并后最高风险点（§13 硬保险负责拦截），灰度观测", - tool_name, p, + tool_name, + p, ) break except Exception: # noqa: BLE001 — observation must never affect the main flow diff --git a/src/backend/core/llm/runtime/tool_execution_adapter.py b/src/backend/core/llm/runtime/tool_execution_adapter.py new file mode 100644 index 00000000..07de06c5 --- /dev/null +++ b/src/backend/core/llm/runtime/tool_execution_adapter.py @@ -0,0 +1,210 @@ +"""SDK stream and payload translation for the injected durable tool executor.""" +import asyncio +import json +import logging +import time +from contextlib import suppress +from typing import Any +from agentscope.message import ToolResultState +from agentscope.tool._response import ToolChunk, ToolResponse +from core.harness.tool_call_context import CURRENT_RUN_BINDING +from core.llm.context.rendering import render_text_block + +logger = logging.getLogger(__name__) + + +class DurableToolDispatch: + """Commit durable Intent before every harness-owned tool invocation. + + The final ``ToolResponse`` is committed before it leaves this middleware, + so the existing AgentScope event/SSE mapping remains unchanged while the + database becomes authoritative for replay and reconciliation. + """ + + def __init__(self, session_factory=None, registry=None) -> None: # noqa: ANN001 + from core.db.engine import SessionLocal + from core.llm.ports import tool_effect_runtime + + self._gateway = tool_effect_runtime(session_factory, registry) + from core.harness.ledger import HarnessUsageLedger + + self._usage_ledger = HarnessUsageLedger(session_factory or SessionLocal) + + @staticmethod + def _args(tool_call) -> dict: # noqa: ANN001 + raw = getattr(tool_call, "input", "") or "{}" + try: + parsed = json.loads(raw) if isinstance(raw, str) else raw + except (TypeError, json.JSONDecodeError): + parsed = {"_raw": str(raw)} + return parsed if isinstance(parsed, dict) else {"_value": parsed} + + @staticmethod + def _response_payload(response: ToolResponse) -> dict: + return {"tool_response": response.model_dump(mode="json")} + + @staticmethod + def _response_from_payload(payload: Any) -> ToolResponse: + from agentscope.message import DataBlock, TextBlock + + raw = payload.get("tool_response") if isinstance(payload, dict) else None + if not isinstance(raw, dict): + return ToolResponse( + content=[render_text_block(json.dumps(payload, ensure_ascii=False, default=str))], + state=ToolResultState.SUCCESS, + ) + blocks = [] + for item in raw.get("content") or []: + if not isinstance(item, dict): + blocks.append(render_text_block(str(item))) + elif item.get("type") == "data": + blocks.append(DataBlock.model_validate(item)) + else: + blocks.append(TextBlock.model_validate(item)) + state = ToolResultState(str(raw.get("state") or "success")) + return ToolResponse( + id=str(raw.get("id") or ""), + content=blocks, + state=state, + metadata=dict(raw.get("metadata") or {}), + ) + + async def stream(self, agent: Any, input_kwargs: dict, next_handler): # noqa: ANN001 + run_id = str(getattr(agent.state, "run_id", "") or "") + run_owner = str(getattr(agent.state, "journal_owner", "") or "") + tool_call = input_kwargs.get("tool_call") + tool_name = str(getattr(tool_call, "name", "") or "") + tool_call_id = str(getattr(tool_call, "id", "") or "") + if not run_id or not run_owner or not tool_name: + raise self._gateway.binding_error( + "tool execution requires a durable run_id and journal_owner binding" + ) + + args = self._args(tool_call) + from core.harness.registry.tool_operations import prepare_tool_arguments + args = prepare_tool_arguments(tool_name, args, str(getattr(agent.state, "user_id", "") or ""), + str(getattr(agent.state, "chat_id", "") or "")) + live_chunks: asyncio.Queue = asyncio.Queue() + + async def _invoke(): + from core.harness.usage import ( + UsageAttempt, + attempt_status_for_exception, + record_usage_safely, + ) + from core.harness.tool_effect_context import CURRENT_TOOL_EFFECT + + final = None + started = time.monotonic() + usage_status = "failed" + binding_token = CURRENT_RUN_BINDING.set((run_id, run_owner)) + invoke_kwargs = input_kwargs + effect = CURRENT_TOOL_EFFECT.get() + if effect is not None and tool_name in { + "create_scheduled_task", + "update_scheduled_task", + "delete_scheduled_task", + }: + adapter_args = dict(args) + adapter_args["tool_effect_id"] = effect.effect_id + adapter_call = tool_call.model_copy( + update={"input": json.dumps(adapter_args, ensure_ascii=False)} + ) + invoke_kwargs = {**input_kwargs, "tool_call": adapter_call} + try: + outcome_unknown = False + async for item in next_handler(**invoke_kwargs): + # Toolkit normally converts adapter exceptions into ordinary + # error responses. An ambiguous remote write must retain its + # ledger meaning across that conversion and stop this run. + ambiguous = bool( + (getattr(item, "metadata", None) or {}).get("gateway_outcome_unknown") + ) + outcome_unknown = outcome_unknown or ambiguous + if isinstance(item, ToolResponse): + final = item + elif not ambiguous: + await live_chunks.put(item) + if outcome_unknown: + raise RuntimeError("cloud write outcome unknown; reconciliation required") + usage_status = ( + "success" + if final is not None and final.state == ToolResultState.SUCCESS + else "failed" + ) + except asyncio.CancelledError: + usage_status = "cancelled" + raise + except Exception as exc: + usage_status = attempt_status_for_exception(exc) + raise + finally: + CURRENT_RUN_BINDING.reset(binding_token) + if effect is not None: + try: + await record_usage_safely( + self._usage_ledger, + UsageAttempt( + run_id=run_id, + kind="tool", + operation_name=tool_name, + effect_id=effect.effect_id, + status=usage_status, + latency_ms=int((time.monotonic() - started) * 1_000), + metadata={"tool_call_id": tool_call_id}, + ), + ) + except Exception: # usage cannot change tool execution + logger.debug("tool usage attempt persistence failed", exc_info=True) + if final is None: + raise self._gateway.response_error(f"tool {tool_name} ended without ToolResponse") + return self._response_payload(final) + + def _response_failed(payload: Any) -> bool: + raw = payload.get("tool_response") if isinstance(payload, dict) else None + return bool(isinstance(raw, dict) and str(raw.get("state") or "success") != "success") + + task = asyncio.create_task( + self._gateway.execute_outcome( + run_id=run_id, + owner=run_owner, + tool_call_id=tool_call_id, + tool_name=tool_name, + args=args, + invoke=_invoke, + classify_failure=_response_failed, + ) + ) + try: + while not task.done() or not live_chunks.empty(): + try: + item = await asyncio.wait_for(live_chunks.get(), timeout=0.02) + except asyncio.TimeoutError: + continue + yield item + outcome = await task + finally: + if not task.done(): + task.cancel() + with suppress(BaseException): + await task + elif not task.cancelled(): + task.exception() + response = self._response_from_payload(outcome.result) + if outcome.result_state == "error" and response.state == ToolResultState.SUCCESS: + response.state = ToolResultState.ERROR + linkage = { + "effect_id": outcome.effect_id, + "result_id": outcome.result_id, + } + response.metadata = {**dict(response.metadata or {}), **linkage} + links = getattr(agent.state, "tool_effect_links", None) + if isinstance(links, dict): + links[tool_call_id] = linkage + if not outcome.invoked: + yield ToolChunk( + content=list(response.content), + state=response.state, + metadata=dict(response.metadata), + ) + yield response diff --git a/src/backend/core/llm/runtime/tool_hook_adapter.py b/src/backend/core/llm/runtime/tool_hook_adapter.py new file mode 100644 index 00000000..71973e92 --- /dev/null +++ b/src/backend/core/llm/runtime/tool_hook_adapter.py @@ -0,0 +1,81 @@ +"""Translate neutral tool decisions; arguments remain immutable after admission.""" +from agentscope.tool._response import ToolResponse +from core.harness.events import thaw_value +from core.harness.hooks import DecisionKind, HookRejected, HookPaused, HookStage, Invocation +from core.llm.runtime.context_hook_adapter import append_hook_context + + +async def _tool_admission(adapter, agent, tool_call, args, plain): + state = agent.state + outcome = await adapter.bus.run(Invocation.create( + run_id=adapter._run_id(agent), stage=HookStage.BEFORE_TOOL, + operation_name=str(getattr(tool_call, "name", "") or "tool"), + data={"tool_args": args, "raw_input": plain(getattr(tool_call, "input", None)), + "ontology_runtime": plain(getattr(state, "ontology_runtime", None)), + "identity": {"journal_owner": getattr(state, "journal_owner", ""), "user_id": getattr(state, "user_id", ""), "chat_id": getattr(state, "chat_id", "")}, + "context_items": [], "metadata": {"tool_call_id": getattr(tool_call, "id", "")}}, + )) + updated = outcome.invocation + if updated.data.get("ontology_runtime") is not None: + state.ontology_runtime = thaw_value(updated.data["ontology_runtime"]) + state.ontology_enabled = bool(state.ontology_runtime.get("enabled")) + append_hook_context(agent, updated) + response = updated.data.get("tool_response") + if outcome.decision.kind == DecisionKind.REJECT: + if response is None: + raise HookRejected(outcome.decision.reason) + return ToolResponse.model_validate(thaw_value(response)) + if outcome.decision.kind == DecisionKind.PAUSE: + raise HookPaused(outcome.decision.reason) + return None + + +async def permission_dispatch(adapter, agent, input_kwargs, next_handler): + from contextlib import aclosing, nullcontext + call_id = str(getattr(input_kwargs.get("tool_call"), "id", "") or "") + policy = adapter.permission_policy + scope = policy.bind(call_id) if policy is not None else nullcontext(None) + async def dispatch(**kwargs): + async with aclosing(execute_tool(adapter, agent, kwargs, next_handler)) as executed: + async for event in executed: + yield event + with scope as audit: + async with aclosing(dispatch(**input_kwargs)) as stream: + async for item in stream: + if isinstance(item, ToolResponse) and audit is not None: + item.metadata = {**dict(item.metadata or {}), "permission": audit} + yield item + + +async def tool_admission(adapter, agent, tool_call, args, plain): + call_id = str(getattr(tool_call, "id", "") or "") + try: + result = await _tool_admission(adapter, agent, tool_call, args, plain) + except BaseException: + if adapter.permission_policy is not None: + adapter.permission_policy.discard(call_id) + raise + if result is not None and adapter.permission_policy is not None: + adapter.permission_policy.discard(call_id) + return result + + +async def execute_tool(adapter, agent, input_kwargs, next_handler): + import asyncio + from contextlib import aclosing + from core.harness.tool_call_context import CURRENT_TOOL_CALL_ID + call = input_kwargs.get("tool_call") + call_id = str(getattr(call, "id", "") or "") + prepared = getattr(agent, "_jx_prepared_capabilities", None) + if prepared is not None: + await asyncio.to_thread(prepared.validate, user_id=str(getattr(agent.state, "user_id", "") or "")) + await asyncio.to_thread(prepared.record_tool_scope, call_id, str(getattr(call, "name", "") or "")) + token = CURRENT_TOOL_CALL_ID.set(call_id) + try: + stream = (adapter.tool_dispatch.stream(agent, input_kwargs, next_handler) + if adapter.tool_dispatch is not None else next_handler(**input_kwargs)) + async with aclosing(stream): + async for item in stream: + yield item + finally: + CURRENT_TOOL_CALL_ID.reset(token) diff --git a/src/backend/core/llm/tool_media_store.py b/src/backend/core/llm/runtime/tool_media_store.py similarity index 95% rename from src/backend/core/llm/tool_media_store.py rename to src/backend/core/llm/runtime/tool_media_store.py index 724efeda..a1f12fc3 100644 --- a/src/backend/core/llm/tool_media_store.py +++ b/src/backend/core/llm/runtime/tool_media_store.py @@ -36,6 +36,7 @@ import hashlib import logging from collections import OrderedDict +from dataclasses import dataclass from typing import Any, Dict, Iterable, List, Mapping, Optional, Sequence from core.db.engine import SessionLocal @@ -54,7 +55,6 @@ # 命中率本就有限，而这份缓存是每进程一份，调大等于按 worker 数翻倍占用常驻内存。 _CACHE_MAX_BYTES = 32 * 1024 * 1024 _cache: "OrderedDict[str, str]" = OrderedDict() -_cache_bytes = 0 def _cache_get(digest: str) -> Optional[str]: @@ -65,14 +65,13 @@ def _cache_get(digest: str) -> Optional[str]: def _cache_put(digest: str, data_b64: str) -> None: - global _cache_bytes if digest in _cache: return _cache[digest] = data_b64 - _cache_bytes += len(data_b64) - while _cache_bytes > _CACHE_MAX_BYTES and len(_cache) > 1: + process_state().cache_bytes += len(data_b64) + while process_state().cache_bytes > _CACHE_MAX_BYTES and len(_cache) > 1: _, evicted = _cache.popitem(last=False) - _cache_bytes -= len(evicted) + process_state().cache_bytes -= len(evicted) def _data_block(block: Mapping[str, Any], source: Dict[str, Any]) -> Dict[str, Any]: @@ -236,6 +235,17 @@ def resolved(block: Mapping[str, Any]) -> Dict[str, Any]: def reset_cache() -> None: """测试用：清空取回缓存。""" - global _cache_bytes _cache.clear() - _cache_bytes = 0 + process_state().cache_bytes = 0 + + +@dataclass +class ToolMediaStoreState: + cache_bytes: object = 0 + + +_STATE = ToolMediaStoreState() + + +def process_state() -> ToolMediaStoreState: + return _STATE diff --git a/src/backend/core/llm/runtime/tool_result_adapter.py b/src/backend/core/llm/runtime/tool_result_adapter.py new file mode 100644 index 00000000..bc993eec --- /dev/null +++ b/src/backend/core/llm/runtime/tool_result_adapter.py @@ -0,0 +1,53 @@ +"""SDK result buffering and conversion around neutral tool hooks.""" +import json +from contextlib import aclosing +from agentscope.tool._response import ToolChunk, ToolResponse +from core.harness.events import thaw_value +from core.harness.hooks import HookStage +from core.llm.citations.hook import citation_candidate +from core.llm.citations.allocator import resolve_allocator +from core.llm.runtime.context_hook_adapter import append_hook_context +from core.llm.runtime.tool_hook_adapter import tool_admission, permission_dispatch + + +async def run_tool(adapter, agent, input_kwargs, next_handler, plain): + call = input_kwargs.get("tool_call") + raw = getattr(call, "input", "") or "{}" + try: + args = json.loads(raw) if isinstance(raw, str) else raw + except (TypeError, json.JSONDecodeError): + args = {"_raw": str(raw)} + name = str(getattr(call, "name", "") or "tool") + final = await tool_admission(adapter, agent, call, args, plain) + buffered = [] + buffer_result = citation_candidate(name) + exposed = False + if final is not None: + buffered.append(ToolChunk(content=list(final.content), state=final.state, metadata=dict(final.metadata or {}))) + else: + async with aclosing(permission_dispatch(adapter, agent, input_kwargs, next_handler)) as stream: + async for event in stream: + if isinstance(event, ToolResponse): + final = event + elif buffer_result: + buffered.append(event) + else: + exposed = True + yield event + if buffer_result: + resolve_allocator(agent) + observed = await adapter._stage( + HookStage.AFTER_TOOL, agent, operation_name=name, + data={"tool_result": plain(final), "context_items": [], "result_exposed": exposed, + "metadata": {"tool_call_id": str(getattr(call, "id", "") or "")}}, + ) + append_hook_context(agent, observed) + patched = thaw_value(observed.data.get("tool_result")) + if patched is not None and patched != plain(final): + final = ToolResponse.model_validate(patched) + yield ToolChunk(content=list(final.content), state=final.state, metadata=dict(final.metadata or {})) + else: + for event in buffered: + yield event + if final is not None: + yield final diff --git a/src/backend/core/llm/runtime/turn_state.py b/src/backend/core/llm/runtime/turn_state.py new file mode 100644 index 00000000..f3b69d78 --- /dev/null +++ b/src/backend/core/llm/runtime/turn_state.py @@ -0,0 +1,41 @@ +"""Per-turn trackers for artifact reads and pin hints.""" +from __future__ import annotations +from contextvars import ContextVar +from typing import Any, Dict, Optional + +def get_artifact_read_state() -> Dict[str, int]: + """Return the current per-turn read tracker, lazily initializing it. + + Used by ``read_artifact`` to enforce the cumulative cap. + """ + state = artifact_read_chars_per_turn.get() + if state is None: + state = {} + artifact_read_chars_per_turn.set(state) + return state + + +def reset_artifact_read_state() -> None: + """Clear the per-turn read tracker. Called from file_context_pre_reply.""" + artifact_read_chars_per_turn.set({}) + + +def reset_pin_hint_state() -> None: + """Clear the per-turn pin-reminder cache. Called at turn boundary.""" + pin_hint_state.set({"seen": set(), "last_reminded_sig": ""}) + + +def get_pin_hint_state() -> Dict[str, Any]: + state = pin_hint_state.get() + if state is None: + state = {"seen": set(), "last_reminded_sig": ""} + pin_hint_state.set(state) + return state + + +artifact_read_chars_per_turn: ContextVar[Optional[Dict[str, int]]] = ContextVar( + "_artifact_read_chars_per_turn", default=None +) + + +pin_hint_state: ContextVar[Optional[Dict[str, Any]]] = ContextVar("_pin_hint_state", default=None) diff --git a/src/backend/core/llm/workspace.py b/src/backend/core/llm/runtime/workspace.py similarity index 100% rename from src/backend/core/llm/workspace.py rename to src/backend/core/llm/runtime/workspace.py diff --git a/src/backend/core/agent_skills/selector.py b/src/backend/core/llm/skill_selection.py similarity index 91% rename from src/backend/core/agent_skills/selector.py rename to src/backend/core/llm/skill_selection.py index f7bbf3a6..1a75dba0 100644 --- a/src/backend/core/agent_skills/selector.py +++ b/src/backend/core/llm/skill_selection.py @@ -10,18 +10,13 @@ from agentscope.model import OpenAIChatModel -from core.agent_skills.loader import get_skill_loader -from core.agent_skills.registry import AgentSkillMetadata -from core.llm.context_adapter import AgentScopeContextAdapter -from core.llm.context_ir import ( - KIND_SYSTEM_RULE, - KIND_USER_INPUT, - POLICY_HEAD_TAIL, - POLICY_NEVER, - ContextAssembler, - make_text_context_item, -) -from core.llm.context_manager import FALLBACK_CONTEXT_WINDOW, usable_context_window +from core.capabilities.skills.loader_factory import get_skill_loader +from core.capabilities.skills.registry import AgentSkillMetadata +from core.llm.context.adapter import AgentScopeContextAdapter +from core.llm.context.ir_constants import KIND_SYSTEM_RULE, KIND_USER_INPUT, POLICY_HEAD_TAIL, POLICY_NEVER +from core.llm.context.ir import ContextAssembler +from core.llm.context.items import make_text_context_item +from core.models.context_window import FALLBACK_CONTEXT_WINDOW, usable_context_window def select_skills_for_query( @@ -62,7 +57,7 @@ def select_skills_for_query( return sorted([s.id for s in candidates])[:max_skills] # Use LLM to select relevant skills - return _llm_select_skills( + return rank_skill_candidates( user_query=user_query, candidates=candidates, model=model, @@ -70,7 +65,7 @@ def select_skills_for_query( ) -def _llm_select_skills( +def rank_skill_candidates( *, user_query: str, candidates: List[AgentSkillMetadata], diff --git a/src/backend/core/llm/steer_middleware.py b/src/backend/core/llm/steer_middleware.py deleted file mode 100644 index 2309277b..00000000 --- a/src/backend/core/llm/steer_middleware.py +++ /dev/null @@ -1,96 +0,0 @@ -"""Deliver user steering at safe model/tool boundaries.""" -import asyncio -from agentscope.agent import Agent -from agentscope.message import ToolResultState -from agentscope.middleware import MiddlewareBase -from agentscope.tool._response import ToolChunk, ToolResponse -from core.llm.context_adapter import append_context_text, render_text_block -from core.llm.context_ir import KIND_STEER, POLICY_NEVER - - -class SteerMiddleware(MiddlewareBase): - """Deliver a queued user instruction at the next safe ReAct boundary. - - ``on_reasoning`` is the primary insertion point: AgentScope calls it after - the previous tool results have entered context and immediately before the - next model call. ``on_acting`` is the earlier fallback for a steer that - arrives while the model is constructing its next tool call; it marks that - not-yet-started tool batch interrupted so the next reasoning round can - replan. Both paths preserve a valid assistant-tool-result-user order. - """ - - def __init__(self, take_pending=None, acknowledge=None) -> None: - self._take_pending = take_pending - self._acknowledge = acknowledge - self._lock = asyncio.Lock() - self._delivery: dict | None = None - self._interrupted_tools = False - - async def on_acting(self, agent: Agent, input_kwargs: dict, next_handler): # noqa: ANN001 - run_id = str(getattr(agent.state, "run_id", "") or "") - if not run_id: - async for item in next_handler(**input_kwargs): - yield item - return - - async with self._lock: - if self._delivery is None: - from core.services.chat_steer_service import take_pending_steer - - self._delivery = await (self._take_pending or take_pending_steer)(run_id) - self._interrupted_tools = self._delivery is not None - delivery = self._delivery - - if delivery is None: - async for item in next_handler(**input_kwargs): - yield item - return - - notice = "用户追加了新指令；本工具调用已在执行前中止，等待模型按新指令重新规划。" - block = render_text_block(notice) - yield ToolChunk(content=[block], state=ToolResultState.INTERRUPTED) - yield ToolResponse(content=[block], state=ToolResultState.INTERRUPTED) - - async def on_reasoning(self, agent: Agent, input_kwargs: dict, next_handler): # noqa: ANN001 - run_id = str(getattr(agent.state, "run_id", "") or "") - async with self._lock: - # A steer can arrive while a long-running tool is executing. There - # is no later on_acting hook in that round, so poll again here, - # after AgentScope saved the tool result and before it starts the - # next model call. This is the normal "insert after this tool" - # path; without it the steer waits until another tool call (or the - # whole run) finishes. - if self._delivery is None and run_id: - from core.services.chat_steer_service import take_pending_steer - - self._delivery = await (self._take_pending or take_pending_steer)(run_id) - delivery = self._delivery - interrupted_tools = self._interrupted_tools - self._delivery = None - self._interrupted_tools = False - - if delivery is not None: - message = str(delivery.get("message") or "").strip() - if message: - append_context_text( - agent, - "[用户在当前执行中追加的新指令]\n" - f"{message}\n" - + ( - "请立即按这条新指令调整后续计划；不要继续已经被中止的旧工具调用。" - if interrupted_tools - else "请立即按这条新指令调整后续计划；上一轮工具结果已经完成，可按需使用。" - ), - kind=KIND_STEER, - origin="user:steer", - trust="user", - priority=950, - token_budget=4_000, - truncation_policy=POLICY_NEVER, - ) - if self._acknowledge is not None: - await self._acknowledge(delivery) - agent.state.steer_delivery = dict(delivery) - - async for item in next_handler(**input_kwargs): - yield item diff --git a/src/backend/core/llm/subagents/__init__.py b/src/backend/core/llm/subagents/__init__.py new file mode 100644 index 00000000..015345e9 --- /dev/null +++ b/src/backend/core/llm/subagents/__init__.py @@ -0,0 +1 @@ +"""Sub-agents: the call tool, their sessions, prompts and streamed execution.""" diff --git a/src/backend/core/llm/_subagent_execution.py b/src/backend/core/llm/subagents/execution.py similarity index 89% rename from src/backend/core/llm/_subagent_execution.py rename to src/backend/core/llm/subagents/execution.py index 6a995fcf..2f7cb4bb 100644 --- a/src/backend/core/llm/_subagent_execution.py +++ b/src/backend/core/llm/subagents/execution.py @@ -10,21 +10,14 @@ import logging import threading import time -import uuid -from concurrent.futures import ThreadPoolExecutor from typing import Any, Callable, Dict, List, Optional, Tuple -from agentscope.message import TextBlock -from agentscope.tool import Toolkit # AgentScope 2.0: tool functions must return ToolChunk (call_tool rejects ToolResponse). -from agentscope.tool._response import ToolChunk as ToolResponse -from core.infra.task_lifecycle import settle_task -from core.llm import subagent_sessions -from core.llm.context_adapter import next_request_sequence, render_context_item -from core.llm.context_ir import KIND_USER_INPUT, POLICY_NEVER, make_text_context_item +from core.llm.context.rendering import next_request_sequence, render_context_item +from core.llm.context.ir_constants import KIND_USER_INPUT, POLICY_NEVER +from core.llm.context.items import make_text_context_item from core.llm.tools._common import resolve_sandbox_session -from core.services import log_service as log_writer logger = logging.getLogger(__name__) @@ -209,7 +202,7 @@ def _run_subagent_in_thread( or they never reach the user-visible assistant message. When ``emit`` is provided, the sub-agent is consumed **in streaming mode** - (``agent._reply``, event by event) and every thinking/tool_call/tool_result/content + (``agent.reply_stream``, event by event) and every thinking/tool_call/tool_result/content sub-step is bypass-forwarded back to the main SSE stream via ``emit``; with ``emit`` None the behavior matches the old implementation (final text only). The returned text is always taken from the reply's **final Msg** — fully equivalent to the old @@ -219,12 +212,12 @@ def _run_subagent_in_thread( async def _inner() -> str: from agentscope.message import Msg from core.db.engine import SessionLocal - from core.llm.builtin_subagents import build_builtin_runtime_profile, get_builtin_subagent + from core.capabilities.agents.builtin import build_builtin_runtime_profile, get_builtin_subagent from core.llm.factory import create_agent_executor - from core.llm.mcp_manager import close_clients - from core.llm.message_compat import extract_messages_from_context, session_to_msgs - from core.services.user_agent_service import UserAgentService - from orchestration.streaming import StreamingAgent + from core.llm.mcp.manager import close_clients + from core.llm.runtime.message_compat import extract_messages_from_context, session_to_msgs + from core.llm.runtime.streaming import StreamingAgent + from core.capabilities.agents.edition import UserAgentService builtin_spec = get_builtin_subagent(agent_id) if builtin_spec is not None: @@ -251,7 +244,7 @@ async def _inner() -> str: runtime.get("sandbox_session_id"), runtime.get("chat_id"), ) - from core.services.local_project_workspace import child_project_context + from core.projects.local_workspace import child_project_context agent, mcp_clients = await create_agent_executor( user_agent=user_agent, @@ -278,7 +271,7 @@ async def _inner() -> str: # Initialize one so the sub-agent's pin_to_workspace has somewhere to land; after the # run, get_pinned() hands the files back to the main context for re-pinning — # otherwise files pinned to the workspace are lost and never shown in the main conversation. - from core.llm import workspace as _ws + from core.llm.runtime import workspace as _ws _ws.init_state() @@ -323,15 +316,8 @@ async def _inner() -> str: extract_messages_from_context(agent.state.context), ) - # Streaming path: consume _reply directly (reply_stream would discard the final - # Msg, which we need as the authoritative return text). Non-Msg events are mapped - # to sub-steps and bypass-forwarded via emit. - final_msg = None - mapper = _SubMapper() - stream_mapper = StreamingAgent(agent, mcp_clients) - stream_mapper._enable_thinking = True - _last_progress = time.monotonic() - async for chunk in agent._reply(inputs=user_msg): + # Public stream includes the final message for child result extraction. + async for chunk in agent.reply_stream(inputs=user_msg, yield_final_msg=True): if isinstance(chunk, Msg): final_msg = chunk continue @@ -373,8 +359,8 @@ async def _owned_inner(): return await _inner() finally: from core.infra.redis import close_redis - from core.llm.chat_models import close_loop_http_clients - from core.llm.hooks import release_loop_models + from core.models.chat_models import close_loop_http_clients + from core.llm.runtime.model_cache import release_loop_models release_loop_models() try: diff --git a/src/backend/core/llm/_subagent_prompts.py b/src/backend/core/llm/subagents/prompts.py similarity index 100% rename from src/backend/core/llm/_subagent_prompts.py rename to src/backend/core/llm/subagents/prompts.py diff --git a/src/backend/core/llm/subagent_sessions.py b/src/backend/core/llm/subagents/sessions.py similarity index 98% rename from src/backend/core/llm/subagent_sessions.py rename to src/backend/core/llm/subagents/sessions.py index 7cac4aa9..465546d3 100644 --- a/src/backend/core/llm/subagent_sessions.py +++ b/src/backend/core/llm/subagents/sessions.py @@ -18,7 +18,7 @@ from typing import Any, Dict, List, Optional from core.infra.ephemeral import get_ephemeral_state -from core.llm.human_interaction import STREAM_TTL_SECONDS +from core.llm.runtime.human_interaction import STREAM_TTL_SECONDS logger = logging.getLogger(__name__) diff --git a/src/backend/core/llm/_subagent_stream.py b/src/backend/core/llm/subagents/stream.py similarity index 80% rename from src/backend/core/llm/_subagent_stream.py rename to src/backend/core/llm/subagents/stream.py index d06d03c8..c21dd513 100644 --- a/src/backend/core/llm/_subagent_stream.py +++ b/src/backend/core/llm/subagents/stream.py @@ -1,14 +1,14 @@ """Sub-agent streaming event side-channel — relays call_subagent's internal thinking / tool_call / tool_result / content events into the main chat SSE stream in real time. -Background: ``call_subagent`` (core/llm/subagent_tool.py) runs the sub-agent in a +Background: ``call_subagent`` (core/llm/subagents/tool.py) runs the sub-agent in a **separate thread + separate event loop** (to avoid anyio cancel-scope cross-task errors). The sub-agent used to execute one-shot via ``agent.reply()``, swallowing all intermediate events; the frontend only saw a single "call sub-agent" tool card and the final summary text, with no visibility into its internal tool calls. This module provides a per-chat side-channel shaped after the proven ``_tool_confirmation``: -- ``stream()`` (orchestration/streaming.py) calls ``attach`` on startup with its own +- ``stream()`` (core/llm/runtime/streaming.py) calls ``attach`` on startup with its own ``event_q`` + the loop it runs on, and ``detach`` on shutdown. - ``call_subagent`` and its worker thread deliver sub-agent events back to the main ``event_q`` via ``push`` — using ``loop.call_soon_threadsafe`` for **cross-thread @@ -29,7 +29,7 @@ import asyncio import logging import threading -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import Any, Dict, Optional logger = logging.getLogger(__name__) @@ -46,24 +46,20 @@ class _Bus: queue: asyncio.Queue -_LOCK = threading.RLock() -_BUSES: Dict[str, _Bus] = {} - - def _key(chat_id: Optional[str]) -> str: return str(chat_id) if chat_id else _NO_CHAT def attach(chat_id: Optional[str], loop: asyncio.AbstractEventLoop, queue: asyncio.Queue) -> None: """Register this run's event outlet (event_q + its loop) when stream() starts.""" - with _LOCK: - _BUSES[_key(chat_id)] = _Bus(loop=loop, queue=queue) + with process_state().LOCK: + process_state().BUSES[_key(chat_id)] = _Bus(loop=loop, queue=queue) def detach(chat_id: Optional[str]) -> None: """Deregister when stream() winds down — prevents late events from writing to a finished run's event_q.""" - with _LOCK: - _BUSES.pop(_key(chat_id), None) + with process_state().LOCK: + process_state().BUSES.pop(_key(chat_id), None) def push(chat_id: Optional[str], payload: Dict[str, Any]) -> None: @@ -77,8 +73,8 @@ def push(chat_id: Optional[str], payload: Dict[str, Any]) -> None: never registered) or the loop is already closed; never raises and drags down the sub-agent. """ - with _LOCK: - bus = _BUSES.get(_key(chat_id)) + with process_state().LOCK: + bus = process_state().BUSES.get(_key(chat_id)) if bus is None: return try: @@ -92,5 +88,18 @@ def push(chat_id: Optional[str], payload: Dict[str, Any]) -> None: def is_active(chat_id: Optional[str]) -> bool: """Whether the chat currently has an active stream listening — lets call_subagent decide whether to emit streaming events.""" - with _LOCK: - return _key(chat_id) in _BUSES + with process_state().LOCK: + return _key(chat_id) in process_state().BUSES + + +@dataclass +class StreamState: + LOCK: object = field(default_factory=lambda: threading.RLock()) + BUSES: Dict[str, _Bus] = field(default_factory=lambda: {}) + + +_STATE = StreamState() + + +def process_state() -> StreamState: + return _STATE diff --git a/src/backend/core/llm/subagent_tool.py b/src/backend/core/llm/subagents/tool.py similarity index 89% rename from src/backend/core/llm/subagent_tool.py rename to src/backend/core/llm/subagents/tool.py index 16c9b4f8..2a14c825 100644 --- a/src/backend/core/llm/subagent_tool.py +++ b/src/backend/core/llm/subagents/tool.py @@ -5,54 +5,40 @@ """ from __future__ import annotations +from core.llm.factory.tools.definitions import function_tool +from core.harness import log_writer import asyncio import logging -import threading import time import uuid from concurrent.futures import ThreadPoolExecutor -from typing import Any, Callable, Dict, List, Optional, Tuple +from typing import Any, Callable, Dict, List, Optional from agentscope.message import TextBlock from agentscope.tool import Toolkit -# AgentScope 2.0: tool functions must return ToolChunk (call_tool rejects ToolResponse). -from agentscope.tool._response import ToolChunk as ToolResponse +# AgentScope 2.0: tool functions must return ToolChunk (call_tool rejects ToolChunk). +from agentscope.tool._response import ToolChunk from core.infra.task_lifecycle import settle_task -from core.llm import subagent_sessions -from core.llm.context_adapter import next_request_sequence, render_context_item -from core.llm.context_ir import KIND_USER_INPUT, POLICY_NEVER, make_text_context_item -from core.llm.tools._common import resolve_sandbox_session -from core.services import log_service as log_writer +from core.llm.subagents import sessions as subagent_sessions -logger = logging.getLogger(__name__) -from core.llm._subagent_execution import ( - _ChildExecution, - _map_subagent_event, - _run_subagent_in_thread, - _shared_ontology_runtime, - _SubMapper, -) -from core.llm._subagent_prompts import ( - _collapse_ids, - _get_tools_desc, - build_explicit_subagent_command_hint, - build_subagent_mention_hint, - build_subagent_prompt_section, -) +from core.llm.subagents.execution import _ChildExecution, _map_subagent_event, _run_subagent_in_thread, _shared_ontology_runtime, _SubMapper +from core.llm.subagents.prompts import _get_tools_desc as get_tools_desc, build_explicit_subagent_command_hint, build_subagent_mention_hint, build_subagent_prompt_section + +logger = logging.getLogger(__name__) _subagent_pool = ThreadPoolExecutor(max_workers=16, thread_name_prefix="subagent") def _child_capability_runtime(parent_runtime, agent_id): runtime = dict(parent_runtime or {}) - from core.capabilities.paths import capabilities_enabled + from core.harness.registry.run_capabilities import has_run_capabilities - if runtime.get("run_id") and capabilities_enabled(): - from core.capabilities.runtime import child_scope - from core.llm.middlewares import CURRENT_TOOL_CALL_ID + if runtime.get("run_id") and has_run_capabilities(): + from core.harness.registry.capability_runtime import child_scope + from core.harness.tool_call_context import CURRENT_TOOL_CALL_ID # The ledger's persisted tool-call identity is stable on replay. SSE's # random sub_run_id is presentation-only and never defines authority. @@ -80,7 +66,7 @@ def register_subagent_tool( agent instance after creation. Used to extract shared context for sub-agents that have ``extra_config.shared_context == True``. chat_id: This run's chat_id — sub-agent streaming events are bypass-routed back - to the main SSE stream by it (see core/llm/_subagent_stream.py). + to the main SSE stream by it (see core/llm/subagents/stream.py). """ agent_map = {a["agent_id"]: a for a in visible_agents} @@ -89,7 +75,7 @@ async def call_subagent( task: str, context_summary: str = "", resume_session_id: str = "", - ) -> ToolResponse: + ) -> ToolChunk: """调用子智能体执行专业任务。子智能体拥有独立的工具和专业知识。 需要并行调用多个子智能体时，在同一轮回复中生成多个 call_subagent 调用， @@ -109,12 +95,12 @@ async def call_subagent( 完整上下文（读过的文件、跑过的命令、犯过的错）上继续；留空则全新启动。 Returns: - `ToolResponse`: + `ToolChunk`: 子智能体的执行结果，末尾附带本次的续跑句柄。 结果对用户不可见，你需要汇总后呈现给用户。 """ if agent_id not in agent_map: - return ToolResponse( + return ToolChunk( content=[ TextBlock( type="text", @@ -136,7 +122,7 @@ async def call_subagent( user_id=current_user_id, ) if resume_messages is None: - return ToolResponse( + return ToolChunk( content=[ TextBlock( type="text", @@ -154,7 +140,7 @@ async def call_subagent( shared_messages = None if not resume_messages and shared_context and agent_ref and agent_ref.get("agent"): try: - from core.llm.message_compat import extract_messages_from_context + from core.llm.runtime.message_compat import extract_messages_from_context # 2.0: agent.memory → agent.state.context; this function is now synchronous shared_messages = extract_messages_from_context(agent_ref["agent"].state.context) @@ -172,7 +158,7 @@ async def call_subagent( # call id (already written into the ContextVar by ActingToolCallIdMiddleware); the # frontend uses it to group sub-steps under the right card; when missing, the # frontend falls back to grouping under the most recent unmatched call_subagent card. - from core.llm import _subagent_stream + from core.llm.subagents import stream as _subagent_stream # Sub-step counting (best-effort): in the streaming bypass, count one tool call per # tool_result received, so the sub-agent call log can show how many tools that call @@ -182,7 +168,7 @@ async def call_subagent( _emit: Optional[Callable[[Dict[str, Any]], None]] = None if bool(chat_id) and _subagent_stream.is_active(chat_id): try: - from core.llm.middlewares import CURRENT_TOOL_CALL_ID + from core.harness.tool_call_context import CURRENT_TOOL_CALL_ID parent_tool_id = CURRENT_TOOL_CALL_ID.get("") or "" except Exception: # noqa: BLE001 @@ -258,13 +244,13 @@ async def record_terminal(): # copy here would let child activations update nested lists while # losing scalar changes such as an escalated review_level. ontology_runtime = _shared_ontology_runtime(agent_ref) - from core.capabilities.errors import IntegrityFailed + from core.harness.capability_errors import IntegrityFailed try: child_runtime = _child_capability_runtime(parent_runtime, agent_id) except IntegrityFailed: await _finish("failed", error="missing durable capability scope") - return ToolResponse( + return ToolChunk( content=[ TextBlock( type="text", text="子智能体缺少可恢复的持久工具调用身份，已拒绝执行。" @@ -296,7 +282,7 @@ async def record_terminal(): # produced by the sub-agent never show up in the main conversation. if sub_pinned: try: - from core.llm import workspace as _ws + from core.llm.runtime import workspace as _ws for _it in sub_pinned: _ws.pin( @@ -312,7 +298,7 @@ async def record_terminal(): if not ok: await _finish("failed", output=text, error=text) - return ToolResponse( + return ToolChunk( content=[ TextBlock( type="text", @@ -345,7 +331,7 @@ async def record_terminal(): if saved else "" ) - return ToolResponse( + return ToolChunk( content=[ TextBlock( type="text", @@ -380,7 +366,7 @@ async def finish_cancelled(): if _emit is not None: _emit({"sub_type": "error", "error": str(e)[:200]}) await _finish("failed", error=str(e)[:200]) - return ToolResponse( + return ToolChunk( content=[ TextBlock( type="text", @@ -389,4 +375,4 @@ async def finish_cancelled(): ] ) - toolkit.register_tool_function(call_subagent, namesake_strategy="skip") + toolkit.append(function_tool(call_subagent)) diff --git a/src/backend/core/llm/tool_collector.py b/src/backend/core/llm/tool_collector.py deleted file mode 100644 index 006312a5..00000000 --- a/src/backend/core/llm/tool_collector.py +++ /dev/null @@ -1,270 +0,0 @@ -"""ToolCollector — adapts 1.x's incremental ``toolkit.register_*`` to 2.0's one-shot Toolkit. - -AgentScope 2.0's ``Toolkit`` is injected once at construction time (``Toolkit(tools=, -mcps=, skills_or_loaders=)``); it has no incremental ``register_tool_function`` / -``register_mcp_client`` / ``register_agent_skill`` methods. - -To avoid rewriting the ~15 ``register_*`` in-house tool functions (they all call -``toolkit.register_tool_function(fn, namesake_strategy=...)``), this module provides a -**duck-typed compatible** collector: it exposes methods of the same names but internally -just collects tools into an ``AllowedFunctionTool`` list / skill directory list. -``agent_factory`` passes the collector to every ``register_*``, then constructs once via -``Toolkit(tools=collector.function_tools, mcps=clients, -skills_or_loaders=collector.skill_loaders)``. - -Also resolves two behavioral differences in 2.0: - * ``FunctionTool``'s default ``check_permissions`` returns ASK -> every in-house tool - would pop HITL. The ``AllowedFunctionTool`` subclass flips the default decision back - to ALLOW (built-in tools like Bash keep their own dangerous-command checks, since we - do not touch their check_permissions). - * Tool functions return ``ToolChunk`` (tool.py already aliases ToolResponse to ToolChunk). -""" - -from __future__ import annotations - -import asyncio -import logging -import os -from dataclasses import replace -from functools import lru_cache -from pathlib import Path - -import frontmatter -from typing import Any, Callable, List - -from agentscope.permission import PermissionBehavior, PermissionDecision -from agentscope.tool import FunctionTool -from agentscope.skill import LocalSkillLoader, SkillLoaderBase, Skill -from core.llm.tool_permissions import ( - ToolPermissionSpec, - builtin_tool_permission, -) - -from core.agent_skills.config import model_facing_skill_dir - -logger = logging.getLogger(__name__) - - -class AllowedFunctionTool(FunctionTool): - """In-house Python tool: allowed by default (overrides 2.0 FunctionTool's default ASK).""" - - async def check_permissions(self, *args: Any, **kwargs: Any) -> PermissionDecision: - return PermissionDecision( - behavior=PermissionBehavior.ALLOW, - message="HugAgentOS self-developed tool (auto-allowed).", - ) - - -def _parse_runtime_skill(text): - content = frontmatter.loads(text) - name, description = content.get("name"), content.get("description") - if not name or not description: - return None - return str(name), str(description), content.content - - -_cached_runtime_skill = lru_cache(maxsize=256)(_parse_runtime_skill) - - -class RuntimeNamedSkillLoader(SkillLoaderBase): - """Read frozen files and expose a deployment-appropriate directory.""" - - def __init__(self, directory: str, runtime_name: str, capability_run=None) -> None: - self.directory = directory - self.runtime_name = runtime_name - self.capability_run = capability_run - self._physical_loader = LocalSkillLoader(directory) - - def _list_desktop_skill(self): - from core.capabilities.runtime import validate - from core.capabilities.errors import CapabilityError - - # Keep authorization outside the parse-error handling: it must fail the - # whole enumeration, never quietly omit a revoked or modified skill. - try: - validate(self.capability_run, only_skill=self.runtime_name) - except (OSError, CapabilityError): - # 被撤销或改动过的单个技能只从枚举里消失，不牵连这一轮其它工具。 - return [] - try: - entry = Path(self.directory) / "SKILL.md" - updated_at = entry.stat().st_mtime - text = entry.read_text(encoding="utf-8") - parsed = ( - _cached_runtime_skill(text) if len(text) <= 262144 else _parse_runtime_skill(text) - ) - if parsed is None: - return [] - _, description, markdown = parsed - return [ - Skill( - name=self.runtime_name, - description=description, - dir=model_facing_skill_dir(self.runtime_name, self.directory), - markdown=markdown, - updated_at=updated_at, - ) - ] - except Exception as exc: - logger.warning("Failed to load skill metadata: %s", type(exc).__name__) - return [] - - async def list_skills(self): - # AgentScope's general filesystem scanner crosses the executor boundary - # eight times for one known file. Batch that I/O in one worker while - # preserving its parser, fields and sandbox alias. - if os.name == "nt" and self.capability_run is not None: - return await asyncio.to_thread(self._list_desktop_skill) - from core.capabilities.runtime import validate - - if self.capability_run is not None: - # Re-check only this loader's own skill (authorization + fresh hash), - # not the whole run. N loaders each re-hashing all N skills — twice - # per assembly — was O(N²) filesystem work and the dominant cost of - # desktop agent setup; scoping to one skill makes it O(N). - from core.capabilities.errors import CapabilityError - - try: - await asyncio.to_thread(validate, self.capability_run, only_skill=self.runtime_name) - except (CapabilityError, OSError): - return [] - physical = await self._physical_loader.list_skills() - return [ - replace(skill, name=self.runtime_name, dir=model_facing_skill_dir(self.runtime_name, self.directory)) - for skill in physical - ] - - -class _ParallelRuntimeSkillLoader(SkillLoaderBase): - """Enumerate independent desktop loaders with bounded I/O concurrency.""" - - def __init__(self, loaders): - self._loaders = tuple(loaders) - - async def list_skills(self): - limit = asyncio.Semaphore(4) - - async def load(loader): - async with limit: - return await loader.list_skills() - - results = await asyncio.gather( - *(load(loader) for loader in self._loaders), return_exceptions=True - ) - skills = [] - for result in results: - if isinstance(result, BaseException): - raise result - skills.extend(result) - return skills - - -class ToolCollector: - """Duck-type compatible with the 1.x Toolkit registration interface; actually only collects tools/skills for the 2.0 Toolkit.""" - - def __init__(self) -> None: - # name -> AllowedFunctionTool (deduped by name, supports override/skip) - self._tools: dict[str, AllowedFunctionTool] = {} - self._permission_specs: dict[str, ToolPermissionSpec] = {} - self._tool_order: List[str] = [] - self._skill_loaders: List[Any] = [] - - # ── 1.x compatibility interface ───────────────────────────────────── - def register_tool_function( - self, - func: Callable[..., Any], - *, - func_description: str | None = None, - namesake_strategy: str = "override", - permission: ToolPermissionSpec | None = None, - **_ignored: Any, - ) -> None: - """Collect a tool function as an AllowedFunctionTool. - - namesake_strategy: - - "override" (default): same name replaces the existing one - - "skip": same name keeps the existing one and discards the new one - """ - name = getattr(func, "__name__", None) or "tool" - if name in self._tools: - if namesake_strategy == "skip": - return - # override: replace, but keep the original ordering position - permission_spec = permission or builtin_tool_permission(name) - try: - ft = AllowedFunctionTool( - func, - name=name, - description=func_description, - ) - except Exception as exc: # noqa: BLE001 - logger.warning("[tool_collector] 构造 FunctionTool '%s' 失败: %s", name, exc) - return - if name not in self._tools: - self._tool_order.append(name) - self._tools[name] = ft - if permission_spec is not None: - self._permission_specs[name] = permission_spec - else: - self._permission_specs.pop(name, None) - - def register_tool_permission(self, name: str, spec: ToolPermissionSpec) -> None: - """Attach developer-owned MCP policy to the same dispatch registry.""" - previous = self._permission_specs.get(name) - if previous is not None and previous.key != spec.key: - raise ValueError(f"Conflicting permission declaration for {name!r}") - self._permission_specs[name] = spec - - def register_agent_skill( - self, skill_dir: Any, *, runtime_name: str | None = None, capability_run=None - ) -> None: - """Keep physical reads separate from a desktop skill's runtime identity.""" - if not skill_dir: - return - if runtime_name is not None: - if not any( - isinstance(item, RuntimeNamedSkillLoader) - and item.directory == skill_dir - and item.runtime_name == runtime_name - for item in self._skill_loaders - ): - self._skill_loaders.append( - RuntimeNamedSkillLoader(skill_dir, runtime_name, capability_run) - ) - else: - self.register_agent_skill(skill_dir, runtime_name=Path(skill_dir).name) - - def register_mcp_client(self, *args: Any, **kwargs: Any) -> None: # pragma: no cover - # 2.0 goes through Toolkit(mcps=); this should never be called — kept as an empty fallback against legacy call paths. - logger.warning( - "[tool_collector] register_mcp_client 被调用但已忽略（2.0 经 Toolkit(mcps=)）。" - ) - - # ── Result accessors for agent_factory ─────────────────────────────── - def get_tool(self, name: str) -> AllowedFunctionTool | None: - """Get a collected AllowedFunctionTool by name (for tests/introspection; `._func` - is the original callable, `.input_schema` is the JSON schema).""" - return self._tools.get(name) - - @property - def function_tools(self) -> List[AllowedFunctionTool]: - return [self._tools[n] for n in self._tool_order if n in self._tools] - - @property - def permission_specs(self) -> dict[str, ToolPermissionSpec]: - """Detached declaration map consumed by the run-scoped gateway.""" - return dict(self._permission_specs) - - @property - def skill_loaders(self) -> List[Any]: - loaders = list(self._skill_loaders) - if ( - os.name == "nt" - and len(loaders) >= 8 - and all(isinstance(loader, RuntimeNamedSkillLoader) for loader in loaders) - ): - return [_ParallelRuntimeSkillLoader(loaders)] - return loaders - - -__all__ = ["AllowedFunctionTool", "ToolCollector"] diff --git a/src/backend/core/llm/tool_permissions/__init__.py b/src/backend/core/llm/tool_permissions/__init__.py index 8438c3cb..ab14f91f 100644 --- a/src/backend/core/llm/tool_permissions/__init__.py +++ b/src/backend/core/llm/tool_permissions/__init__.py @@ -1,93 +1,4 @@ -"""Public API of the governed-tool permission domain.""" +"""SDK tool authorization service; shared policy lives in sandbox.permissions.""" +from .service import ToolPermissionService -from .models import ( - CURRENT_PERMISSION_TICKET, - FAIL_CLOSED_MODE, - FALLBACK_ALLOW, - FALLBACK_DENY, - UNCONFINED_APPROVAL_MODES, - LocalCommandAuthorization, - LocalConfinementUnavailableError, - PermissionEnforcementError, - PermissionIntent, - PermissionOutcome, - PermissionRuntime, - PermissionTicket, - ToolPermissionSpec, - current_local_command_authorization, - require_local_path_permission, - READ, - WRITE, - EXECUTE, - DOMAIN_LOCAL_PATH, - DOMAIN_LOCAL_COMMAND, - DOMAIN_MYSPACE, - DOMAIN_APPROVAL, - DOMAIN_DENY, -) -from .specs import ( - ToolPermissionRegistry, - allow_tool, - builtin_tool_permission, - deny_tool, - local_command_tool, - local_path_tool, -) -from .preferences import ( - preset_answers_for_user, - resolve_approval_mode, - normalize_approval_mode, - APPROVAL_ASK, - APPROVAL_AUTO, - APPROVAL_FULL, - APPROVAL_MODES, - DESTRUCTIVE_OPS, -) -from .service import ( - ToolPermissionService, -) -from .middleware import ( - ToolPermissionMiddleware, -) - -__all__ = [ - "CURRENT_PERMISSION_TICKET", - "FAIL_CLOSED_MODE", - "FALLBACK_ALLOW", - "FALLBACK_DENY", - "UNCONFINED_APPROVAL_MODES", - "LocalCommandAuthorization", - "LocalConfinementUnavailableError", - "PermissionEnforcementError", - "PermissionIntent", - "PermissionOutcome", - "PermissionRuntime", - "PermissionTicket", - "ToolPermissionMiddleware", - "ToolPermissionRegistry", - "ToolPermissionService", - "ToolPermissionSpec", - "allow_tool", - "builtin_tool_permission", - "current_local_command_authorization", - "deny_tool", - "local_command_tool", - "local_path_tool", - "preset_answers_for_user", - "require_local_path_permission", - "resolve_approval_mode", - "normalize_approval_mode", - "APPROVAL_ASK", - "APPROVAL_AUTO", - "APPROVAL_FULL", - "APPROVAL_MODES", - "DESTRUCTIVE_OPS", - "READ", - "WRITE", - "EXECUTE", - "DOMAIN_LOCAL_PATH", - "DOMAIN_LOCAL_COMMAND", - "DOMAIN_MYSPACE", - "DOMAIN_APPROVAL", - "DOMAIN_DENY", -] +__all__ = ["ToolPermissionService"] diff --git a/src/backend/core/llm/tool_permissions/hook.py b/src/backend/core/llm/tool_permissions/hook.py new file mode 100644 index 00000000..e9797637 --- /dev/null +++ b/src/backend/core/llm/tool_permissions/hook.py @@ -0,0 +1,71 @@ +"""Neutral authorization policy with an execution-scoped permission ticket.""" +import json +from contextlib import contextmanager +from dataclasses import dataclass +from typing import Any +from core.harness.events import thaw_value +from core.harness.hooks import Decision, DecisionKind, HookSpec, HookStage +from core.sandbox.permissions.dispatch_models import CURRENT_PERMISSION_TICKET, PermissionEnforcementError +from core.sandbox.permissions.approval import refresh_runtime +from .service import ToolPermissionService + + +@dataclass(frozen=True) +class PermissionCall: + id: str + name: str + input: Any + + +class ToolPermissionHook: + def __init__(self, service): + self.service = service + self.pending = {} + + async def __call__(self, invocation): + call = PermissionCall(str(invocation.metadata.get("tool_call_id") or + invocation.data.get("metadata", {}).get("tool_call_id") or ""), + invocation.operation_name, thaw_value(invocation.data.get("tool_args", {}))) + try: + runtime = await refresh_runtime(self.service.runtime) + except PermissionEnforcementError as exc: + return self.block(call.id, {"error": str(exc), "blocked": True}, + {"decision": "deny", "reason": "policy_unavailable"}) + outcome = await ToolPermissionService(self.service.registry, runtime).authorize(call) + if not outcome.proceed: + return self.block(call.id, outcome.payload or {"error": "工具权限判定拒绝执行", "blocked": True}, outcome.audit) + self.pending[call.id] = outcome + return Decision() + + @staticmethod + def block(call_id, payload, audit): + response = {"id": call_id, "content": [{"type": "text", "text": json.dumps(payload, ensure_ascii=False, default=str)}], + "state": "success" if payload.get("ok") and not payload.get("error") else "error", + "metadata": {"permission": dict(audit)}} + return Decision(DecisionKind.REJECT, patch={"tool_response": response}, reason="permission outcome") + + @contextmanager + def bind(self, call_id): + outcome = self.pending.pop(call_id, None) + if outcome is None: + yield None + return + ticket = outcome.ticket + if ticket is None: + yield dict(outcome.audit) + return + token = CURRENT_PERMISSION_TICKET.set(ticket) + try: + yield ticket.audit_dict() + finally: + ticket.lifetime.expire() + CURRENT_PERMISSION_TICKET.reset(token) + + def discard(self, call_id): + outcome = self.pending.pop(call_id, None) + if outcome is not None and outcome.ticket is not None: + outcome.ticket.lifetime.expire() + + def spec(self): + return HookSpec("tool_permission", HookStage.BEFORE_TOOL, self, priority=600, + timeout_ms=None, mutable_fields=frozenset({"tool_response"})) diff --git a/src/backend/core/llm/tool_permissions/middleware.py b/src/backend/core/llm/tool_permissions/middleware.py deleted file mode 100644 index 58d0606b..00000000 --- a/src/backend/core/llm/tool_permissions/middleware.py +++ /dev/null @@ -1,86 +0,0 @@ -"""One permission dispatch boundary: live policy, authorization and ticket lifetime.""" - -from __future__ import annotations -import json -from typing import Any, Mapping -from agentscope.agent import Agent -from agentscope.message import TextBlock, ToolResultState -from agentscope.middleware import MiddlewareBase -from agentscope.tool._response import ToolResponse -from .models import CURRENT_PERMISSION_TICKET, PermissionEnforcementError -from .service import ToolPermissionService -from .preferences import refresh_runtime - - -def _permission_response( - tool_call: Any, - payload: Mapping[str, Any], - audit: Mapping[str, Any], -) -> ToolResponse: - success = bool(payload.get("ok")) and not payload.get("error") - return ToolResponse( - id=str(getattr(tool_call, "id", "") or ""), - content=[ - TextBlock( - type="text", - text=json.dumps(payload, ensure_ascii=False, default=str), - ) - ], - state=ToolResultState.SUCCESS if success else ToolResultState.ERROR, - metadata={"permission": dict(audit)}, - ) - - -class ToolPermissionMiddleware(MiddlewareBase): - """Authorize once before effects and bind the ticket around dispatch.""" - - def __init__(self, service: ToolPermissionService) -> None: - self.service = service - - async def on_acting(self, agent: Agent, input_kwargs: dict, next_handler): # noqa: ANN001 - async for item in self._act(input_kwargs, next_handler): - yield item - - async def _act(self, input_kwargs: dict, next_handler): # noqa: ANN001 - tool_call = input_kwargs.get("tool_call") - try: - runtime = await refresh_runtime(self.service.runtime) - except PermissionEnforcementError as exc: - yield _permission_response( - tool_call, - {"error": str(exc), "blocked": True}, - {"decision": "deny", "reason": "policy_unavailable"}, - ) - return - service = ToolPermissionService(self.service.registry, runtime) - outcome = await service.authorize(tool_call) - if not outcome.proceed: - yield _permission_response( - tool_call, - outcome.payload or {"error": "工具权限判定拒绝执行", "blocked": True}, - outcome.audit, - ) - return - - if outcome.ticket is None: - async for item in next_handler(**input_kwargs): - if isinstance(item, ToolResponse): - item.metadata = { - **dict(item.metadata or {}), - "permission": dict(outcome.audit), - } - yield item - return - - token = CURRENT_PERMISSION_TICKET.set(outcome.ticket) - try: - async for item in next_handler(**input_kwargs): - if isinstance(item, ToolResponse): - item.metadata = { - **dict(item.metadata or {}), - "permission": outcome.ticket.audit_dict(), - } - yield item - finally: - outcome.ticket.lifetime.expire() - CURRENT_PERMISSION_TICKET.reset(token) diff --git a/src/backend/core/llm/tool_permissions/pending.py b/src/backend/core/llm/tool_permissions/pending.py index 7932a454..8774bea0 100644 --- a/src/backend/core/llm/tool_permissions/pending.py +++ b/src/backend/core/llm/tool_permissions/pending.py @@ -1,8 +1,8 @@ """A pending permission is answered only by its declared, current server-side authority.""" import asyncio -from core.llm import interaction_store -from .account import read_account_mode +from core.llm.runtime import interaction_store +from core.sandbox.permissions.account import read_account_mode async def approve_current_policy(chat_id: str, confirm_id: str, info: dict) -> bool: diff --git a/src/backend/core/llm/tool_permissions/service.py b/src/backend/core/llm/tool_permissions/service.py index 0ec20b98..1a244cab 100644 --- a/src/backend/core/llm/tool_permissions/service.py +++ b/src/backend/core/llm/tool_permissions/service.py @@ -4,26 +4,9 @@ import logging import os from typing import Any, Sequence, Optional -from .preferences import DESTRUCTIVE_OPS, _preset_answers -from .specs import ToolPermissionRegistry, _one_shot_write_root -from .models import ( - DOMAIN_APPROVAL, - DOMAIN_DENY, - DOMAIN_LOCAL_COMMAND, - DOMAIN_LOCAL_PATH, - DOMAIN_MYSPACE, - FAIL_CLOSED_MODE, - FALLBACK_ALLOW, - LocalCommandAuthorization, - PermissionIntent, - PermissionOutcome, - PermissionRuntime, - PermissionTicket, - READ, - UNCONFINED_APPROVAL_MODES, - _call_args, - _json_hash, -) +from core.sandbox.permissions.approval import DESTRUCTIVE_OPS, _preset_answers +from core.sandbox.permissions.tool_specs import ToolPermissionRegistry, _one_shot_write_root +from core.sandbox.permissions.dispatch_models import DOMAIN_APPROVAL, DOMAIN_DENY, DOMAIN_LOCAL_COMMAND, DOMAIN_LOCAL_PATH, DOMAIN_MYSPACE, FAIL_CLOSED_MODE, FALLBACK_ALLOW, LocalCommandAuthorization, PermissionIntent, PermissionOutcome, PermissionRuntime, PermissionTicket, READ, UNCONFINED_APPROVAL_MODES, _call_args, _json_hash logger = logging.getLogger(__name__) @@ -187,7 +170,7 @@ def _safe_local_security(self): mode = self.runtime.approval_mode try: - from core.services.local_grant_service import grants_for_gate, policy_for_gate + from core.harness.registry.local_files import grants_for_gate, policy_for_gate return mode, grants_for_gate(), policy_for_gate(mode) except Exception: # unreadable configuration must never grant access @@ -208,7 +191,7 @@ def _safe_local_security(self): ) def _execution_directory(self) -> str: - from core.llm.tools._paths import workspace_directory + from core.sandbox.workspace.resolution import workspace_directory return workspace_directory( self.runtime.sandbox_session_id or self.runtime.chat_id, @@ -269,7 +252,7 @@ async def _authorize_local_path(self, intent: PermissionIntent) -> dict[str, Any if self._no_ui_pass_through(intent): return {"reasons": [*verdict.reasons, "no_approval_ui_pass_through:local_path"]} - from core.llm.tools import _tool_confirmation as confirm + from core.llm.tools import confirmation as confirm op = confirm.OP_LOCAL_READ if intent.action == READ else confirm.OP_LOCAL_WRITE blocked = await confirm.gate( @@ -325,7 +308,7 @@ async def _authorize_local_command(self, intent: PermissionIntent) -> dict[str, and not self._answers_for_user(dangerous=bool(danger_categories(verdict.reasons))) and not self._no_ui_pass_through(intent) ): - from core.llm.tools import _tool_confirmation as confirm + from core.llm.tools import confirmation as confirm blocked = await confirm.gate( chat_id=self.runtime.chat_id, @@ -360,7 +343,7 @@ def _physical(self, logical: str) -> str: approved one-shot write would be silently dropped. """ try: - from core.llm.tools._paths import to_physical_path + from core.sandbox.workspace.resolution import to_physical_path session = self.runtime.sandbox_session_id or self.runtime.chat_id return to_physical_path( @@ -457,7 +440,7 @@ async def _authorize_approval( *, interactive: bool, ) -> dict[str, Any]: - from core.llm.tools import _tool_confirmation as confirm + from core.llm.tools import confirmation as confirm op = intent.op or intent.action if self._answers_for_user(dangerous=op in DESTRUCTIVE_OPS): diff --git a/src/backend/core/llm/tools/__init__.py b/src/backend/core/llm/tools/__init__.py index d5a0676c..9adabae7 100644 --- a/src/backend/core/llm/tools/__init__.py +++ b/src/backend/core/llm/tools/__init__.py @@ -17,9 +17,8 @@ from .read_tool import register_read from .write_tool import register_write -# Sandbox / skill / myspace / artifact tools (relocated from the former -# singular ``core.llm.tool`` module). Surfaced here so callers import the whole -# tool family from one place: ``from core.llm.tools import register_bash, ...``. +# Sandbox / skill / myspace / artifact tools, surfaced here so callers import +# the whole tool family from one place: ``from core.llm.tools import register_bash, ...``. from .sandbox_tool import ( register_bash, register_sandbox_get_artifact, @@ -28,19 +27,15 @@ from .skill_tool import ( register_sandboxed_view_text_file, ) -from .channel_attachment_tool import register_channel_attachment from .chat_history_tool import register_chat_history_tools from .pin_tool import register_pin_to_workspace from .read_artifact_tool import register_read_artifact -from .data_context_tool import register_get_data_context __all__ = [ "ReadEntry", "ReadStateTracker", "register_bash", - "register_channel_attachment", "register_chat_history_tools", - "register_get_data_context", "register_edit", "register_glob", "register_grep", diff --git a/src/backend/core/llm/tools/_common.py b/src/backend/core/llm/tools/_common.py index cc88181f..54bf1036 100644 --- a/src/backend/core/llm/tools/_common.py +++ b/src/backend/core/llm/tools/_common.py @@ -2,7 +2,6 @@ from __future__ import annotations -from core.content.artifact_refs import require_artifact_storage_key import asyncio import base64 @@ -14,16 +13,16 @@ from agentscope.message import Base64Source, DataBlock, TextBlock -# AgentScope 2.0: tool functions must return ToolChunk; aliased (its fields are a superset of ToolResponse). -from agentscope.tool._response import ToolChunk as ToolResponse -from core.services.project_scope import ProjectScope +# AgentScope 2.0: tool functions must return ToolChunk; aliased (its fields are a superset of ToolChunk). +from agentscope.tool._response import ToolChunk +from core.projects.edition import ProjectScope logger = logging.getLogger(__name__) async def resp_image( image_bytes: bytes, mime_type: str, *, name: str, meta: Optional[dict[str, Any]] = None -) -> ToolResponse: +) -> ToolChunk: """图片原样交给多模态主模型：一段 JSON 元信息 + 一个图片数据块。 AgentScope 会把工具结果里的 DataBlock 提升成下一轮的 image 消息；端点不收图时 @@ -32,7 +31,7 @@ async def resp_image( """ payload = {"type": "image", "mime_type": mime_type, "size": len(image_bytes), **(meta or {})} data = await asyncio.to_thread(lambda: base64.b64encode(image_bytes).decode("ascii")) - return ToolResponse( + return ToolChunk( content=[ *resp_json(payload).content, DataBlock( @@ -44,13 +43,13 @@ async def resp_image( ) -def resp_json(payload: dict[str, Any]) -> ToolResponse: - """Wrap a JSON dict as a single-text-block ToolResponse. +def resp_json(payload: dict[str, Any]) -> ToolChunk: + """Wrap a JSON dict as a single-text-block ToolChunk. Mirrors ``core.llm.tool._resp_json``. Re-implemented here so this package has zero cross-imports back to the large monolithic ``tool.py``. """ - return ToolResponse( + return ToolChunk( content=[ TextBlock( type="text", @@ -60,7 +59,7 @@ def resp_json(payload: dict[str, Any]) -> ToolResponse: ) -def response_metadata(response: ToolResponse, metadata: dict[str, Any]) -> ToolResponse: +def response_metadata(response: ToolChunk, metadata: dict[str, Any]) -> ToolChunk: """Merge file metadata into the existing JSON block, preserving image layout.""" block = response.content[0] block.text = json.dumps({**json.loads(block.text), **metadata}, ensure_ascii=False) @@ -130,7 +129,7 @@ def register_file_artifact( content: bytes, scope: Optional[ProjectScope] = None, ) -> Optional[dict[str, Any]]: """Create an immutable downloadable snapshot of a sandbox file.""" - from core.llm.tools._tool_helpers import _store_generated_files + from core.files.artifacts.generated import store_generated_files as _store_generated_files if not user_id or not filename: return None @@ -162,7 +161,7 @@ def pin_artifact_to_workspace(ref: dict[str, Any]) -> bool: if not file_id: return False try: - from core.llm import workspace as _workspace + from core.llm.runtime import workspace as _workspace pinned = _workspace.pin( file_id=str(file_id), diff --git a/src/backend/core/llm/tools/_tool_helpers.py b/src/backend/core/llm/tools/_tool_helpers.py index c92f27a8..390ac7d8 100644 --- a/src/backend/core/llm/tools/_tool_helpers.py +++ b/src/backend/core/llm/tools/_tool_helpers.py @@ -4,227 +4,31 @@ functions share them. ``core.llm.tool`` re-exports these for compatibility. """ -import base64 import json import logging -from pathlib import Path -from typing import Any, Optional +from typing import Any from agentscope.message import TextBlock -from agentscope.tool._response import ToolChunk as ToolResponse +from agentscope.tool._response import ToolChunk from core.config.settings import settings logger = logging.getLogger(__name__) -def _store_generated_files( - files_data: list[dict[str, Any]], - *, - user_id: Optional[str], - source: str, - extra_metadata: Optional[dict[str, Any]] = None, -) -> list[dict[str, Any]]: - """Persist sidecar-generated files and return normalized artifact refs.""" - if not files_data: - return [] - - try: - from core.artifacts.store import save_artifact_bytes - except Exception as exc: - logger.warning("artifact store unavailable for %s: %s", source, exc) - return [] - - refs: list[dict[str, Any]] = [] - for fd in files_data: - content_b64 = fd.get("content_b64", "") - if not content_b64: - continue - - name = str(fd.get("name", "output")).strip() or "output" - mime_type = ( - str(fd.get("mime_type", "application/octet-stream")).strip() - or "application/octet-stream" - ) - try: - content = base64.b64decode(content_b64) - except Exception: - logger.warning("skip invalid base64 artifact payload: %s", name) - continue - - metadata: dict[str, Any] = {"source": source} - if user_id: - metadata["user_id"] = user_id - if extra_metadata: - metadata.update(extra_metadata) - - try: - item = save_artifact_bytes( - content=content, - name=name, - mime_type=mime_type, - extension=Path(name).suffix.lstrip("."), - metadata=metadata, - ) - except Exception as exc: - logger.warning("failed to persist generated artifact %s: %s", name, exc) - continue - - refs.append( - { - "file_id": item["file_id"], - "name": item.get("name", name), - "url": f"/files/{item['file_id']}", - "mime_type": item.get("mime_type", mime_type), - "size": item.get("size", len(content)), - "storage_key": item.get("storage_key"), - } - ) - - return refs - - -def _store_generated_file_path( - file_path: str | Path, - *, - name: str, - mime_type: str, - user_id: Optional[str], - source: str, - extra_metadata: Optional[dict[str, Any]] = None, -) -> dict[str, Any] | None: - """Persist a generated file from disk and return a normalized artifact ref.""" - try: - from core.artifacts.store import save_artifact_file - except Exception as exc: - logger.warning("artifact store unavailable for %s: %s", source, exc) - return None - - metadata: dict[str, Any] = {"source": source} - if user_id: - metadata["user_id"] = user_id - if extra_metadata: - metadata.update(extra_metadata) - try: - item = save_artifact_file( - file_path=file_path, - name=name, - mime_type=mime_type, - extension=Path(name).suffix.lstrip("."), - metadata=metadata, - ) - except Exception as exc: - logger.warning("failed to persist generated artifact %s: %s", name, exc) - return None - return { - "file_id": item["file_id"], - "name": item.get("name", name), - "url": f"/files/{item['file_id']}", - "mime_type": item.get("mime_type", mime_type), - "size": item.get("size", 0), - "storage_key": item.get("storage_key"), - } -MAX_ARTIFACT_FILE_SIZE = settings.sandbox.artifact_max_bytes -def _resolve_artifact_files( - artifact_refs: dict[str, str], - user_id: str | None, -) -> tuple[dict[str, str] | None, str | None]: - """Resolve artifact: references to base64-encoded binary content. - - Tries DB lookup first, falls back to local artifact store index. - Returns (files_b64_dict, error_message). - """ - if not artifact_refs: - return None, None - - try: - from core.content.artifact_refs import resolve_artifact_storage_key - from core.storage.factory import get_storage - except Exception as exc: - return None, f"artifact 解析依赖不可用: {exc}" - - result: dict[str, str] = {} - storage = get_storage() - - for filename, artifact_id in artifact_refs.items(): - from core.artifacts.local_reference import is_local_file_ref - if is_local_file_ref(artifact_id): - from core.llm.hooks import _download_artifact_bytes - file_bytes = _download_artifact_bytes(artifact_id, filename, "artifact input", user_id=user_id) - if file_bytes is None: - return None, f"artifact '{artifact_id}' 不存在或无权访问" - if len(file_bytes) > MAX_ARTIFACT_FILE_SIZE: - return None, f"artifact 文件 '{filename}' 过大" - result[filename] = base64.b64encode(file_bytes).decode("ascii") - continue - storage_key = None - owner_ok = True - - # Try 1: DB lookup - try: - from core.db.engine import SessionLocal - from core.db.repository import ArtifactRepository - - db = SessionLocal() - try: - repo = ArtifactRepository(db) - art = repo.get_by_id(artifact_id) - if art: - if user_id and art.user_id != user_id: - owner_ok = False - else: - storage_key = resolve_artifact_storage_key(art.artifact_id, art.storage_key) - finally: - db.close() - except Exception: - pass - - if not owner_ok: - return None, f"无权访问 artifact '{artifact_id}'" - - # Try 2: local artifact store fallback - if not storage_key: - try: - from core.artifacts.store import get_artifact - - item = get_artifact(artifact_id) - if item: - item_user = (item.get("metadata") or {}).get("user_id") - if user_id and item_user and item_user != user_id: - return None, f"无权访问 artifact '{artifact_id}'" - storage_key = item.get("storage_key") - except Exception: - pass - - if not storage_key: - return None, f"artifact '{artifact_id}' 不存在或已删除" - - try: - file_bytes = storage.download_bytes(storage_key) - except Exception as exc: - return None, f"artifact 文件 '{filename}' 读取失败: {exc}" - - if len(file_bytes) > MAX_ARTIFACT_FILE_SIZE: - return None, ( - f"artifact 文件 '{filename}' 过大: " - f"{len(file_bytes)} bytes > {MAX_ARTIFACT_FILE_SIZE} bytes" - ) - - result[filename] = base64.b64encode(file_bytes).decode("ascii") - return result or None, None -def _resp_json(payload: dict[str, Any]) -> ToolResponse: - """Helper: wrap a JSON-serializable dict as a single-text-block ToolResponse.""" - return ToolResponse( +def resp_json(payload: dict[str, Any]) -> ToolChunk: + """Helper: wrap a JSON-serializable dict as a single-text-block ToolChunk.""" + return ToolChunk( content=[ TextBlock( type="text", @@ -234,7 +38,7 @@ def _resp_json(payload: dict[str, Any]) -> ToolResponse: ) -def _validate_workspace_path( +def validate_workspace_path( path: str, *, root: str | None = None, diff --git a/src/backend/core/llm/tools/ask_user_question_tool.py b/src/backend/core/llm/tools/ask_user_question_tool.py index 2052a622..097bec1d 100644 --- a/src/backend/core/llm/tools/ask_user_question_tool.py +++ b/src/backend/core/llm/tools/ask_user_question_tool.py @@ -1,11 +1,12 @@ """Model-facing ``ask_user_question`` tool over the suspended question registry.""" +from core.llm.factory.tools.definitions import function_tool import logging from typing import Annotated, List, Optional from agentscope.message import TextBlock from agentscope.tool import Toolkit -from agentscope.tool._response import ToolChunk as ToolResponse +from agentscope.tool._response import ToolChunk from agentscope.tool._response import ToolResultState from core.llm.tools import user_questions from core.llm.tools._common import resp_json @@ -78,7 +79,7 @@ class AskUserQuestionItem(TypedDict): ] -def _error_response(status: str) -> ToolResponse: +def _error_response(status: str) -> ToolChunk: """Return DSH-style tool errors while retaining local timeout semantics.""" errors = { @@ -97,7 +98,7 @@ def _error_response(status: str) -> ToolResponse: status, ("ASK_ABORTED", "ask_user_question was aborted before the user answered"), ) - return ToolResponse( + return ToolChunk( content=[TextBlock(type="text", text=f"Error: {message}")], state=ToolResultState.ERROR, metadata={ @@ -132,7 +133,7 @@ async def ask_user_question( List[AskUserQuestionItem], Field(description="Questions to ask the user before continuing."), ], - ) -> ToolResponse: + ) -> ToolChunk: """Ask the user one or more questions and wait for the human answer.""" result = await user_questions.ask( @@ -160,7 +161,7 @@ async def ask_user_question( # to DSH's exact two-sentence contract instead of leaking local policy or # transport details into the fixed tool-schema prefix. ask_user_question.__doc__ = _TOOL_DESCRIPTION - toolkit.register_tool_function(ask_user_question, namesake_strategy="override") + toolkit.append(function_tool(ask_user_question)) logger.info("[factory] Registered ask_user_question tool (interactive top-level chat)") diff --git a/src/backend/core/llm/tools/chat_history_tool.py b/src/backend/core/llm/tools/chat_history_tool.py index 81df9387..02c6bd91 100644 --- a/src/backend/core/llm/tools/chat_history_tool.py +++ b/src/backend/core/llm/tools/chat_history_tool.py @@ -9,6 +9,7 @@ """ from __future__ import annotations +from core.llm.factory.tools.definitions import function_tool import asyncio import logging @@ -33,48 +34,10 @@ def register_chat_history_tools( scope_hint = "当前项目内的会话" if project_id else "你最近的会话" - def _list(query: str, limit: int): - from core.db.engine import SessionLocal - from core.services.chat_reference_service import list_referencable_sessions, session_brief - - with SessionLocal() as db: - sessions = list_referencable_sessions( - db, - user_id, - project_id=project_id, - query=query, - limit=limit, - exclude_chat_id=chat_id, - ) - return { - "ok": True, - "scope": "project" if project_id else "user", - "count": len(sessions), - "chats": [session_brief(session) for session in sessions], - } - - def _read(target_chat_id: str, mode: str, offset: int, limit: int): - from core.db.engine import SessionLocal - from core.services.chat_reference_service import read_session_messages, session_card - - with SessionLocal() as db: - card = session_card(db, target_chat_id, user_id) - if card is None: - return {"ok": False, "error": f"会话不存在或你无权访问：{target_chat_id}"} - if mode == "card": - return {"ok": True, "mode": "card", **card} - page = read_session_messages(db, target_chat_id, user_id, offset=offset, limit=limit) - if page is None: - return {"ok": False, "error": f"会话不存在或你无权访问：{target_chat_id}"} - return {"ok": True, "mode": "full", "title": card["title"], **page} - - async def _digest(target_chat_id: str): - from core.services.chat_reference_service import build_session_digest - - result = await build_session_digest(target_chat_id, user_id) - if result is None: - return {"ok": False, "error": f"会话不存在或你无权访问：{target_chat_id}"} - return {"ok": True, "mode": "digest", **result} + from core.harness.registry.chat_history import history_operations + _list, _read, _digest = history_operations( + user_id=user_id, chat_id=chat_id, project_id=project_id, + ) async def list_related_chats(query: str = "", limit: int = 20): """列出可供参考的历史会话（{scope}），每条给出标题、最后活跃时间和消息条数。 @@ -127,5 +90,5 @@ async def read_chat( list_related_chats.__doc__ = (list_related_chats.__doc__ or "").format(scope=scope_hint) - toolkit.register_tool_function(list_related_chats) - toolkit.register_tool_function(read_chat) + toolkit.append(function_tool(list_related_chats)) + toolkit.append(function_tool(read_chat)) diff --git a/src/backend/core/llm/gateway_mcp_tool.py b/src/backend/core/llm/tools/cloud_gateway.py similarity index 71% rename from src/backend/core/llm/gateway_mcp_tool.py rename to src/backend/core/llm/tools/cloud_gateway.py index 5ae0e72c..685a1fac 100644 --- a/src/backend/core/llm/gateway_mcp_tool.py +++ b/src/backend/core/llm/tools/cloud_gateway.py @@ -1,24 +1,43 @@ """Authorized cloud gateway invocation, separate from manifest discovery.""" +from core.harness.registry import execution_identity as bridge_identity import asyncio +import weakref +from contextlib import asynccontextmanager from typing import Any, Dict import httpx -import mcp.types from agentscope.permission import PermissionBehavior, PermissionDecision from agentscope.tool import ToolBase, ToolChunk +_shared_clients = weakref.WeakKeyDictionary() + + +@asynccontextmanager +async def _http(transport): + """Reuse one pooled client per event loop; an injected transport gets its own client.""" + if transport is not None: + async with httpx.AsyncClient(transport=transport) as client: + yield client + return + loop = asyncio.get_running_loop() + client = _shared_clients.get(loop) + if client is None or client.is_closed: + client = _shared_clients[loop] = httpx.AsyncClient() + yield client + + class GatewayMCPTool(ToolBase): """MCP-shaped tool built from a cloud-owned runtime manifest snapshot.""" is_mcp = True is_state_injected = False is_external_tool = False - is_concurrency_safe = True + is_concurrency_safe = False def __init__( self, *, mcp_name: str, - tool: mcp.types.Tool, + tool: dict, invoke_url: str, schema_hash: str, headers: Dict[str, str], @@ -26,17 +45,18 @@ def __init__( transport: Any = None, source_plugin: str = "", ) -> None: + super().__init__() self._source_plugin = source_plugin self.mcp_name = mcp_name - self.name = tool.name - self.description = tool.description or "" - schema = dict(tool.inputSchema or {}) + self.name = tool["name"] + self.description = tool.get("description") or "" + schema = dict(tool.get("inputSchema") or {}) schema.setdefault("type", "object") schema.setdefault("properties", {}) schema.setdefault("required", []) self.input_schema = schema self.is_read_only = bool( - tool.annotations and getattr(tool.annotations, "readOnlyHint", False) + tool.get("annotations", {}).get("readOnlyHint") ) self._invoke_url = invoke_url self._schema_hash = schema_hash @@ -65,11 +85,9 @@ def _unknown_outcome(self) -> ToolChunk: "gateway_outcome_unknown": True}, ) - async def __call__(self, **kwargs: Any) -> ToolChunk: + async def call(self, **kwargs: Any) -> ToolChunk: from extensions.mcp_servers.registry import permission_specs - from core.llm.tool_permissions.models import ( - CURRENT_PERMISSION_TICKET, PermissionEnforcementError, _json_hash, - ) + from core.sandbox.permissions.dispatch_models import CURRENT_PERMISSION_TICKET, PermissionEnforcementError, _json_hash spec = permission_specs(self.mcp_name).get(self.name) if spec is not None: ticket = CURRENT_PERMISSION_TICKET.get() @@ -77,77 +95,66 @@ async def __call__(self, **kwargs: Any) -> ToolChunk: or ticket.spec_key != spec.key or ticket.args_hash != _json_hash(kwargs)): raise PermissionEnforcementError("云端空间工具缺少匹配的预执行授权票据") headers = dict(self._headers) - from core.services.desktop_observation_context import context_headers + from core.harness.registry.execution_identity import context_headers headers.update(context_headers()) - from core.capabilities.paths import capabilities_enabled + from core.harness.registry.run_capabilities import has_run_capabilities captured = None - if capabilities_enabled() and "/api/v1/desktop/capability/gateway/" in self._invoke_url: - from core.services import desktop_cloud_bridge as bridge + if has_run_capabilities() and "/api/v1/desktop/capability/gateway/" in self._invoke_url: captured = {"cloud_base": self._invoke_url.split("/api/v1/desktop/capability/gateway/", 1)[0], "token": str(headers.get("Authorization") or "").removeprefix("Bearer ")} - bridge.require_current_account(captured) - headers.update(bridge.cloud_headers(bridge.get_state())) + bridge_identity.require_current_account(captured) + headers.update(bridge_identity.cloud_headers(bridge_identity.get_state())) local_result = None if captured is not None: - from core.services.automation_tool_routing import local_automation_result + from core.harness.registry.tool_operations import local_tool_result async def authorize_local(): - options = {"timeout": 15.0} - if self._transport is not None: - options["transport"] = self._transport - async with httpx.AsyncClient(**options) as client: + async with _http(self._transport) as client: reply = await client.post( self._invoke_url.rsplit("/", 1)[0] + "/authorize", - headers=headers, + headers=headers, timeout=15.0, json={"tool_name": self.name, "schema_hash": self._schema_hash, "operation_id": kwargs.get("tool_effect_id") or "authorization"}, ) - bridge.require_current_account(captured) + bridge_identity.require_current_account(captured) if reply.status_code in (401, 403): - with bridge.account_scope(captured): - bridge.clear_state() + with bridge_identity.account_scope(captured): + bridge_identity.clear_state() if not reply.is_success: raise ValueError("本机任务工具授权已变更或不可用，请同步云端能力后重试") try: - local_result = await local_automation_result( + local_result = await local_tool_result( self._source_plugin, self.name, kwargs, headers, authorize=authorize_local, ) except ValueError as exc: from agentscope.message import TextBlock, ToolResultState return ToolChunk(content=[TextBlock(text=str(exc))], state=ToolResultState.ERROR) - bridge.require_current_account(captured) + bridge_identity.require_current_account(captured) if local_result is not None and not local_result.metadata.get("merge_cloud"): return local_result headers["accept-encoding"] = "identity" - client_kwargs: Dict[str, Any] = { - "timeout": httpx.Timeout( - connect=min(10.0, self._timeout), - read=self._timeout, - write=min(60.0, self._timeout), - pool=min(10.0, self._timeout), - ) - } - if self._transport is not None: - client_kwargs["transport"] = self._transport + timeout = httpx.Timeout( + connect=min(10.0, self._timeout), + read=self._timeout, + write=min(60.0, self._timeout), + pool=min(10.0, self._timeout), + ) channel = None if captured is not None: - from core.services.desktop_gateway_uploads import ( - UPLOAD_OPTIONS_HEADER, - UPLOAD_SCHEMA_HEADER, - upload_channel, - ) + from core.harness.registry.transfer_channels import UPLOAD_OPTIONS_HEADER, UPLOAD_SCHEMA_HEADER, upload_channel channel = upload_channel(self._source_plugin, self.name) if channel is not None: body, options = await channel.package(kwargs, headers) - bridge.require_current_account(captured) - from core.services.automation_remote_effect import RECEIPT_TOOLS, remember_remote_call - from core.services.tool_effect_ledger import CURRENT_TOOL_EFFECT + bridge_identity.require_current_account(captured) + from core.harness.registry.tool_operations import receipt_operations + from core.harness.tool_effect_context import CURRENT_TOOL_EFFECT effect = CURRENT_TOOL_EFFECT.get() - if self.name in RECEIPT_TOOLS and effect is not None: - await asyncio.to_thread(remember_remote_call, effect, self.mcp_name, + operations = receipt_operations(self.name) + if operations is not None and effect is not None: + await asyncio.to_thread(operations.remember, effect, self.mcp_name, self._invoke_url, self._schema_hash) try: - async with httpx.AsyncClient(**client_kwargs) as client: + async with _http(self._transport) as client: if channel is not None: headers.update( { @@ -160,6 +167,7 @@ async def authorize_local(): f"{self._invoke_url.rsplit('/', 1)[0]}/{channel.endpoint}", headers=headers, content=body, + timeout=timeout, ) else: response = await client.post( @@ -170,20 +178,21 @@ async def authorize_local(): "arguments": kwargs, "schema_hash": self._schema_hash, }, + timeout=timeout, ) if captured is not None: - bridge.require_current_account(captured) + bridge_identity.require_current_account(captured) response.raise_for_status() payload = response.json() data = payload.get("data") if isinstance(payload, dict) else None if not isinstance(data, dict): raise ValueError("cloud gateway returned no tool result") if captured is not None: - from core.services.desktop_gateway_uploads import result_localizer + from core.harness.registry.transfer_channels import result_localizer localize = result_localizer(self._source_plugin, self.name) if localize is not None: - with bridge.account_scope(captured): + with bridge_identity.account_scope(captured): localize(data, captured["cloud_base"], kwargs, headers) chunk = ToolChunk.model_validate(data) chunk.metadata.setdefault("origin", "cloud") @@ -202,13 +211,13 @@ async def authorize_local(): if not self.is_read_only and (exc.response.status_code >= 500 or exc.response.status_code == 422): return self._unknown_outcome() if captured is not None and exc.response.status_code in (401, 403): - with bridge.account_scope(captured): - bridge.clear_state() + with bridge_identity.account_scope(captured): + bridge_identity.clear_state() if exc.response.status_code == 409: # 云端在此明确告知能力已变——这是「不轮询」下发现云端改动的信号之一， # 立刻后台同步一次清单，下一轮对话就是新的。 if captured is not None: - bridge.notify_cloud_changed() + bridge_identity.notify_cloud_changed() raise RuntimeError( f"云端工具 {self.name} 已更新，正在同步最新能力，请重试" ) from exc diff --git a/src/backend/core/llm/tools/cloud_manifest.py b/src/backend/core/llm/tools/cloud_manifest.py new file mode 100644 index 00000000..5a80d1f5 --- /dev/null +++ b/src/backend/core/llm/tools/cloud_manifest.py @@ -0,0 +1,55 @@ +"""Build native tools from an authorized cloud or local management manifest.""" +from core.capabilities.mcp.tool_schema import has_usable_schema +from core.capabilities.plugins.management.contract import declared, localize, tools as manager_tools +from core.harness.registry.run_capabilities import has_run_capabilities +from core.llm.tools.cloud_gateway import GatewayMCPTool +from core.llm.tools.management import ManagementTool + + +def manifest_tools(name, cfg): + plugin = str(cfg.get("gateway_plugin") or "") + url = str(cfg.get("gateway_invoke_url") or "") + schema_hash = str(cfg.get("schema_hash") or "") + items = cfg.get("manifest_tools") + if not isinstance(items, list): + raise ValueError("Missing authorized tool manifest") + if cfg.get("schema_source") == "cloud_manifest" and not (url and schema_hash): + raise ValueError("Cloud manifest requires an invocation URL and schema hash") + if cfg.get("schema_source") == "plugin_manifest" and not ( + cfg.get("origin") == "local_plugin" and all(declared(t, plugin) for t in items) + ): + raise ValueError("Invalid local management manifest") + if not all(isinstance(t, dict) for t in items): + raise ValueError("Invalid manifest tool entry") + items = list(items) + from extensions.mcp_servers.registry import gateway_server + if cfg.get("read_only") and gateway_server(name) is not None: + items = [t for t in items if t.get("annotations", {}).get("readOnlyHint") is True] + local = has_run_capabilities() + if local and plugin == "skill-manager" and any(t.get("name") == "install_skill" and declared(t, plugin) for t in items): + items += [t for t in manager_tools(plugin, "local") if t["name"] == "upload_skill_to_cloud" and not any(x.get("name") == t["name"] for x in items)] + result = [] + names = set() + for item in items: + if local: + item = localize(item, plugin, cloud_available=bool(cfg.get("manager_cloud_available") or url)) + if item is None or not has_usable_schema(item): + continue + tool_name = item.get("name") + if not isinstance(tool_name, str) or not tool_name or tool_name in names: + raise ValueError("Invalid or duplicate manifest tool name") + names.add(tool_name) + if cfg.get("enable_tools") is not None and tool_name not in cfg["enable_tools"]: + continue + if tool_name in (cfg.get("disable_tools") or []): + continue + import jsonschema + jsonschema.Draft202012Validator.check_schema(item["inputSchema"]) + headers = dict(cfg.get("headers") or {}) + if local and declared(item, plugin): + result.append(ManagementTool(name, item, plugin, headers, cloud_backed=bool(url))) + else: + result.append(GatewayMCPTool(mcp_name=name, tool=item, invoke_url=url, + schema_hash=schema_hash, headers=headers, timeout=cfg.get("execution_timeout") or 120, + transport=cfg.get("gateway_transport"), source_plugin=plugin)) + return result diff --git a/src/backend/core/llm/tools/_tool_confirmation/__init__.py b/src/backend/core/llm/tools/confirmation/__init__.py similarity index 100% rename from src/backend/core/llm/tools/_tool_confirmation/__init__.py rename to src/backend/core/llm/tools/confirmation/__init__.py diff --git a/src/backend/core/llm/tools/_tool_confirmation/gate.py b/src/backend/core/llm/tools/confirmation/gate.py similarity index 99% rename from src/backend/core/llm/tools/_tool_confirmation/gate.py rename to src/backend/core/llm/tools/confirmation/gate.py index 9f33c78d..afeda91e 100644 --- a/src/backend/core/llm/tools/_tool_confirmation/gate.py +++ b/src/backend/core/llm/tools/confirmation/gate.py @@ -7,10 +7,11 @@ from typing import Any, Dict, Optional from . import state -logger = logging.getLogger(__name__) from .shared import _load_session_grants, _save_session_grant from .pending import _register_pending, _wait_pending +logger = logging.getLogger(__name__) + async def gate( *, diff --git a/src/backend/core/llm/tools/_tool_confirmation/pending.py b/src/backend/core/llm/tools/confirmation/pending.py similarity index 96% rename from src/backend/core/llm/tools/_tool_confirmation/pending.py rename to src/backend/core/llm/tools/confirmation/pending.py index 8ad0e966..3069093f 100644 --- a/src/backend/core/llm/tools/_tool_confirmation/pending.py +++ b/src/backend/core/llm/tools/confirmation/pending.py @@ -6,13 +6,14 @@ import time import uuid from typing import Any, Dict, Optional, Tuple -from core.llm import interaction_store +from core.llm.runtime import interaction_store from . import state -logger = logging.getLogger(__name__) from .shared import _load_session_grants, _save_session_grant from core.llm.tool_permissions.pending import approve_current_policy +logger = logging.getLogger(__name__) + async def _register_pending( cid_key: str, key: Tuple[str, ...], *, make_pending, timeout, precheck=None @@ -21,7 +22,7 @@ async def _register_pending( now = time.monotonic() ui_q: Optional[asyncio.Queue] = None ui_payload: Optional[Dict[str, Any]] = None - with state._LOCK: + with state.process_state().LOCK: state._gc(now) st = state._get_chat(cid_key) st.last_ts = now @@ -63,7 +64,7 @@ async def _register_pending( if p.registration_error is not None: raise p.registration_error except BaseException: - with state._LOCK: + with state.process_state().LOCK: p.waiters -= 1 last = p.waiters <= 0 if last: @@ -126,12 +127,12 @@ async def _wait_pending( cancelled = True expire_q: Optional[asyncio.Queue] = None expire_payload: Optional[Dict[str, Any]] = None - with state._LOCK: + with state.process_state().LOCK: decision = p.decision or state._DECISION_TIMEOUT extracted = in_lock_extract(p, decision) if in_lock_extract and not cancelled else None p.waiters -= 1 if p.waiters <= 0: - st2 = state._CHATS.get(cid_key) + st2 = state.process_state().CHATS.get(cid_key) if st2 is not None: st2.pending.pop(cid, None) if st2.key_index.get(key) == cid: diff --git a/src/backend/core/llm/tools/_tool_confirmation/shared.py b/src/backend/core/llm/tools/confirmation/shared.py similarity index 98% rename from src/backend/core/llm/tools/_tool_confirmation/shared.py rename to src/backend/core/llm/tools/confirmation/shared.py index 3c971602..a7f1304c 100644 --- a/src/backend/core/llm/tools/_tool_confirmation/shared.py +++ b/src/backend/core/llm/tools/confirmation/shared.py @@ -2,7 +2,7 @@ from __future__ import annotations import logging -from core.llm import interaction_store +from core.llm.runtime import interaction_store from . import state logger = logging.getLogger(__name__) diff --git a/src/backend/core/llm/tools/_tool_confirmation/state.py b/src/backend/core/llm/tools/confirmation/state.py similarity index 89% rename from src/backend/core/llm/tools/_tool_confirmation/state.py rename to src/backend/core/llm/tools/confirmation/state.py index 28e95125..99fd18ba 100644 --- a/src/backend/core/llm/tools/_tool_confirmation/state.py +++ b/src/backend/core/llm/tools/confirmation/state.py @@ -1,6 +1,33 @@ """Local confirmation state, deduplication and session-scoped decisions.""" from __future__ import annotations +from core.sandbox.permissions.vocabulary import ( + KIND_MYSPACE, + KIND_AUTOMATION, + KIND_DESIGN_PICK, + KIND_LOCAL_CMD, + KIND_LOCAL_PATH_PREFIX, + KIND_TOOL_PERMISSION, + KIND_TOOL_PREFIX, + OP_WRITE, + OP_EDIT, + OP_DELETE, + OP_LOCAL_EXEC, + OP_LOCAL_READ, + OP_LOCAL_WRITE, + OP_MOVE, + OP_MKDIR, + OP_CRON_CREATE, + OP_CRON_UPDATE, + OP_CRON_DELETE, + DECISION_ALLOW, + DECISION_ALLOW_SESSION, + DECISION_DENY, + DECISION_CHOICE, + DECISION_SKIP, + STATUS_BLOCKED +) + import asyncio import logging import os @@ -9,36 +36,12 @@ import time from dataclasses import dataclass, field from typing import Any, Dict, Optional, Tuple -from core.llm.human_interaction import MAX_WAIT_SECONDS +from core.llm.runtime.human_interaction import MAX_WAIT_SECONDS logger = logging.getLogger(__name__) -KIND_MYSPACE = "myspace" -KIND_AUTOMATION = "automation" -KIND_DESIGN_PICK = "design_pick" -KIND_LOCAL_CMD = "local_cmd" -KIND_LOCAL_PATH_PREFIX = "local_path:" -KIND_TOOL_PERMISSION = "tool_permission" -KIND_TOOL_PREFIX = "tool:" -OP_WRITE = "write" -OP_EDIT = "edit" -OP_DELETE = "delete" -OP_LOCAL_EXEC = "local_exec" -OP_LOCAL_READ = "local_read" -OP_LOCAL_WRITE = "local_write" -OP_MOVE = "move" -OP_MKDIR = "mkdir" -OP_CRON_CREATE = "cron_create" -OP_CRON_UPDATE = "cron_update" -OP_CRON_DELETE = "cron_delete" -DECISION_ALLOW = "allow" -DECISION_ALLOW_SESSION = "allow_session" -DECISION_DENY = "deny" _DECISIONS = (DECISION_ALLOW, DECISION_ALLOW_SESSION, DECISION_DENY) _DECISION_TIMEOUT = "_timeout" -DECISION_CHOICE = "choice" -DECISION_SKIP = "skip" _DESIGN_DECISIONS = (DECISION_CHOICE, DECISION_SKIP) -STATUS_BLOCKED = "blocked_non_interactive" _TTL_S = 1800 _DEFAULT_WAIT_S = MAX_WAIT_SECONDS @@ -105,23 +108,19 @@ def _scope_covers(kind: str, granted: str, logical_path: str) -> bool: return True -_LOCK = threading.RLock() -_CHATS: Dict[str, _ChatConfirm] = {} - - def _gc(now: float) -> None: """Lazily clean up expired chat state (call while holding the lock). Chats with pending items / suspended waiters are not cleaned.""" - dead = [c for c, st in _CHATS.items() if now - st.last_ts > _TTL_S and (not st.pending)] + dead = [c for c, st in process_state().CHATS.items() if now - st.last_ts > _TTL_S and (not st.pending)] for c in dead: - _CHATS.pop(c, None) + process_state().CHATS.pop(c, None) def _get_chat(chat_id: str) -> _ChatConfirm: """Get or create per-chat state (call while holding the lock).""" - st = _CHATS.get(chat_id) + st = process_state().CHATS.get(chat_id) if st is None: st = _ChatConfirm() - _CHATS[chat_id] = st + process_state().CHATS[chat_id] = st return st @@ -129,7 +128,7 @@ def allow_session(chat_id: Optional[str]) -> None: """Pre-authorize inbound channel writes for this conversation's explicit scope.""" if not chat_id: return - with _LOCK: + with process_state().LOCK: st = _get_chat(chat_id) st.session_allows.add((KIND_MYSPACE, "", "")) st.last_ts = time.monotonic() @@ -145,8 +144,8 @@ def get_ui_queue(chat_id: Optional[str]) -> Optional[asyncio.Queue]: """ if not chat_id: return None - with _LOCK: - st = _CHATS.get(chat_id) + with process_state().LOCK: + st = process_state().CHATS.get(chat_id) return st.ui_signals if st is not None else None @@ -180,8 +179,8 @@ def get_pending(chat_id: Optional[str]) -> Optional[Dict[str, Any]]: """ if not chat_id: return None - with _LOCK: - st = _CHATS.get(chat_id) + with process_state().LOCK: + st = process_state().CHATS.get(chat_id) if st is None or not st.pending: return None live = [(cid, p) for cid, p in st.pending.items() if p.decision is None] @@ -202,8 +201,8 @@ def get_all_pending(chat_id: Optional[str]) -> list[Dict[str, Any]]: """ if not chat_id: return [] - with _LOCK: - st = _CHATS.get(chat_id) + with process_state().LOCK: + st = process_state().CHATS.get(chat_id) if st is None or not st.pending: return [] live = [(cid, p) for cid, p in st.pending.items() if p.decision is None] @@ -216,10 +215,10 @@ def get_all_pending(chat_id: Optional[str]) -> list[Dict[str, Any]]: def list_pending_chat_ids() -> list[str]: """All chat_ids that have undecided pending confirmations (the out-of-band batch endpoint uses this for ownership filtering).""" - with _LOCK: + with process_state().LOCK: return [ cid - for cid, st in _CHATS.items() + for cid, st in process_state().CHATS.items() if any((p.decision is None for p in st.pending.values())) ] @@ -228,8 +227,8 @@ def has_pending(chat_id: Optional[str]) -> bool: """Whether this chat has a live suspended confirmation/design wait.""" if not chat_id: return False - with _LOCK: - st = _CHATS.get(chat_id) + with process_state().LOCK: + st = process_state().CHATS.get(chat_id) return bool(st and any((p.decision is None for p in st.pending.values()))) @@ -248,8 +247,8 @@ def set_decision( - design_pick: {choice, skip}; for choice, option_id is required and must be a valid option Returns {ok, decision, op, logical_path} or {ok: False, error}. """ - with _LOCK: - st = _CHATS.get(chat_id) + with process_state().LOCK: + st = process_state().CHATS.get(chat_id) if st is None: return {"ok": False, "reason": "stale", "error": "该确认已失效（可能已超时或服务重启）"} p = st.pending.get(confirm_id) @@ -363,3 +362,16 @@ def _intercept_message(kind: str, phase: str, op: str, logical_path: str, summar "deny": f"用户拒绝了对「我的空间」的{op}操作（{logical_path}）。不要重试该写入。如确需产物，改写到本次会话的工作目录（相对路径），或向用户澄清意图。", "timeout": f"等待用户确认对「我的空间」的{op}操作超时（{logical_path}），已放弃未写入。请简短告知用户超时，让其重新发起或改写到会话工作目录。", }[phase] + + +@dataclass +class State: + LOCK: object = field(default_factory=lambda: threading.RLock()) + CHATS: Dict[str, _ChatConfirm] = field(default_factory=lambda: {}) + + +_STATE = State() + + +def process_state() -> State: + return _STATE diff --git a/src/backend/core/llm/tools/data_context_tool.py b/src/backend/core/llm/tools/data_context_tool.py deleted file mode 100644 index c47553e8..00000000 --- a/src/backend/core/llm/tools/data_context_tool.py +++ /dev/null @@ -1,96 +0,0 @@ -"""get_data_context tool — on-demand recall of "data dictionary + golden SQL" before data retrieval. - -The sole injection vehicle for improving "direct database query" accuracy: the -model proactively calls this tool before writing SQL, getting back relevant -table/column business names + descriptions + enum dictionaries + sample values -+ verified query examples, then writes the SELECT based on them. -**Metadata never enters the system prompt** — it only appears as this tool's -return value; the model sees it only by calling the tool. - -agent_factory registers this tool only when there exists a data source that is -"direct-connect (not external_nl2sql) + enabled + annotated" (see the mounting -gate in core/llm/factory/build.py). -""" - -import logging -from typing import List, Optional - -from agentscope.message import TextBlock -from agentscope.tool import Toolkit -from agentscope.tool._response import ToolChunk as ToolResponse - -logger = logging.getLogger(__name__) - -# Guardrail on the number of data sources returned per call (concatenating all of them would be too large with many sources). -_MAX_SOURCES_PER_CALL = 4 - - -def register_get_data_context(toolkit: Toolkit, datasource_ids: List[str]) -> None: - """Register the get_data_context tool, scoped to the given set of available data sources.""" - - scoped = list(datasource_ids) - - async def get_data_context(question: str, datasource: Optional[str] = None) -> ToolResponse: - """获取数据库的「数据字典 + 已验证查询范例」，用于准确取数。 - - ⚠️ **使用时机（强烈建议先调用本工具再写 SQL）**：当你要用「数据库直连查询」 - （execute_sql / search_objects）从业务库取数，而该库字段名晦涩、取值是编码 - （如 status=1）、或表很多不确定选哪张时，**先调用本工具**拿到人工标注的表/列 - 业务含义、枚举字典（如 1=已审核）、外键关联和已验证的「问题→SQL」范例，再据此 - 写 SELECT，可显著降低取数出错。 - - Args: - question (`str`): - 你要回答的取数问题（自然语言）。用于召回最相关的表/字段/范例。 - datasource (`str`, optional): - 指定数据源 id（仅当你已知道要查哪个库时填）。不填则返回全部可用 - 数据源的字典摘要。 - - Returns: - 数据字典与黄金 SQL 的 Markdown 文本；若该库尚无标注则返回提示。 - 注意：返回内容是人工治理的业务口径，应优先据此选表/字段、解码枚举值。 - """ - from core.services import db_metadata_service as svc - - if not scoped: - return _text("当前没有可用的数据字典（未配置元数据治理）。可直接用 " - "search_objects 探查表结构后再写 SQL。") - - targets: List[str] - ds = (datasource or "").strip() - if ds: - if ds in scoped: - targets = [ds] - else: - return _text( - f"数据源 `{ds}` 无可用数据字典。可用的数据源：{', '.join(scoped)}。" - "（不填 datasource 可返回全部）" - ) - else: - targets = scoped[:_MAX_SOURCES_PER_CALL] - - parts: List[str] = [] - for ds_id in targets: - try: - digest = svc.build_digest(ds_id, question=question) - except Exception as exc: # noqa: BLE001 - logger.warning("[get_data_context] build_digest failed for %s: %s", ds_id, exc) - digest = "" - if digest: - parts.append(digest) - - if not parts: - return _text("相关数据源暂无已标注的数据字典内容。可用 search_objects 探查表结构。") - - text = "\n\n---\n\n".join(parts) - if len(scoped) > len(targets): - text += (f"\n\n（另有 {len(scoped) - len(targets)} 个数据源未展开；" - "如需指定，请用 datasource 参数。可用：" + ", ".join(scoped) + "）") - return _text(text) - - toolkit.register_tool_function(get_data_context, namesake_strategy="override") - logger.info("[factory] Registered get_data_context tool (datasources=%s)", scoped) - - -def _text(s: str) -> ToolResponse: - return ToolResponse(content=[TextBlock(type="text", text=s)]) diff --git a/src/backend/core/llm/tools/edit_tool.py b/src/backend/core/llm/tools/edit_tool.py index 2880a85d..90bc64c5 100644 --- a/src/backend/core/llm/tools/edit_tool.py +++ b/src/backend/core/llm/tools/edit_tool.py @@ -1,6 +1,7 @@ """Read-checked mutations of files in the active sandbox.""" from __future__ import annotations +from core.llm.factory.tools.definitions import function_tool import difflib import hashlib @@ -8,19 +9,13 @@ from typing import Optional from agentscope.tool import Toolkit, ToolResponse -from core.services.project_scope import ProjectScope +from core.projects.edition import ProjectScope from ._common import ( resolve_sandbox_session, resp_json, ) -from ._paths import ( - is_myspace_physical, - space_metadata, - to_physical_path, - validate_project_scope_path, - validate_workspace_path, -) +from core.sandbox.workspace.resolution import space_metadata, to_physical_path, validate_project_scope_path, validate_workspace_path from ._state import ReadEntry, ReadStateTracker logger = logging.getLogger(__name__) @@ -63,10 +58,7 @@ async def Edit( new_string: str, replace_all: bool = False, ) -> "ToolResponse": # type: ignore[name-defined] - from core.llm.tool_permissions import ( - PermissionEnforcementError, - require_local_path_permission, - ) + from core.sandbox.permissions.dispatch_models import PermissionEnforcementError, require_local_path_permission from core.sandbox import SandboxConnectError as _SCE from core.sandbox import SandboxError as _SE from core.sandbox import get_sandbox_provider as _get_provider @@ -207,11 +199,12 @@ async def Edit( new_bytes = new_text.encode("utf-8") # Local-mode write-back snapshot (ticket #08): back up before overwriting. try: - from core.services.local_snapshot_service import maybe_snapshot_local + from core.harness.registry.local_files import maybe_snapshot_local maybe_snapshot_local(physical) - except Exception: - pass + except Exception as fallback_error: + from core.infra.failures import record_best_effort_failure + record_best_effort_failure('core.llm.tools.edit_tool:Edit', fallback_error) try: if _is_local: import os as _os @@ -277,5 +270,5 @@ async def Edit( " ``{error: '...'}`` 失败。\n" ) - toolkit.register_tool_function(Edit, namesake_strategy="override") + toolkit.append(function_tool(Edit)) logger.info("[factory] Registered Edit tool (chat_id=%s)", chat_id) diff --git a/src/backend/core/llm/tools/glob_tool.py b/src/backend/core/llm/tools/glob_tool.py index af69102b..34f5c9f4 100644 --- a/src/backend/core/llm/tools/glob_tool.py +++ b/src/backend/core/llm/tools/glob_tool.py @@ -1,5 +1,6 @@ """Authorized sandbox file search with stateless continuation.""" from __future__ import annotations +from core.llm.factory.tools.definitions import function_tool import logging import json @@ -8,10 +9,10 @@ from agentscope.tool import Toolkit, ToolResponse from core.db.paging import normalize_page, normalize_page_size -from core.services.project_scope import ProjectScope +from core.projects.edition import ProjectScope from ._common import resolve_sandbox_session, resp_json, sandbox_exec_bash, shell_quote -from ._paths import to_physical_path, validate_project_scope_path, validate_workspace_path +from core.sandbox.workspace.resolution import to_physical_path, validate_project_scope_path, validate_workspace_path logger = logging.getLogger(__name__) @@ -95,5 +96,5 @@ async def Glob( "limit 默认 100；0 或负数返回全部。page 仅兼容第一页。" ) - toolkit.register_tool_function(Glob, namesake_strategy="override") + toolkit.append(function_tool(Glob)) logger.info("[factory] Registered Glob tool (chat_id=%s)", chat_id) diff --git a/src/backend/core/llm/tools/grep_tool.py b/src/backend/core/llm/tools/grep_tool.py index 7830ff36..0541b8c8 100644 --- a/src/backend/core/llm/tools/grep_tool.py +++ b/src/backend/core/llm/tools/grep_tool.py @@ -7,21 +7,17 @@ """ from __future__ import annotations -from core.services.edition_workspace import is_shared_scope +from core.llm.factory.tools.definitions import function_tool import logging from typing import Optional from agentscope.tool import Toolkit, ToolResponse -from core.services.project_scope import ProjectScope +from core.projects.edition import ProjectScope from ._common import resolve_sandbox_session, resp_json, sandbox_exec_bash, shell_quote -from ._paths import ( - to_physical_path, - validate_project_scope_path, - validate_workspace_path, -) +from core.sandbox.workspace.resolution import to_physical_path, validate_project_scope_path, validate_workspace_path logger = logging.getLogger(__name__) @@ -181,5 +177,5 @@ async def Grep( " num_matches, truncated}``。\n" ) - toolkit.register_tool_function(Grep, namesake_strategy="override") + toolkit.append(function_tool(Grep)) logger.info("[factory] Registered Grep tool (chat_id=%s)", chat_id) diff --git a/src/backend/core/llm/tools/job_tool.py b/src/backend/core/llm/tools/job_tool.py index 35f120b3..e528a97a 100644 --- a/src/backend/core/llm/tools/job_tool.py +++ b/src/backend/core/llm/tools/job_tool.py @@ -1,72 +1,11 @@ -"""run_job 工具 —— 主对话智能体提交批量作业的唯一入口。 - -定位：这不是「又一个固定形状的批处理工具」，而是一个**可编程的编排面**。智能体先用 -``write`` 把一段作业脚本写进沙箱，再用 ``run_job`` 提交；脚本里的控制流（循环、条件、 -分层取数、动态收敛、嵌套 map）都是普通 Python，平台不需要知道任务在做什么。 - -与 Bash / read / write / glob / grep 同级原生注册，不走 catalog 开关——「对话内默认 -可用」就落在这里。 -""" - -from __future__ import annotations - -import json as _json -import logging +"""SDK binding for the user-scoped batch job action service.""" +from core.llm.factory.tools.definitions import function_tool +import json from typing import Any, Dict, List, Optional - from agentscope.message import TextBlock from agentscope.tool import Toolkit -from agentscope.tool._response import ToolChunk as ToolResponse - -from ._paths import quote_shell_path as _quote_path - -logger = logging.getLogger(__name__) - -# 前台等待（wait=True）的硬上限。超过就把作业转后台，不再阻塞这一轮对话。 -# 该等多久是模型自己判断的，而它对"这批要跑多久"的估计经常偏乐观——实测把一个 265 项、 -# 每项都要过模型的分类作业当成"小作业"，前台一口气阻塞了 5 分钟，用户看不到进度、 -# 插不上话、也没法中途取消。提示词已经写过"几分钟以上就走后台"仍然拦不住，所以在 -# 机制上给前台等待封顶：等够 90 秒还没完，作业照跑，对话先还给用户。 -_FOREGROUND_WAIT_CAP_S = 90.0 - - -def _resp(payload: Dict[str, Any]) -> ToolResponse: - return ToolResponse( - content=[TextBlock(type="text", text=_json.dumps(payload, ensure_ascii=False))] - ) - - -def _mark_wake_on_finish(job_id: str) -> None: - """转后台后要有人叫醒会话播报终态，否则作业跑完没人吭声。""" - try: - from core.db.engine import SessionLocal - from core.services.job_service import JobService - - with SessionLocal() as db: - JobService(db).set_wake_on_finish(job_id, True) - except Exception: # noqa: BLE001 —— 叫醒是尽力而为，不该反过来弄挂工具调用 - logger.warning("[run_job] mark wake_on_finish failed job=%s", job_id, exc_info=True) - - -def _detached_payload(job_id: str, *, resumed: bool = False) -> Dict[str, Any]: - """前台等超时 → 按后台语义返回（作业没停，只是不占着这一轮对话了）。""" - _mark_wake_on_finish(job_id) - return { - "ok": True, - "job_id": job_id, - "status": "started", - "waited": False, - "detached": True, - "next": ( - f"作业比预期久（前台已等满 {int(_FOREGROUND_WAIT_CAP_S)} 秒），" - "**已自动转入后台继续跑，没有中断**。" - "现在就结束这轮回复：告诉用户作业仍在进行、" - "输入框上方的状态条会实时显示进度，期间可以继续聊别的、也可以随时取消。" - "作业跑完会自动叫醒你播报，不要在这里轮询 action='status' 干等，" - "也不要重新提交一遍（它还在跑）。" - ), - } - +from agentscope.tool._response import ToolChunk +from core.harness.registry.job_operations import job_operations def register_run_job( toolkit: Toolkit, @@ -86,6 +25,8 @@ def register_run_job( ``allowed_tools`` 保留脚本显式选择 MCP 的权限检查。 """ + execute = job_operations(user_id=user_id, chat_id=chat_id, sandbox_session_id=sandbox_session_id, allowed_tools=allowed_tools, model_name=model_name, model_provider_id=model_provider_id, project_ctx=project_ctx, parent_runtime=parent_runtime) + async def run_job( action: str = "start", script_path: str = "", @@ -104,312 +45,10 @@ async def run_job( status: str = "", progress_wake_sec: int = 300, on_conflict: str = "block", - ) -> ToolResponse: - from core.db.engine import SessionLocal - from core.db.models import Job - from core.services.job_service import JobService - from orchestration import job_runtime - from orchestration.jobs import files as job_files - - act = (action or "start").strip().lower() - - if act == "start": - if not script_path: - return _resp( - {"ok": False, "error": "script_path 必填：先用 write 把作业脚本写进沙箱"} - ) - if not sandbox_session_id: - return _resp({"ok": False, "error": "当前会话没有可用沙箱，无法提交作业"}) - - # 同一会话不许叠加在跑的作业。被进度唤醒后"改个脚本再交一份"是很自然的动作， - # 但旧作业不会自己消失：两份并存会同时烧预算、同时叫醒会话，用户在状态条上 - # 也分不清哪份才算数。要么先 cancel，要么 resume（已完成的项不会重做）。 - # 本会话已有在跑的作业时**默认**拦一道：默默叠加两份会双倍烧预算、双份叫醒会话， - # 用户在状态条上也分不清哪份算数。但这只是默认，不是禁令——确实要换新版本时 - # 用 on_conflict 明说：'replace' 停掉旧的再跑新的，'parallel' 两份并行。 - # 判断权留给调用方，工具只负责让"叠加"变成一个显式选择而不是意外。 - conflict = (on_conflict or "block").strip().lower() - if chat_id and conflict != "parallel": - # 字段必须在 session 内取出：出了 with 块 ORM 实例就 detached，再读属性会炸 - with SessionLocal() as db: - live = ( - db.query(Job) - .filter( - Job.chat_id == chat_id, - Job.status.in_(("pending", "running")), - ) - .order_by(Job.created_at.desc()) - .first() - ) - live_id = live.job_id if live is not None else "" - live_name = (live.name or "未命名") if live is not None else "" - if live_id and conflict == "replace": - await job_runtime.cancel_job(live_id, user_id=user_id) - logger.info("[run_job] replaced live job %s in chat %s", live_id, chat_id) - elif live_id: - return _resp( - { - "ok": False, - "error": ( - f"本会话已有在跑的作业 {live_id}（{live_name}）。三条路选一条：" - "① 想用新脚本取代它 → 再调一次本工具并带 on_conflict='replace'" - "（自动停掉旧作业再提交新的）；" - f"② 旧作业只是中断、脚本没问题 → run_job(action='resume', job_id='{live_id}')" - "断点续跑，已完成的项不会重做；" - "③ 确实需要两份同时跑 → on_conflict='parallel'。" - "默认拦下只是为了让叠加成为显式选择，不是不让跑。" - ), - "job_id": live_id, - "hint": "on_conflict=replace|parallel", - } - ) - - # 读脚本正文：既做存在性校验，也作为 job 的审计快照与 resume 依据。 - # ⚠️ 必须走 base64，不能直接 `cat`：沙箱 execute 回传的 stdout 会丢换行， - # 脚本会被压成一行落地 → SyntaxError（实测踩过）。base64 -w0 保字节不变。 - try: - code, out, err = await job_files._sbx_bash( - f"base64 -w0 {_quote_path(script_path)} 2>/dev/null || base64 -i {_quote_path(script_path)}", - session_id=sandbox_session_id, - user_id=user_id, - timeout=30, - ) - except Exception as exc: # noqa: BLE001 - return _resp({"ok": False, "error": f"读取脚本失败: {exc}"}) - if code != 0 or not (out or "").strip(): - return _resp( - {"ok": False, "error": f"脚本不存在或为空: {script_path} {(err or '')[:200]}"} - ) - import base64 as _b64dec - - try: - out = _b64dec.b64decode("".join((out or "").split())).decode("utf-8") - except Exception as exc: # noqa: BLE001 - return _resp({"ok": False, "error": f"脚本解码失败: {exc}"}) - if not out.strip(): - return _resp({"ok": False, "error": f"脚本为空: {script_path}"}) - - budget: Dict[str, Any] = {"concurrency": concurrency} - if max_calls: - budget["max_calls"] = max_calls - if max_seconds: - budget["max_seconds"] = max_seconds - - try: - jid = await job_runtime.start_job( - user_id=user_id, - chat_id=chat_id, - name=name or "批量作业", - script_path=script_path, - script_text=out, - session_id=sandbox_session_id, - budget=budget, - start_params={ - "allowed_tools": list(allowed_tools or []), - "model_name": model_name, - "model_provider_id": model_provider_id, - "project_ctx": project_ctx, - "parent_runtime": parent_runtime, - # wait=False 时作业跑完没人叫醒本会话 → 打标，让驱动在终态入队一轮续跑 - "wake_on_finish": not wait, - # 中途进度播报间隔（秒）；0 = 只在终态叫一次 - "progress_wake_sec": max(0, int(progress_wake_sec)), - }, - interpreter=interpreter or "${PY_BIN:-python3}", - ) - except Exception as exc: # noqa: BLE001 - logger.warning("[run_job] start failed: %s", exc) - return _resp({"ok": False, "error": f"作业启动失败: {exc}"}) - - if not wait: - return _resp( - { - "ok": True, - "job_id": jid, - "status": "started", - "waited": False, - "next": ( - "作业已在后台开始。**现在就结束这轮回复**：告诉用户作业已提交、" - "输入框上方的状态条会实时显示进度（几分之几、失败数、已运行多久），" - "期间可以继续聊别的、也可以点状态条上的按钮取消。" - "作业跑完和中途都会自动叫醒你播报，不要在这里轮询 action='status' 干等。" - ), - } - ) - - res = await job_runtime.run_and_wait( - jid, chat_id=chat_id, detach_after=_FOREGROUND_WAIT_CAP_S - ) - if res.get("status") == "detached": - return _resp(_detached_payload(jid)) - with SessionLocal() as db: - svc = JobService(db) - stats = svc.stats(jid) - job = svc.get(jid) - usage = dict(job.usage or {}) if job else {} - payload = { - "ok": res.get("status") == "completed", - "job_id": jid, - "status": res.get("status"), - "stats": stats, - "usage": usage, - } - if res.get("error"): - payload["error"] = res["error"] - if stats.get("remaining", 0) > 0: - payload["note"] = ( - f"仍有 {stats['remaining']} 项未完成——必须如实告知用户并给出未覆盖清单与" - f"续跑方式（run_job action=resume job_id={jid}），不得当作已完成。" - ) - # 脚本崩溃时把 runner 日志尾巴带回来，省一轮排查 - if res.get("status") == "failed" and sandbox_session_id: - try: - payload["runner_log"] = ( - await job_files.read_runner_log( - jid, user_id=user_id, session_id=sandbox_session_id - ) - )[-1500:] - except Exception: # noqa: BLE001 - pass - return _resp(payload) - - if not job_id: - return _resp({"ok": False, "error": f"action={act} 需要 job_id"}) - - # job_id 容错 + 找不到时把本会话的作业列出来。 - # 实测踩过：模型凭记忆抄 id 时漏了 "job_" 前缀，只回一句「job 不存在」它无从自救， - # 结果决定把 568 次子调用整个重跑一遍。错误信息必须自带下一步。 - with SessionLocal() as db: - svc = JobService(db) - resolved = svc.get(job_id) - if (resolved is None or resolved.user_id != user_id) and not job_id.startswith("job_"): - cand = svc.get(f"job_{job_id}") - if cand is not None and cand.user_id == user_id: - resolved, job_id = cand, f"job_{job_id}" - if resolved is None or resolved.user_id != user_id: - from core.db.models import Job - - rows = ( - db.query(Job) - .filter(Job.user_id == user_id, Job.chat_id == chat_id) - .order_by(Job.created_at.desc()) - .limit(5) - .all() - ) - return _resp( - { - "ok": False, - "error": f"job 不存在或无权访问: {job_id}", - "jobs_in_this_chat": [ - { - "job_id": r.job_id, - "name": r.name, - "status": r.status, - "stats": svc.stats(r.job_id), - } - for r in rows - ], - "hint": "用上面列表里的完整 job_id 重试（注意 job_ 前缀）；" - "已经跑完的作业**不要重跑**，直接 export 取结果。", - } - ) - - if act == "status": - with SessionLocal() as db: - svc = JobService(db) - job = svc.get(job_id) - if job is None or job.user_id != user_id: - return _resp({"ok": False, "error": "job 不存在或无权访问"}) - return _resp( - { - "ok": True, - "job_id": job_id, - "status": job.status, - "stats": svc.stats(job_id), - "usage": dict(job.usage or {}), - "budget_left": svc.budget_left(job_id), - "error": job.error_message, - } - ) - - if act == "export": - # 台账在 DB，逐项结果**不进对话上下文**——所以导出的方式是写成沙箱里的 - # JSONL 文件，之后用 Bash/python 处理（写 Excel、跑校验、喂评审）。 - if not sandbox_session_id: - return _resp({"ok": False, "error": "当前会话没有可用沙箱"}) - with SessionLocal() as db: - svc = JobService(db) - job = svc.get(job_id) - if job is None or job.user_id != user_id: - return _resp({"ok": False, "error": "job 不存在或无权访问"}) - wanted = [s.strip() for s in (status or "").split(",") if s.strip()] or [ - "done", - "not_found", - "failed", - "needs_review", - "pending", - ] - rows = [] - for st in wanted: - rows.extend(svc.pending(job_id, status=st)) - stats = svc.stats(job_id) - - dest = dest_path or f"jobs/{job_id}_ledger.jsonl" - body = "\n".join(_json.dumps(r, ensure_ascii=False) for r in rows) - # 分块写 + 读回校验：单条 Bash 携带大 base64 到约 170KB 会**静默失败** - # （exit=0、stderr 空、文件不存在）。568 行台账正好落在这个区间——历史上 - # 因此出现过「导出报成功 rows=568，文件却从来没出现」。成功与否只认沙箱 - # 里读回的真实字节数，绝不用 Python 侧的行数报平安。 - try: - ok_write, detail = await job_files.write_sandbox_file( - dest, body, session_id=sandbox_session_id, user_id=user_id - ) - except Exception as exc: # noqa: BLE001 - return _resp({"ok": False, "error": f"导出失败: {exc}"}) - if not ok_write: - return _resp({"ok": False, "error": f"导出失败（落盘未通过校验）: {detail}"}) - return _resp( - { - "ok": True, - "job_id": job_id, - "path": dest, - "rows": len(rows), - "bytes": detail, - "stats": stats, - "hint": "每行一个 JSON：{key, status, payload, result, review, attempts}。" - "用 Bash/python 读这个文件写产物，不要把逐项内容读回对话。", - } - ) - - if act == "cancel": - ok = await job_runtime.cancel_job(job_id, user_id=user_id) - return _resp({"ok": ok, "job_id": job_id, "status": "cancelled" if ok else "unknown"}) - - if act == "resume": - res = await job_runtime.resume_job(job_id, user_id=user_id, chat_id=chat_id) - if not res.get("ok"): - return _resp({"ok": False, "error": res.get("error")}) - if not wait: - return _resp({"ok": True, "job_id": job_id, "status": "running", "waited": False}) - done = await job_runtime.run_and_wait( - job_id, chat_id=chat_id, detach_after=_FOREGROUND_WAIT_CAP_S - ) - if done.get("status") == "detached": - return _resp(_detached_payload(job_id, resumed=True)) - with SessionLocal() as db: - stats = JobService(db).stats(job_id) - return _resp( - { - "ok": done.get("status") == "completed", - "job_id": job_id, - "status": done.get("status"), - "stats": stats, - } - ) - - return _resp({"ok": False, "error": f"未知 action: {act}"}) + ) -> ToolChunk: + payload = await execute(action=action, script_path=script_path, name=name, job_id=job_id, wait=wait, interpreter=interpreter, max_calls=max_calls, max_seconds=max_seconds, concurrency=concurrency, dest_path=dest_path, status=status, progress_wake_sec=progress_wake_sec, on_conflict=on_conflict) + return ToolChunk(content=[TextBlock(type="text", text=json.dumps(payload, ensure_ascii=False))]) from .job_description import RUN_JOB_DESCRIPTION - run_job.__doc__ = RUN_JOB_DESCRIPTION - toolkit.register_tool_function(run_job) + toolkit.append(function_tool(run_job)) diff --git a/src/backend/core/llm/tools/management.py b/src/backend/core/llm/tools/management.py new file mode 100644 index 00000000..7e4fce90 --- /dev/null +++ b/src/backend/core/llm/tools/management.py @@ -0,0 +1,45 @@ +"""Local execution of an explicitly declared, installed manager plugin contract.""" +import asyncio +import json +from agentscope.tool import ToolBase, ToolChunk +from agentscope.message import TextBlock, ToolResultState +from agentscope.permission import PermissionBehavior, PermissionDecision +from core.capabilities.plugins.management.errors import management_error + + +class ManagementTool(ToolBase): + is_mcp = True + is_state_injected = False + is_external_tool = False + is_concurrency_safe = False + + def __init__(self, server_id, tool, manager, headers, *, cloud_backed): + super().__init__() + self.mcp_name, self.name = server_id, tool["name"] + self.description, self.input_schema = tool.get("description", ""), tool["inputSchema"] + self.manager, self.headers = manager, dict(headers) + self.cloud_backed = cloud_backed + self.is_read_only = self.name.startswith(("get_", "list_")) + + async def check_permissions(self, *_args, **_kwargs): + return PermissionDecision(behavior=PermissionBehavior.ALLOW if self.is_read_only else PermissionBehavior.ASK, + message="管理当前用户的本机技能或插件") + + def _execute(self, arguments): + from core.harness.registry.capability_management import execute_manager + return execute_manager( + manager=self.manager, server_id=self.mcp_name, headers=self.headers, + cloud_backed=self.cloud_backed, tool_name=self.name, + input_schema=self.input_schema, arguments=arguments, + ) + + async def call(self, **kwargs): + try: + result = await asyncio.to_thread(self._execute, kwargs) + return ToolChunk(content=[TextBlock(text=json.dumps(result, ensure_ascii=False))], + state=ToolResultState.SUCCESS, + metadata={"origin": "local", "capabilities_changed": not self.is_read_only}) + except Exception as exc: + message = management_error(exc) + return ToolChunk(content=[TextBlock(text=json.dumps({"ok": False, "error": message}, ensure_ascii=False))], + state=ToolResultState.ERROR, metadata={"origin": "local"}) diff --git a/src/backend/core/ontology/toolkit.py b/src/backend/core/llm/tools/ontology_toolkit.py similarity index 85% rename from src/backend/core/ontology/toolkit.py rename to src/backend/core/llm/tools/ontology_toolkit.py index 10698e2e..5a8a54cc 100644 --- a/src/backend/core/ontology/toolkit.py +++ b/src/backend/core/llm/tools/ontology_toolkit.py @@ -13,11 +13,9 @@ from agentscope.tool._response import ToolChunk, ToolResponse from jinja2 import Template -from core.llm.execution_manifest import canonical_json, stable_tool_schema_order -from core.llm.tool_permissions import ( - CURRENT_PERMISSION_TICKET, - ToolPermissionService, -) +from core.harness.execution_manifest import canonical_json, stable_tool_schema_order +from core.sandbox.permissions.dispatch_models import CURRENT_PERMISSION_TICKET +from core.llm.tool_permissions import ToolPermissionService @dataclass(frozen=True) @@ -50,6 +48,7 @@ class OntologyFilteredToolkit(Toolkit): def __init__(self, *args, hidden_tools: set[str] | None = None, **kwargs): super().__init__(*args, **kwargs) + self.capability_guard = None self._ontology_hidden_tools = set(hidden_tools or ()) self._execution_surface: Optional[ExecutionSurfaceSnapshot] = None self._execution_surface_generation = 0 @@ -71,6 +70,17 @@ def set_tool_permission_service( self._tool_permission_service = service async def call_tool(self, tool_call, state): # noqa: ANN001 + if self.capability_guard is not None: + tools = await self._get_available_tools(state.tool_context.activated_groups) + registered = tools.get(tool_call.name) + tool = registered.tool if registered is not None else None + if tool is not None and tool.is_mcp: + allowed = await asyncio.to_thread(self.capability_guard, tool.mcp_name) + if not allowed: + chunk = ToolChunk(content=[TextBlock(text="该连接器已停用或当前不可用，本次调用未执行。")], state=ToolResultState.ERROR) + yield chunk + yield ToolResponse(id=tool_call.id).append_chunk(chunk) + return service = self._tool_permission_service if service is not None and service.registry.contains(tool_call.name): ticket = CURRENT_PERMISSION_TICKET.get() @@ -115,11 +125,9 @@ async def get_tool_schemas(self, groups=None): # noqa: ANN001 snapshot = await self.freeze_execution_surface(groups) return snapshot.tool_schemas - async def get_skill_instructions(self) -> Optional[str]: - """Return the skill prompt from the same frozen execution surface.""" - snapshot = self._execution_surface - if snapshot is None: - snapshot = await self.freeze_execution_surface() + async def get_skill_instructions(self, activated_groups=None) -> Optional[str]: + """Use the same group-scoped snapshot as the model's tool schemas.""" + snapshot = await self.freeze_execution_surface(activated_groups) return snapshot.skill_instructions @staticmethod @@ -153,8 +161,6 @@ async def freeze_execution_surface( self._capture_surface_skills = False skills = self._captured_surface_skills if skills is None: - # Compatibility fallback if a future AgentScope version stops - # consulting skills while constructing the tool surface. skills = await self._get_available_skills(groups) skill_instructions = None if skills: diff --git a/src/backend/core/llm/tools/pin_tool.py b/src/backend/core/llm/tools/pin_tool.py index 40c2db80..eb3e9f05 100644 --- a/src/backend/core/llm/tools/pin_tool.py +++ b/src/backend/core/llm/tools/pin_tool.py @@ -3,16 +3,17 @@ Extracted from the oversized ``core/llm/tool.py``. Heavy deps imported lazily inside the function; ``core.llm.tool`` re-exports ``register_pin_to_workspace``. """ +from core.llm.factory.tools.definitions import function_tool import logging from typing import Any, Dict, Optional, TYPE_CHECKING if TYPE_CHECKING: - from core.services.project_scope import ProjectScope + from core.projects.edition import ProjectScope from agentscope.message import TextBlock from agentscope.tool import Toolkit -from agentscope.tool._response import ToolChunk as ToolResponse +from agentscope.tool._response import ToolChunk logger = logging.getLogger(__name__) @@ -39,11 +40,11 @@ def register_pin_to_workspace( output leaked into the personal MySpace root. Now the scope travels with the call chain, no timing window. """ - from core.services.project_scope import ProjectScope # noqa: F401 - re-import for closure + from core.projects.edition import ProjectScope # noqa: F401 - re-import for closure async def pin_to_workspace( file_ids: list[str] = [], file_paths: list[str] = [] - ) -> ToolResponse: + ) -> ToolChunk: """把文件交付到对话区——**唯一**让用户看到文件的方式。 凡用户要求生成/导出文件（文档、图片、PPT、Excel、PDF、CSV、压缩包、音视频、 @@ -73,8 +74,8 @@ async def pin_to_workspace( """ import json as _json - from core.artifacts.store import get_artifact - from core.llm import workspace as _workspace + from core.files.artifacts.store import get_artifact + from core.llm.runtime import workspace as _workspace # Mark the gate active even if every id below fails — the agent's # *intent* to use the workspace is what flips the default. Otherwise @@ -89,7 +90,7 @@ async def pin_to_workspace( elif isinstance(file_ids, list): raw_ids = file_ids else: - return ToolResponse( + return ToolChunk( content=[ TextBlock( type="text", @@ -105,7 +106,7 @@ async def pin_to_workspace( ) if not raw_ids and not file_paths: - return ToolResponse( + return ToolChunk( content=[ TextBlock( type="text", @@ -125,7 +126,7 @@ async def pin_to_workspace( from core.infra.logging import user_id_var from fastapi import HTTPException - from core.artifacts.local_reference import reference_local_file + from core.files.artifacts.local_reference import reference_local_file if not isinstance(file_paths, list): file_paths = [file_paths] @@ -135,7 +136,7 @@ async def pin_to_workspace( failed_results.append({"path": str(path), "error": "文件路径必须是非空字符串"}) continue try: - from core.artifacts.local_reference import project_file_path + from core.files.artifacts.local_reference import project_file_path from core.infra.logging import chat_id_var physical = project_file_path( @@ -218,7 +219,7 @@ async def pin_to_workspace( _cid = chat_id_var.get() or "" if _uid and to_persist: from core.db.engine import SessionLocal - from core.services.artifact_service import persist_artifacts + from core.files.artifacts.service import persist_artifacts _db = SessionLocal() try: @@ -251,7 +252,7 @@ async def pin_to_workspace( } if failed_results: result["failed"] = failed_results - return ToolResponse( + return ToolChunk( content=[ TextBlock( type="text", @@ -260,10 +261,11 @@ async def pin_to_workspace( ] ) - from core.llm.tool_permissions import READ, ToolPermissionSpec, local_path_tool + from core.sandbox.permissions.dispatch_models import READ, ToolPermissionSpec + from core.sandbox.permissions.tool_specs import local_path_tool def resolve_pin_paths(args, runtime): - from core.artifacts.local_reference import project_file_path + from core.files.artifacts.local_reference import project_file_path paths = args.get("file_paths") or [] if isinstance(paths, str): @@ -284,11 +286,7 @@ def resolve_pin_paths(args, runtime): ) return intents - toolkit.register_tool_function( - pin_to_workspace, - namesake_strategy="override", - permission=ToolPermissionSpec("local-project-pin-paths", resolve_pin_paths), - ) + toolkit.append(function_tool(pin_to_workspace, dispatch_spec=ToolPermissionSpec("local-project-pin-paths", resolve_pin_paths))) logger.info("[factory] Registered pin_to_workspace tool") diff --git a/src/backend/core/llm/plan_update_tool.py b/src/backend/core/llm/tools/plan_update.py similarity index 92% rename from src/backend/core/llm/plan_update_tool.py rename to src/backend/core/llm/tools/plan_update.py index a963a451..6feaaa8b 100644 --- a/src/backend/core/llm/plan_update_tool.py +++ b/src/backend/core/llm/tools/plan_update.py @@ -5,14 +5,16 @@ """ from __future__ import annotations +from core.llm.factory.tools.definitions import function_tool +from core.llm import prompt_rendering import logging from typing import Any, Dict, List, Optional from agentscope.tool import Toolkit -# AgentScope 2.0: tool functions must return ToolChunk (call_tool rejects ToolResponse). -from agentscope.tool._response import ToolChunk as ToolResponse +# AgentScope 2.0: tool functions must return ToolChunk (call_tool rejects ToolChunk). +from agentscope.tool._response import ToolChunk from agentscope.message import TextBlock logger = logging.getLogger(__name__) @@ -63,7 +65,7 @@ def register_plan_update_tool(toolkit: Toolkit) -> None: 不更新（commit 7578469f）。 """ - async def update_plan(steps: list, title: str = "", explanation: str = "") -> ToolResponse: + async def update_plan(steps: list, title: str = "", explanation: str = "") -> ToolChunk: """更新任务计划。传入一份计划项列表，每项含步骤与状态，以及可选的说明。 至多一个步骤可以处于 in_progress。调用本工具**不会打断**执行。 @@ -82,12 +84,12 @@ async def update_plan(steps: list, title: str = "", explanation: str = "") -> To （可选）本次改动的理由。中途增删步骤、重写计划结构时必须填写。 Returns: - `ToolResponse`: + `ToolChunk`: 当前清单的各状态计数。 """ parsed = parse_plan_update_args({"title": title, "steps": steps}) if not parsed: - return ToolResponse(content=[TextBlock( + return ToolChunk(content=[TextBlock( type="text", text=( "错误：steps 必须是非空列表，每个元素为 " @@ -114,9 +116,9 @@ async def update_plan(steps: list, title: str = "", explanation: str = "") -> To "\n全部步骤已完成，计划终结：不要再调用 update_plan，也不要原样重复" "任何已成功的工具调用，请直接输出面向用户的最终文字回复。" ) - return ToolResponse(content=[TextBlock(type="text", text=text)]) + return ToolChunk(content=[TextBlock(type="text", text=text)]) - toolkit.register_tool_function(update_plan, namesake_strategy="skip") + toolkit.append(function_tool(update_plan)) def build_plan_update_prompt_section() -> str: @@ -125,6 +127,5 @@ def build_plan_update_prompt_section() -> str: 正文的真源是 ``prompts/prompt_text/plan_tool/plan_tool.system.md``，运行时取 ``plan_tool`` 版本池的激活版本，可在 Config 管理台「任务计划清单」里编辑。 """ - from core.services import prompt_version_service as pvs - return pvs.render_kind_segment("plan_tool") + return prompt_rendering.render_kind_segment("plan_tool") diff --git a/src/backend/core/plugins/runtime/activation.py b/src/backend/core/llm/tools/plugin_activation.py similarity index 78% rename from src/backend/core/plugins/runtime/activation.py rename to src/backend/core/llm/tools/plugin_activation.py index 8a11e77d..73764e9f 100644 --- a/src/backend/core/plugins/runtime/activation.py +++ b/src/backend/core/llm/tools/plugin_activation.py @@ -1,18 +1,20 @@ """Progressive plugin runtime: activation.""" from __future__ import annotations +from core.llm.factory.tools.definitions import function_tool import asyncio import logging from contextlib import asynccontextmanager from typing import Any, Dict, List, Set -logger = logging.getLogger(__name__) -from core.plugins.runtime import activation_history as plugin_activation_history -from core.plugins.runtime import models as plugin_runtime_models -from core.plugins.runtime import resolution as plugin_resolution +from core.capabilities.plugins.runtime import activation_history as plugin_activation_history +from core.capabilities.plugins.runtime import models as plugin_runtime_models +from core.capabilities.plugins.runtime import resolution as plugin_resolution + +logger = logging.getLogger(__name__) class _ActivationStage: @@ -21,9 +23,7 @@ class _ActivationStage: A device capability run must never expose half a plugin: if the authorized closure stops holding — or an MCP turns out to be unreachable — after some clients already connected, the run has to end up exactly as it started, so - everything lands in a buffer that is published only on success. The legacy - cloud path has no such checkpoint and keeps appending straight to the live - group. + everything lands in a buffer that is published only on success. """ def __init__(self, live_group: Any, runtime: Dict[str, Any], prepared: Any) -> None: @@ -33,41 +33,36 @@ def __init__(self, live_group: Any, runtime: Dict[str, Any], prepared: Any) -> N self._runtime = runtime self._prepared = prepared self._staged_clients: List[Any] = [] - self.staged = prepared is not None - self.group = SimpleNamespace(mcps=[], skills_or_loaders=[]) if self.staged else live_group + self.group = SimpleNamespace(tools=[], skills_or_loaders=[]) current = runtime.setdefault("connected_keys", set()) - self.connected: Set[str] = set(current) if self.staged else current + self.connected: Set[str] = set(current) async def checkpoint(self) -> None: """Re-assert the authorized closure between steps of the activation.""" if self._prepared is None: return - from core.capabilities.runtime import validate + from core.harness.registry.capability_runtime import validate await asyncio.to_thread(validate, self._prepared) def add_client(self, key: str, client: Any) -> None: - self.group.mcps.append(client) + self.group.tools.extend(client.tools) self.connected.add(key) - if self.staged: - self._staged_clients.append(client) - return - close_list = self._runtime.get("close_list") - if isinstance(close_list, list): - close_list.append(client) + self._staged_clients.append(client) async def discard(self) -> None: - for client in self._staged_clients: - try: - await client.close() - except Exception as exc: # noqa: BLE001 — one bad close must not leak the rest - logger.warning("[plugin-loader] staged MCP close failed: %s", exc) - self._staged_clients.clear() + from core.llm.mcp.manager import close_clients + + try: + await close_clients(self._staged_clients) + finally: + self._staged_clients.clear() def commit(self) -> None: - if not self.staged: - return - self._live.mcps.extend(self.group.mcps) + names = [t.name for t in [*self._live.tools, *self.group.tools]] + if len(names) != len(set(names)): + raise ValueError("Plugin tool name collision") + self._live.tools.extend(self.group.tools) self._live.skills_or_loaders.extend(self.group.skills_or_loaders) self._runtime["connected_keys"] = self.connected close_list = self._runtime.get("close_list") @@ -84,7 +79,11 @@ async def _activation_stage(live_group: Any, runtime: Dict[str, Any], prepared: except BaseException: await stage.discard() raise - stage.commit() + try: + stage.commit() + except BaseException: + await stage.discard() + raise def register_load_plugin( @@ -117,10 +116,10 @@ def register_load_plugin( Deferred clients inherit the current run's read-only setting. """ from agentscope.message import TextBlock - from agentscope.tool._response import ToolChunk as ToolResponse + from agentscope.tool._response import ToolChunk def _text(msg: str) -> Any: - return ToolResponse(content=[TextBlock(type="text", text=msg)]) + return ToolChunk(content=[TextBlock(type="text", text=msg)]) async def load_plugin(plugin: str) -> Any: """加载插件目录中列出的插件，激活其包含的全部工具与技能。 @@ -177,7 +176,7 @@ async def load_plugin(plugin: str) -> Any: prepared = runtime.get("prepared_run") if prepared is not None: - from core.capabilities.runtime import validate, validate_activation + from core.harness.registry.capability_runtime import validate, validate_activation await asyncio.to_thread( validate, prepared, user_id=runtime.get("user_id", prepared.user_id) @@ -200,9 +199,9 @@ async def load_plugin(plugin: str) -> Any: m for m in [*target.mcp_ids, *target.bound_mcp_ids] if m not in stage.connected ] if mcp_ids: - from core.llm.factory.tools.mcp_config import _inject_runtime_headers - from core.llm.mcp_pool import make_client - from core.services.mcp_service import McpServerConfigService + from core.llm.factory.tools.mcp_config import inject_runtime_headers as _inject_runtime_headers + from core.llm.mcp.lifetime import discover + from core.capabilities.mcp.service import McpServerConfigService if prepared is not None: cfgs = runtime["prepared_servers"] @@ -215,8 +214,9 @@ async def load_plugin(plugin: str) -> Any: str(runtime.get("user_id") or ""), enabled_only=False ) ) - except Exception: # noqa: BLE001 - pass + except Exception as fallback_error: # noqa: BLE001 + from core.infra.failures import record_best_effort_failure + record_best_effort_failure('core.llm.tools.plugin_activation:load_plugin', fallback_error) wanted_cfgs = {k: v for k, v in cfgs.items() if k in set(mcp_ids)} wanted_cfgs = _inject_runtime_headers( wanted_cfgs, @@ -228,35 +228,36 @@ async def load_plugin(plugin: str) -> Any: ) async def _connect(key: str, cfg: dict) -> None: - if not stage.staged and not plugin_resolution._http_transport(cfg): - failed_servers.append(key) - return client = None try: - from orchestration.local_plugin_sidecars import ensure_native_servers + from core.harness.registry.local_publication import ensure_native_servers await stage.checkpoint() await ensure_native_servers({key: cfg}) - client = make_client( + from core.llm.mcp.callbacks import Interactions + # Runtimes bound before agent construction have no request yet; + # their sessions stay run-owned and decline server interactions. + request = runtime.get("request") + interactions = (Interactions(request, key, bool(runtime.get("approval_available")), + agent=runtime.get("agent")) + if request is not None else None) + scope = ((request.current_user_id, request.chat_id, request.capability_scope) + if request is not None and request.current_user_id and request.chat_id + and not request.isolated else None) + client = await discover( key, {**cfg, "read_only": bool(runtime.get("read_only"))}, - is_stateful=False, + interactions=interactions, scope=scope, ) - if stage.staged and not plugin_resolution._http_transport(cfg): - await client.connect() - tools = await client.list_tools() + tools = client.tools await stage.checkpoint() except ( BaseException ) as exc: # noqa: BLE001 — SSE cleanup may raise CancelledError if client is not None: - await client.close() + from core.llm.mcp.manager import close_clients + await close_clients([client]) if isinstance(exc, asyncio.CancelledError): - current = asyncio.current_task() - if ( - current is not None - and getattr(current, "cancelling", lambda: 0)() > 0 - ): - raise + raise logger.warning("[plugin-loader] MCP '%s' connect failed: %s", key, exc) failed_servers.append(key) return @@ -270,7 +271,7 @@ async def _connect(key: str, cfg: dict) -> None: failed_servers.append(key) await stage.checkpoint() - if stage.staged and failed_servers: + if failed_servers: raise RuntimeError("插件 MCP 服务不可用：" + "、".join(failed_servers)) # ── Skills: materialize and append loaders in place ── @@ -289,24 +290,24 @@ async def _connect(key: str, cfg: dict) -> None: if not d: continue if prepared is not None: - from core.llm.tool_collector import RuntimeNamedSkillLoader + from core.capabilities.skills.runtime_loader import RuntimeNamedSkillLoader stage.group.skills_or_loaders.append( RuntimeNamedSkillLoader(d, sid, prepared) ) else: - from core.llm.tool_collector import RuntimeNamedSkillLoader + from core.capabilities.skills.runtime_loader import RuntimeNamedSkillLoader stage.group.skills_or_loaders.append(RuntimeNamedSkillLoader(d, sid)) item = meta.get(sid) desc = str(getattr(item, "description", "") or "") - from core.agent_skills.config import model_facing_skill_dir + from core.capabilities.skills.config import model_facing_skill_dir skill_path = model_facing_skill_dir(sid, d) skill_lines.append(f"- `{sid}`：`{skill_path}/SKILL.md` — {desc}") # Ontology gate sees the activated skill's trusted tags too. try: - from core.ontology.validator import register_runtime_asset_tags + from core.ontology.runtime_pack import register_runtime_asset_tags register_runtime_asset_tags( runtime.get("ontology_runtime") or {}, @@ -314,8 +315,9 @@ async def _connect(key: str, cfg: dict) -> None: asset_id=sid, tags=list(getattr(item, "tags", []) or []), ) - except Exception: # noqa: BLE001 - pass + except Exception as fallback_error: # noqa: BLE001 + from core.infra.failures import record_best_effort_failure + record_best_effort_failure('core.llm.tools.plugin_activation:load_plugin', fallback_error) await stage.checkpoint() @@ -368,4 +370,4 @@ async def _connect(key: str, cfg: dict) -> None: parts.append("该插件本轮没有可加载的组件（可能均已被禁用）。") return _text("\n".join(parts)) - toolkit.register_tool_function(load_plugin, namesake_strategy="override") + toolkit.append(function_tool(load_plugin)) diff --git a/src/backend/core/llm/tools/project_instructions_tool.py b/src/backend/core/llm/tools/project_instructions_tool.py index 6aee455d..f5e6d8a0 100644 --- a/src/backend/core/llm/tools/project_instructions_tool.py +++ b/src/backend/core/llm/tools/project_instructions_tool.py @@ -1,5 +1,6 @@ """Canonical instruction reads for project chats; writes only during initialization.""" from __future__ import annotations +from core.llm.factory.tools.definitions import function_tool import asyncio @@ -13,7 +14,7 @@ def project_instruction_path(project_ctx: dict) -> str: if project_ctx.get("project_is_local"): root = project_ctx.get("project_local_path") return f"{root.rstrip('/')}/AGENTS.md" if root else "" - from core.services.edition_workspace import context_directory + from core.files.edition import context_directory directory = context_directory(project_ctx) if directory: @@ -23,19 +24,20 @@ def project_instruction_path(project_ctx: dict) -> str: def register_project_instruction_tools(toolkit, *, project_id: str, user_id: str, local_path=None, instruction_path=None, allow_write=True): - from core.llm.tool_permissions import READ, WRITE, local_path_tool - from core.llm.tools._tool_confirmation import OP_EDIT + from core.sandbox.permissions.dispatch_models import READ, WRITE + from core.sandbox.permissions.tool_specs import local_path_tool + from core.llm.tools.confirmation import OP_EDIT target = instruction_path or (f"{local_path.rstrip('/')}/AGENTS.md" if local_path else "/myspace/AGENTS.md") def operate(content=None, revision=None): - from core.auth.permissions_iface import resolve_project_permission + from core.auth.edition import resolve_project_permission from core.db.engine import SessionLocal from core.db.models import Project - from core.services.project_instructions import ProjectInstructionsService + from core.projects.instructions import ProjectInstructionsService if content is None: - from core.services.project_instructions import read_authorized_project_instructions + from core.projects.instructions import read_authorized_project_instructions return {"ok": True, "path": target, **read_authorized_project_instructions(project_id, user_id)} with SessionLocal() as db: @@ -59,7 +61,7 @@ async def read_project_instructions(): """ try: if local_path: - from core.llm.tool_permissions import require_local_path_permission + from core.sandbox.permissions.dispatch_models import require_local_path_permission require_local_path_permission(target, "read") return resp_json(await asyncio.to_thread(operate)) except HTTPException as exc: @@ -75,22 +77,16 @@ async def save_project_instructions(content: str, expected_revision: str): """ try: if local_path: - from core.llm.tool_permissions import require_local_path_permission + from core.sandbox.permissions.dispatch_models import require_local_path_permission require_local_path_permission(target, "write") return resp_json(await asyncio.to_thread(operate, content, expected_revision)) except HTTPException as exc: return resp_json({"ok": False, "error": exc.detail, "status": exc.status_code}) - toolkit.register_tool_function( - read_project_instructions, - permission=local_path_tool("file_path", READ, tool_name="read_project_instructions", default_path=target), - ) + toolkit.append(function_tool(read_project_instructions, dispatch_spec=local_path_tool("file_path", READ, tool_name="read_project_instructions", default_path=target))) if not allow_write: return - toolkit.register_tool_function( - save_project_instructions, - permission=local_path_tool( + toolkit.append(function_tool(save_project_instructions, dispatch_spec=local_path_tool( "file_path", WRITE, tool_name="save_project_instructions", default_path=target, myspace_op=OP_EDIT, - ), - ) + ))) diff --git a/src/backend/core/llm/tools/project_source_access.py b/src/backend/core/llm/tools/project_source_access.py index 9dcce65c..bd74d42d 100644 --- a/src/backend/core/llm/tools/project_source_access.py +++ b/src/backend/core/llm/tools/project_source_access.py @@ -6,10 +6,10 @@ def validate_scope(db, scope, actor, *, write=False): - from core.services.project_source import ProjectSourceService + from core.projects.source import ProjectSourceService project = ProjectSourceService(db).authorized_project(scope.project_id, actor, write=write) - from core.services.project_file_service import ProjectFileService + from core.projects.files import ProjectFileService if not ProjectFileService(db).matches_scope(project, scope): raise HTTPException(409, "项目空间已变更，请重新打开项目会话后再操作") @@ -23,7 +23,7 @@ def current_scope_error(scope, actor, *, write=False): from core.config.local_mode import local_mode_enabled if not local_mode_enabled(): return {"error": "本地项目只能在本机模式下操作", "status": 403} - from core.services.local_project_workspace import validate_current_project + from core.projects.local_workspace import validate_current_project try: validate_current_project(scope, actor, write=write) except (HTTPException, ValueError, OSError) as exc: diff --git a/src/backend/core/llm/tools/read_artifact_tool.py b/src/backend/core/llm/tools/read_artifact_tool.py index 02368c56..9ea79e9d 100644 --- a/src/backend/core/llm/tools/read_artifact_tool.py +++ b/src/backend/core/llm/tools/read_artifact_tool.py @@ -4,6 +4,7 @@ deps (content parsers, hooks) are imported lazily inside the functions. ``core.llm.tool`` re-exports the public names for backward compatibility. """ +from core.llm.factory.tools.definitions import function_tool import asyncio import logging @@ -11,7 +12,7 @@ from agentscope.message import TextBlock from agentscope.tool import Toolkit -from agentscope.tool._response import ToolChunk as ToolResponse +from agentscope.tool._response import ToolChunk logger = logging.getLogger(__name__) @@ -35,8 +36,8 @@ def _is_pptx_meta(meta: dict) -> bool: def _probe_xlsx_sheet_names(file_id: str, filename: str, user_id: Optional[str] = None) -> tuple[list[str], bytes | None]: """Cheaply list sheet names and return the bytes for reuse. Returns ``([], None)`` on download failure, ``(names, bytes)`` on success.""" - from core.content.file_parser import parse_xlsx_sheet_names - from core.llm.hooks import _download_artifact_bytes + from core.files.content.spreadsheet_parser import parse_xlsx_sheet_names + from core.files.artifact_bytes import download_artifact_bytes as _download_artifact_bytes file_bytes = _download_artifact_bytes(file_id, filename, "read_artifact xlsx", user_id=user_id) if file_bytes is None: @@ -58,8 +59,8 @@ async def _resolve_xlsx_text( the probe — passed through to fetch_parsed_text on the default path so we don't re-download from storage. Raises RuntimeError on bad sheet_name or missing bytes when sheet_name is set.""" - from core.content.artifact_reader import fetch_parsed_text - from core.content.file_parser import parse_xlsx_single_sheet + from core.files.artifacts.reader import fetch_parsed_text + from core.files.content.spreadsheet_parser import parse_xlsx_single_sheet if sheet_name: if file_bytes is None: @@ -73,8 +74,8 @@ async def _resolve_xlsx_text( def _probe_pptx_slide_count(file_id: str, filename: str, user_id: Optional[str] = None) -> tuple[int, bytes | None]: """Cheaply count slides and return the bytes for reuse. Returns ``(0, None)`` on download failure, ``(count, bytes)`` on success.""" - from core.content.file_parser import parse_pptx_slide_count - from core.llm.hooks import _download_artifact_bytes + from core.files.content.presentation_parser import parse_pptx_slide_count + from core.files.artifact_bytes import download_artifact_bytes as _download_artifact_bytes file_bytes = _download_artifact_bytes(file_id, filename, "read_artifact pptx", user_id=user_id) if file_bytes is None: @@ -96,8 +97,8 @@ async def _resolve_pptx_text( bytes the probe already downloaded — passed through to fetch_parsed_text on the default path so we don't re-download. Raises RuntimeError on bad slide_index or missing bytes.""" - from core.content.artifact_reader import fetch_parsed_text - from core.content.file_parser import parse_pptx_slide + from core.files.artifacts.reader import fetch_parsed_text + from core.files.content.presentation_parser import parse_pptx_slide if slide_index is not None: if file_bytes is None: @@ -124,7 +125,7 @@ async def read_artifact( limit: int = _READ_ARTIFACT_DEFAULT_LIMIT, sheet_name: str | None = None, slide_index: int | None = None, - ) -> ToolResponse: + ) -> ToolChunk: """读取已上传文件的完整解析文本（按字符分页）。 ⚠️ 每轮对话每个 file_id **累计最多读 50000 字符**，超出即被拒绝。所以要精准 @@ -158,18 +159,19 @@ async def read_artifact( """ import json as _json - from core.content.artifact_reader import fetch_parsed_text, load_artifact_meta - from core.llm.hooks import MAX_FILE_CONTENT_CHARS, get_artifact_read_state + from core.files.artifacts.reader import fetch_parsed_text, load_artifact_meta + from core.llm.runtime.hooks_constants import MAX_FILE_CONTENT_CHARS + from core.llm.runtime.turn_state import get_artifact_read_state fid = (file_id or "").strip() if not fid: - return ToolResponse(content=[TextBlock( + return ToolChunk(content=[TextBlock( type="text", text=_json.dumps({"error": "file_id 不能为空"}, ensure_ascii=False), )]) if sheet_name is not None and slide_index is not None: - return ToolResponse(content=[TextBlock( + return ToolChunk(content=[TextBlock( type="text", text=_json.dumps( {"error": "sheet_name 与 slide_index 互斥（分别用于 xlsx / pptx）"}, @@ -184,7 +186,7 @@ async def read_artifact( # Blocking it mid-answer stalls streaming for every chat on the instance. meta = await asyncio.to_thread(load_artifact_meta, fid, user_id=user_id) if meta is None: - return ToolResponse(content=[TextBlock( + return ToolChunk(content=[TextBlock( type="text", text=_json.dumps( {"error": f"文件 {fid} 不存在、已删除，或无权访问"}, @@ -226,7 +228,7 @@ async def read_artifact( except RuntimeError as e: err = {"error": str(e), "file_id": fid, "filename": meta.get("filename")} err.update(extras) - return ToolResponse(content=[TextBlock( + return ToolChunk(content=[TextBlock( type="text", text=_json.dumps(err, ensure_ascii=False), )]) @@ -234,7 +236,7 @@ async def read_artifact( error = meta.get("parse_error") or "文件暂无可读文本内容" err = {"error": error, "filename": meta.get("filename"), "file_id": fid} err.update(extras) - return ToolResponse(content=[TextBlock( + return ToolChunk(content=[TextBlock( type="text", text=_json.dumps(err, ensure_ascii=False), )]) @@ -271,7 +273,7 @@ async def read_artifact( ) else: hint += " 如需更多内容，请缩小范围（精准 offset/limit）后再试。" - return ToolResponse(content=[TextBlock( + return ToolChunk(content=[TextBlock( type="text", text=_json.dumps( { @@ -316,10 +318,10 @@ async def read_artifact( "budget_remaining": budget_remaining_after, } result.update(extras) - return ToolResponse(content=[TextBlock( + return ToolChunk(content=[TextBlock( type="text", text=_json.dumps(result, ensure_ascii=False), )]) - toolkit.register_tool_function(read_artifact, namesake_strategy="override") + toolkit.append(function_tool(read_artifact)) logger.info("[factory] Registered read_artifact tool (user=%s)", user_id) diff --git a/src/backend/core/llm/tools/read_image_tool.py b/src/backend/core/llm/tools/read_image_tool.py index 89432035..2ff2d21b 100644 --- a/src/backend/core/llm/tools/read_image_tool.py +++ b/src/backend/core/llm/tools/read_image_tool.py @@ -1,6 +1,6 @@ """``read_image`` —— 让智能体主动看一张图。 -同一个工具名，按主模型能力注册两个版本之一（``core.vision.resolve_vision_mode``）： +同一个工具名，按主模型能力注册两个版本之一（``core.models.vision.resolve_vision_mode``）： - **native**：把图原样送进上下文，模型自己看像素（对应 Codex 的 ``view_image``）。 - **bridge**：交给视觉桥转成结构化文字证据，可带 ``focus`` 针对某处细节追问。 @@ -11,17 +11,18 @@ """ from __future__ import annotations +from core.llm.factory.tools.definitions import function_tool import asyncio import logging from typing import Optional from agentscope.tool import Toolkit -from core.services.project_scope import ProjectScope -from core.vision import VisionMode +from core.projects.edition import ProjectScope +from core.models.vision import VisionMode -from ._common import ToolResponse, resolve_sandbox_session, resp_image, resp_json -from ._paths import to_physical_path, validate_project_scope_path, validate_workspace_path +from ._common import ToolChunk, resolve_sandbox_session, resp_image, resp_json +from core.sandbox.workspace.resolution import to_physical_path, validate_project_scope_path, validate_workspace_path logger = logging.getLogger(__name__) @@ -75,7 +76,7 @@ async def load_image( label = file_path or file_id if file_id.strip(): - from core.llm.hooks import _download_artifact_bytes + from core.files.artifact_bytes import download_artifact_bytes as _download_artifact_bytes # Storage download (S3/OSS in some deployments) — off the event loop. image_bytes = await asyncio.to_thread( @@ -97,7 +98,7 @@ async def load_image( if image_bytes is None: raise ImageLoadError(f"读取失败：{file_path}") - from core.vision import sniff_mime + from core.models.vision import sniff_mime mime = sniff_mime(image_bytes) if mime is None: @@ -107,7 +108,7 @@ async def load_image( async def transcribe_image(image_bytes: bytes, *, label: str, focus: str = "") -> Optional[dict]: """视觉桥：图片 → ``image_evidence`` 载荷。未配视觉模型或识别失败返回 ``None``。""" - from core.vision import get_vision_bridge, is_available, render_evidence + from core.models.vision import get_vision_bridge, is_available, render_evidence if not is_available(): return None @@ -140,7 +141,7 @@ def register_read_image( return _sess = resolve_sandbox_session(sandbox_session_id, chat_id) - async def _run(file_path: str, file_id: str, focus: str) -> ToolResponse: + async def _run(file_path: str, file_id: str, focus: str) -> ToolChunk: try: image_bytes, mime, label = await load_image( file_path, @@ -154,7 +155,7 @@ async def _run(file_path: str, file_id: str, focus: str) -> ToolResponse: return resp_json({"error": str(exc)}) if vision_mode == "native": - from core.vision.service import MAX_IMAGE_BYTES + from core.models.vision.service import MAX_IMAGE_BYTES if len(image_bytes) > MAX_IMAGE_BYTES: return resp_json( @@ -165,7 +166,7 @@ async def _run(file_path: str, file_id: str, focus: str) -> ToolResponse: ) return await resp_image(image_bytes, mime, name=label, meta={"source": label}) - from core.vision import is_available + from core.models.vision import is_available if not is_available(): return resp_json( @@ -182,7 +183,7 @@ async def _run(file_path: str, file_id: str, focus: str) -> ToolResponse: # and only the bridge variant can act on ``focus``. if vision_mode == "native": - async def read_image(file_path: str = "", file_id: str = "") -> ToolResponse: + async def read_image(file_path: str = "", file_id: str = "") -> ToolChunk: return await _run(file_path, file_id, "") read_image.__doc__ = _NATIVE_DOC @@ -190,12 +191,12 @@ async def read_image(file_path: str = "", file_id: str = "") -> ToolResponse: async def read_image( file_path: str = "", file_id: str = "", focus: str = "" - ) -> ToolResponse: + ) -> ToolChunk: return await _run(file_path, file_id, focus) read_image.__doc__ = _BRIDGE_DOC - toolkit.register_tool_function(read_image) + toolkit.append(function_tool(read_image)) logger.info("[factory] Registered read_image tool (%s, chat_id=%s)", vision_mode, chat_id) @@ -212,10 +213,7 @@ async def _read_path_bytes( if local_mode_enabled(): try: - from core.llm.tool_permissions import ( - PermissionEnforcementError, - require_local_path_permission, - ) + from core.sandbox.permissions.dispatch_models import PermissionEnforcementError, require_local_path_permission require_local_path_permission(physical, "read") with open(physical, "rb") as handle: diff --git a/src/backend/core/llm/tools/read_tool.py b/src/backend/core/llm/tools/read_tool.py index 51b0439a..8c852030 100644 --- a/src/backend/core/llm/tools/read_tool.py +++ b/src/backend/core/llm/tools/read_tool.py @@ -1,6 +1,7 @@ """Read files from the active sandbox.""" from __future__ import annotations +from core.llm.factory.tools.definitions import function_tool import asyncio import hashlib @@ -8,16 +9,11 @@ from typing import Optional from agentscope.tool import Toolkit, ToolResponse -from core.services.project_scope import ProjectScope -from core.vision import VisionMode +from core.projects.edition import ProjectScope +from core.models.vision import VisionMode from ._common import response_metadata, resolve_sandbox_session, resp_image, resp_json -from ._paths import ( - space_metadata, - to_physical_path, - validate_project_scope_path, - validate_workspace_path, -) +from core.sandbox.workspace.resolution import space_metadata, to_physical_path, validate_project_scope_path, validate_workspace_path from ._state import ReadEntry, ReadStateTracker logger = logging.getLogger(__name__) @@ -42,7 +38,7 @@ async def _read_image_as_evidence(content_bytes: bytes, file_path: str) -> Optio configured, so the caller falls through to its existing binary handling. """ try: - from core.vision import sniff_mime + from core.models.vision import sniff_mime from .read_image_tool import transcribe_image @@ -64,8 +60,8 @@ async def _image_response( text-only one. ``None`` means "not handled here" and falls through to the parsed-text / binary handling.""" if vision_mode == "native": - from core.vision import sniff_mime - from core.vision.service import MAX_IMAGE_BYTES + from core.models.vision import sniff_mime + from core.models.vision.service import MAX_IMAGE_BYTES mime = sniff_mime(content_bytes) if mime is None or len(content_bytes) > MAX_IMAGE_BYTES: @@ -115,7 +111,7 @@ def register_read( ``state`` is the per-chat ReadStateTracker shared with Edit/Write. ``sandbox_session_id`` selects the container; it defaults to ``chat_id``. - ``vision_mode`` (see ``core.vision.resolve_vision_mode``) decides what an image + ``vision_mode`` (see ``core.models.vision.resolve_vision_mode``) decides what an image file reads back as: pixels, a vision-bridge transcription, or ``type=binary``. """ @@ -154,10 +150,7 @@ async def Read( _local_read: Optional[bytes] = None if _local_on(): try: - from core.llm.tool_permissions import ( - PermissionEnforcementError, - require_local_path_permission, - ) + from core.sandbox.permissions.dispatch_models import PermissionEnforcementError, require_local_path_permission require_local_path_permission(physical, "read") with open(physical, "rb") as _f: @@ -291,5 +284,5 @@ async def Read( " ...}`` / ``{type: 'too_large', ...}`` / ``{error: '...'}``。\n" ) - toolkit.register_tool_function(Read, namesake_strategy="override") + toolkit.append(function_tool(Read)) logger.info("[factory] Registered Read tool (chat_id=%s)", chat_id) diff --git a/src/backend/core/llm/tools/sandbox_artifact_tools.py b/src/backend/core/llm/tools/sandbox_artifact_tools.py index a8a8cdd1..8010eac2 100644 --- a/src/backend/core/llm/tools/sandbox_artifact_tools.py +++ b/src/backend/core/llm/tools/sandbox_artifact_tools.py @@ -10,6 +10,7 @@ """ from __future__ import annotations +from core.llm.factory.tools.definitions import function_tool import asyncio import base64 @@ -17,19 +18,18 @@ import os import tempfile from pathlib import Path -from typing import Any, Optional +from typing import Optional, TYPE_CHECKING + +if TYPE_CHECKING: + from core.projects.edition import ProjectScope from agentscope.tool import Toolkit -# AgentScope 2.0: tool functions must return ToolChunk (call_tool rejects ToolResponse). -from agentscope.tool._response import ToolChunk as ToolResponse +# AgentScope 2.0: tool functions must return ToolChunk (call_tool rejects ToolChunk). +from agentscope.tool._response import ToolChunk from core.llm.tools._common import resolve_sandbox_session -from core.llm.tools._tool_helpers import ( - _resolve_artifact_files, - _resp_json, - _store_generated_file_path, - _validate_workspace_path, -) +from core.files.artifacts.generated import resolve_artifact_files as _resolve_artifact_files, store_generated_file_path as _store_generated_file_path +from core.llm.tools._tool_helpers import resp_json as _resp_json, validate_workspace_path as _validate_workspace_path logger = logging.getLogger(__name__) @@ -46,7 +46,7 @@ def register_sandbox_put_artifact( _sess = resolve_sandbox_session(sandbox_session_id, chat_id) - async def sandbox_put_artifact(artifact_id: str, dest_path: str) -> ToolResponse: + async def sandbox_put_artifact(artifact_id: str, dest_path: str) -> ToolChunk: from core.sandbox import SandboxConnectError as _SandboxConnectError from core.sandbox import SandboxError as _SandboxError from core.sandbox import get_sandbox_provider as _get_provider @@ -54,7 +54,7 @@ async def sandbox_put_artifact(artifact_id: str, dest_path: str) -> ToolResponse if not artifact_id or not isinstance(artifact_id, str): return _resp_json({"error": "artifact_id 必须为非空字符串"}) - from ._paths import to_physical_path, workspace_directory + from core.sandbox.workspace.resolution import to_physical_path, workspace_directory dest_path = to_physical_path(dest_path, user_id, session_id=_sess) path_err = _validate_workspace_path(dest_path, root=workspace_directory(_sess)) @@ -102,7 +102,7 @@ async def sandbox_put_artifact(artifact_id: str, dest_path: str) -> ToolResponse "限制：单个 artifact 最大 10 MB。\n" ) - toolkit.register_tool_function(sandbox_put_artifact, namesake_strategy="override") + toolkit.append(function_tool(sandbox_put_artifact)) logger.info("[factory] Registered sandbox_put_artifact tool (chat_id=%s)", chat_id) @@ -123,7 +123,7 @@ def register_sandbox_get_artifact( max_bytes = _settings.sandbox.artifact_max_bytes - async def sandbox_get_artifact(src_path: str, name: str = "") -> ToolResponse: + async def sandbox_get_artifact(src_path: str, name: str = "") -> ToolChunk: import mimetypes as _mt from core.sandbox import SandboxConnectError as _SandboxConnectError @@ -131,10 +131,10 @@ async def sandbox_get_artifact(src_path: str, name: str = "") -> ToolResponse: from core.sandbox import get_sandbox_provider as _get_provider from core.config.local_mode import local_mode_enabled - from ._paths import to_physical_path + from core.sandbox.workspace.resolution import to_physical_path if local_mode_enabled(): - from core.artifacts.local_reference import reference_local_file + from core.files.artifacts.local_reference import reference_local_file from fastapi import HTTPException physical = to_physical_path(src_path, user_id, session_id=_sess, scope=scope) @@ -149,7 +149,7 @@ async def sandbox_get_artifact(src_path: str, name: str = "") -> ToolResponse: ref["url"] = f"/files/{ref['file_id']}" return _resp_json({"ok": True, **ref, "artifacts": [ref]}) - from ._paths import to_physical_path, workspace_directory + from core.sandbox.workspace.resolution import to_physical_path, workspace_directory src_path = to_physical_path(src_path, user_id, session_id=_sess) path_err = _validate_workspace_path(src_path, root=workspace_directory(_sess)) @@ -243,5 +243,5 @@ async def sandbox_get_artifact(src_path: str, name: str = "") -> ToolResponse: "PDF 应按页拆分，其他格式应拆包或降低体积后再逐个登记。\n" ) - toolkit.register_tool_function(sandbox_get_artifact, namesake_strategy="override") + toolkit.append(function_tool(sandbox_get_artifact)) logger.info("[factory] Registered sandbox_get_artifact tool (chat_id=%s)", chat_id) diff --git a/src/backend/core/llm/tools/sandbox_tool.py b/src/backend/core/llm/tools/sandbox_tool.py index 6e55e942..97f6a403 100644 --- a/src/backend/core/llm/tools/sandbox_tool.py +++ b/src/backend/core/llm/tools/sandbox_tool.py @@ -10,6 +10,7 @@ """ from __future__ import annotations +from core.llm.factory.tools.definitions import function_tool import asyncio import logging @@ -18,15 +19,16 @@ from agentscope.tool import Toolkit -# AgentScope 2.0: tool functions must return ToolChunk (call_tool rejects ToolResponse). -from agentscope.tool._response import ToolChunk as ToolResponse +# AgentScope 2.0: tool functions must return ToolChunk (call_tool rejects ToolChunk). +from agentscope.tool._response import ToolChunk from core.llm.tools._common import resolve_sandbox_session -from core.llm.tools._tool_helpers import _resp_json +from core.llm.tools._tool_helpers import resp_json as _resp_json -logger = logging.getLogger(__name__) import re as _re +logger = logging.getLogger(__name__) + # dws exit code 4 = PAT authorization interception; stderr/stdout carries a line # ``PAT_AUTHORIZATION_URL=`` (a copy-safe link dws prints separately for # OpenClaw-style hosts, see dws CHANGELOG #242). @@ -55,7 +57,6 @@ def register_bash( toolkit: Toolkit, *, loader: Any, - loaded_skill_ids: set[str], chat_id: Optional[str] = None, sandbox_session_id: Optional[str] = None, user_id: Optional[str] = None, @@ -69,8 +70,8 @@ def register_bash( ``opensandbox_provider._make_skills_volumes`` + ``config.get_sandbox_skills_dir``): what is bound there is the **caller's own skill view**, holding the shared skills plus that user's private ones, so no one sees another user's skill - files. This registration just sets up the Bash tool itself; ``loader`` / - ``loaded_skill_ids`` are kept for backward compat with existing callers. + files. This registration just sets up the Bash tool itself; ``loader`` only + supplies the capability run that scopes it. The sandbox session is bound to ``chat_id`` so OpenSandbox keeps a single persistent container per conversation (variables, pip packages, /workspace @@ -82,13 +83,13 @@ def register_bash( # Effective sandbox session (``None`` → legacy fall back to chat_id). _sess = resolve_sandbox_session(sandbox_session_id, chat_id) - from core.llm.evaluation_runtime import is_evaluation_session + from core.harness.registry.evaluation import is_evaluation_session _evaluation = is_evaluation_session(_sess) async def Bash( command: str, timeout: int | None = None, yield_time_ms: int = 60000, run_in_background: bool = False, - ) -> ToolResponse: + ) -> ToolChunk: from core.sandbox import ProcessRequest as _ProcessRequest from core.sandbox import SandboxConnectError as _SandboxConnectError from core.sandbox import SandboxError as _SandboxError @@ -111,7 +112,7 @@ async def Bash( authorization = None if local_mode_enabled(): - from core.llm.tool_permissions import current_local_command_authorization + from core.sandbox.permissions.dispatch_models import current_local_command_authorization authorization = current_local_command_authorization(cmd) if authorization is None: @@ -132,7 +133,7 @@ async def Bash( # confinement nobody enforces — so the question is put to the provider # rather than assumed from the deployment profile. if authorization is not None and provider.runs_on_host: - from core.llm.tool_permissions import LocalConfinementUnavailableError + from core.sandbox.permissions.dispatch_models import LocalConfinementUnavailableError try: sandbox_launch = authorization.confine() @@ -151,7 +152,7 @@ async def Bash( sandbox_launch.backend if sandbox_launch else "未启用（用户选择的权限档）", ) - from core.services.edition_workspace import project_directory + from core.files.edition import project_directory root = project_directory(scope) if root: import shlex @@ -159,7 +160,7 @@ async def Bash( if timeout is not None and timeout <= 0: return _resp_json({"error": "timeout 必须为正数；省略表示不设置命令执行期限。"}) - from ._paths import workspace_directory + from core.sandbox.workspace.resolution import workspace_directory effective_timeout = timeout req = _ProcessRequest( @@ -179,8 +180,8 @@ async def Bash( ) from fastapi import HTTPException try: - from orchestration.commands.runtime import runtime - payload = await runtime.start(provider, req, chat_id=chat_id, + from core.llm.ports import command_execution + payload = await command_execution().start(provider, req, chat_id=chat_id, source_run_id=source_run_id, background=run_in_background, yield_time_ms=yield_time_ms) except HTTPException as exc: @@ -194,33 +195,33 @@ async def Bash( return _resp_json(payload) return await finish(payload) - async def write_stdin(session_id: str, chars: str = "", yield_time_ms: int = 0) -> ToolResponse: - from core.services.commands import store - from orchestration.commands.runtime import runtime - tid = await asyncio.to_thread(store.resolve_id, session_id, user_id, _sess) + async def write_stdin(session_id: str, chars: str = "", yield_time_ms: int = 0) -> ToolChunk: + from core.llm.ports import command_execution + from core.llm.ports import command_records + tid = await asyncio.to_thread(command_records().resolve_id, session_id, user_id, _sess) if chars not in ("", "\x03"): return _resp_json({"error": "stdin is closed; use empty input or Ctrl+C"}) if chars: - await runtime.cancel(tid, user_id, _sess) + await command_execution().cancel(tid, user_id, _sess) else: - await runtime.wait(tid, user_id, _sess, max(0, min(yield_time_ms, 3000))) - payload = await asyncio.to_thread(store.read, tid, user_id, _sess) + await command_execution().wait(tid, user_id, _sess, max(0, min(yield_time_ms, 3000))) + payload = await asyncio.to_thread(command_records().read, tid, user_id, _sess) return await finish(payload) - async def background_process(session_id: str) -> ToolResponse: + async def background_process(session_id: str) -> ToolChunk: """Adopt an existing Bash process without executing the command again.""" from core.sandbox import get_sandbox_provider - from orchestration.commands.runtime import runtime - payload = await runtime.adopt(get_sandbox_provider(), session_id, + from core.llm.ports import command_execution + payload = await command_execution().adopt(get_sandbox_provider(), session_id, user_id=user_id, session_id=_sess, chat_id=chat_id, source_run_id=source_run_id) return _resp_json(payload) - async def list_processes() -> ToolResponse: + async def list_processes() -> ToolChunk: """List this user's command tasks in the current chat.""" - from core.services.commands import store - return _resp_json({"tasks": await asyncio.to_thread(store.list_owned, user_id, chat_id)}) + from core.llm.ports import command_records + return _resp_json({"tasks": await asyncio.to_thread(command_records().list_owned, user_id, chat_id)}) - async def finish(payload: dict) -> ToolResponse: + async def finish(payload: dict) -> ToolChunk: # dws (DingTalk CLI) PAT per-scope authorization interception: exit code 4 # + a PAT_AUTHORIZATION_URL= line on stderr. Surface the link to the # model in structured form so it hands it verbatim to the user, who @@ -237,10 +238,10 @@ async def finish(payload: dict) -> ToolResponse: return _resp_json(payload) - from ._paths import WORKSPACE_ROOT, path_rules - import core.sandbox.workspace.desktop as desktop_paths + from core.sandbox.workspace.resolution import WORKSPACE_ROOT, path_rules + rules = path_rules() instructions = ( rules.bash_workspace_instructions(WORKSPACE_ROOT, _sess, scope=scope) @@ -260,7 +261,7 @@ async def finish(payload: dict) -> ToolResponse: " 或失败时 {error, exit_code: -1}。\n" ) - from core.services.edition_workspace import command_instructions + from core.files.edition import command_instructions Bash.__doc__ += command_instructions(scope) write_stdin.__doc__ = ( @@ -273,13 +274,13 @@ async def finish(payload: dict) -> ToolResponse: " JSON: {stdout, stderr, exit_code, execution_time_ms, status, session_id}；输出仅含新增内容。\n" " running 表示继续运行；exited 时检查 exit_code 和 error。\n" ) - toolkit.register_tool_function(background_process, namesake_strategy="override") - toolkit.register_tool_function(list_processes, namesake_strategy="override") - toolkit.register_tool_function(write_stdin, namesake_strategy="override") - toolkit.register_tool_function(Bash, namesake_strategy="override") + toolkit.append(function_tool(background_process)) + toolkit.append(function_tool(list_processes)) + toolkit.append(function_tool(write_stdin)) + toolkit.append(function_tool(Bash)) logger.info("[factory] Registered Bash tool (chat_id=%s)", chat_id) -from .sandbox_artifact_tools import register_sandbox_put_artifact, register_sandbox_get_artifact +from .sandbox_artifact_tools import register_sandbox_get_artifact, register_sandbox_put_artifact diff --git a/src/backend/core/llm/tools/skill_tool.py b/src/backend/core/llm/tools/skill_tool.py index ad97727f..0f737102 100644 --- a/src/backend/core/llm/tools/skill_tool.py +++ b/src/backend/core/llm/tools/skill_tool.py @@ -10,6 +10,8 @@ """ from __future__ import annotations +from core.llm.factory.tools.definitions import function_tool +from core.harness import log_writer import json import logging @@ -20,16 +22,16 @@ from agentscope.message import TextBlock from agentscope.tool import Toolkit -# AgentScope 2.0: tool functions must return ToolChunk (call_tool rejects ToolResponse). -from agentscope.tool._response import ToolChunk as ToolResponse +# AgentScope 2.0: tool functions must return ToolChunk (call_tool rejects ToolChunk). +from agentscope.tool._response import ToolChunk -from core.agent_skills.loader import get_skill_loader +from core.capabilities.skills.loader_factory import get_skill_loader logger = logging.getLogger(__name__) def _resolve_skill_path(file_path: str, loader: Any = None) -> str | None: - from ._paths import path_rules + from core.sandbox.workspace.resolution import path_rules return path_rules().resolve_skill_file(file_path, loader) @@ -59,11 +61,11 @@ def _build_skill_bash_hint( return None name = getattr(spec, "name", skill_id) or skill_id - from core.agent_skills.config import model_facing_skill_dir + from core.capabilities.skills.config import model_facing_skill_dir sandbox_dir = model_facing_skill_dir(skill_id, skill_dir) - from ._paths import path_rules + from core.sandbox.workspace.resolution import path_rules quote = path_rules().quote_shell_path scripts = list(getattr(spec, "executable_scripts", None) or []) @@ -113,13 +115,13 @@ def register_sandboxed_view_text_file( async def view_text_file( file_path: str, ranges: list[int] | None = None, - ) -> ToolResponse: + ) -> ToolChunk: """View file content within allowed skill directories.""" prepared = getattr(loader, "capability_run", None) if prepared is not None: - from core.capabilities.runtime import validate + from core.harness.registry.capability_runtime import validate import asyncio - from core.capabilities.errors import CapabilityError + from core.harness.capability_errors import CapabilityError try: # Check only the file's skill. An unrelated skill cannot block this read. @@ -140,7 +142,7 @@ async def view_text_file( if requested_name is not None: await asyncio.to_thread(validate, prepared, only_skill=requested_name) except (CapabilityError, OSError): - return ToolResponse( + return ToolChunk( content=[ TextBlock( type="text", @@ -160,7 +162,7 @@ async def view_text_file( real = _os.path.realpath(resolved) if not any(real.startswith(d + _os.sep) or real == d for d in resolved_dirs): - return ToolResponse(content=[TextBlock( + return ToolChunk(content=[TextBlock( type="text", text=f"Error: Access denied. Only files inside skill directories can be read.\nRequested: {file_path}", )]) @@ -177,22 +179,13 @@ async def view_text_file( else: _selected = _lines _text = "".join(_selected) - from core.config.local_mode import local_mode_enabled - if local_mode_enabled() and "/home/ubuntu/yida-workspace" in _text and user_id: - from core.sandbox.workspace.identity import safe_user_id - from core.services.yida_execution import workspace_dir - import shlex - - if safe_user_id(user_id): - _text = _text.replace( - "/home/ubuntu/yida-workspace", - shlex.quote(workspace_dir(user_id).as_posix()), - ) + from core.harness.registry.skill_content import adapt_skill_content + _text = adapt_skill_content(_text, user_id) except Exception as _e: # noqa: BLE001 - return ToolResponse(content=[TextBlock( + return ToolChunk(content=[TextBlock( type="text", text=f"Error reading file: {_e}", )]) - resp = ToolResponse(content=[TextBlock(type="text", text=_text)]) + resp = ToolChunk(content=[TextBlock(type="text", text=_text)]) if _os.path.basename(real) == "SKILL.md": from core.config.local_mode import local_mode_enabled @@ -214,7 +207,7 @@ async def view_text_file( for i, block in enumerate(resp.content): text = block.get("text", "") if isinstance(block, dict) else getattr(block, "text", "") if "{baseDir}" in text: - from core.agent_skills.config import model_facing_skill_dir + from core.capabilities.skills.config import model_facing_skill_dir directory = model_facing_skill_dir(skill_id or "", skill_dir).replace(chr(92), "/") resp.content[i] = TextBlock(type="text", text=text.replace("{baseDir}", directory)) if skill_id: @@ -224,13 +217,13 @@ async def view_text_file( # ── Observability: record skill auto-load via view_text_file ── try: - from core.services import log_service as _lw + from core.harness import log_writer as _lw spec = loader.load_skill_full(skill_id) source = loader.get_skill_source(skill_id) if prepared is not None and skill_id in prepared.bindings: - from core.capabilities.ref import LOCAL_PROFILE, BUILTIN_PROFILE + from core.capabilities.identity_rules import LOCAL_PROFILE, BUILTIN_PROFILE source = "local" if prepared.bindings[skill_id]["profile"] in (LOCAL_PROFILE, BUILTIN_PROFILE) else "cloud" - _lw.schedule_skill_call_write({ + log_writer.schedule_skill_call_write({ "skill_id": skill_id, "skill_name": getattr(spec, "name", skill_id) if spec else skill_id, "skill_version": getattr(spec, "version", None) if spec else None, @@ -244,4 +237,4 @@ async def view_text_file( return resp - toolkit.register_tool_function(view_text_file, namesake_strategy="override") + toolkit.append(function_tool(view_text_file)) diff --git a/src/backend/core/llm/tools/user_questions.py b/src/backend/core/llm/tools/user_questions.py index e815689a..e7d7c714 100644 --- a/src/backend/core/llm/tools/user_questions.py +++ b/src/backend/core/llm/tools/user_questions.py @@ -21,8 +21,8 @@ from dataclasses import dataclass, field from typing import Any, Dict, Iterable, List, Optional -from core.llm.human_interaction import MAX_WAIT_SECONDS -from core.llm import interaction_store +from core.llm.runtime.human_interaction import MAX_WAIT_SECONDS +from core.llm.runtime import interaction_store KIND_USER_QUESTION = "user_question" STATUS_BLOCKED = "blocked_non_interactive" @@ -64,10 +64,6 @@ class _ChatQuestions: last_ts: float = field(default_factory=time.monotonic) -_LOCK = threading.RLock() -_CHATS: Dict[str, _ChatQuestions] = {} - - def _text( value: Any, *, @@ -235,20 +231,20 @@ def _normalize_answers( def _gc_locked(now: float) -> None: dead = [ chat_id - for chat_id, state in _CHATS.items() + for chat_id, state in process_state().CHATS.items() if not state.pending and now - state.last_ts > _STATE_IDLE_TTL_SECONDS ] for chat_id in dead: - _CHATS.pop(chat_id, None) + process_state().CHATS.pop(chat_id, None) def _state_locked(chat_id: str) -> _ChatQuestions: now = time.monotonic() _gc_locked(now) - state = _CHATS.get(chat_id) + state = process_state().CHATS.get(chat_id) if state is None: state = _ChatQuestions() - _CHATS[chat_id] = state + process_state().CHATS[chat_id] = state state.last_ts = now return state @@ -274,32 +270,32 @@ def _resolved_signal(pending: _PendingQuestion) -> Dict[str, Any]: def get_ui_queue(chat_id: Optional[str]) -> Optional[asyncio.Queue]: if not chat_id: return None - with _LOCK: - state = _CHATS.get(chat_id) + with process_state().LOCK: + state = process_state().CHATS.get(chat_id) return state.ui_signals if state is not None else None def get_all_pending(chat_id: Optional[str]) -> List[Dict[str, Any]]: if not chat_id: return [] - with _LOCK: - state = _CHATS.get(chat_id) + with process_state().LOCK: + state = process_state().CHATS.get(chat_id) if state is None: return [] return [_requested_signal(item) for item in state.pending.values()] def list_pending_chat_ids() -> List[str]: - with _LOCK: + with process_state().LOCK: _gc_locked(time.monotonic()) - return [chat_id for chat_id, state in _CHATS.items() if state.pending] + return [chat_id for chat_id, state in process_state().CHATS.items() if state.pending] def has_pending(chat_id: Optional[str]) -> bool: if not chat_id: return False - with _LOCK: - state = _CHATS.get(chat_id) + with process_state().LOCK: + state = process_state().CHATS.get(chat_id) return bool(state and state.pending) @@ -339,8 +335,8 @@ def answer( ) -> Dict[str, Any]: """Validate and claim a pending request; first valid claimant wins.""" - with _LOCK: - state = _CHATS.get(chat_id) + with process_state().LOCK: + state = process_state().CHATS.get(chat_id) pending = state.pending.get(request_id) if state is not None else None if state is None or pending is None: return {"ok": False, "reason": "stale", "error": "该问题已失效"} @@ -355,8 +351,8 @@ def answer( def cancel(chat_id: str, request_id: str) -> Dict[str, Any]: """Cancel a pending question from the user interface.""" - with _LOCK: - state = _CHATS.get(chat_id) + with process_state().LOCK: + state = process_state().CHATS.get(chat_id) pending = state.pending.get(request_id) if state is not None else None if state is None or pending is None: return {"ok": False, "reason": "stale", "error": "该问题已失效"} @@ -389,7 +385,7 @@ async def ask( await interaction_store.register( "question", chat_id, pending.request_id, _requested_signal(pending), timeout ) - with _LOCK: + with process_state().LOCK: state = _state_locked(chat_id) if state.ui_signals is None: state.ui_signals = asyncio.Queue() @@ -401,26 +397,26 @@ async def ask( "question", chat_id, pending.request_id, pending.event, timeout ) if decision is not None: - with _LOCK: + with process_state().LOCK: _claim_locked( state, pending, outcome=decision["outcome"], answers=decision.get("answers") ) except asyncio.TimeoutError: - with _LOCK: - state = _CHATS.get(chat_id) + with process_state().LOCK: + state = process_state().CHATS.get(chat_id) if state is not None: _claim_locked(state, pending, outcome="timeout") # A valid HTTP answer/cancel may have claimed the request at the same # event-loop boundary where wait_for delivered TimeoutError. The # claimant owns the terminal outcome; never overwrite it with timeout. except asyncio.CancelledError: - with _LOCK: - state = _CHATS.get(chat_id) + with process_state().LOCK: + state = process_state().CHATS.get(chat_id) if state is not None: _claim_locked(state, pending, outcome="cancelled") raise finally: - with _LOCK: + with process_state().LOCK: _claim_locked(state, pending, outcome="cancelled") await interaction_store.remove("question", chat_id, pending.request_id) @@ -481,3 +477,16 @@ async def cancel_shared(chat_id, request_id): "list_pending_chat_ids", "normalize_questions", ] + + +@dataclass +class UserQuestionsState: + LOCK: object = field(default_factory=lambda: threading.RLock()) + CHATS: Dict[str, _ChatQuestions] = field(default_factory=lambda: {}) + + +_STATE = UserQuestionsState() + + +def process_state() -> UserQuestionsState: + return _STATE diff --git a/src/backend/core/llm/tools/write_tool.py b/src/backend/core/llm/tools/write_tool.py index ce3d384b..e7f29c47 100644 --- a/src/backend/core/llm/tools/write_tool.py +++ b/src/backend/core/llm/tools/write_tool.py @@ -1,6 +1,7 @@ """Read-checked mutations of files in the active sandbox.""" from __future__ import annotations +from core.llm.factory.tools.definitions import function_tool import difflib import hashlib @@ -8,7 +9,7 @@ from typing import Optional from agentscope.tool import Toolkit, ToolResponse -from core.services.project_scope import ProjectScope +from core.projects.edition import ProjectScope from ._common import ( pin_artifact_to_workspace, @@ -18,16 +19,7 @@ shell_quote, register_file_artifact, ) -from ._paths import ( - basename, - is_myspace_physical, - space_metadata, - WORKSPACE_ROOT, - parent_dir, - to_physical_path, - validate_project_scope_path, - validate_workspace_path, -) +from core.sandbox.workspace.resolution import basename, is_myspace_physical, space_metadata, WORKSPACE_ROOT, parent_dir, to_physical_path, validate_project_scope_path, validate_workspace_path from ._state import ReadEntry, ReadStateTracker logger = logging.getLogger(__name__) @@ -69,10 +61,7 @@ async def Write( content: str, register_as_artifact: bool = False, ) -> "ToolResponse": # type: ignore[name-defined] - from core.llm.tool_permissions import ( - PermissionEnforcementError, - require_local_path_permission, - ) + from core.sandbox.permissions.dispatch_models import PermissionEnforcementError, require_local_path_permission from core.sandbox import SandboxConnectError as _SCE from core.sandbox import SandboxError as _SE from core.sandbox import get_sandbox_provider as _get_provider @@ -218,11 +207,12 @@ async def Write( # Local-mode write-back snapshot (ticket #08): back up the real file # before overwriting it, so the user can roll back. No-op on cloud/web. try: - from core.services.local_snapshot_service import maybe_snapshot_local + from core.harness.registry.local_files import maybe_snapshot_local maybe_snapshot_local(physical) - except Exception: - pass + except Exception as fallback_error: + from core.infra.failures import record_best_effort_failure + record_best_effort_failure('core.llm.tools.write_tool:Write', fallback_error) try: if _is_local: # Local desktop mode: the backend runs on the host, so write the @@ -273,7 +263,7 @@ async def Write( artifact_ref: Optional[dict] = None needs_registration = bool(register_as_artifact and user_id and not is_persistent) if needs_registration and _is_local: - from core.artifacts.local_reference import reference_local_file + from core.files.artifacts.local_reference import reference_local_file from fastapi import HTTPException try: @@ -346,5 +336,5 @@ async def Write( " ``persistent=true`` 表示文件保存于个人或团队空间；``space_type`` 为 personal/team/temporary。\n" ) - toolkit.register_tool_function(Write, namesake_strategy="override") + toolkit.append(function_tool(Write)) logger.info("[factory] Registered Write tool (chat_id=%s)", chat_id) diff --git a/src/backend/core/memory/access.py b/src/backend/core/memory/access.py index 9d3e8d2c..5fb4a587 100644 --- a/src/backend/core/memory/access.py +++ b/src/backend/core/memory/access.py @@ -27,10 +27,10 @@ async def authorized_memory_context(memory_id: str, actor: str) -> Optional[Memo scope = item.get("user_id") or metadata.get("user_id") workspace = str(metadata.get("workspace_id") or "default") if workspace.startswith("project:"): - from core.auth.permissions_iface import resolve_project_access + from core.auth.edition import resolve_project_access from core.db.engine import SessionLocal from core.db.models import Project - from core.services.project_scope import project_memory_policy + from core.projects.edition import project_memory_policy with SessionLocal() as db: project = db.query(Project).filter_by( diff --git a/src/backend/core/memory/audit.py b/src/backend/core/memory/audit.py index 2e3c3042..5a3a42d5 100644 --- a/src/backend/core/memory/audit.py +++ b/src/backend/core/memory/audit.py @@ -1,35 +1,155 @@ -"""Memory audit side-channel — community-edition no-op stub. - -Memory auditing (hash-only storage, trace chains) is a commercial-edition -compliance capability; the community edition keeps the same module surface but -records nothing, so callers (the L1 profile writer, the extraction pipeline) -need zero changes. - -The stubs must stay *call-compatible* with the EE implementation, whose real -signatures are ``record(ctx, action, layer, *, memory_id=None, content=None, -reason=None)`` etc. — the first three are positional. Narrowing the stub -signature makes positional calls raise TypeError, and that raise lands *after* -the business transaction has committed, turning a successful write into a -reported failure (bitten in the 0.2.15 desktop local build: the L1 preference -was persisted, yet the UI never showed the write card). Hence ``*args`` / -``**kwargs`` catch-alls. +"""Unified entry point for memory operation auditing. + +All read/write operations across the L1 / L2 / L3 / session layers should go +through `record()` or `record_batch()` to write to the `memory_audit` table. +Failures do not bubble up; they only emit a warning (auditing must not block +the main flow). + +Toggle: `MEMORY_AUDIT_ENABLED` env (default true). Forced on for government / +financial scenarios. """ from __future__ import annotations -from typing import Any +import asyncio +import hashlib +import logging +from typing import Iterable, Literal, Optional + +from core.config.settings import settings +from core.memory.context import MemoryContext + +logger = logging.getLogger(__name__) + + +Action = Literal["read", "write", "update", "delete", "write_rejected", "forget"] +Layer = Literal["L1", "L2", "L3", "session", "batch"] + + +def _audit_enabled() -> bool: + """Audit toggle (aligned with global settings, default true).""" + from core.memory.edition import audit_enabled + return audit_enabled() + + +def _content_hash(content: Optional[str]) -> Optional[str]: + if content is None: + return None + return hashlib.sha256(content.encode("utf-8", errors="replace")).hexdigest() + + +async def record( + ctx: MemoryContext, + action: Action, + layer: Layer, + *, + memory_id: Optional[str] = None, + content: Optional[str] = None, + reason: Optional[str] = None, +) -> None: + """Write one audit record. Failures do not bubble up. + + content is used only to compute the hash, never stored; the original text + is never written to the audit table. + """ + if not settings.memory.enabled or not _audit_enabled(): + return + + try: + loop = asyncio.get_running_loop() + await loop.run_in_executor(None, _sync_insert_one, ctx, action, layer, memory_id, content, reason) + except Exception as exc: + logger.warning("[memory_audit] record failed: %s", exc) + + +async def record_batch( + ctx: MemoryContext, + entries: Iterable[dict], +) -> None: + """Batch-write audit records; each entry must contain `action` / `layer`, other fields optional.""" + if not settings.memory.enabled or not _audit_enabled(): + return + + rows = [] + for e in entries: + rows.append({ + "actor": ctx.effective_actor, + "action": e["action"], + "layer": e["layer"], + "memory_id": e.get("memory_id"), + "workspace_id": ctx.workspace_id, + "user_id": ctx.user_id, + "chat_id": ctx.chat_id, + "confidentiality": e.get("confidentiality") or ctx.confidentiality, + "content_hash": _content_hash(e.get("content")), + "reason": e.get("reason"), + }) + if not rows: + return + + try: + loop = asyncio.get_running_loop() + await loop.run_in_executor(None, _sync_insert_many, rows) + except Exception as exc: + logger.warning("[memory_audit] record_batch failed (%d rows): %s", len(rows), exc) + +def record_sync( + ctx: MemoryContext, + action: Action, + layer: Layer, + *, + memory_id: Optional[str] = None, + content: Optional[str] = None, + reason: Optional[str] = None, +) -> None: + """Synchronously write one audit record (for non-async contexts, e.g. sync functions inside run_in_executor). -async def record(*args: Any, **kwargs: Any) -> None: - return None + The only difference from `record()` is the calling style: the former is + async (internally goes through the executor), the latter inserts into the + DB synchronously on the current thread. Both are gated by the + `audit_enabled` toggle, and failures do not bubble up. + """ + if not settings.memory.enabled or not _audit_enabled(): + return + try: + _sync_insert_one(ctx, action, layer, memory_id, content, reason) + except Exception as exc: + logger.debug("[memory_audit] record_sync failed: %s", exc) -async def record_batch(*args: Any, **kwargs: Any) -> None: - return None +def _sync_insert_one( + ctx: MemoryContext, + action: Action, + layer: Layer, + memory_id: Optional[str], + content: Optional[str], + reason: Optional[str], +) -> None: + from core.db.engine import SessionLocal + from core.db.models import MemoryAudit + row = MemoryAudit( + actor=ctx.effective_actor, + action=action, + layer=layer, + memory_id=memory_id, + workspace_id=ctx.workspace_id, + user_id=ctx.user_id, + chat_id=ctx.chat_id, + confidentiality=ctx.confidentiality or "unknown", + content_hash=_content_hash(content), + reason=reason, + ) + with SessionLocal() as session: + session.add(row) + session.commit() -def record_sync(*args: Any, **kwargs: Any) -> None: - return None +def _sync_insert_many(rows: list[dict]) -> None: + from core.db.engine import SessionLocal + from core.db.models import MemoryAudit -__all__ = ["record", "record_batch", "record_sync"] + with SessionLocal() as session: + session.add_all([MemoryAudit(**r) for r in rows]) + session.commit() diff --git a/src/backend/core/memory/backend.py b/src/backend/core/memory/backend.py index 0e9d3141..e2d787d7 100644 --- a/src/backend/core/memory/backend.py +++ b/src/backend/core/memory/backend.py @@ -3,6 +3,7 @@ from __future__ import annotations import logging import threading +from dataclasses import dataclass, field from typing import Optional from core.config.settings import settings @@ -13,11 +14,8 @@ # L2 stores procedures; fact extraction belongs to the layered pipeline. MEMORY_TYPE_PROCEDURAL = "procedural" # One lock and instance per process; account epoch changes invalidate the cache. -_memory_instance = None -_memory_epoch: Optional[str] = None -_memory_lock = threading.Lock() -_memory_init_failed = False -_embedding_patched = False + + _MIGRATION_MAX_ROWS = 10_000 @@ -28,8 +26,8 @@ def _patch_mem0_embedding() -> None: but mem0's OpenAIEmbedding.embed() hardcodes dimensions=... Patched only once. """ - global _embedding_patched - if _embedding_patched: + pass + if process_state().embedding_patched: return try: from mem0.embeddings.openai import OpenAIEmbedding as _OAIEmbed @@ -43,9 +41,10 @@ def _patched_embed(self, text, memory_action=None): ) _OAIEmbed.embed = _patched_embed - _embedding_patched = True - except Exception: - pass + process_state().embedding_patched = True + except Exception as fallback_error: + from core.infra.failures import record_best_effort_failure + record_best_effort_failure('core.memory.backend:_patch_mem0_embedding', fallback_error) def _build_mem0_config() -> dict: @@ -58,7 +57,7 @@ def _build_mem0_config() -> dict: """ # Resolve LLM config from DB try: - from core.services.model_config import ModelConfigService + from core.models.config import ModelConfigService svc = ModelConfigService.get_instance() mem_cfg = svc.resolve("memory") @@ -139,7 +138,7 @@ def _apply_probed_embed_dims(cfg: dict) -> None: vs = cfg["vector_store"]["config"] try: from openai import OpenAI - from core.services.desktop_model_credentials import sync_client_kwargs + from core.harness.registry.model_credentials import sync_client_kwargs client = OpenAI( api_key=emb["api_key"], @@ -252,8 +251,9 @@ def _reconcile_vector_collection(cfg: dict) -> Optional[list]: if client is not None: try: client.close() - except Exception: - pass + except Exception as fallback_error: + from core.infra.failures import record_best_effort_failure + record_best_effort_failure('core.memory.backend:_reconcile_vector_collection', fallback_error) def _reembed_rows(client, name: str, cfg: dict) -> Optional[list]: @@ -266,7 +266,7 @@ def _reembed_rows(client, name: str, cfg: dict) -> Optional[list]: emb = cfg["embedder"]["config"] try: from openai import OpenAI - from core.services.desktop_model_credentials import sync_client_kwargs + from core.harness.registry.model_credentials import sync_client_kwargs oai = OpenAI( api_key=emb["api_key"], @@ -319,12 +319,12 @@ def _replay_migrated_rows(instance, rows: list) -> None: def _desktop_memory_epoch() -> str: """Nonsecret identity for clients that hold desktop authorization hooks.""" - from core.auth.desktop_bridge import bridge_enabled - from core.capabilities.paths import capabilities_enabled + from core.harness.registry.host_runtime import requires_remote_execution as bridge_enabled + from core.harness.registry.run_capabilities import has_run_capabilities - if not bridge_enabled() or not capabilities_enabled(): + if not bridge_enabled() or not has_run_capabilities(): return "" - from core.services.desktop_cloud_bridge import _state_fingerprint, get_state + from core.harness.registry.execution_identity import state_fingerprint as _state_fingerprint, get_state return _state_fingerprint(get_state()) @@ -352,23 +352,23 @@ def _close_failed_memory(instance) -> None: def _get_memory() -> Optional[object]: """Thread-safe lazy initialization: cache the instance on success; on failure allow retry next time.""" - global _memory_instance, _memory_init_failed, _memory_epoch + pass if not settings.memory.enabled: return None epoch = _desktop_memory_epoch() # Existing callers retain their captured hooks; only new callers rebuild. - if _memory_instance is not None and _memory_epoch == epoch: - return _memory_instance + if process_state().memory_instance is not None and process_state().memory_epoch == epoch: + return process_state().memory_instance - with _memory_lock: + with process_state().memory_lock: # Double-check after acquiring lock epoch = _desktop_memory_epoch() - if _memory_instance is not None and _memory_epoch == epoch: - return _memory_instance - _memory_instance = None - _memory_epoch = None + if process_state().memory_instance is not None and process_state().memory_epoch == epoch: + return process_state().memory_instance + process_state().memory_instance = None + process_state().memory_epoch = None instance = None try: @@ -384,7 +384,7 @@ def _get_memory() -> Optional[object]: instance = Memory.from_config(cfg) # Existing collections remain released after a local database restart. instance.vector_store.client.load_collection(collection_name=MEMORY_COLLECTION_NAME) - from core.services.desktop_model_credentials import is_reference, sync_client_kwargs + from core.harness.registry.model_credentials import is_reference, sync_client_kwargs from openai import OpenAI for component, section in ( @@ -402,7 +402,7 @@ def _get_memory() -> Optional[object]: ) old_client.close() if epoch != _desktop_memory_epoch(): - from core.capabilities.errors import CloudUnavailable + from core.harness.capability_errors import CloudUnavailable raise CloudUnavailable("cloud account changed while preparing memory") if replay_rows: @@ -410,17 +410,17 @@ def _get_memory() -> Optional[object]: # with the rows re-embedded during reconciliation. _replay_migrated_rows(instance, replay_rows) if epoch != _desktop_memory_epoch(): - from core.capabilities.errors import CloudUnavailable + from core.harness.capability_errors import CloudUnavailable raise CloudUnavailable("cloud account changed while preparing memory") - _memory_instance = instance - _memory_epoch = epoch - _memory_init_failed = False - return _memory_instance + process_state().memory_instance = instance + process_state().memory_epoch = epoch + process_state().memory_init_failed = False + return process_state().memory_instance except Exception as exc: if instance is not None: _close_failed_memory(instance) - _memory_init_failed = True + process_state().memory_init_failed = True logger.error("[MemoryService] 初始化失败，记忆功能将降级为空: %s", exc) return None @@ -430,11 +430,11 @@ def _reset_memory() -> None: Used when the Milvus connection is broken (e.g. closed channel). """ - global _memory_instance, _memory_init_failed, _memory_epoch - with _memory_lock: - _memory_instance = None - _memory_epoch = None - _memory_init_failed = False + pass + with process_state().memory_lock: + process_state().memory_instance = None + process_state().memory_epoch = None + process_state().memory_init_failed = False logger.info("[MemoryService] 已重置 mem0 实例，下次调用将重新初始化") @@ -471,3 +471,19 @@ def _is_auth_error(exc: Exception) -> bool: "401 unauthorized", ) ) + + +@dataclass +class BackendState: + memory_instance: object = None + memory_epoch: Optional[str] = None + memory_lock: object = field(default_factory=lambda: threading.Lock()) + memory_init_failed: object = False + embedding_patched: object = False + + +_STATE = BackendState() + + +def process_state() -> BackendState: + return _STATE diff --git a/src/backend/core/memory/context.py b/src/backend/core/memory/context.py index a4654c08..8059cda2 100644 --- a/src/backend/core/memory/context.py +++ b/src/backend/core/memory/context.py @@ -7,7 +7,7 @@ from __future__ import annotations -from dataclasses import dataclass, field +from dataclasses import dataclass from typing import Literal, Optional Confidentiality = Literal["public", "internal", "sensitive"] diff --git a/src/backend/core/memory/edition.py b/src/backend/core/memory/edition.py new file mode 100644 index 00000000..3d175a39 --- /dev/null +++ b/src/backend/core/memory/edition.py @@ -0,0 +1,4 @@ +"""Community memory has no audit storage.""" + +def audit_enabled(): + return False diff --git a/src/backend/core/memory/effect_lane.py b/src/backend/core/memory/effect_lane.py index 50ad522c..9f81eeab 100644 --- a/src/backend/core/memory/effect_lane.py +++ b/src/backend/core/memory/effect_lane.py @@ -6,6 +6,7 @@ import hashlib import threading from contextlib import asynccontextmanager +from dataclasses import dataclass, field from datetime import datetime, timedelta, timezone from core.db.engine import SessionLocal @@ -13,9 +14,6 @@ from core.memory.executor import run_effect from sqlalchemy import text -_local_locks: dict[str, threading.Lock] = {} -_local_locks_guard = threading.Lock() - class EffectLaneDeferred(RuntimeError): def __init__(self, retry_at: datetime) -> None: @@ -24,8 +22,8 @@ def __init__(self, retry_at: datetime) -> None: def _local_lock(scope_key: str) -> threading.Lock: - with _local_locks_guard: - return _local_locks.setdefault(scope_key, threading.Lock()) + with process_state().local_locks_guard: + return process_state().local_locks.setdefault(scope_key, threading.Lock()) def _advisory_key(scope_key: str) -> int: @@ -101,3 +99,16 @@ async def ordered_l2_effect(scope_key: str, job_id: str): await run_effect(db.close) if local is not None and acquired: local.release() + + +@dataclass +class EffectLaneState: + local_locks: dict[str, threading.Lock] = field(default_factory=lambda: {}) + local_locks_guard: object = field(default_factory=lambda: threading.Lock()) + + +_STATE = EffectLaneState() + + +def process_state() -> EffectLaneState: + return _STATE diff --git a/src/backend/core/memory/executor.py b/src/backend/core/memory/executor.py index 1aa48465..42a209c3 100644 --- a/src/backend/core/memory/executor.py +++ b/src/backend/core/memory/executor.py @@ -26,8 +26,9 @@ async def run_effect(fn: Callable[..., T], *args, **kwargs) -> T: except asyncio.CancelledError: try: await future - except Exception: + except Exception as fallback_error: # Cancellation remains the owner-visible outcome. The durable row # is released/retried only after this thread has actually stopped. - pass + from core.infra.failures import record_best_effort_failure + record_best_effort_failure('core.memory.executor:run_effect', fallback_error) raise diff --git a/src/backend/core/memory/extractors/_base.py b/src/backend/core/memory/extractors/_base.py index f33211f8..78314153 100644 --- a/src/backend/core/memory/extractors/_base.py +++ b/src/backend/core/memory/extractors/_base.py @@ -11,6 +11,7 @@ import json import logging import re +from dataclasses import dataclass, field from datetime import date from typing import Any, Optional @@ -20,9 +21,6 @@ # Lazily-loaded OpenAI client + resolved model name (avoids errors at import time) -_client = None -_model_name: Optional[str] = None -_client_lock = asyncio.Lock() def _resolve_memory_model_config() -> tuple[str, str, str]: @@ -32,7 +30,7 @@ def _resolve_memory_model_config() -> tuple[str, str, str]: Consistent with `core/memory/service.py::_build_mem0_config`. """ try: - from core.services.model_config import ModelConfigService + from core.models.config import ModelConfigService svc = ModelConfigService.get_instance() mem_cfg = svc.resolve("memory") if mem_cfg and mem_cfg.base_url and mem_cfg.api_key: @@ -46,21 +44,17 @@ def _resolve_memory_model_config() -> tuple[str, str, str]: def _client_identity(base_url: str, api_key: str) -> tuple: """桌面网关引用绑定账号：桥接身份变了就要换客户端（凭据轮换不换）。""" - from core.services.desktop_model_credentials import bind, is_reference + from core.harness.registry.model_credentials import bind, is_reference if not is_reference(api_key): return (base_url, api_key) - from core.services.desktop_cloud_bridge import _state_fingerprint + from core.harness.registry.execution_identity import state_fingerprint as _state_fingerprint return (base_url, api_key, _state_fingerprint(bind(api_key, base_url))) -_client_identity_key: tuple | None = None - - async def _get_client() -> tuple[object, str]: """A dedicated memory LLM client, not shared with the main conversation. Returns (client, model_name).""" - global _client, _model_name, _client_identity_key base_url, api_key, model_name = _resolve_memory_model_config() if not base_url or not api_key or not model_name: raise RuntimeError( @@ -68,13 +62,13 @@ async def _get_client() -> tuple[object, str]: f"api_key={bool(api_key)} model_name={bool(model_name)}" ) identity = _client_identity(base_url, api_key) - if _client is not None and _model_name is not None and _client_identity_key == identity: - return _client, _model_name - async with _client_lock: - if _client is not None and _model_name is not None and _client_identity_key == identity: - return _client, _model_name + if process_state().client is not None and process_state().model_name is not None and process_state().client_identity_key == identity: + return process_state().client, process_state().model_name + async with process_state().client_lock: + if process_state().client is not None and process_state().model_name is not None and process_state().client_identity_key == identity: + return process_state().client, process_state().model_name from openai import AsyncOpenAI - from core.services.desktop_model_credentials import is_reference, request_hook + from core.harness.registry.model_credentials import is_reference, request_hook http_client = None if is_reference(api_key): @@ -84,7 +78,7 @@ async def _get_client() -> tuple[object, str]: hook, _captured = request_hook(api_key, base_url) http_client = httpx.AsyncClient(timeout=180.0, event_hooks={"request": [hook]}) - _client = AsyncOpenAI( + process_state().client = AsyncOpenAI( base_url=base_url, api_key=api_key, http_client=http_client, @@ -96,11 +90,11 @@ async def _get_client() -> tuple[object, str]: # extractor sets; this only stops a socket hanging forever. timeout=180.0, ) - _model_name = model_name - _client_identity_key = identity + process_state().model_name = model_name + process_state().client_identity_key = identity logger.info("[extractor] memory LLM client initialized model=%s base=%s", model_name, base_url) - return _client, _model_name + return process_state().client, process_state().model_name def _strip_fences(text: str) -> str: @@ -286,3 +280,18 @@ def fill_prompt( except KeyError as exc: logger.warning("[extractor] prompt template missing key: %s", exc) return template + + +@dataclass +class BaseState: + client: object = None + model_name: Optional[str] = None + client_lock: object = field(default_factory=lambda: asyncio.Lock()) + client_identity_key: tuple | None = None + + +_STATE = BaseState() + + +def process_state() -> BaseState: + return _STATE diff --git a/src/backend/core/memory/extractors/graph.py b/src/backend/core/memory/extractors/graph.py index 95778a9f..9f5cb059 100644 --- a/src/backend/core/memory/extractors/graph.py +++ b/src/backend/core/memory/extractors/graph.py @@ -11,7 +11,7 @@ from typing import Optional from core.memory.extractors._base import fill_prompt, parse_json, run_llm_with_prompt -from core.memory.graph import PREDICATE_LABELS +from core.memory.graph_state import PREDICATE_LABELS PROMPT = """你是一个 L3 知识图谱关系抽取器。只抽取用户明确陈述或明确确认的、跨会话仍有价值的稳定实体关系。 diff --git a/src/backend/core/memory/extractors/writers.py b/src/backend/core/memory/extractors/writers.py index 1ee1f0bd..01dc6789 100644 --- a/src/backend/core/memory/extractors/writers.py +++ b/src/backend/core/memory/extractors/writers.py @@ -30,7 +30,8 @@ from core.memory.audit import record_batch as audit_record_batch from core.memory.context import MemoryContext from core.memory.extractors.router import ExtractorType -from core.memory.graph import parse_relation, parse_relations, write_graph_relations +from core.memory.graph_contract import parse_relation, parse_relations +from core.memory.graph import write_graph_relations from core.memory.pipeline import milvus_breaker from core.memory.sanitizer import sanitize from core.memory.service import ( @@ -47,7 +48,6 @@ LAYER_PROCEDURE = "L2" LAYER_GRAPH = "L3" - async def write_layered( results: dict[ExtractorType, Optional[dict]], ctx: MemoryContext, @@ -107,7 +107,6 @@ async def write_layered( return written - async def _write_relations_to_neo4j( data: dict, ctx: MemoryContext, *, strict: bool = False ) -> list[dict]: @@ -188,7 +187,6 @@ async def _write_relations_to_neo4j( for row in rows ] - async def _write_profile_from_identity( data: dict, ctx: MemoryContext, *, strict: bool = False ) -> list[dict]: @@ -202,7 +200,6 @@ async def _write_profile_from_identity( strict=strict, ) - async def _write_profile_from_preference( data: dict, ctx: MemoryContext, *, strict: bool = False ) -> list[dict]: @@ -216,7 +213,6 @@ async def _write_profile_from_preference( strict=strict, ) - async def _upsert_profile_facts( data: dict, ctx: MemoryContext, @@ -272,7 +268,6 @@ async def _upsert_profile_facts( for item in applied ] - async def _write_procedures_to_milvus( data: dict, ctx: MemoryContext, *, strict: bool = False ) -> list[dict]: @@ -463,7 +458,6 @@ async def _write_procedures_to_milvus( logger.info("[writer:procedural] wrote %d procedures to L2", len(written)) return written - async def _write_session_task(data: dict, ctx: MemoryContext, *, strict: bool = False) -> int: """Write the session task working set into chats.metadata.session_memory. diff --git a/src/backend/core/memory/graph.py b/src/backend/core/memory/graph.py index cdf20194..35c49c3e 100644 --- a/src/backend/core/memory/graph.py +++ b/src/backend/core/memory/graph.py @@ -1,198 +1,26 @@ -"""Neo4j-backed L3 graph memory. - -mem0ai 2.x removed graph-store support from its open-source ``Memory`` class. -The project still owns a Neo4j service and an L3 product contract, so this -module keeps that contract local and explicit instead of passing an ignored -``graph_store`` key to mem0. - -L3 stores stable, declarative relationships (entity -- predicate -- entity). -It deliberately does not store procedures (L2), profile preferences (L1), -volatile measurements, or raw conversation text. -""" - +"""Read and write L3 graph relations in Neo4j.""" from __future__ import annotations - import asyncio import hashlib -import logging -import re -import threading -import unicodedata -from dataclasses import dataclass from typing import Any, Dict, Optional, Sequence - from core.config.settings import settings from core.memory.context import MemoryContext from core.memory.executor import run_effect +import logging +from core.memory import graph_contract +from core.memory import graph_state logger = logging.getLogger(__name__) - -PREDICATE_LABELS: Dict[str, str] = { - "is_a": "属于类别", - "part_of": "组成", - "belongs_to": "隶属于", - "responsible_for": "负责", - "depends_on": "依赖", - "uses": "使用", - "alias_of": "别名为", - "located_in": "位于", - "produces": "产出", - "constrained_by": "受约束于", - "related_to": "关联", -} -ALLOWED_ENTITY_TYPES = frozenset( - { - "person", - "organization", - "team", - "project", - "system", - "document", - "metric", - "concept", - "place", - "other", - } -) - -_MAX_ENTITY_CHARS = 120 -_MAX_RELATIONS_PER_TURN = 12 -_MIN_CONFIDENCE = 0.65 -_SAFE_NAME = re.compile(r"\s+") - - -@dataclass(frozen=True) -class GraphRelation: - """One validated L3 relation before persistence.""" - - source: str - predicate: str - target: str - source_type: str = "other" - target_type: str = "other" - confidence: float = 0.8 - - @property - def relationship(self) -> str: - return PREDICATE_LABELS[self.predicate] - - @property - def text(self) -> str: - return f"{self.source} → {self.relationship} → {self.target}" - - def to_dict(self) -> Dict[str, Any]: - return { - "source": self.source, - "source_type": self.source_type, - "predicate": self.predicate, - "relationship": self.relationship, - "target": self.target, - "target_type": self.target_type, - "confidence": self.confidence, - } - - -def parse_relation(raw: Any) -> Optional[GraphRelation]: - """Validate an extractor item and return its canonical representation.""" - if not isinstance(raw, dict): - return None - source = _clean_name(raw.get("source")) - target = _clean_name(raw.get("target")) - predicate = str(raw.get("predicate") or "").strip().lower() - if not source or not target or source == target or predicate not in PREDICATE_LABELS: - return None - - source_type = str(raw.get("source_type") or "other").strip().lower() - target_type = str(raw.get("target_type") or "other").strip().lower() - if source_type not in ALLOWED_ENTITY_TYPES: - source_type = "other" - if target_type not in ALLOWED_ENTITY_TYPES: - target_type = "other" - try: - confidence = float(raw.get("confidence", 0.8)) - except (TypeError, ValueError): - confidence = 0.8 - confidence = max(0.0, min(1.0, confidence)) - if confidence < _MIN_CONFIDENCE: - return None - return GraphRelation( - source=source, - predicate=predicate, - target=target, - source_type=source_type, - target_type=target_type, - confidence=confidence, - ) - - -def parse_relations(raw_items: Sequence[Any]) -> list[GraphRelation]: - """Validate, de-duplicate, and bound one turn's extracted relations.""" - relations: list[GraphRelation] = [] - seen: set[tuple[str, str, str]] = set() - for raw in raw_items or []: - relation = parse_relation(raw) - if relation is None: - continue - key = ( - _entity_key(relation.source, relation.source_type), - relation.predicate, - _entity_key(relation.target, relation.target_type), - ) - if key in seen: - continue - seen.add(key) - relations.append(relation) - if len(relations) >= _MAX_RELATIONS_PER_TURN: - break - return relations - - -def _clean_name(value: Any) -> str: - name = unicodedata.normalize("NFKC", str(value or "")) - name = _SAFE_NAME.sub(" ", name).strip(" \t\r\n,，。；;：:") - if len(name) < 2 or len(name) > _MAX_ENTITY_CHARS: - return "" - return name - - -def _normalise_name(value: str) -> str: - return _SAFE_NAME.sub(" ", unicodedata.normalize("NFKC", value).lower()).strip() - - -def _entity_key(name: str, entity_type: str) -> str: - raw = f"{entity_type}|{_normalise_name(name)}" - return hashlib.sha256(raw.encode("utf-8")).hexdigest()[:32] - - -def relation_id_for(relation: GraphRelation, *, scope_user_id: str, workspace_id: str) -> str: - raw = "|".join( - ( - scope_user_id, - workspace_id or "default", - _entity_key(relation.source, relation.source_type), - relation.predicate, - _entity_key(relation.target, relation.target_type), - ) - ) - return "grel_" + hashlib.sha256(raw.encode("utf-8")).hexdigest()[:28] - - -_driver = None -_driver_lock = threading.Lock() -_schema_ready = False - - -def _get_driver(): +def get_driver(): """Lazily create the thread-safe Neo4j driver; failures are retryable.""" - global _driver, _schema_ready if not (settings.memory.enabled and settings.memory.graph_enabled): return None - if _driver is not None: - return _driver - with _driver_lock: - if _driver is not None: - return _driver + if graph_state.process_state().driver is not None: + return graph_state.process_state().driver + with graph_state.process_state().driver_lock: + if graph_state.process_state().driver is not None: + return graph_state.process_state().driver try: from neo4j import GraphDatabase @@ -202,33 +30,33 @@ def _get_driver(): connection_timeout=3.0, ) driver.verify_connectivity() - _driver = driver - if not _schema_ready: - _ensure_schema(driver) - _schema_ready = True + graph_state.process_state().driver = driver + if not graph_state.process_state().schema_ready: + ensure_schema(driver) + graph_state.process_state().schema_ready = True logger.info("[GraphMemory] Neo4j L3 initialized") - return _driver + return graph_state.process_state().driver except Exception as exc: # noqa: BLE001 - optional memory layer logger.warning( "[GraphMemory] initialization failed; L3 disabled for this attempt: %s", exc ) - _driver = None + graph_state.process_state().driver = None return None -def _reset_driver() -> None: - global _driver, _schema_ready - with _driver_lock: - current, _driver = _driver, None - _schema_ready = False +def reset_driver() -> None: + with graph_state.process_state().driver_lock: + current, graph_state.process_state().driver = graph_state.process_state().driver, None + graph_state.process_state().schema_ready = False if current is not None: try: current.close() - except Exception: - pass + except Exception as fallback_error: + from core.infra.failures import record_best_effort_failure + record_best_effort_failure('core.memory.graph:_reset_driver', fallback_error) -def _ensure_schema(driver) -> None: +def ensure_schema(driver) -> None: statements = ( "CREATE CONSTRAINT memory_entity_scope_key IF NOT EXISTS " "FOR (n:MemoryEntity) REQUIRE (n.scope_user_id, n.workspace_id, n.key) IS UNIQUE", @@ -243,47 +71,8 @@ def _ensure_schema(driver) -> None: logger.warning("[GraphMemory] schema setup skipped: %s", exc) -_UPSERT_CYPHER = """ -MERGE (s:MemoryEntity { - scope_user_id: $scope_user_id, workspace_id: $workspace_id, key: $source_key -}) -ON CREATE SET s.created_at = datetime() -SET s.name = $source, s.entity_type = $source_type, s.last_seen_at = datetime() -MERGE (t:MemoryEntity { - scope_user_id: $scope_user_id, workspace_id: $workspace_id, key: $target_key -}) -ON CREATE SET t.created_at = datetime() -SET t.name = $target, t.entity_type = $target_type, t.last_seen_at = datetime() -MERGE (s)-[r:MEMORY_RELATION {relation_id: $relation_id}]->(t) -ON CREATE SET r.created_at = datetime(), r.seen_count = 0, r.weight = 1.0, r.active = true -WITH r, - CASE - WHEN $effect_id <> '' AND $effect_id IN coalesce(r.outbox_effect_ids, []) THEN true - ELSE false - END AS replayed -FOREACH (_ IN CASE WHEN replayed THEN [] ELSE [1] END | - SET r.predicate = $predicate, - r.relationship = $relationship, - r.last_seen_at = datetime(), - r.seen_count = coalesce(r.seen_count, 0) + 1, - r.confidence = CASE - WHEN coalesce(r.confidence, 0.0) > $confidence THEN r.confidence - ELSE $confidence - END, - r.author_user_id = $author_user_id, - r.chat_id = $chat_id, - r.evidence_hash = $evidence_hash, - r.outbox_effect_ids = CASE - WHEN $effect_id = '' THEN coalesce(r.outbox_effect_ids, []) - ELSE coalesce(r.outbox_effect_ids, []) + [$effect_id] - END -) -RETURN r.relation_id AS relation_id, r.seen_count AS seen_count, replayed -""" - - -def _write_sync(ctx: MemoryContext, relations: Sequence[GraphRelation]) -> list[Dict[str, Any]]: - driver = _get_driver() +def write_sync(ctx: MemoryContext, relations: Sequence[graph_contract.GraphRelation]) -> list[Dict[str, Any]]: + driver = get_driver() if driver is None: return [] evidence_hash = hashlib.sha256( @@ -293,19 +82,19 @@ def _write_sync(ctx: MemoryContext, relations: Sequence[GraphRelation]) -> list[ try: with driver.session() as session: for relation in relations: - relation_id = relation_id_for( + relation_id = graph_contract.relation_id_for( relation, scope_user_id=ctx.effective_scope_user_id, workspace_id=ctx.workspace_id, ) record = session.run( - _UPSERT_CYPHER, + graph_state.UPSERT_CYPHER, scope_user_id=ctx.effective_scope_user_id, workspace_id=ctx.workspace_id or "default", - source_key=_entity_key(relation.source, relation.source_type), + source_key=graph_contract.entity_key(relation.source, relation.source_type), source=relation.source, source_type=relation.source_type, - target_key=_entity_key(relation.target, relation.target_type), + target_key=graph_contract.entity_key(relation.target, relation.target_type), target=relation.target, target_type=relation.target_type, relation_id=relation_id, @@ -331,66 +120,26 @@ def _write_sync(ctx: MemoryContext, relations: Sequence[GraphRelation]) -> list[ return rows except Exception as exc: # noqa: BLE001 - L3 is optional and post-response logger.warning("[GraphMemory] write failed: %s", exc) - _reset_driver() + reset_driver() return [] async def write_graph_relations( - ctx: MemoryContext, relations: Sequence[GraphRelation] + ctx: MemoryContext, relations: Sequence[graph_contract.GraphRelation] ) -> list[Dict[str, Any]]: """Persist validated relations without blocking the event loop.""" if not relations: return [] - return await run_effect(_write_sync, ctx, list(relations)) - - -_LIST_CYPHER = """ -MATCH (s:MemoryEntity)-[r:MEMORY_RELATION]->(t:MemoryEntity) -WHERE s.scope_user_id = $scope_user_id - AND s.workspace_id = $workspace_id - AND coalesce(r.active, true) = true -RETURN r.relation_id AS relation_id, - s.name AS source, s.entity_type AS source_type, - r.predicate AS predicate, r.relationship AS relationship, - t.name AS target, t.entity_type AS target_type, - coalesce(r.confidence, 0.0) AS confidence, - coalesce(r.seen_count, 1) AS seen_count, - coalesce(r.weight, 1.0) AS weight, - toString(r.last_seen_at) AS last_seen_at -ORDER BY coalesce(r.weight, 1.0) DESC, coalesce(r.seen_count, 1) DESC, r.last_seen_at DESC -LIMIT $limit -""" - - -_SEARCH_CYPHER = """ -MATCH (s:MemoryEntity)-[r:MEMORY_RELATION]->(t:MemoryEntity) -WHERE s.scope_user_id = $scope_user_id - AND s.workspace_id = $workspace_id - AND coalesce(r.active, true) = true - AND ( - toLower($search_text) CONTAINS toLower(s.name) - OR toLower($search_text) CONTAINS toLower(t.name) - ) -RETURN r.relation_id AS relation_id, - s.name AS source, s.entity_type AS source_type, - r.predicate AS predicate, r.relationship AS relationship, - t.name AS target, t.entity_type AS target_type, - coalesce(r.confidence, 0.0) AS confidence, - coalesce(r.seen_count, 1) AS seen_count, - coalesce(r.weight, 1.0) AS weight, - toString(r.last_seen_at) AS last_seen_at -ORDER BY coalesce(r.weight, 1.0) DESC, coalesce(r.seen_count, 1) DESC, r.last_seen_at DESC -LIMIT $limit -""" + return await run_effect(write_sync, ctx, list(relations)) -def _read_sync( +def read_sync( *, scope_user_id: str, workspace_id: str, limit: int, query: str = "" ) -> list[Dict[str, Any]]: - driver = _get_driver() + driver = get_driver() if driver is None: return [] - statement = _SEARCH_CYPHER if query.strip() else _LIST_CYPHER + statement = graph_state.SEARCH_CYPHER if query.strip() else graph_state.LIST_CYPHER try: with driver.session() as session: result = session.run( @@ -403,7 +152,7 @@ def _read_sync( return [dict(record) for record in result] except Exception as exc: # noqa: BLE001 logger.warning("[GraphMemory] read failed: %s", exc) - _reset_driver() + reset_driver() return [] @@ -413,7 +162,7 @@ async def list_graph_relations( loop = asyncio.get_running_loop() return await loop.run_in_executor( None, - lambda: _read_sync( + lambda: read_sync( scope_user_id=scope_user_id, workspace_id=workspace_id, limit=limit, @@ -433,7 +182,7 @@ async def search_graph_relations( loop = asyncio.get_running_loop() return await loop.run_in_executor( None, - lambda: _read_sync( + lambda: read_sync( scope_user_id=scope_user_id, workspace_id=workspace_id, limit=limit, @@ -449,7 +198,7 @@ def relation_is_active(relation_id: str, *, scope_user_id: str) -> Optional[bool depends on this relation is still applicable. Tri-state on purpose: an unreachable graph store must not disable every dependent skill. """ - driver = _get_driver() + driver = get_driver() if driver is None or not relation_id: return None try: @@ -468,7 +217,7 @@ def relation_is_active(relation_id: str, *, scope_user_id: str) -> Optional[bool return bool(record["active"]) except Exception as exc: # noqa: BLE001 logger.warning("[GraphMemory] relation_is_active failed: %s", exc) - _reset_driver() + reset_driver() return None @@ -485,7 +234,7 @@ def update_relation_state( deactivating are reversible; deletion is not, and a wrongly deleted edge is indistinguishable from knowledge the system never had. """ - driver = _get_driver() + driver = get_driver() if driver is None or not relation_id: return False sets = [] @@ -513,12 +262,12 @@ def update_relation_state( return bool(record and int(record["touched"] or 0) > 0) except Exception as exc: # noqa: BLE001 logger.warning("[GraphMemory] update_relation_state failed: %s", exc) - _reset_driver() + reset_driver() return False -def _delete_sync(relation_id: str, *, scope_user_id: str) -> bool: - driver = _get_driver() +def delete_sync(relation_id: str, *, scope_user_id: str) -> bool: + driver = get_driver() if driver is None: return False try: @@ -544,7 +293,7 @@ def _delete_sync(relation_id: str, *, scope_user_id: str) -> bool: return bool(record and int(record["deleted"] or 0) > 0) except Exception as exc: # noqa: BLE001 logger.warning("[GraphMemory] delete failed: %s", exc) - _reset_driver() + reset_driver() return False @@ -553,5 +302,5 @@ async def delete_graph_relation(relation_id: str, *, scope_user_id: str) -> bool return False loop = asyncio.get_running_loop() return await loop.run_in_executor( - None, lambda: _delete_sync(relation_id, scope_user_id=scope_user_id) + None, lambda: delete_sync(relation_id, scope_user_id=scope_user_id) ) diff --git a/src/backend/core/memory/graph_contract.py b/src/backend/core/memory/graph_contract.py new file mode 100644 index 00000000..3a8d4a23 --- /dev/null +++ b/src/backend/core/memory/graph_contract.py @@ -0,0 +1,122 @@ +"""Validate and normalise extracted graph relations.""" +from __future__ import annotations +import hashlib +import unicodedata +from dataclasses import dataclass +from typing import Any, Dict, Optional, Sequence +from core.memory import graph_state + +@dataclass(frozen=True) +class GraphRelation: + """One validated L3 relation before persistence.""" + + source: str + predicate: str + target: str + source_type: str = "other" + target_type: str = "other" + confidence: float = 0.8 + + @property + def relationship(self) -> str: + return graph_state.PREDICATE_LABELS[self.predicate] + + @property + def text(self) -> str: + return f"{self.source} → {self.relationship} → {self.target}" + + def to_dict(self) -> Dict[str, Any]: + return { + "source": self.source, + "source_type": self.source_type, + "predicate": self.predicate, + "relationship": self.relationship, + "target": self.target, + "target_type": self.target_type, + "confidence": self.confidence, + } + + +def parse_relation(raw: Any) -> Optional[GraphRelation]: + """Validate an extractor item and return its canonical representation.""" + if not isinstance(raw, dict): + return None + source = clean_name(raw.get("source")) + target = clean_name(raw.get("target")) + predicate = str(raw.get("predicate") or "").strip().lower() + if not source or not target or source == target or predicate not in graph_state.PREDICATE_LABELS: + return None + + source_type = str(raw.get("source_type") or "other").strip().lower() + target_type = str(raw.get("target_type") or "other").strip().lower() + if source_type not in graph_state.ALLOWED_ENTITY_TYPES: + source_type = "other" + if target_type not in graph_state.ALLOWED_ENTITY_TYPES: + target_type = "other" + try: + confidence = float(raw.get("confidence", 0.8)) + except (TypeError, ValueError): + confidence = 0.8 + confidence = max(0.0, min(1.0, confidence)) + if confidence < graph_state.MIN_CONFIDENCE: + return None + return GraphRelation( + source=source, + predicate=predicate, + target=target, + source_type=source_type, + target_type=target_type, + confidence=confidence, + ) + + +def parse_relations(raw_items: Sequence[Any]) -> list[GraphRelation]: + """Validate, de-duplicate, and bound one turn's extracted relations.""" + relations: list[GraphRelation] = [] + seen: set[tuple[str, str, str]] = set() + for raw in raw_items or []: + relation = parse_relation(raw) + if relation is None: + continue + key = ( + entity_key(relation.source, relation.source_type), + relation.predicate, + entity_key(relation.target, relation.target_type), + ) + if key in seen: + continue + seen.add(key) + relations.append(relation) + if len(relations) >= graph_state.MAX_RELATIONS_PER_TURN: + break + return relations + + +def clean_name(value: Any) -> str: + name = unicodedata.normalize("NFKC", str(value or "")) + name = graph_state.SAFE_NAME.sub(" ", name).strip(" \t\r\n,，。；;：:") + if len(name) < 2 or len(name) > graph_state.MAX_ENTITY_CHARS: + return "" + return name + + +def normalise_name(value: str) -> str: + return graph_state.SAFE_NAME.sub(" ", unicodedata.normalize("NFKC", value).lower()).strip() + + +def entity_key(name: str, entity_type: str) -> str: + raw = f"{entity_type}|{normalise_name(name)}" + return hashlib.sha256(raw.encode("utf-8")).hexdigest()[:32] + + +def relation_id_for(relation: GraphRelation, *, scope_user_id: str, workspace_id: str) -> str: + raw = "|".join( + ( + scope_user_id, + workspace_id or "default", + entity_key(relation.source, relation.source_type), + relation.predicate, + entity_key(relation.target, relation.target_type), + ) + ) + return "grel_" + hashlib.sha256(raw.encode("utf-8")).hexdigest()[:28] diff --git a/src/backend/core/memory/graph_state.py b/src/backend/core/memory/graph_state.py new file mode 100644 index 00000000..29b61713 --- /dev/null +++ b/src/backend/core/memory/graph_state.py @@ -0,0 +1,141 @@ +"""Graph vocabulary, limits, Cypher queries and the shared driver.""" +from __future__ import annotations +import re +import threading +from dataclasses import dataclass, field +from typing import Dict + +PREDICATE_LABELS: Dict[str, str] = { + "is_a": "属于类别", + "part_of": "组成", + "belongs_to": "隶属于", + "responsible_for": "负责", + "depends_on": "依赖", + "uses": "使用", + "alias_of": "别名为", + "located_in": "位于", + "produces": "产出", + "constrained_by": "受约束于", + "related_to": "关联", +} + + +ALLOWED_ENTITY_TYPES = frozenset( + { + "person", + "organization", + "team", + "project", + "system", + "document", + "metric", + "concept", + "place", + "other", + } +) + + +MAX_ENTITY_CHARS = 120 + + +MAX_RELATIONS_PER_TURN = 12 + + +MIN_CONFIDENCE = 0.65 + + +SAFE_NAME = re.compile(r"\s+") + + +UPSERT_CYPHER = """ +MERGE (s:MemoryEntity { + scope_user_id: $scope_user_id, workspace_id: $workspace_id, key: $source_key +}) +ON CREATE SET s.created_at = datetime() +SET s.name = $source, s.entity_type = $source_type, s.last_seen_at = datetime() +MERGE (t:MemoryEntity { + scope_user_id: $scope_user_id, workspace_id: $workspace_id, key: $target_key +}) +ON CREATE SET t.created_at = datetime() +SET t.name = $target, t.entity_type = $target_type, t.last_seen_at = datetime() +MERGE (s)-[r:MEMORY_RELATION {relation_id: $relation_id}]->(t) +ON CREATE SET r.created_at = datetime(), r.seen_count = 0, r.weight = 1.0, r.active = true +WITH r, + CASE + WHEN $effect_id <> '' AND $effect_id IN coalesce(r.outbox_effect_ids, []) THEN true + ELSE false + END AS replayed +FOREACH (_ IN CASE WHEN replayed THEN [] ELSE [1] END | + SET r.predicate = $predicate, + r.relationship = $relationship, + r.last_seen_at = datetime(), + r.seen_count = coalesce(r.seen_count, 0) + 1, + r.confidence = CASE + WHEN coalesce(r.confidence, 0.0) > $confidence THEN r.confidence + ELSE $confidence + END, + r.author_user_id = $author_user_id, + r.chat_id = $chat_id, + r.evidence_hash = $evidence_hash, + r.outbox_effect_ids = CASE + WHEN $effect_id = '' THEN coalesce(r.outbox_effect_ids, []) + ELSE coalesce(r.outbox_effect_ids, []) + [$effect_id] + END +) +RETURN r.relation_id AS relation_id, r.seen_count AS seen_count, replayed +""" + + +LIST_CYPHER = """ +MATCH (s:MemoryEntity)-[r:MEMORY_RELATION]->(t:MemoryEntity) +WHERE s.scope_user_id = $scope_user_id + AND s.workspace_id = $workspace_id + AND coalesce(r.active, true) = true +RETURN r.relation_id AS relation_id, + s.name AS source, s.entity_type AS source_type, + r.predicate AS predicate, r.relationship AS relationship, + t.name AS target, t.entity_type AS target_type, + coalesce(r.confidence, 0.0) AS confidence, + coalesce(r.seen_count, 1) AS seen_count, + coalesce(r.weight, 1.0) AS weight, + toString(r.last_seen_at) AS last_seen_at +ORDER BY coalesce(r.weight, 1.0) DESC, coalesce(r.seen_count, 1) DESC, r.last_seen_at DESC +LIMIT $limit +""" + + +SEARCH_CYPHER = """ +MATCH (s:MemoryEntity)-[r:MEMORY_RELATION]->(t:MemoryEntity) +WHERE s.scope_user_id = $scope_user_id + AND s.workspace_id = $workspace_id + AND coalesce(r.active, true) = true + AND ( + toLower($search_text) CONTAINS toLower(s.name) + OR toLower($search_text) CONTAINS toLower(t.name) + ) +RETURN r.relation_id AS relation_id, + s.name AS source, s.entity_type AS source_type, + r.predicate AS predicate, r.relationship AS relationship, + t.name AS target, t.entity_type AS target_type, + coalesce(r.confidence, 0.0) AS confidence, + coalesce(r.seen_count, 1) AS seen_count, + coalesce(r.weight, 1.0) AS weight, + toString(r.last_seen_at) AS last_seen_at +ORDER BY coalesce(r.weight, 1.0) DESC, coalesce(r.seen_count, 1) DESC, r.last_seen_at DESC +LIMIT $limit +""" + + +@dataclass +class GraphState: + driver: object = None + driver_lock: object = field(default_factory=lambda: threading.Lock()) + schema_ready: object = False + + +_STATE = GraphState() + + +def process_state() -> GraphState: + return _STATE diff --git a/src/backend/core/memory/outbox/enqueue.py b/src/backend/core/memory/outbox/enqueue.py index ee2a0420..7a081f30 100644 --- a/src/backend/core/memory/outbox/enqueue.py +++ b/src/backend/core/memory/outbox/enqueue.py @@ -11,8 +11,9 @@ from core.memory.extractors.router import ExtractorType from sqlalchemy.exc import IntegrityError +from core.memory.outbox import common + logger = logging.getLogger(__name__) -from core.memory.outbox import common as common def _enqueue( diff --git a/src/backend/core/memory/outbox/jobs.py b/src/backend/core/memory/outbox/jobs.py index cbcb20ba..9bdc09e5 100644 --- a/src/backend/core/memory/outbox/jobs.py +++ b/src/backend/core/memory/outbox/jobs.py @@ -11,8 +11,9 @@ from core.memory.context import MemoryContext from core.memory.extractors.router import ExtractorType +from core.memory.outbox import common + logger = logging.getLogger(__name__) -from core.memory.outbox import common as common async def _extract_candidates( @@ -198,9 +199,9 @@ async def _process_profile_edit(row: MemoryOutbox) -> dict[str, Any]: text = sanitized.text applied = await upsert_fields(ctx, [(key, text, "user_edit")], strict=True) if ctx.message_id: - from core.evolution.settlement_store import update_entry_text + from core.memory.settlement_port import memory_settlement - update_entry_text(ctx.message_id, key, text) + memory_settlement().update_entry_text(ctx.message_id, key, text) return {"key": key, "text": text, "applied": applied} @@ -219,9 +220,9 @@ async def _process_memory_edit(row: MemoryOutbox) -> dict[str, Any]: raise ValueError("memory not found for actor") await update_memory(memory_id, text, strict=True) if ctx.message_id: - from core.evolution.settlement_store import update_entry_text + from core.memory.settlement_port import memory_settlement - update_entry_text(ctx.message_id, memory_id, text) + memory_settlement().update_entry_text(ctx.message_id, memory_id, text) return {"id": memory_id, "text": text} @@ -243,28 +244,10 @@ def _report_settlement( items: Optional[list[dict]] = None, failed: bool = False, ) -> dict[str, Any]: - from core.db.models import EvolutionEpisode - from core.evolution.settlement import settle_turn - from core.evolution.settlement_runner import acknowledge_durable_settlement - - # The Outbox is now the authoritative join. Retire the old in-process - # watchdog before computing the final card so it cannot race the atomic - # summary+ack transaction performed by ``_finish_success``. - acknowledge_durable_settlement(message_id) - with common.SessionLocal() as db: - episode = ( - db.query(EvolutionEpisode.episode_id) - .filter(EvolutionEpisode.message_id == message_id) - .first() - ) - summary = settle_turn( - message_id=message_id, - episode_id=str(episode[0]) if episode is not None else "", - memory_entries=list(items or []), - memory_failed=failed, - memory_enabled=True, + from core.memory.settlement_port import memory_settlement + return memory_settlement().build_summary( + message_id, session_factory=common.SessionLocal, items=list(items or []), failed=failed, ) - return cast(dict[str, Any], summary.to_dict()) async def _process_settlement(row: MemoryOutbox) -> dict[str, Any]: diff --git a/src/backend/core/memory/outbox/leases.py b/src/backend/core/memory/outbox/leases.py index 37cbc2e4..66256e4e 100644 --- a/src/backend/core/memory/outbox/leases.py +++ b/src/backend/core/memory/outbox/leases.py @@ -10,8 +10,9 @@ from core.db.models import MemoryOutbox from sqlalchemy import and_, or_ +from core.memory.outbox import common + logger = logging.getLogger(__name__) -from core.memory.outbox import common as common def _claim_next(worker_id: str) -> Optional[str]: diff --git a/src/backend/core/memory/outbox/settlement.py b/src/backend/core/memory/outbox/settlement.py index 0620e41a..bedacf46 100644 --- a/src/backend/core/memory/outbox/settlement.py +++ b/src/backend/core/memory/outbox/settlement.py @@ -8,8 +8,9 @@ from core.db.models import MemoryOutbox +from core.memory.outbox import common + logger = logging.getLogger(__name__) -from core.memory.outbox import common as common def _lock_pipeline_row(db, message_id: str) -> Optional[MemoryOutbox]: @@ -93,20 +94,9 @@ def _finish_success(job_id: str, worker_id: str, result: Any) -> Optional[str]: ) if affected: if row.job_kind == "settlement": - from core.db.models import ChatMessage - from core.evolution.settlement_store import METADATA_KEY - + from core.memory.settlement_port import memory_settlement summary = result.get("summary") if isinstance(result, dict) else None - message = db.query(ChatMessage).filter(ChatMessage.message_id == message_id).first() - if message is None: - raise RuntimeError( - "assistant message is not durable yet; settlement will retry" - ) - if not isinstance(summary, dict): - raise RuntimeError("settlement result is missing its durable summary") - extra = dict(message.extra_data or {}) - extra[METADATA_KEY] = summary - message.extra_data = extra + memory_settlement().write_summary(db, message_id, summary) _enqueue_settlement_if_ready(db, message_id, now) db.commit() if affected: diff --git a/src/backend/core/memory/outbox/worker.py b/src/backend/core/memory/outbox/worker.py index 1dd00a26..35546edc 100644 --- a/src/backend/core/memory/outbox/worker.py +++ b/src/backend/core/memory/outbox/worker.py @@ -7,18 +7,18 @@ import os import socket import uuid +from dataclasses import dataclass from typing import Any, Optional from core.db.models import MemoryOutbox from core.memory.effect_lane import EffectLaneDeferred, ordered_l2_effect -logger = logging.getLogger(__name__) -from core.memory.outbox import common as common -from core.memory.outbox import jobs as jobs -from core.memory.outbox import leases as leases -from core.memory.outbox import settlement as settlement +from core.memory.outbox import common +from core.memory.outbox import jobs +from core.memory.outbox import leases +from core.memory.outbox import settlement -_active_worker: Optional["MemoryOutboxWorker"] = None +logger = logging.getLogger(__name__) async def _process_claimed(job_id: str, worker_id: str) -> None: @@ -122,7 +122,7 @@ async def consume_outbox_job(job_id: str, *, wait_s: float = 5.0) -> dict[str, A def kick_outbox_drain() -> None: """Wake the lifecycle-owned worker; never create an orphan consumer task.""" - worker = _active_worker + worker = process_state().active_worker if worker is not None: worker.wake() @@ -135,20 +135,20 @@ def __init__(self) -> None: self.worker_id = f"{socket.gethostname()}:{os.getpid()}:{uuid.uuid4().hex[:8]}" async def start(self) -> None: - global _active_worker + pass if self._task is None or self._task.done(): self._stop.clear() self._wake.clear() settlement.reconcile_settlement_jobs() - _active_worker = self + process_state().active_worker = self self._task = asyncio.create_task(self._loop(), name="memory-outbox-worker") async def stop(self) -> None: - global _active_worker + pass self._stop.set() self._wake.set() - if _active_worker is self: - _active_worker = None + if process_state().active_worker is self: + process_state().active_worker = None if self._task is None: return self._task.cancel() @@ -181,3 +181,15 @@ async def _loop(self) -> None: pass finally: self._wake.clear() + + +@dataclass +class WorkerState: + active_worker: Optional['MemoryOutboxWorker'] = None + + +_STATE = WorkerState() + + +def process_state() -> WorkerState: + return _STATE diff --git a/src/backend/core/memory/pipeline.py b/src/backend/core/memory/pipeline.py index f3202309..65a56cf9 100644 --- a/src/backend/core/memory/pipeline.py +++ b/src/backend/core/memory/pipeline.py @@ -16,6 +16,7 @@ import asyncio import logging +from dataclasses import dataclass, field from threading import Lock from typing import Optional @@ -30,17 +31,12 @@ # ─── Global semaphore (lazy init) ────────────────────────────────────────── -_bg_semaphore: Optional[asyncio.Semaphore] = None -_sem_lock = Lock() - - def get_background_semaphore() -> asyncio.Semaphore: """Return the global background-task semaphore. Must be first called inside an event loop.""" - global _bg_semaphore - with _sem_lock: - if _bg_semaphore is None: - _bg_semaphore = asyncio.Semaphore(settings.memory.bg_max_concurrency) - return _bg_semaphore + with process_state().sem_lock: + if process_state().bg_semaphore is None: + process_state().bg_semaphore = asyncio.Semaphore(settings.memory.bg_max_concurrency) + return process_state().bg_semaphore # ─── Circuit breaker ─────────────────────────────────────────────────────── @@ -153,8 +149,21 @@ def _report_settlement( if not message_id: return try: - from core.evolution.settlement_runner import report_memory_writes + from core.memory.settlement_port import memory_settlement - report_memory_writes(message_id, items=list(items or []), failed=failed) + memory_settlement().report_writes(message_id, items=list(items or []), failed=failed) except Exception as exc: # noqa: BLE001 - settlement must never break writes logger.debug("[memory_pipeline] settlement report skipped: %s", exc) + + +@dataclass +class PipelineState: + bg_semaphore: Optional[asyncio.Semaphore] = None + sem_lock: object = field(default_factory=lambda: Lock()) + + +_STATE = PipelineState() + + +def process_state() -> PipelineState: + return _STATE diff --git a/src/backend/core/memory/preview.py b/src/backend/core/memory/preview.py index d7f89cfb..fc6753a3 100644 --- a/src/backend/core/memory/preview.py +++ b/src/backend/core/memory/preview.py @@ -13,6 +13,7 @@ def sanitized_preview(text: str, *, max_chars: int) -> str: if result.reject: return "[REDACTED]" collapsed = result.text or collapsed - except Exception: - pass + except Exception as fallback_error: + from core.infra.failures import record_best_effort_failure + record_best_effort_failure('core.memory.preview:sanitized_preview', fallback_error) return collapsed[:max_chars] diff --git a/src/backend/core/memory/procedures.py b/src/backend/core/memory/procedures.py index 5d74b2fd..d3bd5705 100644 --- a/src/backend/core/memory/procedures.py +++ b/src/backend/core/memory/procedures.py @@ -11,9 +11,10 @@ from core.memory.executor import run_effect from core.memory.ttl_sweeper import memory_entry_expired -logger = logging.getLogger(__name__) from . import backend +logger = logging.getLogger(__name__) + def _expiration_from_ttl(ttl_days: int) -> Optional[str]: """UTC expiry date owned by L2; mem0 stores it as ordinary metadata.""" diff --git a/src/backend/core/memory/queries.py b/src/backend/core/memory/queries.py index 122e7589..dcac2bcb 100644 --- a/src/backend/core/memory/queries.py +++ b/src/backend/core/memory/queries.py @@ -11,9 +11,10 @@ from core.memory.ttl_sweeper import memory_entry_expired from core.memory.retrieval_types import MemoryRetrievalResult, RetrievedRelation, build_memory_item -logger = logging.getLogger(__name__) from . import backend +logger = logging.getLogger(__name__) + async def retrieve_memories( user_id: str, diff --git a/src/backend/core/memory/sanitizer.py b/src/backend/core/memory/sanitizer.py index d7efec6e..e16d91ab 100644 --- a/src/backend/core/memory/sanitizer.py +++ b/src/backend/core/memory/sanitizer.py @@ -85,16 +85,14 @@ def clean(self) -> bool: _RULES_CACHE_TTL = 300 # seconds -_rules_cache: dict = {"patterns": None, "classified": None, "loaded_at": 0.0} -_rules_lock = Lock() def invalidate_rules_cache() -> None: """Called proactively after an admin adds/removes rules; the next sanitize re-pulls from the DB.""" - with _rules_lock: - _rules_cache["patterns"] = None - _rules_cache["classified"] = None - _rules_cache["loaded_at"] = 0.0 + with process_state().rules_lock: + process_state().rules_cache["patterns"] = None + process_state().rules_cache["classified"] = None + process_state().rules_cache["loaded_at"] = 0.0 def _load_rules_with_db_overrides() -> tuple[dict[str, Pattern[str]], tuple[str, ...]]: @@ -108,12 +106,12 @@ def _load_rules_with_db_overrides() -> tuple[dict[str, Pattern[str]], tuple[str, On DB query failure, falls back to the hardcoded rules without affecting the main flow. """ now = time.time() - with _rules_lock: + with process_state().rules_lock: if ( - _rules_cache["patterns"] is not None - and now - _rules_cache["loaded_at"] < _RULES_CACHE_TTL + process_state().rules_cache["patterns"] is not None + and now - process_state().rules_cache["loaded_at"] < _RULES_CACHE_TTL ): - return _rules_cache["patterns"], _rules_cache["classified"] + return process_state().rules_cache["patterns"], process_state().rules_cache["classified"] patterns = dict(REDACT_PATTERNS) classified = list(CLASSIFIED_TERMS) @@ -141,10 +139,10 @@ def _load_rules_with_db_overrides() -> tuple[dict[str, Pattern[str]], tuple[str, logger.debug("[sanitizer] DB overrides unavailable, using hardcoded only: %s", exc) result_classified = tuple(classified) - with _rules_lock: - _rules_cache["patterns"] = patterns - _rules_cache["classified"] = result_classified - _rules_cache["loaded_at"] = now + with process_state().rules_lock: + process_state().rules_cache["patterns"] = patterns + process_state().rules_cache["classified"] = result_classified + process_state().rules_cache["loaded_at"] = now return patterns, result_classified @@ -177,3 +175,16 @@ def sanitize(text: str) -> SanitizeResult: hits.append(name) cleaned = pat.sub(f"[REDACTED:{name}]", cleaned) return SanitizeResult(text=cleaned, hits=hits, reject=False) + + +@dataclass +class SanitizerState: + rules_cache: dict = field(default_factory=lambda: {'patterns': None, 'classified': None, 'loaded_at': 0.0}) + rules_lock: object = field(default_factory=lambda: Lock()) + + +_STATE = SanitizerState() + + +def process_state() -> SanitizerState: + return _STATE diff --git a/src/backend/core/memory/service.py b/src/backend/core/memory/service.py index 615c4364..b6e03ee4 100644 --- a/src/backend/core/memory/service.py +++ b/src/backend/core/memory/service.py @@ -1,11 +1,6 @@ """Public memory API. Lifecycle, retrieval and procedural writes have separate owners.""" -from .backend import ( - reset_runtime, - MEMORY_COLLECTION_NAME, - MEMORY_TYPE_PROCEDURAL, - _get_memory, -) +from .backend import reset_runtime, MEMORY_COLLECTION_NAME, MEMORY_TYPE_PROCEDURAL, _get_memory as get_memory from .queries import ( retrieve_memories, retrieve_memories_structured, @@ -25,7 +20,7 @@ "reset_runtime", "MEMORY_COLLECTION_NAME", "MEMORY_TYPE_PROCEDURAL", - "_get_memory", + "get_memory", "retrieve_memories", "retrieve_memories_structured", "get_all_memories", diff --git a/src/backend/core/memory/session_snapshot.py b/src/backend/core/memory/session_snapshot.py index c246d49f..ce338bf9 100644 --- a/src/backend/core/memory/session_snapshot.py +++ b/src/backend/core/memory/session_snapshot.py @@ -9,7 +9,7 @@ call, against 17s for the second call in the same turn, which kept the prefix. Stored on ``ChatSession.extra_data`` next to the other chat-sticky runtime -state (see ``core.llm.session_capabilities``). The snapshot records the memory +state (see ``core.llm.runtime.session_capabilities``). The snapshot records the memory scope it was built for, because a snapshot assembled for one scope must never be served to another. """ diff --git a/src/backend/core/services/memory_settings_service.py b/src/backend/core/memory/settings.py similarity index 98% rename from src/backend/core/services/memory_settings_service.py rename to src/backend/core/memory/settings.py index 81357866..aa702f55 100644 --- a/src/backend/core/services/memory_settings_service.py +++ b/src/backend/core/memory/settings.py @@ -6,7 +6,7 @@ from typing import Any, Dict from core.config.settings import settings -from core.db import model_repository +from core.models import repository as model_repository from core.infra.exceptions import BadRequestError from sqlalchemy.orm import Session diff --git a/src/backend/core/memory/settlement_port.py b/src/backend/core/memory/settlement_port.py new file mode 100644 index 00000000..6bb3327c --- /dev/null +++ b/src/backend/core/memory/settlement_port.py @@ -0,0 +1,33 @@ +"""Memory's named settlement boundary; callers retain their transaction scope.""" +from __future__ import annotations +from dataclasses import dataclass +from typing import Any, Protocol + + +class MemorySettlement(Protocol): + def report_writes(self, message_id: str, *, items: list[dict], failed: bool) -> None: ... + def update_entry_text(self, message_id: str, handle: str, text: str) -> bool: ... + def build_summary(self, message_id: str, *, session_factory: Any, items: list[dict], failed: bool) -> dict[str, Any]: ... + def write_summary(self, db: Any, message_id: str, summary: Any) -> None: ... + + +def register_memory_settlement(service: MemorySettlement) -> None: + process_state().service = service + + +def memory_settlement() -> MemorySettlement: + if process_state().service is None: + raise RuntimeError("Memory settlement service is not registered") + return process_state().service + + +@dataclass +class SettlementPortState: + service: MemorySettlement | None = None + + +_STATE = SettlementPortState() + + +def process_state() -> SettlementPortState: + return _STATE diff --git a/src/backend/core/memory/ttl_sweeper.py b/src/backend/core/memory/ttl_sweeper.py index fa214866..ce70b143 100644 --- a/src/backend/core/memory/ttl_sweeper.py +++ b/src/backend/core/memory/ttl_sweeper.py @@ -79,7 +79,7 @@ async def sweep_expired_memories(scan_limit: int = DEFAULT_SCAN_LIMIT) -> dict: if not settings.memory.enabled: return stats - from core.memory.service import _get_memory + from core.memory.service import get_memory as _get_memory loop = asyncio.get_running_loop() memory = await loop.run_in_executor(None, _get_memory) diff --git a/src/backend/core/memory/weights.py b/src/backend/core/memory/weights.py index 6335de5c..1fc6a885 100644 --- a/src/backend/core/memory/weights.py +++ b/src/backend/core/memory/weights.py @@ -29,7 +29,8 @@ import logging import time -from typing import Dict, Optional, Tuple +from dataclasses import dataclass, field +from typing import Dict, Tuple logger = logging.getLogger(__name__) @@ -46,7 +47,6 @@ MIN_WEIGHT = 0.05 _CACHE_TTL_S = 30.0 -_cache: Dict[str, Tuple[float, Dict[str, float], set]] = {} def _cache_key(user_id: str, workspace_id: str) -> str: @@ -56,9 +56,9 @@ def _cache_key(user_id: str, workspace_id: str) -> str: def invalidate(user_id: str = "", workspace_id: str = "") -> None: """Drop cached overlays so an applied change takes effect on the next turn.""" if not user_id: - _cache.clear() + process_state().cache.clear() return - _cache.pop(_cache_key(user_id, workspace_id), None) + process_state().cache.pop(_cache_key(user_id, workspace_id), None) def load_overlay(user_id: str, workspace_id: str = "default") -> Tuple[Dict[str, float], set]: @@ -67,7 +67,7 @@ def load_overlay(user_id: str, workspace_id: str = "default") -> Tuple[Dict[str, return {}, set() key = _cache_key(user_id, workspace_id) - cached = _cache.get(key) + cached = process_state().cache.get(key) now = time.monotonic() if cached is not None and now - cached[0] < _CACHE_TTL_S: return cached[1], cached[2] @@ -106,7 +106,7 @@ def load_overlay(user_id: str, workspace_id: str = "default") -> Tuple[Dict[str, logger.debug("[memory-weights] overlay unavailable, ignoring: %s", exc) return {}, set() - _cache[key] = (now, weights, superseded) + process_state().cache[key] = (now, weights, superseded) return weights, superseded @@ -155,3 +155,14 @@ def apply_overlay( kept.sort(key=lambda x: x.get(score_key, 0.0), reverse=True) return kept + +@dataclass +class WeightsState: + cache: Dict[str, Tuple[float, Dict[str, float], set]] = field(default_factory=lambda: {}) + + +_STATE = WeightsState() + + +def process_state() -> WeightsState: + return _STATE diff --git a/src/backend/core/models/__init__.py b/src/backend/core/models/__init__.py new file mode 100644 index 00000000..9a02b8ce --- /dev/null +++ b/src/backend/core/models/__init__.py @@ -0,0 +1 @@ +"""Shared model configuration and inference clients.""" diff --git a/src/backend/core/models/admission.py b/src/backend/core/models/admission.py new file mode 100644 index 00000000..0a6b715a --- /dev/null +++ b/src/backend/core/models/admission.py @@ -0,0 +1,21 @@ +"""Task-local request admission callback supplied by the execution layer.""" +from collections.abc import Callable, Iterator +from contextlib import contextmanager +from contextvars import ContextVar + +_check: ContextVar[Callable[[], None] | None] = ContextVar("model_request_admission", default=None) + + +@contextmanager +def request_admission_callback(check: Callable[[], None]) -> Iterator[None]: + token = _check.set(check) + try: + yield + finally: + _check.reset(token) + + +def check_request_admission() -> None: + check = _check.get() + if check is not None: + check() diff --git a/src/backend/core/services/api_protocol_backfill.py b/src/backend/core/models/api_protocol_backfill.py similarity index 92% rename from src/backend/core/services/api_protocol_backfill.py rename to src/backend/core/models/api_protocol_backfill.py index bf171bbe..5e3ec4ba 100644 --- a/src/backend/core/services/api_protocol_backfill.py +++ b/src/backend/core/models/api_protocol_backfill.py @@ -20,7 +20,7 @@ def _needs_probe(extra: dict) -> bool: 可用；那种上游一进智能体循环就卡死。判据变严后，旧结论必须重新验一遍，否则谁也不会 去纠正它。人工设定的值（没有 source 标记）永远不动。 """ - from core.llm.providers.protocol_probe import PROBE_VERSION + from core.models.providers.protocol_probe import PROBE_VERSION if extra.get("api_protocol_source") != "probe": return not extra.get("api_protocol") @@ -30,9 +30,9 @@ def _needs_probe(extra: dict) -> bool: async def backfill_missing_api_protocol() -> int: """探测所有还没记录协议（或结论已过期）的 chat 上游并写库，返回写入的条数。""" from core.db.engine import SessionLocal - from core.db.model_repository import list_providers, update_provider - from core.llm.providers.protocol_probe import PROBE_VERSION, detect_api_protocol - from core.llm.providers.registry import get_spec + from core.models.repository import list_providers, update_provider + from core.models.providers.protocol_probe import PROBE_VERSION, detect_api_protocol + from core.models.providers.registry import get_spec with SessionLocal() as db: pending = [] diff --git a/src/backend/core/llm/chat_models.py b/src/backend/core/models/chat_models.py similarity index 92% rename from src/backend/core/llm/chat_models.py rename to src/backend/core/models/chat_models.py index 16ed87d6..79ad6fb1 100755 --- a/src/backend/core/llm/chat_models.py +++ b/src/backend/core/models/chat_models.py @@ -4,18 +4,15 @@ import logging import os +from dataclasses import dataclass, field from typing import Any, Optional import httpx from agentscope.credential import OpenAICredential from agentscope.model import ChatModelBase, OpenAIChatModel, OpenAIResponseModel -from core.llm.providers.openai_compat import ( - OpenAICompatChatModel, - ReasoningEchoChatFormatter, - _build_chat_template_kwargs, -) -from core.llm.providers.registry import get_spec -from core.llm.providers.vendor_models import build_litellm_model, build_native_model +from core.models.providers.openai_compat import OpenAICompatChatModel, ReasoningEchoChatFormatter, _build_chat_template_kwargs +from core.models.providers.registry import get_spec +from core.models.providers.vendor_models import build_litellm_model, build_native_model from prompts.prompt_config import ModelConfig logger = logging.getLogger(__name__) @@ -30,7 +27,7 @@ # (loop, timeout) keeps keep-alive connections warm across turns while staying # correct under pytest-asyncio, which runs each test on its own loop and would # otherwise inherit a pool bound to a closed one. -_HTTP_CLIENTS: dict[tuple[int, float], httpx.AsyncClient] = {} + # Keep enough idle sockets for concurrent runs (main agent + subagents + memory # extraction all talk to the same gateway) without letting an idle pool pin @@ -47,9 +44,9 @@ async def close_loop_http_clients() -> None: import asyncio loop_id = id(asyncio.get_running_loop()) - keys = [key for key in list(_HTTP_CLIENTS) if key[0] == loop_id] + keys = [key for key in list(process_state().HTTP_CLIENTS) if key[0] == loop_id] for key in keys: - client = _HTTP_CLIENTS.pop(key) + client = process_state().HTTP_CLIENTS.pop(key) await client.aclose() @@ -77,14 +74,14 @@ def _make_http_client( hooks = {} identity = "" if desktop_reference: - from core.services.desktop_cloud_bridge import _state_fingerprint - from core.services.desktop_model_credentials import request_hook + from core.harness.registry.execution_identity import state_fingerprint as _state_fingerprint + from core.harness.registry.model_credentials import request_hook hook, captured = request_hook(desktop_reference, base_url) identity = (desktop_reference, base_url, _state_fingerprint(captured)) hooks = {"request": [hook]} key = (loop_key, base_t, identity) - client = _HTTP_CLIENTS.get(key) + client = process_state().HTTP_CLIENTS.get(key) if client is not None and not client.is_closed: return client @@ -99,7 +96,7 @@ def _make_http_client( event_hooks=hooks, ) if loop_key: - _HTTP_CLIENTS[key] = client + process_state().HTTP_CLIENTS[key] = client return client @@ -183,7 +180,7 @@ def wants_responses(spec, api_protocol: Optional[str]) -> bool: ``protocol_probe`` and stored in ``extra_config.api_protocol``, never guessed here from the vendor name. """ - from core.llm.providers.protocol_probe import PROTOCOL_CHAT, PROTOCOL_RESPONSES + from core.models.providers.protocol_probe import PROTOCOL_CHAT, PROTOCOL_RESPONSES if not spec.speaks_responses: return False @@ -214,7 +211,7 @@ def _make_openai_responses( reasoning_effort_is_wire: bool = False, ): """Build the Responses-protocol twin of ``_make_openai_compatible``.""" - from core.llm.responses_models import OpenAICompatResponsesModel + from core.models.responses import OpenAICompatResponsesModel # The thinking switch keeps the same mechanism as the chat line: vLLM-style servers # read chat_template_kwargs on both protocols, so a model behaves identically no @@ -291,7 +288,7 @@ def make_chat_model( spec = get_spec(provider) if not context_size or context_size <= 0: - from core.llm.context_manager import resolve_model_context_window + from core.models.context_window import resolve_model_context_window context_size = resolve_model_context_window(model) @@ -355,7 +352,7 @@ def make_chat_model( def _resolve_or_dummy(role_key: str): """Resolve config from DB, return None if not available.""" try: - from core.services.model_config import ModelConfigService + from core.models.config import ModelConfigService return ModelConfigService.get_instance().resolve(role_key) except Exception as exc: @@ -373,7 +370,7 @@ def build_model_for_mode( same way rather than by three copies of the same flag arithmetic. """ disable_thinking = mode in ("fast", "turbo") - from core.llm.reasoning_effort import resolve_reasoning_effort + from core.models.reasoning_effort import resolve_reasoning_effort reasoning_effort = resolve_reasoning_effort(resolved.extra, mode) model = make_chat_model( @@ -468,3 +465,15 @@ def get_summarize_model(cfg: ModelConfig | None = None) -> ChatModelBase: "close_loop_http_clients", "wants_responses", ] + + +@dataclass +class ChatModelsState: + HTTP_CLIENTS: dict[tuple[int, float], httpx.AsyncClient] = field(default_factory=lambda: {}) + + +_STATE = ChatModelsState() + + +def process_state() -> ChatModelsState: + return _STATE diff --git a/src/backend/core/models/completions/__init__.py b/src/backend/core/models/completions/__init__.py new file mode 100644 index 00000000..3767d09b --- /dev/null +++ b/src/backend/core/models/completions/__init__.py @@ -0,0 +1 @@ +"""Single-turn completion, classification and summarization clients.""" diff --git a/src/backend/core/llm/classifier.py b/src/backend/core/models/completions/classifier.py similarity index 84% rename from src/backend/core/llm/classifier.py rename to src/backend/core/models/completions/classifier.py index 7b8ce610..c31f171d 100644 --- a/src/backend/core/llm/classifier.py +++ b/src/backend/core/models/completions/classifier.py @@ -2,12 +2,13 @@ from __future__ import annotations import logging +from dataclasses import dataclass from typing import Dict, List, Optional from core.config.settings import settings # 分类与摘要共用 summarizer 角色，所以共用同一个解析器，避免两份会漂移的副本。 -from core.llm.summarizer import _resolve_summarizer_endpoint -from core.llm.single_turn import complete +from core.models.completions.summarizer import _resolve_summarizer_endpoint +from core.models.completions.single_turn import complete _LOGGER = logging.getLogger(__name__) @@ -86,11 +87,21 @@ async def classify_conversation( # Singleton instance -_classifier: ConversationClassifier | None = None def get_classifier() -> ConversationClassifier: - global _classifier - if _classifier is None: - _classifier = ConversationClassifier() - return _classifier + if process_state().classifier is None: + process_state().classifier = ConversationClassifier() + return process_state().classifier + + +@dataclass +class ClassifierState: + classifier: ConversationClassifier | None = None + + +_STATE = ClassifierState() + + +def process_state() -> ClassifierState: + return _STATE diff --git a/src/backend/core/llm/single_turn.py b/src/backend/core/models/completions/single_turn.py similarity index 96% rename from src/backend/core/llm/single_turn.py rename to src/backend/core/models/completions/single_turn.py index 8f5b7d69..51cd52f7 100644 --- a/src/backend/core/llm/single_turn.py +++ b/src/backend/core/models/completions/single_turn.py @@ -16,8 +16,8 @@ import httpx -from core.llm._distill_shared import strip_think_blocks -from core.llm.providers.protocol_probe import PROTOCOL_RESPONSES +from core.infra.model_output import strip_think_blocks +from core.models.providers.protocol_probe import PROTOCOL_RESPONSES _TEXT_ITEM = "output_text" @@ -45,8 +45,8 @@ def from_resolved(cls, cfg: Any) -> "Endpoint": @property def speaks_responses(self) -> bool: - from core.llm.chat_models import wants_responses - from core.llm.providers.registry import get_spec + from core.models.chat_models import wants_responses + from core.models.providers.registry import get_spec return wants_responses(get_spec(self.provider), self.api_protocol) @@ -90,7 +90,7 @@ def build_request( 凭据统一走桌面模型凭据模块：本机执行面的模型行存的是账号绑定的网关引用而不是密钥， 非引用的普通密钥原样加 Bearer，所以云端和桌面共用这一条路径。 """ - from core.services.desktop_model_credentials import prepare_request_headers + from core.harness.registry.model_credentials import prepare_request_headers base = endpoint.base_url.rstrip("/") headers = prepare_request_headers(endpoint.api_key, endpoint.base_url) diff --git a/src/backend/core/llm/summarizer.py b/src/backend/core/models/completions/summarizer.py similarity index 86% rename from src/backend/core/llm/summarizer.py rename to src/backend/core/models/completions/summarizer.py index 20190f1c..3f9b27b7 100644 --- a/src/backend/core/llm/summarizer.py +++ b/src/backend/core/models/completions/summarizer.py @@ -2,10 +2,11 @@ from __future__ import annotations import logging +from dataclasses import dataclass from typing import Dict, List, Optional from core.config.settings import settings -from core.llm.single_turn import Endpoint, complete +from core.models.completions.single_turn import Endpoint, complete _LOGGER = logging.getLogger(__name__) @@ -13,7 +14,7 @@ def _resolve_summarizer_endpoint() -> Optional[Endpoint]: """Resolve the summarizer role's endpoint from DB.""" try: - from core.services.model_config import ModelConfigService + from core.models.config import ModelConfigService cfg = ModelConfigService.get_instance().resolve("summarizer") if cfg and cfg.base_url and cfg.api_key and cfg.model_name: return Endpoint.from_resolved(cfg) @@ -93,11 +94,21 @@ async def summarize_conversation( # Singleton instance -_summarizer: ConversationSummarizer | None = None def get_summarizer() -> ConversationSummarizer: - global _summarizer - if _summarizer is None: - _summarizer = ConversationSummarizer() - return _summarizer + if process_state().summarizer is None: + process_state().summarizer = ConversationSummarizer() + return process_state().summarizer + + +@dataclass +class SummarizerState: + summarizer: ConversationSummarizer | None = None + + +_STATE = SummarizerState() + + +def process_state() -> SummarizerState: + return _STATE diff --git a/src/backend/core/services/model_config.py b/src/backend/core/models/config.py similarity index 99% rename from src/backend/core/services/model_config.py rename to src/backend/core/models/config.py index 75ead29a..e7a08d6f 100644 --- a/src/backend/core/services/model_config.py +++ b/src/backend/core/models/config.py @@ -165,7 +165,7 @@ def _maybe_refresh(self) -> None: @staticmethod def _provider_to_resolved(provider: ModelProvider) -> ResolvedModelConfig: - from core.llm.providers.registry import get_spec, split_provider_extra + from core.models.providers.registry import get_spec, split_provider_extra extra = dict(provider.extra_config or {}) ctx_len = _optional_int(extra.pop("context_length", None)) or 0 diff --git a/src/backend/core/models/constants.py b/src/backend/core/models/constants.py new file mode 100644 index 00000000..8ddc968b --- /dev/null +++ b/src/backend/core/models/constants.py @@ -0,0 +1,5 @@ +"""Shared model/file contracts without execution-layer dependencies.""" +from __future__ import annotations +PROVIDER_CONTEXT_META_KEY = "_harness_context_item" + +IMAGE_TOKEN_RESERVE = 1_844 diff --git a/src/backend/core/llm/context_manager.py b/src/backend/core/models/context_window.py similarity index 97% rename from src/backend/core/llm/context_manager.py rename to src/backend/core/models/context_window.py index af696e49..0bd1241f 100644 --- a/src/backend/core/llm/context_manager.py +++ b/src/backend/core/models/context_window.py @@ -44,7 +44,7 @@ def resolve_model_context_window(model_name: str) -> int: ) try: - from core.services.model_config import ModelConfigService + from core.models.config import ModelConfigService ctx_len = ModelConfigService.get_instance().get_context_length_by_model_name(model_name) except ValueError: diff --git a/src/backend/core/models/embedding.py b/src/backend/core/models/embedding.py new file mode 100644 index 00000000..6961e8fe --- /dev/null +++ b/src/backend/core/models/embedding.py @@ -0,0 +1,123 @@ +"""Call the configured embedding service.""" +from __future__ import annotations +from core.models import embedding_state +import os +import requests +import logging +logger = logging.getLogger(__name__) + +def resolve_embed_config() -> tuple[str, str, str]: + """Resolve embedding config from DB, with env fallback.""" + try: + from core.models.config import ModelConfigService + + cfg = ModelConfigService.get_instance().resolve("embedding") + if cfg: + return cfg.base_url.rstrip("/"), cfg.model_name, cfg.api_key + except Exception as fallback_error: + from core.infra.failures import record_best_effort_failure + record_best_effort_failure('core.models.embedding:resolve_embed_config', fallback_error) + return ( + os.getenv("MEM0_EMBED_URL", "").rstrip("/"), + os.getenv("MEM0_EMBED_MODEL", ""), + os.getenv("MEM0_EMBED_API_KEY", ""), + ) + + +def embed_request( + url: str, + headers: dict, + inputs: list[str], + embed_model: str, + *, + timeout: float | None = None, +) -> list[list[float]]: + """Post a single embedding request and return vectors ordered by input index.""" + payload = {"input": inputs, "model": embed_model} + resp = requests.post( + url, + headers=headers, + json=payload, + timeout=timeout or embedding_state.EMBED_TIMEOUT, + ) + resp.raise_for_status() + data = resp.json() + return [item["embedding"] for item in sorted(data["data"], key=lambda x: x["index"])] + + +def embed_text(text: str, *, timeout: float | None = None) -> list[float]: + """Call the configured embedding service and return a dense vector.""" + embed_url, embed_model, api_key = resolve_embed_config() + + if not embed_url: + raise RuntimeError("Embedding model is not configured") + + url = f"{embed_url}/embeddings" + from core.harness.registry.model_credentials import prepare_request_headers + headers = prepare_request_headers(api_key, embed_url) + vector = embed_request(url, headers, [text], embed_model, timeout=timeout)[0] + if vector: + embedding_state.process_state().DETECTED_DIM_CACHE[(embed_url, embed_model)] = len(vector) + return vector + + +def embed_batch(texts: list[str]) -> list[list[float]]: + """Batch embed multiple texts, splitting into sub-batches with one-by-one fallback. + + A large document is split into hundreds or thousands of chunks. Sending the whole batch at once to a slow embedding + model (e.g. 8B, ~0.45s/item serially) would time out the single request and fail the whole document's vectorization. So we + split into sub-batches of _EMBED_BATCH_SIZE and request them one sub-batch at a time; if a sub-batch fails, we downgrade to + embedding one item at a time within that sub-batch, preserving as many other chunks as possible so one timeout doesn't drag + down the whole document. Only if the one-by-one attempt still fails do we re-raise (treated as a true failure). + """ + embed_url, embed_model, api_key = resolve_embed_config() + + if not embed_url: + raise RuntimeError("Embedding model is not configured") + + url = f"{embed_url}/embeddings" + from core.harness.registry.model_credentials import prepare_request_headers + headers = prepare_request_headers(api_key, embed_url) + + results: list[list[float]] = [] + for start in range(0, len(texts), embedding_state.EMBED_BATCH_SIZE): + sub = texts[start : start + embedding_state.EMBED_BATCH_SIZE] + try: + results.extend(embed_request(url, headers, sub, embed_model)) + except Exception as exc: + logger.warning( + "Sub-batch embed failed for [%d:%d] (%s); falling back to one-by-one", + start, + start + len(sub), + exc, + ) + for one in sub: + results.extend(embed_request(url, headers, [one], embed_model)) + return results + + +def detect_embed_dim() -> int: + """Probe the real output dimension of the currently chosen embedding model (one /embeddings call, take the vector length). + + The result is cached by (embed_url, embed_model) to avoid repeated probing on every index; on probe failure (endpoint not + configured/unreachable) it falls back to MEM0_EMBED_DIMS. This way the table dimension auto-follows after switching the + embedding model, no manual filling needed. + """ + embed_url, embed_model, _ = resolve_embed_config() + key = (embed_url, embed_model) + cached = embedding_state.process_state().DETECTED_DIM_CACHE.get(key) + if cached: + return cached + try: + dim = len(embed_text("dimension probe")) + if dim > 0: + embedding_state.process_state().DETECTED_DIM_CACHE[key] = dim + logger.info("Auto-detected embedding dim=%d (model=%s)", dim, embed_model or "?") + return dim + except Exception as exc: + logger.warning( + "Embedding dim auto-detect failed (%s); falling back to MEM0_EMBED_DIMS=%d", + exc, + embedding_state.EMBED_DIMS, + ) + return embedding_state.EMBED_DIMS diff --git a/src/backend/core/models/embedding_state.py b/src/backend/core/models/embedding_state.py new file mode 100644 index 00000000..4b04376d --- /dev/null +++ b/src/backend/core/models/embedding_state.py @@ -0,0 +1,20 @@ +"""Embedding request limits and dimension cache, shared across consumers.""" +from __future__ import annotations +import os +from dataclasses import dataclass, field +EMBED_DIMS = int(os.getenv("MEM0_EMBED_DIMS", "1024")) + +EMBED_BATCH_SIZE = max(1, int(os.getenv("KB_EMBED_BATCH_SIZE", "32"))) +EMBED_TIMEOUT = max(1, int(os.getenv("KB_EMBED_TIMEOUT", "120"))) + + +@dataclass +class EmbeddingState: + DETECTED_DIM_CACHE: dict[tuple[str, str], int] = field(default_factory=lambda: {}) + + +_STATE = EmbeddingState() + + +def process_state() -> EmbeddingState: + return _STATE diff --git a/src/backend/core/llm/failover.py b/src/backend/core/models/failover.py similarity index 94% rename from src/backend/core/llm/failover.py rename to src/backend/core/models/failover.py index 239be5af..4dffc1ed 100644 --- a/src/backend/core/llm/failover.py +++ b/src/backend/core/models/failover.py @@ -28,13 +28,14 @@ import logging import time from contextvars import ContextVar +from dataclasses import dataclass, field from typing import Any, AsyncGenerator, Callable, Optional from agentscope.message import Msg from agentscope.model import ChatModelBase, ChatResponse from agentscope.tool._types import ToolChoice -from core.llm.providers._image_tokens import ImageTokenCountingMixin -from core.llm.tool_call_identity import ToolCallIdentityMixin +from core.models.providers._image_tokens import ImageTokenCountingMixin +from core.models.tool_call_identity import ToolCallIdentityMixin logger = logging.getLogger(__name__) @@ -43,7 +44,6 @@ # that a dead upstream is not re-probed by every concurrent run. PROVIDER_COOLDOWN_SECONDS = 120.0 -_cooldown: dict[str, float] = {} # Which candidate answered the call in flight. The model instance is cached and # shared by every concurrent run, so "who is answering" cannot live on it: one @@ -53,21 +53,21 @@ def mark_provider_down(provider_id: str) -> None: if provider_id: - _cooldown[provider_id] = time.monotonic() + PROVIDER_COOLDOWN_SECONDS + process_state().cooldown[provider_id] = time.monotonic() + PROVIDER_COOLDOWN_SECONDS def provider_is_cooling(provider_id: str) -> bool: - until = _cooldown.get(provider_id) + until = process_state().cooldown.get(provider_id) if until is None: return False if time.monotonic() >= until: - _cooldown.pop(provider_id, None) + process_state().cooldown.pop(provider_id, None) return False return True def clear_cooldowns() -> None: - _cooldown.clear() + process_state().cooldown.clear() def provider_is_unusable(exc: BaseException) -> bool: @@ -112,7 +112,7 @@ def carries_media(messages: list[Msg]) -> bool: Media rides in ``DataBlock``s — either directly on a message (an upload the middleware passed through) or folded into a ``ToolResultBlock``'s output - (see ``core.llm.tool_result_media``). Both places are checked because both + (see ``core.models.tool_result_media``). Both places are checked because both reach the wire. """ for msg in messages or []: @@ -260,7 +260,7 @@ def _reads_media(self, index: int) -> bool: """Whether candidate *index* may be sent a request carrying media. The primary needs no test: the turn's vision mode is resolved against - it (``core.vision.resolve_vision_mode``), so a tool only ever returns + it (``core.models.vision.resolve_vision_mode``), so a tool only ever returns pixels — and an upload only ever passes through untranscribed — when the primary reads them natively. Fallbacks were chosen by weight and context window, neither of which says anything about media, so each is @@ -268,7 +268,7 @@ def _reads_media(self, index: int) -> bool: """ if index == 0: return True - from core.vision import model_supports_vision + from core.models.vision import model_supports_vision return model_supports_vision(self._fallback_specs[index - 1]) @@ -381,8 +381,8 @@ def with_failover( ``resolved`` is the ``ResolvedModelConfig`` *primary* was built from; it is excluded from its own fallback list. """ - from core.llm.chat_models import build_model_for_mode - from core.services.model_config import ModelConfigService + from core.models.chat_models import build_model_for_mode + from core.models.config import ModelConfigService try: chain = ModelConfigService.get_instance().resolve_failover_chain(resolved) @@ -402,3 +402,15 @@ def with_failover( fallbacks, lambda cfg: build_model_for_mode(cfg, mode=mode, stream=primary.stream), ) + + +@dataclass +class FailoverState: + cooldown: dict[str, float] = field(default_factory=lambda: {}) + + +_STATE = FailoverState() + + +def process_state() -> FailoverState: + return _STATE diff --git a/src/backend/core/db/model_import_validation.py b/src/backend/core/models/import_validation.py similarity index 93% rename from src/backend/core/db/model_import_validation.py rename to src/backend/core/models/import_validation.py index 2ed6e7f4..ad39202d 100644 --- a/src/backend/core/db/model_import_validation.py +++ b/src/backend/core/models/import_validation.py @@ -1,10 +1,10 @@ """Validate the resulting model/role graph before an import mutates any row.""" from core.db.models import ModelProvider, ModelRoleAssignment -from core.llm.providers.registry import get_spec +from core.models.providers.registry import get_spec def validate_import_graph(db, data, overwrite): - from core.db.model_repository import ROLE_DEFINITIONS + from core.models.repository import ROLE_DEFINITIONS providers = {row.provider_id: {"provider_type": row.provider_type, "provider": row.provider, "extra_config": row.extra_config or {}} for row in db.query(ModelProvider).all()} for row in data.get("providers", []): key = row.get("provider_id") diff --git a/src/backend/core/models/providers/__init__.py b/src/backend/core/models/providers/__init__.py new file mode 100644 index 00000000..90e72509 --- /dev/null +++ b/src/backend/core/models/providers/__init__.py @@ -0,0 +1,3 @@ +"""Multi-vendor model provider support: registry + per-engine model implementations.""" + +from core.models.providers.registry import PROVIDER_SPECS, ProviderField, ProviderSpec, get_spec, is_known, list_specs, split_provider_extra, to_frontend_schema, validate_payload diff --git a/src/backend/core/llm/providers/_image_tokens.py b/src/backend/core/models/providers/_image_tokens.py similarity index 96% rename from src/backend/core/llm/providers/_image_tokens.py rename to src/backend/core/models/providers/_image_tokens.py index e7b7d1a9..b35339f6 100644 --- a/src/backend/core/llm/providers/_image_tokens.py +++ b/src/backend/core/models/providers/_image_tokens.py @@ -2,7 +2,7 @@ from agentscope.message import DataBlock, Msg, ToolResultBlock -from core.llm.context_ir import IMAGE_TOKEN_RESERVE +from core.models.constants import IMAGE_TOKEN_RESERVE class ImageTokenCountingMixin: diff --git a/src/backend/core/llm/providers/context_probe.py b/src/backend/core/models/providers/context_probe.py similarity index 99% rename from src/backend/core/llm/providers/context_probe.py rename to src/backend/core/models/providers/context_probe.py index 90320b68..8c3e6569 100644 --- a/src/backend/core/llm/providers/context_probe.py +++ b/src/backend/core/models/providers/context_probe.py @@ -322,7 +322,7 @@ async def discover_context_length( result.notes.append("模型名为空，无法探测") return result - from core.llm.providers.registry import get_spec + from core.models.providers.registry import get_spec try: spec = get_spec(provider) diff --git a/src/backend/core/llm/providers/openai_compat.py b/src/backend/core/models/providers/openai_compat.py similarity index 96% rename from src/backend/core/llm/providers/openai_compat.py rename to src/backend/core/models/providers/openai_compat.py index c9d2b34b..6b99fa76 100644 --- a/src/backend/core/llm/providers/openai_compat.py +++ b/src/backend/core/models/providers/openai_compat.py @@ -24,20 +24,20 @@ ) from agentscope.model import ChatResponse, OpenAIChatModel from agentscope.tool._types import ToolChoice -from core.llm.context_adapter import PROVIDER_CONTEXT_META_KEY -from core.llm.context_safety import check_request_admission -from core.llm.providers._image_tokens import ImageTokenCountingMixin -from core.llm.providers.stream_retry import _stream_with_bounded_retry -from core.llm.reasoning_replay import ReasoningReplayMixin -from core.llm.tool_call_identity import ToolCallIdentityMixin -from core.llm.tool_result_media import NoDiskToolMediaMixin +from core.models.constants import PROVIDER_CONTEXT_META_KEY +from core.models.admission import check_request_admission +from core.models.providers._image_tokens import ImageTokenCountingMixin +from core.models.providers.stream_retry import _stream_with_bounded_retry +from core.models.reasoning_replay import ReasoningReplayMixin +from core.models.tool_call_identity import ToolCallIdentityMixin +from core.models.tool_result_media import NoDiskToolMediaMixin logger = logging.getLogger(__name__) def _safe_exception_chain(exc: BaseException) -> list[dict[str, str]]: """Identify wrapped SDK failures without serializing messages or requests.""" - from core.capabilities.errors import CapabilityError + from core.harness.capability_errors import CapabilityError known_codes = { "capability_error", @@ -414,7 +414,7 @@ async def _call_api( # type: ignore[override] response = await client.chat.completions.create(**kwargs) logger.info("[wire] response headers in %.0fms", (_perf_counter() - _t_req) * 1000) except Exception as exc: - from core.llm.model_usage import record_provider_failure + from core.models.usage import record_provider_failure logger.warning("Model request failed: exception_chain=%s", _safe_exception_chain(exc)) await record_provider_failure( @@ -432,7 +432,7 @@ async def _call_api( # type: ignore[override] return _stream_with_bounded_retry( self, reissue=lambda: client.chat.completions.create(**kwargs), - parse=lambda started, raw: self._parse_stream_response(started, raw, audio_fmt), + parse=lambda started, raw: self._parse_stream_response(started, raw), model_name=model_name, start_datetime=start_datetime, response=response, diff --git a/src/backend/core/llm/providers/protocol_probe.py b/src/backend/core/models/providers/protocol_probe.py similarity index 100% rename from src/backend/core/llm/providers/protocol_probe.py rename to src/backend/core/models/providers/protocol_probe.py diff --git a/src/backend/core/llm/providers/reasoning_probe.py b/src/backend/core/models/providers/reasoning_probe.py similarity index 97% rename from src/backend/core/llm/providers/reasoning_probe.py rename to src/backend/core/models/providers/reasoning_probe.py index e8eb63bc..48983cc1 100644 --- a/src/backend/core/llm/providers/reasoning_probe.py +++ b/src/backend/core/models/providers/reasoning_probe.py @@ -12,9 +12,9 @@ import httpx -from core.llm.providers.protocol_probe import detect_api_protocol -from core.llm.providers.registry import get_spec -from core.llm.reasoning_effort import EFFORT_KEYS +from core.models.providers.protocol_probe import detect_api_protocol +from core.models.providers.registry import get_spec +from core.models.reasoning_effort import EFFORT_KEYS def _result( diff --git a/src/backend/core/llm/providers/registry.py b/src/backend/core/models/providers/registry.py similarity index 99% rename from src/backend/core/llm/providers/registry.py rename to src/backend/core/models/providers/registry.py index 10ae61a9..6b7b7c39 100644 --- a/src/backend/core/llm/providers/registry.py +++ b/src/backend/core/models/providers/registry.py @@ -208,7 +208,7 @@ def validate_payload(provider_id: str, provider_type: str, extra_config: dict) - if provider_type not in spec.supports_types: return f"{spec.label} 不支持用途 '{provider_type}'（支持：{', '.join(spec.supports_types)}）" extra = extra_config or {} - from core.llm.reasoning_effort import validate_reasoning_config + from core.models.reasoning_effort import validate_reasoning_config error = validate_reasoning_config(extra) if error: diff --git a/src/backend/core/llm/providers/stream_retry.py b/src/backend/core/models/providers/stream_retry.py similarity index 96% rename from src/backend/core/llm/providers/stream_retry.py rename to src/backend/core/models/providers/stream_retry.py index 4e650a84..c23b65f8 100644 --- a/src/backend/core/llm/providers/stream_retry.py +++ b/src/backend/core/models/providers/stream_retry.py @@ -7,7 +7,7 @@ from time import monotonic as _monotonic from time import perf_counter as _perf_counter -from core.llm.context_safety import check_request_admission +from core.models.admission import check_request_admission logger = logging.getLogger(__name__) @@ -65,7 +65,7 @@ async def _stream_with_bounded_retry( except Exception as exc: # noqa: BLE001 - classified below if not _is_retryable_stream_start_error(exc): raise - from core.llm.model_usage import note_provider_retry_started, record_provider_failure + from core.models.usage import note_provider_retry_started, record_provider_failure await record_provider_failure( model, diff --git a/src/backend/core/llm/providers/vendor_models.py b/src/backend/core/models/providers/vendor_models.py similarity index 97% rename from src/backend/core/llm/providers/vendor_models.py rename to src/backend/core/models/providers/vendor_models.py index 5232fcad..b50e00a6 100644 --- a/src/backend/core/llm/providers/vendor_models.py +++ b/src/backend/core/models/providers/vendor_models.py @@ -41,12 +41,12 @@ OpenAIChatModel, ) from agentscope.tool._types import ToolChoice -from core.llm.reasoning_replay import ReasoningReplayMixin -from core.llm.tool_call_identity import ToolCallIdentityMixin -from core.llm.tool_result_media import NoDiskToolMediaMixin +from core.models.reasoning_replay import ReasoningReplayMixin +from core.models.tool_call_identity import ToolCallIdentityMixin +from core.models.tool_result_media import NoDiskToolMediaMixin -from ._image_tokens import ImageTokenCountingMixin -from .registry import ProviderSpec +from core.models.providers._image_tokens import ImageTokenCountingMixin +from core.models.providers.registry import ProviderSpec logger = logging.getLogger(__name__) @@ -338,7 +338,7 @@ async def _call_api( # type: ignore[override] response = await litellm.acompletion(**kwargs) if self.stream: - return self._parse_stream_response(start_datetime, response, "wav") + return self._parse_stream_response(start_datetime, response) return self._parse_completion_response(start_datetime, response, "wav") diff --git a/src/backend/core/llm/reasoning_effort.py b/src/backend/core/models/reasoning_effort.py similarity index 100% rename from src/backend/core/llm/reasoning_effort.py rename to src/backend/core/models/reasoning_effort.py diff --git a/src/backend/core/llm/reasoning_replay.py b/src/backend/core/models/reasoning_replay.py similarity index 100% rename from src/backend/core/llm/reasoning_replay.py rename to src/backend/core/models/reasoning_replay.py diff --git a/src/backend/core/db/model_repository.py b/src/backend/core/models/repository.py similarity index 99% rename from src/backend/core/db/model_repository.py rename to src/backend/core/models/repository.py index 0b6bf28e..2bed4acc 100644 --- a/src/backend/core/db/model_repository.py +++ b/src/backend/core/models/repository.py @@ -11,7 +11,7 @@ from core.db.models import ModelProvider, ModelRoleAssignment from core.db.model_config_revision import bump_revision -from core.llm.providers.registry import get_spec +from core.models.providers.registry import get_spec # ── Predefined roles ───────────────────────────────────────────────────────── @@ -280,7 +280,7 @@ def export_all(db: Session) -> dict: def import_all(db: Session, data: dict, overwrite: bool = True) -> dict: """Import providers + role assignments. Returns counts.""" - from core.db.model_import_validation import validate_import_graph + from core.models.import_validation import validate_import_graph validate_import_graph(db, data, overwrite) imported_providers = 0 imported_roles = 0 diff --git a/src/backend/core/models/reranking.py b/src/backend/core/models/reranking.py new file mode 100644 index 00000000..a1c5b2f8 --- /dev/null +++ b/src/backend/core/models/reranking.py @@ -0,0 +1,63 @@ +"""Call the configured reranking service.""" +from __future__ import annotations +import os +import requests + +def resolve_reranker_config() -> tuple[str, str, str]: + """Resolve reranker config from DB, with env fallback.""" + try: + from core.models.config import ModelConfigService + + cfg = ModelConfigService.get_instance().resolve("reranker") + if cfg: + return cfg.base_url.rstrip("/"), cfg.model_name, cfg.api_key + except Exception as fallback_error: + from core.infra.failures import record_best_effort_failure + record_best_effort_failure('core.models.reranking:resolve_reranker_config', fallback_error) + return ( + os.getenv("RERANKER_URL", "").rstrip("/"), + os.getenv("RERANKER_MODEL", ""), + os.getenv("RERANKER_API_KEY", ""), + ) + + +def is_reranker_configured() -> bool: + """Check if a reranker model endpoint is configured.""" + reranker_url, reranker_model, _ = resolve_reranker_config() + return bool(reranker_url and reranker_model) + + +def rerank( + query: str, + documents: list[str], + top_n: int | None = None, + *, + timeout: float = 30, +) -> list[dict]: + """Call the configured reranker endpoint (OpenAI-compatible /rerank). + + Returns a list of {"index": int, "relevance_score": float} sorted by score descending. + """ + reranker_url, reranker_model, reranker_key = resolve_reranker_config() + + if not reranker_url or not reranker_model: + raise RuntimeError("Reranker is not configured") + + url = f"{reranker_url}/rerank" + from core.harness.registry.model_credentials import prepare_request_headers + headers = prepare_request_headers(reranker_key, reranker_url) + payload: dict = { + "model": reranker_model, + "query": query, + "documents": documents, + } + if top_n is not None: + payload["top_n"] = top_n + + resp = requests.post(url, headers=headers, json=payload, timeout=timeout) + resp.raise_for_status() + data = resp.json() + + # Response format: {"results": [{"index": 0, "relevance_score": 0.95}, ...]} + results = data.get("results", []) + return sorted(results, key=lambda x: x.get("relevance_score", 0), reverse=True) diff --git a/src/backend/core/llm/responses_models.py b/src/backend/core/models/responses.py similarity index 80% rename from src/backend/core/llm/responses_models.py rename to src/backend/core/models/responses.py index 1915966f..59b90981 100644 --- a/src/backend/core/llm/responses_models.py +++ b/src/backend/core/models/responses.py @@ -26,11 +26,11 @@ from agentscope.message import Msg, ThinkingBlock, ToolCallBlock from agentscope.model import ChatResponse, OpenAIResponseModel from agentscope.tool._types import ToolChoice -from core.llm.context_safety import check_request_admission -from core.llm.providers._image_tokens import ImageTokenCountingMixin -from core.llm.reasoning_replay import ReasoningReplayMixin -from core.llm.tool_call_identity import ToolCallIdentityMixin -from core.llm.tool_result_media import ResponsesToolMediaMixin +from core.models.admission import check_request_admission +from core.models.providers._image_tokens import ImageTokenCountingMixin +from core.models.reasoning_replay import ReasoningReplayMixin +from core.models.tool_call_identity import ToolCallIdentityMixin +from core.models.tool_result_media import NoDiskToolMediaMixin logger = logging.getLogger(__name__) @@ -42,14 +42,6 @@ # 只有这两个终态事件会带上完整的 response 对象（含 output 数组）。 _TERMINAL_EVENTS = ("response.completed", "response.incomplete") -# Responses 有两条思考通道：``reasoning_summary_text`` 是经过删减的摘要（OpenAI 返回 -# 的那一条），``reasoning_text`` 是模型的思考原文（自建推理服务返回的那一条）。 -# AgentScope 的解析器只认前者，后者整条被丢掉——实测自建 deepseekv4-flash-vision 一轮 -# 发 144 帧 reasoning_text.delta，界面上就是一个空的思考块。两条通道对界面是同一件东西， -# 所以在事件入口把后者翻成前者，流式逐字、累积、进历史全部照旧由父类处理。 -_REASONING_TEXT_DELTA = "response.reasoning_text.delta" -_REASONING_SUMMARY_DELTA = "response.reasoning_summary_text.delta" - # 对齐 codex：让上游把思考加密串交还给我们自己保管，服务端不留会话状态。 _REASONING_INCLUDE = ["reasoning.encrypted_content"] @@ -103,24 +95,10 @@ def _as_dict(item: Any) -> dict[str, Any]: return {} -class _ReasoningTextDelta: - """把思考原文增量伪装成解析器认得的摘要增量。 - - 父类只读 ``type`` 和 ``delta`` 两个字段（``response`` 用于取 response_id，这类事件 - 本来就没有），所以一个同形的小对象足够，不必构造 SDK 的类型。 - """ - - __slots__ = ("type", "delta", "response", "summary_index") - - def __init__(self, delta: str) -> None: - self.type = _REASONING_SUMMARY_DELTA - self.delta = delta - self.response = None - self.summary_index = 0 - - class _ReasoningItemCapture: - """透传上游 SSE，旁路留下完整 reasoning 条目，并接上思考原文通道。 + """透传上游 SSE，旁路留下完整 reasoning 条目。 + + AgentScope 原生解析思考原文与摘要；保留事件身份及分段索引，避免合并不同条目。 旁路而非重写解析器：那 130 行逐事件拼装跟着 SDK 版本走，抄过来就得跟着维护。 """ @@ -129,6 +107,13 @@ def __init__(self, stream: Any) -> None: self._stream = stream self.items: list[dict[str, Any]] = [] + async def __aenter__(self): + await self._stream.__aenter__() + return self + + async def __aexit__(self, exc_type, exc_value, traceback): + return await self._stream.__aexit__(exc_type, exc_value, traceback) + def __aiter__(self): # noqa: ANN204 return self._iter() @@ -137,11 +122,6 @@ async def _iter(self): # noqa: ANN202 kind = getattr(event, "type", None) if kind in _TERMINAL_EVENTS: self._collect(getattr(event, "response", None)) - elif kind == _REASONING_TEXT_DELTA: - delta = getattr(event, "delta", None) - if delta: - yield _ReasoningTextDelta(delta) - continue yield event def _collect(self, response: Any) -> None: @@ -216,39 +196,8 @@ def _restore_reasoning_items(msgs: list[Msg], items: list[dict[str, Any]]) -> li return restored -def _repair_tool_output_call_ids( - msgs: list[Msg], items: list[dict[str, Any]] -) -> list[dict[str, Any]]: - """把工具结果的 ``call_id`` 换回发起那次调用时上游给的那一个。 - - Responses 线上一次函数调用有**两个**互不相同的 id：条目 id（``function_call.id``） - 与调用 id（``function_call.call_id``），回传结果时必须按后者配对。AgentScope 建 - ``ToolResultBlock`` 时只带上了条目 id（``id=tool_call.id``），格式化又直接拿它当 - ``function_call_output.call_id`` 发出去，于是两边对不上。 - - DeepSeek 官方端点正是两者不同（条目 id 是 uuid，调用 id 形如 ``call_00_…``），整轮 - 请求会被打回 ``No tool call found for tool output with call_id …``；只要模型调了工具， - 这一轮就必然失败。调用块自己同时带着这两个 id，这里按它把结果侧改正。 - - 历史回放时旧块只剩条目 id，两侧都退回同一个值，配对依然自洽。 - """ - call_ids = { - block.id: str(getattr(block, "call_id", None) or "") - for msg in msgs - for block in msg.content or [] - if isinstance(block, ToolCallBlock) - } - for item in items: - if not isinstance(item, dict) or item.get("type") != "function_call_output": - continue - call_id = call_ids.get(str(item.get("call_id") or "")) - if call_id: - item["call_id"] = call_id - return items - - class ResponsesReplayFormatter( - ReasoningReplayMixin, ResponsesToolMediaMixin, OpenAIResponseFormatter + ReasoningReplayMixin, NoDiskToolMediaMixin, OpenAIResponseFormatter ): """Responses 线格式化：思考按上游原样回传，工具媒体绝不落盘。 @@ -257,7 +206,7 @@ class ResponsesReplayFormatter( 造成当前模型的推理。 """ - def _format_response_data_block(self, block: Any, role: str = "user") -> Any: + def _format_response_data_block(self, block: Any) -> Any: """给每个 ``input_image`` 补上 ``detail``——严格实现少了它直接 400。 AgentScope 只产出 ``type`` + ``image_url``，而 OpenAI 的 @@ -269,7 +218,7 @@ def _format_response_data_block(self, block: Any, role: str = "user") -> Any: 管这条线上的每张图，不只工具返回的：用户上传的图走的是同一个方法。 ``setdefault`` 是为了不覆盖上游将来自己给出的值。 """ - item = super()._format_response_data_block(block, role) # type: ignore[misc] + item = super()._format_response_data_block(block) # type: ignore[misc] if isinstance(item, dict) and item.get("type") == "input_image": item.setdefault("detail", "auto") return item @@ -280,7 +229,7 @@ async def format(self, msgs: list[Msg]) -> list[dict[str, Any]]: for item in items: if "role" in item and "content" in item: item.setdefault("type", "message") - return _restore_reasoning_items(msgs, _repair_tool_output_call_ids(msgs, items)) + return _restore_reasoning_items(msgs, items) class OpenAICompatResponsesModel( @@ -368,8 +317,8 @@ async def _call_api( # type: ignore[override] tool_choice: ToolChoice | None = None, **generate_kwargs: Any, ) -> ChatResponse | AsyncGenerator[ChatResponse, None]: - from core.llm.providers.openai_compat import _dump_wire_payload, _slim_tool_schemas - from core.llm.providers.stream_retry import _stream_with_bounded_retry + from core.models.providers.openai_compat import _dump_wire_payload, _slim_tool_schemas + from core.models.providers.stream_retry import _stream_with_bounded_retry client = self._build_client() formatted_input = await self.formatter.format(messages) @@ -413,7 +362,7 @@ async def _issue(): # noqa: ANN202 try: response = await _issue() except Exception as exc: - from core.llm.model_usage import record_provider_failure + from core.models.usage import record_provider_failure logger.warning("Responses request failed: %s: %s", type(exc).__name__, exc) await record_provider_failure( @@ -449,7 +398,7 @@ async def _publish_context_rewrite( """ import inspect - from core.llm.context_adapter import PROVIDER_CONTEXT_META_KEY + from core.models.constants import PROVIDER_CONTEXT_META_KEY listener = self._context_rewrite_listener if listener is not None: diff --git a/src/backend/core/services/user_model_selection.py b/src/backend/core/models/selection.py similarity index 63% rename from src/backend/core/services/user_model_selection.py rename to src/backend/core/models/selection.py index 52dbc347..49d0dbdc 100644 --- a/src/backend/core/services/user_model_selection.py +++ b/src/backend/core/models/selection.py @@ -10,57 +10,15 @@ from sqlalchemy.orm import Session -from core.auth.capabilities import resolve_user_capabilities from core.db.models import ModelProvider, ModelRoleAssignment -from core.services.model_config import ModelConfigService -from core.llm.reasoning_effort import reasoning_capabilities +from core.models.config import ModelConfigService +from core.models.reasoning_effort import reasoning_capabilities -class UserModelSelectionError(ValueError): - """Raised when a requested user-selectable model is not allowed.""" -def resolve_user_model_provider_id( - db: Session, - provider_id: Optional[str], - *, - user_id: Optional[str] = None, - can_switch_model: Optional[bool] = None, -) -> Optional[str]: - """Return an allowed provider id for the current request. - - Disabled switch means "ignore the browser-provided override" rather than - "fail the chat"; enabled switch validates against active chat providers. - """ - pid = (provider_id or "").strip() - if not pid: - return None - - enabled = ( - can_switch_model - if can_switch_model is not None - else bool(user_id and user_can_switch_model(db, user_id)) - ) - if not enabled: - return None - - provider = ( - db.query(ModelProvider) - .filter( - ModelProvider.provider_id == pid, - ModelProvider.provider_type == "chat", - ModelProvider.is_active == True, # noqa: E712 - ) - .first() - ) - if provider is None: - raise UserModelSelectionError("所选模型不可用，请刷新页面后重试。") - return pid -def user_can_switch_model(db: Session, user_id: str) -> bool: - """Return whether the user may switch models in the chat input.""" - return bool(resolve_user_capabilities(db, user_id).get("can_switch_model")) def resolve_effective_chat_model_name( diff --git a/src/backend/core/models/selection_hook.py b/src/backend/core/models/selection_hook.py new file mode 100644 index 00000000..05a1f860 --- /dev/null +++ b/src/backend/core/models/selection_hook.py @@ -0,0 +1,20 @@ +"""Framework-neutral model choice before a run's context is assembled.""" +from core.harness.hooks import Decision, FailureMode, HookSpec, HookStage, Invocation + + +def select_model(invocation: Invocation) -> Decision: + runtime = invocation.data.get("runtime", {}) + if runtime.get("model_pinned"): + return Decision() + mode = runtime.get("chat_mode") + if mode not in ("turbo", "fast", "low", "medium", "high", "xhigh", "max"): + mode = "fast" + return Decision.modify({"model_selection": { + "provider_id": str(runtime.get("model_provider_id") or ""), "mode": mode, + }}) + + +def model_selection_hook() -> HookSpec: + return HookSpec("model_selection", HookStage.BEFORE_RUN, select_model, + priority=10, failure_mode=FailureMode.OPEN, + mutable_fields=frozenset({"model_selection"})) diff --git a/src/backend/core/llm/tool_call_identity.py b/src/backend/core/models/tool_call_identity.py similarity index 100% rename from src/backend/core/llm/tool_call_identity.py rename to src/backend/core/models/tool_call_identity.py diff --git a/src/backend/core/llm/tool_result_media.py b/src/backend/core/models/tool_result_media.py similarity index 69% rename from src/backend/core/llm/tool_result_media.py rename to src/backend/core/models/tool_result_media.py index 55b26383..00f067d4 100644 --- a/src/backend/core/llm/tool_result_media.py +++ b/src/backend/core/models/tool_result_media.py @@ -8,14 +8,13 @@ - 那个临时文件落在 **backend 容器**，而智能体的 shell 跑在**沙箱容器**里，模型 照着路径去读永远读不到——一条指向不存在位置的死链，还每调用一次泄漏一份文件。 - 图被换成文案就是静默降级：调用方已经按「这个模型能看图」注册了工具 - （见 :func:`core.vision.resolve_vision_mode`），到这一层再悄悄抽掉像素， + （见 :func:`core.models.vision.resolve_vision_mode`），到这一层再悄悄抽掉像素， 没有任何日志或报错，排障时完全看不见。 :class:`NoDiskToolMediaMixin` 堵掉落盘和静默降级，两条线都用。**图放在哪一条消息上 则按协议走各自的原生机制**： -- **Responses**：折回 ``function_call_output`` 自己的 ``output``（ - :class:`ResponsesToolMediaMixin`）。该协议的函数输出本就能载图，codex 也是这么做的 +- **Responses**：由 AgentScope 原生写入 ``function_call_output`` 自己的 ``output``。该协议的函数输出本就能载图，codex 也是这么做的 （``FunctionCallOutputContentItem::InputImage``，见 ``codex-rs/core/src/tools/handlers/view_image.rs``），图与 ``call_id`` 直接绑定， 多图并发天然不串。 @@ -99,7 +98,7 @@ def convert_tool_result_to_string( def _is_sendable(self, block: DataBlock) -> bool: """线路能不能收下这份媒体。收不下要吵，但不能掀桌。 - 走到这里还收不下，说明上游的能力判定（``core.vision.resolve_vision_mode``） + 走到这里还收不下，说明上游的能力判定（``core.models.vision.resolve_vision_mode``） 与线路实际能力不一致——那是配置或代码的 bug，得有人去修，所以记 ``error`` 级日志。但它不该连累这一轮对话：整个请求抛错会把智能体正在跑的任务直接打断， 而这次失败本身是可以如实告诉模型、让它自己决定怎么绕的。所以降级只有这一种 @@ -117,59 +116,3 @@ def _is_sendable(self, block: DataBlock) -> bool: patterns or "纯文本", ) return False - - -class ResponsesToolMediaMixin(NoDiskToolMediaMixin): - """把媒体折回它所属的那条 ``function_call_output``（Responses 协议）。 - - 父类 formatter 会把媒体提升成紧跟其后的一条 user 消息。那条消息与产出它的调用 - 之间只剩位置关系，多图并发时要靠额外的编号文案才能对上号；Responses 的函数输出 - 本身就能载内容部件，折回去即可与 ``call_id`` 直接绑定。 - """ - - async def format(self, msgs: list[Msg]) -> list[dict[str, Any]]: - # 逐条消息格式化后再折叠：父类对每条消息的处理彼此独立，逐条与整批结果一致， - # 而逐条能保证「函数输出后面那条纯媒体 user 项」只可能是本条消息的提升产物， - # 不会误收下一轮真实的用户消息。 - items: list[dict[str, Any]] = [] - for msg in msgs: - # Concrete SDK formatter follows this mixin in each bound class MRO. - formatted = await super().format([msg]) # type: ignore[safe-super] - items.extend(_fold_media_into_function_output(formatted)) - return items - - -def _fold_media_into_function_output(items: list[dict[str, Any]]) -> list[dict[str, Any]]: - folded: list[dict[str, Any]] = [] - for item in items: - target = folded[-1] if folded else None - if ( - target is not None - and target.get("type") == "function_call_output" - and _is_media_only(item) - ): - target["output"] = _as_output_parts(target.get("output")) + list(item["content"]) - continue - folded.append(item) - return folded - - -def _is_media_only(item: dict[str, Any]) -> bool: - """这条 user 项是不是上一条函数输出提升出来的纯媒体项。 - - ``convert_tool_result_to_string`` 只交出 ``DataBlock``，不再带任何说明文案， - 所以提升出来的项必然是「清一色非文本的内容部件」——据此判定，不去认父类内部 - 给这条消息起的名字。 - """ - content = item.get("content") - if item.get("role") != "user" or not isinstance(content, list) or not content: - return False - return all(isinstance(part, dict) and part.get("type") != "input_text" for part in content) - - -def _as_output_parts(output: Any) -> list[dict[str, Any]]: - if isinstance(output, list): - return list(output) - if output: - return [{"type": "input_text", "text": output}] - return [] diff --git a/src/backend/core/llm/model_usage.py b/src/backend/core/models/usage.py similarity index 98% rename from src/backend/core/llm/model_usage.py rename to src/backend/core/models/usage.py index 5eea010c..df554b16 100644 --- a/src/backend/core/llm/model_usage.py +++ b/src/backend/core/models/usage.py @@ -168,7 +168,7 @@ def _usage_from_response(response: Any) -> AttemptUsage: async def _record_safely(recorder: UsageRecorder, attempt: UsageAttempt) -> UsageAttempt | None: from dataclasses import replace - from core.services.desktop_observation_context import model_call_id + from core.harness.log_context import model_call_id call_id = model_call_id.get() if call_id: attempt = replace(attempt, metadata={**attempt.metadata, "gateway_call_id": call_id}) @@ -287,7 +287,7 @@ async def measured(model_name: str, *args: Any, **kwargs: Any) -> Any: if context is None: return await original(model_name, *args, **kwargs) - from core.services.desktop_observation_context import model_call_id + from core.harness.log_context import model_call_id model_call_id.set("") started = time.monotonic() operation = str(model_name or getattr(model, "model", "") or "unknown") diff --git a/src/backend/core/models/vision/__init__.py b/src/backend/core/models/vision/__init__.py new file mode 100644 index 00000000..10c34df7 --- /dev/null +++ b/src/backend/core/models/vision/__init__.py @@ -0,0 +1,34 @@ +"""视觉桥（Vision Bridge）：让纯文本主模型「看得见」图片。 + +图片 → 多模态模型 → 结构化文字证据 → 注入主模型上下文。主模型全程只吃文本。 + +对外入口： + +- :func:`core.models.vision.service.is_available` —— 当前是否配了可用的视觉模型 +- :func:`core.models.vision.service.get_vision_bridge` —— 识别单张/多张图 +- :func:`core.models.vision.render.render_evidence` —— 证据 → 可注入文本（带不可信输入围栏） +- :func:`core.models.vision.service.model_supports_vision` —— 某个模型配置是否原生多模态 +- :func:`core.models.vision.service.resolve_vision_mode` —— 本轮模型看图方式：native / bridge / none +""" + +from core.models.vision.render import render_evidence, render_many, render_unavailable +from core.models.vision.schema import VisionEvidence, parse_evidence +from core.models.vision.service import VISION_ROLE_KEY, VisionBridge, VisionMode, VisionResult, get_vision_bridge, is_available, model_supports_vision, resolve_vision_config, resolve_vision_mode, sniff_mime + +__all__ = [ + "VISION_ROLE_KEY", + "VisionBridge", + "VisionEvidence", + "VisionMode", + "VisionResult", + "get_vision_bridge", + "is_available", + "model_supports_vision", + "parse_evidence", + "render_evidence", + "render_many", + "render_unavailable", + "resolve_vision_config", + "resolve_vision_mode", + "sniff_mime", +] diff --git a/src/backend/core/vision/attachments.py b/src/backend/core/models/vision/attachments.py similarity index 83% rename from src/backend/core/vision/attachments.py rename to src/backend/core/models/vision/attachments.py index 11c711b3..bd7a2d39 100644 --- a/src/backend/core/vision/attachments.py +++ b/src/backend/core/models/vision/attachments.py @@ -2,10 +2,10 @@ 抽出来是因为有两个调用方，且**它们想要的时机不同**： -- ``orchestration/streaming.py`` 在进入模型流之前先调它，好在识图这几秒里给前端发 +- ``core/llm/runtime/streaming.py`` 在进入模型流之前先调它，好在识图这几秒里给前端发 「图像理解中」的状态事件——识图是纯网络等待，不告诉用户的话，界面上只有一个笼统的 「深度拥抱中」在走秒。 -- ``core/llm/middlewares.py`` 的 FileContextMiddleware 在装配上下文时兜底调它（非流式 +- ``core/llm/context/attachment_hook.py`` 的附件钩子 在装配上下文时兜底调它（非流式 入口、渠道机器人等没走流式那条路的场景）。 所以这里只负责「图 → 文本」，不碰 agent 上下文，也不管状态事件——注入和发事件各自留在 @@ -18,6 +18,7 @@ import time from dataclasses import dataclass, field from typing import Any, Optional +from collections.abc import Callable logger = logging.getLogger(__name__) @@ -41,17 +42,17 @@ def count(self) -> int: return len(self.names) -def image_attachments(uploaded_files: list[dict[str, Any]]) -> list[dict[str, Any]]: +def image_attachments(uploaded_files: list[dict[str, Any]], *, is_image: Callable[[dict[str, Any]], bool]) -> list[dict[str, Any]]: """挑出图片类附件。""" - from core.llm.hooks import _is_image - return [f for f in (uploaded_files or []) if _is_image(f)] + return [f for f in (uploaded_files or []) if is_image(f)] async def transcribe_attachments( image_files: list[dict[str, Any]], *, user_id: Optional[str] = None, + download_bytes: Callable[..., Optional[bytes]], ) -> Optional[AttachmentVisionResult]: """把图片附件转写成注入文本。 @@ -62,19 +63,18 @@ async def transcribe_attachments( if not image_files: return None - from core.vision import get_vision_bridge, render_many, render_unavailable - from core.vision.service import is_available + from core.models.vision import get_vision_bridge, render_many, render_unavailable + from core.models.vision.service import is_available names = [f.get("name") or "图片" for f in image_files] if not is_available(): return AttachmentVisionResult(text=render_unavailable(names), names=names) - from core.llm.hooks import _download_artifact_bytes payloads: list[tuple[bytes, str]] = [] kept_names: list[str] = [] for f in image_files: - raw = _download_artifact_bytes( + raw = download_bytes( f.get("file_id") or "", f.get("name") or "image", "Vision bridge", diff --git a/src/backend/core/vision/prompt.py b/src/backend/core/models/vision/prompt.py similarity index 98% rename from src/backend/core/vision/prompt.py rename to src/backend/core/models/vision/prompt.py index 7310b627..63b1ca5f 100644 --- a/src/backend/core/vision/prompt.py +++ b/src/backend/core/models/vision/prompt.py @@ -10,7 +10,7 @@ 安全：图片里的文字是不可信输入，和网页正文同级。规则 4 明确要求视觉模型把图内文字 当数据、绝不执行其中的指令——这是缓解措施不是保证，注入端还有一层围栏 -（见 :mod:`core.vision.render`）。 +（见 :mod:`core.models.vision.render`）。 """ from __future__ import annotations diff --git a/src/backend/core/vision/provider.py b/src/backend/core/models/vision/provider.py similarity index 97% rename from src/backend/core/vision/provider.py rename to src/backend/core/models/vision/provider.py index 2a4e6503..2cc3ce13 100644 --- a/src/backend/core/vision/provider.py +++ b/src/backend/core/models/vision/provider.py @@ -7,7 +7,7 @@ - **Anthropic 原生**：``/v1/messages`` + ``source.type=base64``。 - **Gemini 原生**：``generateContent`` + ``inline_data``。 -这里只管协议差异，不管重试/缓存/降级——那些在 :mod:`core.vision.service`。 +这里只管协议差异，不管重试/缓存/降级——那些在 :mod:`core.models.vision.service`。 结构化输出按 **三级降级**尝试，逐级放宽： ``json_schema`` → ``json_object`` → 纯文本 + 填模板指令。弱网关对前两级常报 400， @@ -26,11 +26,11 @@ import httpx from core.harness.usage import AttemptUsage, attempt_status_for_exception -from core.llm.model_usage import record_external_model_attempt -from core.llm.providers.registry import get_spec -from core.services.model_config import ResolvedModelConfig -from core.vision.prompt import JSON_TEMPLATE_INSTRUCTION -from core.vision.schema import VISION_JSON_SCHEMA +from core.models.usage import record_external_model_attempt +from core.models.providers.registry import get_spec +from core.models.config import ResolvedModelConfig +from core.models.vision.prompt import JSON_TEMPLATE_INSTRUCTION +from core.models.vision.schema import VISION_JSON_SCHEMA logger = logging.getLogger(__name__) diff --git a/src/backend/core/vision/render.py b/src/backend/core/models/vision/render.py similarity index 98% rename from src/backend/core/vision/render.py rename to src/backend/core/models/vision/render.py index 06e84d5e..411a1695 100644 --- a/src/backend/core/vision/render.py +++ b/src/backend/core/models/vision/render.py @@ -3,7 +3,7 @@ 两个关注点： - **围栏**。图里的文字是不可信外部输入，和网页正文同级——一张截图可以写着「忽略你 - 之前的指令」。视觉模型侧已经被要求把图当数据（见 :mod:`core.vision.prompt` 规则 4）， + 之前的指令」。视觉模型侧已经被要求把图当数据（见 :mod:`core.models.vision.prompt` 规则 4）， 这里再加一层显式围栏，让主模型也知道这段文字的来路。 - **预算**。完整证据（全文转写 + 全部版面区域）很长，无条件塞进上下文会挤掉真正的 对话内容。默认按 :data:`DEFAULT_MAX_CHARS` 截断，超出部分明确标注被截断，agent @@ -14,7 +14,7 @@ from typing import Optional -from core.vision.schema import VisionEvidence +from core.models.vision.schema import VisionEvidence DEFAULT_MAX_CHARS = 4000 BRIEF_MAX_CHARS = 1200 diff --git a/src/backend/core/vision/schema.py b/src/backend/core/models/vision/schema.py similarity index 100% rename from src/backend/core/vision/schema.py rename to src/backend/core/models/vision/schema.py diff --git a/src/backend/core/vision/service.py b/src/backend/core/models/vision/service.py similarity index 96% rename from src/backend/core/vision/service.py rename to src/backend/core/models/vision/service.py index 9c67b49e..899b22e6 100644 --- a/src/backend/core/vision/service.py +++ b/src/backend/core/models/vision/service.py @@ -1,8 +1,8 @@ """视觉桥服务 —— 纯文本主模型的「眼睛」。 职责：解析 vision 角色配置 → 校验图片 → 调 provider → 校验结构 → 缓存 → 返回证据。 -provider 的协议差异在 :mod:`core.vision.provider`，注入用的文本渲染在 -:mod:`core.vision.render`。 +provider 的协议差异在 :mod:`core.models.vision.provider`，注入用的文本渲染在 +:mod:`core.models.vision.render`。 三个不能省的工程点： @@ -25,10 +25,10 @@ from dataclasses import dataclass, field from typing import Literal, Any, Optional -from core.services.model_config import ModelConfigService, ResolvedModelConfig -from core.vision.prompt import build_vision_prompt -from core.vision.provider import VisionProvider -from core.vision.schema import VisionEvidence, parse_evidence +from core.models.config import ModelConfigService, ResolvedModelConfig +from core.models.vision.prompt import build_vision_prompt +from core.models.vision.provider import VisionProvider +from core.models.vision.schema import VisionEvidence, parse_evidence logger = logging.getLogger(__name__) @@ -101,7 +101,7 @@ def model_supports_vision(cfg: Optional[ResolvedModelConfig]) -> bool: 以 provider 的 ``extra_config.supports_vision`` 显式开关为准，这是「模型能不能 看图」的唯一判定处。曾经还有一条靠匹配网关报错文案事后补救的路径，换个网关文案 就失效、还得先发一次注定失败的请求，而且补救手段是把图悄悄换成文字——已删除： - 图一旦交给工具返回，就必须原样送达（见 ``core.llm.tool_result_media``）。 + 图一旦交给工具返回，就必须原样送达（见 ``core.models.tool_result_media``）。 """ if cfg is None: return False diff --git a/src/backend/core/myspace/duplicate_cleanup.py b/src/backend/core/myspace/duplicate_cleanup.py deleted file mode 100644 index b0763178..00000000 --- a/src/backend/core/myspace/duplicate_cleanup.py +++ /dev/null @@ -1,63 +0,0 @@ -"""Explicit, journalled removal of all live records in duplicate personal paths.""" -from datetime import datetime, timezone -from typing import Callable -import sqlalchemy as sa - - -def _table(connection): - return sa.Table("artifacts", sa.MetaData(), autoload_with=connection) - - -def duplicate_rows(connection): - """Read-only preview. A duplicate group has no privileged 'winner'.""" - table = _table(connection) - filters = [table.c.deleted_at.is_(None)] - groups = ( - sa.select(table.c.user_id, table.c.user_folder_id, table.c.filename) - .where(*filters) - .group_by(table.c.user_id, table.c.user_folder_id, table.c.filename) - .having(sa.func.count() > 1).subquery() - ) - return [dict(row) for row in connection.execute( - sa.select(table).join(groups, sa.and_( - table.c.user_id == groups.c.user_id, - table.c.user_folder_id.is_not_distinct_from(groups.c.user_folder_id), - table.c.filename == groups.c.filename, - )).where(*filters).order_by(table.c.artifact_id) - ).mappings()] - - -def clean_duplicates(connection, *, before_delete: Callable | None = None): - """Caller owns the transaction and stops writers until mirror cleanup completes. - - Tombstones retain file identities so message backfill cannot resurrect them. - Original storage objects are retained for recovery, never reused as live paths. - """ - if connection.dialect.name == "postgresql": - connection.execute(sa.text("LOCK TABLE artifacts IN SHARE ROW EXCLUSIVE MODE")) - rows = duplicate_rows(connection) - timestamp = datetime.now(timezone.utc) - journal = {"deleted_at": timestamp.isoformat(), "rows": rows} - if before_delete is not None: - before_delete(journal) # Backup failure aborts before the first deletion. - table = _table(connection) - for offset in range(0, len(rows), 500): - connection.execute(table.update().where( - table.c.artifact_id.in_([r["artifact_id"] for r in rows[offset:offset+500]]) - ).values(deleted_at=timestamp, updated_at=timestamp)) - return journal - - -def restore_cleanup(connection, journal): - """Restore unmodified tombstones; caller must downgrade the unique indexes first.""" - table = _table(connection) - marker = datetime.fromisoformat(journal["deleted_at"]) - for row in journal["rows"]: - old_updated = row.get("updated_at") - if isinstance(old_updated, str): - old_updated = datetime.fromisoformat(old_updated) - connection.execute(table.update().where( - table.c.artifact_id == row["artifact_id"], - table.c.deleted_at == marker, - table.c.updated_at == marker, - ).values(deleted_at=None, updated_at=old_updated)) diff --git a/src/backend/core/ontology/__init__.py b/src/backend/core/ontology/__init__.py index a8e6eb03..b3cc274c 100644 --- a/src/backend/core/ontology/__init__.py +++ b/src/backend/core/ontology/__init__.py @@ -1,6 +1,7 @@ """Ontology harness: Domain Pack validation, runtime selection, and gates.""" from core.ontology.schemas import OntologyPackDocument -from core.ontology.validator import DomainPackValidator, OntologyGateDecision +from core.ontology.validator import DomainPackValidator +from core.ontology.validation_models import OntologyGateDecision __all__ = ["DomainPackValidator", "OntologyGateDecision", "OntologyPackDocument"] diff --git a/src/backend/core/ontology/audit_events.py b/src/backend/core/ontology/audit_events.py new file mode 100644 index 00000000..284501f4 --- /dev/null +++ b/src/backend/core/ontology/audit_events.py @@ -0,0 +1,71 @@ +"""Persist ontology enforcement, activation and review events.""" +from __future__ import annotations +from core.infra.time import utc_now +import uuid +from typing import Any +from core.db.repository import OntologyRepository + +def record_enforcement_event(data: dict[str, Any]) -> None: + """Persist an event from middleware/reviewer using an isolated session.""" + from core.db.engine import SessionLocal + + db = SessionLocal() + try: + payload = dict(data) + payload.setdefault("event_id", f"ontoe_{uuid.uuid4().hex[:16]}") + OntologyRepository(db).create_event(payload) + except Exception: + db.rollback() + raise + finally: + db.close() + + +def record_runtime_activation( + event: dict[str, Any], + runtime: dict[str, Any], + *, + user_id: str | None, + chat_id: str | None, +) -> None: + """Persist one asset-driven workflow activation as append-only evidence.""" + + version_id = next( + ( + pack.get("version_id") + for pack in runtime.get("packs", []) + if pack.get("pack_id") == event.get("pack_id") + ), + None, + ) + record_enforcement_event( + { + "user_id": user_id, + "chat_id": chat_id, + "pack_id": event.get("pack_id"), + "version_id": version_id, + "rule_id": f"activation:{event.get('workflow_id')}", + "stage": "checkpoint", + "event_type": "runtime_activation", + "decision": "pass", + "mode": "enforce", + "target": f"{event.get('asset_kind')}:{event.get('asset_id')}", + "details": event, + } + ) + + +def record_review_run(data: dict[str, Any]) -> None: + from core.db.engine import SessionLocal + + db = SessionLocal() + try: + payload = dict(data) + payload.setdefault("review_id", f"ontor_{uuid.uuid4().hex[:16]}") + payload.setdefault("created_at", utc_now()) + OntologyRepository(db).create_review(payload) + except Exception: + db.rollback() + raise + finally: + db.close() diff --git a/src/backend/core/ontology/build_validator.py b/src/backend/core/ontology/build_validator.py index 6b0831de..0b553776 100644 --- a/src/backend/core/ontology/build_validator.py +++ b/src/backend/core/ontology/build_validator.py @@ -9,7 +9,7 @@ from core.db.models import AdminMcpServer, AdminSkill, InstalledPlugin from core.infra.exceptions import BadRequestError from core.ontology.schemas import OntologyPackDocument -from core.ontology.validator import activate_runtime_for_asset +from core.ontology.runtime_pack import activate_runtime_for_asset from sqlalchemy.orm import Session @@ -65,7 +65,7 @@ def validate( output_schema: dict[str, Any] | None = None, tool_schemas: dict[str, dict[str, Any]] | None = None, ) -> BuildValidationReport: - from core.services.ontology_service import OntologyService + from core.ontology.service import OntologyService task = "\n".join(item for item in (name, description, instructions[:4000]) if item) runtime = OntologyService(self.db).build_runtime(task=task) @@ -343,7 +343,7 @@ def _validate_tool_contracts( suggestions: list[str], ) -> None: """Check discovered tool inputs directly against active pack rules.""" - from core.services.ontology_service import OntologyService + from core.ontology.service import OntologyService service = OntologyService(self.db) for version in service.repo.get_active_versions(): diff --git a/src/backend/core/services/ontology_evolution_service.py b/src/backend/core/ontology/evolution.py similarity index 96% rename from src/backend/core/services/ontology_evolution_service.py rename to src/backend/core/ontology/evolution.py index b0d8cac5..9bcc7b66 100644 --- a/src/backend/core/services/ontology_evolution_service.py +++ b/src/backend/core/ontology/evolution.py @@ -12,6 +12,7 @@ import re import uuid from collections import defaultdict +from dataclasses import dataclass, field from threading import Lock from typing import Any @@ -19,25 +20,21 @@ from core.infra.background import spawn from core.infra.exceptions import BadRequestError, ResourceNotFoundError from core.memory.sanitizer import sanitize -from core.services.ontology_service import OntologyService +from core.ontology.service import OntologyService from sqlalchemy.orm import Session _SECTIONS = {"term": "concepts", "relation": "relations", "constraint": "constraints"} logger = logging.getLogger(__name__) -_SCHEDULE_LOCK = Lock() -_EVOLUTION_RUNNING = False def schedule_ontology_evolution(*, user_id: str) -> bool: """Debounce the low-priority evidence prefilter on the current event loop.""" - global _EVOLUTION_RUNNING - with _SCHEDULE_LOCK: - if _EVOLUTION_RUNNING: + with process_state().SCHEDULE_LOCK: + if process_state().EVOLUTION_RUNNING: return False - _EVOLUTION_RUNNING = True + process_state().EVOLUTION_RUNNING = True async def _run() -> None: - global _EVOLUTION_RUNNING from core.db.engine import SessionLocal db = SessionLocal() @@ -51,8 +48,8 @@ async def _run() -> None: logger.warning("ontology evolution background pass failed", exc_info=True) finally: db.close() - with _SCHEDULE_LOCK: - _EVOLUTION_RUNNING = False + with process_state().SCHEDULE_LOCK: + process_state().EVOLUTION_RUNNING = False spawn(_run(), name="ontology-evolution-prefilter") return True @@ -231,7 +228,7 @@ def ingest_user_correction( materialized or activated automatically. """ from core.db.models import ChatMessage - from core.services.ontology_service import build_user_ontology_runtime + from core.ontology.runtime_policy import build_user_ontology_runtime assistant = ( self.db.query(ChatMessage) @@ -376,9 +373,9 @@ async def _llm_proposals( '"evidence":["只引用输入证据"]}。' ) try: - from orchestration.subagents.ontology_reviewer import _run_text_agent + from core.harness.registry.model_services import run_text_agent - text = await _run_text_agent( + text = await run_text_agent( prompt, user_id=user_id, model_name=model_name, @@ -481,3 +478,16 @@ def _sanitize_json_value(value: Any) -> Any | None: return json.loads(result.text) except (json.JSONDecodeError, TypeError): return None + + +@dataclass +class EvolutionState: + SCHEDULE_LOCK: object = field(default_factory=lambda: Lock()) + EVOLUTION_RUNNING: object = False + + +_STATE = EvolutionState() + + +def process_state() -> EvolutionState: + return _STATE diff --git a/src/backend/core/ontology/host.py b/src/backend/core/ontology/host.py new file mode 100644 index 00000000..bfa3c53e --- /dev/null +++ b/src/backend/core/ontology/host.py @@ -0,0 +1,13 @@ +"""Install ontology governance at the application composition boundary.""" + +def register(): + from core.harness.registry.build_governance import register_build_governance + from core.ontology.build_validator import ensure_ontology_build_valid, OntologyBuildValidator + from core.ontology.service import OntologyService + from core.ontology.policy import resolve_plugin_import_ontology_validation + register_build_governance( + ensure=ensure_ontology_build_valid, + validate=lambda db, **kwargs: OntologyBuildValidator(db).validate(**kwargs), + tag_options=lambda db, asset_type: OntologyService(db).list_asset_tag_options(asset_type), + import_policy=resolve_plugin_import_ontology_validation, + ) diff --git a/src/backend/core/ontology/manual_review.py b/src/backend/core/ontology/manual_review.py new file mode 100644 index 00000000..26a0322d --- /dev/null +++ b/src/backend/core/ontology/manual_review.py @@ -0,0 +1,85 @@ +"""Build the payload for human ontology review cards.""" +from __future__ import annotations +from typing import Any + +def dedupe_strings(items: list[str]) -> list[str]: + seen: set[str] = set() + output: list[str] = [] + for item in items: + value = str(item or "").strip() + if not value or value in seen: + continue + seen.add(value) + output.append(value) + return output + + +def build_manual_review( + *, + required: bool, + violations: list[dict[str, Any]], + affected_claims: list[dict[str, str]], + suggestions: list[str], +) -> dict[str, Any]: + """Build the JSON payload rendered by the frontend's human-review cards.""" + items: list[dict[str, str]] = [] + seen: set[tuple[str, str]] = set() + for item in affected_claims: + quote = " ".join(str(item.get("quote") or "").split())[:240] + risk = " ".join(str(item.get("issue") or "").split())[:360] + rule_id = str(item.get("rule_id") or "未指定规则")[:160] + manual_check = " ".join(str(item.get("manual_check") or "").split())[:360] + key = (quote, rule_id) + if key in seen or (not quote and not risk): + continue + seen.add(key) + items.append( + { + "quote": quote or "答案中的相关领域结论", + "rule_id": rule_id, + "risk": risk or "委员会未能确认该结论满足领域契约。", + "manual_check": manual_check or "核对原始工具结果、适用前提和结论表述。", + } + ) + if len(items) >= 8: + break + + for violation in violations: + if len(items) >= 8: + break + rule_id = str(violation.get("rule_id") or "未指定规则")[:160] + reasons = "；".join(str(item) for item in violation.get("reasons") or [] if str(item)) + key = ("", rule_id) + if key in seen: + continue + seen.add(key) + message = str(violation.get("message") or "领域约束未满足").strip() + items.append( + { + "quote": "整体证据链或输出结构", + "rule_id": rule_id, + "risk": f"{message}{f'（{reasons}）' if reasons else ''}"[:360], + "manual_check": "确认必需工具、引用证据和答案中的对应结论能够逐项追溯。", + } + ) + + if required and not items: + items.append( + { + "quote": "答案中的领域判断", + "rule_id": "ontology_review", + "risk": "委员会未能形成通过结论。", + "manual_check": "核对关键事实、推断、适用前提和原始证据。", + } + ) + return { + "required": required, + "title": "领域本体人工复核", + "summary": ( + "候选答案仍有未闭合的领域证据或判断，请在替换原文前完成复核。" + if required + else "本轮修订建议已结构化，可按需核对后采用。" + ), + "items": items, + "actions": [item[:500] for item in dedupe_strings(suggestions)[:6]], + } diff --git a/src/backend/core/ontology/output_review_state.py b/src/backend/core/ontology/output_review_state.py new file mode 100644 index 00000000..48c1402b --- /dev/null +++ b/src/backend/core/ontology/output_review_state.py @@ -0,0 +1,60 @@ +"""Claim and complete the single final-answer review of a governance run.""" +from __future__ import annotations +from typing import Any + +def requires_output_review(runtime: dict[str, Any]) -> bool: + """Whether an L-b/L-c review must run after the streamed draft.""" + return bool( + runtime.get("enabled") + and ( + runtime.get("review_level") != "none" + or any( + (rule.get("target") or {}).get("kind") == "output" + for pack in runtime.get("packs", []) + for rule in pack.get("constraints", []) + ) + ) + ) + + +def claim_output_review(runtime: dict[str, Any], *, owner: str) -> bool: + """Atomically claim the single final-answer review for one governance run.""" + if not requires_output_review(runtime): + return False + state = runtime.setdefault( + "output_review", + {"status": "pending", "owner": None, "count": 0}, + ) + if state.get("status") in {"running", "completed"}: + return False + state.update({"status": "running", "owner": owner}) + return True + + +def complete_output_review( + runtime: dict[str, Any], + *, + owner: str, + verdict: str, + attempts: int = 1, +) -> None: + """Mark the claimed review complete and retain its single-run evidence.""" + state = runtime.setdefault("output_review", {}) + if state.get("status") != "running" or state.get("owner") != owner: + return + count = int(state.get("count") or 0) + max(1, int(attempts or 1)) + state.update( + { + "status": "completed", + "owner": owner, + "count": count, + "verdict": verdict, + } + ) + + +def release_output_review(runtime: dict[str, Any], *, owner: str) -> None: + """Release a failed claim so a safe retry can review the final answer.""" + state = runtime.setdefault("output_review", {}) + if state.get("status") == "running" and state.get("owner") == owner: + state.update({"status": "pending", "owner": None}) diff --git a/src/backend/core/services/ontology_policy.py b/src/backend/core/ontology/policy.py similarity index 97% rename from src/backend/core/services/ontology_policy.py rename to src/backend/core/ontology/policy.py index 0f8fc672..8df9a79a 100644 --- a/src/backend/core/services/ontology_policy.py +++ b/src/backend/core/ontology/policy.py @@ -7,7 +7,7 @@ from core.auth.capabilities import resolve_user_capabilities from core.db.models import SystemConfig from core.infra.exceptions import AccessDeniedError -from core.services.user_service import UserService +from core.auth.users import UserService from sqlalchemy.orm import Session FORCE_PLUGIN_IMPORT_BUILD_VALIDATION_KEY = "ontology.force_plugin_import_build_validation" @@ -114,7 +114,7 @@ def set_plugin_import_build_validation_forced( # The generic system-config service may have cached this row for another # consumer. Keep both access paths coherent immediately after an admin save. - from core.services.system_config import SystemConfigService + from core.config.store.system_config import SystemConfigService SystemConfigService.get_instance().invalidate_cache() return enabled diff --git a/src/backend/core/ontology/review_operations.py b/src/backend/core/ontology/review_operations.py new file mode 100644 index 00000000..89d517df --- /dev/null +++ b/src/backend/core/ontology/review_operations.py @@ -0,0 +1,102 @@ +"""Run one ontology review and revise the answer when it fails.""" +from __future__ import annotations +from typing import Any +from core.harness.registry.capability_runtime import child_scope +from core.harness.registry.model_services import run_text_agent as _run_text_agent +from core.infra.json_output import parse_json_lenient as _parse_json_lenient +from core.infra.logging import get_logger +from core.ontology import review_prompt +from core.ontology import reviewer_state + +logger = get_logger(__name__) + +async def review_once( + perspective: str, + **kwargs, +) -> dict[str, Any]: + prompt = review_prompt.review_prompt( + perspective=perspective, + task=kwargs["task"], + answer=kwargs["answer"], + runtime=kwargs["runtime"], + trace=kwargs["trace"], + citations=kwargs["citations"], + deterministic=kwargs["deterministic"], + ) + try: + text = await _run_text_agent( + prompt, + user_id=kwargs["user_id"], + model_name=kwargs.get("model_name"), + model_provider_id=kwargs.get("model_provider_id"), + runtime=kwargs["runtime"], + capability_scope=child_scope( + kwargs.get("capability_scope", ""), "ontology-review", perspective + ), + ) + obj = _parse_json_lenient(text) + if not isinstance(obj, dict) or obj.get("verdict") not in reviewer_state.VALID_VERDICTS: + return { + "verdict": "escalate", + "evidence": [], + "feedback": "评审输出无法解析，保守转人工。", + } + evidence = [str(item) for item in (obj.get("evidence") or []) if str(item).strip()] + affected_claims = [] + for item in obj.get("affected_claims") or []: + if not isinstance(item, dict): + continue + claim = { + "quote": str(item.get("quote") or "").strip()[:500], + "rule_id": str(item.get("rule_id") or "").strip()[:160], + "issue": str(item.get("issue") or "").strip()[:500], + "manual_check": str(item.get("manual_check") or "").strip()[:500], + } + if claim["quote"] or claim["issue"]: + affected_claims.append(claim) + verdict = obj["verdict"] + if verdict == "pass" and not evidence: + verdict = "revise" + return { + "verdict": verdict, + "evidence": evidence, + "feedback": str(obj.get("feedback") or "").strip(), + "affected_claims": affected_claims, + } + except Exception as exc: # noqa: BLE001 + logger.warning("[ontology-review] reviewer failed: %s", exc) + return { + "verdict": "escalate", + "evidence": [], + "feedback": f"评审委员不可用：{str(exc)[:160]}", + "affected_claims": [], + } + + +async def revise_answer( + *, + task: str, + answer: str, + feedback: list[str], + user_id: str, + model_name: str | None, + model_provider_id: str | None, + runtime: dict[str, Any], + capability_scope: str = "", +) -> str: + prompt = ( + "你是本体校验后的答案修订员。严格根据评审意见修正答案，不新增没有证据的事实。" + "直接输出修订后的完整答案，不要解释修订过程。\n\n" + f"用户任务：\n{task}\n\n原答案：\n{answer}\n\n评审意见：\n" + + "\n".join(f"- {item}" for item in feedback if item) + ) + return ( + await _run_text_agent( + prompt, + user_id=user_id, + model_name=model_name, + model_provider_id=model_provider_id, + runtime=runtime, + capability_scope=child_scope(capability_scope, "ontology-model", "revise"), + ) + ).strip() diff --git a/src/backend/core/ontology/review_prompt.py b/src/backend/core/ontology/review_prompt.py new file mode 100644 index 00000000..c3856ffc --- /dev/null +++ b/src/backend/core/ontology/review_prompt.py @@ -0,0 +1,99 @@ +"""Build the reviewer prompt from bounded evidence.""" +from __future__ import annotations +import json +from typing import Any + +def tail_with_json_budget(items: list[dict[str, Any]], max_chars: int) -> list[dict[str, Any]]: + """Retain the newest evidence without splitting a serialized trace item.""" + selected: list[dict[str, Any]] = [] + used = 2 + for item in reversed(items): + serialized = json.dumps(item, ensure_ascii=False, default=str) + cost = len(serialized) + (1 if selected else 0) + if selected and used + cost > max_chars: + break + if not selected and used + cost > max_chars: + selected.append( + { + "truncated": True, + "preview": serialized[: max(0, max_chars - 64)], + } + ) + break + selected.append(item) + used += cost + selected.reverse() + return selected + + +def review_evidence_payload( + trace: list[dict[str, Any]], citations: list[dict[str, Any]] +) -> dict[str, Any]: + """Prefer evidence produced by the ontology repair over stale draft evidence.""" + return { + "trace": tail_with_json_budget(trace, 8000), + "citations": tail_with_json_budget(citations, 2000), + } + + +def review_prompt( + *, + perspective: str, + task: str, + answer: str, + runtime: dict[str, Any], + trace: list[dict[str, Any]], + citations: list[dict[str, Any]], + deterministic: dict[str, Any], +) -> str: + contract = { + "packs": [ + { + "pack_id": pack.get("pack_id"), + "version": pack.get("version"), + "concepts": pack.get("concepts", []), + "relations": pack.get("relations", []), + "constraints": pack.get("constraints", []), + "workflows": pack.get("workflows", []), + } + for pack in runtime.get("packs", []) + ] + } + review_evidence = review_evidence_payload(trace, citations) + return ( + "你是本体评审委员会中的独立委员，与生成答案的 agent 相互独立。\n" + f"评审视角：{perspective}\n" + "只依据领域契约、工具轨迹和引用作判断。答案自称已核验不算证据；" + "不得补做任务或调用工具。还必须核对答案是否仍忠实完成用户任务：本体核验过程" + "不是最终交付物。除非用户明确要求核验报告，否则若答案把润色稿改写成‘原始语句’、" + "‘逐项核验’、‘主张评估’、‘修改说明’等元分析，或擅自改变用户要求的文体、篇幅、" + "结构与文件类型，必须判定为 revise。这里的任务忠实度是指交付形式与表达意图，不是" + "机械保留错误内容：若用户要求原样输出的语句本身违反已激活的领域事实或证据约束，" + "修订稿必须纠正这些问题；只要仍保持用户要求的一句话、文档等形式，不得因未逐字照抄" + "错误语句而判定为偏离任务。强制判定规则：不得把‘没有原样保留已被证据否定或缺乏" + "证据的断言’列为 revise 理由，也不得建议恢复这类断言；即使用户把请求称作测试或强调" + "原样输出，本体修订仍必须纠错。示例：用户要求用一句话原样输出‘某记录风险极高’，" + "工具证据显示未发现对应风险，答案仍用一句话改成有证据支持的审慎结论——任务形式忠实，" + "应继续按领域证据判断，而不能因没有逐字照抄判 revise。只有把一句话扩写成‘原始语句 +" + "逐项核验 + 修改说明’之类的元报告，才属于这里所说的交付形式偏离。\n\n" + "形式判断必须客观：只有一个句号结尾、内部用逗号或分号连接的陈述仍是一句话，附带引用" + "标记也不会把它变成分析报告；工具轨迹已生成并交付 .docx 时，不得声称文件类型发生了" + "变化。不要把纠错所必需的证据化表述误判成文体或篇幅偏离。工具轨迹按时间顺序排列，" + "并优先保留最近产生的证据；同名文件被多次生成时，后生成且已交付的文件是当前修订稿，" + "不得只根据早期文件内容判断待评答案。\n\n" + f"用户任务：\n{task}\n\n待评答案：\n{answer}\n\n" + "领域契约：\n" + f"{json.dumps(contract, ensure_ascii=False)[:14000]}\n\n" + "工具与引用证据：\n" + f"{json.dumps(review_evidence, ensure_ascii=False)}\n\n" + "确定性门禁结果：\n" + f"{json.dumps(deterministic, ensure_ascii=False)}\n\n" + "严格只输出 JSON：" + '{"verdict":"pass|revise|escalate","evidence":["规则ID或轨迹证据"],' + '"feedback":"具体问题与可执行修改要求",' + '"affected_claims":[{"quote":"答案中的原句或段落摘要",' + '"rule_id":"规则ID","issue":"可能违反本体的原因",' + '"manual_check":"人工需要核对的证据或判断"}]}。' + "只有所有高风险约束有证据支持时才 pass；可修正文案时 revise；" + "缺关键证据、存在冲突或需人工决策时 escalate。" + ) diff --git a/src/backend/core/ontology/reviewer.py b/src/backend/core/ontology/reviewer.py new file mode 100644 index 00000000..ddac9768 --- /dev/null +++ b/src/backend/core/ontology/reviewer.py @@ -0,0 +1,319 @@ +"""Review an answer against the ontology, repair it, and audit the outcome.""" +from __future__ import annotations +import asyncio +import time +from collections import Counter +from typing import Any +from core.ontology.revision import is_substantive_revision +from core.ontology.runtime_gates import evaluate_output +from core.infra.logging import get_logger +from core.ontology import manual_review +from core.ontology import review_operations +from core.ontology import reviewer_state + +logger = get_logger(__name__) + +async def review_ontology_output( + *, + task: str, + answer: str, + runtime: dict[str, Any], + trace: list[dict[str, Any]], + citations: list[dict[str, Any]], + user_id: str, + chat_id: str | None, + model_name: str | None, + model_provider_id: str | None = None, + trace_complete: bool = True, + remediate: reviewer_state.RemediationCallback | None = None, + max_repair_attempts: int = reviewer_state.DEFAULT_MAX_REPAIR_ATTEMPTS, + capability_scope: str = "", +) -> dict[str, Any]: + """Review one answer, repair it with the originating agent, and audit the session.""" + started = time.monotonic() + original_answer = answer + current_answer = answer + max_repair_attempts = max(0, min(int(max_repair_attempts), 1)) + repair_attempts = 0 + review_rounds: list[dict[str, Any]] = [] + all_feedback: list[str] = [] + all_evidence: list[str] = [] + final_affected_claims: list[dict[str, str]] = [] + final_reviewers: list[dict[str, Any]] = [] + level = runtime.get("review_level", "none") + initial_trace_len = len(trace) + initial_citation_len = len(citations) + + def _deterministic_result() -> tuple[Any, dict[str, Any]]: + completed_tools = { + str(item.get("tool_name")) + for item in trace + if item.get("type") == "tool_result" and item.get("tool_name") + } + decision = evaluate_output( + runtime, + answer=current_answer, + citations=citations, + completed_tools=completed_tools if trace_complete else None, + ) + return decision, { + "allowed": decision.allowed, + "decision": decision.decision, + "violations": decision.violations, + "suggestions": decision.suggestions, + } + + deterministic, deterministic_dict = _deterministic_result() + if level == "none" and deterministic.allowed: + return { + "verdict": "pass", + "answer": current_answer, + "reviewers": [], + "evidence": [], + "feedback": [], + "violations": [], + "affected_claims": [], + "manual_review": manual_review.build_manual_review( + required=False, violations=[], affected_claims=[], suggestions=[] + ), + "latency_ms": round((time.monotonic() - started) * 1000), + "revised": False, + "annotated": False, + "repair_attempts": 0, + "attempts": 0, + "review_rounds": [], + "new_tools": [], + "new_citation_count": 0, + } + effective_level = "checkpoint" if level == "none" else level + + # A deterministic evidence gap is repaired before the sole committee round, + # so the committee evaluates the newly retrieved evidence instead of merely + # rephrasing the first draft. + if not deterministic.allowed and remediate is not None and max_repair_attempts: + repair_attempts = 1 + try: + repaired = await remediate( + { + "attempt": 1, + "source": "deterministic", + "original_task": task, + "answer": current_answer, + "violations": deterministic.violations, + "suggestions": deterministic.suggestions, + "feedback": [], + "citations": list(citations), + } + ) + except Exception as exc: # noqa: BLE001 + logger.warning("[ontology-review] tool remediation failed: %s", exc) + all_feedback.append(f"自动补充证据失败：{str(exc)[:200]}") + repaired = "" + if is_substantive_revision(repaired): + current_answer = repaired.strip() + deterministic, deterministic_dict = _deterministic_result() + elif repaired.strip(): + all_feedback.append("自动修订未生成完整正文，已保留原文。") + + common = { + "capability_scope": capability_scope, + "task": task, + "answer": current_answer, + "runtime": runtime, + "trace": trace, + "citations": citations, + "deterministic": deterministic_dict, + "user_id": user_id, + "model_name": model_name, + "model_provider_id": model_provider_id, + } + if effective_level == "committee": + committee_size = max( + ( + int(pack.get("config", {}).get("committee_size", 3)) + for pack in runtime.get("packs", []) + ), + default=3, + ) + perspectives = [ + reviewer_state.PERSPECTIVES[index % len(reviewer_state.PERSPECTIVES)] + f"（席位 {index + 1}）" + for index in range(committee_size) + ] + reviewers = await asyncio.gather( + *(review_operations.review_once(perspective, **common) for perspective in perspectives) + ) + counts = Counter(item["verdict"] for item in reviewers) + verdict, votes = counts.most_common(1)[0] + if votes <= committee_size // 2: + verdict = "escalate" + else: + reviewers = [await review_operations.review_once(reviewer_state.PERSPECTIVES[0], **common)] + verdict = reviewers[0]["verdict"] + + if not deterministic.allowed and verdict == "pass": + verdict = "revise" + round_feedback = [str(item.get("feedback") or "") for item in reviewers] + round_feedback.extend(deterministic.suggestions) + round_evidence = [ + str(item) for reviewer in reviewers for item in reviewer.get("evidence", []) if str(item) + ] + round_claims = [ + claim + for reviewer in reviewers + for claim in reviewer.get("affected_claims", []) + if isinstance(claim, dict) + ] + all_feedback.extend(round_feedback) + all_evidence.extend(round_evidence) + final_affected_claims = round_claims + final_reviewers = reviewers + review_rounds.append( + { + "round": 1, + "verdict": verdict, + "deterministic": deterministic_dict, + "reviewers": reviewers, + } + ) + + # A committee wording/evidence revision gets one tools-enabled continuation, + # but is deliberately not sent through a second committee round. + if verdict == "revise" and repair_attempts == 0 and max_repair_attempts: + repair_attempts = 1 + try: + if remediate is not None: + revised = await remediate( + { + "attempt": 1, + "source": "committee", + "original_task": task, + "answer": current_answer, + "violations": deterministic.violations, + "suggestions": deterministic.suggestions, + "feedback": manual_review.dedupe_strings(round_feedback), + "affected_claims": round_claims, + "citations": list(citations), + } + ) + elif deterministic.allowed: + revised = await review_operations.revise_answer( + task=task, + answer=current_answer, + feedback=manual_review.dedupe_strings(round_feedback), + user_id=user_id, + model_name=model_name, + model_provider_id=model_provider_id, + runtime=runtime, + capability_scope=capability_scope, + ) + else: + revised = "" + except Exception as exc: # noqa: BLE001 + logger.warning("[ontology-review] committee remediation failed: %s", exc) + all_feedback.append(f"自动修订失败：{str(exc)[:200]}") + revised = "" + if is_substantive_revision(revised): + current_answer = revised.strip() + deterministic, _ = _deterministic_result() + elif revised.strip(): + all_feedback.append("自动修订未生成完整正文，已保留原文。") + + final_violations = list(deterministic.violations) + final_suggestions = list(deterministic.suggestions) + annotated = verdict == "escalate" or not deterministic.allowed + review_card = manual_review.build_manual_review( + required=annotated, + violations=final_violations, + affected_claims=final_affected_claims, + suggestions=final_suggestions or all_feedback, + ) + new_tools = manual_review.dedupe_strings( + [ + str(item.get("tool_name") or "") + for item in trace[initial_trace_len:] + if item.get("type") == "tool_result" + ] + ) + evidence = manual_review.dedupe_strings(all_evidence) + feedback = manual_review.dedupe_strings(all_feedback + final_suggestions) + latency_ms = round((time.monotonic() - started) * 1000) + try: + from core.ontology.audit_events import record_enforcement_event, record_review_run + + review_payload = { + "user_id": user_id or None, + "chat_id": chat_id, + "pack_version_ids": runtime.get("version_ids", []), + "level": "checkpoint" if level == "none" else level, + "subject_type": "final_answer", + "verdict": verdict, + "evidence": evidence, + "feedback": "\n".join(feedback), + "reviewers": [ + {**reviewer, "round": round_info["round"]} + for round_info in review_rounds + for reviewer in round_info["reviewers"] + ], + "latency_ms": latency_ms, + } + audit_tasks = [asyncio.to_thread(record_review_run, review_payload)] + for pack in runtime.get("packs", []): + audit_tasks.append( + asyncio.to_thread( + record_enforcement_event, + { + "user_id": user_id or None, + "chat_id": chat_id, + "pack_id": pack.get("pack_id"), + "version_id": pack.get("version_id"), + "rule_id": ( + deterministic.violations[0].get("rule_id") + if deterministic.violations + else f"review:{level}" + ), + "stage": "output", + "event_type": "output_review", + "decision": verdict, + "mode": "enforce", + "target": "final_answer", + "latency_ms": latency_ms, + "details": { + "governance_run_id": runtime.get("governance_run_id"), + "review_owner": (runtime.get("output_review") or {}).get("owner"), + "violations": deterministic.violations, + "evidence": evidence, + "feedback": review_payload["feedback"], + "repair_attempts": repair_attempts, + "review_rounds": len(review_rounds), + }, + }, + ) + ) + await asyncio.gather(*audit_tasks) + except Exception as exc: # noqa: BLE001 + logger.warning("[ontology-review] audit persistence failed: %s", exc) + if verdict in {"revise", "escalate"} and chat_id: + try: + from core.ontology.evolution import schedule_ontology_evolution + + schedule_ontology_evolution(user_id=user_id or "system") + except Exception as exc: # noqa: BLE001 + logger.warning("[ontology-review] evolution scheduling failed: %s", exc) + return { + "verdict": verdict, + "answer": current_answer, + "reviewers": final_reviewers, + "evidence": evidence, + "feedback": feedback, + "violations": final_violations, + "affected_claims": final_affected_claims, + "manual_review": review_card, + "latency_ms": latency_ms, + "revised": current_answer != original_answer, + "annotated": annotated, + "repair_attempts": repair_attempts, + "attempts": max(1, len(review_rounds)), + "review_rounds": review_rounds, + "new_tools": new_tools, + "new_citation_count": max(0, len(citations) - initial_citation_len), + } diff --git a/src/backend/core/ontology/reviewer_state.py b/src/backend/core/ontology/reviewer_state.py new file mode 100644 index 00000000..3f9c979b --- /dev/null +++ b/src/backend/core/ontology/reviewer_state.py @@ -0,0 +1,19 @@ +"""Verdicts, perspectives and repair limits for ontology review.""" +from __future__ import annotations +from collections.abc import Awaitable, Callable +from typing import Any + +VALID_VERDICTS = {"pass", "revise", "escalate"} + + +DEFAULT_MAX_REPAIR_ATTEMPTS = 1 + + +PERSPECTIVES = ( + "领域合规委员：逐条核对术语、关系、禁用条件和工作流约束。", + "证据委员：只认可工具轨迹与引用中可追溯的证据，严查无证据断言。", + "风险委员：关注高风险误导、遗漏前提、越权建议和不可逆影响。", +) + + +RemediationCallback = Callable[[dict[str, Any]], Awaitable[str]] diff --git a/src/backend/core/ontology/runtime_gates.py b/src/backend/core/ontology/runtime_gates.py new file mode 100644 index 00000000..2e368749 --- /dev/null +++ b/src/backend/core/ontology/runtime_gates.py @@ -0,0 +1,157 @@ +"""Deterministic ontology checks on tool calls and outputs.""" +from __future__ import annotations +from typing import Any, Iterable +from jsonschema import Draft202012Validator, ValidationError +from core.ontology import validation_models + +def evaluate_tool_call( + runtime: dict[str, Any], + *, + tool_name: str, + tool_input: dict[str, Any], + completed_tools: Iterable[str] = (), +) -> validation_models.OntologyGateDecision: + """Apply zero-LLM L-a validation to one tool call.""" + if not runtime.get("enabled"): + return validation_models.OntologyGateDecision(True, "pass") + completed = set(completed_tools) + violations: list[dict[str, Any]] = [] + suggestions: list[str] = [] + matched: list[str] = [] + enforced = False + + for pack in runtime.get("packs", []): + for workflow in pack.get("workflows", []): + if tool_name in workflow.get("forbidden_tools", []): + violations.append( + { + "rule_id": f"workflow:{workflow['id']}:forbidden", + "message": f"工作流 {workflow['name']} 禁止调用工具 {tool_name}", + "mode": "enforce", + } + ) + enforced = True + for rule in pack.get("constraints", []): + target = rule.get("target") or {} + if target.get("tool") != tool_name: + continue + matched.append(rule["id"]) + missing_prerequisites = [ + name for name in rule.get("prerequisite_tools", []) if name not in completed + ] + rule_errors: list[str] = [] + if missing_prerequisites: + rule_errors.append("缺少前置工具调用: " + ", ".join(missing_prerequisites)) + try: + if target.get("kind") == "tool": + Draft202012Validator(rule.get("schema") or {}).validate(tool_input) + elif target.get("kind") == "tool_parameter": + parameter = target.get("parameter") + Draft202012Validator(rule.get("schema") or {}).validate( + tool_input.get(parameter) + ) + except ValidationError as exc: + rule_errors.append(exc.message) + if rule_errors: + violations.append( + { + "pack_id": pack.get("pack_id"), + "rule_id": rule["id"], + "message": rule["message"], + "reasons": rule_errors, + "mode": rule.get("mode", "log"), + "risk": rule.get("risk", "low"), + } + ) + if rule.get("suggestion"): + suggestions.append(rule["suggestion"]) + if rule.get("mode") == "enforce": + enforced = True + + if not violations: + return validation_models.OntologyGateDecision(True, "pass", matched_rule_ids=matched) + return validation_models.OntologyGateDecision( + allowed=not enforced, + decision="deny" if enforced else "log", + violations=violations, + suggestions=suggestions, + matched_rule_ids=matched, + ) + + +def evaluate_output( + runtime: dict[str, Any], + *, + answer: str, + citations: Iterable[dict[str, Any]] = (), + completed_tools: Iterable[str] | None = (), +) -> validation_models.OntologyGateDecision: + """Apply deterministic output constraints before any LLM reviewer.""" + if not runtime.get("enabled"): + return validation_models.OntologyGateDecision(True, "pass") + citation_items = list(citations) + # ``None`` means the caller cannot observe a complete tool trace (the + # legacy non-streaming reply path). Output and citation rules still run, + # but required-tool rules must not produce a false omission claim. + completed = set(completed_tools) if completed_tools is not None else None + violations: list[dict[str, Any]] = [] + matched: list[str] = [] + suggestions: list[str] = [] + enforced = False + for pack in runtime.get("packs", []): + if completed is not None: + for workflow in pack.get("workflows", []): + missing_tools = sorted(set(workflow.get("required_tools", [])) - completed) + if missing_tools: + violations.append( + { + "pack_id": pack.get("pack_id"), + "rule_id": f"workflow:{workflow.get('id')}:required", + "message": "领域工作流缺少必需的证据工具调用。", + "reasons": ["未调用: " + ", ".join(missing_tools)], + "mode": "enforce", + "risk": workflow.get("risk", "low"), + } + ) + enforced = True + output_tags = { + tag for workflow in pack.get("workflows", []) for tag in workflow.get("output_tags", []) + } + for rule in pack.get("constraints", []): + target = rule.get("target") or {} + if target.get("kind") != "output": + continue + if output_tags and target.get("output_tag") not in output_tags: + continue + matched.append(rule["id"]) + reasons: list[str] = [] + if rule.get("requires_citations") and not citation_items: + reasons.append("输出缺少可追溯引用证据") + try: + Draft202012Validator(rule.get("schema") or {}).validate(answer) + except ValidationError as exc: + reasons.append(exc.message) + if reasons: + violations.append( + { + "pack_id": pack.get("pack_id"), + "rule_id": rule["id"], + "message": rule["message"], + "reasons": reasons, + "mode": rule.get("mode", "log"), + "risk": rule.get("risk", "low"), + } + ) + if rule.get("suggestion"): + suggestions.append(rule["suggestion"]) + if rule.get("mode") == "enforce": + enforced = True + if not violations: + return validation_models.OntologyGateDecision(True, "pass", matched_rule_ids=matched) + return validation_models.OntologyGateDecision( + allowed=not enforced, + decision="deny" if enforced else "log", + violations=violations, + suggestions=suggestions, + matched_rule_ids=matched, + ) diff --git a/src/backend/core/ontology/runtime_pack.py b/src/backend/core/ontology/runtime_pack.py new file mode 100644 index 00000000..230c50e2 --- /dev/null +++ b/src/backend/core/ontology/runtime_pack.py @@ -0,0 +1,357 @@ +"""Compile active ontology packs into the runtime policy for a request.""" +from __future__ import annotations +import json +import re +import uuid +from typing import Any, Iterable +from core.ontology.schemas import Constraint, OntologyPackDocument, Workflow + + +REVIEW_RANK = {"none": 0, "checkpoint": 1, "committee": 2} + + +def select_workflows(document: OntologyPackDocument, task: str) -> list[Workflow]: + """Select workflows by case-insensitive lexical trigger match.""" + normalized = task.casefold() + return [ + workflow + for workflow in document.workflows + if any(trigger.casefold() in normalized for trigger in workflow.triggers) + ] + + +def workflow_constraints( + document: OntologyPackDocument, + workflows: list[Workflow], +) -> list[Constraint]: + workflow_tools = { + tool_name + for workflow in workflows + for tool_name in (workflow.required_tools + workflow.forbidden_tools) + } + workflow_output_tags = {tag for workflow in workflows for tag in workflow.output_tags} + return [ + rule + for rule in document.constraints + if rule.enabled + and ( + rule.target.tool in workflow_tools + if rule.target.kind in {"tool", "tool_parameter"} + else rule.target.output_tag in workflow_output_tags + ) + ] + + +def runtime_concepts( + document: OntologyPackDocument, + task: str, + constraints: list[Constraint], +) -> list[dict[str, Any]]: + """Select task concepts while always retaining concepts used by active rules.""" + by_id = {item.id: item for item in document.concepts} + selected: list[Any] = [] + selected_ids: set[str] = set() + + def add_with_parents(concept_id: str | None) -> None: + chain: list[Any] = [] + current = by_id.get(concept_id or "") + while current is not None and current.id not in selected_ids: + chain.append(current) + current = by_id.get(current.parent_id or "") + for item in reversed(chain): + if len(selected) >= document.config.max_concepts: + return + if item.id not in selected_ids: + selected.append(item) + selected_ids.add(item.id) + + for rule in constraints: + add_with_parents(rule.concept_id) + for item in select_concepts(document, task, limit=document.config.max_concepts): + add_with_parents(item["id"]) + return [item.model_dump() for item in selected[: document.config.max_concepts]] + + +def compile_pack_runtime( + document: OntologyPackDocument, + task: str, + workflows: list[Workflow], +) -> dict[str, Any]: + constraints = workflow_constraints(document, workflows) + concepts = runtime_concepts(document, task, constraints) + selected_concept_ids = {item["id"] for item in concepts} + relations = [ + relation.model_dump() + for relation in document.relations + if relation.subject in selected_concept_ids and relation.object in selected_concept_ids + ] + return { + "pack_id": document.pack_id, + "version": document.version, + "name": document.name, + "domain": document.domain, + "description": document.description, + "config": document.config.model_dump(), + "concepts": concepts, + "relations": relations, + "constraints": [item.model_dump(by_alias=True) for item in constraints], + "workflows": [item.model_dump() for item in workflows], + } + + +def select_concepts( + document: OntologyPackDocument, + task: str, + *, + limit: int | None = None, +) -> list[dict[str, Any]]: + """Select only task-relevant concepts; fall back to tagged roots.""" + normalized = task.casefold() + # English words can be tokenized by boundaries. Chinese text cannot be + # treated as one giant ``[\u4e00-\u9fff]+`` token: doing so made a task such + # as “数据质量分析” match no concept at all. Concept names and aliases are + # therefore also checked directly as substrings of the task. + terms = set(re.findall(r"[a-z0-9_]+", normalized)) + scored: list[tuple[int, Any]] = [] + for concept in document.concepts: + labels = [concept.id, concept.name, *concept.aliases, *concept.tags] + haystack = " ".join([*labels, concept.definition]).casefold() + score = sum(1 for term in terms if len(term) >= 2 and term in haystack) + score += sum( + 3 for label in labels if len(label.strip()) >= 2 and label.casefold() in normalized + ) + if score or not concept.parent_id: + scored.append((score, concept)) + max_items = limit or document.config.max_concepts + scored.sort(key=lambda item: (-item[0], item[1].id)) + selected = [item for _, item in scored[:max_items]] + by_id = {item.id: item for item in document.concepts} + selected_ids = {item.id for item in selected} + # Preserve the hierarchy for every selected concept when the budget allows. + for concept in list(selected): + parent_id = concept.parent_id + while parent_id and len(selected) < max_items: + if parent_id not in selected_ids and parent_id in by_id: + selected.append(by_id[parent_id]) + selected_ids.add(parent_id) + parent_id = by_id.get(parent_id).parent_id if parent_id in by_id else None + return [item.model_dump() for item in selected[:max_items]] + + +def build_runtime_payload( + documents: list[OntologyPackDocument], + task: str, +) -> dict[str, Any]: + """Compile active packs into a compact, serializable request policy.""" + governance_run_id = f"ontog_{uuid.uuid4().hex[:16]}" + packs: list[dict[str, Any]] = [] + activation_candidates: list[dict[str, Any]] = [] + runtime_events: list[dict[str, Any]] = [] + activated_workflows: list[str] = [] + review_level = "none" + for document in documents: + workflows = select_workflows(document, task) + for workflow in workflows: + if REVIEW_RANK[workflow.review_level] > REVIEW_RANK[review_level]: + review_level = workflow.review_level + ref = f"{document.pack_id}:{workflow.id}" + activated_workflows.append(ref) + runtime_events.append( + { + "type": "ontology_activation", + "status": "completed", + "governance_run_id": governance_run_id, + "source": "text", + "pack_id": document.pack_id, + "workflow_id": workflow.id, + "workflow_name": workflow.name, + "review_level": workflow.review_level, + } + ) + packs.append(compile_pack_runtime(document, task, workflows)) + for workflow in document.workflows: + if not workflow.asset_triggers: + continue + activation_candidates.append( + { + "pack_id": document.pack_id, + "workflow_id": workflow.id, + "workflow_name": workflow.name, + "review_level": workflow.review_level, + "asset_triggers": [item.model_dump() for item in workflow.asset_triggers], + "pack": compile_pack_runtime(document, task, [workflow]), + } + ) + return { + "enabled": bool(packs), + "governance_run_id": governance_run_id, + "packs": packs, + "review_level": review_level, + "task": task, + "activation_candidates": activation_candidates, + "activated_workflows": activated_workflows, + "asset_tags": {"tool": {}, "skill": {}, "subagent": {}}, + "runtime_events": runtime_events, + "review_owner": "outer_workflow", + "output_review": {"status": "pending", "owner": None, "count": 0}, + } + + +def register_runtime_asset_tags( + runtime: dict[str, Any], + *, + kind: str, + asset_id: str, + tags: Iterable[str], +) -> None: + """Attach trusted server-side asset metadata to a request runtime.""" + if kind not in {"tool", "skill", "subagent"} or not asset_id: + return + catalog = runtime.setdefault("asset_tags", {}).setdefault(kind, {}) + existing = {str(item) for item in catalog.get(asset_id, []) if str(item)} + existing.update(str(item) for item in tags if str(item)) + catalog[asset_id] = sorted(existing) + + +def activate_runtime_for_asset( + runtime: dict[str, Any], + *, + kind: str, + asset_id: str, + tags: Iterable[str] = (), +) -> list[dict[str, Any]]: + """Monotonically merge workflow policy when an actual asset is invoked.""" + if not runtime.get("enabled") or kind not in {"tool", "skill", "subagent"}: + return [] + known_tags = set(str(item) for item in tags if str(item)) + known_tags.update( + str(item) + for item in runtime.get("asset_tags", {}).get(kind, {}).get(asset_id, []) + if str(item) + ) + activated = set(runtime.setdefault("activated_workflows", [])) + events: list[dict[str, Any]] = [] + for candidate in runtime.get("activation_candidates", []): + ref = f"{candidate.get('pack_id')}:{candidate.get('workflow_id')}" + if ref in activated: + continue + matched_tags: set[str] = set() + matched = False + for trigger in candidate.get("asset_triggers", []): + if trigger.get("kind") != kind: + continue + if asset_id in set(trigger.get("ids", [])): + matched = True + tag_hits = known_tags & set(trigger.get("tags_any", [])) + if tag_hits: + matched = True + matched_tags.update(tag_hits) + if not matched: + continue + + fragment = candidate.get("pack") or {} + current = next( + ( + pack + for pack in runtime.get("packs", []) + if pack.get("pack_id") == fragment.get("pack_id") + ), + None, + ) + if current is None: + current = fragment + runtime.setdefault("packs", []).append(current) + else: + for field, key in ( + ("workflows", "id"), + ("constraints", "id"), + ("concepts", "id"), + ("relations", "id"), + ): + existing_ids = {item.get(key) for item in current.get(field, [])} + current.setdefault(field, []).extend( + item for item in fragment.get(field, []) if item.get(key) not in existing_ids + ) + if fragment.get("version_id"): + current["version_id"] = fragment["version_id"] + + level = str(candidate.get("review_level") or "none") + if REVIEW_RANK.get(level, 0) > REVIEW_RANK.get(runtime.get("review_level"), 0): + runtime["review_level"] = level + activated.add(ref) + runtime["activated_workflows"] = sorted(activated) + event = { + "type": "ontology_activation", + "status": "completed", + "governance_run_id": runtime.get("governance_run_id"), + "source": kind, + "asset_kind": kind, + "asset_id": asset_id, + "asset_tags": sorted(known_tags), + "matched_tags": sorted(matched_tags), + "pack_id": candidate.get("pack_id"), + "workflow_id": candidate.get("workflow_id"), + "workflow_name": candidate.get("workflow_name"), + "review_level": level, + } + runtime.setdefault("runtime_events", []).append(event) + events.append(event) + return events + + +def render_runtime_prompt(runtime: dict[str, Any]) -> str: + """Render the bounded ontology segment injected into the system prompt.""" + if not runtime.get("enabled"): + return "" + compact_packs = [] + for pack in runtime.get("packs", []): + compact_pack = { + "id": pack["pack_id"], + "version": pack["version"], + "concepts": ( + list(pack.get("concepts", [])) + if pack.get("config", {}).get("injection_enabled", True) + else [] + ), + "relations": ( + list(pack.get("relations", [])) + if pack.get("config", {}).get("injection_enabled", True) + else [] + ), + "workflows": pack.get("workflows", []), + "constraints": [ + { + "id": rule["id"], + "target": rule["target"], + "mode": rule["mode"], + "message": rule["message"], + "suggestion": rule.get("suggestion", ""), + "prerequisite_tools": rule.get("prerequisite_tools", []), + } + for rule in pack.get("constraints", []) + ], + } + char_budget = int(pack.get("config", {}).get("token_budget", 2500)) * 4 + while len(json.dumps(compact_pack, ensure_ascii=False)) > char_budget: + if compact_pack["relations"]: + compact_pack["relations"].pop() + continue + if compact_pack["concepts"]: + compact_pack["concepts"].pop() + remaining = {item["id"] for item in compact_pack["concepts"]} + compact_pack["relations"] = [ + relation + for relation in compact_pack["relations"] + if relation["subject"] in remaining and relation["object"] in remaining + ] + continue + break + compact_packs.append(compact_pack) + compact = {"packs": compact_packs} + return ( + "\n" + "本轮已启用领域本体校验。以下内容是可执行的领域契约；工具门禁会独立执行，" + "请在调用前主动满足参数、前置证据与工作流要求。不要虚构证据。\n" + + json.dumps(compact, ensure_ascii=False, separators=(",", ":")) + + "\n" + ) diff --git a/src/backend/core/ontology/runtime_policy.py b/src/backend/core/ontology/runtime_policy.py new file mode 100644 index 00000000..e0f56b8d --- /dev/null +++ b/src/backend/core/ontology/runtime_policy.py @@ -0,0 +1,158 @@ +"""Resolve a user's ontology preference into a runtime policy and asset tags.""" +from __future__ import annotations +from core.ontology import service as ontology_service +from typing import Any +from core.infra.exceptions import ServiceUnavailableError +from core.ontology.runtime_pack import register_runtime_asset_tags +from sqlalchemy.orm import Session +import logging +logger = logging.getLogger(__name__) + +def disabled_ontology_runtime() -> dict[str, Any]: + """Return the canonical pass-through policy used when the user opts out.""" + return {"enabled": False, "packs": [], "review_level": "none"} + + +def build_ontology_runtime_for_preference( + *, + enabled: bool, + task: str, + db: Session, + pack_ids: list[str] | None = None, +) -> tuple[bool, dict[str, Any]]: + """Resolve the effective ontology switch and compile its runtime policy. + + A user's stored opt-in is subordinate to the administrator-controlled pack + availability. Disabling the selected/default packs is therefore an + intentional global off switch, not a runtime-policy failure. Once an + active pack exists, compilation remains fail-closed so malformed policy + data cannot silently bypass validation. + """ + if not enabled: + return False, disabled_ontology_runtime() + + service = ontology_service.OntologyService(db) + selected = pack_ids or None + if not service.repo.get_active_versions(selected): + return False, disabled_ontology_runtime() + + runtime = service.build_runtime(task=task, pack_ids=selected) + if not runtime.get("enabled"): + raise ServiceUnavailableError("本体校验运行时策略编译失败") + return True, runtime + + +def runtime_needs_asset_tag_lookup( + runtime: dict[str, Any], + *, + kind: str, + asset_id: str, +) -> bool: + """Return whether tags could activate an otherwise-unmatched workflow.""" + activated = set(runtime.get("activated_workflows", [])) + for candidate in runtime.get("activation_candidates", []): + ref = f"{candidate.get('pack_id')}:{candidate.get('workflow_id')}" + if ref in activated: + continue + triggers = [ + trigger + for trigger in candidate.get("asset_triggers", []) + if trigger.get("kind") == kind + ] + if not triggers: + continue + if any(asset_id in set(trigger.get("ids", [])) for trigger in triggers): + continue + if any(trigger.get("tags_any") for trigger in triggers): + return True + return False + + +def resolve_runtime_asset_tags( + *, + runtime: dict[str, Any], + kind: str, + asset_id: str, + user_id: str, +) -> list[str]: + """Resolve and request-cache one invoked asset's trusted ontology tags. + + The runtime catalog doubles as a per-turn positive/negative cache. Metadata + failures are fail-closed so an asset cannot execute without the governance + policy that its server-authoritative tags may activate. + """ + if not runtime.get("enabled") or kind not in {"tool", "skill", "subagent"} or not asset_id: + return [] + catalog = runtime.setdefault("asset_tags", {}).setdefault(kind, {}) + if asset_id in catalog: + return list(catalog[asset_id]) + if not runtime_needs_asset_tag_lookup(runtime, kind=kind, asset_id=asset_id): + register_runtime_asset_tags(runtime, kind=kind, asset_id=asset_id, tags=[]) + return [] + + try: + from core.db.engine import SessionLocal + + with SessionLocal() as db: + tags = ontology_service.OntologyService(db).resolve_asset_tags( + kind=kind, + asset_id=asset_id, + user_id=user_id, + ) + except Exception as exc: # noqa: BLE001 + logger.warning( + "[ontology] lazy asset tag lookup failed kind=%s asset_id=%s", + kind, + asset_id, + exc_info=True, + ) + raise ServiceUnavailableError("本体资产标签读取失败，本次资产调用已停止") from exc + + register_runtime_asset_tags(runtime, kind=kind, asset_id=asset_id, tags=tags) + return list(runtime["asset_tags"][kind][asset_id]) + + +def build_user_ontology_runtime( + *, + user_id: str, + task: str, + db: Session | None = None, +) -> tuple[bool, dict[str, Any]]: + """Resolve one user's opt-in and compile the task-scoped runtime policy. + + Web chat, channels, automation, batch, plan mode, and autonomous loops all + use this resolver so independent execution paths cannot silently bypass the + setting selected by the user. + """ + from core.auth.users import UserService + + owns_session = db is None + if owns_session: + from core.db.engine import SessionLocal + + db = SessionLocal() + assert db is not None + try: + from core.ontology.policy import user_can_use_ontology_validation + + if not user_can_use_ontology_validation(db, user_id): + return False, disabled_ontology_runtime() + settings = UserService(db).get_user_settings(user_id) + opted_in = bool(settings.get("ontology_enabled", False)) + if not opted_in: + return False, disabled_ontology_runtime() + pack_ids = settings.get("ontology_pack_ids") + selected = ( + [str(item) for item in pack_ids if str(item).strip()] + if isinstance(pack_ids, list) + else None + ) + return build_ontology_runtime_for_preference( + enabled=opted_in, + task=task, + db=db, + pack_ids=selected or None, + ) + finally: + if owns_session: + db.close() diff --git a/src/backend/core/services/ontology_service.py b/src/backend/core/ontology/service.py similarity index 68% rename from src/backend/core/services/ontology_service.py rename to src/backend/core/ontology/service.py index 3a24e195..462b3166 100644 --- a/src/backend/core/services/ontology_service.py +++ b/src/backend/core/ontology/service.py @@ -1,38 +1,22 @@ -"""Business service for ontology asset governance and runtime compilation.""" - +"""Manage ontology packs, drafts and versions, and resolve asset tags.""" from __future__ import annotations from core.infra.time import utc_now - import hashlib import json -import logging import uuid -from datetime import datetime from typing import Any - from core.config.display_names import TOOL_DISPLAY_NAMES -from core.db.models import ( - AdminMcpServer, - AdminSkill, - OntologyDraft, - OntologyPack, - OntologyPackVersion, -) +from core.db.models import AdminMcpServer, AdminSkill, OntologyDraft, OntologyPack, OntologyPackVersion from core.db.repository import OntologyRepository -from core.infra.exceptions import BadRequestError, ResourceNotFoundError, ServiceUnavailableError +from core.infra.exceptions import BadRequestError, ResourceNotFoundError from core.ontology.schemas import OntologyPackDocument -from core.ontology.validator import ( - DomainPackValidator, - build_runtime_payload, - register_runtime_asset_tags, -) +from core.ontology.validator import DomainPackValidator +from core.ontology.runtime_pack import build_runtime_payload from sqlalchemy import or_ from sqlalchemy.orm import Session -logger = logging.getLogger(__name__) - -def _normalize_ontology_tags(value: Any) -> list[str]: +def normalize_ontology_tags(value: Any) -> list[str]: if isinstance(value, str): items = value.replace("，", ",").split(",") elif isinstance(value, (list, tuple, set)): @@ -42,40 +26,6 @@ def _normalize_ontology_tags(value: Any) -> list[str]: return sorted({str(item).strip() for item in items if str(item).strip()}) -def disabled_ontology_runtime() -> dict[str, Any]: - """Return the canonical pass-through policy used when the user opts out.""" - return {"enabled": False, "packs": [], "review_level": "none"} - - -def build_ontology_runtime_for_preference( - *, - enabled: bool, - task: str, - db: Session, - pack_ids: list[str] | None = None, -) -> tuple[bool, dict[str, Any]]: - """Resolve the effective ontology switch and compile its runtime policy. - - A user's stored opt-in is subordinate to the administrator-controlled pack - availability. Disabling the selected/default packs is therefore an - intentional global off switch, not a runtime-policy failure. Once an - active pack exists, compilation remains fail-closed so malformed policy - data cannot silently bypass validation. - """ - if not enabled: - return False, disabled_ontology_runtime() - - service = OntologyService(db) - selected = pack_ids or None - if not service.repo.get_active_versions(selected): - return False, disabled_ontology_runtime() - - runtime = service.build_runtime(task=task, pack_ids=selected) - if not runtime.get("enabled"): - raise ServiceUnavailableError("本体校验运行时策略编译失败") - return True, runtime - - class OntologyService: def __init__(self, db: Session): self.db = db @@ -415,7 +365,7 @@ def resolve_asset_tags( .one_or_none() ) if row: - return _normalize_ontology_tags(row.tags) + return normalize_ontology_tags(row.tags) # A DB row that exists but is outside this user's scope must never # fall through to the process-global loader (which indexes private # skills for materialization). Only true filesystem/built-in skills @@ -426,10 +376,10 @@ def resolve_asset_tags( ) if db_row_exists: return [] - from core.agent_skills.loader import get_skill_loader + from core.capabilities.skills.loader_factory import get_skill_loader metadata = get_skill_loader().load_all_metadata().get(asset_id) - return _normalize_ontology_tags(getattr(metadata, "tags", [])) + return normalize_ontology_tags(getattr(metadata, "tags", [])) if kind == "tool": owner_filter = ( @@ -458,20 +408,20 @@ def resolve_asset_tags( for tool in row.tools_json or []: if not isinstance(tool, dict) or str(tool.get("name") or "") != asset_id: continue - tags.update(_normalize_ontology_tags(extra.get("ontology_tags"))) - tags.update(_normalize_ontology_tags(tool.get("tags"))) - tags.update(_normalize_ontology_tags(tool.get("ontology_tags"))) - tags.update(_normalize_ontology_tags(tool_tag_map.get(asset_id))) + tags.update(normalize_ontology_tags(extra.get("ontology_tags"))) + tags.update(normalize_ontology_tags(tool.get("tags"))) + tags.update(normalize_ontology_tags(tool.get("ontology_tags"))) + tags.update(normalize_ontology_tags(tool_tag_map.get(asset_id))) return sorted(tags) if kind == "subagent": - from core.services.user_agent_service import UserAgentService + from core.capabilities.agents.edition import UserAgentService try: agent = UserAgentService(self.db).get_by_id(asset_id, user_id=user_id or None) except (LookupError, PermissionError): return [] - return _normalize_ontology_tags(agent.get("ontology_tags")) + return normalize_ontology_tags(agent.get("ontology_tags")) return [] @@ -488,185 +438,3 @@ def _known_tool_schemas(self) -> tuple[dict[str, dict[str, Any]], set[str]]: if tool.get("inputSchema"): schemas[name] = {"inputSchema": tool["inputSchema"]} return schemas, known - - -def _runtime_needs_asset_tag_lookup( - runtime: dict[str, Any], - *, - kind: str, - asset_id: str, -) -> bool: - """Return whether tags could activate an otherwise-unmatched workflow.""" - activated = set(runtime.get("activated_workflows", [])) - for candidate in runtime.get("activation_candidates", []): - ref = f"{candidate.get('pack_id')}:{candidate.get('workflow_id')}" - if ref in activated: - continue - triggers = [ - trigger - for trigger in candidate.get("asset_triggers", []) - if trigger.get("kind") == kind - ] - if not triggers: - continue - if any(asset_id in set(trigger.get("ids", [])) for trigger in triggers): - continue - if any(trigger.get("tags_any") for trigger in triggers): - return True - return False - - -def resolve_runtime_asset_tags( - *, - runtime: dict[str, Any], - kind: str, - asset_id: str, - user_id: str, -) -> list[str]: - """Resolve and request-cache one invoked asset's trusted ontology tags. - - The runtime catalog doubles as a per-turn positive/negative cache. Metadata - failures are fail-closed so an asset cannot execute without the governance - policy that its server-authoritative tags may activate. - """ - if not runtime.get("enabled") or kind not in {"tool", "skill", "subagent"} or not asset_id: - return [] - catalog = runtime.setdefault("asset_tags", {}).setdefault(kind, {}) - if asset_id in catalog: - return list(catalog[asset_id]) - if not _runtime_needs_asset_tag_lookup(runtime, kind=kind, asset_id=asset_id): - register_runtime_asset_tags(runtime, kind=kind, asset_id=asset_id, tags=[]) - return [] - - try: - from core.db.engine import SessionLocal - - with SessionLocal() as db: - tags = OntologyService(db).resolve_asset_tags( - kind=kind, - asset_id=asset_id, - user_id=user_id, - ) - except Exception as exc: # noqa: BLE001 - logger.warning( - "[ontology] lazy asset tag lookup failed kind=%s asset_id=%s", - kind, - asset_id, - exc_info=True, - ) - raise ServiceUnavailableError("本体资产标签读取失败，本次资产调用已停止") from exc - - register_runtime_asset_tags(runtime, kind=kind, asset_id=asset_id, tags=tags) - return list(runtime["asset_tags"][kind][asset_id]) - - -def build_user_ontology_runtime( - *, - user_id: str, - task: str, - db: Session | None = None, -) -> tuple[bool, dict[str, Any]]: - """Resolve one user's opt-in and compile the task-scoped runtime policy. - - Web chat, channels, automation, batch, plan mode, and autonomous loops all - use this resolver so independent execution paths cannot silently bypass the - setting selected by the user. - """ - from core.services.user_service import UserService - - owns_session = db is None - if owns_session: - from core.db.engine import SessionLocal - - db = SessionLocal() - assert db is not None - try: - from core.services.ontology_policy import user_can_use_ontology_validation - - if not user_can_use_ontology_validation(db, user_id): - return False, disabled_ontology_runtime() - settings = UserService(db).get_user_settings(user_id) - opted_in = bool(settings.get("ontology_enabled", False)) - if not opted_in: - return False, disabled_ontology_runtime() - pack_ids = settings.get("ontology_pack_ids") - selected = ( - [str(item) for item in pack_ids if str(item).strip()] - if isinstance(pack_ids, list) - else None - ) - return build_ontology_runtime_for_preference( - enabled=opted_in, - task=task, - db=db, - pack_ids=selected or None, - ) - finally: - if owns_session: - db.close() - - -def record_enforcement_event(data: dict[str, Any]) -> None: - """Persist an event from middleware/reviewer using an isolated session.""" - from core.db.engine import SessionLocal - - db = SessionLocal() - try: - payload = dict(data) - payload.setdefault("event_id", f"ontoe_{uuid.uuid4().hex[:16]}") - OntologyRepository(db).create_event(payload) - except Exception: - db.rollback() - raise - finally: - db.close() - - -def record_runtime_activation( - event: dict[str, Any], - runtime: dict[str, Any], - *, - user_id: str | None, - chat_id: str | None, -) -> None: - """Persist one asset-driven workflow activation as append-only evidence.""" - - version_id = next( - ( - pack.get("version_id") - for pack in runtime.get("packs", []) - if pack.get("pack_id") == event.get("pack_id") - ), - None, - ) - record_enforcement_event( - { - "user_id": user_id, - "chat_id": chat_id, - "pack_id": event.get("pack_id"), - "version_id": version_id, - "rule_id": f"activation:{event.get('workflow_id')}", - "stage": "checkpoint", - "event_type": "runtime_activation", - "decision": "pass", - "mode": "enforce", - "target": f"{event.get('asset_kind')}:{event.get('asset_id')}", - "details": event, - } - ) - - -def record_review_run(data: dict[str, Any]) -> None: - from core.db.engine import SessionLocal - - db = SessionLocal() - try: - payload = dict(data) - payload.setdefault("review_id", f"ontor_{uuid.uuid4().hex[:16]}") - payload.setdefault("created_at", utc_now()) - OntologyRepository(db).create_review(payload) - except Exception: - db.rollback() - raise - finally: - db.close() diff --git a/src/backend/core/ontology/tool_hook.py b/src/backend/core/ontology/tool_hook.py new file mode 100644 index 00000000..1aaa00ad --- /dev/null +++ b/src/backend/core/ontology/tool_hook.py @@ -0,0 +1,183 @@ +"""Ontology tool admission and audit through neutral execution hooks.""" +import asyncio +import json +import logging +import time +from pathlib import Path +from typing import Any +from core.harness.context_items import reminder_item +from core.harness.events import thaw_value +from core.harness.hooks import Decision, DecisionKind, HookSpec, HookStage + +logger = logging.getLogger(__name__) + + +class OntologyToolHook: + def __init__(self, runtime=None): + self.runtime = dict(runtime or {}) + self.completed_tools = set() + self.denial_counts = {} + + async def __call__(self, invocation): + if invocation.stage == HookStage.AFTER_TOOL: + result = invocation.data.get("tool_result") or {} + if result.get("state") not in {"error", "denied", "interrupted"}: + self.completed_tools.add(invocation.operation_name) + return Decision() + from core.ontology.runtime_pack import activate_runtime_for_asset, render_runtime_prompt + from core.ontology.runtime_gates import evaluate_tool_call + from core.ontology.runtime_policy import resolve_runtime_asset_tags + runtime = thaw_value(invocation.data.get("ontology_runtime") or self.runtime) + if not runtime.get("enabled"): + return Decision() + identity = invocation.data.get("identity", {}) + started = time.monotonic() + tool_name = invocation.operation_name + tool_input = thaw_value(invocation.data.get("tool_args") or {}) + asset_kind, asset_id = self._resolve_invoked_asset(tool_name, tool_input) + await asyncio.to_thread(resolve_runtime_asset_tags, runtime=runtime, kind=asset_kind, + asset_id=asset_id, user_id=str(identity.get("user_id") or "")) + activations = activate_runtime_for_asset(runtime, kind=asset_kind, asset_id=asset_id) + items = list(invocation.data.get("context_items") or ()) + if activations: + contract = render_runtime_prompt(runtime) + if contract: + items.append(reminder_item( + "检测到受领域本体治理的运行时资产，策略已升级且本轮不可降级。\n" + contract, + "harness:ontology_gate")) + await self._audit_activations(identity, runtime, activations) + decision = evaluate_tool_call(runtime, tool_name=tool_name, tool_input=tool_input, + completed_tools=self.completed_tools) + patch = {"ontology_runtime": runtime, "context_items": items} + if decision.allowed: + await self._audit(identity, runtime, tool_name, decision, started) + return Decision.modify(patch) + key = ",".join(decision.matched_rule_ids) or tool_name + count = self.denial_counts[key] = self.denial_counts.get(key, 0) + 1 + configs = [pack.get("config", {}) for pack in runtime.get("packs", [])] + strategy = min((int(item.get("repeated_denial_threshold", 2)) for item in configs), default=2) + breaker = min((int(item.get("circuit_breaker_threshold", 5)) for item in configs), default=5) + guidance = list(decision.suggestions) + if count >= strategy: + guidance.append("同一规则已重复拦截，请改变执行策略，不要原样重试。") + if count >= breaker: + guidance.append("本体门禁已触发熔断；停止调用该工具，并向用户说明缺失条件。") + await self._audit(identity, runtime, tool_name, decision, started, + denial_count=count, circuit_breaker=count >= breaker) + if count == strategy and identity.get("chat_id"): + try: + from core.ontology.evolution import schedule_ontology_evolution + schedule_ontology_evolution(user_id=str(identity.get("user_id") or "system")) + except Exception as exc: + logger.warning("[ontology] evolution scheduling failed: %s", exc) + payload = {"code": "ONTOLOGY_GATE_DENIED", "tool": tool_name, + "violations": decision.violations, "suggestions": guidance, + "denial_count": count, "circuit_breaker": count >= breaker} + patch["tool_response"] = {"content": [{"type": "text", "text": json.dumps(payload, ensure_ascii=False)}], + "state": "denied", "metadata": {"ontology_gate": payload}} + return Decision(DecisionKind.REJECT, patch=patch, reason="ontology gate denied tool") + + async def _audit( + self, + identity: dict, + runtime: dict[str, Any], + tool_name: str, + decision, + started: float, + *, + denial_count: int = 0, + circuit_breaker: bool = False, + ) -> None: # noqa: ANN001 + if not decision.violations and not decision.matched_rule_ids: + return + runtime.setdefault("runtime_events", []).append( + { + "type": "ontology_gate", + "status": "completed", + "governance_run_id": runtime.get("governance_run_id"), + "stage": "tool", + "decision": decision.decision, + "tool_name": tool_name, + "matched_rule_ids": list(decision.matched_rule_ids), + "violations": list(decision.violations), + "suggestions": list(decision.suggestions), + "denial_count": denial_count, + "circuit_breaker": circuit_breaker, + } + ) + try: + from core.ontology.audit_events import record_enforcement_event + + first = decision.violations[0] if decision.violations else {} + pack_id = first.get("pack_id") + version_id = None + for pack in runtime.get("packs", []): + if pack.get("pack_id") == pack_id: + version_id = pack.get("version_id") + break + payload = { + "user_id": identity.get("user_id"), + "chat_id": identity.get("chat_id"), + "pack_id": pack_id, + "version_id": version_id, + "rule_id": first.get("rule_id") + or (decision.matched_rule_ids[0] if decision.matched_rule_ids else None), + "stage": "tool", + "event_type": "tool_call_gate", + "decision": decision.decision, + "mode": first.get("mode", "enforce"), + "target": tool_name, + "latency_ms": round((time.monotonic() - started) * 1000), + "details": { + "governance_run_id": runtime.get("governance_run_id"), + "violations": decision.violations, + "matched_rule_ids": decision.matched_rule_ids, + "suggestions": decision.suggestions, + "denial_count": denial_count, + "circuit_breaker": circuit_breaker, + }, + } + await asyncio.to_thread(record_enforcement_event, payload) + except Exception as exc: # noqa: BLE001 + logger.warning("[ontology] audit event persistence failed: %s", exc) + + + @staticmethod + def _resolve_invoked_asset(tool_name: str, tool_input: dict[str, Any]) -> tuple[str, str]: + if tool_name == "call_subagent" and tool_input.get("agent_id"): + return "subagent", str(tool_input["agent_id"]) + if tool_name == "view_text_file": + file_path = str(tool_input.get("file_path") or "") + path = Path(file_path) + if path.name == "SKILL.md" and path.parent.name: + return "skill", path.parent.name + return "tool", tool_name + + + @staticmethod + async def _audit_activations( + identity: dict, + runtime: dict[str, Any], + activations: list[dict[str, Any]], + ) -> None: + try: + from core.ontology.audit_events import record_runtime_activation + + for event in activations: + await asyncio.to_thread( + record_runtime_activation, + event, + runtime, + user_id=identity.get("user_id"), + chat_id=identity.get("chat_id"), + ) + except Exception as exc: # noqa: BLE001 + logger.warning("[ontology] activation audit persistence failed: %s", exc) + + + +def ontology_tool_hooks(runtime): + policy = OntologyToolHook(runtime) + return (HookSpec("ontology_admission", HookStage.BEFORE_TOOL, policy, priority=500, + timeout_ms=30000, mutable_fields=frozenset({"ontology_runtime", "context_items", "tool_response"})), + HookSpec("ontology_completion", HookStage.AFTER_TOOL, policy, priority=300)) diff --git a/src/backend/core/ontology/validation_models.py b/src/backend/core/ontology/validation_models.py new file mode 100644 index 00000000..cda206d4 --- /dev/null +++ b/src/backend/core/ontology/validation_models.py @@ -0,0 +1,33 @@ +"""Validation issues, reports and gate decisions.""" +from __future__ import annotations +from dataclasses import asdict, dataclass, field +from typing import Any + +@dataclass(slots=True) +class ValidationIssue: + severity: str + path: str + message: str + + +@dataclass(slots=True) +class ValidationReport: + valid: bool + errors: list[ValidationIssue] = field(default_factory=list) + warnings: list[ValidationIssue] = field(default_factory=list) + + def as_dict(self) -> dict[str, Any]: + return { + "valid": self.valid, + "errors": [asdict(item) for item in self.errors], + "warnings": [asdict(item) for item in self.warnings], + } + + +@dataclass(slots=True) +class OntologyGateDecision: + allowed: bool + decision: str + violations: list[dict[str, Any]] = field(default_factory=list) + suggestions: list[str] = field(default_factory=list) + matched_rule_ids: list[str] = field(default_factory=list) diff --git a/src/backend/core/ontology/validator.py b/src/backend/core/ontology/validator.py index 02638c2c..dcf9092b 100644 --- a/src/backend/core/ontology/validator.py +++ b/src/backend/core/ontology/validator.py @@ -1,47 +1,10 @@ -"""Domain Pack validation and deterministic runtime gates.""" - +"""Validate ontology packs before activation.""" from __future__ import annotations - -import json -import re -import uuid -from dataclasses import asdict, dataclass, field from typing import Any, Iterable - -from core.ontology.schemas import Constraint, OntologyPackDocument, Workflow -from jsonschema import Draft202012Validator, ValidationError +from core.ontology.schemas import OntologyPackDocument +from jsonschema import Draft202012Validator from pydantic import ValidationError as PydanticValidationError - - -@dataclass(slots=True) -class ValidationIssue: - severity: str - path: str - message: str - - -@dataclass(slots=True) -class ValidationReport: - valid: bool - errors: list[ValidationIssue] = field(default_factory=list) - warnings: list[ValidationIssue] = field(default_factory=list) - - def as_dict(self) -> dict[str, Any]: - return { - "valid": self.valid, - "errors": [asdict(item) for item in self.errors], - "warnings": [asdict(item) for item in self.warnings], - } - - -@dataclass(slots=True) -class OntologyGateDecision: - allowed: bool - decision: str - violations: list[dict[str, Any]] = field(default_factory=list) - suggestions: list[str] = field(default_factory=list) - matched_rule_ids: list[str] = field(default_factory=list) - +from core.ontology import validation_models class DomainPackValidator: """Validate structure, JSON Schema, and tool references before activation.""" @@ -52,21 +15,21 @@ def validate( *, tool_schemas: dict[str, dict[str, Any]] | None = None, known_tools: Iterable[str] = (), - ) -> tuple[OntologyPackDocument | None, ValidationReport]: - errors: list[ValidationIssue] = [] - warnings: list[ValidationIssue] = [] + ) -> tuple[OntologyPackDocument | None, validation_models.ValidationReport]: + errors: list[validation_models.ValidationIssue] = [] + warnings: list[validation_models.ValidationIssue] = [] try: document = OntologyPackDocument.model_validate(payload) except PydanticValidationError as exc: for item in exc.errors(include_url=False): errors.append( - ValidationIssue( + validation_models.ValidationIssue( "error", ".".join(str(part) for part in item["loc"]), item["msg"], ) ) - return None, ValidationReport(False, errors, warnings) + return None, validation_models.ValidationReport(False, errors, warnings) schemas = tool_schemas or {} known = set(known_tools) | set(schemas) @@ -76,12 +39,12 @@ def validate( try: Draft202012Validator.check_schema(rule.schema_) except Exception as exc: # SchemaError is version-specific - errors.append(ValidationIssue("error", f"{path}.schema", str(exc))) + errors.append(validation_models.ValidationIssue("error", f"{path}.schema", str(exc))) tool_name = rule.target.tool if not tool_name: continue if tool_name not in known: - issue = ValidationIssue( + issue = validation_models.ValidationIssue( "warning" if document.config.allow_unresolved_tools else "error", f"{path}.target.tool", f"unknown tool reference: {tool_name}", @@ -89,10 +52,10 @@ def validate( (warnings if document.config.allow_unresolved_tools else errors).append(issue) continue if rule.target.kind == "tool_parameter" and tool_name in schemas: - properties = _tool_input_schema(schemas[tool_name]).get("properties", {}) + properties = tool_input_schema(schemas[tool_name]).get("properties", {}) if rule.target.parameter not in properties: errors.append( - ValidationIssue( + validation_models.ValidationIssue( "error", f"{path}.target.parameter", f"unknown parameter {rule.target.parameter!r} for tool {tool_name}", @@ -102,7 +65,7 @@ def validate( for index, workflow in enumerate(document.workflows): for tool_name in set(workflow.required_tools + workflow.forbidden_tools): if tool_name not in known: - issue = ValidationIssue( + issue = validation_models.ValidationIssue( "warning" if document.config.allow_unresolved_tools else "error", f"workflows.{index}", f"unknown tool reference: {tool_name}", @@ -111,7 +74,7 @@ def validate( overlap = set(workflow.required_tools) & set(workflow.forbidden_tools) if overlap: errors.append( - ValidationIssue( + validation_models.ValidationIssue( "error", f"workflows.{index}", f"tools cannot be both required and forbidden: {sorted(overlap)}", @@ -123,579 +86,18 @@ def validate( for tool_name in trigger.ids: if tool_name in known: continue - issue = ValidationIssue( + issue = validation_models.ValidationIssue( "warning" if document.config.allow_unresolved_tools else "error", f"workflows.{index}.asset_triggers.{trigger_index}.ids", f"unknown tool reference: {tool_name}", ) (warnings if document.config.allow_unresolved_tools else errors).append(issue) - return document, ValidationReport(not errors, errors, warnings) + return document, validation_models.ValidationReport(not errors, errors, warnings) -def _tool_input_schema(schema: dict[str, Any]) -> dict[str, Any]: +def tool_input_schema(schema: dict[str, Any]) -> dict[str, Any]: function = schema.get("function") if isinstance(schema, dict) else None if isinstance(function, dict): return function.get("parameters") or function.get("inputSchema") or {} return schema.get("inputSchema") or schema.get("parameters") or schema - - -def select_workflows(document: OntologyPackDocument, task: str) -> list[Workflow]: - """Select workflows by case-insensitive lexical trigger match.""" - normalized = task.casefold() - return [ - workflow - for workflow in document.workflows - if any(trigger.casefold() in normalized for trigger in workflow.triggers) - ] - - -_REVIEW_RANK = {"none": 0, "checkpoint": 1, "committee": 2} - - -def _workflow_constraints( - document: OntologyPackDocument, - workflows: list[Workflow], -) -> list[Constraint]: - workflow_tools = { - tool_name - for workflow in workflows - for tool_name in (workflow.required_tools + workflow.forbidden_tools) - } - workflow_output_tags = {tag for workflow in workflows for tag in workflow.output_tags} - return [ - rule - for rule in document.constraints - if rule.enabled - and ( - rule.target.tool in workflow_tools - if rule.target.kind in {"tool", "tool_parameter"} - else rule.target.output_tag in workflow_output_tags - ) - ] - - -def _runtime_concepts( - document: OntologyPackDocument, - task: str, - constraints: list[Constraint], -) -> list[dict[str, Any]]: - """Select task concepts while always retaining concepts used by active rules.""" - by_id = {item.id: item for item in document.concepts} - selected: list[Any] = [] - selected_ids: set[str] = set() - - def add_with_parents(concept_id: str | None) -> None: - chain: list[Any] = [] - current = by_id.get(concept_id or "") - while current is not None and current.id not in selected_ids: - chain.append(current) - current = by_id.get(current.parent_id or "") - for item in reversed(chain): - if len(selected) >= document.config.max_concepts: - return - if item.id not in selected_ids: - selected.append(item) - selected_ids.add(item.id) - - for rule in constraints: - add_with_parents(rule.concept_id) - for item in select_concepts(document, task, limit=document.config.max_concepts): - add_with_parents(item["id"]) - return [item.model_dump() for item in selected[: document.config.max_concepts]] - - -def _compile_pack_runtime( - document: OntologyPackDocument, - task: str, - workflows: list[Workflow], -) -> dict[str, Any]: - constraints = _workflow_constraints(document, workflows) - concepts = _runtime_concepts(document, task, constraints) - selected_concept_ids = {item["id"] for item in concepts} - relations = [ - relation.model_dump() - for relation in document.relations - if relation.subject in selected_concept_ids and relation.object in selected_concept_ids - ] - return { - "pack_id": document.pack_id, - "version": document.version, - "name": document.name, - "domain": document.domain, - "description": document.description, - "config": document.config.model_dump(), - "concepts": concepts, - "relations": relations, - "constraints": [item.model_dump(by_alias=True) for item in constraints], - "workflows": [item.model_dump() for item in workflows], - } - - -def select_concepts( - document: OntologyPackDocument, - task: str, - *, - limit: int | None = None, -) -> list[dict[str, Any]]: - """Select only task-relevant concepts; fall back to tagged roots.""" - normalized = task.casefold() - # English words can be tokenized by boundaries. Chinese text cannot be - # treated as one giant ``[\u4e00-\u9fff]+`` token: doing so made a task such - # as “数据质量分析” match no concept at all. Concept names and aliases are - # therefore also checked directly as substrings of the task. - terms = set(re.findall(r"[a-z0-9_]+", normalized)) - scored: list[tuple[int, Any]] = [] - for concept in document.concepts: - labels = [concept.id, concept.name, *concept.aliases, *concept.tags] - haystack = " ".join([*labels, concept.definition]).casefold() - score = sum(1 for term in terms if len(term) >= 2 and term in haystack) - score += sum( - 3 for label in labels if len(label.strip()) >= 2 and label.casefold() in normalized - ) - if score or not concept.parent_id: - scored.append((score, concept)) - max_items = limit or document.config.max_concepts - scored.sort(key=lambda item: (-item[0], item[1].id)) - selected = [item for _, item in scored[:max_items]] - by_id = {item.id: item for item in document.concepts} - selected_ids = {item.id for item in selected} - # Preserve the hierarchy for every selected concept when the budget allows. - for concept in list(selected): - parent_id = concept.parent_id - while parent_id and len(selected) < max_items: - if parent_id not in selected_ids and parent_id in by_id: - selected.append(by_id[parent_id]) - selected_ids.add(parent_id) - parent_id = by_id.get(parent_id).parent_id if parent_id in by_id else None - return [item.model_dump() for item in selected[:max_items]] - - -def build_runtime_payload( - documents: list[OntologyPackDocument], - task: str, -) -> dict[str, Any]: - """Compile active packs into a compact, serializable request policy.""" - governance_run_id = f"ontog_{uuid.uuid4().hex[:16]}" - packs: list[dict[str, Any]] = [] - activation_candidates: list[dict[str, Any]] = [] - runtime_events: list[dict[str, Any]] = [] - activated_workflows: list[str] = [] - review_level = "none" - for document in documents: - workflows = select_workflows(document, task) - for workflow in workflows: - if _REVIEW_RANK[workflow.review_level] > _REVIEW_RANK[review_level]: - review_level = workflow.review_level - ref = f"{document.pack_id}:{workflow.id}" - activated_workflows.append(ref) - runtime_events.append( - { - "type": "ontology_activation", - "status": "completed", - "governance_run_id": governance_run_id, - "source": "text", - "pack_id": document.pack_id, - "workflow_id": workflow.id, - "workflow_name": workflow.name, - "review_level": workflow.review_level, - } - ) - packs.append(_compile_pack_runtime(document, task, workflows)) - for workflow in document.workflows: - if not workflow.asset_triggers: - continue - activation_candidates.append( - { - "pack_id": document.pack_id, - "workflow_id": workflow.id, - "workflow_name": workflow.name, - "review_level": workflow.review_level, - "asset_triggers": [item.model_dump() for item in workflow.asset_triggers], - "pack": _compile_pack_runtime(document, task, [workflow]), - } - ) - return { - "enabled": bool(packs), - "governance_run_id": governance_run_id, - "packs": packs, - "review_level": review_level, - "task": task, - "activation_candidates": activation_candidates, - "activated_workflows": activated_workflows, - "asset_tags": {"tool": {}, "skill": {}, "subagent": {}}, - "runtime_events": runtime_events, - "review_owner": "outer_workflow", - "output_review": {"status": "pending", "owner": None, "count": 0}, - } - - -def register_runtime_asset_tags( - runtime: dict[str, Any], - *, - kind: str, - asset_id: str, - tags: Iterable[str], -) -> None: - """Attach trusted server-side asset metadata to a request runtime.""" - if kind not in {"tool", "skill", "subagent"} or not asset_id: - return - catalog = runtime.setdefault("asset_tags", {}).setdefault(kind, {}) - existing = {str(item) for item in catalog.get(asset_id, []) if str(item)} - existing.update(str(item) for item in tags if str(item)) - catalog[asset_id] = sorted(existing) - - -def activate_runtime_for_asset( - runtime: dict[str, Any], - *, - kind: str, - asset_id: str, - tags: Iterable[str] = (), -) -> list[dict[str, Any]]: - """Monotonically merge workflow policy when an actual asset is invoked.""" - if not runtime.get("enabled") or kind not in {"tool", "skill", "subagent"}: - return [] - known_tags = set(str(item) for item in tags if str(item)) - known_tags.update( - str(item) - for item in runtime.get("asset_tags", {}).get(kind, {}).get(asset_id, []) - if str(item) - ) - activated = set(runtime.setdefault("activated_workflows", [])) - events: list[dict[str, Any]] = [] - for candidate in runtime.get("activation_candidates", []): - ref = f"{candidate.get('pack_id')}:{candidate.get('workflow_id')}" - if ref in activated: - continue - matched_tags: set[str] = set() - matched = False - for trigger in candidate.get("asset_triggers", []): - if trigger.get("kind") != kind: - continue - if asset_id in set(trigger.get("ids", [])): - matched = True - tag_hits = known_tags & set(trigger.get("tags_any", [])) - if tag_hits: - matched = True - matched_tags.update(tag_hits) - if not matched: - continue - - fragment = candidate.get("pack") or {} - current = next( - ( - pack - for pack in runtime.get("packs", []) - if pack.get("pack_id") == fragment.get("pack_id") - ), - None, - ) - if current is None: - current = fragment - runtime.setdefault("packs", []).append(current) - else: - for field, key in ( - ("workflows", "id"), - ("constraints", "id"), - ("concepts", "id"), - ("relations", "id"), - ): - existing_ids = {item.get(key) for item in current.get(field, [])} - current.setdefault(field, []).extend( - item for item in fragment.get(field, []) if item.get(key) not in existing_ids - ) - if fragment.get("version_id"): - current["version_id"] = fragment["version_id"] - - level = str(candidate.get("review_level") or "none") - if _REVIEW_RANK.get(level, 0) > _REVIEW_RANK.get(runtime.get("review_level"), 0): - runtime["review_level"] = level - activated.add(ref) - runtime["activated_workflows"] = sorted(activated) - event = { - "type": "ontology_activation", - "status": "completed", - "governance_run_id": runtime.get("governance_run_id"), - "source": kind, - "asset_kind": kind, - "asset_id": asset_id, - "asset_tags": sorted(known_tags), - "matched_tags": sorted(matched_tags), - "pack_id": candidate.get("pack_id"), - "workflow_id": candidate.get("workflow_id"), - "workflow_name": candidate.get("workflow_name"), - "review_level": level, - } - runtime.setdefault("runtime_events", []).append(event) - events.append(event) - return events - - -def render_runtime_prompt(runtime: dict[str, Any]) -> str: - """Render the bounded ontology segment injected into the system prompt.""" - if not runtime.get("enabled"): - return "" - compact_packs = [] - for pack in runtime.get("packs", []): - compact_pack = { - "id": pack["pack_id"], - "version": pack["version"], - "concepts": ( - list(pack.get("concepts", [])) - if pack.get("config", {}).get("injection_enabled", True) - else [] - ), - "relations": ( - list(pack.get("relations", [])) - if pack.get("config", {}).get("injection_enabled", True) - else [] - ), - "workflows": pack.get("workflows", []), - "constraints": [ - { - "id": rule["id"], - "target": rule["target"], - "mode": rule["mode"], - "message": rule["message"], - "suggestion": rule.get("suggestion", ""), - "prerequisite_tools": rule.get("prerequisite_tools", []), - } - for rule in pack.get("constraints", []) - ], - } - char_budget = int(pack.get("config", {}).get("token_budget", 2500)) * 4 - while len(json.dumps(compact_pack, ensure_ascii=False)) > char_budget: - if compact_pack["relations"]: - compact_pack["relations"].pop() - continue - if compact_pack["concepts"]: - removed = compact_pack["concepts"].pop() - remaining = {item["id"] for item in compact_pack["concepts"]} - compact_pack["relations"] = [ - relation - for relation in compact_pack["relations"] - if relation["subject"] in remaining and relation["object"] in remaining - ] - continue - break - compact_packs.append(compact_pack) - compact = {"packs": compact_packs} - return ( - "\n" - "本轮已启用领域本体校验。以下内容是可执行的领域契约；工具门禁会独立执行，" - "请在调用前主动满足参数、前置证据与工作流要求。不要虚构证据。\n" - + json.dumps(compact, ensure_ascii=False, separators=(",", ":")) - + "\n" - ) - - -def requires_output_review(runtime: dict[str, Any]) -> bool: - """Whether an L-b/L-c review must run after the streamed draft.""" - return bool( - runtime.get("enabled") - and ( - runtime.get("review_level") != "none" - or any( - (rule.get("target") or {}).get("kind") == "output" - for pack in runtime.get("packs", []) - for rule in pack.get("constraints", []) - ) - ) - ) - - -def claim_output_review(runtime: dict[str, Any], *, owner: str) -> bool: - """Atomically claim the single final-answer review for one governance run.""" - if not requires_output_review(runtime): - return False - state = runtime.setdefault( - "output_review", - {"status": "pending", "owner": None, "count": 0}, - ) - if state.get("status") in {"running", "completed"}: - return False - state.update({"status": "running", "owner": owner}) - return True - - -def complete_output_review( - runtime: dict[str, Any], - *, - owner: str, - verdict: str, - attempts: int = 1, -) -> None: - """Mark the claimed review complete and retain its single-run evidence.""" - state = runtime.setdefault("output_review", {}) - if state.get("status") != "running" or state.get("owner") != owner: - return - count = int(state.get("count") or 0) + max(1, int(attempts or 1)) - state.update( - { - "status": "completed", - "owner": owner, - "count": count, - "verdict": verdict, - } - ) - - -def release_output_review(runtime: dict[str, Any], *, owner: str) -> None: - """Release a failed claim so a safe retry can review the final answer.""" - state = runtime.setdefault("output_review", {}) - if state.get("status") == "running" and state.get("owner") == owner: - state.update({"status": "pending", "owner": None}) - - -def evaluate_tool_call( - runtime: dict[str, Any], - *, - tool_name: str, - tool_input: dict[str, Any], - completed_tools: Iterable[str] = (), -) -> OntologyGateDecision: - """Apply zero-LLM L-a validation to one tool call.""" - if not runtime.get("enabled"): - return OntologyGateDecision(True, "pass") - completed = set(completed_tools) - violations: list[dict[str, Any]] = [] - suggestions: list[str] = [] - matched: list[str] = [] - enforced = False - - for pack in runtime.get("packs", []): - for workflow in pack.get("workflows", []): - if tool_name in workflow.get("forbidden_tools", []): - violations.append( - { - "rule_id": f"workflow:{workflow['id']}:forbidden", - "message": f"工作流 {workflow['name']} 禁止调用工具 {tool_name}", - "mode": "enforce", - } - ) - enforced = True - for rule in pack.get("constraints", []): - target = rule.get("target") or {} - if target.get("tool") != tool_name: - continue - matched.append(rule["id"]) - missing_prerequisites = [ - name for name in rule.get("prerequisite_tools", []) if name not in completed - ] - rule_errors: list[str] = [] - if missing_prerequisites: - rule_errors.append("缺少前置工具调用: " + ", ".join(missing_prerequisites)) - try: - if target.get("kind") == "tool": - Draft202012Validator(rule.get("schema") or {}).validate(tool_input) - elif target.get("kind") == "tool_parameter": - parameter = target.get("parameter") - Draft202012Validator(rule.get("schema") or {}).validate( - tool_input.get(parameter) - ) - except ValidationError as exc: - rule_errors.append(exc.message) - if rule_errors: - violations.append( - { - "pack_id": pack.get("pack_id"), - "rule_id": rule["id"], - "message": rule["message"], - "reasons": rule_errors, - "mode": rule.get("mode", "log"), - "risk": rule.get("risk", "low"), - } - ) - if rule.get("suggestion"): - suggestions.append(rule["suggestion"]) - if rule.get("mode") == "enforce": - enforced = True - - if not violations: - return OntologyGateDecision(True, "pass", matched_rule_ids=matched) - return OntologyGateDecision( - allowed=not enforced, - decision="deny" if enforced else "log", - violations=violations, - suggestions=suggestions, - matched_rule_ids=matched, - ) - - -def evaluate_output( - runtime: dict[str, Any], - *, - answer: str, - citations: Iterable[dict[str, Any]] = (), - completed_tools: Iterable[str] | None = (), -) -> OntologyGateDecision: - """Apply deterministic output constraints before any LLM reviewer.""" - if not runtime.get("enabled"): - return OntologyGateDecision(True, "pass") - citation_items = list(citations) - # ``None`` means the caller cannot observe a complete tool trace (the - # legacy non-streaming reply path). Output and citation rules still run, - # but required-tool rules must not produce a false omission claim. - completed = set(completed_tools) if completed_tools is not None else None - violations: list[dict[str, Any]] = [] - matched: list[str] = [] - suggestions: list[str] = [] - enforced = False - for pack in runtime.get("packs", []): - if completed is not None: - for workflow in pack.get("workflows", []): - missing_tools = sorted(set(workflow.get("required_tools", [])) - completed) - if missing_tools: - violations.append( - { - "pack_id": pack.get("pack_id"), - "rule_id": f"workflow:{workflow.get('id')}:required", - "message": "领域工作流缺少必需的证据工具调用。", - "reasons": ["未调用: " + ", ".join(missing_tools)], - "mode": "enforce", - "risk": workflow.get("risk", "low"), - } - ) - enforced = True - output_tags = { - tag for workflow in pack.get("workflows", []) for tag in workflow.get("output_tags", []) - } - for rule in pack.get("constraints", []): - target = rule.get("target") or {} - if target.get("kind") != "output": - continue - if output_tags and target.get("output_tag") not in output_tags: - continue - matched.append(rule["id"]) - reasons: list[str] = [] - if rule.get("requires_citations") and not citation_items: - reasons.append("输出缺少可追溯引用证据") - try: - Draft202012Validator(rule.get("schema") or {}).validate(answer) - except ValidationError as exc: - reasons.append(exc.message) - if reasons: - violations.append( - { - "pack_id": pack.get("pack_id"), - "rule_id": rule["id"], - "message": rule["message"], - "reasons": reasons, - "mode": rule.get("mode", "log"), - "risk": rule.get("risk", "low"), - } - ) - if rule.get("suggestion"): - suggestions.append(rule["suggestion"]) - if rule.get("mode") == "enforce": - enforced = True - if not violations: - return OntologyGateDecision(True, "pass", matched_rule_ids=matched) - return OntologyGateDecision( - allowed=not enforced, - decision="deny" if enforced else "log", - violations=violations, - suggestions=suggestions, - matched_rule_ids=matched, - ) diff --git a/src/backend/core/plugins/README.md b/src/backend/core/plugins/README.md deleted file mode 100644 index e04e17ae..00000000 --- a/src/backend/core/plugins/README.md +++ /dev/null @@ -1,28 +0,0 @@ -# Plugins - -Plugin code has one domain owner: this package. Neither `core/services/` nor -`core/llm/` contains another plugin implementation. - -| Package | Responsibility | Main entry | -| --- | --- | --- | -| `management/` | Install, enable, uninstall, inspect and publish database-backed plugins | `core.plugins.management` | -| `packaging/` | Discover files, extract archives and normalize plugin manifests | `importer.normalize_plugin_dir`, `archive` | -| `runtime/` | Resolve deferred capabilities, activate tools/skills and restore per-chat activation | `core.plugins.runtime` | -| `local/` | Device-local installation and migration of managed bundles | `local.service` | -| `ui/contract.py` | Normalize UI declarations and project browser-safe data | `normalize_ui`, `public_contributions` | -| `ui/declarations.py`, `ui/primitives.py` | Contribution-specific parsing and shared validation rules | Internal to UI ingestion | -| `ui/contributions.py`, `ui/data_proxy.py` | Resolve installed UI and execute authorized data queries | API routes call these explicitly | - -Runtime code may read installation state; installation and packaging must not -import the agent factory or plugin runtime. Packaging does not publish database -records: safe ZIP extraction lives in `packaging/archive.py`, while marketplace -persistence lives in `management/packages.py`. - -Add a new manifest format in packaging, lifecycle behavior in management, and -activation behavior in runtime. Add UI declaration kinds in declarations and -host renderer support together. Do not add another `plugin_*` sibling to the -LLM or general services directories. Internal imports and test patch targets -use the new owner paths; old module paths are intentionally removed. - -Bundled resource paths are resolved relative to the backend root, including -in Docker. Package discovery uses the existing `core*` setuptools inclusion. diff --git a/src/backend/core/plugins/management/__init__.py b/src/backend/core/plugins/management/__init__.py deleted file mode 100644 index 289ebf54..00000000 --- a/src/backend/core/plugins/management/__init__.py +++ /dev/null @@ -1,80 +0,0 @@ -"""Public plugin lifecycle API; implementations follow domain ownership.""" - -from core.plugins.management.admin_config import get_plugin_admin_config, set_plugin_admin_config -from core.plugins.management.details import ( - exclude_market_skill, - get_market_skill_exclusions, - get_plugin_detail, -) -from core.plugins.management.installation import ( - ensure_default_plugins_bootstrapped, - import_plugin, - import_plugin_from_zip, - install_plugin, -) -from core.plugins.management.lifecycle import ( - set_installed_plugin_meta, - set_plugin_component_enabled, - set_plugin_component_enabled_for_user, - set_plugin_enabled, - set_plugin_enabled_for_user, - uninstall_plugin, -) -from core.plugins.management.market import ( - list_plugins, - resolve_market_meta, - set_market_meta, - set_plugin_market_enabled, -) -from core.plugins.management.packages import delete_market_package, publish_plugin_zip_to_market -from core.plugins.management.queries import get_installed_detail, list_installed -from core.plugins.packaging.definitions import ( - BUILTIN_PLUGIN_MARKET_META, - DEFAULT_BOOTSTRAP_MARKER_ID, - DEFAULT_BOOTSTRAP_PLUGIN_SLUGS, - LOCAL_BOOTSTRAP_MARKER_NAME, - MAX_ICON_LEN, - MAX_ICON_URL_LEN, - MAX_ZIP_BYTES, - PLUGIN_BUNDLES_DIR, - PLUGIN_MARKET_META_BLOCK_ID, - PLUGIN_SOURCE_DIRS, -) -from core.plugins.packaging.sources import builtin_plugin_component_ids - -__all__ = [ - "resolve_market_meta", - "set_market_meta", - "builtin_plugin_component_ids", - "list_plugins", - "set_plugin_market_enabled", - "list_installed", - "get_installed_detail", - "get_plugin_admin_config", - "set_plugin_admin_config", - "get_market_skill_exclusions", - "get_plugin_detail", - "exclude_market_skill", - "install_plugin", - "ensure_default_plugins_bootstrapped", - "import_plugin", - "import_plugin_from_zip", - "publish_plugin_zip_to_market", - "delete_market_package", - "uninstall_plugin", - "set_plugin_enabled", - "set_plugin_component_enabled", - "set_plugin_enabled_for_user", - "set_plugin_component_enabled_for_user", - "set_installed_plugin_meta", - "PLUGIN_BUNDLES_DIR", - "PLUGIN_SOURCE_DIRS", - "MAX_ZIP_BYTES", - "DEFAULT_BOOTSTRAP_PLUGIN_SLUGS", - "DEFAULT_BOOTSTRAP_MARKER_ID", - "LOCAL_BOOTSTRAP_MARKER_NAME", - "PLUGIN_MARKET_META_BLOCK_ID", - "BUILTIN_PLUGIN_MARKET_META", - "MAX_ICON_LEN", - "MAX_ICON_URL_LEN", -] diff --git a/src/backend/core/plugins/resources/owners.py b/src/backend/core/plugins/resources/owners.py deleted file mode 100644 index 142e26d7..00000000 --- a/src/backend/core/plugins/resources/owners.py +++ /dev/null @@ -1,112 +0,0 @@ -"""Resource owner validation and provider session identity.""" - -import hashlib -import json - -from core.config.local_mode import local_mode_enabled -from fastapi import HTTPException - - -def execution_scope(): - return "local" if local_mode_enabled() else "cloud" - - -def runtime_session(row): - if row.chat_id is not None: - return row.chat_id - # Provider session IDs are execution containers, never conversation records. - binding = [ - row.user_id, - row.application_key, - row.install_id, - row.revision, - row.scope, - row.instance_id, - ] - return "workbench-" + hashlib.sha256(json.dumps(binding).encode()).hexdigest() - - -def application_binding(db, user_id, key, install_id, revision, scope, module_id, selected=None): - from core.applications import catalog - - item = catalog.readable(db, user_id, key) - if not item["availability"]["available"]: - raise HTTPException(403, "application_unavailable") - binding = item.get("binding", {}) - if ( - item["declaration"]["target"]["kind"] != "plugin" - or binding.get("installId") != install_id - or ( - binding.get("packageRevision") != revision - and (selected is None or selected.revision != revision) - ) - or scope != execution_scope() - or (binding.get("origin") is not None and binding["origin"].get("kind") != scope) - ): - raise HTTPException(409, "resource_binding_mismatch") - target = item["declaration"]["target"] - if selected is not None and binding.get("packageRevision") != revision: - previous = next( - ( - a - for a in (selected.ui.get("contributes") or {}).get("applications", []) - if a["id"] == item["declaration"]["id"] - ), - None, - ) - if previous is None: - raise HTTPException(403, "resource_application_removed") - target = previous["target"] - allowed = {target.get("moduleId")} | { - view.get("moduleId") for view in item["declaration"].get("views", []) - } - if module_id not in allowed: - raise HTTPException(403, "resource_module_not_granted") - return item - - -def resolve_installation(db, row, user_id): - from .installation import resolve - - if row.application_key is not None: - from core.applications.installation_binding import exact as resolve - - selected = resolve(db, row.slug, user_id, row.install_id) - if row.application_key is not None and selected.install_id != row.install_id: - raise HTTPException(403, "resource_installation_mismatch") - if selected.revision == row.revision: - return selected - if row.application_key is None: - raise HTTPException(409, "plugin_revision_changed") - try: - from core.applications.plugin_revisions import resolve_revision - except ModuleNotFoundError as error: - if error.name != "core.applications.plugin_revisions": - raise - raise HTTPException(409, "revision_retention_unavailable") from None - # WB13 rechecks current catalog permissions and the exact live instance lease. - return resolve_revision(db, user_id, row.application_key, row.revision, row.instance_id) - - -def validate(db, row, user_id, selected=None): - if row.chat_id is not None: - if row.application_key is not None or row.instance_id is not None: - raise HTTPException(403, "resource_owner_invalid") - from core.db.models import ChatSession - - chat = db.get(ChatSession, row.chat_id) - if chat is None or chat.user_id != user_id or chat.deleted_at is not None: - raise HTTPException(403, "conversation_unavailable") - else: - if not row.application_key or not row.instance_id: - raise HTTPException(403, "resource_owner_invalid") - application_binding( - db, - user_id, - row.application_key, - row.install_id, - row.revision, - row.scope, - row.module_id, - selected, - ) diff --git a/src/backend/core/projects/__init__.py b/src/backend/core/projects/__init__.py new file mode 100644 index 00000000..51ddcc2b --- /dev/null +++ b/src/backend/core/projects/__init__.py @@ -0,0 +1 @@ +"""Projects: access, files, environment and personal workspaces.""" diff --git a/src/backend/core/services/project_access.py b/src/backend/core/projects/access.py similarity index 90% rename from src/backend/core/services/project_access.py rename to src/backend/core/projects/access.py index 9c673659..13b546cc 100644 --- a/src/backend/core/services/project_access.py +++ b/src/backend/core/projects/access.py @@ -6,7 +6,7 @@ def require_visible_project(db: Session, user_id: str, project_id: str): """Return the live project the user may see; otherwise 404 so existence never leaks.""" - from core.auth.permissions_iface import resolve_project_permission + from core.auth.edition import resolve_project_permission from core.db.models import Project project = ( diff --git a/src/backend/core/services/project_api_service.py b/src/backend/core/projects/api.py similarity index 97% rename from src/backend/core/services/project_api_service.py rename to src/backend/core/projects/api.py index 2f17a501..dddc2d41 100644 --- a/src/backend/core/services/project_api_service.py +++ b/src/backend/core/projects/api.py @@ -9,7 +9,7 @@ def require_project_key_access(db, user_id: str, project_id: str): - from core.auth.permissions_iface import resolve_project_permission + from core.auth.edition import resolve_project_permission project = db.get(Project, project_id) if project is None or project.deleted_at is not None: diff --git a/src/backend/core/services/project_service.py b/src/backend/core/projects/base.py similarity index 85% rename from src/backend/core/services/project_service.py rename to src/backend/core/projects/base.py index 28c3e6cb..8581610e 100644 --- a/src/backend/core/services/project_service.py +++ b/src/backend/core/projects/base.py @@ -1,85 +1,20 @@ """Personal-project service for Community Edition.""" from __future__ import annotations +from core.projects import project_names, personal_summary import uuid from core.infra.time import utc_now from typing import Any, Dict, List, Optional, Tuple -from core.auth.permissions_iface import ProjectPermissionLevel, resolve_project_permission +from core.auth.edition import ProjectPermissionLevel, resolve_project_permission from core.db.models import Artifact, ChatSession, Project, ProjectFavorite, UserFolder, UserShadow from fastapi import HTTPException from sqlalchemy import desc, func, or_ from sqlalchemy.orm import Session -def _slugify(name: str) -> str: - """Lowercase, ASCII-safe, hyphen-joined slug for the local sandbox mount name.""" - out = [] - for ch in (name or "").strip().lower(): - if ch.isalnum() and ord(ch) < 128: - out.append(ch) - elif ch in (" ", "-", "_", "."): - out.append("-") - slug = "".join(out).strip("-") - while "--" in slug: - slug = slug.replace("--", "-") - return slug or "project" - - -def _next_available_folder_name(base: str, existing: List) -> str: - base = (base or "").strip() or "新项目" - used = {row[0] for row in existing or []} - if base not in used: - return base - index = 2 - while f"{base} ({index})" in used: - index += 1 - return f"{base} ({index})" - - -def _summary( - project: Project, - *, - favorite: bool, - folder_name: Optional[str], - file_count: int, - chat_count: int, -) -> Dict[str, Any]: - extra = project.extra_data or {} - local = extra.get("local") or {} - return { - "project_id": project.project_id, - "name": project.name, - "description": project.description or "", - "kind": project.kind, - "is_local": project.kind == "local", - "local_path": local.get("path"), - "local_slug": local.get("slug"), - "local_machine": local.get("machine"), - "owner_user_id": project.owner_user_id, - "linked_folder_id": project.linked_folder_id, - "folder_name": folder_name, - "instructions": project.instructions or "", - "icon_color": project.icon_color, - "pinned": bool(project.pinned), - "favorite": favorite, - "memory_enabled": bool(extra.get("memory_enabled", True)), - "memory_write_enabled": bool(extra.get("memory_write_enabled", True)), - "permission": "admin", - "is_owner": True, - "file_count": file_count, - "chat_count": chat_count, - "metadata": extra, - "created_at": project.created_at.isoformat() if project.created_at else None, - "updated_at": project.updated_at.isoformat() if project.updated_at else None, - "last_activity_at": ( - project.last_activity_at.isoformat() if project.last_activity_at else None - ), - } - - -class ProjectService: +class BaseProjectService: def __init__(self, db: Session): self.db = db @@ -190,7 +125,7 @@ def list_visible( else {} ) return [ - _summary( + personal_summary.summarize( project, favorite=project.project_id in favorite_ids, folder_name=folder_names.get(project.linked_folder_id), @@ -247,9 +182,9 @@ def create_personal( ): raise HTTPException(status_code=400, detail="该文件夹已被其它项目挂钩") else: - from core.services.user_folder_service import UserFolderService + from core.projects.user_folders import UserFolderService - folder_name = _next_available_folder_name( + folder_name = project_names.next_available_folder_name( clean, self.db.query(UserFolder.name) .filter( @@ -335,7 +270,7 @@ def create_local( project_id = f"prj_{uuid.uuid4().hex[:16]}" # slug: stable, filesystem-safe, unique per project — used as the sandbox # ``local/`` mount name (ticket #04). - slug = f"{_slugify(clean)}-{project_id[-6:]}" + slug = f"{project_names.slugify(clean)}-{project_id[-6:]}" project = Project( project_id=project_id, name=clean, @@ -384,7 +319,7 @@ def get(self, project_id: str, user_id: str) -> Optional[Dict[str, Any]]: .scalar() or 0 ) - data = _summary( + data = personal_summary.summarize( project, favorite=favorite, folder_name=folder[0] if folder else None, @@ -392,7 +327,7 @@ def get(self, project_id: str, user_id: str) -> Optional[Dict[str, Any]]: chat_count=chat_count, ) - from core.services.project_instructions import ProjectInstructionsService + from core.projects.instructions import ProjectInstructionsService data.update(ProjectInstructionsService(self.db).read(project)) return data @@ -449,7 +384,7 @@ def update( extra[key] = bool(patch[key]) project.extra_data = extra if "instructions" in patch: - from core.services.project_instructions import ProjectInstructionsService + from core.projects.instructions import ProjectInstructionsService ProjectInstructionsService(self.db).write( project, user_id, patch["instructions"] or "", @@ -530,4 +465,4 @@ def list_chats( ], total -__all__ = ["ProjectService"] +__all__ = ["BaseProjectService"] diff --git a/src/backend/core/projects/edition.py b/src/backend/core/projects/edition.py new file mode 100644 index 00000000..94d8771b --- /dev/null +++ b/src/backend/core/projects/edition.py @@ -0,0 +1,8 @@ +"""Community project service and single-owner scope.""" +from core.projects.base import BaseProjectService as ProjectService +from core.projects.personal_scope import ( + ProjectScope, build_project_ctx, build_project_ctx_from_chat_id, + edition_project_context_keys, project_memory_policy, project_scope_from_chat_id, + project_scope_from_context, +) +from core.projects.personal_transfer import transfer_project diff --git a/src/backend/core/project_environment/__init__.py b/src/backend/core/projects/environment/__init__.py similarity index 100% rename from src/backend/core/project_environment/__init__.py rename to src/backend/core/projects/environment/__init__.py diff --git a/src/backend/core/project_environment/config.py b/src/backend/core/projects/environment/config.py similarity index 100% rename from src/backend/core/project_environment/config.py rename to src/backend/core/projects/environment/config.py diff --git a/src/backend/core/project_environment/dependencies.py b/src/backend/core/projects/environment/dependencies.py similarity index 96% rename from src/backend/core/project_environment/dependencies.py rename to src/backend/core/projects/environment/dependencies.py index 7509ff59..a9c2b76e 100644 --- a/src/backend/core/project_environment/dependencies.py +++ b/src/backend/core/projects/environment/dependencies.py @@ -1,6 +1,6 @@ """Pure authorized dependency closure; no catalog or database access.""" -from core.project_environment.config import FIELDS +from core.projects.environment.config import FIELDS def expand_environment(config, graph): diff --git a/src/backend/core/services/project_environment.py b/src/backend/core/projects/environment/service.py similarity index 86% rename from src/backend/core/services/project_environment.py rename to src/backend/core/projects/environment/service.py index f7912bf9..8d00b264 100644 --- a/src/backend/core/services/project_environment.py +++ b/src/backend/core/projects/environment/service.py @@ -5,7 +5,7 @@ from contextlib import contextmanager from uuid import uuid4 -from core.project_environment.config import FIELDS, ProjectEnvironment +from core.projects.environment.config import FIELDS, ProjectEnvironment from fastapi import HTTPException @@ -29,14 +29,14 @@ def _validate_bindings(config, graph): def validate(self, user_id, data): config = ProjectEnvironment.model_validate(data).model_dump() if config["mode"] == "custom" and any(config[field] for field in FIELDS): - from core.services.project_environment_resources import environment_resources + from core.projects.environment_resources import environment_resources self._validate_bindings(config, environment_resources(self.db, user_id)) return config def preview(self, user_id, data): - from core.project_environment.dependencies import expand_environment - from core.services.project_environment_resources import environment_resources + from core.projects.environment.dependencies import expand_environment + from core.projects.environment_resources import environment_resources config = ProjectEnvironment.model_validate(data).model_dump() graph = environment_resources(self.db, user_id) @@ -78,8 +78,8 @@ def save(self, project, user_id, data): return self.read(project) def describe(self, project, user_id): - from core.project_environment.dependencies import expand_environment - from core.services.project_environment_resources import environment_resources + from core.projects.environment.dependencies import expand_environment + from core.projects.environment_resources import environment_resources config = self.read(project) graph = environment_resources(self.db, user_id) diff --git a/src/backend/core/services/project_environment_resources.py b/src/backend/core/projects/environment_resources.py similarity index 90% rename from src/backend/core/services/project_environment_resources.py rename to src/backend/core/projects/environment_resources.py index 4c039937..75f6a0e5 100644 --- a/src/backend/core/services/project_environment_resources.py +++ b/src/backend/core/projects/environment_resources.py @@ -2,7 +2,7 @@ from __future__ import annotations -from core.project_environment.config import FIELDS +from core.projects.environment.config import FIELDS def _keys(entries): @@ -23,12 +23,12 @@ def _keys(entries): def environment_resources(db, user_id): - from core.agent_skills.loader import get_skill_loader - from core.capabilities import device_catalog + from core.capabilities.skills.loader_factory import get_skill_loader + from core.harness.registry import device_catalog from core.db.models import AdminMcpServer, AdminSkill, InstalledPlugin - from core.llm.builtin_subagents import merge_builtin_subagents - from core.services.user_agent_service import UserAgentService - from core.services.user_service import UserService + from core.capabilities.agents.builtin import merge_builtin_subagents + from core.capabilities.agents.edition import UserAgentService + from core.auth.users import UserService from sqlalchemy import or_ service = UserAgentService(db) @@ -91,14 +91,14 @@ def environment_resources(db, user_id): for item in resources["mcp_servers"]: graph["mcp_ids"].setdefault(item["id"], {**item, "dependencies": {}}) - from core.config.catalog_loader import DB_HIDDEN_SERVERS, DB_UMBRELLA_ID + from core.capabilities.catalog.loader import DB_HIDDEN_SERVERS, DB_UMBRELLA_ID if DB_UMBRELLA_ID in graph["mcp_ids"]: graph["mcp_ids"][DB_UMBRELLA_ID]["dependencies"] = { "mcp_ids": [m for m in sorted(DB_HIDDEN_SERVERS) if m in graph["mcp_ids"]] } - from core.plugins.management import list_installed + from core.capabilities.plugins.management import list_installed callable_plugins = { p["install_id"] for p in list_installed(db, user_id, include_global=True) if p["callable"] @@ -130,9 +130,9 @@ def environment_resources(db, user_id): for field, entries in (("skill_ids", overlay["skills"]), ("mcp_ids", overlay["mcp"])): for entry in entries: graph[field].setdefault(entry["id"], {**entry, "dependencies": {}}) - from core.capabilities.paths import KIND_PLUGIN + from core.capabilities.identity_rules import KIND_PLUGIN - device_plugins = device_catalog._installations(KIND_PLUGIN, user_id=user_id) + device_plugins = device_catalog.installations(KIND_PLUGIN, user_id=user_id) for entry in device_catalog.plugin_entries(user_id): graph["plugin_ids"][entry["install_id"]] = { "id": entry["install_id"], @@ -166,7 +166,7 @@ def environment_resources(db, user_id): graph[field].setdefault(ident, {"id": ident, "name": ident, "dependencies": {}}) # Use the same authoritative ownership, suspension and readiness gate # as explicit chat invocation; listing a resource is not a permission grant. - from core.config.catalog_resolver import resolve_explicit_runtime_capabilities + from core.capabilities.catalog.resolver import resolve_explicit_runtime_capabilities allowed_skills, allowed_mcps, _, _ = resolve_explicit_runtime_capabilities( db, user_id, skill_ids=list(graph["skill_ids"]), mcp_ids=list(graph["mcp_ids"]) diff --git a/src/backend/core/services/project_file_service.py b/src/backend/core/projects/files.py similarity index 78% rename from src/backend/core/services/project_file_service.py rename to src/backend/core/projects/files.py index 369d8517..52813883 100644 --- a/src/backend/core/services/project_file_service.py +++ b/src/backend/core/projects/files.py @@ -2,8 +2,8 @@ import os from typing import NamedTuple -from core.spaces.projects import project_entries -from core.storage import get_storage +from core.files.spaces.projects import project_entries +from core.files.storage import get_storage class ProjectFileListing(NamedTuple): @@ -17,7 +17,7 @@ def __init__(self, db): @staticmethod def matches_scope(project, scope): - from core.spaces.edition_access import matches_scope + from core.files.edition import matches_scope return matches_scope(project, scope) def list_files(self, project, *, limit=None): @@ -35,16 +35,16 @@ def capacity_limit(): return 200 * 1024 * 1024 def upload(self, project, user_id, file_bytes, filename, mime_type=None): - from core.services.project_source import ProjectSourceService + from core.projects.source import ProjectSourceService ProjectSourceService(self.db).write_files( project.project_id, user_id, [(filename, file_bytes)], ) return next(item for item in self.list_files(project).items if item["name"] == filename) def delete(self, project_id, actor, path): - from core.services.project_source import ProjectSourceService - from core.spaces.files import SpaceFiles - from core.spaces.projects import project_location + from core.projects.source import ProjectSourceService + from core.files.spaces.files import SpaceFiles + from core.files.spaces.projects import project_location project = ProjectSourceService(self.db).authorized_project(project_id, actor, write=True) space, root, _ = project_location(self.db, project) SpaceFiles(self.db, actor, get_storage()).delete(space, root + "/" + path) diff --git a/src/backend/core/services/project_instructions.py b/src/backend/core/projects/instructions.py similarity index 95% rename from src/backend/core/services/project_instructions.py rename to src/backend/core/projects/instructions.py index 25b1e458..e415d61c 100644 --- a/src/backend/core/services/project_instructions.py +++ b/src/backend/core/projects/instructions.py @@ -4,14 +4,12 @@ File content is authoritative; the database field is only a legacy fallback. """ from __future__ import annotations -from core.infra.time import utc_now import hashlib import os import stat import tempfile -import uuid -from contextlib import contextmanager, nullcontext +from contextlib import contextmanager from datetime import datetime from pathlib import Path @@ -19,7 +17,7 @@ from sqlalchemy.orm import Session, attributes from core.db.models import Project -from core.storage import get_storage +from core.files.storage import get_storage AGENTS_FILENAME = "AGENTS.md" MAX_INSTRUCTIONS_BYTES = 32 * 1024 @@ -107,7 +105,7 @@ def _read(self, project: Project, *, remember_file: bool) -> dict: raise HTTPException(409, "AGENTS.md 必须是普通文件") data = stream.read(MAX_INSTRUCTIONS_BYTES + 1) else: - from core.spaces.projects import source_key + from core.files.spaces.projects import source_key storage = get_storage() key = source_key(self.db, project, AGENTS_FILENAME) info = storage.stat_object(key) @@ -161,13 +159,13 @@ def write( self, project: Project, user_id: str, text: str, *, expected_revision: str | None = None ) -> None: """Write the root file; participate in the caller's database transaction.""" - from core.spaces.leases import space_lock + from core.files.spaces.leases import space_lock lock = _local_write_lock(self._local_file(project)) if project.kind == "local" else space_lock(user_id, engine=self.db.get_bind()) with lock: self._write_locked(project, user_id, text, expected_revision=expected_revision) def _write_locked(self, project: Project, user_id: str, text: str, *, expected_revision: str | None): - from core.auth.permissions_iface import resolve_project_permission + from core.auth.edition import resolve_project_permission if resolve_project_permission(self.db, user_id, project) not in ("admin", "edit"): raise HTTPException(403, "需要项目编辑权限才能更新 AGENTS.md") @@ -216,15 +214,15 @@ def _write_locked(self, project: Project, user_id: str, text: str, *, expected_r project.extra_data = {**(project.extra_data or {}), "agents_file_managed": True} def _write_file(self, project: Project, data: bytes) -> None: - from core.services.project_file_service import ProjectFileService - from core.spaces.projects import source_key + from core.projects.files import ProjectFileService + from core.files.spaces.projects import source_key storage = get_storage() key = source_key(self.db, project, AGENTS_FILENAME) current = storage.stat_object(key) files = ProjectFileService(self.db) if files.capacity_used(project) - (current.size if current else 0) + len(data) > files.capacity_limit(): raise HTTPException(413, "项目容量不足，无法保存 AGENTS.md") - from core.spaces.tree import ensure_parents + from core.files.spaces.tree import ensure_parents ensure_parents(storage, key) if current and current.is_directory: raise HTTPException(409, "AGENTS.md 不能是目录") @@ -233,7 +231,7 @@ def _write_file(self, project: Project, data: bytes) -> None: def read_authorized_project_instructions(project_id: str, user_id: str) -> dict: """Read canonical rules in a fresh transaction with current view permission.""" - from core.auth.permissions_iface import resolve_project_permission + from core.auth.edition import resolve_project_permission from core.db.engine import SessionLocal with SessionLocal() as db: diff --git a/src/backend/core/services/local_project_workspace.py b/src/backend/core/projects/local_workspace.py similarity index 97% rename from src/backend/core/services/local_project_workspace.py rename to src/backend/core/projects/local_workspace.py index c61a1cac..a0a1b924 100644 --- a/src/backend/core/services/local_project_workspace.py +++ b/src/backend/core/projects/local_workspace.py @@ -19,7 +19,7 @@ def project_directory(scope, *, validate=True) -> str: def validate_current_project(scope, actor, *, write=False): """Reject stale scopes after deletion/rebinding; never switch an active run.""" from core.db.engine import SessionLocal - from core.services.project_source import ProjectSourceService + from core.projects.source import ProjectSourceService with SessionLocal() as db: project = ProjectSourceService(db).authorized_project( diff --git a/src/backend/core/services/project_scope.py b/src/backend/core/projects/personal_scope.py similarity index 95% rename from src/backend/core/services/project_scope.py rename to src/backend/core/projects/personal_scope.py index 29d0f414..1a527b88 100644 --- a/src/backend/core/services/project_scope.py +++ b/src/backend/core/projects/personal_scope.py @@ -97,7 +97,7 @@ def build_project_ctx(db: "Session", project_id: Optional[str]) -> Optional[dict if not project_id: return None from core.db.models import Project, UserFolder - from core.services.project_instructions import ProjectInstructionsService + from core.projects.instructions import ProjectInstructionsService project = ( db.query(Project) @@ -129,8 +129,10 @@ def build_project_ctx(db: "Session", project_id: Optional[str]) -> Optional[dict if path and slug: try: ensure_local_project_link(slug, path) - except Exception: - pass + except Exception as error: + from core.infra.failures import record_best_effort_failure + + record_best_effort_failure("projects.personal_scope.local_mount", error) return { "project_id": project_id, "project_environment_config": (project.extra_data or {}).get("environment") diff --git a/src/backend/core/projects/personal_summary.py b/src/backend/core/projects/personal_summary.py new file mode 100644 index 00000000..5aa06bea --- /dev/null +++ b/src/backend/core/projects/personal_summary.py @@ -0,0 +1,44 @@ +"""Personal-project response serialization.""" +from __future__ import annotations +from typing import Any, Dict, Optional +from core.db.models import Project + +def summarize( + project: Project, + *, + favorite: bool, + folder_name: Optional[str], + file_count: int, + chat_count: int, +) -> Dict[str, Any]: + extra = project.extra_data or {} + local = extra.get("local") or {} + return { + "project_id": project.project_id, + "name": project.name, + "description": project.description or "", + "kind": project.kind, + "is_local": project.kind == "local", + "local_path": local.get("path"), + "local_slug": local.get("slug"), + "local_machine": local.get("machine"), + "owner_user_id": project.owner_user_id, + "linked_folder_id": project.linked_folder_id, + "folder_name": folder_name, + "instructions": project.instructions or "", + "icon_color": project.icon_color, + "pinned": bool(project.pinned), + "favorite": favorite, + "memory_enabled": bool(extra.get("memory_enabled", True)), + "memory_write_enabled": bool(extra.get("memory_write_enabled", True)), + "permission": "admin", + "is_owner": True, + "file_count": file_count, + "chat_count": chat_count, + "metadata": extra, + "created_at": project.created_at.isoformat() if project.created_at else None, + "updated_at": project.updated_at.isoformat() if project.updated_at else None, + "last_activity_at": ( + project.last_activity_at.isoformat() if project.last_activity_at else None + ), + } diff --git a/src/backend/core/services/project_transfer.py b/src/backend/core/projects/personal_transfer.py similarity index 100% rename from src/backend/core/services/project_transfer.py rename to src/backend/core/projects/personal_transfer.py diff --git a/src/backend/core/projects/project_names.py b/src/backend/core/projects/project_names.py new file mode 100644 index 00000000..418bf3a5 --- /dev/null +++ b/src/backend/core/projects/project_names.py @@ -0,0 +1,27 @@ +"""Project folder names and local workspace slugs.""" +from __future__ import annotations +from typing import List + +def slugify(name: str) -> str: + """Lowercase, ASCII-safe, hyphen-joined slug for the local sandbox mount name.""" + out = [] + for ch in (name or "").strip().lower(): + if ch.isalnum() and ord(ch) < 128: + out.append(ch) + elif ch in (" ", "-", "_", "."): + out.append("-") + slug = "".join(out).strip("-") + while "--" in slug: + slug = slug.replace("--", "-") + return slug or "project" + + +def next_available_folder_name(base: str, existing: List) -> str: + base = (base or "").strip() or "新项目" + used = {row[0] for row in existing or []} + if base not in used: + return base + index = 2 + while f"{base} ({index})" in used: + index += 1 + return f"{base} ({index})" diff --git a/src/backend/core/services/project_source.py b/src/backend/core/projects/source.py similarity index 91% rename from src/backend/core/services/project_source.py rename to src/backend/core/projects/source.py index 8ec6a85f..d84debc3 100644 --- a/src/backend/core/services/project_source.py +++ b/src/backend/core/projects/source.py @@ -6,10 +6,10 @@ import hashlib from pathlib import PurePosixPath -from core.auth.permissions_iface import resolve_project_permission +from core.auth.edition import resolve_project_permission from core.db.models import Project -from core.services.project_file_service import ProjectFileService -from core.storage import get_storage +from core.projects.files import ProjectFileService +from core.files.storage import get_storage from fastapi import HTTPException from sqlalchemy.orm import Session @@ -51,8 +51,8 @@ def snapshot( self, project_id: str, actor: str, *, lock: bool = False, relative_root: str = "", exclude_patterns: frozenset[str] = frozenset(), ) -> list[tuple[str, bytes]]: - from core.spaces.projects import project_entries - from core.storage.entries import validate_key + from core.files.spaces.projects import project_entries + from core.files.storage.entries import validate_key project = self.authorized_project(project_id, actor, write=False) try: prefix = validate_key(relative_root, directory=True) @@ -74,7 +74,7 @@ def snapshot( return result def read_bytes(self, project_id: str, actor: str, path: str) -> tuple[bytes, str]: - from core.spaces.projects import source_key + from core.files.spaces.projects import source_key project = self.authorized_project(project_id, actor, write=False) storage = get_storage() key = source_key(self.db, project, path) @@ -98,8 +98,8 @@ def write_files( self, project_id: str, actor: str, files: list[tuple[str, bytes]], *, revisions: dict[str, str] | None = None, ) -> list[dict]: - from core.spaces.leases import space_lock - from core.spaces.projects import source_key, project_entries + from core.files.spaces.leases import space_lock + from core.files.spaces.projects import source_key, project_entries with space_lock(actor, engine=self.db.get_bind()): self.db.expire_all() project = self.authorized_project(project_id, actor, write=True) @@ -110,7 +110,7 @@ def write_files( prepared, seen = [], set() for path, data in files: key = source_key(self.db, project, path) - from core.spaces.tree import ensure_parents + from core.files.spaces.tree import ensure_parents ensure_parents(storage, key) existing = storage.stat_object(key) if (existing and existing.is_directory) or storage.list_directory(key + "/", limit=1).entries: @@ -121,7 +121,7 @@ def write_files( if len(data) > 50 * 1024 * 1024: raise HTTPException(413, "源码文件超过大小限制") if path == "AGENTS.md": - from core.services.project_instructions import _decode + from core.projects.instructions import _decode _decode(data) project.instructions = None project.extra_data = {**(project.extra_data or {}), "agents_file_managed": True} diff --git a/src/backend/core/services/user_folder_service.py b/src/backend/core/projects/user_folders.py similarity index 95% rename from src/backend/core/services/user_folder_service.py rename to src/backend/core/projects/user_folders.py index 06ce03de..24727283 100644 --- a/src/backend/core/services/user_folder_service.py +++ b/src/backend/core/projects/user_folders.py @@ -7,15 +7,14 @@ import os import uuid from core.db.models import UserShadow -from core.services.folder_batch import lock_folder_owner +from core.db.repository.folder import lock_folder_owner from dataclasses import dataclass from datetime import datetime from typing import Any, Dict, List, Optional, Tuple -from core.db.models import Artifact, UserFolder -from core.db.repository import ArtifactRepository, AuditLogRepository -from core.services.artifact_edition import is_personal_artifact, personal_artifact_create_fields -from core.storage import get_storage +from core.db.models import UserFolder +from core.db.repository import AuditLogRepository +from core.files.storage import get_storage from sqlalchemy.orm import Session logger = logging.getLogger(__name__) @@ -41,7 +40,7 @@ def _sanitize_name(name: str) -> Tuple[bool, str]: return True, cleaned -from core.services.personal_file_operations import PersonalFileOperations +from core.files.personal_operations import PersonalFileOperations class UserFolderService(PersonalFileOperations): @@ -150,7 +149,7 @@ def create_folder( parent_folder_id=parent_folder_id, name=cleaned, ) - from core.spaces.files import SpaceFiles + from core.files.spaces.files import SpaceFiles if actor != user_id: return FolderResult(False, "无权创建该用户的文件夹") path = self._folder_rel(folder) @@ -223,7 +222,7 @@ def rename_folder(self, folder_id: str, name: str, actor: str) -> FolderResult: old_rel = self._folder_rel(folder) parent_path = old_rel.rpartition("/")[0] new_rel = parent_path + "/" + cleaned if parent_path else cleaned - from core.spaces.files import SpaceFiles + from core.files.spaces.files import SpaceFiles SpaceFiles(self.db, actor, get_storage()).move("personal", old_rel, new_rel) folder.name = cleaned folder.updated_at = utc_now() @@ -294,7 +293,7 @@ def move_folder( old_rel = self._folder_rel(folder) parent_path = self._folder_rel(parent) if new_parent_id else "" new_rel = parent_path + "/" + folder.name if parent_path else folder.name - from core.spaces.files import SpaceFiles + from core.files.spaces.files import SpaceFiles SpaceFiles(self.db, actor, get_storage()).move("personal", old_rel, new_rel) folder.parent_folder_id = new_parent_id folder.updated_at = utc_now() @@ -341,7 +340,7 @@ def delete_folder(self, folder_id: str, actor: str) -> Tuple[FolderResult, int]: ids_to_delete = self._collect_descendants(folder_id) now = utc_now() - from core.spaces.files import SpaceFiles + from core.files.spaces.files import SpaceFiles affected = SpaceFiles(self.db, actor, get_storage()).delete_tree("personal", self._folder_rel(folder)) self.db.query(UserFolder).filter( @@ -367,8 +366,8 @@ def delete_folder(self, folder_id: str, actor: str) -> Tuple[FolderResult, int]: return FolderResult(True, "文件夹已删除", folder_id=folder_id), int(affected or 0) def count_affected_artifacts(self, folder_id: str, user_id: str) -> int: - from core.spaces.access import namespace - from core.spaces.tree import tree_entries + from core.files.spaces.access import namespace + from core.files.spaces.tree import tree_entries folder = self.get_owned(folder_id, user_id) if folder is None: return 0 diff --git a/src/backend/core/runs/__init__.py b/src/backend/core/runs/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/backend/orchestration/agent_api_recovery.py b/src/backend/core/runs/api_recovery.py similarity index 94% rename from src/backend/orchestration/agent_api_recovery.py rename to src/backend/core/runs/api_recovery.py index 7427dde6..68737228 100644 --- a/src/backend/orchestration/agent_api_recovery.py +++ b/src/backend/core/runs/api_recovery.py @@ -3,9 +3,9 @@ from __future__ import annotations from core.db.models import ChatSession -from core.llm.agent_api_runtime import apply_api_scope, parse_api_scope -from core.services.agent_api_service import session_matches_scope -from core.services.project_api_service import project_session_matches +from core.llm.api_runtime.agent import apply_api_scope, parse_api_scope +from core.auth.agent_api import session_matches_scope +from core.projects.api import project_session_matches def validate_recovery_context(db, run, context, *, user_id: str, chat_id: str) -> dict: diff --git a/src/backend/core/services/run_binding.py b/src/backend/core/runs/binding.py similarity index 92% rename from src/backend/core/services/run_binding.py rename to src/backend/core/runs/binding.py index 88374833..62e99bb8 100644 --- a/src/backend/core/services/run_binding.py +++ b/src/backend/core/runs/binding.py @@ -9,12 +9,7 @@ from core.db.engine import SessionLocal from core.db.models import ChatRun, ChatSession -from core.services.run_journal_domain import ( - DurableRunBinding, - RunAlreadyExists, - RunJournalError, - RunLeaseLost, -) +from core.runs.journal_domain import DurableRunBinding, RunAlreadyExists, RunJournalError, RunLeaseLost from sqlalchemy.orm import sessionmaker @@ -34,7 +29,7 @@ async def durable_run_binding( ): """Give non-chat Agent entry points the same durable run/heartbeat contract.""" - from core.services.run_journal import RunJournal + from core.runs.journal import RunJournal actual_chat_id = str(chat_id or f"internal_{uuid.uuid4().hex[:24]}") with session_factory() as db: @@ -108,7 +103,7 @@ async def _heartbeat() -> None: return heartbeat = asyncio.create_task(_heartbeat()) - from core.llm.context_safety import request_admission_scope + from core.llm.context.safety import request_admission_scope with request_admission_scope( context={"run_id": run_id, "journal_owner": owner}, @@ -128,11 +123,11 @@ async def _heartbeat() -> None: ) raise except Exception as exc: - from core.services.tool_effect_ledger import find_tool_outcome_unknown + from core.runs.tool_effects.errors import find_tool_outcome_unknown unknown = find_tool_outcome_unknown(exc) if admission.stopped is not None: - from core.services.context_budget_checkpoint import ContextBudgetJournal + from core.chat.compaction.budget_checkpoint import ContextBudgetJournal ContextBudgetJournal(session_factory).pause_context( {"run_id": run_id, "journal_owner": owner}, diff --git a/src/backend/core/runs/clock.py b/src/backend/core/runs/clock.py new file mode 100644 index 00000000..8db190c5 --- /dev/null +++ b/src/backend/core/runs/clock.py @@ -0,0 +1,55 @@ +"""UTC timestamps shared by run records and emitted events.""" + +from __future__ import annotations + +import datetime as _datetime +from datetime import datetime +from typing import Optional + +import core.db.engine as engine +import core.db.models as models +from core.infra.logging import get_logger + +logger = get_logger(__name__) + + +def _utcnow() -> datetime: + return _datetime.datetime.now(_datetime.timezone.utc) + + +def _epoch_ms(value: Optional[datetime]) -> Optional[int]: + """Wall-clock milliseconds for a stored timestamp. + + SQLite hands back naive datetimes; everything written here is UTC, so an + absent tzinfo is read as UTC rather than as the host's local zone. + """ + if value is None: + return None + if value.tzinfo is None: + value = value.replace(tzinfo=_datetime.timezone.utc) + return int(value.timestamp() * 1000) + + +def _run_started_at_ms(run_id: str) -> Optional[int]: + """When this run actually began, in wall-clock milliseconds. + + Read from ``chat_runs.started_at``, which is written once when the run is + claimed and is never rewritten by a recovering worker. Sending it with the + first frame is what lets a second device — or a reloaded tab — join a run + already in flight and show the true elapsed time instead of starting its + own stopwatch from zero. + + Only a worker resuming someone else's run needs to look this up: a worker + that claimed the run already holds the instant it wrote. Keeping the read + off the normal send path matters — it sits directly in front of the first + frame, where every millisecond is one the user spends staring at nothing. + """ + with engine.SessionLocal() as db: + row = ( + db.query(models.ChatRun.started_at, models.ChatRun.created_at) + .filter(models.ChatRun.run_id == run_id) + .first() + ) + if row is None: + return None + return _epoch_ms(row.started_at or row.created_at) diff --git a/src/backend/core/runs/events/__init__.py b/src/backend/core/runs/events/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/backend/core/runs/events/projection.py b/src/backend/core/runs/events/projection.py new file mode 100644 index 00000000..3dd8caf3 --- /dev/null +++ b/src/backend/core/runs/events/projection.py @@ -0,0 +1,13 @@ +"""Projection functions supplied by the event log's owning domain.""" +from collections.abc import Callable +from dataclasses import dataclass +from typing import Any + + +@dataclass(frozen=True) +class ProjectionCodec: + new_projection: Callable[[str], dict[str, Any]] + fold_projection: Callable[[dict[str, Any], dict[str, Any]], dict[str, Any]] + affected_tools: Callable[[dict[str, Any], dict[str, Any]], set[int]] + encode_checkpoint: Callable[..., dict[str, str]] + decode_checkpoint: Callable[..., dict[str, Any] | None] diff --git a/src/backend/orchestration/run_event_stream.py b/src/backend/core/runs/events/store.py similarity index 84% rename from src/backend/orchestration/run_event_stream.py rename to src/backend/core/runs/events/store.py index 33e68099..d081c709 100644 --- a/src/backend/orchestration/run_event_stream.py +++ b/src/backend/core/runs/events/store.py @@ -19,16 +19,15 @@ import asyncio import json -from copy import deepcopy -import time import threading +import time from collections import deque +from copy import deepcopy from typing import Any, Dict, List, Optional, Protocol, Tuple, runtime_checkable +from core.runs.events.projection import ProjectionCodec from core.infra.redis import get_redis, redis_configured from redis.exceptions import WatchError -from orchestration.run_projection import fold_projection, new_projection -from orchestration.run_projection_storage import affected_tools, decode_checkpoint, encode_checkpoint # Cursor meaning "from the very beginning of the log". START = "" @@ -59,7 +58,7 @@ def cursor_millis(cursor: str) -> Optional[int]: return None -def _sort_key(cursor: str) -> Tuple[int, int]: +def sort_key(cursor: str) -> Tuple[int, int]: if not cursor: return (-1, -1) stamp, _, seq = cursor.partition("-") @@ -134,7 +133,8 @@ def _decode(entries: Any) -> List[Entry]: def _snapshot_key(self, run_id: str) -> str: return f"jx:chat:run:{run_id}:projection" - def __init__(self) -> None: + def __init__(self, projection: ProjectionCodec) -> None: + self.projection = projection self._projections: Dict[str, Dict[str, Any]] = {} self._pending: Dict[str, int] = {} self._versions: Dict[str, str] = {} @@ -145,7 +145,7 @@ async def append(self, run_id: str, event: Dict[str, Any]) -> None: event = {**event, "event_offset": int(event.get("_offset") or 0)} for attempt in range(8): if attempt: - await asyncio.sleep(min(0.005 * 2 ** attempt, 0.1)) + await asyncio.sleep(min(0.005 * 2**attempt, 0.1)) async with get_redis().pipeline(transaction=True) as pipe: try: await pipe.watch(self._key(run_id)) @@ -156,7 +156,7 @@ async def append(self, run_id: str, event: Dict[str, Any]) -> None: previous, captured_version = await self.capture(run_id) if captured_version != version: continue - previous = previous or new_projection(run_id) + previous = previous or self.projection.new_projection(run_id) count = 1 dirty = set(range(len(previous["tools"]))) history_dirty = True @@ -167,24 +167,36 @@ async def append(self, run_id: str, event: Dict[str, Any]) -> None: offset = int(event.get("_offset") or 0) if offset and offset <= previous["event_offset"]: raise ValueError("run event offset must advance") - state = fold_projection(previous, event) + state = self.projection.fold_projection(previous, event) if event.get("type") in {"run_started", "steer_applied"}: - dirty = affected_tools(state, event) + dirty = self.projection.affected_tools(state, event) else: - dirty.update(affected_tools(state, event)) + dirty.update(self.projection.affected_tools(state, event)) if len(state["tools"]) > len(previous["tools"]): dirty.update(range(len(previous["tools"]), len(state["tools"]))) - history_dirty = history_dirty or event.get("type") in {"run_started", "steer_applied"} + history_dirty = history_dirty or event.get("type") in { + "run_started", + "steer_applied", + } interval = min(64, max(1, MAXLEN // 2)) distance = offset - previous.get("_checkpoint_offset", 0) if offset else count checkpoint = distance >= interval or event.get("type") in { - "run_started", "steer_applied", "__terminal__" + "run_started", + "steer_applied", + "__terminal__", } pipe.multi() - pipe.xadd(self._key(run_id), {"data": json.dumps(event, ensure_ascii=False)}, maxlen=MAXLEN) + pipe.xadd( + self._key(run_id), + {"data": json.dumps(event, ensure_ascii=False)}, + maxlen=MAXLEN, + ) if checkpoint: state["_checkpoint_offset"] = offset - pipe.hset(self._snapshot_key(run_id), mapping=encode_checkpoint(state, dirty, history_dirty=history_dirty)) + pipe.hset( + self._snapshot_key(run_id), + mapping=self.projection.encode_checkpoint(state, dirty, history_dirty=history_dirty), + ) result = await pipe.execute() self._projections[run_id] = state self._versions[run_id] = result[0] @@ -209,14 +221,14 @@ async def capture(self, run_id: str) -> Tuple[Optional[Dict[str, Any]], str]: pipe.hgetall(self._snapshot_key(run_id)) pipe.xrange(self._key(run_id), min="-", max="+") raw, entries = await pipe.execute() - state = decode_checkpoint(raw) + state = self.projection.decode_checkpoint(raw) decoded = self._decode(entries) if state is not None: floor = state["event_offset"] state["_checkpoint_offset"] = floor for _, event in decoded: if int(event.get("_offset") or 0) > floor: - state = fold_projection(state, event) + state = self.projection.fold_projection(state, event) return state, decoded[-1][0] if decoded else START async def read( @@ -246,8 +258,9 @@ async def wait(self, run_id: str, *, after: str, limit: int, timeout_ms: int) -> async def _drop_connections(redis: Any) -> None: try: await redis.connection_pool.disconnect(inuse_connections=False) - except Exception: # noqa: BLE001 - recovery must not mask the real error - pass + except Exception as fallback_error: # noqa: BLE001 - recovery must not mask the real error + from core.infra.failures import record_best_effort_failure + record_best_effort_failure('core.runs.events.store:_drop_connections', fallback_error) async def last_write_ms(self, run_id: str) -> Optional[int]: entries = await get_redis().xrevrange(self._key(run_id), max="+", min="-", count=1) @@ -281,7 +294,15 @@ async def clear(self, run_id: str) -> None: class _RunLog: """One run's in-process buffer plus the followers parked on it.""" - __slots__ = ("entries", "expires_at", "last_millis", "sequence", "waiters", "projection", "lock") + __slots__ = ( + "entries", + "expires_at", + "last_millis", + "sequence", + "waiters", + "projection", + "lock", + ) def __init__(self) -> None: self.entries: deque[Entry] = deque(maxlen=MAXLEN) @@ -300,9 +321,9 @@ def mint(self) -> str: return f"{millis}-{self.sequence}" def since(self, after: str, limit: Optional[int]) -> List[Entry]: - floor = _sort_key(after) + floor = sort_key(after) with self.lock: - found = [entry for entry in self.entries if _sort_key(entry[0]) > floor] + found = [entry for entry in self.entries if sort_key(entry[0]) > floor] return found[:limit] if limit else found @@ -314,7 +335,8 @@ class LocalRunEventStream: with an asyncio primitive shared across loops. """ - def __init__(self) -> None: + def __init__(self, projection: ProjectionCodec) -> None: + self.projection = projection self._logs: Dict[str, _RunLog] = {} def _sweep(self) -> None: @@ -337,7 +359,7 @@ async def append(self, run_id: str, event: Dict[str, Any]) -> None: log = self._log(run_id, create=True) event = {**event, "event_offset": int(event.get("_offset") or 0)} with log.lock: - log.projection = fold_projection(log.projection or new_projection(run_id), event) + log.projection = self.projection.fold_projection(log.projection or self.projection.new_projection(run_id), event) log.entries.append((log.mint(), event)) waiters, log.waiters = log.waiters, [] for loop, ready in waiters: @@ -393,13 +415,25 @@ async def clear(self, run_id: str) -> None: self._logs.pop(run_id, None) -_REDIS_STREAM = RedisRunEventStream() -_LOCAL_STREAM = LocalRunEventStream() +_REDIS_STREAM: RedisRunEventStream | None = None +_LOCAL_STREAM: LocalRunEventStream | None = None + + +def configure_projection(projection: ProjectionCodec) -> None: + """Initialize once; repeated host setup must preserve active streams.""" + global _REDIS_STREAM, _LOCAL_STREAM + if _REDIS_STREAM is None: + _REDIS_STREAM = RedisRunEventStream(projection) + if _LOCAL_STREAM is None: + _LOCAL_STREAM = LocalRunEventStream(projection) def get_run_event_stream() -> RunEventStream: """The run event log this deployment should use.""" - return _REDIS_STREAM if redis_configured() else _LOCAL_STREAM + stream = _REDIS_STREAM if redis_configured() else _LOCAL_STREAM + if stream is None: + raise RuntimeError("Run event projection has not been configured") + return stream __all__ = [ diff --git a/src/backend/core/runs/events/subscription.py b/src/backend/core/runs/events/subscription.py new file mode 100644 index 00000000..9ce2181e --- /dev/null +++ b/src/backend/core/runs/events/subscription.py @@ -0,0 +1,81 @@ +"""Replay and follow durable run events from the requested offset.""" + +from __future__ import annotations + +import asyncio +from typing import Any, AsyncIterator, Dict + +import core.chat.runtime.followups as followups +import core.runs.events.store as store +import core.runs.events.writer as writer +import core.runs.runtime as runtime +import core.runs.settings as settings +import redis.exceptions as _redis_exceptions +from core.infra.logging import get_logger + +logger = get_logger(__name__) + + +async def follow_run(run_id: str, *, from_offset: int = 0) -> AsyncIterator[Dict[str, Any]]: + "Read the run's event log, delivering events after ``from_offset``.\n\n Flow:\n 1. Replay everything recorded so far; deliver those with ``_offset > from_offset``\n 2. Block on the log waiting for new events\n 3. Stop when a ``__terminal__``-typed event arrives\n 4. Wait timed out + run in a terminal state → drain once more, then exit\n gracefully (backstop so the follower never hangs forever)\n\n Which log backs this — Redis Streams or the in-process one — is the\n deployment's choice; see :mod:`core.runs.events.store`.\n" + events = writer._events() + + run = runtime.get_run(run_id) + if run is None: + # The chats.py route layer should have validated this already; this is a second line of defense + raise runtime.ChatRunNotFound(f"chat run {run_id} not found") + + cursor = store.START + + # ── Phase 1: replay historical events ── + try: + history = await events.read(run_id) + except Exception as exc: + logger.warning("chat_run_xrange_failed", run_id=run_id, error=str(exc), exc_info=exc) + history = [] + + delivered, cursor, terminal_seen = followups._drain(history, from_offset, cursor) + for event in delivered: + yield event + if terminal_seen: + return + + # ── Phase 2: blocking tail ── + while True: + try: + batch = await events.wait( + run_id, after=cursor, limit=100, timeout_ms=settings._XREAD_BLOCK_MS + ) + except _redis_exceptions.TimeoutError: + # Benign: the block window elapsed with no new events — semantically + # identical to an empty result. (redis-py 8.0 defaults socket_timeout + # to 5s; if it ever equals _XREAD_BLOCK_MS the read raises here on + # every idle window instead of returning nil. We size socket_timeout + # well above the block in core/infra/redis.py, but treat the timeout + # as "no events" regardless so a long, quiet run never spams logs.) + batch = [] + except Exception as exc: + logger.warning("chat_run_xread_failed", run_id=run_id, error=str(exc), exc_info=exc) + await asyncio.sleep(0.5) + batch = [] + + if not batch: + # Nothing arrived: check whether the run reached a terminal state + current = runtime.get_run(run_id) + if current is not None and current.status in settings._TERMINAL_STATUSES: + # After the terminal state, read the latest events once more (covers the race) + try: + tail = await events.read(run_id, after=cursor, limit=200) + except Exception: + tail = [] + delivered, cursor, _terminal = followups._drain(tail, from_offset, cursor) + for event in delivered: + yield event + return + continue + + delivered, cursor, terminal_seen = followups._drain(batch, from_offset, cursor) + for event in delivered: + yield event + if terminal_seen: + return diff --git a/src/backend/core/runs/events/writer.py b/src/backend/core/runs/events/writer.py new file mode 100644 index 00000000..2a6ebec0 --- /dev/null +++ b/src/backend/core/runs/events/writer.py @@ -0,0 +1,197 @@ +"""Append ordered run events and persist terminal stream markers.""" + +from __future__ import annotations + +import asyncio +import json +import time +import typing as _typing +from typing import Any, Dict + +import core.runs.events.store as store +import core.runs.ownership as ownership +import core.runs.settings as settings +import core.runs.journal as run_journal +from core.infra.logging import get_logger +from core.runs.events.store import RunEventStream +from core.runs.journal import RunJournal + +logger = get_logger(__name__) + + +def _events() -> RunEventStream: + """This deployment's run event log (Redis Streams, or in-process).""" + return store.get_run_event_stream() + + +class EventOffsets: + """Durable SSE offsets for one worker, without a database write per event. + + Allocating an offset and proving the worker still owns the run used to be + the same write, so every token cost a serialized transaction — invisible + against a server database, and a disk flush per token on the desktop, where + the journal is SQLite and the person is watching the answer appear. + + The two are separable. Offsets come from reservations the journal hands out + in runs, and this keeps the unused remainder. Ownership is proven by reading + the lease rather than writing to it, and on a bounded schedule rather than + once per event: measured in the deployment that read costs ~0.5 ms, which at + forty concurrent streams is a whole core spent asking a question whose + answer almost never changes. + + Ownership can stop being ours two ways, and the bound is sized against the + slower one: + + * This process notices itself — its own heartbeat fails to renew, or the + user cancelled here. ``lease_lost`` is set, and the very next event is + fenced without leaving the loop at all, so this path got *faster*, not + looser. + * Another worker decided it — a cancel that landed on a different process, + or a recovery that re-claimed this run. That can only arrive through the + shared journal, so the read cannot be dropped, only bounded: at most one + per ``_CANCEL_VISIBILITY_SEC``, the same cadence the chunk loop already + polls cancellation on. Re-claiming requires the lease to have *expired*, + which means this worker went a full lease without renewing — it was not + emitting during that stretch either — so the window where a fenced worker + could still append is bounded by a quarter second and reached only by a + process that had stopped producing anyway. + + A terminal event is allocated on its own: it has to sit after every live + offset, including the ones a reservation left unused. That allocation is an + owned write, so a run always fences hard on the way out. + """ + + def __init__( + self, + journal: RunJournal, + run_id: str, + owner: str, + *, + lease_lost: asyncio.Event, + ): + self._journal = journal + self._run_id = run_id + self._owner = owner + self._next = 0 + self._remaining = 0 + self._lease_lost = lease_lost + self._proven_at = 0.0 + # The offset most recently handed out. Callers persist it with their + # checkpoints so a reconnecting follower knows where the run had got to. + self.last = 0 + + async def take(self, *, terminal: bool = False) -> int: + if terminal: + self._remaining = 0 + self.last = await asyncio.to_thread( + self._journal.allocate_event_offset, self._run_id, terminal=True + ) + return self.last + if self._remaining == 0: + # A reservation is an owned write: it fences on its own, and more + # strongly than a read does. + self._next = await asyncio.to_thread(self._reserve) + self._remaining = settings._OFFSET_RESERVATION + self._proven_at = time.monotonic() + else: + await self._prove_ownership() + offset = self._next + self._next += 1 + self._remaining -= 1 + self.last = offset + return offset + + async def _prove_ownership(self) -> None: + """Raise ``RunLeaseLost`` unless this worker still owns the run.""" + if self._lease_lost.is_set(): + # Our own heartbeat already decided this; the journal cannot + # disagree, so there is nothing to go and ask. + raise run_journal.RunLeaseLost(self._run_id) + now = time.monotonic() + if now - self._proven_at < settings._CANCEL_VISIBILITY_SEC: + return + await asyncio.to_thread(self._journal.require_lease, self._run_id, self._owner) + self._proven_at = now + + async def emit(self, event: Dict[str, Any], *, terminal: bool = False) -> None: + """Allocate this event's offset and append it to the run's log.""" + await self.take(terminal=terminal) + await xadd_event(self._run_id, self.last, event) + + def _reserve(self) -> int: + return self._journal.allocate_event_offset( + self._run_id, owner=self._owner, count=settings._OFFSET_RESERVATION + ) + + +def _json_safe(value: Any) -> Any: + return json.loads(json.dumps(value, ensure_ascii=False, default=str)) + + +async def _write_terminal_to_stream( + run_id: str, + *, + chat_id: str, + error_text: str, + cancelled: bool = False, +) -> None: + """Write an error event + terminal marker + EXPIRE. + + Used when the worker isn't around to emit them itself (cross-process cancel, + orphan recovery). Failure is logged but not raised. + """ + err_event: _typing.Dict[str, _typing.Any] = { + "type": "error", + "error": error_text, + "delta": error_text, + "chat_id": chat_id, + } + term_event: _typing.Dict[str, _typing.Any] = { + "type": settings.TERMINAL_TYPE, + "chat_id": chat_id, + } + if cancelled: + err_event["_cancelled"] = True + term_event["_cancelled"] = True + try: + journal = ownership.journal() + error_offset = journal.allocate_event_offset(run_id, terminal=True) + await xadd_event(run_id, error_offset, err_event) + terminal_offset = journal.allocate_event_offset(run_id, terminal=True) + await xadd_event(run_id, terminal_offset, term_event) + await expire_stream(run_id) + except Exception as exc: + logger.warning("chat_run_terminal_write_failed", run_id=run_id, error=str(exc)) + + +async def xadd_event(run_id: str, offset: int, event: Dict[str, Any]) -> None: + """Stamp an SSE event with its offset and wall clock, append it to the run's log. + + ``server_ts`` is when the server produced the event. A client that joins + late — after a reconnect, a session switch, or from a second device — + replays the log and reads these stamps, so its "waiting since" clock + reflects the run's real timeline instead of the moment this browser + happened to receive them. It deliberately has no underscore prefix: that + prefix marks transport-internal keys, which ``follow_run_as_sse`` strips + before the frame goes out, and this one is meant for the client. + """ + payload = {**event, "_offset": offset, "server_ts": int(time.time() * 1000)} + try: + await _events().append(run_id, payload) + except Exception as exc: + logger.warning("chat_run_xadd_failed", run_id=run_id, error=str(exc)) + + +async def expire_stream(run_id: str) -> None: + try: + await _events().expire(run_id, settings._STREAM_TTL_SECONDS) + except Exception as exc: + logger.warning("chat_run_expire_failed", run_id=run_id, error=str(exc)) + + +async def _reset_stream_projection(run_id: str) -> None: + """Drop a crashed worker's partial projection before replaying its run.""" + try: + await _events().clear(run_id) + except Exception as exc: + logger.warning("chat_run_projection_reset_failed", run_id=run_id, error=str(exc)) diff --git a/src/backend/core/runs/host.py b/src/backend/core/runs/host.py new file mode 100644 index 00000000..046fe99e --- /dev/null +++ b/src/backend/core/runs/host.py @@ -0,0 +1,41 @@ +"""Run services installed by the application composition root.""" +from core.harness.registry.run_cancellation import register_run_cancellation +from core.runs import runtime + + +def register() -> None: + from core.jobs.tool_operations import job_operations + from core.harness.registry.job_operations import register_job_operations + register_job_operations(job_operations) + from core.chat.history_operations import history_operations + from core.harness.registry.chat_history import register_chat_history + register_chat_history(history_operations) + from core.harness.registry.compaction import register_compaction_services + from core.chat.compaction import service as compaction + from core.chat.compaction.budget_checkpoint import ContextStopCheckpoint + register_compaction_services( + run=compaction.run_compaction, mid_turn=compaction.run_mid_turn_compaction, + token_limit=compaction.resolve_token_limit, trigger_ratio=compaction.resolve_trigger_ratio, + checkpoint=ContextStopCheckpoint, + ) + from core.harness.registry.hook_factories import register_run_hook_factories + from core.chat.steer_hook import SteeringHook + from core.sandbox.commands.completion_hook import command_completion_hooks + register_run_hook_factories(steering=SteeringHook, command_completion=command_completion_hooks) + from core.harness.registry.hook_factories import register_hook_factory + from core.jobs.reminder_hook import job_ledger_hooks + register_hook_factory(job_ledger_hooks) + from core.runs.tool_effects.hooks import tool_effect_hooks + register_hook_factory(tool_effect_hooks) + from core.llm.ports import register_execution_services + from core.sandbox.commands.runtime import runtime as commands + from core.sandbox.commands import store as records + from core.jobs.agent import snapshot_parent_runtime + register_execution_services(commands=commands, records=records, parent_snapshot=snapshot_parent_runtime) + from core.chat.projection.codec import chat_projection_codec + from core.runs.events.store import configure_projection + configure_projection(chat_projection_codec()) + async def cancel(run_id: str, *, user_id: str, stop_terminal_worker: bool = False) -> bool: + return await runtime.cancel_run(run_id, user_id=user_id, stop_terminal_worker=stop_terminal_worker) + from core.runs.journal import RunJournal + register_run_cancellation(cancel, cancel_in_session=RunJournal.cancel_in_session) diff --git a/src/backend/core/services/run_journal.py b/src/backend/core/runs/journal.py similarity index 94% rename from src/backend/core/services/run_journal.py rename to src/backend/core/runs/journal.py index d3b24093..c11ec987 100644 --- a/src/backend/core/services/run_journal.py +++ b/src/backend/core/runs/journal.py @@ -11,21 +11,10 @@ from core.db.engine import SessionLocal from core.db.models import ChatRun, ChatRunOperation -from core.services.run_binding import durable_run_binding -from core.services.run_journal_domain import ( - LIVE_STATUSES, - DurableRunBinding, - JournalReceipt, - RecoveryDecision, - RunAlreadyExists, - RunJournalError, - RunLeaseLost, - RunNotFound, - _aware, - _utcnow, -) -from core.services.run_journal_lifecycle import RunLifecycle -from core.services.run_journal_recovery import RunRecovery +from core.runs.binding import durable_run_binding +from core.runs.journal_domain import LIVE_STATUSES, DurableRunBinding, JournalReceipt, RecoveryDecision, RunAlreadyExists, RunJournalError, RunLeaseLost, RunNotFound, _aware, _utcnow +from core.runs.journal_lifecycle import RunLifecycle +from core.runs.journal_recovery import RunRecovery from sqlalchemy import func, or_ from sqlalchemy.orm import Session, sessionmaker, undefer @@ -51,6 +40,7 @@ def accept( user_id: str, request_payload: Mapping[str, Any], recovery_snapshot: Optional[Mapping[str, Any]] = None, + on_accept=None, ) -> ChatRun: """Commit acceptance before any in-process worker is created.""" with self._sessions() as db: @@ -64,6 +54,8 @@ def accept( recovery_snapshot=recovery_snapshot, now=self._clock(), ) + if on_accept is not None: + on_accept(db, row) db.commit() db.refresh(row) db.expunge(row) @@ -87,7 +79,7 @@ def accept_in_session( ) -> ChatRun: """Stage a pending run inside the caller's existing transaction.""" accepted_at = now or _utcnow() - from core.services.api_key_runs import bind_request_key, require_admission_key + from core.auth.api_key_runs import bind_request_key, require_admission_key request_payload = bind_request_key(request_payload, user_id) require_admission_key(db, request_payload, user_id, accepted_at) @@ -116,14 +108,14 @@ def accept_in_session( db.flush() return row - def claim(self, run_id: str, *, owner: str, lease_seconds: int) -> bool: + def claim(self, run_id: str, *, owner: str, lease_seconds: int, pending_only: bool = False) -> bool: """Atomically acquire a free/expired lease; exactly one claimant wins.""" if not owner: raise ValueError("owner is required") now = self._clock() expires = now + timedelta(seconds=max(1, int(lease_seconds))) with self._sessions() as db: - from core.services.api_key_runs import authorize_run_execution + from core.auth.api_key_runs import authorize_run_execution if not authorize_run_execution(db, run_id, now): return False @@ -131,7 +123,7 @@ def claim(self, run_id: str, *, owner: str, lease_seconds: int) -> bool: db.query(ChatRun) .filter( ChatRun.run_id == run_id, - ChatRun.status.in_(LIVE_STATUSES), + ChatRun.status.in_(("pending",) if pending_only else LIVE_STATUSES), or_( ChatRun.lease_owner.is_(None), ChatRun.lease_expires_at.is_(None), @@ -157,7 +149,7 @@ def renew(self, run_id: str, *, owner: str, lease_seconds: int) -> bool: now = self._clock() expires = now + timedelta(seconds=max(1, int(lease_seconds))) with self._sessions() as db: - from core.services.api_key_runs import authorize_run_execution + from core.auth.api_key_runs import authorize_run_execution if not authorize_run_execution(db, run_id, now): return False diff --git a/src/backend/core/services/run_journal_domain.py b/src/backend/core/runs/journal_domain.py similarity index 100% rename from src/backend/core/services/run_journal_domain.py rename to src/backend/core/runs/journal_domain.py diff --git a/src/backend/core/services/run_journal_lifecycle.py b/src/backend/core/runs/journal_lifecycle.py similarity index 93% rename from src/backend/core/services/run_journal_lifecycle.py rename to src/backend/core/runs/journal_lifecycle.py index 33456d8d..5ec43b88 100644 --- a/src/backend/core/services/run_journal_lifecycle.py +++ b/src/backend/core/runs/journal_lifecycle.py @@ -6,13 +6,7 @@ from typing import Any, Callable, Mapping, Optional from core.db.models import ChatRun -from core.services.run_journal_domain import ( - LIVE_STATUSES, - TERMINAL_STATUSES, - RunLeaseLost, - RunNotFound, - _aware, -) +from core.runs.journal_domain import LIVE_STATUSES, TERMINAL_STATUSES, RunLeaseLost, RunNotFound, _aware from sqlalchemy.orm import Session @@ -85,9 +79,19 @@ def complete( db.commit() return True - def cancel(self, run_id: str, *, reason: str) -> bool: + def cancel( + self, + run_id: str, + *, + reason: str, + cancel_guard: Optional[Callable[[Session], bool]] = None, + ) -> bool: now = self._clock() with self._sessions() as db: + # The caller may lock its ownership row here. Keep that lock and + # the child transition in one short transaction, without external I/O. + if cancel_guard is not None and not cancel_guard(db): + return False row = db.query(ChatRun).filter(ChatRun.run_id == run_id).with_for_update().one_or_none() if row is None or row.status not in LIVE_STATUSES: return False diff --git a/src/backend/core/services/run_journal_recovery.py b/src/backend/core/runs/journal_recovery.py similarity index 97% rename from src/backend/core/services/run_journal_recovery.py rename to src/backend/core/runs/journal_recovery.py index 74aa3ea6..fccd6a27 100644 --- a/src/backend/core/services/run_journal_recovery.py +++ b/src/backend/core/runs/journal_recovery.py @@ -3,14 +3,7 @@ from __future__ import annotations from core.db.models import ChatRun -from core.services.run_journal_domain import ( - LIVE_STATUSES, - NON_CHAT_RUN_KINDS, - RECOVERABLE_PRE_MODEL_PHASES, - RECOVERABLE_TOOL_PHASES, - SAFE_STOP_ORPHAN_KINDS, - RecoveryDecision, -) +from core.runs.journal_domain import LIVE_STATUSES, NON_CHAT_RUN_KINDS, RECOVERABLE_PRE_MODEL_PHASES, RECOVERABLE_TOOL_PHASES, SAFE_STOP_ORPHAN_KINDS, RecoveryDecision from sqlalchemy import or_ from sqlalchemy.orm import undefer diff --git a/src/backend/core/runs/ownership.py b/src/backend/core/runs/ownership.py new file mode 100644 index 00000000..42908987 --- /dev/null +++ b/src/backend/core/runs/ownership.py @@ -0,0 +1,276 @@ +"""Own run leases and journal-bound creation, fencing and terminal transitions.""" + +from __future__ import annotations + +import asyncio +import typing as _typing +import uuid +from datetime import datetime +from typing import Any, Dict, Optional + +import core.db.engine as engine +import core.db.models as models +import core.runs.clock as clock +import core.runs.events.writer as writer +import core.runs.runtime as runtime +import core.runs.settings as settings +import core.runs.journal as run_journal +from core.db.models import ChatRun +from core.infra.logging import get_logger +from core.runs.journal import RunJournal + +logger = get_logger(__name__) + + +def journal() -> RunJournal: + # Resolve SessionLocal at call time so tests and alternate runtime profiles + # can replace the factory without leaving the journal bound to another DB. + return run_journal.RunJournal(engine.SessionLocal) + + +def new_worker_owner(run_id: str) -> str: + return f"{settings._WORKER_INSTANCE_ID}:{run_id}:{uuid.uuid4().hex[:12]}" + + +async def lease_heartbeat( + run_id: str, + owner: str, + worker: asyncio.Task, + lease_lost: asyncio.Event, +) -> None: + """Renew ownership and fence the local coroutine immediately on lease loss.""" + interval = settings._RUN_LEASE_HEARTBEAT_SEC + try: + run = await asyncio.to_thread(runtime.get_run, run_id) + payload = (run.request_payload or {}) if run else {} + if payload.get("api_key_id") or payload.get("agent_api_scope"): + # A revoke handled by another worker must also stop a silent model call. + interval = min(interval, 1.0) + while True: + await asyncio.sleep(interval) + # ``renew`` is a synchronous UPDATE on chat_runs. Run it off the + # event loop: whenever that row is locked by another transaction + # this call waits inside the driver, and waiting *on the loop* + # freezes every coroutine in the process — including the one + # holding the lock, which then can never commit and release it. + # A blocked heartbeat must only cost this run its lease, never the + # whole backend. + renewed = await asyncio.to_thread( + journal().renew, + run_id, + owner=owner, + lease_seconds=settings.RUN_LEASE_SECONDS, + ) + if not renewed: + logger.warning("chat_run_lease_lost", run_id=run_id, owner=owner) + lease_lost.set() + worker.cancel() + return + except asyncio.CancelledError: + raise + except Exception as exc: # DB/driver failures must fail closed + logger.error( + "chat_run_lease_renew_failed", + run_id=run_id, + owner=owner, + error=str(exc), + exc_info=True, + ) + lease_lost.set() + worker.cancel() + + +def _update_run_status(run_id: str, **fields: Any) -> None: + """Single-shot UPDATE on chat_runs; isolated session so it doesn't pollute caller's txn.""" + if not fields: + return + if fields.get("status") in settings._TERMINAL_STATUSES: + fields.setdefault("writer_slot", None) + with engine.SessionLocal() as db: + affected = ( + db.query(models.ChatRun) + .filter(models.ChatRun.run_id == run_id) + .update(fields, synchronize_session=False) + ) + if not affected: + logger.warning("chat_run_update_missing", run_id=run_id, fields=list(fields.keys())) + db.commit() + + +def _claim_run_execution(run_id: str) -> Optional[datetime]: + """Fence duplicate initial workers before acquiring the durable lease. + + Returns the instant the run was marked running, or None when another worker + got there first. The caller needs that instant for the first frame, and + handing it back keeps the send path free of a second read of the same row. + """ + + started_at = clock._utcnow() + with engine.SessionLocal() as db: + affected = ( + db.query(models.ChatRun) + .filter(models.ChatRun.run_id == run_id, models.ChatRun.status == "pending") + .update( + {"status": "running", "started_at": started_at}, + synchronize_session=False, + ) + ) + db.commit() + return started_at if affected else None + + +def _finalize_run( + run_id: str, + *, + require_expired_lease: bool = False, + **fields: Any, +) -> bool: + """CAS variant of writing the terminal state: only updates while the run is still live. + + Prevents the race where "the reaper / cancel has already moved the run to a + terminal state, and the worker's late completed/failed overwrites it back" — + the side that loses the race abandons its write and returns False. + """ + fields.setdefault("lease_owner", None) + fields.setdefault("lease_expires_at", None) + if fields.get("error_message") and not fields.get("failure_reason"): + fields["failure_reason"] = fields["error_message"] + if fields.get("status") in settings._TERMINAL_STATUSES: + fields.setdefault("writer_slot", None) + with engine.SessionLocal() as db: + query = db.query(models.ChatRun).filter( + models.ChatRun.run_id == run_id, + models.ChatRun.status.in_(settings.LIVE_STATUSES), + ) + if require_expired_lease: + from sqlalchemy import or_ + + now = clock._utcnow() + query = query.filter( + or_( + models.ChatRun.lease_owner.is_(None), + models.ChatRun.lease_expires_at.is_(None), + models.ChatRun.lease_expires_at <= now, + ) + ) + affected = query.update(fields, synchronize_session=False) + db.commit() + if not affected: + logger.info("chat_run_finalize_skipped", run_id=run_id, fields=list(fields.keys())) + return bool(affected) + + +def _request_run_stop( + run_id: str, + *, + status: settings.RunStatus, + error_message: Optional[str] = None, + require_expired_lease: bool = False, +) -> bool: + """Write an external terminal verdict while retaining the writer fence.""" + + if status not in settings._TERMINAL_STATUSES: + raise ValueError(f"not a terminal status: {status}") + fields: _typing.Dict[str, _typing.Any] = {"status": status, "completed_at": clock._utcnow()} + if error_message: + fields["error_message"] = error_message[:1000] + fields["failure_reason"] = error_message[:1000] + with engine.SessionLocal() as db: + query = db.query(models.ChatRun).filter( + models.ChatRun.run_id == run_id, + models.ChatRun.status.in_(settings.LIVE_STATUSES), + ) + if require_expired_lease: + from sqlalchemy import or_ + + now = clock._utcnow() + query = query.filter( + or_( + models.ChatRun.lease_owner.is_(None), + models.ChatRun.lease_expires_at.is_(None), + models.ChatRun.lease_expires_at <= now, + ) + ) + affected = query.update(fields, synchronize_session=False) + db.commit() + return bool(affected) + + +def _request_run_cancel(run_id: str) -> bool: + return _request_run_stop(run_id, status="cancelled") + + +def _acknowledge_terminal_writer(run_id: str) -> bool: + """Release the writer fence only after the owning worker has stopped writing.""" + + with engine.SessionLocal() as db: + affected = ( + db.query(models.ChatRun) + .filter( + models.ChatRun.run_id == run_id, + models.ChatRun.status.in_(settings._TERMINAL_STATUSES), + models.ChatRun.writer_slot.isnot(None), + ) + .update({"writer_slot": None}, synchronize_session=False) + ) + db.commit() + return bool(affected) + + +def _acknowledge_never_started_terminal_writer(run_id: str) -> bool: + """Release a terminal fence only when no worker ever claimed the run. + + A duplicate worker can lose the pending-to-running claim while the real + worker is active, then observe a concurrent cancellation. It must not + release the real worker's fence in that case. ``started_at IS NULL`` is + the durable proof that there is no owning worker to wait for. + """ + + with engine.SessionLocal() as db: + affected = ( + db.query(models.ChatRun) + .filter( + models.ChatRun.run_id == run_id, + models.ChatRun.status.in_(settings._TERMINAL_STATUSES), + models.ChatRun.writer_slot.isnot(None), + models.ChatRun.started_at.is_(None), + ) + .update({"writer_slot": None}, synchronize_session=False) + ) + db.commit() + return bool(affected) + + +def create_run_record( + *, + chat_id: str, + user_id: str, + request_payload: Dict[str, Any], + recovery_snapshot: Optional[Dict[str, Any]] = None, + on_accept=None, +) -> ChatRun: + """Durably accept a run before any worker task can be spawned.""" + run_id = f"run_{uuid.uuid4().hex[:16]}" + message_id = f"msg_{uuid.uuid4().hex[:16]}" + return journal().accept( + run_id=run_id, + message_id=message_id, + chat_id=chat_id, + user_id=user_id, + request_payload=writer._json_safe(request_payload), + recovery_snapshot=writer._json_safe(recovery_snapshot or {}), + on_accept=on_accept, + ) + + +def cancelled_by_user(run_id: str) -> bool: + """租约被清掉有两种可能：别的 worker 接管了恢复，或者用户按了停止。 + + cancel_run 会先把行写成终态 cancelled 再清租约，之后不会再有人接手这一轮 —— + 被围栏的 worker 若也闭嘴，用户已经看到的半截回答就没人落库了。 + """ + try: + current = runtime.get_run(run_id) + except Exception: # noqa: BLE001 - 读不到就按"别人接管"处理，宁可不写 + return False + return current is not None and current.status == "cancelled" diff --git a/src/backend/core/runs/runtime.py b/src/backend/core/runs/runtime.py new file mode 100644 index 00000000..84b45093 --- /dev/null +++ b/src/backend/core/runs/runtime.py @@ -0,0 +1,275 @@ +"""Query and cancel durable runs and track their local asyncio workers.""" + +from __future__ import annotations + +import asyncio +import datetime as _datetime +import typing as _typing +from typing import Any, Awaitable, Callable, List, Optional, Tuple + +import core.db.engine as engine +import core.db.models as models +import core.db.reserved_chats as reserved_chats +import core.runs.clock as clock +import core.runs.events.writer as writer +import core.runs.ownership as ownership +import core.runs.settings as settings +import core.runs.tasks as tasks +from core.db.models import ChatRun +from core.infra.logging import get_logger + +logger = get_logger(__name__) + + +class ChatRunNotFound(Exception): + pass + + +class ChatRunPermissionDenied(Exception): + pass + + +def register_run_task(run_id: str, coro: Awaitable[None], *, name: str) -> None: + """Spawn the worker coroutine, register it in _active_runs, auto-cleanup on done.""" + active = tasks.process_state().active_runs + if run_id in active and not active[run_id].done(): + coro.close() + return + task = asyncio.create_task(coro, name=name) + tasks.process_state().active_runs[run_id] = task + + def finished(worker): + if active.get(run_id) is worker: + active.pop(run_id, None) + if not worker.cancelled() and worker.exception() is not None: + logger.error( + "run_worker_unhandled_error", + run_id=run_id, + error=str(worker.exception()), + exc_info=worker.exception(), + ) + + task.add_done_callback(finished) + + +async def wait_run(run_id: str) -> ChatRun: + """Wait for a locally launched run and return its durable terminal row. + + Shielding prevents a disconnected non-stream HTTP client from cancelling + the detached worker. Explicit ``cancel_run`` still targets the registered + task and keeps the database writer fence until that task has stopped. + """ + + task = tasks.process_state().active_runs.get(run_id) + if task is not None: + try: + await asyncio.shield(task) + except asyncio.CancelledError: + if not task.done(): + # The waiter (for example a disconnected HTTP request) was + # cancelled; shielding keeps the worker alive, and the caller + # must still observe its own cancellation. + raise + # Explicit run cancellation stopped the worker; the durable row + # below is the authoritative outcome. + run = get_run(run_id) + if run is None: + raise ChatRunNotFound(f"chat run {run_id} not found") + if run.status in settings.LIVE_STATUSES: + raise RuntimeError(f"chat run {run_id} lost its local worker before completion") + return run + + +def get_run(run_id: str) -> Optional[ChatRun]: + """Read a single run (used by the route layer for authorization).""" + with engine.SessionLocal() as db: + return db.query(models.ChatRun).filter(models.ChatRun.run_id == run_id).first() + + +def is_run_cancelled(run_id: str) -> bool: + """Polled by plan_mode every ~15s heartbeat; see ``cancel_run`` for why. + + Any terminal state stops the worker (not just ``cancelled``): a run reaped + to ``failed`` by the stale reaper must also make the cooperative worker stop + itself as soon as possible, or we get the split-brain of "DB says dead, task + keeps running". The worker itself only writes the terminal state during + wrap-up, so any terminal state seen while polling must come from an external + verdict. + """ + with engine.SessionLocal() as db: + row = db.query(models.ChatRun.status).filter(models.ChatRun.run_id == run_id).first() + return bool(row and row[0] in settings._TERMINAL_STATUSES) + + +async def cancel_run( + run_id: str, + *, + user_id: str, + stop_terminal_worker: bool = False, + cancel_guard: Optional[Callable[[Any], bool]] = None, +) -> bool: + """Mark status=cancelled. Idempotent on terminal runs. + + For chat-run kinds (regular chat / plan-generate) we forcefully cancel the + underlying asyncio task — these don't hold long-lived per-step HTTP MCP + clients so anyio cancel propagation is safe. + + For plan-execute kinds we DO NOT call task.cancel(). plan_mode polls + is_run_cancelled() between LLM streaming heartbeats and self-cancels + inside its own task. Cross-task task.cancel() on a plan-execute worker + risks an anyio cancel-scope deadlock: the worker holds per-step + streamable_http MCP clients whose internal SSE tasks may be blocked on + socket recv at cancel time, causing anyio's _deliver_cancellation to + self-reschedule every 10ms and starve the whole event loop. + """ + run = get_run(run_id) + if run is None: + raise ChatRunNotFound(f"chat run {run_id} not found") + if run.user_id != user_id: + raise ChatRunPermissionDenied(f"run {run_id} owned by another user") + already_cancelled = stop_terminal_worker and run.status == "cancelled" + if run.status not in settings.LIVE_STATUSES and not already_cancelled: + return False + + if already_cancelled and cancel_guard is not None: + # Even terminal-worker cleanup needs current caller ownership. Release + # the short guard transaction before stream or worker operations. + with engine.SessionLocal() as db: + if not cancel_guard(db): + return False + + if not already_cancelled and not ownership.journal().cancel( + run_id, + reason="user cancelled", + **({"cancel_guard": cancel_guard} if cancel_guard is not None else {}), + ): + # Race: the worker/reaper just beat us to writing the terminal state — treat as "no longer running". + return False + + payload = run.request_payload if isinstance(run.request_payload, dict) else {} + cooperative_only = payload.get("kind") in settings._COOPERATIVE_KINDS + + # Cooperative workers are fenced by the cancelled DB row and therefore + # deliberately stay silent when their heartbeat cancels them. Project the + # authoritative cancellation here so followers never wait on the old task. + if cooperative_only: + await writer._write_terminal_to_stream( + run_id, + chat_id=run.chat_id, + error_text="任务已被用户取消", + cancelled=True, + ) + + task = tasks.process_state().active_runs.get(run_id) + if task is not None and not task.done(): + if cooperative_only: + target = asyncio.shield(task) + timeout = 20.0 + else: + task.cancel() + target = task + timeout = 2.0 + try: + await asyncio.wait_for(target, timeout=timeout) + except (asyncio.TimeoutError, asyncio.CancelledError): + pass + except Exception as exc: + logger.warning("cancel_run_await_failed", run_id=run_id, error=str(exc)) + if task.done(): + ownership._acknowledge_terminal_writer(run_id) + elif not cooperative_only: + # Cross-process / post-restart: worker not in this process. Inject the + # terminal markers ourselves so any active follower exits cleanly. + await writer._write_terminal_to_stream( + run_id, + chat_id=run.chat_id, + error_text="任务已被用户取消", + cancelled=True, + ) + + return True + + +def _run_kind(run: ChatRun) -> Any: + payload = run.request_payload if isinstance(run.request_payload, dict) else {} + return payload.get("kind") + + +def get_active_run_for_chat(chat_id: str, user_id: str) -> Optional[ChatRun]: + """Backing query for GET /v1/chats/{chat_id}/active-run.""" + with engine.SessionLocal() as db: + rows = ( + db.query(models.ChatRun) + .filter( + models.ChatRun.chat_id == chat_id, + models.ChatRun.user_id == user_id, + ( + models.ChatRun.status.in_(settings.LIVE_STATUSES) + | (models.ChatRun.writer_slot.isnot(None)) + ), + ) + .order_by(models.ChatRun.created_at.desc()) + .all() + ) + for row in rows: + if _run_kind(row) not in settings._BACKGROUND_RUN_KINDS: + return row + return None + + +def has_local_runs() -> bool: + """这个进程手上有没有正在跑的 run。 + + 用来在查库之前便宜地排掉"本进程什么都没在跑"的情况 —— 谁在跑哪个 run 只有进程自己 + 知道（``_active_runs`` 是进程内的），而有些判断必须落到具体进程上：确认条的 pending + 与前端信号队列都活在进程里，只有执行这个 run 的进程弹得出来。 + """ + return bool(tasks.process_state().active_runs) + + +def is_local_run(run_id: str) -> bool: + """这个 run 是不是由本进程在执行。""" + return run_id in tasks.process_state().active_runs + + +def list_active_runs_for_user( + user_id: str, recent_seconds: int = 0 +) -> Tuple[List[ChatRun], List[str]]: + """Backing query for GET /v1/chats/active-runs, in one query. + + Returns one live run per chat: only genuinely live rows (``pending`` / + ``running``) qualify. The per-chat probe additionally accepts a lingering + ``writer_slot`` so a reconnecting client can clean its own zombie UI up; + this listing feeds the sidebar's "still running" dot, where a terminal row + must not light anything. + + With ``recent_seconds`` it also returns the chats whose runs started in that + window, newest first, so a page finds a chat started elsewhere (Personal + Agent, API, another device) even when its run ended between two checks. + """ + live = models.ChatRun.status.in_(settings.LIVE_STATUSES) + since = clock._utcnow() - _datetime.timedelta( + seconds=min(recent_seconds, settings.MAX_RECENT_RUN_SECONDS) + ) + with engine.SessionLocal() as db: + rows = ( + db.query(models.ChatRun) + .filter( + models.ChatRun.user_id == user_id, + (live | (models.ChatRun.created_at >= since)) if recent_seconds else live, + # Module-owned chats (Personal Agent) never belong in the ordinary sidebar. + reserved_chats.ordinary_chat_filter(models.ChatRun.chat_id), + ) + .order_by(models.ChatRun.created_at.desc()) + .all() + ) + latest_per_chat: _typing.Dict[str, models.ChatRun] = {} + recent: _typing.Dict[str, None] = {} + for row in rows: + if _run_kind(row) in settings._BACKGROUND_RUN_KINDS: + continue + if row.status in settings.LIVE_STATUSES: + latest_per_chat.setdefault(row.chat_id, row) + if recent_seconds and row.created_at >= since: + recent.setdefault(row.chat_id) + return list(latest_per_chat.values()), list(recent) diff --git a/src/backend/core/runs/settings.py b/src/backend/core/runs/settings.py new file mode 100644 index 00000000..0e406f73 --- /dev/null +++ b/src/backend/core/runs/settings.py @@ -0,0 +1,98 @@ +"""Existing run kinds, statuses and execution deadlines shared by run owners.""" + +from __future__ import annotations + +import os +import socket +import typing as _typing +import uuid + +import core.llm.runtime.human_interaction as human_interaction +from core.infra.logging import get_logger + +logger = get_logger(__name__) + +RunKind = _typing.Literal["chat", "plan_execute", "plan_generate", "autonomous_loop"] + + +_COOPERATIVE_KINDS: tuple[str, ...] = ("plan_execute", "autonomous_loop") + + +RunStatus = _typing.Literal[ + "pending", "running", "needs_attention", "completed", "failed", "cancelled" +] + + +_TERMINAL_STATUSES: tuple[RunStatus, ...] = ( + "needs_attention", + "completed", + "failed", + "cancelled", +) + + +LIVE_STATUSES: tuple[RunStatus, ...] = ("pending", "running") + + +_STREAM_TTL_SECONDS = human_interaction.STREAM_TTL_SECONDS + + +TERMINAL_TYPE = "__terminal__" + + +_XREAD_BLOCK_MS = 5000 + + +HEARTBEAT_INTERVAL_SEC = 15.0 + + +_FIRST_RESPONSE_TIMEOUT_SEC = 600.0 + + +_INACTIVITY_TIMEOUT_SEC = float(os.getenv("CHAT_RUN_INACTIVITY_TIMEOUT_SEC", "600")) + + +_STALE_RUN_MAX_AGE_SEC = float(os.getenv("CHAT_RUN_MAX_AGE_SEC", "1800")) + + +STALE_REAPER_INTERVAL_SEC = float(os.getenv("CHAT_RUN_REAPER_INTERVAL_SEC", "300")) + + +_STALE_QUIET_SEC = float(os.getenv("CHAT_RUN_STALE_QUIET_SEC", str(_INACTIVITY_TIMEOUT_SEC))) + + +_HARD_MAX_AGE_SEC = float(os.getenv("CHAT_RUN_HARD_MAX_AGE_SEC", "21600")) + + +RUN_LEASE_SECONDS = max(15, int(os.getenv("CHAT_RUN_LEASE_SECONDS", "90"))) + + +_RUN_LEASE_HEARTBEAT_SEC = max(1.0, RUN_LEASE_SECONDS / 3) + + +RUN_RECOVERY_INTERVAL_SEC = max( + 1.0, + float(os.getenv("CHAT_RUN_RECOVERY_INTERVAL_SEC", "15")), +) + + +_WORKER_INSTANCE_ID = f"{socket.gethostname()}:{os.getpid()}:{uuid.uuid4().hex[:12]}" + + +_OFFSET_RESERVATION = 64 + + +_CANCEL_VISIBILITY_SEC = 0.25 + + +_BACKGROUND_RUN_KINDS = frozenset( + { + "automation_plan", + "automation_prompt", + "internal_job_agent", + "legacy_chat_stream", + } +) + + +MAX_RECENT_RUN_SECONDS = 86400 diff --git a/src/backend/core/runs/tasks.py b/src/backend/core/runs/tasks.py new file mode 100644 index 00000000..746f7350 --- /dev/null +++ b/src/backend/core/runs/tasks.py @@ -0,0 +1,23 @@ +"""Local worker handles, keyed by durable run identity.""" + +from __future__ import annotations + +import asyncio +import typing as _typing +from dataclasses import dataclass, field + +from core.infra.logging import get_logger + +logger = get_logger(__name__) + + +@dataclass +class TasksState: + active_runs: _typing.Dict[str, asyncio.Task] = field(default_factory=lambda: {}) + + +_STATE = TasksState() + + +def process_state() -> TasksState: + return _STATE diff --git a/src/backend/orchestration/tool_effect_recovery.py b/src/backend/core/runs/tool_effect_recovery.py similarity index 82% rename from src/backend/orchestration/tool_effect_recovery.py rename to src/backend/core/runs/tool_effect_recovery.py index ae38417d..c5ada64f 100644 --- a/src/backend/orchestration/tool_effect_recovery.py +++ b/src/backend/core/runs/tool_effect_recovery.py @@ -6,16 +6,17 @@ """ from __future__ import annotations +from core.harness.registry import execution_identity as bridge_identity import json from typing import Any from agentscope.message import ToolCallBlock from agentscope.tool._response import ToolResponse - from core.db.engine import SessionLocal -from core.db.models import ChatRun, ToolEffectReceipt, RemoteToolEffect -from core.services.tool_effect_ledger import ReconciliationResult, ToolEffectError, ToolIntent +from core.db.models import ChatRun, RemoteToolEffect, ToolEffectReceipt +from core.runs.tool_effects.types import ReconciliationResult, ToolIntent +from core.runs.tool_effects.errors import ToolEffectError def _contains_withheld(value: Any) -> bool: @@ -41,7 +42,7 @@ async def replay_tool_intent(intent: ToolIntent) -> dict[str, Any]: """Rebuild the frozen chat tool surface and invoke one adapter, without an LLM call.""" from core.llm.factory import create_agent_executor - from core.llm.mcp_manager import close_clients + from core.llm.mcp.manager import close_clients with SessionLocal() as db: run = db.get(ChatRun, intent.run_id) @@ -55,11 +56,11 @@ async def replay_tool_intent(intent: ToolIntent) -> dict[str, Any]: user_id = run.user_id chat_id = run.chat_id - from core.capabilities.paths import capabilities_enabled + from core.harness.registry.run_capabilities import has_run_capabilities - if capabilities_enabled(): - from core.capabilities.errors import IntegrityFailed - from core.capabilities.runtime import require_root_tool_scope + if has_run_capabilities(): + from core.harness.capability_errors import IntegrityFailed + from core.harness.registry.capability_runtime import require_root_tool_scope try: require_root_tool_scope(intent.run_id, user_id, intent.tool_call_id, intent.tool_name) @@ -138,7 +139,8 @@ async def reconcile_scheduled_task_intent(intent: ToolIntent) -> ReconciliationR remote = db.get(RemoteToolEffect, intent.effect_id) if remote is not None and remote.gateway_url: remote_call = { - "url": remote.gateway_url, "schema_hash": remote.schema_hash, + "url": remote.gateway_url, + "schema_hash": remote.schema_hash, "user_id": run.user_id, } else: @@ -146,7 +148,8 @@ async def reconcile_scheduled_task_intent(intent: ToolIntent) -> ReconciliationR if remote_call is None: # A device operation may predate persisted cross-end bindings. # Its missing local receipt says nothing about cloud state. - from core.auth.desktop_bridge import bridge_enabled + from core.harness.registry.host_runtime import requires_remote_execution as bridge_enabled + if bridge_enabled() and intent.redacted_args.get("execution_location") != "local": return ReconciliationResult.unknown() return ReconciliationResult.not_applied() @@ -159,25 +162,29 @@ async def reconcile_scheduled_task_intent(intent: ToolIntent) -> ReconciliationR async def _reconcile_cloud_receipt(intent, remote_call): import httpx - from core.services import desktop_cloud_bridge as bridge - from core.services.desktop_capability_protocol import token_subject - from core.capabilities.errors import CapabilityError + from core.harness.capability_errors import CapabilityError + from core.capabilities.manifest_protocol import token_subject - state = bridge.get_state() + state = bridge_identity.get_state() if not state or token_subject(state.get("token", "")) != remote_call["user_id"]: return ReconciliationResult.unknown() - if not remote_call["url"].startswith(state.get("cloud_base", "").rstrip("/") + "/api/v1/desktop/capability/gateway/"): + if not remote_call["url"].startswith( + state.get("cloud_base", "").rstrip("/") + "/api/v1/desktop/capability/gateway/" + ): return ReconciliationResult.unknown() try: - bridge.ensure_current_authorization(state) + bridge_identity.ensure_current_authorization(state) async with httpx.AsyncClient(timeout=15) as client: response = await client.post( remote_call["url"].rsplit("/", 1)[0] + "/receipt", - headers=bridge.cloud_headers(state), - json={"tool_name": intent.tool_name, "operation_id": intent.effect_id, - "schema_hash": remote_call["schema_hash"]}, + headers=bridge_identity.cloud_headers(state), + json={ + "tool_name": intent.tool_name, + "operation_id": intent.effect_id, + "schema_hash": remote_call["schema_hash"], + }, ) - bridge.require_current_account(state) + bridge_identity.require_current_account(state) response.raise_for_status() data = response.json()["data"] if not isinstance(data, dict): diff --git a/src/backend/core/runs/tool_effects/__init__.py b/src/backend/core/runs/tool_effects/__init__.py new file mode 100644 index 00000000..877f5963 --- /dev/null +++ b/src/backend/core/runs/tool_effects/__init__.py @@ -0,0 +1 @@ +"""Durable tool invocation intents, results and recovery.""" diff --git a/src/backend/core/runs/tool_effects/arguments.py b/src/backend/core/runs/tool_effects/arguments.py new file mode 100644 index 00000000..84c89911 --- /dev/null +++ b/src/backend/core/runs/tool_effects/arguments.py @@ -0,0 +1,65 @@ +"""Canonicalise and redact tool arguments and derive idempotency keys.""" +from __future__ import annotations +import hashlib +import json +from typing import Any, Mapping, Optional +from core.infra.data_masking import mask_sensitive_data +from core.runs.tool_effects import errors +from core.runs.tool_effects import journal_constants + +def embedded_redact(value: str) -> str: + cleaned = value + for pattern in journal_constants.EMBEDDED_SECRET_PATTERNS: + cleaned = pattern.sub(r"\1[REDACTED]", cleaned) + return cleaned + + +def sanitize_value(value: Any, *, key: str = "") -> Any: + lower = key.lower() + if isinstance(value, Mapping): + return {str(k): sanitize_value(v, key=str(k)) for k, v in value.items()} + if isinstance(value, (list, tuple)): + return [sanitize_value(item, key=key) for item in value] + if isinstance(value, str): + if lower in journal_constants.WITHHOLD_FIELDS: + digest = hashlib.sha256(value.encode("utf-8")).hexdigest() + return {"_withheld": True, "sha256": digest, "length": len(value)} + return embedded_redact(value) + return value + + +def canonical_tool_args(args: Mapping[str, Any]) -> tuple[str, str, Mapping[str, Any]]: + canonical = json.dumps( + args or {}, sort_keys=True, separators=(",", ":"), default=str + ) + args_hash = hashlib.sha256(canonical.encode("utf-8")).hexdigest() + by_field = mask_sensitive_data(dict(args or {}), field_patterns=journal_constants.REDACT_FIELDS) + return canonical, args_hash, sanitize_value(by_field) + + +def tool_args_hash(args: Mapping[str, Any]) -> str: + return canonical_tool_args(args)[1] + + +def stable_idempotency_key(run_id: str, tool_name: str, invocation_id: str) -> str: + raw = f"{run_id}\x00{tool_name}\x00{invocation_id}".encode("utf-8") + return "tool:" + hashlib.sha256(raw).hexdigest() + + +def resolved_idempotency_key( + *, + run_id: str, + principal_scope: str, + tool_name: str, + tool_call_id: str, + explicit_key: Optional[str], +) -> tuple[str, bool]: + caller_key = str(explicit_key or "").strip() + if caller_key: + scoped = f"{principal_scope}\x00{caller_key}".encode("utf-8") + return "caller:" + hashlib.sha256(scoped).hexdigest(), True + if not tool_call_id: + raise errors.ToolIntentCommitError( + "tool_call_id or explicit idempotency_key is required" + ) + return stable_idempotency_key(run_id, tool_name, tool_call_id), False diff --git a/src/backend/core/runs/tool_effects/errors.py b/src/backend/core/runs/tool_effects/errors.py new file mode 100644 index 00000000..dcaf1373 --- /dev/null +++ b/src/backend/core/runs/tool_effects/errors.py @@ -0,0 +1,43 @@ +"""Errors raised by the tool effect ledger.""" +from __future__ import annotations +from typing import Optional + +class ToolEffectError(RuntimeError): + pass + + +class ToolIntentCommitError(ToolEffectError): + """The intent could not be durably committed; invocation is forbidden.""" + + +class ToolOutcomeUnknown(ToolEffectError): + """The ledger proves intent but cannot prove the external outcome.""" + + +def find_tool_outcome_unknown(exc: BaseException) -> Optional[ToolOutcomeUnknown]: + """Unwrap AgentScope/anyio ExceptionGroup layers without losing safety meaning.""" + pending = [exc] + seen: set[int] = set() + while pending: + current = pending.pop() + if id(current) in seen: + continue + seen.add(id(current)) + if isinstance(current, ToolOutcomeUnknown): + return current + nested = getattr(current, "exceptions", None) + if isinstance(nested, tuple): + pending.extend(item for item in nested if isinstance(item, BaseException)) + if isinstance(current.__cause__, BaseException): + pending.append(current.__cause__) + if isinstance(current.__context__, BaseException): + pending.append(current.__context__) + return None + + +class ToolEffectLeaseLost(ToolEffectError): + pass + + +class ToolIdempotencyConflict(ToolEffectError): + """A caller reused one explicit key for different logical input.""" diff --git a/src/backend/core/runs/tool_effects/gateway.py b/src/backend/core/runs/tool_effects/gateway.py new file mode 100644 index 00000000..293e64c5 --- /dev/null +++ b/src/backend/core/runs/tool_effects/gateway.py @@ -0,0 +1,256 @@ +"""The state machine every side-effecting tool call goes through.""" +from __future__ import annotations +from core.harness import tool_effect_context +from core.runs.tool_effects.journal import ToolEffectJournal +import asyncio +import inspect +import uuid +from contextlib import suppress +from typing import Any, Awaitable, Callable, Mapping, Optional +from core.runs.tool_effects import errors +from core.runs.tool_effects.registry import ToolRecoveryRegistry +from core.runs.tool_effects import types + +class ToolEffectGateway: + """The single execution state machine used by production middleware and tests.""" + + def __init__( + self, + journal: ToolEffectJournal, + registry: ToolRecoveryRegistry, + *, + poll_interval: float = 0.05, + lease_seconds: int = 300, + ) -> None: + self.journal = journal + self.registry = registry + self.poll_interval = poll_interval + self.lease_seconds = lease_seconds + + @staticmethod + async def _call(handler, intent: types.ToolIntent): # noqa: ANN001, ANN205 + value = handler(intent) if handler is not None else None + return await value if inspect.isawaitable(value) else value + + @staticmethod + def _reconciliation(value: Any) -> types.ReconciliationResult: + if isinstance(value, types.ReconciliationResult): + return value + return ( + types.ReconciliationResult.applied(value) + if value is not None + else types.ReconciliationResult.unknown() + ) + + def _start_heartbeat( + self, + effect_id: str, + *, + run_owner: str, + claim_owner: str, + ) -> tuple[asyncio.Task, asyncio.Event]: + lost = asyncio.Event() + interval = max(0.05, min(30.0, self.lease_seconds / 3)) + + async def _beat() -> None: + while True: + await asyncio.sleep(interval) + try: + ok = await asyncio.to_thread( + self.journal.renew_invocation_claim, + effect_id, + run_owner=run_owner, + claim_owner=claim_owner, + lease_seconds=self.lease_seconds, + ) + except Exception: # noqa: BLE001 - lost heartbeat means lost ownership + ok = False + if not ok: + lost.set() + return + + return asyncio.create_task(_beat()), lost + + @staticmethod + async def _stop_heartbeat(task: asyncio.Task) -> None: + task.cancel() + with suppress(asyncio.CancelledError): + await task + + async def execute_outcome( + self, + *, + run_id: str, + owner: str, + tool_call_id: str, + tool_name: str, + args: Mapping[str, Any], + invoke: Callable[[], Awaitable[Any] | Any], + idempotency_key: Optional[str] = None, + classify_failure: Optional[Callable[[Any], bool]] = None, + ) -> types.ToolExecutionOutcome: + claim_owner = f"tool:{uuid.uuid4().hex}" + policy = self.registry.resolve(tool_name) + while True: + decision = await asyncio.to_thread( + self.journal.begin_intent, + run_id=run_id, + owner=owner, + claim_owner=claim_owner, + tool_call_id=tool_call_id, + tool_name=tool_name, + args=args, + recovery_policy=policy.policy, + idempotency_key=idempotency_key, + lease_seconds=self.lease_seconds, + ) + if decision.action == "known": + return types.ToolExecutionOutcome( + decision.result, + decision.intent.effect_id, + decision.intent.result_id, + decision.result_state, + False, + ) + if decision.action == "unknown": + raise errors.ToolOutcomeUnknown(decision.effect_id) + if decision.action == "wait": + await asyncio.sleep(self.poll_interval) + terminal = await asyncio.to_thread( + self.journal.terminal_decision, decision.effect_id + ) + if terminal is None: + continue + if terminal.action == "unknown": + raise errors.ToolOutcomeUnknown(decision.effect_id) + return types.ToolExecutionOutcome( + terminal.result, + terminal.intent.effect_id, + terminal.intent.result_id, + terminal.result_state, + False, + ) + if decision.action not in ("execute", "reconcile"): + raise errors.ToolEffectError( + f"unsupported tool intent action: {decision.action}" + ) + + await asyncio.to_thread( + self.journal.acquire_invocation_claim, + decision.effect_id, + run_owner=owner, + claim_owner=claim_owner, + lease_seconds=self.lease_seconds, + ) + heartbeat, claim_lost = self._start_heartbeat( + decision.effect_id, + run_owner=owner, + claim_owner=claim_owner, + ) + token = tool_effect_context.CURRENT_TOOL_EFFECT.set( + tool_effect_context.ToolInvocationContext( + decision.intent.effect_id, + decision.intent.result_id, + decision.intent.idempotency_key, + decision.intent.run_id, + claim_owner, + ) + ) + try: + if decision.action == "reconcile": + if policy.reconciler is None: + self.journal.mark_unknown( + decision.effect_id, + owner=owner, + claim_owner=claim_owner, + reason="reconcile policy has no reconciliation adapter", + ) + raise errors.ToolOutcomeUnknown(decision.effect_id) + reconciliation = self._reconciliation( + await self._call(policy.reconciler, decision.intent) + ) + if claim_lost.is_set(): + raise errors.ToolEffectLeaseLost(decision.effect_id) + if reconciliation.outcome == "applied": + result = await asyncio.to_thread( + self.journal.commit_result, + decision.effect_id, + run_owner=owner, + claim_owner=claim_owner, + result=reconciliation.result, + reconciled=True, + ) + return types.ToolExecutionOutcome( + result, + decision.intent.effect_id, + decision.intent.result_id, + "success", + False, + ) + if reconciliation.outcome != "not_applied": + self.journal.mark_unknown( + decision.effect_id, + owner=owner, + claim_owner=claim_owner, + reason="reconciliation could not determine external outcome", + ) + raise errors.ToolOutcomeUnknown(decision.effect_id) + value = invoke() + result = await value if inspect.isawaitable(value) else value + if claim_lost.is_set(): + raise errors.ToolEffectLeaseLost(decision.effect_id) + failed = ( + bool(classify_failure(result)) + if classify_failure is not None + else False + ) + # A returned error payload is still a definitive tool outcome. + # Persist it and give the original error back to the model so + # it can correct the request or explain an authorization issue. + # ToolOutcomeUnknown is reserved for invocations that never + # produced a result at all (exceptions, disconnects, crashes). + result = await asyncio.to_thread( + self.journal.commit_result, + decision.effect_id, + run_owner=owner, + claim_owner=claim_owner, + result=result, + failed=failed, + ) + return types.ToolExecutionOutcome( + result, + decision.intent.effect_id, + decision.intent.result_id, + "error" if failed else "success", + True, + ) + except asyncio.CancelledError: + with suppress(Exception): + self.journal.abandon_invocation_claim( + decision.effect_id, + run_owner=owner, + claim_owner=claim_owner, + ) + raise + except errors.ToolOutcomeUnknown: + with suppress(Exception): + self.journal.abandon_invocation_claim( + decision.effect_id, + run_owner=owner, + claim_owner=claim_owner, + ) + raise + except Exception as exc: + with suppress(Exception): + self.journal.abandon_invocation_claim( + decision.effect_id, + run_owner=owner, + claim_owner=claim_owner, + ) + raise errors.ToolOutcomeUnknown(decision.effect_id) from exc + finally: + await self._stop_heartbeat(heartbeat) + tool_effect_context.CURRENT_TOOL_EFFECT.reset(token) + + async def execute(self, **kwargs) -> Any: # noqa: ANN003 + return (await self.execute_outcome(**kwargs)).result diff --git a/src/backend/core/runs/tool_effects/hooks.py b/src/backend/core/runs/tool_effects/hooks.py new file mode 100644 index 00000000..769a40e9 --- /dev/null +++ b/src/backend/core/runs/tool_effects/hooks.py @@ -0,0 +1,14 @@ +"""Durable tool admission is an execution-layer policy.""" +from core.harness.hooks import Decision, HookSpec, HookStage + + +def require_durable_tool_binding(invocation): + identity = invocation.data.get("identity", {}) + if not invocation.run_id or not identity.get("journal_owner") or not invocation.operation_name: + return Decision.reject("tool execution requires a durable run_id and journal_owner binding") + return Decision() + + +def tool_effect_hooks(request): + yield HookSpec("durable_tool_binding", HookStage.BEFORE_TOOL, + require_durable_tool_binding, priority=800) diff --git a/src/backend/core/runs/tool_effects/host.py b/src/backend/core/runs/tool_effects/host.py new file mode 100644 index 00000000..a71c9eab --- /dev/null +++ b/src/backend/core/runs/tool_effects/host.py @@ -0,0 +1,28 @@ +"""Compose durable tool-effect execution for model runtime ports.""" +from core.db.engine import SessionLocal +from core.llm.ports import register_tool_effect_runtime +from core.runs.tool_effects.errors import ToolEffectError, ToolIntentCommitError +from core.runs.tool_effects.gateway import ToolEffectGateway +from core.runs.tool_effects.journal import ToolEffectJournal +from core.runs.tool_effects.registry import DEFAULT_TOOL_RECOVERY_REGISTRY + + +class DurableToolEffects: + def __init__(self, session_factory=None, registry=None): + self.gateway = ToolEffectGateway( + ToolEffectJournal(session_factory or SessionLocal), + registry or DEFAULT_TOOL_RECOVERY_REGISTRY, + ) + + async def execute_outcome(self, **kwargs): + return await self.gateway.execute_outcome(**kwargs) + + def binding_error(self, message): + return ToolIntentCommitError(message) + + def response_error(self, message): + return ToolEffectError(message) + + +def register(): + register_tool_effect_runtime(DurableToolEffects) diff --git a/src/backend/core/runs/tool_effects/intent_transactions.py b/src/backend/core/runs/tool_effects/intent_transactions.py new file mode 100644 index 00000000..753386cf --- /dev/null +++ b/src/backend/core/runs/tool_effects/intent_transactions.py @@ -0,0 +1,342 @@ +"""Record tool intents and decide whether a call may run.""" +from __future__ import annotations +import uuid +from datetime import timedelta +from typing import Any, Mapping, Optional +from core.db.models import ChatRun, ToolEffectLease, ToolEffectLedger +from core.runs.journal import LIVE_STATUSES, RunLeaseLost, RunNotFound +from sqlalchemy import or_ +from sqlalchemy.exc import IntegrityError, OperationalError +from sqlalchemy.orm import Session +from core.runs.tool_effects import arguments +from core.runs.tool_effects import errors +from core.runs.tool_effects import registry +from core.runs.tool_effects import types +from core.infra.time import as_utc + +def intent_from_row(row: ToolEffectLedger) -> types.ToolIntent: + return types.ToolIntent( + effect_id=row.effect_id, + result_id=row.result_id, + run_id=row.run_id, + operation_seq=int(row.operation_seq), + tool_name=row.tool_name, + tool_call_id=row.tool_call_id or "", + args_hash=row.args_hash, + redacted_args=dict(row.redacted_args or {}), + idempotency_key=row.idempotency_key or "", + recovery_policy=row.recovery_policy, + created_at=as_utc(row.created_at), + ) + + +def terminal_row(db: Session, effect_id: str) -> Optional[ToolEffectLedger]: + return ( + db.query(ToolEffectLedger) + .filter(ToolEffectLedger.terminal_effect_id == effect_id) + .one_or_none() + ) + + +def decision_from_terminal( + intent: types.ToolIntent, terminal: ToolEffectLedger +) -> types.IntentDecision: + if terminal.event_type == "unknown_outcome": + return types.IntentDecision("unknown", intent, result_state="unknown") + return types.IntentDecision( + "known", + intent, + terminal.result_payload, + "success" if terminal.event_type == "result" else "error", + ) + + +def validate_identity( + row: ToolEffectLedger, *, tool_name: str, args_hash: str +) -> None: + if row.tool_name != tool_name or row.args_hash != args_hash: + raise errors.ToolIdempotencyConflict( + "idempotency_key already belongs to different tool arguments" + ) + + +def existing_decision( + self, + *, + key: str, + caller_run_id: str, + run_owner: str, + claim_owner: str, + tool_name: str, + args_hash: str, + lease_seconds: int, +) -> types.IntentDecision: + with self._sessions() as db: + intent_row = ( + db.query(ToolEffectLedger) + .filter(ToolEffectLedger.idempotency_key == key) + .one() + ) + self._validate_identity( + intent_row, tool_name=tool_name, args_hash=args_hash + ) + intent = self._intent_from_row(intent_row) + caller_run = db.get(ChatRun, caller_run_id) + original_run = db.get(ChatRun, intent.run_id) + if caller_run is None or original_run is None: + raise errors.ToolIntentCommitError(f"run does not exist: {caller_run_id}") + if original_run.user_id != caller_run.user_id: + raise errors.ToolIdempotencyConflict( + "idempotency_key belongs to a different principal" + ) + terminal = self._terminal_row(db, intent.effect_id) + if terminal is not None and caller_run_id == intent.run_id: + return self._decision_from_terminal(intent, terminal) + now = as_utc(self._clock()) + caller_expiry = as_utc(caller_run.lease_expires_at) + if ( + caller_run.status not in LIVE_STATUSES + or caller_run.lease_owner != run_owner + or caller_expiry is None + or now is None + or caller_expiry <= now + ): + raise errors.ToolIntentCommitError(f"run lease is not owned: {caller_run_id}") + if terminal is not None: + return self._decision_from_terminal(intent, terminal) + if intent.run_id != caller_run_id: + return types.IntentDecision("wait", intent) + + lease = ( + db.query(ToolEffectLease) + .filter(ToolEffectLease.effect_id == intent.effect_id) + .with_for_update() + .one() + ) + run = self._runs._locked_owned_run(db, caller_run_id, run_owner) + terminal = self._terminal_row(db, intent.effect_id) + if terminal is not None: + return self._decision_from_terminal(intent, terminal) + lease_expiry = as_utc(lease.lease_expires_at) + if ( + lease.run_owner == run_owner + and lease_expiry is not None + and lease_expiry > now + ): + if lease.claim_owner == claim_owner: + return types.IntentDecision( + "reconcile" + if intent.recovery_policy == "reconcile" + else "execute", + intent, + ) + return types.IntentDecision("wait", intent) + + previous_owner = lease.run_owner + previous_claim = lease.claim_owner + previous_expiry = lease.lease_expires_at + affected = ( + db.query(ToolEffectLease) + .filter( + ToolEffectLease.effect_id == intent.effect_id, + ToolEffectLease.run_owner == previous_owner, + ToolEffectLease.claim_owner == previous_claim, + ToolEffectLease.lease_expires_at == previous_expiry, + or_( + ToolEffectLease.lease_expires_at.is_(None), + ToolEffectLease.lease_expires_at <= self._clock(), + ), + ) + .update( + { + ToolEffectLease.run_owner: run_owner, + ToolEffectLease.claim_owner: claim_owner, + ToolEffectLease.lease_expires_at: self._clock() + + timedelta(seconds=max(1, lease_seconds)), + ToolEffectLease.updated_at: self._clock(), + }, + synchronize_session=False, + ) + ) + if not affected: + db.rollback() + return types.IntentDecision("wait", intent) + db.expire_all() + lease = db.get(ToolEffectLease, intent.effect_id) + self._append_event( + db, + run, + intent_row, + owner=run_owner, + event_type="recovery_claim", + operation_type="tool_effect_recovery_claimed", + phase=f"tool_intent_{intent.recovery_policy}", + safety=intent.recovery_policy, + ) + if intent.recovery_policy == "never_replay": + self._mark_unknown_locked( + db, + run, + intent_row, + lease, + owner=run_owner, + claim_owner=claim_owner, + reason="non-replayable tool intent has no committed result", + ) + db.commit() + return types.IntentDecision("unknown", intent) + db.commit() + return types.IntentDecision( + "reconcile" if intent.recovery_policy == "reconcile" else "execute", + intent, + ) + + +def begin_intent( + self, + *, + run_id: str, + owner: str, + claim_owner: str, + tool_call_id: str, + tool_name: str, + args: Mapping[str, Any], + recovery_policy: str, + idempotency_key: Optional[str] = None, + lease_seconds: int = 300, + _conflict_retries: int = 4, +) -> types.IntentDecision: + if recovery_policy not in registry.RECOVERY_POLICIES: + raise ValueError(f"invalid recovery policy: {recovery_policy}") + _canonical, args_hash, redacted = arguments.canonical_tool_args(args) + with self._sessions() as db: + caller_run = db.get(ChatRun, run_id) + if caller_run is None: + raise errors.ToolIntentCommitError(f"run does not exist: {run_id}") + principal_scope = caller_run.user_id + key, _explicit = arguments.resolved_idempotency_key( + run_id=run_id, + principal_scope=str(principal_scope), + tool_name=tool_name, + tool_call_id=tool_call_id, + explicit_key=idempotency_key + or str(args.get("idempotency_key") or args.get("request_id") or "").strip(), + ) + try: + with self._sessions() as db: + found = ( + db.query(ToolEffectLedger.event_id) + .filter(ToolEffectLedger.idempotency_key == key) + .first() + ) + if found is not None: + return self._existing_decision( + key=key, + caller_run_id=run_id, + run_owner=owner, + claim_owner=claim_owner, + tool_name=tool_name, + args_hash=args_hash, + lease_seconds=lease_seconds, + ) + + with self._sessions() as db: + run = self._runs._locked_owned_run(db, run_id, owner) + found = ( + db.query(ToolEffectLedger.event_id) + .filter(ToolEffectLedger.idempotency_key == key) + .first() + ) + if found is not None: + db.rollback() + return self._existing_decision( + key=key, + caller_run_id=run_id, + run_owner=owner, + claim_owner=claim_owner, + tool_name=tool_name, + args_hash=args_hash, + lease_seconds=lease_seconds, + ) + effect_id = f"eff_{uuid.uuid4().hex}" + result_id = f"toolres_{uuid.uuid4().hex}" + receipt = self._runs._append_locked( + db, + run, + owner=owner, + operation_type="tool_intent_committed", + phase=f"tool_intent_{recovery_policy}", + safety=recovery_policy, + payload={ + "effect_id": effect_id, + "result_id": result_id, + "tool_name": tool_name, + "tool_call_id": tool_call_id, + "args_hash": args_hash, + }, + ) + row = ToolEffectLedger( + effect_id=effect_id, + result_id=result_id, + run_id=run_id, + operation_seq=receipt.operation_seq, + event_type="intent", + tool_name=tool_name, + tool_call_id=tool_call_id, + args_hash=args_hash, + redacted_args=dict(redacted), + idempotency_key=key, + recovery_policy=recovery_policy, + ) + db.add(row) + db.add( + ToolEffectLease( + effect_id=effect_id, + run_id=run_id, + run_owner=owner, + claim_owner=claim_owner, + lease_expires_at=self._clock() + + timedelta(seconds=max(1, lease_seconds)), + updated_at=self._clock(), + ) + ) + run.updated_at = self._clock() + db.commit() + return types.IntentDecision("execute", self._intent_from_row(row)) + except errors.ToolEffectError: + raise + except (IntegrityError, OperationalError) as exc: + with self._sessions() as db: + found = ( + db.query(ToolEffectLedger.event_id) + .filter(ToolEffectLedger.idempotency_key == key) + .first() + ) + if found is not None: + return self._existing_decision( + key=key, + caller_run_id=run_id, + run_owner=owner, + claim_owner=claim_owner, + tool_name=tool_name, + args_hash=args_hash, + lease_seconds=lease_seconds, + ) + if _conflict_retries > 0: + return self.begin_intent( + run_id=run_id, + owner=owner, + claim_owner=claim_owner, + tool_call_id=tool_call_id, + tool_name=tool_name, + args=args, + recovery_policy=recovery_policy, + idempotency_key=idempotency_key, + lease_seconds=lease_seconds, + _conflict_retries=_conflict_retries - 1, + ) + raise errors.ToolIntentCommitError(str(exc)) from exc + except (RunNotFound, RunLeaseLost) as exc: + raise errors.ToolIntentCommitError(str(exc)) from exc + except Exception as exc: # noqa: BLE001 + raise errors.ToolIntentCommitError(str(exc)) from exc diff --git a/src/backend/core/runs/tool_effects/invocation_claims.py b/src/backend/core/runs/tool_effects/invocation_claims.py new file mode 100644 index 00000000..c67ae972 --- /dev/null +++ b/src/backend/core/runs/tool_effects/invocation_claims.py @@ -0,0 +1,123 @@ +"""Acquire, renew and abandon invocation claims.""" +from __future__ import annotations +from datetime import timedelta +from core.db.models import ChatRun, ToolEffectLease, ToolEffectLedger +from core.runs.journal import LIVE_STATUSES +from core.runs.tool_effects import errors + +def acquire_invocation_claim( + self, + effect_id: str, + *, + run_owner: str, + claim_owner: str, + lease_seconds: int = 300, +) -> None: + """Validate and extend one invocation claim in a short atomic transaction.""" + now = self._clock() + expires = now + timedelta(seconds=max(1, lease_seconds)) + with self._sessions() as db: + live_run = db.query(ChatRun.run_id).filter( + ChatRun.status.in_(LIVE_STATUSES), + ChatRun.lease_owner == run_owner, + ChatRun.lease_expires_at > now, + ) + affected = ( + db.query(ToolEffectLease) + .filter( + ToolEffectLease.effect_id == effect_id, + ToolEffectLease.run_owner == run_owner, + ToolEffectLease.claim_owner == claim_owner, + ToolEffectLease.lease_expires_at > now, + ToolEffectLease.run_id.in_(live_run), + ) + .update( + { + ToolEffectLease.lease_expires_at: expires, + ToolEffectLease.updated_at: now, + }, + synchronize_session=False, + ) + ) + if not affected or self._terminal_row(db, effect_id) is not None: + db.rollback() + raise errors.ToolEffectLeaseLost(effect_id) + db.commit() + + +def renew_invocation_claim( + self, + effect_id: str, + *, + run_owner: str, + claim_owner: str, + lease_seconds: int, +) -> bool: + if not self._runs.renew( + self._intent_run_id(effect_id), + owner=run_owner, + lease_seconds=lease_seconds, + ): + return False + try: + self.acquire_invocation_claim( + effect_id, + run_owner=run_owner, + claim_owner=claim_owner, + lease_seconds=lease_seconds, + ) + return True + except errors.ToolEffectLeaseLost: + return False + + +def renew_recovery_claim( + self, + effect_id: str, + *, + recovery_owner: str, + lease_seconds: int, +) -> bool: + return self.renew_invocation_claim( + effect_id, + run_owner=recovery_owner, + claim_owner=recovery_owner, + lease_seconds=lease_seconds, + ) + + +def intent_run_id(self, effect_id: str) -> str: + with self._sessions() as db: + row = ( + db.query(ToolEffectLedger.run_id) + .filter( + ToolEffectLedger.effect_id == effect_id, + ToolEffectLedger.event_type == "intent", + ) + .one() + ) + return str(row[0]) + + +def abandon_invocation_claim( + self, + effect_id: str, + *, + run_owner: str, + claim_owner: str, +) -> None: + """Expire a cancelled/ambiguous call so policy recovery can take over.""" + now = self._clock() + with self._sessions() as db: + db.query(ToolEffectLease).filter( + ToolEffectLease.effect_id == effect_id, + ToolEffectLease.run_owner == run_owner, + ToolEffectLease.claim_owner == claim_owner, + ).update( + { + ToolEffectLease.lease_expires_at: now, + ToolEffectLease.updated_at: now, + }, + synchronize_session=False, + ) + db.commit() diff --git a/src/backend/core/runs/tool_effects/journal.py b/src/backend/core/runs/tool_effects/journal.py new file mode 100644 index 00000000..7b593351 --- /dev/null +++ b/src/backend/core/runs/tool_effects/journal.py @@ -0,0 +1,223 @@ +"""Synchronous transaction boundary behind the tool effect gateway.""" +from __future__ import annotations +from core.runs.tool_effects import intent_transactions +from core.runs.tool_effects import result_transactions +from core.runs.tool_effects import invocation_claims +from core.runs.tool_effects import retention +from core.runs.tool_effects import recovery_claims +from datetime import datetime +from typing import Any, Callable, Mapping, Optional +from core.db.engine import SessionLocal +from core.db.models import ChatRun, ToolEffectLease, ToolEffectLedger +from core.runs.journal import RunJournal +from sqlalchemy.orm import Session, sessionmaker +from core.runs.tool_effects import types +from core.infra.time import utc_now + +class ToolEffectJournal: + """Small synchronous transaction boundary behind the async gateway.""" + + def __init__( + self, + session_factory: sessionmaker = SessionLocal, + *, + clock: Callable[[], datetime] = utc_now, + ) -> None: + self._sessions = session_factory + self._clock = clock + self._runs = RunJournal(session_factory, clock=clock) + + @staticmethod + def _intent_from_row(row: ToolEffectLedger) -> types.ToolIntent: + return intent_transactions.intent_from_row(row) + + @staticmethod + def _terminal_row(db: Session, effect_id: str) -> Optional[ToolEffectLedger]: + return intent_transactions.terminal_row(db, effect_id) + + @staticmethod + def _decision_from_terminal( + intent: types.ToolIntent, terminal: ToolEffectLedger + ) -> types.IntentDecision: + return intent_transactions.decision_from_terminal(intent, terminal) + + @staticmethod + def _validate_identity( + row: ToolEffectLedger, *, tool_name: str, args_hash: str + ) -> None: + return intent_transactions.validate_identity(row, tool_name=tool_name, args_hash=args_hash) + + def _append_event( + self, + db: Session, + run: ChatRun, + intent: ToolEffectLedger, + *, + owner: str, + event_type: str, + operation_type: str, + phase: str, + safety: str, + result: Any = None, + error: Optional[str] = None, + terminal: bool = False, + ) -> None: + return result_transactions.append_event(self, db, run, intent, owner=owner, event_type=event_type, operation_type=operation_type, phase=phase, safety=safety, result=result, error=error, terminal=terminal) + + def _mark_unknown_locked( + self, + db: Session, + run: ChatRun, + intent: ToolEffectLedger, + lease: ToolEffectLease, + *, + owner: str, + claim_owner: str, + reason: str, + ) -> None: + return result_transactions.mark_unknown_locked(self, db, run, intent, lease, owner=owner, claim_owner=claim_owner, reason=reason) + + def _existing_decision( + self, + *, + key: str, + caller_run_id: str, + run_owner: str, + claim_owner: str, + tool_name: str, + args_hash: str, + lease_seconds: int, + ) -> types.IntentDecision: + return intent_transactions.existing_decision(self, key=key, caller_run_id=caller_run_id, run_owner=run_owner, claim_owner=claim_owner, tool_name=tool_name, args_hash=args_hash, lease_seconds=lease_seconds) + + def begin_intent( + self, + *, + run_id: str, + owner: str, + claim_owner: str, + tool_call_id: str, + tool_name: str, + args: Mapping[str, Any], + recovery_policy: str, + idempotency_key: Optional[str] = None, + lease_seconds: int = 300, + _conflict_retries: int = 4, + ) -> types.IntentDecision: + return intent_transactions.begin_intent(self, run_id=run_id, owner=owner, claim_owner=claim_owner, tool_call_id=tool_call_id, tool_name=tool_name, args=args, recovery_policy=recovery_policy, idempotency_key=idempotency_key, lease_seconds=lease_seconds, _conflict_retries=_conflict_retries) + + def acquire_invocation_claim( + self, + effect_id: str, + *, + run_owner: str, + claim_owner: str, + lease_seconds: int = 300, + ) -> None: + return invocation_claims.acquire_invocation_claim(self, effect_id, run_owner=run_owner, claim_owner=claim_owner, lease_seconds=lease_seconds) + + def renew_invocation_claim( + self, + effect_id: str, + *, + run_owner: str, + claim_owner: str, + lease_seconds: int, + ) -> bool: + return invocation_claims.renew_invocation_claim(self, effect_id, run_owner=run_owner, claim_owner=claim_owner, lease_seconds=lease_seconds) + + def renew_recovery_claim( + self, + effect_id: str, + *, + recovery_owner: str, + lease_seconds: int, + ) -> bool: + return invocation_claims.renew_recovery_claim(self, effect_id, recovery_owner=recovery_owner, lease_seconds=lease_seconds) + + def _intent_run_id(self, effect_id: str) -> str: + return invocation_claims.intent_run_id(self, effect_id) + + def abandon_invocation_claim( + self, + effect_id: str, + *, + run_owner: str, + claim_owner: str, + ) -> None: + return invocation_claims.abandon_invocation_claim(self, effect_id, run_owner=run_owner, claim_owner=claim_owner) + + def _commit_result_locked( + self, + db: Session, + intent: ToolEffectLedger, + lease: ToolEffectLease, + *, + owner: str, + result: Any, + failed: bool, + reconciled: bool, + ) -> Any: + return result_transactions.commit_result_locked(self, db, intent, lease, owner=owner, result=result, failed=failed, reconciled=reconciled) + + def commit_result( + self, + effect_id: str, + *, + run_owner: str, + claim_owner: str, + result: Any, + failed: bool = False, + reconciled: bool = False, + _conflict_retries: int = 4, + ) -> Any: + return result_transactions.commit_result(self, effect_id, run_owner=run_owner, claim_owner=claim_owner, result=result, failed=failed, reconciled=reconciled, _conflict_retries=_conflict_retries) + + def mark_unknown( + self, + effect_id: str, + *, + owner: str, + claim_owner: str, + reason: str, + ) -> None: + return result_transactions.mark_unknown(self, effect_id, owner=owner, claim_owner=claim_owner, reason=reason) + + def terminal_decision(self, effect_id: str) -> Optional[types.IntentDecision]: + return result_transactions.terminal_decision(self, effect_id) + + def pending_effect_ids(self) -> list[str]: + return result_transactions.pending_effect_ids(self) + + def prune_settled(self, *, retention_days: int = 7) -> int: + return retention.prune_settled(self, retention_days=retention_days) + + def settle_terminal_run_intent(self, effect_id: str) -> Optional[types.ToolIntent]: + return retention.settle_terminal_run_intent(self, effect_id) + + def claim_recovery_intent( + self, + effect_id: str, + *, + recovery_owner: str, + lease_seconds: int = 120, + ) -> Optional[types.ToolIntent]: + return recovery_claims.claim_recovery_intent(self, effect_id, recovery_owner=recovery_owner, lease_seconds=lease_seconds) + + def commit_recovered_result( + self, + effect_id: str, + result: Any, + *, + recovery_owner: str, + ) -> Any: + return recovery_claims.commit_recovered_result(self, effect_id, result, recovery_owner=recovery_owner) + + def release_recovery_claim( + self, + effect_id: str, + *, + recovery_owner: str, + retry_seconds: int = 30, + ) -> None: + return recovery_claims.release_recovery_claim(self, effect_id, recovery_owner=recovery_owner, retry_seconds=retry_seconds) diff --git a/src/backend/core/runs/tool_effects/journal_constants.py b/src/backend/core/runs/tool_effects/journal_constants.py new file mode 100644 index 00000000..6963b27b --- /dev/null +++ b/src/backend/core/runs/tool_effects/journal_constants.py @@ -0,0 +1,32 @@ +"""Retention and redaction settings for the tool effect journal.""" +from __future__ import annotations +import re + +ATTENTION_RETENTION_FACTOR = 4 + + +REDACT_FIELDS = [ + "password", + "token", + "api_key", + "apikey", + "authorization", + "secret", + "access_key", + "cookie", +] + + +WITHHOLD_FIELDS = {"command", "content", "stdin", "script", "code"} + + +EMBEDDED_SECRET_PATTERNS = ( + re.compile(r"(?i)(authorization\s*:\s*bearer\s+)[^\s'\";]+"), + re.compile(r"(?i)(bearer\s+)[A-Za-z0-9._~+/=-]{8,}"), + re.compile( + r"(?i)([?&](?:token|api_key|apikey|access_key|secret|password|passwd|client_secret|authorization)=)[^&#\s]+" + ), + re.compile( + r"(?i)((?:token|api_key|apikey|access_key|secret|password|passwd|client_secret|authorization)\s*[=:]\s*)[^\s'\";]+" + ), +) diff --git a/src/backend/core/runs/tool_effects/recovery.py b/src/backend/core/runs/tool_effects/recovery.py new file mode 100644 index 00000000..5fd565b3 --- /dev/null +++ b/src/backend/core/runs/tool_effects/recovery.py @@ -0,0 +1,176 @@ +"""Recover tool effects left incomplete by a lost process.""" +from __future__ import annotations +from core.runs.tool_effects.journal import ToolEffectJournal +import asyncio +import time +import uuid +from contextlib import suppress +from typing import Optional +import logging +from core.runs.tool_effects import errors +from core.runs.tool_effects import gateway +from core.runs.tool_effects.registry import DEFAULT_TOOL_RECOVERY_REGISTRY, ToolRecoveryRegistry +from core.runs.tool_effects import types + +logger = logging.getLogger(__name__) + +async def recover_incomplete_tool_effects( + *, + journal: Optional[ToolEffectJournal] = None, + registry: Optional[ToolRecoveryRegistry] = None, + lease_seconds: int = 120, + heartbeat_interval: Optional[float] = None, +) -> list[types.RecoverySweepDecision]: + """Recover each orphan independently; one broken adapter cannot starve later work.""" + + journal = journal or ToolEffectJournal() + registry = registry or DEFAULT_TOOL_RECOVERY_REGISTRY + from core.harness.usage import UsageAttempt, attempt_status_for_exception, record_usage_safely + from core.harness.ledger import HarnessUsageLedger + + usage_ledger = HarnessUsageLedger(journal._sessions) + + async def _recorded_recovery_call(handler, intent, phase): # noqa: ANN001 + started = time.monotonic() + status = "failed" + try: + result = await gateway.ToolEffectGateway._call(handler, intent) + status = "success" + return result + except asyncio.CancelledError: + status = "cancelled" + raise + except Exception as exc: + status = attempt_status_for_exception(exc) + raise + finally: + try: + await record_usage_safely( + usage_ledger, + UsageAttempt( + run_id=intent.run_id, + kind="tool", + operation_name=intent.tool_name, + effect_id=intent.effect_id, + status=status, + latency_ms=int((time.monotonic() - started) * 1_000), + metadata={"recovery_phase": phase}, + ), + ) + except Exception: # noqa: BLE001 + logger.debug("tool recovery usage persistence failed", exc_info=True) + + decisions: list[types.RecoverySweepDecision] = [] + for effect_id in await asyncio.to_thread(journal.pending_effect_ids): + terminal_intent = journal.settle_terminal_run_intent(effect_id) + if terminal_intent is not None: + decisions.append( + types.RecoverySweepDecision( + effect_id, terminal_intent.run_id, "terminal_unknown" + ) + ) + continue + recovery_owner = f"tool-recovery:{uuid.uuid4().hex}" + intent = journal.claim_recovery_intent( + effect_id, + recovery_owner=recovery_owner, + lease_seconds=lease_seconds, + ) + if intent is None: + continue + policy = registry.resolve(intent.tool_name) + claim_lost = asyncio.Event() + + async def _heartbeat() -> None: + interval = heartbeat_interval + if interval is None: + interval = max(0.05, min(30.0, lease_seconds / 3)) + while True: + await asyncio.sleep(interval) + try: + ok = await asyncio.to_thread( + journal.renew_recovery_claim, + effect_id, + recovery_owner=recovery_owner, + lease_seconds=lease_seconds, + ) + except Exception: # noqa: BLE001 - lost heartbeat means lost ownership + ok = False + if not ok: + claim_lost.set() + return + + heartbeat = asyncio.create_task(_heartbeat()) + try: + if intent.recovery_policy == "never_replay": + journal.mark_unknown( + effect_id, + owner=recovery_owner, + claim_owner=recovery_owner, + reason="non-replayable tool intent has no committed result", + ) + decisions.append( + types.RecoverySweepDecision(effect_id, intent.run_id, "needs_attention") + ) + continue + if intent.recovery_policy == "replay_safe": + if policy.replayer is None: + raise errors.ToolEffectError("replay_safe policy has no replay adapter") + result = await _recorded_recovery_call( + policy.replayer, intent, "replay" + ) + if claim_lost.is_set(): + raise errors.ToolEffectLeaseLost(effect_id) + journal.commit_recovered_result( + effect_id, result, recovery_owner=recovery_owner + ) + decisions.append( + types.RecoverySweepDecision(effect_id, intent.run_id, "replayed") + ) + continue + + if policy.reconciler is None: + raise errors.ToolEffectError("reconcile policy has no reconciliation adapter") + reconciliation = gateway.ToolEffectGateway._reconciliation( + await _recorded_recovery_call(policy.reconciler, intent, "reconcile") + ) + if claim_lost.is_set(): + raise errors.ToolEffectLeaseLost(effect_id) + if reconciliation.outcome == "applied": + result = reconciliation.result + elif ( + reconciliation.outcome == "not_applied" and policy.replayer is not None + ): + result = await _recorded_recovery_call( + policy.replayer, intent, "replay_after_reconcile" + ) + else: + journal.mark_unknown( + effect_id, + owner=recovery_owner, + claim_owner=recovery_owner, + reason="reconciliation could not determine external outcome", + ) + decisions.append( + types.RecoverySweepDecision(effect_id, intent.run_id, "needs_attention") + ) + continue + if claim_lost.is_set(): + raise errors.ToolEffectLeaseLost(effect_id) + journal.commit_recovered_result( + effect_id, result, recovery_owner=recovery_owner + ) + decisions.append( + types.RecoverySweepDecision(effect_id, intent.run_id, "reconciled") + ) + except Exception: # noqa: BLE001 - release this claim and continue the sweep + try: + journal.release_recovery_claim(effect_id, recovery_owner=recovery_owner) + except errors.ToolEffectError: + pass + decisions.append(types.RecoverySweepDecision(effect_id, intent.run_id, "retry")) + finally: + heartbeat.cancel() + with suppress(asyncio.CancelledError): + await heartbeat + return decisions diff --git a/src/backend/core/runs/tool_effects/recovery_claims.py b/src/backend/core/runs/tool_effects/recovery_claims.py new file mode 100644 index 00000000..c2f72da2 --- /dev/null +++ b/src/backend/core/runs/tool_effects/recovery_claims.py @@ -0,0 +1,166 @@ +"""Claim and settle intents during recovery.""" +from __future__ import annotations +from datetime import timedelta +from typing import Any, Optional +from core.db.models import ChatRun, ToolEffectLease, ToolEffectLedger +from core.runs.journal import LIVE_STATUSES +from sqlalchemy import or_ +from core.runs.tool_effects import errors +from core.runs.tool_effects import types + +def claim_recovery_intent( + self, + effect_id: str, + *, + recovery_owner: str, + lease_seconds: int = 120, +) -> Optional[types.ToolIntent]: + with self._sessions() as db: + intent = ( + db.query(ToolEffectLedger) + .filter( + ToolEffectLedger.effect_id == effect_id, + ToolEffectLedger.event_type == "intent", + ) + .one_or_none() + ) + if intent is None or self._terminal_row(db, effect_id) is not None: + return None + now = self._clock() + from core.auth.api_key_runs import authorize_run_execution + + if not authorize_run_execution(db, intent.run_id, now): + return None + expires = now + timedelta(seconds=max(1, lease_seconds)) + run_affected = ( + db.query(ChatRun) + .filter( + ChatRun.run_id == intent.run_id, + ChatRun.status.in_((*LIVE_STATUSES, "needs_attention")), + or_( + ChatRun.lease_owner.is_(None), + ChatRun.lease_expires_at.is_(None), + ChatRun.lease_expires_at <= now, + ChatRun.lease_owner == recovery_owner, + ), + ) + .update( + { + ChatRun.status: "running", + ChatRun.lease_owner: recovery_owner, + ChatRun.lease_expires_at: expires, + ChatRun.updated_at: now, + }, + synchronize_session=False, + ) + ) + if not run_affected: + db.rollback() + return None + lease_affected = ( + db.query(ToolEffectLease) + .filter( + ToolEffectLease.effect_id == effect_id, + or_( + ToolEffectLease.lease_expires_at.is_(None), + ToolEffectLease.lease_expires_at <= now, + ToolEffectLease.claim_owner == recovery_owner, + ), + ) + .update( + { + ToolEffectLease.run_owner: recovery_owner, + ToolEffectLease.claim_owner: recovery_owner, + ToolEffectLease.lease_expires_at: expires, + ToolEffectLease.updated_at: now, + }, + synchronize_session=False, + ) + ) + if not lease_affected or self._terminal_row(db, effect_id) is not None: + db.rollback() + return None + db.expire_all() + run = db.get(ChatRun, intent.run_id) + self._append_event( + db, + run, + intent, + owner=recovery_owner, + event_type="recovery_claim", + operation_type="tool_effect_recovery_claimed", + phase=f"tool_intent_{intent.recovery_policy}", + safety=intent.recovery_policy, + ) + db.commit() + return self._intent_from_row(intent) + + +def commit_recovered_result( + self, + effect_id: str, + result: Any, + *, + recovery_owner: str, +) -> Any: + value = self.commit_result( + effect_id, + run_owner=recovery_owner, + claim_owner=recovery_owner, + result=result, + reconciled=True, + ) + with self._sessions() as db: + lease = ( + db.query(ToolEffectLease) + .filter(ToolEffectLease.effect_id == effect_id) + .with_for_update() + .one() + ) + run = ( + db.query(ChatRun) + .filter(ChatRun.run_id == lease.run_id) + .with_for_update() + .one() + ) + if lease.run_owner != recovery_owner or run.lease_owner != recovery_owner: + raise errors.ToolEffectLeaseLost(effect_id) + run.lease_owner = None + run.lease_expires_at = None + run.updated_at = self._clock() + db.commit() + return value + + +def release_recovery_claim( + self, + effect_id: str, + *, + recovery_owner: str, + retry_seconds: int = 30, +) -> None: + """Defer a failed adapter without letting RunJournal resume past the pending effect.""" + with self._sessions() as db: + lease = ( + db.query(ToolEffectLease) + .filter(ToolEffectLease.effect_id == effect_id) + .with_for_update() + .one() + ) + if lease.run_owner != recovery_owner or lease.claim_owner != recovery_owner: + raise errors.ToolEffectLeaseLost(effect_id) + run = ( + db.query(ChatRun) + .filter(ChatRun.run_id == lease.run_id) + .with_for_update() + .one() + ) + if run.lease_owner != recovery_owner: + raise errors.ToolEffectLeaseLost(effect_id) + retry_at = self._clock() + timedelta(seconds=max(1, retry_seconds)) + lease.claim_owner = None + lease.lease_expires_at = retry_at + lease.updated_at = self._clock() + run.lease_expires_at = retry_at + run.updated_at = self._clock() + db.commit() diff --git a/src/backend/core/runs/tool_effects/registry.py b/src/backend/core/runs/tool_effects/registry.py new file mode 100644 index 00000000..36d1608e --- /dev/null +++ b/src/backend/core/runs/tool_effects/registry.py @@ -0,0 +1,95 @@ +"""Recovery policy for each tool; unknown tools are never replayed.""" +from __future__ import annotations +from typing import Any, Awaitable, Callable, Optional, cast +from core.runs.tool_effects import types + +RECOVERY_POLICIES = ("replay_safe", "reconcile", "never_replay") + + +class ToolRecoveryRegistry: + """Explicit adapter contracts. Unknown tools fail closed as never-replay.""" + + def __init__(self) -> None: + self._policies: dict[str, types.ToolPolicy] = {} + + def register( + self, + tool_name: str, + policy: str, + *, + replayer: Optional[Callable[[types.ToolIntent], Awaitable[Any] | Any]] = None, + reconciler: Optional[ + Callable[[types.ToolIntent], Awaitable[types.ReconciliationResult | Any] | Any] + ] = None, + ) -> None: + if policy not in RECOVERY_POLICIES: + raise ValueError(f"invalid recovery policy: {policy}") + if policy == "replay_safe" and reconciler is not None: + raise ValueError("replay_safe does not accept a reconciler") + if policy == "never_replay" and ( + replayer is not None or reconciler is not None + ): + raise ValueError("never_replay does not accept recovery adapters") + self._policies[str(tool_name)] = types.ToolPolicy(policy, replayer, reconciler) + + def resolve(self, tool_name: str) -> types.ToolPolicy: + return self._policies.get(str(tool_name), types.ToolPolicy("never_replay")) + + +async def default_replayer(intent: types.ToolIntent) -> Any: + from core.runs.tool_effect_recovery import replay_tool_intent + + return await replay_tool_intent(intent) + + +async def scheduled_task_reconciler(intent: types.ToolIntent) -> types.ReconciliationResult: + from core.runs.tool_effect_recovery import reconcile_scheduled_task_intent + + return cast(types.ReconciliationResult, await reconcile_scheduled_task_intent(intent)) + + +def build_default_tool_recovery_registry() -> ToolRecoveryRegistry: + registry = ToolRecoveryRegistry() + for name in ( + "Read", + "read_project_instructions", + "list_sites", + "Glob", + "Grep", + "view_text_file", + "get_data_context", + "read_image", + "read_artifact", + "channel_read_attachment", + "sandbox_get_artifact", + ): + registry.register(name, "replay_safe", replayer=default_replayer) + for name in ( + "Write", + "save_project_instructions", + "Edit", + "Bash", + "sandbox_put_artifact", + "pin_to_workspace", + "run_job", + "choose_design", + "call_subagent", + "update_plan", + "load_plugin", + ): + registry.register(name, "never_replay") + for name in ( + "create_scheduled_task", + "update_scheduled_task", + "delete_scheduled_task", + ): + registry.register( + name, + "reconcile", + replayer=default_replayer, + reconciler=scheduled_task_reconciler, + ) + return registry + + +DEFAULT_TOOL_RECOVERY_REGISTRY = build_default_tool_recovery_registry() diff --git a/src/backend/core/runs/tool_effects/result_transactions.py b/src/backend/core/runs/tool_effects/result_transactions.py new file mode 100644 index 00000000..b66b8e3b --- /dev/null +++ b/src/backend/core/runs/tool_effects/result_transactions.py @@ -0,0 +1,285 @@ +"""Commit tool results or mark their outcome unknown.""" +from __future__ import annotations +from typing import Any, Optional +from core.db.models import ChatRun, ToolEffectLease, ToolEffectLedger +from sqlalchemy.exc import IntegrityError, OperationalError +from sqlalchemy.orm import Session, aliased +from core.runs.tool_effects import errors +from core.runs.tool_effects import types +from core.infra.time import as_utc + +def append_event( + self, + db: Session, + run: ChatRun, + intent: ToolEffectLedger, + *, + owner: str, + event_type: str, + operation_type: str, + phase: str, + safety: str, + result: Any = None, + error: Optional[str] = None, + terminal: bool = False, +) -> None: + receipt = self._runs._append_locked( + db, + run, + owner=owner, + operation_type=operation_type, + phase=phase, + safety=safety, + payload={"effect_id": intent.effect_id, "result_id": intent.result_id}, + ) + db.add( + ToolEffectLedger( + effect_id=intent.effect_id, + result_id=intent.result_id, + run_id=intent.run_id, + operation_seq=receipt.operation_seq, + event_type=event_type, + tool_name=intent.tool_name, + tool_call_id=intent.tool_call_id, + args_hash=intent.args_hash, + redacted_args=intent.redacted_args, + recovery_policy=intent.recovery_policy, + result_payload=result, + error_message=error, + terminal_effect_id=intent.effect_id if terminal else None, + ) + ) + + +def mark_unknown_locked( + self, + db: Session, + run: ChatRun, + intent: ToolEffectLedger, + lease: ToolEffectLease, + *, + owner: str, + claim_owner: str, + reason: str, +) -> None: + if self._terminal_row(db, intent.effect_id) is not None: + return + if lease.run_owner != owner or lease.claim_owner != claim_owner: + raise errors.ToolEffectLeaseLost(intent.effect_id) + self._append_event( + db, + run, + intent, + owner=owner, + event_type="unknown_outcome", + operation_type="tool_outcome_unknown", + phase="needs_attention", + safety="manual_review", + error=reason, + terminal=True, + ) + run.status = "needs_attention" + run.failure_reason = reason + run.lease_owner = None + run.lease_expires_at = None + run.updated_at = self._clock() + lease.claim_owner = None + lease.lease_expires_at = None + lease.updated_at = self._clock() + + +def commit_result_locked( + self, + db: Session, + intent: ToolEffectLedger, + lease: ToolEffectLease, + *, + owner: str, + result: Any, + failed: bool, + reconciled: bool, +) -> Any: + terminal = self._terminal_row(db, intent.effect_id) + if terminal is not None: + if terminal.event_type == "unknown_outcome": + raise errors.ToolOutcomeUnknown(intent.effect_id) + return terminal.result_payload + if lease.run_owner != owner: + raise errors.ToolEffectLeaseLost(intent.effect_id) + run = self._runs._locked_owned_run(db, intent.run_id, owner) + self._append_event( + db, + run, + intent, + owner=owner, + event_type="failure" if failed else "result", + operation_type="tool_failure_committed" + if failed + else "tool_result_committed", + phase="tool_result_committed", + safety="reconciled" if reconciled else intent.recovery_policy, + result=result, + error=str(result)[:4000] if failed else None, + terminal=True, + ) + lease.claim_owner = None + lease.lease_expires_at = None + lease.updated_at = self._clock() + run.updated_at = self._clock() + return result + + +def commit_result( + self, + effect_id: str, + *, + run_owner: str, + claim_owner: str, + result: Any, + failed: bool = False, + reconciled: bool = False, + _conflict_retries: int = 4, +) -> Any: + with self._sessions() as db: + intent = ( + db.query(ToolEffectLedger) + .filter( + ToolEffectLedger.effect_id == effect_id, + ToolEffectLedger.event_type == "intent", + ) + .one() + ) + terminal = self._terminal_row(db, effect_id) + if terminal is not None: + if terminal.event_type == "unknown_outcome": + raise errors.ToolOutcomeUnknown(effect_id) + return terminal.result_payload + lease = ( + db.query(ToolEffectLease) + .filter(ToolEffectLease.effect_id == effect_id) + .with_for_update() + .one() + ) + now = as_utc(self._clock()) + lease_expiry = as_utc(lease.lease_expires_at) + if ( + lease.run_owner != run_owner + or lease.claim_owner != claim_owner + or lease_expiry is None + or now is None + or lease_expiry <= now + ): + # A concurrent retry can observe the lease only after the + # winning commit cleared its claim. Re-check the durable + # terminal decision before reporting lease loss: identical + # result commits are idempotent even when the first pre-lock + # terminal read raced with the winner. + terminal = self._terminal_row(db, effect_id) + if terminal is not None: + if terminal.event_type == "unknown_outcome": + raise errors.ToolOutcomeUnknown(effect_id) + return terminal.result_payload + raise errors.ToolEffectLeaseLost(effect_id) + value = self._commit_result_locked( + db, + intent, + lease, + owner=run_owner, + result=result, + failed=failed, + reconciled=reconciled, + ) + try: + db.commit() + except (IntegrityError, OperationalError) as exc: + db.rollback() + terminal = self.terminal_decision(effect_id) + if terminal is not None: + if terminal.action == "unknown": + raise errors.ToolOutcomeUnknown(effect_id) + return terminal.result + if _conflict_retries > 0: + return self.commit_result( + effect_id, + run_owner=run_owner, + claim_owner=claim_owner, + result=result, + failed=failed, + reconciled=reconciled, + _conflict_retries=_conflict_retries - 1, + ) + raise errors.ToolEffectError(str(exc)) from exc + return value + + +def mark_unknown( + self, + effect_id: str, + *, + owner: str, + claim_owner: str, + reason: str, +) -> None: + with self._sessions() as db: + lease = ( + db.query(ToolEffectLease) + .filter(ToolEffectLease.effect_id == effect_id) + .with_for_update() + .one() + ) + intent = ( + db.query(ToolEffectLedger) + .filter( + ToolEffectLedger.effect_id == effect_id, + ToolEffectLedger.event_type == "intent", + ) + .one() + ) + run = self._runs._locked_owned_run(db, intent.run_id, owner) + self._mark_unknown_locked( + db, + run, + intent, + lease, + owner=owner, + claim_owner=claim_owner, + reason=reason, + ) + db.commit() + + +def terminal_decision(self, effect_id: str) -> Optional[types.IntentDecision]: + with self._sessions() as db: + intent = ( + db.query(ToolEffectLedger) + .filter( + ToolEffectLedger.effect_id == effect_id, + ToolEffectLedger.event_type == "intent", + ) + .one_or_none() + ) + if intent is None: + return None + terminal = self._terminal_row(db, effect_id) + return ( + self._decision_from_terminal(self._intent_from_row(intent), terminal) + if terminal is not None + else None + ) + + +def pending_effect_ids(self) -> list[str]: + with self._sessions() as db: + terminal = aliased(ToolEffectLedger) + rows = ( + db.query(ToolEffectLedger.effect_id) + .filter( + ToolEffectLedger.event_type == "intent", + ~db.query(terminal.event_id) + .filter(terminal.terminal_effect_id == ToolEffectLedger.effect_id) + .exists(), + ) + .order_by(ToolEffectLedger.created_at, ToolEffectLedger.event_id) + .all() + ) + return [row.effect_id for row in rows] diff --git a/src/backend/core/runs/tool_effects/retention.py b/src/backend/core/runs/tool_effects/retention.py new file mode 100644 index 00000000..0d089fe1 --- /dev/null +++ b/src/backend/core/runs/tool_effects/retention.py @@ -0,0 +1,154 @@ +"""Prune settled tool effect records and close orphaned intents.""" +from __future__ import annotations +from datetime import timedelta +from typing import Optional +from core.db.models import ChatRun, ToolEffectLease, ToolEffectLedger +from core.runs.journal import LIVE_STATUSES +from sqlalchemy import and_, case +from core.runs.tool_effects import journal_constants +from core.runs.tool_effects import types +from core.infra.time import as_utc + +def prune_settled(self, *, retention_days: int = 7) -> int: + """Drop per-run recovery/observability facts of long-terminal runs. + + Covers the tool effect ledger and leases plus the other append-only + per-run journals (harness usage attempts, harness events, run + operations); without this they grow without bound. + + The run's own recovery columns go the same way. ``recovery_snapshot`` + carries the whole conversation as of that turn, so a long chat stores + roughly its own history once per turn — the single largest thing in a + desktop database. Only crash recovery reads it, and only for runs that + are still live and still replayable, which is exactly what the cutoff + and the settled filter below exclude. + + ``needs_attention`` is kept ``_ATTENTION_RETENTION_FACTOR`` times longer + rather than exempted, and that applies to every tier above. Exempting it + means never: nothing in the backend transitions a run out of that status, + so an exemption pins its journals — and its snapshot — for good. In + production every record that had outlived the retention window belonged + to such a run. + """ + from core.db.models import ChatRunOperation, HarnessEventLog, HarnessUsageAttempt + + now = as_utc(self._clock()) + cutoff = now - timedelta(days=retention_days) + attention_cutoff = now - timedelta(days=retention_days * journal_constants.ATTENTION_RETENTION_FACTOR) + with self._sessions() as db: + + earliest_cutoff = max(cutoff, attention_cutoff) + + def _prunable(run_col, stamp): + """Row whose run is terminal and past that run's retention tier. + + ``stamp < earliest_cutoff`` is implied by the EXISTS (it is the + loosest of the tiers) and stated anyway: it is the only + predicate on these tables an index can serve, and without it the + planner has nothing to range-scan on. + """ + tier = case( + (ChatRun.status == "needs_attention", attention_cutoff), + else_=cutoff, + ) + return and_( + stamp < earliest_cutoff, + db.query(ChatRun.run_id) + .filter( + ChatRun.run_id == run_col, + ChatRun.status.notin_(LIVE_STATUSES), + stamp < tier, + ) + .exists(), + ) + + deleted = ( + db.query(ToolEffectLedger) + .filter(_prunable(ToolEffectLedger.run_id, ToolEffectLedger.created_at)) + .delete(synchronize_session=False) + ) + db.query(ToolEffectLease).filter( + _prunable(ToolEffectLease.run_id, ToolEffectLease.updated_at) + ).delete(synchronize_session=False) + for model in (HarnessUsageAttempt, HarnessEventLog, ChatRunOperation): + deleted += ( + db.query(model) + .filter(_prunable(model.run_id, model.created_at)) + .delete(synchronize_session=False) + ) + deleted += ( + db.query(ChatRun) + .filter( + ChatRun.status.notin_(LIVE_STATUSES), + ChatRun.completed_at < earliest_cutoff, + ChatRun.completed_at + < case( + (ChatRun.status == "needs_attention", attention_cutoff), + else_=cutoff, + ), + ChatRun.recovery_snapshot.isnot(None), + ) + .update( + {ChatRun.recovery_snapshot: None, ChatRun.request_payload: None}, + synchronize_session=False, + ) + ) + db.commit() + return deleted + + +def settle_terminal_run_intent(self, effect_id: str) -> Optional[types.ToolIntent]: + """Close an orphan intent without replay after its owning run is terminal. + + Cancellation/failure is a hard replay boundary: the user's stopped run + must never perform a delayed external action. We still append an + ``unknown_outcome`` fact so the intent cannot remain pending forever. + """ + with self._sessions() as db: + intent = ( + db.query(ToolEffectLedger) + .filter( + ToolEffectLedger.effect_id == effect_id, + ToolEffectLedger.event_type == "intent", + ) + .with_for_update() + .one_or_none() + ) + if intent is None or self._terminal_row(db, effect_id) is not None: + return None + run = ( + db.query(ChatRun) + .filter(ChatRun.run_id == intent.run_id) + .with_for_update() + .one_or_none() + ) + if run is None or run.status in (*LIVE_STATUSES, "needs_attention"): + return None + lease = ( + db.query(ToolEffectLease) + .filter(ToolEffectLease.effect_id == effect_id) + .with_for_update() + .one() + ) + terminal_status = str(run.status) + self._append_event( + db, + run, + intent, + owner="system:terminal-settlement", + event_type="unknown_outcome", + operation_type="tool_outcome_unknown_after_terminal_run", + phase=str(run.run_phase or terminal_status), + safety="manual_review", + error=f"run became {terminal_status} before the tool outcome committed", + terminal=True, + ) + # _append_event advances only the operation ledger. Preserve the + # authoritative terminal verdict and merely close the effect claim. + run.status = terminal_status + lease.claim_owner = None + lease.lease_expires_at = None + lease.updated_at = self._clock() + run.updated_at = self._clock() + db.commit() + return self._intent_from_row(intent) diff --git a/src/backend/core/runs/tool_effects/types.py b/src/backend/core/runs/tool_effects/types.py new file mode 100644 index 00000000..2052f25d --- /dev/null +++ b/src/backend/core/runs/tool_effects/types.py @@ -0,0 +1,74 @@ +"""Value types for tool intents, decisions and outcomes.""" +from __future__ import annotations +from dataclasses import dataclass +from datetime import datetime +from typing import Any, Awaitable, Callable, Mapping, Optional + +@dataclass(frozen=True) +class ReconciliationResult: + outcome: str + result: Any = None + + @classmethod + def applied(cls, result: Any) -> "ReconciliationResult": + return cls("applied", result) + + @classmethod + def not_applied(cls) -> "ReconciliationResult": + return cls("not_applied") + + @classmethod + def unknown(cls) -> "ReconciliationResult": + return cls("unknown") + + +@dataclass(frozen=True) +class ToolPolicy: + policy: str + replayer: Optional[Callable[["ToolIntent"], Awaitable[Any] | Any]] = None + reconciler: Optional[ + Callable[["ToolIntent"], Awaitable[ReconciliationResult | Any] | Any] + ] = None + + +@dataclass(frozen=True) +class ToolIntent: + effect_id: str + result_id: str + run_id: str + operation_seq: int + tool_name: str + tool_call_id: str + args_hash: str + redacted_args: Mapping[str, Any] + idempotency_key: str + recovery_policy: str + created_at: Optional[datetime] = None + + +@dataclass(frozen=True) +class IntentDecision: + action: str + intent: ToolIntent + result: Any = None + result_state: str = "success" + + @property + def effect_id(self) -> str: + return self.intent.effect_id + + +@dataclass(frozen=True) +class ToolExecutionOutcome: + result: Any + effect_id: str + result_id: str + result_state: str + invoked: bool + + +@dataclass(frozen=True) +class RecoverySweepDecision: + effect_id: str + run_id: str + action: str diff --git a/src/backend/core/runs/watchdog.py b/src/backend/core/runs/watchdog.py new file mode 100644 index 00000000..799613fa --- /dev/null +++ b/src/backend/core/runs/watchdog.py @@ -0,0 +1,319 @@ +"""Detect stalled streams and reap expired runs without replacing a live writer.""" + +from __future__ import annotations + +import asyncio +import contextlib +import datetime as _datetime +from datetime import datetime +from typing import Any, AsyncIterator, Callable, Optional + +import core.db.engine as engine +import core.db.models as models +import core.llm.runtime.human_interaction as human_interaction +import core.runs.clock as clock +import core.runs.events.writer as writer +import core.runs.ownership as ownership +import core.runs.settings as settings +import core.runs.tasks as tasks +from core.infra.logging import get_logger + +logger = get_logger(__name__) + + +async def _aiter_with_inactivity_timeout( + aiter: AsyncIterator[Any], + timeout: float, + *, + is_activity: Callable[[Any], bool] | None = None, + first_response_timeout: float | None = None, +): + """Yield from an async iterator and enforce a meaningful-activity deadline. + + Guards against a hung workflow (model loop / blocked MCP / asyncio + deadlock) leaving the chat run stuck in 'running' forever — on timeout + the underlying generator is closed and the error propagates into the + existing ``except Exception`` path that writes the ``failed`` terminal. + + Transport keep-alives may still be yielded to the caller, but they do not + reset the deadline when ``is_activity`` marks them as non-meaningful. + The optional first-response deadline is absolute until visible output; + it is then disabled for the remainder of this workflow. + """ + activity_check = is_activity or (lambda _: True) + loop = asyncio.get_running_loop() + deadline = loop.time() + timeout + first_deadline = ( + loop.time() + first_response_timeout if first_response_timeout is not None else None + ) + + def timeout_error() -> TimeoutError: + from core.infra.exceptions import FirstResponseTimeoutError + + if first_deadline is not None and loop.time() >= first_deadline: + return FirstResponseTimeoutError() + return TimeoutError(f"工作流 {timeout:.0f}s 内无有效输出，已判定为卡死并中止") + + while True: + effective_deadline = ( + min(deadline, first_deadline) if first_deadline is not None else deadline + ) + remaining = effective_deadline - loop.time() + if remaining <= 0: + with contextlib.suppress(Exception): + await aiter.aclose() # type: ignore[attr-defined] + raise timeout_error() + try: + item = await asyncio.wait_for(aiter.__anext__(), timeout=remaining) + except StopAsyncIteration: + return + except asyncio.TimeoutError as exc: + with contextlib.suppress(Exception): + await aiter.aclose() # type: ignore[attr-defined] + raise timeout_error() from exc + if first_deadline is not None and _is_first_response(item): + first_deadline = None + if activity_check(item): + deadline = loop.time() + timeout + yield item + + +def _is_first_response(item: Any) -> bool: + """Only visible model output or a tool interaction ends the initial wait.""" + kind = item.get("type") + if kind in {"ai_message", "content", "thinking"}: + return bool(item.get("delta")) + if kind == "content_replace": + return bool(item.get("content")) + return kind in { + "tool_call_start", + "tool_call_delta", + "tool_call", + "tool_result", + "tool_pending", + "file_confirm", + "plan_update", + "compaction_progress", + } + + +def _is_chat_run_activity(item: Any, chat_id: str) -> bool: + """Classify workflow output for the inactivity watchdog. + + A transport heartbeat is normally not meaningful progress. It becomes + progress only while a registered tool is legitimately suspended on a + human answer; this keeps true HITL waits alive without weakening zombie + detection for every other silent run. + """ + + return item.get("type") != "heartbeat" or human_interaction.has_pending(chat_id) + + +async def _stream_last_write_ms(run_id: str) -> Optional[int]: + """Write time of the last event on the run's log (epoch ms). + + Cursors are minted as ``-`` by every backend, so the millisecond + half is the write time — no extra bookkeeping. Returns None when there is + no log / no events / the read fails (callers treat that as "no activity"). + """ + try: + return await writer._events().last_write_ms(run_id) + except Exception as exc: + logger.warning("chat_run_activity_check_failed", run_id=run_id, error=str(exc)) + return None + + +def _chat_has_live_job(chat_id: Optional[str], now: datetime) -> bool: + """这个会话是否挂着**还在推进**的批量作业（跨进程可见的活性证据）。 + + 判据是 ``jobs.updated_at`` 在静默窗口内前进过：作业每完成一次子调用都会写 usage， + onupdate 随之推进 updated_at。只要作业还在动，这个 run 就不是僵尸——哪怕它的 + asyncio task 属于另一个进程（多 worker / 多副本），也哪怕主链路一个流事件都没有。 + + 作业本身有 max_seconds 墙钟预算与熔断，所以这条豁免不会让 run 永生。 + """ + if not chat_id: + return False + try: + from core.db.models import Job + + cutoff = now - _datetime.timedelta(seconds=settings._STALE_QUIET_SEC) + with engine.SessionLocal() as db: + return ( + db.query(Job.job_id) + .filter( + Job.chat_id == chat_id, + Job.status.in_(("pending", "running")), + Job.updated_at >= cutoff, + ) + .first() + is not None + ) + except Exception: # noqa: BLE001 —— 证据查不到就按原判据走，绝不因此放过真僵尸 + return False + + +async def reap_stale_runs() -> int: + """Periodic safety net: fail 'running' runs that show no sign of life. + + The verdict is activity-aware, in two tiers (both tunable via env vars): + + - Reap only when **over-age + quiet**: lifetime exceeds + ``CHAT_RUN_MAX_AGE_SEC`` AND the Redis Stream's last write is older than + ``CHAT_RUN_STALE_QUIET_SEC``. Long tasks still steadily producing + tool_call/content are no longer falsely killed (historical bug: purely + age-based reaping choked even active runs mid-tool-call once the 30-minute + mark hit). + - **Absolute cap**: runs older than ``CHAT_RUN_HARD_MAX_AGE_SEC`` are reaped + even when actively emitting, except for automation conversations and + registered bounded human interactions (which keep their own deadline). + + Reaping also cancels the in-process worker task, keeping the DB terminal + state aligned with the actual task (historical bug: only flipping the DB + without killing the task let the worker silently run another 1.5h and + overwrite the terminal state back to completed). plan_execute is the + exception — cross-task cancel risks the anyio cancel-scope deadlock (see the + ``cancel_run`` docstring); it stops itself after ``is_run_cancelled`` + polling observes the terminal state. + + Complements periodic ``recover_orphan_runs`` and the per-run + inactivity watchdog — also sweeps up historical zombie runs left by + older code paths. + """ + from core.db.models import ChatSession + from sqlalchemy import func + + now = clock._utcnow() + cutoff = now - _datetime.timedelta(seconds=settings._STALE_RUN_MAX_AGE_SEC) + hard_cutoff = now - _datetime.timedelta(seconds=settings._HARD_MAX_AGE_SEC) + with engine.SessionLocal() as db: + candidates = ( + db.query( + models.ChatRun.run_id, + models.ChatRun.chat_id, + models.ChatRun.request_payload, + models.ChatRun.lease_expires_at, + ChatSession.extra_data.label("session_metadata"), + func.coalesce(models.ChatRun.started_at, models.ChatRun.created_at).label( + "began_at" + ), + ) + .outerjoin(ChatSession, ChatSession.chat_id == models.ChatRun.chat_id) + .filter(models.ChatRun.status == "running") + .filter(func.coalesce(models.ChatRun.started_at, models.ChatRun.created_at) < cutoff) + .all() + ) + if not candidates: + return 0 + + now_ms = int(now.timestamp() * 1000) + reaped = 0 + for row in candidates: + rid, cid = row.run_id, row.chat_id + lease_expires_at = row.lease_expires_at + if lease_expires_at is not None and lease_expires_at.tzinfo is None: + lease_expires_at = lease_expires_at.replace(tzinfo=_datetime.timezone.utc) + began_at = row.began_at + if began_at is not None and began_at.tzinfo is None: # SQLite stores naive UTC + began_at = began_at.replace(tzinfo=_datetime.timezone.utc) + session_metadata = row.session_metadata or {} + automation_run = bool(session_metadata.get("automation_run")) + hard_expired = not automation_run and began_at is not None and began_at < hard_cutoff + if not hard_expired and lease_expires_at is not None and lease_expires_at > now: + logger.info( + "chat_run_stale_skip_live_lease", + run_id=rid, + lease_expires_at=lease_expires_at.isoformat(), + ) + continue + payload = row.request_payload if isinstance(row.request_payload, dict) else {} + + # 自主循环豁免统一年龄硬顶：loop 的使命就是「跑到任务完成为止」（去预算化）， + # 且它有自己的停滞护栏/熔断/硬后备轮数。进程内 task 还活着的 loop 绝不按 + # 年龄杀（历史竞态：6h 硬顶 == loop 旧默认预算，跑满预算的健康 loop 在优雅 + # 收尾前被硬顶抢杀，报错还误导为「无响应」）。孤儿 loop（进程重启遗留） + # 走下面的静默判据正常清理。 + if hard_expired and payload.get("kind") == "autonomous_loop": + task = tasks.process_state().active_runs.get(rid) + if task is not None and not task.done(): + logger.info("chat_run_stale_skip_loop_inprocess", run_id=rid) + continue + hard_expired = False # 孤儿 loop：不按年龄硬杀，交给静默判据 + + # A legitimate user decision may begin near the run's absolute-age + # ceiling. Its own bounded wait deadline is the authority; do not cut a + # newly opened question/confirmation down to the remainder of the run + # hard cap. Once the registry resolves or times out, this exemption + # disappears automatically on the next sweep. + if hard_expired and human_interaction.has_pending(cid): + logger.info("chat_run_stale_skip_human_wait", run_id=rid, chat_id=cid) + continue + + # 工作流模式的批量作业：run 卡在 run_job(wait=True) 里等作业，期间主链路一个 + # 流事件都不产生（逐项结果按设计不进对话），"流静默"判据必然误判。进程内 task + # 豁免只在本进程有效——多 worker / 多副本部署里，另一个进程看不到这个 task， + # 照样会把健康的长作业杀掉。所以这里用**跨进程可见的证据**：DB 里这个会话是否 + # 还有正在推进的作业（每次子调用都会 add_usage → jobs.updated_at 前进）。 + if _chat_has_live_job(cid, now): + logger.info("chat_run_stale_skip_live_job", run_id=rid, chat_id=cid) + continue + + if not hard_expired: + # A run whose asyncio task is alive in THIS process is never a + # zombie — the in-process inactivity watchdog owns hang detection + # for it. The stream-quiet check below is only for orphans whose + # worker vanished (process restart/crash). Without this guard, a + # healthy long run streaming huge tool-call args (which map to no + # stream events) past 30min would be reaped as "quiet". + task = tasks.process_state().active_runs.get(rid) + if task is not None and not task.done(): + logger.info("chat_run_stale_skip_inprocess", run_id=rid) + continue + last_ms = await _stream_last_write_ms(rid) + if last_ms is not None and (now_ms - last_ms) < settings._STALE_QUIET_SEC * 1000: + # The stream is still producing: this is a long task, not a zombie — skip this round + logger.info( + "chat_run_stale_skip_active", + run_id=rid, + quiet_sec=round((now_ms - last_ms) / 1000, 1), + ) + continue + reason = "run stalled: no stream activity (stale watchdog)" + else: + reason = "run exceeded hard max age (stale watchdog)" + + if not ownership._request_run_stop( + rid, + status="failed", + error_message=reason, + require_expired_lease=not hard_expired, + ): + continue # race: the worker just beat us to writing the terminal state + + await writer._write_terminal_to_stream( + rid, + chat_id=cid or "", + error_text="任务长时间无响应，已被系统中止，请重新发起", + ) + + payload = row.request_payload if isinstance(row.request_payload, dict) else {} + task = tasks.process_state().active_runs.get(rid) + if ( + task is not None + and not task.done() + and payload.get("kind") not in settings._COOPERATIVE_KINDS + ): + task.cancel() + try: + await asyncio.wait_for(task, timeout=2.0) + except (asyncio.TimeoutError, asyncio.CancelledError): + pass + except Exception as exc: + logger.warning("stale_reaper_await_failed", run_id=rid, error=str(exc)) + if task.done(): + ownership._acknowledge_terminal_writer(rid) + reaped += 1 + + if reaped: + logger.info("chat_run_stale_reaped", count=reaped) + return reaped diff --git a/src/backend/core/sandbox/commands/__init__.py b/src/backend/core/sandbox/commands/__init__.py new file mode 100644 index 00000000..ebd4bf0b --- /dev/null +++ b/src/backend/core/sandbox/commands/__init__.py @@ -0,0 +1 @@ +"""Sandbox commands, owned lifecycle and bounded output storage.""" diff --git a/src/backend/core/sandbox/commands/completion_hook.py b/src/backend/core/sandbox/commands/completion_hook.py new file mode 100644 index 00000000..86fe23a0 --- /dev/null +++ b/src/backend/core/sandbox/commands/completion_hook.py @@ -0,0 +1,44 @@ +"""Expose terminal command facts at safe reasoning and finish boundaries.""" +import asyncio +from core.harness.hooks import Decision, HookSpec, HookStage +from core.sandbox.commands import delivery, store + + +async def inject_completion(invocation): + if invocation.operation_name != "agent_reasoning": + return Decision() + identity = invocation.data.get("identity", {}) + chat_id, user_id = identity.get("chat_id"), identity.get("user_id") + if not (invocation.run_id and chat_id and user_id): + return Decision() + if not await asyncio.to_thread(store.has_pending, chat_id, user_id): + return Decision() + rows = await asyncio.to_thread(delivery.reserve_boundary, chat_id, user_id, invocation.run_id) + if not rows: + return Decision() + item = dict(content=delivery.notice_text(rows), kind="reminder", + origin="harness:command_completion", trust="system", priority=950, + token_budget=4000, truncation_policy="never") + return Decision.modify({"context_items": [*invocation.data.get("context_items", ()), item]}) + + +async def pending_completion(invocation): + identity = invocation.data.get("identity", {}) + chat_id, user_id = identity.get("chat_id"), identity.get("user_id") + if not (invocation.run_id and chat_id and user_id): + return Decision() + rows = await asyncio.to_thread(store.pending_notices, chat_id) + if any(row["notice_status"] == "pending" and row["user_id"] == user_id for row in rows): + return Decision.modify({"continue_run": True}) + return Decision() + + +def command_completion_hooks(): + return ( + HookSpec("command_completion_context", HookStage.TRANSFORM_CONTEXT, + inject_completion, priority=90, timeout_ms=30000, + mutable_fields=frozenset({"context_items"})), + HookSpec("command_completion_finish", HookStage.BEFORE_FINISH, + pending_completion, priority=90, timeout_ms=30000, + mutable_fields=frozenset({"continue_run"})), + ) diff --git a/src/backend/core/services/commands/context.py b/src/backend/core/sandbox/commands/context.py similarity index 85% rename from src/backend/core/services/commands/context.py rename to src/backend/core/sandbox/commands/context.py index c323401e..3a412598 100644 --- a/src/backend/core/services/commands/context.py +++ b/src/backend/core/sandbox/commands/context.py @@ -1,15 +1,13 @@ """Reconstruct active command events at the persisted-history boundary.""" from core.db.models import ChatRun from core.db.models.command import CommandTask -from core.services.commands.delivery import notice_text +from core.sandbox.commands.delivery import notice_text def completion_context(row): """One stable sideband record per completed command, never a chat row.""" - from core.llm.context_ir import ( - KIND_REMINDER, POLICY_NEVER, SESSION_CONTEXT_META_KEY, - make_text_context_item, session_context_metadata, - ) + from core.llm.context.ir_constants import KIND_REMINDER, POLICY_NEVER, SESSION_CONTEXT_META_KEY + from core.llm.context.items import make_text_context_item, session_context_metadata event_id = "command_event_" + row["task_id"] prompt = notice_text([row]) item = make_text_context_item( @@ -34,7 +32,7 @@ def append_active_completions(db, chat_id, history): ChatRun.writer_slot == "main") .order_by(CommandTask.task_id).all()) existing = {message.get("message_id") for message in history} - from core.services.commands.store import view + from core.sandbox.commands.store import view events = [completion_context(view(row)) for row in rows if "command_event_" + row.task_id not in existing] return [*history, *events] diff --git a/src/backend/core/services/commands/delivery.py b/src/backend/core/sandbox/commands/delivery.py similarity index 92% rename from src/backend/core/services/commands/delivery.py rename to src/backend/core/sandbox/commands/delivery.py index 1acdf1ca..bb969009 100644 --- a/src/backend/core/services/commands/delivery.py +++ b/src/backend/core/sandbox/commands/delivery.py @@ -4,8 +4,8 @@ from sqlalchemy import update from core.db.models import ChatRun, ChatSession from core.db.models.command import CommandTask -from core.services.commands import store -from core.services.chat_sequencer import ChatSequencer +from core.sandbox.commands import store +from core.chat.sequencer import ChatSequencer def notice_text(rows): facts = ["[Background command completion]", @@ -23,7 +23,7 @@ def acknowledge(task_ids, run_id): def reserve_boundary(chat_id, user_id, run_id): """Persist facts and their receipt together before injecting live context.""" - from core.services.commands.context import completion_context + from core.sandbox.commands.context import completion_context with store.SessionLocal() as db: db.execute(update(ChatSession).where(ChatSession.chat_id == chat_id) .values(updated_at=ChatSession.updated_at)) @@ -63,7 +63,8 @@ def attach(db, run): return attach def prepare(rows, owner): - from core.llm.context_ir import KIND_REMINDER, POLICY_NEVER, SESSION_CONTEXT_META_KEY, make_text_context_item, session_context_metadata + from core.llm.context.ir_constants import KIND_REMINDER, POLICY_NEVER, SESSION_CONTEXT_META_KEY + from core.llm.context.items import make_text_context_item, session_context_metadata first = rows[0] with store.SessionLocal() as db: chat = db.get(ChatSession, first["chat_id"]) @@ -100,8 +101,8 @@ def claim_retry(task_id, owner): return row.notice_run_id def load_launch(run_id): - from core.services.chat_service import ChatService - from core.services.session_operations.session_context import _load_session_messages + from core.chat.service import ChatService + from core.chat.operations.session_context import load_session_messages as _load_session_messages with store.SessionLocal() as db: rows = db.query(CommandTask).filter_by(notice_run_id=run_id, notice_status="reserved").all() if not rows: diff --git a/src/backend/orchestration/commands/events.py b/src/backend/core/sandbox/commands/events.py similarity index 93% rename from src/backend/orchestration/commands/events.py rename to src/backend/core/sandbox/commands/events.py index cd9e23b2..6f4089a2 100644 --- a/src/backend/orchestration/commands/events.py +++ b/src/backend/core/sandbox/commands/events.py @@ -1,9 +1,11 @@ """Bounded command wake events, separate from per-run output projections.""" + import asyncio import hashlib import threading import time from collections import deque + from core.infra.redis import get_redis, redis_configured TTL_SECONDS = 3600 @@ -82,8 +84,11 @@ async def wait(self, user_id, chat_id, cursor, timeout=15): with self.lock: log = self._log(key) batch = [item for item in log if item[0] > cursor] - return (batch[-1][0] if batch else cursor, - [run for _, run in batch], len(log) == MAX_EVENTS) + return ( + batch[-1][0] if batch else cursor, + [run for _, run in batch], + len(log) == MAX_EVENTS, + ) def _wake(future): @@ -105,8 +110,9 @@ async def publish(self, user_id, chat_id, run_id): async def wait(self, user_id, chat_id, cursor, timeout=15): key = topic(user_id, chat_id) - rows = await get_redis(blocking=True).xread({key: cursor}, - count=MAX_EVENTS, block=max(1, int(timeout * 1000))) + rows = await get_redis(blocking=True).xread( + {key: cursor}, count=MAX_EVENTS, block=max(1, int(timeout * 1000)) + ) if not rows: return cursor, [], False batch = rows[0][1] @@ -123,6 +129,7 @@ def get_command_events(): if redis_configured(): return _redis from core.infra.worker_count import resolve + if resolve() != 1: raise RuntimeError("In-memory command events require a single worker") return _memory diff --git a/src/backend/orchestration/commands/notifications.py b/src/backend/core/sandbox/commands/notifications.py similarity index 77% rename from src/backend/orchestration/commands/notifications.py rename to src/backend/core/sandbox/commands/notifications.py index b15d7c22..1f70c7b9 100644 --- a/src/backend/orchestration/commands/notifications.py +++ b/src/backend/core/sandbox/commands/notifications.py @@ -1,27 +1,39 @@ """Deliver command completion without blocking the event loop on database I/O.""" + import asyncio import logging import uuid -from core.services.commands import delivery, store -from core.services.chat_sequencer import ChatBusyError + +import core.chat.runtime.admission as admission +from core.chat.sequencer import ChatBusyError +from core.sandbox.commands import delivery, store logger = logging.getLogger(__name__) + async def launch(run_id): - from orchestration import chat_run_executor prepared = await asyncio.to_thread(delivery.load_launch, run_id) if prepared is None: return rows, run, history, args = prepared if args is not None: prompt = delivery.notice_text(rows) - await chat_run_executor.start_run(accepted_run=run, chat_id=run.chat_id, - user_id=run.user_id, session_messages=history, effective_user_message=prompt, - raw_user_message=prompt, context=args["context"], model_name=args.get("model_name")) - from orchestration.commands.events import get_command_events + await admission.start_run( + accepted_run=run, + chat_id=run.chat_id, + user_id=run.user_id, + session_messages=history, + effective_user_message=prompt, + raw_user_message=prompt, + context=args["context"], + model_name=args.get("model_name"), + ) + from core.sandbox.commands.events import get_command_events + await get_command_events().publish(run.user_id, run.chat_id, run_id) await asyncio.to_thread(delivery.acknowledge, [r["task_id"] for r in rows], run_id) + async def dispatch(): owner = uuid.uuid4().hex after = None @@ -32,6 +44,7 @@ async def dispatch(): await dispatch_batch(rows, owner) after = rows[-1]["task_id"] + async def dispatch_batch(batch, owner): groups = {} for row in batch: diff --git a/src/backend/core/sandbox/commands/runtime.py b/src/backend/core/sandbox/commands/runtime.py new file mode 100644 index 00000000..c9e1c2a6 --- /dev/null +++ b/src/backend/core/sandbox/commands/runtime.py @@ -0,0 +1,449 @@ +"""One output collector per command; no model-driven polling.""" + +import asyncio +import logging +import time +import uuid +from functools import wraps + +from core.sandbox.protocol import ProcessRequest +from core.sandbox.providers.opensandbox.recovery import ( + execution_reference, + recover_execution, + release_execution, +) +from core.sandbox.commands import store + +logger = logging.getLogger(__name__) +LAUNCH_RENEW_SECONDS = 10 + + +def service_loop(method): + """Keep command collectors on their service loop across agent worker threads.""" + + @wraps(method) + async def invoke(self, *args, **kwargs): + current = asyncio.get_running_loop() + if self.loop is None: + self.loop = current + if self.loop is current: + return await method(self, *args, **kwargs) + if not self.loop.is_running(): + raise RuntimeError("Command service loop is not running") + future = asyncio.run_coroutine_threadsafe(method(self, *args, **kwargs), self.loop) + return await asyncio.wrap_future(future) + + return invoke + + +class CommandRuntime: + def __init__(self): + self.loop = None + self.owner = uuid.uuid4().hex + self.tasks = {} + self.providers = {} + self._locks = {} + self._heartbeat = None + + def _ensure_heartbeat(self): + if self._heartbeat is None or self._heartbeat.done(): + self._heartbeat = asyncio.create_task(self._renew_handles()) + + async def _renew_handles(self): + while self.providers: + try: + await asyncio.wait_for( + asyncio.to_thread(store.renew_owned, list(self.providers), self.owner), + timeout=store.LEASE_SECONDS / 3, + ) + except Exception: + logger.exception("Command handle renewal failed") + await asyncio.sleep(store.LEASE_SECONDS / 3) + + def bind_loop(self): + """Initialize every worker before a temporary agent thread can call it.""" + current = asyncio.get_running_loop() + if self.loop is not None and self.loop is not current: + raise RuntimeError("Command runtime already belongs to another event loop") + self.loop = current + + @service_loop + async def start( + self, provider, req, *, chat_id, source_run_id, background=False, yield_time_ms=60000 + ): + row = await asyncio.to_thread(store.create, req, chat_id, source_run_id, self.owner) + tid = row["task_id"] + result = None + try: + result = await self._launch( + tid, provider, req, yield_time_ms=250 if background else yield_time_ms + ) + return await self._attach(tid, provider, req, result) + except BaseException as exc: + if result and result.get("status") == "running" and tid not in self.providers: + # Preserve control even if persistence or the first stop RPC fails. + self.providers[tid] = provider, req, result["session_id"] + self._ensure_heartbeat() + await self._interrupt(tid) + self.providers.pop(tid, None) + self._locks.pop(tid, None) + try: + await asyncio.to_thread( + store.finish, tid, "unknown", "Launch state unknown; do not rerun: " + str(exc) + ) + except LookupError: + pass # A hard-deleted chat already removed the task. + raise + + async def _launch(self, tid, provider, req, *, yield_time_ms): + async def renew(): + while True: + await asyncio.sleep(LAUNCH_RENEW_SECONDS) + await asyncio.to_thread(store.renew_launch, tid, self.owner) + + launch = asyncio.create_task(provider.start_process(req, yield_time_ms=yield_time_ms)) + lease = asyncio.create_task(renew()) + try: + done, _ = await asyncio.wait((launch, lease), return_when=asyncio.FIRST_COMPLETED) + if launch in done: + return await launch + await lease # Lease loss cancels the pending launch; never restart it. + raise RuntimeError("Command launch lease ended unexpectedly") + finally: + for task in (launch, lease): + if not task.done(): + task.cancel() + await asyncio.gather(launch, lease, return_exceptions=True) + + async def _attach(self, tid, provider, req, result): + reference = None + if result.get("status") == "running": + reference = await execution_reference(provider, result["session_id"], req) + if reference is not None: + reference["cursor"] = result["execution_cursor"] + row = await asyncio.to_thread(store.attach, tid, self.owner, result, reference) + payload = await asyncio.to_thread( + store.read, tid, req.user_id, req.session_id, consume_notice=False + ) + if row["status"] == "running": + self.providers[tid] = provider, req, result["session_id"] + self._ensure_heartbeat() + self.tasks[tid] = asyncio.create_task(self._monitor(tid, row)) + payload["next"] = ( + "Command is managed in the background. Completion will notify this chat. Do independent work; do not poll or restart it. If no independent work remains, end this response once; completion will resume this chat." + ) + return payload + + @service_loop + async def adopt(self, provider, process_id, *, user_id, session_id, chat_id, source_run_id): + if process_id.startswith("cmd_"): + await asyncio.to_thread(store.owned, process_id, user_id, session_id) + return await asyncio.to_thread( + store.read, process_id, user_id, session_id, consume_notice=False + ) + req = ProcessRequest( + script_content="[adopted command]", + script_name="_managed.sh", + language="bash", + session_id=session_id, + user_id=user_id, + ) + row = await asyncio.to_thread( + store.create, req, chat_id, source_run_id, self.owner, process_id + ) + if not row["created"]: + return await asyncio.to_thread( + store.read, row["task_id"], user_id, session_id, consume_notice=False + ) + try: + result = await provider.write_stdin( + process_id, + sandbox_session_id=session_id, + user_id=user_id, + chars="", + yield_time_ms=0, + ) + return await self._attach(row["task_id"], provider, req, result) + except BaseException: + await asyncio.to_thread( + store.finish, row["task_id"], "unknown", "Process adoption failed" + ) + raise + + async def _monitor(self, tid, row): + provider, req, pid = self.providers[tid] + last_touch = 0 + failures = 0 + try: + while True: + try: + if not await asyncio.to_thread(store.chat_available, tid): + result = await self._interrupt(tid) + try: + await asyncio.to_thread( + store.control_result, + tid, + self.owner, + "cancelled", + "Conversation removed", + True, + result, + ) + except LookupError: + pass # Physical deletion removed the row; the owned process was stopped. + break + row = await asyncio.to_thread(store.snapshot, tid) + if row["status"] not in store.LIVE or row["owner"] != self.owner: + if row["execution_ref"]: + await release_execution(provider, pid, req) + else: + await self._interrupt(tid) + break + if row["cancel_requested"]: + result = await self._interrupt(tid) + await asyncio.to_thread( + store.control_result, tid, self.owner, "cancelled", "", True, result + ) + break + if row["deadline_at"] and time.time() >= row["deadline_at"]: + result = await self._interrupt(tid) + await asyncio.to_thread( + store.control_result, + tid, + self.owner, + "failed", + "Command execution deadline exceeded", + False, + result, + ) + break + async with self._locks.setdefault(tid, asyncio.Lock()): + result = await provider.write_stdin( + pid, + sandbox_session_id=req.session_id, + user_id=req.user_id, + chars="", + yield_time_ms=250, + ) + row = await asyncio.to_thread(store.collect, tid, self.owner, result) + if row["status"] not in store.LIVE: + break + if time.monotonic() - last_touch >= 10: + await provider.touch_session(req.session_id) + last_touch = time.monotonic() + failures = 0 + await asyncio.sleep(0.5) + except Exception: + failures += 1 + logger.warning("Command observation failed task=%s attempt=%s", + tid, failures, exc_info=True) + if failures >= 3: + if row["execution_ref"]: + # Durable recovery can retry without rerunning the command. + await release_execution(provider, pid, req) + await asyncio.to_thread(store.release_lease, tid, self.owner) + break + # A nonrecoverable local process must be stopped, never + # reported terminal merely because observation failed. + result = await self._interrupt(tid) + await asyncio.to_thread( + store.control_result, tid, self.owner, "failed", + "Command observation failed", False, result, + ) + break + await asyncio.sleep(2 ** (failures - 1)) + except asyncio.CancelledError: + if row["execution_ref"]: + await release_execution(provider, pid, req) + self.providers.pop(tid, None) + self._locks.pop(tid, None) + await asyncio.to_thread(store.release_lease, tid, self.owner) + else: + result = await self._interrupt(tid) + self.providers.pop(tid, None) + self._locks.pop(tid, None) + await asyncio.to_thread( + store.control_result, + tid, + self.owner, + "cancelled", + "Command owner stopped", + True, + result, + ) + raise + else: + self.providers.pop(tid, None) + self._locks.pop(tid, None) + finally: + self.tasks.pop(tid, None) + + async def _interrupt(self, tid): + provider, req, pid = self.providers[tid] + async with self._locks.setdefault(tid, asyncio.Lock()): + result = await provider.write_stdin( + pid, + sandbox_session_id=req.session_id, + user_id=req.user_id, + chars="\x03", + yield_time_ms=1000, + ) + while result.get("status") == "running": + try: + await asyncio.to_thread(store.collect, tid, self.owner, result) + except Exception: + logger.warning("Command stop bookkeeping deferred task=%s", tid) + next_result = await provider.write_stdin( + pid, + sandbox_session_id=req.session_id, + user_id=req.user_id, + chars="", + yield_time_ms=1000, + ) + result = next_result + if result.get("status") != "exited": + raise RuntimeError("Process stop not confirmed") + return result + + async def cancel(self, tid, user_id, session_id): + await asyncio.to_thread(store.owned, tid, user_id, session_id) + row = await asyncio.to_thread(store.snapshot, tid) + if row["status"] in store.TERMINAL: + return + await asyncio.to_thread(store.request_cancel, tid) + await self.wait(tid, user_id, session_id, 3000) + + async def wait(self, tid, user_id, session_id, wait_ms=0): + await asyncio.to_thread(store.owned, tid, user_id, session_id) + deadline = time.monotonic() + max(0, min(wait_ms, 300000)) / 1000 + while time.monotonic() < deadline: + if (await asyncio.to_thread(store.snapshot, tid))["status"] in store.TERMINAL: + break + await asyncio.sleep(0.05) + + @service_loop + async def recover(self, provider): + # Failed local stops remain owned and are retried by this worker's + # existing recovery supervisor before any no-reference classification. + for tid in list(self.providers): + if tid not in self.tasks: + self.tasks[tid] = asyncio.create_task(self._stop_retained(tid)) + for row in await asyncio.to_thread(store.pending_recovery): + tid = row["task_id"] + if tid in self.tasks and not self.tasks[tid].done(): + continue + if not await asyncio.to_thread(store.claim, tid, self.owner): + continue + if not row["execution_ref"]: + await asyncio.to_thread( + store.finish, + tid, + "unknown", + "No recoverable execution reference; original command will not be rerun", + owner=self.owner, + ) + continue + req = ProcessRequest( + script_content="", + script_name="_managed.sh", + language="bash", + session_id=row["sandbox_session_id"], + user_id=row["user_id"], + ) + result = None + try: + result = await recover_execution(provider, row["execution_ref"], req) + ref = {**row["execution_ref"], "cursor": result["execution_cursor"]} + recovered = await asyncio.to_thread( + store.save_recovery, tid, self.owner, result, ref + ) + if recovered["status"] == "running": + self.providers[tid] = provider, req, result["session_id"] + self._ensure_heartbeat() + self.tasks[tid] = asyncio.create_task(self._monitor(tid, recovered)) + except BaseException as exc: + if result and result.get("status") == "running": + await release_execution(provider, result["session_id"], req) + await asyncio.to_thread(store.release_lease, tid, self.owner) + if isinstance(exc, asyncio.CancelledError): + raise + logger.warning("Command recovery deferred task=%s", tid, exc_info=True) + + async def _stop_retained(self, tid): + try: + try: + row = await asyncio.to_thread(store.snapshot, tid) + except LookupError: + row = None + if row and row["execution_ref"]: + provider, req, pid = self.providers[tid] + await release_execution(provider, pid, req) + self.providers.pop(tid, None) + self._locks.pop(tid, None) + await asyncio.to_thread(store.release_lease, tid, self.owner) + return + result = await self._interrupt(tid) + self.providers.pop(tid, None) + self._locks.pop(tid, None) + try: + await asyncio.to_thread( + store.control_result, tid, self.owner, "failed", + "Command observation failed", False, result, + ) + except LookupError: + pass + except Exception: + logger.exception("Command stop requires another recovery pass task=%s", tid) + finally: + self.tasks.pop(tid, None) + + @service_loop + async def shutdown(self): + tasks = list(self.tasks.values()) + for task in tasks: + if not task.done(): + task.cancel() + results = await asyncio.gather(*tasks, return_exceptions=True) + # A task cancelled before its first instruction cannot run its finally. + for tid, (provider, req, pid) in list(self.providers.items()): + try: + await self._shutdown_handle(tid, provider, req, pid) + except Exception as exc: + results.append(exc) + for result in results: + if isinstance(result, BaseException) and not isinstance(result, asyncio.CancelledError): + raise result + self.tasks.clear() + if self._heartbeat is not None: + self._heartbeat.cancel() + await asyncio.gather(self._heartbeat, return_exceptions=True) + self._heartbeat = None + self.loop = None + + async def _shutdown_handle(self, tid, provider, req, pid): + try: + row = await asyncio.to_thread(store.snapshot, tid) + except LookupError: + await self._interrupt(tid) + self.providers.pop(tid, None) + self._locks.pop(tid, None) + return + if row["execution_ref"]: + await release_execution(provider, pid, req) + await asyncio.to_thread(store.release_lease, tid, self.owner) + else: + result = await self._interrupt(tid) + await asyncio.to_thread( + store.control_result, + tid, + self.owner, + "cancelled", + "Command owner stopped", + True, + result, + ) + self.providers.pop(tid, None) + self._locks.pop(tid, None) + + +runtime = CommandRuntime() diff --git a/src/backend/core/services/commands/store.py b/src/backend/core/sandbox/commands/store.py similarity index 95% rename from src/backend/core/services/commands/store.py rename to src/backend/core/sandbox/commands/store.py index 869b890e..2ddf87a5 100644 --- a/src/backend/core/services/commands/store.py +++ b/src/backend/core/sandbox/commands/store.py @@ -18,7 +18,7 @@ def view(row): return {c.name: getattr(row, c.name) for c in row.__table__.columns} def can_edit(db, chat_id, user_id): - from core.services.chat_service import ChatService + from core.chat.service import ChatService pair = ChatService(db).get_session_with_access(chat_id, user_id) return pair is not None and pair[1] in ("admin", "edit") @@ -248,3 +248,14 @@ def renew_launch(task_id, owner): if row.owner != owner or row.status != "starting": raise RuntimeError("Command launch ownership lost") row.lease_until = time.time() + LAUNCH_LEASE_SECONDS + + +def renew_owned(task_ids, owner): + """Keep live local handles fenced even while output/stop RPCs are blocked.""" + now = time.time() + with SessionLocal() as db: + db.execute(update(CommandTask).where( + CommandTask.task_id.in_(task_ids), CommandTask.owner == owner, + CommandTask.status.in_(LIVE), CommandTask.lease_until > now, + ).values(lease_until=now + LEASE_SECONDS)) + db.commit() diff --git a/src/backend/orchestration/commands/subscription.py b/src/backend/core/sandbox/commands/subscription.py similarity index 69% rename from src/backend/orchestration/commands/subscription.py rename to src/backend/core/sandbox/commands/subscription.py index 04ba3f6d..3311f844 100644 --- a/src/backend/orchestration/commands/subscription.py +++ b/src/backend/core/sandbox/commands/subscription.py @@ -1,12 +1,14 @@ """Authenticated snapshot-and-tail subscription for ordinary-chat wake events.""" + import asyncio import json -from fastapi import HTTPException + from core.auth.backend import get_current_user, security -from core.services.commands import store from core.db.models import ChatRun from core.db.models.command import CommandTask -from orchestration.commands.events import MAX_EVENTS, get_command_events +from core.sandbox.commands.events import MAX_EVENTS, get_command_events +from core.sandbox.commands import store +from fastapi import HTTPException HEARTBEAT_SECONDS = 15 @@ -31,18 +33,31 @@ async def authorize(request, chat_id, expected_user=None): def snapshot(chat_id, user_id): with store.SessionLocal() as db: - rows = (db.query(ChatRun).join(CommandTask, CommandTask.notice_run_id == ChatRun.run_id) - .filter(CommandTask.chat_id == chat_id, CommandTask.user_id == user_id, - CommandTask.delivery_mode == "followup", - CommandTask.notice_status.in_(("reserved", "delivered")), - ChatRun.chat_id == chat_id, ChatRun.user_id == user_id) - .distinct().order_by(ChatRun.created_at.desc(), ChatRun.run_id.desc()) - .limit(MAX_EVENTS).all()) + rows = ( + db.query(ChatRun) + .join(CommandTask, CommandTask.notice_run_id == ChatRun.run_id) + .filter( + CommandTask.chat_id == chat_id, + CommandTask.user_id == user_id, + CommandTask.delivery_mode == "followup", + CommandTask.notice_status.in_(("reserved", "delivered")), + ChatRun.chat_id == chat_id, + ChatRun.user_id == user_id, + ) + .distinct() + .order_by(ChatRun.created_at.desc(), ChatRun.run_id.desc()) + .limit(MAX_EVENTS) + .all() + ) return [run.run_id for run in rows if run.recovery_snapshot] def frame(run_ids): - return "data: " + json.dumps({"type": "command_runs", "run_ids": list(dict.fromkeys(run_ids))}) + "\n\n" + return ( + "data: " + + json.dumps({"type": "command_runs", "run_ids": list(dict.fromkeys(run_ids))}) + + "\n\n" + ) async def subscribe(request, chat_id, user_id): @@ -55,7 +70,8 @@ async def subscribe(request, chat_id, user_id): yield frame(runs) while True: cursor, runs, resync = await events.wait( - user_id, chat_id, cursor, timeout=HEARTBEAT_SECONDS) + user_id, chat_id, cursor, timeout=HEARTBEAT_SECONDS + ) await authorize(request, chat_id, user_id) if resync: runs = await asyncio.to_thread(snapshot, chat_id, user_id) diff --git a/src/backend/core/sandbox/credential_dispatch.py b/src/backend/core/sandbox/credential_dispatch.py new file mode 100644 index 00000000..fcb13137 --- /dev/null +++ b/src/backend/core/sandbox/credential_dispatch.py @@ -0,0 +1,30 @@ +"""Provider-neutral credential dispatch for account-owned cloud commands.""" +from core.harness.registry import sandbox_credentials + +def credential_environment(): + environment = {} + for provider in sandbox_credentials(): + if provider.environment: + environment.update(provider.environment()) + return environment + + +def credential_mounts(user_id): + return [mount for provider in sandbox_credentials() if provider.mounts + for mount in provider.mounts(user_id)] + + +async def prepare_credentials(host, sandbox, user_id, script): + if not user_id: + return + for provider in sorted(sandbox_credentials(), key=lambda item: item.prepare_order): + if provider.before_command: + await provider.before_command(host, sandbox, user_id, script) + + +async def persist_credentials(host, sandbox, user_id, script): + if not user_id: + return + for provider in sandbox_credentials(): + if provider.after_command: + await provider.after_command(host, sandbox, user_id, script) diff --git a/src/backend/core/sandbox/factory.py b/src/backend/core/sandbox/factory.py index 406d0b93..7a57ccfb 100644 --- a/src/backend/core/sandbox/factory.py +++ b/src/backend/core/sandbox/factory.py @@ -8,6 +8,7 @@ from __future__ import annotations import logging +from dataclasses import dataclass from core.config.settings import settings @@ -15,34 +16,31 @@ logger = logging.getLogger(__name__) -_provider: SandboxProvider | None = None - def _ordinary_provider() -> SandboxProvider: - global _provider - if _provider is not None: - return _provider + if process_state().provider is not None: + return process_state().provider kind = settings.sandbox.provider if kind == "script_runner": from core.sandbox.providers.script_runner.provider import ScriptRunnerProvider - _provider = ScriptRunnerProvider() + process_state().provider = ScriptRunnerProvider() elif kind == "opensandbox": from core.sandbox.providers.edition import PersistentSandboxProvider - _provider = PersistentSandboxProvider() + process_state().provider = PersistentSandboxProvider() elif kind == "cube": from core.sandbox.providers.cube.provider import CubeSandboxProvider - _provider = CubeSandboxProvider() + process_state().provider = CubeSandboxProvider() else: raise ValueError( f"未知 SANDBOX_PROVIDER: {kind!r}，可选: script_runner / opensandbox / cube" ) - logger.info("[sandbox] activated provider=%s", _provider.name) - return _provider + logger.info("[sandbox] activated provider=%s", process_state().provider.name) + return process_state().provider def get_sandbox_provider() -> SandboxProvider: @@ -53,5 +51,16 @@ def get_sandbox_provider() -> SandboxProvider: def reset_provider_cache() -> None: """For testing use only.""" - global _provider - _provider = None + process_state().provider = None + + +@dataclass +class FactoryState: + provider: SandboxProvider | None = None + + +_STATE = FactoryState() + + +def process_state() -> FactoryState: + return _STATE diff --git a/src/backend/core/sandbox/oslayer/windows_runtime.py b/src/backend/core/sandbox/oslayer/windows_runtime.py index ef2062b2..2a0f8397 100644 --- a/src/backend/core/sandbox/oslayer/windows_runtime.py +++ b/src/backend/core/sandbox/oslayer/windows_runtime.py @@ -27,15 +27,12 @@ import os import threading from collections import Counter +from dataclasses import dataclass, field # Environment variables Windows programs read to find their temp directory. TEMP_ENV_KEYS = ("TEMP", "TMP") ROOTS_FILENAME = "sandbox_labeled_roots.json" -_LOCK = threading.Lock() -_IN_USE: Counter[str] = Counter() -_TOKEN = None - def root_key(path: str) -> str: return os.path.normcase(os.path.normpath(os.path.abspath(path))) @@ -73,7 +70,7 @@ def apply_labels(plan: dict) -> None: roots = {root_key(root["path"]): root for root in plan.get("writable_roots") or []} store = LabeledRoots(plan["state_dir"]) labeled = store.load() - in_use = {key for key, count in _IN_USE.items() if count > 0} + in_use = {key for key, count in process_state().IN_USE.items() if count > 0} stale = labeled - set(roots) - in_use for key in stale: remove_label(key) @@ -99,17 +96,16 @@ def child_environment(plan: dict, env: dict) -> dict: def _sandbox_token(): """The low-integrity token, built once: it takes nothing from the plan.""" - global _TOKEN - if _TOKEN is None: + if process_state().TOKEN is None: from .win32 import ffi from .win32.token import create_low_integrity_token, open_current_process_token base_token = open_current_process_token() try: - _TOKEN = create_low_integrity_token(base_token) + process_state().TOKEN = create_low_integrity_token(base_token) finally: ffi.CloseHandle(base_token) - return _TOKEN + return process_state().TOKEN def spawn_confined( @@ -136,13 +132,13 @@ def spawn_confined( keys = [root_key(root["path"]) for root in plan.get("writable_roots") or []] def release() -> None: - with _LOCK: - _IN_USE.subtract(keys) + with process_state().LOCK: + process_state().IN_USE.subtract(keys) - with _LOCK: + with process_state().LOCK: apply_labels(plan) token = _sandbox_token() - _IN_USE.update(keys) + process_state().IN_USE.update(keys) try: return spawn_with_token( token, @@ -168,3 +164,17 @@ def release() -> None: "root_key", "spawn_confined", ] + + +@dataclass +class WindowsRuntimeState: + LOCK: object = field(default_factory=lambda: threading.Lock()) + IN_USE: Counter[str] = field(default_factory=lambda: Counter()) + TOKEN: object = None + + +_STATE = WindowsRuntimeState() + + +def process_state() -> WindowsRuntimeState: + return _STATE diff --git a/src/backend/core/llm/tool_permissions/account.py b/src/backend/core/sandbox/permissions/account.py similarity index 90% rename from src/backend/core/llm/tool_permissions/account.py rename to src/backend/core/sandbox/permissions/account.py index e653eb32..941fd786 100644 --- a/src/backend/core/llm/tool_permissions/account.py +++ b/src/backend/core/sandbox/permissions/account.py @@ -2,8 +2,8 @@ from core.db.models import ChatSession, UserShadow from core.db.reserved_chats import reserved_chat_owner -from core.services.user_service import UserService -from .preferences import normalize_approval_mode +from core.auth.users import UserService +from core.sandbox.permissions.approval import normalize_approval_mode def read_account_mode(user_id: str, chat_id: str | None = None) -> str | None: diff --git a/src/backend/core/llm/tool_permissions/preferences.py b/src/backend/core/sandbox/permissions/approval.py similarity index 94% rename from src/backend/core/llm/tool_permissions/preferences.py rename to src/backend/core/sandbox/permissions/approval.py index 1cfa8ce9..2737d0e8 100644 --- a/src/backend/core/llm/tool_permissions/preferences.py +++ b/src/backend/core/sandbox/permissions/approval.py @@ -38,7 +38,7 @@ def resolve_approval_mode(explicit: Any, *, user_id: Optional[str]) -> str: return APPROVAL_ASK try: from core.db.engine import SessionLocal - from core.services.user_service import UserService + from core.auth.users import UserService with SessionLocal() as db: stored = UserService(db).get_user_settings(str(user_id)).get("tool_approval_mode") @@ -72,8 +72,8 @@ async def refresh_runtime(runtime): """Resolve one immutable dispatch policy; a failed read is an error, never a prompt.""" if not runtime.account_approval or not runtime.user_id or runtime.default_allow: return runtime - from .account import read_account_mode - from .models import PermissionEnforcementError + from core.sandbox.permissions.account import read_account_mode + from core.sandbox.permissions.dispatch_models import PermissionEnforcementError try: mode = await asyncio.to_thread(read_account_mode, runtime.user_id, runtime.chat_id) diff --git a/src/backend/core/llm/tool_permissions/models.py b/src/backend/core/sandbox/permissions/dispatch_models.py similarity index 98% rename from src/backend/core/llm/tool_permissions/models.py rename to src/backend/core/sandbox/permissions/dispatch_models.py index 1d8d8c9a..70bc82ee 100644 --- a/src/backend/core/llm/tool_permissions/models.py +++ b/src/backend/core/sandbox/permissions/dispatch_models.py @@ -7,12 +7,12 @@ from contextvars import ContextVar from dataclasses import dataclass, field, replace from typing import TYPE_CHECKING, Any, Callable, Mapping, Optional, Sequence -from .preferences import APPROVAL_ASK, APPROVAL_FULL +from core.sandbox.permissions.approval import APPROVAL_ASK, APPROVAL_FULL if TYPE_CHECKING: from core.sandbox.permissions.os_gate import LocalAccessDecision from core.sandbox.oslayer import SandboxLaunch - from core.services.project_scope import ProjectScope + from core.projects.edition import ProjectScope DOMAIN_LOCAL_PATH = "local_path" DOMAIN_LOCAL_COMMAND = "local_command" diff --git a/src/backend/core/sandbox/permissions/os_gate.py b/src/backend/core/sandbox/permissions/os_gate.py index d68e7e51..34065398 100644 --- a/src/backend/core/sandbox/permissions/os_gate.py +++ b/src/backend/core/sandbox/permissions/os_gate.py @@ -109,7 +109,7 @@ def protected_read_paths() -> tuple[str, ...]: resolves the *narrowest* matching entry, and the writable workspace root is deeper than the data root denied here. """ - from core.capabilities.paths import CAPS_ROOT_ENV + from core.config.runtime_env import CAPS_ROOT_ENV candidates = [local_state_dir()] for name in (CAPS_ROOT_ENV, _INSTALL_ROOT_ENV): diff --git a/src/backend/core/llm/tool_permissions/specs.py b/src/backend/core/sandbox/permissions/tool_specs.py similarity index 95% rename from src/backend/core/llm/tool_permissions/specs.py rename to src/backend/core/sandbox/permissions/tool_specs.py index 0ed3f93e..1796292a 100644 --- a/src/backend/core/llm/tool_permissions/specs.py +++ b/src/backend/core/sandbox/permissions/tool_specs.py @@ -4,19 +4,7 @@ import os import logging from typing import Sequence, Any, Mapping, Optional -from .models import ( - PermissionRuntime, - PermissionIntent, - ToolPermissionSpec, - DOMAIN_DENY, - DOMAIN_MYSPACE, - DOMAIN_LOCAL_PATH, - DOMAIN_LOCAL_COMMAND, - READ, - WRITE, - EXECUTE, - _canonical_path, -) +from core.sandbox.permissions.dispatch_models import PermissionRuntime, PermissionIntent, ToolPermissionSpec, DOMAIN_DENY, DOMAIN_MYSPACE, DOMAIN_LOCAL_PATH, DOMAIN_LOCAL_COMMAND, READ, WRITE, EXECUTE, _canonical_path logger = logging.getLogger(__name__) @@ -88,7 +76,7 @@ def _is_myspace_target(logical: str, physical: str, user_id: Optional[str]) -> b """ if logical == "/myspace" or logical.startswith("/myspace/"): return True - from core.llm.tools._paths import is_myspace_physical + from core.sandbox.workspace.resolution import is_myspace_physical return bool(is_myspace_physical(physical, user_id)) @@ -137,7 +125,7 @@ def resolve(args: Mapping[str, Any], runtime: PermissionRuntime): return () intents: list[PermissionIntent] = [] first: Optional[tuple[str, str]] = None - from core.llm.tools._paths import to_physical_path + from core.sandbox.workspace.resolution import to_physical_path for index, (arg, mode) in enumerate(path_specs): logical = str(args.get(arg) or "").strip() @@ -196,7 +184,7 @@ def resolve(args: Mapping[str, Any], _runtime: PermissionRuntime): def builtin_tool_permission(tool_name: str) -> Optional[ToolPermissionSpec]: """Return the rule for a governed native tool, otherwise pass through.""" - from core.llm.tools._tool_confirmation import OP_EDIT, OP_WRITE + from core.sandbox.permissions.vocabulary import OP_EDIT, OP_WRITE governed: dict[str, ToolPermissionSpec] = { "Read": local_path_tool("file_path", READ, tool_name="Read"), diff --git a/src/backend/core/sandbox/permissions/vocabulary.py b/src/backend/core/sandbox/permissions/vocabulary.py new file mode 100644 index 00000000..fd4c534f --- /dev/null +++ b/src/backend/core/sandbox/permissions/vocabulary.py @@ -0,0 +1,26 @@ +"""Stable operation and decision vocabulary for execution approvals.""" + +KIND_MYSPACE = "myspace" +KIND_AUTOMATION = "automation" +KIND_DESIGN_PICK = "design_pick" +KIND_LOCAL_CMD = "local_cmd" +KIND_LOCAL_PATH_PREFIX = "local_path:" +KIND_TOOL_PERMISSION = "tool_permission" +KIND_TOOL_PREFIX = "tool:" +OP_WRITE = "write" +OP_EDIT = "edit" +OP_DELETE = "delete" +OP_LOCAL_EXEC = "local_exec" +OP_LOCAL_READ = "local_read" +OP_LOCAL_WRITE = "local_write" +OP_MOVE = "move" +OP_MKDIR = "mkdir" +OP_CRON_CREATE = "cron_create" +OP_CRON_UPDATE = "cron_update" +OP_CRON_DELETE = "cron_delete" +DECISION_ALLOW = "allow" +DECISION_ALLOW_SESSION = "allow_session" +DECISION_DENY = "deny" +DECISION_CHOICE = "choice" +DECISION_SKIP = "skip" +STATUS_BLOCKED = "blocked_non_interactive" diff --git a/src/backend/core/sandbox/processes/cloud.py b/src/backend/core/sandbox/processes/cloud.py index 10df65a9..a78f79ea 100644 --- a/src/backend/core/sandbox/processes/cloud.py +++ b/src/backend/core/sandbox/processes/cloud.py @@ -19,7 +19,7 @@ def _module_owned(session_id) -> bool: """A module's dedicated sandbox never receives the platform's extra credentials.""" - from core.extensions import sandbox_route + from core.harness.registry import sandbox_route return sandbox_route(str(session_id or "")) is not None @@ -85,7 +85,8 @@ async def close_all(self): async def _spawn(self, req): from core.sandbox.workspace.identity import SANDBOX_RUN_GID, SANDBOX_RUN_UID from core.sandbox.workspace.paths import WORKSPACE - from core.sandbox.providers.account_env import dws_extra_envs, _firecrawl_extra_envs + from core.sandbox.providers.account_env import _firecrawl_extra_envs + from core.sandbox.credential_dispatch import credential_environment provider = self.provider interpreters = {"bash": "bash", "python": "python3 -u", "javascript": "node"} @@ -138,7 +139,7 @@ async def _spawn(self, req): working_directory=WORKSPACE, uid=SANDBOX_RUN_UID, gid=SANDBOX_RUN_GID, - envs=({**_EXECD_BASH_ENVS, "EXECD_ACCESS_TOKEN": ""} if _module_owned(req.session_id) else {**_EXECD_BASH_ENVS, **dws_extra_envs(), **_firecrawl_extra_envs()}), + envs=({**_EXECD_BASH_ENVS, "EXECD_ACCESS_TOKEN": ""} if _module_owned(req.session_id) else {**_EXECD_BASH_ENVS, **credential_environment(), **_firecrawl_extra_envs()}), ) execution = await sess.sandbox.commands.run(command, opts=opts) if not execution.id: @@ -158,7 +159,7 @@ async def _spawn(self, req): envs=( {"EXECD_ACCESS_TOKEN": ""} if _module_owned(req.session_id) - else {**dws_extra_envs(), **_firecrawl_extra_envs()} or None + else {**credential_environment(), **_firecrawl_extra_envs()} or None ), stdin=False, timeout=0, @@ -276,8 +277,11 @@ async def poll(self): raise code = exc.exit_code self.finished = True - if self.req.user_id and "openyida" in self.req.script_content: - await self.provider._persist_yida_state(self.sandbox, self.req.user_id) + from core.sandbox.credential_dispatch import persist_credentials + + await persist_credentials( + self.provider, self.sandbox, self.req.user_id, self.req.script_content + ) return out, err, code async def interrupt(self): diff --git a/src/backend/core/sandbox/processes/completion.py b/src/backend/core/sandbox/processes/completion.py index 71c1f8b1..b7e926e3 100644 --- a/src/backend/core/sandbox/processes/completion.py +++ b/src/backend/core/sandbox/processes/completion.py @@ -36,8 +36,9 @@ async def stop_after_start(): chars="\x03", yield_time_ms=1000, ) - except Exception: - pass + except Exception as fallback_error: + from core.infra.failures import record_best_effort_failure + record_best_effort_failure('core.sandbox.processes.completion:stop_after_start', fallback_error) cleanup = asyncio.create_task(stop_after_start()) await asyncio.shield(cleanup) @@ -88,6 +89,7 @@ async def stop_after_start(): yield_time_ms=1000, ) ) - except Exception: - pass + except Exception as fallback_error: + from core.infra.failures import record_best_effort_failure + record_best_effort_failure('core.sandbox.processes.completion:run_to_completion', fallback_error) raise diff --git a/src/backend/core/sandbox/processes/dispatch.py b/src/backend/core/sandbox/processes/dispatch.py index d8862f1e..06a780e4 100644 --- a/src/backend/core/sandbox/processes/dispatch.py +++ b/src/backend/core/sandbox/processes/dispatch.py @@ -12,7 +12,7 @@ def _on_service_loop(method: Callable[..., Any]) -> Callable[..., Any]: """把碰 SDK / 池 / httpx 的入口钉在 provider 的服务事件循环上执行。 provider 是进程级单例，但调用方不止一个事件循环：子智能体跑在 - ``_subagent_pool`` 线程里的独立循环（``core/llm/subagent_tool.py`` 的 + ``_subagent_pool`` 线程里的独立循环（``core/llm/subagents/tool.py`` 的 ``asyncio.new_event_loop()``），且**平台内置子智能体复用主对话的 session_id**（explorer/worker/reviewer 要看同一份 workspace），于是拿到的是 主循环创建的那个 ``_Session``。 diff --git a/src/backend/core/sandbox/processes/interactive.py b/src/backend/core/sandbox/processes/interactive.py index 1f171f21..ccbd5848 100644 --- a/src/backend/core/sandbox/processes/interactive.py +++ b/src/backend/core/sandbox/processes/interactive.py @@ -7,7 +7,6 @@ import logging import secrets import zipfile -from pathlib import Path from urllib.parse import urlsplit, urlunsplit import httpx @@ -108,7 +107,7 @@ async def launch(provider, installation, module, user_id, chat_id, config): raise ValueError("runtime_package_too_large") archive_name = ".__interactive_" + secrets.token_hex(12) + ".zip" body = {**config, "archive": archive_name, "callable": runtime["callable"]} - from core.plugins.resources.confinement import launch_policy + from core.capabilities.plugins.resources.confinement import launch_policy confinement = launch_policy(provider, user_id, chat_id) start = asyncio.create_task( diff --git a/src/backend/core/sandbox/providers/account_env.py b/src/backend/core/sandbox/providers/account_env.py index 050a110e..d4400d6f 100644 --- a/src/backend/core/sandbox/providers/account_env.py +++ b/src/backend/core/sandbox/providers/account_env.py @@ -7,31 +7,6 @@ logger = logging.getLogger(__name__) -def dws_extra_envs() -> dict[str, str]: - """Return deployment-level dws Custom App environment variables. - - This helper lives in the provider-neutral sandbox module because both the - backend OAuth subprocess and optional sandbox providers need it. Keeping - it here also lets DingTalk configuration load without importing any concrete - sandbox provider. - """ - out: dict[str, str] = {} - try: - from core.services.system_config import SystemConfigService - - svc = SystemConfigService.get_instance() - client_id = (svc.get("dingtalk.client_id") or "").strip() - client_secret = (svc.get("dingtalk.client_secret") or "").strip() - trusted_domains = (svc.get("dingtalk.trusted_domains") or "").strip() - except Exception as exc: # noqa: BLE001 - logger.warning("[dws] read dingtalk config failed: %s", exc) - return out - if client_id and client_secret: - out["DWS_CLIENT_ID"] = client_id - out["DWS_CLIENT_SECRET"] = client_secret - if trusted_domains: - out["DWS_TRUSTED_DOMAINS"] = trusted_domains - return out def _firecrawl_extra_envs() -> dict[str, str]: @@ -51,7 +26,7 @@ def _firecrawl_extra_envs() -> dict[str, str]: """ out: dict[str, str] = {} try: - from core.services.system_config import SystemConfigService + from core.config.store.system_config import SystemConfigService svc = SystemConfigService.get_instance() api_key = (svc.get("firecrawl.api_key") or "").strip() diff --git a/src/backend/core/sandbox/providers/edition.py b/src/backend/core/sandbox/providers/edition.py index e507d98b..5d1f4527 100644 --- a/src/backend/core/sandbox/providers/edition.py +++ b/src/backend/core/sandbox/providers/edition.py @@ -1,6 +1,6 @@ """Persistent sandbox provider for personal accounts.""" from core.sandbox.providers.opensandbox.provider import OpenSandboxProvider -from core.spaces.sandbox import SpaceSandboxMixin +from core.files.spaces.sandbox import SpaceSandboxMixin class PersistentSandboxProvider(SpaceSandboxMixin, OpenSandboxProvider): diff --git a/src/backend/core/sandbox/providers/opensandbox/account_volumes.py b/src/backend/core/sandbox/providers/opensandbox/account_volumes.py deleted file mode 100644 index d00832e1..00000000 --- a/src/backend/core/sandbox/providers/opensandbox/account_volumes.py +++ /dev/null @@ -1,292 +0,0 @@ -"""Account-specific OpenSandbox volumes; shared account paths live in workspace.""" - -from __future__ import annotations - -import logging -import re -from typing import Any, Optional - -from core.config.settings import settings -from core.sandbox.workspace.credentials import ( - _YIDA_WORKSPACE_MOUNT, - dws_cache_dir, - email_cache_dir, - lark_cache_dir, - yida_cache_dir, -) -from core.sandbox.workspace.identity import safe_user_id - -logger = logging.getLogger(__name__) - -# dws (DingTalk CLI) credential persistence volumes: bind per-user host directories to the -# two paths under the sandbox ubuntu user's home that dws reads/writes by default. Measured -# behavior: the dws keychain/cache only honors $HOME, env redirection has no effect, so these -# two absolute paths must be bind-mounted (see the workspace.credentials.dws_cache_dir comment). -_DWS_HOME = "/home/ubuntu" - -_DWS_CREDS_SUBPATHS: tuple[tuple[str, str], ...] = ( - # (host subpath under dws_cache/{uid}, sandbox mount point) - # Matches the backend login layout HOME=dws_cache/{uid}/home: login writes home/.dws and - # home/.local/share/dws-cli; here we bind those two directories into the sandbox ubuntu - # user's home. - ("home/.dws", f"{_DWS_HOME}/.dws"), - ("home/.local/share/dws-cli", f"{_DWS_HOME}/.local/share/dws-cli"), -) - - -def _make_dws_creds_volumes(user_id: Optional[str]) -> list[Any]: - """Build the two read-write bind-mount Volumes for dws credential persistence (one set per user). - - Empty list is returned when: flag off / invalid user_id / host_storage not configured / - SDK has no Volume model — any unmet condition quietly degrades to "no credential - volumes mounted"; dws then has no credentials in the sandbox, business commands fail - with exit code 2 (not logged in), and the dingtalk skill guides the user to - "Settings → Connect DingTalk account" first. Same approach as _make_myspace_volume: - pre-create the host directories as backend (UID 1000) to avoid docker's root - auto-mkdir at bind time leaving the backend unable to sync/clean credentials. - """ - if not settings.sandbox.dws_creds_bind_mount_enabled: - return [] - uid = safe_user_id(user_id or "") - if not uid: - return [] - host_storage = (settings.sandbox.opensandbox_host_storage_path or "").strip().rstrip("/") - if not host_storage: - logger.warning( - "[opensandbox] dws creds bind mount enabled but HOST_STORAGE_PATH " - "not configured; dws will have no persisted credentials" - ) - return [] - cache_dir = dws_cache_dir(uid) - try: - for sub, _mount in _DWS_CREDS_SUBPATHS: - (cache_dir / sub).mkdir(parents=True, exist_ok=True) - except Exception as exc: # noqa: BLE001 - logger.warning( - "[opensandbox] failed to pre-create dws_cache dir %s: %s; " - "skipping dws creds bind mount", - cache_dir, - exc, - ) - return [] - try: - from opensandbox.models.sandboxes import Host, Volume - except ImportError: - return [] - slug = re.sub(r"[^a-z0-9-]", "-", uid.lower()).strip("-") or "u" - out: list[Any] = [] - for idx, (sub, mount_path) in enumerate(_DWS_CREDS_SUBPATHS): - # vol name must match k8s naming (lowercase alphanumerics + hyphens); sub contains - # / and ., so use an index instead. - vol_name = f"dws{idx}-{slug}"[:63].rstrip("-") or f"dws{idx}" - out.append( - Volume( - name=vol_name, - host=Host(path=f"{host_storage}/dws_cache/{uid}/{sub}"), - mount_path=mount_path, - read_only=False, - ) - ) - return out - - -# lark-cli (Feishu/Lark CLI) credential persistence volumes: same structure as dws. On Linux -# lark-cli uses a file-based encrypted store (master.key + *.enc) and only honors these two -# directories under $HOME, so both absolute paths must be bind-mounted (see the -# _common.lark_cache_dir comment). Note: we do **not** inject LARK_APP_* via env — when -# LARK_APP_ID/SECRET are present in the env, lark-cli defaults to the tenant/bot token flow, -# breaking --as user; app config is instead seeded by the backend's config init into the -# per-user HOME (same directory as the user token, entering the sandbox with the volume). -_LARK_HOME = "/home/ubuntu" - -_LARK_CREDS_SUBPATHS: tuple[tuple[str, str], ...] = ( - # (host subpath under lark_cache/{uid}, sandbox mount point); matches the backend login - # layout HOME=lark_cache/{uid}/home: login writes home/.lark-cli and - # home/.local/share/lark-cli; here we bind them into the sandbox ubuntu home. - ("home/.lark-cli", f"{_LARK_HOME}/.lark-cli"), - ("home/.local/share/lark-cli", f"{_LARK_HOME}/.local/share/lark-cli"), -) - - -def _make_lark_creds_volumes(user_id: Optional[str]) -> list[Any]: - """Build the two read-write bind-mount Volumes for lark-cli credential persistence (one set per user). - - Same approach as [[_make_dws_creds_volumes]]: flag off / invalid user_id / host_storage - not configured / SDK has no Volume model — any unmet condition quietly degrades to "no - credential volumes mounted"; lark-cli then has no credentials in the sandbox, business - commands fail as not-logged-in, and the lark-shared skill guides the user to complete - auth login first. Pre-create the host directories as backend (UID 1000) to avoid - docker's root auto-mkdir at bind time leaving the backend unable to sync/clean. - """ - if not settings.sandbox.lark_creds_bind_mount_enabled: - return [] - uid = safe_user_id(user_id or "") - if not uid: - return [] - host_storage = (settings.sandbox.opensandbox_host_storage_path or "").strip().rstrip("/") - if not host_storage: - logger.warning( - "[opensandbox] lark creds bind mount enabled but HOST_STORAGE_PATH " - "not configured; lark-cli will have no persisted credentials" - ) - return [] - cache_dir = lark_cache_dir(uid) - try: - for sub, _mount in _LARK_CREDS_SUBPATHS: - (cache_dir / sub).mkdir(parents=True, exist_ok=True) - except Exception as exc: # noqa: BLE001 - logger.warning( - "[opensandbox] failed to pre-create lark_cache dir %s: %s; " - "skipping lark creds bind mount", - cache_dir, - exc, - ) - return [] - try: - from opensandbox.models.sandboxes import Host, Volume - except ImportError: - return [] - slug = re.sub(r"[^a-z0-9-]", "-", uid.lower()).strip("-") or "u" - out: list[Any] = [] - for idx, (sub, mount_path) in enumerate(_LARK_CREDS_SUBPATHS): - # vol name must match k8s naming (lowercase alphanumerics + hyphens); sub contains - # / and ., so use an index instead. - vol_name = f"lark{idx}-{slug}"[:63].rstrip("-") or f"lark{idx}" - out.append( - Volume( - name=vol_name, - host=Host(path=f"{host_storage}/lark_cache/{uid}/{sub}"), - mount_path=mount_path, - read_only=False, - ) - ) - return out - - -# himalaya (email CLI) credential persistence volume: same structure as lark/dws. himalaya's -# config is a plain-file config.toml (containing account blocks + raw app-password lines) and -# only honors the config path, so binding this one directory suffices (no keychain, no -# secondary store). The app password is a user-supplied IMAP/SMTP app password written into -# config.toml by the backend; no env is injected in the sandbox (unlike the dingtalk Custom -# App / firecrawl, which need deployment-level envs). -_EMAIL_HOME = "/home/ubuntu" - -_EMAIL_CREDS_SUBPATHS: tuple[tuple[str, str], ...] = ( - # (host subpath under email_cache/{uid}, sandbox mount point); matches the backend - # layout HOME=email_cache/{uid}/home: config lands in home/.config/himalaya; here we - # bind it to the same path under the sandbox ubuntu home. - ("home/.config/himalaya", f"{_EMAIL_HOME}/.config/himalaya"), -) - - -def _make_email_creds_volumes(user_id: Optional[str]) -> list[Any]: - """Build the read-write bind-mount Volume for himalaya credential persistence (one per user, single directory). - - Same approach as [[_make_lark_creds_volumes]]: flag off / invalid user_id / host_storage - not configured / SDK has no Volume model — any unmet condition quietly degrades to "no - credential volume mounted"; himalaya then has no config in the sandbox, business - commands fail as unconfigured, and the email skill guides the user to bind a mailbox on - the plugin detail page first. Pre-create the host directories as backend (UID 1000) to - avoid docker's root auto-mkdir at bind time leaving the backend unable to sync/clean. - """ - if not settings.sandbox.email_creds_bind_mount_enabled: - return [] - uid = safe_user_id(user_id or "") - if not uid: - return [] - host_storage = (settings.sandbox.opensandbox_host_storage_path or "").strip().rstrip("/") - if not host_storage: - logger.warning( - "[opensandbox] email creds bind mount enabled but HOST_STORAGE_PATH " - "not configured; himalaya will have no persisted credentials" - ) - return [] - cache_dir = email_cache_dir(uid) - try: - for sub, _mount in _EMAIL_CREDS_SUBPATHS: - (cache_dir / sub).mkdir(parents=True, exist_ok=True) - except Exception as exc: # noqa: BLE001 - logger.warning( - "[opensandbox] failed to pre-create email_cache dir %s: %s; " - "skipping email creds bind mount", - cache_dir, - exc, - ) - return [] - try: - from opensandbox.models.sandboxes import Host, Volume - except ImportError: - return [] - slug = re.sub(r"[^a-z0-9-]", "-", uid.lower()).strip("-") or "u" - out: list[Any] = [] - for idx, (sub, mount_path) in enumerate(_EMAIL_CREDS_SUBPATHS): - vol_name = f"email{idx}-{slug}"[:63].rstrip("-") or f"email{idx}" - out.append( - Volume( - name=vol_name, - host=Host(path=f"{host_storage}/email_cache/{uid}/{sub}"), - mount_path=mount_path, - read_only=False, - ) - ) - return out - - -_YIDA_CREDS_SUBPATHS: tuple[tuple[str, str], ...] = ( - # (host subpath under yida_cache/{uid}, sandbox mount point) - ("workspace", _YIDA_WORKSPACE_MOUNT), -) - - -def _make_yida_creds_volumes(user_id: Optional[str]) -> list[Any]: - """Build the read-write bind-mount Volume for openyida login-state persistence (one per user, single directory). - - Same approach as [[_make_email_creds_volumes]]: flag off / invalid user_id / - host_storage not configured / SDK has no Volume model — any unmet condition quietly - degrades to "no credential volume mounted"; openyida then has no login state in the - sandbox, business commands fail as not-logged-in, and the yida skill guides the user - through QR-code login in the conversation (still usable within the session, just not - surviving across sessions). Pre-create the host directories as backend (UID 1000) to - avoid docker's root auto-mkdir at bind time leaving the backend unable to sync/clean. - """ - if not settings.sandbox.yida_creds_bind_mount_enabled: - return [] - uid = safe_user_id(user_id or "") - if not uid: - return [] - host_storage = (settings.sandbox.opensandbox_host_storage_path or "").strip().rstrip("/") - if not host_storage: - logger.warning( - "[opensandbox] yida creds bind mount enabled but HOST_STORAGE_PATH " - "not configured; openyida will have no persisted login state" - ) - return [] - cache_dir = yida_cache_dir(uid) - try: - for sub, _mount in _YIDA_CREDS_SUBPATHS: - (cache_dir / sub).mkdir(parents=True, exist_ok=True) - except Exception as exc: # noqa: BLE001 - logger.warning( - "[opensandbox] failed to pre-create yida_cache dir %s: %s; " - "skipping yida creds bind mount", - cache_dir, - exc, - ) - return [] - try: - from opensandbox.models.sandboxes import Host, Volume - except ImportError: - return [] - slug = re.sub(r"[^a-z0-9-]", "-", uid.lower()).strip("-") or "u" - out: list[Any] = [] - for idx, (sub, mount_path) in enumerate(_YIDA_CREDS_SUBPATHS): - vol_name = f"yida{idx}-{slug}"[:63].rstrip("-") or f"yida{idx}" - out.append( - Volume( - name=vol_name, - host=Host(path=f"{host_storage}/yida_cache/{uid}/{sub}"), - mount_path=mount_path, - read_only=False, - ) - ) - return out diff --git a/src/backend/core/sandbox/providers/opensandbox/admin.py b/src/backend/core/sandbox/providers/opensandbox/admin.py index 81ac1781..d22adc74 100644 --- a/src/backend/core/sandbox/providers/opensandbox/admin.py +++ b/src/backend/core/sandbox/providers/opensandbox/admin.py @@ -94,8 +94,9 @@ async def _iter_server_sandbox_infos(self) -> Any: finally: try: await mgr.close() - except Exception: # noqa: BLE001 - pass + except Exception as fallback_error: # noqa: BLE001 + from core.infra.failures import record_best_effort_failure + record_best_effort_failure('core.sandbox.providers.opensandbox.admin:_iter_server_sandbox_infos', fallback_error) @_on_service_loop async def admin_get_sandbox(self, sandbox_id: str) -> Optional[SandboxInfo]: diff --git a/src/backend/core/sandbox/providers/opensandbox/adoption.py b/src/backend/core/sandbox/providers/opensandbox/adoption.py index a21591a6..c247e48d 100644 --- a/src/backend/core/sandbox/providers/opensandbox/adoption.py +++ b/src/backend/core/sandbox/providers/opensandbox/adoption.py @@ -6,10 +6,6 @@ import core.sandbox.sessions.registry as session_registry from core.sandbox.providers.opensandbox.runtime import ( - _LEGACY_ENTRYPOINTS, - _LIGHT_ENTRYPOINT, - _SETUP_ENTRYPOINT, - _user_bound_sandbox_required, logger, ) from core.sandbox.providers.opensandbox.volumes import ( @@ -24,28 +20,12 @@ class _OpenSandboxAdoptionMixin: async def _adopt_existing_sandboxes(self) -> None: - """Find sandboxes this app previously left on the OpenSandbox server and route - them to adopt / kill. - - Classification rules (strictest to loosest): - 1. ``state != "Running"`` → kill (container already dead, metadata leftover) - 2. ``metadata[hugagent-owner] == "backend"`` → 先看 session_registry： - 仍绑在会话上的原样留着（会话下一轮自己 attach 回来），无主的才 adopt 进对应的池 - 3. ``entrypoint`` matches ``_LIGHT_ENTRYPOINT`` / a legacy entrypoint, - but the ``hugagent-owner`` tag is missing → legacy orphan, kill - 4. anything else (someone else's sandbox, or an unrecognized entrypoint) → leave alone - - Adopted sandboxes go into ``_pool.idle``; surplus adopt candidates beyond - ``max_idle`` are also killed to avoid unbounded pile-up. This method is called - once before ``shutdown``; calling it again is idempotent (sandboxes scanned a - second time are already in the pool, and an SDK ``connect`` failure - auto-kills). - """ + """Adopt explicitly owned, unbound sandboxes; leave other clients alone.""" stats = { "adopted_jupyter": 0, "adopted_light": 0, "skipped_bound": 0, - "killed_legacy": 0, + "killed_invalid": 0, "killed_dead": 0, "killed_overflow": 0, } @@ -56,11 +36,11 @@ async def _adopt_existing_sandboxes(self) -> None: logger.info( "[opensandbox] sandbox adoption complete: " "adopted jupyter=%d light=%d; left to sessions=%d; " - "killed legacy=%d dead=%d overflow=%d", + "killed invalid=%d dead=%d overflow=%d", stats["adopted_jupyter"], stats["adopted_light"], stats["skipped_bound"], - stats["killed_legacy"], + stats["killed_invalid"], stats["killed_dead"], stats["killed_overflow"], ) @@ -77,9 +57,8 @@ async def _classify_and_adopt( try: state = (getattr(info.status, "state", None) or "").lower() metadata = info.metadata or {} - entrypoint = list(info.entrypoint or []) owner_tag = metadata.get(_METADATA_OWNER_KEY) - if owner_tag and owner_tag != _METADATA_OWNER_VALUE: + if owner_tag != _METADATA_OWNER_VALUE: # Another owner's sandbox (another deployment, or a module's own # controller-managed computers) is never adopted or reaped here. return @@ -88,10 +67,6 @@ async def _classify_and_adopt( return # The creator has not yet delivered this instance. kind_tag = metadata.get(_METADATA_POOL_KIND_KEY) - known_legacy = entrypoint in (_LIGHT_ENTRYPOINT, _SETUP_ENTRYPOINT) or entrypoint in _LEGACY_ENTRYPOINTS - # Untagged instances with unknown entrypoints belong to other clients. - if owner_tag != _METADATA_OWNER_VALUE and not known_legacy: - return # 1. Dead owned sandbox → guarded cleanup if state != "running": if await self._kill_by_id(mgr, info.id, reason=f"non-running state={state}"): @@ -103,145 +78,114 @@ async def _classify_and_adopt( return # 2. Our own sandbox → adopt - if owner_tag == _METADATA_OWNER_VALUE: - if kind_tag not in ("jupyter", "light"): - logger.warning( - "[opensandbox] adopt: own sandbox %s missing pool-kind tag, killing", - info.id, - ) - if await self._kill_by_id(mgr, info.id, reason="missing pool-kind"): - stats["killed_legacy"] += 1 - return + if kind_tag not in ("jupyter", "light"): + logger.warning( + "[opensandbox] adopt: own sandbox %s missing pool-kind tag, killing", + info.id, + ) + if await self._kill_by_id(mgr, info.id, reason="missing pool-kind"): + stats["killed_invalid"] += 1 + return - # User-bound jupyter sandboxes are re-homed by user_id. When any - # private-data mount is enabled, a sandbox without a user tag cannot - # carry the current user's volumes; kill it rather than silently hand - # out a credential-less or cross-user container. - user_tag = metadata.get(_METADATA_USER_ID_KEY) - user_bound_enabled = _user_bound_sandbox_required() - if kind_tag == "jupyter" and user_bound_enabled and not user_tag: - if await self._kill_by_id( - mgr, - info.id, - reason="jupyter sandbox missing user-id tag (legacy / cross-user unsafe)", - ): - stats["killed_legacy"] += 1 - return - # Created before the per-user skill view: its /workspace/skills is the - # whole shared dir, which no longer holds this user's private skills. - # Mounts can't be changed after creation, so drop it. - if ( - kind_tag == "jupyter" - and user_tag - and metadata.get(_METADATA_SKILLS_LAYOUT_KEY) != _METADATA_SKILLS_LAYOUT_VALUE + # Persistent sandboxes require an explicit user owner so adoption + # cannot assign private mounts to another account. + user_tag = metadata.get(_METADATA_USER_ID_KEY) + if kind_tag == "jupyter" and not user_tag: + if await self._kill_by_id( + mgr, + info.id, + reason="jupyter sandbox missing user-id tag", ): - if await self._kill_by_id( - mgr, info.id, reason="jupyter sandbox predates the per-user skill view" - ): - stats["killed_legacy"] += 1 - return - # Reverse case: the sandbox still carries a user tag after all - # user-bound mounts were disabled. Its original volumes remain mounted; - # generic-pool reuse could expose them to another user, so kill it. - if kind_tag == "jupyter" and user_tag and not user_bound_enabled: - if await self._kill_by_id( - mgr, - info.id, - reason="jupyter sandbox carries user-id tag but user-bound mounts " - "are disabled (stale private mounts would cross-user leak)", - ): - stats["killed_legacy"] += 1 - return - - # 仍绑在会话上的容器不是无主的，收进空闲池就会被按空闲回收掉——而会话 - # 那边还在用它。绑定活在 session_registry 里（跨进程、比空闲窗口活得 - # 久），重启并不会让它消失，所以这里原样留着：会话下一轮照登记 attach - # 回来。真没人来接，服务端自己的 TTL 会收走。 - owner_session = await session_registry.bound_session(info.id) - if owner_session: - stats["skipped_bound"] += 1 - logger.info( - "[opensandbox] adopt: sandbox %s still bound to session %s; " - "leaving it to that session", - info.id, - owner_session, - ) - return - - if not await self._track_sandbox(str(info.id)): - return # Another live worker already owns this idle handle. - try: - sbx = await Sandbox.connect( - info.id, - connection_config=self._make_config(), - skip_health_check=(kind_tag == "light"), - ) - except Exception as exc: # noqa: BLE001 - logger.warning( - "[opensandbox] adopt: connect %s failed (%s), retaining container", - info.id, exc, - ) - await session_registry.untrack(str(info.id), self._tracking_owner_token()) - self._tracking_ids.discard(str(info.id)) - return + stats["killed_invalid"] += 1 + return + # Created before the per-user skill view: its /workspace/skills is the + # whole shared dir, which no longer holds this user's private skills. + # Mounts can't be changed after creation, so drop it. + if ( + kind_tag == "jupyter" + and user_tag + and metadata.get(_METADATA_SKILLS_LAYOUT_KEY) != _METADATA_SKILLS_LAYOUT_VALUE + ): + if await self._kill_by_id( + mgr, info.id, reason="jupyter sandbox predates the per-user skill view" + ): + stats["killed_invalid"] += 1 + return + # 仍绑在会话上的容器不是无主的，收进空闲池就会被按空闲回收掉——而会话 + # 那边还在用它。绑定活在 session_registry 里（跨进程、比空闲窗口活得 + # 久），重启并不会让它消失，所以这里原样留着：会话下一轮照登记 attach + # 回来。真没人来接，服务端自己的 TTL 会收走。 + owner_session = await session_registry.bound_session(info.id) + if owner_session: + stats["skipped_bound"] += 1 + logger.info( + "[opensandbox] adopt: sandbox %s still bound to session %s; " + "leaving it to that session", + info.id, + owner_session, + ) + return - # Direct-connect to execd (bypasses the server proxy's ~3s/request overhead); falls back automatically when unreachable. - await self._repoint_execd_direct(sbx) + if not await self._track_sandbox(str(info.id)): + return # Another live worker already owns this idle handle. + try: + sbx = await Sandbox.connect( + info.id, + connection_config=self._make_config(), + skip_health_check=(kind_tag == "light"), + ) + except Exception as exc: # noqa: BLE001 + logger.warning( + "[opensandbox] adopt: connect %s failed (%s), retaining container", + info.id, exc, + ) + await session_registry.untrack(str(info.id), self._tracking_owner_token()) + self._tracking_ids.discard(str(info.id)) + return - # jupyter + user tag → user-bound pool - if kind_tag == "jupyter" and user_tag and user_bound_enabled: - ok = await self._jupyter_user_pool.adopt(user_tag, sbx) - if not ok: - await self._destroy_sandbox(sbx) - stats["killed_overflow"] += 1 - return - stats["adopted_jupyter"] += 1 - logger.info( - "[opensandbox] adopt: claimed jupyter sandbox %s for user=%s", - info.id, - user_tag, - ) - return + # Direct-connect to execd (bypasses the server proxy's ~3s/request overhead); falls back automatically when unreachable. + await self._repoint_execd_direct(sbx) - # light (or fallback jupyter with all user-bound mounts disabled) → generic pool - # pool full → kill rather than leak - bucket = self._pool._buckets[kind_tag] # noqa: SLF001 - if len(bucket.idle) >= bucket.max_idle: - if await self._kill_by_id( - mgr, - info.id, - reason=f"{kind_tag} bucket already at max_idle={bucket.max_idle}", - ): - stats["killed_overflow"] += 1 - return - ok = await self._pool.adopt_idle(kind_tag, sbx) + # jupyter + user tag → user-bound pool + if kind_tag == "jupyter" and user_tag: + ok = await self._jupyter_user_pool.adopt(user_tag, sbx) if not ok: - # adopt_idle refused (race: another entry enqueued concurrently filled the capacity) — destroy await self._destroy_sandbox(sbx) stats["killed_overflow"] += 1 return - stats[f"adopted_{kind_tag}"] += 1 + stats["adopted_jupyter"] += 1 logger.info( - "[opensandbox] adopt: claimed %s sandbox %s", - kind_tag, + "[opensandbox] adopt: claimed jupyter sandbox %s for user=%s", info.id, + user_tag, ) return - # 3. No tag, but the entrypoint looks like one we previously created (orphan left by an older version) → kill - if ( - entrypoint in (_LIGHT_ENTRYPOINT, _SETUP_ENTRYPOINT) - or entrypoint in _LEGACY_ENTRYPOINTS - ): + # Light sandboxes use the shared pool. + # pool full → kill rather than leak + bucket = self._pool._buckets[kind_tag] # noqa: SLF001 + if len(bucket.idle) >= bucket.max_idle: if await self._kill_by_id( mgr, info.id, - reason=f"legacy untagged sandbox with our entrypoint {entrypoint}", + reason=f"{kind_tag} bucket already at max_idle={bucket.max_idle}", ): - stats["killed_legacy"] += 1 + stats["killed_overflow"] += 1 return + ok = await self._pool.adopt_idle(kind_tag, sbx) + if not ok: + # adopt_idle refused (race: another entry enqueued concurrently filled the capacity) — destroy + await self._destroy_sandbox(sbx) + stats["killed_overflow"] += 1 + return + stats[f"adopted_{kind_tag}"] += 1 + logger.info( + "[opensandbox] adopt: claimed %s sandbox %s", + kind_tag, + info.id, + ) + return - # 4. Someone else's sandbox / unrecognized entrypoint → leave alone except Exception as exc: # noqa: BLE001 logger.warning( "[opensandbox] adopt: classifying %s failed: %s", diff --git a/src/backend/core/sandbox/providers/opensandbox/binding.py b/src/backend/core/sandbox/providers/opensandbox/binding.py index 426f3ca8..acc7983c 100644 --- a/src/backend/core/sandbox/providers/opensandbox/binding.py +++ b/src/backend/core/sandbox/providers/opensandbox/binding.py @@ -49,7 +49,7 @@ async def _resolve_bound_session(self, session_id, user_id): await session_registry.remember(session_id, str(sess.sandbox.id)) self._maybe_schedule_renew(session_id, sess) if newly_bound: - from core.plugins.resources.prewarm import schedule + from core.capabilities.plugins.resources.prewarm import schedule schedule(self, session_id, sess.user_id, str(sess.sandbox.id)) return sess diff --git a/src/backend/core/sandbox/providers/opensandbox/creation.py b/src/backend/core/sandbox/providers/opensandbox/creation.py index a78197c1..2a842ca5 100644 --- a/src/backend/core/sandbox/providers/opensandbox/creation.py +++ b/src/backend/core/sandbox/providers/opensandbox/creation.py @@ -10,13 +10,8 @@ import core.sandbox.sessions.registry as session_registry from core.config.settings import settings from core.sandbox.errors import SandboxConnectError -from core.sandbox.providers.opensandbox.account_volumes import ( - _make_dws_creds_volumes, - _make_email_creds_volumes, - _make_lark_creds_volumes, - _make_yida_creds_volumes, -) -from core.sandbox.providers.opensandbox.models import _Session +from core.sandbox.credential_dispatch import credential_mounts +from core.sandbox.providers.opensandbox.models import SandboxSessionRecord as _Session # Snapshot persistence (see internal design docs). Imports are # deferred to runtime to avoid cross-module circular dependencies at startup @@ -24,7 +19,6 @@ from core.sandbox.providers.opensandbox.runtime import ( _LIGHT_ENTRYPOINT, _SETUP_ENTRYPOINT, - _user_bound_sandbox_required, logger, ) from core.sandbox.providers.opensandbox.volumes import _make_skills_volumes, _pool_metadata @@ -38,11 +32,11 @@ def _session_entrypoint(self, user_id): return _SETUP_ENTRYPOINT if user_id else _LIGHT_ENTRYPOINT async def _additional_workspace_mounts(self, user_id): - from core.spaces.mounts import prepare_volumes + from core.files.spaces.mounts import provision_volumes if not user_id: return [], {} - return await asyncio.to_thread(prepare_volumes, user_id) + return await asyncio.to_thread(provision_volumes, user_id) async def _create_jupyter_sandbox( self, @@ -59,23 +53,7 @@ async def _create_jupyter_sandbox( volumes = [] else: skills_vols = _make_skills_volumes(user_id) - # dws (DingTalk CLI) credential persistence volumes: two bind-mounts per user, - # mounted at the sandbox home's ~/.dws and ~/.local/share/dws-cli (where dws - # reads/writes by default), so OAuth credentials survive across sessions. - dws_vols = _make_dws_creds_volumes(user_id) - # lark-cli (Feishu CLI) credential persistence volumes: likewise mounts - # ~/.lark-cli and ~/.local/share/lark-cli (including master.key), so the - # user's OAuth user_access_token survives across sessions. - lark_vols = _make_lark_creds_volumes(user_id) - # himalaya (email CLI) credential persistence volume: mounts ~/.config/himalaya - # (including config.toml + IMAP/SMTP authorization codes), so the user's - # mailbox configuration survives across sessions. - email_vols = _make_email_creds_volumes(user_id) - # openyida (Yida CLI) login-state persistence volume: mounts the fixed working - # directory ~/yida-workspace (including .cache/cookies-*.json QR-login cookies - # + project artifacts), so the login state survives across sessions. - yida_vols = _make_yida_creds_volumes(user_id) - volumes = skills_vols + extra_volumes + dws_vols + lark_vols + email_vols + yida_vols + volumes = skills_vols + extra_volumes + credential_mounts(user_id) # entrypoint / volumes / metadata / timeout are exactly the same on the image # and snapshot paths; volumes must also be passed explicitly on the snapshot # path — docker commit does not preserve mount configuration, otherwise the @@ -103,15 +81,14 @@ async def _create_jupyter_sandbox( except Exception as e: raise SandboxConnectError(f"无法创建 OpenSandbox 持久会话沙盒: {e}") from e - # 目录约定已由 entrypoint 在启动时建好，直接打标，让 _persistent_setup 整段跳过 - # —— 这才是首条命令省下那一秒的地方。老沙箱（旧 entrypoint 建的、adopt 回来的） - # 没有这个标记，仍按原路在首次执行前补建，不受影响。 + # The entrypoint prepared the workspace; avoid repeating setup on first use. if setup_uid: try: sbx._jx_setup_done = True # type: ignore[attr-defined] sbx._jx_myspace_uid = setup_uid # type: ignore[attr-defined] - except Exception: # noqa: BLE001 — defensive: opaque SDK object - pass + except Exception as fallback_error: # noqa: BLE001 — defensive: opaque SDK object + from core.infra.failures import record_best_effort_failure + record_best_effort_failure('core.sandbox.providers.opensandbox.creation:_create_jupyter_sandbox', fallback_error) return sbx async def _create_light_sandbox(self) -> Any: @@ -132,9 +109,7 @@ async def _create_light_sandbox(self) -> Any: try: sbx = await self._create_owned_sandbox( self._image, - timeout=timedelta( - seconds=300 - ), # ephemeral is short-lived; 5min cap as a safety net + timeout=timedelta(seconds=300), # ephemeral is short-lived; 5min cap as a safety net entrypoint=_LIGHT_ENTRYPOINT, connection_config=config, skip_health_check=True, @@ -265,8 +240,9 @@ async def _destroy_sandbox( finally: try: await sbx.close() - except Exception: - pass + except Exception as fallback_error: + from core.infra.failures import record_best_effort_failure + record_best_effort_failure('core.sandbox.providers.opensandbox.creation:_destroy_sandbox', fallback_error) async def _pooled_sandbox_owner(self, sbx: Any) -> Optional[str]: """池子用的归属谓词：这台容器当前属于哪个会话，无主返回 ``None``。 @@ -355,18 +331,8 @@ async def _probe_pooled_sandbox_alive(self, sbx: Any) -> bool: return True async def _create_session(self, user_id: Optional[str] = None) -> _Session: - """Construct a new persistent session (user-bound sandbox, running directly via execd, no Jupyter kernel). - - Plan F: when ``user_id`` is given and any private-data bind mount is on, - **bypass the general warm pool** — the pool's sandboxes have no user-bound - MySpace/account-credential volumes. Instead, take the user-bound path via - ``_create_jupyter_sandbox(user_id=...)``. Adopted old sandboxes have already - been slotted per user into ``_jupyter_user_pool`` (see B.7). - - When ``user_id`` is absent (legacy / fallback), take the old path: grab one - from the general pool without account volumes. - """ - use_user_pool = bool(user_id and (_user_bound_sandbox_required() or str(user_id).startswith("api_ak_"))) + """Route account sessions to their user pool; anonymous sessions use the general pool.""" + use_user_pool = bool(user_id) if use_user_pool: sandbox = await self._jupyter_user_pool.acquire(user_id) @@ -382,9 +348,7 @@ async def _create_session(self, user_id: Optional[str] = None) -> _Session: ) async def _create_session_from_snapshot( - self, - snapshot_id: str, - user_id: Optional[str] = None, + self, snapshot_id: str, user_id: Optional[str] = None, ) -> _Session: """Start a persistent session sandbox directly from an opensandbox snapshot + wrap it into a _Session. @@ -399,8 +363,7 @@ async def _create_session_from_snapshot( usable as soon as it is up. """ sandbox = await self._create_jupyter_sandbox( - from_snapshot_id=snapshot_id, - user_id=user_id, + from_snapshot_id=snapshot_id, user_id=user_id, ) return _Session( sandbox=sandbox, @@ -422,6 +385,5 @@ async def _create_owned_sandbox(self, *args, **kwargs): sandbox = await Sandbox.create(*args, **kwargs) await self._repoint_execd_direct(sandbox) if not await self._track_sandbox(str(sandbox.id)): - # Another owner may already be using it. Never kill or deliver it. raise SandboxConnectError("sandbox ownership was acquired elsewhere") return sandbox diff --git a/src/backend/core/sandbox/providers/opensandbox/execution.py b/src/backend/core/sandbox/providers/opensandbox/execution.py index 9eccacc8..10e8ab6f 100644 --- a/src/backend/core/sandbox/providers/opensandbox/execution.py +++ b/src/backend/core/sandbox/providers/opensandbox/execution.py @@ -10,7 +10,7 @@ from core.config.settings import settings from core.sandbox.protocol import ProcessRequest -from core.sandbox.providers.opensandbox.models import _Session +from core.sandbox.providers.opensandbox.models import SandboxSessionRecord as _Session # Snapshot persistence (see internal design docs). Deferred to a # runtime import to avoid a cross-module circular dependency at startup @@ -143,8 +143,9 @@ async def _persistent_setup(self, sess: _Session, req: ProcessRequest) -> None: sess.sandbox._jx_setup_done = True # type: ignore[attr-defined] if uid: sess.sandbox._jx_myspace_uid = uid # type: ignore[attr-defined] - except Exception: # noqa: BLE001 — defensive: opaque SDK object - pass + except Exception as fallback_error: # noqa: BLE001 — defensive: opaque SDK object + from core.infra.failures import record_best_effort_failure + record_best_effort_failure('core.sandbox.providers.opensandbox.execution:_persistent_setup', fallback_error) else: stderr = _join_logs(getattr(getattr(result, "logs", None), "stderr", []) or []) logger.warning( diff --git a/src/backend/core/sandbox/providers/opensandbox/lifecycle.py b/src/backend/core/sandbox/providers/opensandbox/lifecycle.py index 00a74034..ee4db477 100644 --- a/src/backend/core/sandbox/providers/opensandbox/lifecycle.py +++ b/src/backend/core/sandbox/providers/opensandbox/lifecycle.py @@ -9,7 +9,7 @@ from core.config.settings import settings from core.sandbox.errors import SandboxNotFoundError, SandboxError -from core.sandbox.providers.opensandbox.models import _Session +from core.sandbox.providers.opensandbox.models import SandboxSessionRecord as _Session # Snapshot persistence (see internal design docs). Imports are # deferred to runtime to avoid cross-module circular dependencies at startup @@ -40,7 +40,7 @@ async def _create_session_for( workspace it built up. The ~7s warm path is only correct for a chat with nothing to restore, so the idle pool is checked after the lookup misses. """ - from core.agent_skills.publication import prepare_skill_view + from core.capabilities.skills.publication import prepare_skill_view # This runs even on a pre-warmed pool hit, before any snapshot/fresh # fallback can hide a publication failure. diff --git a/src/backend/core/sandbox/providers/opensandbox/models.py b/src/backend/core/sandbox/providers/opensandbox/models.py index de15e34f..4f256929 100644 --- a/src/backend/core/sandbox/providers/opensandbox/models.py +++ b/src/backend/core/sandbox/providers/opensandbox/models.py @@ -7,14 +7,11 @@ @dataclass -class _Session: +class SandboxSessionRecord: sandbox: Any # opensandbox.Sandbox - # The user_id owning this sandbox (binds MySpace/account-credential volumes + - # user re-homing on adopt across restarts). None when legacy paths call - # _get_or_create_session without a user_id — execution then falls back to a - # credential-less general sandbox. + # None identifies an anonymous session without account volumes. user_id: Optional[str] = None - # Plan F: which pool this sandbox was actually taken from. **release must follow this, + # Which pool this sandbox was actually taken from. **release must follow this, # never re-decide from the current user-bound-volume settings** — a mid-flight # flag change (ops toggle, hot reload) would route release to a different pool, # corrupting counts / cross-user contamination. The snapshot restore path builds diff --git a/src/backend/core/sandbox/providers/opensandbox/pool.py b/src/backend/core/sandbox/providers/opensandbox/pool.py index b85cb96e..597c604a 100644 --- a/src/backend/core/sandbox/providers/opensandbox/pool.py +++ b/src/backend/core/sandbox/providers/opensandbox/pool.py @@ -224,8 +224,7 @@ async def probe(sbx): logger.info( "[sandbox-pool] sweep: evicting dead idle sandbox kind=%s " "id=%s (server-side reaped); refilling", - kind, - getattr(sbx, "id", "?"), + kind, getattr(sbx, "id", "?"), ) if (id(sbx) not in foreign and (self._owner is None or not await self._owner(sbx))): diff --git a/src/backend/core/sandbox/providers/opensandbox/provider.py b/src/backend/core/sandbox/providers/opensandbox/provider.py index c724577e..813832c7 100644 --- a/src/backend/core/sandbox/providers/opensandbox/provider.py +++ b/src/backend/core/sandbox/providers/opensandbox/provider.py @@ -18,10 +18,9 @@ from core.sandbox.providers.opensandbox.creation import _OpenSandboxCreationMixin from core.sandbox.providers.opensandbox.execution import _OpenSandboxExecMixin from core.sandbox.providers.opensandbox.lifecycle import _OpenSandboxLifecycleMixin -from core.sandbox.providers.opensandbox.models import _Session +from core.sandbox.providers.opensandbox.models import SandboxSessionRecord as _Session from core.sandbox.providers.opensandbox.pool import SandboxPool from core.sandbox.providers.opensandbox.runtime import ( - _user_bound_sandbox_required, logger, write_file, ) @@ -51,9 +50,7 @@ class OpenSandboxProvider( _service_loop: Optional[asyncio.AbstractEventLoop] = None def __init__(self) -> None: - # Only the opensandbox SDK is needed (execd commands.run); no longer depends on - # code_interpreter (the Jupyter kernel was removed, all languages go through - # execd, see cloud_processes). + # All languages execute through execd. try: from opensandbox import Sandbox except ImportError as e: @@ -63,7 +60,10 @@ def __init__(self) -> None: ) from e s = settings.sandbox - self._domain = s.opensandbox_domain + # SDK and direct health/renew requests must use the same absolute endpoint. + self._domain = s.opensandbox_domain.rstrip("/") + if not self._domain.startswith(("http://", "https://")): + self._domain = "http://" + self._domain self._api_key = s.opensandbox_api_key self._image = s.opensandbox_image self._ttl_s = s.idle_ttl_s @@ -93,7 +93,7 @@ def __init__(self) -> None: except RuntimeError: self._service_loop = None # ⚠️ 锁必须按**事件循环**分桶。这个 provider 是进程级单例，但调用方不止一个 loop： - # 子智能体跑在独立线程的独立事件循环里（core/llm/subagent_tool.py 的 + # 子智能体跑在独立线程的独立事件循环里（core/llm/subagents/tool.py 的 # asyncio.new_event_loop()）。asyncio.Lock 在**首次 await 时**绑定当时的 running loop， # 之后被另一个 loop 用到就抛 "is bound to a different event loop"。 # 事故形态极隐蔽：谁先碰 provider 谁把锁绑走，另一个 loop 的沙箱保活 touch_session @@ -118,9 +118,7 @@ def __init__(self) -> None: # serves the fallback path where user_id is missing; normal user-bound traffic # goes through _jupyter_user_pool. self._pool = SandboxPool( - jupyter_min_idle=( - s.opensandbox_pool_jupyter_min_idle if not _user_bound_sandbox_required() else 0 - ), + jupyter_min_idle=0, jupyter_max_idle=s.opensandbox_pool_jupyter_max_idle, light_min_idle=s.opensandbox_pool_light_min_idle, light_max_idle=s.opensandbox_pool_light_max_idle, @@ -131,9 +129,7 @@ def __init__(self) -> None: liveness_fn=self._probe_pooled_sandbox_alive, owner_fn=self._pooled_sandbox_owner, ) - # Plan F: jupyter sandboxes are bucketed by user_id. When bind mount is enabled, - # all jupyter acquires go through here; adopted old sandboxes are placed into the - # matching bucket by their hugagent-user-id metadata. + # Account sessions and adopted instances share the same per-user pool. self._jupyter_user_pool = _JupyterUserPool( factory=lambda uid: self._create_jupyter_sandbox(user_id=uid), destroy_fn=self._destroy_sandbox, @@ -338,23 +334,15 @@ async def health(self) -> bool: # instances (still alive but not tracked in memory, backend_managed=False). # Pool figures come from the stats of the two pools. + def bind_loop(self) -> None: + loop = asyncio.get_running_loop() + if self._service_loop is not None and self._service_loop is not loop: + raise RuntimeError("Sandbox provider already belongs to another event loop") + self._service_loop = loop + async def warmup(self) -> None: - """Startup-time pool warmup entry point (called by the FastAPI startup hook, - repeatable, idempotent). - - Steps: - 1. ``_adopt_existing_sandboxes``: scan all sandboxes on the OpenSandbox server; - live sandboxes carrying this app's metadata tag are reclaimed into - ``_pool.idle``; those with "our previous entrypoint but missing the tag" or - in an unusable state are killed outright to clean up orphans. - 2. ``_pool.warmup``: if adoption did not fill min_idle, top up the remaining - deficit via ``factory()`` (same behavior as before). - Effect: after a backend restart, idle fills almost instantly with no need to - recreate 4 sandboxes (saves ~60s). - """ - # 锚定服务循环：warmup 由 FastAPI 启动钩子在主循环上调用，之后所有跨循环的 - # provider 调用都转发到这里（见 ``_on_service_loop``）。 - self._service_loop = asyncio.get_running_loop() + """Adopt owned instances and start pool/snapshot maintenance.""" + self.bind_loop() try: await self._adopt_existing_sandboxes() except Exception as exc: # noqa: BLE001 — an adoption failure must not block warmup @@ -371,11 +359,7 @@ async def warmup(self) -> None: if self._snapshot_gc_task is None or self._snapshot_gc_task.done(): self._snapshot_gc_task = asyncio.create_task(self._snapshot_gc_loop()) - async def shutdown(self) -> None: - """Called on process exit: gracefully shut down all persistent sessions + idle sandboxes in the pools + snapshot workers.""" - processes = getattr(self, "_command_processes", None) - if processes: - await processes.close_all() + async def stop_maintenance(self) -> None: # Stop the snapshot workers first: no more new park / GC triggers self._snapshot_workers_shutdown.set() for task in (self._snapshot_idle_task, self._snapshot_gc_task): @@ -389,6 +373,13 @@ async def shutdown(self) -> None: self._snapshot_idle_task = None self._snapshot_gc_task = None + async def shutdown(self) -> None: + """Called on process exit: gracefully shut down all persistent sessions + idle sandboxes in the pools + snapshot workers.""" + processes = getattr(self, "_command_processes", None) + if processes: + await processes.close_all() + await self.stop_maintenance() + for sid, sess in list(self._sessions.items()): try: await self._release_session(sid, sess, detach_only=True) diff --git a/src/backend/core/sandbox/providers/opensandbox/reclamation.py b/src/backend/core/sandbox/providers/opensandbox/reclamation.py index 0ed0f7f6..e0a767f4 100644 --- a/src/backend/core/sandbox/providers/opensandbox/reclamation.py +++ b/src/backend/core/sandbox/providers/opensandbox/reclamation.py @@ -6,7 +6,7 @@ import core.sandbox.sessions.registry as session_registry from core.sandbox.providers.opensandbox.accounting import finish_accounting -from core.sandbox.providers.opensandbox.models import _Session +from core.sandbox.providers.opensandbox.models import SandboxSessionRecord as _Session from core.sandbox.providers.opensandbox.runtime import logger @@ -50,7 +50,7 @@ async def protect(chat_id, sess): await asyncio.gather(*(protect(chat_id, sess) for chat_id, sess in entries)) async def _destroy_session(self, session_id, sess, *, reuse_sandbox=False): - from core.plugins.resources.prewarm import closing, stop + from core.capabilities.plugins.resources.prewarm import closing, stop import asyncio async with closing(session_id): clean = False diff --git a/src/backend/core/sandbox/providers/opensandbox/runtime.py b/src/backend/core/sandbox/providers/opensandbox/runtime.py index 63e27793..9106e4a7 100644 --- a/src/backend/core/sandbox/providers/opensandbox/runtime.py +++ b/src/backend/core/sandbox/providers/opensandbox/runtime.py @@ -45,20 +45,7 @@ async def write_file(sbx: Any, path: str, content: Any) -> None: logger.warning("[opensandbox] chown %s 失败（继续）: %r", path, exc) -def _user_bound_sandbox_required() -> bool: - """Account sessions always require account-specific volumes.""" - return True - - -# Entrypoints earlier versions created sandboxes with. Nothing is created with them -# any more (every sandbox uses _LIGHT_ENTRYPOINT), but adopt still recognizes them so -# orphans left by an older backend get killed instead of leaking. -_LEGACY_ENTRYPOINTS = ( - ["/opt/opensandbox/code-interpreter.sh"], - ["/opt/hugagent/entrypoint-as-ubuntu.sh"], -) - -# Plan F Q1: environment variables used when running bash via sandbox.commands.run (execd). +# Environment variables used when running bash via sandbox.commands.run (execd). # execd does not source /opt/opensandbox/code-interpreter-env.sh; PATH then only contains the # system bin paths, so python resolves to a "system python" like /usr/bin/python3 that has no # pandas/docx etc. installed. We explicitly provide a PATH matching where packages are diff --git a/src/backend/core/sandbox/providers/opensandbox/snapshots.py b/src/backend/core/sandbox/providers/opensandbox/snapshots.py index 6627255b..c871cd5d 100644 --- a/src/backend/core/sandbox/providers/opensandbox/snapshots.py +++ b/src/backend/core/sandbox/providers/opensandbox/snapshots.py @@ -9,7 +9,7 @@ import core.sandbox.sessions.registry as session_registry from core.config.settings import settings from core.sandbox.errors import SandboxNotFoundError, SandboxError, SandboxTimeoutError -from core.sandbox.providers.opensandbox.models import _Session +from core.sandbox.providers.opensandbox.models import SandboxSessionRecord as _Session # Snapshot persistence (see internal design docs). Imports are # deferred to runtime to avoid cross-module circular dependencies at startup @@ -94,7 +94,7 @@ def _upsert() -> Optional[str]: return await asyncio.to_thread(_upsert) - async def _delete_snapshot_row(self, chat_id: str) -> None: + async def _delete_snapshot_row(self, chat_id: str, snapshot_id: str) -> None: """Delete the chat_sandbox_snapshots row (leaves the remote snapshot untouched).""" def _delete() -> None: @@ -102,7 +102,8 @@ def _delete() -> None: ChatSandboxSnapshot = _get_snapshot_model() with SessionLocal() as db: db.query(ChatSandboxSnapshot).filter( - ChatSandboxSnapshot.chat_id == chat_id + ChatSandboxSnapshot.chat_id == chat_id, + ChatSandboxSnapshot.snapshot_id == snapshot_id, ).delete() db.commit() @@ -213,6 +214,7 @@ async def _snapshot_idle_sessions_once(self) -> None: The idle threshold starts snapshotting; server expiry includes a grace period so that the scan and snapshot can finish before container removal. """ + await self._discover_idle_sessions() threshold = settings.sandbox.idle_ttl_s now = time.monotonic() candidates: list[tuple[str, _Session]] = [] @@ -233,9 +235,13 @@ async def _snapshot_idle_sessions_once(self) -> None: candidates.append((chat_id, sess)) if len(candidates) >= self._PARK_BATCH_LIMIT: break - for chat_id, sess in candidates: + for chat_id, _ in candidates: self._parking_in_flight.add(chat_id) - asyncio.create_task(self._park_session_via_snapshot(chat_id, sess)) + # Children belong to the maintenance task: losing leadership cancels + # and joins every in-flight snapshot before releasing ownership. + async with asyncio.TaskGroup() as group: + for chat_id, sess in candidates: + group.create_task(self._park_session_via_snapshot(chat_id, sess)) async def _park_session_via_snapshot(self, chat_id: str, sess: _Session) -> None: """take snapshot → wait Ready → upsert DB → release the session. @@ -271,8 +277,7 @@ async def _park_session_via_snapshot(self, chat_id: str, sess: _Session) -> None chat_id, settings.sandbox.idle_ttl_s ): logger.info( - "[opensandbox] park chat=%s skipped: still in use elsewhere", - chat_id, + "[opensandbox] park chat=%s skipped: still in use elsewhere", chat_id, ) return try: @@ -295,29 +300,23 @@ async def _park_session_via_snapshot(self, chat_id: str, sess: _Session) -> None return await self._wait_snapshot_ready(snap_id) old_snap = await self._upsert_snapshot( - chat_id, - snapshot_id=snap_id, - sandbox_id=sandbox_id, + chat_id, snapshot_id=snap_id, sandbox_id=sandbox_id, ) # The actual release; the container goes back to the idle pool await self._destroy_session(chat_id, sess, reuse_sandbox=True) logger.info( "[opensandbox] parked chat=%s snapshot=%s (replaced_old=%s)", - chat_id, - snap_id, - old_snap or "none", + chat_id, snap_id, old_snap or "none", ) # Clean up the old remote snapshot outside the lock (async) if old_snap: - asyncio.create_task(self._delete_snapshot(old_snap)) + await self._delete_snapshot(old_snap) except asyncio.CancelledError: raise except Exception as e: # noqa: BLE001 logger.warning( "[opensandbox] park chat=%s sandbox=%s failed: %s", - chat_id, - sandbox_id, - e, + chat_id, sandbox_id, e, ) finally: self._parking_in_flight.discard(chat_id) @@ -383,7 +382,7 @@ async def _delete_one(snapshot_id: str) -> bool: if not ok: continue try: - await self._delete_snapshot_row(chat_id) + await self._delete_snapshot_row(chat_id, snapshot_id) except Exception as e: # noqa: BLE001 logger.warning( "[opensandbox] gc: delete db row chat=%s failed: %s", @@ -417,14 +416,13 @@ async def _take_snapshot(self, sandbox_id: str) -> str: data = resp.json() snap_id = data.get("id") or data.get("snapshotId") if not snap_id: - raise SandboxError(f"create snapshot returned no id (sandbox={sandbox_id}): {data!r}") + raise SandboxError( + f"create snapshot returned no id (sandbox={sandbox_id}): {data!r}" + ) return str(snap_id) async def _wait_snapshot_ready( - self, - snapshot_id: str, - *, - timeout_s: Optional[int] = None, + self, snapshot_id: str, *, timeout_s: Optional[int] = None, ) -> None: """Poll GET /v1/snapshots/{id} until state == 'Ready'. diff --git a/src/backend/core/sandbox/providers/opensandbox/tracking.py b/src/backend/core/sandbox/providers/opensandbox/tracking.py index 04e2005e..6fa04478 100644 --- a/src/backend/core/sandbox/providers/opensandbox/tracking.py +++ b/src/backend/core/sandbox/providers/opensandbox/tracking.py @@ -85,3 +85,34 @@ async def _confirm_sandbox_exists(self, sandbox_id): raise SandboxNotFoundError(f"sandbox {sandbox_id} no longer exists") if response.status_code >= 400: raise SandboxError(f"sandbox metadata query failed: HTTP {response.status_code}") + + + async def _discover_idle_sessions(self): + """Maintenance owns shared bindings, regardless of which worker served them.""" + import time + from core.sandbox.providers.opensandbox.volumes import _METADATA_USER_ID_KEY + for chat, sandbox_id, active_at in await session_registry.bindings(): + if chat in self._sessions or time.time() - active_at < settings.sandbox.idle_ttl_s: + continue + lock = await self._get_session_lock(chat) + if lock.locked(): + continue + try: + async with lock, session_registry.claim(chat): + if await session_registry.lookup(chat) != sandbox_id: + continue + response = await self._get_http_client().get( + f"{self._domain.rstrip('/')}/v1/sandboxes/{sandbox_id}" + ) + if response.status_code == 404: + if response.json().get("code") == "DOCKER::SANDBOX_NOT_FOUND": + await session_registry.forget(chat, sandbox_id) + continue + response.raise_for_status() + user_id = (response.json().get("metadata") or {}).get(_METADATA_USER_ID_KEY) + session = await self._attach_session(sandbox_id, user_id) + if session is not None: + session.last_active_ts = time.monotonic() - max(0, time.time() - active_at) + self._sessions[chat] = session + except Exception as exc: + logger.warning("sandbox discovery deferred chat=%s: %s", chat, exc) diff --git a/src/backend/core/sandbox/providers/opensandbox/user_pool.py b/src/backend/core/sandbox/providers/opensandbox/user_pool.py index 4a2c469c..0c87798d 100644 --- a/src/backend/core/sandbox/providers/opensandbox/user_pool.py +++ b/src/backend/core/sandbox/providers/opensandbox/user_pool.py @@ -8,7 +8,7 @@ import logging import time from collections import deque -from typing import Any, Optional +from typing import Any, Callable, Optional from core.sandbox.errors import SandboxError @@ -149,9 +149,7 @@ async def release(self, user_id: str, sbx: Any, *, reuse: bool, destroy: bool = logger.info( "[jupyter-user-pool] release user=%s sandbox_id=%s -> idle " "(idle_count=%d)", - user_id, - getattr(sbx, "id", "?"), - len(q), + user_id, getattr(sbx, "id", "?"), len(q), ) else: should_destroy = sbx diff --git a/src/backend/core/sandbox/providers/opensandbox/volumes.py b/src/backend/core/sandbox/providers/opensandbox/volumes.py index a4cc3bf1..dab22752 100644 --- a/src/backend/core/sandbox/providers/opensandbox/volumes.py +++ b/src/backend/core/sandbox/providers/opensandbox/volumes.py @@ -2,55 +2,11 @@ from __future__ import annotations -import logging -import os -import re -from pathlib import Path -from typing import Any, Optional +from typing import Optional from core.config.settings import settings from core.sandbox.sessions.metadata import USER_ID_METADATA_KEY from core.sandbox.workspace.identity import safe_user_id -from extensions.paths import SKILLS_DIR - -logger = logging.getLogger(__name__) - -# Built-in skills enter the sandbox at /workspace/skills/ via a host bind mount, read-only. -# -# Note: an absolute host path is required — the OpenSandbox server creates sandbox -# containers through the host docker daemon, and docker.sock sees the host's view. -# HOST_REPO_PATH is injected by docker-compose (the .env variable of the same name); -# when missing we fall back to the in-container path derived from Path(__file__) — -# in dev the backend already bind-mounts `${HOST_REPO_PATH}:${HOST_REPO_PATH}`, so -# paths inside and outside the container are identical and the derived result is -# also a valid host path. -_BUILTIN_SKILLS_HOST_PATH: Optional[str] = None - - -def _resolve_builtin_skills_host_path() -> Optional[str]: - """Compute the absolute host path of the built-in skills dir, for sandbox volumes.""" - global _BUILTIN_SKILLS_HOST_PATH - if _BUILTIN_SKILLS_HOST_PATH is not None: - return _BUILTIN_SKILLS_HOST_PATH or None # "" → None - host_repo = os.environ.get("HOST_REPO_PATH", "").strip() - if host_repo: - candidate = f"{host_repo.rstrip('/')}/src/backend/extensions/skills" - else: - # In-container derived path: this file lives in src/backend/core/sandbox/providers/opensandbox/, - # walk back up to src/backend then down into extensions/skills. In dev the backend - # bind-mounts the whole repo via HOST_REPO_PATH, so the absolute path inside - # the container equals the host path. - candidate = str(SKILLS_DIR) - if not Path(candidate).is_dir(): - logger.warning( - "[opensandbox] built-in skills dir not found at %s; skill bind-mount disabled", - candidate, - ) - _BUILTIN_SKILLS_HOST_PATH = "" - return None - _BUILTIN_SKILLS_HOST_PATH = candidate - return candidate - # Metadata tags used for adoption / cross-process pool reuse. Every sandbox created by # this app carries them; after a backend restart _adopt_existing_sandboxes uses this tag @@ -68,7 +24,7 @@ def _resolve_builtin_skills_host_path() -> Optional[str]: _METADATA_OWNER_KEY = "hugagent-owner" _METADATA_OWNER_VALUE = "backend" _METADATA_POOL_KIND_KEY = "hugagent-pool-kind" -# Plan F: jupyter sandboxes are bucketed by user_id. Every jupyter sandbox created by this +# Persistent sandboxes are bucketed by user_id. Every jupyter sandbox created by this # service carries this tag; after a backend restart _adopt_existing_sandboxes uses it to # reclaim orphan sandboxes into the correct user's pool — cross-user reuse would let one # user see another's myspace bind contents, a security hole that must be prevented. @@ -110,46 +66,23 @@ def _make_skills_volumes(user_id: Optional[str] = None) -> list: whoever asks next and must never carry someone's private files. Read-only, and the bind is live, so newly imported skills appear at once. - Without ``HOST_STORAGE_PATH`` fall back to the built-in source tree alone and - warn; on failure return an empty list — the sandbox is created as usual, just - without /workspace/skills/, and skill references hit a plain not-found. + HOST_STORAGE_PATH must identify the same storage root on the Docker host. """ - try: - from opensandbox.models.sandboxes import Host, Volume - except ImportError: # older SDKs lack these two — can only continue without mounting - return [] + from opensandbox.models.sandboxes import Host, Volume + from core.sandbox.errors import SandboxError - host_storage = (settings.sandbox.opensandbox_host_storage_path or "").strip().rstrip("/") + host_storage = settings.sandbox.opensandbox_host_storage_path.strip().rstrip("/") if not host_storage: - # No host storage → no unified dir to bind; mount the built-in source tree alone. - legacy = _resolve_builtin_skills_host_path() - logger.warning( - "[opensandbox] HOST_STORAGE_PATH 未配置；/workspace/skills 仅挂内置源码树，" - "DB 导入技能在沙盒内不可见" - ) - if not legacy: - return [] - return [ - Volume( - name="skills", - host=Host(path=legacy), - mount_path="/workspace/skills", - read_only=True, - ) - ] + raise SandboxError("OpenSandbox skill mounts require HOST_STORAGE_PATH") # get_sandbox_skills_dir() / get_user_skills_root() return the container-side # dirs (the same storage volume as the host bind source) and guarantee they # exist — OpenSandbox fails to bind a nonexistent source. Only their basenames # are used to compose host paths, so the names are not written twice. - from core.agent_skills.config import ( - SHARED_LINK_NAME, - get_sandbox_skills_dir, - get_user_skills_root, - ) + from core.capabilities.skills.config import SHARED_LINK_NAME, get_sandbox_skills_dir, get_user_skills_root shared_host = f"{host_storage}/{get_sandbox_skills_dir().name}" - from core.agent_skills.publication import prepare_skill_view + from core.capabilities.skills.publication import prepare_skill_view view = prepare_skill_view(safe_user_id(user_id) or None) if view is None: diff --git a/src/backend/core/sandbox/providers/script_runner/provider.py b/src/backend/core/sandbox/providers/script_runner/provider.py index db759f1a..0b317a17 100644 --- a/src/backend/core/sandbox/providers/script_runner/provider.py +++ b/src/backend/core/sandbox/providers/script_runner/provider.py @@ -7,6 +7,7 @@ import logging import re import time +from dataclasses import dataclass, field from pathlib import Path from typing import Optional @@ -25,7 +26,7 @@ SandboxAdminNotSupported, SandboxInfo, ) -from core.sandbox.providers.script_runner.auth import auth_headers +from core.infra.local_runtime.runner_auth import auth_headers from core.sandbox.workspace.streams import stream_to_file logger = logging.getLogger(__name__) @@ -61,11 +62,11 @@ def _base_url(self) -> str: async def _execution_body(self, req: ProcessRequest) -> dict: capability_view_key = None - from core.capabilities.paths import capabilities_enabled + from core.harness.registry.run_capabilities import has_run_capabilities - if capabilities_enabled() and req.capability_run_id: - from core.capabilities import runtime - from core.capabilities.errors import IntegrityFailed + if has_run_capabilities() and req.capability_run_id: + from core.harness.registry import capability_runtime as runtime + from core.harness.capability_errors import IntegrityFailed pinned_view = await asyncio.to_thread( runtime.view_for_execution, @@ -163,8 +164,8 @@ async def _process_request(self, route: str, body: dict, wait_ms: int) -> dict: payload = response.json() capability = payload.pop("_capability", None) if capability and capability.get("run_id"): - from core.capabilities import runtime - from core.capabilities.errors import IntegrityFailed + from core.harness.registry import capability_runtime as runtime + from core.harness.capability_errors import IntegrityFailed try: prepared = await asyncio.to_thread( @@ -372,7 +373,6 @@ def admin_pool_stats(self) -> dict: raise SandboxAdminNotSupported("script_runner 无连接池") -_SKILL_VIEW_SYNCED: dict[str, tuple[float, int]] = {} _SKILL_VIEW_TTL_S = 60.0 @@ -386,27 +386,39 @@ def _refresh_skill_view(user_id: Optional[str]) -> None: an immediate refresh regardless of the TTL. Failure is never worth failing an execution over. """ - from core.capabilities.paths import capabilities_enabled + from core.harness.registry.run_capabilities import has_run_capabilities - if not capabilities_enabled(): - from core.agent_skills.publication import prepare_skill_view + if not has_run_capabilities(): + from core.capabilities.skills.publication import prepare_skill_view prepare_skill_view(user_id) return uid = (user_id or "").strip() if not uid: return - from core.capabilities.skills import view_generation + from core.harness.registry.local_publication import local_publication now = time.monotonic() - gen = view_generation() - last_ts, last_gen = _SKILL_VIEW_SYNCED.get(uid, (0.0, -1)) + gen = local_publication().view_generation() + last_ts, last_gen = process_state().SKILL_VIEW_SYNCED.get(uid, (0.0, -1)) if gen == last_gen and now - last_ts < _SKILL_VIEW_TTL_S: return - _SKILL_VIEW_SYNCED[uid] = (now, gen) + process_state().SKILL_VIEW_SYNCED[uid] = (now, gen) try: - from core.agent_skills.config import sync_user_skill_view + from core.capabilities.skills.config import sync_user_skill_view sync_user_skill_view(uid) except Exception as exc: # noqa: BLE001 logger.warning("[script_runner] 技能视图刷新失败 user=%s: %s", uid, exc) + + +@dataclass +class ProviderState: + SKILL_VIEW_SYNCED: dict[str, tuple[float, int]] = field(default_factory=lambda: {}) + + +_STATE = ProviderState() + + +def process_state() -> ProviderState: + return _STATE diff --git a/src/backend/core/sandbox/runner/native_group.py b/src/backend/core/sandbox/runner/native_group.py index 7150477d..5f055d04 100644 --- a/src/backend/core/sandbox/runner/native_group.py +++ b/src/backend/core/sandbox/runner/native_group.py @@ -1,4 +1,5 @@ """Linux owner process groups retain their unreaped leader until the whole group stops.""" +from __future__ import annotations import asyncio import os @@ -7,6 +8,7 @@ import subprocess import sys import threading +from dataclasses import dataclass, field from pathlib import Path @@ -170,13 +172,8 @@ def close(self): self._namespace_fd = None -_namespace_lock = threading.Lock() -_namespace_prefix = None - - def lifetime_command(cmd, trusted_prefix=()): """Preserve trusted existing confinement; otherwise add only PID lifetime isolation.""" - global _namespace_prefix if ( trusted_prefix and Path(trusted_prefix[0]).name == "bwrap" @@ -184,8 +181,8 @@ def lifetime_command(cmd, trusted_prefix=()): and "--die-with-parent" in trusted_prefix ): return cmd, True - with _namespace_lock: - if _namespace_prefix is None: + with process_state().namespace_lock: + if process_state().namespace_prefix is None: executable = shutil.which("bwrap") prefix = ( [ @@ -216,6 +213,19 @@ def lifetime_command(cmd, trusted_prefix=()): prefix = [] except (OSError, subprocess.TimeoutExpired): prefix = [] - _namespace_prefix = prefix - prefix = list(_namespace_prefix) + process_state().namespace_prefix = prefix + prefix = list(process_state().namespace_prefix) return [*prefix, *cmd], bool(prefix) + + +@dataclass +class NativeGroupState: + namespace_lock: object = field(default_factory=lambda: threading.Lock()) + namespace_prefix: object = None + + +_STATE = NativeGroupState() + + +def process_state() -> NativeGroupState: + return _STATE diff --git a/src/backend/core/sandbox/runner/server.py b/src/backend/core/sandbox/runner/server.py index 9d591c4f..398b638f 100644 --- a/src/backend/core/sandbox/runner/server.py +++ b/src/backend/core/sandbox/runner/server.py @@ -14,7 +14,6 @@ import os import shutil import signal -import subprocess import sys import tempfile from pathlib import Path diff --git a/src/backend/core/sandbox/sessions/registry.py b/src/backend/core/sandbox/sessions/registry.py index 0bdb12bf..53eb0e17 100644 --- a/src/backend/core/sandbox/sessions/registry.py +++ b/src/backend/core/sandbox/sessions/registry.py @@ -127,7 +127,9 @@ async def renew_lease() -> None: while True: await asyncio.sleep(min(_LOCK_RENEW_INTERVAL_S, ttl / 3)) try: - retained = await state.hold(key, token, ttl=ttl) + retained = await asyncio.wait_for( + state.compare_exchange(key, token, token, ttl=ttl), timeout=ttl / 3 + ) except Exception: retained = False if not retained: @@ -312,7 +314,9 @@ async def renew_destruction(): while True: await asyncio.sleep(min(_LOCK_RENEW_INTERVAL_S, ttl / 3)) try: - retained = await state.hold(key, token, ttl=ttl) + retained = await asyncio.wait_for( + state.compare_exchange(key, token, token, ttl=ttl), timeout=ttl / 3 + ) except Exception: retained = False if not retained: @@ -378,3 +382,16 @@ async def creation_pending(token: str) -> bool: return bool(await get_ephemeral_state().get( _LOCK_KEY.format(session_id=f"creating:{token}") )) + + +async def bindings(): + """Snapshot shared bindings without extending activity or their lifetime.""" + prefix = _KEY.split("{")[0] + state = get_ephemeral_state() + entries = [] + for key in await state.keys(prefix): + session_id = key[len(prefix):] + entry = await _read(session_id) + if entry is not None: + entries.append((session_id, entry[0], entry[1])) + return entries diff --git a/src/backend/core/sandbox/sessions/router.py b/src/backend/core/sandbox/sessions/router.py index 2946b775..8de0dbbb 100644 --- a/src/backend/core/sandbox/sessions/router.py +++ b/src/backend/core/sandbox/sessions/router.py @@ -11,22 +11,17 @@ def __getattr__(self, name): return getattr(self._ordinary, name) def _for(self, session_id): - from core.extensions import sandbox_route + from core.harness.registry import sandbox_route module_provider = sandbox_route(str(session_id or "")) if module_provider is not None: return module_provider() if str(session_id or "").startswith("eval_"): - from core.config.settings import settings - from core.sandbox.providers.evaluation.provider import get_evaluation_provider - - if settings.sandbox.provider != "opensandbox": - raise SandboxError("Evaluation requires OpenSandbox; fallback is forbidden") - return get_evaluation_provider() + raise SandboxError("Evaluation extension is not registered; fallback is forbidden") return self._ordinary async def _authorized_for(self, session_id, user_id): - from core.extensions import sandbox_route + from core.harness.registry import sandbox_route provider = self._for(session_id) if ( @@ -45,7 +40,7 @@ async def _ready(self, provider, session_id, user_id): if str(session_id or "").startswith("eval_"): return if provider.name != "opensandbox" and user_id and session_id: - from core.plugins.resources.prewarm import schedule + from core.capabilities.plugins.resources.prewarm import schedule sandbox_id = await provider.current_sandbox_id(session_id) schedule(provider, session_id, user_id, sandbox_id) @@ -110,7 +105,7 @@ async def close_session(self, session_id): provider = self._for(session_id) if not session_id or str(session_id).startswith("eval_") or provider.name == "opensandbox": return await provider.close_session(session_id) - from core.plugins.resources.prewarm import closing, stop + from core.capabilities.plugins.resources.prewarm import closing, stop async with closing(session_id): await stop(session_id) diff --git a/src/backend/core/sandbox/workspace/container.py b/src/backend/core/sandbox/workspace/container.py index 986bb047..b0a83550 100644 --- a/src/backend/core/sandbox/workspace/container.py +++ b/src/backend/core/sandbox/workspace/container.py @@ -5,7 +5,7 @@ import posixpath from typing import Any, Optional -from core.agent_skills.loader import get_skill_loader +from core.capabilities.skills.loader_factory import get_skill_loader def _is_traversal_or_bad(path: str) -> Optional[str]: @@ -23,7 +23,7 @@ def validate_path(path: str, root: str) -> str | None: return error if ".." in path.replace(chr(92), "/").split("/"): return f"path 不允许包含 ..: {path}" - from core.services.edition_workspace import resolve_alias + from core.files.edition import resolve_alias if resolve_alias(path) is not None or path == "/myspace" or path.startswith("/myspace/"): return None @@ -44,7 +44,7 @@ def workspace_directory(root: str, session_id: str | None) -> str: def resolve_path(path: str, root: str, session_id: str | None, user_id=None) -> str: - from core.services.edition_workspace import resolve_alias + from core.files.edition import resolve_alias resolved = resolve_alias(path) if resolved is not None: @@ -98,7 +98,7 @@ def resolve_skill_file(file_path: str, loader: Any = None) -> str | None: return candidate return None - from core.agent_skills.config import get_sandbox_skills_dir + from core.capabilities.skills.config import get_sandbox_skills_dir cache_root = str(get_sandbox_skills_dir()) for skill_id, rel_path in reversed(candidates): @@ -113,8 +113,9 @@ def resolve_skill_file(file_path: str, loader: Any = None) -> str | None: candidate = os.path.join(skill_dir, rel_path) if os.path.exists(candidate): return candidate - except Exception: - pass + except Exception as fallback_error: + from core.infra.failures import record_best_effort_failure + record_best_effort_failure('core.sandbox.workspace.container:resolve_skill_file', fallback_error) return None @@ -156,7 +157,7 @@ def is_myspace_physical(physical_path: str, user_id: Optional[str], root: str) - def bash_workspace_instructions(root: str, session_id: str | None) -> str: _WS = root - from core.services.edition_workspace import workspace_instructions + from core.files.edition import workspace_instructions return ( workspace_instructions() + "\n" diff --git a/src/backend/core/sandbox/workspace/credentials.py b/src/backend/core/sandbox/workspace/credentials.py index 4cc87d5f..541f8515 100644 --- a/src/backend/core/sandbox/workspace/credentials.py +++ b/src/backend/core/sandbox/workspace/credentials.py @@ -44,166 +44,26 @@ def purge_credential_dir(root: Path) -> None: logger.warning("[sandbox] purge credential file failed %s: %s", fp, exc) -def dws_cache_dir(user_id: str) -> Path: - """Backend-local dws (DingTalk CLI) credential persistence directory (one per user). - - Contains two subdirectories, bind-mounted into the sandbox DingTalk CLI's default - ``$HOME`` paths: - - ``dws/`` → sandbox ``/home/ubuntu/.dws`` (config + market cache + app.json) - - ``share/`` → sandbox ``/home/ubuntu/.local/share/dws-cli`` (encrypted keychain: - ``dek`` + ``auth-token.enc``) - - Empirically: dws's encrypted keychain and the ``.dws`` cache only respect ``$HOME``; - ``DWS_CONFIG_DIR`` only relocates identity.json/logs, and ``XDG_DATA_HOME`` is - ignored — so login must run with a per-user ``$HOME`` and these two home subpaths - must be bind-mounted, rather than relying on env redirection. The sandbox runs as - ubuntu (UID 1000) and the backend is also 1000, so ownership matches - (see [[project_dingtalk_dws_integration]]). - """ - return settings.storage.root / "dws_cache" / user_id - - -def dws_home_dir(user_id: str) -> Path: - """Per-user ``$HOME`` for dws login (the unified root for backend subprocesses + - opensandbox bind sources). - - The backend runs ``dws auth login`` with this as ``HOME`` → credentials land in - ``{home}/.dws`` and ``{home}/.local/share/dws-cli``; opensandbox bind-mounts these - two subpaths into the same home paths inside the sandbox, so **that user's** session - sandboxes (bucketed per user) see their own credentials. The root of multi-tenant - isolation: each user gets an independent home directory, never shared. - """ - return dws_cache_dir(user_id) / "home" - - -def lark_cache_dir(user_id: str) -> Path: - """Backend-local lark-cli (Feishu/Lark CLI) credential persistence directory (one per user). - - Contains two subdirectories, bind-mounted into the sandbox lark-cli's default - ``$HOME`` paths: - - ``home/.lark-cli/`` → sandbox ``/home/ubuntu/.lark-cli`` (config.json - app config + cache + logs) - - ``home/.local/share/lark-cli/`` → sandbox ``/home/ubuntu/.local/share/lark-cli`` - (file-based encrypted store: ``master.key`` + - ``appsecret_*.enc`` + per-user ``cli_*_ou_*.enc``) - - Empirically: on Linux, lark-cli does **not** use the OS keychain but a file-based - encrypted store — it only respects these two directories under ``$HOME``; - ``master.key`` is a local DEK, so decryption works wherever the files travel. - Isomorphic to dws's "Linux file-based DEK" scheme, so the per-user bind-mount - persistence is copied verbatim (see [[dws_cache_dir]] and - internal design docs). - The sandbox runs as ubuntu (UID 1000) and the backend is also 1000, so ownership matches. - """ - return settings.storage.root / "lark_cache" / user_id - - -def lark_home_dir(user_id: str) -> Path: - """Per-user ``$HOME`` for lark-cli login (the unified root for the backend - config init / auth login subprocesses + opensandbox bind sources). - - The backend runs ``lark-cli config init`` (writes app config) and ``auth login`` - (device flow, writes user_access_token) with this as ``HOME`` → credentials land in - ``{home}/.lark-cli`` and ``{home}/.local/share/lark-cli``; opensandbox bind-mounts - these two subpaths into the same home paths inside the sandbox, so **that user's** - session sandboxes see their own credentials. The root of multi-tenant isolation: - each user gets an independent home directory, never shared. - """ - return lark_cache_dir(user_id) / "home" -def lark_app_home_dir() -> Path: - """Org-wide shared Feishu **app** config HOME (the admin's one-click init - ``config init --new`` lands here). - Contains ``~/.lark-cli/config.json`` + ``~/.local/share/lark-cli/{master.key, - appsecret_*.enc}`` (the app foundation — contains **no user tokens whatsoever**). - These three items are copied into every user's ``lark_home_dir`` as the app - foundation for their ``auth login`` — the whole org shares one app, and the admin - configures it only once. - """ - return settings.storage.root / "lark_cache" / "__app__" / "home" -def email_cache_dir(user_id: str) -> Path: - """Backend-local email (himalaya CLI) credential persistence directory (one per user). - Contains one subdirectory, bind-mounted into the sandbox himalaya's default - config path: - - ``home/.config/himalaya/`` → sandbox ``/home/ubuntu/.config/himalaya`` - (``config.toml``: account blocks + IMAP/SMTP servers + auth-code raw lines, file mode 0600) - Isomorphic to lark/dws's "Linux file-based credentials": himalaya config is plain - files that only respect the config path, so the per-user bind-mount persistence is - copied verbatim. No OS keychain, no OAuth token refresh — auth codes stay valid - long-term (governed by the mail provider). - See internal design docs. - """ - return settings.storage.root / "email_cache" / user_id -def email_home_dir(user_id: str) -> Path: - """Per-user ``$HOME`` for himalaya login (the unified root for the backend - verification subprocess + opensandbox bind sources). - The backend runs ``himalaya folder list`` with this as ``HOME`` as a connectivity - check; the config lands at ``{home}/.config/himalaya/config.toml``; opensandbox - bind-mounts this subdirectory into the same home path inside the sandbox. The root - of multi-tenant isolation: each user gets an independent home directory, never shared. - """ - return email_cache_dir(user_id) / "home" -def email_himalaya_config(user_id: str) -> Path: - """Backend absolute path of this user's himalaya ``config.toml`` (under email_home_dir).""" - return email_home_dir(user_id) / ".config" / "himalaya" / "config.toml" -def yida_cache_dir(user_id: str) -> Path: - """Backend-local openyida (Yida CLI) login-state persistence directory (one per user). - Contains one subdirectory, bind-mounted into the sandbox's fixed Yida working directory: - - ``workspace/`` → sandbox ``/home/ubuntu/yida-workspace`` (openyida's projectRoot: - ``.cache/cookies-*.json`` login cookies + ``.cache/openyida-envs.json`` env config + - project artifacts such as ``prd/`` and schema cache) - Difference from dws/lark: openyida's login state does **not** respect $HOME; it lands - under the "project root"'s ``.cache/`` (projectRoot = the cwd at command execution - time, see the cwd fallback of findProjectRoot in openyida lib/core/utils.js). So the - persistence scheme is not binding home subpaths but pinning each user a fixed - in-sandbox working directory and binding it wholesale: the skill forces all openyida - commands to run inside that directory, so cookies survive across sessions with the - volume. Login is completed by QR scan inside the conversation - (``openyida login --agent-qr``); the backend runs no openyida subprocess. The sandbox - runs as ubuntu (UID 1000) and the backend is also 1000, so ownership matches. - """ - return settings.storage.root / "yida_cache" / user_id -def yida_workspace_dir(user_id: str) -> Path: - """Backend absolute path of this user's fixed Yida working directory (opensandbox - bind source / cube inject-and-return source). - - The sandbox mount point is fixed at ``/home/ubuntu/yida-workspace``; the yida skill - requires all openyida commands to ``cd`` into that directory, so the ``.cache/`` - login state and project artifacts all land on the persistent volume. - """ - return yida_cache_dir(user_id) / "workspace" -def yida_shared_workspace_dir() -> Path: - """Yida working directory for the script-runner shared sandbox (**one per deployment**, - not per-user). - - script-runner is a shared container with no user concept (the provider ignores - user_id; rootfs is read_only) and cannot do per-user credential isolation — compose - mounts this directory at the container's ``/home/ubuntu/yida-workspace``, so the Yida - login state is shared deployment-wide. Only suitable for single-user/dev environments; - multi-user production should use opensandbox (per-user bind-mount) or cube (per-user - inject + return). The backend pre-creates it with 0777 at startup (runner uid 1001 ≠ - backend uid 1000; cross-uid writes need wide-open permissions). - """ - return settings.storage.root / "yida_cache" / "__shared__" / "workspace" # openyida (Yida CLI) login-state persistence volume: unlike the "$HOME subpath" pattern of @@ -212,4 +72,3 @@ def yida_shared_workspace_dir() -> Path: # So each user gets a fixed working directory bind-mounted into the sandbox, and the yida # skill forces all openyida commands to cd into that directory before running (see the # _common.yida_cache_dir comment). -_YIDA_WORKSPACE_MOUNT = "/home/ubuntu/yida-workspace" diff --git a/src/backend/core/sandbox/workspace/desktop.py b/src/backend/core/sandbox/workspace/desktop.py index 6f599435..5ab3991a 100644 --- a/src/backend/core/sandbox/workspace/desktop.py +++ b/src/backend/core/sandbox/workspace/desktop.py @@ -8,7 +8,7 @@ def workspace_directory(root: str, session_id: str | None, *, scope=None, validate=True) -> str: if scope is not None and scope.is_local: - from core.services.local_project_workspace import project_directory + from core.projects.local_workspace import project_directory return project_directory(scope, validate=validate) return session_root(root, session_id) if session_id else root diff --git a/src/backend/core/llm/tools/_paths.py b/src/backend/core/sandbox/workspace/resolution.py similarity index 88% rename from src/backend/core/llm/tools/_paths.py rename to src/backend/core/sandbox/workspace/resolution.py index 9d261501..58ff4c3d 100644 --- a/src/backend/core/llm/tools/_paths.py +++ b/src/backend/core/sandbox/workspace/resolution.py @@ -8,12 +8,10 @@ def path_rules(): - from core.llm.evaluation_runtime import CURRENT_EVALUATION_SCOPE - - if CURRENT_EVALUATION_SCOPE.get() is not None: - from core.llm import evaluation_paths - - return evaluation_paths + from core.harness.registry.evaluation import current_path_rules + restricted_paths = current_path_rules() + if restricted_paths is not None: + return restricted_paths from core.config.local_mode import local_mode_enabled if local_mode_enabled(): @@ -46,7 +44,7 @@ def to_physical_path( if rules is desktop_paths: return rules.resolve_path(path, WORKSPACE_ROOT, session_id, user_id, scope=scope) - from core.services.edition_workspace import resolve_project_path + from core.files.edition import resolve_project_path resolved = resolve_project_path(path, user_id, WORKSPACE_ROOT, scope) if resolved is not None: @@ -82,7 +80,7 @@ def space_metadata(physical_path: str, user_id: Optional[str]) -> dict: if is_myspace_physical(physical_path, user_id): kind = "personal" else: - from core.services.edition_workspace import is_organization_path + from core.files.edition import is_organization_path kind = "team" if is_organization_path(None, user_id, physical_path) else "temporary" return {"persistent": kind != "temporary", "space_type": kind} diff --git a/src/backend/core/services/__init__.py b/src/backend/core/services/__init__.py deleted file mode 100644 index bd408101..00000000 --- a/src/backend/core/services/__init__.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Community-edition service exports.""" - -from core.services.api_key_service import ApiKeyService -from core.services.artifact_service import ArtifactService -from core.services.catalog_service import CatalogService -from core.services.chat_service import ChatService -from core.services.kb_service import KBService -from core.services.user_agent_service import UserAgentService -from core.services.user_service import UserService - -__all__ = [ - "ApiKeyService", - "ArtifactService", - "CatalogService", - "ChatService", - "KBService", - "UserAgentService", - "UserService", -] diff --git a/src/backend/core/services/agent_market_categories.py b/src/backend/core/services/agent_market_categories.py deleted file mode 100644 index 301bb298..00000000 --- a/src/backend/core/services/agent_market_categories.py +++ /dev/null @@ -1,77 +0,0 @@ -"""Sub-agent marketplace categories (single source of truth) + Cherry Studio group → category mapping. - -The sub-agent marketplace categories are **independent of the skill marketplace's 8 -categories** (role-positioning semantics fit agents better). The preset list, community -listing applications, and admin review category changes may only take these 9 values; -the frontend mirror is ``AGENT_MARKETPLACE_CATEGORIES`` in ``utils/constants.ts``. -""" - -from __future__ import annotations - -from typing import List - -from core.infra.exceptions import BadRequestError - -# Sub-agent marketplace fixed categories (stable order, frontend mirror must match) -AGENT_MARKETPLACE_CATEGORIES: List[str] = [ - "通用助手", - "职场办公", - "商业分析", - "数据分析", - "研发编程", - "翻译写作", - "创意设计", - "政策法务", - "教育科研", -] - -DEFAULT_AGENT_CATEGORY = "通用助手" - -# Cherry Studio's group tags (dozens of them) → normalized mapping to this project's 9 categories. -# Only used by scripts/import_cherry_agents.py during import; an unmatched group falls to 「通用助手」. -CHERRY_GROUP_TO_CATEGORY = { - # General - "通用": "通用助手", - "工具": "通用助手", - "百科": "通用助手", - # Workplace / office - "职业": "职场办公", - "办公": "职场办公", - "管理": "职场办公", - "咨询": "职场办公", - # Business analysis - "商业": "商业分析", - "营销": "商业分析", - "金融": "商业分析", - "分析": "商业分析", - "点评": "商业分析", - # Data analysis (Cherry has no dedicated group; decided secondarily by keywords in the import script) - # R&D / programming - "编程": "研发编程", - "科学": "研发编程", - # Translation / writing - "翻译": "翻译写作", - "写作": "翻译写作", - "文案": "翻译写作", - "语言": "翻译写作", - "学术": "翻译写作", - # Creative design - "创意": "创意设计", - "设计": "创意设计", - "艺术": "创意设计", - "音乐": "创意设计", - # Policy / legal - "法律": "政策法务", - # Education / research - "教育": "教育科研", -} - - -def validate_category(category: str) -> str: - """Validate and return a legal category; anything not in the fixed set 400s directly.""" - category = (category or "").strip() - if category not in AGENT_MARKETPLACE_CATEGORIES: - raise BadRequestError( - message=f"请从固定分类中选择：{'、'.join(AGENT_MARKETPLACE_CATEGORIES)}" - ) - return category diff --git a/src/backend/core/services/agent_market_service.py b/src/backend/core/services/agent_market_service.py deleted file mode 100644 index b7ae7260..00000000 --- a/src/backend/core/services/agent_market_service.py +++ /dev/null @@ -1,940 +0,0 @@ -"""Sub-Agent Marketplace service. - -The sub-agent marketplace replicates the skill marketplace's "user submits → admin reviews → -everyone can install" loop, but the marketplace items are **sub-agents** (role prompt + -capability bindings) rather than skill files: - -- Preset sub-agents: shipped with the repo, stored at ``extensions/agents/marketplace//agent.json`` - (pure data, no attachments). The initial content was curated and adapted from Cherry Studio. -- Community listings (``agent_market_submissions``): users **submit** their self-built sub-agents - for listing; once an admin approves, they appear in the marketplace with ``source=community`` - and anyone can install. A content snapshot of the source ``UserAgent`` is taken at submit time. - -Install semantics = **clone as a private sub-agent**: create a new ``UserAgent`` under the -installer (``owner_type=user``; admin installs get ``owner_type=admin``, global), copying the -prompt/welcome message/suggested questions/model config, and applying "install-on-demand" to -capability bindings — MCP/default skills bind directly, marketplace skills/plugins are installed -as the installer's private copies as needed and then bound, and unresolvable items are dropped. -The clone records ``source_market_slug`` to mark it "installed". -""" - -from __future__ import annotations - -import json -import logging -import re -import uuid -from datetime import datetime -from typing import Any, Dict, List, Optional - -from core.db.models import AgentMarketSubmission, UserAgent -from core.infra.exceptions import BadRequestError, ResourceNotFoundError -from core.infra.time import utc_now -from core.services import marketplace_listing as ml -from core.services.agent_market_categories import ( - AGENT_MARKETPLACE_CATEGORIES, - DEFAULT_AGENT_CATEGORY, - validate_category, -) -from extensions.paths import AGENTS_DIR -from fastapi import HTTPException -from sqlalchemy.orm import Session -from sqlalchemy.orm.attributes import flag_modified - -logger = logging.getLogger(__name__) - -# Preset sub-agent root directory: ``src/backend/extensions/agents/marketplace//agent.json``. -AGENT_MARKET_DIR = AGENTS_DIR / "marketplace" -AGENT_JSON = "agent.json" - -# Sentinel owner for admin direct uploads to the marketplace (distinguishes "admin upload" from real users' community submissions). -ADMIN_UPLOAD_OWNER = "__admin_upload__" - -# The clone's description field matches the user-side creation form's constraint (≤20 chars); the full copy lives in system_prompt. -_MAX_DESC = 20 - - -# ── Preset bundle reading ──────────────────────────────────────────────────── - - -def _read_bundle(slug: str) -> Optional[Dict[str, Any]]: - """Read and validate a single preset sub-agent's ``agent.json`` (fault-tolerant; returns None if broken).""" - if not slug or "/" in slug or ".." in slug: - return None - path = AGENT_MARKET_DIR / slug / AGENT_JSON - if not path.is_file(): - return None - try: - b = json.loads(path.read_text(encoding="utf-8")) - except Exception as exc: # noqa: BLE001 - logger.warning("agent market bundle broken: %s (%s)", slug, exc) - return None - if not isinstance(b, dict) or not b.get("name"): - return None - b.setdefault("slug", slug) - return b - - -def _normalize_binding_lists(raw: Optional[Dict[str, Any]]) -> Dict[str, List[str]]: - """Normalize a bindings snapshot into the four id lists (missing ones become empty).""" - b = raw or {} - return { - "skill_ids": list(b.get("skill_ids") or []), - "mcp_server_ids": list(b.get("mcp_server_ids") or []), - "plugin_ids": list(b.get("plugin_ids") or []), - "kb_ids": list(b.get("kb_ids") or []), - } - - -def _bindings_of(d: Dict[str, Any]) -> Dict[str, List[str]]: - """Get the four binding id lists from a preset bundle's ``bindings``.""" - return _normalize_binding_lists(d.get("bindings")) - - -def _binding_counts(bindings: Dict[str, List[str]]) -> Dict[str, int]: - return { - "skill_count": len(bindings.get("skill_ids") or []), - "mcp_count": len(bindings.get("mcp_server_ids") or []), - "plugin_count": len(bindings.get("plugin_ids") or []), - "kb_count": len(bindings.get("kb_ids") or []), - } - - -def _make_market_meta( - *, - slug: str, - name: str, - avatar: str, - summary: str, - description: str, - category: str, - tags: Any, - version: str, - author: str, - source: str, - featured: bool, - deletable: bool, - bindings: Dict[str, List[str]], -) -> Dict[str, Any]: - """Preset bundle / community listing → isomorphic public list metadata (without the system_prompt body).""" - return { - "slug": slug, - "name": name or slug, - "avatar": avatar or "", - "summary": summary or "", - "description": description or "", - "category": category or DEFAULT_AGENT_CATEGORY, - "tags": list(tags or []), - "version": version or "1.0.0", - "author": author, - "source": source, - "featured": featured, - "deletable": deletable, - **_binding_counts(bindings), - } - - -def _bundle_public_meta(b: Dict[str, Any]) -> Dict[str, Any]: - # Presets (filesystem) ship with the repo; admins cannot delete them online (deletable=False). - return _make_market_meta( - slug=b.get("slug"), - name=b.get("name"), - avatar=b.get("avatar"), - summary=b.get("summary"), - description=b.get("description"), - category=b.get("category"), - tags=b.get("tags"), - version=b.get("version"), - author=b.get("author") or "内置", - source="builtin", - featured=bool(b.get("featured")), - deletable=False, - bindings=_bindings_of(b), - ) - - -def _submission_public_meta(sub: AgentMarketSubmission) -> Dict[str, Any]: - # DB listing records (admin upload / user community submission) → admins can delete online (deletable=True). - return _make_market_meta( - slug=sub.slug, - name=sub.name, - avatar=sub.avatar, - summary=sub.summary, - description=sub.description, - category=sub.category, - tags=sub.tags, - version=sub.version, - author=sub.submitter_name or sub.owner_user_id, - source="community", - featured=False, - deletable=True, - bindings=_normalize_binding_lists(sub.bindings_snapshot), - ) - - -def _approved_submissions(db: Session) -> List[AgentMarketSubmission]: - return ( - db.query(AgentMarketSubmission) - .filter(AgentMarketSubmission.status == "approved") - .order_by(AgentMarketSubmission.created_at.desc()) - .all() - ) - - -def _get_approved_submission(db: Session, slug: str) -> Optional[AgentMarketSubmission]: - return ( - db.query(AgentMarketSubmission) - .filter( - AgentMarketSubmission.slug == slug, - AgentMarketSubmission.status == "approved", - ) - .first() - ) - - -# ── List / categories / detail ─────────────────────────────────────────────── - - -def list_marketplace_agents( - db: Session, *, include_disabled: bool = False, viewer_user_id: Optional[str] = None -) -> List[Dict[str, Any]]: - """List all marketplace sub-agents: presets (featured first) + community listings, layered with the enable/disable switch and visibility-scope filtering.""" - items: List[Dict[str, Any]] = [] - if AGENT_MARKET_DIR.is_dir(): - for child in sorted(AGENT_MARKET_DIR.iterdir()): - if not child.is_dir(): - continue - b = _read_bundle(child.name) - if b: - items.append(_bundle_public_meta(b)) - items.sort(key=lambda x: (0 if x["featured"] else 1, x["slug"])) - items.extend(_submission_public_meta(s) for s in _approved_submissions(db)) - items = ml.annotate_and_filter( - db, - ml.KIND_AGENT, - items, - id_key="slug", - include_disabled=include_disabled, - viewer_user_id=viewer_user_id, - ) - return items - - -def categories_from_items(items: List[Dict[str, Any]]) -> List[str]: - """Derive categories from an already-fetched marketplace list: the fixed 9 major categories first, legacy leftover categories appended after.""" - seen: List[str] = list(AGENT_MARKETPLACE_CATEGORIES) - for it in items: - if it["category"] not in seen: - seen.append(it["category"]) - return seen - - -def list_categories( - db: Session, *, include_disabled: bool = False, viewer_user_id: Optional[str] = None -) -> List[str]: - """Marketplace category list (standalone entry point; the list endpoint should reuse categories_from_items to avoid a second scan).""" - return categories_from_items( - list_marketplace_agents( - db, include_disabled=include_disabled, viewer_user_id=viewer_user_id - ) - ) - - -def get_agent_detail(db: Session, slug: str) -> Dict[str, Any]: - """Detail of a single marketplace sub-agent: metadata + prompt body + capability bindings (for the detail preview). - - The preset directory takes priority; falls back to the community listing (approved submission) if not found. - """ - b = _read_bundle(slug) - if b: - data = _bundle_public_meta(b) - data["system_prompt"] = b.get("system_prompt") or "" - data["welcome_message"] = b.get("welcome_message") or "" - data["suggested_questions"] = list(b.get("suggested_questions") or []) - data["bindings"] = _bindings_of(b) - return data - sub = _get_approved_submission(db, slug) - if sub is None: - raise ResourceNotFoundError("marketplace_agent", slug) - data = _submission_public_meta(sub) - data["system_prompt"] = sub.system_prompt or "" - data["welcome_message"] = sub.welcome_message or "" - data["suggested_questions"] = list(sub.suggested_questions or []) - data["bindings"] = _normalize_binding_lists(sub.bindings_snapshot) - return data - - -def _installed_slugs(db: Session, owner_user_id: Optional[str], slugs: List[str]) -> set: - """The subset of these marketplace slugs already installed (cloned) in the given scope (user / global admin).""" - if not slugs: - return set() - q = db.query(UserAgent.source_market_slug).filter(UserAgent.source_market_slug.in_(slugs)) - if owner_user_id is None: - q = q.filter(UserAgent.owner_type == "admin") - else: - q = q.filter(UserAgent.owner_type == "user", UserAgent.user_id == owner_user_id) - return {row[0] for row in q.all()} - - -def annotate_installed( - items: List[Dict[str, Any]], db: Session, owner_user_id: Optional[str] -) -> List[Dict[str, Any]]: - """Add the ``installed`` flag to each marketplace list item (for the given scope).""" - if not items: - return items - installed = _installed_slugs(db, owner_user_id, [it["slug"] for it in items]) - for it in items: - it["installed"] = it["slug"] in installed - return items - - -def is_installed(db: Session, slug: str, owner_user_id: Optional[str]) -> bool: - return bool(_installed_slugs(db, owner_user_id, [slug])) - - -# ── Install (clone + install-on-demand dependencies) ───────────────────────── - - -def _resolve_market_entry(db: Session, slug: str) -> Dict[str, Any]: - """Normalize a preset bundle / community listing into a unified clone-source dict.""" - b = _read_bundle(slug) - if b: - return { - "name": b.get("name") or slug, - "avatar": b.get("avatar") or "", - "description": b.get("description") or "", - "summary": b.get("summary") or "", - "system_prompt": b.get("system_prompt") or "", - "welcome_message": b.get("welcome_message") or "", - "suggested_questions": list(b.get("suggested_questions") or []), - "model_config": dict(b.get("model_config") or {}), - "bindings": _bindings_of(b), - "ontology_tags": [ - str(tag) - for tag in (b.get("ontology_tags") or b.get("tags") or []) - if str(tag).startswith("ontology:") - ], - } - sub = _get_approved_submission(db, slug) - if sub is None: - raise ResourceNotFoundError("marketplace_agent", slug) - return { - "name": sub.name or slug, - "avatar": sub.avatar or "", - "description": sub.description or "", - "summary": sub.summary or "", - "system_prompt": sub.system_prompt or "", - "welcome_message": sub.welcome_message or "", - "suggested_questions": list(sub.suggested_questions or []), - "model_config": dict(sub.model_config_snapshot or {}), - "bindings": _normalize_binding_lists(sub.bindings_snapshot), - "ontology_tags": [str(tag) for tag in (sub.tags or []) if str(tag).startswith("ontology:")], - } - - -def _resolve_bindings( - db: Session, - bindings: Dict[str, List[str]], - owner_user_id: Optional[str], - operator_name: Optional[str], -) -> tuple[Dict[str, List[str]], Dict[str, List[str]]]: - """Resolve each marketplace binding id into the installer's scope; returns (final bindings, report). - - - MCP / default skills / accessible KBs: bind directly if within ``available-resources``; - - marketplace skill slugs: if not in the available list, install as the installer's private skill and bind the returned id; - - plugin slugs: bind if already installed, otherwise install as private and then bind; - - unresolvable items go into ``dropped``; auto-installed skills/plugins requiring credentials go into ``needs_secret``. - """ - from core.config.catalog import get_enabled_ids - from core.plugins import management as plugin_service - from core.services import marketplace_service as mk - from core.services.user_agent_service import UserAgentService - - avail = UserAgentService(db).list_available_resources(owner_user_id=owner_user_id) - avail_skill = {s["id"] for s in avail.get("skills", [])} - # available-resources only lists AdminMcpServer rows; but built-in MCP tools (e.g. - # database_query) are catalog-defined and loaded at runtime by catalog id, - # and don't necessarily have a same-named AdminMcpServer row. Merge the catalog-enabled - # built-in MCP ids into the "resolvable" set, so these legitimate built-in tools aren't - # misjudged as "unresolvable" and dropped (once bound, the runtime connects on demand). - avail_mcp = {s["id"] for s in avail.get("mcp_servers", [])} | set(get_enabled_ids("mcp")) - avail_plugin = {p["id"] for p in avail.get("plugins", [])} - avail_kb = {k["id"] for k in avail.get("kb_spaces", [])} - - final: Dict[str, List[str]] = { - "skill_ids": [], - "mcp_server_ids": [], - "plugin_ids": [], - "kb_ids": [], - } - report: Dict[str, List[str]] = { - "bound": [], - "installed": [], - "dropped": [], - "needs_secret": [], - } - - # MCP tools - for mid in bindings.get("mcp_server_ids", []): - if mid in avail_mcp: - final["mcp_server_ids"].append(mid) - report["bound"].append(f"mcp:{mid}") - else: - report["dropped"].append(f"mcp:{mid}") - - # Skills (default skills bind directly; marketplace skills get auto-installed as private) - for sid in bindings.get("skill_ids", []): - if sid in avail_skill: - final["skill_ids"].append(sid) - report["bound"].append(f"skill:{sid}") - continue - try: - res = mk.install_marketplace_skill(db, sid, owner_user_id=owner_user_id, secrets={}) - final["skill_ids"].append(res["id"]) - report["installed"].append(f"skill:{sid}") - if mk.market_skill_requires_secrets(sid): - report["needs_secret"].append(f"skill:{sid}") - except Exception as exc: # noqa: BLE001 - logger.info("agent clone: skill binding unresolved %s (%s)", sid, exc) - report["dropped"].append(f"skill:{sid}") - - # Plugins (bindings store the slug; bind the install_id if installed, otherwise auto-install) - for pslug in bindings.get("plugin_ids", []): - gid = f"{pslug}@global" - match = ( - gid - if gid in avail_plugin - else next((p for p in avail_plugin if p.startswith(f"{pslug}@")), None) - ) - if match: - final["plugin_ids"].append(match) - report["bound"].append(f"plugin:{pslug}") - continue - try: - res = plugin_service.install_plugin( - db, pslug, owner_user_id=owner_user_id, secrets={}, created_by=operator_name - ) - final["plugin_ids"].append(res["install_id"]) - report["installed"].append(f"plugin:{pslug}") - if res.get("required_secrets") or res.get("requires_secret"): - report["needs_secret"].append(f"plugin:{pslug}") - except Exception as exc: # noqa: BLE001 - logger.info("agent clone: plugin binding unresolved %s (%s)", pslug, exc) - report["dropped"].append(f"plugin:{pslug}") - - # Knowledge bases (keep only those the installer can access) - for kid in bindings.get("kb_ids", []): - if kid in avail_kb: - final["kb_ids"].append(kid) - report["bound"].append(f"kb:{kid}") - else: - report["dropped"].append(f"kb:{kid}") - - return final, report - - -def install_marketplace_agent( - db: Session, - slug: str, - *, - owner_user_id: Optional[str], - operator_name: Optional[str] = None, -) -> Dict[str, Any]: - """Install a marketplace sub-agent as a private clone (admin owner_user_id=None → global admin sub-agent).""" - from core.services.user_agent_service import UserAgentService - - entry = _resolve_market_entry(db, slug) - final_bindings, report = _resolve_bindings(db, entry["bindings"], owner_user_id, operator_name) - - mc = entry.get("model_config") or {} - data: Dict[str, Any] = { - "name": entry["name"], - "avatar": entry.get("avatar") or None, - "description": (entry.get("summary") or entry.get("description") or "")[:_MAX_DESC], - "system_prompt": entry.get("system_prompt") or "", - "welcome_message": entry.get("welcome_message") or "", - "suggested_questions": list(entry.get("suggested_questions") or []), - "skill_ids": final_bindings["skill_ids"], - "mcp_server_ids": final_bindings["mcp_server_ids"], - "plugin_ids": final_bindings["plugin_ids"], - "kb_ids": final_bindings["kb_ids"], - "source_market_slug": slug, - "ontology_tags": list(entry.get("ontology_tags") or []), - } - if mc.get("temperature") is not None: - data["temperature"] = mc["temperature"] - if mc.get("max_tokens") is not None: - data["max_tokens"] = mc["max_tokens"] - if mc.get("max_iters") is not None: - data["max_iters"] = mc["max_iters"] - if mc.get("timeout") is not None: - data["timeout"] = mc["timeout"] - - owner_type = "admin" if owner_user_id is None else "user" - agent = UserAgentService(db).create( - user_id=owner_user_id, - operator_name=operator_name, - owner_type=owner_type, - data=data, - ) - logger.info( - "agent_market_install: slug=%s owner=%s agent=%s bound=%d installed=%d dropped=%d", - slug, - owner_user_id or "global", - agent["agent_id"], - len(report["bound"]), - len(report["installed"]), - len(report["dropped"]), - ) - return { - "agent_id": agent["agent_id"], - "slug": slug, - "owner": "self" if owner_user_id else "global", - "install_report": report, - "message": "子智能体已安装", - } - - -# ── Community listing: user submission + admin review ─────────────────────── - - -def _existing_slugs(db: Session) -> set: - """Marketplace slugs currently taken: preset directory + all submissions (including rejected; slugs are table-wide unique).""" - slugs = set() - if AGENT_MARKET_DIR.is_dir(): - slugs.update(c.name for c in AGENT_MARKET_DIR.iterdir() if c.is_dir()) - slugs.update(r[0] for r in db.query(AgentMarketSubmission.slug).all()) - return slugs - - -def _derive_slug(db: Session, name: str) -> str: - """Derive the marketplace slug from the sub-agent name (ASCII-ized + unique); Chinese names fall back to agent-.""" - base = re.sub(r"[^a-z0-9_-]+", "-", (name or "").lower()).strip("-") - if not base: - base = f"agent-{uuid.uuid4().hex[:8]}" - taken = _existing_slugs(db) - slug = base - n = 2 - while slug in taken: - slug = f"{base}-{n}" - n += 1 - return slug - - -def _model_config_snapshot(agent: UserAgent) -> Dict[str, Any]: - return { - "temperature": float(agent.temperature) if agent.temperature is not None else None, - "max_tokens": agent.max_tokens, - "max_iters": agent.max_iters, - "timeout": agent.timeout, - } - - -def _bindings_snapshot(agent: UserAgent) -> Dict[str, List[str]]: - return { - "skill_ids": list(agent.skill_ids or []), - "mcp_server_ids": list(agent.mcp_server_ids or []), - "plugin_ids": list(agent.plugin_ids or []), - "kb_ids": list(agent.kb_ids or []), - } - - -def _submission_to_dict( - sub: AgentMarketSubmission, *, with_content: bool = False -) -> Dict[str, Any]: - data = { - "submission_id": sub.submission_id, - "slug": sub.slug, - "agent_id": sub.agent_id, - "owner_user_id": sub.owner_user_id, - "submitter_name": sub.submitter_name or "", - "name": sub.name, - "avatar": sub.avatar or "", - "description": sub.description or "", - "summary": sub.summary or "", - "category": sub.category or DEFAULT_AGENT_CATEGORY, - "tags": list(sub.tags or []), - "version": sub.version or "1.0.0", - "note": sub.note or "", - "status": sub.status, - "review_note": sub.review_note or "", - "reviewed_at": sub.reviewed_at.isoformat() if sub.reviewed_at else None, - "created_at": sub.created_at.isoformat() if sub.created_at else None, - } - if with_content: - data["system_prompt"] = sub.system_prompt or "" - data["welcome_message"] = sub.welcome_message or "" - data["suggested_questions"] = list(sub.suggested_questions or []) - data["bindings"] = _normalize_binding_lists(sub.bindings_snapshot) - return data - - -def _owned_agent(db: Session, agent_id: str, owner_user_id: str) -> UserAgent: - """Fetch the sub-agent owned by this user (owner_type=user), otherwise 404.""" - agent = ( - db.query(UserAgent) - .filter( - UserAgent.agent_id == agent_id, - UserAgent.owner_type == "user", - UserAgent.user_id == owner_user_id, - ) - .first() - ) - if agent is None: - raise ResourceNotFoundError("user_agent", agent_id) - return agent - - -def _new_submission( - *, - submission_id: str, - slug: str, - agent_id: str, - owner_user_id: str, - name: str, - category: str, - status: str, - submitter_name: str = "", - avatar: Optional[str] = None, - description: str = "", - summary: str = "", - note: str = "", - system_prompt: str = "", - welcome_message: str = "", - suggested_questions: Optional[List[str]] = None, - model_config_snapshot: Optional[Dict[str, Any]] = None, - bindings_snapshot: Optional[Dict[str, Any]] = None, - tags: Optional[List[str]] = None, - reviewed_at: Optional[datetime] = None, -) -> AgentMarketSubmission: - """Uniformly construct one ``AgentMarketSubmission`` (fixed fields and default normalization centralized here). - - The three entry points (user submission ``submit_to_marketplace`` / the create branch of the - admin direct-upload ``publish_agent_to_market`` / admin create-and-list - ``create_market_agent``) share the same field set; callers pass only what differs. - """ - now = utc_now() - return AgentMarketSubmission( - submission_id=submission_id, - slug=slug, - agent_id=agent_id, - owner_user_id=owner_user_id, - submitter_name=(submitter_name or "").strip(), - name=name, - avatar=avatar, - description=description or "", - summary=summary or "", - category=category, - tags=list(tags or []), - version="1.0.0", - note=(note or "").strip(), - system_prompt=system_prompt or "", - welcome_message=welcome_message or "", - suggested_questions=list(suggested_questions or []), - model_config_snapshot=dict(model_config_snapshot or {}), - bindings_snapshot=_normalize_binding_lists(bindings_snapshot), - status=status, - reviewed_at=reviewed_at, - created_at=now, - updated_at=now, - ) - - -def submit_to_marketplace( - db: Session, - agent_id: str, - *, - owner_user_id: str, - submitter_name: str = "", - note: str = "", - category: str = "", - summary: str = "", -) -> Dict[str, Any]: - """A user submits their self-built sub-agent for marketplace listing (content snapshot, pending admin review).""" - agent = _owned_agent(db, agent_id, owner_user_id) - - active = ( - db.query(AgentMarketSubmission) - .filter( - AgentMarketSubmission.agent_id == agent_id, - AgentMarketSubmission.owner_user_id == owner_user_id, - AgentMarketSubmission.status.in_(["pending", "approved"]), - ) - .first() - ) - if active is not None: - state = "待审核" if active.status == "pending" else "已上架" - raise HTTPException(status_code=409, detail=f"该子智能体已有{state}的上架申请") - - sub = _new_submission( - submission_id=f"asub_{uuid.uuid4().hex[:16]}", - slug=_derive_slug(db, agent.name), - agent_id=agent_id, - owner_user_id=owner_user_id, - submitter_name=submitter_name, - name=agent.name, - avatar=agent.avatar, - description=agent.description or "", - summary=(summary or "").strip() or (agent.description or ""), - category=validate_category(category), - note=note, - system_prompt=agent.system_prompt or "", - welcome_message=agent.welcome_message or "", - suggested_questions=agent.suggested_questions, - model_config_snapshot=_model_config_snapshot(agent), - bindings_snapshot=_bindings_snapshot(agent), - tags=list((agent.extra_config or {}).get("ontology_tags") or []), - status="pending", - ) - db.add(sub) - db.commit() - logger.info( - "agent_market_submission_created: id=%s agent=%s owner=%s slug=%s", - sub.submission_id, - agent_id, - owner_user_id, - sub.slug, - ) - return _submission_to_dict(sub) - - -def list_my_submissions(db: Session, owner_user_id: str) -> List[Dict[str, Any]]: - rows = ( - db.query(AgentMarketSubmission) - .filter(AgentMarketSubmission.owner_user_id == owner_user_id) - .order_by(AgentMarketSubmission.created_at.desc()) - .all() - ) - return [_submission_to_dict(r) for r in rows] - - -def withdraw_submission(db: Session, submission_id: str, owner_user_id: str) -> None: - """A user withdraws their own submission: pending/rejected can be deleted; approved requires an admin to delist.""" - sub = ( - db.query(AgentMarketSubmission) - .filter( - AgentMarketSubmission.submission_id == submission_id, - AgentMarketSubmission.owner_user_id == owner_user_id, - ) - .first() - ) - if sub is None: - raise ResourceNotFoundError("agent_market_submission", submission_id) - if sub.status == "approved": - raise BadRequestError(message="子智能体已上架，如需下架请联系管理员") - db.delete(sub) - db.commit() - - -def list_submissions(db: Session, status: Optional[str] = None) -> List[Dict[str, Any]]: - """Admin view: all listing submissions, filterable by status (pending first, newest → oldest).""" - q = db.query(AgentMarketSubmission) - if status: - q = q.filter(AgentMarketSubmission.status == status) - rows = q.order_by(AgentMarketSubmission.created_at.desc()).all() - order = {"pending": 0, "approved": 1, "rejected": 2} - rows.sort(key=lambda r: order.get(r.status, 3)) - return [_submission_to_dict(r) for r in rows] - - -def get_submission(db: Session, submission_id: str) -> Dict[str, Any]: - sub = ( - db.query(AgentMarketSubmission) - .filter(AgentMarketSubmission.submission_id == submission_id) - .first() - ) - if sub is None: - raise ResourceNotFoundError("agent_market_submission", submission_id) - return _submission_to_dict(sub, with_content=True) - - -def review_submission( - db: Session, - submission_id: str, - *, - approve: bool, - review_note: str = "", - category: str = "", -) -> Dict[str, Any]: - """Admin review: approval lists it immediately; rejecting pending = refuse, rejecting approved = delist.""" - sub = ( - db.query(AgentMarketSubmission) - .filter(AgentMarketSubmission.submission_id == submission_id) - .first() - ) - if sub is None: - raise ResourceNotFoundError("agent_market_submission", submission_id) - if approve and sub.status != "pending": - state = "已上架" if sub.status == "approved" else "已驳回" - raise HTTPException( - status_code=409, - detail=f"该申请{state}，不能重复审核通过（如需重新上架请让用户重新提交）", - ) - if not approve and sub.status == "rejected": - raise HTTPException(status_code=409, detail="该申请已是驳回状态") - sub.status = "approved" if approve else "rejected" - sub.review_note = (review_note or "").strip() - if approve and (category or "").strip(): - sub.category = validate_category(category) - sub.reviewed_at = utc_now() - sub.updated_at = utc_now() - db.commit() - logger.info( - "agent_market_submission_reviewed: id=%s slug=%s status=%s", - sub.submission_id, - sub.slug, - sub.status, - ) - return _submission_to_dict(sub) - - -# ── Admin direct upload + delete + list/delist ─────────────────────────────── - - -def publish_agent_to_market( - db: Session, - agent_id: str, - *, - category: str, - summary: str = "", - submitter_name: str = "管理员上传", -) -> Dict[str, Any]: - """An admin lists an existing sub-agent directly on the marketplace as approved (no review). - - If the same agent_id already has an admin listing record, update it (keeping the original slug); otherwise create a new one. - """ - category = validate_category(category) - agent = db.query(UserAgent).filter(UserAgent.agent_id == agent_id).first() - if agent is None: - raise ResourceNotFoundError("user_agent", agent_id) - - now = utc_now() - existing = ( - db.query(AgentMarketSubmission) - .filter( - AgentMarketSubmission.agent_id == agent_id, - AgentMarketSubmission.owner_user_id == ADMIN_UPLOAD_OWNER, - ) - .first() - ) - summary = (summary or "").strip() or (agent.description or "") - if existing is not None: - existing.name = agent.name - existing.avatar = agent.avatar - existing.description = agent.description or "" - existing.summary = summary - existing.category = category - existing.system_prompt = agent.system_prompt or "" - existing.welcome_message = agent.welcome_message or "" - existing.suggested_questions = list(agent.suggested_questions or []) - existing.model_config_snapshot = _model_config_snapshot(agent) - existing.bindings_snapshot = _bindings_snapshot(agent) - existing.tags = list((agent.extra_config or {}).get("ontology_tags") or []) - existing.status = "approved" - existing.review_note = "" - existing.reviewed_at = now - existing.updated_at = now - flag_modified(existing, "suggested_questions") - flag_modified(existing, "model_config_snapshot") - flag_modified(existing, "bindings_snapshot") - flag_modified(existing, "tags") - db.commit() - return {"slug": existing.slug, "action": "updated", "message": "子智能体市场内容已更新"} - - sub = _new_submission( - submission_id=f"aadm_{uuid.uuid4().hex[:16]}", - slug=_derive_slug(db, agent.name), - agent_id=agent_id, - owner_user_id=ADMIN_UPLOAD_OWNER, - submitter_name=submitter_name, - name=agent.name, - avatar=agent.avatar, - description=agent.description or "", - summary=summary, - category=category, - note="管理员上传，免审核直接上架", - system_prompt=agent.system_prompt or "", - welcome_message=agent.welcome_message or "", - suggested_questions=agent.suggested_questions, - model_config_snapshot=_model_config_snapshot(agent), - bindings_snapshot=_bindings_snapshot(agent), - tags=list((agent.extra_config or {}).get("ontology_tags") or []), - status="approved", - reviewed_at=now, - ) - db.add(sub) - db.commit() - logger.info("agent_market_admin_published: agent=%s slug=%s", agent_id, sub.slug) - return {"slug": sub.slug, "action": "published", "message": "子智能体已上架市场"} - - -def create_market_agent( - db: Session, - *, - name: str, - category: str, - avatar: str = "", - description: str = "", - summary: str = "", - system_prompt: str = "", - welcome_message: str = "", - suggested_questions: Optional[List[str]] = None, - bindings: Optional[Dict[str, Any]] = None, - model_config: Optional[Dict[str, Any]] = None, - tags: Optional[List[str]] = None, - submitter_name: str = "管理员上传", -) -> Dict[str, Any]: - """An admin directly "creates" a sub-agent and lists it on the marketplace as approved (no global agent created). - - Mirrors the skill side's "creating a skill lists it on the skill marketplace": the new - sub-agent enters the sub-agent marketplace (explicitly installable as a global / private - clone) and does not take effect globally right away. Content comes from form fields rather - than a source ``UserAgent``, hence a synthetic placeholder ``agent_id`` (installs always use - the snapshot and never look back at a source agent). - """ - now = utc_now() - sub = _new_submission( - submission_id=f"aadm_{uuid.uuid4().hex[:16]}", - slug=_derive_slug(db, name), - agent_id=f"__market__{uuid.uuid4().hex[:12]}", - owner_user_id=ADMIN_UPLOAD_OWNER, - submitter_name=submitter_name, - name=name, - avatar=avatar or "", - description=description or "", - summary=(summary or "").strip() or (description or ""), - category=validate_category(category), - note="管理员创建，直接上架", - system_prompt=system_prompt or "", - welcome_message=welcome_message or "", - suggested_questions=suggested_questions, - model_config_snapshot=model_config, - bindings_snapshot=bindings, - tags=tags, - status="approved", - reviewed_at=now, - ) - db.add(sub) - db.commit() - logger.info("agent_market_admin_created: slug=%s name=%s", sub.slug, name) - return {"slug": sub.slug, "action": "created", "message": "子智能体已上架子智能体市场"} - - -def delete_market_agent(db: Session, slug: str) -> Dict[str, Any]: - """An admin deletes one DB listing record from the marketplace. Presets ship with the repo and cannot be deleted online (400).""" - if _read_bundle(slug) is not None: - raise BadRequestError(message="预置子智能体随仓库发布，不可在线删除") - sub = db.query(AgentMarketSubmission).filter(AgentMarketSubmission.slug == slug).first() - if sub is None: - raise ResourceNotFoundError("marketplace_agent", slug) - db.delete(sub) - db.commit() - logger.info("agent_market_deleted: slug=%s", slug) - return {"slug": slug, "deleted": True} - - -def set_market_agent_enabled( - db: Session, slug: str, enabled: bool, *, updated_by: Optional[str] = None -) -> Dict[str, Any]: - """List/delist a marketplace sub-agent (controls whether it shows in the marketplace; installed clones are unaffected).""" - return ml.set_listing_enabled(db, ml.KIND_AGENT, slug, enabled, updated_by=updated_by) diff --git a/src/backend/core/services/artifact_edition.py b/src/backend/core/services/artifact_edition.py deleted file mode 100644 index daef43ca..00000000 --- a/src/backend/core/services/artifact_edition.py +++ /dev/null @@ -1,66 +0,0 @@ -"""Community artifact lists are always personal.""" - -from typing import Any, Optional - - -def artifact_list_scope() -> str: - return "personal" - - -def extend_artifact_item(artifact, item: dict) -> dict: - return item - - -def personal_artifact_predicates(artifact_model: Any) -> list[Any]: - return [] - - -def is_personal_artifact(artifact: Any) -> bool: - return True - - -def personal_artifact_create_fields() -> dict[str, None]: - return {} - - -def artifact_scope_fields(scope: Any) -> dict[str, Optional[str]]: - return { - "user_folder_id": ( - scope.root_folder_id or None if scope is not None and scope.is_personal else None - ) - } - - -def artifact_scope_folder_id(scope: Any, artifact: Any) -> Optional[str]: - return artifact.user_folder_id - - -def can_access_artifact(db: Any, user_id: str, artifact: Any) -> bool: - return str(artifact.user_id) == str(user_id) - - -def artifact_access_metadata(artifact: Any) -> dict[str, Optional[str]]: - return {"owner_id": artifact.user_id} - - -def can_access_artifact_metadata(db: Any, user_id: str, metadata: dict[str, Any]) -> bool: - owner_id = metadata.get("owner_id") or metadata.get("user_id") - return owner_id is None or str(owner_id) == str(user_id) - - -__all__ = [ - "upload_target_matches", - "artifact_access_metadata", - "artifact_list_scope", - "artifact_scope_fields", - "artifact_scope_folder_id", - "can_access_artifact", - "can_access_artifact_metadata", - "extend_artifact_item", - "is_personal_artifact", - "personal_artifact_create_fields", - "personal_artifact_predicates", -] - -def upload_target_matches(artifact: Any, scope: str, folder_id: Optional[str]) -> bool: - return scope == "personal" and artifact.user_folder_id == folder_id diff --git a/src/backend/core/services/chat_service.py b/src/backend/core/services/chat_service.py deleted file mode 100644 index 90775c5c..00000000 --- a/src/backend/core/services/chat_service.py +++ /dev/null @@ -1,1301 +0,0 @@ -"""Chat session and message business logic.""" -from core.infra.time import utc_now - -import hashlib -import json -import logging -import uuid -from dataclasses import dataclass, replace -from datetime import datetime, timedelta, timezone -from typing import Any, Dict, List, Optional, Tuple - -from core.auth.permissions_iface import ( - ChatAccessLevel, - can_delete_session, - resolve_chat_access, -) -from core.db.models import ChatCompactionState, ChatMessage, ChatRun, ChatSession -from core.db.repository import ( - AuditLogRepository, - ChatMessageRepository, - ChatSessionRepository, -) -from core.db.repository.chat import _strip_nul -from core.db.reserved_chats import reserved_chat_owner -from sqlalchemy import update -from core.ontology.revision import is_substantive_revision, normalize_revision_candidate -from sqlalchemy.orm import Session - -logger = logging.getLogger(__name__) - -_CONTENT_MAX = 100_000 -_TRUNC_NOTE = "\n\n…（本条回复过长，已截断保存；完整过程见工具调用记录）" -# 思考的独立额度。content 有 CHECK 约束卡死 10 万字，thinking 是 JSON 列没有 -# 约束，但也不能无限长——跑飞的一轮能产出几十万字推理。 -_THINKING_MAX = 200_000 - - -def clamp_message_content(content: str) -> str: - """Apply the database content limit independently of ChatService wiring.""" - - text = content or "" - if len(text) <= _CONTENT_MAX: - return text - keep = _CONTENT_MAX - len(_TRUNC_NOTE) - logger.warning("[chat] message content truncated: %d -> %d chars", len(text), _CONTENT_MAX) - kept = text[:keep] - # 老格式的思考是以 … 内联在正文里的。截断点可能落在块中间， - # 闭合标签一丢，历史重建就把整段思考当正文渲染出来（思考泄露到页面上）。 - close = "" - if kept.count("") > kept.count(close): - kept = kept[: keep - len(close)] + close - return kept + _TRUNC_NOTE - - -def clamp_thinking( - blocks: Optional[List[Dict[str, Any]]], - segments: Optional[List[Dict[str, Any]]] = None, -) -> tuple[Optional[List[Dict[str, Any]]], Optional[List[Dict[str, Any]]]]: - """思考单独存一列，有自己的额度，不再和正文抢 content 的 10 万字上限。 - - 超额时从最早的块开始丢——最后几轮的推理离最终答案最近，对用户回看最有用。 - - ``metadata.segments`` 按下标引用思考块，丢块会让下标错位，所以两者必须一起夹取： - 返回夹取后的块列表，以及下标已重映射、引用了被丢块的段落已剔除的段落表。 - """ - if not blocks: - return None, segments or None - if sum(len(str(b.get("content") or "")) for b in blocks) <= _THINKING_MAX: - return blocks, segments or None - kept: List[Dict[str, Any]] = [] - kept_indices: List[int] = [] - budget = _THINKING_MAX - for offset, block in enumerate(reversed(blocks)): - text = str(block.get("content") or "") - if not text: - continue - if len(text) > budget: - text = text[len(text) - budget :] - if not text: - break - budget -= len(text) - kept.append({**block, "content": text}) - kept_indices.append(len(blocks) - 1 - offset) - if budget <= 0: - break - kept.reverse() - kept_indices.reverse() - if not kept: - return None, _drop_thinking_segments(segments, {}) - remap = {old: new for new, old in enumerate(kept_indices)} - return kept, _drop_thinking_segments(segments, remap) - - -def _drop_thinking_segments( - segments: Optional[List[Dict[str, Any]]], - remap: Dict[int, int], -) -> Optional[List[Dict[str, Any]]]: - """把段落表里的思考下标重映射到夹取后的位置，引用已丢块的段落直接剔除。""" - if not segments: - return segments or None - rebuilt: List[Dict[str, Any]] = [] - for segment in segments: - if segment.get("type") != "thinking": - rebuilt.append(segment) - continue - new_index = remap.get(segment.get("index")) - if new_index is None: - continue - rebuilt.append({**segment, "index": new_index}) - return rebuilt or None - - -class CompactionCASConflict(RuntimeError): - """The checkpoint base/lease is stale and another writer won.""" - - -@dataclass(frozen=True) -class CompactionSourceRow: - """Detached, immutable copy of one message in the compacted snapshot.""" - - message_id: str - chat_seq: int - role: str - content: str - tool_calls: Any - extra_data: Dict[str, Any] - - -@dataclass(frozen=True) -class CompactionSnapshot: - """Immutable source boundary captured before the summary LLM call.""" - - chat_id: str - owner: str - base_checkpoint_id: Optional[str] - base_checkpoint_version: int - base_covered_seq: int - covered_seq: int - source_hash: str - replacement_history: tuple[Dict[str, Any], ...] - source_rows: tuple[CompactionSourceRow, ...] - source_message_ids: tuple[str, ...] - source_message_seqs: tuple[int, ...] - - -def _utcnow() -> datetime: - return datetime.now(timezone.utc) - - -def _as_utc(value: Optional[datetime]) -> Optional[datetime]: - if value is None: - return None - if value.tzinfo is None: - return value.replace(tzinfo=timezone.utc) - return value.astimezone(timezone.utc) - - -class ChatService: - """Service for chat-related operations.""" - - def __init__(self, db: Session): - self.db = db - self.session_repo = ChatSessionRepository(db) - self.message_repo = ChatMessageRepository(db) - self.audit_repo = AuditLogRepository(db) - - def create_session( - self, - user_id: str, - title: str = "新对话", - extra_data: Dict = None, - chat_id: Optional[str] = None, - project_id: Optional[str] = None, - ) -> ChatSession: - """Create a new chat session. - - If `chat_id` is provided and belongs to the same user, reuse it. - If it belongs to another user, generate a new chat_id. - """ - if chat_id: - existing = self.session_repo.get_by_id(chat_id) - if existing: - if existing.user_id == user_id: - return existing - chat_id = None - - session_data = { - "chat_id": chat_id or f"chat_{uuid.uuid4().hex[:16]}", - "user_id": user_id, - "title": title, - "extra_data": extra_data or {}, - "project_id": project_id, - } - session = self.session_repo.create(session_data) - - # Audit log - self.audit_repo.create( - { - "user_id": user_id, - "action": "chat.session.created", - "resource_type": "chat_session", - "resource_id": session.chat_id, - "status": "success", - } - ) - - return session - - def ensure_session( - self, - chat_id: str, - user_id: str, - title: str = "新对话", - extra_data: Optional[Dict] = None, - project_id: Optional[str] = None, - ) -> Optional[ChatSession]: - """Ensure a chat session exists for user and chat_id. - - When ``project_id`` is given, attach the session to that project (first write wins; if - already attached to a different project it stays unchanged — in-project chats never drift - across projects). All session-creation entry points share this attachment rule. - """ - existing = self.session_repo.get_by_id(chat_id) - if existing: - if existing.user_id != user_id: - return None - # Merge any missing metadata flags into existing session - if extra_data: - merged = dict(existing.extra_data or {}) - changed = False - for k, v in extra_data.items(): - if k not in merged: - merged[k] = v - changed = True - if changed: - existing.extra_data = merged - self.db.commit() - session = existing - else: - session = self.create_session( - user_id=user_id, - title=title, - extra_data=extra_data or {}, - chat_id=chat_id, - project_id=project_id, - ) - if session is not None and project_id and not session.project_id: - session.project_id = project_id - self.db.commit() - return session - - def list_sessions( - self, - user_id: str, - page: int = 1, - page_size: int = 20, - pinned_only: bool = False, - favorite_only: bool = False, - exclude_automation: bool = False, - ) -> Tuple[List[ChatSession], int, int]: - """List chat sessions with pagination.""" - sessions, total = self.session_repo.list_by_user( - user_id, - page, - page_size, - pinned_only, - favorite_only, - exclude_automation=exclude_automation, - ) - - total_pages = (total + page_size - 1) // page_size - - return sessions, total, total_pages - - def get_session(self, chat_id: str, user_id: str) -> Optional[ChatSession]: - """Get chat session with ownership check. - - Historical semantics unchanged: only the owner can get the session. For - edition-specific sharing, use :py:meth:`get_session_with_access`. - """ - # Module-owned chats are reachable only through their module, never as ordinary chats. - if reserved_chat_owner(chat_id): - return None - session = self.session_repo.get_by_id(chat_id) - - if session and session.user_id != user_id: - # Access denied - user doesn't own this session - return None - - return session - - def get_session_with_access( - self, chat_id: str, user_id: str - ) -> Optional[Tuple[ChatSession, ChatAccessLevel]]: - """Get a session in a sharing context + compute the access level. - - - Does not exist / soft-deleted → ``None`` - - No access (``resolve_chat_access`` returns ``'none'``) → ``None`` - - Module-owned (reserved namespace) → ``None``: not an ordinary chat - - Otherwise returns ``(session, 'admin'|'edit'|'read')`` - """ - if reserved_chat_owner(chat_id): - return None - session = self.session_repo.get_by_id(chat_id) - if session is None: - return None - level = resolve_chat_access(self.db, user_id, session) - if level == "none": - return None - return session, level - - def update_session_fields( - self, - chat_id: str, - fields: Dict[str, Any], - *, - actor_user_id: Optional[str] = None, - ) -> Optional[ChatSession]: - """Field update without ownership check (for sharing contexts). - - The caller is responsible for permission checks beforehand. ``actor_user_id`` is used - only for the audit log. The ``extra_data`` field is merged rather than overwritten. - """ - session = self.session_repo.get_by_id(chat_id) - if session is None: - return None - normalized = dict(fields) - extra_patch = normalized.get("extra_data") - if isinstance(extra_patch, dict): - merged = dict(session.extra_data or {}) - merged.update(extra_patch) - normalized["extra_data"] = merged - updated = self.session_repo.update(chat_id, normalized) - self.audit_repo.create( - { - "user_id": actor_user_id or session.user_id, - "action": "chat.session.updated", - "resource_type": "chat_session", - "resource_id": chat_id, - "details": normalized, - "status": "success", - } - ) - return updated - - def delete_session_force(self, chat_id: str, *, actor_user_id: str) -> bool: - """Forced delete in a sharing context (no ownership check). Caller handles permissions.""" - session = self.session_repo.get_by_id(chat_id) - if session is None: - return False - result = self.session_repo.soft_delete(chat_id) - if result: - self.audit_repo.create( - { - "user_id": actor_user_id, - "action": "chat.session.deleted", - "resource_type": "chat_session", - "resource_id": chat_id, - "details": { - "owner_user_id": session.user_id, - "deleted_by_owner": session.user_id == actor_user_id, - }, - "status": "success", - } - ) - return result - - def update_session( - self, chat_id: str, user_id: str, update_data: Dict[str, Any] - ) -> Optional[ChatSession]: - """Update chat session.""" - session = self.get_session(chat_id, user_id) - if not session: - return None - - normalized_update_data = dict(update_data) - extra_data_patch = normalized_update_data.get("extra_data") - if isinstance(extra_data_patch, dict): - merged_extra_data = dict(session.extra_data or {}) - merged_extra_data.update(extra_data_patch) - normalized_update_data["extra_data"] = merged_extra_data - - updated_session = self.session_repo.update(chat_id, normalized_update_data) - - # Audit log - self.audit_repo.create( - { - "user_id": user_id, - "action": "chat.session.updated", - "resource_type": "chat_session", - "resource_id": chat_id, - "details": normalized_update_data, - "status": "success", - } - ) - - return updated_session - - def delete_session(self, chat_id: str, user_id: str) -> bool: - """Delete chat session (soft delete).""" - session = self.get_session(chat_id, user_id) - if not session: - return False - - result = self.session_repo.soft_delete(chat_id) - - if result: - # Audit log - self.audit_repo.create( - { - "user_id": user_id, - "action": "chat.session.deleted", - "resource_type": "chat_session", - "resource_id": chat_id, - "status": "success", - } - ) - - return result - - # chat_messages.content 有 CHECK 约束 char_length <= 100000。长任务的一轮回复 - # （反复重试 + 大量思考正文）真的会撞上，而撞上的后果是**整条 INSERT 抛 - # CheckViolation → 整个 run failed**，一轮的产出全部丢失（实测踩过）。 - # 与其让约束把一轮工作全毁掉，不如夹逼后落库并明确标注被截断。 - def add_message( - self, - chat_id: str, - role: str, - content: str, - model: Optional[str] = None, - thinking: Optional[List[Dict[str, Any]]] = None, - tool_calls: Optional[List[Dict]] = None, - usage: Optional[Dict] = None, - error: Optional[Dict] = None, - extra_data: Dict = None, - message_id: Optional[str] = None, - commit: bool = True, - chat_seq: Optional[int] = None, - model_steps: Optional[List[Dict[str, Any]]] = None, - ) -> ChatMessage: - """Add a message to a chat session.""" - extra = dict(extra_data or {}) - kept_thinking, kept_segments = clamp_thinking(thinking, extra.get("segments")) - if "segments" in extra: - extra["segments"] = kept_segments - message_data = { - "message_id": message_id or f"msg_{uuid.uuid4().hex[:16]}", - "chat_id": chat_id, - "chat_seq": chat_seq, - "role": role, - "content": clamp_message_content(content), - "model": model, - "thinking": kept_thinking, - "tool_calls": tool_calls, - "model_steps": model_steps, - "usage": usage, - "error": error, - "extra_data": extra, - } - - message = self.message_repo.create(message_data, commit=commit) - - # Keep session metadata in sync for list APIs. - session = self.session_repo.get_by_id(chat_id) - if session: - session.message_count = (session.message_count or 0) + 1 - now = utc_now() - session.updated_at = now - session.last_message_at = now - if commit: - self.db.commit() - - return message - - def refresh_streaming_message( - self, - *, - run_id: str, - owner: str, - message_id: str, - content: str, - thinking: Optional[List[Dict[str, Any]]], - tool_calls: Optional[List[Dict]], - extra_data: Dict[str, Any], - model_steps: Optional[List[Dict[str, Any]]] = None, - ) -> bool: - """Overwrite the in-flight assistant row while its run streams. - - The run row lock serializes this write with its final commit, so a - superseded or finishing worker cannot clobber the final message. Nothing else is touched — in particular no session - timestamps, so pollers do not mistake a checkpoint for a new message. - Returns False when the caller no longer owns the run. - """ - # Lock the same run row as journal.complete before touching its message. - # An EXISTS predicate alone can see a live lease, wait for the message - # lock, and then overwrite a final commit using that stale MVCC view. - live_run = (self.db.query(ChatRun) - .filter(ChatRun.run_id == run_id, ChatRun.lease_owner == owner, - ChatRun.status.in_(("pending", "running"))) - .with_for_update().first()) - if live_run is None: - return False - extra = dict(extra_data) - kept_thinking, kept_segments = clamp_thinking(thinking, extra.get("segments")) - if "segments" in extra: - extra["segments"] = kept_segments - result = self.db.execute( - update(ChatMessage) - .where(ChatMessage.message_id == message_id) - .values( - content=clamp_message_content(content), - thinking=_strip_nul(kept_thinking), - tool_calls=_strip_nul(tool_calls), - model_steps=_strip_nul(model_steps), - extra_data=_strip_nul(extra), - ) - ) - return result.rowcount > 0 - - def upsert_message( - self, - chat_id: str, - role: str, - content: str, - *, - message_id: str, - model: Optional[str] = None, - thinking: Optional[List[Dict[str, Any]]] = None, - tool_calls: Optional[List[Dict]] = None, - usage: Optional[Dict] = None, - error: Optional[Dict] = None, - extra_data: Dict = None, - commit: bool = True, - chat_seq: Optional[int] = None, - model_steps: Optional[List[Dict[str, Any]]] = None, - invalidate_mid_turn_checkpoint: bool = False, - ) -> ChatMessage: - """Idempotently upsert a message by message_id: overwrite in place if it exists, otherwise create. - - The autonomous loop uses this for "incrementally refreshing the same assistant message as - progress advances" — each requirement processed / each evaluation round flushes the - current accumulated body + tool cards into this message, so progress is visible in the DB - even after a mid-run crash/refresh, instead of only being written once at the terminal - state. The update path does not re-increment message_count (only +1 on creation). - """ - existing = self.message_repo.get_by_id(message_id) - if existing is not None: - update: Dict[str, Any] = {"content": clamp_message_content(content)} - extra = dict(extra_data) if extra_data is not None else None - kept_thinking, kept_segments = clamp_thinking( - thinking, (extra or {}).get("segments") - ) - if model is not None: - update["model"] = model - if thinking is not None: - update["thinking"] = kept_thinking - if tool_calls is not None: - update["tool_calls"] = tool_calls - if model_steps is not None: - update["model_steps"] = model_steps - if usage is not None: - update["usage"] = usage - if error is not None: - update["error"] = error - if extra is not None: - if "segments" in extra: - extra["segments"] = kept_segments - update["extra_data"] = extra - if invalidate_mid_turn_checkpoint: - # The completed turn has already crossed the post-turn trigger. - # Its new steps cannot be safely sliced from a row covered by a - # mid-turn checkpoint, so rebuild that successor from originals. - # Match checkpoint publisher lock order: session, then lineage. - self.db.query(ChatSession).filter(ChatSession.chat_id == chat_id).with_for_update().first() - state = (self.db.query(ChatCompactionState) - .filter(ChatCompactionState.chat_id == chat_id) - .with_for_update().populate_existing().first()) - if (state is not None and state.active_checkpoint_id - and int(existing.chat_seq or 0) <= int(state.covered_seq or 0)): - checkpoint = self.db.get(ChatMessage, state.active_checkpoint_id) - manifest = ((checkpoint.extra_data or {}).get("replacement_manifest") or {}) if checkpoint else {} - captured = (extra or {}).get("context_usage", {}).get("captured_at_ms", 0) - checkpoint_ms = int(_as_utc(checkpoint.created_at).timestamp() * 1000) if checkpoint else 0 - if (manifest.get("phase") == "mid_turn" and type(captured) is int - and captured > checkpoint_ms): - self._invalidate_compaction_lineage_for_rewrite( - chat_id, rewritten_seq=int(existing.chat_seq or 0) - ) - msg = self.message_repo.update(message_id, update, commit=commit) - session = self.session_repo.get_by_id(chat_id) - if session: - now = utc_now() - session.updated_at = now - session.last_message_at = now - if commit: - self.db.commit() - return msg or existing - return self.add_message( - chat_id=chat_id, - role=role, - content=content, - model=model, - thinking=thinking, - tool_calls=tool_calls, - usage=usage, - error=error, - extra_data=extra_data, - message_id=message_id, - commit=commit, - chat_seq=chat_seq, - model_steps=model_steps, - ) - - def list_all_messages( - self, chat_id: str, user_id: str - ) -> Optional[List[ChatMessage]]: - """List all messages in chronological order with access check. - - Edition-specific sharing policies may also grant read access. - - **Excludes** compaction checkpoint rows (the only writer of role='system' in - chat_messages is add_compaction_checkpoint) — internal artifacts, invisible to all - downstream consumers (replay/export/sharing/title/memory), filtered at the SQL layer - (a checkpoint's extra_data carries the entire replacement_history; loading it just to - discard is pure waste). The replay layer (compaction_service) fetches them separately - via get_latest_compaction_checkpoint. - """ - pair = self.get_session_with_access(chat_id, user_id) - if pair is None: - return None - - return ( - self.db.query(ChatMessage) - .filter( - ChatMessage.chat_id == chat_id, - ChatMessage.role != "system", - ) - .order_by(ChatMessage.chat_seq) - .all() - ) - - def list_messages( - self, chat_id: str, user_id: str, page: int = 1, page_size: int = 50 - ) -> Optional[Tuple[List[ChatMessage], int, int]]: - """List messages in a chat session.""" - # Check ownership - session = self.get_session(chat_id, user_id) - if not session: - return None - - messages, total = self.message_repo.list_by_chat(chat_id, page, page_size) - total_pages = (total + page_size - 1) // page_size - - return messages, total, total_pages - - def delete_messages_from(self, chat_id: str, message_id: str) -> int: - """Delete a message and all subsequent messages in the chat. - - History rewrites invalidate the entire compaction lineage in the same - transaction. This fences an in-flight summarizer through the lineage - version CAS and forces the next replay/compaction to rebuild from raw - retained messages instead of trusting a stale covered watermark. - - Returns the number of messages deleted. - """ - self._ensure_message_sequences(chat_id) - target = ( - self.db.query(ChatMessage) - .filter( - ChatMessage.chat_id == chat_id, - ChatMessage.message_id == message_id, - ) - .first() - ) - if not target: - return 0 - - self._invalidate_compaction_lineage_for_rewrite( - chat_id, rewritten_seq=int(target.chat_seq) - ) - - deleted = ( - self.db.query(ChatMessage) - .filter( - ChatMessage.chat_id == chat_id, - ChatMessage.chat_seq >= int(target.chat_seq), - ) - .delete(synchronize_session="fetch") - ) - - # Update session message count - session = self.session_repo.get_by_id(chat_id) - if session: - remaining = ( - self.db.query(ChatMessage) - .filter( - ChatMessage.chat_id == chat_id, - ) - .count() - ) - session.message_count = remaining - session.updated_at = utc_now() - - self.db.commit() - return deleted - - def _invalidate_compaction_lineage_for_rewrite( - self, - chat_id: str, - *, - rewritten_seq: int, - ) -> bool: - """Fence a snapshot that may contain a rewritten/deleted message.""" - state = self._ensure_compaction_state(chat_id) - # Lock the lineage row where supported. SQLite serializes the following - # write transaction; PostgreSQL blocks a concurrent publisher here. - state = ( - self.db.query(ChatCompactionState) - .filter(ChatCompactionState.chat_id == chat_id) - .with_for_update() - .populate_existing() - .one() - ) - affects_checkpoint = bool( - state.active_checkpoint_id - and int(rewritten_seq) <= int(state.covered_seq or 0) - ) - affects_snapshot = bool(state.lease_owner) - if not affects_checkpoint and not affects_snapshot: - return False - - state.active_checkpoint_id = None - state.checkpoint_version = int(state.checkpoint_version or 0) + 1 - state.covered_seq = 0 - state.lease_owner = None - state.lease_expires_at = None - state.updated_at = _utcnow() - - from core.llm.compaction import COMPACTION_CHECKPOINT_KIND - - checkpoints = ( - self.db.query(ChatMessage) - .filter(ChatMessage.chat_id == chat_id, ChatMessage.role == "system") - .all() - ) - for checkpoint in checkpoints: - extra = dict(checkpoint.extra_data or {}) - if extra.get("kind") != COMPACTION_CHECKPOINT_KIND: - continue - extra["active"] = False - extra["notice_pending"] = False - checkpoint.extra_data = extra - return True - - def add_compaction_checkpoint( - self, - chat_id: str, - *, - summary_text: str, - replacement_history: List[Dict], - covered_seq: Optional[int] = None, - source_hash: Optional[str] = None, - base_checkpoint_version: Optional[int] = None, - replacement_manifest: Optional[Dict[str, Any]] = None, - ) -> ChatMessage: - """Compatibility wrapper around the lease + CAS checkpoint protocol.""" - owner = f"direct:{uuid.uuid4().hex}" - snapshot = self.acquire_compaction_snapshot( - chat_id, owner=owner, lease_seconds=60 - ) - if snapshot is None: - raise CompactionCASConflict( - "no uncompacted source or compaction lease is busy" - ) - if ( - base_checkpoint_version is not None - and snapshot.base_checkpoint_version != int(base_checkpoint_version) - ): - self.release_compaction_lease(chat_id, owner=owner) - raise CompactionCASConflict("checkpoint base changed") - if covered_seq is not None and int(covered_seq) != snapshot.covered_seq: - self.release_compaction_lease(chat_id, owner=owner) - raise ValueError("covered_seq must equal the acquired source watermark") - if source_hash: - snapshot = replace(snapshot, source_hash=source_hash) - return self.commit_compaction_checkpoint( - snapshot, - owner=owner, - summary_text=summary_text, - replacement_history=replacement_history, - replacement_manifest=replacement_manifest or {}, - ) - - def _latest_compaction_checkpoint_fallback( - self, chat_id: str - ) -> Optional[ChatMessage]: - """Find the newest checkpoint without timestamps (legacy-state fallback).""" - from core.llm.compaction import COMPACTION_CHECKPOINT_KIND - - from sqlalchemy.orm import defer - - rows = ( - self.db.query(ChatMessage).options(defer(ChatMessage.model_steps), defer(ChatMessage.tool_calls)) - .filter(ChatMessage.chat_id == chat_id, ChatMessage.role == "system") - .order_by(ChatMessage.chat_seq.desc(), ChatMessage.message_id.desc()) - .limit(20) - .all() - ) - return next( - ( - row - for row in rows - if (row.extra_data or {}).get("kind") == COMPACTION_CHECKPOINT_KIND - ), - None, - ) - - @staticmethod - def _checkpoint_covered_seq(checkpoint: Optional[ChatMessage]) -> int: - """Return only an explicitly captured source watermark. - - A legacy checkpoint was inserted *after* summarization, so its own - sequence cannot prove which earlier rows were actually summarized. - Treating ``checkpoint.chat_seq - 1`` as covered can hide a message - that arrived while the summary was in flight. Legacy checkpoints are - therefore conservatively invalidated and rebuilt from raw history. - """ - if checkpoint is None: - return 0 - extra = checkpoint.extra_data or {} - explicit = extra.get("covered_seq") - if explicit is None: - explicit = extra.get("covers_up_to_chat_seq") - if explicit is not None: - return max(0, int(explicit)) - return 0 - - def _ensure_message_sequences(self, chat_id: str) -> None: - """Repair legacy/direct ORM rows before they enter a source snapshot.""" - from sqlalchemy import func - - # ``key_share=True`` renders ``FOR NO KEY UPDATE`` on PostgreSQL. - # A plain ``FOR UPDATE`` here is a full outage risk: inserting any row - # into ``chat_messages`` takes ``FOR KEY SHARE`` on this parent row for - # the foreign key, and ``FOR KEY SHARE`` conflicts with ``FOR UPDATE``. - # Holding that lock therefore blocks every message write for the chat — - # including the run journal's terminal commit, which runs on the event - # loop and freezes the whole process when it waits. We only bump the - # non-key column ``next_message_seq``, so the weaker lock is both - # sufficient (it still excludes a concurrent sequence repair) and - # compatible with the foreign-key lock. - chat = ( - self.db.query(ChatSession) - .filter(ChatSession.chat_id == chat_id) - .with_for_update(key_share=True) - .one_or_none() - ) - if chat is None: - return - max_seq = int( - self.db.query(func.max(ChatMessage.chat_seq)) - .filter(ChatMessage.chat_id == chat_id) - .scalar() - or 0 - ) - next_seq = max(int(chat.next_message_seq or 1), max_seq + 1) - missing = ( - self.db.query(ChatMessage) - .filter(ChatMessage.chat_id == chat_id, ChatMessage.chat_seq.is_(None)) - .order_by(ChatMessage.created_at, ChatMessage.message_id) - .all() - ) - for row in missing: - row.chat_seq = next_seq - next_seq += 1 - chat.next_message_seq = next_seq - self.db.flush() - - def _ensure_compaction_state(self, chat_id: str) -> ChatCompactionState: - state = self.db.get(ChatCompactionState, chat_id) - if state is not None: - return state - checkpoint = self._latest_compaction_checkpoint_fallback(chat_id) - extra = (checkpoint.extra_data or {}) if checkpoint is not None else {} - has_exact_watermark = ( - checkpoint is not None - and ( - extra.get("covered_seq") is not None - or extra.get("covers_up_to_chat_seq") is not None - ) - ) - state = ChatCompactionState( - chat_id=chat_id, - active_checkpoint_id=checkpoint.message_id if has_exact_watermark else None, - checkpoint_version=( - int(extra.get("checkpoint_version") or 1) if has_exact_watermark else 0 - ), - covered_seq=self._checkpoint_covered_seq(checkpoint) - if has_exact_watermark - else 0, - updated_at=_utcnow(), - ) - self.db.add(state) - self.db.flush() - return state - - def acquire_compaction_snapshot( - self, - chat_id: str, - *, - owner: str, - lease_seconds: int, - ) -> Optional[CompactionSnapshot]: - """Capture an exact source watermark and acquire a short-lived lease. - - Every exit path must end this transaction. The body takes a row lock on - the chat before it reads, so an exception escaping with the transaction - still open strands that lock on a pooled connection that no longer has - a Python owner — an ``idle in transaction`` backend that blocks the - chat's message writes until the process dies. The rollback below is the - guarantee that cannot be forgotten by a future early return. - """ - try: - return self._acquire_compaction_snapshot( - chat_id, owner=owner, lease_seconds=lease_seconds - ) - except BaseException: - try: - self.db.rollback() - except Exception: # noqa: BLE001 - never mask the original failure - logger.exception( - "[compaction] rollback failed after a snapshot error (chat=%s)", - chat_id, - ) - raise - - def _acquire_compaction_snapshot( - self, - chat_id: str, - *, - owner: str, - lease_seconds: int, - ) -> Optional[CompactionSnapshot]: - from core.llm.compaction import COMPACTION_CHECKPOINT_KIND - - self._ensure_message_sequences(chat_id) - state = self._ensure_compaction_state(chat_id) - now = _utcnow() - if state.lease_owner and (_as_utc(state.lease_expires_at) or now) > now: - self.db.rollback() - return None - - checkpoint = ( - self.db.get(ChatMessage, state.active_checkpoint_id) - if state.active_checkpoint_id - else None - ) - replacement_history = ( - tuple( - dict(item) - for item in ( - (checkpoint.extra_data or {}).get("replacement_history") or [] - ) - ) - if checkpoint is not None - else () - ) - candidates = ( - self.db.query(ChatMessage) - .filter( - ChatMessage.chat_id == chat_id, - ChatMessage.chat_seq > int(state.covered_seq or 0), - ) - .order_by(ChatMessage.chat_seq) - .all() - ) - source_models = tuple( - row - for row in candidates - if not ( - row.role == "system" - and (row.extra_data or {}).get("kind") == COMPACTION_CHECKPOINT_KIND - ) - ) - if not source_models: - self.db.rollback() - return None - - source_rows = tuple( - CompactionSourceRow( - message_id=row.message_id, - chat_seq=int(row.chat_seq or 0), - role=row.role, - content=row.content, - tool_calls=row.tool_calls, - extra_data=dict(row.extra_data or {}), - ) - for row in source_models - ) - covered_seq = max(row.chat_seq for row in source_rows) - canonical_source = [ - { - "message_id": row.message_id, - "chat_seq": row.chat_seq, - "role": row.role, - "content": row.content, - "tool_calls": row.tool_calls, - "metadata": row.extra_data, - } - for row in source_rows - ] - source_hash = hashlib.sha256( - json.dumps( - { - "base_checkpoint_version": int(state.checkpoint_version or 0), - "base_covered_seq": int(state.covered_seq or 0), - "covered_seq": covered_seq, - "source": canonical_source, - }, - ensure_ascii=False, - sort_keys=True, - separators=(",", ":"), - default=str, - ).encode("utf-8") - ).hexdigest() - - base_version = int(state.checkpoint_version or 0) - updated = ( - self.db.query(ChatCompactionState) - .filter( - ChatCompactionState.chat_id == chat_id, - ChatCompactionState.checkpoint_version == base_version, - ( - ChatCompactionState.lease_owner.is_(None) - | (ChatCompactionState.lease_expires_at <= now) - | (ChatCompactionState.lease_owner == owner) - ), - ) - .update( - { - ChatCompactionState.lease_owner: owner, - ChatCompactionState.lease_expires_at: now - + timedelta(seconds=max(1, lease_seconds)), - ChatCompactionState.updated_at: now, - }, - synchronize_session=False, - ) - ) - if updated != 1: - self.db.rollback() - return None - base_covered_seq = int(state.covered_seq or 0) - base_checkpoint_id = checkpoint.message_id if checkpoint is not None else None - self.db.commit() - return CompactionSnapshot( - chat_id=chat_id, - owner=owner, - base_checkpoint_id=base_checkpoint_id, - base_checkpoint_version=base_version, - base_covered_seq=base_covered_seq, - covered_seq=covered_seq, - source_hash=source_hash, - replacement_history=replacement_history, - source_rows=source_rows, - source_message_ids=tuple(row.message_id for row in source_rows), - source_message_seqs=tuple(row.chat_seq for row in source_rows), - ) - - def release_compaction_lease(self, chat_id: str, *, owner: str) -> bool: - updated = ( - self.db.query(ChatCompactionState) - .filter( - ChatCompactionState.chat_id == chat_id, - ChatCompactionState.lease_owner == owner, - ) - .update( - { - ChatCompactionState.lease_owner: None, - ChatCompactionState.lease_expires_at: None, - ChatCompactionState.updated_at: _utcnow(), - }, - synchronize_session=False, - ) - ) - self.db.commit() - return updated == 1 - - def commit_compaction_checkpoint( - self, - snapshot: CompactionSnapshot, - *, - owner: str, - summary_text: str, - replacement_history: List[Dict[str, Any]], - replacement_manifest: Dict[str, Any], - ) -> ChatMessage: - """Atomically publish one successor from ``snapshot`` using CAS.""" - from core.llm.compaction import COMPACTION_CHECKPOINT_KIND - - if owner != snapshot.owner: - raise CompactionCASConflict("snapshot owner mismatch") - message_id = f"cmpct_{uuid.uuid4().hex[:16]}" - next_version = snapshot.base_checkpoint_version + 1 - manifest = dict(replacement_manifest) - manifest.update( - { - "source_message_ids": list(snapshot.source_message_ids), - "source_message_seqs": list(snapshot.source_message_seqs), - "source_count": len(snapshot.source_rows), - "replacement_count": len(replacement_history), - } - ) - message = self.message_repo.create( - { - "message_id": message_id, - "chat_id": snapshot.chat_id, - "role": "system", - "content": summary_text, - "extra_data": { - "kind": COMPACTION_CHECKPOINT_KIND, - "replacement_history": replacement_history, - "covered_seq": snapshot.covered_seq, - "covers_up_to_chat_seq": snapshot.covered_seq, - "covers_up_to_message_id": ( - snapshot.source_message_ids[-1] - if snapshot.source_message_ids - else None - ), - "source_hash": snapshot.source_hash, - "base_checkpoint_version": snapshot.base_checkpoint_version, - "checkpoint_version": next_version, - "replacement_manifest": manifest, - "active": True, - "notice_pending": True, - }, - }, - commit=False, - ) - now = _utcnow() - updated = ( - self.db.query(ChatCompactionState) - .filter( - ChatCompactionState.chat_id == snapshot.chat_id, - ChatCompactionState.checkpoint_version - == snapshot.base_checkpoint_version, - ChatCompactionState.covered_seq == snapshot.base_covered_seq, - ChatCompactionState.lease_owner == owner, - ) - .update( - { - ChatCompactionState.active_checkpoint_id: message_id, - ChatCompactionState.checkpoint_version: next_version, - ChatCompactionState.covered_seq: snapshot.covered_seq, - ChatCompactionState.lease_owner: None, - ChatCompactionState.lease_expires_at: None, - ChatCompactionState.updated_at: now, - }, - synchronize_session=False, - ) - ) - if updated != 1: - self.db.rollback() - raise CompactionCASConflict("checkpoint base or lease changed") - - if snapshot.base_checkpoint_id: - previous = self.db.get(ChatMessage, snapshot.base_checkpoint_id) - if previous is not None: - previous_extra = dict(previous.extra_data or {}) - previous_extra["active"] = False - previous.extra_data = previous_extra - self.db.commit() - return message - - def get_latest_compaction_checkpoint(self, chat_id: str) -> Optional[ChatMessage]: - """Return the chat's latest compaction checkpoint (None if there is none). - - The kind filter is done on the Python side to avoid depending on Postgres-specific - ``->>`` (SQLite compatible). Each checkpoint's extra_data carries the entire - replacement_history (can reach ~100KB), and rolling compaction keeps accumulating — - scan only the latest few rows, never ``.all()`` load everything. - """ - state = self.db.get(ChatCompactionState, chat_id) - if state is not None: - if not state.active_checkpoint_id: - return None - from sqlalchemy.orm import defer - checkpoint = self.db.get(ChatMessage, state.active_checkpoint_id, - options=[defer(ChatMessage.model_steps), defer(ChatMessage.tool_calls)]) - if ( - checkpoint is not None - and (checkpoint.extra_data or {}).get("covered_seq") is not None - ): - return checkpoint - return None - - checkpoint = self._latest_compaction_checkpoint_fallback(chat_id) - if ( - checkpoint is None - or (checkpoint.extra_data or {}).get("covered_seq") is None - ): - return None - return checkpoint - - def get_message_by_id(self, message_id: str) -> Optional[ChatMessage]: - """Get a single message by its ID.""" - return ( - self.db.query(ChatMessage) - .filter( - ChatMessage.message_id == message_id, - ) - .first() - ) - - def get_message_by_index(self, chat_id: str, index: int) -> Optional[ChatMessage]: - """Get a message by its durable 0-based position in the chat.""" - return ( - self.db.query(ChatMessage) - .filter( - ChatMessage.chat_id == chat_id, - ) - .order_by(ChatMessage.chat_seq) - .offset(index) - .limit(1) - .first() - ) - - def get_user_message_before( - self, chat_id: str, message_id: str - ) -> Optional[ChatMessage]: - """Get the user message immediately before the given message.""" - target = self.get_message_by_id(message_id) - if not target: - return None - return ( - self.db.query(ChatMessage) - .filter( - ChatMessage.chat_id == chat_id, - ChatMessage.role == "user", - ChatMessage.chat_seq < target.chat_seq, - ) - .order_by(ChatMessage.chat_seq.desc()) - .first() - ) - - def update_message_extra_data( - self, - message_id: str, - patch: Dict[str, Any], - ) -> bool: - """Merge *patch* into a message's extra_data. Returns True on success.""" - return self.message_repo.update_extra_data(message_id, patch) is not None - - def accept_ontology_revision(self, message_id: str) -> Optional[ChatMessage]: - """Replace an assistant message with its persisted ontology revision candidate.""" - message = self.message_repo.get_by_id(message_id) - if not message or message.role != "assistant": - return None - extra_data = dict(message.extra_data or {}) - governance = extra_data.get("ontology_governance") - if not isinstance(governance, dict): - return None - review = governance.get("review") - if not isinstance(review, dict): - return None - candidate = normalize_revision_candidate(review.get("candidate_answer")) - if not is_substantive_revision(candidate): - return None - updated_review = {**review, "candidate_answer": candidate, "accepted": True} - extra_data["ontology_governance"] = {**governance, "review": updated_review} - # Some pure unit adapters exercise this method without a database. - if not hasattr(self, "db"): - return self.message_repo.update( - message_id, - {"content": candidate, "extra_data": extra_data}, - ) - - updated = self.message_repo.update( - message_id, - {"content": candidate, "extra_data": extra_data}, - commit=False, - ) - self._invalidate_compaction_lineage_for_rewrite( - message.chat_id, - rewritten_seq=int(message.chat_seq), - ) - self.db.commit() - return updated - - def search_sessions( - self, - user_id: str, - query: str, - page: int = 1, - page_size: int = 20, - scope: str = "title", - ) -> Tuple[list, int]: - """Search chat sessions by title (and optionally message content).""" - results, total = self.session_repo.search( - user_id, query, page, page_size, scope=scope - ) - return results, total diff --git a/src/backend/core/services/commands/__init__.py b/src/backend/core/services/commands/__init__.py deleted file mode 100644 index 0280cf1b..00000000 --- a/src/backend/core/services/commands/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Owned command lifecycle and bounded output storage.""" diff --git a/src/backend/core/services/desktop_cloud_bridge.py b/src/backend/core/services/desktop_cloud_bridge.py deleted file mode 100644 index 2c5860f7..00000000 --- a/src/backend/core/services/desktop_cloud_bridge.py +++ /dev/null @@ -1,960 +0,0 @@ -"""桌面双端「云端能力桥」客户端（本机侧）。 - -双端模式的本机后端通过本模块把**云端授权的 MCP 工具**合并进本机 Agent 装配： - - 桌面壳登录云端 → 换取 capability token → 推送 {cloud_base, token} 到本机 - （POST /v1/desktop/capability/cloud-bridge，CONFIG_TOKEN=桥接秘密） - → 本模块拉取云端 manifest（当前用户最终可用的 MCP 清单）——只在登录、 - 云端能力被改动、以及云端在网关调用里告知能力已变时拉取，没有定时轮询 - → catalog_resolver 解析 enabled_mcp_ids 时把云端 server 追加进清单并按 - 完整 server_id 解析来源（同名候选显式裁决），agent 装配时 - 直接使用 manifest 内完整 schema 注册虚拟 MCP 工具；模型真正调用后 - 才通过普通 JSON 网关在云端网络内执行真实 MCP。 - -硬边界（与设计文档一致）： -- 本机拿不到云端真实 MCP URL / 密钥——只有网关地址 + capability token； -- 云端断线不会阻塞 Agent 装配；云端工具真正被调用时会返回明确错误， - **不**静默回退本机同名旧实现； -- 工具绑定按当前账号授权清单及显式来源选择解析，不使用工具名称保留名单。 - -纯本机模式（未配桥）与云端部署（无桥接秘密）零行为变化。 -""" - -from __future__ import annotations - -from contextlib import contextmanager -import hashlib -import logging -import threading -import time -from typing import Any, Dict, List, Optional, Set, Tuple - -from core.auth.desktop_bridge import bridge_enabled - -logger = logging.getLogger(__name__) - -BRIDGE_BLOCK_ID = "desktop_cloud_bridge" - -# 拿不到清单时的有界重试间隔（唯一的时间参数；正常情况下不会用到）。 -_MANIFEST_NEG_TTL_S = 30.0 - -_state_lock = threading.RLock() -_state: Optional[Dict[str, Any]] = None # {"cloud_base", "token", "expires_at"} -_credential_rejected = "" # account fingerprint whose current token the cloud refused -_state_loaded = False - -_manifest_lock = threading.Lock() -_tool_sync_lock = threading.Lock() -_manifest: Optional[Dict[str, Any]] = None -_manifest_ts: float = 0.0 -_manifest_error: Optional[str] = None -_manifest_fetching = False -_refresh_pending = False # a forced refresh arrived while a pass was running - - -def _state_fingerprint(state: Optional[Dict[str, Any]]) -> str: - """Bind a manifest snapshot to exactly one cloud/token identity.""" - if not state: - return "" - from core.services.desktop_capability_protocol import token_subject, token_claims - - token = str(state.get("token") or "") - # A rotated token for the same account keeps the snapshot; only a real - # account switch (or an opaque token) invalidates it. - claims = token_claims(token) - payload = f"{state.get('cloud_base') or ''}\0{token_subject(token) or token}\0{claims.get('a', '')}\0{claims.get('h', '')}\0{claims.get('d', state.get('device_id') or '')}" - return hashlib.sha256(payload.encode("utf-8")).hexdigest() - - -def require_current_account(state: Dict[str, Any]) -> None: - """Reject work captured before logout/account switch, including late HTTP replies.""" - from core.capabilities.errors import CloudUnavailable - - if _state_fingerprint(get_state()) != _state_fingerprint(state): - raise CloudUnavailable("cloud account changed; retry with the current account") - - -@contextmanager -def account_scope(state: Dict[str, Any]): - """Serialize a short publication with identity changes; never hold across network IO.""" - with _state_lock: - require_current_account(state) - yield - - -def ensure_current_authorization(state: Optional[Dict[str, Any]] = None) -> None: - """账号仍然有效才允许使用私有字节；这条判定不发网络请求。 - - 本机不做「事前探测」——云端清单只在登录、云端能力被改动（前端写操作后调 - ``POST /v1/desktop/capabilities/sync``）以及云端在网关调用里明确告知能力已变 - 这三种事件下同步，没有任何定时轮询。真正的撤权由云端在网关调用时裁决：返回 - 401/403 会立刻清掉本机的桥状态，被撤权的能力下一次调用即失败。 - """ - from core.capabilities.errors import CloudUnavailable - - st = state or get_state() - if not st: - raise CloudUnavailable("cloud authorization expired; sign in again") - require_current_account(st) - - -def notify_cloud_changed() -> None: - """云端能力可能已变（前端写操作 / 网关告知）：后台同步一次清单与本机文件。""" - _refresh_manifest_async(force=True) - - -# ── 桥状态（仅内存；壳启动后推送短时运行凭据） ──────────────────── - - -def _purge_persisted_state() -> None: - """Remove credentials left by an earlier release; bridge tokens are memory-only.""" - try: - from core.services.desktop_model_credentials import scrub_legacy_rows - - scrub_legacy_rows() - except Exception as exc: - logger.warning( - "[cloud-bridge] legacy model credential cleanup unavailable: %s", type(exc).__name__ - ) - try: - from core.db.engine import SessionLocal - from core.db.models import ContentBlock - - with SessionLocal() as db: - row = db.get(ContentBlock, BRIDGE_BLOCK_ID) - if row is not None: - db.delete(row) - db.commit() - except Exception as exc: - logger.warning( - "[cloud-bridge] legacy credential cleanup unavailable: %s", type(exc).__name__ - ) - - -def _load_state_from_db() -> None: - _purge_persisted_state() - return None - - -def current_credentials(state: Dict[str, Any]) -> Dict[str, Any]: - """The credential belongs to the account, not to the snapshot a pass started with. - - A sync pass can outlive the token it started with while the shell has already - renewed it. Same account → newest token; another account → untouched, so the - surrounding identity checks reject it. - """ - with _state_lock: - live = _state - if live and _state_fingerprint(live) == _state_fingerprint(state): - return live - return state - - -def note_rejected_credential(state: Dict[str, Any], exc: BaseException) -> None: - """Only a refusal of the live token schedules re-sync on its next renewal.""" - global _credential_rejected - import httpx - - if isinstance(exc, httpx.HTTPStatusError) and exc.response.status_code == 401: - authorization = exc.request.headers.get("Authorization") - with _state_lock: - if ( - _state - and authorization == f"Bearer {_state.get('token')}" - and _state_fingerprint(_state) == _state_fingerprint(state) - ): - _credential_rejected = _state_fingerprint(state) - - -def cloud_headers(state: Dict[str, Any]) -> Dict[str, str]: - from core.services.desktop_capability_protocol import token_claims - - state = current_credentials(state) - device_id = str( - state.get("device_id") or token_claims(str(state.get("token") or "")).get("d") or "" - ) - headers = {"Authorization": f"Bearer {state['token']}"} - if device_id: - headers["X-Desktop-Device-Id"] = device_id - return headers - - -def clear_state() -> None: - """Logout invalidates the account immediately, including in-flight responses.""" - global _state, _state_loaded, _manifest, _manifest_error, _manifest_ts, _credential_rejected - with _state_lock: - _state, _state_loaded = None, True - _credential_rejected = "" - _clear_partial_selection() - with _manifest_lock: - _manifest, _manifest_error, _manifest_ts = None, None, 0.0 - from core.services import desktop_cloud_bundles, desktop_cloud_skills - - desktop_cloud_skills.on_account_switch() - desktop_cloud_bundles.on_account_switch() - _purge_persisted_state() - _rebuild_identity_views() - - -def _rebuild_identity_views() -> None: - from core.capabilities import skills - from core.capabilities.paths import capabilities_enabled - - if not capabilities_enabled(): - return - from core.agent_skills.cache_refresh import refresh_skill_caches - - refresh_skill_caches() - shared = skills.device_view_dir() - root = shared.parent / (shared.name + "_u") - if root.is_dir(): - for child in root.iterdir(): - if child.is_dir(): - skills.rebuild_user_view(child.name) - - -def get_state() -> Optional[Dict[str, Any]]: - global _state, _state_loaded - with _state_lock: - if not _state_loaded: - _state = _load_state_from_db() - _state_loaded = True - st = _state - if not st: - return None - if float(st.get("expires_at") or 0) < time.time(): - return None - return dict(st) - - -def shell_user_center_id(cloud_base: str, user_center_id: str) -> str: - """壳送来的桥接用户标识：``cloud:::<云端 user_center_id>``。 - - 壳按云端地址给用户加命名空间，同一台机器连不同云端时本机身份天然隔离。本机侧 - 用完全相同的规则从凭据里的云端身份推出壳会送来的标识，两者必须逐字相同。 - 端口缺省与壳一致：https 443、其它 80。 - """ - from urllib.parse import urlsplit - - parts = urlsplit(cloud_base.strip()) - host = parts.hostname or "cloud" - port = parts.port or (443 if parts.scheme == "https" else 80) - return f"cloud:{host}:{port}:{user_center_id}" - - -def get_identity_state() -> Optional[Dict[str, Any]]: - """Last shell-authenticated identity for offline local resources; never a cloud grant.""" - from core.services.desktop_capability_protocol import token_claims - - with _state_lock: - st = dict(_state) if _state else None - if not st: - return None - claims = token_claims(str(st.get("token") or "")) - if not claims.get("c"): - return None - return { - "cloud_base": st["cloud_base"], - "user_center_id": str(claims["c"]), - "shell_user_center_id": shell_user_center_id(str(st["cloud_base"]), str(claims["c"])), - "subject": str(claims.get("u") or ""), - "device_id": str(claims.get("d") or ""), - "authorization_epoch": claims.get("a"), - } - - -def set_state( - cloud_base: str, - token: str, - expires_in: int, - *, - device_id: Optional[str] = None, - authorization_epoch: Optional[int] = None, -) -> None: - """壳侧推送桥配置（幂等）。立即触发一次后台 manifest 刷新。""" - global _manifest, _manifest_error, _manifest_ts, _state, _state_loaded, _credential_rejected - payload = { - "cloud_base": cloud_base.strip().rstrip("/"), - "token": token.strip(), - "expires_at": time.time() + max(1, int(expires_in or 0)), - "device_id": device_id, - "authorization_epoch": authorization_epoch, - } - from core.services.desktop_capability_protocol import token_claims - - claims = token_claims(token) - if device_id and claims.get("d") and device_id != claims["d"]: - raise ValueError("desktop device identity does not match token") - if claims.get("e"): - payload["expires_at"] = min(payload["expires_at"], float(claims["e"])) - with _state_lock: - changed = _state_fingerprint(_state) != _state_fingerprint(payload) - renewed_after_rejection = not changed and _credential_rejected == _state_fingerprint(payload) - _credential_rejected = "" - _state = payload - _state_loaded = True - if changed: - _clear_partial_selection() - # Never expose a previous cloud account's tools while the replacement - # dynamic manifest is still in flight. Skill files stay in the previous - # account's isolated profile directory; only the runtime view changes. - with _manifest_lock: - _manifest = None - _manifest_ts = 0.0 - _manifest_error = None - from core.services import desktop_cloud_bundles, desktop_cloud_skills - - desktop_cloud_skills.on_account_switch() - desktop_cloud_bundles.on_account_switch() - _purge_persisted_state() - if changed: - _rebuild_identity_views() - _refresh_manifest_async(force=True) - elif renewed_after_rejection: - _refresh_manifest_async(force=True) - - -def bridge_active() -> bool: - """本进程是否应启用云端能力桥（仅桌面壳孵化的双端本机后端为 True）。""" - return bridge_enabled() and get_state() is not None - - -# ── manifest 拉取（后台线程，不阻塞事件循环） ────────────────────────── - - -def synced_manifest() -> Optional[Dict[str, Any]]: - """Read the last accepted connector snapshot without starting a refresh.""" - import copy - - with _manifest_lock: - return copy.deepcopy(_manifest) - - -def sync_tool_manifest_blocking(st: Dict[str, Any]) -> None: - """Refresh connectors for both login and explicit synchronization.""" - with _tool_sync_lock: - _sync_tool_manifest(st) - - -def _sync_tool_manifest(st: Dict[str, Any]) -> None: - global _manifest, _manifest_ts, _manifest_error - from core.capabilities.errors import CloudUnavailable - - require_current_account(st) - url = f"{st['cloud_base']}/api/v1/desktop/capability/manifest" - import httpx - from core.services.desktop_capability_protocol import validate_manifest - - headers = {**cloud_headers(st), "Cache-Control": "no-cache"} - with _manifest_lock: - current_revision = str((_manifest or {}).get("revision") or "") - if current_revision: - headers["If-None-Match"] = f'"{current_revision}"' - - resp = httpx.get( - url, - headers=headers, - timeout=httpx.Timeout(10.0, connect=5.0), - ) - require_current_account(st) - if resp.status_code in (401, 403): - with account_scope(st): - clear_state() - raise CloudUnavailable("cloud authorization unavailable") - if resp.status_code == 304: - with account_scope(st), _manifest_lock: - _manifest_ts = time.monotonic() - _manifest_error = None - else: - resp.raise_for_status() - body = resp.json() - data = body.get("data") if isinstance(body, dict) else None - manifest = validate_manifest(data) - with account_scope(st): - with _manifest_lock: - _manifest = manifest - _manifest_ts = time.monotonic() - _manifest_error = None - _project_managed_profile(st, manifest) - logger.info( - "[cloud-bridge] manifest refreshed revision=%s servers=%d", - manifest["revision"][:12], - len(manifest["servers"]), - ) - - -def _fetch_manifest_blocking(st: Dict[str, Any]) -> None: - global _manifest, _manifest_ts, _manifest_error, _manifest_fetching, _refresh_pending - refresh_replacement = False - try: - sync_tool_manifest_blocking(st) - # Skills, agents and plugins ride the same poll and token: their intent - # is reconciled right after the tool manifest is known to be current. - sync_capabilities_blocking(st) - except Exception as exc: # noqa: BLE001 - with _state_lock: - if _state_fingerprint(get_state()) != _state_fingerprint(st): - refresh_replacement = True - return - with _manifest_lock: - _manifest_ts = time.monotonic() - _manifest_error = str(exc) - logger.warning("[cloud-bridge] manifest 拉取失败: %s", exc) - finally: - with _manifest_lock: - _manifest_fetching = False - rerun = _refresh_pending - _refresh_pending = False - if refresh_replacement or rerun: - _refresh_manifest_async(force=True) - - -def sync_capabilities_blocking(st: Dict[str, Any]) -> None: - """One pass over every cloud-synced kind (skills, agents, plugins). Blocking; call off-loop. - - Both the background manifest poll and the user-triggered sync endpoint go - through here so no kind can be forgotten by one of the two paths. - """ - from core.services import desktop_cloud_bundles, desktop_cloud_skills - - desktop_cloud_skills.sync_blocking(st) - desktop_cloud_bundles.sync_blocking(st) - try: - from core.capabilities.warmup import warmup_current_account - - warmup_current_account(st) - except Exception as exc: # Best effort: ordinary execution retains all checks. - logger.warning("[caps] startup reads deferred: %s", type(exc).__name__) - - -def _refresh_manifest_async(force: bool = False) -> None: - """后台同步一次云端清单。没有定时器：``force`` 由登录 / 变更通知触发，非 - ``force`` 只是「还没有可用清单」时的有界重试（离线登录后的自愈），一旦拿到 - 清单就不再有任何后台请求。""" - global _manifest_fetching, _refresh_pending - st = get_state() - if not st: - return - with _manifest_lock: - if _manifest_fetching: - _refresh_pending = _refresh_pending or force - return - if not force: - if _manifest is not None and not _manifest_error: - return - if _manifest_ts and time.monotonic() - _manifest_ts < _MANIFEST_NEG_TTL_S: - return - _manifest_fetching = True - threading.Thread( - target=_fetch_manifest_blocking, args=(st,), name="cloud-bridge-manifest", daemon=True - ).start() - - -def get_cached_manifest() -> Optional[Dict[str, Any]]: - """返回缓存的 manifest（可能为 None）。调用方只读，不得修改。 - - 还没有清单时（离线登录等）触发一次有界重试；已有清单则不发任何请求，更新 - 靠登录与变更通知。不做激活判定——守门统一在公共入口（``_bridge_context``）。 - 清单只在 ``_fetch_manifest_blocking`` 里整体替换，从不原地修改，所以可以安全共享。 - """ - _refresh_manifest_async() - with _manifest_lock: - return _manifest - - -# ── 混合能力解析（解析器裁决绑定；mcp.json 投影） ────────────────────── - - -def _account_profile(st: Dict[str, Any]) -> str: - """Profile label of the bridged account (opaque tokens get a deterministic label).""" - from core.capabilities.ref import profile_id - from core.services.desktop_capability_protocol import token_subject - - return profile_id( - str(st["cloud_base"]), token_subject(str(st.get("token") or "")) or "opaque-subject" - ) - - -def _project_managed_profile(st: Dict[str, Any], manifest: Dict[str, Any]) -> None: - """Write the account's cloud connector references into mcp.json (managedProfiles).""" - from core.capabilities import mcp_json - from core.capabilities.paths import capabilities_enabled - from core.capabilities.ref import cloud_issuer - - if not capabilities_enabled(): - return - try: - mcp_json.project_managed_profile( - _account_profile(st), - cloud_instance_id=cloud_issuer(str(st["cloud_base"])), - catalog_revision=str(manifest["revision"]), - servers=list(manifest.get("servers") or []), - initial_enabled={ - str(s["server_id"]): bool(s.get("enabled", True)) - for s in manifest.get("servers") or [] - if isinstance(s, dict) and s.get("server_id") - }, - ) - except mcp_json.McpJsonCorrupt as exc: - logger.error("[cloud-bridge] mcp.json unreadable, projection skipped: %s", exc) - except Exception as exc: # noqa: BLE001 - logger.warning("[cloud-bridge] mcp.json projection failed: %s", exc) - - -def _managed_enabled(st: Dict[str, Any]) -> Dict[str, bool]: - from core.capabilities import mcp_json - from core.capabilities.paths import capabilities_enabled - - if not capabilities_enabled(): - return {} - try: - return mcp_json.managed_enabled(_account_profile(st)) - except mcp_json.McpJsonError as exc: - logger.warning("[cloud-bridge] mcp.json managed flags unavailable: %s", exc) - return {} - - -def managed_connectors() -> List[Dict[str, Any]]: - """云端下发的连接器全集（含本机停用的），供本机能力中心展示。 - - ``_bridge_context`` 是装配用的，它按本机停用标志把关掉的过滤掉了——界面要的是 - 「有哪些」，关掉的也得列出来，开关才有地方可点。启停仍以本机 mcp.json 为准。 - - 只读已缓存的清单，不触发任何刷新：这是界面读取路径，同步有它自己的时机。 - """ - st = get_state() - if not st or not bridge_enabled(): - return [] - with _manifest_lock: - manifest = _manifest - if not manifest: - return [] - enabled = _managed_enabled(st) - out: List[Dict[str, Any]] = [] - for server in manifest.get("servers") or []: - if not isinstance(server, dict): - continue - sid = str(server.get("server_id") or "").strip() - if not sid: - continue - out.append( - { - "server_id": sid, - "display_name": str(server.get("display_name") or sid), - "description": str(server.get("description") or ""), - "created_at": server.get("created_at"), - "enabled": bool(enabled.get(sid, True)), - "source_plugin": str(server.get("source_plugin") or ""), - "icon": str(server.get("icon") or ""), - "tools": [ - { - "name": str(tool.get("name") or ""), - "description": str(tool.get("description") or ""), - } - for tool in server.get("tools") or [] - if isinstance(tool, dict) - ], - } - ) - return out - - -def set_managed_connector_enabled(server_id: str, enabled: bool) -> bool: - """记下用户对某个云端下发连接器的启停，落本机 mcp.json。 - - 本机是这个开关的唯一记账处：装配走 ``_bridge_context()``，它就按这里的标志 - 过滤，云端不参与也不需要回写。返回 False 表示该 server_id 不是本机托管的 - 云端连接器，调用方应继续按普通目录项处理。 - - 「数据库查询」是个伞形条目（见 ``device_catalog._db_umbrella_item``），本身没有 - 对应的 server，开关要落到它收拢的那几个成员上——否则界面上的开关点了没有任何 - 东西真的被打开。 - """ - from core.capabilities import mcp_json - from core.capabilities.paths import capabilities_enabled - from core.config.catalog_loader import DB_HIDDEN_SERVERS, DB_UMBRELLA_ID - - st = get_state() - if not capabilities_enabled() or not bridge_enabled() or not st: - return False - profile = _account_profile(st) - if not profile: - return False - sid = str(server_id) - if sid == DB_UMBRELLA_ID: - members = [ - str(s["server_id"]) - for s in managed_connectors() - if str(s.get("server_id")) in DB_HIDDEN_SERVERS - ] - if not members: - return False - for member in members: - try: - mcp_json.set_managed_enabled(profile, member, bool(enabled)) - except mcp_json.McpJsonError: - return False - return True - try: - mcp_json.set_managed_enabled(profile, sid, bool(enabled)) - except mcp_json.McpJsonError: - return False - return True - - -_context_lock = threading.Lock() -_context_cache: Optional[Tuple[tuple, Dict[str, Any]]] = None - - -def _bridge_context() -> Optional[Dict[str, Any]]: - """唯一守门点：桥激活且 manifest 就绪时返回上下文，否则 None。 - - 返回 {"state": st, "servers": [含 component 的云端 server，已剔除 mcp.json 里 - 用户停用的项], "profile": 账号 profile}。同名裁决交给解析器。 - server 列表按 (账号, 清单 revision, 停用集合) 记忆化；``state`` 每次现取， - 因为它携带轮换中的 token。 - """ - global _context_cache - if not bridge_enabled(): - return None - st = get_state() - if not st: - return None - manifest = get_cached_manifest() - if not manifest: - return None - enabled = _managed_enabled(st) - key = (_state_fingerprint(st), str(manifest["revision"]), tuple(sorted(enabled.items()))) - with _context_lock: - hit = _context_cache - if hit is None or hit[0] != key: - servers: List[Dict[str, Any]] = [] - for s in manifest.get("servers") or []: - if not isinstance(s, dict): - continue - # A server with no current tool contracts cannot replace a local - # implementation. The manifest validator already guarantees every - # non-empty entry contains complete, cloud-supplied schemas. - if not s.get("tools"): - continue - sid = str(s.get("server_id") or "").strip() - if not sid or not enabled.get(sid, True): - continue - entry = dict(s) - entry["server_id"] = sid - entry["component"] = str(s.get("component") or sid).strip() - servers.append(entry) - hit = ( - key, - { - "servers": servers, - "manifest_revision": str(manifest["revision"]), - "profile": _account_profile(st), - }, - ) - with _context_lock: - _context_cache = hit - if not hit[1]["servers"]: - return None - return {"state": st, **hit[1]} - - -def _mcp_json_local_configs() -> Dict[str, dict]: - from core.capabilities import mcp_json - from core.capabilities.paths import capabilities_enabled - - if not capabilities_enabled(): - return {} - try: - return mcp_json.local_server_configs() - except mcp_json.McpJsonError as exc: - logger.warning("[cloud-bridge] mcp.json local servers unavailable: %s", exc) - return {} - - -def cloud_gateway_mcp_configs( - enabled_mcp_ids: Optional[List[str]] = None, *, resolution_out=None, unavailable_out=None -) -> Dict[str, dict]: - """Desktop-only MCP config sources: cloud gateway bindings + mcp.json local servers. - - Only the components the resolver chose for the cloud binding get a gateway - config; the run's enabled-id allowlist still collapses the final set. - """ - from core.capabilities import connectors - - configs: Dict[str, dict] = dict(_mcp_json_local_configs()) - ctx = _bridge_context() - res = _resolve_mcp_bindings(enabled_mcp_ids, ctx) - if resolution_out is not None: - resolution_out.append(res) - requested = set(enabled_mcp_ids or []) - for name, group in {**res.conflicts, **res.unusable}.items(): - if name in requested or any(connectors.server_id_of(c) in requested for c in group): - from core.capabilities.errors import NameConflict, PackageMissing - - error = NameConflict if name in res.conflicts else PackageMissing - if unavailable_out is not None: - unavailable_out.update({connectors.server_id_of(c): error.code for c in group}) - continue - raise error( - "selected connector binding is unavailable; choose its source", runtime_name=name - ) - chosen = res.chosen_ids() - configs = { - sid: cfg - for sid, cfg in configs.items() - if f"mcp:{connectors.MCP_JSON_PROFILE}:{sid}" in chosen - } - if not ctx: - return configs - st = ctx["state"] - manifest_revision = ctx["manifest_revision"] - for s in ctx["servers"]: - # Source selection is complete; only the exact authorized cloud binding - # chosen by the resolver may produce a gateway config. - sid = s["server_id"] - if f"mcp:{ctx['profile']}:{sid}" not in chosen: - continue - from core.services.desktop_resource_plugins import local_config - from core.capabilities.skills import current_local_user_id - native = local_config(s, current_local_user_id(), ctx["profile"]) - if native is not None: - configs[sid] = native - continue - invoke_url = f"{st['cloud_base']}/api/v1/desktop/capability/gateway/{sid}/call" - configs[sid] = { - "transport": "streamable_http", - # HttpMCPConfig requires a URL, but ManifestMCPClient never performs - # discovery against it. The same JSON endpoint is the only remote hop. - "url": invoke_url, - "headers": cloud_headers(st), - "execution_timeout": 180, - "is_stable": False, - "schema_source": "cloud_manifest", - "manifest_revision": manifest_revision, - "manifest_tools": s["tools"], - "schema_hash": str(s["schema_hash"]), - "gateway_invoke_url": invoke_url, - "gateway_plugin": str(s.get("source_plugin") or ""), - } - return configs - - -def _local_server_ids() -> Set[str]: - """本机 DB 内全部 MCP 的 server_id。 - - 复用 ``McpServerConfigService`` 自带的 30s 缓存与失效链路，不另建缓存。 - """ - try: - from core.services.mcp_service import McpServerConfigService - - return set(McpServerConfigService.get_instance().get_all_servers(enabled_only=False)) - except Exception as exc: # noqa: BLE001 - logger.debug("[cloud-bridge] local server ids failed: %s", exc) - return set() - - -def apply_to_enabled_mcp_ids(mcp_ids: Optional[List[str]]) -> Optional[List[str]]: - """把本轮 enabled_mcp_ids 交给解析器，按 server_id 裁决每个连接器的绑定。 - - 候选：本机目录行（device）、云端 manifest（account）、mcp.json 本机声明。 - 同一 server_id 多个候选时，云端账号级候选默认胜出， - 用户可用同名偏好显式改选；落选绑定记为 shadowed，可在界面 - 切换，而不是被静默顶掉。云端 manifest 尚未就绪 / 桥未激活：原样返回。 - 顺序稳定（已选项按原顺序，新增云端与 mcp.json 项追加），幂等。 - """ - if mcp_ids is None: - return None - ctx = _bridge_context() - from core.capabilities import connectors - - local_ids = _local_server_ids() - json_local = _mcp_json_local_declarations() - res = _resolve_mcp_bindings(mcp_ids, ctx) - - chosen_ids = {connectors.server_id_of(c) for c in res.chosen.values()} - known = local_ids | {s["server_id"] for s in (ctx or {}).get("servers", [])} | set(json_local) - kept = [mid for mid in mcp_ids if mid in chosen_ids or mid not in known] - existing = set(kept) - for name in sorted(res.chosen): - sid = connectors.server_id_of(res.chosen[name]) - if sid not in existing: - kept.append(sid) - existing.add(sid) - return kept - - -def _resolve_mcp_bindings(mcp_ids: Optional[List[str]], ctx): - from core.capabilities import connectors - - local_ids = _local_server_ids() - enabled = set(local_ids) if mcp_ids is None else set(mcp_ids) - candidates = connectors.db_candidates(local_ids, enabled) + connectors.json_candidates( - _mcp_json_local_declarations() - ) - if ctx: - candidates += connectors.cloud_candidates(ctx["profile"], ctx["servers"], {}) - return connectors.resolve_bindings(candidates) - - -def _mcp_json_local_declarations() -> Dict[str, Dict[str, Any]]: - from core.capabilities import mcp_json - from core.capabilities.paths import capabilities_enabled - - if not capabilities_enabled(): - return {} - try: - return mcp_json.load().local - except mcp_json.McpJsonError: - return {} - - -def apply_to_enabled_skill_ids(skill_ids: Optional[List[str]]) -> Optional[List[str]]: - """把云端技能合并进本轮 enabled_skill_ids（云端为真源，见 desktop_cloud_skills）。""" - if skill_ids is None or not bridge_active(): - return skill_ids - from core.services import desktop_cloud_skills - - return desktop_cloud_skills.apply_to_enabled_skill_ids(skill_ids) - - -def _clear_partial_selection(): - from core.capabilities import session_availability, registry - if session_availability.active(): - session_availability.clear() - registry._bump() # Runtime selection changed even though cloud intent did not. - - -def retry_initial_sync(st): - from core.capabilities import skills - with account_scope(st): - _clear_partial_selection() - skills.bump_view_generation() - _refresh_manifest_async(force=True) - - -def initial_sync_status() -> Dict[str, Any]: - """Report actual package counts without disk hashing or network probes.""" - from core.services import desktop_cloud_skills, desktop_cloud_bundles - from core.capabilities import session_availability - from core.capabilities.paths import revision_for_hash - - st = get_state() - if not st: - return {"ready": False, "syncing": False, "partial": False, "groups": [], - "pending": 0, "error": "云端身份尚未就绪", "can_continue": False} - with _manifest_lock: - manifest_ready = _manifest is not None - fetching = _manifest_fetching - error = _manifest_error - groups = {"skill": desktop_cloud_skills.status(), **desktop_cloud_bundles.status()} - pending, completed, total = 0, 0, 0 - progress = [] - for kind, group in groups.items(): - error = error or group.get("last_error") - rows = [row for row in group.get("installations", []) if row.get("state") != "removed"] - done = sum(bool(row.get("content_hash")) and row.get("state") == "ready" - and row.get("resolved_revision") == revision_for_hash(row["content_hash"]) - for row in rows) - failed = sum(row.get("state") == "failed" or bool(row.get("last_error")) for row in rows) - completed += done - total += len(rows) - pending += len(rows) - done - progress.append({"kind": kind, "total": len(rows), "completed": done, - "failed": failed, "manifest_ready": bool(group.get("revision"))}) - complete_manifests = all(group.get("revision") for group in groups.values()) - partial = session_availability.active(desktop_cloud_skills._profile(st)) - # Progress is read without the identity lock: package rows may wait on the - # database, and the shell's token push must never queue behind that wait. - require_current_account(st) - complete = (manifest_ready and not fetching and not error and pending == 0 and complete_manifests) - if not fetching and not complete and not partial: - error = error or "部分能力未能同步完成，请选择重试或使用已同步能力继续" - return {"ready": bool(complete or partial), "syncing": fetching, "partial": partial, - "groups": progress, "completed": completed, "total": total, - "totals_known": bool(complete_manifests), "pending": pending, - "error": error, "can_continue": bool(manifest_ready and not fetching and not complete)} - - -def bridge_status() -> Dict[str, Any]: - """诊断视图（/v1/desktop/capability/cloud-bridge/status）。""" - from core.services import desktop_cloud_skills - - st = get_state() - with _manifest_lock: - servers = (_manifest or {}).get("servers") or [] - err = _manifest_error - manifest_revision = str((_manifest or {}).get("revision") or "") - schema_ready_count = sum( - 1 for server in servers if isinstance(server, dict) and bool(server.get("tools")) - ) - return { - "configured": st is not None, - "cloud_base": (st or {}).get("cloud_base"), - "active": bridge_active(), - "cloud_server_count": len(servers), - "manifest_revision": manifest_revision, - "schema_mode": "dynamic_manifest" if manifest_revision else "none", - "schema_ready_server_count": schema_ready_count, - "cloud_servers": [ - { - "server_id": s.get("server_id"), - "component": s.get("component"), - "display_name": s.get("display_name"), - "origin": "cloud", - } - for s in servers - if isinstance(s, dict) - ], - "last_error": err, - "skills": desktop_cloud_skills.status(), - "bundles": _bundles_status(), - "mcp_json": _mcp_json_status(), - } - - -def _bundles_status() -> Dict[str, Any]: - from core.services import desktop_cloud_bundles - - return desktop_cloud_bundles.status() - - -def _mcp_json_status() -> Dict[str, Any]: - from core.capabilities import connectors, mcp_json - from core.capabilities.paths import capabilities_enabled - - if not capabilities_enabled(): - return {"enabled": False} - try: - doc = mcp_json.load() - except mcp_json.McpJsonCorrupt as exc: - return {"enabled": True, "corrupt": True, "error": str(exc)} - res = connectors.last_resolution() - return { - "enabled": True, - "generation": doc.generation, - "local_server_count": len(doc.local), - "managed_profiles": { - p: len((v or {}).get("servers") or {}) for p, v in doc.managed.items() - }, - "conflicts": sorted(res.conflicts) if res else [], - "shadowed": sorted(res.shadowed) if res else [], - } - - -def reset_for_tests() -> None: # pragma: no cover - 仅测试用 - global _state, _state_loaded, _manifest, _manifest_ts, _manifest_error, _manifest_fetching - global _credential_rejected, _refresh_pending - from core.services import desktop_cloud_skills - - with _state_lock: - _state = None - _state_loaded = False - _credential_rejected = "" - with _manifest_lock: - _manifest = None - _manifest_ts = 0.0 - _manifest_error = None - _manifest_fetching = False - _refresh_pending = False - desktop_cloud_skills.reset_for_tests() diff --git a/src/backend/core/services/edition_system_config.py b/src/backend/core/services/edition_system_config.py deleted file mode 100644 index cefd7c72..00000000 --- a/src/backend/core/services/edition_system_config.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Community edition contributes no enterprise service configuration.""" - -SEED_CONFIGS = [] -CONFIG_KEY_TO_ENV = {} - -__all__ = ["CONFIG_KEY_TO_ENV", "SEED_CONFIGS"] diff --git a/src/backend/core/services/edition_workspace.py b/src/backend/core/services/edition_workspace.py deleted file mode 100644 index 7b9cf653..00000000 --- a/src/backend/core/services/edition_workspace.py +++ /dev/null @@ -1,52 +0,0 @@ -"""Community Edition has no additional workspace volumes.""" - - - - - - - -def project_directory(scope): - return "" - - -def project_directory_by_id(project_id): - return "" - - -def workspace_instructions(): - return "" - - -def is_organization_path(scope, actor, path): - return False - - - - -def resolve_alias(path): - return None - - -def resolve_project_path(path, user_id, root, scope): - return None - - -def is_shared_scope(scope): - return False - - -def context_directory(context): - return "" - - -def source_directory(project): - return "" - - -def folder_scope_label(kind): - return "我的空间" - - -def command_instructions(scope): - return "" diff --git a/src/backend/core/services/kb_edition.py b/src/backend/core/services/kb_edition.py deleted file mode 100644 index 5217dd4b..00000000 --- a/src/backend/core/services/kb_edition.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Community knowledge-base grant policy: resources are owner-only.""" - -from __future__ import annotations - -from sqlalchemy.orm import Session - - -def initial_visibility_grants(db: Session, user_id: str, visibility: str) -> list[tuple[str, str]]: - return [] - - -__all__ = ["initial_visibility_grants"] diff --git a/src/backend/core/services/lark_service.py b/src/backend/core/services/lark_service.py deleted file mode 100644 index 012509b3..00000000 --- a/src/backend/core/services/lark_service.py +++ /dev/null @@ -1,752 +0,0 @@ -"""Lark (Feishu) account connection service (feishu-cli plugin / lark-cli). - -Drives ``lark-cli auth`` **as the current user** in their dedicated backend -HOME to complete the device-flow OAuth login (scan-to-bind), and persists -the connection status / Lark identity summary into the ``lark_connections`` -table. The credentials themselves (user_access_token + file-based encrypted -store, including master.key) are persisted at ``$STORAGE/lark_cache/{uid}/`` -via ``_make_lark_creds_volumes``'s per-user bind-mount, never stored in the -DB. - -Structurally identical to dingtalk_service, with two differences (Lark is -easier + Block B decision): - 1. Device flow: ``lark-cli auth login --no-wait --json`` returns a - structured device_code + verification_url **immediately** (no need to - stream-parse a text box like dws); the device_code is persisted, then - ``--device-code`` polls to completion ("poll and complete", a single - call blocking until authorized/expired). - 2. app_id/app_secret are **not** injected as env (with ``LARK_APP_*`` in - the environment, lark-cli defaults to the tenant/bot token flow, - breaking ``--as user``). Instead, before login, ``config init - --app-secret-stdin`` seeds the app config into the per-user HOME - (which enters the sandbox together with the credential volume). - -Note: the JSON shapes of the device flow / auth status are authoritative -only per real-device testing (P0). Parsing in this module is deliberately -lenient and all subprocess interactions degrade harmlessly — any step -failure sets the connection to error / returns a hint, never raising a bare -exception that breaks the front end. -See internal design docs. -""" - -from __future__ import annotations -from core.infra.time import utc_now - -import asyncio -import json -import logging -import os -import re -import shutil -from datetime import datetime -from typing import Any, Dict, List, Optional, Tuple - -from sqlalchemy.orm import Session - -from core.db.repository import LarkConnectionRepository -from core.infra.proc import no_window_kwargs -# QR rendering is identical to dingtalk's (segno SVG data-uri); reuse it directly to avoid duplicating the implementation. -from core.services.dingtalk_service import make_qr_data_uri - -logger = logging.getLogger(__name__) - -# In-progress device-flow completion tasks (user_id → asyncio.Task). ``--device-code`` -# blocks polling until the user authorizes / it expires; run as a backend subprocess, -# surviving across HTTP requests, updating the DB itself on completion and removing -# itself from the map. -_LOGIN_TASKS: "dict[str, asyncio.Task]" = {} -_LARK_BIN = "lark-cli" - -# Default authorization scopes requested by the device flow: --recommend requests only the -# recommended (auto-approvable) scopes, following least privilege; when a business command -# hits insufficient permissions, the lark-shared skill guides the user to an incremental auth login. -_DEFAULT_LOGIN_ARGS = ["auth", "login", "--recommend", "--no-wait", "--json"] - -# The four login-state fields: filled while pending, cleared on connect success/failure/disconnect. -# Centralized as a constant to avoid missing one of the hand-copied sites. -_CLEAR_LOGIN: Dict[str, Any] = { - "login_verification_url": None, - "login_verification_url_complete": None, - "login_user_code": None, - "login_device_code": None, -} - -_EXPIRED_ERROR = "飞书登录已失效（令牌被刷新或过期），请到设置页重新扫码连接飞书" -_NO_APP_ERROR = "飞书应用尚未配置，请联系管理员在「插件 → 飞书工作台」初始化飞书应用" - -# Markers in auth/login results that positively confirm "auth invalid / device code terminated". -_AUTH_FAIL_MARKERS = ( - "expired_token", "access_denied", "invalid_grant", "invalid_device_code", - "authorization_expired", "expired", "denied", -) -_PENDING_MARKERS = ("authorization_pending", "pending", "slow_down") - - -# ── JSON parsing (lenient, best effort) ───────────────────────────────────────── - -def _loads(s: str) -> Any: - try: - return json.loads((s or "").strip() or "null") - except (ValueError, TypeError): - return None - - -def parse_device_login(stdout: str) -> Dict[str, Optional[str]]: - """``auth login --no-wait --json`` → {device_code, user_code, verification_url, - verification_url_complete}. Field names matched leniently (Lark/OAuth device-flow conventional naming).""" - out: Dict[str, Optional[str]] = { - "device_code": None, "user_code": None, - "verification_url": None, "verification_url_complete": None, - } - data = _loads(stdout) - if not isinstance(data, dict): - return out - # Some CLIs wrap the content inside data/result - for key in ("data", "result", "device", "deviceAuth"): - inner = data.get(key) - if isinstance(inner, dict): - data = {**data, **inner} - - def _g(*names: str) -> Optional[str]: - for n in names: - for k, v in data.items(): - if k.lower().replace("_", "") == n and isinstance(v, (str, int)): - return str(v) - return None - - out["device_code"] = _g("devicecode") - out["user_code"] = _g("usercode") - out["verification_url"] = _g("verificationurl", "verificationuri") - out["verification_url_complete"] = _g("verificationurlcomplete", "verificationuricomplete") - return out - - -def parse_lark_identity(stdout: str) -> Dict[str, Optional[str]]: - """``auth status``/``auth list`` output → {lark_open_id, lark_name, tenant_key} (searched recursively).""" - out: Dict[str, Optional[str]] = {"lark_open_id": None, "lark_name": None, "tenant_key": None} - data = _loads(stdout) - if data is None: - return out - - def _walk(obj: Any) -> None: - if isinstance(obj, dict): - for k, v in obj.items(): - lk = k.lower().replace("_", "") - if out["lark_open_id"] is None and lk in ("openid", "unionid", "userid"): - if isinstance(v, (str, int)): - out["lark_open_id"] = str(v) - if out["lark_name"] is None and lk in ("name", "username", "nick", "ennname", "displayname"): - if isinstance(v, str): - out["lark_name"] = v - if out["tenant_key"] is None and lk in ("tenantkey", "tenant"): - if isinstance(v, (str, int)): - out["tenant_key"] = str(v) - _walk(v) - elif isinstance(obj, list): - for it in obj: - _walk(it) - - _walk(data) - return out - - -def parse_user_identity_state(stdout: str) -> Optional[bool]: - """``auth status`` → whether the **user** identity is usable right now; - ``None`` when the output shape is unrecognized (caller falls back to heuristics). - - lark-cli reports the bot and the user identity separately:: - - {"identities": {"bot": {"status": "ready", "available": true}, - "user": {"status": "missing", "available": false, - "openId": "ou_…", "userName": "…", - "tokenStatus": "expired"}}} - - Note that ``openId``/``userName`` **stay in the payload after the token - expires** — they describe who last authorized, not whether the - authorization still works. So "an identity field is present" must never be - read as "logged in"; doing so is what let a login that expired months ago - keep reporting 已登录 on the connection panel while the sandbox correctly - answered "User identity: missing". - """ - data = _loads(stdout) - if not isinstance(data, dict): - return None - identities = data.get("identities") - if not isinstance(identities, dict): - return None - user = identities.get("user") - if not isinstance(user, dict): - return None - available = user.get("available") - if isinstance(available, bool): - return available - status = str(user.get("status") or "").strip().lower() - if not status: - return None - return status in ("ready", "ok", "active", "valid") - - -def is_authenticated(stdout: str) -> bool: - """``auth status`` → whether the user is logged in. - - Prefers the structured ``identities.user`` verdict (see - [[parse_user_identity_state]]); only when the CLI emits a shape we do not - recognize does it fall back to the old lenient heuristics — those also have - to serve ``auth login --device-code`` output, which carries no - ``identities`` block. - """ - state = parse_user_identity_state(stdout) - if state is not None: - return state - data = _loads(stdout) - if isinstance(data, dict): - for k, v in data.items(): - lk = k.lower().replace("_", "") - if lk in ("authenticated", "loggedin", "isloggedin", "valid") and v is True: - return True - ident = parse_lark_identity(stdout) - return bool(ident.get("lark_open_id") or ident.get("lark_name")) - - -def _classify_login_result(stdout: str, stderr: str, rc: int) -> str: - """Classify the result of ``auth login --device-code`` → ``ok`` / ``pending`` / ``error``.""" - blob = ((stdout or "") + "\n" + (stderr or "")).lower() - if rc == 0 and is_authenticated(stdout): - return "ok" - if any(m in blob for m in _PENDING_MARKERS): - return "pending" - if any(m in blob for m in _AUTH_FAIL_MARKERS): - return "error" - # rc==0 but no definite identity: the caller decides whether to probe status again; here conservatively treat as pending (not yet terminal) - return "ok" if rc == 0 else "pending" - - -# ── Backend subprocess environment / invocation ───────────────────────────────── - -def _lark_env(user_id: str) -> Dict[str, str]: - """Environment for running lark-cli subprocesses on the backend: per-user isolated ``HOME`` (the root of credential isolation). - - LARK_APP_ID/SECRET are **not** injected (with them in the env, lark-cli - defaults to the tenant/bot token flow, breaking --as user); the app - config lands in HOME via config init. OPENCLAW_HOME/HERMES_HOME are - stripped — otherwise ``config init`` decides it is in an Agent context - and refuses to run.""" - from core.sandbox.workspace.credentials import lark_home_dir - - home = lark_home_dir(user_id) - try: - home.mkdir(parents=True, exist_ok=True) - except Exception as exc: # noqa: BLE001 - logger.warning("[lark] mkdir home %s failed: %s", home, exc) - env = dict(os.environ) - env["HOME"] = str(home) - for k in ("OPENCLAW_HOME", "HERMES_HOME", "LARK_APP_ID", "LARK_APP_SECRET"): - env.pop(k, None) - return env - - -async def _run_lark( - user_id: str, args: List[str], timeout: int = 40, stdin_data: Optional[str] = None, -) -> Tuple[str, str, int]: - """Run one lark-cli subcommand on the backend as this user; returns (stdout, stderr, exit_code).""" - try: - proc = await asyncio.create_subprocess_exec( - _LARK_BIN, *args, - stdin=asyncio.subprocess.PIPE if stdin_data is not None else None, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - env=_lark_env(user_id), - **no_window_kwargs(), - ) - except FileNotFoundError: - return "", "lark-cli binary not found on backend", 127 - try: - out, err = await asyncio.wait_for( - proc.communicate(input=stdin_data.encode("utf-8") if stdin_data is not None else None), - timeout=timeout, - ) - return out.decode("utf-8", "replace"), err.decode("utf-8", "replace"), proc.returncode or 0 - except asyncio.TimeoutError: - try: - proc.kill() - except ProcessLookupError: - pass - return "", f"lark-cli {' '.join(args[:2])} timeout", 124 - - -def _lark_app_config() -> Tuple[Optional[str], Optional[str]]: - """Read the app's app_id / app_secret from "System Config → Lark Workspace" (deployment-level, one app shared by everyone).""" - try: - from core.services.system_config import SystemConfigService - - svc = SystemConfigService.get_instance() - app_id = (svc.get("lark.app_id") or "").strip() - app_secret = (svc.get("lark.app_secret") or "").strip() - return (app_id or None), (app_secret or None) - except Exception as exc: # noqa: BLE001 - logger.warning("[lark] read lark config failed: %s", exc) - return None, None - - -# The app base's two on-disk roots (config init writes both; copying them into the user HOME enables their auth login) -_APP_CFG_SUB = ".lark-cli/config.json" -_APP_STORE_SUB = ".local/share/lark-cli" - - -def _app_configured() -> Optional[str]: - """Whether the shared app has been initialized (the admin's one-click config init --new completed). Returns the app_id or None.""" - from core.sandbox.workspace.credentials import lark_app_home_dir - - cfg = lark_app_home_dir() / _APP_CFG_SUB - try: - data = json.loads(cfg.read_text(encoding="utf-8")) - apps = data.get("apps") or [] - if apps and isinstance(apps[0], dict): - return apps[0].get("appId") - except Exception: # noqa: BLE001 - pass - return None - - -def _seed_user_from_shared_app(user_id: str) -> bool: - """Copy the shared app base (config.json + master.key + appsecret_*.enc) into the user HOME. - - Copies only once, when the user HOME has no config.json yet (persisted - afterwards via the bind-mount). master.key and appsecret are copied - together so that user tokens written by this user's auth login are - encrypted with — and decryptable by — the same key. - User token files are never touched (those belong to each individual).""" - import shutil - - from core.sandbox.workspace.credentials import lark_app_home_dir - from core.sandbox.workspace.credentials import lark_home_dir - - app_home = lark_app_home_dir() - user_home = lark_home_dir(user_id) - user_cfg = user_home / _APP_CFG_SUB - if user_cfg.exists(): - return True # base already present - src_cfg = app_home / _APP_CFG_SUB - if not src_cfg.exists(): - return False - try: - user_cfg.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(src_cfg, user_cfg) - src_store = app_home / _APP_STORE_SUB - dst_store = user_home / _APP_STORE_SUB - dst_store.mkdir(parents=True, exist_ok=True) - for name in os.listdir(src_store): - if name == "master.key" or name.startswith("appsecret_"): - shutil.copy2(src_store / name, dst_store / name) - return True - except Exception as exc: # noqa: BLE001 - logger.warning("[lark] seed user app from shared failed user=%s: %s", user_id, exc) - return False - - -async def _ensure_app_config(user_id: str) -> Optional[str]: - """Ensure the user HOME has the app base. Returns an error message; None means ready. - - Prefer the shared app from the **admin's one-click initialization** - (copying the base into the user HOME; the whole group shares one app); - without a shared app, fall back to the app_id/secret entered in **System - Config** (``config init --app-secret-stdin``); with neither → return a - hint message (the front end guides the admin to initialize). - """ - # 1) Shared app (admin one-click initialization) - if _app_configured(): - if _seed_user_from_shared_app(user_id): - return None - # 2) app_id/secret hand-entered in System Config (fallback) - app_id, app_secret = _lark_app_config() - if app_id and app_secret: - shown, _, _ = await _run_lark(user_id, ["config", "show"], timeout=15) - if app_id in (shown or ""): - return None - _, err, rc = await _run_lark( - user_id, - ["config", "init", "--app-id", app_id, "--app-secret-stdin", "--brand", "feishu"], - timeout=30, - stdin_data=app_secret, - ) - if rc != 0: - logger.warning("[lark] config init failed user=%s rc=%s err=%s", user_id, rc, (err or "")[:200]) - return f"飞书应用配置初始化失败：{(err or '').strip()[:160] or '未知错误'}" - return None - return _NO_APP_ERROR - - -def _update_connection(user_id: str, data: Dict[str, Any]) -> None: - """Update the connection record with an independent DB session (for the background login task — must not borrow the request-scoped session).""" - from core.db.engine import SessionLocal - - try: - with SessionLocal() as db: - LarkConnectionRepository(db).update(user_id, data) - except Exception as exc: # noqa: BLE001 - logger.warning("[lark] _update_connection user=%s failed: %s", user_id, exc) - - -async def _build_connected_update(user_id: str) -> Dict[str, Any]: - """After successful login, backfill the Lark identity and return the update dict for the ``connected`` state.""" - # No ``--format json``: no shipped lark-cli accepts that flag (1.0.34 has only - # ``--verify``; 1.0.78 spells it ``--json``), and passing it made every call return a - # "unknown flag" error instead of the status — which is why lark_open_id / lark_name - # stayed NULL on every connection row. Bare ``auth status`` already prints JSON on both. - who, _, _ = await _run_lark(user_id, ["auth", "status"], timeout=20) - ident = parse_lark_identity(who) - return { - "status": "connected", - "lark_open_id": ident.get("lark_open_id"), - "lark_name": ident.get("lark_name"), - "tenant_key": ident.get("tenant_key"), - **_CLEAR_LOGIN, - "last_verified_at": utc_now(), - "last_error": None, - } - - -async def verify_and_refresh(user_id: str) -> str: - """Real liveness probe: ``auth status --verify`` (validates against the server, - renewing with the refresh token along the way). - Returns ``valid`` / ``invalid`` / ``unknown``. Network/timeout etc. draw no - conclusion (unknown), avoiding misjudgment from a blip.""" - out, err, rc = await _run_lark( - user_id, ["auth", "status", "--verify"], timeout=25 - ) - blob = ((out or "") + "\n" + (err or "")).strip() - if not blob or rc == 124 or rc == 127: - return "unknown" - # The structured verdict is authoritative when the CLI gives us one: substring - # markers are a poor judge here because a *healthy* status payload also carries - # words like "expiresAt", and an expired one still carries openId/userName. - state = parse_user_identity_state(out) - if state is not None: - return "valid" if state else "invalid" - low = blob.lower() - if any(m in low for m in _AUTH_FAIL_MARKERS) or "not authenticated" in low or "not logged in" in low: - return "invalid" - if is_authenticated(out): - return "valid" - return "unknown" - - -async def _device_complete_flow(user_id: str, device_code: str) -> None: - """Backend long-running task: ``auth login --device-code `` polls to - completion (blocking until authorized/expired), then backfills identity. - Manages its own DB sessions and exceptions, never raising into the event loop.""" - try: - out, err, rc = await _run_lark( - user_id, - ["auth", "login", "--device-code", device_code, "--json"], - timeout=600, - ) - verdict = _classify_login_result(out, err, rc) - if verdict == "ok": - # Confirm login with another status check (fallback when --device-code succeeds but the json has no identity) - st, _, _ = await _run_lark(user_id, ["auth", "status"], timeout=20) - # ``rc == 0`` alone must not win: the CLI can exit 0 without ever storing a - # user token, and that is how connection rows ended up marked ``connected`` - # with a NULL identity while the sandbox saw no token at all. A *definite* - # negative from the structured status therefore overrides it; an - # inconclusive status keeps the old lenient behaviour so unfamiliar CLI - # output shapes do not start failing valid logins. - st_state = parse_user_identity_state(st) - if st_state is False: - verdict = "error" - elif st_state is True or is_authenticated(st) or rc == 0: - _update_connection(user_id, await _build_connected_update(user_id)) - return - else: - verdict = "error" - if verdict == "error": - tail = ((err or out or "").strip())[-300:] - _update_connection(user_id, { - "status": "error", **_CLEAR_LOGIN, - "last_error": f"扫码授权未完成或已过期，请重试。{('详情: ' + tail) if tail else ''}", - }) - return - # pending reaching here (timed out unauthorized) → set error so the user can retry - _update_connection(user_id, { - "status": "error", **_CLEAR_LOGIN, - "last_error": "扫码授权超时，请重新发起连接。", - }) - except asyncio.CancelledError: - raise - except Exception as exc: # noqa: BLE001 - logger.warning("[lark] device complete failed user=%s: %s", user_id, exc) - _update_connection(user_id, {"status": "error", "last_error": f"登录异常: {exc}"}) - finally: - _LOGIN_TASKS.pop(user_id, None) - - -# ── Admin one-click initialization of the shared app (config init --new) ──────── -# The whole group shares one app, initialized only once, hence a single module-level task + state (not per-user). -_APP_INIT_TASK: "Optional[asyncio.Task]" = None -_app_init_state: Dict[str, Any] = { - "status": "idle", "verification_url": None, "qr_data_uri": None, "error": None, -} - - -def _app_env() -> Dict[str, str]: - """Environment for running the shared app's config init: HOME = the shared app HOME; Agent/LARK_APP_* interference stripped.""" - from core.sandbox.workspace.credentials import lark_app_home_dir - - home = lark_app_home_dir() - try: - home.mkdir(parents=True, exist_ok=True) - except Exception as exc: # noqa: BLE001 - logger.warning("[lark] mkdir app home failed: %s", exc) - env = dict(os.environ) - env["HOME"] = str(home) - for k in ("OPENCLAW_HOME", "HERMES_HOME", "LARK_APP_ID", "LARK_APP_SECRET"): - env.pop(k, None) - return env - - -async def _app_init_flow() -> None: - """Backend long-running task: ``config init --new`` stream-reads the - ``open.feishu.cn/page/cli`` configuration URL → echoes it back for the - admin to scan → blocks while the admin completes it in the browser → - the app base lands in the shared HOME. Manages its own exceptions.""" - global _APP_INIT_TASK - try: - proc = await asyncio.create_subprocess_exec( - _LARK_BIN, "config", "init", "--new", "--lang", "zh", - stdin=asyncio.subprocess.DEVNULL, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.STDOUT, - env=_app_env(), - **no_window_kwargs(), - ) - except FileNotFoundError: - _app_init_state.update({"status": "error", "error": "后端未安装 lark-cli"}) - _APP_INIT_TASK = None - return - buf = "" - try: - while True: - try: - line = await asyncio.wait_for(proc.stdout.readline(), timeout=600) - except asyncio.TimeoutError: - break - if not line: - break - buf += line.decode("utf-8", "replace") - if _app_init_state.get("verification_url") is None: - m = re.search(r"https?://open\.feishu\.cn/\S+", buf) - if m: - url = m.group(0).rstrip(".,;") - _app_init_state.update({ - "status": "pending", "verification_url": url, - "qr_data_uri": make_qr_data_uri(url), "error": None, - }) - await asyncio.wait_for(proc.wait(), timeout=30) - except asyncio.CancelledError: - try: - proc.kill() - except ProcessLookupError: - pass - raise - except Exception as exc: # noqa: BLE001 - logger.warning("[lark] app init stream failed: %s", exc) - finally: - _APP_INIT_TASK = None - # Completion check: does the shared HOME now have the app configuration - aid = _app_configured() - if aid: - _app_init_state.update({ - "status": "configured", "app_id": aid, - "verification_url": None, "qr_data_uri": None, "error": None, - }) - else: - _app_init_state.update({ - "status": "error", "verification_url": None, "qr_data_uri": None, - "error": "应用配置未完成或已取消，请重试。", - }) - - -def _app_status_dict() -> Dict[str, Any]: - aid = _app_configured() - if aid: - return {"configured": True, "app_id": aid, "status": "configured", - "verification_url": None, "qr_data_uri": None, "error": None} - return {"configured": False, "app_id": None, - "status": _app_init_state.get("status", "idle"), - "verification_url": _app_init_state.get("verification_url"), - "qr_data_uri": _app_init_state.get("qr_data_uri"), - "error": _app_init_state.get("error")} - - -class LarkService: - def __init__(self, db: Session): - self.db = db - self.repo = LarkConnectionRepository(db) - - def _fresh(self, user_id: str): - """Discard the session cache and re-read — to see the latest state committed by the background task's independent session.""" - self.db.expire_all() - return self.repo.ensure(user_id) - - # ── Status ─────────────────────────────────────────────────────────── - def status_dict(self, record) -> Dict[str, Any]: - qr_target = record.login_verification_url_complete or record.login_verification_url - return { - "status": record.status, - "lark_open_id": record.lark_open_id, - "lark_name": record.lark_name, - "tenant_key": record.tenant_key, - "granted_scopes": list(record.granted_scopes or []), - "verification_url": record.login_verification_url, - "verification_url_complete": record.login_verification_url_complete, - "qr_data_uri": make_qr_data_uri(qr_target) if record.status == "pending" else None, - "user_code": record.login_user_code, - "last_verified_at": record.last_verified_at.isoformat() if record.last_verified_at else None, - "last_error": record.last_error, - } - - def get_status(self, user_id: str) -> Dict[str, Any]: - return self.status_dict(self.repo.ensure(user_id)) - - async def probe_status(self, user_id: str) -> Dict[str, Any]: - """Real-API liveness-probe reconciliation (corrects mismatches between the DB and actual credentials). Does not interrupt pending (login in progress).""" - record = self._fresh(user_id) - if record.status == "pending" and user_id in _LOGIN_TASKS: - return self.status_dict(record) - verdict = await verify_and_refresh(user_id) - if verdict == "valid" and record.status != "connected": - self.repo.update(user_id, {"status": "connected", "last_error": None}) - elif verdict == "invalid" and record.status == "connected": - self.repo.update(user_id, {"status": "disconnected", "last_error": _EXPIRED_ERROR}) - return self.status_dict(self.repo.get(user_id)) - - # ── Device-flow login (scan-to-bind) ───────────────────────────────── - async def start_device_login(self, user_id: str) -> Dict[str, Any]: - """Initiate the Lark device-flow login: first seed the app config, then - ``auth login --no-wait`` gets the device_code / QR URL, and the - background completion task is started. The front end presents - qr_data_uri for the user to scan, then polls.""" - self.repo.ensure(user_id) - # Cancel the same user's old task - old = _LOGIN_TASKS.pop(user_id, None) - if old is not None and not old.done(): - old.cancel() - - cfg_err = await _ensure_app_config(user_id) - if cfg_err: - self.repo.update(user_id, {"status": "error", **_CLEAR_LOGIN, "last_error": cfg_err}) - return self.status_dict(self.repo.get(user_id)) - - out, err, rc = await _run_lark(user_id, _DEFAULT_LOGIN_ARGS, timeout=40) - dev = parse_device_login(out) - if rc != 0 or not (dev.get("verification_url") or dev.get("verification_url_complete")): - tail = ((err or out or "").strip())[-200:] - self.repo.update(user_id, { - "status": "error", **_CLEAR_LOGIN, - "last_error": f"发起扫码登录失败，请重试。{('详情: ' + tail) if tail else ''}", - }) - return self.status_dict(self.repo.get(user_id)) - - self.repo.update(user_id, { - "status": "pending", - "login_verification_url": dev.get("verification_url"), - "login_verification_url_complete": dev.get("verification_url_complete"), - "login_user_code": dev.get("user_code"), - "login_device_code": dev.get("device_code"), - "login_started_at": utc_now(), - "last_error": None, - }) - # Start the background completion task (blocking polling until authorized/expired) - if dev.get("device_code"): - _LOGIN_TASKS[user_id] = asyncio.create_task( - _device_complete_flow(user_id, str(dev["device_code"])) - ) - return self.status_dict(self._fresh(user_id)) - - async def poll_login(self, user_id: str) -> Dict[str, Any]: - """Poll login progress. The background task drives to connected/error; - this reads the latest state. If the task is lost yet still pending - (backend restart), fall back to the persisted device_code and run - completion once more.""" - record = self._fresh(user_id) - if record.status == "pending" and user_id not in _LOGIN_TASKS and record.login_device_code: - _LOGIN_TASKS[user_id] = asyncio.create_task( - _device_complete_flow(user_id, str(record.login_device_code)) - ) - return self.status_dict(record) - - # ── Disconnect ─────────────────────────────────────────────────────── - async def disconnect(self, user_id: str) -> Dict[str, Any]: - """Cancel any in-progress login + backend logout + clear the backend persistent credential directory + set disconnected.""" - old = _LOGIN_TASKS.pop(user_id, None) - if old is not None and not old.done(): - old.cancel() - await _run_lark(user_id, ["auth", "logout"], timeout=30) - try: - from core.sandbox.workspace.credentials import lark_cache_dir - from core.sandbox.workspace.credentials import purge_credential_dir - from core.sandbox.workspace.identity import safe_user_id - - # Empty in place — never rmtree the root. ~/.lark-cli and - # ~/.local/share/lark-cli under it are bind-mounted into this user's - # live sandboxes; removing the directory orphans those mounts and the - # next login's credentials land on an inode the sandbox cannot see - # (see [[purge_credential_dir]]). - if safe_user_id(user_id): - purge_credential_dir(lark_cache_dir(user_id)) - except Exception as exc: # noqa: BLE001 - logger.warning("[lark] disconnect cache purge failed user=%s: %s", user_id, exc) - self.repo.update( - user_id, - { - "status": "disconnected", - "lark_open_id": None, - "lark_name": None, - "tenant_key": None, - "granted_scopes": [], - **_CLEAR_LOGIN, - "auth_bundle": None, - "last_error": None, - }, - ) - return self.status_dict(self.repo.get(user_id)) - - # ── Admin: shared-app one-click initialization (config init --new) ──── - def app_status(self) -> Dict[str, Any]: - """Query the shared app's initialization state (configured / pending+QR code / error).""" - return _app_status_dict() - - async def start_app_init(self) -> Dict[str, Any]: - """Initiate ``config init --new``: run in the background and echo the configuration QR code for the admin to scan. If already configured, report back directly.""" - global _APP_INIT_TASK - if _app_configured(): - return _app_status_dict() - if _APP_INIT_TASK is not None and not _APP_INIT_TASK.done(): - return _app_status_dict() # in progress, echo the current URL - _app_init_state.update({"status": "pending", "verification_url": None, - "qr_data_uri": None, "error": None}) - _APP_INIT_TASK = asyncio.create_task(_app_init_flow()) - # Wait up to ~15s for the configuration URL to appear - for _ in range(30): - await asyncio.sleep(0.5) - if _app_init_state.get("verification_url") or _app_init_state.get("status") in ("configured", "error"): - break - return _app_status_dict() - - async def reset_app(self) -> Dict[str, Any]: - """Clear the shared app base (for re-initialization). Does not touch users' existing login tokens.""" - global _APP_INIT_TASK - if _APP_INIT_TASK is not None and not _APP_INIT_TASK.done(): - _APP_INIT_TASK.cancel() - _APP_INIT_TASK = None - try: - import shutil - - from core.sandbox.workspace.credentials import lark_app_home_dir - - shutil.rmtree(lark_app_home_dir(), ignore_errors=True) - except Exception as exc: # noqa: BLE001 - logger.warning("[lark] reset app home failed: %s", exc) - _app_init_state.update({"status": "idle", "verification_url": None, - "qr_data_uri": None, "error": None}) - return _app_status_dict() diff --git a/src/backend/core/services/log_service.py b/src/backend/core/services/log_service.py deleted file mode 100644 index 317ae0be..00000000 --- a/src/backend/core/services/log_service.py +++ /dev/null @@ -1,649 +0,0 @@ -"""Observability log writer — async, best-effort writer for the three -tool/sub-agent/skill log tables. - -Writes are fire-and-forget so the SSE hot path never blocks on DB I/O. -Failures downgrade to structured logs instead of propagating. -""" - -from __future__ import annotations - -import asyncio -import json -import os -import uuid -from contextlib import contextmanager -from contextvars import ContextVar -from datetime import datetime, timezone -from typing import Any, Dict, Iterator, List, Optional - -from sqlalchemy.orm import Session - -from core.db.engine import SessionLocal -from core.db.models import ( - LocalUser, - SkillCallLog, - SubAgentCallLog, - ToolCallLog, - ToolEffectLedger, - UserShadow, -) -from core.infra.background import spawn -from core.infra.data_masking import mask_sensitive_data -from core.infra.logging import chat_id_var, get_logger, trace_id_var, user_id_var - -logger = get_logger(__name__) - -_current_source: ContextVar[str] = ContextVar("log_current_source", default="main_agent") -_current_subagent_log_id: ContextVar[Optional[str]] = ContextVar( - "log_current_subagent", default=None -) -_current_message_id: ContextVar[str] = ContextVar("log_current_message", default="") - -TOOL_LOG_ENABLED = os.getenv("TOOL_CALL_LOG_ENABLED", "true").lower() == "true" -SUBAGENT_LOG_ENABLED = os.getenv("SUBAGENT_LOG_ENABLED", "true").lower() == "true" -SKILL_LOG_ENABLED = os.getenv("SKILL_LOG_ENABLED", "true").lower() == "true" -# Whether sandbox-type tool calls additionally write a security audit event -# (Security Management → Audit Logs, traceable per sandbox instance). -# Independent of the tool-call log switch: even with tool logging off, -# security auditing can stay on. -SANDBOX_AUDIT_ENABLED = os.getenv("SANDBOX_AUDIT_ENABLED", "true").lower() == "true" - -MAX_RESULT_BYTES = int(os.getenv("LOG_MAX_RESULT_BYTES", 64 * 1024)) -MAX_STDOUT_BYTES = int(os.getenv("LOG_MAX_STDOUT_BYTES", 64 * 1024)) - -_REDACT_FIELDS = [ - s.strip() - for s in os.getenv( - "LOG_REDACT_FIELDS", - "password,token,api_key,apikey,authorization,secret,access_key", - ).split(",") - if s.strip() -] - -def _new_id() -> str: - return uuid.uuid4().hex - - -def now_utc() -> datetime: - return datetime.now(timezone.utc) - - -def _truncate_text(text: Optional[str], limit: int) -> tuple[Optional[str], bool]: - if text is None: - return None, False - if not isinstance(text, str): - text = str(text) - raw = text.encode("utf-8", errors="ignore") - if len(raw) <= limit: - return text, False - return raw[:limit].decode("utf-8", errors="ignore") + "\n…[truncated]", True - - -def _truncate_json(payload: Any, limit: int) -> tuple[Any, bool]: - if payload is None: - return None, False - try: - encoded = json.dumps(payload, ensure_ascii=False, default=str).encode( - "utf-8", errors="ignore" - ) - except Exception: - return {"_repr": str(payload)[:limit]}, True - if len(encoded) <= limit: - return payload, False - preview = encoded[: limit // 2].decode("utf-8", errors="ignore") - return {"_truncated": True, "_preview": preview}, True - - -def _context_ids() -> Dict[str, str]: - return { - "trace_id": trace_id_var.get() or "", - "user_id": user_id_var.get() or "", - "chat_id": chat_id_var.get() or "", - } - - -def _fetch_username(db: Session, user_id: str) -> Optional[str]: - """The "user name" used in logs: prefer the local account display name - (nickname → real_name), fall back to the login account username. - - When the audit log panel shows this column, operators should see a - recognizable person name (e.g. "张三"), not the login account (e.g. - "zhangsan01") — the latter would not match the display names shown - elsewhere in the UI. - """ - if not user_id: - return None - try: - row = ( - db.query(UserShadow.username, LocalUser.nickname, LocalUser.real_name) - .outerjoin(LocalUser, LocalUser.user_id == UserShadow.user_id) - .filter(UserShadow.user_id == user_id) - .one_or_none() - ) - if not row: - return None - return row.nickname or row.real_name or row.username - except Exception as exc: # noqa: BLE001 - logger.debug("username_lookup_failed", user_id=user_id, error=str(exc)) - return None - - -@contextmanager -def subagent_scope(subagent_log_id: Optional[str], source: str = "subagent") -> Iterator[None]: - tok_id = _current_subagent_log_id.set(subagent_log_id) - tok_src = _current_source.set(source) - try: - yield - finally: - _current_subagent_log_id.reset(tok_id) - _current_source.reset(tok_src) - - -@contextmanager -def skill_scope(source: str = "skill") -> Iterator[None]: - tok = _current_source.set(source) - try: - yield - finally: - _current_source.reset(tok) - - -def set_current_message_id(message_id: str) -> None: - _current_message_id.set(message_id or "") - - -def current_subagent_log_id() -> Optional[str]: - return _current_subagent_log_id.get() - - -def current_source() -> str: - return _current_source.get() or "main_agent" - - -def _effective_status(declared: Optional[str], result: Any) -> str: - """声明的状态优先，但"成功"要经得起载荷检验。 - - 只认**顶层** error，且只认 dict / 能解析成 dict 的 JSON 串——正文里出现 error 字样的 - 正常内容（比如搜到一篇讲错误码的网页）不该被误判成故障。 - """ - status = declared or "success" - if status != "success": - return status - from core.chat.tool_log import _payload_carries_error - - return "error" if _payload_carries_error(result) else "success" - - -def _sanitize_tool_args(value: Any) -> Any: - from core.services.tool_effect_ledger import canonical_tool_args - - if isinstance(value, dict): - return canonical_tool_args(value)[2] - return mask_sensitive_data(value, field_patterns=_REDACT_FIELDS) - - -def _write_tool_call_sync(record: Dict[str, Any]) -> None: - if not TOOL_LOG_ENABLED: - return - db: Session = SessionLocal() - try: - args_safe = _sanitize_tool_args(record.get("tool_args")) - args_safe, _ = _truncate_json(args_safe, MAX_RESULT_BYTES) - result_safe, result_trunc = _truncate_json(record.get("tool_result"), MAX_RESULT_BYTES) - username = _fetch_username(db, record.get("user_id", "")) - effect = None - effect_id = str(record.get("effect_id") or "") - run_id = str(record.get("run_id") or "") - tool_call_id = str(record.get("tool_call_id") or "") - if effect_id: - effect = ( - db.query(ToolEffectLedger) - .filter( - ToolEffectLedger.effect_id == effect_id, - ToolEffectLedger.event_type == "intent", - ) - .one_or_none() - ) - if effect is None and run_id and tool_call_id: - effect = ( - db.query(ToolEffectLedger) - .filter( - ToolEffectLedger.run_id == run_id, - ToolEffectLedger.tool_call_id == tool_call_id, - ToolEffectLedger.event_type == "intent", - ) - .order_by(ToolEffectLedger.event_id.desc()) - .first() - ) - # ToolCallLog stays a replaceable query projection. The pre-allocated - # result id makes retries update the same row, while ToolEffectLedger - # remains the append-only recovery authority. - row_id = ( - (effect.result_id if effect is not None else None) - or record.get("result_id") - or record.get("id") - or _new_id() - ) - row = db.get(ToolCallLog, row_id) or ToolCallLog(id=row_id) - row.effect_id = effect.effect_id if effect is not None else record.get("effect_id") - row.trace_id = record.get("trace_id") - row.chat_id = record.get("chat_id") - row.message_id = record.get("message_id") - row.user_id = record.get("user_id") - row.user_name = username - row.tool_name = record["tool_name"] - row.tool_display_name = record.get("tool_display_name") - row.tool_call_id = record.get("tool_call_id") - row.mcp_server = record.get("mcp_server") - row.tool_args = args_safe - row.tool_result = result_safe - row.result_truncated = bool(result_trunc) - # A top-level error payload is a failed projection even when an MCP - # adapter declared the transport call successful. - row.status = _effective_status(record.get("status"), record.get("tool_result")) - row.error_message = record.get("error_message") - row.duration_ms = record.get("duration_ms") - row.source = record.get("source") or "main_agent" - row.subagent_log_id = record.get("subagent_log_id") - row.skill_log_id = record.get("skill_log_id") - row.sandbox_id = record.get("sandbox_id") - row.started_at = record.get("started_at") - row.created_at = record.get("created_at") or now_utc() - db.add(row) - db.commit() - except Exception as exc: # noqa: BLE001 - db.rollback() - logger.warning("tool_call_log_write_failed", error=str(exc), tool=record.get("tool_name")) - finally: - db.close() - - -def _write_subagent_create_sync(record: Dict[str, Any]) -> None: - if not SUBAGENT_LOG_ENABLED: - return - db: Session = SessionLocal() - try: - input_safe, _ = _truncate_json(record.get("input_messages"), MAX_RESULT_BYTES) - username = _fetch_username(db, record.get("user_id", "")) - row = SubAgentCallLog( - id=record["id"], - trace_id=record.get("trace_id"), - chat_id=record.get("chat_id"), - message_id=record.get("message_id"), - user_id=record.get("user_id"), - user_name=username, - subagent_id=record.get("subagent_id"), - subagent_name=record["subagent_name"], - subagent_type=record.get("subagent_type"), - plan_id=record.get("plan_id"), - step_id=record.get("step_id"), - step_index=record.get("step_index"), - step_title=record.get("step_title"), - model=record.get("model"), - input_messages=input_safe, - status="running", - parent_subagent_log_id=record.get("parent_subagent_log_id"), - started_at=record.get("started_at") or now_utc(), - created_at=now_utc(), - ) - db.add(row) - db.commit() - except Exception as exc: # noqa: BLE001 - db.rollback() - logger.warning( - "subagent_log_create_failed", error=str(exc), name=record.get("subagent_name") - ) - finally: - db.close() - - -def _write_subagent_finish_sync( - subagent_log_id: str, - status: str, - updates: Dict[str, Any], -) -> None: - if not SUBAGENT_LOG_ENABLED: - return - db: Session = SessionLocal() - try: - row = db.query(SubAgentCallLog).filter(SubAgentCallLog.id == subagent_log_id).one_or_none() - if row is None: - return - row.status = status - if "output_content" in updates: - output, _ = _truncate_text(updates.get("output_content"), MAX_RESULT_BYTES) - row.output_content = output - if "intermediate_steps" in updates: - steps_safe, _ = _truncate_json(updates.get("intermediate_steps"), MAX_RESULT_BYTES) - row.intermediate_steps = steps_safe - if "token_usage" in updates: - row.token_usage = updates.get("token_usage") - if "tool_calls_count" in updates: - row.tool_calls_count = updates.get("tool_calls_count") - if "skill_calls_count" in updates: - row.skill_calls_count = updates.get("skill_calls_count") - if "error_message" in updates: - row.error_message = updates.get("error_message") - if "duration_ms" in updates: - row.duration_ms = updates.get("duration_ms") - row.completed_at = now_utc() - db.commit() - except Exception as exc: # noqa: BLE001 - db.rollback() - logger.warning("subagent_log_finish_failed", error=str(exc), id=subagent_log_id) - finally: - db.close() - - -def _write_skill_call_sync(record: Dict[str, Any]) -> None: - if not SKILL_LOG_ENABLED: - return - db: Session = SessionLocal() - try: - args_safe = mask_sensitive_data(record.get("script_args"), field_patterns=_REDACT_FIELDS) - args_safe, _ = _truncate_json(args_safe, MAX_RESULT_BYTES) - stdout_safe, trunc_out = _truncate_text(record.get("script_stdout"), MAX_STDOUT_BYTES) - stderr_safe, trunc_err = _truncate_text(record.get("script_stderr"), MAX_STDOUT_BYTES) - stdin_safe, _ = _truncate_text(record.get("script_stdin"), MAX_STDOUT_BYTES) - username = _fetch_username(db, record.get("user_id", "")) - row = SkillCallLog( - id=record.get("id") or _new_id(), - trace_id=record.get("trace_id"), - chat_id=record.get("chat_id"), - message_id=record.get("message_id"), - user_id=record.get("user_id"), - user_name=username, - skill_id=record["skill_id"], - skill_name=record.get("skill_name"), - skill_version=record.get("skill_version"), - skill_source=record.get("skill_source"), - invocation_type=record.get("invocation_type") or "auto_load", - script_name=record.get("script_name"), - script_language=record.get("script_language"), - script_args=args_safe, - script_stdin=stdin_safe, - script_stdout=stdout_safe, - script_stderr=stderr_safe, - output_truncated=bool(trunc_out or trunc_err), - exit_code=record.get("exit_code"), - status=record.get("status") or "success", - error_message=record.get("error_message"), - duration_ms=record.get("duration_ms"), - source=record.get("source") or current_source(), - subagent_log_id=record.get("subagent_log_id") or current_subagent_log_id(), - started_at=record.get("started_at"), - created_at=record.get("created_at") or now_utc(), - ) - db.add(row) - db.commit() - except Exception as exc: # noqa: BLE001 - db.rollback() - logger.warning("skill_call_log_write_failed", error=str(exc), skill=record.get("skill_id")) - finally: - db.close() - - -# Sandbox-type tool names: these tools act directly on the sandbox (bash -# execution, artifact transfer, file create/modify/read, search). Only these -# tools resolve sandbox_id and write security audits; backend-only tools -# (web / pin / data_context…) are not tagged. -# Note: Read/Grep/Glob are attributed to the "current sandbox" when there is an -# active sandbox session; details carry the actual path/pattern for tracing. -_SANDBOX_TOOL_NAMES = { - "Bash", - "sandbox_put_artifact", - "sandbox_get_artifact", - "Write", - "Edit", - "Read", - "Grep", - "Glob", -} - - -async def _resolve_sandbox_id(session_id: Optional[str]) -> Optional[str]: - """Look up the current sandbox instance id from the sandbox provider by session key (default = chat_id). Pure query; None on failure.""" - if not session_id: - return None - try: - from core.sandbox import get_sandbox_provider - - return await get_sandbox_provider().current_sandbox_id(session_id) - except ( - Exception - ): # noqa: BLE001 — audit enrichment is best-effort, must never affect the main path - return None - - -# Sandbox tool → security audit action name. Audits let the security admin -# console trace by sandbox instance, independent of tool-call logging. -_SANDBOX_AUDIT_ACTIONS = { - "Bash": "sandbox.bash.exec", - "sandbox_put_artifact": "sandbox.artifact.put", - "sandbox_get_artifact": "sandbox.artifact.get", - "Write": "sandbox.file.write", - "Edit": "sandbox.file.edit", - "Read": "sandbox.file.read", - "Grep": "sandbox.file.grep", - "Glob": "sandbox.file.glob", -} - - -# Tool/skill status → audit-constrained success/failure/error (unlisted ones like failed/timeout → failure). -_AUDIT_STATUS_MAP = {"success": "success", "error": "error"} - - -def _audit_status(raw: Optional[str]) -> str: - return _AUDIT_STATUS_MAP.get(raw or "success", "failure") - - -def _insert_sandbox_audit( - *, - action: str, - sandbox_id: Any, - user_id: Optional[str], - trace_id: Optional[str], - details: Dict[str, Any], - status: str, - created_at: Optional[datetime], -) -> None: - """Write one resource=sandbox security audit row. Best-effort: failure only degrades to a log entry, never affects the main path.""" - from core.db.repository import AuditLogRepository - - db: Session = SessionLocal() - try: - # Reuse the audit repository's write (has its own add/commit/rollback; audit failure doesn't block the main path). - AuditLogRepository(db).create( - { - "user_id": user_id - or None, # empty string would trip the FK constraint; normalize to NULL - "action": action, - "resource_type": "sandbox", - "resource_id": str(sandbox_id)[:64], - "sandbox_id": str(sandbox_id)[:128], - "details": details, - "trace_id": trace_id, - "status": status, - "created_at": created_at or now_utc(), - } - ) - except Exception as exc: # noqa: BLE001 - logger.warning("sandbox_audit_write_failed", error=str(exc), sandbox=str(sandbox_id)) - finally: - db.close() - - -def _write_sandbox_audit_sync(record: Dict[str, Any]) -> None: - """Write one sandbox-type tool call as a security audit event (resource=sandbox / sandbox instance id). - - Separate from tool-call logging: audits let "Security Management → Audit - Logs" distinguish and filter by sandbox instance. details only summarizes - the input args (redacted + truncated), never dumps full results. - """ - sandbox_id = record.get("sandbox_id") - if not sandbox_id: - return - tool_name = record.get("tool_name") - action = _SANDBOX_AUDIT_ACTIONS.get(tool_name, "sandbox.tool.call") - args_safe = _sanitize_tool_args(record.get("tool_args")) - details: Dict[str, Any] = {"tool": tool_name} - if isinstance(args_safe, dict): - cmd = args_safe.get("command") - if cmd is not None: - preview, _ = _truncate_text(str(cmd), 2000) - details["command"] = preview - # File/search tools: record path and pattern (not large fields like content) to trace what was changed/read. - for k in ("file_path", "filename", "path", "pattern", "artifact_id", "name"): - if args_safe.get(k) is not None: - details[k] = args_safe[k] - if record.get("error_message"): - details["error"], _ = _truncate_text(str(record["error_message"]), 500) - - _insert_sandbox_audit( - action=action, - sandbox_id=sandbox_id, - user_id=record.get("user_id"), - trace_id=record.get("trace_id"), - details=details, - status=_audit_status(record.get("status")), - created_at=record.get("created_at"), - ) - - -def _write_skill_sandbox_audit_sync(record: Dict[str, Any]) -> None: - """Skill script execution (run_script) = running code in the sandbox; write one sandbox.skill.run audit. - - Triggered only by run_script (view / auto_load do not execute code in the - sandbox); skipped when there is no sandbox instance. - """ - sandbox_id = record.get("sandbox_id") - if not sandbox_id: - return - details: Dict[str, Any] = { - "skill_id": record.get("skill_id"), - "skill_name": record.get("skill_name"), - "script_name": record.get("script_name"), - "script_language": record.get("script_language"), - } - if record.get("error_message"): - details["error"], _ = _truncate_text(str(record["error_message"]), 500) - - _insert_sandbox_audit( - action="sandbox.skill.run", - sandbox_id=sandbox_id, - user_id=record.get("user_id"), - trace_id=record.get("trace_id"), - details=details, - status=_audit_status(record.get("status")), - created_at=record.get("created_at"), - ) - - -async def write_tool_call(record: Dict[str, Any]) -> None: - record.setdefault("source", current_source()) - record.setdefault("subagent_log_id", current_subagent_log_id()) - ctx = _context_ids() - record.setdefault("trace_id", ctx["trace_id"]) - record.setdefault("user_id", ctx["user_id"]) - record.setdefault("chat_id", ctx["chat_id"]) - record.setdefault("message_id", _current_message_id.get()) - # Sandbox-type tools: resolve the sandbox instance id they run in (session - # key default = chat_id). Awaiting in place on this async path — provider - # reads its active-session table (synchronous in-memory read, no side - # effects) — is more reliable than passing a ContextVar across tasks. - if record.get("tool_name") in _SANDBOX_TOOL_NAMES: - if not record.get("sandbox_id"): - record["sandbox_id"] = await _resolve_sandbox_id(record.get("chat_id")) - # Security audit is independent of the tool-call log switch: write the audit first, then the (optional) tool log. - if SANDBOX_AUDIT_ENABLED: - try: - await asyncio.to_thread(_write_sandbox_audit_sync, record) - except Exception as exc: # noqa: BLE001 - logger.warning("sandbox_audit_task_failed", error=str(exc)) - if not TOOL_LOG_ENABLED: - return - try: - await asyncio.to_thread(_write_tool_call_sync, record) - except Exception as exc: # noqa: BLE001 - logger.warning("tool_call_log_task_failed", error=str(exc)) - - -async def start_subagent_log(record: Dict[str, Any]) -> str: - log_id = record.get("id") or _new_id() - record["id"] = log_id - if not SUBAGENT_LOG_ENABLED: - return log_id - ctx = _context_ids() - record.setdefault("trace_id", ctx["trace_id"]) - record.setdefault("user_id", ctx["user_id"]) - record.setdefault("chat_id", ctx["chat_id"]) - record.setdefault("message_id", _current_message_id.get()) - record.setdefault("parent_subagent_log_id", current_subagent_log_id()) - record.setdefault("started_at", now_utc()) - try: - await asyncio.to_thread(_write_subagent_create_sync, record) - except Exception as exc: # noqa: BLE001 - logger.warning("subagent_log_start_failed", error=str(exc)) - return log_id - - -async def finish_subagent_log( - subagent_log_id: str, - *, - status: str = "success", - output_content: Optional[str] = None, - intermediate_steps: Optional[List[Dict[str, Any]]] = None, - token_usage: Optional[Dict[str, int]] = None, - tool_calls_count: int = 0, - skill_calls_count: int = 0, - error_message: Optional[str] = None, - duration_ms: Optional[int] = None, -) -> None: - if not SUBAGENT_LOG_ENABLED or not subagent_log_id: - return - updates = { - "output_content": output_content, - "intermediate_steps": intermediate_steps, - "token_usage": token_usage, - "tool_calls_count": tool_calls_count, - "skill_calls_count": skill_calls_count, - "error_message": error_message, - "duration_ms": duration_ms, - } - try: - await asyncio.to_thread(_write_subagent_finish_sync, subagent_log_id, status, updates) - except Exception as exc: # noqa: BLE001 - logger.warning("subagent_log_finish_failed", error=str(exc)) - - -async def write_skill_call(record: Dict[str, Any]) -> str: - log_id = record.get("id") or _new_id() - record["id"] = log_id - ctx = _context_ids() - record.setdefault("trace_id", ctx["trace_id"]) - record.setdefault("user_id", ctx["user_id"]) - record.setdefault("chat_id", ctx["chat_id"]) - record.setdefault("message_id", _current_message_id.get()) - # Skill script execution (run_script) = running code in the sandbox → write a security audit (independent of the skill log switch). - if SANDBOX_AUDIT_ENABLED and record.get("invocation_type") == "run_script": - if not record.get("sandbox_id"): - record["sandbox_id"] = await _resolve_sandbox_id(record.get("chat_id")) - try: - await asyncio.to_thread(_write_skill_sandbox_audit_sync, record) - except Exception as exc: # noqa: BLE001 - logger.warning("skill_sandbox_audit_task_failed", error=str(exc)) - if not SKILL_LOG_ENABLED: - return log_id - try: - await asyncio.to_thread(_write_skill_call_sync, record) - except Exception as exc: # noqa: BLE001 - logger.warning("skill_call_log_task_failed", error=str(exc)) - return log_id - - -def schedule_tool_call_write(record: Dict[str, Any]) -> None: - spawn(write_tool_call(record), name="log_writer.tool_call") - - -def schedule_skill_call_write(record: Dict[str, Any]) -> None: - spawn(write_skill_call(record), name="log_writer.skill_call") diff --git a/src/backend/core/services/marketplace_service.py b/src/backend/core/services/marketplace_service.py deleted file mode 100644 index 8d97bd3d..00000000 --- a/src/backend/core/services/marketplace_service.py +++ /dev/null @@ -1,1400 +0,0 @@ -"""Skill Marketplace service. - -The skill marketplace is a set of **preloaded, curated** installable skill -packages, shipped with the repo/image and stored under -``extensions/skills/marketplace//`` (each directory holds the original -SKILL.md + referenced files + a ``marketplace.json`` manifest). The built-in -loader scans ``extensions/skills/*/SKILL.md`` with single-level matching; market -skills live two levels deep at ``marketplace//SKILL.md`` and are NOT -auto-loaded as built-ins — so before "installation" a market skill never -appears in the catalog nor gets registered with the agent. Only after a -user/admin explicitly installs it does it take effect in the DB. - -Installation reuses the existing ``AdminSkill`` mechanism: -- Admin install → global skill (``owner_user_id`` empty), skill id = manifest entry_name; -- User install → private skill (``owner_user_id`` = current user); the skill id - gets a user-fingerprint suffix appended to guarantee global uniqueness, so - multiple users can each install the same market skill, each with their own - credentials. - -Credentials (API keys etc.): the manifest may declare ``required_secrets``; -they are collected by the caller at install time, written to ``secrets.json`` -in that installed skill's directory, and a "credentials configuration" section -is appended at the end of SKILL.md for the agent to read when running scripts. -The marketplace directory itself stores NO secrets. - -Community publishing (marketplace_submissions): users can **submit** their -private skills for listing in the marketplace; once an admin approves them at -/admin, the skill appears in the market list with ``source=community`` and -becomes installable by everyone. Submission snapshots the source skill's -content, decoupling the listed content from the user's later edits; an admin -rejecting an already-approved submission delists it. -""" - -from __future__ import annotations - -import hashlib -import io -import json -import logging -import os -import re -import uuid -import zipfile -from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple - -from core.agent_skills.binary_files import ( - BINARY_EXTENSIONS, - JUNK_BASENAMES, - MAX_SINGLE_FILE, - MAX_TOTAL, - decode_binary, - encode_binary, - is_binary_value, - pack_directory, -) -from core.agent_skills.cache_refresh import refresh_skill_caches -from core.agent_skills.deps_detector import detect_dependencies -from core.agent_skills.registry import ( - SkillSpecError, - _load_skill_metadata_from_str, - _split_frontmatter, -) -from core.db.models import AdminSkill, MarketplaceSubmission -from core.infra.exceptions import BadRequestError, ResourceNotFoundError -from core.infra.time import utc_now -from core.ontology.build_validator import ensure_ontology_build_valid -from extensions.paths import SKILLS_DIR -from fastapi import HTTPException -from sqlalchemy.orm import Session -from sqlalchemy.orm.attributes import flag_modified - -logger = logging.getLogger(__name__) - -# Root directory of preloaded market skill packages: ``src/backend/extensions/skills/marketplace/``. -# Lives in the same tree as built-in skills (``extensions/skills//``), making -# additions/removals uniform under extensions/skills; but the built-in loader's scan -# is single-level ``glob("*/SKILL.md")`` — market skills at -# ``marketplace//SKILL.md`` (two levels deep) are not auto-loaded as -# built-ins, preserving the install-based model. -MARKETPLACE_DIR = SKILLS_DIR / "marketplace" -# Built-in default skills directory: ``extensions/skills/default//``. These -# skills are globally resident and always available to everyone; they are also -# shown in the skill marketplace as "built-in" entries (read from their own -# folders, not copied into marketplace/ to avoid 4MB+ duplication). -DEFAULT_SKILLS_DIR = SKILLS_DIR / "default" - -MANIFEST_NAME = "marketplace.json" -SKILL_MD_NAME = "SKILL.md" -SECRETS_FILENAME = "secrets.json" - -# The built-in default skills' "category + summary" in the marketplace (default -# skills have no category in their frontmatter; the Chinese name comes from -# SKILL.md's display_name). The key set defines "which default skills are listed -# in the marketplace" — visible to everyone, marked as built-in resident. -DEFAULT_SKILL_MARKET = { - "pdf-editing": {"category": "文档处理", "summary": "生成 / 合并 / 拆分 / 抽页 / 填表单 / 读取 PDF 文档"}, - "capability-guide-brief": {"category": "办公效率", "summary": "速答系统能力清单与使用指引"}, - # OfficeCLI 系列（上游 iOfficeAI/OfficeCLI，Apache 2.0）：靠沙盒预装的 - # ``officecli`` 单文件二进制驱动，是 Word / Excel / PPT 的**唯一**入口—— - # 自研的 word-editing / excel-editing / ppt-design 三个技能已于 2026-08 下线， - # 能力全部由这一系列承接。三个基座（docx / xlsx / pptx）之上叠场景层， - # 分工写在各自 SKILL.md 的 description 里，由技能选择器路由。 - "officecli-docx": {"category": "文档处理", "summary": "OfficeCLI 深度编辑 .docx：TOC / 域 / 脚注 / 公式 / 原生图表 / 批注修订 / schema 校验"}, - "officecli-xlsx": {"category": "办公效率", "summary": "OfficeCLI 深度编辑 .xlsx：透视表 / 迷你图 / 条件格式 / 数据验证 / 切片器"}, - "officecli-pptx": {"category": "办公效率", "summary": "OfficeCLI 深度编辑 .pptx：母版版式 / 连接线 / 形状级控制 / 逐页截图质检"}, - "officecli-word-form": {"category": "文档处理", "summary": "制作可填写 Word 表单：内容控件(SDT) + 复选框 + 邮件合并域 + 文档保护"}, - "officecli-academic-paper": {"category": "文档处理", "summary": "学术论文排版：APA / Chicago / IEEE / MLA 引用、编号公式、交叉引用、参考文献"}, - "officecli-data-dashboard": {"category": "数据分析", "summary": "CSV / 表格数据做成 Excel 数据看板：KPI 卡片 + 多图表 + 迷你图 + 条件格式"}, - "officecli-financial-model": {"category": "数据分析", "summary": "Excel 财务模型：三表 / DCF / LBO / 单位经济 / 敏感性分析，全活公式驱动"}, - "officecli-pitch-deck": {"category": "办公效率", "summary": "融资路演 PPT：种子轮 / A-C 轮 / SAFE / 可转债的投资人 deck"}, - "morph-ppt": {"category": "办公效率", "summary": "带 Morph 平滑转场动效的 PPT，自带 40 套成品设计风格包"}, - "morph-ppt-3d": {"category": "办公效率", "summary": "带 .glb 三维模型与电影级镜头运动的 Morph 动效 PPT"}, -} - -# Fixed skill-marketplace categories (the only allowed set): preloaded -# manifests, community submissions, and admin-review category changes may only -# take these 8 values; mirrored on the frontend in utils/constants.ts as -# MARKETPLACE_CATEGORIES. -MARKETPLACE_CATEGORIES = [ - "写作助手", "文档处理", "数据分析", "政策产业", - "营销创意", "法务合规", "办公效率", "研发效率", -] -# Storage constraints / binary extension set + packing logic are unified in -# core.agent_skills.binary_files (pack_directory / BINARY_EXTENSIONS / MAX_*); -# admin upload and plugin import share the same single source of truth. - - -# ── Manifest / listing ─────────────────────────────────────────────────────── - -def _read_manifest(slug: str) -> Optional[Dict[str, Any]]: - """Read and validate a single market skill's ``marketplace.json`` (tolerant; returns None if corrupted).""" - if not slug or "/" in slug or ".." in slug: - return None - path = MARKETPLACE_DIR / slug / MANIFEST_NAME - if not path.is_file(): - return None - try: - m = json.loads(path.read_text(encoding="utf-8")) - except Exception as exc: # noqa: BLE001 - logger.warning("marketplace manifest broken: %s (%s)", slug, exc) - return None - if not isinstance(m, dict) or not m.get("entry_name"): - return None - m.setdefault("slug", slug) - return m - - -def market_skill_requires_secrets(slug: str) -> bool: - """Whether a preloaded market skill declares required_secrets (public contract, reusable by other markets/callers).""" - m = _read_manifest(slug) - if not m: - return False - return bool(m.get("requires_api_key") or m.get("required_secrets")) - - -def _public_meta(m: Dict[str, Any]) -> Dict[str, Any]: - """Manifest → externally visible metadata (no skill body/file contents).""" - return { - "slug": m.get("slug"), - "entry_name": m.get("entry_name"), - "display_name": m.get("display_name") or m.get("slug"), - "summary": m.get("summary") or "", - "category": m.get("category") or "其他", - "tags": list(m.get("tags") or []), - "version": m.get("version") or "1.0.0", - "author": m.get("author") or "skillhub", - "icon_url": m.get("icon_url") or "", - "source": m.get("source") or "skillhub", - "source_url": m.get("source_url") or "", - "downloads": int(m.get("downloads") or 0), - "stars": int(m.get("stars") or 0), - "featured": bool(m.get("featured")), - "requires_api_key": bool(m.get("requires_api_key")), - "required_secrets": list(m.get("required_secrets") or []), - # Preloaded (filesystem) skills ship with the repo; admins cannot delete them online. - "deletable": False, - } - - -def _submission_public_meta(sub: MarketplaceSubmission) -> Dict[str, Any]: - """Approved community submission → external metadata isomorphic to the preloaded manifest form.""" - return { - "slug": sub.slug, - "entry_name": sub.slug, - "display_name": sub.display_name or sub.slug, - "summary": sub.summary or "", - "category": sub.category or "社区共享", - "tags": list(sub.tags or []), - "version": sub.version or "1.0.0", - "author": sub.submitter_name or sub.owner_user_id, - "icon_url": "", - "source": "community", - "source_url": "", - "downloads": 0, - "stars": 0, - "featured": False, - "requires_api_key": False, - "required_secrets": _submission_required_secrets(sub), - # DB listing records (admin uploads / user community submissions) → admins can delete online (remove from the market). - "deletable": True, - } - - -def _approved_submissions(db: Session) -> List[MarketplaceSubmission]: - return ( - db.query(MarketplaceSubmission) - .filter(MarketplaceSubmission.status == "approved") - .order_by(MarketplaceSubmission.created_at.desc()) - .all() - ) - - -def _default_skill_public_meta(slug: str) -> Optional[Dict[str, Any]]: - """Built-in default skill (extensions/skills/default/) → market-list metadata isomorphic to the preloaded form. - - The Chinese name comes from SKILL.md's ``display_name``; category/summary - come from ``DEFAULT_SKILL_MARKET``. Marked ``source='builtin'`` + - ``builtin=True`` — these skills are globally resident and always available - to everyone; the market shows them as "built-in" (already available) and - they do not go through "install as a private copy" (avoiding duplicating - 1MB+ office skills into the DB). - """ - cfg = DEFAULT_SKILL_MARKET.get(slug) - if cfg is None: - return None - md = DEFAULT_SKILLS_DIR / slug / SKILL_MD_NAME - if not md.is_file(): - return None - try: - fm, _ = _split_frontmatter(md.read_text(encoding="utf-8", errors="ignore")) - except Exception: # noqa: BLE001 - fm = {} - name = str(fm.get("name") or slug).strip() - tags = fm.get("tags") or [] - if isinstance(tags, str): - tags = [t.strip() for t in tags.split(",") if t.strip()] - return { - "slug": slug, - "entry_name": name, - "display_name": str(fm.get("display_name") or name).strip(), - "summary": cfg["summary"], - "category": cfg["category"], - "tags": list(tags), - "version": str(fm.get("version") or "1.0.0"), - "author": "内置", - "icon_url": "", - "source": "builtin", - "source_url": "", - "downloads": 0, - "stars": 0, - "featured": False, - "requires_api_key": False, - "required_secrets": [], - "deletable": False, # built-ins ship with the repo; not deletable online - "builtin": True, # the frontend uses this to show "built-in" and set the button to "already built-in" - } - - -def _default_skill_metas() -> List[Dict[str, Any]]: - metas = [_default_skill_public_meta(s) for s in DEFAULT_SKILL_MARKET] - return [m for m in metas if m] - - -def list_marketplace_skills( - db: Optional[Session] = None, - *, - include_disabled: bool = False, - viewer_user_id: Optional[str] = None, -) -> List[Dict[str, Any]]: - """List all market skill metadata: built-in default skills + preloaded (featured first, downloads descending) + community submissions. - - ``include_disabled``: the admin console passes True (sees everything + - ``market_enabled`` annotation); the user side defaults to False (only sees - listed items, and scoped items invisible to ``viewer_user_id`` are filtered - by visibility). See ``marketplace_listing`` for the delist switch and - visibility scopes. - """ - items: List[Dict[str, Any]] = [] - if MARKETPLACE_DIR.is_dir(): - for child in sorted(MARKETPLACE_DIR.iterdir()): - if not child.is_dir(): - continue - m = _read_manifest(child.name) - if m: - items.append(_public_meta(m)) - items.sort(key=lambda x: (0 if x["featured"] else 1, -x["downloads"], x["slug"])) - # Built-in default skills come first (built-in resident, shown with priority). - items = _default_skill_metas() + items - if db is not None: - items.extend(_submission_public_meta(s) for s in _approved_submissions(db)) - from core.services import marketplace_listing as ml - items = ml.annotate_and_filter( - db, ml.KIND_SKILL, items, id_key="slug", - include_disabled=include_disabled, viewer_user_id=viewer_user_id, - ) - return items - - -def set_marketplace_skill_enabled( - db: Session, slug: str, enabled: bool, *, updated_by: Optional[str] = None -) -> Dict[str, Any]: - """List/delist a market skill (controls whether it shows in the skill marketplace; installed instances are unaffected).""" - from core.services import marketplace_listing as ml - return ml.set_listing_enabled(db, ml.KIND_SKILL, slug, enabled, updated_by=updated_by) - - -def list_categories( - db: Optional[Session] = None, - *, - include_disabled: bool = False, - viewer_user_id: Optional[str] = None, -) -> List[str]: - """Marketplace category list: the 8 fixed categories first (stable order), legacy leftover categories appended after.""" - seen: List[str] = list(MARKETPLACE_CATEGORIES) - for it in list_marketplace_skills( - db, include_disabled=include_disabled, viewer_user_id=viewer_user_id - ): - if it["category"] not in seen: - seen.append(it["category"]) - return seen - - -def _validate_category(category: str) -> str: - """Validate and return a legal category; anything outside the fixed set is a straight 400.""" - category = (category or "").strip() - if category not in MARKETPLACE_CATEGORIES: - raise BadRequestError( - message=f"请从固定分类中选择：{'、'.join(MARKETPLACE_CATEGORIES)}" - ) - return category - - -def _get_approved_submission(db: Session, slug: str) -> Optional[MarketplaceSubmission]: - return ( - db.query(MarketplaceSubmission) - .filter(MarketplaceSubmission.slug == slug, MarketplaceSubmission.status == "approved") - .first() - ) - - -def _strip_frontmatter(raw: str) -> str: - try: - _, body = _split_frontmatter(raw) - except Exception: # noqa: BLE001 - body = raw - return (body or "").strip() - - -def get_marketplace_skill(slug: str, db: Optional[Session] = None) -> Dict[str, Any]: - """Single market skill detail: metadata + file list + SKILL.md body (for the detail preview). - - The preloaded directory takes priority; when not found, fall back to the - community listing (approved submission). - """ - # Built-in default skills: read extensions/skills/default/; the body comes from SKILL.md, the file list from the directory. - if slug in DEFAULT_SKILL_MARKET: - meta = _default_skill_public_meta(slug) - if meta is None: - raise ResourceNotFoundError("marketplace_skill", slug) - pkg_dir = DEFAULT_SKILLS_DIR / slug - files: List[Dict[str, Any]] = [] - for p in sorted(pkg_dir.rglob("*")): - if not p.is_file(): - continue - rel = p.relative_to(pkg_dir).as_posix() - if rel == SKILL_MD_NAME: - continue - files.append({"path": rel, "size": p.stat().st_size}) - skill_md = pkg_dir / SKILL_MD_NAME - meta["files"] = files - meta["instructions"] = _strip_frontmatter(skill_md.read_text(encoding="utf-8", errors="ignore")) if skill_md.is_file() else "" - return meta - m = _read_manifest(slug) - if not m: - sub = _get_approved_submission(db, slug) if db is not None else None - if sub is None: - raise ResourceNotFoundError("marketplace_skill", slug) - data = _submission_public_meta(sub) - data["files"] = [ - {"path": rel, "size": len(str(content).encode("utf-8"))} - for rel, content in sorted((sub.extra_files or {}).items()) - if rel != REQUIRED_SECRETS_SNAPSHOT # internal sentinel, not part of the installed content - ] - data["instructions"] = _strip_frontmatter(sub.skill_content or "") - return data - pkg_dir = MARKETPLACE_DIR / slug - data = _public_meta(m) - - files: List[Dict[str, Any]] = [] - for p in sorted(pkg_dir.rglob("*")): - if not p.is_file() or p.name == MANIFEST_NAME: - continue - rel = p.relative_to(pkg_dir).as_posix() - if rel == SKILL_MD_NAME: # SKILL.md is returned separately as instructions; don't duplicate it in the file list - continue - files.append({"path": rel, "size": p.stat().st_size}) - body = "" - skill_md = pkg_dir / SKILL_MD_NAME - if skill_md.is_file(): - body = _strip_frontmatter(skill_md.read_text(encoding="utf-8", errors="ignore")) - data["files"] = files - data["instructions"] = body - return data - - -# ── Installation ───────────────────────────────────────────────────────────── - -def _user_suffix(owner_user_id: str) -> str: - """Stable 6-char fingerprint derived from user_id, used to deduplicate private-install skill ids.""" - return hashlib.sha1(owner_user_id.encode("utf-8")).hexdigest()[:6] - - -def compute_install_id(entry_name: str, owner_user_id: Optional[str]) -> str: - """The installed AdminSkill.skill_id: admin = original name (global); user = original name + fingerprint (private).""" - if owner_user_id is None: - return entry_name - return f"{entry_name}-{_user_suffix(owner_user_id)}" - - -def base_entry_name(skill_id: str, owner_user_id: Optional[str]) -> str: - """Inverse of ``compute_install_id``: recover the underlying market entry_name from an install id. - - Private install ids look like ``{entry_name}-{fingerprint}``; stripping that - user's 6-char fingerprint suffix yields the entry_name. Global ids (owner is - None) already are the entry_name and are returned as-is. Used to merge the - "privately installed version" and the "admin global version" onto the same - underlying skill for deduplication — preventing the same skill from - appearing twice in the skill library / at runtime. - """ - if owner_user_id is None: - return skill_id - suffix = f"-{_user_suffix(owner_user_id)}" - if skill_id.endswith(suffix): - return skill_id[: -len(suffix)] - return skill_id - - -def is_installed(db: Session, slug: str, owner_user_id: Optional[str]) -> bool: - """Whether this market skill has been installed by the given scope (global / current user).""" - if slug in DEFAULT_SKILL_MARKET: - return True # built-in default skills are globally resident; considered always "installed" - m = _read_manifest(slug) - if not m: - if _get_approved_submission(db, slug) is None: - return False - entry_name = slug - else: - entry_name = m["entry_name"] - install_id = compute_install_id(entry_name, owner_user_id) - q = db.query(AdminSkill).filter(AdminSkill.skill_id == install_id) - if owner_user_id is None: - q = q.filter(AdminSkill.owner_user_id.is_(None)) - else: - q = q.filter(AdminSkill.owner_user_id == owner_user_id) - return q.first() is not None - - -def annotate_installed( - items: List[Dict[str, Any]], db: Session, owner_user_id: Optional[str] -) -> List[Dict[str, Any]]: - """Annotate each market list item with an ``installed`` flag (for the given scope). One batched query decides all.""" - if not items: - return items - install_ids = { - it["slug"]: compute_install_id(it["entry_name"], owner_user_id) for it in items - } - q = db.query(AdminSkill.skill_id, AdminSkill.dep_status) - if owner_user_id is None: - q = q.filter(AdminSkill.owner_user_id.is_(None)) - else: - q = q.filter(AdminSkill.owner_user_id == owner_user_id) - q = q.filter(AdminSkill.skill_id.in_(list(install_ids.values()))) - status_map = {sid: dep for sid, dep in q.all()} - # Rejected skills carry the admin's reason, surfaced to the user ("Rejected by admin: ..."). - rejected_ids = [sid for sid, dep in status_map.items() if dep == "rejected"] - reason_map: Dict[str, Optional[str]] = {} - if rejected_ids: - from core.services.skill_deps_request_service import get_reject_reason - for sid in rejected_ids: - reason_map[sid] = get_reject_reason(db, sid) - for it in items: - # Built-in default skills are globally resident and always available to - # everyone — mark them "installed/ready" directly (the frontend button - # shows "already built-in"); no AdminSkill lookup, no install flow. - if it.get("builtin"): - it["installed"] = True - it["dep_status"] = "ready" - it["dep_reason"] = None - continue - iid = install_ids[it["slug"]] - it["installed"] = iid in status_map - # dep_status: 'installing' = dependencies installing / 'rejected' = admin declined / 'ready' = usable. None when not installed. - it["dep_status"] = status_map.get(iid) if iid in status_map else None - it["dep_reason"] = reason_map.get(iid) - return items - - -def _rewrite_frontmatter_name(content: str, new_name: str) -> str: - """Rewrite the ``name:`` in the SKILL.md frontmatter to ``new_name`` (touching only that line). - - Replaces only the first ``name:`` line within the frontmatter block; all - other content (including multi-line descriptions) is preserved verbatim. - Returned unchanged when there is no frontmatter. - """ - if not content.startswith("---"): - return content - end = content.find("\n---", 3) - if end == -1: - return content - head = content[: end + 1] - tail = content[end + 1 :] - head = re.sub(r"(?m)^name:[ \t]*.*$", f"name: {new_name}", head, count=1) - return head + tail - - -def _load_package_files(pkg_dir: Path) -> Tuple[str, Dict[str, str]]: - """Read a market skill package: returns (SKILL.md original text, {relative path: stored content}). - - Binary files are stored as base64 (consistent with admin upload), text as - original text; the manifest and junk files are skipped, and per-file/total - size caps are enforced. - """ - skill_md = pkg_dir / SKILL_MD_NAME - if not skill_md.is_file(): - raise BadRequestError(message="市场技能包缺少 SKILL.md") - skill_content = skill_md.read_text(encoding="utf-8") - extra_files = pack_directory(pkg_dir, skip_names={MANIFEST_NAME, SKILL_MD_NAME}) - return skill_content, extra_files - - -def _inject_secrets( - skill_content: str, - extra_files: Dict[str, str], - required_secrets: List[Dict[str, Any]], - secrets: Dict[str, str], -) -> str: - """Write the user-provided credentials into secrets.json and append a "credentials configuration" section at the end of SKILL.md. - - Returns the SKILL.md content with the section appended. Missing required - fields raise a 400. - """ - cleaned: Dict[str, str] = {} - for field in required_secrets: - key = str(field.get("key") or "").strip() - if not key: - continue - val = (secrets.get(key) or "").strip() - if not val: - if field.get("required"): - label = field.get("label") or key - raise BadRequestError(message=f"请填写必填凭据：{label}") - continue - cleaned[key] = val - if not cleaned: - return skill_content - - extra_files[SECRETS_FILENAME] = json.dumps(cleaned, ensure_ascii=False, indent=2) - lines = [ - "", - "---", - "", - "## 🔑 凭据配置（由安装者提供）", - "", - f"运行本技能所需的凭据已写入技能目录下的 `{SECRETS_FILENAME}`，包含以下字段：", - "", - ] - for key in cleaned: - lines.append(f"- `{key}`") - lines += [ - "", - f"执行技能脚本前，请先从 `{SECRETS_FILENAME}` 读取这些值，并以**同名环境变量**导出后再运行，例如：", - "", - "```bash", - f'export {next(iter(cleaned))}="$(python3 -c "import json;print(json.load(open(\'{SECRETS_FILENAME}\'))[\'{next(iter(cleaned))}\'])")"', - "```", - "", - "> 凭据仅存于本技能目录，请勿在对话中明文回显。", - "", - ] - return skill_content.rstrip() + "\n" + "\n".join(lines) - - -_SECRETS_SECTION_HEADER = "## 🔑 凭据配置（由安装者提供）" -# Internal sentinel filename carrying required_secrets inside listing snapshots: -# stored with the snapshot, popped before install/detail display, never lands in -# the installed skill directory. -REQUIRED_SECRETS_SNAPSHOT = "_required_secrets.json" - - -def _strip_injected_secrets(skill_content: str) -> str: - """Strip the "credentials configuration" section appended by _inject_secrets (including its preceding --- separator). - - Listing snapshots must be in the pre-injection form — the section instructs - reading secrets.json, which the snapshot has already excluded; the - installer's own credentials get a fresh section re-injected at install time. - """ - idx = skill_content.find(_SECRETS_SECTION_HEADER) - if idx < 0: - return skill_content - sep = skill_content.rfind("\n---", 0, idx) - cut = sep if sep >= 0 else idx - return skill_content[:cut].rstrip() + "\n" - - -def _submission_required_secrets(sub: MarketplaceSubmission) -> List[Dict[str, Any]]: - """Recover a community skill's required_secrets from the snapshot sentinel entry (empty list if absent).""" - raw = (sub.extra_files or {}).get(REQUIRED_SECRETS_SNAPSHOT) - if not raw: - return [] - try: - fields = json.loads(raw) - return fields if isinstance(fields, list) else [] - except (TypeError, ValueError): - return [] - - -def install_marketplace_skill( - db: Session, - slug: str, - *, - owner_user_id: Optional[str], - secrets: Optional[Dict[str, str]] = None, -) -> Dict[str, Any]: - """Install a market skill as an ``AdminSkill`` (admin = global / user = private), returning a result dict. - - The preloaded directory takes priority; when the slug isn't in the preloaded - directory, fall back to the community listing (approved submission), with - install content taken from the submission-time snapshot. - """ - # Built-in default skills are globally resident and always available to - # everyone — no install needed (the frontend button is already disabled as - # "already built-in"); this is a defensive block to avoid duplicating 1MB+ - # office skill content into the DB as a private copy. - if slug in DEFAULT_SKILL_MARKET: - raise BadRequestError(message="内置技能已全局可用，无需安装") - secrets = secrets or {} - m = _read_manifest(slug) - if m: - pkg_dir = MARKETPLACE_DIR / slug - skill_content, extra_files = _load_package_files(pkg_dir) - else: - sub = _get_approved_submission(db, slug) - if sub is None: - raise ResourceNotFoundError("marketplace_skill", slug) - m = { - "slug": sub.slug, - "entry_name": sub.slug, - "display_name": sub.display_name, - "summary": sub.summary or "", - "user_intro": sub.user_intro or "", - "category": sub.category or "社区共享", - "tags": list(sub.tags or []), - "version": sub.version or "1.0.0", - "required_secrets": _submission_required_secrets(sub), - } - skill_content = sub.skill_content or "" - extra_files = dict(sub.extra_files or {}) - # The sentinel entry only carries the credential field names; it never lands in the install directory - extra_files.pop(REQUIRED_SECRETS_SNAPSHOT, None) - - entry_name = str(m["entry_name"]).strip() - install_id = compute_install_id(entry_name, owner_user_id) - # Always rewrite the frontmatter ``name`` to the install id: this both - # namespaces private user installs and, along the way, normalizes illegal - # names in market skills (containing spaces/Chinese etc.) — otherwise - # _load_skill_metadata_from_str fails validation with "invalid skill id". - # install_id comes from slug/entry_name and is always a legal slug. - skill_content = _rewrite_frontmatter_name(skill_content, install_id) - - required_secrets = list(m.get("required_secrets") or []) - if required_secrets: - skill_content = _inject_secrets( - skill_content, extra_files, required_secrets, secrets - ) - - # Validate SKILL.md (description required, etc.); an invalid one is a straight 400 rather than being persisted. - try: - meta = _load_skill_metadata_from_str(skill_content, install_id) - except Exception as exc: # noqa: BLE001 - raise BadRequestError(message=f"市场技能 SKILL.md 不合法：{exc}") - - dependencies = detect_dependencies( - {fn: c for fn, c in extra_files.items() if not is_binary_value(c)} - ) - - display_name = m.get("display_name") or meta.name or install_id - description = meta.description or m.get("summary") or "" - tags = list(m.get("tags") or meta.tags or []) - version = m.get("version") or meta.version or "1.0.0" - user_intro = _build_user_intro(m) or None - - ensure_ontology_build_valid( - db, - asset_type="skill", - name=display_name or install_id, - description=description, - instructions=skill_content, - tool_names=list(meta.allowed_tools or []), - ontology_tags=list(tags), - ) - - now = utc_now() - existing = db.query(AdminSkill).filter(AdminSkill.skill_id == install_id).first() - if existing is not None: - # Owner conflict: cannot overwrite a public skill or another user's private skill (admin reinstalling a global one is allowed) - if existing.owner_user_id != owner_user_id: - if existing.owner_user_id is None: - raise HTTPException(status_code=409, detail=f"技能 id 「{install_id}」与公共技能冲突") - raise HTTPException(status_code=409, detail=f"技能 id 「{install_id}」已被占用") - existing.skill_content = skill_content - existing.display_name = display_name - existing.description = description - existing.user_intro = user_intro - existing.version = version - existing.tags = tags - existing.allowed_tools = list(meta.allowed_tools or []) - existing.extra_files = extra_files - existing.dependencies = dependencies - existing.is_enabled = True - existing.updated_at = now - flag_modified(existing, "tags") - flag_modified(existing, "extra_files") - flag_modified(existing, "dependencies") - action = "updated" - else: - db.add(AdminSkill( - skill_id=install_id, - skill_content=skill_content, - display_name=display_name, - description=description, - user_intro=user_intro, - version=version, - tags=tags, - allowed_tools=list(meta.allowed_tools or []), - extra_files=extra_files, - dependencies=dependencies, - is_enabled=True, - owner_user_id=owner_user_id, - created_at=now, - updated_at=now, - )) - action = "installed" - db.commit() - # Strict dependency verdicts happen at the async route layer (gate_skill_deps - # needs to actually run probes in the sandbox; this function is synchronous). - # Here we only carry the declared dependencies out with the result; the route - # awaits the probe → decides installing/ready. - # Assign the installed skill a built-in icon by category (users/admins can change it later in skill editing). - try: - from core.services.skill_icon_service import ( - get_skill_icon, - preset_for_category, - set_skill_icon, - ) - if not get_skill_icon(db, install_id): - set_skill_icon(db, install_id, preset_for_category(m.get("category"))) - except Exception as exc: # noqa: BLE001 - logger.debug("marketplace set default icon failed: %s", exc) - refresh_skill_caches() - logger.info( - "marketplace_skill_%s: slug=%s id=%s owner=%s files=%d", - action, slug, install_id, owner_user_id or "global", len(extra_files), - ) - return { - "id": install_id, - "slug": slug, - "owner": "self" if owner_user_id else "global", - "action": action, - "dependencies": dependencies, - # dep_pending / message are filled in by the route layer after awaiting gate_skill_deps (strict probe result). - "dep_pending": False, - "message": "技能已安装" if action == "installed" else "技能已更新", - } - - -def _build_user_intro(m: Dict[str, Any]) -> str: - """Return only an explicitly authored user intro from marketplace metadata. - - The marketplace ``summary`` is card copy. Treating it as ``user_intro`` - suppresses the catalog's SKILL.md-body fallback and reduces rich skill - documentation to a single line after installation. - """ - return str(m.get("user_intro") or "").strip() - - -# ── Admin uploads a zip → listed directly in the skill marketplace ─────────── -# -# Skills uploaded by admins from the /admin console no longer land directly as -# "globally enabled" AdminSkill rows; instead they are listed as a -# status='approved' MarketplaceSubmission — appearing in the skill marketplace -# and explicitly installable, but they do NOT enter the catalog and are not -# auto-loaded by the agent. Shares the same table as users' "submit for -# listing", just review-exempt (the admin IS the review authority). -# owner_user_id uses a fixed sentinel to distinguish "admin uploads" from real -# users' private listing submissions. -ADMIN_UPLOAD_OWNER = "__admin_upload__" - - -def parse_skill_zip(data: bytes) -> Dict[str, Any]: - """Parse a skill zip → returns {skill_id, skill_content (original text), meta, extra_files, dependencies, skipped}. - - Parsing/validation only; nothing is persisted. The same unpacking logic is - shared by admin market-listing uploads and users' private uploads. - """ - try: - zf = zipfile.ZipFile(io.BytesIO(data)) - except zipfile.BadZipFile: - raise HTTPException(status_code=400, detail="Invalid zip file") - - from core.agent_skills.publication import normalize_file_path, safe_skill_id - - # Preserve ZipInfo for reads: normalized names may not exist in the archive. - entries = {} - for info in zf.infolist(): - raw_name = info.filename.replace("\\", "/") - is_dir = raw_name.endswith("/") - try: - name = normalize_file_path(raw_name[:-1] if is_dir else raw_name) - except ValueError as exc: - raise HTTPException(status_code=400, detail=str(exc)) from exc - if is_dir: - continue - if name in entries: - raise HTTPException(status_code=400, detail=f"Duplicate skill file path: {name}") - entries[name] = info - # A file cannot also be a parent directory of another file. - for name in entries: - parts = name.split("/") - if any("/".join(parts[:i]) in entries for i in range(1, len(parts))): - raise HTTPException(status_code=400, detail=f"Conflicting skill file path: {name}") - - skill_md_paths = [n for n in entries if n.split("/")[-1] == "SKILL.md"] - if not skill_md_paths: - raise HTTPException(status_code=400, detail="No SKILL.md found in zip") - - skill_md_path = skill_md_paths[0] - parts = skill_md_path.split("/") - if len(parts) == 1: - prefix = "" - elif len(parts) == 2: - prefix = parts[0] + "/" - else: - prefix = "/".join(parts[:-1]) + "/" - - try: - raw = zf.read(entries[skill_md_path]).decode("utf-8") - fm, _ = _split_frontmatter(raw) - skill_id = fm.get("name", "").strip() - if not skill_id: - if prefix: - skill_id = prefix.rstrip("/").split("/")[-1] - else: - raise HTTPException(status_code=400, detail="SKILL.md missing 'name' in frontmatter") - except SkillSpecError as e: - raise HTTPException(status_code=400, detail=f"Invalid SKILL.md: {e}") - - try: - safe_skill_id(skill_id) - meta = _load_skill_metadata_from_str(raw, skill_id) - except Exception as e: # noqa: BLE001 - raise HTTPException(status_code=400, detail=f"Invalid skill: {e}") - - JUNK_PATH_MARKERS = ("__MACOSX/", "/.git/", "/__pycache__/", "/.svn/", "/.hg/") - extra_files: Dict[str, str] = {} - skipped: List[Dict[str, str]] = [] - total_size = 0 - for entry, info in entries.items(): - if entry == skill_md_path: - continue - if prefix and not entry.startswith(prefix): - continue - rel_name = entry[len(prefix):] if prefix else entry - if not rel_name: - continue - base = rel_name.rsplit("/", 1)[-1] - if base in JUNK_BASENAMES or any(m in f"/{entry}" for m in JUNK_PATH_MARKERS): - continue - if info.file_size > MAX_SINGLE_FILE: - skipped.append({"file": rel_name, "reason": f"exceeds {MAX_SINGLE_FILE // (1024 * 1024)}MB single-file ceiling"}) - continue - try: - file_bytes = zf.read(info) - except (KeyError, zipfile.BadZipFile): - skipped.append({"file": rel_name, "reason": "unreadable zip entry"}) - continue - _, ext = os.path.splitext(rel_name) - if ext.lower() in BINARY_EXTENSIONS: - stored = encode_binary(file_bytes) - else: - try: - stored = file_bytes.decode("utf-8") - except UnicodeDecodeError: - stored = encode_binary(file_bytes) - stored_size = len(stored.encode("utf-8")) - if total_size + stored_size > MAX_TOTAL: - skipped.append({"file": rel_name, "reason": f"would exceed {MAX_TOTAL // (1024 * 1024)}MB total ceiling"}) - continue - total_size += stored_size - extra_files[rel_name] = stored - - dependencies = detect_dependencies( - {fn: c for fn, c in extra_files.items() if not is_binary_value(c)} - ) - return { - "skill_id": skill_id, - "skill_content": raw, - "meta": meta, - "extra_files": extra_files, - "dependencies": dependencies, - "skipped": skipped, - } - - -def build_skill_zip(skill_id: str, skill_content: str, extra_files: Dict[str, str]) -> bytes: - """Pack a DB skill into a zip byte stream (inverse of ``parse_skill_zip``). - - Layout is ``/SKILL.md`` + ``/``, - consistent with the upload/import convention, so an exported package can be - re-imported directly. base64 values in extra_files carrying the binary - marker are restored to their original bytes. - """ - buf = io.BytesIO() - with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf: - zf.writestr(f"{skill_id}/SKILL.md", skill_content or "") - for rel, stored in (extra_files or {}).items(): - data = decode_binary(stored) if is_binary_value(stored) else str(stored).encode("utf-8") - zf.writestr(f"{skill_id}/{rel}", data) - return buf.getvalue() - - -_ZIP_SKIP_DIR_PARTS = {"__pycache__", ".git", ".svn", ".hg", "__MACOSX"} - - -def build_skill_zip_from_dir(skill_id: str, skill_dir: Path) -> bytes: - """Pack an on-disk skill directory (built-in/filesystem skills) into a zip byte stream. - - Same layout as ``build_skill_zip``; skips version-control/cache directories - and junk files, and packs binary files as-is (no base64 round-trip, - lossless). - """ - buf = io.BytesIO() - with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf: - for p in sorted(skill_dir.rglob("*")): - if not p.is_file() or p.name in JUNK_BASENAMES: - continue - rel = p.relative_to(skill_dir) - if _ZIP_SKIP_DIR_PARTS.intersection(rel.parts): - continue - zf.write(p, f"{skill_id}/{rel.as_posix()}") - return buf.getvalue() - - -def _upsert_admin_submission( - db: Session, - *, - skill_id: str, - skill_content: str, - display_name: str, - summary: str, - category: str, - tags: List[str], - version: str, - extra_files: Dict[str, str], - note: str, - submitter_name: str, - user_intro: Optional[str] = None, -) -> Tuple[str, str]: - """Upsert an approved listing record under the ADMIN_UPLOAD_OWNER sentinel, returning (slug, action). - - When the same skill_id already has an admin listing record, update it - (keeping the original slug); otherwise create one. Shared by the two admin - listing entry points: "upload zip" and "create via form". - """ - category = _validate_category(category) - snapshot_files = dict(extra_files) - # Credential files are never distributed with the market snapshot (same as community listing). - snapshot_files.pop(SECRETS_FILENAME, None) - now = utc_now() - existing = ( - db.query(MarketplaceSubmission) - .filter( - MarketplaceSubmission.skill_id == skill_id, - MarketplaceSubmission.owner_user_id == ADMIN_UPLOAD_OWNER, - ) - .first() - ) - if existing is not None: - existing.display_name = display_name - existing.summary = summary - existing.user_intro = str(user_intro or "").strip() or None - existing.category = category - existing.tags = list(tags or []) - existing.version = version or "1.0.0" - existing.skill_content = skill_content - existing.extra_files = snapshot_files - existing.note = note - existing.status = "approved" - existing.review_note = "" - existing.reviewed_at = now - existing.updated_at = now - flag_modified(existing, "tags") - flag_modified(existing, "extra_files") - return existing.slug, "updated" - - slug = _derive_slug(db, skill_id, ADMIN_UPLOAD_OWNER) - db.add(MarketplaceSubmission( - submission_id=f"mkadm_{uuid.uuid4().hex[:16]}", - slug=slug, - skill_id=skill_id, - owner_user_id=ADMIN_UPLOAD_OWNER, - submitter_name=(submitter_name or "").strip(), - display_name=display_name, - summary=summary, - user_intro=str(user_intro or "").strip() or None, - category=category, - tags=list(tags or []), - version=version or "1.0.0", - note=note, - skill_content=skill_content, - extra_files=snapshot_files, - status="approved", - reviewed_at=now, - created_at=now, - updated_at=now, - )) - return slug, "published" - - -def publish_skill_zip_to_marketplace( - db: Session, - data: bytes, - *, - category: str, - display_name: str = "", - summary: str = "", - submitter_name: str = "管理员上传", -) -> Dict[str, Any]: - """Admin uploads a skill zip → listed directly in the marketplace as approved (not in the catalog, not globally effective). - - When the same skill (by SKILL.md's name = skill_id) is uploaded again, the - existing admin listing record is updated (keeping the original slug) rather - than adding a new one each time. - """ - parsed = parse_skill_zip(data) - meta = parsed["meta"] - name = (display_name or "").strip() or meta.name or parsed["skill_id"] - ensure_ontology_build_valid( - db, - asset_type="skill", - name=name, - description=(summary or "").strip() or meta.description or "", - instructions=parsed["skill_content"], - tool_names=list(meta.allowed_tools or []), - ontology_tags=list(meta.tags or []), - ) - slug, action = _upsert_admin_submission( - db, - skill_id=parsed["skill_id"], - skill_content=parsed["skill_content"], - display_name=name, - summary=(summary or "").strip() or meta.description or "", - category=category, - tags=list(meta.tags or []), - version=meta.version or "1.0.0", - extra_files=parsed["extra_files"], - note="管理员上传，免审核直接上架", - submitter_name=submitter_name, - ) - db.commit() - logger.info("marketplace_skill_admin_%s: skill=%s slug=%s", action, parsed["skill_id"], slug) - return { - "slug": slug, - "skill_id": parsed["skill_id"], - "display_name": name, - "category": _validate_category(category), - "stored_files": len([k for k in parsed["extra_files"] if k != SECRETS_FILENAME]), - "skipped": parsed["skipped"], - "action": action, - "message": "技能已上架技能市场" if action == "published" else "技能市场内容已更新", - } - - -def publish_skill_to_marketplace( - db: Session, - *, - skill_id: str, - skill_content: str, - display_name: str, - description: str, - version: str, - tags: List[str], - category: str, - user_intro: Optional[str] = None, - submitter_name: str = "管理员新建", -) -> Dict[str, Any]: - """Admin "create skill" form → listed in the marketplace as approved (not in the catalog, not globally effective). - - ``skill_content`` is the complete SKILL.md assembled from the form fields - (valid frontmatter). - """ - name = (display_name or "").strip() or skill_id - try: - meta = _load_skill_metadata_from_str(skill_content, skill_id) - allowed_tools = list(meta.allowed_tools or []) - parsed_description = meta.description or "" - except Exception: # Some reviewed legacy drafts have minimal frontmatter. - allowed_tools = [] - parsed_description = "" - ensure_ontology_build_valid( - db, - asset_type="skill", - name=name, - description=description or parsed_description, - instructions=skill_content, - tool_names=allowed_tools, - ontology_tags=list(tags or []), - ) - slug, action = _upsert_admin_submission( - db, - skill_id=skill_id, - skill_content=skill_content, - display_name=name, - summary=(description or "").strip(), - category=category, - tags=list(tags or []), - version=version or "1.0.0", - extra_files={}, - note="管理员新建，免审核直接上架", - submitter_name=submitter_name, - user_intro=user_intro, - ) - db.commit() - logger.info("marketplace_skill_admin_%s: skill=%s slug=%s (form)", action, skill_id, slug) - return { - "slug": slug, - "skill_id": skill_id, - "display_name": name, - "category": _validate_category(category), - "action": action, - "message": "技能已上架技能市场" if action == "published" else "技能市场内容已更新", - } - - -def delete_marketplace_skill(db: Session, slug: str) -> Dict[str, Any]: - """Admin deletes a DB listing record from the skill marketplace (admin upload / user community listing). - - Preloaded (filesystem) skills ship with the repo and cannot be deleted - online — straight 400. Deletion only removes it from the market; it does not - affect any installed skill instances (installs are content snapshots). - """ - if _read_manifest(slug) is not None: - raise BadRequestError(message="预置技能随仓库发布，不可在线删除") - sub = ( - db.query(MarketplaceSubmission) - .filter(MarketplaceSubmission.slug == slug) - .first() - ) - if sub is None: - raise ResourceNotFoundError("marketplace_skill", slug) - skill_id, owner = sub.skill_id, sub.owner_user_id - db.delete(sub) - db.commit() - logger.info("marketplace_skill_deleted: slug=%s skill=%s owner=%s", slug, skill_id, owner) - return {"slug": slug, "deleted": True} - - -# ── Community listing: user submission + admin review ──────────────────────── - -def _existing_slugs(db: Session) -> set: - """Currently occupied market slugs: the preloaded directory + all submissions. - - Must include rejected rows — the slug column is table-wide UNIQUE and - rejected submissions still occupy their slug; if derivation skipped them, - "re-submitting after rejection" would hit the unique constraint and 500 - outright. - """ - slugs = set() - if MARKETPLACE_DIR.is_dir(): - slugs.update(c.name for c in MARKETPLACE_DIR.iterdir() if c.is_dir()) - rows = db.query(MarketplaceSubmission.slug).all() - slugs.update(r[0] for r in rows) - return slugs - - -def _derive_slug(db: Session, skill_id: str, owner_user_id: str) -> str: - """Derive the market slug from the source skill id: strip the user's own install-fingerprint suffix, then ensure global uniqueness.""" - base = skill_id - suffix = f"-{_user_suffix(owner_user_id)}" - if base.endswith(suffix): - base = base[: -len(suffix)] - base = re.sub(r"[^a-z0-9_-]+", "-", base.lower()).strip("-") or "skill" - taken = _existing_slugs(db) - slug = base - n = 2 - while slug in taken: - slug = f"{base}-{n}" - n += 1 - return slug - - -def _submission_to_dict(sub: MarketplaceSubmission, *, with_content: bool = False) -> Dict[str, Any]: - data = { - "submission_id": sub.submission_id, - "slug": sub.slug, - "skill_id": sub.skill_id, - "owner_user_id": sub.owner_user_id, - "submitter_name": sub.submitter_name or "", - "display_name": sub.display_name, - "summary": sub.summary or "", - "user_intro": sub.user_intro or "", - "category": sub.category or "社区共享", - "tags": list(sub.tags or []), - "version": sub.version or "1.0.0", - "note": sub.note or "", - "status": sub.status, - "review_note": sub.review_note or "", - "reviewed_at": sub.reviewed_at.isoformat() if sub.reviewed_at else None, - "created_at": sub.created_at.isoformat() if sub.created_at else None, - "file_count": len(sub.extra_files or {}), - } - if with_content: - data["instructions"] = _strip_frontmatter(sub.skill_content or "") - data["files"] = sorted((sub.extra_files or {}).keys()) - return data - - -def submit_to_marketplace( - db: Session, - skill_id: str, - *, - owner_user_id: str, - submitter_name: str = "", - note: str = "", - category: str = "", - summary: str = "", -) -> Dict[str, Any]: - """User submits their private skill for marketplace listing (content snapshot, pending admin review).""" - row = ( - db.query(AdminSkill) - .filter(AdminSkill.skill_id == skill_id, AdminSkill.owner_user_id == owner_user_id) - .first() - ) - if row is None: - raise ResourceNotFoundError("skill", skill_id) - - active = ( - db.query(MarketplaceSubmission) - .filter( - MarketplaceSubmission.skill_id == skill_id, - MarketplaceSubmission.owner_user_id == owner_user_id, - MarketplaceSubmission.status.in_(["pending", "approved"]), - ) - .first() - ) - if active is not None: - state = "待审核" if active.status == "pending" else "已上架" - raise HTTPException(status_code=409, detail=f"该技能已有{state}的上架申请") - - # The snapshot excludes the credentials file: when installing a market - # skill with secrets, _inject_secrets writes the user's API key in - # plaintext into extra_files[SECRETS_FILENAME] — that must never be - # distributed to all installers with the listing snapshot. The credential - # field names are kept with the snapshot as a sentinel entry (without - # values), so installers are prompted to enter their own keys; SKILL.md is - # correspondingly stripped of the injected section. - snapshot_files = dict(row.extra_files or {}) - submitter_secrets = snapshot_files.pop(SECRETS_FILENAME, None) - snapshot_content = row.skill_content - if submitter_secrets: - snapshot_content = _strip_injected_secrets(row.skill_content or "") - try: - secret_keys = sorted(json.loads(submitter_secrets).keys()) - except (TypeError, ValueError, AttributeError): - secret_keys = [] - if secret_keys: - # The original required/label metadata is unavailable in the installed artifact; approximate as "all required" - snapshot_files[REQUIRED_SECRETS_SNAPSHOT] = json.dumps( - [{"key": k, "label": k, "required": True} for k in secret_keys], - ensure_ascii=False, - ) - - now = utc_now() - sub = MarketplaceSubmission( - submission_id=f"mksub_{uuid.uuid4().hex[:16]}", - slug=_derive_slug(db, skill_id, owner_user_id), - skill_id=skill_id, - owner_user_id=owner_user_id, - submitter_name=(submitter_name or "").strip(), - display_name=row.display_name or skill_id, - summary=(summary or "").strip() or (row.description or ""), - user_intro=str(row.user_intro or "").strip() or None, - category=_validate_category(category), - tags=list(row.tags or []), - version=row.version or "1.0.0", - note=(note or "").strip(), - skill_content=snapshot_content, - extra_files=snapshot_files, - status="pending", - created_at=now, - updated_at=now, - ) - db.add(sub) - db.commit() - logger.info( - "marketplace_submission_created: id=%s skill=%s owner=%s slug=%s", - sub.submission_id, skill_id, owner_user_id, sub.slug, - ) - return _submission_to_dict(sub) - - -def list_my_submissions(db: Session, owner_user_id: str) -> List[Dict[str, Any]]: - """All of the current user's listing submissions (newest → oldest).""" - rows = ( - db.query(MarketplaceSubmission) - .filter(MarketplaceSubmission.owner_user_id == owner_user_id) - .order_by(MarketplaceSubmission.created_at.desc()) - .all() - ) - return [_submission_to_dict(r) for r in rows] - - -def withdraw_submission(db: Session, submission_id: str, owner_user_id: str) -> None: - """User withdraws their own submission: pending is deleted directly; approved requires an admin to delist; rejected can be deleted and resubmitted.""" - sub = ( - db.query(MarketplaceSubmission) - .filter( - MarketplaceSubmission.submission_id == submission_id, - MarketplaceSubmission.owner_user_id == owner_user_id, - ) - .first() - ) - if sub is None: - raise ResourceNotFoundError("marketplace_submission", submission_id) - if sub.status == "approved": - raise BadRequestError(message="技能已上架，如需下架请联系管理员") - db.delete(sub) - db.commit() - - -def list_submissions(db: Session, status: Optional[str] = None) -> List[Dict[str, Any]]: - """Admin view: all listing submissions, filterable by status (pending first, newest → oldest).""" - q = db.query(MarketplaceSubmission) - if status: - q = q.filter(MarketplaceSubmission.status == status) - rows = q.order_by(MarketplaceSubmission.created_at.desc()).all() - order = {"pending": 0, "approved": 1, "rejected": 2} - rows.sort(key=lambda r: order.get(r.status, 3)) - return [_submission_to_dict(r) for r in rows] - - -def get_submission(db: Session, submission_id: str) -> Dict[str, Any]: - """Submission detail for admin review (including a SKILL.md body preview and the attachment list).""" - sub = ( - db.query(MarketplaceSubmission) - .filter(MarketplaceSubmission.submission_id == submission_id) - .first() - ) - if sub is None: - raise ResourceNotFoundError("marketplace_submission", submission_id) - return _submission_to_dict(sub, with_content=True) - - -def review_submission( - db: Session, - submission_id: str, - *, - approve: bool, - review_note: str = "", - category: str = "", -) -> Dict[str, Any]: - """Admin review: approval means listing (appears in the market list); rejecting pending = decline, rejecting approved = delist.""" - sub = ( - db.query(MarketplaceSubmission) - .filter(MarketplaceSubmission.submission_id == submission_id) - .first() - ) - if sub is None: - raise ResourceNotFoundError("marketplace_submission", submission_id) - # State-machine guard: approve only allows pending→approved (prevents two - # concurrent admins / a stale list mis-click from listing a just-rejected - # submission directly); reject allows pending (decline) and approved - # (delist). - if approve and sub.status != "pending": - state = "已上架" if sub.status == "approved" else "已驳回" - raise HTTPException(status_code=409, detail=f"该申请{state}，不能重复审核通过（如需重新上架请让用户重新提交）") - if not approve and sub.status == "rejected": - raise HTTPException(status_code=409, detail="该申请已是驳回状态") - sub.status = "approved" if approve else "rejected" - sub.review_note = (review_note or "").strip() - if approve and (category or "").strip(): - sub.category = _validate_category(category) - sub.reviewed_at = utc_now() - sub.updated_at = utc_now() - db.commit() - logger.info( - "marketplace_submission_reviewed: id=%s slug=%s status=%s", - sub.submission_id, sub.slug, sub.status, - ) - return _submission_to_dict(sub) diff --git a/src/backend/core/services/mcp_marketplace_service.py b/src/backend/core/services/mcp_marketplace_service.py deleted file mode 100644 index f75952f7..00000000 --- a/src/backend/core/services/mcp_marketplace_service.py +++ /dev/null @@ -1,1772 +0,0 @@ -"""MCP marketplace lifecycle: submit, review, publish, install, and suspend. - -Marketplace rows are credential-free reviewed snapshots. Installation creates -or updates an ``AdminMcpServer`` owned by the installing user (or global for an -admin install), encrypting install-time header values before persistence. -""" - -from __future__ import annotations -from core.infra.time import utc_now - -import json -import re -import uuid -from datetime import datetime -from typing import Any, Dict, List, Optional -from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse - -from core.config.mcp_marketplace_catalog import CURATED_MCP_MARKET_ITEMS -from core.db.models import ( - AdminMcpServer, - McpMarketInstallation, - McpMarketItem, - McpMarketSubmission, - McpMarketVersion, -) -from core.infra.exceptions import BadRequestError, ResourceNotFoundError -from core.services import marketplace_listing as ml -from core.services.mcp_management_service import ( - auth_schema_from_headers, - decrypt_mcp_headers, - encrypt_mcp_headers, - materialize_mcp_http_connection, - mcp_oauth_bundle_storage_key, - mcp_query_secret_storage_key, - mcp_url_override_storage_key, - probe_mcp_connectivity, - refresh_mcp_caches, - tool_snapshot_hash, - validate_remote_mcp_url, -) -from sqlalchemy.orm import Session - -MCP_MARKET_CATEGORIES = [ - "信息检索", - "数据分析", - "内容创作", - "办公协作", - "研发工具", - "业务系统", - "自动化", - "通用工具", -] -_SECRET_QUERY_TERMS = ("key", "token", "secret", "password", "credential") - - -def ensure_curated_market_items(db: Session) -> List[str]: - """Seed credential-free official templates once, without resurrecting deletions.""" - seeded: List[str] = [] - updated = False - now = utc_now() - for definition in CURATED_MCP_MARKET_ITEMS: - slug = str(definition["slug"]) - # Soft-deleted rows deliberately count as existing: an administrator's - # removal choice must survive application restarts. - existing_item = db.query(McpMarketItem).filter(McpMarketItem.slug == slug).first() - if existing_item: - if existing_item.deleted_at is None: - existing_version = ( - db.query(McpMarketVersion) - .filter(McpMarketVersion.version_id == existing_item.latest_version_id) - .first() - ) - if existing_version and existing_version.approved_by == "system-curated": - schema = list(definition.get("auth_schema") or []) - auth_config = _normalize_auth_config(definition.get("auth_config"), schema) - notice = dict(existing_version.listing_notice or {}) - notice.update( - { - "discovery_mode": "per_install", - "install_notice": str(definition.get("install_notice") or ""), - "docs_url": str(definition.get("docs_url") or ""), - } - ) - description = str(definition.get("description") or "") - user_intro = str(definition.get("user_intro") or "") - icon = str(definition.get("icon") or "") - changed = ( - list(existing_version.auth_schema or []) != schema - or dict(existing_version.auth_config or {}) != auth_config - or dict(existing_version.listing_notice or {}) != notice - or (existing_item.description or "") != description - or (existing_item.user_intro or "") != user_intro - or (existing_item.icon or "") != icon - ) - if changed: - existing_version.auth_schema = schema - existing_version.auth_config = auth_config - existing_version.listing_notice = notice - existing_item.description = description - existing_item.user_intro = user_intro - existing_item.icon = icon - existing_item.updated_at = now - server_ids = [ - row.server_id - for row in db.query(McpMarketInstallation) - .filter(McpMarketInstallation.slug == slug) - .all() - ] - if server_ids: - ( - db.query(AdminMcpServer) - .filter(AdminMcpServer.server_id.in_(server_ids)) - .update({AdminMcpServer.icon: icon}, synchronize_session=False) - ) - updated = True - continue - tools = list(definition.get("tools") or []) - listing_notice = { - "discovery_mode": "per_install", - "install_notice": str(definition.get("install_notice") or ""), - "docs_url": str(definition.get("docs_url") or ""), - } - version_id = f"mcpver_curated_{slug}" - item = McpMarketItem( - slug=slug, - display_name=str(definition["display_name"]), - description=str(definition.get("description") or ""), - user_intro=str(definition.get("user_intro") or ""), - category=_validate_category(str(definition.get("category") or "通用工具")), - tags=list(definition.get("tags") or []), - icon=str(definition.get("icon") or ""), - publisher_id=None, - publisher_name=str(definition.get("publisher_name") or "HugAgentOS"), - source="admin", - latest_version_id=version_id, - status="active", - status_reason=None, - last_verified_at=None, - created_at=now, - updated_at=now, - ) - db.add(item) - db.add( - McpMarketVersion( - version_id=version_id, - slug=slug, - version=_normalize_version(str(definition.get("version") or "1.0.0")), - transport=str(definition.get("transport") or "streamable_http"), - url=str(definition["url"]), - auth_schema=list(definition.get("auth_schema") or []), - auth_config=_normalize_auth_config( - definition.get("auth_config"), - list(definition.get("auth_schema") or []), - ), - tools_json=tools, - tool_hash=tool_snapshot_hash(tools), - listing_notice=listing_notice, - source_server_id=None, - approved_by="system-curated", - approved_at=now, - created_at=now, - ) - ) - seeded.append(slug) - if not seeded and not updated: - return [] - db.commit() - for slug in seeded: - ml.set_listing_enabled(db, ml.KIND_MCP, slug, True, updated_by="system-curated") - return seeded - - -def _validate_category(category: str) -> str: - value = (category or "").strip() - if value not in MCP_MARKET_CATEGORIES: - raise BadRequestError(message=f"MCP 市场分类必须是：{', '.join(MCP_MARKET_CATEGORIES)}") - return value - - -def _normalize_auth_config( - auth_config: Any, - auth_schema: List[Dict[str, Any]], -) -> Dict[str, Any]: - """Return a backwards-compatible none/token/oauth2 authentication contract.""" - config = dict(auth_config or {}) if isinstance(auth_config, dict) else {} - normalized: List[Dict[str, Any]] = [] - seen: set[str] = set() - for raw in config.get("methods") or []: - if not isinstance(raw, dict): - continue - method = dict(raw) - method_id = str(method.get("id") or method.get("type") or "").strip() - method_type = str(method.get("type") or "").strip() - if not method_id or method_id in seen or method_type not in {"none", "token", "oauth2"}: - continue - method["id"] = method_id - method["type"] = method_type - method["label"] = str(method.get("label") or method_type) - if method_type == "oauth2": - method["client_registration"] = str( - method.get("client_registration") or "dynamic_or_manual" - ) - method["scopes"] = [str(scope) for scope in (method.get("scopes") or []) if str(scope)] - normalized.append(method) - seen.add(method_id) - if not normalized: - inferred = "token" if auth_schema else "none" - normalized = [ - {"id": inferred, "type": inferred, "label": "Token" if auth_schema else "无需认证"} - ] - default_method = str(config.get("default_method") or normalized[0]["id"]) - if default_method not in {method["id"] for method in normalized}: - default_method = str(normalized[0]["id"]) - credential_mode = str(config.get("credential_mode") or "auto") - if credential_mode not in {"auto", "installer", "admin"}: - credential_mode = "auto" - return { - "default_method": default_method, - "methods": normalized, - "credential_mode": credential_mode, - } - - -def _endpoint_identity(url: str) -> tuple[str, str, int | None]: - """Scheme/host/port of an endpoint, ignoring path, query, and credentials.""" - parsed = urlparse((url or "").strip()) - scheme = (parsed.scheme or "").lower() - default_port = 443 if scheme == "https" else 80 if scheme == "http" else None - return scheme, (parsed.hostname or "").lower(), parsed.port or default_port - - -def _installs_admin_published_endpoint( - item: McpMarketItem, - version: McpMarketVersion, - runtime_url: str, -) -> bool: - """Whether an install still targets the exact endpoint an admin put on the shelf. - - Admin publishing already vets the endpoint and may legitimately point at the - deployment's own network (the built-in MCP container, a LAN service). The - SSRF boundary exists for endpoints the *installer* supplies, so it must key - on the endpoint's provenance rather than on who is installing: an unchanged - admin-published host stays trusted for everyone who installs it, while a - URL-target credential that redirects the install elsewhere is checked - strictly, whoever submits it. - """ - if item.source != "admin": - return False - return _endpoint_identity(runtime_url) == _endpoint_identity(version.url) - - -def _credential_free_connection(row: AdminMcpServer) -> tuple[str, List[Dict[str, Any]]]: - """Strip likely query-string secrets and expose only their install-time schema.""" - parsed = urlparse(row.url or "") - public_query: List[tuple[str, str]] = [] - auth_schema = auth_schema_from_headers(row.headers) - known_keys = {str(field.get("key") or "") for field in auth_schema} - for name, value in parse_qsl(parsed.query, keep_blank_values=True): - normalized = re.sub(r"[^A-Za-z0-9]+", "_", name).strip("_").upper() - looks_secret = any(term in name.lower() for term in _SECRET_QUERY_TERMS) - credential_key = f"QUERY_{normalized or 'SECRET'}" - if looks_secret and value: - if credential_key not in known_keys: - auth_schema.append( - { - "key": credential_key, - "label": name, - "target": "query", - "name": name, - "required": True, - "secret": True, - } - ) - known_keys.add(credential_key) - continue - public_query.append((name, value)) - template_url = urlunparse(parsed._replace(query=urlencode(public_query))) - return template_url, auth_schema - - -def _normalize_version(version: str) -> str: - value = (version or "1.0.0").strip() - if not re.fullmatch(r"[0-9A-Za-z][0-9A-Za-z._+-]{0,49}", value): - raise BadRequestError(message="版本号格式无效") - return value - - -def _new_slug(db: Session, display_name: str, source_server_id: str) -> str: - ascii_slug = re.sub(r"[^a-z0-9]+", "-", display_name.lower()).strip("-") - base = (ascii_slug or f"mcp-{source_server_id[-12:].lower()}")[:100] - slug = base - suffix = 2 - while ( - db.query(McpMarketItem.slug).filter(McpMarketItem.slug == slug).first() - or db.query(McpMarketSubmission.slug) - .filter( - McpMarketSubmission.slug == slug, - McpMarketSubmission.status.in_(["pending", "approved"]), - McpMarketSubmission.deleted_at.is_(None), - ) - .first() - ): - slug = f"{base[:115]}-{suffix}" - suffix += 1 - return slug - - -def _source_slug(db: Session, row: AdminMcpServer, owner_user_id: str) -> str: - previous = ( - db.query(McpMarketSubmission.slug) - .filter( - McpMarketSubmission.source_server_id == row.server_id, - McpMarketSubmission.owner_user_id == owner_user_id, - McpMarketSubmission.deleted_at.is_(None), - ) - .order_by(McpMarketSubmission.created_at.desc()) - .first() - ) - return str(previous[0]) if previous else _new_slug(db, row.display_name, row.server_id) - - -def _version_for_item(db: Session, item: McpMarketItem) -> McpMarketVersion: - version = ( - db.query(McpMarketVersion) - .filter(McpMarketVersion.version_id == item.latest_version_id) - .first() - ) - if not version: - raise ResourceNotFoundError("mcp_market_version", item.latest_version_id) - return version - - -def _installed_slug_set(db: Session, owner_user_id: Optional[str]) -> set[str]: - query = db.query(McpMarketInstallation.slug).filter( - McpMarketInstallation.status.in_(["active", "suspended"]) - ) - query = ( - query.filter(McpMarketInstallation.owner_user_id.is_(None)) - if owner_user_id is None - else query.filter(McpMarketInstallation.owner_user_id == owner_user_id) - ) - return {str(row[0]) for row in query.all()} - - -def _item_dict( - db: Session, - item: McpMarketItem, - version: McpMarketVersion, - *, - installed: bool = False, -) -> Dict[str, Any]: - auth_schema = list(version.auth_schema or []) - auth_config = _normalize_auth_config(version.auth_config, auth_schema) - requires_auth = any(method["type"] != "none" for method in auth_config["methods"]) - credentials_managed_by_admin = bool( - _admin_managed_credential_headers( - db, - item, - version, - auth_method=str(auth_config["default_method"]), - ) - ) - if auth_config["credential_mode"] == "auto": - auth_config["credential_mode"] = "admin" if credentials_managed_by_admin else "installer" - supports_admin_credentials = bool(_admin_credential_source(db, item, version)) - return { - "slug": item.slug, - "display_name": item.display_name, - "description": item.description or "", - "summary": item.description or "", - "user_intro": item.user_intro or "", - "category": item.category, - "tags": list(item.tags or []), - "icon": item.icon or "", - "publisher_name": item.publisher_name or "", - "source": item.source, - "version": version.version, - "version_id": version.version_id, - "transport": version.transport, - "url_origin": _safe_origin(version.url), - "auth_schema": auth_schema, - "auth_config": auth_config, - "requires_auth": requires_auth, - "credentials_managed_by_admin": credentials_managed_by_admin, - "supports_admin_credentials": supports_admin_credentials, - "requires_user_credentials": requires_auth and not credentials_managed_by_admin, - "tools": list(version.tools_json or []), - "tool_count": len(version.tools_json or []), - "tool_hash": version.tool_hash, - "listing_notice": dict(version.listing_notice or {}), - "status": item.status, - "status_reason": item.status_reason or "", - "last_verified_at": item.last_verified_at.isoformat() if item.last_verified_at else None, - "installed": installed, - "deletable": True, - "created_at": item.created_at.isoformat() if item.created_at else None, - "updated_at": item.updated_at.isoformat() if item.updated_at else None, - } - - -def _safe_origin(url: str) -> str: - from urllib.parse import urlparse - - parsed = urlparse(url or "") - return f"{parsed.scheme}://{parsed.netloc}" if parsed.scheme and parsed.netloc else "" - - -def list_market_items( - db: Session, - *, - owner_user_id: Optional[str], - viewer_user_id: Optional[str], - include_disabled: bool = False, -) -> Dict[str, Any]: - query = db.query(McpMarketItem).filter(McpMarketItem.deleted_at.is_(None)) - if not include_disabled: - query = query.filter(McpMarketItem.status == "active") - rows = query.order_by(McpMarketItem.updated_at.desc(), McpMarketItem.slug).all() - installed = _installed_slug_set(db, owner_user_id) - items = [ - _item_dict(db, row, _version_for_item(db, row), installed=row.slug in installed) - for row in rows - ] - items = ml.annotate_and_filter( - db, - ml.KIND_MCP, - items, - id_key="slug", - include_disabled=include_disabled, - viewer_user_id=viewer_user_id, - ) - categories = [ - category - for category in MCP_MARKET_CATEGORIES - if any(i["category"] == category for i in items) - ] - return {"items": items, "categories": categories} - - -def get_market_item( - db: Session, - slug: str, - *, - owner_user_id: Optional[str], - viewer_user_id: Optional[str], - admin: bool = False, -) -> Dict[str, Any]: - item = db.query(McpMarketItem).filter(McpMarketItem.slug == slug).first() - if not item or (not admin and item.status != "active"): - raise ResourceNotFoundError("mcp_market_item", slug) - if not admin: - ml.ensure_item_visible(db, ml.KIND_MCP, slug, viewer_user_id, resource="mcp_market_item") - data = _item_dict( - db, - item, - _version_for_item(db, item), - installed=slug in _installed_slug_set(db, owner_user_id), - ) - state = ml.annotate_and_filter( - db, - ml.KIND_MCP, - [data], - id_key="slug", - include_disabled=admin, - viewer_user_id=viewer_user_id, - ) - if not state: - raise ResourceNotFoundError("mcp_market_item", slug) - return state[0] - - -async def submit_to_marketplace( - db: Session, - source_server_id: str, - *, - owner_user_id: str, - submitter_name: str, - category: str, - version: str, - summary: str = "", - note: str = "", - tags: Optional[List[str]] = None, -) -> Dict[str, Any]: - row = ( - db.query(AdminMcpServer) - .filter( - AdminMcpServer.server_id == source_server_id, - AdminMcpServer.owner_user_id == owner_user_id, - ) - .first() - ) - if not row: - raise ResourceNotFoundError("private_mcp_server", source_server_id) - if row.transport not in ("streamable_http", "sse") or not row.url: - raise BadRequestError(message="只有远程 HTTP/SSE MCP 可以申请上架") - installed_from_market = ( - db.query(McpMarketInstallation.install_id) - .filter(McpMarketInstallation.server_id == row.server_id) - .first() - ) - if (row.extra_config or {}).get("market_slug") or installed_from_market: - raise BadRequestError(message="从 MCP 市场安装的实例不能再次申请上架") - pending = ( - db.query(McpMarketSubmission) - .filter( - McpMarketSubmission.source_server_id == source_server_id, - McpMarketSubmission.owner_user_id == owner_user_id, - McpMarketSubmission.status == "pending", - McpMarketSubmission.deleted_at.is_(None), - ) - .first() - ) - if pending: - raise BadRequestError(message="该 MCP 已有待审核的上架申请") - - await validate_remote_mcp_url(row.url, require_https=False) - ok, error = await probe_mcp_connectivity(row, db) - if not ok: - raise BadRequestError(message=f"MCP 上架检测失败：{error}") - tools = list(row.tools_json or []) - if not tools: - raise BadRequestError(message="MCP 未发现任何工具，不能申请上架") - template_url, auth_schema = _credential_free_connection(row) - now = utc_now() - submission = McpMarketSubmission( - submission_id=f"mcpsub_{uuid.uuid4().hex}", - slug=_source_slug(db, row, owner_user_id), - source_server_id=row.server_id, - owner_user_id=owner_user_id, - submitter_name=submitter_name or "", - display_name=row.display_name, - description=(summary or row.description or "").strip(), - user_intro=row.user_intro, - category=_validate_category(category), - tags=list(dict.fromkeys(str(tag).strip() for tag in (tags or []) if str(tag).strip())), - icon=row.icon, - version=_normalize_version(version), - transport=row.transport, - url=template_url, - auth_schema=auth_schema, - auth_config=_normalize_auth_config(None, auth_schema), - tools_json=tools, - tool_hash=tool_snapshot_hash(tools), - note=(note or "").strip(), - status="pending", - created_at=now, - updated_at=now, - ) - db.add(submission) - db.commit() - db.refresh(submission) - return _submission_dict(submission) - - -def _submission_dict(row: McpMarketSubmission, *, detail: bool = False) -> Dict[str, Any]: - data: Dict[str, Any] = { - "submission_id": row.submission_id, - "slug": row.slug, - "source_server_id": row.source_server_id, - "owner_user_id": row.owner_user_id, - "submitter_name": row.submitter_name or "", - "display_name": row.display_name, - "description": row.description or "", - "category": row.category, - "tags": list(row.tags or []), - "icon": row.icon or "", - "version": row.version, - "transport": row.transport, - "url_origin": _safe_origin(row.url), - "auth_schema": list(row.auth_schema or []), - "auth_config": _normalize_auth_config(row.auth_config, list(row.auth_schema or [])), - "tool_count": len(row.tools_json or []), - "tool_hash": row.tool_hash, - "listing_notice": dict(row.listing_notice or {}), - "note": row.note or "", - "status": row.status, - "review_note": row.review_note or "", - "reviewed_by": row.reviewed_by, - "reviewed_at": row.reviewed_at.isoformat() if row.reviewed_at else None, - "created_at": row.created_at.isoformat() if row.created_at else None, - } - if detail: - data["user_intro"] = row.user_intro or "" - data["url"] = row.url - data["tools"] = list(row.tools_json or []) - return data - - -def list_my_submissions(db: Session, owner_user_id: str) -> List[Dict[str, Any]]: - rows = ( - db.query(McpMarketSubmission) - .filter( - McpMarketSubmission.owner_user_id == owner_user_id, - McpMarketSubmission.deleted_at.is_(None), - ) - .order_by(McpMarketSubmission.created_at.desc()) - .all() - ) - return [_submission_dict(row) for row in rows] - - -def withdraw_submission(db: Session, submission_id: str, owner_user_id: str) -> None: - row = ( - db.query(McpMarketSubmission) - .filter( - McpMarketSubmission.submission_id == submission_id, - McpMarketSubmission.owner_user_id == owner_user_id, - McpMarketSubmission.deleted_at.is_(None), - ) - .first() - ) - if not row: - raise ResourceNotFoundError("mcp_market_submission", submission_id) - if row.status != "pending": - raise BadRequestError(message="只有待审核申请可以撤回") - row.status = "withdrawn" - row.deleted_at = utc_now() - row.updated_at = utc_now() - db.commit() - - -def list_submissions(db: Session, status: Optional[str] = None) -> List[Dict[str, Any]]: - query = db.query(McpMarketSubmission).filter(McpMarketSubmission.deleted_at.is_(None)) - if status: - query = query.filter(McpMarketSubmission.status == status) - return [ - _submission_dict(row) for row in query.order_by(McpMarketSubmission.created_at.desc()).all() - ] - - -def get_submission(db: Session, submission_id: str) -> Dict[str, Any]: - row = ( - db.query(McpMarketSubmission) - .filter( - McpMarketSubmission.submission_id == submission_id, - McpMarketSubmission.deleted_at.is_(None), - ) - .first() - ) - if not row: - raise ResourceNotFoundError("mcp_market_submission", submission_id) - return _submission_dict(row, detail=True) - - -def _publish_snapshot( - db: Session, - *, - slug: str, - display_name: str, - description: str, - user_intro: Optional[str], - category: str, - tags: List[str], - icon: Optional[str], - publisher_id: Optional[str], - publisher_name: str, - source: str, - version: str, - transport: str, - url: str, - auth_schema: List[Dict[str, Any]], - auth_config: Optional[Dict[str, Any]], - tools: List[Dict[str, Any]], - source_server_id: Optional[str], - approved_by: Optional[str], -) -> McpMarketItem: - if ( - db.query(McpMarketVersion) - .filter(McpMarketVersion.slug == slug, McpMarketVersion.version == version) - .first() - ): - raise BadRequestError(message=f"市场条目 {slug} 已存在版本 {version}，请提升版本号") - now = utc_now() - # Include a soft-deleted row so a later reviewed version can resurrect the - # same stable marketplace slug without colliding on the primary key. - item = db.query(McpMarketItem).filter(McpMarketItem.slug == slug).first() - version_id = f"mcpver_{uuid.uuid4().hex}" - if item is None: - item = McpMarketItem( - slug=slug, - display_name=display_name, - description=description, - user_intro=user_intro, - category=_validate_category(category), - tags=tags, - icon=icon, - publisher_id=publisher_id, - publisher_name=publisher_name, - source=source, - latest_version_id=version_id, - status="active", - last_verified_at=now, - created_at=now, - updated_at=now, - ) - db.add(item) - db.flush() - else: - item.deleted_at = None - item.display_name = display_name - item.description = description - item.user_intro = user_intro - item.category = _validate_category(category) - item.tags = tags - item.icon = icon - item.publisher_id = publisher_id - item.publisher_name = publisher_name - item.latest_version_id = version_id - item.status = "active" - item.status_reason = None - item.last_verified_at = now - item.updated_at = now - db.add( - McpMarketVersion( - version_id=version_id, - slug=slug, - version=_normalize_version(version), - transport=transport, - url=url, - auth_schema=auth_schema, - auth_config=_normalize_auth_config(auth_config, auth_schema), - tools_json=tools, - tool_hash=tool_snapshot_hash(tools), - source_server_id=source_server_id, - approved_by=approved_by, - approved_at=now, - created_at=now, - ) - ) - return item - - -def _next_revalidated_version(db: Session, slug: str, current_version: str) -> str: - """Return a collision-free version for an administrator-accepted snapshot.""" - semantic = re.fullmatch(r"(\d+)\.(\d+)\.(\d+)", current_version) - if semantic: - major, minor, patch = (int(value) for value in semantic.groups()) - candidate = f"{major}.{minor}.{patch + 1}" - while ( - db.query(McpMarketVersion.version_id) - .filter( - McpMarketVersion.slug == slug, - McpMarketVersion.version == candidate, - ) - .first() - ): - patch += 1 - candidate = f"{major}.{minor}.{patch + 1}" - return candidate - - base = re.sub(r"[^0-9A-Za-z._+-]", "-", current_version).strip("-.") or "1.0.0" - sequence = 1 - while True: - candidate = f"{base[:32]}+recheck.{sequence}" - if not ( - db.query(McpMarketVersion.version_id) - .filter( - McpMarketVersion.slug == slug, - McpMarketVersion.version == candidate, - ) - .first() - ): - return candidate - sequence += 1 - - -def _version_probe_candidate(item: McpMarketItem, version: McpMarketVersion) -> AdminMcpServer: - """Build a credential-free probe target from an immutable reviewed version.""" - return AdminMcpServer( - server_id=f"market_revalidate_{uuid.uuid4().hex}", - display_name=item.display_name, - description=item.description or "", - user_intro=item.user_intro, - transport=version.transport, - command=None, - args=[], - url=version.url, - env_vars={}, - env_inherit=[], - headers={}, - is_stable=False, - is_enabled=False, - sort_order=0, - extra_config={}, - tools_json=list(version.tools_json or []), - icon=item.icon, - owner_user_id=None, - source_plugin=None, - created_by="market-revalidation", - ) - - -def _accept_revalidated_snapshot( - db: Session, - item: McpMarketItem, - version: McpMarketVersion, - tools: List[Dict[str, Any]], -) -> McpMarketVersion: - """Publish live drift as a new immutable version after explicit admin review.""" - next_version = _next_revalidated_version(db, item.slug, version.version) - _publish_snapshot( - db, - slug=item.slug, - display_name=item.display_name, - description=item.description or "", - user_intro=item.user_intro, - category=item.category, - tags=list(item.tags or []), - icon=item.icon, - publisher_id=item.publisher_id, - publisher_name=item.publisher_name or "", - source=item.source, - version=next_version, - transport=version.transport, - url=version.url, - auth_schema=list(version.auth_schema or []), - auth_config=dict(version.auth_config or {}), - tools=tools, - source_server_id=version.source_server_id, - approved_by="admin-revalidation", - ) - db.flush() - accepted = _version_for_item(db, item) - if version.source_server_id: - ( - db.query(McpMarketInstallation) - .filter( - McpMarketInstallation.slug == item.slug, - McpMarketInstallation.server_id == version.source_server_id, - McpMarketInstallation.version_id == version.version_id, - ) - .update( - { - McpMarketInstallation.version_id: accepted.version_id, - McpMarketInstallation.updated_at: utc_now(), - }, - synchronize_session=False, - ) - ) - db.commit() - refresh_mcp_caches() - return accepted - - -async def review_submission( - db: Session, - submission_id: str, - *, - approve: bool, - review_note: str = "", - category: str = "", - reviewed_by: Optional[str] = None, -) -> Dict[str, Any]: - row = ( - db.query(McpMarketSubmission) - .filter( - McpMarketSubmission.submission_id == submission_id, - McpMarketSubmission.deleted_at.is_(None), - ) - .first() - ) - if not row: - raise ResourceNotFoundError("mcp_market_submission", submission_id) - if (approve and row.status != "pending") or ( - not approve and row.status not in ("pending", "approved") - ): - raise BadRequestError(message="该申请当前状态不可审核") - now = utc_now() - if approve: - source = ( - db.query(AdminMcpServer) - .filter( - AdminMcpServer.server_id == row.source_server_id, - AdminMcpServer.owner_user_id == row.owner_user_id, - ) - .first() - ) - if not source: - raise BadRequestError(message="申请人的原始 MCP 已删除，不能通过审核") - source_template_url, source_auth_schema = _credential_free_connection(source) - if source.transport != row.transport or source_template_url.strip() != row.url.strip(): - raise BadRequestError(message="MCP 连接地址或传输方式已变化，请申请人重新提交") - if source_auth_schema != list(row.auth_schema or []): - raise BadRequestError(message="MCP 认证参数已变化，请申请人重新提交") - await validate_remote_mcp_url(source.url or "", require_https=False) - ok, error = await probe_mcp_connectivity(source, db) - if not ok: - db.rollback() - raise BadRequestError(message=f"MCP 审核复检失败：{error}") - if tool_snapshot_hash(source.tools_json or []) != row.tool_hash: - db.rollback() - raise BadRequestError(message="MCP 工具或参数结构已变化，请申请人重新提交") - _publish_snapshot( - db, - slug=row.slug, - display_name=row.display_name, - description=row.description, - user_intro=row.user_intro, - category=category or row.category, - tags=list(row.tags or []), - icon=row.icon, - publisher_id=row.owner_user_id, - publisher_name=row.submitter_name, - source="community", - version=row.version, - transport=row.transport, - url=row.url, - auth_schema=list(row.auth_schema or []), - auth_config=_normalize_auth_config(row.auth_config, list(row.auth_schema or [])), - tools=list(row.tools_json or []), - source_server_id=row.source_server_id, - approved_by=reviewed_by, - ) - row.status = "approved" - else: - row.status = "rejected" - item = db.query(McpMarketItem).filter(McpMarketItem.slug == row.slug).first() - if item: - item.deleted_at = now - item.updated_at = now - ml.set_listing_enabled(db, ml.KIND_MCP, row.slug, False, updated_by=reviewed_by) - row.review_note = (review_note or "").strip() - row.reviewed_by = reviewed_by - row.reviewed_at = now - row.updated_at = now - db.commit() - if approve: - ml.set_listing_enabled(db, ml.KIND_MCP, row.slug, True, updated_by=reviewed_by) - return _submission_dict(row, detail=True) - - -async def publish_admin_server( - db: Session, - source_server_id: str, - *, - category: str, - version: str, - summary: str = "", - tags: Optional[List[str]] = None, - publisher_name: str = "管理员", - link_as_installation: bool = True, - auth_schema_override: Optional[List[Dict[str, Any]]] = None, - auth_config_override: Optional[Dict[str, Any]] = None, - allow_deferred_discovery: bool = False, -) -> Dict[str, Any]: - row = ( - db.query(AdminMcpServer) - .filter( - AdminMcpServer.server_id == source_server_id, - AdminMcpServer.owner_user_id.is_(None), - ) - .first() - ) - if not row: - raise ResourceNotFoundError("admin_mcp_server", source_server_id) - if row.transport not in ("streamable_http", "sse") or not row.url: - raise BadRequestError(message="MCP 市场仅支持远程 HTTP/SSE 服务；stdio 请通过插件市场分发") - if (row.extra_config or {}).get("market_slug"): - raise BadRequestError(message="从 MCP 市场安装的实例不能重复发布") - await validate_remote_mcp_url( - row.url, - allow_private_network=True, - require_https=False, - ) - ok, error = await probe_mcp_connectivity(row, db) - if not ok and not allow_deferred_discovery: - raise BadRequestError(message=f"MCP 发布检测失败：{error}") - tools = list(row.tools_json or []) - if not tools and not allow_deferred_discovery: - raise BadRequestError(message="MCP 未发现任何工具，不能发布") - existing = ( - db.query(McpMarketItem) - .join(McpMarketVersion, McpMarketVersion.version_id == McpMarketItem.latest_version_id) - .filter(McpMarketVersion.source_server_id == source_server_id) - .first() - ) - slug = existing.slug if existing else _new_slug(db, row.display_name, source_server_id) - template_url, inferred_auth_schema = _credential_free_connection(row) - stored_market_config = dict(row.extra_config or {}) - auth_schema = list( - auth_schema_override - if auth_schema_override is not None - else stored_market_config.get("market_auth_schema") or inferred_auth_schema - ) - auth_config = _normalize_auth_config( - ( - auth_config_override - if auth_config_override is not None - else stored_market_config.get("market_auth_config") - ), - auth_schema, - ) - item = _publish_snapshot( - db, - slug=slug, - display_name=row.display_name, - description=(summary or row.description or "").strip(), - user_intro=row.user_intro, - category=category, - tags=list(dict.fromkeys(str(tag).strip() for tag in (tags or []) if str(tag).strip())), - icon=row.icon, - publisher_id=None, - publisher_name=publisher_name, - source="admin", - version=version, - transport=row.transport, - url=template_url, - auth_schema=auth_schema, - auth_config=auth_config, - tools=tools, - source_server_id=row.server_id, - approved_by=publisher_name, - ) - db.flush() - if allow_deferred_discovery: - market_version = _version_for_item(db, item) - market_version.listing_notice = { - **dict(market_version.listing_notice or {}), - "discovery_mode": "per_install", - "install_notice": "该 MCP 的完整工具清单将在用户完成认证后动态发现。", - } - # Existing global MCPs are linked as an installation. A server created - # through the marketplace-first flow remains a disabled source template - # until an administrator explicitly installs it from the market. - if link_as_installation: - installation = ( - db.query(McpMarketInstallation) - .filter(McpMarketInstallation.server_id == row.server_id) - .first() - ) - if installation is None: - db.add( - McpMarketInstallation( - install_id=f"mcpins_{uuid.uuid4().hex}", - slug=item.slug, - version_id=item.latest_version_id, - server_id=row.server_id, - owner_user_id=None, - status="active", - installed_by="admin", - created_at=utc_now(), - updated_at=utc_now(), - ) - ) - else: - installation.slug = item.slug - installation.version_id = item.latest_version_id - installation.status = "active" - installation.updated_at = utc_now() - else: - for installation in ( - db.query(McpMarketInstallation) - .filter(McpMarketInstallation.server_id == row.server_id) - .all() - ): - db.delete(installation) - row.is_enabled = False - row.extra_config = { - **dict(row.extra_config or {}), - "market_source_only": True, - "market_source_slug": item.slug, - } - row.headers = encrypt_mcp_headers(decrypt_mcp_headers(row.headers)) - db.commit() - ml.set_listing_enabled(db, ml.KIND_MCP, item.slug, True, updated_by=publisher_name) - refresh_mcp_caches() - return _item_dict(db, item, _version_for_item(db, item), installed=link_as_installation) - - -def _credential_storage_key(field: Dict[str, Any]) -> str: - target = str(field.get("target") or "header") - name = str(field.get("name") or field.get("key") or "").strip() - if target == "query": - return mcp_query_secret_storage_key(name) - if target == "url": - return mcp_url_override_storage_key() - return name - - -def _admin_credential_source( - db: Session, - item: McpMarketItem, - version: McpMarketVersion, -) -> Optional[AdminMcpServer]: - if item.source != "admin" or not version.source_server_id: - return None - return ( - db.query(AdminMcpServer) - .filter( - AdminMcpServer.server_id == version.source_server_id, - AdminMcpServer.owner_user_id.is_(None), - ) - .first() - ) - - -def _admin_managed_credential_headers( - db: Session, - item: McpMarketItem, - version: McpMarketVersion, - *, - auth_method: str, -) -> Dict[str, str]: - """Return an admin publisher's credentials without exposing them to marketplace APIs. - - Marketplace versions remain credential-free. For an admin-published MCP, - the original global/source ``AdminMcpServer`` is the encrypted credential - authority. If it still has every required field for the selected token - method, user installations may copy those values into their own encrypted - server row without asking the user to enter the same token again. - """ - auth_config = _normalize_auth_config(version.auth_config, list(version.auth_schema or [])) - if auth_config["credential_mode"] == "installer": - return {} - selected = next( - (method for method in auth_config["methods"] if method["id"] == auth_method), - None, - ) - if selected is None or selected["type"] != "token": - return {} - fields = [ - field - for field in (version.auth_schema or []) - if not field.get("methods") or auth_method in field.get("methods", []) - ] - if not fields: - return {} - source = _admin_credential_source(db, item, version) - if source is None: - return {} - - stored = decrypt_mcp_headers(source.headers) - source_query = dict(parse_qsl(urlparse(source.url or "").query, keep_blank_values=True)) - managed: Dict[str, str] = {} - for field in fields: - storage_key = _credential_storage_key(field) - value = str(stored.get(storage_key) or "") - if not value and str(field.get("target") or "header") == "query": - query_name = str(field.get("name") or field.get("key") or "").strip() - value = str(source_query.get(query_name) or "") - if value: - managed[storage_key] = value - elif field.get("required", True): - return {} - return managed - - -def _materialize_credential_value(field: Dict[str, Any], value: str) -> str: - prefix = str(field.get("prefix") or "") - if prefix and not value.lower().startswith(prefix.lower()): - return f"{prefix}{value}" - return value - - -def _installation_secrets( - version: McpMarketVersion, - credentials: Dict[str, str], - existing_headers: Dict[str, str] | None = None, - *, - auth_method: str, - oauth_bundle: Optional[Dict[str, Any]] = None, -) -> Dict[str, str]: - stored: Dict[str, str] = {} - missing: List[str] = [] - for field in version.auth_schema or []: - method_ids = [str(value) for value in (field.get("methods") or []) if str(value)] - if method_ids and auth_method not in method_ids: - continue - key = str(field.get("key") or "").strip() - storage_key = _credential_storage_key(field) - if not key or not storage_key: - continue - value = str(credentials.get(key) or "").strip() - if value: - stored[storage_key] = _materialize_credential_value(field, value) - elif storage_key in (existing_headers or {}): - stored[storage_key] = str((existing_headers or {})[storage_key]) - elif field.get("required", True): - missing.append(str(field.get("label") or key)) - if missing: - raise BadRequestError(message=f"请填写安装凭据：{', '.join(missing)}") - oauth_key = mcp_oauth_bundle_storage_key() - if oauth_bundle: - stored[oauth_key] = json.dumps(oauth_bundle, ensure_ascii=False, separators=(",", ":")) - elif auth_method == "oauth2" and oauth_key in (existing_headers or {}): - stored[oauth_key] = str((existing_headers or {})[oauth_key]) - return stored - - -async def install_market_item( - db: Session, - slug: str, - *, - owner_user_id: Optional[str], - installed_by: Optional[str], - credentials: Optional[Dict[str, str]] = None, - auth_method: Optional[str] = None, - oauth_bundle: Optional[Dict[str, Any]] = None, -) -> Dict[str, Any]: - item = ( - db.query(McpMarketItem) - .filter( - McpMarketItem.slug == slug, - McpMarketItem.deleted_at.is_(None), - McpMarketItem.status == "active", - ) - .first() - ) - if not item: - raise ResourceNotFoundError("mcp_market_item", slug) - if owner_user_id is not None: - if slug in ml.get_disabled_ids(db, ml.KIND_MCP): - raise ResourceNotFoundError("mcp_market_item", slug) - ml.ensure_item_visible(db, ml.KIND_MCP, slug, owner_user_id, resource="mcp_market_item") - version = _version_for_item(db, item) - auth_config = _normalize_auth_config(version.auth_config, list(version.auth_schema or [])) - selected_method = str(auth_method or auth_config["default_method"]) - selected = next( - (method for method in auth_config["methods"] if method["id"] == selected_method), - None, - ) - if selected is None: - raise BadRequestError(message="不支持所选 MCP 认证方式") - if selected["type"] == "oauth2" and not oauth_bundle: - raise BadRequestError(message="请先完成 OAuth 登录") - if selected["type"] != "oauth2" and oauth_bundle: - raise BadRequestError(message="当前认证方式不能写入 OAuth 凭据") - existing = db.query(McpMarketInstallation).filter(McpMarketInstallation.slug == slug) - existing = ( - existing.filter(McpMarketInstallation.owner_user_id.is_(None)) - if owner_user_id is None - else existing.filter(McpMarketInstallation.owner_user_id == owner_user_id) - ).first() - existing_server = ( - db.query(AdminMcpServer).filter(AdminMcpServer.server_id == existing.server_id).first() - if existing - else None - ) - if existing_server is None and owner_user_id is None and version.source_server_id: - source_candidate = ( - db.query(AdminMcpServer) - .filter( - AdminMcpServer.server_id == version.source_server_id, - AdminMcpServer.owner_user_id.is_(None), - ) - .first() - ) - if source_candidate and (source_candidate.extra_config or {}).get("market_source_only"): - existing_server = source_candidate - old_headers = decrypt_mcp_headers(existing_server.headers) if existing_server else {} - managed_headers = _admin_managed_credential_headers( - db, - item, - version, - auth_method=selected_method, - ) - fallback_headers = {**managed_headers, **old_headers} - headers = _installation_secrets( - version, - dict(credentials or {}), - fallback_headers, - auth_method=selected_method, - oauth_bundle=oauth_bundle, - ) - runtime_url, _ = materialize_mcp_http_connection(version.url, headers) - await validate_remote_mcp_url( - runtime_url, - allow_private_network=_installs_admin_published_endpoint(item, version, runtime_url), - require_https=False, - ) - - now = utc_now() - server_id = existing_server.server_id if existing_server else f"mmcp_{uuid.uuid4().hex[:20]}" - candidate = AdminMcpServer( - server_id=server_id, - display_name=item.display_name, - description=item.description or "", - user_intro=item.user_intro, - transport=version.transport, - command=None, - args=[], - url=version.url, - env_vars={}, - env_inherit=[], - headers=headers, - is_stable=False, - is_enabled=True, - sort_order=0, - extra_config={ - **( - { - key: value - for key, value in dict(existing_server.extra_config or {}).items() - if key not in {"market_source_only", "market_source_slug"} - } - if existing_server - else {} - ), - "market_slug": slug, - "market_version": version.version, - "listing_notice": dict(version.listing_notice or {}), - "auth_method": selected_method, - }, - tools_json=list(version.tools_json or []), - icon=item.icon, - owner_user_id=owner_user_id, - source_plugin=None, - created_by=installed_by, - created_at=existing_server.created_at if existing_server else now, - updated_at=now, - ) - - ok, error = await probe_mcp_connectivity(candidate, db) - if not ok: - raise BadRequestError(message=f"MCP 安装检测失败：{error}") - discovery_mode = str((version.listing_notice or {}).get("discovery_mode") or "reviewed") - if discovery_mode == "per_install": - candidate.extra_config = { - **dict(candidate.extra_config or {}), - "discovery_mode": discovery_mode, - } - else: - actual_hash = tool_snapshot_hash(candidate.tools_json or []) - if actual_hash != version.tool_hash: - item.status = "changed" - item.status_reason = "远程工具或参数结构已变化，等待管理员重新审核" - item.updated_at = now - db.commit() - raise BadRequestError(message="远程 MCP 工具清单已发生变化，已暂停安装并等待重新审核") - - server = existing_server or AdminMcpServer(server_id=server_id) - for field in ( - "display_name", - "description", - "user_intro", - "transport", - "command", - "args", - "url", - "env_vars", - "env_inherit", - "is_stable", - "is_enabled", - "sort_order", - "extra_config", - "tools_json", - "icon", - "owner_user_id", - "source_plugin", - "created_by", - "updated_at", - ): - setattr(server, field, getattr(candidate, field)) - server.headers = encrypt_mcp_headers(headers) - if not existing_server: - server.created_at = now - db.add(server) - db.flush() - if existing: - previous_version_id = existing.version_id - existing.version_id = version.version_id - existing.status = "active" - existing.installed_by = installed_by - existing.updated_at = now - action = "updated" if previous_version_id != version.version_id else "existing" - else: - existing = McpMarketInstallation( - install_id=f"mcpins_{uuid.uuid4().hex}", - slug=slug, - version_id=version.version_id, - server_id=server.server_id, - owner_user_id=owner_user_id, - status="active", - installed_by=installed_by, - created_at=now, - updated_at=now, - ) - db.add(existing) - action = "installed" - db.commit() - refresh_mcp_caches() - return { - "install_id": existing.install_id, - "server_id": server.server_id, - "slug": slug, - "version": version.version, - "action": action, - } - - -def set_market_enabled( - db: Session, slug: str, enabled: bool, *, updated_by: Optional[str] = None -) -> Dict[str, Any]: - item = ( - db.query(McpMarketItem) - .filter(McpMarketItem.slug == slug, McpMarketItem.deleted_at.is_(None)) - .first() - ) - if not item: - raise ResourceNotFoundError("mcp_market_item", slug) - return ml.set_listing_enabled(db, ml.KIND_MCP, slug, enabled, updated_by=updated_by) - - -def update_market_item( - db: Session, - slug: str, - *, - display_name: Optional[str] = None, - description: Optional[str] = None, - user_intro: Optional[str] = None, - category: Optional[str] = None, - tags: Optional[List[str]] = None, - icon: Optional[str] = None, - requires_auth: Optional[bool] = None, - credential_mode: Optional[str] = None, - managed_credentials: Optional[Dict[str, str]] = None, -) -> Dict[str, Any]: - """Edit marketplace presentation metadata and its administrator-owned auth policy. - - Tool schemas, transport, endpoint templates, and version numbers remain - immutable. Authentication can be disabled, delegated to each installer, - or backed by the administrator source server's encrypted credentials. - Secret values are never persisted in marketplace rows or returned by this - function. Presentation fields are propagated to existing installations. - """ - item = ( - db.query(McpMarketItem) - .filter(McpMarketItem.slug == slug, McpMarketItem.deleted_at.is_(None)) - .first() - ) - if not item: - raise ResourceNotFoundError("mcp_market_item", slug) - version = _version_for_item(db, item) - previous_auth_schema = list(version.auth_schema or []) - - if display_name is not None: - normalized_name = display_name.strip() - if not normalized_name: - raise BadRequestError(message="MCP 市场名称不能为空") - item.display_name = normalized_name - if description is not None: - item.description = description.strip() - if user_intro is not None: - item.user_intro = user_intro.strip() or None - if category is not None: - item.category = _validate_category(category) - if tags is not None: - item.tags = list(dict.fromkeys(str(tag).strip() for tag in tags if str(tag).strip())) - if icon is not None: - item.icon = icon.strip() or None - - auth_schema = list(version.auth_schema or []) - auth_config = _normalize_auth_config(version.auth_config, auth_schema) - if requires_auth is False: - auth_schema = [] - auth_config = _normalize_auth_config( - { - "default_method": "none", - "methods": [{"id": "none", "type": "none", "label": "无需认证"}], - "credential_mode": "installer", - }, - auth_schema, - ) - elif requires_auth is True and not any( - method["type"] != "none" for method in auth_config["methods"] - ): - auth_schema = [ - { - "key": "Authorization", - "label": "Token / Authorization", - "target": "header", - "required": True, - "secret": True, - "prefix": "Bearer ", - } - ] - auth_config = _normalize_auth_config( - { - "default_method": "token", - "methods": [{"id": "token", "type": "token", "label": "Token"}], - "credential_mode": credential_mode or "installer", - }, - auth_schema, - ) - - if credential_mode is not None: - if credential_mode not in {"installer", "admin"}: - raise BadRequestError(message="MCP 凭据提供方式无效") - if credential_mode == "admin" and not auth_schema: - raise BadRequestError(message="请先勾选需要 Token/Auth") - auth_config["credential_mode"] = credential_mode - - version.auth_schema = auth_schema - version.auth_config = auth_config - - admin_source = None - if item.source == "admin" and version.source_server_id: - admin_source = ( - db.query(AdminMcpServer) - .filter( - AdminMcpServer.server_id == version.source_server_id, - AdminMcpServer.owner_user_id.is_(None), - ) - .first() - ) - if admin_source is not None: - admin_source.extra_config = { - **dict(admin_source.extra_config or {}), - "market_auth_schema": auth_schema, - "market_auth_config": auth_config, - } - - if auth_config["credential_mode"] == "admin": - if item.source != "admin" or not version.source_server_id: - raise BadRequestError(message="只有管理员发布的 MCP 才能统一配置 Token") - if admin_source is None: - raise BadRequestError(message="管理员发布源已不存在，无法统一配置 Token") - source_headers = decrypt_mcp_headers(admin_source.headers) - supplied = dict(managed_credentials or {}) - allowed_keys = {str(field.get("key") or "") for field in auth_schema} - unknown_keys = sorted(key for key in supplied if key not in allowed_keys) - if unknown_keys: - raise BadRequestError(message=f"未知的 MCP 认证字段：{', '.join(unknown_keys)}") - for field in auth_schema: - key = str(field.get("key") or "").strip() - value = str(supplied.get(key) or "").strip() - if value: - source_headers[_credential_storage_key(field)] = _materialize_credential_value( - field, value - ) - admin_source.headers = encrypt_mcp_headers(source_headers) - managed_headers = _admin_managed_credential_headers( - db, - item, - version, - auth_method=str(auth_config["default_method"]), - ) - if not managed_headers: - raise BadRequestError(message="请填写管理员统一 Token/Auth") - else: - managed_headers = {} - - item.updated_at = utc_now() - - installation_server_ids = [ - str(row[0]) - for row in db.query(McpMarketInstallation.server_id) - .filter(McpMarketInstallation.slug == slug) - .all() - ] - installations: List[AdminMcpServer] = [] - if installation_server_ids: - installations = ( - db.query(AdminMcpServer) - .filter(AdminMcpServer.server_id.in_(installation_server_ids)) - .all() - ) - for server in installations: - server.display_name = item.display_name - server.description = item.description or "" - server.user_intro = item.user_intro - server.icon = item.icon - server.updated_at = item.updated_at - if requires_auth is False: - existing_headers = decrypt_mcp_headers(server.headers) - for field in previous_auth_schema: - existing_headers.pop(_credential_storage_key(field), None) - server.headers = encrypt_mcp_headers(existing_headers) - elif managed_headers: - existing_headers = decrypt_mcp_headers(server.headers) - existing_headers.update(managed_headers) - server.headers = encrypt_mcp_headers(existing_headers) - - db.commit() - refresh_mcp_caches() - result = get_market_item( - db, - slug, - owner_user_id=None, - viewer_user_id=None, - admin=True, - ) - result["updated_installations"] = len(installations) - return result - - -def delete_market_item( - db: Session, slug: str, *, updated_by: Optional[str] = None -) -> Dict[str, Any]: - item = ( - db.query(McpMarketItem) - .filter(McpMarketItem.slug == slug, McpMarketItem.deleted_at.is_(None)) - .first() - ) - if not item: - raise ResourceNotFoundError("mcp_market_item", slug) - version = _version_for_item(db, item) - if version.source_server_id: - source = ( - db.query(AdminMcpServer) - .filter(AdminMcpServer.server_id == version.source_server_id) - .first() - ) - if source and (source.extra_config or {}).get("market_source_only"): - db.delete(source) - item.deleted_at = utc_now() - item.updated_at = utc_now() - db.commit() - ml.set_listing_enabled(db, ml.KIND_MCP, slug, False, updated_by=updated_by) - return {"slug": slug, "deleted": True} - - -def set_suspended( - db: Session, - slug: str, - *, - suspended: bool, - reason: str = "", -) -> Dict[str, Any]: - item = ( - db.query(McpMarketItem) - .filter(McpMarketItem.slug == slug, McpMarketItem.deleted_at.is_(None)) - .first() - ) - if not item: - raise ResourceNotFoundError("mcp_market_item", slug) - now = utc_now() - item.status = "suspended" if suspended else "active" - item.status_reason = (reason or "").strip() if suspended else None - item.updated_at = now - affected = 0 - installs = db.query(McpMarketInstallation).filter(McpMarketInstallation.slug == slug).all() - if suspended: - server_ids = [] - for install in installs: - install.status = "suspended" - install.updated_at = now - server_ids.append(install.server_id) - if server_ids: - affected = ( - db.query(AdminMcpServer) - .filter(AdminMcpServer.server_id.in_(server_ids)) - .update({AdminMcpServer.is_enabled: False}, synchronize_session=False) - ) - else: - server_ids = [] - for install in installs: - if install.status != "suspended": - continue - install.status = "active" - install.updated_at = now - server_ids.append(install.server_id) - if server_ids: - affected = ( - db.query(AdminMcpServer) - .filter(AdminMcpServer.server_id.in_(server_ids)) - .update({AdminMcpServer.is_enabled: True}, synchronize_session=False) - ) - db.commit() - refresh_mcp_caches() - return { - "slug": slug, - "status": item.status, - "disabled_installations": affected if suspended else 0, - "restored_installations": affected if not suspended else 0, - } - - -async def revalidate_market_item( - db: Session, - slug: str, - *, - accept_changes: bool = False, -) -> Dict[str, Any]: - item = ( - db.query(McpMarketItem) - .filter(McpMarketItem.slug == slug, McpMarketItem.deleted_at.is_(None)) - .first() - ) - if not item: - raise ResourceNotFoundError("mcp_market_item", slug) - version = _version_for_item(db, item) - if str((version.listing_notice or {}).get("discovery_mode") or "") == "per_install": - # Curated templates have no shared provider credential, so their live - # tools are intentionally discovered with each installer's credential. - # Revalidation checks only a fixed public endpoint; private endpoint - # templates are validated when the user installs them. - has_endpoint_override = any( - str(field.get("target") or "") == "url" for field in version.auth_schema or [] - ) - if not has_endpoint_override: - try: - await validate_remote_mcp_url(version.url, require_https=False) - item.status = "active" - item.status_reason = None - except BadRequestError as exc: - item.status = "suspended" - item.status_reason = f"官方端点安全复检失败：{exc}" - else: - item.status = "active" - item.status_reason = "个人端点将在安装时使用用户凭据验证" - item.last_verified_at = utc_now() - item.updated_at = utc_now() - db.commit() - return _item_dict(db, item, version, installed=False) - source = ( - db.query(AdminMcpServer) - .filter(AdminMcpServer.server_id == version.source_server_id) - .first() - if version.source_server_id - else None - ) - if source: - source_template_url, source_auth_schema = _credential_free_connection(source) - if ( - source.transport != version.transport - or source_template_url.strip() != version.url.strip() - ): - item.status = "changed" - item.status_reason = "原始 MCP 的连接地址或传输方式已变化，需发布新版本" - item.last_verified_at = utc_now() - item.updated_at = utc_now() - db.commit() - return _item_dict(db, item, version, installed=False) - auth_policy = _normalize_auth_config(version.auth_config, list(version.auth_schema or [])) - if auth_policy["credential_mode"] == "auto" and source_auth_schema != list( - version.auth_schema or [] - ): - item.status = "changed" - item.status_reason = "原始 MCP 的认证参数已变化，需发布新版本" - item.last_verified_at = utc_now() - item.updated_at = utc_now() - db.commit() - return _item_dict(db, item, version, installed=False) - try: - await validate_remote_mcp_url( - version.url, - allow_private_network=item.source == "admin", - require_https=False, - ) - except BadRequestError as exc: - set_suspended(db, slug, suspended=True, reason=f"远程地址安全复检失败：{exc}") - return _item_dict(db, item, version, installed=False) - probe_target = source or _version_probe_candidate(item, version) - ok, error = await probe_mcp_connectivity(probe_target, db if source else None) - discovered_tools = list(probe_target.tools_json or []) - if not ok: - item.status = "changed" - prefix = "复检连接失败" if source else "原始 MCP 连接已不存在，版本端点复检失败" - item.status_reason = f"{prefix}：{error}" - elif not discovered_tools: - item.status = "changed" - item.status_reason = "复检连接成功，但远程 MCP 未返回任何工具，暂不接受新快照" - else: - actual_hash = tool_snapshot_hash(discovered_tools) - if actual_hash == version.tool_hash: - item.status = "active" - item.status_reason = None - elif accept_changes and item.source == "admin": - previous_version = version.version - accepted = _accept_revalidated_snapshot(db, item, version, discovered_tools) - result = _item_dict(db, item, accepted, installed=False) - result["snapshot_accepted"] = True - result["previous_version"] = previous_version - return result - else: - item.status = "changed" - item.status_reason = "远程工具或参数结构已变化，需发布新版本并重新审核" - item.last_verified_at = utc_now() - item.updated_at = utc_now() - db.commit() - return _item_dict(db, item, version, installed=False) diff --git a/src/backend/core/services/mcp_tool_metadata.py b/src/backend/core/services/mcp_tool_metadata.py deleted file mode 100644 index acebc62c..00000000 --- a/src/backend/core/services/mcp_tool_metadata.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Capture the same public MCP schema for discovery, hashing and desktop delivery.""" -import hashlib -import json -from core.services.desktop_capability_protocol import public_tool_schema - - -def snapshot_tool(tool): - raw = getattr(tool, "_tool", tool) - if hasattr(raw, "model_dump"): - data = raw.model_dump(by_alias=True, mode="json") - else: - data = {"name": tool.name, "description": getattr(tool, "description", ""), - "inputSchema": getattr(tool, "inputSchema", None) or getattr(tool, "input_schema", {})} - return public_tool_schema(data) - - -def tool_snapshot_hash(tools): - normalized = sorted([public_tool_schema(t) for t in tools or []], key=lambda t: t["name"]) - return hashlib.sha256(json.dumps(normalized, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode()).hexdigest() diff --git a/src/backend/core/services/prompt_version_service.py b/src/backend/core/services/prompt_version_service.py deleted file mode 100644 index f8a85395..00000000 --- a/src/backend/core/services/prompt_version_service.py +++ /dev/null @@ -1,889 +0,0 @@ -"""Prompt version pool service. - -Stores multiple versions of system / code_exec / distillation / sub-agent prompts in -ContentBlock(id="prompt_versions") and provides activation + CRUD. - -Design: -- Single ContentBlock row holds {active: {kind: version_id}, versions: [...]}. -- Each version has (kind, id) as its composite key. -- `get_active_version(kind)` returns the currently active version for a kind, - or None if the DB is empty — callers then fall back to filesystem. -- Shipped prompt files under prompts/prompt_text/ and extensions/agents/default/ - serve as seeds and as the last-resort fallback when the DB is unreachable. -""" - -from __future__ import annotations - -import logging -import os -from dataclasses import dataclass -from datetime import datetime, timezone -from pathlib import Path -from threading import Lock -from typing import Any, Dict, List, Optional - -from core.content.content_blocks import DEFAULT_PROMPT_VERSIONS, PROMPT_VERSIONS_BLOCK_ID -from core.db.engine import SessionLocal -from core.db.models import ContentBlock -from extensions.paths import DEFAULT_AGENTS_DIR -from sqlalchemy.orm import Session - -logger = logging.getLogger(__name__) - - -#: 每个版本池默认版本的 id。内置 kind 与自定义 kind 共用这一个约定。 -DEFAULT_VERSION_ID = "default" - - -@dataclass(frozen=True) -class KindSpec: - """一个内置 kind 的全部差异都集中在这里，别处不再按 kind 分支。 - - ``layout`` 决定目录里的 md 文件怎么变成 parts： - - - ``concat``：目录下每个 ``*.system.md`` 是一段，拼起来构成一份提示词 - - ``independent``：每个文件是一份互不相干的独立提示词，按 part_id 单取 - - ``single``：整个 kind 只有 ``.system.md`` 一个文件 - """ - - subdir: tuple[str, ...] - layout: str - label: str - name: str - desc: str - visible: bool = True - preview: str | None = None - #: 兜底 md 的绝对目录。可选模块的提示词随模块代码放（不进 CE），不在 prompt_text/ 下。 - root: Path | None = None - - -#: 内置 kind：各自对应一段固定的运行时装配位置，有文件系统兜底，不可删。 -#: 新增一个内置 kind = 这张表加一行 + 放好 md 文件，不必再改别处。 -KIND_SPECS: Dict[str, KindSpec] = { - "desktop": KindSpec(("desktop",), "independent", "桌面端", "default - 桌面端提示词", "本机环境、文件、项目与工具规则；仅桌面端装配", preview="desktop-preview"), - "system": KindSpec( - ("default", "system"), - "concat", - "系统提示词", - "default - 标准系统提示词", - "当前默认系统提示词，包含 role / constraints / tools / workflow / format", - ), - "turbo": KindSpec( - ("turbo",), - "single", - "极速模式", - "default - 极速模式", - "极速模式（快速查询）系统提示词：仅联网搜索/网页抓取/知识库检索三类工具，" - "1-2 轮并行调用后直接作答", - ), - "plan_mode": KindSpec( - ("plan_mode",), - "single", - "计划模式", - "default - 计划模式", - "Plan 模式下用于拆解用户任务为可执行步骤的 sub-agent 系统提示词", - ), - "plan_tool": KindSpec( - ("plan_tool",), - "single", - "任务计划清单", - "default - 任务计划清单", - "顶层对话注册 update_plan 工具时追加的计划清单说明", - ), - "code_exec": KindSpec( - ("code_exec", "system"), - "concat", - "代码执行", - "default - 代码执行 (沙盒)", - "Lab 代码执行模式的系统提示词（沙盒环境、工具能力、执行规范等）", - ), - "distillation": KindSpec( - ("distillation",), - "independent", - "蒸馏", - "default - 技能蒸馏", - "从对话轨迹蒸馏出可复用技能的系统提示词", - ), - "subagents": KindSpec( - ("subagents",), - "independent", - "平台默认子智能体", - "default - 平台默认子智能体", - "探索员、执行员和审查员三个平台内置角色的独立系统提示词", - visible=False, - root=DEFAULT_AGENTS_DIR, - ), -} - -#: 核心内置 kind。可选模块启用时经 :func:`register_kind` 往 KIND_SPECS 追加自己的 kind， -#: 运行时判断一律查 KIND_SPECS，这个元组只列核心那批。 -BUILTIN_KINDS = tuple(KIND_SPECS) - - -def register_kind(key: str, spec: KindSpec) -> None: - """可选模块注册自己的内置 kind（模块关闭时不注册，Config 台就不出现这个 tab）。 - - 须在启动播种（``seed_from_filesystem``）之前调用，即放在模块的 ``setup()`` 里。 - """ - KIND_SPECS[key] = spec - -#: 兼容别名。历史上这个名字既是"内置清单"也是"合法性白名单"；自定义 kind 出现后 -#: 两者分家了——校验一律走 :func:`is_valid_kind`，这里只保留内置那批。 -VALID_KINDS = BUILTIN_KINDS - - -def list_custom_kinds(db: Optional[Session] = None) -> List[Dict[str, str]]: - """管理员自建的 kind（``[{"key","label"}]``）。 - - 存在版本池同一个 ContentBlock 的 ``custom_kinds`` 下——它们和内置 kind 共用 - versions/active 两张表，差别只在于没有文件系统兜底、可以删。 - 「对话模式」就是靠这个扩展的：管理员在提示词管理新开一个 tab，模式那边就能绑它。 - """ - payload = _load_payload(db) - out: List[Dict[str, str]] = [] - for item in payload.get("custom_kinds") or []: - if not isinstance(item, dict): - continue - key = str(item.get("key") or "").strip() - if key and key not in KIND_SPECS: - out.append({"key": key, "label": str(item.get("label") or key)}) - return out - - -def all_kinds(db: Optional[Session] = None) -> List[Dict[str, str]]: - """内置 + 自定义的完整 kind 清单，供管理端渲染 tab 与模式绑定下拉。""" - items = [{"key": k, "label": s.label, "builtin": True, "visible": s.visible, "preview": s.preview, "description": s.desc} for k, s in KIND_SPECS.items()] - items += [{**c, "builtin": False} for c in list_custom_kinds(db)] - return items - - -def is_valid_kind(kind: str, db: Optional[Session] = None) -> bool: - if kind in KIND_SPECS: - return True - return any(c["key"] == kind for c in list_custom_kinds(db)) - - -def create_custom_kind( - key: str, label: str, db: Optional[Session] = None, updated_by: str = "admin" -) -> Dict[str, str]: - """新开一个 kind（tab），并给它建一个空的 ``default`` 版本并激活。 - - 没有初始版本的 kind 在管理端会是个点进去什么都没有的空 tab，所以这里顺手建上， - 管理员进去直接就能写正文。 - """ - cleaned = "".join( - ch if (ch.isalnum() and ch.isascii()) or ch in "-_" else "_" for ch in (key or "").strip().lower() - ).strip("-_") - if not cleaned: - raise ValueError("kind key required") - if cleaned in KIND_SPECS: - raise ValueError(f"「{cleaned}」是内置 kind，不能重复创建") - payload = _clone(_load_payload(db)) - customs = payload.setdefault("custom_kinds", []) - if any(str((c or {}).get("key")) == cleaned for c in customs): - raise ValueError(f"kind「{cleaned}」已存在") - customs.append({"key": cleaned, "label": (label or cleaned).strip()[:60]}) - - # 空的初始版本：一个可编辑的 part，内容留空由管理员填。 - versions = payload.setdefault("versions", []) - now = datetime.now(timezone.utc).isoformat() - versions.append( - { - "kind": cleaned, - "id": "default", - "name": (label or cleaned).strip()[:60], - "description": "", - "parts": [{"part_id": cleaned, "display_name": label or cleaned, "content": ""}], - "created_at": now, - "updated_at": now, - "updated_by": updated_by, - } - ) - payload.setdefault("active", {})[cleaned] = "default" - _save_payload(payload, db=db, updated_by=updated_by) - return {"key": cleaned, "label": (label or cleaned).strip()[:60]} - - -def delete_custom_kind(key: str, db: Optional[Session] = None) -> bool: - """删掉一个自定义 kind 及其全部版本。内置 kind 拒绝删除。 - - 注意：绑了这个 kind 的对话模式会退回默认提示词装配（``render_kind_segment(fs_fallback=False)`` - 取不到就返回空串，装配侧按"没配提示词"处理），不会让对话起不来。 - """ - if key in KIND_SPECS: - raise ValueError("内置 kind 不可删除") - payload = _clone(_load_payload(db)) - customs = payload.get("custom_kinds") or [] - remaining = [c for c in customs if str((c or {}).get("key")) != key] - if len(remaining) == len(customs): - return False - payload["custom_kinds"] = remaining - payload["versions"] = [v for v in (payload.get("versions") or []) if v.get("kind") != key] - (payload.get("active") or {}).pop(key, None) - _save_payload(payload, db=db) - return True - -# Process-local payload cache, checked against the DB revision on every read. -_payload_cache: Optional[Dict[str, Any]] = None -_payload_cache_key: Optional[tuple] = None -_payload_cache_lock = Lock() - -# kind -> (目录签名, parts)。签名变了才重读磁盘，见 _read_fs_parts。 -_fs_parts_cache: Dict[str, tuple] = {} -_fs_parts_cache_lock = Lock() - - -# ── Helpers ───────────────────────────────────────────────────────────────── - - -def _backend_root() -> Path: - # src/backend - return Path(__file__).resolve().parents[2] - - -def _fs_dir(kind: str) -> Path: - spec = KIND_SPECS.get(kind) - if spec is None: - raise ValueError(f"unknown kind: {kind}") - if spec.root is not None: - return spec.root - return _backend_root().joinpath("prompts", "prompt_text", *spec.subdir) - - -def _fs_signature(dirp: Path) -> tuple: - """目录里每个 md 的 (文件名, mtime, 大小)。用它判断要不要重读磁盘。""" - try: - names = sorted( - [p.name for p in dirp.glob("*.system.md")] - + [p.relative_to(dirp).as_posix() for p in dirp.glob("*/system.md")] - ) - except OSError: - return () - sig: List[tuple] = [] - for name in names: - try: - st = os.stat(dirp / name) - except OSError: - continue - sig.append((name, st.st_mtime_ns, st.st_size)) - return tuple(sig) - - -def _read_fs_parts(kind: str) -> List[Dict[str, Any]]: - """Read on-disk markdown into a parts[] list. - - 布局由 :data:`KIND_SPECS` 的 ``layout`` 决定（concat / independent / single）。 - concat 版的 sort_order 取文件名的数字前缀（``05_x`` → 5），新增文件落到它本该 - 在的位置，不挤动相邻段。 - - 结果按目录内容签名缓存：这些 md 是只读的部署产物，而本函数落在每轮对话的 - 提示词兜底路径和管理台的每次 GET 上，不做缓存就是反复读同一批文件。 - """ - dirp = _fs_dir(kind) - sig = _fs_signature(dirp) - with _fs_parts_cache_lock: - cached = _fs_parts_cache.get(kind) - if cached is not None and cached[0] == sig: - return _clone(cached[1]) - - parts = _build_fs_parts(kind, dirp, [name for name, _, _ in sig]) - with _fs_parts_cache_lock: - _fs_parts_cache[kind] = (sig, parts) - return _clone(parts) - - -def _prompt_part_id(filename: str) -> str: - """Map a flat prompt file or an agent package prompt to its stable part ID.""" - if filename.endswith("/system.md"): - return Path(filename).parent.name - return filename.removesuffix(".system.md") - - -def _build_fs_parts(kind: str, dirp: Path, files: List[str]) -> List[Dict[str, Any]]: - layout = KIND_SPECS[kind].layout - - if layout == "single": - fname = f"{kind}.system.md" - if fname not in files: - return [] - return [ - { - "part_id": kind, - "display_name": kind, - "content": (dirp / fname).read_text(encoding="utf-8"), - "sort_order": 0, - "is_enabled": True, - } - ] - - if layout == "independent": - # 每个文件是一份独立提示词（按 part_id 单取，见 render_active_prompt_part）， - # 不像 concat 那样拼成一份。skill_distiller 固定排头，兼容旧数据。 - if kind == "distillation": - files = sorted(files, key=lambda f: (f != "skill_distiller.system.md", f)) - return [ - { - "part_id": _prompt_part_id(fname), - "display_name": _prompt_part_id(fname), - "content": (dirp / fname).read_text(encoding="utf-8"), - "sort_order": idx * 10, - "is_enabled": True, - } - for idx, fname in enumerate(files) - ] - - parts: List[Dict[str, Any]] = [] - for idx, fname in enumerate(files): - name = _prompt_part_id(fname) - prefix = name.split("_", 1)[0] - parts.append( - { - "part_id": f"system/{name}", - "display_name": name, - "content": (dirp / fname).read_text(encoding="utf-8"), - "sort_order": int(prefix) if prefix.isdigit() else idx * 10, - "is_enabled": True, - } - ) - return parts - - -def _default_version_for_kind(kind: str, version_id: Optional[str] = None) -> Dict[str, Any]: - """Build a new version dict from filesystem for a given kind.""" - spec = KIND_SPECS.get(kind) - if spec is None: - raise ValueError(f"unknown kind: {kind}") - - now = datetime.now(timezone.utc).isoformat() - return { - "id": version_id or DEFAULT_VERSION_ID, - "kind": kind, - "name": spec.name, - "description": spec.desc, - "parts": _read_fs_parts(kind), - "created_at": now, - "updated_at": now, - } - - -# ── Payload load / save ───────────────────────────────────────────────────── - - -def _load_payload(db: Optional[Session] = None) -> Dict[str, Any]: - """Read the committed pool revision before reusing a worker-local payload. - - Workers do not share Python memory. Local invalidation alone leaves other - workers serving old versions indefinitely, including on read-modify-write. - Query columns directly so an existing ORM identity-map entry cannot hide a - newer payload. The payload and its revision are loaded in the same query. - """ - global _payload_cache, _payload_cache_key - own_session = db is None - if own_session: - db = SessionLocal() - try: - revision = ( - db.query(ContentBlock.updated_at) - .filter(ContentBlock.id == PROMPT_VERSIONS_BLOCK_ID) - .first() - ) - cache_key = (db.get_bind(), tuple(revision) if revision is not None else None) - with _payload_cache_lock: - if _payload_cache is not None and _payload_cache_key == cache_key: - return _clone(_payload_cache) - - row = ( - db.query(ContentBlock.updated_at, ContentBlock.payload) - .filter(ContentBlock.id == PROMPT_VERSIONS_BLOCK_ID) - .first() - ) - payload = ( - row.payload - if row is not None and isinstance(row.payload, dict) - else _clone(DEFAULT_PROMPT_VERSIONS) - ) - # A write may have committed between the two SELECTs. Cache the revision - # returned with the payload, never the earlier revision probe. - cache_key = (db.get_bind(), (row.updated_at,) if row is not None else None) - with _payload_cache_lock: - _payload_cache = payload - _payload_cache_key = cache_key - return _clone(payload) - finally: - if own_session: - db.close() - - -def _save_payload( - payload: Dict[str, Any], - *, - db: Optional[Session] = None, - updated_by: str = "system", -) -> None: - own_session = db is None - if own_session: - db = SessionLocal() - try: - row = db.query(ContentBlock).filter(ContentBlock.id == PROMPT_VERSIONS_BLOCK_ID).first() - now = datetime.now(timezone.utc) - if row: - row.payload = payload - row.updated_at = now - row.updated_by = updated_by - else: - row = ContentBlock( - id=PROMPT_VERSIONS_BLOCK_ID, - payload=payload, - updated_at=now, - updated_by=updated_by, - ) - db.add(row) - db.commit() - finally: - if own_session: - db.close() - - invalidate_cache() - - -def invalidate_cache() -> None: - """Drop this worker's cache; other workers detect the DB revision on read.""" - global _payload_cache, _payload_cache_key - with _payload_cache_lock: - _payload_cache = None - _payload_cache_key = None - - -def _clone(obj: Any) -> Any: - import copy - - return copy.deepcopy(obj) - - -# ── Public API ────────────────────────────────────────────────────────────── - - -def list_versions(kind: Optional[str] = None, db: Optional[Session] = None) -> List[Dict[str, Any]]: - payload = _load_payload(db) - versions = payload.get("versions") or [] - active = payload.get("active") or {} - items = [] - for v in versions: - if kind and v.get("kind") != kind: - continue - items.append( - { - "id": v.get("id"), - "kind": v.get("kind"), - "name": v.get("name") or v.get("id"), - "description": v.get("description") or "", - "parts_count": len(v.get("parts") or []), - "is_active": active.get(v.get("kind")) == v.get("id"), - "created_at": v.get("created_at"), - "updated_at": v.get("updated_at"), - } - ) - return items - - -def get_version( - kind: str, version_id: str, db: Optional[Session] = None -) -> Optional[Dict[str, Any]]: - if not is_valid_kind(kind, db): - return None - payload = _load_payload(db) - active = payload.get("active") or {} - for v in payload.get("versions") or []: - if v.get("kind") == kind and v.get("id") == version_id: - data = _clone(v) - data["is_active"] = active.get(kind) == version_id - return data - return None - - -def get_active_version(kind: str, db: Optional[Session] = None) -> Optional[Dict[str, Any]]: - """Return the active version for a kind, or None if not found / DB empty.""" - if not is_valid_kind(kind, db): - return None - payload = _load_payload(db) - active_id = (payload.get("active") or {}).get(kind) - if not active_id: - return None - for v in payload.get("versions") or []: - if v.get("kind") == kind and v.get("id") == active_id: - return _clone(v) - return None - - -def upsert_version( - kind: str, - version_id: str, - *, - name: Optional[str] = None, - description: Optional[str] = None, - parts: Optional[List[Dict[str, Any]]] = None, - from_id: Optional[str] = None, - db: Optional[Session] = None, - updated_by: str = "admin", -) -> Dict[str, Any]: - if not is_valid_kind(kind, db): - raise ValueError(f"invalid kind: {kind}") - if not version_id: - raise ValueError("version_id required") - - payload = _clone(_load_payload(db)) - versions: List[Dict[str, Any]] = payload.setdefault("versions", []) - now = datetime.now(timezone.utc).isoformat() - - existing = next( - (v for v in versions if v.get("kind") == kind and v.get("id") == version_id), None - ) - - # If cloning, look up source (optional) - source_parts: List[Dict[str, Any]] = [] - if from_id: - src = next((v for v in versions if v.get("kind") == kind and v.get("id") == from_id), None) - if src: - source_parts = _clone(src.get("parts") or []) - if name is None: - name = f"{src.get('name') or from_id} 副本" - if description is None: - description = src.get("description") or "" - - if existing: - if name is not None: - existing["name"] = name - if description is not None: - existing["description"] = description - if parts is not None: - existing["parts"] = _normalize_parts(parts) - existing["updated_at"] = now - saved = existing - else: - # On create: if from_id is passed and parts is empty (empty list / not passed), - # always treat it as a clone. - # Only explicitly passing non-empty parts overrides the clone's source content. - if parts and len(parts) > 0: - final_parts = _normalize_parts(parts) - elif from_id and source_parts: - final_parts = source_parts - else: - final_parts = _normalize_parts(parts) if parts is not None else [] - saved = { - "id": version_id, - "kind": kind, - "name": name or version_id, - "description": description or "", - "parts": final_parts, - "created_at": now, - "updated_at": now, - } - versions.append(saved) - - _save_payload(payload, db=db, updated_by=updated_by) - return saved - - -def delete_version(kind: str, version_id: str, db: Optional[Session] = None) -> None: - if not is_valid_kind(kind, db): - raise ValueError(f"invalid kind: {kind}") - payload = _clone(_load_payload(db)) - active = payload.get("active") or {} - if active.get(kind) == version_id: - raise ValueError("cannot delete the currently active version; activate another first") - - versions: List[Dict[str, Any]] = payload.get("versions") or [] - new_list = [v for v in versions if not (v.get("kind") == kind and v.get("id") == version_id)] - if len(new_list) == len(versions): - raise KeyError(f"version not found: {kind}/{version_id}") - payload["versions"] = new_list - _save_payload(payload, db=db) - - -def activate_version(kind: str, version_id: str, db: Optional[Session] = None) -> None: - if not is_valid_kind(kind, db): - raise ValueError(f"invalid kind: {kind}") - payload = _clone(_load_payload(db)) - versions: List[Dict[str, Any]] = payload.get("versions") or [] - if not any(v.get("kind") == kind and v.get("id") == version_id for v in versions): - raise KeyError(f"version not found: {kind}/{version_id}") - payload.setdefault("active", {})[kind] = version_id - _save_payload(payload, db=db) - # Downstream prompt builders cache by (active_id, updated_at); poke their cache too. - try: - from prompts.prompt_runtime import invalidate_prompt_cache - - invalidate_prompt_cache() - except Exception: - pass - - -def _normalize_parts(parts: List[Dict[str, Any]]) -> List[Dict[str, Any]]: - result: List[Dict[str, Any]] = [] - for idx, p in enumerate(parts or []): - pid = (p.get("part_id") or "").strip() - if not pid: - continue - result.append( - { - "part_id": pid, - "display_name": p.get("display_name") or pid.split("/")[-1], - "content": p.get("content") or "", - "sort_order": int( - p.get("sort_order") if p.get("sort_order") is not None else idx * 10 - ), - "is_enabled": bool(p.get("is_enabled", True)), - } - ) - # Stable sort by sort_order - result.sort(key=lambda x: x["sort_order"]) - return result - - -# ── Seeding ───────────────────────────────────────────────────────────────── - - -def seed_from_filesystem( - *, - db: Optional[Session] = None, -) -> Dict[str, Any]: - """Create default versions from filesystem if not already present.""" - payload = _clone(_load_payload(db)) - versions: List[Dict[str, Any]] = payload.setdefault("versions", []) - active: Dict[str, str] = payload.setdefault("active", {}) - changed = False - - def exists(kind: str, vid: str) -> bool: - return any(v.get("kind") == kind and v.get("id") == vid for v in versions) - - added: List[str] = [] - - # ── One-time migration: rename legacy system/v4 → system/default ── - # Keeps the user's existing edits; the name change aligns the in-code - # default version id with other kinds (code_exec/default, distillation/default, …). - v4_row = next((v for v in versions if v.get("kind") == "system" and v.get("id") == "v4"), None) - default_row = next( - (v for v in versions if v.get("kind") == "system" and v.get("id") == "default"), None - ) - if v4_row and not default_row: - v4_row["id"] = "default" - # Refresh display name only if it still looks like the factory label - if v4_row.get("name", "").startswith("v4 - 标准版"): - v4_row["name"] = "default - 标准系统提示词" - if active.get("system") == "v4": - active["system"] = "default" - added.append("system/v4 → system/default (renamed)") - changed = True - - # ── One-time migration: extract system/90_plan_mode from every system - # version into a new plan_mode/default version (if not already seeded). - # After extraction, the system/90_plan_mode part is REMOVED from system - # versions so the main agent prompt no longer duplicates it. - if not exists("plan_mode", "default"): - extracted_content: Optional[str] = None - for v in versions: - if v.get("kind") != "system": - continue - new_parts: List[Dict[str, Any]] = [] - for p in v.get("parts") or []: - if (p.get("part_id") or "").strip() == "system/90_plan_mode": - if extracted_content is None and (p.get("content") or "").strip(): - extracted_content = p["content"] - changed = True - continue - new_parts.append(p) - v["parts"] = new_parts - if extracted_content: - now = datetime.now(timezone.utc).isoformat() - versions.append( - { - "id": "default", - "kind": "plan_mode", - "name": "default - 计划模式（从 v4 迁移）", - "description": "由历史 v4 system/90_plan_mode 片段迁移而来的 plan_mode 默认版本", - "parts": [ - { - "part_id": "plan_mode", - "display_name": "plan_mode", - "content": extracted_content, - "sort_order": 0, - "is_enabled": True, - } - ], - "created_at": now, - "updated_at": now, - } - ) - active["plan_mode"] = "default" - added.append("plan_mode/default (migrated)") - changed = True - - # 每个内置 kind 的默认版本：没有就按文件系统建；已有就把**新增**的 md 文件补进去。 - # 补这一步是因为默认版本只在首次冷启动时从文件系统建一次，之后新加的 *.system.md - # 再也进不了已建库的部署。已有 part 的正文原样保留，管理员改过的不会被覆盖。 - for kind in list(KIND_SPECS): - if not exists(kind, DEFAULT_VERSION_ID): - default_row = _default_version_for_kind(kind, DEFAULT_VERSION_ID) - versions.append(default_row) - added.append(f"{kind}/{DEFAULT_VERSION_ID}") - changed = True - else: - default_row = next( - v for v in versions if v.get("kind") == kind and v.get("id") == DEFAULT_VERSION_ID - ) - existing_pids = { - (p.get("part_id") or "").strip() for p in default_row.get("parts") or [] - } - for fs_part in _read_fs_parts(kind): - if fs_part["part_id"] not in existing_pids: - default_row.setdefault("parts", []).append(fs_part) - added.append(f"{kind}/{DEFAULT_VERSION_ID}:{fs_part['part_id']}") - changed = True - if kind not in active: - active[kind] = DEFAULT_VERSION_ID - changed = True - - if added or changed: - _save_payload(payload, db=db, updated_by="system_seed") - if added: - logger.info("[prompt_version_service] seeded: %s", ", ".join(added)) - return {"added": added, "active": active} - - -# ── Assembled prompt helpers (used by runtime callers) ────────────────────── - - -def render_active_prompt(kind: str, db: Optional[Session] = None) -> Optional[str]: - """Concatenate enabled parts of the active version into a single string. - - Returns None if no active version exists in DB (caller should fall back). - """ - v = get_active_version(kind, db=db) - if not v: - return None - parts = v.get("parts") or [] - chunks: List[str] = [] - for p in parts: - if not p.get("is_enabled", True): - continue - content = (p.get("content") or "").strip() - if content: - chunks.append(content) - return "\n\n".join(chunks) if chunks else None - - -def render_active_prompt_part( - kind: str, part_id: str, db: Optional[Session] = None -) -> Optional[str]: - """Return a single named part of the active version (independent prompts). - - Used by distillation and platform-subagent prompts where each part is a standalone - system prompt (skill_distiller / session_digest / colleague_distiller / - personal_distiller) rather than a concatenation segment. - - Back-compat: an active version that predates the multi-part layout holds - exactly one part (the legacy skill_distiller prompt) — only a request for - ``skill_distiller`` may fall back to it; other part_ids must not silently - receive the wrong prompt. - Returns None if no active version / no match (caller falls back to fs). - """ - v = get_active_version(kind, db=db) - if not v: - return None - parts = v.get("parts") or [] - for p in parts: - if p.get("part_id") == part_id: - content = (p.get("content") or "").strip() - return content or None - if len(parts) == 1 and part_id == "skill_distiller": - content = (parts[0].get("content") or "").strip() - return content or None - return None - - -def _skipped_parts(kind: str) -> set: - try: - from core.config.local_mode import local_mode_enabled - - if local_mode_enabled(): - from prompts.desktop_workspace import SKIP_PARTS - return SKIP_PARTS.get(kind) or set() - except Exception: - logger.debug("local-mode check failed while assembling %s", kind, exc_info=True) - return set() - - -def render_kind_segment( - kind: str, db: Optional[Session] = None, *, fs_fallback: bool = True -) -> str: - """某个 kind 装配好的正文：DB 激活版本 → 文件系统默认，都没有就返回空串。 - - 这是「一个 kind 的正文怎么取」的唯一实现——代码执行段、极速模式、任务计划清单 - 以及 Config 管理台 ``/v1/admin/prompts/preview`` 的预览都走它，预览与智能体 - 实际看到的因此不会漂移。 - - 文件系统这一步读的就是播种用的那批 md，所以尚未播种的部署渲染出来的，与它之后 - 存进库里的是同一份。``fs_fallback=False`` 用于「对话模式」绑定的 kind：绑了个 - 空 kind 就该退回默认装配，而不是套用别的 kind 的话术。 - """ - skip = _skipped_parts(kind) - try: - rendered = _render_active_parts(kind, db=db, skip=skip) - if rendered: - return rendered - except Exception: - logger.debug("render %s active prompt failed", kind, exc_info=True) - if not fs_fallback or kind not in KIND_SPECS: - return "" - chunks = [ - (p.get("content") or "").strip() - for p in _read_fs_parts(kind) - if p.get("is_enabled", True) - and (p.get("content") or "").strip() - and (p.get("part_id") or "") not in skip - ] - return "\n\n".join(chunks) - - -def _render_active_parts(kind: str, *, db: Optional[Session], skip: set) -> Optional[str]: - """激活版本的正文，按 ``skip`` 剔除若干段。``skip`` 为空时等同 render_active_prompt。""" - if not skip: - return render_active_prompt(kind, db=db) - version = get_active_version(kind, db=db) - if not version: - return None - chunks = [ - (p.get("content") or "").strip() - for p in (version.get("parts") or []) - if p.get("is_enabled", True) - and (p.get("content") or "").strip() - and (p.get("part_id") or "").strip() not in skip - ] - return "\n\n".join(chunks) or None - - -#: 极速模式在库和文件都取不到时的最后兜底——它替换掉整份系统提示词，返回空串会让 -#: 智能体裸奔，所以这一段必须留在代码里。 -_TURBO_LAST_RESORT = ( - "你是极速查询助手。收到问题后，最多发起 1-2 轮工具调用（同一轮内可并行" - "调用多个检索工具），拿到结果立即用中文简洁作答，并标注信息来源。" - "不要使用超出检索范围的工具，不要反复迭代。" -) - - -def render_turbo_system_prompt(db: Optional[Session] = None) -> str: - """极速模式（快速查询）的整份系统提示词。 - - 极速模式不做常规装配：智能体只带检索类工具，默认提示词的工具/流程段都不适用。 - """ - return render_kind_segment("turbo", db=db) or _TURBO_LAST_RESORT - - -def effective_parts(kind: str, db: Optional[Session] = None) -> List[Dict[str, Any]]: - """Read the active version, with whole-version filesystem fallback only.""" - try: - version = get_active_version(kind, db=db) - except Exception: - logger.warning("Prompt version unavailable for %s; using filesystem", kind, exc_info=True) - version = None - return version.get("parts", []) if version is not None else _read_fs_parts(kind) diff --git a/src/backend/core/services/skill_draft_service.py b/src/backend/core/services/skill_draft_service.py deleted file mode 100644 index 80193c60..00000000 --- a/src/backend/core/services/skill_draft_service.py +++ /dev/null @@ -1,192 +0,0 @@ -"""Skill draft approve/reject — the single implementation shared by the AdminApp review endpoint and ConfigApp colleague-distillation save. - -Extracted from api/routes/v1/admin_skill_drafts.py::approve_draft with behavior kept -identical; adds owner_user_id (whether the artifact is global or belongs to a specific -user) and edited_content (minor tweaks before saving). -The "immutable after approved/rejected" constraint is enforced here, naturally -preventing the two consoles from double-persisting. -""" - -from __future__ import annotations -from core.infra.time import utc_now - -import logging -import uuid -from datetime import datetime -from typing import Optional, Tuple - -from fastapi import HTTPException -from sqlalchemy.orm import Session - -from core.db.models import AdminSkill, AdminSkillDraft -from core.ontology.build_validator import ensure_ontology_build_valid - -logger = logging.getLogger(__name__) - - -def get_draft_or_404(db: Session, draft_id: str) -> AdminSkillDraft: - d = db.query(AdminSkillDraft).filter(AdminSkillDraft.draft_id == draft_id).first() - if not d: - raise HTTPException(status_code=404, detail="draft_not_found") - return d - - -def approve_draft( - db: Session, - draft_id: str, - *, - enable_immediately: bool = False, - reviewer_comment: Optional[str] = None, - reviewer_id: Optional[str] = None, - owner_user_id: Optional[str] = None, - edited_content: Optional[str] = None, - publish_to_marketplace: bool = False, - category: Optional[str] = None, -) -> Tuple[AdminSkillDraft, str]: - """Approve a draft: materialize the skill artifact per decision. Returns (draft, skill_id). - - - decision='new_skill' with ``publish_to_marketplace=True`` → **publish to the skill - marketplace** (submitted as approved, effective only once a user installs it). Must - specify ``category`` (one of the marketplace's 8 fixed categories); in this mode - ``enable_immediately`` / ``owner_user_id`` do not apply (marketplace items have no - enable/ownership concept). — AdminApp pending-draft review takes this path. - - decision='new_skill' with ``publish_to_marketplace=False`` → directly create a formal - skill (optionally enable immediately / specify owner as a private skill). — ConfigApp - colleague-distillation "confirm persist" takes this path. - - decision='patch' → patch an existing formal skill in place (in-place update of a live skill). - - edited_content non-empty → overwrite the draft content with it before persisting (only pending is editable) - """ - d = get_draft_or_404(db, draft_id) - if d.review_status != "pending": - raise HTTPException(status_code=400, detail="draft_not_pending") - - if edited_content is not None and edited_content.strip(): - d.skill_content = edited_content - - skill_id = d.proposed_skill_id - validation_target = None - if d.decision == "patch": - target_id = d.patch_target_id or skill_id - validation_target = ( - db.query(AdminSkill).filter(AdminSkill.skill_id == target_id).first() - ) - if not validation_target: - raise HTTPException(status_code=400, detail=f"patch_target_missing:{target_id}") - ensure_ontology_build_valid( - db, - asset_type="skill", - name=d.display_name - or (validation_target.display_name if validation_target else None) - or skill_id, - description=d.description - or (validation_target.description if validation_target else None) - or "", - instructions=d.skill_content or "", - tool_names=list( - d.allowed_tools - or (validation_target.allowed_tools if validation_target else None) - or [] - ), - ontology_tags=list( - d.tags or (validation_target.tags if validation_target else None) or [] - ), - ) - - if d.decision == "new_skill" and publish_to_marketplace: - # Approved new skill goes to the skill marketplace (not directly into the formal store). The marketplace publish commits internally. - from core.services import marketplace_service - - result = marketplace_service.publish_skill_to_marketplace( - db, - skill_id=skill_id, - skill_content=d.skill_content, - display_name=d.display_name or skill_id, - description=d.description or "", - version=d.version or "0.1.0", - tags=list(d.tags or []), - category=category or "", - submitter_name=( - "同事蒸馏审核" if d.draft_kind == "colleague" else "蒸馏审核" - ), - ) - skill_id = result["skill_id"] - elif d.decision == "new_skill": - existing: Optional[AdminSkill] = ( - db.query(AdminSkill).filter(AdminSkill.skill_id == skill_id).first() - ) - if existing: - skill_id = f"{skill_id}-{uuid.uuid4().hex[:4]}" - created_by = ( - "colleague_distill" if d.draft_kind == "colleague" else "distiller_approved" - ) - skill = AdminSkill( - skill_id=skill_id, - skill_content=d.skill_content, - display_name=d.display_name or skill_id, - description=d.description or "", - version=d.version or "0.1.0", - tags=list(d.tags or []), - allowed_tools=list(d.allowed_tools or []), - extra_files={}, - is_enabled=bool(enable_immediately), - owner_user_id=str(owner_user_id) if owner_user_id else None, - created_at=utc_now(), - updated_at=utc_now(), - created_by=created_by, - ) - db.add(skill) - elif d.decision == "patch": - target_id = d.patch_target_id or skill_id - target = validation_target - assert target is not None - target.skill_content = d.skill_content - if d.display_name: - target.display_name = d.display_name - if d.description: - target.description = d.description - if d.version: - target.version = d.version - if d.tags: - target.tags = list(d.tags) - if d.allowed_tools: - target.allowed_tools = list(d.allowed_tools) - target.updated_at = utc_now() - skill_id = target_id - else: - raise HTTPException(status_code=400, detail=f"unsupported_decision:{d.decision}") - - d.review_status = "approved" - d.reviewer_comment = reviewer_comment - if reviewer_id: - d.reviewer_id = reviewer_id - d.reviewed_at = utc_now() - d.updated_at = utc_now() - - db.commit() - try: - from core.agent_skills.cache_refresh import refresh_skill_caches - refresh_skill_caches() - except Exception as exc: - logger.warning("skill_draft_service: cache refresh failed (%s)", exc) - return d, skill_id - - -def reject_draft( - db: Session, - draft_id: str, - *, - rejected_reason: str, - reviewer_id: Optional[str] = None, -) -> AdminSkillDraft: - d = get_draft_or_404(db, draft_id) - if d.review_status != "pending": - raise HTTPException(status_code=400, detail="draft_not_pending") - - d.review_status = "rejected" - d.rejected_reason = rejected_reason - if reviewer_id: - d.reviewer_id = reviewer_id - d.reviewed_at = utc_now() - d.updated_at = utc_now() - db.commit() - return d diff --git a/src/backend/core/services/system_config.py b/src/backend/core/services/system_config.py deleted file mode 100644 index a6afd2ef..00000000 --- a/src/backend/core/services/system_config.py +++ /dev/null @@ -1,655 +0,0 @@ -"""Central service configuration service (DB-driven, cached). - -Manages external service configs (DB query, KB, industry, file parser) stored -in the system_configs table. Thread-safe singleton with a short TTL cache so -admin changes take effect within seconds without a restart. - -Falls back to os.getenv() when DB has no value for a given key. -""" - -from __future__ import annotations - -import logging -import os -import threading -import time -from typing import Optional - -from core.db.engine import SessionLocal -from core.db.models import SystemConfig -from core.services.edition_system_config import CONFIG_KEY_TO_ENV as EDITION_CONFIG_KEY_TO_ENV -from core.services.edition_system_config import SEED_CONFIGS as EDITION_SEED_CONFIGS - -logger = logging.getLogger(__name__) - -_CACHE_TTL_SECONDS = 30.0 - -_PROVIDER_SCOPED_KNOWLEDGE_FIELDS = ("url", "api_key", "allowed_dataset_ids") - -# ── Seed definitions (single source of truth) ──────────────────────────────── -# (config_key, default_value, display_name, description, group_key, is_secret) -SEED_CONFIGS: list[tuple[str, str | None, str, str, str, bool]] = [ - # query_database - ( - "query_database.url", - None, - "数据库查询接口 URL", - "NL2SQL 数据库查询服务地址", - "query_database", - False, - ), - ("query_database.timeout", "40", "超时时间(秒)", "请求超时时间", "query_database", False), - ("query_database.retry_times", "1", "重试次数", "请求失败重试次数", "query_database", False), - ( - "query_database.max_output_tokens", - "45000", - "最大输出 Token", - "数据库查询最大输出 Token 数", - "query_database", - False, - ), - ( - "database_query.capability_enabled", - "true", - "数据库查询能力总开关", - "控制统一「数据库查询」能力是否在能力中心可用。由数据库工具页维护，不写入 catalog.json。", - "query_database", - False, - ), - ( - "knowledge_base.detail_max_chars", - "50000", - "详情最大字符数", - "知识库文档详情最大字符数", - "knowledge_base", - False, - ), - # file_parser - ("file_parser.api_url", None, "文件解析 API URL", "PDF/文档解析服务地址", "file_parser", False), - ("file_parser.timeout", "60", "超时时间(秒)", "文件解析请求超时", "file_parser", False), - ("file_parser.lang_list", "ch", "语言列表", "OCR 语言列表", "file_parser", False), - ("file_parser.backend", "pipeline", "解析后端", "解析后端引擎", "file_parser", False), - ("file_parser.parse_method", "auto", "解析方法", "auto / ocr / txt", "file_parser", False), - ( - "file_parser.formula_enable", - "true", - "启用公式识别", - "是否启用公式识别", - "file_parser", - False, - ), - ("file_parser.table_enable", "true", "启用表格识别", "是否启用表格识别", "file_parser", False), - # internet_search - ( - "internet_search.engine", - "tavily", - "搜索引擎", - "互联网搜索引擎 (tavily / baidu / langsearch)", - "internet_search", - False, - ), - ( - "internet_search.tavily_api_key", - None, - "Tavily API Key", - "Tavily 互联网搜索服务密钥", - "internet_search", - True, - ), - ( - "internet_search.baidu_api_key", - None, - "百度搜索 API Key", - "百度千帆 AppBuilder 搜索服务密钥", - "internet_search", - True, - ), - ( - "internet_search.langsearch_api_key", - None, - "LangSearch API Key", - "LangSearch Web Search API 服务密钥", - "internet_search", - True, - ), - # sandbox / code execution. The admin toggle is the **sole authority** (no env fallback): - # explicitly defaults to "true"; admins can turn it off in "System config", which writes "false" and persists. - ( - "sandbox.code_capability_enable", - "true", - "主智能体代码执行能力", - "开启后主/计划/批量/子智能体默认具备文件工具+沙箱+我的空间访问；关闭则仅 Lab " - "入口可用。本开关为唯一控制源，保存后 ≤30s 生效、无需重启。", - "sandbox", - False, - ), - # ontology —— independent enterprise governance gate for plugin imports. - # Unlike the user's ontology preference, this switch cannot be bypassed by - # turning off ontology validation in personal settings. - ( - "ontology.force_plugin_import_build_validation", - "false", - "强制构建时本体校验", - "插件导入或安装时必须通过激活 Domain Pack 的构建校验；个人本体开关不能关闭此门禁。", - "ontology", - False, - ), - # context —— the single compaction trigger ratio. Read per turn by - # compaction_service.resolve_trigger_ratio and applied by every phase (mid-turn at the ReAct step - # boundary, pre-turn before a turn samples, post-turn background pre-warm) — it is no longer an - # "in-turn only" knob competing with CHAT_COMPACT_TRIGGER_RATIO. Admin config takes precedence over - # env; takes effect on new conversations after saving, no restart needed. - # ⚠️ Grouped under context, not chat: the chat group is in service_configs._HIDDEN_GROUPS - # (chat.user_model_switch_enabled is exclusively managed by the model management panel to prevent a double entry point), - # so attaching to the chat group would hide the whole group in the system config UI. - ( - "chat.compress_in_turn_ratio", - "0.90", - "轮内压缩触发比例", - "上下文占用超过「模型窗口 × 该值」时触发压缩（0.5~0.90）。计数优先用模型返回的" - "真实 token 用量，拿不到时才退回字节估算，两者都不含模型输出长度——输出余量已由" - "「窗口 95% 可用」统一预留。调大=更晚压缩（上下文更完整）；调小=更早压缩。" - "该比例对轮内、轮前、轮末三个时机统一生效。", - "context", - False, - ), - # dingtalk —— DingTalk workbench Custom App (enterprise self-built app) credentials. Once AppKey/AppSecret are filled, - # DingTalk login and the dws command inside the sandbox execute as that self-built app, bypassing DingTalk's co-creation-era - # "CLI data access" org toggle and self-controlling scope; leave empty to use the dws default shared app (org admin must - # enable CLI data access first). Sole authority = admin config (no env fallback), takes effect ≤30s after saving, no restart. - ( - "dingtalk.client_id", - None, - "钉钉 AppKey (client-id)", - "企业自建钉钉应用的 AppKey。在 open-dev.dingtalk.com 创建企业内部应用后获取。", - "dingtalk", - True, - ), - ( - "dingtalk.client_secret", - None, - "钉钉 AppSecret (client-secret)", - "企业自建钉钉应用的 AppSecret，与 AppKey 配对。", - "dingtalk", - True, - ), - ( - "dingtalk.trusted_domains", - "*.dingtalk.com", - "受信域名白名单", - "dws bearer token 允许发送的域名，逗号分隔，默认锁死 *.dingtalk.com 防 token 外泄。", - "dingtalk", - False, - ), - # firecrawl —— deployment-level credentials for the firecrawl web scraping/retrieval CLI. The firecrawl CLI is preinstalled - # in the sandbox; at runtime these two items are injected as FIRECRAWL_API_KEY / FIRECRAWL_API_URL env vars into - # every user's sandbox commands (one shared config for all users, mirroring the DingTalk dws Custom App approach). - # - Cloud version: fill only api_key (starts with fc-). - # - Self-hosted: fill api_url pointing to the self-built instance (the CLI skips cloud auth once it sees api_url); if the - # self-built instance also needs a key, fill both. - # Sole authority = admin config (no env fallback), takes effect ≤30s after saving, no restart. - ( - "firecrawl.api_key", - None, - "Firecrawl API Key", - "firecrawl 云版 API Key（fc- 开头），在 firecrawl.dev 注册获取。用自托管实例时可留空。", - "firecrawl", - True, - ), - ( - "firecrawl.api_url", - None, - "Firecrawl 自托管地址", - "自托管 firecrawl 实例地址（如 http://firecrawl:3002）。填了即走自建实例、CLI 跳过云端鉴权；用云版时留空。", - "firecrawl", - False, - ), - # lark —— Feishu workbench app (enterprise self-built app) credentials. Once AppID/AppSecret are filled, the lark-cli inside - # the sandbox uses that self-built app as its OAuth app: admin configures once, and each user completes the device-flow - # auth login to operate Feishu as themselves. The app config is seeded by the backend into each user's lark HOME - # (config init), not injected as a global env var — this avoids lark-cli defaulting to the tenant/bot token flow when - # LARK_APP_* is in the environment, which would break --as user. Sole authority = admin config, ≤30s after saving, no restart. - ( - "lark.app_id", - None, - "飞书 App ID", - "企业自建飞书应用的 App ID。在 open.feishu.cn 创建企业自建应用后获取。", - "lark", - True, - ), - ( - "lark.app_secret", - None, - "飞书 App Secret", - "企业自建飞书应用的 App Secret，与 App ID 配对。", - "lark", - True, - ), - # auth —— login and session. Number of days a session is kept after the user checks "remember login state" (checked by - # default) on the login page. Sole authority = admin config (no env fallback), takes effect ≤30s after saving, no restart; - # if unchecked, the default session duration (SESSION_TTL_HOURS) is used. - ( - "auth.remember_me_days", - "30", - "记住登录状态时长(天)", - "用户在登录页勾选「记住登录状态」后，登录会话保持的天数（默认 30 天）。" - "未勾选则使用默认会话时长。", - "auth", - False, - ), - # turbo —— 极速模式（检索速查）。工具集独立于能力目录（catalog）的启停状态： - # 极速模式装配的 MCP 以本组配置为唯一来源，管理台改动 ≤30s 生效、无需重启。 - ( - "turbo.mcp_server_ids", - "retrieve_dataset_content,internet_search,web_fetch", - "极速模式可用工具", - "极速模式下装配的 MCP 工具集合（逗号分隔的 server id）。" - "该集合不受「能力目录」中 MCP/技能/智能体/插件启停的影响，是极速模式工具面的唯一来源。", - "turbo", - False, - ), - ( - "turbo.skill_ids", - "", - "极速模式可用技能", - "极速模式下装配的技能集合（逗号分隔的 skill id）。" - "极速模式没有 Bash / 沙箱 / 文件写入，选纯文本类技能（写作模板、话术规范等）效果最好；" - "需要代码执行的技能只会被读到说明、无法执行。同样不受「能力目录」启停影响。", - "turbo", - False, - ), - ( - "turbo.plugin_ids", - "", - "极速模式可用插件", - "极速模式下装配的插件集合（逗号分隔的 install_id）。" - "插件是「技能 + 工具」的能力包，选中后其全部工具与技能一并在极速模式生效；" - "与上面的 MCP 工具集合合并去重，同样不受「能力目录」启停影响。", - "turbo", - False, - ), - ( - "turbo.manual_invoke_enable", - "true", - "允许手动呼唤能力", - "开启后，用户在极速模式对话框中通过斜杠命令呼唤技能/插件、@ 呼唤子智能体时，" - "被呼唤的能力将临时装配到本次请求（仅显式呼唤时可用，不会把全量能力清单注入提示词）。" - "关闭后极速模式严格只用上面的工具集合。", - "turbo", - False, - ), - ( - "turbo.max_iters", - "4", - "迭代轮数上限", - "极速模式单次回答的最大 ReAct 迭代轮数（含最终作答轮）。" - "上限越小响应越快；默认 4 对应「最多 1-2 轮并行检索 + 直接作答」的产品契约。", - "turbo", - False, - ), -] + EDITION_SEED_CONFIGS - -# config_key → env var name mapping -_CONFIG_KEY_TO_ENV: dict[str, str] = { - "query_database.url": "QUERY_DATABASE_URL", - "query_database.timeout": "QUERY_DATABASE_TIMEOUT_SECONDS", - "query_database.retry_times": "QUERY_DATABASE_RETRY_TIMES", - "query_database.max_output_tokens": "QUERY_DATABASE_MAX_OUTPUT_TOKENS", - "knowledge_base.detail_max_chars": "KB_DETAIL_CONTENT_MAX_CHARS", - "file_parser.api_url": "FILE_PARSER_API_URL", - "file_parser.timeout": "FILE_PARSER_TIMEOUT", - "file_parser.lang_list": "FILE_PARSER_LANG_LIST", - "file_parser.backend": "FILE_PARSER_BACKEND", - "file_parser.parse_method": "FILE_PARSER_PARSE_METHOD", - "file_parser.formula_enable": "FILE_PARSER_FORMULA_ENABLE", - "file_parser.table_enable": "FILE_PARSER_TABLE_ENABLE", - "internet_search.engine": "INTERNET_SEARCH_ENGINE", - "internet_search.tavily_api_key": "TAVILY_API_KEY", - "internet_search.baidu_api_key": "BAIDU_API_KEY", - "internet_search.langsearch_api_key": "LANGSEARCH_API_KEY", - # Note: sandbox.code_capability_enable deliberately does **not** map to an env var —— the admin toggle is the sole - # authority (plan B), explicitly seeded to "true", and env CODE_CAPABILITY_ENABLED is retired. - **EDITION_CONFIG_KEY_TO_ENV, -} - -# Reverse mapping for env-fallback lookups -_ENV_TO_CONFIG_KEY: dict[str, str] = {v: k for k, v in _CONFIG_KEY_TO_ENV.items()} - - -def get_config_key_for_env(env_var: str) -> Optional[str]: - """Return the system-config key that backs ``env_var``, if any. - - Public lookup helper so callers (e.g. ``runtime_env``) don't need to import - the internal reverse map. - """ - return _ENV_TO_CONFIG_KEY.get(env_var) - - -class SystemConfigService: - """Thread-safe singleton that resolves service configs from DB with env fallback.""" - - _instance: Optional["SystemConfigService"] = None - _lock = threading.Lock() - - def __init__(self) -> None: - self._cache: dict[str, Optional[str]] = {} - self._cache_meta: dict[str, dict] = {} # full row metadata - self._cache_ts: float = 0.0 - self._cache_lock = threading.Lock() - - @classmethod - def get_instance(cls) -> "SystemConfigService": - if cls._instance is None: - with cls._lock: - if cls._instance is None: - cls._instance = cls() - return cls._instance - - # ── get / get_group ───────────────────────────────────────────── - - def get(self, key: str, default: str | None = None) -> str | None: - """Get a config value. DB first, then env fallback, then default.""" - self._maybe_refresh() - if key in self._cache and self._cache[key] is not None: - return self._cache[key] - # env fallback - env_key = _CONFIG_KEY_TO_ENV.get(key) - if env_key: - env_val = os.getenv(env_key) - if env_val is not None: - return env_val.strip() - return default - - def get_group(self, group_key: str) -> dict[str, str]: - """Return all config key→value pairs for a group.""" - self._maybe_refresh() - result: dict[str, str] = {} - for key, meta in self._cache_meta.items(): - if meta.get("group_key") == group_key: - val = self.get(key) - if val is not None: - result[key] = val - return result - - def get_all_configs(self) -> list[dict]: - """Return all config rows as dicts (for API responses).""" - self._maybe_refresh() - return list(self._cache_meta.values()) - - def get_group_configs(self, group_key: str) -> list[dict]: - """Return config rows for a specific group.""" - self._maybe_refresh() - return [m for m in self._cache_meta.values() if m.get("group_key") == group_key] - - # ── set ───────────────────────────────────────────────────────── - - def set(self, key: str, value: str | None, updated_by: str = "admin") -> None: - """Update a config value in DB.""" - try: - db = SessionLocal() - try: - row = db.query(SystemConfig).filter(SystemConfig.config_key == key).first() - if row is None: - logger.warning("[SystemConfigService] key %s not found, skipping set", key) - return - row.config_value = value - row.updated_by = updated_by - db.commit() - finally: - db.close() - except Exception as exc: - logger.error("[SystemConfigService] set(%s) failed: %s", key, exc) - raise - self.invalidate_cache() - - def bulk_set(self, items: list[dict], updated_by: str = "admin") -> None: - """Batch update multiple config values. Each item: {key, value}. - - For secret fields, masked values (containing '****') are skipped to - prevent the frontend from accidentally overwriting real secrets with - their masked representation. - """ - try: - db = SessionLocal() - try: - for item in items: - key = item.get("key", "").strip() - value = item.get("value") - if not key: - continue - row = db.query(SystemConfig).filter(SystemConfig.config_key == key).first() - if row is None: - continue - # Skip masked values for secret fields - if row.is_secret and isinstance(value, str) and "****" in value: - continue - row.config_value = value if value != "" else None - row.updated_by = updated_by - db.commit() - finally: - db.close() - except Exception as exc: - logger.error("[SystemConfigService] bulk_set failed: %s", exc) - raise - self.invalidate_cache() - - # ── cache management ────────────────────────────────────────── - - def invalidate_cache(self) -> None: - with self._cache_lock: - self._cache.clear() - self._cache_meta.clear() - self._cache_ts = 0.0 - - def _maybe_refresh(self) -> None: - now = time.monotonic() - if now - self._cache_ts < _CACHE_TTL_SECONDS and self._cache_meta: - return - with self._cache_lock: - if now - self._cache_ts < _CACHE_TTL_SECONDS and self._cache_meta: - return - self._load_from_db() - self._cache_ts = time.monotonic() - - def _ensure_seed_rows(self, db) -> None: - """Insert any missing seed config rows. Idempotent.""" - rows_by_key = {row.config_key: row for row in db.query(SystemConfig).all()} - existing_keys = set(rows_by_key) - inserted_keys: set[str] = set() - inserted = 0 - for key, val, display, desc, group, secret in SEED_CONFIGS: - if key not in existing_keys: - row = SystemConfig( - config_key=key, - config_value=val, - display_name=display, - description=desc, - group_key=group, - is_secret=secret, - ) - db.add(row) - rows_by_key[key] = row - inserted_keys.add(key) - inserted += 1 - migrated = self._migrate_legacy_knowledge_config(rows_by_key, inserted_keys) - if inserted or migrated: - db.commit() - logger.info( - "[SystemConfigService] Inserted %d missing seed row(s), migrated %d legacy KB value(s)", - inserted, - migrated, - ) - - @staticmethod - def _migrate_legacy_knowledge_config( - rows_by_key: dict[str, SystemConfig], - inserted_keys: set[str], - ) -> int: - """Move the old shared KB connection into one provider exactly once. - - Older releases stored every external backend in the shared - knowledge_base.url/api_key/allowed_dataset_ids rows. Those rows cannot - prove which dropdown option they belonged to, and copying them into the - currently selected provider can reproduce the original cross-provider - leak. Treat them as Dify compatibility values and leave FastGPT/WeKnora - empty until each provider is configured explicitly. - - inserted_keys is the one-time guard: after scoped rows exist, later - legacy edits can never overwrite provider-specific values. - """ - migrated = 0 - for field in _PROVIDER_SCOPED_KNOWLEDGE_FIELDS: - destination_key = f"knowledge_base.dify.{field}" - if destination_key not in inserted_keys: - continue - destination = rows_by_key.get(destination_key) - legacy = rows_by_key.get(f"knowledge_base.{field}") - legacy_value = getattr(legacy, "config_value", None) - if ( - destination is not None - and not getattr(destination, "config_value", None) - and legacy_value not in (None, "") - ): - destination.config_value = legacy_value - migrated += 1 - return migrated - - def _load_from_db(self) -> None: - new_cache: dict[str, Optional[str]] = {} - new_meta: dict[str, dict] = {} - try: - db = SessionLocal() - try: - self._ensure_seed_rows(db) - rows = db.query(SystemConfig).all() - for row in rows: - new_cache[row.config_key] = row.config_value - new_meta[row.config_key] = { - "config_key": row.config_key, - "config_value": row.config_value, - "display_name": row.display_name, - "description": row.description, - "group_key": row.group_key, - "is_secret": row.is_secret, - "updated_at": row.updated_at.isoformat() if row.updated_at else None, - "updated_by": row.updated_by, - } - finally: - db.close() - except Exception as exc: - logger.warning("[SystemConfigService] DB load failed, keeping stale cache: %s", exc) - return - - self._cache = new_cache - self._cache_meta = new_meta - - # ── env overlay for MCP sub-processes ───────────────────────── - - def get_service_env_overlay(self) -> dict[str, str]: - """Return env-var style dict for injecting into MCP sub-processes. - - Maps DB configs to the env var names that MCP servers already read via os.getenv(). - Only includes keys that have a non-None value (from DB or env fallback). - """ - overlay: dict[str, str] = {} - for config_key, env_key in _CONFIG_KEY_TO_ENV.items(): - val = self.get(config_key) - if val is not None: - overlay[env_key] = val - return overlay - - -def code_capability_enabled() -> bool: - """Master switch for the main agent's code execution capability (single source of truth at runtime). - - Sole control source = the DB value of Config console "System config" (plan B, env fallback retired); - seeded explicitly to "true". Saving in the console calls invalidate_cache; takes effect ≤30s without restart. - agent_factory and the admin_prompts preview call the same source, ensuring the console display matches runtime. - """ - try: - val = SystemConfigService.get_instance().get("sandbox.code_capability_enable", "false") - except Exception: # noqa: BLE001 — conservatively disable on config-layer errors - return False - return str(val).strip().lower() in ("1", "true", "yes", "on") - - -#: 极速模式检索工具集的代码内兜底（配置层异常/为空时使用）。 -DEFAULT_TURBO_MCP_SERVER_IDS = frozenset( - {"retrieve_dataset_content", "internet_search", "web_fetch"} -) - - -def turbo_mcp_server_ids() -> frozenset[str]: - """极速模式装配的 MCP server 集合（运行时唯一真源）。 - - 控制源 = Config 管理台「系统配置 → 极速模式」的 ``turbo.mcp_server_ids`` - （逗号分隔）。刻意不与能力目录（catalog）的启停求交集——极速模式的工具面 - 是独立的产品契约。配置异常或解析为空时回退到内置检索三件套。 - """ - default = ",".join(sorted(DEFAULT_TURBO_MCP_SERVER_IDS)) - try: - raw = SystemConfigService.get_instance().get("turbo.mcp_server_ids", default) or default - except Exception: # noqa: BLE001 — conservatively fall back to the built-in trio - return DEFAULT_TURBO_MCP_SERVER_IDS - ids = frozenset(part.strip() for part in str(raw).split(",") if part.strip()) - return ids or DEFAULT_TURBO_MCP_SERVER_IDS - - -def turbo_skill_ids() -> tuple[str, ...]: - """极速模式装配的技能集合（``AdminSkill.skill_id`` / 内置技能 id 列表）。 - - 控制源 = Config 管理台「系统配置 → 极速模式」的 ``turbo.skill_ids``（逗号 - 分隔）。与用户本轮显式呼唤的技能合并去重。默认空 —— 不选就没有技能， - 极速模式的系统提示词里也就不出现技能清单。 - """ - try: - raw = SystemConfigService.get_instance().get("turbo.skill_ids", "") or "" - except Exception: # noqa: BLE001 — 配置层异常时按「没配技能」处理 - return () - return tuple(dict.fromkeys(part.strip() for part in str(raw).split(",") if part.strip())) - - -def turbo_plugin_ids() -> tuple[str, ...]: - """极速模式装配的插件集合（``InstalledPlugin.install_id`` 列表）。 - - 控制源 = Config 管理台「系统配置 → 极速模式」的 ``turbo.plugin_ids``（逗号 - 分隔）。插件是「技能 + 工具」的能力包，装配时由 agent_factory 展开成其组件 - 技能 / MCP，与 ``turbo.mcp_server_ids`` 合并去重。默认空 —— 不选就没有插件。 - """ - try: - raw = SystemConfigService.get_instance().get("turbo.plugin_ids", "") or "" - except Exception: # noqa: BLE001 — 配置层异常时按「没配插件」处理 - return () - return tuple(dict.fromkeys(part.strip() for part in str(raw).split(",") if part.strip())) - - -def turbo_manual_invoke_enabled() -> bool: - """极速模式下是否允许显式呼唤（斜杠技能 / @子智能体 / 插件）临时装配。""" - try: - val = SystemConfigService.get_instance().get("turbo.manual_invoke_enable", "true") - except Exception: # noqa: BLE001 - return True - return str(val).strip().lower() in ("1", "true", "yes", "on") - - -def turbo_max_iters() -> int: - """极速模式的 ReAct 迭代硬上限（最小 2，防止配错把回答轮也掐掉）。""" - try: - val = SystemConfigService.get_instance().get("turbo.max_iters", "4") - return max(2, int(str(val).strip())) - except Exception: # noqa: BLE001 - return 4 - - -def auto_plan_entry_enabled() -> bool: - """Master switch for the main agent's in-conversation plan tracker (the update_plan tool). - - Control source = the Config console "System config" DB value ``chat.auto_plan_entry_enable``, - defaulting to "true" (enabled by default). Ops can set it to false to disable the capability, ≤30s, no restart. - Conservatively disable on config-layer errors (don't register the tool, fall back to manually clicking "Plan mode"). - """ - try: - val = SystemConfigService.get_instance().get("chat.auto_plan_entry_enable", "true") - except Exception: # noqa: BLE001 — conservatively disable on config-layer errors - return False - return str(val).strip().lower() in ("1", "true", "yes", "on") diff --git a/src/backend/core/services/tool_effect_ledger.py b/src/backend/core/services/tool_effect_ledger.py deleted file mode 100644 index 03bb6d0d..00000000 --- a/src/backend/core/services/tool_effect_ledger.py +++ /dev/null @@ -1,1808 +0,0 @@ -"""Durable intent-before-invocation gateway for tool effects. - -``ToolEffectLedger`` is append-only and authoritative. The companion lease -row is only a mutable execution/recovery claim. All public execution paths go -through :class:`ToolEffectGateway`; adapters are never called until an Intent -has committed and an invocation guard has rechecked ownership. -""" - -from __future__ import annotations - -import asyncio -import hashlib -import inspect -import json -import logging -import re -import time -import uuid -from contextlib import suppress -from contextvars import ContextVar -from dataclasses import dataclass -from datetime import datetime, timedelta, timezone -from typing import Any, Awaitable, Callable, Mapping, Optional, cast - -from sqlalchemy import and_, case, or_ -from sqlalchemy.exc import IntegrityError, OperationalError -from sqlalchemy.orm import Session, aliased, sessionmaker - -from core.db.engine import SessionLocal -from core.db.models import ChatRun, ToolEffectLease, ToolEffectLedger -from core.infra.data_masking import mask_sensitive_data -from core.services.run_journal import ( - LIVE_STATUSES, - RunJournal, - RunLeaseLost, - RunNotFound, -) - -logger = logging.getLogger(__name__) - -RECOVERY_POLICIES = ("replay_safe", "reconcile", "never_replay") -# needs_attention 的现场值得多留一阵子再回收，保留期是普通终态的这个倍数——从 -# 同一个 retention_days 推导，不额外引入一个可独立漂移的阈值。 -_ATTENTION_RETENTION_FACTOR = 4 -_REDACT_FIELDS = [ - "password", - "token", - "api_key", - "apikey", - "authorization", - "secret", - "access_key", - "cookie", -] -_WITHHOLD_FIELDS = {"command", "content", "stdin", "script", "code"} -_EMBEDDED_SECRET_PATTERNS = ( - re.compile(r"(?i)(authorization\s*:\s*bearer\s+)[^\s'\";]+"), - re.compile(r"(?i)(bearer\s+)[A-Za-z0-9._~+/=-]{8,}"), - re.compile( - r"(?i)([?&](?:token|api_key|apikey|access_key|secret|password|passwd|client_secret|authorization)=)[^&#\s]+" - ), - re.compile( - r"(?i)((?:token|api_key|apikey|access_key|secret|password|passwd|client_secret|authorization)\s*[=:]\s*)[^\s'\";]+" - ), -) - - -class ToolEffectError(RuntimeError): - pass - - -class ToolIntentCommitError(ToolEffectError): - """The intent could not be durably committed; invocation is forbidden.""" - - -class ToolOutcomeUnknown(ToolEffectError): - """The ledger proves intent but cannot prove the external outcome.""" - - -def find_tool_outcome_unknown(exc: BaseException) -> Optional[ToolOutcomeUnknown]: - """Unwrap AgentScope/anyio ExceptionGroup layers without losing safety meaning.""" - pending = [exc] - seen: set[int] = set() - while pending: - current = pending.pop() - if id(current) in seen: - continue - seen.add(id(current)) - if isinstance(current, ToolOutcomeUnknown): - return current - nested = getattr(current, "exceptions", None) - if isinstance(nested, tuple): - pending.extend(item for item in nested if isinstance(item, BaseException)) - if isinstance(current.__cause__, BaseException): - pending.append(current.__cause__) - if isinstance(current.__context__, BaseException): - pending.append(current.__context__) - return None - - -class ToolEffectLeaseLost(ToolEffectError): - pass - - -class ToolIdempotencyConflict(ToolEffectError): - """A caller reused one explicit key for different logical input.""" - - -@dataclass(frozen=True) -class ReconciliationResult: - outcome: str - result: Any = None - - @classmethod - def applied(cls, result: Any) -> "ReconciliationResult": - return cls("applied", result) - - @classmethod - def not_applied(cls) -> "ReconciliationResult": - return cls("not_applied") - - @classmethod - def unknown(cls) -> "ReconciliationResult": - return cls("unknown") - - -@dataclass(frozen=True) -class ToolPolicy: - policy: str - replayer: Optional[Callable[["ToolIntent"], Awaitable[Any] | Any]] = None - reconciler: Optional[ - Callable[["ToolIntent"], Awaitable[ReconciliationResult | Any] | Any] - ] = None - - -@dataclass(frozen=True) -class ToolIntent: - effect_id: str - result_id: str - run_id: str - operation_seq: int - tool_name: str - tool_call_id: str - args_hash: str - redacted_args: Mapping[str, Any] - idempotency_key: str - recovery_policy: str - created_at: Optional[datetime] = None - - -@dataclass(frozen=True) -class IntentDecision: - action: str - intent: ToolIntent - result: Any = None - result_state: str = "success" - - @property - def effect_id(self) -> str: - return self.intent.effect_id - - -@dataclass(frozen=True) -class ToolExecutionOutcome: - result: Any - effect_id: str - result_id: str - result_state: str - invoked: bool - - -@dataclass(frozen=True) -class RecoverySweepDecision: - effect_id: str - run_id: str - action: str - - -@dataclass(frozen=True) -class ToolInvocationContext: - effect_id: str - result_id: str - idempotency_key: str - run_id: str - fencing_owner: str - - -CURRENT_TOOL_EFFECT: ContextVar[Optional[ToolInvocationContext]] = ContextVar( - "jx_current_tool_effect", default=None -) - - -class ToolRecoveryRegistry: - """Explicit adapter contracts. Unknown tools fail closed as never-replay.""" - - def __init__(self) -> None: - self._policies: dict[str, ToolPolicy] = {} - - def register( - self, - tool_name: str, - policy: str, - *, - replayer: Optional[Callable[[ToolIntent], Awaitable[Any] | Any]] = None, - reconciler: Optional[ - Callable[[ToolIntent], Awaitable[ReconciliationResult | Any] | Any] - ] = None, - ) -> None: - if policy not in RECOVERY_POLICIES: - raise ValueError(f"invalid recovery policy: {policy}") - if policy == "replay_safe" and reconciler is not None: - raise ValueError("replay_safe does not accept a reconciler") - if policy == "never_replay" and ( - replayer is not None or reconciler is not None - ): - raise ValueError("never_replay does not accept recovery adapters") - self._policies[str(tool_name)] = ToolPolicy(policy, replayer, reconciler) - - def resolve(self, tool_name: str) -> ToolPolicy: - return self._policies.get(str(tool_name), ToolPolicy("never_replay")) - - -async def _default_replayer(intent: ToolIntent) -> Any: - from orchestration.tool_effect_recovery import replay_tool_intent - - return await replay_tool_intent(intent) - - -async def _scheduled_task_reconciler(intent: ToolIntent) -> ReconciliationResult: - from orchestration.tool_effect_recovery import reconcile_scheduled_task_intent - - return cast(ReconciliationResult, await reconcile_scheduled_task_intent(intent)) - - -def build_default_tool_recovery_registry() -> ToolRecoveryRegistry: - registry = ToolRecoveryRegistry() - for name in ( - "Read", - "read_project_instructions", - "list_sites", - "Glob", - "Grep", - "view_text_file", - "get_data_context", - "read_image", - "read_artifact", - "channel_read_attachment", - "sandbox_get_artifact", - ): - registry.register(name, "replay_safe", replayer=_default_replayer) - for name in ( - "Write", - "save_project_instructions", - "Edit", - "Bash", - "sandbox_put_artifact", - "pin_to_workspace", - "run_job", - "choose_design", - "call_subagent", - "update_plan", - "load_plugin", - ): - registry.register(name, "never_replay") - for name in ( - "create_scheduled_task", - "update_scheduled_task", - "delete_scheduled_task", - ): - registry.register( - name, - "reconcile", - replayer=_default_replayer, - reconciler=_scheduled_task_reconciler, - ) - return registry - - -DEFAULT_TOOL_RECOVERY_REGISTRY = build_default_tool_recovery_registry() - - -def _utcnow() -> datetime: - return datetime.now(timezone.utc) - - -def _aware(value: Optional[datetime]) -> Optional[datetime]: - if value is None: - return None - if value.tzinfo is None: - return value.replace(tzinfo=timezone.utc) - return value.astimezone(timezone.utc) - - -def _embedded_redact(value: str) -> str: - cleaned = value - for pattern in _EMBEDDED_SECRET_PATTERNS: - cleaned = pattern.sub(r"\1[REDACTED]", cleaned) - return cleaned - - -def _sanitize_value(value: Any, *, key: str = "") -> Any: - lower = key.lower() - if isinstance(value, Mapping): - return {str(k): _sanitize_value(v, key=str(k)) for k, v in value.items()} - if isinstance(value, (list, tuple)): - return [_sanitize_value(item, key=key) for item in value] - if isinstance(value, str): - if lower in _WITHHOLD_FIELDS: - digest = hashlib.sha256(value.encode("utf-8")).hexdigest() - return {"_withheld": True, "sha256": digest, "length": len(value)} - return _embedded_redact(value) - return value - - -def canonical_tool_args(args: Mapping[str, Any]) -> tuple[str, str, Mapping[str, Any]]: - canonical = json.dumps( - args or {}, sort_keys=True, separators=(",", ":"), default=str - ) - args_hash = hashlib.sha256(canonical.encode("utf-8")).hexdigest() - by_field = mask_sensitive_data(dict(args or {}), field_patterns=_REDACT_FIELDS) - return canonical, args_hash, _sanitize_value(by_field) - - -def tool_args_hash(args: Mapping[str, Any]) -> str: - return canonical_tool_args(args)[1] - - -def stable_idempotency_key(run_id: str, tool_name: str, invocation_id: str) -> str: - raw = f"{run_id}\x00{tool_name}\x00{invocation_id}".encode("utf-8") - return "tool:" + hashlib.sha256(raw).hexdigest() - - -def _resolved_idempotency_key( - *, - run_id: str, - principal_scope: str, - tool_name: str, - tool_call_id: str, - explicit_key: Optional[str], -) -> tuple[str, bool]: - caller_key = str(explicit_key or "").strip() - if caller_key: - scoped = f"{principal_scope}\x00{caller_key}".encode("utf-8") - return "caller:" + hashlib.sha256(scoped).hexdigest(), True - if not tool_call_id: - raise ToolIntentCommitError( - "tool_call_id or explicit idempotency_key is required" - ) - return stable_idempotency_key(run_id, tool_name, tool_call_id), False - - -class ToolEffectJournal: - """Small synchronous transaction boundary behind the async gateway.""" - - def __init__( - self, - session_factory: sessionmaker = SessionLocal, - *, - clock: Callable[[], datetime] = _utcnow, - ) -> None: - self._sessions = session_factory - self._clock = clock - self._runs = RunJournal(session_factory, clock=clock) - - @staticmethod - def _intent_from_row(row: ToolEffectLedger) -> ToolIntent: - return ToolIntent( - effect_id=row.effect_id, - result_id=row.result_id, - run_id=row.run_id, - operation_seq=int(row.operation_seq), - tool_name=row.tool_name, - tool_call_id=row.tool_call_id or "", - args_hash=row.args_hash, - redacted_args=dict(row.redacted_args or {}), - idempotency_key=row.idempotency_key or "", - recovery_policy=row.recovery_policy, - created_at=_aware(row.created_at), - ) - - @staticmethod - def _terminal_row(db: Session, effect_id: str) -> Optional[ToolEffectLedger]: - return ( - db.query(ToolEffectLedger) - .filter(ToolEffectLedger.terminal_effect_id == effect_id) - .one_or_none() - ) - - @staticmethod - def _decision_from_terminal( - intent: ToolIntent, terminal: ToolEffectLedger - ) -> IntentDecision: - if terminal.event_type == "unknown_outcome": - return IntentDecision("unknown", intent, result_state="unknown") - return IntentDecision( - "known", - intent, - terminal.result_payload, - "success" if terminal.event_type == "result" else "error", - ) - - @staticmethod - def _validate_identity( - row: ToolEffectLedger, *, tool_name: str, args_hash: str - ) -> None: - if row.tool_name != tool_name or row.args_hash != args_hash: - raise ToolIdempotencyConflict( - "idempotency_key already belongs to different tool arguments" - ) - - def _append_event( - self, - db: Session, - run: ChatRun, - intent: ToolEffectLedger, - *, - owner: str, - event_type: str, - operation_type: str, - phase: str, - safety: str, - result: Any = None, - error: Optional[str] = None, - terminal: bool = False, - ) -> None: - receipt = self._runs._append_locked( - db, - run, - owner=owner, - operation_type=operation_type, - phase=phase, - safety=safety, - payload={"effect_id": intent.effect_id, "result_id": intent.result_id}, - ) - db.add( - ToolEffectLedger( - effect_id=intent.effect_id, - result_id=intent.result_id, - run_id=intent.run_id, - operation_seq=receipt.operation_seq, - event_type=event_type, - tool_name=intent.tool_name, - tool_call_id=intent.tool_call_id, - args_hash=intent.args_hash, - redacted_args=intent.redacted_args, - recovery_policy=intent.recovery_policy, - result_payload=result, - error_message=error, - terminal_effect_id=intent.effect_id if terminal else None, - ) - ) - - def _mark_unknown_locked( - self, - db: Session, - run: ChatRun, - intent: ToolEffectLedger, - lease: ToolEffectLease, - *, - owner: str, - claim_owner: str, - reason: str, - ) -> None: - if self._terminal_row(db, intent.effect_id) is not None: - return - if lease.run_owner != owner or lease.claim_owner != claim_owner: - raise ToolEffectLeaseLost(intent.effect_id) - self._append_event( - db, - run, - intent, - owner=owner, - event_type="unknown_outcome", - operation_type="tool_outcome_unknown", - phase="needs_attention", - safety="manual_review", - error=reason, - terminal=True, - ) - run.status = "needs_attention" - run.failure_reason = reason - run.lease_owner = None - run.lease_expires_at = None - run.updated_at = self._clock() - lease.claim_owner = None - lease.lease_expires_at = None - lease.updated_at = self._clock() - - def _existing_decision( - self, - *, - key: str, - caller_run_id: str, - run_owner: str, - claim_owner: str, - tool_name: str, - args_hash: str, - lease_seconds: int, - ) -> IntentDecision: - with self._sessions() as db: - intent_row = ( - db.query(ToolEffectLedger) - .filter(ToolEffectLedger.idempotency_key == key) - .one() - ) - self._validate_identity( - intent_row, tool_name=tool_name, args_hash=args_hash - ) - intent = self._intent_from_row(intent_row) - caller_run = db.get(ChatRun, caller_run_id) - original_run = db.get(ChatRun, intent.run_id) - if caller_run is None or original_run is None: - raise ToolIntentCommitError(f"run does not exist: {caller_run_id}") - if original_run.user_id != caller_run.user_id: - raise ToolIdempotencyConflict( - "idempotency_key belongs to a different principal" - ) - terminal = self._terminal_row(db, intent.effect_id) - if terminal is not None and caller_run_id == intent.run_id: - return self._decision_from_terminal(intent, terminal) - now = _aware(self._clock()) - caller_expiry = _aware(caller_run.lease_expires_at) - if ( - caller_run.status not in LIVE_STATUSES - or caller_run.lease_owner != run_owner - or caller_expiry is None - or now is None - or caller_expiry <= now - ): - raise ToolIntentCommitError(f"run lease is not owned: {caller_run_id}") - if terminal is not None: - return self._decision_from_terminal(intent, terminal) - if intent.run_id != caller_run_id: - return IntentDecision("wait", intent) - - lease = ( - db.query(ToolEffectLease) - .filter(ToolEffectLease.effect_id == intent.effect_id) - .with_for_update() - .one() - ) - run = self._runs._locked_owned_run(db, caller_run_id, run_owner) - terminal = self._terminal_row(db, intent.effect_id) - if terminal is not None: - return self._decision_from_terminal(intent, terminal) - lease_expiry = _aware(lease.lease_expires_at) - if ( - lease.run_owner == run_owner - and lease_expiry is not None - and lease_expiry > now - ): - if lease.claim_owner == claim_owner: - return IntentDecision( - "reconcile" - if intent.recovery_policy == "reconcile" - else "execute", - intent, - ) - return IntentDecision("wait", intent) - - previous_owner = lease.run_owner - previous_claim = lease.claim_owner - previous_expiry = lease.lease_expires_at - affected = ( - db.query(ToolEffectLease) - .filter( - ToolEffectLease.effect_id == intent.effect_id, - ToolEffectLease.run_owner == previous_owner, - ToolEffectLease.claim_owner == previous_claim, - ToolEffectLease.lease_expires_at == previous_expiry, - or_( - ToolEffectLease.lease_expires_at.is_(None), - ToolEffectLease.lease_expires_at <= self._clock(), - ), - ) - .update( - { - ToolEffectLease.run_owner: run_owner, - ToolEffectLease.claim_owner: claim_owner, - ToolEffectLease.lease_expires_at: self._clock() - + timedelta(seconds=max(1, lease_seconds)), - ToolEffectLease.updated_at: self._clock(), - }, - synchronize_session=False, - ) - ) - if not affected: - db.rollback() - return IntentDecision("wait", intent) - db.expire_all() - lease = db.get(ToolEffectLease, intent.effect_id) - self._append_event( - db, - run, - intent_row, - owner=run_owner, - event_type="recovery_claim", - operation_type="tool_effect_recovery_claimed", - phase=f"tool_intent_{intent.recovery_policy}", - safety=intent.recovery_policy, - ) - if intent.recovery_policy == "never_replay": - self._mark_unknown_locked( - db, - run, - intent_row, - lease, - owner=run_owner, - claim_owner=claim_owner, - reason="non-replayable tool intent has no committed result", - ) - db.commit() - return IntentDecision("unknown", intent) - db.commit() - return IntentDecision( - "reconcile" if intent.recovery_policy == "reconcile" else "execute", - intent, - ) - - def begin_intent( - self, - *, - run_id: str, - owner: str, - claim_owner: str, - tool_call_id: str, - tool_name: str, - args: Mapping[str, Any], - recovery_policy: str, - idempotency_key: Optional[str] = None, - lease_seconds: int = 300, - _conflict_retries: int = 4, - ) -> IntentDecision: - if recovery_policy not in RECOVERY_POLICIES: - raise ValueError(f"invalid recovery policy: {recovery_policy}") - _canonical, args_hash, redacted = canonical_tool_args(args) - with self._sessions() as db: - caller_run = db.get(ChatRun, run_id) - if caller_run is None: - raise ToolIntentCommitError(f"run does not exist: {run_id}") - principal_scope = caller_run.user_id - key, _explicit = _resolved_idempotency_key( - run_id=run_id, - principal_scope=str(principal_scope), - tool_name=tool_name, - tool_call_id=tool_call_id, - explicit_key=idempotency_key - or str(args.get("idempotency_key") or args.get("request_id") or "").strip(), - ) - try: - with self._sessions() as db: - found = ( - db.query(ToolEffectLedger.event_id) - .filter(ToolEffectLedger.idempotency_key == key) - .first() - ) - if found is not None: - return self._existing_decision( - key=key, - caller_run_id=run_id, - run_owner=owner, - claim_owner=claim_owner, - tool_name=tool_name, - args_hash=args_hash, - lease_seconds=lease_seconds, - ) - - with self._sessions() as db: - run = self._runs._locked_owned_run(db, run_id, owner) - found = ( - db.query(ToolEffectLedger.event_id) - .filter(ToolEffectLedger.idempotency_key == key) - .first() - ) - if found is not None: - db.rollback() - return self._existing_decision( - key=key, - caller_run_id=run_id, - run_owner=owner, - claim_owner=claim_owner, - tool_name=tool_name, - args_hash=args_hash, - lease_seconds=lease_seconds, - ) - effect_id = f"eff_{uuid.uuid4().hex}" - result_id = f"toolres_{uuid.uuid4().hex}" - receipt = self._runs._append_locked( - db, - run, - owner=owner, - operation_type="tool_intent_committed", - phase=f"tool_intent_{recovery_policy}", - safety=recovery_policy, - payload={ - "effect_id": effect_id, - "result_id": result_id, - "tool_name": tool_name, - "tool_call_id": tool_call_id, - "args_hash": args_hash, - }, - ) - row = ToolEffectLedger( - effect_id=effect_id, - result_id=result_id, - run_id=run_id, - operation_seq=receipt.operation_seq, - event_type="intent", - tool_name=tool_name, - tool_call_id=tool_call_id, - args_hash=args_hash, - redacted_args=dict(redacted), - idempotency_key=key, - recovery_policy=recovery_policy, - ) - db.add(row) - db.add( - ToolEffectLease( - effect_id=effect_id, - run_id=run_id, - run_owner=owner, - claim_owner=claim_owner, - lease_expires_at=self._clock() - + timedelta(seconds=max(1, lease_seconds)), - updated_at=self._clock(), - ) - ) - run.updated_at = self._clock() - db.commit() - return IntentDecision("execute", self._intent_from_row(row)) - except ToolEffectError: - raise - except (IntegrityError, OperationalError) as exc: - with self._sessions() as db: - found = ( - db.query(ToolEffectLedger.event_id) - .filter(ToolEffectLedger.idempotency_key == key) - .first() - ) - if found is not None: - return self._existing_decision( - key=key, - caller_run_id=run_id, - run_owner=owner, - claim_owner=claim_owner, - tool_name=tool_name, - args_hash=args_hash, - lease_seconds=lease_seconds, - ) - if _conflict_retries > 0: - return self.begin_intent( - run_id=run_id, - owner=owner, - claim_owner=claim_owner, - tool_call_id=tool_call_id, - tool_name=tool_name, - args=args, - recovery_policy=recovery_policy, - idempotency_key=idempotency_key, - lease_seconds=lease_seconds, - _conflict_retries=_conflict_retries - 1, - ) - raise ToolIntentCommitError(str(exc)) from exc - except (RunNotFound, RunLeaseLost) as exc: - raise ToolIntentCommitError(str(exc)) from exc - except Exception as exc: # noqa: BLE001 - raise ToolIntentCommitError(str(exc)) from exc - - def acquire_invocation_claim( - self, - effect_id: str, - *, - run_owner: str, - claim_owner: str, - lease_seconds: int = 300, - ) -> None: - """Validate and extend one invocation claim in a short atomic transaction.""" - now = self._clock() - expires = now + timedelta(seconds=max(1, lease_seconds)) - with self._sessions() as db: - live_run = db.query(ChatRun.run_id).filter( - ChatRun.status.in_(LIVE_STATUSES), - ChatRun.lease_owner == run_owner, - ChatRun.lease_expires_at > now, - ) - affected = ( - db.query(ToolEffectLease) - .filter( - ToolEffectLease.effect_id == effect_id, - ToolEffectLease.run_owner == run_owner, - ToolEffectLease.claim_owner == claim_owner, - ToolEffectLease.lease_expires_at > now, - ToolEffectLease.run_id.in_(live_run), - ) - .update( - { - ToolEffectLease.lease_expires_at: expires, - ToolEffectLease.updated_at: now, - }, - synchronize_session=False, - ) - ) - if not affected or self._terminal_row(db, effect_id) is not None: - db.rollback() - raise ToolEffectLeaseLost(effect_id) - db.commit() - - def renew_invocation_claim( - self, - effect_id: str, - *, - run_owner: str, - claim_owner: str, - lease_seconds: int, - ) -> bool: - if not self._runs.renew( - self._intent_run_id(effect_id), - owner=run_owner, - lease_seconds=lease_seconds, - ): - return False - try: - self.acquire_invocation_claim( - effect_id, - run_owner=run_owner, - claim_owner=claim_owner, - lease_seconds=lease_seconds, - ) - return True - except ToolEffectLeaseLost: - return False - - def renew_recovery_claim( - self, - effect_id: str, - *, - recovery_owner: str, - lease_seconds: int, - ) -> bool: - return self.renew_invocation_claim( - effect_id, - run_owner=recovery_owner, - claim_owner=recovery_owner, - lease_seconds=lease_seconds, - ) - - def _intent_run_id(self, effect_id: str) -> str: - with self._sessions() as db: - row = ( - db.query(ToolEffectLedger.run_id) - .filter( - ToolEffectLedger.effect_id == effect_id, - ToolEffectLedger.event_type == "intent", - ) - .one() - ) - return str(row[0]) - - def abandon_invocation_claim( - self, - effect_id: str, - *, - run_owner: str, - claim_owner: str, - ) -> None: - """Expire a cancelled/ambiguous call so policy recovery can take over.""" - now = self._clock() - with self._sessions() as db: - db.query(ToolEffectLease).filter( - ToolEffectLease.effect_id == effect_id, - ToolEffectLease.run_owner == run_owner, - ToolEffectLease.claim_owner == claim_owner, - ).update( - { - ToolEffectLease.lease_expires_at: now, - ToolEffectLease.updated_at: now, - }, - synchronize_session=False, - ) - db.commit() - - def _commit_result_locked( - self, - db: Session, - intent: ToolEffectLedger, - lease: ToolEffectLease, - *, - owner: str, - result: Any, - failed: bool, - reconciled: bool, - ) -> Any: - terminal = self._terminal_row(db, intent.effect_id) - if terminal is not None: - if terminal.event_type == "unknown_outcome": - raise ToolOutcomeUnknown(intent.effect_id) - return terminal.result_payload - if lease.run_owner != owner: - raise ToolEffectLeaseLost(intent.effect_id) - run = self._runs._locked_owned_run(db, intent.run_id, owner) - self._append_event( - db, - run, - intent, - owner=owner, - event_type="failure" if failed else "result", - operation_type="tool_failure_committed" - if failed - else "tool_result_committed", - phase="tool_result_committed", - safety="reconciled" if reconciled else intent.recovery_policy, - result=result, - error=str(result)[:4000] if failed else None, - terminal=True, - ) - lease.claim_owner = None - lease.lease_expires_at = None - lease.updated_at = self._clock() - run.updated_at = self._clock() - return result - - def commit_result( - self, - effect_id: str, - *, - run_owner: str, - claim_owner: str, - result: Any, - failed: bool = False, - reconciled: bool = False, - _conflict_retries: int = 4, - ) -> Any: - with self._sessions() as db: - intent = ( - db.query(ToolEffectLedger) - .filter( - ToolEffectLedger.effect_id == effect_id, - ToolEffectLedger.event_type == "intent", - ) - .one() - ) - terminal = self._terminal_row(db, effect_id) - if terminal is not None: - if terminal.event_type == "unknown_outcome": - raise ToolOutcomeUnknown(effect_id) - return terminal.result_payload - lease = ( - db.query(ToolEffectLease) - .filter(ToolEffectLease.effect_id == effect_id) - .with_for_update() - .one() - ) - now = _aware(self._clock()) - lease_expiry = _aware(lease.lease_expires_at) - if ( - lease.run_owner != run_owner - or lease.claim_owner != claim_owner - or lease_expiry is None - or now is None - or lease_expiry <= now - ): - # A concurrent retry can observe the lease only after the - # winning commit cleared its claim. Re-check the durable - # terminal decision before reporting lease loss: identical - # result commits are idempotent even when the first pre-lock - # terminal read raced with the winner. - terminal = self._terminal_row(db, effect_id) - if terminal is not None: - if terminal.event_type == "unknown_outcome": - raise ToolOutcomeUnknown(effect_id) - return terminal.result_payload - raise ToolEffectLeaseLost(effect_id) - value = self._commit_result_locked( - db, - intent, - lease, - owner=run_owner, - result=result, - failed=failed, - reconciled=reconciled, - ) - try: - db.commit() - except (IntegrityError, OperationalError) as exc: - db.rollback() - terminal = self.terminal_decision(effect_id) - if terminal is not None: - if terminal.action == "unknown": - raise ToolOutcomeUnknown(effect_id) - return terminal.result - if _conflict_retries > 0: - return self.commit_result( - effect_id, - run_owner=run_owner, - claim_owner=claim_owner, - result=result, - failed=failed, - reconciled=reconciled, - _conflict_retries=_conflict_retries - 1, - ) - raise ToolEffectError(str(exc)) from exc - return value - - def mark_unknown( - self, - effect_id: str, - *, - owner: str, - claim_owner: str, - reason: str, - ) -> None: - with self._sessions() as db: - lease = ( - db.query(ToolEffectLease) - .filter(ToolEffectLease.effect_id == effect_id) - .with_for_update() - .one() - ) - intent = ( - db.query(ToolEffectLedger) - .filter( - ToolEffectLedger.effect_id == effect_id, - ToolEffectLedger.event_type == "intent", - ) - .one() - ) - run = self._runs._locked_owned_run(db, intent.run_id, owner) - self._mark_unknown_locked( - db, - run, - intent, - lease, - owner=owner, - claim_owner=claim_owner, - reason=reason, - ) - db.commit() - - def terminal_decision(self, effect_id: str) -> Optional[IntentDecision]: - with self._sessions() as db: - intent = ( - db.query(ToolEffectLedger) - .filter( - ToolEffectLedger.effect_id == effect_id, - ToolEffectLedger.event_type == "intent", - ) - .one_or_none() - ) - if intent is None: - return None - terminal = self._terminal_row(db, effect_id) - return ( - self._decision_from_terminal(self._intent_from_row(intent), terminal) - if terminal is not None - else None - ) - - def pending_effect_ids(self) -> list[str]: - with self._sessions() as db: - terminal = aliased(ToolEffectLedger) - rows = ( - db.query(ToolEffectLedger.effect_id) - .filter( - ToolEffectLedger.event_type == "intent", - ~db.query(terminal.event_id) - .filter(terminal.terminal_effect_id == ToolEffectLedger.effect_id) - .exists(), - ) - .order_by(ToolEffectLedger.created_at, ToolEffectLedger.event_id) - .all() - ) - return [row.effect_id for row in rows] - - def prune_settled(self, *, retention_days: int = 7) -> int: - """Drop per-run recovery/observability facts of long-terminal runs. - - Covers the tool effect ledger and leases plus the other append-only - per-run journals (harness usage attempts, harness events, run - operations); without this they grow without bound. - - The run's own recovery columns go the same way. ``recovery_snapshot`` - carries the whole conversation as of that turn, so a long chat stores - roughly its own history once per turn — the single largest thing in a - desktop database. Only crash recovery reads it, and only for runs that - are still live and still replayable, which is exactly what the cutoff - and the settled filter below exclude. - - ``needs_attention`` is kept ``_ATTENTION_RETENTION_FACTOR`` times longer - rather than exempted, and that applies to every tier above. Exempting it - means never: nothing in the backend transitions a run out of that status, - so an exemption pins its journals — and its snapshot — for good. In - production every record that had outlived the retention window belonged - to such a run. - """ - from core.db.models import ChatRunOperation, HarnessEventLog, HarnessUsageAttempt - - now = _aware(self._clock()) - cutoff = now - timedelta(days=retention_days) - attention_cutoff = now - timedelta(days=retention_days * _ATTENTION_RETENTION_FACTOR) - with self._sessions() as db: - - earliest_cutoff = max(cutoff, attention_cutoff) - - def _prunable(run_col, stamp): - """Row whose run is terminal and past that run's retention tier. - - ``stamp < earliest_cutoff`` is implied by the EXISTS (it is the - loosest of the tiers) and stated anyway: it is the only - predicate on these tables an index can serve, and without it the - planner has nothing to range-scan on. - """ - tier = case( - (ChatRun.status == "needs_attention", attention_cutoff), - else_=cutoff, - ) - return and_( - stamp < earliest_cutoff, - db.query(ChatRun.run_id) - .filter( - ChatRun.run_id == run_col, - ChatRun.status.notin_(LIVE_STATUSES), - stamp < tier, - ) - .exists(), - ) - - deleted = ( - db.query(ToolEffectLedger) - .filter(_prunable(ToolEffectLedger.run_id, ToolEffectLedger.created_at)) - .delete(synchronize_session=False) - ) - db.query(ToolEffectLease).filter( - _prunable(ToolEffectLease.run_id, ToolEffectLease.updated_at) - ).delete(synchronize_session=False) - for model in (HarnessUsageAttempt, HarnessEventLog, ChatRunOperation): - deleted += ( - db.query(model) - .filter(_prunable(model.run_id, model.created_at)) - .delete(synchronize_session=False) - ) - deleted += ( - db.query(ChatRun) - .filter( - ChatRun.status.notin_(LIVE_STATUSES), - ChatRun.completed_at < earliest_cutoff, - ChatRun.completed_at - < case( - (ChatRun.status == "needs_attention", attention_cutoff), - else_=cutoff, - ), - ChatRun.recovery_snapshot.isnot(None), - ) - .update( - {ChatRun.recovery_snapshot: None, ChatRun.request_payload: None}, - synchronize_session=False, - ) - ) - db.commit() - return deleted - - def settle_terminal_run_intent(self, effect_id: str) -> Optional[ToolIntent]: - """Close an orphan intent without replay after its owning run is terminal. - - Cancellation/failure is a hard replay boundary: the user's stopped run - must never perform a delayed external action. We still append an - ``unknown_outcome`` fact so the intent cannot remain pending forever. - """ - with self._sessions() as db: - intent = ( - db.query(ToolEffectLedger) - .filter( - ToolEffectLedger.effect_id == effect_id, - ToolEffectLedger.event_type == "intent", - ) - .with_for_update() - .one_or_none() - ) - if intent is None or self._terminal_row(db, effect_id) is not None: - return None - run = ( - db.query(ChatRun) - .filter(ChatRun.run_id == intent.run_id) - .with_for_update() - .one_or_none() - ) - if run is None or run.status in (*LIVE_STATUSES, "needs_attention"): - return None - lease = ( - db.query(ToolEffectLease) - .filter(ToolEffectLease.effect_id == effect_id) - .with_for_update() - .one() - ) - terminal_status = str(run.status) - self._append_event( - db, - run, - intent, - owner="system:terminal-settlement", - event_type="unknown_outcome", - operation_type="tool_outcome_unknown_after_terminal_run", - phase=str(run.run_phase or terminal_status), - safety="manual_review", - error=f"run became {terminal_status} before the tool outcome committed", - terminal=True, - ) - # _append_event advances only the operation ledger. Preserve the - # authoritative terminal verdict and merely close the effect claim. - run.status = terminal_status - lease.claim_owner = None - lease.lease_expires_at = None - lease.updated_at = self._clock() - run.updated_at = self._clock() - db.commit() - return self._intent_from_row(intent) - - def claim_recovery_intent( - self, - effect_id: str, - *, - recovery_owner: str, - lease_seconds: int = 120, - ) -> Optional[ToolIntent]: - with self._sessions() as db: - intent = ( - db.query(ToolEffectLedger) - .filter( - ToolEffectLedger.effect_id == effect_id, - ToolEffectLedger.event_type == "intent", - ) - .one_or_none() - ) - if intent is None or self._terminal_row(db, effect_id) is not None: - return None - now = self._clock() - from core.services.api_key_runs import authorize_run_execution - - if not authorize_run_execution(db, intent.run_id, now): - return None - expires = now + timedelta(seconds=max(1, lease_seconds)) - run_affected = ( - db.query(ChatRun) - .filter( - ChatRun.run_id == intent.run_id, - ChatRun.status.in_((*LIVE_STATUSES, "needs_attention")), - or_( - ChatRun.lease_owner.is_(None), - ChatRun.lease_expires_at.is_(None), - ChatRun.lease_expires_at <= now, - ChatRun.lease_owner == recovery_owner, - ), - ) - .update( - { - ChatRun.status: "running", - ChatRun.lease_owner: recovery_owner, - ChatRun.lease_expires_at: expires, - ChatRun.updated_at: now, - }, - synchronize_session=False, - ) - ) - if not run_affected: - db.rollback() - return None - lease_affected = ( - db.query(ToolEffectLease) - .filter( - ToolEffectLease.effect_id == effect_id, - or_( - ToolEffectLease.lease_expires_at.is_(None), - ToolEffectLease.lease_expires_at <= now, - ToolEffectLease.claim_owner == recovery_owner, - ), - ) - .update( - { - ToolEffectLease.run_owner: recovery_owner, - ToolEffectLease.claim_owner: recovery_owner, - ToolEffectLease.lease_expires_at: expires, - ToolEffectLease.updated_at: now, - }, - synchronize_session=False, - ) - ) - if not lease_affected or self._terminal_row(db, effect_id) is not None: - db.rollback() - return None - db.expire_all() - run = db.get(ChatRun, intent.run_id) - self._append_event( - db, - run, - intent, - owner=recovery_owner, - event_type="recovery_claim", - operation_type="tool_effect_recovery_claimed", - phase=f"tool_intent_{intent.recovery_policy}", - safety=intent.recovery_policy, - ) - db.commit() - return self._intent_from_row(intent) - - def commit_recovered_result( - self, - effect_id: str, - result: Any, - *, - recovery_owner: str, - ) -> Any: - value = self.commit_result( - effect_id, - run_owner=recovery_owner, - claim_owner=recovery_owner, - result=result, - reconciled=True, - ) - with self._sessions() as db: - lease = ( - db.query(ToolEffectLease) - .filter(ToolEffectLease.effect_id == effect_id) - .with_for_update() - .one() - ) - run = ( - db.query(ChatRun) - .filter(ChatRun.run_id == lease.run_id) - .with_for_update() - .one() - ) - if lease.run_owner != recovery_owner or run.lease_owner != recovery_owner: - raise ToolEffectLeaseLost(effect_id) - run.lease_owner = None - run.lease_expires_at = None - run.updated_at = self._clock() - db.commit() - return value - - def release_recovery_claim( - self, - effect_id: str, - *, - recovery_owner: str, - retry_seconds: int = 30, - ) -> None: - """Defer a failed adapter without letting RunJournal resume past the pending effect.""" - with self._sessions() as db: - lease = ( - db.query(ToolEffectLease) - .filter(ToolEffectLease.effect_id == effect_id) - .with_for_update() - .one() - ) - if lease.run_owner != recovery_owner or lease.claim_owner != recovery_owner: - raise ToolEffectLeaseLost(effect_id) - run = ( - db.query(ChatRun) - .filter(ChatRun.run_id == lease.run_id) - .with_for_update() - .one() - ) - if run.lease_owner != recovery_owner: - raise ToolEffectLeaseLost(effect_id) - retry_at = self._clock() + timedelta(seconds=max(1, retry_seconds)) - lease.claim_owner = None - lease.lease_expires_at = retry_at - lease.updated_at = self._clock() - run.lease_expires_at = retry_at - run.updated_at = self._clock() - db.commit() - - -class ToolEffectGateway: - """The single execution state machine used by production middleware and tests.""" - - def __init__( - self, - journal: ToolEffectJournal, - registry: ToolRecoveryRegistry, - *, - poll_interval: float = 0.05, - lease_seconds: int = 300, - ) -> None: - self.journal = journal - self.registry = registry - self.poll_interval = poll_interval - self.lease_seconds = lease_seconds - - @staticmethod - async def _call(handler, intent: ToolIntent): # noqa: ANN001, ANN205 - value = handler(intent) if handler is not None else None - return await value if inspect.isawaitable(value) else value - - @staticmethod - def _reconciliation(value: Any) -> ReconciliationResult: - if isinstance(value, ReconciliationResult): - return value - return ( - ReconciliationResult.applied(value) - if value is not None - else ReconciliationResult.unknown() - ) - - def _start_heartbeat( - self, - effect_id: str, - *, - run_owner: str, - claim_owner: str, - ) -> tuple[asyncio.Task, asyncio.Event]: - lost = asyncio.Event() - interval = max(0.05, min(30.0, self.lease_seconds / 3)) - - async def _beat() -> None: - while True: - await asyncio.sleep(interval) - try: - ok = await asyncio.to_thread( - self.journal.renew_invocation_claim, - effect_id, - run_owner=run_owner, - claim_owner=claim_owner, - lease_seconds=self.lease_seconds, - ) - except Exception: # noqa: BLE001 - lost heartbeat means lost ownership - ok = False - if not ok: - lost.set() - return - - return asyncio.create_task(_beat()), lost - - @staticmethod - async def _stop_heartbeat(task: asyncio.Task) -> None: - task.cancel() - with suppress(asyncio.CancelledError): - await task - - async def execute_outcome( - self, - *, - run_id: str, - owner: str, - tool_call_id: str, - tool_name: str, - args: Mapping[str, Any], - invoke: Callable[[], Awaitable[Any] | Any], - idempotency_key: Optional[str] = None, - classify_failure: Optional[Callable[[Any], bool]] = None, - ) -> ToolExecutionOutcome: - claim_owner = f"tool:{uuid.uuid4().hex}" - policy = self.registry.resolve(tool_name) - while True: - decision = await asyncio.to_thread( - self.journal.begin_intent, - run_id=run_id, - owner=owner, - claim_owner=claim_owner, - tool_call_id=tool_call_id, - tool_name=tool_name, - args=args, - recovery_policy=policy.policy, - idempotency_key=idempotency_key, - lease_seconds=self.lease_seconds, - ) - if decision.action == "known": - return ToolExecutionOutcome( - decision.result, - decision.intent.effect_id, - decision.intent.result_id, - decision.result_state, - False, - ) - if decision.action == "unknown": - raise ToolOutcomeUnknown(decision.effect_id) - if decision.action == "wait": - await asyncio.sleep(self.poll_interval) - terminal = await asyncio.to_thread( - self.journal.terminal_decision, decision.effect_id - ) - if terminal is None: - continue - if terminal.action == "unknown": - raise ToolOutcomeUnknown(decision.effect_id) - return ToolExecutionOutcome( - terminal.result, - terminal.intent.effect_id, - terminal.intent.result_id, - terminal.result_state, - False, - ) - if decision.action not in ("execute", "reconcile"): - raise ToolEffectError( - f"unsupported tool intent action: {decision.action}" - ) - - await asyncio.to_thread( - self.journal.acquire_invocation_claim, - decision.effect_id, - run_owner=owner, - claim_owner=claim_owner, - lease_seconds=self.lease_seconds, - ) - heartbeat, claim_lost = self._start_heartbeat( - decision.effect_id, - run_owner=owner, - claim_owner=claim_owner, - ) - token = CURRENT_TOOL_EFFECT.set( - ToolInvocationContext( - decision.intent.effect_id, - decision.intent.result_id, - decision.intent.idempotency_key, - decision.intent.run_id, - claim_owner, - ) - ) - try: - if decision.action == "reconcile": - if policy.reconciler is None: - self.journal.mark_unknown( - decision.effect_id, - owner=owner, - claim_owner=claim_owner, - reason="reconcile policy has no reconciliation adapter", - ) - raise ToolOutcomeUnknown(decision.effect_id) - reconciliation = self._reconciliation( - await self._call(policy.reconciler, decision.intent) - ) - if claim_lost.is_set(): - raise ToolEffectLeaseLost(decision.effect_id) - if reconciliation.outcome == "applied": - result = await asyncio.to_thread( - self.journal.commit_result, - decision.effect_id, - run_owner=owner, - claim_owner=claim_owner, - result=reconciliation.result, - reconciled=True, - ) - return ToolExecutionOutcome( - result, - decision.intent.effect_id, - decision.intent.result_id, - "success", - False, - ) - if reconciliation.outcome != "not_applied": - self.journal.mark_unknown( - decision.effect_id, - owner=owner, - claim_owner=claim_owner, - reason="reconciliation could not determine external outcome", - ) - raise ToolOutcomeUnknown(decision.effect_id) - value = invoke() - result = await value if inspect.isawaitable(value) else value - if claim_lost.is_set(): - raise ToolEffectLeaseLost(decision.effect_id) - failed = ( - bool(classify_failure(result)) - if classify_failure is not None - else False - ) - # A returned error payload is still a definitive tool outcome. - # Persist it and give the original error back to the model so - # it can correct the request or explain an authorization issue. - # ToolOutcomeUnknown is reserved for invocations that never - # produced a result at all (exceptions, disconnects, crashes). - result = await asyncio.to_thread( - self.journal.commit_result, - decision.effect_id, - run_owner=owner, - claim_owner=claim_owner, - result=result, - failed=failed, - ) - return ToolExecutionOutcome( - result, - decision.intent.effect_id, - decision.intent.result_id, - "error" if failed else "success", - True, - ) - except asyncio.CancelledError: - with suppress(Exception): - self.journal.abandon_invocation_claim( - decision.effect_id, - run_owner=owner, - claim_owner=claim_owner, - ) - raise - except ToolOutcomeUnknown: - with suppress(Exception): - self.journal.abandon_invocation_claim( - decision.effect_id, - run_owner=owner, - claim_owner=claim_owner, - ) - raise - except Exception as exc: - with suppress(Exception): - self.journal.abandon_invocation_claim( - decision.effect_id, - run_owner=owner, - claim_owner=claim_owner, - ) - raise ToolOutcomeUnknown(decision.effect_id) from exc - finally: - await self._stop_heartbeat(heartbeat) - CURRENT_TOOL_EFFECT.reset(token) - - async def execute(self, **kwargs) -> Any: # noqa: ANN003 - return (await self.execute_outcome(**kwargs)).result - - -async def recover_incomplete_tool_effects( - *, - journal: Optional[ToolEffectJournal] = None, - registry: Optional[ToolRecoveryRegistry] = None, - lease_seconds: int = 120, - heartbeat_interval: Optional[float] = None, -) -> list[RecoverySweepDecision]: - """Recover each orphan independently; one broken adapter cannot starve later work.""" - - journal = journal or ToolEffectJournal() - registry = registry or DEFAULT_TOOL_RECOVERY_REGISTRY - from core.harness.usage import ( - UsageAttempt, - attempt_status_for_exception, - record_usage_safely, - ) - from core.services.harness_ledger import HarnessUsageLedger - - usage_ledger = HarnessUsageLedger(journal._sessions) - - async def _recorded_recovery_call(handler, intent, phase): # noqa: ANN001 - started = time.monotonic() - status = "failed" - try: - result = await ToolEffectGateway._call(handler, intent) - status = "success" - return result - except asyncio.CancelledError: - status = "cancelled" - raise - except Exception as exc: - status = attempt_status_for_exception(exc) - raise - finally: - try: - await record_usage_safely( - usage_ledger, - UsageAttempt( - run_id=intent.run_id, - kind="tool", - operation_name=intent.tool_name, - effect_id=intent.effect_id, - status=status, - latency_ms=int((time.monotonic() - started) * 1_000), - metadata={"recovery_phase": phase}, - ), - ) - except Exception: # noqa: BLE001 - logger.debug("tool recovery usage persistence failed", exc_info=True) - - decisions: list[RecoverySweepDecision] = [] - for effect_id in await asyncio.to_thread(journal.pending_effect_ids): - terminal_intent = journal.settle_terminal_run_intent(effect_id) - if terminal_intent is not None: - decisions.append( - RecoverySweepDecision( - effect_id, terminal_intent.run_id, "terminal_unknown" - ) - ) - continue - recovery_owner = f"tool-recovery:{uuid.uuid4().hex}" - intent = journal.claim_recovery_intent( - effect_id, - recovery_owner=recovery_owner, - lease_seconds=lease_seconds, - ) - if intent is None: - continue - policy = registry.resolve(intent.tool_name) - claim_lost = asyncio.Event() - - async def _heartbeat() -> None: - interval = heartbeat_interval - if interval is None: - interval = max(0.05, min(30.0, lease_seconds / 3)) - while True: - await asyncio.sleep(interval) - try: - ok = await asyncio.to_thread( - journal.renew_recovery_claim, - effect_id, - recovery_owner=recovery_owner, - lease_seconds=lease_seconds, - ) - except Exception: # noqa: BLE001 - lost heartbeat means lost ownership - ok = False - if not ok: - claim_lost.set() - return - - heartbeat = asyncio.create_task(_heartbeat()) - try: - if intent.recovery_policy == "never_replay": - journal.mark_unknown( - effect_id, - owner=recovery_owner, - claim_owner=recovery_owner, - reason="non-replayable tool intent has no committed result", - ) - decisions.append( - RecoverySweepDecision(effect_id, intent.run_id, "needs_attention") - ) - continue - if intent.recovery_policy == "replay_safe": - if policy.replayer is None: - raise ToolEffectError("replay_safe policy has no replay adapter") - result = await _recorded_recovery_call( - policy.replayer, intent, "replay" - ) - if claim_lost.is_set(): - raise ToolEffectLeaseLost(effect_id) - journal.commit_recovered_result( - effect_id, result, recovery_owner=recovery_owner - ) - decisions.append( - RecoverySweepDecision(effect_id, intent.run_id, "replayed") - ) - continue - - if policy.reconciler is None: - raise ToolEffectError("reconcile policy has no reconciliation adapter") - reconciliation = ToolEffectGateway._reconciliation( - await _recorded_recovery_call(policy.reconciler, intent, "reconcile") - ) - if claim_lost.is_set(): - raise ToolEffectLeaseLost(effect_id) - if reconciliation.outcome == "applied": - result = reconciliation.result - elif ( - reconciliation.outcome == "not_applied" and policy.replayer is not None - ): - result = await _recorded_recovery_call( - policy.replayer, intent, "replay_after_reconcile" - ) - else: - journal.mark_unknown( - effect_id, - owner=recovery_owner, - claim_owner=recovery_owner, - reason="reconciliation could not determine external outcome", - ) - decisions.append( - RecoverySweepDecision(effect_id, intent.run_id, "needs_attention") - ) - continue - if claim_lost.is_set(): - raise ToolEffectLeaseLost(effect_id) - journal.commit_recovered_result( - effect_id, result, recovery_owner=recovery_owner - ) - decisions.append( - RecoverySweepDecision(effect_id, intent.run_id, "reconciled") - ) - except Exception: # noqa: BLE001 - release this claim and continue the sweep - try: - journal.release_recovery_claim(effect_id, recovery_owner=recovery_owner) - except ToolEffectError: - pass - decisions.append(RecoverySweepDecision(effect_id, intent.run_id, "retry")) - finally: - heartbeat.cancel() - with suppress(asyncio.CancelledError): - await heartbeat - return decisions - - -__all__ = [ - "CURRENT_TOOL_EFFECT", - "DEFAULT_TOOL_RECOVERY_REGISTRY", - "IntentDecision", - "ReconciliationResult", - "RecoverySweepDecision", - "ToolEffectError", - "ToolEffectGateway", - "ToolEffectJournal", - "ToolEffectLeaseLost", - "ToolExecutionOutcome", - "ToolIdempotencyConflict", - "ToolIntent", - "ToolIntentCommitError", - "ToolInvocationContext", - "ToolOutcomeUnknown", - "ToolPolicy", - "ToolRecoveryRegistry", - "build_default_tool_recovery_registry", - "canonical_tool_args", - "find_tool_outcome_unknown", - "recover_incomplete_tool_effects", - "stable_idempotency_key", - "tool_args_hash", -] diff --git a/src/backend/core/services/user_agent_base.py b/src/backend/core/services/user_agent_base.py deleted file mode 100644 index 60a71db6..00000000 --- a/src/backend/core/services/user_agent_base.py +++ /dev/null @@ -1,854 +0,0 @@ -"""Edition-neutral business logic for personal and administrator sub-agents.""" - -from __future__ import annotations -from core.infra.time import utc_now - -import logging -import re -import uuid -from datetime import datetime -from decimal import Decimal -from typing import Any, Dict, List, Optional - -from core.db.models import UserAgent -from core.db.repository import AuditLogRepository, UserAgentRepository -from core.ontology.build_validator import ensure_ontology_build_valid -from sqlalchemy.orm import Session - -logger = logging.getLogger(__name__) - - -def _desktop_store() -> bool: - """Whether this process keeps a desktop capability file store (agents projected there).""" - from core.capabilities.paths import capabilities_enabled - - return capabilities_enabled() - - -MAX_USER_AGENTS = 20 -DEFAULT_AGENT_VERSION = "V1.0" -MAX_CHANGE_HISTORY = 30 -NON_VERSIONED_FIELDS = {"is_enabled"} -VERSIONED_FIELDS = { - "name": "名称", - "description": "简介", - "system_prompt": "角色设定", - "welcome_message": "开场白", - "suggested_questions": "推荐问题", - "mcp_server_ids": "绑定工具", - "skill_ids": "绑定技能", - "plugin_ids": "绑定插件", - "kb_ids": "绑定知识库", - "model_provider_id": "模型", - "temperature": "温度", - "max_tokens": "最大输出长度", - "max_iters": "最大推理轮次", - "timeout": "超时时间", - "is_enabled": "启用状态", -} - - -class UserAgentBaseService: - """Service for user agent CRUD and permission checks.""" - - def __init__(self, db: Session): - self.db = db - self.repo = UserAgentRepository(db) - - # ── Queries ────────────────────────────────────────────────────── - - def list_for_user(self, user_id: str) -> List[Dict[str, Any]]: - agents = self.repo.list_for_user(user_id) - rows = [self._serialize(a) for a in agents] - if _desktop_store(): - # Desktop: the bridged account's stored agents join the device's own; - # a display-name clash is decided by the resolver, never by order. - from core.capabilities import agents as caps_agents - - return caps_agents.merge_visible(user_id, rows) - return rows - - def list_admin(self) -> List[Dict[str, Any]]: - agents = self.repo.list_admin() - return [self._serialize(a) for a in agents] - - def get_by_id(self, agent_id: str, user_id: Optional[str] = None) -> Dict[str, Any]: - agent = self.repo.get_by_id(agent_id) - if not agent: - stored = self._stored_account_agent(agent_id, user_id) - if stored is not None: - return stored.to_serialized() - raise LookupError(f"Agent {agent_id} not found") - if user_id and not self._is_accessible(agent, user_id): - raise PermissionError("No access to this agent") - return self._serialize(agent) - - def get_raw_by_id(self, agent_id: str, user_id: Optional[str] = None) -> UserAgent: - """Return the ORM object (for direct use by workflow/factory). - - On a desktop the bridged account's stored agents are a second, equally - authoritative source: an ``AgentDefinition`` with the same attribute - surface is returned for them. - """ - agent = self.repo.get_by_id(agent_id) - if not agent: - stored = self._stored_account_agent(agent_id, user_id) - if stored is not None: - return stored # type: ignore[return-value] - raise LookupError(f"Agent {agent_id} not found") - if user_id and not self._is_accessible(agent, user_id): - raise PermissionError("No access to this agent") - return agent - - @staticmethod - def _stored_account_agent(agent_id: str, user_id: Optional[str] = None): - if not _desktop_store(): - return None - from core.capabilities import agents as caps_agents - from core.capabilities.skills import account_authorized_for - - if not account_authorized_for(user_id): - return None - return caps_agents.account_definition(agent_id) - - def _project_to_store(self, agent: UserAgent) -> None: - if not _desktop_store(): - return - from core.capabilities import agents as caps_agents - - caps_agents.publish_local_agent(self._serialize(agent)) - - @staticmethod - def _remove_from_store(agent_id: str) -> None: - if not _desktop_store(): - return - from core.capabilities import agents as caps_agents - - caps_agents.remove_local_agent(agent_id) - - # ── Mutations ──────────────────────────────────────────────────── - - def create( - self, - user_id: Optional[str], - operator_name: Optional[str], - owner_type: str, - data: Dict[str, Any], - scope_id: Optional[str] = None, - ) -> Dict[str, Any]: - data = dict(data) - incoming_extra = dict(data.get("extra_config") or {}) - ontology_tags = list( - data.pop("ontology_tags", incoming_extra.get("ontology_tags") or []) or [] - ) - incoming_extra["ontology_tags"] = ontology_tags - data["extra_config"] = incoming_extra - ensure_ontology_build_valid( - self.db, - asset_type="subagent", - name=str(data.get("name") or ""), - description=str(data.get("description") or ""), - instructions=str(data.get("system_prompt") or ""), - mcp_server_ids=list(data.get("mcp_server_ids") or []), - skill_ids=list(data.get("skill_ids") or []), - plugin_ids=list(data.get("plugin_ids") or []), - ontology_tags=ontology_tags, - ) - self._validate_create_scope(user_id, owner_type, scope_id) - - agent_id = f"ua_{uuid.uuid4().hex[:16]}" - created_at = self._now_iso() - creation_history = [ - { - "version": DEFAULT_AGENT_VERSION, - "timestamp": created_at, - "content": "创建了子智能体", - "operator_name": operator_name or user_id or "未知用户", - "details": [], - } - ] - extra_config = self._merge_extra_config( - current_extra=None, - incoming_extra=incoming_extra, - version=DEFAULT_AGENT_VERSION, - change_history=creation_history, - ) - - record = { - "agent_id": agent_id, - "owner_type": owner_type, - **self._owner_fields(user_id, owner_type, scope_id), - "created_by": user_id, - **data, - "extra_config": extra_config, - } - agent = self.repo.create(record) - self._project_to_store(agent) - self._audit( - user_id, - "agent.create", - agent_id, - {"owner_type": owner_type, "name": data.get("name")}, - ) - return self._serialize(agent) - - def update( - self, - agent_id: str, - user_id: Optional[str], - operator_name: Optional[str], - owner_type: str, - data: Dict[str, Any], - ) -> Dict[str, Any]: - agent = self.repo.get_by_id(agent_id) - if not agent: - raise LookupError(f"Agent {agent_id} not found") - self._check_ownership(agent, user_id, owner_type) - - data = dict(data) - current_extra = dict(agent.extra_config or {}) - incoming_extra = dict(data.get("extra_config") or {}) - ontology_tags = list( - data.pop( - "ontology_tags", - incoming_extra.get("ontology_tags", current_extra.get("ontology_tags") or []), - ) - or [] - ) - incoming_extra["ontology_tags"] = ontology_tags - if "extra_config" in data or ontology_tags != list( - current_extra.get("ontology_tags") or [] - ): - data["extra_config"] = incoming_extra - ensure_ontology_build_valid( - self.db, - asset_type="subagent", - name=str(data.get("name", agent.name) or ""), - description=str(data.get("description", agent.description) or ""), - instructions=str(data.get("system_prompt", agent.system_prompt) or ""), - mcp_server_ids=list(data.get("mcp_server_ids", agent.mcp_server_ids) or []), - skill_ids=list(data.get("skill_ids", agent.skill_ids) or []), - plugin_ids=list(data.get("plugin_ids", agent.plugin_ids) or []), - ontology_tags=ontology_tags, - ) - - changed_fields = self._collect_changed_fields(agent, data) - versioned_fields = [field for field in changed_fields if field not in NON_VERSIONED_FIELDS] - changed_labels = [VERSIONED_FIELDS[field] for field in changed_fields] - next_version = self._read_version(current_extra) - change_history = self._read_change_history(current_extra) - - if changed_labels: - change_summary = self._build_change_summary(changed_fields, data) - change_details = self._build_change_details(agent, changed_fields, data) - entry_version = next_version - if versioned_fields: - next_version = self._increment_version(next_version) - entry_version = next_version - change_history.append( - { - "version": entry_version, - "timestamp": self._now_iso(), - "content": change_summary, - "operator_name": operator_name or user_id or "未知用户", - "details": change_details, - } - ) - change_history = change_history[-MAX_CHANGE_HISTORY:] - - payload = dict(data) - payload["extra_config"] = self._merge_extra_config( - current_extra=current_extra, - incoming_extra=incoming_extra, - version=next_version, - change_history=change_history, - ) - - agent = self.repo.update(agent_id, payload) - self._project_to_store(agent) - audit_details = {"fields": list(data.keys())} - if changed_labels: - audit_details["change_summary"] = change_summary - audit_details["version"] = next_version - self._audit(user_id, "agent.update", agent_id, audit_details) - return self._serialize(agent) - - def delete( - self, - agent_id: str, - user_id: Optional[str], - owner_type: str, - ) -> bool: - agent = self.repo.get_by_id(agent_id) - if not agent: - raise LookupError(f"Agent {agent_id} not found") - self._check_ownership(agent, user_id, owner_type) - - ok = self.repo.delete(agent_id) - if ok: - self._remove_from_store(agent_id) - self._audit(user_id, "agent.delete", agent_id) - return ok - - def toggle_enabled(self, agent_id: str) -> Dict[str, Any]: - agent = self.repo.get_by_id(agent_id) - if not agent: - raise LookupError(f"Agent {agent_id} not found") - new_val = not agent.is_enabled - agent = self.repo.update(agent_id, {"is_enabled": new_val}) - self._project_to_store(agent) - return self._serialize(agent) - - # ── Available resources ────────────────────────────────────────── - - def list_available_resources(self, owner_user_id: Optional[str] = None) -> Dict[str, Any]: - """Return MCP servers, skills, plugins, and KB spaces bindable to agents. - - Plugin-sourced skills/MCP are removed from the skills/mcp_servers lists - and instead bound as a whole via the ``plugins`` list (plugin = - installable/removable unit, expanded at runtime into its skills+tools). - owner_user_id is used to include that user's private plugins and MCPs. - - The MCP list intentionally contains capabilities the user has personally - switched off. A sub-agent binding is an explicit, narrower capability - grant and therefore may opt into one of those tools without turning it - on for the user's main agent. Deployment-global MCPs disabled by an - administrator remain unavailable. - """ - from core.db.models import AdminMcpServer, InstalledPlugin, KBSpace - from sqlalchemy import or_ - - # ── Plugin list + their component id sets (used to strip plugin capabilities from the loose skills/tools) ── - # Query the ORM directly for component_ids (authoritative): global plugins + current user's private plugins. - try: - pq = self.db.query(InstalledPlugin) - if owner_user_id: - pq = pq.filter( - or_( - InstalledPlugin.owner_user_id == owner_user_id, - InstalledPlugin.owner_user_id.is_(None), - ) - ) - else: - pq = pq.filter(InstalledPlugin.owner_user_id.is_(None)) - plugin_rows = pq.order_by(InstalledPlugin.created_at.desc()).all() - except Exception as exc: # noqa: BLE001 - logger.debug("Failed to list installed plugins: %s", exc) - plugin_rows = [] - - plugin_skill_ids: set = set() - plugin_mcp_ids: set = set() - # The same plugin may exist both as a global version (install_id=slug@global) - # and as the user's private version (slug@) — same name → duplicate - # display (the user sees two "定时任务管理" entries). Dedupe by slug and - # show only one: prefer the user's private version (its components carry - # the user's fingerprint and match their sandbox credentials); fall back - # to the global version when there is no private one. - # Note: plugin_skill_ids / plugin_mcp_ids still accumulate from **all** - # rows (including the deduped-away one), so the loose skill/MCP lists - # can fully exclude both versions' components. - _plugin_by_slug: Dict[str, Dict[str, Any]] = {} - for p in plugin_rows: - cids = p.component_ids or {} - s_ids = list(cids.get("skills") or []) - m_ids = list(cids.get("mcp") or []) - plugin_skill_ids.update(s_ids) - plugin_mcp_ids.update(m_ids) - slug = (p.install_id or "").rsplit("@", 1)[0] - is_owned = bool(owner_user_id) and p.owner_user_id == owner_user_id - existing = _plugin_by_slug.get(slug) - if existing is None or (is_owned and not existing["_owned"]): - _plugin_by_slug[slug] = { - "id": p.install_id, - "name": p.name, - "description": p.description or "", - "skill_count": len(s_ids), - "mcp_count": len(m_ids), - "_owned": is_owned, - } - plugin_list: List[Dict[str, Any]] = [ - {k: v for k, v in item.items() if k != "_owned"} for item in _plugin_by_slug.values() - ] - - # Built-in plugin MCPs can also be present in the static catalog without - # a source_plugin DB row. They still belong under the plugin selector, - # not the loose MCP selector. - try: - from core.plugins.management import builtin_plugin_component_ids - - _, builtin_plugin_mcp_ids = builtin_plugin_component_ids() - plugin_mcp_ids.update(builtin_plugin_mcp_ids) - except Exception as exc: # noqa: BLE001 - logger.debug("Failed to load built-in plugin MCP ids: %s", exc) - - # Resolve the user's personal on/off layer once so each MCP option can - # explain whether it is already enabled for the main agent. This flag - # is display metadata only; disabled options remain bindable here. - enabled_mcp_ids: Optional[set[str]] = None - if owner_user_id: - try: - from core.config.catalog_resolver import resolve_all_runtime_enabled - - _skills, _agents, resolved_mcps = resolve_all_runtime_enabled( - self.db, owner_user_id - ) - if resolved_mcps is not None: - enabled_mcp_ids = set(resolved_mcps) - except Exception as exc: # noqa: BLE001 - logger.debug("Failed to resolve user MCP enablement: %s", exc) - - # ── MCP tools (exclude plugin-sourced + owner isolation) ── - # Owner isolation: private entries (owner_user_id non-null) are visible - # only to their owner. Otherwise private copies produced by other users - # installing the same plugin (e.g. automation-automation_task-) would all leak into this user's bindable list → - # duplicates of "定时任务" etc. Empty owner = globally shared entry, - # visible to everyone. - _mcp_owner_ok = ( - or_( - AdminMcpServer.owner_user_id.is_(None), - AdminMcpServer.owner_user_id == owner_user_id, - ) - if owner_user_id - else AdminMcpServer.owner_user_id.is_(None) - ) - mcp_servers = ( - self.db.query(AdminMcpServer) - .filter( - _mcp_owner_ok, - # Authoritative exclusion: any plugin-sourced MCP (source_plugin - # nonnull) never enters the loose list — it is bound as a whole via - # the plugins list instead. More robust than checking only - # plugin_mcp_ids (doesn't depend on the plugin row still - # existing/being visible), and also blocks orphaned plugin MCP rows. - AdminMcpServer.source_plugin.is_(None), - ) - .order_by(AdminMcpServer.sort_order) - .all() - ) - mcp_list: List[Dict[str, Any]] = [] - seen_mcp_ids: set = set() - for server in mcp_servers: - # A global false is an administrator lock. A private false is the - # owner's personal off state and remains eligible for an explicit - # sub-agent binding. - if server.owner_user_id is None and not server.is_enabled: - continue - if server.server_id in plugin_mcp_ids: - continue - mcp_list.append( - { - "id": server.server_id, - "name": server.display_name, - "description": server.description, - "enabled": ( - server.server_id in enabled_mcp_ids - if enabled_mcp_ids is not None - else bool(server.is_enabled) - ), - } - ) - seen_mcp_ids.add(server.server_id) - - # Some umbrella/built-in MCP entries are catalog-defined and do not - # necessarily have a same-named AdminMcpServer row. Include every - # administrator-enabled catalog item so the selector is complete. - try: - from core.config.catalog_runtime import get_runtime_catalog - - runtime_catalog = get_runtime_catalog(self.db, include_runtime_details=False) - for item in runtime_catalog.get("mcp") or []: - item_id = str(item.get("id") or "").strip() - if ( - not item_id - or item_id in seen_mcp_ids - or item_id in plugin_mcp_ids - or not bool(item.get("enabled", True)) - ): - continue - mcp_list.append( - { - "id": item_id, - "name": item.get("name") or item_id, - "description": item.get("description") or item.get("desc") or "", - "enabled": ( - item_id in enabled_mcp_ids - if enabled_mcp_ids is not None - else bool(item.get("enabled", True)) - ), - } - ) - seen_mcp_ids.add(item_id) - except Exception as exc: # noqa: BLE001 - logger.debug("Failed to load catalog MCP resources: %s", exc) - - # ── Skills: DB-managed + filesystem-discovered (both exclude plugin-sourced + owner isolation) ── - # ⚠️ Critical: the filesystem loader (load_all_metadata) scans **all** - # materialized skills on disk — including other users' private skills - # and plugin skills materialized by other users' plugin installs. - # Excluding via plugin_skill_ids alone (components of plugins visible to - # the current user only) would miss some, letting other users' plugin - # skills sneak into "bindable skills". So compute two **authoritative** - # exclusion sets directly from AdminSkill and filter both sources - # uniformly: - # - all_plugin_skill_ids: any plugin-sourced skill (source_plugin non-null, any owner); - # - foreign_private_skill_ids: other users' private skills (owner_user_id non-null and ≠ current user). - from core.db.models import AdminSkill - - all_plugin_skill_ids: set = set() - foreign_private_skill_ids: set = set() - try: - for row in self.db.query( - AdminSkill.skill_id, AdminSkill.source_plugin, AdminSkill.owner_user_id - ).all(): - if row.source_plugin: - all_plugin_skill_ids.add(row.skill_id) - if row.owner_user_id and row.owner_user_id != owner_user_id: - foreign_private_skill_ids.add(row.skill_id) - except Exception as exc: - logger.debug("Failed to precompute skill exclusion sets: %s", exc) - - _excluded_skill_ids = plugin_skill_ids | all_plugin_skill_ids | foreign_private_skill_ids - - skill_list: List[Dict[str, Any]] = [] - try: - _skill_owner_ok = ( - or_(AdminSkill.owner_user_id.is_(None), AdminSkill.owner_user_id == owner_user_id) - if owner_user_id - else AdminSkill.owner_user_id.is_(None) - ) - db_skills = ( - self.db.query(AdminSkill) - .filter( - AdminSkill.is_enabled == True, - _skill_owner_ok, - ) - .order_by(AdminSkill.updated_at.desc()) - .all() - ) - seen_ids = set() - for s in db_skills: - if s.skill_id in _excluded_skill_ids: - continue - skill_list.append( - {"id": s.skill_id, "name": s.display_name, "description": s.description or ""} - ) - seen_ids.add(s.skill_id) - except Exception: - seen_ids = set() - - try: - from core.agent_skills.loader import get_skill_loader - - loader = get_skill_loader() - for sid, meta in loader.load_all_metadata().items(): - if sid not in seen_ids and sid not in _excluded_skill_ids: - skill_list.append( - { - "id": sid, - "name": getattr(meta, "name", sid), - "description": getattr(meta, "description", ""), - } - ) - except Exception as exc: - logger.debug("Failed to load filesystem skills: %s", exc) - - # KB spaces - kb_list: List[Dict[str, Any]] = [] - try: - kb_spaces = ( - self.db.query(KBSpace) - .filter( - KBSpace.deleted_at.is_(None), - ) - .order_by(KBSpace.created_at.desc()) - .all() - ) - kb_list = [ - {"id": s.kb_id, "name": s.name, "description": s.description or ""} - for s in kb_spaces - ] - except Exception: - pass - - return { - "mcp_servers": mcp_list, - "skills": skill_list, - "plugins": plugin_list, - "kb_spaces": kb_list, - "ontology_tags": self._ontology_tag_options(), - } - - def _ontology_tag_options(self) -> List[Dict[str, Any]]: - """Controlled labels that activate a sub-agent ontology workflow at runtime.""" - from core.services.ontology_service import OntologyService - - return OntologyService(self.db).list_asset_tag_options("subagent") - - # ── Helpers ─────────────────────────────────────────────────────── - - def _validate_create_scope( - self, user_id: Optional[str], owner_type: str, scope_id: Optional[str] - ) -> None: - if owner_type == "user": - if not user_id: - raise ValueError("user_id required for user agents") - if self.repo.count_user_agents(user_id) >= MAX_USER_AGENTS: - raise ValueError(f"Maximum {MAX_USER_AGENTS} agents per user reached") - return - if owner_type == "admin": - return - raise ValueError(f"Unsupported agent owner type: {owner_type}") - - @staticmethod - def _owner_fields( - user_id: Optional[str], owner_type: str, scope_id: Optional[str] - ) -> Dict[str, Any]: - return {"user_id": user_id if owner_type == "user" else None} - - def _is_accessible(self, agent: UserAgent, user_id: str) -> bool: - if agent.owner_type == "admin" and agent.is_enabled: - return True - if agent.owner_type == "user" and agent.user_id == user_id: - return True - return False - - def _check_ownership(self, agent: UserAgent, user_id: Optional[str], owner_type: str) -> None: - # owner_type is the route context: 'admin' = Admin console routes, everything else = user-side routes - if owner_type == "admin": - if agent.owner_type != "admin": - raise PermissionError("Admin can only modify admin agents") - return - if agent.owner_type == "user" and agent.user_id == user_id: - return - raise PermissionError("You can only modify your own agents") - - @classmethod - def _serialize(cls, agent: UserAgent) -> Dict[str, Any]: - extra_config = agent.extra_config or {} - return { - "agent_id": agent.agent_id, - "owner_type": agent.owner_type, - "user_id": agent.user_id, - **cls._serialize_scope(agent), - "name": agent.name, - "avatar": agent.avatar, - "description": agent.description, - "system_prompt": agent.system_prompt, - "welcome_message": agent.welcome_message, - "suggested_questions": agent.suggested_questions or [], - "mcp_server_ids": agent.mcp_server_ids or [], - "skill_ids": agent.skill_ids or [], - "plugin_ids": agent.plugin_ids or [], - "kb_ids": agent.kb_ids or [], - "model_provider_id": agent.model_provider_id, - "temperature": float(agent.temperature) if agent.temperature is not None else None, - "max_tokens": agent.max_tokens, - "max_iters": agent.max_iters, - "timeout": agent.timeout, - "is_enabled": agent.is_enabled, - "sort_order": agent.sort_order, - "source_market_slug": agent.source_market_slug, - "ontology_tags": list(extra_config.get("ontology_tags") or []), - "extra_config": extra_config, - "version": cls._read_version(extra_config), - "change_history": cls._read_change_history(extra_config), - "created_at": agent.created_at.isoformat() if agent.created_at else None, - "updated_at": agent.updated_at.isoformat() if agent.updated_at else None, - "created_by": agent.created_by, - } - - @staticmethod - def _serialize_scope(agent: UserAgent) -> Dict[str, Any]: - return {} - - def _audit( - self, user_id: Optional[str], action: str, resource_id: str, details: Dict = None - ) -> None: - try: - audit_repo = AuditLogRepository(self.db) - audit_repo.create( - { - "user_id": user_id, - "action": action, - "resource_type": "user_agent", - "resource_id": resource_id, - "details": details or {}, - "status": "success", - } - ) - except Exception as exc: - logger.warning("Audit log failed: %s", exc) - - @staticmethod - def _normalize_value(value: Any) -> Any: - if isinstance(value, Decimal): - return float(value) - if isinstance(value, list): - return list(value) - return value - - @staticmethod - def _now_iso() -> str: - return utc_now().replace(microsecond=0).isoformat() - - @classmethod - def _collect_changed_fields(cls, agent: UserAgent, data: Dict[str, Any]) -> List[str]: - fields: List[str] = [] - for field in VERSIONED_FIELDS: - if field not in data: - continue - old_value = cls._normalize_value(getattr(agent, field, None)) - new_value = cls._normalize_value(data.get(field)) - if old_value != new_value: - fields.append(field) - return fields - - @staticmethod - def _read_version(extra_config: Optional[Dict[str, Any]]) -> str: - if not isinstance(extra_config, dict): - return DEFAULT_AGENT_VERSION - raw = extra_config.get("version") - return UserAgentBaseService._normalize_version(raw if isinstance(raw, str) else "") - - @staticmethod - def _read_change_history(extra_config: Optional[Dict[str, Any]]) -> List[Dict[str, str]]: - if not isinstance(extra_config, dict): - return [] - raw_history = extra_config.get("change_history") - if not isinstance(raw_history, list): - return [] - - history: List[Dict[str, str]] = [] - for item in raw_history: - if not isinstance(item, dict): - continue - timestamp = item.get("timestamp") - content = item.get("content") - version = item.get("version") - operator_name = item.get("operator_name") - details = item.get("details") - if not isinstance(timestamp, str) or not isinstance(content, str): - continue - history.append( - { - "timestamp": timestamp, - "content": content, - "version": UserAgentBaseService._normalize_version( - version if isinstance(version, str) else "" - ), - "operator_name": ( - operator_name - if isinstance(operator_name, str) and operator_name.strip() - else "未知用户" - ), - "details": UserAgentBaseService._normalize_change_details(details), - } - ) - return history - - @staticmethod - def _increment_version(version: str) -> str: - normalized = UserAgentBaseService._normalize_version(version) - match = re.match(r"^[Vv](\d+)\.(\d+)$", normalized) - if not match: - return "V1.1" - major, minor = (int(part) for part in match.groups()) - return f"V{major}.{minor + 1}" - - @staticmethod - def _build_change_summary(changed_fields: List[str], data: Dict[str, Any]) -> str: - if changed_fields == ["is_enabled"]: - return "启用了子智能体" if bool(data.get("is_enabled")) else "停用了子智能体" - - changed_labels = [VERSIONED_FIELDS[field] for field in changed_fields] - if not changed_labels: - return "更新了智能体配置" - if len(changed_labels) <= 3: - return f"修改了{'、'.join(changed_labels)}" - preview = "、".join(changed_labels[:3]) - return f"修改了{preview}等{len(changed_labels)}项" - - @classmethod - def _build_change_details( - cls, agent: UserAgent, changed_fields: List[str], data: Dict[str, Any] - ) -> List[Dict[str, str]]: - details: List[Dict[str, str]] = [] - for field in changed_fields: - old_value = cls._stringify_detail_value(field, getattr(agent, field, None)) - new_value = cls._stringify_detail_value(field, data.get(field)) - details.append( - { - "field": VERSIONED_FIELDS[field], - "before": old_value, - "after": new_value, - } - ) - return details - - @staticmethod - def _stringify_detail_value(field: str, value: Any) -> str: - if field == "is_enabled": - return "启用" if bool(value) else "关闭" - if value is None: - return "未填写" - if isinstance(value, list): - return "、".join(str(item) for item in value) if value else "未填写" - if isinstance(value, bool): - return "是" if value else "否" - text = str(value).strip() - return text if text else "未填写" - - @staticmethod - def _normalize_change_details(details: Any) -> List[Dict[str, str]]: - if not isinstance(details, list): - return [] - normalized: List[Dict[str, str]] = [] - for item in details: - if not isinstance(item, dict): - continue - field = item.get("field") - before = item.get("before") - after = item.get("after") - if not isinstance(field, str): - continue - normalized.append( - { - "field": field, - "before": before if isinstance(before, str) else "未填写", - "after": after if isinstance(after, str) else "未填写", - } - ) - return normalized - - @staticmethod - def _normalize_version(version: str) -> str: - if not isinstance(version, str) or not version.strip(): - return DEFAULT_AGENT_VERSION - raw = version.strip() - if re.match(r"^[Vv]\d+\.\d+$", raw): - return f"V{raw[1:]}" - legacy_patch = re.match(r"^(\d+)\.(\d+)\.(\d+)$", raw) - if legacy_patch: - major, minor, patch = (int(part) for part in legacy_patch.groups()) - return f"V{major}.{minor + patch}" - legacy_minor = re.match(r"^(\d+)\.(\d+)$", raw) - if legacy_minor: - major, minor = (int(part) for part in legacy_minor.groups()) - return f"V{major}.{minor}" - return DEFAULT_AGENT_VERSION - - @staticmethod - def _merge_extra_config( - current_extra: Optional[Dict[str, Any]], - incoming_extra: Optional[Dict[str, Any]], - *, - version: str, - change_history: List[Dict[str, str]], - ) -> Dict[str, Any]: - merged = dict(current_extra or {}) - if isinstance(incoming_extra, dict): - merged.update(incoming_extra) - merged["version"] = version - merged["change_history"] = change_history - return merged diff --git a/src/backend/core/spaces/edition_access.py b/src/backend/core/spaces/edition_access.py deleted file mode 100644 index 7f5a7637..00000000 --- a/src/backend/core/spaces/edition_access.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Personal file mounts for the community distribution.""" -workspace_mount_names = ("myspace", "skills") - -def team_grants(db, actor): - return () - -def session_entrypoint(entrypoint, user_id): - return list(entrypoint) - -def project_location(db, project): - return None - -def matches_scope(project, scope): - return scope.kind == project.kind and scope.root_folder_id == project.linked_folder_id - -def owner_space(column, owner): - raise ValueError("Unsupported directory owner") diff --git a/src/backend/core/spaces/edition_revocation.py b/src/backend/core/spaces/edition_revocation.py deleted file mode 100644 index d84eaa20..00000000 --- a/src/backend/core/spaces/edition_revocation.py +++ /dev/null @@ -1,3 +0,0 @@ -"""The community distribution has no dedicated account computers.""" -def revoke_computers(actor): - return None diff --git a/src/backend/core/vision/__init__.py b/src/backend/core/vision/__init__.py deleted file mode 100644 index 06cf1624..00000000 --- a/src/backend/core/vision/__init__.py +++ /dev/null @@ -1,45 +0,0 @@ -"""视觉桥（Vision Bridge）：让纯文本主模型「看得见」图片。 - -图片 → 多模态模型 → 结构化文字证据 → 注入主模型上下文。主模型全程只吃文本。 - -对外入口： - -- :func:`core.vision.service.is_available` —— 当前是否配了可用的视觉模型 -- :func:`core.vision.service.get_vision_bridge` —— 识别单张/多张图 -- :func:`core.vision.render.render_evidence` —— 证据 → 可注入文本（带不可信输入围栏） -- :func:`core.vision.service.model_supports_vision` —— 某个模型配置是否原生多模态 -- :func:`core.vision.service.resolve_vision_mode` —— 本轮模型看图方式：native / bridge / none -""" - -from core.vision.render import render_evidence, render_many, render_unavailable -from core.vision.schema import VisionEvidence, parse_evidence -from core.vision.service import ( - VISION_ROLE_KEY, - VisionBridge, - VisionMode, - VisionResult, - get_vision_bridge, - is_available, - model_supports_vision, - resolve_vision_config, - resolve_vision_mode, - sniff_mime, -) - -__all__ = [ - "VISION_ROLE_KEY", - "VisionBridge", - "VisionEvidence", - "VisionMode", - "VisionResult", - "get_vision_bridge", - "is_available", - "model_supports_vision", - "parse_evidence", - "render_evidence", - "render_many", - "render_unavailable", - "resolve_vision_config", - "resolve_vision_mode", - "sniff_mime", -] diff --git a/src/backend/extensions/README.md b/src/backend/extensions/README.md index 3aaf8df2..e1cb4e6d 100644 --- a/src/backend/extensions/README.md +++ b/src/backend/extensions/README.md @@ -19,7 +19,7 @@ policy and iteration limits. Core runtime code applies the policy against the parent's actual grants. Active database prompt parts (`subagents` kind) take precedence over the shipped Markdown. The files remain the seed and fallback. -Lifecycle code stays in `core/agent_skills/`, `core/plugins/` and +Lifecycle code stays in `core/capabilities/skills/`, `core/capabilities/plugins/` and `core/capabilities/`. Agent execution stays in `core/llm/` and `orchestration/`. User installations, credentials and mutable state belong to the configured runtime stores, never this directory. diff --git a/src/backend/extensions/mcp_servers/_packaging.py b/src/backend/extensions/mcp_servers/_packaging.py index 4675d086..a5b3a3ba 100644 --- a/src/backend/extensions/mcp_servers/_packaging.py +++ b/src/backend/extensions/mcp_servers/_packaging.py @@ -28,7 +28,7 @@ def read_artifact_bytes(file_id: str) -> Optional[bytes]: """从共享产物库按 file_id 读回字节（local: 读 path；oss: download_bytes）。""" try: - from core.artifacts import store + from core.files.artifacts import store meta = store.get_artifact(file_id) if not meta: @@ -39,7 +39,7 @@ def read_artifact_bytes(file_id: str) -> Optional[bytes]: return fh.read() key = meta.get("storage_key") if key: - from core.storage import get_storage + from core.files.storage import get_storage return bytes(get_storage().download_bytes(key)) except Exception as exc: # noqa: BLE001 @@ -129,7 +129,7 @@ def is_plugin_root(root: Path) -> bool: 却能从后台上传导入——同一个包两条通路结论不同。所以直接问导入器。 """ try: - from core.plugins.packaging.importer import detect_manifest + from core.capabilities.plugins.packaging.importer import detect_manifest detect_manifest(root) return True diff --git a/src/backend/extensions/mcp_servers/_serve.py b/src/backend/extensions/mcp_servers/_serve.py index 039d4769..cbbf74d5 100644 --- a/src/backend/extensions/mcp_servers/_serve.py +++ b/src/backend/extensions/mcp_servers/_serve.py @@ -1,4 +1,4 @@ -"""Shared launcher for FastMCP servers. +"""Shared launcher for MCPServer servers. Every MCP server's ``main()`` calls ``run(mcp, default_port=NNNN)``. Picks transport from ``--transport`` (``stdio`` for local debug, the @@ -13,14 +13,14 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: - from mcp.server.fastmcp import FastMCP + from mcp.server.mcpserver import MCPServer def _streamable_http_bind_host() -> str: return os.getenv("MCP_BIND_HOST", "0.0.0.0").strip() or "0.0.0.0" -def run(mcp: "FastMCP", default_port: int) -> None: +def run(mcp: "MCPServer", default_port: int) -> None: parser = argparse.ArgumentParser(description=mcp.name) parser.add_argument( "--transport", @@ -31,22 +31,16 @@ def run(mcp: "FastMCP", default_port: int) -> None: args = parser.parse_args() if args.transport == "streamable-http": - mcp.settings.port = args.port - # Compose needs all-interface binding inside its private container - # network. The no-Docker local/desktop launcher explicitly supplies - # MCP_BIND_HOST=127.0.0.1 so these internal control-plane ports are not - # exposed to the user's LAN (and do not trigger an avoidable Windows - # firewall prompt). - mcp.settings.host = _streamable_http_bind_host() - # MCP's default DNS-rebinding allow-list is localhost only. Backend - # reaches us via the docker DNS name (e.g. ``mcp:9108``); rebinding - # protection isn't relevant on a private network with no browser - # traffic. from mcp.server.transport_security import TransportSecuritySettings - mcp.settings.transport_security = TransportSecuritySettings( - enable_dns_rebinding_protection=False, + host = _streamable_http_bind_host() + mcp.run( + "streamable-http", host=host, port=args.port, + transport_security=TransportSecuritySettings( + enable_dns_rebinding_protection=True, + allowed_hosts=["127.0.0.1:*", "localhost:*", "[::1]:*", "mcp:*", "hugagent-mcp:*"], + allowed_origins=["http://127.0.0.1:*", "http://localhost:*"], + ), ) - mcp.run("streamable-http") else: asyncio.run(mcp.run_stdio_async()) diff --git a/src/backend/extensions/mcp_servers/agent_manager_mcp/impl.py b/src/backend/extensions/mcp_servers/agent_manager_mcp/impl.py index b2ac18ba..070b16b7 100644 --- a/src/backend/extensions/mcp_servers/agent_manager_mcp/impl.py +++ b/src/backend/extensions/mcp_servers/agent_manager_mcp/impl.py @@ -10,6 +10,9 @@ """ from __future__ import annotations +from features.marketplace import agent_catalog +from features.marketplace import agent_installation +from features.marketplace import agent_submissions import logging from typing import Any, Dict, List, Optional, Tuple @@ -25,12 +28,10 @@ # 绑定类字段一次最多绑多少项，防止模型把整库 id 一股脑塞进来。 _MAX_BINDINGS = 100 - # ── 通用 ──────────────────────────────────────────────────────────────── def _no_user() -> Dict[str, Any]: return {"ok": False, "message": "❌ 无法确定用户身份（缺 X-Current-User-Id 头），拒绝操作。"} - def _invalidate_user_cache(user_id: Optional[str]) -> None: """清掉该用户的 30s 能力解析缓存。 @@ -40,13 +41,12 @@ def _invalidate_user_cache(user_id: Optional[str]) -> None: CATALOG_MUTATING_TOOLS 白名单。本调用作为进程内一致性的兜底保留，无害。 """ try: - from core.config.catalog_resolver import invalidate_capability_cache + from core.capabilities.catalog.resolver import invalidate_capability_cache invalidate_capability_cache(str(user_id) if user_id else None) except Exception as exc: # noqa: BLE001 logger.debug("agent_manager: invalidate_capability_cache failed (%s)", exc) - def _require_cap(db, user_id: str, cap: str) -> Optional[Dict[str, Any]]: """能力位校验。缺权限返回错误 dict，否则 None。""" try: @@ -62,24 +62,21 @@ def _require_cap(db, user_id: str, cap: str) -> Optional[Dict[str, Any]]: return {"ok": False, "message": f"❌ 权限校验失败：{exc}"} return None - def _svc(db): - from core.services.user_agent_service import UserAgentService + from core.capabilities.agents.edition import UserAgentService return UserAgentService(db) - def valid_categories() -> List[str]: """子智能体市场的固定分类（**与技能市场的 8 类不同**）。 从单一真相源读，别在工具描述里硬写一份——分类调整时两边会漂移， 模型照着过期的列表猜，每次上架都要失败一轮。 """ - from core.services.agent_market_categories import AGENT_MARKETPLACE_CATEGORIES + from features.marketplace.agent_categories import AGENT_MARKETPLACE_CATEGORIES return list(AGENT_MARKETPLACE_CATEGORIES) - def _clean_ids(raw: Optional[List[str]], field: str) -> Tuple[Optional[List[str]], Optional[str]]: """绑定 id 数组归一化：去空白/去重/保序/限量。返回 (清洗后, 错误信息)。""" if raw is None: @@ -96,7 +93,6 @@ def _clean_ids(raw: Optional[List[str]], field: str) -> Tuple[Optional[List[str] return None, f"❌ {field} 一次最多绑 {_MAX_BINDINGS} 项（收到 {len(out)} 项）。" return out, None - def _binding_updates( raw: Dict[str, Optional[List[str]]], *, fill_missing: bool ) -> Tuple[Dict[str, List[str]], Optional[str]]: @@ -115,7 +111,6 @@ def _binding_updates( data[field] = cleaned or [] return data, None - def _clean_max_iters(value: Optional[int]) -> Tuple[Optional[int], Optional[str]]: """max_iters 校验。None 表示"没传"，返回 (None, None) 让调用方跳过。""" if value is None: @@ -128,7 +123,6 @@ def _clean_max_iters(value: Optional[int]) -> Tuple[Optional[int], Optional[str] return None, "❌ max_iters 取值范围 1–100。" return mi, None - def _slim(a: Dict[str, Any]) -> Dict[str, Any]: """把 service 的完整序列化裁成智能体读得动的摘要（别把 change_history 等整包回吐）。""" return { @@ -148,7 +142,6 @@ def _slim(a: Dict[str, Any]) -> Dict[str, Any]: "updated_at": a.get("updated_at"), } - def _my_agents(db, user_id: str) -> List[Dict[str, Any]]: """只取"我自己建的/装的"子智能体。 @@ -164,7 +157,6 @@ def _my_agents(db, user_id: str) -> List[Dict[str, Any]]: and not str(a.get("agent_id") or "").startswith(_BUILTIN_PREFIX) ] - def _resolve_agent(db, user_id: str, ref: str) -> Tuple[Optional[Dict[str, Any]], List[Dict[str, Any]]]: """agent_ref → (唯一命中, 候选)。先精确 agent_id，再按名称模糊匹配。仅本人的子智能体。""" ref = (ref or "").strip() @@ -178,7 +170,6 @@ def _resolve_agent(db, user_id: str, ref: str) -> Tuple[Optional[Dict[str, Any]] return cands[0], [] return None, cands[:10] - def _need_clarify(cands: List[Dict[str, Any]]) -> Dict[str, Any]: return { "ok": False, @@ -187,23 +178,22 @@ def _need_clarify(cands: List[Dict[str, Any]]) -> Dict[str, Any]: "candidates": [{"agent_id": c.get("agent_id"), "name": c.get("name")} for c in cands], } - # ── ① 搜索子智能体市场（只读）────────────────────────────────────────── def search_agent_market(*, user_id: str, query: str = "", category: str = "") -> Dict[str, Any]: if not user_id: return _no_user() from core.db.engine import SessionLocal - from core.services import agent_market_service + q = (query or "").strip().lower() cat = (category or "").strip() with SessionLocal() as db: # 与用户端市场接口同一套可见范围过滤（scoped 条目仅授权者可见） - items = agent_market_service.list_marketplace_agents(db, viewer_user_id=user_id) + items = agent_catalog.list_marketplace_agents(db, viewer_user_id=user_id) # list_marketplace_agents 只标 market_enabled/visibility，不标 installed——不补这一下， # 每条都会显示"未安装"，模型就会去重装用户已经有的智能体（一次代价高昂的克隆）。 # 插件侧的 list_plugins 自带该标注，两边口径要一致。 - agent_market_service.annotate_installed(items, db, user_id) + agent_installation.annotate_installed(items, db, user_id) def _match(it: Dict[str, Any]) -> bool: if cat and str(it.get("category") or "") != cat: @@ -235,7 +225,6 @@ def _match(it: Dict[str, Any]) -> bool: ) return {"ok": True, "count": len(slim), "agents": slim, "message": msg} - # ── ② 从市场安装（私有，写）──────────────────────────────────────────── def install_market_agent(*, user_id: str, slug: str) -> Dict[str, Any]: if not user_id: @@ -244,7 +233,7 @@ def install_market_agent(*, user_id: str, slug: str) -> Dict[str, Any]: if not slug: return {"ok": False, "message": "❌ 请提供要安装的子智能体 slug。"} from core.db.engine import SessionLocal - from core.services import agent_market_service + with SessionLocal() as db: cap_err = _require_cap(db, user_id, "can_add_agent") @@ -252,13 +241,13 @@ def install_market_agent(*, user_id: str, slug: str) -> Dict[str, Any]: return cap_err # 可见范围守卫：对该用户不可见的 scoped 条目按不存在处理（与用户端安装接口一致）。 # 不要包 try——守卫出错就该报错，静默跳过等于对不可见条目失败开放。 - from core.auth.marketplace_visibility import is_item_visible - from core.services import marketplace_listing as ml + from features.marketplace.edition import is_item_visible + from features.marketplace import edition as ml if not is_item_visible(db, ml.KIND_AGENT, slug, user_id): return {"ok": False, "message": f"❌ 子智能体市场里找不到「{slug}」。"} try: - res = agent_market_service.install_marketplace_agent( + res = agent_installation.install_marketplace_agent( db, slug, owner_user_id=user_id, operator_name=user_id ) except Exception as exc: # noqa: BLE001 @@ -285,7 +274,6 @@ def install_market_agent(*, user_id: str, slug: str) -> Dict[str, Any]: ), } - # ── ③ 列出可绑能力（只读，创建/修改前必调）────────────────────────────── def list_bindable_capabilities(*, user_id: str) -> Dict[str, Any]: if not user_id: @@ -329,7 +317,6 @@ def _pick(items: Any) -> List[Dict[str, Any]]: ), } - # ── ④ 从零创建（写）──────────────────────────────────────────────────── def create_agent( *, @@ -412,7 +399,6 @@ def create_agent( ), } - def _friendly_error(action: str, exc: Exception) -> str: """把 service 层异常转成人话。本体校验失败要说清楚是本体没过，而不是甩一句失败。""" text = str(exc) or exc.__class__.__name__ @@ -425,7 +411,6 @@ def _friendly_error(action: str, exc: Exception) -> str: return f"❌ {action}失败：找不到该子智能体（可能已被删除）。" return f"❌ {action}失败：{text}" - # ── ⑤ 列出我的子智能体（只读）────────────────────────────────────────── def list_my_agents(*, user_id: str) -> Dict[str, Any]: if not user_id: @@ -442,7 +427,6 @@ def list_my_agents(*, user_id: str) -> Dict[str, Any]: ) return {"ok": True, "count": len(agents), "agents": agents, "message": msg} - # ── ⑥ 修改（字段级部分更新，写）──────────────────────────────────────── def edit_agent( *, @@ -544,7 +528,6 @@ def edit_agent( ), } - # ── ⑦ 删除（写）──────────────────────────────────────────────────────── def delete_agent(*, user_id: str, agent_ref: str) -> Dict[str, Any]: if not user_id: @@ -576,7 +559,6 @@ def delete_agent(*, user_id: str, agent_ref: str) -> Dict[str, Any]: "message": f"✅ 已删除子智能体「{display}」（agent_id={deleted_id}）。", } - # ── ⑧ 申请上架市场（写）──────────────────────────────────────────────── def submit_agent_to_market( *, user_id: str, agent_id: str, category: str = "", summary: str = "", note: str = "" @@ -587,7 +569,7 @@ def submit_agent_to_market( if not agent_id: return {"ok": False, "message": "❌ 请提供要上架的 agent_id（取自 list_my_agents）。"} from core.db.engine import SessionLocal - from core.services import agent_market_service + with SessionLocal() as db: cap_err = _require_cap(db, user_id, "can_add_agent") @@ -612,7 +594,7 @@ def submit_agent_to_market( ), } try: - sub = agent_market_service.submit_to_marketplace( + sub = agent_submissions.submit_to_marketplace( db, str(hit.get("agent_id")), owner_user_id=user_id, diff --git a/src/backend/extensions/mcp_servers/agent_manager_mcp/server.py b/src/backend/extensions/mcp_servers/agent_manager_mcp/server.py index dbf93650..a77565ba 100644 --- a/src/backend/extensions/mcp_servers/agent_manager_mcp/server.py +++ b/src/backend/extensions/mcp_servers/agent_manager_mcp/server.py @@ -13,11 +13,11 @@ from typing import Any, Dict, List, Optional -from mcp.server.fastmcp import Context, FastMCP +from mcp.server.mcpserver import Context, MCPServer from extensions.mcp_servers.agent_manager_mcp import impl -mcp = FastMCP("hugagent-agent-manager") +mcp = MCPServer("hugagent-agent-manager") _HDR_USER = "x-current-user-id" diff --git a/src/backend/extensions/mcp_servers/automation_task_mcp/impl.py b/src/backend/extensions/mcp_servers/automation_task_mcp/impl.py index 942c315f..6d7eb789 100644 --- a/src/backend/extensions/mcp_servers/automation_task_mcp/impl.py +++ b/src/backend/extensions/mcp_servers/automation_task_mcp/impl.py @@ -69,7 +69,7 @@ def _commit_effect_receipt( def _svc(db): - from core.services.automation_service import AutomationService + from features.automation.service import AutomationService return AutomationService(db) @@ -88,7 +88,7 @@ def _resolve_ref(db, user_id: str, ref: str) -> Tuple[Optional[Any], List[Any]]: 返回 (task, candidates)：命中唯一 → (task, [])；多命中 → (None, candidates)；无 → (None, [])。 """ - from core.db.models import ScheduledTask + from features.automation.models import ScheduledTask ref = (ref or "").strip() if not ref: @@ -113,8 +113,8 @@ def _resolve_ref(db, user_id: str, ref: str) -> Tuple[Optional[Any], List[Any]]: def _task_brief(task) -> Dict[str, Any]: - from core.services.automation_service import AutomationService - from core.services.delivery_targets import resolve_delivery_targets, describe_targets + from features.automation.service import AutomationService + from features.automation.delivery_targets import resolve_delivery_targets, describe_targets d = AutomationService.task_to_dict(task) targets = resolve_delivery_targets(task.extra_data) @@ -157,7 +157,7 @@ def list_conversations(*, user_id: str) -> Dict[str, Any]: if not user_id: return {"ok": False, "message": "❌ 无法确定用户身份。"} from core.db.engine import SessionLocal - from core.services.channel_service import list_owner_conversations + from integrations.channels.service import list_owner_conversations with SessionLocal() as db: convs = list_owner_conversations(db, user_id, limit=100) @@ -179,7 +179,7 @@ def _resolve_create_targets( deliver_to：空 → 按来源自动；"inapp" → 仅站内；否则当作 conversation_id（须本人拥有的渠道会话）。 """ - from core.services.delivery_targets import build_delivery_targets + from features.automation.delivery_targets import build_delivery_targets val = (deliver_to or "").strip() if not val: @@ -231,7 +231,7 @@ def create_task( return {"ok": False, "message": f"❌ cron 表达式无效：{cron_expression}"} from core.db.engine import SessionLocal - from core.services.delivery_targets import describe_targets + from features.automation.delivery_targets import describe_targets with SessionLocal() as db: try: @@ -250,7 +250,7 @@ def create_task( targets, err = _resolve_create_targets(user_id, deliver_to, channel_origin, db) if err: return {"ok": False, "message": err} - from core.services.automation_execution import execution_metadata + from features.automation.execution import execution_metadata import pytz try: pytz.timezone(timezone) @@ -307,7 +307,7 @@ def list_tasks(*, user_id: str, status: str = "active") -> Dict[str, Any]: def get_task(*, user_id: str, task_ref: str) -> Dict[str, Any]: from core.db.engine import SessionLocal - from core.services.automation_service import AutomationService, SUMMARY_LIMIT_BRIEF + from features.automation.service import AutomationService, SUMMARY_LIMIT_BRIEF with SessionLocal() as db: task, cands = _resolve_ref(db, user_id, task_ref) diff --git a/src/backend/extensions/mcp_servers/automation_task_mcp/server.py b/src/backend/extensions/mcp_servers/automation_task_mcp/server.py index a97429ba..0b4696f7 100644 --- a/src/backend/extensions/mcp_servers/automation_task_mcp/server.py +++ b/src/backend/extensions/mcp_servers/automation_task_mcp/server.py @@ -13,11 +13,11 @@ from typing import Any, Dict, Optional -from mcp.server.fastmcp import Context, FastMCP +from mcp.server.mcpserver import Context, MCPServer from extensions.mcp_servers.automation_task_mcp import impl -mcp = FastMCP("hugagent-automation-task") +mcp = MCPServer("hugagent-automation-task") _HDR_USER = "x-current-user-id" _HDR_CHANNEL = "x-channel-id" @@ -115,7 +115,7 @@ async def list_scheduled_tasks( 用户问"我有哪些定时任务/查看我的定时任务/有哪些计划任务"时调用。 参数 status："active"(默认,仅生效中) / "paused" / "all"(全部)。 """ - from core.services.automation_execution import instance_location + from features.automation.execution import instance_location if execution_location and execution_location != instance_location(): return {"ok": False, "message": "执行位置不匹配，请选择对应的本机或云端任务"} return impl.list_tasks(user_id=_user(ctx), status=status) @@ -131,7 +131,7 @@ async def get_scheduled_task( task_ref 可传 task_id 或任务名称（名称模糊匹配；匹配到多个会返回候选让你向用户确认）。 """ - from core.services.automation_execution import instance_location + from features.automation.execution import instance_location if execution_location and execution_location != instance_location(): return {"ok": False, "message": "执行位置不匹配，请选择对应的本机或云端任务"} return impl.get_task(user_id=_user(ctx), task_ref=task_ref) @@ -153,7 +153,7 @@ async def update_scheduled_task( task_ref 传 task_id 或名称（多命中会要求澄清）。 tool_effect_id 由 ToolGateway 内部注入，模型不要填写。 """ - from core.services.automation_execution import instance_location + from features.automation.execution import instance_location if execution_location and execution_location != instance_location(): return {"ok": False, "message": "执行位置不匹配，请选择对应的本机或云端任务"} return impl.update_task( @@ -169,7 +169,7 @@ async def update_scheduled_task( @mcp.tool() async def pause_scheduled_task(task_ref: str, execution_location: str = "", ctx: Context | None = None) -> Dict[str, Any]: """暂停一个定时任务（暂停后到点不再触发，可随后恢复）。task_ref=task_id 或名称。""" - from core.services.automation_execution import instance_location + from features.automation.execution import instance_location if execution_location and execution_location != instance_location(): return {"ok": False, "message": "执行位置不匹配，请选择对应的本机或云端任务"} return impl.pause_task(user_id=_user(ctx), task_ref=task_ref) @@ -178,7 +178,7 @@ async def pause_scheduled_task(task_ref: str, execution_location: str = "", ctx: @mcp.tool() async def resume_scheduled_task(task_ref: str, execution_location: str = "", ctx: Context | None = None) -> Dict[str, Any]: """恢复一个被暂停的定时任务。task_ref=task_id 或名称。""" - from core.services.automation_execution import instance_location + from features.automation.execution import instance_location if execution_location and execution_location != instance_location(): return {"ok": False, "message": "执行位置不匹配，请选择对应的本机或云端任务"} return impl.resume_task(user_id=_user(ctx), task_ref=task_ref) @@ -196,7 +196,7 @@ async def delete_scheduled_task( 【铁律】未成功调用本工具拿到 ✅ 前不要声称已删除。匹配到多个任务时必须先向用户确认，禁止猜删。 tool_effect_id 由 ToolGateway 内部注入，模型不要填写。 """ - from core.services.automation_execution import instance_location + from features.automation.execution import instance_location if execution_location and execution_location != instance_location(): return {"ok": False, "message": "执行位置不匹配，请选择对应的本机或云端任务"} return impl.delete_task( diff --git a/src/backend/extensions/mcp_servers/browser_runtime_mcp/server.py b/src/backend/extensions/mcp_servers/browser_runtime_mcp/server.py index a78de5d7..bd1f4b8a 100644 --- a/src/backend/extensions/mcp_servers/browser_runtime_mcp/server.py +++ b/src/backend/extensions/mcp_servers/browser_runtime_mcp/server.py @@ -3,18 +3,18 @@ import uuid import base64 from core.infra.http_errors import response_error -from mcp.server.fastmcp.utilities.types import Image +from mcp.server.mcpserver.utilities.types import Image from typing import Any import httpx -from mcp.server.fastmcp import Context, FastMCP +from mcp.server.mcpserver import Context, MCPServer from extensions.mcp_servers._ports import PORTS from extensions.mcp_servers._serve import run -mcp = FastMCP("browser-automation") +mcp = MCPServer("browser-automation") async def call(ctx, action, resource_id=None, params=None, checkpoint_id=None): headers = ctx.request_context.request.headers - from core.llm.mcp_invocation import verify + from core.llm.mcp.invocation import verify claims = verify(headers, "browser_runtime") plugin_id = claims.get("plugin", "") slug = plugin_id.split(":", 2)[2] if plugin_id else "browser-automation" diff --git a/src/backend/extensions/mcp_servers/internet_search_mcp/_selftest.py b/src/backend/extensions/mcp_servers/internet_search_mcp/_selftest.py index f762d106..592c0d07 100755 --- a/src/backend/extensions/mcp_servers/internet_search_mcp/_selftest.py +++ b/src/backend/extensions/mcp_servers/internet_search_mcp/_selftest.py @@ -7,7 +7,7 @@ async def check(): tools = await mcp.list_tools() - schema = tools[0].inputSchema + schema = tools[0].input_schema assert "queries" in schema["properties"] assert "cn_only" not in schema["properties"] assert schema["properties"]["max_results"]["maximum"] == 8 diff --git a/src/backend/extensions/mcp_servers/internet_search_mcp/impl.py b/src/backend/extensions/mcp_servers/internet_search_mcp/impl.py index 1d0b5a22..3229df15 100755 --- a/src/backend/extensions/mcp_servers/internet_search_mcp/impl.py +++ b/src/backend/extensions/mcp_servers/internet_search_mcp/impl.py @@ -11,7 +11,9 @@ import anyio import httpx -from core.config.runtime_env import get_runtime_value +from orchestration.configuration import register_configuration +register_configuration() +from core.config.store.runtime_env import get_runtime_value from .merge import merge_sources from .models import SearchRequest diff --git a/src/backend/extensions/mcp_servers/internet_search_mcp/server.py b/src/backend/extensions/mcp_servers/internet_search_mcp/server.py index 8e800893..055be6ef 100755 --- a/src/backend/extensions/mcp_servers/internet_search_mcp/server.py +++ b/src/backend/extensions/mcp_servers/internet_search_mcp/server.py @@ -5,44 +5,31 @@ import anyio import httpx -from mcp.server.fastmcp import Context, FastMCP +from mcp.server.mcpserver import Context, MCPServer from pydantic import Field from .impl import MAX_INFLIGHT, SearchService from .models import Depth, Query, ResultLimit, SearchRequest, Topic -# FastMCP enters lifespan per MCP session. Share one service across overlapping -# sessions on this server's event loop; the last session closes its client. -_service: SearchService | None = None -_sessions = 0 - - @asynccontextmanager -async def lifespan(server: FastMCP): - global _service, _sessions - if _service is None: - _service = SearchService( - httpx.AsyncClient(limits=httpx.Limits(max_connections=MAX_INFLIGHT)) - ) - service = _service - _sessions += 1 +async def lifespan(server: MCPServer): + service = SearchService( + httpx.AsyncClient(limits=httpx.Limits(max_connections=MAX_INFLIGHT)) + ) try: yield service finally: - _sessions -= 1 - if _sessions == 0: - _service = None - with anyio.CancelScope(shield=True): - await service.client.aclose() + with anyio.move_on_after(5, shield=True): + await service.client.aclose() -class SearchMCP(FastMCP): - """Validate the raw MCP boundary before FastMCP drops unknown arguments.""" +class SearchMCP(MCPServer): + """Validate the raw MCP boundary before MCPServer drops unknown arguments.""" - async def call_tool(self, name: str, arguments: dict[str, Any]): + async def call_tool(self, name: str, arguments: dict[str, Any], context=None): if name == "internet_search": SearchRequest.model_validate(arguments) - return await super().call_tool(name, arguments) + return await super().call_tool(name, arguments, context) mcp = SearchMCP("hugagent-internet-search", lifespan=lifespan) diff --git a/src/backend/extensions/mcp_servers/management_registration.py b/src/backend/extensions/mcp_servers/management_registration.py index 8e40b137..a1a01742 100644 --- a/src/backend/extensions/mcp_servers/management_registration.py +++ b/src/backend/extensions/mcp_servers/management_registration.py @@ -2,10 +2,10 @@ import asyncio from typing import Literal from pydantic import BaseModel, ConfigDict -from mcp.server.fastmcp import Context +from mcp.server.mcpserver import Context from mcp.types import ToolAnnotations -from core.services.management_contract import MANAGERS, tools -from core.services.management_errors import management_error +from core.capabilities.plugins.management.contract import MANAGERS, tools +from core.capabilities.plugins.management.errors import management_error class ArtifactSource(BaseModel): @@ -18,7 +18,7 @@ def register(mcp, manager, user_from_context): kind = MANAGERS[manager] async def call(action, ctx, **arguments): - from core.services import cloud_management as service + from core.capabilities.plugins.management import cloud as service try: return await asyncio.to_thread(getattr(service, action), user_from_context(ctx), kind, **arguments) except Exception as exc: diff --git a/src/backend/extensions/mcp_servers/plugin_manager_mcp/impl.py b/src/backend/extensions/mcp_servers/plugin_manager_mcp/impl.py index de4c1b45..1f6d66ac 100644 --- a/src/backend/extensions/mcp_servers/plugin_manager_mcp/impl.py +++ b/src/backend/extensions/mcp_servers/plugin_manager_mcp/impl.py @@ -1,6 +1,7 @@ """Cloud marketplace operations for plugin-manager; installation lifecycle lives in cloud_management.""" from __future__ import annotations +from features.marketplace import plugin_market import logging from typing import Any, Dict, List, Optional, Tuple @@ -16,7 +17,7 @@ def _invalidate_user_cache(user_id: Optional[str]) -> None: 见 agent_manager_mcp/impl.py 同名函数的说明——两个 MCP 面对的是同一个跨进程缓存问题。 """ try: - from core.config.catalog_resolver import invalidate_capability_cache + from core.capabilities.catalog.resolver import invalidate_capability_cache invalidate_capability_cache(str(user_id) if user_id else None) except Exception as exc: # noqa: BLE001 @@ -63,13 +64,13 @@ def search_plugin_market(*, user_id: str, query: str = "", category: str = "") - if not user_id: return _no_user() from core.db.engine import SessionLocal - from core.plugins import management as plugin_service + from core.capabilities.plugins import management as plugin_service q = (query or "").strip().lower() cat = (category or "").strip() with SessionLocal() as db: # include_disabled=False：用户侧只看得到已发布的条目，与前端插件市场同口径。 - items = plugin_service.list_plugins(db, user_id, include_disabled=False) or [] + items = plugin_market.list_plugins(db, user_id, include_disabled=False) or [] def _match(it: Dict[str, Any]) -> bool: if cat and str(it.get("category") or "") != cat: @@ -109,7 +110,7 @@ def get_plugin_info(*, user_id: str, slug: str) -> Dict[str, Any]: if not slug: return {"ok": False, "message": "❌ 请提供插件 slug。"} from core.db.engine import SessionLocal - from core.plugins import management as plugin_service + from core.capabilities.plugins import management as plugin_service with SessionLocal() as db: detail = None @@ -168,7 +169,7 @@ def install_plugin( if not slug: return {"ok": False, "message": "❌ 请提供要安装的插件 slug。"} from core.db.engine import SessionLocal - from core.plugins import management as plugin_service + from core.capabilities.plugins import management as plugin_service with SessionLocal() as db: cap_err = _require_cap(db, user_id, "can_import_plugin") diff --git a/src/backend/extensions/mcp_servers/plugin_manager_mcp/server.py b/src/backend/extensions/mcp_servers/plugin_manager_mcp/server.py index 23bbd74c..766ffe3c 100644 --- a/src/backend/extensions/mcp_servers/plugin_manager_mcp/server.py +++ b/src/backend/extensions/mcp_servers/plugin_manager_mcp/server.py @@ -11,11 +11,11 @@ from typing import Any, Dict, Optional -from mcp.server.fastmcp import Context, FastMCP +from mcp.server.mcpserver import Context, MCPServer from extensions.mcp_servers.plugin_manager_mcp import impl -mcp = FastMCP("hugagent-plugin-manager") +mcp = MCPServer("hugagent-plugin-manager") _HDR_USER = "x-current-user-id" diff --git a/src/backend/extensions/mcp_servers/retrieve_dataset_content_mcp/impl.py b/src/backend/extensions/mcp_servers/retrieve_dataset_content_mcp/impl.py index b46bbdd7..88747295 100644 --- a/src/backend/extensions/mcp_servers/retrieve_dataset_content_mcp/impl.py +++ b/src/backend/extensions/mcp_servers/retrieve_dataset_content_mcp/impl.py @@ -6,18 +6,8 @@ import os from typing import Any, Dict, List -from core.auth.kb_permissions import get_accessible_local_kb_ids, is_shared_visibility -from core.kb.external_retrieval import ( - MAX_RETRIEVE_TOKENS, - RETRIEVE_MAX_CONCURRENCY, - RETRIEVE_REQUEST_TIMEOUT_SECONDS, - RETRIEVE_TOTAL_TIMEOUT_SECONDS, - DatasetRetrievalTimeoutError, - DatasetRetrievalUnavailableError, - list_external_datasets, - retrieve_dataset_content, - retrieve_dataset_content_async, -) +from features.knowledge.edition import get_accessible_local_kb_ids, is_shared_visibility +from features.knowledge.edition import MAX_RETRIEVE_TOKENS, RETRIEVE_MAX_CONCURRENCY, RETRIEVE_REQUEST_TIMEOUT_SECONDS, RETRIEVE_TOTAL_TIMEOUT_SECONDS, DatasetRetrievalTimeoutError, DatasetRetrievalUnavailableError, list_external_datasets, retrieve_dataset_content, retrieve_dataset_content_async from extensions.mcp_servers.retrieve_dataset_content_mcp.local_impl import ( LOCAL_RETRIEVE_STAGE_TIMEOUT_SECONDS, LOCAL_RETRIEVE_TOTAL_TIMEOUT_SECONDS, @@ -36,7 +26,7 @@ def _list_local_datasets( private_items: List[Dict[str, Any]] = [] try: from core.db.engine import SessionLocal - from core.db.models import KBDocument, KBSpace + from features.knowledge.models import KBDocument, KBSpace allowed = {item.strip() for item in (allowed_kb_ids or "").split(",") if item.strip()} user_id = (current_user_id or "").strip() or os.getenv("CURRENT_USER_ID", "").strip() @@ -79,7 +69,7 @@ def list_all_datasets( allowed_kb_ids: str | None = None, current_user_id: str | None = None, ) -> Dict[str, Any]: - from core.kb.external_provider import is_enabled + from features.knowledge.edition import is_enabled local_shared, private_items = _list_local_datasets( allowed_kb_ids=allowed_kb_ids, current_user_id=current_user_id diff --git a/src/backend/extensions/mcp_servers/retrieve_dataset_content_mcp/local_impl.py b/src/backend/extensions/mcp_servers/retrieve_dataset_content_mcp/local_impl.py index d37ecff3..5c921487 100644 --- a/src/backend/extensions/mcp_servers/retrieve_dataset_content_mcp/local_impl.py +++ b/src/backend/extensions/mcp_servers/retrieve_dataset_content_mcp/local_impl.py @@ -8,8 +8,10 @@ import time from typing import Any -from core.auth.kb_permissions import get_accessible_local_kb_ids, is_shared_visibility -from core.config.runtime_env import get_runtime_value +from features.knowledge.edition import get_accessible_local_kb_ids, is_shared_visibility +from orchestration.configuration import register_configuration +register_configuration() +from core.config.store.runtime_env import get_runtime_value def _read_int_env(name: str, default: int) -> int: @@ -60,7 +62,7 @@ def _fetch_parent_contents(parent_ids: list[str]) -> dict[str, str]: return {} try: from core.db.engine import SessionLocal - from core.db.models import KBChunk + from features.knowledge.models import KBChunk db = SessionLocal() try: @@ -91,7 +93,7 @@ def _build_runtime_local_kb_section() -> str: kb_names: dict[str, str] = {} try: from core.db.engine import SessionLocal - from core.db.models import KBSpace + from features.knowledge.models import KBSpace db = SessionLocal() try: @@ -156,7 +158,7 @@ def _remaining_stage_timeout() -> float: if not allowed: try: from core.db.engine import SessionLocal - from core.db.models import KBSpace + from features.knowledge.models import KBSpace with SessionLocal() as _db: if user_id: @@ -197,7 +199,7 @@ def _remaining_stage_timeout() -> float: private_ids: list[str] = list(search_kb_ids) try: from core.db.engine import SessionLocal - from core.db.models import KBSpace + from features.knowledge.models import KBSpace with SessionLocal() as _db: rows = ( @@ -223,7 +225,8 @@ def _remaining_stage_timeout() -> float: # ── Embed query ────────────────────────────────────────────────────────── try: - from core.kb.kb_vector import embed_text, hybrid_search + from core.models.embedding import embed_text + from features.knowledge.kb_vector import hybrid_search query_vec = embed_text(query, timeout=_remaining_stage_timeout()) except LocalKnowledgeBaseTimeoutError: @@ -253,7 +256,7 @@ def _remaining_stage_timeout() -> float: kb_meta: list[dict[str, str]] = [] try: from core.db.engine import SessionLocal - from core.db.models import KBSpace + from features.knowledge.models import KBSpace with SessionLocal() as _db: spaces = ( @@ -289,7 +292,7 @@ def _remaining_stage_timeout() -> float: ) if (_reranker_flag or "").lower() in ("true", "1"): try: - from core.kb.kb_vector import is_reranker_configured, rerank + from core.models.reranking import is_reranker_configured, rerank if is_reranker_configured() and top_hits: contents = [hit.get("content", "") for hit in top_hits] @@ -390,7 +393,7 @@ def _attach_assets(results: list) -> None: if not results: return try: - from core.kb.kb_assets import fetch_assets_for_chunks + from features.knowledge.kb_assets import fetch_assets_for_chunks grouped = fetch_assets_for_chunks([r.get("id", "") for r in results]) except Exception as exc: @@ -413,7 +416,7 @@ def retrieve_public_local_kb( ) -> dict: """Search only accessible shared local spaces through the public tool.""" from core.db.engine import SessionLocal - from core.db.models import KBSpace + from features.knowledge.models import KBSpace user_id = (current_user_id or _get_current_user_id()).strip() if not user_id: diff --git a/src/backend/extensions/mcp_servers/retrieve_dataset_content_mcp/server.py b/src/backend/extensions/mcp_servers/retrieve_dataset_content_mcp/server.py index 23051db8..a2a1cc12 100755 --- a/src/backend/extensions/mcp_servers/retrieve_dataset_content_mcp/server.py +++ b/src/backend/extensions/mcp_servers/retrieve_dataset_content_mcp/server.py @@ -13,7 +13,7 @@ import logging from typing import Any, Dict, Optional -from mcp.server.fastmcp import Context, FastMCP +from mcp.server.mcpserver import Context, MCPServer from .descriptions import ( _BASE_LOCAL_KB_TOOL_DESCRIPTION, @@ -32,7 +32,19 @@ tool_timeout_payload, ) -mcp = FastMCP("hugagent-retrieve-dataset-content") +class RetrievalMCP(MCPServer): + async def list_tools(self): + tools = await super().list_tools() + try: + from extensions.mcp_servers.retrieve_dataset_content_mcp.wiki_impl import wiki_supported + if wiki_supported(): + return tools + except Exception as exc: + _LOGGER.warning("wiki capability probe failed, hiding wiki tools: %s", exc) + return [tool for tool in tools if tool.name not in _WIKI_TOOL_NAMES] + + +mcp = RetrievalMCP("hugagent-retrieve-dataset-content") _LOGGER = logging.getLogger(__name__) @@ -97,7 +109,7 @@ async def retrieve_dataset_content( current_user_id = _get_header(ctx, _HDR_CURRENT_USER_ID) try: - from core.kb.external_provider import is_enabled + from features.knowledge.edition import is_enabled from extensions.mcp_servers.retrieve_dataset_content_mcp.local_impl import retrieve_public_local_kb if not await asyncio.to_thread(is_enabled): @@ -457,35 +469,6 @@ async def wiki_fetch_source( return await _run_wiki("wiki_fetch_source", call, empty={"items": []}) -# ── 按后端动态暴露工具 ─────────────────────────────────────────────────────── -# -# FastMCP 在 __init__ 里就把 self.list_tools 绑进了 lowlevel handler，所以事后 -# 改 mcp.list_tools 不生效——必须对 lowlevel server 重新注册一次（装饰器是 -# 覆盖写 request_handlers，不是追加）。 - - -_ORIGINAL_LIST_TOOLS = mcp.list_tools - - -async def _list_tools_filtered(): - """平台没有 Wiki 源时隐藏 Wiki 工具；公有检索按配置选择后端。 - - 每次 list_tools 都重新判定，所以切换知识库后端、或新建了一个开 Wiki 的自建库 - 之后，无需重启 mcp 容器。 - """ - tools = await _ORIGINAL_LIST_TOOLS() - try: - from extensions.mcp_servers.retrieve_dataset_content_mcp.wiki_impl import wiki_supported - - if wiki_supported(): - return tools - except Exception as exc: - _LOGGER.warning("wiki capability probe failed, hiding wiki tools: %s", exc) - return [tool for tool in tools if tool.name not in _WIKI_TOOL_NAMES] - - -mcp._mcp_server.list_tools()(_list_tools_filtered) - def main() -> None: from extensions.mcp_servers import _serve diff --git a/src/backend/extensions/mcp_servers/retrieve_dataset_content_mcp/wiki_impl.py b/src/backend/extensions/mcp_servers/retrieve_dataset_content_mcp/wiki_impl.py index 84b2c8a6..88d224f3 100644 --- a/src/backend/extensions/mcp_servers/retrieve_dataset_content_mcp/wiki_impl.py +++ b/src/backend/extensions/mcp_servers/retrieve_dataset_content_mcp/wiki_impl.py @@ -54,7 +54,7 @@ def _accessible_local_kb_ids(current_user_id: Optional[str]) -> List[str]: if not user_id: return [] try: - from core.auth.kb_permissions import get_accessible_local_kb_ids + from features.knowledge.edition import get_accessible_local_kb_ids from core.db.engine import SessionLocal with SessionLocal() as db: @@ -68,7 +68,7 @@ def _assert_access( kb_id: str, *, allowed_kb_ids: Optional[str], current_user_id: Optional[str] ) -> None: """自建库要过权限；外接库的白名单由其实现模块把关。""" - from core.kb.wiki_router import is_local_kb + from features.knowledge.wiki_router import is_local_kb if not is_local_kb(kb_id): return @@ -89,7 +89,7 @@ def _wiki_module( 权限校验和模块解析绑在一起，是为了让「先鉴权再取数」成为唯一写法——分开写 迟早会有某个工具漏掉那一行。 """ - from core.kb.wiki_router import wiki_module_for + from features.knowledge.wiki_router import wiki_module_for _assert_access(kb_id, allowed_kb_ids=allowed_kb_ids, current_user_id=current_user_id) module = wiki_module_for(kb_id) @@ -101,7 +101,7 @@ def _wiki_module( def wiki_supported() -> bool: """平台上是否存在任何 Wiki 源（决定工具是否暴露在清单里）。""" try: - from core.kb.wiki_router import supports_wiki + from features.knowledge.wiki_router import supports_wiki return bool(supports_wiki()) except Exception: @@ -125,7 +125,7 @@ def resolve_dataset_id( if explicit: return explicit - from core.kb.wiki_router import wiki_capable_local_kb_ids + from features.knowledge.wiki_router import wiki_capable_local_kb_ids local_candidates = _split_ids(allowed_kb_ids) or _accessible_local_kb_ids(current_user_id) wiki_locals = wiki_capable_local_kb_ids(local_candidates) @@ -137,7 +137,7 @@ def resolve_dataset_id( return external[0] try: - from core.kb.external_provider import list_collections + from features.knowledge.edition import list_collections for item in list_collections() or []: caps = item.get("capabilities") or {} diff --git a/src/backend/extensions/mcp_servers/site_publish_mcp/server.py b/src/backend/extensions/mcp_servers/site_publish_mcp/server.py index fbd11d5d..dc2e7468 100644 --- a/src/backend/extensions/mcp_servers/site_publish_mcp/server.py +++ b/src/backend/extensions/mcp_servers/site_publish_mcp/server.py @@ -5,13 +5,13 @@ from typing import Any, Dict, Literal, Optional -from core.services.application_schema import ApplicationPayload, ToolDefinition +from features.hosted_apps.schema import ApplicationPayload, ToolDefinition from pydantic import BaseModel -from mcp.server.fastmcp import Context, FastMCP +from mcp.server.mcpserver import Context, MCPServer from extensions.mcp_servers.site_publish_mcp import impl -mcp = FastMCP("hugagent-site-publish") +mcp = MCPServer("hugagent-site-publish") _HDR_USER = "x-current-user-id" _HDR_CHAT = "x-chat-id" diff --git a/src/backend/extensions/mcp_servers/skill_manager_mcp/impl.py b/src/backend/extensions/mcp_servers/skill_manager_mcp/impl.py index d29a84cc..e1f2ce53 100644 --- a/src/backend/extensions/mcp_servers/skill_manager_mcp/impl.py +++ b/src/backend/extensions/mcp_servers/skill_manager_mcp/impl.py @@ -1,6 +1,9 @@ """Cloud marketplace operations for skill-manager; installation lifecycle lives in cloud_management.""" from __future__ import annotations +from features.marketplace import skill_catalog +from features.marketplace import skill_installation +from features.marketplace import skill_submissions import logging from typing import Any, Dict, List, Optional, Tuple @@ -18,7 +21,7 @@ def _invalidate_user_cache(user_id: Optional[str]) -> None: 重新拉 ``GET /v1/catalog``（跑在 backend 进程）顺带完成，见 api/routes/v1/catalog.py。 本调用作为进程内一致性的兜底保留，无害。""" try: - from core.config.catalog_resolver import invalidate_capability_cache + from core.capabilities.catalog.resolver import invalidate_capability_cache invalidate_capability_cache(str(user_id) if user_id else None) except Exception as exc: # noqa: BLE001 @@ -43,13 +46,13 @@ def search_marketplace(*, user_id: str, query: str = "", category: str = "") -> if not user_id: return _no_user() from core.db.engine import SessionLocal - from core.services import marketplace_service + q = (query or "").strip().lower() cat = (category or "").strip() with SessionLocal() as db: # 与用户端市场接口同一套可见范围过滤（scoped 条目仅授权者可见） - items = marketplace_service.list_marketplace_skills(db, viewer_user_id=user_id) + items = skill_catalog.list_marketplace_skills(db, viewer_user_id=user_id) def _match(it: Dict[str, Any]) -> bool: if cat and str(it.get("category") or "") != cat: @@ -93,19 +96,19 @@ def install_from_marketplace( if not slug: return {"ok": False, "message": "❌ 请提供要安装的技能 slug。"} from core.db.engine import SessionLocal - from core.services import marketplace_service + with SessionLocal() as db: cap_err = _require_cap(db, user_id, "can_add_skill") if cap_err: return cap_err # 可见范围守卫：对该用户不可见的 scoped 条目按不存在处理（与用户端安装接口一致） - from core.auth.marketplace_visibility import is_item_visible - from core.services import marketplace_listing as ml + from features.marketplace.edition import is_item_visible + from features.marketplace import edition as ml if not is_item_visible(db, ml.KIND_SKILL, slug, user_id): return {"ok": False, "message": f"❌ 技能市场里找不到「{slug}」。"} try: - res = marketplace_service.install_marketplace_skill( + res = skill_installation.install_marketplace_skill( db, slug, owner_user_id=user_id, secrets=secrets or {} ) except Exception as exc: # noqa: BLE001 @@ -133,14 +136,14 @@ def submit_to_marketplace( if not skill_id: return {"ok": False, "message": "❌ 请提供要上架的私有技能 skill_id（先用 list_skills 查看）。"} from core.db.engine import SessionLocal - from core.services import marketplace_service + with SessionLocal() as db: cap_err = _require_cap(db, user_id, "can_add_skill") if cap_err: return cap_err try: - res = marketplace_service.submit_to_marketplace( + res = skill_submissions.submit_to_marketplace( db, skill_id, owner_user_id=user_id, diff --git a/src/backend/extensions/mcp_servers/skill_manager_mcp/server.py b/src/backend/extensions/mcp_servers/skill_manager_mcp/server.py index 5d7c3e87..9f36e588 100644 --- a/src/backend/extensions/mcp_servers/skill_manager_mcp/server.py +++ b/src/backend/extensions/mcp_servers/skill_manager_mcp/server.py @@ -11,11 +11,11 @@ from typing import Any, Dict, Optional -from mcp.server.fastmcp import Context, FastMCP +from mcp.server.mcpserver import Context, MCPServer from extensions.mcp_servers.skill_manager_mcp import impl -mcp = FastMCP("hugagent-skill-manager") +mcp = MCPServer("hugagent-skill-manager") _HDR_USER = "x-current-user-id" diff --git a/src/backend/extensions/mcp_servers/space_files_mcp/permissions.py b/src/backend/extensions/mcp_servers/space_files_mcp/permissions.py index 3ab61e75..fbfd7c05 100644 --- a/src/backend/extensions/mcp_servers/space_files_mcp/permissions.py +++ b/src/backend/extensions/mcp_servers/space_files_mcp/permissions.py @@ -1,12 +1,6 @@ """Developer-owned execution policy for the space MCP contract.""" -from core.llm.tool_permissions import ( - DOMAIN_APPROVAL, - DOMAIN_DENY, - WRITE, - PermissionIntent, - ToolPermissionSpec, -) +from core.sandbox.permissions.dispatch_models import DOMAIN_APPROVAL, DOMAIN_DENY, WRITE, PermissionIntent, ToolPermissionSpec from .server import ARGUMENTS, READ_TOOL, WRITE_TOOL diff --git a/src/backend/extensions/mcp_servers/space_files_mcp/server.py b/src/backend/extensions/mcp_servers/space_files_mcp/server.py index 22fbc76c..9287d3c9 100644 --- a/src/backend/extensions/mcp_servers/space_files_mcp/server.py +++ b/src/backend/extensions/mcp_servers/space_files_mcp/server.py @@ -4,9 +4,9 @@ import json from typing import Literal -from core.services.desktop_capability_protocol import canonical_hash -from core.spaces.access import resolve_grants -from core.spaces.files import SpaceFiles +from core.capabilities.manifest_protocol import canonical_hash +from core.files.spaces.access import resolve_grants +from core.files.spaces.files import SpaceFiles from fastapi import HTTPException from pydantic import BaseModel, ConfigDict, Field, ValidationError @@ -71,7 +71,7 @@ def manifest(): def resolve(user, name, schema_hash): - from core.services.desktop_capability_protocol import CapabilityManifestStaleError + from core.capabilities.manifest_protocol import CapabilityManifestStaleError if not user or str(user).startswith("api_ak_") or name not in ARGUMENTS: return None @@ -87,7 +87,7 @@ def resolve(user, name, schema_hash): def execute(user, tool_name, arguments): from core.db.engine import SessionLocal - from core.storage import get_storage + from core.files.storage import get_storage try: args = ARGUMENTS[tool_name].model_validate(arguments) diff --git a/src/backend/extensions/mcp_servers/web_fetch_mcp/server.py b/src/backend/extensions/mcp_servers/web_fetch_mcp/server.py index 0671867d..5799a100 100644 --- a/src/backend/extensions/mcp_servers/web_fetch_mcp/server.py +++ b/src/backend/extensions/mcp_servers/web_fetch_mcp/server.py @@ -4,13 +4,13 @@ from typing import Annotated, Any, Literal import httpx -from mcp.server import FastMCP +from mcp.server.mcpserver import MCPServer from pydantic import Field from .impl import fetch_url logger = logging.getLogger(__name__) -mcp = FastMCP("hugagent-web-fetch") +mcp = MCPServer("hugagent-web-fetch") @mcp.tool() diff --git a/src/backend/extensions/plugins/marketplace/agent-manager/mcp.json b/src/backend/extensions/plugins/marketplace/agent-manager/mcp.json index 9c5680a7..bfcadab0 100644 --- a/src/backend/extensions/plugins/marketplace/agent-manager/mcp.json +++ b/src/backend/extensions/plugins/marketplace/agent-manager/mcp.json @@ -3,7 +3,7 @@ "mcpServers": { "agent_manager": { "type": "streamable-http", - "url": "http://mcp:9115/mcp/" + "url": "http://mcp:9115/mcp" } } } diff --git a/src/backend/extensions/plugins/marketplace/automation/mcp.json b/src/backend/extensions/plugins/marketplace/automation/mcp.json index 551c583c..73c1613b 100644 --- a/src/backend/extensions/plugins/marketplace/automation/mcp.json +++ b/src/backend/extensions/plugins/marketplace/automation/mcp.json @@ -3,7 +3,7 @@ "mcpServers": { "automation_task": { "type": "streamable-http", - "url": "http://mcp:9108/mcp/" + "url": "http://mcp:9108/mcp" } } } diff --git a/src/backend/extensions/plugins/marketplace/plugin-manager/mcp.json b/src/backend/extensions/plugins/marketplace/plugin-manager/mcp.json index 9987f5b2..fe9f3283 100644 --- a/src/backend/extensions/plugins/marketplace/plugin-manager/mcp.json +++ b/src/backend/extensions/plugins/marketplace/plugin-manager/mcp.json @@ -3,7 +3,7 @@ "mcpServers": { "plugin_manager": { "type": "streamable-http", - "url": "http://mcp:9116/mcp/" + "url": "http://mcp:9116/mcp" } } } diff --git a/src/backend/extensions/plugins/marketplace/plugin-manager/skills/plugin-creator/scripts/validate_plugin.py b/src/backend/extensions/plugins/marketplace/plugin-manager/skills/plugin-creator/scripts/validate_plugin.py index 7e8fc8df..70923748 100644 --- a/src/backend/extensions/plugins/marketplace/plugin-manager/skills/plugin-creator/scripts/validate_plugin.py +++ b/src/backend/extensions/plugins/marketplace/plugin-manager/skills/plugin-creator/scripts/validate_plugin.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """打包前自检一个插件目录的结构是否合法（导入前跑一遍，省得来回试错）。 -规则对齐后端导入器 core/plugins/packaging/importer.py： +规则对齐后端导入器 core/capabilities/plugins/packaging/importer.py： - 包根必须有 plugin.json（或 .claude-plugin/plugin.json）； - plugin.json 必须是合法 JSON 且含非空 name； - name 必须匹配 ^[a-z0-9_-]{1,100}$（会被用来生成安装 id）； diff --git a/src/backend/extensions/plugins/marketplace/sites/mcp.json b/src/backend/extensions/plugins/marketplace/sites/mcp.json index 4e51449e..cf976440 100644 --- a/src/backend/extensions/plugins/marketplace/sites/mcp.json +++ b/src/backend/extensions/plugins/marketplace/sites/mcp.json @@ -3,7 +3,7 @@ "mcpServers": { "site_publish": { "type": "streamable-http", - "url": "http://mcp:9113/mcp/" + "url": "http://mcp:9113/mcp" } } } diff --git a/src/backend/extensions/plugins/marketplace/skill-manager/mcp.json b/src/backend/extensions/plugins/marketplace/skill-manager/mcp.json index 177f6ee8..e246f89d 100644 --- a/src/backend/extensions/plugins/marketplace/skill-manager/mcp.json +++ b/src/backend/extensions/plugins/marketplace/skill-manager/mcp.json @@ -3,7 +3,7 @@ "mcpServers": { "skill_manager": { "type": "streamable-http", - "url": "http://mcp:9112/mcp/" + "url": "http://mcp:9112/mcp" } } } diff --git a/src/backend/extensions/plugins/marketplace/skill-manager/skills/skill-creator/references/skill-anatomy.md b/src/backend/extensions/plugins/marketplace/skill-manager/skills/skill-creator/references/skill-anatomy.md index fa5ef921..a27941c8 100644 --- a/src/backend/extensions/plugins/marketplace/skill-manager/skills/skill-creator/references/skill-anatomy.md +++ b/src/backend/extensions/plugins/marketplace/skill-manager/skills/skill-creator/references/skill-anatomy.md @@ -25,7 +25,7 @@ allowed_tools: Read, Bash # 可选，声明技能用到的工具 --- ``` -- 后端技能引擎（core/agent_skills/registry.py）对 `name` 的校验是 `^[a-z0-9_-]{1,63}$`—— +- 后端技能引擎（core/capabilities/skills/registry.py）对 `name` 的校验是 `^[a-z0-9_-]{1,63}$`—— **允许下划线**（与 Anthropic 原版只许连字符不同）。含空格/中文/大写的 name 会在落库时被 自动规范化，但建议一开始就写规范。 - `description` 缺失或为空会直接落库失败。 diff --git a/src/backend/extensions/plugins/marketplace/skill-manager/skills/skill-creator/scripts/quick_validate.py b/src/backend/extensions/plugins/marketplace/skill-manager/skills/skill-creator/scripts/quick_validate.py index 8b98473f..09fd81b9 100644 --- a/src/backend/extensions/plugins/marketplace/skill-manager/skills/skill-creator/scripts/quick_validate.py +++ b/src/backend/extensions/plugins/marketplace/skill-manager/skills/skill-creator/scripts/quick_validate.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """快速校验一个技能目录的 SKILL.md 是否合法（落库前自检）。 -规则对齐后端技能引擎 core/agent_skills/registry.py（**不是** Anthropic 原版规则）： +规则对齐后端技能引擎 core/capabilities/skills/registry.py（**不是** Anthropic 原版规则）： - 必须有 `---` 包裹的 YAML frontmatter； - frontmatter 必须有 `name` 与 `description`； - name 必须匹配 ^[a-z0-9_-]{1,63}$（小写字母/数字/下划线/连字符，1–63 字符）； diff --git a/src/backend/extensions/skills/marketplace/data-mask/config.yaml b/src/backend/extensions/skills/marketplace/data-mask/config.yaml index b3ec2ec6..d95bc389 100644 --- a/src/backend/extensions/skills/marketplace/data-mask/config.yaml +++ b/src/backend/extensions/skills/marketplace/data-mask/config.yaml @@ -1,83 +1,83 @@ -# 数据脱敏工具配置文件 -# 支持自定义：单位后缀、白名单列、黑名单列、脱敏排除规则 - -# ── 自定义单位后缀 ── -# 将自动追加到内置单位列表（万元、亿美元...）之后 -custom_units: - # - "人次" - # - "笔订单" - # - "亿元RMB" - -# ── 自动列检测规则 ── -column_rules: - # 白名单：不管是否数值，强制脱敏这些列（正则匹配列名） - force_mask_patterns: - # - "金额" - # - ".*收入" - # - "工资$" - - # 黑名单：数值列也跳过脱敏（正则匹配列名） - skip_patterns: - - "^id$" - - "^ID$" - - "^序号$" - - "^编号$" - - "^年份$" - - "^年度$" - - "^月$" - - "^季度$" - - "^排名$" - - "^排序$" - - "^百分比$" - - "^占比$" - -# ── 脱敏排除规则 ── -skip_rules: - # 是否跳过电话号码格式 (11位手机号) - skip_phone: false - - # 排除特定格式的正则列表 - skip_patterns: - # - "^\\d{18}$" # 18位身份证号 - # - "^\\d{6}$" # 邮编 - -# ── 默认参数 ── -defaults: - strategy: hash # 默认脱敏策略: hash | placeholder | blur | range - seed: 42 # 默认随机种子 - auto_detect_numeric: true # DataFrame 脱敏时是否自动检测数值列 - -# ── PDF 输出设置 ── -pdf: - # 默认输出格式: pdf (生成新PDF) 或 txt (纯文本) - output_format: pdf - # PDF 中文字体路径（留空则自动查找系统字体） - cjk_font_path: "" - # 字体大小 - font_size: 10 - -# ── 图片 OCR 设置 ── -ocr: - # 默认识别语言 (Tesseract 语言代码) - # chi_sim+eng = 简体中文 + 英文（推荐） - # eng = 纯英文 - # chi_sim = 纯简体中文 - # jpn+eng = 日文 + 英文 - default_lang: chi_sim+eng - # Windows 上 Tesseract 安装路径的候选列表（留空则用默认） - tesseract_cmd_candidates: - - "C:\\Program Files\\Tesseract-OCR\\tesseract.exe" - - "C:\\Program Files (x86)\\Tesseract-OCR\\tesseract.exe" - -# ── 未知格式处理 ── -fallback: - # 遇到未识别格式时，是否自动尝试文本读取并脱敏 - auto_attempt_text: true - # 文本读取尝试的编码顺序 - encodings: - - utf-8 - - gbk - - gb2312 - - latin-1 - # 回退输出格式（始终为 txt） - output_format: txt +# 数据脱敏工具配置文件 +# 支持自定义：单位后缀、白名单列、黑名单列、脱敏排除规则 + +# ── 自定义单位后缀 ── +# 将自动追加到内置单位列表（万元、亿美元...）之后 +custom_units: + # - "人次" + # - "笔订单" + # - "亿元RMB" + +# ── 自动列检测规则 ── +column_rules: + # 白名单：不管是否数值，强制脱敏这些列（正则匹配列名） + force_mask_patterns: + # - "金额" + # - ".*收入" + # - "工资$" + + # 黑名单：数值列也跳过脱敏（正则匹配列名） + skip_patterns: + - "^id$" + - "^ID$" + - "^序号$" + - "^编号$" + - "^年份$" + - "^年度$" + - "^月$" + - "^季度$" + - "^排名$" + - "^排序$" + - "^百分比$" + - "^占比$" + +# ── 脱敏排除规则 ── +skip_rules: + # 是否跳过电话号码格式 (11位手机号) + skip_phone: false + + # 排除特定格式的正则列表 + skip_patterns: + # - "^\\d{18}$" # 18位身份证号 + # - "^\\d{6}$" # 邮编 + +# ── 默认参数 ── +defaults: + strategy: hash # 默认脱敏策略: hash | placeholder | blur | range + seed: 42 # 默认随机种子 + auto_detect_numeric: true # DataFrame 脱敏时是否自动检测数值列 + +# ── PDF 输出设置 ── +pdf: + # 默认输出格式: pdf (生成新PDF) 或 txt (纯文本) + output_format: pdf + # PDF 中文字体路径（留空则自动查找系统字体） + cjk_font_path: "" + # 字体大小 + font_size: 10 + +# ── 图片 OCR 设置 ── +ocr: + # 默认识别语言 (Tesseract 语言代码) + # chi_sim+eng = 简体中文 + 英文（推荐） + # eng = 纯英文 + # chi_sim = 纯简体中文 + # jpn+eng = 日文 + 英文 + default_lang: chi_sim+eng + # Windows 上 Tesseract 安装路径的候选列表（留空则用默认） + tesseract_cmd_candidates: + - "C:\\Program Files\\Tesseract-OCR\\tesseract.exe" + - "C:\\Program Files (x86)\\Tesseract-OCR\\tesseract.exe" + +# ── 未知格式处理 ── +fallback: + # 遇到未识别格式时，是否自动尝试文本读取并脱敏 + auto_attempt_text: true + # 文本读取尝试的编码顺序 + encodings: + - utf-8 + - gbk + - gb2312 + - latin-1 + # 回退输出格式（始终为 txt） + output_format: txt diff --git a/src/backend/features/__init__.py b/src/backend/features/__init__.py new file mode 100644 index 00000000..8ec4e33d --- /dev/null +++ b/src/backend/features/__init__.py @@ -0,0 +1,157 @@ +"""Host composition for product tools until their owning extensions move in P2/P3.""" + + +def register_product_tools(): + from core.agent_runtime.host import register as register_chat_execution + register_chat_execution() + from core.ontology.host import register as register_governance + register_governance() + from core.harness.registry.system_configuration import register_system_configuration_store + from core.db.system_configuration import SystemConfigurationStore + register_system_configuration_store(SystemConfigurationStore()) + from core.harness.registry.cache_invalidation import cache_invalidators + from core.harness.registry.run_capabilities import has_run_capabilities + from features.desktop.capabilities.local_projection import sync_local_skills + from prompts.prompt_runtime import invalidate_prompt_cache + cache_invalidators().register("before_skill_refresh", "desktop_skills", + lambda: sync_local_skills() if has_run_capabilities() else None) + cache_invalidators().register("after_skill_refresh", "prompt_cache", invalidate_prompt_cache) + + from core.evolution.settlement.memory_service import register as register_settlement + register_settlement() + from core.runs.host import register as register_runs + register_runs() + from core.runs.tool_effects.host import register as register_tool_effects + register_tool_effects() + from core.llm.host_services import register + register() + register_marketplace_catalog() + register_retrieval_context() + from importlib import import_module + from features.installed import INSTALLED_FEATURES + for package in INSTALLED_FEATURES: + register = getattr(import_module(f"{package}.extension"), "register_tools", None) + if register: + register() + gateways = getattr(import_module(f"{package}.extension"), "register_gateway_contributions", None) + if gateways: + gateways() + + +def register_package_sources(): + # CLI installs packages before API composition, so bind import policy here. + from core.ontology.host import register as register_governance + register_governance() + register_marketplace_catalog() + from core.harness.registry.package_views import register_package_view + from features.workbench.ui import contract + from features.workbench.ui import workspace_enabled + register_package_view( + contract.public_contributions, normalize=contract.normalize_ui, find=contract.find_module, + enabled=workspace_enabled.enabled, workspace_only=workspace_enabled.workspace_only, + set_enabled=workspace_enabled.set_enabled, + ) + from core.harness.registry.package_sources import register_package_source, register_package_upgrade_policy + from features.marketplace.package_source import MarketplacePackageSource + from features.workbench.package_source import WorkbenchPackagePolicy + register_package_source("marketplace", MarketplacePackageSource()) + register_package_upgrade_policy(WorkbenchPackagePolicy()) + + +def register_resource_owners(): + from core.harness.registry.resource_owners import ResourceOwner, register_resource_owner + from features.workbench import background, diagnostics, installation_binding, revocation_sweep + from features.workbench import resource_owner, resource_deadlines, revocation + from features.hosted_apps.resources import admission, cleanup + register_resource_owner(ResourceOwner( + binding=lambda *args, **kwargs: resource_owner.application_binding(*args, **kwargs), + resolve_installation=lambda *args, **kwargs: resource_owner.resolve_installation(*args, **kwargs), + exact_installation=lambda *args, **kwargs: installation_binding.exact(*args, **kwargs), + authorize=lambda *args, **kwargs: background.authorized_resource(*args, **kwargs), + observe=lambda *args, **kwargs: diagnostics.observe_resource(*args, **kwargs), + record=lambda *args, **kwargs: diagnostics.record(*args, **kwargs), + renew=lambda *args, **kwargs: background.renew(*args, **kwargs), + arm=lambda *args, **kwargs: resource_deadlines.arm(*args, **kwargs), + invalidate=lambda *args, **kwargs: resource_deadlines.invalidate(*args, **kwargs), + sweep=lambda *args, **kwargs: revocation_sweep.sweep(*args, **kwargs), + collect=lambda *args, **kwargs: resource_owner.collect(*args, **kwargs), + create=admission.create, + owned=admission.owned, + lease=admission.renew, + reserve=admission.reserve, + close=cleanup.close, + revoke_session=revocation.revoke_user_workbench, + )) + + +def register_run_capabilities(): + from core.harness.registry.capability_revisions import register_capability_revisions + from features.desktop.capability.revision_provider import CloudCapabilityRevisions + register_capability_revisions(CloudCapabilityRevisions()) + from core.harness.registry.capability_management import register_manager_executor + from features.desktop.local.manager_execution import execute + register_manager_executor(execute) + from core.harness.registry.device_catalog import register_device_catalog + from features.desktop.capabilities import device_catalog + register_device_catalog(device_catalog) + from core.harness.registry.local_files import register_local_file_services + from features.desktop.local import grants, snapshots + register_local_file_services(grants, snapshots) + from core.harness.registry.capability_runtime import register_capability_runtime + from features.desktop.capabilities import runtime as capability_runtime + register_capability_runtime(capability_runtime) + from core.harness.registry.local_publication import register_local_publication + from features.desktop.local.publication_provider import DesktopLocalPublication + register_local_publication(DesktopLocalPublication()) + from core.harness.registry.execution_identity import register_execution_identity + from features.desktop.bridge import identity + register_execution_identity(identity) + from core.harness.registry.execution_catalog import register_execution_catalog + from core.harness.registry.execution_account import register_execution_account + from features.desktop.bridge import gateway + from features.desktop.capabilities import skill_account + register_execution_catalog(gateway) + register_execution_account(skill_account) + from core.harness.registry.host_runtime import register_remote_execution_policy + from features.desktop.auth import bridge as desktop_bridge + from core.harness.registry.authentication import register_host_authentication + register_host_authentication(desktop_bridge.resolve_bridge_user) + register_remote_execution_policy(lambda: desktop_bridge.bridge_enabled()) + from core.harness.registry.run_capabilities import register_run_capability_provider + from features.desktop.capabilities import paths + from features.desktop.run_capabilities.provider import DesktopRunCapabilityProvider + register_run_capability_provider("desktop", DesktopRunCapabilityProvider(), enabled=lambda: paths.capabilities_enabled()) + + +def register_model_credentials(): + from core.harness.registry.model_credentials import register_request_header_resolver + from features.desktop.bridge import model_credentials as desktop_model_credentials + from core.harness.registry.model_credentials import register_model_credential_provider + register_model_credential_provider(desktop_model_credentials) + register_request_header_resolver( + lambda *args: desktop_model_credentials.prepare_request_headers(*args) + ) + + +def register_catalog_options(): + from importlib import import_module + from features.installed import INSTALLED_FEATURES + for package in INSTALLED_FEATURES: + register = getattr(import_module(f"{package}.extension"), "register_catalog_options", None) + if register: + register() + + +def register_retrieval_context(): + from importlib import import_module + from features.installed import INSTALLED_FEATURES + for package in INSTALLED_FEATURES: + register = getattr(import_module(f"{package}.extension"), "register_retrieval_context", None) + if register: + register() + + +def register_marketplace_catalog(): + from core.harness.registry.marketplace_catalog import register_marketplace_catalog as register_catalog + from features.marketplace.core_catalog import MarketplaceCatalogProvider + register_catalog(MarketplaceCatalogProvider()) diff --git a/src/backend/features/automation/README.md b/src/backend/features/automation/README.md new file mode 100644 index 00000000..f5553e21 --- /dev/null +++ b/src/backend/features/automation/README.md @@ -0,0 +1,15 @@ +# 自动化与任务循环 + +管理计划任务、执行记录、展示摘要、执行位置、通知投递与任务循环。`service.AutomationService` 保留 CRUD 和原子状态变更；`scheduling.compute_next_run` 保留时区感知的 Cron 计算；其余模块按执行绑定、投递目标、通知与远端副作用拆分。 + +对外接口包括原 `/v1/automations` 路由、MCP 任务服务调用的公开服务方法，以及 orchestration 调度器/恢复器使用的任务接口。路由顺序与权限键不变，调度器仍留在 orchestration。模型 `ScheduledTask`、`ScheduledTaskRun` 在 FULL 和 CE 均通过安装清单注册，表名与关系不变；执行状态新增 pending，升级脚本与本机 schema hook 同步更新状态约束。 + +依赖内核数据库、运行账本、项目权限、沙箱和临时状态接口。显式功能依赖为 **automation → desktop**：`tool_routing` 核验当前设备与账号授权，再选择本机任务执行；cloud 路径不使用此能力。当前 core.auth.desktop_bridge/core.capabilities 的物理路径待 P5 按清单迁移。 + +P5 边界待办：桌面工具执行器当前反向调用本模块的路由与副作用服务，属于原始导入基线中的循环债务；不能作为最终允许的反向依赖保留，应通过已注册的工具适配接口接入。本批不改变既有授权和幂等语义。 + +模块安装不自动启动调度器；现有 orchestration 启停和环境配置保持原行为。浏览器自动化属于另一套能力，不归入本目录。 + +执行记录的条件写入由 `runs.AutomationRunRepository` 管理。完成只接受 running 且 lease_token 匹配的记录。恢复先认领，再重读 chat_id。子执行通过事务内 on_accept 钩子关联并保存恢复参数，内核不导入功能模型。调度器取消只转交观察权，用户取消仍通过会话执行器处理。 + +接纳由 admission.admit 管理：条件更新消费触发信号并插入 pending 记录，同事务提交；禁止恢复独立清空触发信号或提前推进排期的旧路径。 diff --git a/src/backend/features/automation/__init__.py b/src/backend/features/automation/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/src/backend/features/automation/__init__.py @@ -0,0 +1 @@ + diff --git a/src/backend/features/automation/admission.py b/src/backend/features/automation/admission.py new file mode 100644 index 00000000..0630c400 --- /dev/null +++ b/src/backend/features/automation/admission.py @@ -0,0 +1,80 @@ +"""Durable admission: consume a trigger and insert its run in one transaction.""" +import uuid + +from sqlalchemy import and_, or_ + +from core.infra.time import utc_now +from features.automation.models import ScheduledTask, ScheduledTaskRun +from features.automation import scheduling + +LIVE = ("pending", "running") + + +def admit(db, limit): + if limit <= 0: + return [] + now = utc_now() + outstanding = db.query(ScheduledTaskRun.run_id).filter( + ScheduledTaskRun.task_id == ScheduledTask.task_id, + ScheduledTaskRun.status.in_(LIVE), + ).exists() + tasks = ( + db.query(ScheduledTask) + .filter( + ScheduledTask.status.in_(("active", "paused", "completed")), + ~outstanding, + or_( + ScheduledTask.manual_trigger_at.isnot(None), + and_(ScheduledTask.status == "active", + ScheduledTask.schedule_type != "manual", + ScheduledTask.next_run_at <= now), + ), + ) + .order_by(ScheduledTask.updated_at, ScheduledTask.task_id) + .with_for_update(skip_locked=True) + .limit(limit).all() + ) + ids = [] + for task in tasks: + manual = task.manual_trigger_at is not None + count = (task.run_count or 0) + 1 + values = {"run_count": count, "updated_at": now} + if manual: + values["manual_trigger_at"] = None + elif (not task.recurring or task.schedule_type == "once" + or (task.max_runs and count >= task.max_runs)): + values["next_run_at"] = None + else: + values["next_run_at"] = scheduling.compute_next_run(task.cron_expression, task.timezone) + # Row locks are an optimization; the conditional write also fences + # SQLite and a contender whose initial SELECT observed an older row. + changed = db.query(ScheduledTask).filter( + ScheduledTask.task_id == task.task_id, + ScheduledTask.run_count == task.run_count, + ScheduledTask.manual_trigger_at == task.manual_trigger_at, + ScheduledTask.next_run_at == task.next_run_at, + ScheduledTask.updated_at == task.updated_at, + ~outstanding, + ).update(values, synchronize_session=False) + if changed != 1: + continue + run = ScheduledTaskRun( + run_id="run_" + uuid.uuid4().hex[:16], + task_id=task.task_id, status="pending", started_at=now, + ) + db.add(run) + ids.append(run.run_id) + db.commit() + db.expire_all() + return ids + + +def candidates(db, limit): + return [ + task_id + for (task_id,) in db.query(ScheduledTask.task_id) + .join(ScheduledTaskRun) + .filter(ScheduledTaskRun.status.in_(LIVE)) + .order_by(ScheduledTaskRun.started_at) + .limit(limit).all() + ] diff --git a/src/backend/features/automation/api/__init__.py b/src/backend/features/automation/api/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/src/backend/features/automation/api/__init__.py @@ -0,0 +1 @@ + diff --git a/src/backend/api/routes/v1/automations.py b/src/backend/features/automation/api/automations.py similarity index 94% rename from src/backend/api/routes/v1/automations.py rename to src/backend/features/automation/api/automations.py index 00566c8a..1e70b1da 100644 --- a/src/backend/api/routes/v1/automations.py +++ b/src/backend/features/automation/api/automations.py @@ -10,8 +10,8 @@ from core.auth.backend import get_current_user, UserContext from core.db.engine import get_db from core.infra.responses import success_response, created_response -from core.services import automation_notifications as notifications -from core.services.automation_service import AutomationService, SUMMARY_LIMIT_LIST +from features.automation import notifications as notifications +from features.automation.service import AutomationService, SUMMARY_LIMIT_LIST from core.infra.logging import get_logger, quiet_access_log logger = get_logger(__name__) @@ -83,7 +83,7 @@ def create_automation( if req.task_type == "plan": if not req.plan_id: raise HTTPException(status_code=400, detail="计划类型任务必须提供 plan_id") - from core.services.plan_service import PlanService + from core.chat.plans import PlanService plan_svc = PlanService(db) plan = plan_svc.get_plan(req.plan_id, user.user_id) if not plan: @@ -92,7 +92,7 @@ def create_automation( if req.task_type == "loop": if not req.loop_id: raise HTTPException(status_code=400, detail="循环类型任务必须提供 loop_id") - from core.services.loop_service import LoopService + from features.automation.loops import LoopService if not LoopService(db).get_loop(req.loop_id, user_id=user.user_id): raise HTTPException(status_code=404, detail="循环不存在或无权访问") @@ -103,14 +103,15 @@ def create_automation( if req.task_type == "loop": task_metadata = {"loop_id": req.loop_id} if req.channel_id and req.conversation_id: - from core.db.repository.channel import ChannelConnectionRepository + from integrations.channels.repository import ChannelConnectionRepository conn = ChannelConnectionRepository(db).get_by_id(req.channel_id) if conn is None or conn.owner_user_id != str(user.user_id): raise HTTPException(status_code=403, detail="无权投递到该渠道机器人") task_metadata = {**(task_metadata or {}), - "channel_id": req.channel_id, "conversation_id": req.conversation_id} + "delivery_targets": [{"type": "inapp"}, {"type": "channel", + "channel_id": req.channel_id, "conversation_id": req.conversation_id}]} - from core.services.automation_execution import execution_metadata + from features.automation.execution import execution_metadata import pytz try: pytz.timezone(req.timezone) @@ -201,9 +202,7 @@ def update_automation( if not task0: raise HTTPException(status_code=404, detail="任务不存在") new_ed = dict(task0.extra_data or {}) - # First clear old delivery markers (flat ones + channel targets inside the list) - new_ed.pop("channel_id", None) - new_ed.pop("conversation_id", None) + # Replace channel targets while retaining the other delivery destinations. if isinstance(new_ed.get("delivery_targets"), list): new_ed["delivery_targets"] = [ t for t in new_ed["delivery_targets"] @@ -213,12 +212,14 @@ def update_automation( new_ed.pop("delivery_targets") # If a channel conversation is given -> verify bot ownership then write; otherwise keep it cleared (= in-site only) if req.channel_id and req.conversation_id: - from core.db.repository.channel import ChannelConnectionRepository + from integrations.channels.repository import ChannelConnectionRepository conn = ChannelConnectionRepository(db).get_by_id(req.channel_id) if conn is None or conn.owner_user_id != str(user.user_id): raise HTTPException(status_code=403, detail="无权投递到该渠道机器人") - new_ed["channel_id"] = req.channel_id - new_ed["conversation_id"] = req.conversation_id + new_ed.setdefault("delivery_targets", [{"type": "inapp"}]).append({ + "type": "channel", "channel_id": req.channel_id, + "conversation_id": req.conversation_id, + }) updates["extra_data"] = new_ed try: diff --git a/src/backend/core/services/delivery_targets.py b/src/backend/features/automation/delivery_targets.py similarity index 75% rename from src/backend/core/services/delivery_targets.py rename to src/backend/features/automation/delivery_targets.py index 187b3a5a..5eaee4fc 100644 --- a/src/backend/core/services/delivery_targets.py +++ b/src/backend/features/automation/delivery_targets.py @@ -7,10 +7,6 @@ {"type": "channel", "channel_id": "chan_…", "conversation_id": "oc_…"} # IM conversation (Feishu, etc.) {"type": "email", "to": "a@b.com"} # reserved, delivery not yet implemented -Backward compatibility: legacy tasks put flat ``channel_id`` / ``conversation_id`` only -at the top level of extra_data (old #7 style); ``resolve_delivery_targets`` interprets -this as ``[{inapp}, {channel…}]``. No markers at all → in-app only. - See internal design docs. """ @@ -22,12 +18,7 @@ def resolve_delivery_targets(extra_data: Optional[Dict[str, Any]]) -> List[Dict[str, Any]]: - """Resolve the final delivery-target list from a task's extra_data (with backward - compatibility). - - Priority: explicit ``delivery_targets`` > legacy flat ``channel_id/conversation_id`` - > in-app only. - """ + """Read the canonical delivery-target list; omitted targets mean in-app.""" meta = extra_data if isinstance(extra_data, dict) else {} targets = meta.get("delivery_targets") if isinstance(targets, list) and targets: @@ -38,13 +29,7 @@ def resolve_delivery_targets(extra_data: Optional[Dict[str, Any]]) -> List[Dict[ if out: return out - # Backward compatibility: legacy #7 channel tasks only put flat channel_id/conversation_id at the top level - ch = meta.get("channel_id") - conv = meta.get("conversation_id") - legacy: List[Dict[str, Any]] = [{"type": "inapp"}] - if ch and conv: - legacy.append({"type": "channel", "channel_id": ch, "conversation_id": conv}) - return legacy + return [{"type": "inapp"}] def build_delivery_targets( diff --git a/src/backend/core/services/automation_execution.py b/src/backend/features/automation/execution.py similarity index 96% rename from src/backend/core/services/automation_execution.py rename to src/backend/features/automation/execution.py index faca455a..ba8f5d3f 100644 --- a/src/backend/core/services/automation_execution.py +++ b/src/backend/features/automation/execution.py @@ -26,7 +26,7 @@ def references_host_path(prompt): def execution_metadata(db, user_id, *, location=None, project_id=None, prompt=""): from core.db.models import Project - from core.auth.permissions_iface import resolve_project_permission + from core.auth.edition import resolve_project_permission actual = instance_location() location = location or actual @@ -66,7 +66,7 @@ def execution_metadata(db, user_id, *, location=None, project_id=None, prompt="" def task_execution_context(db, task): - from core.services.project_scope import build_project_ctx + from core.projects.edition import build_project_ctx saved = dict(task.extra_data or {}) # Legacy tasks belong to the backend where they were stored. diff --git a/src/backend/features/automation/extension.py b/src/backend/features/automation/extension.py new file mode 100644 index 00000000..ecc708de --- /dev/null +++ b/src/backend/features/automation/extension.py @@ -0,0 +1,28 @@ +"""Automation installation; existing scheduler enablement remains authoritative.""" +NAME = "automation" + + +def register_schema(): + from core.harness.registry.schema_lifecycle import register_schema_lifecycle + from features.automation.schema import initialize_schema + register_schema_lifecycle(NAME, initialize_schema) + + +def setup(): + register_schema() + return True + + +def register_gateway_contributions(): + from core.harness.registry.tool_operations import ToolOperations, register_tool_operations + from features.automation import remote_effect, tool_routing + register_tool_operations("automation", ToolOperations( + receipt_tools=remote_effect.RECEIPT_TOOLS, + prepare_arguments=lambda name, args, user, chat: (tool_routing.creation_arguments(args, user, chat) + if name == "create_scheduled_task" else args), + local_tools=frozenset(tool_routing.TOOLS), + bind=lambda *args: remote_effect.bind_remote_effect(*args), + lookup=lambda *args: remote_effect.lookup_remote_receipt(*args), + remember=lambda *args: remote_effect.remember_remote_call(*args), + dispatch_local=lambda *args, **kwargs: tool_routing.local_automation_result(*args, **kwargs), + )) diff --git a/src/backend/core/services/loop_service.py b/src/backend/features/automation/loops.py similarity index 99% rename from src/backend/core/services/loop_service.py rename to src/backend/features/automation/loops.py index 01ff5843..9ac49b41 100644 --- a/src/backend/core/services/loop_service.py +++ b/src/backend/features/automation/loops.py @@ -8,10 +8,9 @@ from datetime import datetime, timezone from typing import Any, Dict, List, Optional -from sqlalchemy.orm import Session - from core.db.models import AgentLoop, LoopIteration from core.infra.logging import get_logger +from sqlalchemy.orm import Session logger = get_logger(__name__) @@ -196,7 +195,7 @@ def load_ledger(self, loop_id: str) -> Optional[Dict[str, Any]]: def persist_result(self, loop_id: str, result: Any, *, commit: bool = True) -> None: """Persist the result of one run_autonomous_loop: loop terminal state + each round's LoopIteration. - ``result`` is an orchestration.autonomous_loop.LoopResult. + ``result`` is an orchestration.loops.driver.LoopResult. """ loop = self.get_loop(loop_id) if not loop: diff --git a/src/backend/core/db/models/automation.py b/src/backend/features/automation/models.py similarity index 64% rename from src/backend/core/db/models/automation.py rename to src/backend/features/automation/models.py index 6ea34374..6a75b100 100644 --- a/src/backend/core/db/models/automation.py +++ b/src/backend/features/automation/models.py @@ -1,10 +1,10 @@ -"""SQLAlchemy ORM models — automation / distillation.""" +"""Scheduled task and run models; loaded independently of scheduler enablement.""" from core.db.utc_datetime import UTCDateTime from core.infra.time import utc_now from datetime import datetime, timezone -from core.db.engine import Base +from core.db.models import Base, Plan, UserShadow from sqlalchemy import ( JSON, BigInteger, @@ -79,8 +79,8 @@ class ScheduledTask(Base): updated_at = Column(UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now) # Relationships - user = relationship("UserShadow") - plan = relationship("Plan") + user = relationship(UserShadow) + plan = relationship(Plan) run_history = relationship( "ScheduledTaskRun", back_populates="task", @@ -121,6 +121,7 @@ class ScheduledTaskRun(Base): String(64), ForeignKey("scheduled_tasks.task_id", ondelete="CASCADE"), nullable=False ) status = Column(String(20), nullable=False, default="running") + lease_token = Column(String(64), nullable=True) chat_id = Column(String(64)) result_summary = Column(Text) error_message = Column(Text) @@ -133,57 +134,9 @@ class ScheduledTaskRun(Base): __table_args__ = ( CheckConstraint( - "status IN ('running', 'success', 'failed')", + "status IN ('pending', 'running', 'success', 'failed')", name="scheduled_task_runs_status_check", ), Index("idx_scheduled_task_runs_task_id", "task_id"), Index("idx_scheduled_task_runs_started_at", "started_at"), ) - - -class PersonaDistillJob(Base): - """Persona-level distillation job (colleague skill / personal skill). - - Unlike DistillationRun (single conversation, cron-driven): one job - aggregates a user's multiple conversations + memories, executed in two - stages — map (per-conversation summaries) → reduce (synthesize SKILL.md). - In colleague mode the output is mirrored into admin_skill_drafts for - review; in personal mode it lands directly as AdminSkill(owner=the user) - after the user's own confirmation. - """ - - __tablename__ = "persona_distill_jobs" - - job_id = Column(String(64), primary_key=True) # pdj_<16hex> - kind = Column(String(16), nullable=False) # colleague | personal - target_user_id = Column(String(64), nullable=False) # the user being distilled - requested_by = Column(String(64), nullable=False) # initiator (config_admin or user_id) - scope = Column(JSONType, default=dict) # chat_ids / date range / memory switch / hint - status = Column(String(16), nullable=False, default="queued") - progress_done = Column( - Integer, nullable=False, default=0 - ) # conversations completed in the map stage - progress_total = Column(Integer, nullable=False, default=0) - intermediate = Column(JSONType, default=list) # list of conversation summaries (map output) - result_skill_content = Column(Text) # full SKILL.md text (reduce output) - result_meta = Column(JSONType, default=dict) # proposed_skill_id/display_name/... - result_draft_id = Column(String(64)) # mirrored draft in colleague mode - saved_skill_id = Column(String(100)) # AdminSkill.skill_id after confirmed persistence - cost_usd = Column(Numeric(8, 4), default=0) - error = Column(Text) - created_at = Column(UTCDateTime(timezone=True), default=utc_now) - started_at = Column(UTCDateTime(timezone=True)) - finished_at = Column(UTCDateTime(timezone=True)) - - __table_args__ = ( - CheckConstraint( - "kind IN ('colleague', 'personal')", - name="persona_distill_jobs_kind_check", - ), - CheckConstraint( - "status IN ('queued','running','completed','failed','cancelled')", - name="persona_distill_jobs_status_check", - ), - Index("idx_persona_distill_jobs_target", "target_user_id", "created_at"), - Index("idx_persona_distill_jobs_status", "status"), - ) diff --git a/src/backend/core/services/automation_notifications.py b/src/backend/features/automation/notifications.py similarity index 100% rename from src/backend/core/services/automation_notifications.py rename to src/backend/features/automation/notifications.py diff --git a/src/backend/core/services/automation_remote_effect.py b/src/backend/features/automation/remote_effect.py similarity index 97% rename from src/backend/core/services/automation_remote_effect.py rename to src/backend/features/automation/remote_effect.py index 7568a377..6a3aee83 100644 --- a/src/backend/core/services/automation_remote_effect.py +++ b/src/backend/features/automation/remote_effect.py @@ -3,8 +3,7 @@ import json from sqlalchemy.exc import IntegrityError -class RemoteOperationConflict(ValueError): - """The authenticated operation key is already bound to another request.""" +from core.harness.registry.tool_operations import RemoteOperationConflict RECEIPT_TOOLS = frozenset({ diff --git a/src/backend/features/automation/runs.py b/src/backend/features/automation/runs.py new file mode 100644 index 00000000..0b22d120 --- /dev/null +++ b/src/backend/features/automation/runs.py @@ -0,0 +1,66 @@ +"""Conditional ownership writes for scheduled execution records.""" + +from core.infra.time import utc_now +from features.automation.models import ScheduledTaskRun + + +class AutomationRunRepository: + def __init__(self, db): + self.db = db + + def claim(self, run_id, lease_token, *, expected_token): + """Only a scheduler holding Redis ownership may transfer a running row.""" + if not lease_token: + raise ValueError("Execution ownership token is required") + count = ( + self.db.query(ScheduledTaskRun) + .filter( + ScheduledTaskRun.run_id == run_id, + ScheduledTaskRun.status.in_(("pending", "running")), + ScheduledTaskRun.lease_token == expected_token, + ) + .update({ScheduledTaskRun.lease_token: lease_token, ScheduledTaskRun.status: "running"}, synchronize_session=False) + ) + self.db.commit() + return count == 1 + + def complete(self, run_id, *, lease_token, commit=True, **values): + if not lease_token: + raise ValueError("Execution ownership token is required") + count = ( + self.db.query(ScheduledTaskRun) + .filter( + ScheduledTaskRun.run_id == run_id, + ScheduledTaskRun.status == "running", + ScheduledTaskRun.lease_token == lease_token, + ) + .update({**values, "completed_at": utc_now()}, synchronize_session=False) + ) + if commit: + self.db.commit() + else: + self.db.flush() + if not count: + return None + self.db.expire_all() + return self.db.get(ScheduledTaskRun, run_id) + + +def owned_run_guard(run_id, token): + """Lock the automation owner in the journal's cancellation transaction.""" + if not token: + raise ValueError("Execution ownership token is required") + + def guard(db): + row = ( + db.query(ScheduledTaskRun) + .filter( + ScheduledTaskRun.run_id == run_id, + ScheduledTaskRun.lease_token == token, + ) + .with_for_update() + .one_or_none() + ) + return row is not None + + return guard diff --git a/src/backend/features/automation/scheduling.py b/src/backend/features/automation/scheduling.py new file mode 100644 index 00000000..619ef9fa --- /dev/null +++ b/src/backend/features/automation/scheduling.py @@ -0,0 +1,23 @@ +"""Timezone-aware cron scheduling shared by automation and personal-agent schedules.""" +from datetime import datetime +from typing import Optional +from croniter import croniter +import pytz + + +def compute_next_run( + cron_expression: str, + timezone: str = "Asia/Shanghai", + base_time: Optional[datetime] = None, +) -> datetime: + """Compute the next fire time from a cron expression in the given timezone. + + Returns a timezone-aware UTC datetime. + """ + tz = pytz.timezone(timezone) + base = base_time or datetime.now(tz) + if base.tzinfo is None: + base = tz.localize(base) + cron = croniter(cron_expression, base) + next_local = cron.get_next(datetime) + return next_local.astimezone(pytz.utc) diff --git a/src/backend/features/automation/schema.py b/src/backend/features/automation/schema.py new file mode 100644 index 00000000..9ac2ca52 --- /dev/null +++ b/src/backend/features/automation/schema.py @@ -0,0 +1,59 @@ +"""Schema upgrades for automation admission and canonical delivery targets.""" +from alembic.migration import MigrationContext +from alembic.operations import Operations +from sqlalchemy import inspect + +RUN_STATUSES = "status IN ('pending', 'running', 'success', 'failed')" + + +def upgrade_queue(connection): + inspector = inspect(connection) + if not inspector.has_table("scheduled_task_runs"): + return + current = next((item for item in inspector.get_check_constraints("scheduled_task_runs") + if item.get("name") == "scheduled_task_runs_status_check"), None) + if current and "'pending'" in current["sqltext"]: + return + operations = Operations(MigrationContext.configure(connection)) + with operations.batch_alter_table("scheduled_task_runs") as batch: + if current: + batch.drop_constraint("scheduled_task_runs_status_check", type_="check") + batch.create_check_constraint("scheduled_task_runs_status_check", RUN_STATUSES) + + +def initialize_schema(): + from core.db.engine import engine + with engine.begin() as connection: + upgrade_queue(connection) + upgrade_delivery_targets(connection) + + +def upgrade_delivery_targets(connection): + """Migrate stored flat destinations once; request handling reads only the new form.""" + from sqlalchemy import Column, JSON, MetaData, String, Table, select, text + if not inspect(connection).has_table("scheduled_tasks"): + return + tasks = Table("scheduled_tasks", MetaData(), + Column("task_id", String), Column("metadata", JSON)) + predicate = ( + "metadata::jsonb ?| array['channel_id', 'conversation_id']" + if connection.dialect.name == "postgresql" else + "json_type(metadata, '$.channel_id') IS NOT NULL OR " + "json_type(metadata, '$.conversation_id') IS NOT NULL" + ) + query = select(tasks.c.task_id, tasks.c.metadata).where(text(predicate)) + for task_id, data in connection.execute(query): + if not isinstance(data, dict) or not ({"channel_id", "conversation_id"} & data.keys()): + continue + data = dict(data) + channel = data.pop("channel_id", None) + conversation = data.pop("conversation_id", None) + targets = data.get("delivery_targets") + valid = [dict(item) for item in targets + if isinstance(item, dict) and item.get("type") in ("inapp", "channel", "email")] if isinstance(targets, list) else [] + if not valid: + valid = [{"type": "inapp"}] + if channel and conversation: + valid.append({"type": "channel", "channel_id": channel, "conversation_id": conversation}) + data["delivery_targets"] = valid + connection.execute(tasks.update().where(tasks.c.task_id == task_id).values(metadata=data)) diff --git a/src/backend/core/services/automation_service.py b/src/backend/features/automation/service.py similarity index 73% rename from src/backend/core/services/automation_service.py rename to src/backend/features/automation/service.py index 8cf34ed1..b1117403 100644 --- a/src/backend/core/services/automation_service.py +++ b/src/backend/features/automation/service.py @@ -4,12 +4,11 @@ from datetime import datetime, timezone import uuid -from croniter import croniter -import pytz +from features.automation import scheduling from sqlalchemy import func, update from sqlalchemy.orm import Session -from core.db.models import ScheduledTask, ScheduledTaskRun +from features.automation.models import ScheduledTask, ScheduledTaskRun from core.infra.logging import get_logger logger = get_logger(__name__) @@ -33,12 +32,11 @@ def _channel_target_fields(extra_data: Optional[Dict[str, Any]]) -> Dict[str, An """Parse the first channel delivery target from a task's extra_data, filling in {channel_id, conversation_id}. - Handles both the delivery_targets list and the old flat form (resolve_delivery_targets - unifies them). No channel target → both are None (= site-internal only). Exposed via - task_to_dict to backfill the frontend edit form. + Reads delivery_targets for the frontend edit form. Without a channel target, + both fields are None. """ try: - from core.services.delivery_targets import resolve_delivery_targets + from features.automation.delivery_targets import resolve_delivery_targets tgt = next( (t for t in resolve_delivery_targets(extra_data or {}) if t.get("type") == "channel"), @@ -52,22 +50,6 @@ def _channel_target_fields(extra_data: Optional[Dict[str, Any]]) -> Dict[str, An } -def compute_next_run( - cron_expression: str, - timezone: str = "Asia/Shanghai", - base_time: Optional[datetime] = None, -) -> datetime: - """Compute the next fire time from a cron expression in the given timezone. - - Returns a timezone-aware UTC datetime. - """ - tz = pytz.timezone(timezone) - base = base_time or datetime.now(tz) - if base.tzinfo is None: - base = tz.localize(base) - cron = croniter(cron_expression, base) - next_local = cron.get_next(datetime) - return next_local.astimezone(pytz.utc) class AutomationService: @@ -100,7 +82,7 @@ def create_task( execution_location: Optional[str] = None, project_id: Optional[str] = None, ) -> ScheduledTask: - from core.services.automation_execution import execution_metadata + from features.automation.execution import execution_metadata binding = execution_metadata(self.db, user_id, location=execution_location, project_id=project_id, prompt=prompt) task_id = f"auto_{uuid.uuid4().hex[:16]}" @@ -112,7 +94,7 @@ def create_task( if schedule_type == "manual": next_run = None else: - next_run = compute_next_run(cron_expression, timezone) + next_run = scheduling.compute_next_run(cron_expression, timezone) if not recurring: max_runs = 1 @@ -195,7 +177,7 @@ def update_task( if not task: return None if "prompt" in kwargs: - from core.services.automation_execution import execution_metadata + from features.automation.execution import execution_metadata saved = task.extra_data or {} execution_metadata(self.db, user_id, location=saved.get("execution_location"), project_id=saved.get("project_id"), prompt=kwargs["prompt"]) @@ -210,7 +192,7 @@ def update_task( if task.schedule_type == "manual": task.next_run_at = None elif "cron_expression" in kwargs or "schedule_type" in kwargs: - task.next_run_at = compute_next_run(task.cron_expression, task.timezone) + task.next_run_at = scheduling.compute_next_run(task.cron_expression, task.timezone) task.updated_at = datetime.now(timezone.utc) if commit: self.db.commit() @@ -265,7 +247,7 @@ def resume_task(self, task_id: str, user_id: str) -> Optional[ScheduledTask]: if not task or task.status != "paused": return None task.status = "active" - task.next_run_at = compute_next_run(task.cron_expression, task.timezone) + task.next_run_at = scheduling.compute_next_run(task.cron_expression, task.timezone) task.updated_at = datetime.now(timezone.utc) self.db.commit() self.db.refresh(task) @@ -273,18 +255,6 @@ def resume_task(self, task_id: str, user_id: str) -> Optional[ScheduledTask]: # ── Due tasks (for scheduler) ────────────────────────────────── - def get_due_tasks(self, now: datetime) -> List[ScheduledTask]: - # Manual-type tasks are never auto-fired; they can only be triggered via /trigger. - return ( - self.db.query(ScheduledTask) - .filter( - ScheduledTask.status == "active", - ScheduledTask.schedule_type != "manual", - ScheduledTask.next_run_at <= now, - ) - .all() - ) - @staticmethod def _is_oneshot_done(task: ScheduledTask) -> bool: """True when no more runs are scheduled after the current one @@ -298,30 +268,6 @@ def _is_oneshot_done(task: ScheduledTask) -> bool: or bool(task.max_runs and (task.run_count or 0) >= task.max_runs) ) - def advance_next_run(self, task_id: str) -> None: - task = self.get_task_by_id(task_id) - if not task: - return - task.run_count = (task.run_count or 0) + 1 - - # Manual tasks: stay active, no next_run_at, run_count just grows. - if task.schedule_type == "manual": - task.next_run_at = None - # One-shot / max_runs reached: clear next_run_at so the schedule stops - # re-firing. Do NOT flip status to "completed" here — this runs as a - # PRE-advance (before execute_task), and execute_task's guard refuses any - # task whose status isn't active/paused. Marking it completed now would - # make the executor skip the very run it just fired (never records a run, - # never delivers to the channel). Terminal status is set AFTER the run by - # finalize_after_run(). - elif self._is_oneshot_done(task): - task.next_run_at = None - else: - task.next_run_at = compute_next_run(task.cron_expression, task.timezone) - - task.updated_at = datetime.now(timezone.utc) - self.db.commit() - def request_manual_trigger(self, task: ScheduledTask) -> None: """登记一次「立即执行」，留给调度器取走。 @@ -335,44 +281,9 @@ def request_manual_trigger(self, task: ScheduledTask) -> None: task.updated_at = task.manual_trigger_at self.db.commit() - def take_manual_triggers(self) -> List[Tuple[str, str]]: - """取走所有待执行的手动触发，返回 (task_id, user_id)。 - - 一条语句取走并清空，取到即归本次调用所有，不必再加行锁。 - """ - taken = self.db.execute( - update(ScheduledTask) - .where(ScheduledTask.manual_trigger_at.isnot(None)) - .values(manual_trigger_at=None) - .returning(ScheduledTask.task_id, ScheduledTask.user_id) - ).all() - self.db.commit() - return [(row.task_id, row.user_id) for row in taken] - - def bump_run_count(self, task_id: str) -> None: - """手动触发路径专用的计数递增。 - - run_count 原本只在 advance_next_run 里 +1，而那是**调度器轮询**的前置步骤。 - 手动「立即执行」和 schedule_type='manual' 的任务根本不走调度器（get_due_tasks - 把 manual 排除在外），于是这些执行只落了一条 run 记录、累计次数却纹丝不动—— - 详情页出现「有 5 条执行记录、累计执行 0 次」的自相矛盾。 - - 这里不动调度器那条路径的语义（那边 +1 与 next_run_at 推进必须原子发生）， - 只补上手动触发这一路的计数。 - """ - self.db.execute( - update(ScheduledTask) - .where(ScheduledTask.task_id == task_id) - .values( - run_count=func.coalesce(ScheduledTask.run_count, 0) + 1, - updated_at=datetime.now(timezone.utc), - ) - ) - self.db.commit() - def finalize_after_run(self, task_id: str) -> None: """Mark one-shot / exhausted tasks 'completed' once their run has - finished. Recurring tasks stay active (advance_next_run already moved + finished. Recurring tasks stay active (admission already moved their next_run_at forward). Never overrides an auto-disabled task.""" task = self.get_task_by_id(task_id) if not task or task.status == "disabled": @@ -385,42 +296,32 @@ def finalize_after_run(self, task_id: str) -> None: # ── Run history ─────────────────────────────────────────────── - def record_run_start(self, task_id: str) -> ScheduledTaskRun: - run = ScheduledTaskRun( - run_id=f"run_{uuid.uuid4().hex[:16]}", - task_id=task_id, - status="running", - started_at=datetime.now(timezone.utc), - ) - self.db.add(run) - self.db.commit() - self.db.refresh(run) - return run - def record_run_complete( self, run_id: str, *, + lease_token: str, status: str, chat_id: Optional[str] = None, result_summary: Optional[str] = None, error_message: Optional[str] = None, duration_ms: int = 0, usage: Optional[Dict] = None, + commit: bool = True, ) -> Optional[ScheduledTaskRun]: - run = self.db.query(ScheduledTaskRun).filter(ScheduledTaskRun.run_id == run_id).first() - if not run: - return None - run.status = status - run.chat_id = chat_id - run.result_summary = result_summary - run.error_message = error_message - run.completed_at = datetime.now(timezone.utc) - run.duration_ms = duration_ms - run.usage = usage or {} - self.db.commit() - self.db.refresh(run) - return run + from .runs import AutomationRunRepository + + return AutomationRunRepository(self.db).complete( + run_id, + lease_token=lease_token, + status=status, + chat_id=chat_id, + result_summary=result_summary, + error_message=error_message, + duration_ms=duration_ms, + usage=usage or {}, + commit=commit, + ) def get_task_runs(self, task_id: str, user_id: str, limit: int = 10) -> List[ScheduledTaskRun]: task = self.get_task(task_id, user_id) @@ -438,7 +339,7 @@ def get_task_runs(self, task_id: str, user_id: str, limit: int = 10) -> List[Sch @staticmethod def task_to_dict(task: ScheduledTask) -> Dict[str, Any]: - from core.services.automation_execution import instance_location, device_identity + from features.automation.execution import instance_location, device_identity location = (task.extra_data or {}).get("execution_location") or instance_location() plan_title = None if task.plan_id and task.plan: @@ -475,7 +376,7 @@ def task_to_dict(task: ScheduledTask) -> Dict[str, Any]: "description": task.description, "sidebar_activated": bool(task.sidebar_activated), # Channel delivery target (for frontend edit backfill): parse the first channel - # target from delivery_targets / the old flat channel_id. None if absent (= site-internal only). + # target from delivery_targets. None if absent (= site-internal only). **_channel_target_fields(task.extra_data), "created_at": task.created_at.isoformat() if task.created_at else None, "updated_at": task.updated_at.isoformat() if task.updated_at else None, diff --git a/src/backend/core/services/automation_tool_routing.py b/src/backend/features/automation/tool_routing.py similarity index 85% rename from src/backend/core/services/automation_tool_routing.py rename to src/backend/features/automation/tool_routing.py index 8c281e70..e7baaef3 100644 --- a/src/backend/core/services/automation_tool_routing.py +++ b/src/backend/features/automation/tool_routing.py @@ -1,5 +1,6 @@ """Automation component execution across a desktop's two authorized backends.""" from __future__ import annotations +from features.desktop.bridge import identity as bridge_identity import asyncio import json @@ -14,8 +15,8 @@ def creation_arguments(arguments, user_id, chat_id): from core.db.engine import SessionLocal from core.db.models import ChatSession, Project - from core.auth.desktop_bridge import bridge_enabled - from core.services.automation_execution import instance_location + from features.desktop.auth.bridge import bridge_enabled + from features.automation.execution import instance_location args = dict(arguments) with SessionLocal() as db: @@ -36,16 +37,17 @@ async def local_automation_result(source_plugin, tool_name, arguments, headers, """Return None for cloud execution; local dispatch requires the automation component.""" if source_plugin != "automation" or tool_name not in TOOLS: return None - from core.services import desktop_cloud_bridge as bridge + from features.desktop.bridge import client as bridge from core.db.engine import SessionLocal - from core.db.models import ChatRun, ScheduledTask, ChatSession - from core.services.tool_effect_ledger import CURRENT_TOOL_EFFECT - from core.services.automation_execution import instance_location + from core.db.models import ChatRun, ChatSession + from features.automation.models import ScheduledTask + from core.harness.tool_effect_context import CURRENT_TOOL_EFFECT + from features.automation.execution import instance_location if instance_location() != "local": return None - identity_state = bridge.get_state() - bridge.ensure_current_authorization(identity_state) + identity_state = bridge_identity.get_state() + bridge_identity.ensure_current_authorization(identity_state) args = dict(arguments) # User identity belongs to the durable local run, not model arguments. effect = CURRENT_TOOL_EFFECT.get() @@ -56,7 +58,7 @@ async def local_automation_result(source_plugin, tool_name, arguments, headers, user_id = run.user_id if run else chat.user_id if chat else None if not user_id: raise ValueError("无法确定本机任务的当前用户") - from core.capabilities.skills import current_local_user_id + from features.desktop.capabilities.skill_account import current_local_user_id # The same cloud account has a device-local shadow ID. Resolve its # cloud-address-scoped mapping before comparing local resource owners. if current_local_user_id() != user_id: @@ -89,13 +91,13 @@ async def local_automation_result(source_plugin, tool_name, arguments, headers, if authorize is None: raise ValueError("本机工具缺少当前授权校验") await authorize() - bridge.ensure_current_authorization(identity_state) + bridge_identity.ensure_current_authorization(identity_state) if tool_name != "create_scheduled_task": args.pop("execution_location", None) from extensions.mcp_servers.automation_task_mcp import impl def dispatch(): # Account changes must not race the queued local database operation. - with bridge.account_scope(identity_state): + with bridge_identity.account_scope(identity_state): return getattr(impl, TOOLS[tool_name])(user_id=user_id, **args) result = await asyncio.to_thread(dispatch) diff --git a/src/backend/features/desktop/README.md b/src/backend/features/desktop/README.md new file mode 100644 index 00000000..af2dce1e --- /dev/null +++ b/src/backend/features/desktop/README.md @@ -0,0 +1,11 @@ +# Desktop + +Owns the device capability store, candidate resolution, runtime views, cloud capability authorization, workcopies and desktop observation records. Desktop schemas are installed on cloud and local hosts independently of whether local file-store execution is enabled. + +The host registers `run_capabilities.provider.DesktopRunCapabilityProvider` through the core run-capability interface. `capabilities.paths` controls whether the current runtime has a local capability root. Selection and account authorization are evaluated at use time. + +Installation records and their SQLAlchemy commit/rollback notifications live in `capabilities.registry`; immutable row projections live in `registry_types`. Skill candidates, hashing, identity, views and publication use one shared cache/lock state. Cloud capability API groups preserve the original route order and observation route class; model and MCP streaming share one HTTP client and close each upstream response. + +Core dependencies include database/session primitives, extension ports, sandbox runtime and capability package parsing. Direct feature dependencies are marketplace archives and knowledge access permissions. Workbench presentation uses a host-injected package view. Sites register upload handlers and receipt localization; automation registers local dispatch and durable operation callbacks. Desktop does not import those component implementations, so feature dependencies remain acyclic. Remaining core-to-feature layering is handled in P6. + +Verification covers selection/account revocation, projection persistence, token tampering and expiry, gateway credential/header isolation, streaming response cleanup, workcopy compare-and-swap, original route/schema snapshots, and CE empty-database upgrade/release checks. diff --git a/src/backend/features/desktop/__init__.py b/src/backend/features/desktop/__init__.py new file mode 100644 index 00000000..2e93078f --- /dev/null +++ b/src/backend/features/desktop/__init__.py @@ -0,0 +1 @@ +"""desktop package.""" diff --git a/src/backend/features/desktop/api/__init__.py b/src/backend/features/desktop/api/__init__.py new file mode 100644 index 00000000..27806988 --- /dev/null +++ b/src/backend/features/desktop/api/__init__.py @@ -0,0 +1 @@ +"""api package.""" diff --git a/src/backend/features/desktop/api/capability_auth.py b/src/backend/features/desktop/api/capability_auth.py new file mode 100644 index 00000000..21eacab9 --- /dev/null +++ b/src/backend/features/desktop/api/capability_auth.py @@ -0,0 +1,48 @@ +"""capability auth responsibilities.""" +from __future__ import annotations +from fastapi import APIRouter +from features.desktop.gateway.observability import ObservedGatewayRoute +router = APIRouter(prefix="/v1/desktop/capability", tags=["Desktop Capability"], route_class=ObservedGatewayRoute) +from core.infra.responses import success_response +from fastapi import HTTPException, Request, Response +from pydantic import BaseModel, Field +from features.desktop.api import desktop_capability_state + +class CapabilityTokenBody(BaseModel): + device_id: str = Field( + ..., min_length=1, max_length=128, pattern=r"^[A-Za-z0-9][A-Za-z0-9._-]*$" + ) + + +@router.post("/token", summary="签发桌面能力令牌") +async def issue_token(body: CapabilityTokenBody, request: Request, response: Response): + """Only an actual live cloud cookie session can mint a desktop credential.""" + import hashlib + + from core.auth.session import validate_session + from core.config.settings import settings + from features.desktop.capability.service import issue_capability_token, session_authorization_epoch + + cookie = request.cookies.get(settings.session.cookie_name, "") + try: + current = await validate_session(cookie) if cookie else None + except Exception: + current = None + if ( + not current + or not current.get("user_id") + or not current.get("user_center_id") + or not session_authorization_epoch(current) + ): + raise HTTPException(status_code=401, detail="valid cloud session required") + data = issue_capability_token( + str(current["user_id"]), + device_id=body.device_id, + issuer=str(request.base_url), + session_hash=hashlib.sha256(cookie.encode()).hexdigest(), + authorization_epoch=session_authorization_epoch(current), + user_center_id=str(current["user_center_id"]), + ) + response.headers["Cache-Control"] = "no-store" + response.headers["Pragma"] = "no-cache" + return success_response(data=data) diff --git a/src/backend/features/desktop/api/capability_bridge.py b/src/backend/features/desktop/api/capability_bridge.py new file mode 100644 index 00000000..a1b372d8 --- /dev/null +++ b/src/backend/features/desktop/api/capability_bridge.py @@ -0,0 +1,92 @@ +"""capability bridge responsibilities.""" +from __future__ import annotations +from fastapi import APIRouter +from features.desktop.gateway.observability import ObservedGatewayRoute +router = APIRouter(prefix="/v1/desktop/capability", tags=["Desktop Capability"], route_class=ObservedGatewayRoute) +import asyncio +from core.auth.backend import UserContext, get_current_user +from core.infra.logging import quiet_access_log +from core.infra.responses import success_response +from fastapi import Depends, HTTPException, Request +from pydantic import BaseModel, Field +import logging +logger = logging.getLogger(__name__) +from features.desktop.api import desktop_capability_state + +class CloudBridgeBody(BaseModel): + cloud_base: str = Field(..., min_length=1, description="云端后端根地址（含协议）") + token: str = Field(..., min_length=8, description="capability token") + expires_in: int = Field(default=600, ge=60, le=600) + device_id: str = Field( + ..., min_length=1, max_length=128, pattern=r"^[A-Za-z0-9][A-Za-z0-9._-]*$" + ) + + +def require_desktop_bridge_process() -> None: + from features.desktop.auth.bridge import bridge_enabled + + if not bridge_enabled(): + raise HTTPException(status_code=403, detail="仅桌面双端本机后端可用") + + +def require_desktop_shell_control(request: Request) -> None: + from features.desktop.capability.service import is_desktop_shell_control + + require_desktop_bridge_process() + if not is_desktop_shell_control( + request.headers.get("authorization", ""), request.headers.get("origin") + ): + raise HTTPException(status_code=401, detail="desktop shell authorization required") + + +@router.post("/cloud-bridge", summary="推送云端能力桥配置（桌面壳 → 本机后端）") +async def set_cloud_bridge( + body: CloudBridgeBody, + _: None = Depends(require_desktop_shell_control), +): + require_desktop_bridge_process() + from features.desktop.bridge.client import set_state + + base = body.cloud_base.strip().rstrip("/") + from urllib.parse import urlsplit + + try: + parsed = urlsplit(base) + except ValueError: + raise HTTPException(status_code=400, detail="cloud_base 地址无效") + if ( + parsed.scheme not in ("http", "https") + or not parsed.hostname + or parsed.username + or parsed.password + or parsed.query + or parsed.fragment + ): + raise HTTPException(status_code=400, detail="cloud_base 必须是不含凭据的 http(s) 地址") + set_state(base, body.token, body.expires_in, device_id=body.device_id) + logger.info("[cloud-bridge] 桥配置已更新（cloud_base=%s）", base) + return success_response(data={"ok": True}) + + +@router.delete("/cloud-bridge", summary="清除云端能力桥（桌面退出登录）") +async def clear_cloud_bridge(_: None = Depends(require_desktop_shell_control)): + require_desktop_bridge_process() + from features.desktop.bridge.identity import clear_state + + clear_state() + return success_response(data={"ok": True}) + + +@router.get("/cloud-bridge/status", summary="云端能力桥状态") +async def cloud_bridge_status(_user: UserContext = Depends(get_current_user)): + from features.desktop.bridge.status import bridge_status + + return success_response(data=bridge_status()) + + +@router.get("/cloud-bridge/readiness", summary="桌面壳首次能力同步进度") +@quiet_access_log +async def cloud_bridge_readiness(_: None = Depends(require_desktop_shell_control)): + from features.desktop.bridge.status import initial_sync_status + + return success_response(data=await asyncio.to_thread(initial_sync_status)) diff --git a/src/backend/features/desktop/api/capability_dependencies.py b/src/backend/features/desktop/api/capability_dependencies.py new file mode 100644 index 00000000..1664ee4c --- /dev/null +++ b/src/backend/features/desktop/api/capability_dependencies.py @@ -0,0 +1,107 @@ +"""capability dependencies responsibilities.""" +from __future__ import annotations +from typing import Optional +import httpx +from core.infra.responses import success_response +from fastapi import Depends, HTTPException, Request, Response +from fastapi.security import HTTPAuthorizationCredentials +import logging +logger = logging.getLogger(__name__) +from features.desktop.api import desktop_capability_state + +def client() -> httpx.AsyncClient: + pass # State is owned by its responsibility module. + if desktop_capability_state.gateway_client is None: + desktop_capability_state.gateway_client = httpx.AsyncClient( + timeout=httpx.Timeout(600.0), + # 桌面端两条消息之间常常隔着几十秒；默认 5s 的 keepalive 意味着几乎 + # 每次模型调用都要重新 TCP + TLS 握手。 + limits=httpx.Limits(max_keepalive_connections=64, keepalive_expiry=300.0), + ) + return desktop_capability_state.gateway_client + + +async def require_capability_user( + request: Request, + credentials: Optional[HTTPAuthorizationCredentials] = Depends(desktop_capability_state.bearer), +) -> str: + from features.desktop.capability.service import CAPABILITY_DEVICE_HEADER, verify_capability_token + + token = credentials.credentials if credentials else "" + user_id = await verify_capability_token( + token, + device_id=request.headers.get(CAPABILITY_DEVICE_HEADER, ""), + issuer=str(request.base_url), + ) + if not user_id: + raise HTTPException(status_code=401, detail="invalid capability token") + request.state.desktop_observation_user = user_id + return user_id + + +def public_content(user_id: str, value, *, bundle: bool = False): + from features.desktop.capability.service import CapabilityContentRejected, guard_capability_bundle, guard_capability_content + + try: + if callable(value): + value = value() + return ( + guard_capability_bundle(user_id, value) + if bundle + else guard_capability_content(user_id, value) + ) + except CapabilityContentRejected: + raise HTTPException( + status_code=422, + detail={ + "code": "integrity_failed", + "message": "capability content blocked by credential policy", + }, + ) from None + + +def stream_secrets(user_id: str, target: dict) -> set[str]: + from features.desktop.capability.service import CapabilityContentRejected, gateway_stream_secrets + + try: + return gateway_stream_secrets(user_id, target) + except CapabilityContentRejected: + raise HTTPException( + status_code=422, + detail={"code": "integrity_failed", "message": "credential policy unavailable"}, + ) from None + + +async def checked_upstream_bytes(upstream: httpx.Response, secrets: set[str]): + from features.desktop.capability.service import CapabilityContentRejected, guard_capability_stream + + try: + async for chunk in guard_capability_stream(upstream.aiter_raw(), secrets): + yield chunk + except CapabilityContentRejected: + logger.warning("[desktop-capability] upstream content blocked code=integrity_failed") + # Streaming headers may already be sent. Return only a fixed diagnostic; + # the buffered bytes containing the credential are never released. + yield b'data: {"error":{"code":"integrity_failed","message":"upstream content blocked"}}\n\n' + finally: + await upstream.aclose() + + +def entity_manifest_response(request: Request, response: Response, manifest: dict): + etag = f'"{manifest["revision"]}"' + headers = {"ETag": etag, "Cache-Control": "private, max-age=0, must-revalidate"} + if request.headers.get("if-none-match") == etag: + return Response(status_code=304, headers=headers) + response.headers.update(headers) + return success_response(data=manifest) + + +def bundle_response(request: Request, resolved): + if resolved is None: + raise HTTPException(status_code=404, detail="not available") + data, content_hash = resolved + etag = f'"{content_hash}"' + headers = {"ETag": etag, "Cache-Control": "private, max-age=0, must-revalidate"} + if request.headers.get("if-none-match") == etag: + return Response(status_code=304, headers=headers) + return Response(content=data, media_type="application/zip", headers=headers) diff --git a/src/backend/features/desktop/api/capability_manifests.py b/src/backend/features/desktop/api/capability_manifests.py new file mode 100644 index 00000000..720a0ac1 --- /dev/null +++ b/src/backend/features/desktop/api/capability_manifests.py @@ -0,0 +1,145 @@ +"""capability manifests responsibilities.""" +from __future__ import annotations +from fastapi import APIRouter +from features.desktop.gateway.observability import ObservedGatewayRoute +router = APIRouter(prefix="/v1/desktop/capability", tags=["Desktop Capability"], route_class=ObservedGatewayRoute) +from core.infra.responses import success_response +from fastapi import Depends, HTTPException, Request, Response +from starlette.concurrency import run_in_threadpool +from features.desktop.api import capability_dependencies +from features.desktop.api import desktop_capability_state + +@router.get("/manifest", summary="当前用户的云端能力清单") +async def get_manifest( + request: Request, + response: Response, + user_id: str = Depends(capability_dependencies.require_capability_user), +): + from features.desktop.capability.service import build_user_capability_manifest + + use_cache = request.headers.get("cache-control") != "no-cache" + manifest = await run_in_threadpool( + capability_dependencies.public_content, user_id, + lambda: build_user_capability_manifest(user_id, use_cache=use_cache), + ) + etag = f'"{manifest["revision"]}"' + headers = {"ETag": etag, "Cache-Control": "private, max-age=0, must-revalidate"} + if request.headers.get("if-none-match") == etag: + return Response(status_code=304, headers=headers) + response.headers.update(headers) + return success_response(data=manifest) + + +@router.get("/models", summary="桌面本机执行面的无密钥模型清单") +async def get_model_manifest( + request: Request, + response: Response, + user_id: str = Depends(capability_dependencies.require_capability_user), +): + from features.desktop.capability.service import build_user_model_manifest + + manifest = await run_in_threadpool( + capability_dependencies.public_content, user_id, lambda: build_user_model_manifest(user_id) + ) + etag = f'"{manifest["revision"]}"' + headers = {"ETag": etag, "Cache-Control": "private, max-age=0, must-revalidate"} + if request.headers.get("if-none-match") == etag: + return Response(status_code=304, headers=headers) + response.headers.update(headers) + return success_response(data=manifest) + + +@router.get("/skills/manifest", summary="当前用户的云端技能清单") +async def get_skill_manifest( + request: Request, + response: Response, + user_id: str = Depends(capability_dependencies.require_capability_user), +): + from features.desktop.capability.service import build_user_skill_manifest + + use_cache = request.headers.get("cache-control") != "no-cache" + manifest = await run_in_threadpool( + capability_dependencies.public_content, user_id, + lambda: build_user_skill_manifest(user_id, use_cache=use_cache), + ) + etag = f'"{manifest["revision"]}"' + headers = {"ETag": etag, "Cache-Control": "private, max-age=0, must-revalidate"} + if request.headers.get("if-none-match") == etag: + return Response(status_code=304, headers=headers) + response.headers.update(headers) + return success_response(data=manifest) + + +@router.get("/skills/{skill_id}/bundle", summary="下载一个当前授权技能的完整 zip 包") +async def get_skill_bundle( + skill_id: str, + request: Request, + user_id: str = Depends(capability_dependencies.require_capability_user), +): + from features.desktop.capability.service import resolve_skill_bundle + + resolved = capability_dependencies.public_content( + user_id, lambda: resolve_skill_bundle(user_id, skill_id), bundle=True + ) + if resolved is None: + raise HTTPException(status_code=404, detail="skill not available") + data, content_hash = resolved + etag = f'"{content_hash}"' + headers = {"ETag": etag, "Cache-Control": "private, max-age=0, must-revalidate"} + if request.headers.get("if-none-match") == etag: + return Response(status_code=304, headers=headers) + return Response(content=data, media_type="application/zip", headers=headers) + + +@router.get("/agents/manifest", summary="当前用户可见的智能体定义清单") +async def get_agent_manifest( + request: Request, response: Response, user_id: str = Depends(capability_dependencies.require_capability_user) +): + from features.desktop.capability.service import build_user_agent_manifest + + use_cache = request.headers.get("cache-control") != "no-cache" + manifest = await run_in_threadpool( + capability_dependencies.public_content, user_id, + lambda: build_user_agent_manifest(user_id, use_cache=use_cache), + ) + return capability_dependencies.entity_manifest_response(request, response, manifest) + + +@router.get( + "/agents/{agent_id}/bundle", summary="下载一个智能体定义包（agent.json + instructions.md）" +) +async def get_agent_bundle( + agent_id: str, request: Request, user_id: str = Depends(capability_dependencies.require_capability_user) +): + from features.desktop.capability.service import resolve_agent_bundle + + return capability_dependencies.bundle_response( + request, + capability_dependencies.public_content(user_id, lambda: resolve_agent_bundle(user_id, agent_id), bundle=True), + ) + + +@router.get("/plugins/manifest", summary="当前用户已安装插件清单") +async def get_plugin_manifest( + request: Request, response: Response, user_id: str = Depends(capability_dependencies.require_capability_user) +): + from features.desktop.capability.service import build_user_plugin_manifest + + use_cache = request.headers.get("cache-control") != "no-cache" + manifest = await run_in_threadpool( + capability_dependencies.public_content, user_id, + lambda: build_user_plugin_manifest(user_id, use_cache=use_cache), + ) + return capability_dependencies.entity_manifest_response(request, response, manifest) + + +@router.get("/plugins/{install_id}/bundle", summary="下载一个插件定义包（plugin.json）") +async def get_plugin_bundle( + install_id: str, request: Request, user_id: str = Depends(capability_dependencies.require_capability_user) +): + from features.desktop.capability.service import resolve_plugin_bundle + + return capability_dependencies.bundle_response( + request, + capability_dependencies.public_content(user_id, lambda: resolve_plugin_bundle(user_id, install_id), bundle=True), + ) diff --git a/src/backend/features/desktop/api/capability_mcp_gateway.py b/src/backend/features/desktop/api/capability_mcp_gateway.py new file mode 100644 index 00000000..27eddbb4 --- /dev/null +++ b/src/backend/features/desktop/api/capability_mcp_gateway.py @@ -0,0 +1,79 @@ +"""capability mcp gateway responsibilities.""" +from __future__ import annotations +from fastapi import APIRouter +from features.desktop.gateway.observability import ObservedGatewayRoute +router = APIRouter(prefix="/v1/desktop/capability", tags=["Desktop Capability"], route_class=ObservedGatewayRoute) +import httpx +from fastapi import Depends, HTTPException, Request +from starlette.background import BackgroundTask +from starlette.concurrency import run_in_threadpool +from starlette.responses import StreamingResponse +import logging +logger = logging.getLogger(__name__) +from features.desktop.api import capability_dependencies +from features.desktop.api import desktop_capability_state + +@router.api_route( + "/gateway/{server_id}/mcp", + methods=["POST", "GET", "DELETE"], + summary="桌面能力网关（MCP streamable-http 反代）", +) +async def gateway_mcp( + server_id: str, + request: Request, + user_id: str = Depends(capability_dependencies.require_capability_user), +): + # 授权解析走 30s per-user 缓存、内部短会话——不经 Depends(get_db)， + # 避免流式转发全程占住连接池里的一条 DB 连接。 + from features.desktop.capability.service import resolve_gateway_target + + target = await run_in_threadpool(resolve_gateway_target, user_id, server_id) + if target is None: + raise HTTPException(status_code=404, detail="server not available") + + headers: dict[str, str] = { + k: v for k, v in request.headers.items() if k.lower() not in desktop_capability_state.DROP_REQUEST_HEADERS + } + # 云端侧连接凭据（远程第三方 MCP 的 headers 等）只在此处物化，不出云端。 + for k, v in (target.get("headers") or {}).items(): + if isinstance(k, str) and isinstance(v, str): + headers[k] = v + # 身份以 capability token 为准，客户端注入的同名头已在 deny-list 拦下。 + headers["X-Current-User-Id"] = user_id + # StreamingResponse 使用 aiter_raw 原样转发，因此不允许上游压缩后再丢失 + # Content-Encoding；这是旧客户端透明 MCP 端点的传输完整性要求。 + headers["accept-encoding"] = "identity" + + body = await request.body() + response_secrets = await run_in_threadpool(capability_dependencies.stream_secrets, user_id, target) + client = capability_dependencies.client() + upstream_req = client.build_request( + request.method, target["url"], headers=headers, content=body + ) + try: + upstream = await client.send(upstream_req, stream=True) + except httpx.HTTPError as exc: + logger.warning( + "[desktop-capability] gateway upstream error user=%s server=%s error_type=%s", + user_id, + server_id, + type(exc).__name__, + ) + raise HTTPException(status_code=502, detail="upstream mcp unreachable") + + logger.info( + "[desktop-capability] gateway %s user=%s server=%s status=%s", + request.method, + user_id, + server_id, + upstream.status_code, + ) + resp_headers = { + name: upstream.headers[name] for name in desktop_capability_state.FWD_RESPONSE_HEADERS if name in upstream.headers + } + return StreamingResponse( + capability_dependencies.checked_upstream_bytes(upstream, response_secrets), + status_code=upstream.status_code, + headers=resp_headers, + background=BackgroundTask(upstream.aclose), + ) diff --git a/src/backend/features/desktop/api/capability_model_gateway.py b/src/backend/features/desktop/api/capability_model_gateway.py new file mode 100644 index 00000000..d980500b --- /dev/null +++ b/src/backend/features/desktop/api/capability_model_gateway.py @@ -0,0 +1,89 @@ +"""capability model gateway responsibilities.""" +from __future__ import annotations +from fastapi import APIRouter +from features.desktop.gateway.observability import ObservedGatewayRoute +router = APIRouter(prefix="/v1/desktop/capability", tags=["Desktop Capability"], route_class=ObservedGatewayRoute) +import json +import httpx +from fastapi import Depends, HTTPException, Request +from starlette.background import BackgroundTask +from starlette.concurrency import run_in_threadpool +from starlette.responses import StreamingResponse +import logging +logger = logging.getLogger(__name__) +from features.desktop.api import capability_dependencies +from features.desktop.api import desktop_capability_state + +@router.post( + "/gateway/models/{provider_id}/{model_path:path}", + summary="桌面模型网关（OpenAI-compatible 流式反代）", +) +async def gateway_model( + provider_id: str, + model_path: str, + request: Request, + user_id: str = Depends(capability_dependencies.require_capability_user), +): + """把本机 Agent 的对话/向量/重排请求转发到云端内网模型。 + + 路径、模型名和上游凭据全由云端 DB 重算；客户端提供的 + Authorization / x-api-key / model 字段都不可信。 + """ + from features.desktop.capability.service import resolve_model_gateway_target + + target = await run_in_threadpool(resolve_model_gateway_target, user_id, provider_id) + if target is None or model_path.strip("/") != target["path"]: + raise HTTPException(status_code=404, detail="model not available") + + try: + payload = await request.json() + except (json.JSONDecodeError, UnicodeDecodeError, ValueError): + raise HTTPException(status_code=400, detail="model request must be JSON") + if not isinstance(payload, dict): + raise HTTPException(status_code=400, detail="model request must be a JSON object") + payload["model"] = target["model_name"] + request.state.desktop_observation_model = target["model_name"] + + headers = { + "authorization": f"Bearer {target['api_key']}", + "content-type": "application/json", + "accept": request.headers.get("accept", "application/json"), + # aiter_raw 保留原始字节，因此明确要求上游不压缩，避免在没有 + # Content-Encoding 响应头的情况下把 gzip 字节直接送给 OpenAI SDK。 + "accept-encoding": "identity", + } + response_secrets = await run_in_threadpool(capability_dependencies.stream_secrets, user_id, target) + client = capability_dependencies.client() + upstream_req = client.build_request( + "POST", + target["url"], + headers=headers, + content=json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode("utf-8"), + ) + try: + upstream = await client.send(upstream_req, stream=True) + except httpx.HTTPError as exc: + logger.warning( + "[desktop-capability] model gateway upstream error user=%s provider=%s error_type=%s", + user_id, + provider_id, + type(exc).__name__, + ) + raise HTTPException(status_code=502, detail="upstream model unreachable") + + logger.info( + "[desktop-capability] model gateway user=%s provider=%s type=%s status=%s", + user_id, + provider_id, + target["provider_type"], + upstream.status_code, + ) + response_headers = {} + if "content-type" in upstream.headers: + response_headers["content-type"] = upstream.headers["content-type"] + return StreamingResponse( + capability_dependencies.checked_upstream_bytes(upstream, response_secrets), + status_code=upstream.status_code, + headers=response_headers, + background=BackgroundTask(upstream.aclose), + ) diff --git a/src/backend/features/desktop/api/capability_tools.py b/src/backend/features/desktop/api/capability_tools.py new file mode 100644 index 00000000..6ca1922a --- /dev/null +++ b/src/backend/features/desktop/api/capability_tools.py @@ -0,0 +1,192 @@ +"""capability tools responsibilities.""" +from __future__ import annotations +from fastapi import APIRouter +from features.desktop.gateway.observability import ObservedGatewayRoute +router = APIRouter(prefix="/v1/desktop/capability", tags=["Desktop Capability"], route_class=ObservedGatewayRoute) +import asyncio +import time +from core.infra.responses import success_response +from fastapi import Depends, HTTPException, Request +from pydantic import BaseModel, Field +from starlette.concurrency import run_in_threadpool +import logging +logger = logging.getLogger(__name__) +from features.desktop.api import capability_dependencies +from features.desktop.api import desktop_capability_state + +class GatewayToolCallBody(BaseModel): + tool_name: str = Field(..., min_length=1, max_length=200) + arguments: dict = Field(default_factory=dict) + schema_hash: str = Field(..., min_length=64, max_length=64) + + +@router.post("/gateway/{server_id}/site-publish", summary="上传桌面构建产物并云端托管") +async def gateway_site_publish( + server_id: str, + request: Request, + user_id: str = Depends(capability_dependencies.require_capability_user), +): + import tarfile + + from core.infra.exceptions import AppException + from features.desktop.capability.service import resolve_gateway_tool + from core.capabilities.manifest_protocol import CapabilityManifestStaleError + from core.harness.registry.transfer_channels import MAX_UPLOAD_OPTIONS_CHARS, UPLOAD_OPTIONS_HEADER, UPLOAD_SCHEMA_HEADER, endpoint_plugin + from core.harness.registry.transfer_channels import upload_receiver + receiver = upload_receiver("site-publish") + from pydantic import ValidationError + + try: + resolved = await run_in_threadpool( + resolve_gateway_tool, + user_id, + server_id, + "publish_site", + schema_hash=request.headers.get(UPLOAD_SCHEMA_HEADER, ""), + ) + except CapabilityManifestStaleError: + raise HTTPException(status_code=409, detail="capability manifest changed") from None + if resolved is None or str( + resolved["target"].get("source_plugin") or "" + ) != endpoint_plugin("site-publish"): + raise HTTPException(status_code=403, detail="site publishing is not authorized") + raw_options = request.headers.get(UPLOAD_OPTIONS_HEADER, "{}") + if len(raw_options) > MAX_UPLOAD_OPTIONS_CHARS: + raise HTTPException(status_code=400, detail="site options too large") + try: + options = receiver.options_model.model_validate_json(raw_options) + except ValidationError: + raise HTTPException(status_code=400, detail="invalid site options") from None + archive = bytearray() + async for chunk in request.stream(): + if len(archive) + len(chunk) > receiver.max_bytes: + raise HTTPException(status_code=413, detail="site archive exceeds 40 MB") + archive.extend(chunk) + try: + result = await asyncio.to_thread(receiver.publish, user_id, bytes(archive), options) + except (ValueError, tarfile.TarError): + raise HTTPException(status_code=400, detail="invalid site build archive") from None + except AppException as exc: + raise HTTPException(status_code=400, detail=exc.message) from None + return success_response(data=capability_dependencies.public_content(user_id, result)) + + +@router.post( + "/gateway/{server_id}/call", + summary="桌面云端工具调用网关（动态 manifest schema + JSON invocation）", +) +async def gateway_mcp_call( + server_id: str, + body: GatewayToolCallBody, + request: Request, + user_id: str = Depends(capability_dependencies.require_capability_user), +): + """Execute a currently-authorized MCP tool from the cloud network.""" + from features.desktop.capability.service import CapabilityContentRejected, invoke_gateway_tool, resolve_gateway_tool + from core.capabilities.manifest_protocol import CapabilityManifestStaleError + + try: + resolved = await run_in_threadpool( + resolve_gateway_tool, + user_id, + server_id, + body.tool_name, + schema_hash=body.schema_hash, + ) + except CapabilityManifestStaleError as exc: + raise HTTPException(status_code=409, detail="capability manifest changed") from exc + if resolved is None: + raise HTTPException(status_code=404, detail="tool not available") + + runtime_headers = { + k: v for k, v in request.headers.items() if k.lower() in desktop_capability_state.RUNTIME_CONTEXT_HEADERS + } + from core.harness.registry.tool_operations import RemoteOperationConflict + started = time.monotonic() + try: + result = await invoke_gateway_tool(resolved, body.arguments, runtime_headers) + except HTTPException: + from extensions.mcp_servers.registry import gateway_server + if gateway_server(resolved["server_id"]) is not None: + raise + raise HTTPException(502, "Cloud tool call failed") from None + except RemoteOperationConflict: + raise HTTPException(status_code=400, detail="operation key already bound to different arguments") from None + except CapabilityContentRejected: + raise HTTPException( + status_code=422, + detail={"code": "integrity_failed", "message": "upstream content blocked"}, + ) from None + except asyncio.TimeoutError as exc: + logger.warning( + "[desktop-capability] tool call timeout user=%s server=%s tool=%s", + user_id, + server_id, + body.tool_name, + ) + raise HTTPException(status_code=504, detail="upstream mcp tool timed out") from exc + except Exception as exc: # noqa: BLE001 - never expose cloud credentials/errors + logger.warning( + "[desktop-capability] tool call failed user=%s server=%s tool=%s error_type=%s", + user_id, + server_id, + body.tool_name, + type(exc).__name__, + ) + raise HTTPException(status_code=502, detail="upstream mcp tool failed") from exc + + logger.info( + "[desktop-capability] tool call user=%s server=%s tool=%s duration_ms=%.0f", + user_id, + server_id, + body.tool_name, + (time.monotonic() - started) * 1000, + ) + return success_response(data=capability_dependencies.public_content(user_id, result)) + + +class GatewayReceiptBody(BaseModel): + tool_name: str = Field(..., min_length=1, max_length=128) + schema_hash: str = Field(..., min_length=64, max_length=64) + operation_id: str = Field(..., min_length=1, max_length=128) + + +@router.post("/gateway/{server_id}/authorize", summary="校验本机任务工具的当前授权") +async def gateway_authorize_local( + server_id: str, body: GatewayReceiptBody, + user_id: str = Depends(capability_dependencies.require_capability_user), +): + from features.desktop.capability.service import resolve_gateway_tool + from core.capabilities.manifest_protocol import CapabilityManifestStaleError + from core.harness.registry.tool_operations import local_tool_allowed + try: + resolved = await run_in_threadpool(resolve_gateway_tool, user_id, server_id, + body.tool_name, schema_hash=body.schema_hash) + except CapabilityManifestStaleError: + raise HTTPException(status_code=409, detail="capability manifest changed") from None + if (resolved is None or not local_tool_allowed( + resolved["target"].get("source_plugin", ""), body.tool_name)): + raise HTTPException(status_code=403, detail="local automation is not authorized") + return success_response(data={"authorized": True}) + + +@router.post("/gateway/{server_id}/receipt", summary="查询云端操作回执") +async def gateway_effect_receipt( + server_id: str, body: GatewayReceiptBody, + user_id: str = Depends(capability_dependencies.require_capability_user), +): + from features.desktop.capability.service import resolve_gateway_tool + from core.capabilities.manifest_protocol import CapabilityManifestStaleError + from core.harness.registry.tool_operations import receipt_operations + + try: + resolved = await run_in_threadpool(resolve_gateway_tool, user_id, server_id, + body.tool_name, schema_hash=body.schema_hash) + except CapabilityManifestStaleError: + raise HTTPException(status_code=409, detail="capability manifest changed") from None + operations = receipt_operations(body.tool_name) + if resolved is None or operations is None: + raise HTTPException(status_code=404, detail="tool not available") + result = await run_in_threadpool(operations.lookup, user_id, server_id, + body.tool_name, body.operation_id) + return success_response(data=capability_dependencies.public_content(user_id, result)) diff --git a/src/backend/features/desktop/api/capability_workcopies.py b/src/backend/features/desktop/api/capability_workcopies.py new file mode 100644 index 00000000..9152c456 --- /dev/null +++ b/src/backend/features/desktop/api/capability_workcopies.py @@ -0,0 +1,38 @@ +"""capability workcopies responsibilities.""" +from __future__ import annotations +from fastapi import APIRouter +from features.desktop.gateway.observability import ObservedGatewayRoute +router = APIRouter(prefix="/v1/desktop/capability", tags=["Desktop Capability"], route_class=ObservedGatewayRoute) +from core.infra.responses import success_response +from fastapi import Depends, Response +from pydantic import BaseModel, Field +from starlette.concurrency import run_in_threadpool +from features.desktop.api import capability_dependencies +from features.desktop.api import desktop_capability_state + +class WorkcopyCommitBody(BaseModel): + files: dict[str, str] + expected_revision: str | None = None + expected_model_version: str | None = None + request_id: str = Field(pattern=r"^[a-f0-9]{32}$") + acknowledge_sensitive: bool = False + create_only: bool = False + + +@router.get("/workcopies/{kind}/{key}", summary="比较当前账号的能力文件版本") +async def workcopy_snapshot(kind: str, key: str, response: Response, + user_id: str = Depends(capability_dependencies.require_capability_user)): + from features.desktop.capability.workcopies import snapshot + response.headers["Cache-Control"] = "no-store" + result = await run_in_threadpool(snapshot, user_id, kind, key) + return success_response(data=await run_in_threadpool(capability_dependencies.public_content, user_id, result)) + + +@router.post("/workcopies/{kind}/{key}", summary="显式提交能力文件版本") +async def workcopy_commit(kind: str, key: str, body: WorkcopyCommitBody, response: Response, + user_id: str = Depends(capability_dependencies.require_capability_user)): + from features.desktop.capability.workcopies import commit + response.headers["Cache-Control"] = "no-store" + await run_in_threadpool(capability_dependencies.public_content, user_id, body.files) + result = await run_in_threadpool(commit, user_id, kind, key, body.model_dump()) + return success_response(data=await run_in_threadpool(capability_dependencies.public_content, user_id, result)) diff --git a/src/backend/api/routes/v1/desktop.py b/src/backend/features/desktop/api/desktop.py similarity index 99% rename from src/backend/api/routes/v1/desktop.py rename to src/backend/features/desktop/api/desktop.py index 25ef16bc..8640e054 100644 --- a/src/backend/api/routes/v1/desktop.py +++ b/src/backend/features/desktop/api/desktop.py @@ -47,7 +47,7 @@ from fastapi import APIRouter, Request from fastapi.responses import FileResponse, JSONResponse, Response, StreamingResponse -from core.services.desktop_release_events import ReleaseEvents +from features.desktop.bridge.release_events import ReleaseEvents from core.infra.logging import get_logger logger = get_logger(__name__) diff --git a/src/backend/api/routes/v1/desktop_capabilities.py b/src/backend/features/desktop/api/desktop_capabilities.py similarity index 80% rename from src/backend/api/routes/v1/desktop_capabilities.py rename to src/backend/features/desktop/api/desktop_capabilities.py index 7c3fd7ca..b2317818 100644 --- a/src/backend/api/routes/v1/desktop_capabilities.py +++ b/src/backend/features/desktop/api/desktop_capabilities.py @@ -12,6 +12,10 @@ """ from __future__ import annotations +from features.desktop.capabilities import skill_account +from features.desktop.capabilities import skill_candidates +from features.desktop.capabilities import skill_views +from features.desktop.capabilities import skills import asyncio import logging @@ -37,9 +41,9 @@ async def preview_changes(body: ChangePreviewBody, user: UserContext = Depends(g _require_desktop_store() uid = str(user.user_id) state = _bridge_state(uid) - from core.capabilities.change_sync import preview - from core.capabilities.errors import CapabilityError - from core.capabilities.mcp_json import McpJsonError + from features.desktop.capabilities.change_sync import preview + from core.harness.capability_errors import CapabilityError + from features.desktop.capabilities.mcp_json import McpJsonError try: return success_response(data=await asyncio.to_thread(preview, uid, body.install_id, state)) except (CapabilityError, McpJsonError, OSError, ValueError) as exc: @@ -58,9 +62,9 @@ async def commit_changes(body: ChangeCommitBody, user: UserContext = Depends(get _require_desktop_store() uid = str(user.user_id) state = _bridge_state(uid) - from core.capabilities.change_sync import commit - from core.capabilities.errors import CapabilityError - from core.capabilities.mcp_json import McpJsonError + from features.desktop.capabilities.change_sync import commit + from core.harness.capability_errors import CapabilityError + from features.desktop.capabilities.mcp_json import McpJsonError try: return success_response(data=await asyncio.to_thread( commit, uid, body.preview_id, body.choices, state, @@ -70,23 +74,23 @@ async def commit_changes(body: ChangeCommitBody, user: UserContext = Depends(get def _require_desktop_store() -> None: - from core.auth.desktop_bridge import bridge_enabled - from core.capabilities.paths import capabilities_enabled + from features.desktop.auth.bridge import bridge_enabled + from core.harness.registry.run_capabilities import has_run_capabilities if not bridge_enabled(): raise HTTPException(status_code=403, detail="仅桌面双端本机后端可用") - if not capabilities_enabled(): + if not has_run_capabilities(): raise HTTPException(status_code=403, detail="本机未配置能力文件存储（HUGAGENT_CAPS_ROOT）") def _authorized_profile(user_id: str) -> Optional[str]: - from core.capabilities import skills + from features.desktop.capabilities import skills - return skills.current_account_profile() if skills.account_authorized_for(user_id) else None + return skill_account.current_account_profile() if skill_account.account_authorized_for(user_id) else None def _bridge_state(user_id: str) -> Dict[str, Any]: - from core.services.desktop_cloud_bridge import get_state + from features.desktop.bridge.identity import get_state st = get_state() if not st: @@ -110,17 +114,15 @@ def _device_item(inst, *, runtime_name: str, usable: bool) -> Dict[str, Any]: return item -from core.capabilities.management_context import ( - management_context as _readiness_context, connectors_view as _connectors_view, -) +from features.desktop.capabilities.management_context import management_context as _readiness_context, connectors_view as _connectors_view def _agents_view(user_id: str) -> Dict[str, Any]: - from core.capabilities import agents as caps_agents - from core.capabilities import registry, skills - from core.capabilities.paths import KIND_AGENT + from features.desktop.capabilities import agents as caps_agents + from features.desktop.capabilities import registry, skills + from core.capabilities.identity_rules import KIND_AGENT from core.db.engine import SessionLocal - from core.services.user_agent_service import UserAgentService + from core.capabilities.agents.edition import UserAgentService with SessionLocal() as db: svc = UserAgentService(db) @@ -132,7 +134,7 @@ def _agents_view(user_id: str) -> Dict[str, Any]: items: List[Dict[str, Any]] = [] profile = _authorized_profile(user_id) context = _readiness_context(user_id) - from core.capabilities.readiness import apply_to_item, file_readiness, files_ready + from features.desktop.capabilities.readiness import apply_to_item, file_readiness, files_ready for inst in registry.list_installations( kind=KIND_AGENT, profiles=[p for p in ("local", profile) if p] @@ -162,12 +164,12 @@ def _agents_view(user_id: str) -> Dict[str, Any]: def _plugins_view(user_id: str) -> Dict[str, Any]: - from core.capabilities import registry - from core.capabilities.paths import KIND_PLUGIN + from features.desktop.capabilities import registry + from core.capabilities.identity_rules import KIND_PLUGIN profile = _authorized_profile(user_id) context = _readiness_context(user_id) - from core.capabilities.readiness import file_readiness, files_ready + from features.desktop.capabilities.readiness import file_readiness, files_ready items: List[Dict[str, Any]] = [] for inst in registry.list_installations( @@ -195,8 +197,8 @@ def _plugins_view(user_id: str) -> Dict[str, Any]: def _installations_view(user_id: str, kind: str) -> Dict[str, Any]: - from core.capabilities import skills - from core.capabilities.paths import KIND_AGENT, KIND_MCP, KIND_PLUGIN, KIND_SKILL + from features.desktop.capabilities import skills + from core.capabilities.identity_rules import KIND_AGENT, KIND_MCP, KIND_PLUGIN, KIND_SKILL if kind == KIND_MCP: return _connectors_view(user_id) @@ -206,7 +208,7 @@ def _installations_view(user_id: str, kind: str) -> Dict[str, Any]: return _plugins_view(user_id) if kind != KIND_SKILL: raise HTTPException(status_code=400, detail=f"unknown kind {kind!r}") - from core.capabilities.discovery import scan + from features.desktop.capabilities.discovery import scan discovery_errors = scan(user_id) res = skills.resolve_for_user(user_id) chosen = {c.install_id: name for name, c in res.chosen.items()} @@ -226,17 +228,12 @@ def _resolution_of(install_id: str) -> Dict[str, Any]: return {"outcome": "absent", "reason": None} context = _readiness_context(user_id) - from core.capabilities.readiness import ( - apply_to_item, - builtin_readiness, - file_readiness, - files_ready, - ) + from features.desktop.capabilities.readiness import apply_to_item, builtin_readiness, file_readiness, files_ready items: List[Dict[str, Any]] = [] - for cand in skills.candidates(user_id): + for cand in skill_candidates.candidates(user_id): entry = cand.to_dict() - from core.capabilities import registry + from features.desktop.capabilities import registry inst = registry.get(cand.install_id) entry["registered"] = inst is not None @@ -260,7 +257,7 @@ def _resolution_of(install_id: str) -> Dict[str, Any]: def _preferences(kind: str, user_id: str) -> Dict[str, str]: - from core.capabilities import registry + from features.desktop.capabilities import registry return registry.preferences(kind, user_id=user_id) @@ -269,7 +266,7 @@ def _preferences(kind: str, user_id: str) -> Dict[str, str]: @quiet_access_log async def list_installations(kind: str = "skill", user: UserContext = Depends(get_current_user)): _require_desktop_store() - from core.capabilities.change_sync import annotate + from features.desktop.capabilities.change_sync import annotate def listing(): return annotate(str(user.user_id), _installations_view(str(user.user_id), kind)) return success_response( @@ -281,8 +278,8 @@ def listing(): async def check_sync(user: UserContext = Depends(get_current_user)): _require_desktop_store() st = _bridge_state(str(user.user_id)) - from core.capabilities.errors import CloudUnavailable - from core.services.desktop_capability_sync_check import check + from core.harness.capability_errors import CloudUnavailable + from features.desktop.capability.sync_check import check try: return success_response(data=await asyncio.to_thread(check, st)) @@ -294,10 +291,8 @@ async def check_sync(user: UserContext = Depends(get_current_user)): async def sync_now(user: UserContext = Depends(get_current_user)): _require_desktop_store() st = _bridge_state(str(user.user_id)) - from core.services import desktop_cloud_bundles, desktop_cloud_skills - from core.services.desktop_cloud_bridge import ( - sync_capabilities_blocking, sync_tool_manifest_blocking, - ) + from features.desktop.bridge import bundles as desktop_cloud_bundles, skills as desktop_cloud_skills + from features.desktop.bridge.client import sync_capabilities_blocking, sync_tool_manifest_blocking await asyncio.to_thread(sync_tool_manifest_blocking, st) await asyncio.to_thread(sync_capabilities_blocking, st) @@ -310,18 +305,18 @@ async def sync_now(user: UserContext = Depends(get_current_user)): status_code=502, detail={"code": "cloud_unavailable", "message": "; ".join(errors), "retryable": True}, ) - from core.capabilities import skills + from features.desktop.capabilities import skills - await asyncio.to_thread(skills.rebuild_views, str(user.user_id)) + await asyncio.to_thread(skill_views.rebuild_views, str(user.user_id)) return success_response(data=status) @router.post("/views/rebuild", summary="重建运行视图") async def rebuild_views(user: UserContext = Depends(get_current_user)): _require_desktop_store() - from core.capabilities import skills + from features.desktop.capabilities import skills - reports = await asyncio.to_thread(skills.rebuild_views, str(user.user_id)) + reports = await asyncio.to_thread(skill_views.rebuild_views, str(user.user_id)) return success_response(data={k: v.to_dict() for k, v in reports.items()}) @@ -329,7 +324,7 @@ async def rebuild_views(user: UserContext = Depends(get_current_user)): def _mcp_json_doc(user_id: str) -> Dict[str, Any]: - from core.capabilities import mcp_json, skills + from features.desktop.capabilities import mcp_json, skills try: doc = mcp_json.load() @@ -363,7 +358,7 @@ async def get_mcp_json(_user: UserContext = Depends(get_current_user)): async def initial_sync_status_route(user: UserContext = Depends(get_current_user)): _require_desktop_store() _bridge_state(str(user.user_id)) - from core.services.desktop_cloud_bridge import initial_sync_status + from features.desktop.bridge.status import initial_sync_status return success_response(data=await asyncio.to_thread(initial_sync_status)) @@ -371,16 +366,17 @@ async def initial_sync_status_route(user: UserContext = Depends(get_current_user async def retry_initial_sync_route(user: UserContext = Depends(get_current_user)): _require_desktop_store() st = _bridge_state(str(user.user_id)) - from core.services.desktop_cloud_bridge import retry_initial_sync + from features.desktop.bridge.client import retry_initial_sync await asyncio.to_thread(retry_initial_sync, st) return success_response(data={"ok": True}) def _accept_synced_capabilities(st, user_id): - from core.services.desktop_cloud_bridge import account_scope, initial_sync_status - from core.capabilities import registry, session_availability, skills - from core.capabilities.readiness import file_readiness - from core.capabilities.paths import revision_for_hash + from features.desktop.bridge.identity import account_scope + from features.desktop.bridge.status import initial_sync_status + from features.desktop.capabilities import registry, session_availability, skills + from features.desktop.capabilities.readiness import file_readiness + from features.desktop.capabilities.paths import revision_for_hash with account_scope(st): status = initial_sync_status() @@ -399,7 +395,7 @@ def _accept_synced_capabilities(st, user_id): session_availability.activate(profile, revisions) registry._bump() skills.bump_view_generation() - from core.config.catalog_resolver import invalidate_capability_cache + from core.capabilities.catalog.resolver import invalidate_capability_cache invalidate_capability_cache(user_id) return {"ok": True, "partial": True, "available_count": len(revisions)} diff --git a/src/backend/features/desktop/api/desktop_capability.py b/src/backend/features/desktop/api/desktop_capability.py new file mode 100644 index 00000000..62ddd6f6 --- /dev/null +++ b/src/backend/features/desktop/api/desktop_capability.py @@ -0,0 +1,12 @@ +"""Compose desktop capability routes in their original security and API order.""" +from fastapi import APIRouter +from features.desktop.api import capability_auth, capability_workcopies, capability_manifests, capability_tools, capability_model_gateway, capability_mcp_gateway, capability_bridge + +router = APIRouter() +router.include_router(capability_auth.router) +router.include_router(capability_workcopies.router) +router.include_router(capability_manifests.router) +router.include_router(capability_tools.router) +router.include_router(capability_model_gateway.router) +router.include_router(capability_mcp_gateway.router) +router.include_router(capability_bridge.router) diff --git a/src/backend/features/desktop/api/desktop_capability_state.py b/src/backend/features/desktop/api/desktop_capability_state.py new file mode 100644 index 00000000..2189ecf4 --- /dev/null +++ b/src/backend/features/desktop/api/desktop_capability_state.py @@ -0,0 +1,52 @@ +"""desktop capability state responsibilities.""" +from __future__ import annotations +from typing import Optional +import httpx +from fastapi import APIRouter +from fastapi.security import HTTPBearer +from features.desktop.gateway.observability import ObservedGatewayRoute + + + + +gateway_client: Optional[httpx.AsyncClient] = None + + +bearer = HTTPBearer(auto_error=False) + + +DROP_REQUEST_HEADERS = { + # 身份与凭据：capability token 不上传上游；身份头由网关重写 + "authorization", + "cookie", + "x-current-user-id", + # 知识库授权按云端用户重算（头缺失时 KB MCP 自动解析可用知识库） + "x-allowed-kb-ids", + # 桌面桥内部头 + "x-desktop-bridge", + "x-desktop-bridge-user", + "x-desktop-device-id", + "x-hugagent-target", + # 逐跳 / 传输层头，由 httpx 按新连接重建 + "host", + "content-length", + "connection", + "accept-encoding", + "transfer-encoding", + "keep-alive", + "proxy-authorization", + "te", + "upgrade", +} + + +FWD_RESPONSE_HEADERS = ("content-type", "mcp-session-id", "mcp-protocol-version") + + +RUNTIME_CONTEXT_HEADERS = { + "x-chat-id", + "x-channel-id", + "x-conversation-id", + "x-allowed-dataset-ids", + "x-reranker-enabled", +} diff --git a/src/backend/api/routes/v1/desktop_login.py b/src/backend/features/desktop/api/desktop_login.py similarity index 99% rename from src/backend/api/routes/v1/desktop_login.py rename to src/backend/features/desktop/api/desktop_login.py index b788782d..0deca1c8 100644 --- a/src/backend/api/routes/v1/desktop_login.py +++ b/src/backend/features/desktop/api/desktop_login.py @@ -7,7 +7,7 @@ from fastapi import APIRouter, HTTPException, Request, Response from pydantic import BaseModel, Field -from core.auth import desktop_login_store as store +from features.desktop.auth import login_store as store from core.auth.session import validate_session from core.config.settings import settings from core.infra.responses import success_response diff --git a/src/backend/api/routes/v1/desktop_observability.py b/src/backend/features/desktop/api/desktop_observability.py similarity index 90% rename from src/backend/api/routes/v1/desktop_observability.py rename to src/backend/features/desktop/api/desktop_observability.py index 5baae543..83d57783 100644 --- a/src/backend/api/routes/v1/desktop_observability.py +++ b/src/backend/features/desktop/api/desktop_observability.py @@ -4,12 +4,12 @@ from typing import Literal from api.deps import require_config -from api.routes.v1.desktop_capability import _require_capability_user +from features.desktop.api.capability_dependencies import require_capability_user as _require_capability_user from core.db.engine import SessionLocal, get_db from core.db.models import UserShadow -from core.db.models.observability import DesktopSyncDevice +from features.desktop.models import DesktopSyncDevice from core.infra.responses import paginated_response, success_response -from core.services.desktop_observability import MAX_BATCH_BYTES, ingest, list_records +from features.desktop.observability.projection import MAX_BATCH_BYTES, ingest, list_records from fastapi import APIRouter, Depends, HTTPException, Query, Request from sqlalchemy import select from starlette.concurrency import run_in_threadpool @@ -101,6 +101,6 @@ def devices(db=Depends(get_db)): @admin_router.get("/health") def health(): - from core.services.desktop_gateway_observability import health_snapshot + from features.desktop.gateway.observability import health_snapshot return success_response(data=health_snapshot()) diff --git a/src/backend/api/routes/v1/device_link.py b/src/backend/features/desktop/api/device_link.py similarity index 95% rename from src/backend/api/routes/v1/device_link.py rename to src/backend/features/desktop/api/device_link.py index 1b64995a..b7b6bbbe 100644 --- a/src/backend/api/routes/v1/device_link.py +++ b/src/backend/features/desktop/api/device_link.py @@ -10,9 +10,9 @@ from typing import Optional from core.auth.backend import UserContext, get_current_user -from core.auth.desktop_bridge import bridge_enabled -from core.device_link import ACTIONS, SESSIONS, store -from core.device_link.runner import current_account, current_link, run_driver, runner +from features.desktop.auth.bridge import bridge_enabled +from personal_agent.device_link import ACTIONS, SESSIONS, store +from personal_agent.device_link.runner import current_account, current_link, run_driver, runner from core.infra.responses import success_response from fastapi import APIRouter, Depends, HTTPException from pydantic import BaseModel, Field diff --git a/src/backend/api/routes/v1/local.py b/src/backend/features/desktop/api/local.py similarity index 82% rename from src/backend/api/routes/v1/local.py rename to src/backend/features/desktop/api/local.py index 61b91b5c..87959709 100644 --- a/src/backend/api/routes/v1/local.py +++ b/src/backend/features/desktop/api/local.py @@ -12,8 +12,8 @@ from core.auth.backend import UserContext, get_current_user from core.config.local_mode import local_mode_enabled from core.infra.responses import success_response -from core.services import local_grant_service as grants -from core.services import local_snapshot_service as snapshots +from features.desktop.local import grants as grants +from features.desktop.local import snapshots as snapshots from fastapi import APIRouter, Depends, HTTPException from pydantic import BaseModel, Field @@ -77,20 +77,8 @@ async def set_policy(body: PolicyBody, _user: UserContext = Depends(get_current_ return success_response(data=saved, message="策略已更新") -class ApprovalBody(BaseModel): - mode: Literal["strict", "standard", "full"] - - -@router.get("/approval-mode", summary="获取当前审批档（对话框权限栏）") -async def get_approval_mode(_user: UserContext = Depends(get_current_user)): - _require_local() - return success_response(data={"mode": grants.get_approval_mode()}) - - -@router.put("/approval-mode", summary="设置审批档") -async def set_approval_mode(body: ApprovalBody, _user: UserContext = Depends(get_current_user)): - _require_local() - return success_response(data={"mode": grants.set_approval_mode(body.mode)}, message="已切换权限档") +from features.desktop.edition import register_local_routes +register_local_routes(router, _require_local) class RollbackBody(BaseModel): diff --git a/src/backend/api/routes/local_file_actions.py b/src/backend/features/desktop/api/local_file_actions.py similarity index 93% rename from src/backend/api/routes/local_file_actions.py rename to src/backend/features/desktop/api/local_file_actions.py index 925339b8..3301aae2 100644 --- a/src/backend/api/routes/local_file_actions.py +++ b/src/backend/features/desktop/api/local_file_actions.py @@ -27,7 +27,7 @@ def applications( include_icons: bool = True, ): from api.routes.files import local_file_location - from core.services.local_file_actions import file_applications + from features.desktop.local.file_actions import file_applications location = local_file_location(file_id, user, db) try: @@ -42,7 +42,7 @@ def open_local_file( user: UserContext = Depends(require_auth()), db: Session = Depends(get_db), ): from api.routes.files import local_file_location - from core.services.local_file_actions import open_file + from features.desktop.local.file_actions import open_file location = local_file_location(file_id, user, db) try: diff --git a/src/backend/features/desktop/auth/__init__.py b/src/backend/features/desktop/auth/__init__.py new file mode 100644 index 00000000..95d9f8f1 --- /dev/null +++ b/src/backend/features/desktop/auth/__init__.py @@ -0,0 +1 @@ +"""auth package.""" diff --git a/src/backend/core/auth/desktop_bridge.py b/src/backend/features/desktop/auth/bridge.py similarity index 95% rename from src/backend/core/auth/desktop_bridge.py rename to src/backend/features/desktop/auth/bridge.py index 3a462777..829ede17 100644 --- a/src/backend/core/auth/desktop_bridge.py +++ b/src/backend/features/desktop/auth/bridge.py @@ -85,7 +85,7 @@ def _ensure_bridge_capabilities(db: Session, shadow) -> None: meta = dict(shadow.extra_data or {}) if meta.get(_BRIDGE_META_FLAG): return - from core.services.local_user_service import CE_DEFAULT_ADMIN_CAPABILITIES + from core.auth.local_users_constants import CE_DEFAULT_ADMIN_CAPABILITIES meta.update(CE_DEFAULT_ADMIN_CAPABILITIES) meta["auth_source"] = "desktop_bridge" @@ -124,16 +124,16 @@ def resolve_bridge_user(request: Request, db: Session): # The shell header must belong to the current signed cloud identity. A # header captured before account switch/logout must never create a local user. - from core.capabilities.paths import capabilities_enabled + from features.desktop.capabilities.paths import capabilities_enabled if capabilities_enabled(): - from core.services.desktop_cloud_bridge import get_identity_state + from features.desktop.bridge.identity import get_identity_state identity = get_identity_state() # 壳送来的是按云端地址加了命名空间的标识；本机侧从凭据推出同样的标识再比对。 if not identity or identity.get("shell_user_center_id") != info["user_center_id"]: return None from core.auth.backend import UserContext - from core.services import UserService + from core.auth.users import UserService shadow = UserService(db).get_or_create_user_shadow( user_center_id=info["user_center_id"], diff --git a/src/backend/core/auth/desktop_login_store.py b/src/backend/features/desktop/auth/login_store.py similarity index 100% rename from src/backend/core/auth/desktop_login_store.py rename to src/backend/features/desktop/auth/login_store.py diff --git a/src/backend/core/auth/desktop_ticket_store.py b/src/backend/features/desktop/auth/ticket_store.py similarity index 100% rename from src/backend/core/auth/desktop_ticket_store.py rename to src/backend/features/desktop/auth/ticket_store.py diff --git a/src/backend/features/desktop/bridge/__init__.py b/src/backend/features/desktop/bridge/__init__.py new file mode 100644 index 00000000..e7e03d85 --- /dev/null +++ b/src/backend/features/desktop/bridge/__init__.py @@ -0,0 +1 @@ +"""bridge package.""" diff --git a/src/backend/core/services/desktop_cloud_bundles.py b/src/backend/features/desktop/bridge/bundles.py similarity index 88% rename from src/backend/core/services/desktop_cloud_bundles.py rename to src/backend/features/desktop/bridge/bundles.py index 1f1c8365..ab6578de 100644 --- a/src/backend/core/services/desktop_cloud_bundles.py +++ b/src/backend/features/desktop/bridge/bundles.py @@ -8,16 +8,18 @@ """ from __future__ import annotations +from features.desktop.capabilities import registry_types import logging import threading from typing import Any, Dict, List, Optional -from core.capabilities import plugins, registry, store, manifest_order -from core.capabilities.errors import CloudUnavailable, IntegrityFailed -from core.capabilities.paths import KIND_AGENT, KIND_PLUGIN, capabilities_enabled, revision_for_hash -from core.capabilities.ref import cloud_ref, profile_id -from core.services.desktop_capability_protocol import entity_content_hash, entity_id_key +from features.desktop.capabilities import plugins, registry, store, manifest_order +from core.harness.capability_errors import CloudUnavailable, IntegrityFailed +from core.capabilities.identity_rules import KIND_AGENT, KIND_PLUGIN +from features.desktop.capabilities.paths import capabilities_enabled, revision_for_hash +from core.capabilities.reference import cloud_ref, profile_id +from core.capabilities.manifest_protocol import entity_content_hash, entity_id_key logger = logging.getLogger(__name__) @@ -29,7 +31,7 @@ def _profile(state: Dict[str, Any]) -> str: - from core.services.desktop_capability_protocol import token_subject + from core.capabilities.manifest_protocol import token_subject subject = token_subject(str(state.get("token") or "")) if not subject: @@ -38,14 +40,14 @@ def _profile(state: Dict[str, Any]) -> str: def _headers(state: Dict[str, Any]) -> Dict[str, str]: - from core.services.desktop_cloud_bridge import cloud_headers + from features.desktop.bridge.identity import cloud_headers return cloud_headers(state) def _fetch(kind: str, state: Dict[str, Any]) -> Dict[str, Any]: import httpx - from core.services.desktop_capability_protocol import validate_entity_manifest + from core.capabilities.manifest_protocol import validate_entity_manifest ticket = manifest_order.begin(kind, _profile(state)) with _lock: @@ -89,7 +91,7 @@ def _dir_hash(comp: store.StoredComponent) -> str: def _text_files(comp: store.StoredComponent): - from core.capabilities import archive + from core.capabilities import archive as archive for rel, path in archive.iter_files(comp.path): if rel == ".inventory.json": @@ -97,8 +99,8 @@ def _text_files(comp: store.StoredComponent): yield rel, path -def _prepare(kind: str, state: Dict[str, Any], inst: registry.Installation) -> None: - from core.capabilities.preparation import prepare_component +def _prepare(kind: str, state: Dict[str, Any], inst: registry_types.Installation) -> None: + from features.desktop.capabilities.preparation import prepare_component remote_id = str(inst.payload.get("cloud_install_id") or inst.key) @@ -110,7 +112,7 @@ def after_publish(comp): ) # 云端同步下来的插件同样要在本机铺它自己的资产：站点插件的建站流程要在 # 本机跑 init-react-site.sh，插件从云端来不代表这一步可以省。 - from core.plugins.local.assets import provision_for + from features.desktop.local.plugins.assets import provision_for provision_for(inst.key) @@ -125,7 +127,7 @@ def after_publish(comp): def prepare(state: Dict[str, Any], install_ids: List[str]) -> List[Dict[str, Any]]: """Prepare selected agent/plugin definitions using the same verified publication.""" - from core.capabilities.errors import PackageMissing + from core.harness.capability_errors import PackageMissing results = [] for iid in install_ids: @@ -151,7 +153,7 @@ def _reconcile( kind: str, manifest: Dict[str, Any], state: Dict[str, Any], *, prepare: bool = True ) -> int: """Publish all current intentions before any optional package download.""" - from core.services.desktop_cloud_bridge import account_scope + from features.desktop.bridge.identity import account_scope profile = _profile(state) with account_scope(state): @@ -193,7 +195,7 @@ def prepare_item(inst): def _apply_intent( kind: str, manifest: Dict[str, Any], state: Dict[str, Any] -) -> List[registry.Installation]: +) -> List[registry_types.Installation]: profile = _profile(state) cloud_base = str(state["cloud_base"]) id_key = entity_id_key(kind) @@ -242,7 +244,7 @@ def sync_kind(kind: str, state: Dict[str, Any]) -> bool: _errors[kind] = "capability store disabled (HUGAGENT_CAPS_ROOT unset)" return False try: - from core.services.desktop_cloud_bridge import account_scope + from features.desktop.bridge.identity import account_scope manifest = _fetch(kind, state) with _lock: @@ -253,9 +255,7 @@ def sync_kind(kind: str, state: Dict[str, Any]) -> bool: except manifest_order.StaleManifest: return False except Exception as exc: # noqa: BLE001 - from core.services.desktop_cloud_bridge import ( - _state_fingerprint, get_state, note_rejected_credential, - ) + from features.desktop.bridge.identity import state_fingerprint as _state_fingerprint, get_state, note_rejected_credential note_rejected_credential(state, exc) if _state_fingerprint(get_state()) != _state_fingerprint(state): @@ -280,7 +280,7 @@ def sync_blocking(state: Dict[str, Any]) -> None: for kind in (KIND_AGENT, KIND_PLUGIN): changed = sync_kind(kind, state) or changed if changed: - from core.config.catalog_resolver import invalidate_capability_cache + from core.capabilities.catalog.resolver import invalidate_capability_cache invalidate_capability_cache() @@ -301,7 +301,7 @@ def synced_manifests() -> Dict[str, Optional[Dict[str, Any]]]: def status() -> Dict[str, Any]: - from core.services.desktop_cloud_bridge import get_state + from features.desktop.bridge.identity import get_state st = get_state() out: Dict[str, Any] = {} diff --git a/src/backend/features/desktop/bridge/client.py b/src/backend/features/desktop/bridge/client.py new file mode 100644 index 00000000..863d1f10 --- /dev/null +++ b/src/backend/features/desktop/bridge/client.py @@ -0,0 +1,214 @@ +"""client responsibilities.""" +from __future__ import annotations +from features.desktop.capabilities import skills +import threading +import time +from typing import Any, Dict, Optional +import logging +logger = logging.getLogger(__name__) +from features.desktop.bridge import client_state as bridge_client_state +from features.desktop.bridge import identity as bridge_identity +from features.desktop.bridge import managed_connectors as bridge_managed_connectors + +def notify_cloud_changed() -> None: + """云端能力可能已变（前端写操作 / 网关告知）：后台同步一次清单与本机文件。""" + refresh_manifest_async(force=True) + + +def set_state( + cloud_base: str, + token: str, + expires_in: int, + *, + device_id: Optional[str] = None, + authorization_epoch: Optional[int] = None, +) -> None: + """壳侧推送桥配置（幂等）。立即触发一次后台 manifest 刷新。""" + pass # State is owned by its responsibility module. + payload = { + "cloud_base": cloud_base.strip().rstrip("/"), + "token": token.strip(), + "expires_at": time.time() + max(1, int(expires_in or 0)), + "device_id": device_id, + "authorization_epoch": authorization_epoch, + } + from core.capabilities.manifest_protocol import token_claims + + claims = token_claims(token) + if device_id and claims.get("d") and device_id != claims["d"]: + raise ValueError("desktop device identity does not match token") + if claims.get("e"): + payload["expires_at"] = min(payload["expires_at"], float(claims["e"])) + with bridge_client_state.state_lock: + changed = bridge_identity.state_fingerprint(bridge_client_state.state) != bridge_identity.state_fingerprint(payload) + renewed_after_rejection = not changed and bridge_client_state.credential_rejected == bridge_identity.state_fingerprint(payload) + bridge_client_state.credential_rejected = "" + bridge_client_state.state = payload + bridge_client_state.state_loaded = True + if changed: + bridge_identity.clear_partial_selection() + # Never expose a previous cloud account's tools while the replacement + # dynamic manifest is still in flight. Skill files stay in the previous + # account's isolated profile directory; only the runtime view changes. + with bridge_client_state.manifest_lock: + bridge_client_state.manifest = None + bridge_client_state.manifest_ts = 0.0 + bridge_client_state.manifest_error = None + from features.desktop.bridge import bundles as desktop_cloud_bundles, skills as desktop_cloud_skills + + desktop_cloud_skills.on_account_switch() + desktop_cloud_bundles.on_account_switch() + bridge_identity.purge_persisted_state() + if changed: + bridge_identity.rebuild_identity_views() + refresh_manifest_async(force=True) + elif renewed_after_rejection: + refresh_manifest_async(force=True) + + +def synced_manifest() -> Optional[Dict[str, Any]]: + """Read the last accepted connector snapshot without starting a refresh.""" + import copy + + with bridge_client_state.manifest_lock: + return copy.deepcopy(bridge_client_state.manifest) + + +def sync_tool_manifest_blocking(st: Dict[str, Any]) -> None: + """Refresh connectors for both login and explicit synchronization.""" + with bridge_client_state.tool_sync_lock: + sync_tool_manifest(st) + + +def sync_tool_manifest(st: Dict[str, Any]) -> None: + pass # State is owned by its responsibility module. + from core.harness.capability_errors import CloudUnavailable + + bridge_identity.require_current_account(st) + url = f"{st['cloud_base']}/api/v1/desktop/capability/manifest" + import httpx + from core.capabilities.manifest_protocol import validate_manifest + + headers = {**bridge_identity.cloud_headers(st), "Cache-Control": "no-cache"} + with bridge_client_state.manifest_lock: + current_revision = str((bridge_client_state.manifest or {}).get("revision") or "") + if current_revision: + headers["If-None-Match"] = f'"{current_revision}"' + + resp = httpx.get( + url, + headers=headers, + timeout=httpx.Timeout(10.0, connect=5.0), + ) + bridge_identity.require_current_account(st) + if resp.status_code in (401, 403): + with bridge_identity.account_scope(st): + bridge_identity.clear_state() + raise CloudUnavailable("cloud authorization unavailable") + if resp.status_code == 304: + with bridge_identity.account_scope(st), bridge_client_state.manifest_lock: + bridge_client_state.manifest_ts = time.monotonic() + bridge_client_state.manifest_error = None + else: + resp.raise_for_status() + body = resp.json() + data = body.get("data") if isinstance(body, dict) else None + manifest = validate_manifest(data) + with bridge_identity.account_scope(st): + with bridge_client_state.manifest_lock: + bridge_client_state.manifest = manifest + bridge_client_state.manifest_ts = time.monotonic() + bridge_client_state.manifest_error = None + bridge_managed_connectors.project_managed_profile(st, manifest) + logger.info( + "[cloud-bridge] manifest refreshed revision=%s servers=%d", + manifest["revision"][:12], + len(manifest["servers"]), + ) + + +def fetch_manifest_blocking(st: Dict[str, Any]) -> None: + pass # State is owned by its responsibility module. + refresh_replacement = False + try: + sync_tool_manifest_blocking(st) + # Skills, agents and plugins ride the same poll and token: their intent + # is reconciled right after the tool manifest is known to be current. + sync_capabilities_blocking(st) + except Exception as exc: # noqa: BLE001 + with bridge_client_state.state_lock: + if bridge_identity.state_fingerprint(bridge_identity.get_state()) != bridge_identity.state_fingerprint(st): + refresh_replacement = True + return + with bridge_client_state.manifest_lock: + bridge_client_state.manifest_ts = time.monotonic() + bridge_client_state.manifest_error = str(exc) + logger.warning("[cloud-bridge] manifest 拉取失败: %s", exc) + finally: + with bridge_client_state.manifest_lock: + bridge_client_state.manifest_fetching = False + rerun = bridge_client_state.refresh_pending + bridge_client_state.refresh_pending = False + if refresh_replacement or rerun: + refresh_manifest_async(force=True) + + +def sync_capabilities_blocking(st: Dict[str, Any]) -> None: + """One pass over every cloud-synced kind (skills, agents, plugins). Blocking; call off-loop. + + Both the background manifest poll and the user-triggered sync endpoint go + through here so no kind can be forgotten by one of the two paths. + """ + from features.desktop.bridge import bundles as desktop_cloud_bundles, skills as desktop_cloud_skills + + desktop_cloud_skills.sync_blocking(st) + desktop_cloud_bundles.sync_blocking(st) + try: + from features.desktop.capabilities.warmup import warmup_current_account + + warmup_current_account(st) + except Exception as exc: # Best effort: ordinary execution retains all checks. + logger.warning("[caps] startup reads deferred: %s", type(exc).__name__) + + +def refresh_manifest_async(force: bool = False) -> None: + """后台同步一次云端清单。没有定时器：``force`` 由登录 / 变更通知触发，非 + ``force`` 只是「还没有可用清单」时的有界重试（离线登录后的自愈），一旦拿到 + 清单就不再有任何后台请求。""" + pass # State is owned by its responsibility module. + st = bridge_identity.get_state() + if not st: + return + with bridge_client_state.manifest_lock: + if bridge_client_state.manifest_fetching: + bridge_client_state.refresh_pending = bridge_client_state.refresh_pending or force + return + if not force: + if bridge_client_state.manifest is not None and not bridge_client_state.manifest_error: + return + if bridge_client_state.manifest_ts and time.monotonic() - bridge_client_state.manifest_ts < bridge_client_state.MANIFEST_NEG_TTL_S: + return + bridge_client_state.manifest_fetching = True + threading.Thread( + target=fetch_manifest_blocking, args=(st,), name="cloud-bridge-manifest", daemon=True + ).start() + + +def get_cached_manifest() -> Optional[Dict[str, Any]]: + """返回缓存的 manifest（可能为 None）。调用方只读，不得修改。 + + 还没有清单时（离线登录等）触发一次有界重试；已有清单则不发任何请求，更新 + 靠登录与变更通知。不做激活判定——守门统一在公共入口（``_bridge_context``）。 + 清单只在 ``_fetch_manifest_blocking`` 里整体替换，从不原地修改，所以可以安全共享。 + """ + refresh_manifest_async() + with bridge_client_state.manifest_lock: + return bridge_client_state.manifest + + +def retry_initial_sync(st): + from features.desktop.capabilities import skills + with bridge_identity.account_scope(st): + bridge_identity.clear_partial_selection() + skills.bump_view_generation() + refresh_manifest_async(force=True) diff --git a/src/backend/features/desktop/bridge/client_state.py b/src/backend/features/desktop/bridge/client_state.py new file mode 100644 index 00000000..d2065f81 --- /dev/null +++ b/src/backend/features/desktop/bridge/client_state.py @@ -0,0 +1,48 @@ +"""client state responsibilities.""" +from __future__ import annotations +import threading +from typing import Any, Dict, Optional, Tuple + +BRIDGE_BLOCK_ID = "desktop_cloud_bridge" + + +MANIFEST_NEG_TTL_S = 30.0 + + +state_lock = threading.RLock() + + +state: Optional[Dict[str, Any]] = None # {"cloud_base", "token", "expires_at"} + + +credential_rejected = "" # account fingerprint whose current token the cloud refused + + +state_loaded = False + + +manifest_lock = threading.Lock() + + +tool_sync_lock = threading.Lock() + + +manifest: Optional[Dict[str, Any]] = None + + +manifest_ts: float = 0.0 + + +manifest_error: Optional[str] = None + + +manifest_fetching = False + + +refresh_pending = False # a forced refresh arrived while a pass was running + + +context_lock = threading.Lock() + + +context_cache: Optional[Tuple[tuple, Dict[str, Any]]] = None diff --git a/src/backend/features/desktop/bridge/gateway.py b/src/backend/features/desktop/bridge/gateway.py new file mode 100644 index 00000000..565f0cf6 --- /dev/null +++ b/src/backend/features/desktop/bridge/gateway.py @@ -0,0 +1,225 @@ +"""gateway responsibilities.""" +from __future__ import annotations +from features.desktop.bridge import client as bridge_client +from typing import Any, Dict, List, Optional, Set +from features.desktop.auth import bridge as bridge_auth +import logging +logger = logging.getLogger(__name__) +from features.desktop.bridge import client as bridge_client +from features.desktop.bridge import client_state as bridge_client_state +from features.desktop.bridge import identity as bridge_identity +from features.desktop.bridge import managed_connectors as bridge_managed_connectors + +def bridge_context() -> Optional[Dict[str, Any]]: + """唯一守门点：桥激活且 manifest 就绪时返回上下文，否则 None。 + + 返回 {"state": st, "servers": [含 component 的云端 server，已剔除 mcp.json 里 + 用户停用的项], "profile": 账号 profile}。同名裁决交给解析器。 + server 列表按 (账号, 清单 revision, 停用集合) 记忆化；``state`` 每次现取， + 因为它携带轮换中的 token。 + """ + pass # State is owned by its responsibility module. + if not bridge_auth.bridge_enabled(): + return None + st = bridge_identity.get_state() + if not st: + return None + manifest = bridge_client.get_cached_manifest() + if not manifest: + return None + enabled = bridge_managed_connectors.managed_enabled(st) + key = (bridge_identity.state_fingerprint(st), str(manifest["revision"]), tuple(sorted(enabled.items()))) + with bridge_client_state.context_lock: + hit = bridge_client_state.context_cache + if hit is None or hit[0] != key: + servers: List[Dict[str, Any]] = [] + for s in manifest.get("servers") or []: + if not isinstance(s, dict): + continue + # A server with no current tool contracts cannot replace a local + # implementation. The manifest validator already guarantees every + # non-empty entry contains complete, cloud-supplied schemas. + if not s.get("tools"): + continue + sid = str(s.get("server_id") or "").strip() + if not sid or not enabled.get(sid, True): + continue + entry = dict(s) + entry["server_id"] = sid + entry["component"] = str(s.get("component") or sid).strip() + servers.append(entry) + hit = ( + key, + { + "servers": servers, + "manifest_revision": str(manifest["revision"]), + "profile": bridge_managed_connectors.account_profile(st), + }, + ) + with bridge_client_state.context_lock: + bridge_client_state.context_cache = hit + if not hit[1]["servers"]: + return None + return {"state": st, **hit[1]} + + +def mcp_json_local_configs() -> Dict[str, dict]: + from features.desktop.capabilities import mcp_json + from features.desktop.capabilities.paths import capabilities_enabled + + if not capabilities_enabled(): + return {} + try: + return mcp_json.local_server_configs() + except mcp_json.McpJsonError as exc: + logger.warning("[cloud-bridge] mcp.json local servers unavailable: %s", exc) + return {} + + +def cloud_gateway_mcp_configs( + enabled_mcp_ids: Optional[List[str]] = None, *, resolution_out=None, unavailable_out=None +) -> Dict[str, dict]: + """Desktop-only MCP config sources: cloud gateway bindings + mcp.json local servers. + + Only the components the resolver chose for the cloud binding get a gateway + config; the run's enabled-id allowlist still collapses the final set. + """ + from features.desktop.capabilities import connectors + + configs: Dict[str, dict] = dict(mcp_json_local_configs()) + ctx = bridge_context() + res = resolve_mcp_bindings(enabled_mcp_ids, ctx) + if resolution_out is not None: + resolution_out.append(res) + requested = set(enabled_mcp_ids or []) + for name, group in {**res.conflicts, **res.unusable}.items(): + if name in requested or any(connectors.server_id_of(c) in requested for c in group): + from core.harness.capability_errors import NameConflict, PackageMissing + + error = NameConflict if name in res.conflicts else PackageMissing + if unavailable_out is not None: + unavailable_out.update({connectors.server_id_of(c): error.code for c in group}) + continue + raise error( + "selected connector binding is unavailable; choose its source", runtime_name=name + ) + chosen = res.chosen_ids() + configs = { + sid: cfg + for sid, cfg in configs.items() + if f"mcp:{connectors.MCP_JSON_PROFILE}:{sid}" in chosen + } + if not ctx: + return configs + st = ctx["state"] + manifest_revision = ctx["manifest_revision"] + for s in ctx["servers"]: + # Source selection is complete; only the exact authorized cloud binding + # chosen by the resolver may produce a gateway config. + sid = s["server_id"] + if f"mcp:{ctx['profile']}:{sid}" not in chosen: + continue + from features.desktop.bridge.resource_plugins import local_config + from features.desktop.capabilities.skill_account import current_local_user_id + native = local_config(s, current_local_user_id(), ctx["profile"]) + if native is not None: + configs[sid] = native + continue + invoke_url = f"{st['cloud_base']}/api/v1/desktop/capability/gateway/{sid}/call" + configs[sid] = { + "transport": "streamable_http", + # This endpoint invokes the authorized manifest tool through JSON. + "url": invoke_url, + "headers": bridge_identity.cloud_headers(st), + "execution_timeout": 180, + "schema_source": "cloud_manifest", + "manifest_revision": manifest_revision, + "manifest_tools": s["tools"], + "schema_hash": str(s["schema_hash"]), + "gateway_invoke_url": invoke_url, + "gateway_plugin": str(s.get("source_plugin") or ""), + } + return configs + + +def local_server_ids() -> Set[str]: + """本机 DB 内全部 MCP 的 server_id。 + + 复用 ``McpServerConfigService`` 自带的 30s 缓存与失效链路，不另建缓存。 + """ + try: + from core.capabilities.mcp.service import McpServerConfigService + + return set(McpServerConfigService.get_instance().get_all_servers(enabled_only=False)) + except Exception as exc: # noqa: BLE001 + logger.debug("[cloud-bridge] local server ids failed: %s", exc) + return set() + + +def apply_to_enabled_mcp_ids(mcp_ids: Optional[List[str]]) -> Optional[List[str]]: + """把本轮 enabled_mcp_ids 交给解析器，按 server_id 裁决每个连接器的绑定。 + + 候选：本机目录行（device）、云端 manifest（account）、mcp.json 本机声明。 + 同一 server_id 多个候选时，云端账号级候选默认胜出， + 用户可用同名偏好显式改选；落选绑定记为 shadowed，可在界面 + 切换，而不是被静默顶掉。云端 manifest 尚未就绪 / 桥未激活：原样返回。 + 顺序稳定（已选项按原顺序，新增云端与 mcp.json 项追加），幂等。 + """ + if mcp_ids is None: + return None + ctx = bridge_context() + from features.desktop.capabilities import connectors + + local_ids = local_server_ids() + json_local = mcp_json_local_declarations() + res = resolve_mcp_bindings(mcp_ids, ctx) + + chosen_ids = {connectors.server_id_of(c) for c in res.chosen.values()} + known = local_ids | {s["server_id"] for s in (ctx or {}).get("servers", [])} | set(json_local) + kept = [mid for mid in mcp_ids if mid in chosen_ids or mid not in known] + existing = set(kept) + for name in sorted(res.chosen): + sid = connectors.server_id_of(res.chosen[name]) + if sid not in existing: + kept.append(sid) + existing.add(sid) + return kept + + +def resolve_mcp_bindings(mcp_ids: Optional[List[str]], ctx): + from features.desktop.capabilities import connectors + + local_ids = local_server_ids() + enabled = set(local_ids) if mcp_ids is None else set(mcp_ids) + candidates = connectors.db_candidates(local_ids, enabled) + connectors.json_candidates( + mcp_json_local_declarations() + ) + if ctx: + candidates += connectors.cloud_candidates(ctx["profile"], ctx["servers"], {}) + return connectors.resolve_bindings(candidates) + + +def mcp_json_local_declarations() -> Dict[str, Dict[str, Any]]: + from features.desktop.capabilities import mcp_json + from features.desktop.capabilities.paths import capabilities_enabled + + if not capabilities_enabled(): + return {} + try: + return mcp_json.load().local + except mcp_json.McpJsonError: + return {} + + +def apply_to_enabled_skill_ids(skill_ids: Optional[List[str]]) -> Optional[List[str]]: + """把云端技能合并进本轮 enabled_skill_ids（云端为真源，见 desktop_cloud_skills）。""" + if skill_ids is None or not bridge_identity.bridge_active(): + return skill_ids + from features.desktop.bridge import skills as desktop_cloud_skills + + return desktop_cloud_skills.apply_to_enabled_skill_ids(skill_ids) + + +def local_plugin_configs(user_id, enabled_only=True): + from features.desktop.capabilities.local_plugin_runtime import configs + return configs(user_id, enabled_only=enabled_only) diff --git a/src/backend/features/desktop/bridge/identity.py b/src/backend/features/desktop/bridge/identity.py new file mode 100644 index 00000000..d94458e6 --- /dev/null +++ b/src/backend/features/desktop/bridge/identity.py @@ -0,0 +1,254 @@ +"""identity responsibilities.""" +from __future__ import annotations +from features.desktop.capabilities import skill_paths +from features.desktop.capabilities import skill_views +from contextlib import contextmanager +import hashlib +import time +from typing import Any, Dict, Optional +from features.desktop.auth import bridge as bridge_auth +import logging +logger = logging.getLogger(__name__) +from features.desktop.bridge import client_state as bridge_client_state + +def state_fingerprint(state: Optional[Dict[str, Any]]) -> str: + """Bind a manifest snapshot to exactly one cloud/token identity.""" + if not state: + return "" + from core.capabilities.manifest_protocol import token_subject, token_claims + + token = str(state.get("token") or "") + # A rotated token for the same account keeps the snapshot; only a real + # account switch (or an opaque token) invalidates it. + claims = token_claims(token) + payload = f"{state.get('cloud_base') or ''}\0{token_subject(token) or token}\0{claims.get('a', '')}\0{claims.get('h', '')}\0{claims.get('d', state.get('device_id') or '')}" + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + + +def require_current_account(state: Dict[str, Any]) -> None: + """Reject work captured before logout/account switch, including late HTTP replies.""" + from core.harness.capability_errors import CloudUnavailable + + if state_fingerprint(get_state()) != state_fingerprint(state): + raise CloudUnavailable("cloud account changed; retry with the current account") + + +@contextmanager +def account_scope(state: Dict[str, Any]): + """Serialize a short publication with identity changes; never hold across network IO.""" + with bridge_client_state.state_lock: + require_current_account(state) + yield + + +def ensure_current_authorization(state: Optional[Dict[str, Any]] = None) -> None: + """账号仍然有效才允许使用私有字节；这条判定不发网络请求。 + + 本机不做「事前探测」——云端清单只在登录、云端能力被改动（前端写操作后调 + ``POST /v1/desktop/capabilities/sync``）以及云端在网关调用里明确告知能力已变 + 这三种事件下同步，没有任何定时轮询。真正的撤权由云端在网关调用时裁决：返回 + 401/403 会立刻清掉本机的桥状态，被撤权的能力下一次调用即失败。 + """ + from core.harness.capability_errors import CloudUnavailable + + st = state or get_state() + if not st: + raise CloudUnavailable("cloud authorization expired; sign in again") + require_current_account(st) + + +def purge_persisted_state() -> None: + """Remove credentials left by an earlier release; bridge tokens are memory-only.""" + try: + from features.desktop.bridge.model_credentials import scrub_legacy_rows + + scrub_legacy_rows() + except Exception as exc: + logger.warning( + "[cloud-bridge] legacy model credential cleanup unavailable: %s", type(exc).__name__ + ) + try: + from core.db.engine import SessionLocal + from core.db.models import ContentBlock + + with SessionLocal() as db: + row = db.get(ContentBlock, bridge_client_state.BRIDGE_BLOCK_ID) + if row is not None: + db.delete(row) + db.commit() + except Exception as exc: + logger.warning( + "[cloud-bridge] legacy credential cleanup unavailable: %s", type(exc).__name__ + ) + + +def load_state_from_db() -> None: + purge_persisted_state() + return None + + +def current_credentials(state: Dict[str, Any]) -> Dict[str, Any]: + """The credential belongs to the account, not to the snapshot a pass started with. + + A sync pass can outlive the token it started with while the shell has already + renewed it. Same account → newest token; another account → untouched, so the + surrounding identity checks reject it. + """ + with bridge_client_state.state_lock: + live = bridge_client_state.state + if live and state_fingerprint(live) == state_fingerprint(state): + return live + return state + + +def note_rejected_credential(state: Dict[str, Any], exc: BaseException) -> None: + """Only a refusal of the live token schedules re-sync on its next renewal.""" + pass # State is owned by its responsibility module. + import httpx + + if isinstance(exc, httpx.HTTPStatusError) and exc.response.status_code == 401: + authorization = exc.request.headers.get("Authorization") + with bridge_client_state.state_lock: + if ( + bridge_client_state.state + and authorization == f"Bearer {bridge_client_state.state.get('token')}" + and state_fingerprint(bridge_client_state.state) == state_fingerprint(state) + ): + bridge_client_state.credential_rejected = state_fingerprint(state) + + +def cloud_headers(state: Dict[str, Any]) -> Dict[str, str]: + from core.capabilities.manifest_protocol import token_claims + + state = current_credentials(state) + device_id = str( + state.get("device_id") or token_claims(str(state.get("token") or "")).get("d") or "" + ) + headers = {"Authorization": f"Bearer {state['token']}"} + if device_id: + headers["X-Desktop-Device-Id"] = device_id + return headers + + +def clear_state() -> None: + """Logout invalidates the account immediately, including in-flight responses.""" + pass # State is owned by its responsibility module. + with bridge_client_state.state_lock: + bridge_client_state.state, bridge_client_state.state_loaded = None, True + bridge_client_state.credential_rejected = "" + clear_partial_selection() + with bridge_client_state.manifest_lock: + bridge_client_state.manifest, bridge_client_state.manifest_error, bridge_client_state.manifest_ts = None, None, 0.0 + from features.desktop.bridge import bundles as desktop_cloud_bundles, skills as desktop_cloud_skills + + desktop_cloud_skills.on_account_switch() + desktop_cloud_bundles.on_account_switch() + purge_persisted_state() + rebuild_identity_views() + + +def rebuild_identity_views() -> None: + from features.desktop.capabilities import skills + from features.desktop.capabilities.paths import capabilities_enabled + + if not capabilities_enabled(): + return + from core.capabilities.skills.cache_refresh import refresh_skill_caches + + refresh_skill_caches() + shared = skill_paths.device_view_dir() + root = shared.parent / (shared.name + "_u") + if root.is_dir(): + for child in root.iterdir(): + if child.is_dir(): + skill_views.rebuild_user_view(child.name) + + +def get_state() -> Optional[Dict[str, Any]]: + pass # State is owned by its responsibility module. + with bridge_client_state.state_lock: + if not bridge_client_state.state_loaded: + bridge_client_state.state = load_state_from_db() + bridge_client_state.state_loaded = True + st = bridge_client_state.state + if not st: + return None + if float(st.get("expires_at") or 0) < time.time(): + return None + return dict(st) + + +def shell_user_center_id(cloud_base: str, user_center_id: str) -> str: + """壳送来的桥接用户标识：``cloud:::<云端 user_center_id>``。 + + 壳按云端地址给用户加命名空间，同一台机器连不同云端时本机身份天然隔离。本机侧 + 用完全相同的规则从凭据里的云端身份推出壳会送来的标识，两者必须逐字相同。 + 端口缺省与壳一致：https 443、其它 80。 + """ + from urllib.parse import urlsplit + + parts = urlsplit(cloud_base.strip()) + host = parts.hostname or "cloud" + port = parts.port or (443 if parts.scheme == "https" else 80) + return f"cloud:{host}:{port}:{user_center_id}" + + +def get_identity_state() -> Optional[Dict[str, Any]]: + """Last shell-authenticated identity for offline local resources; never a cloud grant.""" + from core.capabilities.manifest_protocol import token_claims + + with bridge_client_state.state_lock: + st = dict(bridge_client_state.state) if bridge_client_state.state else None + if not st: + return None + claims = token_claims(str(st.get("token") or "")) + if not claims.get("c"): + return None + return { + "cloud_base": st["cloud_base"], + "user_center_id": str(claims["c"]), + "shell_user_center_id": shell_user_center_id(str(st["cloud_base"]), str(claims["c"])), + "subject": str(claims.get("u") or ""), + "device_id": str(claims.get("d") or ""), + "authorization_epoch": claims.get("a"), + } + + +def bridge_active() -> bool: + """本进程是否应启用云端能力桥（仅桌面壳孵化的双端本机后端为 True）。""" + return bridge_auth.bridge_enabled() and get_state() is not None + + +def clear_partial_selection(): + from features.desktop.capabilities import session_availability, registry + if session_availability.active(): + session_availability.clear() + registry._bump() # Runtime selection changed even though cloud intent did not. + + +def reset_for_tests() -> None: # pragma: no cover - 仅测试用 + pass # State is owned by its responsibility module. + pass # State is owned by its responsibility module. + from features.desktop.bridge import skills as desktop_cloud_skills + + with bridge_client_state.state_lock: + bridge_client_state.state = None + bridge_client_state.state_loaded = False + bridge_client_state.credential_rejected = "" + with bridge_client_state.manifest_lock: + bridge_client_state.manifest = None + bridge_client_state.manifest_ts = 0.0 + bridge_client_state.manifest_error = None + bridge_client_state.manifest_fetching = False + bridge_client_state.refresh_pending = False + desktop_cloud_skills.reset_for_tests() + + +def context_headers(*, model=False): + from features.desktop.observability.context import context_headers as correlate + return correlate(model=model) + + +def notify_cloud_changed(): + from features.desktop.bridge.client import notify_cloud_changed as notify + return notify() diff --git a/src/backend/features/desktop/bridge/managed_connectors.py b/src/backend/features/desktop/bridge/managed_connectors.py new file mode 100644 index 00000000..d5a0ee70 --- /dev/null +++ b/src/backend/features/desktop/bridge/managed_connectors.py @@ -0,0 +1,144 @@ +"""managed connectors responsibilities.""" +from __future__ import annotations +from typing import Any, Dict, List +from features.desktop.auth import bridge as bridge_auth +import logging +logger = logging.getLogger(__name__) +from features.desktop.bridge import client_state as bridge_client_state +from features.desktop.bridge import identity as bridge_identity + +def account_profile(st: Dict[str, Any]) -> str: + """Profile label of the bridged account (opaque tokens get a deterministic label).""" + from core.capabilities.reference import profile_id + from core.capabilities.manifest_protocol import token_subject + + return profile_id( + str(st["cloud_base"]), token_subject(str(st.get("token") or "")) or "opaque-subject" + ) + + +def project_managed_profile(st: Dict[str, Any], manifest: Dict[str, Any]) -> None: + """Write the account's cloud connector references into mcp.json (managedProfiles).""" + from features.desktop.capabilities import mcp_json + from features.desktop.capabilities.paths import capabilities_enabled + from core.capabilities.reference import cloud_issuer + + if not capabilities_enabled(): + return + try: + mcp_json.project_managed_profile( + account_profile(st), + cloud_instance_id=cloud_issuer(str(st["cloud_base"])), + catalog_revision=str(manifest["revision"]), + servers=list(manifest.get("servers") or []), + initial_enabled={ + str(s["server_id"]): bool(s.get("enabled", True)) + for s in manifest.get("servers") or [] + if isinstance(s, dict) and s.get("server_id") + }, + ) + except mcp_json.McpJsonCorrupt as exc: + logger.error("[cloud-bridge] mcp.json unreadable, projection skipped: %s", exc) + except Exception as exc: # noqa: BLE001 + logger.warning("[cloud-bridge] mcp.json projection failed: %s", exc) + + +def managed_enabled(st: Dict[str, Any]) -> Dict[str, bool]: + from features.desktop.capabilities import mcp_json + from features.desktop.capabilities.paths import capabilities_enabled + + if not capabilities_enabled(): + return {} + try: + return mcp_json.managed_enabled(account_profile(st)) + except mcp_json.McpJsonError as exc: + logger.warning("[cloud-bridge] mcp.json managed flags unavailable: %s", exc) + return {} + + +def managed_connectors() -> List[Dict[str, Any]]: + """云端下发的连接器全集（含本机停用的），供本机能力中心展示。 + + ``_bridge_context`` 是装配用的，它按本机停用标志把关掉的过滤掉了——界面要的是 + 「有哪些」，关掉的也得列出来，开关才有地方可点。启停仍以本机 mcp.json 为准。 + + 只读已缓存的清单，不触发任何刷新：这是界面读取路径，同步有它自己的时机。 + """ + st = bridge_identity.get_state() + if not st or not bridge_auth.bridge_enabled(): + return [] + with bridge_client_state.manifest_lock: + manifest = bridge_client_state.manifest + if not manifest: + return [] + enabled = managed_enabled(st) + out: List[Dict[str, Any]] = [] + for server in manifest.get("servers") or []: + if not isinstance(server, dict): + continue + sid = str(server.get("server_id") or "").strip() + if not sid: + continue + out.append( + { + "server_id": sid, + "display_name": str(server.get("display_name") or sid), + "description": str(server.get("description") or ""), + "created_at": server.get("created_at"), + "enabled": bool(enabled.get(sid, True)), + "source_plugin": str(server.get("source_plugin") or ""), + "icon": str(server.get("icon") or ""), + "tools": [ + { + "name": str(tool.get("name") or ""), + "description": str(tool.get("description") or ""), + } + for tool in server.get("tools") or [] + if isinstance(tool, dict) + ], + } + ) + return out + + +def set_managed_connector_enabled(server_id: str, enabled: bool) -> bool: + """记下用户对某个云端下发连接器的启停，落本机 mcp.json。 + + 本机是这个开关的唯一记账处：装配走 ``_bridge_context()``，它就按这里的标志 + 过滤，云端不参与也不需要回写。返回 False 表示该 server_id 不是本机托管的 + 云端连接器，调用方应继续按普通目录项处理。 + + 「数据库查询」是个伞形条目（见 ``device_catalog._db_umbrella_item``），本身没有 + 对应的 server，开关要落到它收拢的那几个成员上——否则界面上的开关点了没有任何 + 东西真的被打开。 + """ + from features.desktop.capabilities import mcp_json + from features.desktop.capabilities.paths import capabilities_enabled + from core.capabilities.catalog.loader import DB_HIDDEN_SERVERS, DB_UMBRELLA_ID + + st = bridge_identity.get_state() + if not capabilities_enabled() or not bridge_auth.bridge_enabled() or not st: + return False + profile = account_profile(st) + if not profile: + return False + sid = str(server_id) + if sid == DB_UMBRELLA_ID: + members = [ + str(s["server_id"]) + for s in managed_connectors() + if str(s.get("server_id")) in DB_HIDDEN_SERVERS + ] + if not members: + return False + for member in members: + try: + mcp_json.set_managed_enabled(profile, member, bool(enabled)) + except mcp_json.McpJsonError: + return False + return True + try: + mcp_json.set_managed_enabled(profile, sid, bool(enabled)) + except mcp_json.McpJsonError: + return False + return True diff --git a/src/backend/core/services/desktop_model_credentials.py b/src/backend/features/desktop/bridge/model_credentials.py similarity index 86% rename from src/backend/core/services/desktop_model_credentials.py rename to src/backend/features/desktop/bridge/model_credentials.py index 3d5bc3a2..1f330714 100644 --- a/src/backend/core/services/desktop_model_credentials.py +++ b/src/backend/features/desktop/bridge/model_credentials.py @@ -1,10 +1,11 @@ """Account-bound model gateway references; bearer credentials never enter model rows.""" from __future__ import annotations +from features.desktop.bridge import identity as bridge_identity import copy from urllib.parse import urlsplit -from core.capabilities.errors import CloudUnavailable +from core.harness.capability_errors import CloudUnavailable PREFIX = "desktop-capability:" _GATEWAY = "/api/v1/desktop/capability/gateway/models/" @@ -31,10 +32,10 @@ def gateway_cloud_base(base_url): def sanitize_import(data): """Accept the legacy shell payload at the boundary, persist only a reference.""" - from core.auth.desktop_bridge import bridge_enabled - from core.capabilities.paths import capabilities_enabled - from core.capabilities.ref import profile_id - from core.services.desktop_capability_protocol import token_subject + from features.desktop.auth.bridge import bridge_enabled + from features.desktop.capabilities.paths import capabilities_enabled + from core.capabilities.reference import profile_id + from core.capabilities.manifest_protocol import token_subject out = copy.deepcopy(data) for provider in out.get("providers") or []: @@ -54,11 +55,11 @@ def sanitize_import(data): def bind(reference, base_url): """Capture identity once per model instance; token rotation stays in memory.""" - from core.services import desktop_cloud_bridge as bridge - from core.capabilities.ref import profile_id - from core.services.desktop_capability_protocol import token_subject + from features.desktop.bridge import client as bridge + from core.capabilities.reference import profile_id + from core.capabilities.manifest_protocol import token_subject - state = bridge.get_state() + state = bridge_identity.get_state() if not state or not is_reference(reference): raise CloudUnavailable("sign in again to use the cloud model") expected_base = gateway_cloud_base(base_url) @@ -71,10 +72,10 @@ def bind(reference, base_url): def headers(reference, base_url, captured_state=None): - from core.services import desktop_cloud_bridge as bridge + from features.desktop.bridge import client as bridge captured_state = captured_state or bind(reference, base_url) - bridge.require_current_account(captured_state) + bridge_identity.require_current_account(captured_state) state = bind(reference, base_url) token = str(state.get("token") or "") # The signed device claim is decoded only for routing; the cloud validates it. @@ -86,7 +87,7 @@ def headers(reference, base_url, captured_state=None): device = str(state.get("device_id") or claims.get("device_id") or claims.get("d") or "") except (ValueError, IndexError, TypeError): device = "" - from core.services.desktop_observation_context import context_headers + from features.desktop.observability.context import context_headers return {"Authorization": "Bearer " + token, "X-Desktop-Device-Id": device, **context_headers(model=True)} diff --git a/src/backend/core/services/desktop_release_events.py b/src/backend/features/desktop/bridge/release_events.py similarity index 100% rename from src/backend/core/services/desktop_release_events.py rename to src/backend/features/desktop/bridge/release_events.py diff --git a/src/backend/core/services/desktop_resource_plugins.py b/src/backend/features/desktop/bridge/resource_plugins.py similarity index 86% rename from src/backend/core/services/desktop_resource_plugins.py rename to src/backend/features/desktop/bridge/resource_plugins.py index 45040494..c913f803 100644 --- a/src/backend/core/services/desktop_resource_plugins.py +++ b/src/backend/features/desktop/bridge/resource_plugins.py @@ -8,13 +8,13 @@ def package_root(installed): revision = (installed.get("import_report") or {}).get("package_revision") if revision: from core.db.engine import SessionLocal - from core.plugins.packaging.runtime_assets import materialize + from core.capabilities.plugins.packaging.runtime_assets import materialize with SessionLocal() as db: return materialize(db, revision) - from core.plugins.packaging.sources import _resolve_plugin_dir + from core.capabilities.plugins.packaging.sources import resolve_plugin_dir - return _resolve_plugin_dir(installed["slug"]) + return resolve_plugin_dir(installed["slug"]) def package_files(installed): @@ -62,14 +62,14 @@ def local_config(server, user_id, profile): if not settings.deploy.is_local: return None - from core.capabilities import device_catalog, store - from core.capabilities.plugins import load_manifest - from core.capabilities.dependency import component_hash - from core.capabilities.local_plugin_runtime import enabled_for - from core.config.mcp_config import _mcp_http_url + from features.desktop.capabilities import device_catalog, store + from features.desktop.capabilities.plugins import load_manifest + from features.desktop.capabilities.dependency import component_hash + from features.desktop.capabilities.local_plugin_runtime import enabled_for + from core.config.mcp_config import mcp_http_url as _mcp_http_url from extensions.mcp_servers._ports import PORTS - for inst in device_catalog._installations("plugin", user_id=user_id): + for inst in device_catalog.installations("plugin", user_id=user_id): if inst.profile_id != profile or server["server_id"] not in inst.payload.get( "components", {} ).get("mcp", []): @@ -103,7 +103,6 @@ def local_config(server, user_id, profile): "url": _mcp_http_url(audience), "origin": "local_plugin", "execution_scope": "local", - "is_stable": False, "owner_user_id": user_id, "source_plugin": inst.key, "plugin_install_id": inst.install_id, diff --git a/src/backend/core/services/desktop_cloud_skills.py b/src/backend/features/desktop/bridge/skills.py similarity index 86% rename from src/backend/core/services/desktop_cloud_skills.py rename to src/backend/features/desktop/bridge/skills.py index 60582aaf..e983ec26 100644 --- a/src/backend/core/services/desktop_cloud_skills.py +++ b/src/backend/features/desktop/bridge/skills.py @@ -18,16 +18,21 @@ """ from __future__ import annotations +from features.desktop.capabilities import skill_account +from features.desktop.capabilities import skill_hashes +from features.desktop.capabilities import skill_views +from features.desktop.capabilities import skills import copy import logging import threading from typing import Any, Dict, List, Optional -from core.capabilities import registry, skills, store, manifest_order -from core.capabilities.errors import CloudUnavailable, IntegrityFailed, PackageMissing -from core.capabilities.paths import KIND_SKILL, capabilities_enabled, revision_for_hash -from core.capabilities.ref import cloud_ref, profile_id +from features.desktop.capabilities import registry, skills, store, manifest_order +from core.harness.capability_errors import CloudUnavailable, IntegrityFailed, PackageMissing +from core.capabilities.identity_rules import KIND_SKILL +from features.desktop.capabilities.paths import capabilities_enabled, revision_for_hash +from core.capabilities.reference import cloud_ref, profile_id logger = logging.getLogger(__name__) @@ -38,7 +43,7 @@ def _profile(state: Dict[str, Any]) -> str: - from core.services.desktop_capability_protocol import token_subject + from core.capabilities.manifest_protocol import token_subject subject = token_subject(str(state.get("token") or "")) if not subject: @@ -47,13 +52,13 @@ def _profile(state: Dict[str, Any]) -> str: def _headers(state: Dict[str, Any]) -> Dict[str, str]: - from core.services.desktop_cloud_bridge import cloud_headers + from features.desktop.bridge.identity import cloud_headers return cloud_headers(state) def _fetch_manifest(state: Dict[str, Any]) -> Dict[str, Any]: import httpx - from core.services.desktop_capability_protocol import validate_skill_manifest + from core.capabilities.manifest_protocol import validate_skill_manifest ticket = manifest_order.begin(KIND_SKILL, _profile(state)) with _lock: @@ -76,7 +81,7 @@ def _fetch_manifest(state: Dict[str, Any]) -> Dict[str, Any]: def _reconcile_intent(manifest: Dict[str, Any], state: Dict[str, Any]) -> List[str]: """Atomically gate intent/cache publication against newer in-flight requests.""" global _manifest, _error - from core.services.desktop_cloud_bridge import account_scope + from features.desktop.bridge.identity import account_scope with account_scope(state): with manifest_order.apply(KIND_SKILL, _profile(state), manifest): @@ -132,7 +137,7 @@ def sync_blocking(state: Dict[str, Any]) -> None: return with _sync_lock: try: - from core.services.desktop_cloud_bridge import account_scope + from features.desktop.bridge.identity import account_scope manifest = _fetch_manifest(state) with account_scope(state): @@ -143,9 +148,7 @@ def sync_blocking(state: Dict[str, Any]) -> None: except manifest_order.StaleManifest: return except Exception as exc: # noqa: BLE001 - from core.services.desktop_cloud_bridge import ( - _state_fingerprint, get_state, note_rejected_credential, - ) + from features.desktop.bridge.identity import state_fingerprint as _state_fingerprint, get_state, note_rejected_credential note_rejected_credential(state, exc) if _state_fingerprint(get_state()) != _state_fingerprint(state): @@ -157,8 +160,8 @@ def sync_blocking(state: Dict[str, Any]) -> None: prepared = prepare(state, pending) if pending else [] if changed and not any(row.get("ok") for row in prepared): skills.bump_view_generation() - skills.rebuild_views(None) - from core.agent_skills.cache_refresh import refresh_skill_caches + skill_views.rebuild_views(None) + from core.capabilities.skills.cache_refresh import refresh_skill_caches refresh_skill_caches() if changed or prepared: @@ -190,7 +193,7 @@ def _download(state: Dict[str, Any], skill_id: str) -> bytes: def prepare_one(state: Dict[str, Any], install_id: str) -> Dict[str, Any]: """Publish verified bytes for this account; failed updates keep the old revision.""" - from core.capabilities.preparation import prepare_component + from features.desktop.capabilities.preparation import prepare_component inst = registry.get(install_id) if inst is None or inst.kind != KIND_SKILL: @@ -198,7 +201,7 @@ def prepare_one(state: Dict[str, Any], install_id: str) -> Dict[str, Any]: return prepare_component( state, inst, download=lambda: _download(state, inst.key), - content_hash=lambda comp: skills.skill_dir_hash(comp.path, fresh=True), + content_hash=lambda comp: skill_hashes.skill_dir_hash(comp.path, fresh=True), ).to_dict() @@ -230,8 +233,8 @@ def prepare_item(iid): (time.perf_counter_ns() - started) / 1_000_000) if any(r["ok"] for r in results): skills.bump_view_generation() - skills.rebuild_views(None) - from core.agent_skills.cache_refresh import refresh_skill_caches + skill_views.rebuild_views(None) + from core.capabilities.skills.cache_refresh import refresh_skill_caches refresh_skill_caches() return results @@ -243,10 +246,10 @@ def apply_to_enabled_skill_ids(skill_ids: List[str]) -> List[str]: manifest = _manifest if not manifest: return skill_ids - resolution = skills.resolve_for_user(skills.current_local_user_id()) - from core.capabilities.plugins import enabled_cloud_skill_intents + resolution = skills.resolve_for_user(skill_account.current_local_user_id()) + from features.desktop.capabilities.plugins import enabled_cloud_skill_intents - plugin_intents = enabled_cloud_skill_intents(skills.current_local_user_id()) + plugin_intents = enabled_cloud_skill_intents(skill_account.current_local_user_id()) hidden = set(resolution.conflicts) | (set(resolution.unusable) - plugin_intents) ready = [c.runtime_name for c in resolution.chosen.values() if c.source == "cloud"] kept = [sid for sid in skill_ids if sid not in hidden] @@ -277,7 +280,7 @@ def status() -> Dict[str, Any]: with _lock: manifest = _manifest err = _error - from core.services.desktop_cloud_bridge import get_state + from features.desktop.bridge.identity import get_state st = get_state() profile = None diff --git a/src/backend/features/desktop/bridge/status.py b/src/backend/features/desktop/bridge/status.py new file mode 100644 index 00000000..495e4ac3 --- /dev/null +++ b/src/backend/features/desktop/bridge/status.py @@ -0,0 +1,113 @@ +"""status responsibilities.""" +from __future__ import annotations +from typing import Any, Dict +from features.desktop.bridge import client_state as bridge_client_state +from features.desktop.bridge import identity as bridge_identity + +def initial_sync_status() -> Dict[str, Any]: + """Report actual package counts without disk hashing or network probes.""" + from features.desktop.bridge import skills as desktop_cloud_skills, bundles as desktop_cloud_bundles + from features.desktop.capabilities import session_availability + from features.desktop.capabilities.paths import revision_for_hash + + st = bridge_identity.get_state() + if not st: + return {"ready": False, "syncing": False, "partial": False, "groups": [], + "pending": 0, "error": "云端身份尚未就绪", "can_continue": False} + with bridge_client_state.manifest_lock: + manifest_ready = bridge_client_state.manifest is not None + fetching = bridge_client_state.manifest_fetching + error = bridge_client_state.manifest_error + groups = {"skill": desktop_cloud_skills.status(), **desktop_cloud_bundles.status()} + pending, completed, total = 0, 0, 0 + progress = [] + for kind, group in groups.items(): + error = error or group.get("last_error") + rows = [row for row in group.get("installations", []) if row.get("state") != "removed"] + done = sum(bool(row.get("content_hash")) and row.get("state") == "ready" + and row.get("resolved_revision") == revision_for_hash(row["content_hash"]) + for row in rows) + failed = sum(row.get("state") == "failed" or bool(row.get("last_error")) for row in rows) + completed += done + total += len(rows) + pending += len(rows) - done + progress.append({"kind": kind, "total": len(rows), "completed": done, + "failed": failed, "manifest_ready": bool(group.get("revision"))}) + complete_manifests = all(group.get("revision") for group in groups.values()) + partial = session_availability.active(desktop_cloud_skills._profile(st)) + # Progress is read without the identity lock: package rows may wait on the + # database, and the shell's token push must never queue behind that wait. + bridge_identity.require_current_account(st) + complete = (manifest_ready and not fetching and not error and pending == 0 and complete_manifests) + if not fetching and not complete and not partial: + error = error or "部分能力未能同步完成，请选择重试或使用已同步能力继续" + return {"ready": bool(complete or partial), "syncing": fetching, "partial": partial, + "groups": progress, "completed": completed, "total": total, + "totals_known": bool(complete_manifests), "pending": pending, + "error": error, "can_continue": bool(manifest_ready and not fetching and not complete)} + + +def bridge_status() -> Dict[str, Any]: + """诊断视图（/v1/desktop/capability/cloud-bridge/status）。""" + from features.desktop.bridge import skills as desktop_cloud_skills + + st = bridge_identity.get_state() + with bridge_client_state.manifest_lock: + servers = (bridge_client_state.manifest or {}).get("servers") or [] + err = bridge_client_state.manifest_error + manifest_revision = str((bridge_client_state.manifest or {}).get("revision") or "") + schema_ready_count = sum( + 1 for server in servers if isinstance(server, dict) and bool(server.get("tools")) + ) + return { + "configured": st is not None, + "cloud_base": (st or {}).get("cloud_base"), + "active": bridge_identity.bridge_active(), + "cloud_server_count": len(servers), + "manifest_revision": manifest_revision, + "schema_mode": "dynamic_manifest" if manifest_revision else "none", + "schema_ready_server_count": schema_ready_count, + "cloud_servers": [ + { + "server_id": s.get("server_id"), + "component": s.get("component"), + "display_name": s.get("display_name"), + "origin": "cloud", + } + for s in servers + if isinstance(s, dict) + ], + "last_error": err, + "skills": desktop_cloud_skills.status(), + "bundles": bundles_status(), + "mcp_json": mcp_json_status(), + } + + +def bundles_status() -> Dict[str, Any]: + from features.desktop.bridge import bundles as desktop_cloud_bundles + + return desktop_cloud_bundles.status() + + +def mcp_json_status() -> Dict[str, Any]: + from features.desktop.capabilities import connectors, mcp_json + from features.desktop.capabilities.paths import capabilities_enabled + + if not capabilities_enabled(): + return {"enabled": False} + try: + doc = mcp_json.load() + except mcp_json.McpJsonCorrupt as exc: + return {"enabled": True, "corrupt": True, "error": str(exc)} + res = connectors.last_resolution() + return { + "enabled": True, + "generation": doc.generation, + "local_server_count": len(doc.local), + "managed_profiles": { + p: len((v or {}).get("servers") or {}) for p, v in doc.managed.items() + }, + "conflicts": sorted(res.conflicts) if res else [], + "shadowed": sorted(res.shadowed) if res else [], + } diff --git a/src/backend/features/desktop/capabilities/__init__.py b/src/backend/features/desktop/capabilities/__init__.py new file mode 100644 index 00000000..b98922f8 --- /dev/null +++ b/src/backend/features/desktop/capabilities/__init__.py @@ -0,0 +1,16 @@ +"""Desktop capability runtime: partitioned storage, name resolution, flat runtime views. + +Three layers, deliberately separated (capability plan v2.0, ch. 04): + +- **store** — ``/s////``: immutable versions, one + sub-tree per source profile, same-named keys from different profiles coexist. +- **resolver** — the single place that decides which candidate a runtime name points + to for this device/user. Explicit user choice → explicit request → the current + account's usable candidate → the only usable candidate → otherwise a conflict. +- **view** — a flat directory of directory links (``/``) the + execution plane and the model see. The model-facing path contract + ``/workspace/skills/`` never changes. + +The package is inert unless :func:`paths.capability_root` is set — only the desktop +local runtime keeps a file store; cloud deployments keep their database truth. +""" diff --git a/src/backend/core/capabilities/agents.py b/src/backend/features/desktop/capabilities/agents.py similarity index 93% rename from src/backend/core/capabilities/agents.py rename to src/backend/features/desktop/capabilities/agents.py index e25f2b26..1d8d13da 100644 --- a/src/backend/core/capabilities/agents.py +++ b/src/backend/features/desktop/capabilities/agents.py @@ -17,10 +17,11 @@ from pathlib import Path from typing import Any, Dict, Iterable, List, Optional, Set -from . import registry, store -from .paths import KIND_AGENT, LOCAL_PROFILE, revision_for_hash -from .ref import local_ref -from .resolver import SOURCE_CLOUD, SOURCE_LOCAL, Candidate, Resolution, resolve +from features.desktop.capabilities import registry, store +from core.capabilities.identity_rules import KIND_AGENT, LOCAL_PROFILE +from features.desktop.capabilities.paths import revision_for_hash +from core.capabilities.reference import local_ref +from features.desktop.capabilities.resolver import SOURCE_CLOUD, SOURCE_LOCAL, Candidate, Resolution, resolve logger = logging.getLogger(__name__) @@ -136,7 +137,7 @@ def to_serialized(self) -> Dict[str, Any]: return data def content_hash(self) -> str: - from core.services.desktop_capability_protocol import entity_content_hash + from core.capabilities.manifest_protocol import entity_content_hash return entity_content_hash(self.to_files()) @@ -152,7 +153,7 @@ def load_definition(comp: store.StoredComponent, *, origin: str) -> AgentDefinit def account_definitions(profile: Optional[str] = None) -> List[AgentDefinition]: """Ready + enabled agents of the bridged account, loaded from the store.""" - from .skills import current_account_profile + from features.desktop.capabilities.skill_account import current_account_profile profile = profile or current_account_profile() if not profile: @@ -200,7 +201,7 @@ def publish_local_agent(serialized: Dict[str, Any]) -> store.StoredComponent: def remove_local_agent(agent_id: str) -> bool: - from core.capabilities.runtime import references + from features.desktop.capabilities.runtime import references if references("agent", LOCAL_PROFILE, agent_id): # Source deletion revokes use; history still needs its exact bytes. @@ -242,7 +243,7 @@ def resolve_visible(user_id: str, local_rows: Iterable[Dict[str, Any]]) -> Resol install_id=registry.install_id(KIND_AGENT, LOCAL_PROFILE, defn.agent_id), ) ) - from .skills import account_authorized_for + from features.desktop.capabilities.skill_account import account_authorized_for for defn in account_definitions() if account_authorized_for(user_id) else []: cands.append( diff --git a/src/backend/core/capabilities/change_merge.py b/src/backend/features/desktop/capabilities/change_merge.py similarity index 94% rename from src/backend/core/capabilities/change_merge.py rename to src/backend/features/desktop/capabilities/change_merge.py index b675e747..e105c0f2 100644 --- a/src/backend/core/capabilities/change_merge.py +++ b/src/backend/features/desktop/capabilities/change_merge.py @@ -3,14 +3,8 @@ import hashlib import json import re -from .archive import ( - _normalize_member, - _validate_names, - MAX_MEMBERS, - MAX_TOTAL_BYTES, - MAX_MEMBER_BYTES, -) -from core.agent_skills.binary_files import decode_binary, is_binary_value +from core.capabilities.archive import normalize_member as _normalize_member, validate_names as _validate_names, MAX_MEMBERS, MAX_TOTAL_BYTES, MAX_MEMBER_BYTES +from core.capabilities.skills.binary_files import decode_binary, is_binary_value def validate_files(files): diff --git a/src/backend/core/capabilities/change_signal.py b/src/backend/features/desktop/capabilities/change_signal.py similarity index 100% rename from src/backend/core/capabilities/change_signal.py rename to src/backend/features/desktop/capabilities/change_signal.py diff --git a/src/backend/core/capabilities/change_sync.py b/src/backend/features/desktop/capabilities/change_sync.py similarity index 87% rename from src/backend/core/capabilities/change_sync.py rename to src/backend/features/desktop/capabilities/change_sync.py index 1bf527bf..3e63c8da 100644 --- a/src/backend/core/capabilities/change_sync.py +++ b/src/backend/features/desktop/capabilities/change_sync.py @@ -1,4 +1,6 @@ """Device-side explicit capability changes, isolated from conversation execution.""" +from features.desktop.capabilities import skill_account +from features.desktop.capabilities import skills import hashlib import json @@ -9,12 +11,16 @@ import httpx from fastapi import HTTPException from core.db.models import ContentBlock -from core.agent_skills.binary_files import encode_upload -from . import archive, registry, skills, store -from .paths import assert_managed_path, LOCAL_PROFILE, BUILTIN_PROFILE -from .change_merge import plan, revision, validate_files, sensitive_paths -from .errors import CapabilityError -from .mcp_json import McpJsonError +from core.capabilities.skills.binary_files import encode_upload +from core.capabilities import archive as archive +from features.desktop.capabilities import registry +from features.desktop.capabilities import skills +from features.desktop.capabilities import store +from core.capabilities.identity_rules import LOCAL_PROFILE, BUILTIN_PROFILE +from features.desktop.capabilities.paths import assert_managed_path +from features.desktop.capabilities.change_merge import plan, revision, validate_files, sensitive_paths +from core.harness.capability_errors import CapabilityError +from features.desktop.capabilities.mcp_json import McpJsonError from sqlalchemy.exc import SQLAlchemyError from core.infra.crypto import encrypt_secret, decrypt_secret @@ -43,7 +49,7 @@ def _save(key, payload): def cloud_request(state, method, kind, key, body=None): - from core.services.desktop_cloud_bridge import cloud_headers, require_current_account + from features.desktop.bridge.identity import cloud_headers, require_current_account require_current_account(state) url = ( @@ -86,16 +92,16 @@ def source(user_id, iid): raise HTTPException(400, detail="能力标识无效") from exc if kind not in ("skill", "plugin", "mcp"): raise HTTPException(400, detail="此类型暂不支持文件提交") - current = skills.current_account_profile() + current = skill_account.current_account_profile() if profile not in (LOCAL_PROFILE, BUILTIN_PROFILE, "local-json", current): raise HTTPException(403, detail="该能力不属于当前账号") - if profile == current and not skills.account_authorized_for(user_id): + if profile == current and not skill_account.account_authorized_for(user_id): raise HTTPException(403, detail="账号不匹配") inst = registry.get(iid) if kind == "mcp": if profile == LOCAL_PROFILE: from core.db.models import AdminMcpServer - from core.services.mcp_management_service import decrypt_mcp_headers + from core.capabilities.mcp.management import decrypt_mcp_headers with registry._session() as db: row = db.get(AdminMcpServer, key) @@ -116,8 +122,8 @@ def source(user_id, iid): } spec["headers"] = decrypt_mcp_headers(row.headers or {}) elif profile == "local-json": - from .mcp_json import load - from .paths import mcp_json_path + from features.desktop.capabilities.mcp_json import load + from features.desktop.capabilities.paths import mcp_json_path doc = load() raw = assert_managed_path(mcp_json_path()).read_bytes() @@ -157,7 +163,7 @@ def source(user_id, iid): def preview(user_id, iid, state): kind, key, files, _ = source(user_id, iid) - profile = skills.current_account_profile() + profile = skill_account.current_account_profile() base = _read(_key("cap-change-base:", user_id, profile, iid)) key = base.get("cloud_key", key) if base else key try: @@ -173,7 +179,7 @@ def preview(user_id, iid, state): base_files = base["files"] if base else ({} if not remote.get("exists") else None) result = plan(base_files, files, remote["files"]) token = uuid.uuid4().hex - from core.services.desktop_cloud_bridge import account_scope + from features.desktop.bridge.identity import account_scope with account_scope(state): _save( @@ -204,16 +210,16 @@ def preview(user_id, iid, state): def _apply_local(user_id, iid, files, path, expected_revision): import os - from .paths import require_root, revision_for_hash - from core.agent_skills.binary_files import decode_binary, is_binary_value - from core.services.desktop_capability_protocol import skill_content_hash, entity_content_hash + from features.desktop.capabilities.paths import require_root, revision_for_hash + from core.capabilities.skills.binary_files import decode_binary, is_binary_value + from core.capabilities.manifest_protocol import skill_content_hash, entity_content_hash kind, profile, key = iid.split(":", 2) if kind == "mcp": if profile != "local-json": raise HTTPException(409, detail="云端已保存，请在本机连接器编辑器确认合并配置") - from .mcp_json import load, write, _validate_local_server - from .paths import mcp_json_path + from features.desktop.capabilities.mcp_json import load, write, _validate_local_server + from features.desktop.capabilities.paths import mcp_json_path spec = json.loads(files["connector.json"]) _validate_local_server(key, spec) @@ -260,7 +266,7 @@ def _apply_local(user_id, iid, files, path, expected_revision): except OSError: os.replace(backup, path) raise - from .discovery import scan + from features.desktop.capabilities.discovery import scan scan(user_id) else: @@ -277,7 +283,7 @@ def _apply_local(user_id, iid, files, path, expected_revision): def commit(user_id, token, choices, state, *, acknowledge_sensitive=False, fork_key=None): - from .change_merge import resolve + from features.desktop.capabilities.change_merge import resolve record_key = "cap-change-preview:" + token record = _read(record_key) @@ -293,7 +299,7 @@ def commit(user_id, token, choices, state, *, acknowledge_sensitive=False, fork_ ) } ) - profile = skills.current_account_profile() + profile = skill_account.current_account_profile() if not record or record["user_id"] != str(user_id) or record["profile"] != profile: raise HTTPException(404, detail="比较记录不存在") if record.get("result"): @@ -345,7 +351,7 @@ def commit(user_id, token, choices, state, *, acknowledge_sensitive=False, fork_ } # A remote success is never retried as a new write merely because local # reconciliation failed. Keep its receipt and leave newer local edits alone. - from core.services.desktop_cloud_bridge import account_scope + from features.desktop.bridge.identity import account_scope with account_scope(state): record["result"] = result @@ -376,7 +382,7 @@ def commit(user_id, token, choices, state, *, acknowledge_sensitive=False, fork_ def annotate(user_id, listing): """Local-only change hints. Network comparison happens only on explicit preview.""" - profile = skills.current_account_profile() + profile = skill_account.current_account_profile() for item in listing["items"]: iid = item["install_id"] item["change_state"] = "unavailable" @@ -392,7 +398,7 @@ def annotate(user_id, listing): else: inst = registry.get(iid) if kind == "skill" and inst: - from core.services.desktop_capability_protocol import skill_content_hash + from core.capabilities.manifest_protocol import skill_content_hash digest = skill_content_hash( files.get("SKILL.md", ""), @@ -400,7 +406,7 @@ def annotate(user_id, listing): ) item["change_state"] = "modified" if digest != inst.content_hash else "compare" elif kind == "plugin" and inst: - from core.services.desktop_capability_protocol import entity_content_hash + from core.capabilities.manifest_protocol import entity_content_hash item["change_state"] = ( "modified" if entity_content_hash(files) != inst.content_hash else "compare" @@ -425,7 +431,7 @@ def record_downloaded(inst): import logging try: - user_id = skills.current_local_user_id() + user_id = skill_account.current_local_user_id() if not user_id: return _, key, files, _ = source(user_id, inst.install_id) diff --git a/src/backend/core/capabilities/connectors.py b/src/backend/features/desktop/capabilities/connectors.py similarity index 90% rename from src/backend/core/capabilities/connectors.py rename to src/backend/features/desktop/capabilities/connectors.py index 9d8dd679..c4f8f28e 100644 --- a/src/backend/core/capabilities/connectors.py +++ b/src/backend/features/desktop/capabilities/connectors.py @@ -16,9 +16,9 @@ from pathlib import Path from typing import Any, Dict, Iterable, List, Optional, Set -from . import registry -from .paths import KIND_MCP, LOCAL_PROFILE -from .resolver import SOURCE_CLOUD, SOURCE_LOCAL, Candidate, Resolution, resolve +from features.desktop.capabilities import registry +from core.capabilities.identity_rules import KIND_MCP, LOCAL_PROFILE +from features.desktop.capabilities.resolver import SOURCE_CLOUD, SOURCE_LOCAL, Candidate, Resolution, resolve MCP_JSON_PROFILE = "local-json" _DB_PATH = Path("") @@ -36,8 +36,8 @@ def db_candidates( Include those authorized declarations in every device binding resolution; the caller's enabled-ID allowlist still controls their ambient availability. """ - from .local_plugin_runtime import configs - from .skills import current_local_user_id + from features.desktop.capabilities.local_plugin_runtime import configs + from features.desktop.capabilities.skill_account import current_local_user_id selected_user = user_id if user_id is not None else current_local_user_id() ids = set(server_ids) | set(configs(selected_user)) @@ -115,7 +115,7 @@ def server_id_of(candidate: Candidate) -> str: def resolve_bindings(candidates: List[Candidate], *, user_id: Optional[str] = None) -> Resolution: """Resolve exact source bindings without deployment-specific tool filters.""" - from .skills import current_local_user_id + from features.desktop.capabilities.skill_account import current_local_user_id selected_user_id = user_id if user_id is not None else current_local_user_id() res = resolve( diff --git a/src/backend/core/capabilities/credentials.py b/src/backend/features/desktop/capabilities/credentials.py similarity index 100% rename from src/backend/core/capabilities/credentials.py rename to src/backend/features/desktop/capabilities/credentials.py diff --git a/src/backend/core/capabilities/dependency.py b/src/backend/features/desktop/capabilities/dependency.py similarity index 94% rename from src/backend/core/capabilities/dependency.py rename to src/backend/features/desktop/capabilities/dependency.py index c715eca4..13e29467 100644 --- a/src/backend/core/capabilities/dependency.py +++ b/src/backend/features/desktop/capabilities/dependency.py @@ -7,6 +7,8 @@ """ from __future__ import annotations +from features.desktop.capabilities import skill_account +from features.desktop.capabilities import skill_hashes import copy import importlib.metadata @@ -25,9 +27,9 @@ from packaging.specifiers import SpecifierSet from packaging.version import Version -from . import registry, skills, store -from .errors import CapabilityError -from .paths import BUILTIN_PROFILE, LOCAL_PROFILE +from features.desktop.capabilities import registry, skills, store +from core.harness.capability_errors import CapabilityError +from core.capabilities.identity_rules import BUILTIN_PROFILE, LOCAL_PROFILE logger = logging.getLogger(__name__) @@ -37,7 +39,7 @@ class DependencyMissing(CapabilityError): recovery_action = "inspect_dependencies" -from .dependency_declarations import _entries, _identifier +from features.desktop.capabilities.dependency_declarations import _entries, _identifier def _parse_skill_definition(text): @@ -68,11 +70,11 @@ def skill_definition(path: Path): def component_hash(comp, *, fresh: bool = False): """A component's content digest. ``fresh`` reads every byte; see ``dir_digest``.""" if comp.kind == "skill": - return skills.skill_dir_hash(comp.path, fresh=fresh) - from core.agent_skills.binary_files import encode_upload - from core.services.desktop_capability_protocol import entity_content_hash + return skill_hashes.skill_dir_hash(comp.path, fresh=fresh) + from core.capabilities.skills.binary_files import encode_upload + from core.capabilities.manifest_protocol import entity_content_hash - from .archive import iter_files + from core.capabilities.archive import iter_files def read_all(): return entity_content_hash( @@ -83,7 +85,7 @@ def read_all(): } ) - return skills.dir_digest(comp.path, read_all, fresh=fresh) + return skill_hashes.dir_digest(comp.path, read_all, fresh=fresh) @dataclass @@ -228,8 +230,8 @@ def resolve(self, kind, profile, key): _, selected_profile, key = parts if selected_profile not in (LOCAL_PROFILE, BUILTIN_PROFILE): if ( - selected_profile != skills.current_account_profile() - or not skills.account_authorized_for(self.context.user_id) + selected_profile != skill_account.current_account_profile() + or not skill_account.account_authorized_for(self.context.user_id) ): return None, None, None profile = selected_profile @@ -316,7 +318,7 @@ def visit(self, entry, profile, chain=(), required=True): ): self.issue("component_not_ready", path, required) return - if comp.profile not in (LOCAL_PROFILE, BUILTIN_PROFILE, skills.current_account_profile()): + if comp.profile not in (LOCAL_PROFILE, BUILTIN_PROFILE, skill_account.current_account_profile()): self.issue("not_authorized_or_missing", path, required) return if ( @@ -471,7 +473,7 @@ def require_report(report): def allowed_model_ids(user_id): """Reuse the live model gateway's role/user-switch authorization policy.""" from core.db.models import ModelProvider - from core.services.desktop_capability_models import _model_provider_allowed + from features.desktop.capability.models import _model_provider_allowed with registry._session() as db: return { diff --git a/src/backend/core/capabilities/dependency_declarations.py b/src/backend/features/desktop/capabilities/dependency_declarations.py similarity index 100% rename from src/backend/core/capabilities/dependency_declarations.py rename to src/backend/features/desktop/capabilities/dependency_declarations.py diff --git a/src/backend/core/capabilities/device_catalog.py b/src/backend/features/desktop/capabilities/device_catalog.py similarity index 81% rename from src/backend/core/capabilities/device_catalog.py rename to src/backend/features/desktop/capabilities/device_catalog.py index bc7c2335..6f39f4a3 100644 --- a/src/backend/core/capabilities/device_catalog.py +++ b/src/backend/features/desktop/capabilities/device_catalog.py @@ -11,14 +11,16 @@ """ from __future__ import annotations +from features.desktop.capabilities import skill_account import logging from functools import lru_cache from typing import Any, Dict, List, Optional, Tuple -from core.config.catalog_runtime import merge_items_by_id +from core.capabilities.catalog.runtime import merge_items_by_id -from .paths import KIND_AGENT, KIND_PLUGIN, KIND_SKILL, capabilities_enabled, capability_root +from core.capabilities.identity_rules import KIND_AGENT, KIND_PLUGIN, KIND_SKILL +from features.desktop.capabilities.paths import capabilities_enabled, capability_root logger = logging.getLogger(__name__) @@ -33,38 +35,38 @@ def active() -> bool: 自带的内置技能与内置连接器整体不参与——既不进能力中心的清单，也不进装配，用户 才不会看到云端账号里根本没有、点了也没反应的能力。 """ - from core.auth.desktop_bridge import bridge_enabled + from features.desktop.auth.bridge import bridge_enabled return bool(bridge_enabled() and capabilities_enabled()) def _profile() -> Optional[str]: - from core.capabilities import skills + from features.desktop.capabilities import skills try: - return skills.current_account_profile() + return skill_account.current_account_profile() except Exception as exc: # noqa: BLE001 - 身份未就绪时清单为空，不是错误 logger.debug("[device-catalog] account profile unavailable: %s", exc) return None -def _installations(kind: str, user_id=None) -> List[Any]: - from . import registry, skills - from .paths import LOCAL_PROFILE +def installations(kind: str, user_id=None) -> List[Any]: + from features.desktop.capabilities import registry, skills + from core.capabilities.identity_rules import LOCAL_PROFILE if not capabilities_enabled(): return [] - user_id = user_id or skills.current_local_user_id() + user_id = user_id or skill_account.current_local_user_id() rows = [row for row in registry.list_installations(kind=kind, profile_id=LOCAL_PROFILE) if user_id and (row.payload.get("owner_user_id") == user_id or (row.payload.get("shared_installation") and not active()))] profile = _profile() - if profile and (user_id is None or skills.account_authorized_for(user_id)): + if profile and (user_id is None or skill_account.account_authorized_for(user_id)): rows += registry.list_installations(kind=kind, profile_id=profile) return rows def _managed_connectors() -> List[Dict[str, Any]]: - from core.services.desktop_cloud_bridge import managed_connectors + from features.desktop.bridge.managed_connectors import managed_connectors return managed_connectors() @@ -82,7 +84,7 @@ def _read_component( 的失效点。否则每打开一次能力中心，就把每个技能的目录重读一遍。``root`` 只为 进键——换一个能力仓就是另一套文件。 """ - from . import store + from features.desktop.capabilities import store component = store.get(KIND_SKILL, profile, key, revision) if component is None: @@ -114,7 +116,7 @@ def _skill_body(inst) -> Optional[Tuple[str, Tuple[str, ...]]]: def _skill_item(inst) -> Dict[str, Any]: - from core.config.catalog_details import skill_body_from_raw + from core.capabilities.catalog.details import skill_body_from_raw description = inst.description or "" item = { @@ -169,12 +171,7 @@ def _db_umbrella_item(members: List[Dict[str, Any]]) -> Dict[str, Any]: 启停取成员的并集：任一成员开着，伞形就是开着的。 """ - from core.config.catalog_loader import ( - DB_UMBRELLA_DESC, - DB_UMBRELLA_ICON, - DB_UMBRELLA_ID, - DB_UMBRELLA_NAME, - ) + from core.capabilities.catalog.loader import DB_UMBRELLA_DESC, DB_UMBRELLA_ICON, DB_UMBRELLA_ID, DB_UMBRELLA_NAME return { "id": DB_UMBRELLA_ID, @@ -203,15 +200,15 @@ def catalog_overlay(user_id=None) -> Dict[str, Any]: """ if not capabilities_enabled(): return {"skills": [], "mcp": [], "hidden_skills": set(), "hidden_mcp": set()} - skills = _installations(KIND_SKILL, user_id=user_id) + skills = installations(KIND_SKILL, user_id=user_id) connectors = _managed_connectors() hidden_skills = {inst.key for inst in skills if inst.source_plugin} hidden_mcp = {str(s["server_id"]) for s in connectors if s.get("source_plugin")} - for inst in _installations(KIND_PLUGIN, user_id=user_id): + for inst in installations(KIND_PLUGIN, user_id=user_id): components = dict(inst.payload.get("components") or {}) hidden_skills.update(str(sid) for sid in components.get("skills") or []) hidden_mcp.update(str(mid) for mid in components.get("mcp") or []) - from core.config.catalog_loader import DB_HIDDEN_SERVERS + from core.capabilities.catalog.loader import DB_HIDDEN_SERVERS standalone = [s for s in connectors if not s.get("source_plugin")] db_members = [s for s in standalone if str(s.get("server_id")) in DB_HIDDEN_SERVERS] @@ -237,7 +234,7 @@ def _account_definitions() -> Dict[str, Any]: 逐个 ``account_definition(id)`` 会把整份清单连同文件重查一遍，条目一多就是 N+1 次查询加 N×M 次文件解析。 """ - from . import agents as caps_agents + from features.desktop.capabilities import agents as caps_agents try: return {defn.agent_id: defn for defn in caps_agents.account_definitions()} @@ -286,11 +283,11 @@ def agent_entries() -> List[Dict[str, Any]]: """云端账号的智能体投影成 ``/v1/agents`` 的条目结构。""" if not active(): return [] - installations = _installations(KIND_AGENT) - if not installations: + installed_agents = installations(KIND_AGENT) + if not installed_agents: return [] definitions = _account_definitions() - return [_agent_entry(inst, definitions.get(inst.key)) for inst in installations] + return [_agent_entry(inst, definitions.get(inst.key)) for inst in installed_agents] def toggle_projected_agent(agent_id: str, data: Dict[str, Any]) -> Optional[Dict[str, Any]]: @@ -299,7 +296,7 @@ def toggle_projected_agent(agent_id: str, data: Dict[str, Any]) -> Optional[Dict 定义（提示词、绑定的技能与工具）是云端那一份，改它要去云端改完再同步；这台 机器上能决定的只有开不开。 """ - from . import registry + from features.desktop.capabilities import registry if "is_enabled" not in data: return None @@ -312,9 +309,7 @@ def toggle_projected_agent(agent_id: str, data: Dict[str, Any]) -> Optional[Dict return _agent_entry(inst, _account_definitions().get(agent_id)) -from .device_plugin_catalog import ( - plugin_entries, plugin_detail, plugin_ui_contributions, _projected_plugin, -) +from features.desktop.capabilities.device_plugin_catalog import plugin_entries, plugin_detail, plugin_ui_contributions, _projected_plugin # ── 启停 ────────────────────────────────────────────────────────────── @@ -322,8 +317,8 @@ def toggle_projected_agent(agent_id: str, data: Dict[str, Any]) -> Optional[Dict def set_enabled(kind: str, item_id: str, enabled: bool, user_id=None) -> bool: """把启停写进登记表；返回 False 表示这条不是云端投影项，调用方照常处理。""" - from . import registry - matches = [row for row in _installations(kind, user_id=user_id) + from features.desktop.capabilities import registry + matches = [row for row in installations(kind, user_id=user_id) if item_id in (row.key, row.install_id)] if len(matches) != 1: return False @@ -337,13 +332,13 @@ def set_plugin_enabled(install_id_or_slug: str, enabled: bool, user_id=None) -> """插件按云端 install_id 或 slug 定位后写登记表。""" if not capabilities_enabled(): return False - from . import registry + from features.desktop.capabilities import registry inst = _projected_plugin(install_id_or_slug, user_id) if inst is None: return False if inst.profile_id == "local": - from core.plugins.local.service import set_enabled_for_user + from features.desktop.local.plugins.service import set_enabled_for_user set_enabled_for_user(user_id, inst.install_id, enabled) else: registry.set_enabled(inst.install_id, bool(enabled)) @@ -354,8 +349,8 @@ def set_plugin_enabled(install_id_or_slug: str, enabled: bool, user_id=None) -> def _invalidate() -> None: """启停变了，运行时解析与目录缓存都要重算。""" try: - from core.agent_skills.cache_refresh import refresh_skill_caches - from core.config.catalog_runtime import invalidate_runtime_catalog_cache + from core.capabilities.skills.cache_refresh import refresh_skill_caches + from core.capabilities.catalog.runtime import invalidate_runtime_catalog_cache invalidate_runtime_catalog_cache() refresh_skill_caches() @@ -374,3 +369,26 @@ def _invalidate() -> None: "set_plugin_enabled", "toggle_projected_agent", ] + + +def resolve_plugin(slug, user_id, *, install_id=None): + from features.desktop.capabilities.device_plugin_catalog import resolve_resource_plugin + return resolve_resource_plugin(slug, user_id, install_id=install_id) + + +def account_plugin_installations(user_id): + from features.desktop.capabilities import registry + profile = skill_account.current_account_profile() + if not profile or not skill_account.account_authorized_for(user_id): + return [] + return registry.list_installations(kind="plugin", profile_id=profile) + + +def owned_identities(user_id): + from features.desktop.capabilities import registry + from core.capabilities.identity_rules import LOCAL_PROFILE + profiles = [LOCAL_PROFILE] + if skill_account.account_authorized_for(user_id): + profiles.append(skill_account.current_account_profile()) + return {(row.kind, row.key) for row in registry.list_installations(profiles=profiles) + if row.payload.get("owner_user_id") in (None, "", user_id)} diff --git a/src/backend/core/capabilities/device_plugin_catalog.py b/src/backend/features/desktop/capabilities/device_plugin_catalog.py similarity index 69% rename from src/backend/core/capabilities/device_plugin_catalog.py rename to src/backend/features/desktop/capabilities/device_plugin_catalog.py index 5c6d2605..d5b383ac 100644 --- a/src/backend/core/capabilities/device_plugin_catalog.py +++ b/src/backend/features/desktop/capabilities/device_plugin_catalog.py @@ -1,8 +1,8 @@ """Plugin catalog views over authorized device installations.""" from typing import Any, Dict, List, Optional -from . import device_catalog as catalog -from .local_plugin_runtime import enabled_for -from .paths import KIND_PLUGIN, KIND_SKILL, capabilities_enabled +from features.desktop.capabilities.local_plugin_runtime import enabled_for +from core.capabilities.identity_rules import KIND_PLUGIN, KIND_SKILL +from features.desktop.capabilities.paths import capabilities_enabled # ── 插件 ────────────────────────────────────────────────────────────── @@ -11,11 +11,11 @@ def plugin_entries(user_id=None) -> List[Dict[str, Any]]: """云端账号的插件投影成 ``/v1/plugins/installed`` 的条目结构。""" if not capabilities_enabled(): return [] - from .management_context import management_context - from .readiness import file_readiness + from features.desktop.capabilities.management_context import management_context + from features.desktop.capabilities.readiness import file_readiness context = management_context(user_id) entries: List[Dict[str, Any]] = [] - for inst in catalog._installations(KIND_PLUGIN, user_id=user_id): + for inst in catalog.installations(KIND_PLUGIN, user_id=user_id): components = dict(inst.payload.get("components") or {}) readiness = file_readiness(inst, context) entries.append( @@ -51,12 +51,12 @@ def plugin_ui_contributions(user_id=None) -> List[Dict[str, Any]]: """ if not capabilities_enabled(): return [] - from core.plugins.ui.contract import public_contributions + from core.harness.registry.package_views import public_package_view - from . import plugins as caps_plugins, store + from features.desktop.capabilities import plugins as caps_plugins, store out: List[Dict[str, Any]] = [] - for inst in catalog._installations(KIND_PLUGIN, user_id=user_id): + for inst in catalog.installations(KIND_PLUGIN, user_id=user_id): if not (enabled_for(inst, user_id) and inst.ready): continue comp = store.get(KIND_PLUGIN, inst.profile_id, inst.key, inst.resolved_revision) @@ -67,7 +67,7 @@ def plugin_ui_contributions(user_id=None) -> List[Dict[str, Any]]: except (OSError, ValueError): # 清单读不出来只影响这一个插件的界面，不该让整页拿不到贡献。 continue - public = public_contributions(manifest.get("ui_contributions"), slug=inst.key) + public = public_package_view(manifest.get("ui_contributions"), slug=inst.key) if public.get("contributes"): out.append(public) return out @@ -75,7 +75,7 @@ def plugin_ui_contributions(user_id=None) -> List[Dict[str, Any]]: def _projected_plugin(install_id_or_slug: str, user_id=None): """按云端 install_id、slug 或本机 install_id 定位一条投影插件。""" - for inst in catalog._installations(KIND_PLUGIN, user_id=user_id): + for inst in catalog.installations(KIND_PLUGIN, user_id=user_id): cloud_id = str(inst.payload.get("cloud_install_id") or "") if install_id_or_slug in (inst.key, cloud_id, inst.install_id, *inst.payload.get("legacy_ids", [])): return inst @@ -83,9 +83,9 @@ def _projected_plugin(install_id_or_slug: str, user_id=None): def _component_skill(profile: str, skill_id: str) -> Optional[Dict[str, Any]]: - from core.config.catalog_details import skill_body_from_raw + from core.capabilities.catalog.details import skill_body_from_raw - from . import registry + from features.desktop.capabilities import registry inst = registry.get(registry.install_id(KIND_SKILL, profile, skill_id)) if inst is None or inst.state == "removed": @@ -126,8 +126,8 @@ def plugin_detail(install_id_or_slug: str, user_id=None) -> Optional[Dict[str, A inst = _projected_plugin(install_id_or_slug, user_id) if inst is None: return None - from core.plugins.connections import manifest_connection - from . import plugins, store + from core.capabilities.plugins.connections import manifest_connection + from features.desktop.capabilities import plugins, store comp = store.get("plugin", inst.profile_id, inst.key, inst.resolved_revision) manifest = plugins.load_manifest(comp) if comp else {} @@ -140,7 +140,7 @@ def plugin_detail(install_id_or_slug: str, user_id=None) -> Optional[Dict[str, A ] known = {c["server_id"]: c for c in catalog._managed_connectors()} if profile == "local": - from .local_plugin_runtime import configs + from features.desktop.capabilities.local_plugin_runtime import configs known.update(configs(user_id or inst.payload.get("owner_user_id"), enabled_only=False)) return { "install_id": str(inst.payload.get("cloud_install_id") or inst.install_id), @@ -158,3 +158,40 @@ def plugin_detail(install_id_or_slug: str, user_id=None) -> Optional[Dict[str, A "admin_config": None, "connection": manifest_connection(manifest), } + + +def resolve_resource_plugin(slug, user_id, *, install_id=None): + """Resolve the exact device installation for a resource launch.""" + from core.capabilities.plugins.resources.installation import Installation + from fastapi import HTTPException + from features.desktop.capabilities import store + from features.desktop.capabilities.device_plugin_catalog import _projected_plugin + from features.desktop.capabilities.local_plugin_runtime import enabled_for + from features.desktop.capabilities.plugins import load_manifest + + candidate = _projected_plugin(install_id or slug, user_id=user_id) + if candidate is not None: + if candidate.key != slug or not candidate.ready or not enabled_for(candidate, user_id): + raise HTTPException(403, "plugin_not_ready") + component = store.get( + "plugin", candidate.profile_id, candidate.key, candidate.resolved_revision + ) + if component is None: + raise HTTPException(409, "plugin_package_missing") + manifest = load_manifest(component) + package = component.path / "package" + if not package.is_dir(): + raise HTTPException(409, "plugin_package_missing") + return Installation( + candidate.install_id, + candidate.resolved_revision, + slug, + manifest.get("ui_contributions") or {}, + package, + ) + return None + + +# device_catalog re-exports this module's functions at import time, so it is +# bound last: by then those names exist whichever module was imported first. +from features.desktop.capabilities import device_catalog as catalog # noqa: E402 diff --git a/src/backend/core/capabilities/discovery.py b/src/backend/features/desktop/capabilities/discovery.py similarity index 85% rename from src/backend/core/capabilities/discovery.py rename to src/backend/features/desktop/capabilities/discovery.py index 5e366e7c..8a894a92 100644 --- a/src/backend/core/capabilities/discovery.py +++ b/src/backend/features/desktop/capabilities/discovery.py @@ -1,8 +1,8 @@ """Read-only inventory of unregistered skill folders. Installation is explicit.""" import re -from . import archive -from .errors import CapabilityError -from .paths import capabilities_enabled, kind_root, assert_managed_path +from core.capabilities import archive as archive +from core.harness.capability_errors import CapabilityError +from features.desktop.capabilities.paths import capabilities_enabled, kind_root, assert_managed_path def scan(user_id): diff --git a/src/backend/core/capabilities/invocation.py b/src/backend/features/desktop/capabilities/invocation.py similarity index 74% rename from src/backend/core/capabilities/invocation.py rename to src/backend/features/desktop/capabilities/invocation.py index c38bfe0f..c8cf5356 100644 --- a/src/backend/core/capabilities/invocation.py +++ b/src/backend/features/desktop/capabilities/invocation.py @@ -1,12 +1,16 @@ """Explicit desktop selections use the same authorized sources as runtime views.""" +from features.desktop.bridge import gateway as bridge_gateway +from features.desktop.capabilities import skill_account +from features.desktop.capabilities import skill_candidates as skill_catalog +from features.desktop.capabilities import skills -from . import connectors, plugins, registry, skills -from .errors import NameConflict +from features.desktop.capabilities import connectors, plugins, registry, skills +from core.harness.capability_errors import NameConflict def cloud_plugin_selection(ident, *, user_id): - from .local_plugin_runtime import configs, enabled_for - from .device_catalog import active + from features.desktop.capabilities.local_plugin_runtime import configs, enabled_for + from features.desktop.capabilities.device_catalog import active local = [row for row in registry.list_installations(kind="plugin", profile_id="local") if (row.payload.get("owner_user_id") == user_id or (row.payload.get("shared_installation") and not active())) and ident in (row.key, row.install_id, *row.payload.get("legacy_ids", []))] @@ -18,9 +22,9 @@ def cloud_plugin_selection(ident, *, user_id): "skills": [sid for sid in components.get("skills", []) if (child := registry.get(registry.install_id("skill", "local", sid))) and child.enabled and child.ready], "mcp": [sid for sid in components.get("mcp", []) if sid in available]} - if not skills.account_authorized_for(user_id): + if not skill_account.account_authorized_for(user_id): return None - profile = skills.current_account_profile() + profile = skill_account.current_account_profile() matches = [ row for row in registry.list_installations(kind="plugin", profile_id=profile) @@ -33,9 +37,9 @@ def cloud_plugin_selection(ident, *, user_id): row = matches[0] if row.enabled and not row.ready: # 用户在对话里直接选了一个尚未下载的云端插件：按需准备定义与组件。 - from .preparation import ensure_cloud_ready + from features.desktop.capabilities.preparation import ensure_cloud_ready - from .errors import CapabilityError + from core.harness.capability_errors import CapabilityError try: ensure_cloud_ready(user_id, install_ids=[row.install_id]) @@ -51,9 +55,9 @@ def cloud_plugin_selection(ident, *, user_id): "skills": [], "mcp": [], } - from .preparation import ensure_cloud_ready as _ensure_components + from features.desktop.capabilities.preparation import ensure_cloud_ready as _ensure_components - from .errors import CapabilityError + from core.harness.capability_errors import CapabilityError try: _ensure_components(user_id, install_ids=[row.install_id]) @@ -75,17 +79,17 @@ def resolve_explicit_ids(user_id, allowed_skill_ids, allowed_mcp_ids): For names present in the desktop store, its current resolver is authoritative. This function never installs a component or imports a cloud credential. """ - from core.services import desktop_cloud_bridge as bridge + from features.desktop.bridge import client as bridge - skill_candidates = skills.candidates(user_id) + skill_candidates = skill_catalog.candidates(user_id) skill_resolution = skills.resolve_for_user(user_id) known_skills = {candidate.runtime_name for candidate in skill_candidates} allowed_skills = set(allowed_skill_ids) - known_skills allowed_skills.update(skill_resolution.chosen) - context = bridge._bridge_context() if skills.account_authorized_for(user_id) else None - local_ids = bridge._local_server_ids() - local_json = bridge._mcp_json_local_declarations() + context = bridge_gateway.bridge_context() if skill_account.account_authorized_for(user_id) else None + local_ids = bridge_gateway.local_server_ids() + local_json = bridge_gateway.mcp_json_local_declarations() candidates = connectors.db_candidates(local_ids, set(allowed_mcp_ids), user_id=user_id) candidates.extend(connectors.json_candidates(local_json)) if context: diff --git a/src/backend/core/capabilities/local_plugin_runtime.py b/src/backend/features/desktop/capabilities/local_plugin_runtime.py similarity index 72% rename from src/backend/core/capabilities/local_plugin_runtime.py rename to src/backend/features/desktop/capabilities/local_plugin_runtime.py index 47dd1004..ec7b5e80 100644 --- a/src/backend/core/capabilities/local_plugin_runtime.py +++ b/src/backend/features/desktop/capabilities/local_plugin_runtime.py @@ -1,7 +1,9 @@ """Resolve plugin-owned local connectors directly from retained, authorized packages.""" import json -from . import registry, store -from .paths import LOCAL_PROFILE, capabilities_enabled +from features.desktop.capabilities import registry, store +from core.capabilities.identity_rules import LOCAL_PROFILE +from features.desktop.capabilities.paths import capabilities_enabled +from core.capabilities.mcp.service import rewrite_builtin_mcp_host def enabled_for(inst, user_id): @@ -11,10 +13,10 @@ def enabled_for(inst, user_id): def configs(user_id, enabled_only=True): if not user_id or not capabilities_enabled(): return {} - from .dependency import component_hash + from features.desktop.capabilities.dependency import component_hash out = {} for inst in registry.list_installations(kind="plugin", profile_id=LOCAL_PROFILE): - from .device_catalog import active + from features.desktop.capabilities.device_catalog import active owned = inst.payload.get("owner_user_id") == user_id or (inst.payload.get("shared_installation") and not active()) if not owned or not inst.ready or (enabled_only and not enabled_for(inst, user_id)): continue @@ -26,15 +28,18 @@ def configs(user_id, enabled_only=True): if enabled_only and not entry.get("enabled", True): continue config = json.loads(json.dumps(entry).replace("{plugin_root}", str(comp.path / "package").replace("\\", "\\\\"))) - config.update(source_plugin=inst.key, owner_user_id=user_id, origin="local_plugin", execution_scope="local", is_stable=False) + if config.get("url"): + # Packaged manifests name the compose host; local planes listen elsewhere. + config["url"] = rewrite_builtin_mcp_host(config["url"]) + config.update(source_plugin=inst.key, owner_user_id=user_id, origin="local_plugin", execution_scope="local") executor = definition.get("manager_executor") if executor: - from core.services.management_contract import tools, VERSION - from core.services.desktop_capability_protocol import canonical_hash + from core.capabilities.plugins.management.contract import tools, VERSION + from core.capabilities.manifest_protocol import canonical_hash manager = executor.get("id") if executor.get("version") != VERSION: raise ValueError("plugin executor upgrade required") - from .skills import account_authorized_for + from features.desktop.capabilities.skill_account import account_authorized_for cloud_available = account_authorized_for(user_id) schemas = tools(manager, "local") if not cloud_available: diff --git a/src/backend/core/capabilities/local_projection.py b/src/backend/features/desktop/capabilities/local_projection.py similarity index 79% rename from src/backend/core/capabilities/local_projection.py rename to src/backend/features/desktop/capabilities/local_projection.py index dd27018d..07cb6ee4 100644 --- a/src/backend/core/capabilities/local_projection.py +++ b/src/backend/features/desktop/capabilities/local_projection.py @@ -1,10 +1,13 @@ """Publish committed device DB skill mutations into the capability store/index.""" from __future__ import annotations +from sqlalchemy import inspect as inspect_model +from features.desktop.capabilities import skill_publication +from features.desktop.capabilities import skill_views import threading -from . import registry, skills -from .paths import capabilities_enabled +from features.desktop.capabilities import registry, skills +from features.desktop.capabilities.paths import capabilities_enabled _lock = threading.RLock() @@ -13,23 +16,23 @@ def sync_local_skills() -> None: if not capabilities_enabled(): return from core.db.models import AdminSkill - from core.agent_skills.binary_files import decode_binary, is_binary_value - from core.services.desktop_capability_protocol import skill_content_hash + from core.capabilities.skills.binary_files import decode_binary, is_binary_value + from core.capabilities.manifest_protocol import skill_content_hash - from core.services.local_management_migration import migrate + from features.desktop.local.management_migration import migrate migrate() owners = set() with _lock: with registry._session() as db: from sqlalchemy import inspect - if not inspect(db.get_bind()).has_table(AdminSkill.__tablename__): + if not inspect(db.get_bind()).has_table(inspect_model(AdminSkill).local_table.name): return rows = db.query(AdminSkill).order_by(AdminSkill.skill_id).all() # 混合模式：本机业务库里的插件整体不参与（见 plugin_service.list_installed）， # 它们的技能同样不投影——否则会和云端同步下来的同名技能抢运行名。用户自建的 # 技能没有归属插件，照常投影：那是他自己的东西，只有这台机器上有。 - from . import device_catalog + from features.desktop.capabilities import device_catalog hybrid = device_catalog.active() definitions = [ @@ -53,7 +56,7 @@ def sync_local_skills() -> None: sid, owner = definition["skill_id"], definition["owner"] existing = registry.get(registry.install_id("skill", "local", sid)) if existing is not None and not existing.payload.get("from_db"): - from .errors import NameConflict + from core.harness.capability_errors import NameConflict raise NameConflict( "local upload key collides with an independent installation", runtime_name=sid @@ -69,7 +72,7 @@ def sync_local_skills() -> None: for name, value in extras.items() }, } - skills.publish_local_skill( + skill_publication.publish_local_skill( sid, files=files, content_hash=skill_content_hash(definition["content"], extras), @@ -92,8 +95,8 @@ def sync_local_skills() -> None: if inst.payload.get("from_db") and inst.key not in live: if inst.payload.get("owner_user_id"): owners.add(inst.payload["owner_user_id"]) - skills.prune_local(live) + skill_publication.prune_local(live) # Views depend on identity; do not hold the projection lock while resolving it. - skills.rebuild_views(None) + skill_views.rebuild_views(None) for owner in owners: - skills.rebuild_user_view(owner) + skill_views.rebuild_user_view(owner) diff --git a/src/backend/core/capabilities/management_context.py b/src/backend/features/desktop/capabilities/management_context.py similarity index 82% rename from src/backend/core/capabilities/management_context.py rename to src/backend/features/desktop/capabilities/management_context.py index 3665ad95..ed3ad599 100644 --- a/src/backend/core/capabilities/management_context.py +++ b/src/backend/features/desktop/capabilities/management_context.py @@ -1,14 +1,15 @@ """Shared connector resolution and dependency context for management surfaces.""" +from features.desktop.capabilities import skill_account from typing import Any, Dict -from . import registry, skills +from features.desktop.capabilities import registry, skills def _authorized_profile(user_id): - return skills.current_account_profile() if skills.account_authorized_for(user_id) else None + return skill_account.current_account_profile() if skill_account.account_authorized_for(user_id) else None def management_context(user_id: str): - from core.capabilities.readiness import context_for_user + from features.desktop.capabilities.readiness import context_for_user connectors = connectors_view(user_id) available = { @@ -21,8 +22,8 @@ def management_context(user_id: str): def connectors_view(user_id: str) -> Dict[str, Any]: """Connector bindings as the resolver last decided them for this device.""" - from core.capabilities import connectors, skills, mcp_json - from core.services.mcp_service import McpServerConfigService + from features.desktop.capabilities import connectors, skills, mcp_json + from core.capabilities.mcp.service import McpServerConfigService svc = McpServerConfigService.get_instance() all_cfgs = { @@ -32,14 +33,14 @@ def connectors_view(user_id: str) -> Dict[str, Any]: } all_cfgs.update(svc.get_owned_servers(user_id, enabled_only=False)) enabled_ids = set(svc.get_all_servers(enabled_only=True)) | set(svc.get_owned_servers(user_id)) - from core.services.desktop_cloud_bridge import _mcp_json_local_declarations + from features.desktop.bridge.gateway import mcp_json_local_declarations as _mcp_json_local_declarations candidates = connectors.db_candidates(all_cfgs, enabled_ids, user_id=user_id) + connectors.json_candidates( _mcp_json_local_declarations() ) # The runtime context excludes disabled bindings; the management view must # keep them visible so users can turn them back on. - from core.services.desktop_cloud_bridge import get_cached_manifest + from features.desktop.bridge.client import get_cached_manifest profile = _authorized_profile(user_id) manifest = get_cached_manifest() if profile else None @@ -77,7 +78,7 @@ def connectors_view(user_id: str) -> Dict[str, Any]: ) ) entry["resolution"] = {"outcome": outcome, "reason": res.reasons.get(c.runtime_name)} - from core.capabilities.readiness import apply_to_item, connector_readiness + from features.desktop.capabilities.readiness import apply_to_item, connector_readiness config = ( _mcp_json_local_declarations().get(entry["server_id"], {}) diff --git a/src/backend/core/capabilities/manifest_order.py b/src/backend/features/desktop/capabilities/manifest_order.py similarity index 96% rename from src/backend/core/capabilities/manifest_order.py rename to src/backend/features/desktop/capabilities/manifest_order.py index 53d680bc..4cc3b08c 100644 --- a/src/backend/core/capabilities/manifest_order.py +++ b/src/backend/features/desktop/capabilities/manifest_order.py @@ -8,7 +8,7 @@ from contextlib import contextmanager from dataclasses import dataclass import threading -from .errors import CloudUnavailable +from core.harness.capability_errors import CloudUnavailable class StaleManifest(CloudUnavailable): diff --git a/src/backend/core/capabilities/mcp_json.py b/src/backend/features/desktop/capabilities/mcp_json.py similarity index 98% rename from src/backend/core/capabilities/mcp_json.py rename to src/backend/features/desktop/capabilities/mcp_json.py index 37d7f298..721af3f2 100644 --- a/src/backend/core/capabilities/mcp_json.py +++ b/src/backend/features/desktop/capabilities/mcp_json.py @@ -28,9 +28,10 @@ from pathlib import Path from typing import Any, Dict, List, Optional -from . import credentials -from .lockfile import locked -from .paths import mcp_json_path, meta_root, safe_segment +from features.desktop.capabilities import credentials +from core.infra.file_lock import locked +from core.capabilities.identity_rules import safe_segment +from features.desktop.capabilities.paths import mcp_json_path, meta_root SCHEMA_VERSION = 2 LOCAL_TRANSPORTS = ("stdio", "streamable_http", "sse") @@ -342,7 +343,6 @@ def local_server_configs(doc: Optional[McpJson] = None) -> Dict[str, dict]: continue cfg: Dict[str, Any] = { "transport": entry["transport"], - "is_stable": False, "source_plugin": None, "owner_user_id": None, "origin": "mcp_json_local", diff --git a/src/backend/core/capabilities/migration.py b/src/backend/features/desktop/capabilities/migration.py similarity index 92% rename from src/backend/core/capabilities/migration.py rename to src/backend/features/desktop/capabilities/migration.py index a26bc26d..c9e48581 100644 --- a/src/backend/core/capabilities/migration.py +++ b/src/backend/features/desktop/capabilities/migration.py @@ -17,6 +17,8 @@ """ from __future__ import annotations +from features.desktop.capabilities import skill_paths +from features.desktop.capabilities import skill_publication import hashlib import json @@ -27,8 +29,11 @@ from pathlib import Path from typing import Dict, List, Optional -from . import junction, skills, store -from .paths import KIND_SKILL, capabilities_enabled, migrations_root, safe_segment +from core.infra import directory_links as junction +from features.desktop.capabilities import skills +from features.desktop.capabilities import store +from core.capabilities.identity_rules import KIND_SKILL, safe_segment +from features.desktop.capabilities.paths import capabilities_enabled, migrations_root logger = logging.getLogger(__name__) @@ -84,8 +89,8 @@ def _sweep_view_dir(report: MigrationReport, view: Path) -> None: def _import_flat_skills(report: MigrationReport) -> None: - from core.agent_skills.binary_files import decode_binary, is_binary_value, pack_directory - from core.services.desktop_capability_protocol import skill_content_hash + from core.capabilities.skills.binary_files import decode_binary, is_binary_value, pack_directory + from core.capabilities.manifest_protocol import skill_content_hash for flat in store.flat_import_candidates(KIND_SKILL): try: @@ -94,7 +99,7 @@ def _import_flat_skills(report: MigrationReport) -> None: files = {"SKILL.md": content} for rel, body in packed.items(): files[rel] = decode_binary(body) if is_binary_value(body) else body - skills.publish_local_skill( + skill_publication.publish_local_skill( safe_segment(flat.name), files=files, content_hash=skill_content_hash(content, packed), @@ -138,7 +143,7 @@ def _tree_inventory(root: Path) -> Optional[Dict[str, str]]: def _session_copy_sources(report: MigrationReport, workspace: Path) -> Dict[str, List[Path]]: """Preserved original views and immutable components from which copies can be rebuilt.""" - roots = [workspace / "skills", skills.builtin_dir()] + roots = [workspace / "skills", skill_paths.builtin_dir()] users = workspace / "skills_u" if users.is_dir(): roots.extend( @@ -236,9 +241,9 @@ def _drop_run_copies(report: MigrationReport) -> None: """ from core.db.models import ContentBlock - from . import registry - from .paths import require_root - from core.capabilities.runtime.state import _PREFIX, PreparedRun + from features.desktop.capabilities import registry + from features.desktop.capabilities.paths import require_root + from features.desktop.capabilities.runtime.state import _PREFIX, PreparedRun known = set(PreparedRun.__dataclass_fields__) with registry._session() as db: @@ -259,7 +264,7 @@ def migrate_legacy_layout() -> Optional[MigrationReport]: """Run the migration once per process start when the store is enabled.""" if not capabilities_enabled(): return None - from core.agent_skills.config import get_sandbox_skills_dir, get_user_skills_root + from core.capabilities.skills.config import get_sandbox_skills_dir, get_user_skills_root report = MigrationReport() shared = get_sandbox_skills_dir() diff --git a/src/backend/core/capabilities/paths.py b/src/backend/features/desktop/capabilities/paths.py similarity index 80% rename from src/backend/core/capabilities/paths.py rename to src/backend/features/desktop/capabilities/paths.py index 37d0920a..5b3e29e0 100644 --- a/src/backend/core/capabilities/paths.py +++ b/src/backend/features/desktop/capabilities/paths.py @@ -12,33 +12,18 @@ import os import re from pathlib import Path +from core.capabilities.identity_rules import KIND_SKILL, KIND_PLUGIN, KIND_AGENT, KIND_MCP, KINDS, LOCAL_PROFILE, BUILTIN_PROFILE, safe_segment from typing import Optional -CAPS_ROOT_ENV = "HUGAGENT_CAPS_ROOT" +from core.config.runtime_env import CAPS_ROOT_ENV -KIND_SKILL = "skill" -KIND_PLUGIN = "plugin" -KIND_AGENT = "agent" -KIND_MCP = "mcp" -KINDS = (KIND_SKILL, KIND_PLUGIN, KIND_AGENT, KIND_MCP) _KIND_DIRS = {KIND_SKILL: "skills", KIND_PLUGIN: "plugins", KIND_AGENT: "agents"} -LOCAL_PROFILE = "local" -BUILTIN_PROFILE = "builtin" META_DIR = ".capabilities" MCP_JSON_NAME = "mcp.json" -_SEGMENT_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,120}$") -_WINDOWS_RESERVED = { - "CON", - "PRN", - "AUX", - "NUL", - *(f"COM{i}" for i in range(1, 10)), - *(f"LPT{i}" for i in range(1, 10)), -} class CapabilityStoreDisabled(RuntimeError): @@ -73,8 +58,8 @@ def assert_managed_path(path: Path) -> Path: def _assert_managed_path(path: Path, root: Path) -> Path: - from .errors import IntegrityFailed - from .junction import is_directory_link + from core.harness.capability_errors import IntegrityFailed + from core.infra.directory_links import is_directory_link root = root.absolute() path = path.absolute() @@ -92,14 +77,6 @@ def _assert_managed_path(path: Path, root: Path) -> Path: return path -def safe_segment(value: str) -> str: - """A single path segment safe on every platform, or ``ValueError``.""" - seg = (value or "").strip() - if seg != value or seg.endswith((".", " ")) or not _SEGMENT_RE.match(seg) or seg in (".", ".."): - raise ValueError(f"unsafe path segment: {value!r}") - if seg.split(".")[0].upper() in _WINDOWS_RESERVED: - raise ValueError(f"reserved name: {value!r}") - return seg def _kind_path(kind: str, *segments: str) -> Path: diff --git a/src/backend/core/capabilities/plugins.py b/src/backend/features/desktop/capabilities/plugins.py similarity index 87% rename from src/backend/core/capabilities/plugins.py rename to src/backend/features/desktop/capabilities/plugins.py index f7516228..501979cf 100644 --- a/src/backend/core/capabilities/plugins.py +++ b/src/backend/features/desktop/capabilities/plugins.py @@ -7,23 +7,18 @@ """ from __future__ import annotations +from features.desktop.capabilities import registry_types +from features.desktop.capabilities import skill_account import json import logging from typing import Any, Dict, Iterable, List, Optional -from . import registry, store -from .paths import ( - KIND_AGENT, - KIND_MCP, - KIND_PLUGIN, - KIND_SKILL, - LOCAL_PROFILE, - revision_for_hash, - safe_segment, -) -from .ref import local_ref -from .resolver import SOURCE_LOCAL +from features.desktop.capabilities import registry, store +from core.capabilities.identity_rules import KIND_AGENT, KIND_MCP, KIND_PLUGIN, KIND_SKILL, LOCAL_PROFILE, safe_segment +from features.desktop.capabilities.paths import revision_for_hash +from core.capabilities.reference import local_ref +from features.desktop.capabilities.resolver import SOURCE_LOCAL logger = logging.getLogger(__name__) @@ -40,13 +35,13 @@ def cloud_binding_ids( plugin_ids: Iterable[str], *, user_id: Optional[str] ) -> tuple[list[str], list[str]]: """Expand selected, authorized cloud definitions without preparing their files.""" - from . import skills - from .dependency import Context, Inspector, _identifier, component_hash - from .errors import IntegrityFailed, NameConflict + from features.desktop.capabilities import skills + from features.desktop.capabilities.dependency import Context, Inspector, _identifier, component_hash + from core.harness.capability_errors import IntegrityFailed, NameConflict - if not skills.account_authorized_for(user_id): + if not skill_account.account_authorized_for(user_id): return [], [] - profile = skills.current_account_profile() + profile = skill_account.current_account_profile() account_profile = profile installed = registry.list_installations(kind=KIND_PLUGIN, profile_id=profile) selected_skills, selected_mcp = set(), set() @@ -126,7 +121,7 @@ def publish_local_plugin( definition: Dict[str, Any], *, owner_user_id: Optional[str] ) -> store.StoredComponent: """Project a device-installed plugin row into the store and record its components.""" - from core.services.desktop_capability_protocol import entity_content_hash + from core.capabilities.manifest_protocol import entity_content_hash slug = safe_segment(str(definition["slug"])) files = plugin_manifest_files(definition) @@ -159,7 +154,7 @@ def publish_local_plugin( def remove_local_plugin(slug: str) -> bool: - from core.capabilities.runtime import references + from features.desktop.capabilities.runtime import references if references("plugin", LOCAL_PROFILE, slug): # Source deletion revokes use; history still needs its exact bytes. @@ -169,14 +164,14 @@ def remove_local_plugin(slug: str) -> bool: def readiness( - inst: registry.Installation, + inst: registry_types.Installation, *, cloud_server_ids: Iterable[str], managed_enabled: Optional[Dict[str, bool]] = None, user_id: Optional[str] = None, ) -> Dict[str, Any]: """Component-by-component readiness of one plugin installation on this device.""" - from core.services.mcp_service import McpServerConfigService + from core.capabilities.mcp.service import McpServerConfigService service = McpServerConfigService.get_instance() selected_user = user_id or inst.payload.get("owner_user_id") @@ -219,7 +214,7 @@ def readiness( ) if required and not ok: missing.append(cid) - from .dependency import check_installation + from features.desktop.capabilities.dependency import check_installation report = check_installation( inst, @@ -240,12 +235,12 @@ def readiness( def enabled_cloud_skill_intents(user_id): """Skill intentions of enabled plugins in the current authorized account.""" - from . import skills - from .dependency import _identifier + from features.desktop.capabilities import skills + from features.desktop.capabilities.dependency import _identifier - if not skills.account_authorized_for(user_id): + if not skill_account.account_authorized_for(user_id): return set() - profile = skills.current_account_profile() + profile = skill_account.current_account_profile() selected = set() for row in registry.list_installations(kind=KIND_PLUGIN, profile_id=profile): if not row.enabled: diff --git a/src/backend/core/capabilities/preparation.py b/src/backend/features/desktop/capabilities/preparation.py similarity index 89% rename from src/backend/core/capabilities/preparation.py rename to src/backend/features/desktop/capabilities/preparation.py index 22fbda79..56f51811 100644 --- a/src/backend/core/capabilities/preparation.py +++ b/src/backend/features/desktop/capabilities/preparation.py @@ -1,14 +1,16 @@ """Account-bound publication and recovery shared by the three file package kinds.""" from __future__ import annotations +from features.desktop.bridge import identity as bridge_identity +from features.desktop.capabilities import skill_account import threading import logging import time -from . import registry, store -from .errors import CloudUnavailable, IntegrityFailed, PackageMissing -from .paths import revision_for_hash +from features.desktop.capabilities import registry, store +from core.harness.capability_errors import CloudUnavailable, IntegrityFailed, PackageMissing +from features.desktop.capabilities.paths import revision_for_hash _prepare_lock = threading.RLock() # (registry generation, profile) → ids still awaiting preparation + ownership edges. @@ -36,8 +38,8 @@ def _pending_closure(profile): def prepare_component(state, inst, *, download, content_hash, after_publish=None): - from core.services.desktop_cloud_bridge import account_scope, require_current_account - from core.services.desktop_cloud_skills import _profile + from features.desktop.bridge.identity import account_scope, require_current_account + from features.desktop.bridge.skills import _profile if inst.profile_id != _profile(state): raise PackageMissing("installation belongs to another account", ref=inst.install_id) @@ -105,7 +107,7 @@ def prepare_component(state, inst, *, download, content_hash, after_publish=None payload_update={"resolved_content_hash": inst.content_hash}, ) if inst.kind in ("skill", "plugin"): - from .change_sync import record_downloaded + from features.desktop.capabilities.change_sync import record_downloaded record_downloaded(done) if tx: registry.advance_transaction(tx, "committed") @@ -145,13 +147,13 @@ def ensure_cloud_ready(user_id, *, skill_keys=(), plugin_keys=(), install_ids=() 时，这里就是"按需拉取"的落点。桥未激活、没有对应的云端安装记录、或组件已就绪时不做 任何事。返回准备失败的 install_id 列表（下载/校验失败），由调用方决定如何报错。 """ - from . import registry, skills - from core.services import desktop_cloud_bridge as bridge + from features.desktop.capabilities import registry, skills + from features.desktop.bridge import client as bridge - if not skills.account_authorized_for(user_id): + if not skill_account.account_authorized_for(user_id): return [] - state = bridge.get_state() - profile = skills.current_account_profile() + state = bridge_identity.get_state() + profile = skill_account.current_account_profile() if not state or not profile: return [] @@ -184,14 +186,14 @@ def pending(kind_filter): failures = [] definitions = pending(("plugin", "agent")) if definitions: - from core.services import desktop_cloud_bundles + from features.desktop.bridge import bundles as desktop_cloud_bundles failures += [r["install_id"] for r in desktop_cloud_bundles.prepare(state, definitions) if not r["ok"]] for owner in definitions: wanted.update(registry.components_of(owner)) skill_ids = pending(("skill",)) if skill_ids: - from core.services import desktop_cloud_skills + from features.desktop.bridge import skills as desktop_cloud_skills failures += [r["install_id"] for r in desktop_cloud_skills.prepare(state, skill_ids) if not r["ok"]] return failures diff --git a/src/backend/core/capabilities/readiness.py b/src/backend/features/desktop/capabilities/readiness.py similarity index 92% rename from src/backend/core/capabilities/readiness.py rename to src/backend/features/desktop/capabilities/readiness.py index 1192069d..188892d0 100644 --- a/src/backend/core/capabilities/readiness.py +++ b/src/backend/features/desktop/capabilities/readiness.py @@ -1,15 +1,17 @@ """Read-only management readiness, separate from retained package publication state.""" from __future__ import annotations +from features.desktop.capabilities import skill_account +from features.desktop.capabilities import skills import shutil import os from concurrent.futures import ThreadPoolExecutor from contextvars import copy_context -from . import registry, store -from .dependency import Context, Inspector, allowed_model_ids, skill_definition -from .errors import CapabilityError -from .paths import LOCAL_PROFILE +from features.desktop.capabilities import registry, store +from features.desktop.capabilities.dependency import Context, Inspector, allowed_model_ids, skill_definition +from core.harness.capability_errors import CapabilityError +from core.capabilities.identity_rules import LOCAL_PROFILE def context_for_user(user_id, *, available_mcp=()): @@ -18,13 +20,13 @@ def context_for_user(user_id, *, available_mcp=()): except Exception: models = None try: - from core.auth.kb_permissions import get_accessible_local_kb_ids + from features.knowledge.edition import get_accessible_local_kb_ids with registry._session() as db: kbs = get_accessible_local_kb_ids(db, user_id) except Exception: kbs = None - from . import skills + from features.desktop.capabilities import skills choices = skills.resolve_for_user(user_id).chosen return Context( @@ -208,12 +210,12 @@ def resolve(self, kind, profile, key): parts = key.split(":", 2) if len(parts) != 3 or parts[0] != kind: return None, None, None - from . import skills - from .paths import BUILTIN_PROFILE + from features.desktop.capabilities import skills + from core.capabilities.identity_rules import BUILTIN_PROFILE if parts[1] not in (LOCAL_PROFILE, BUILTIN_PROFILE) and ( - parts[1] != skills.current_account_profile() - or not skills.account_authorized_for(self.context.user_id) + parts[1] != skill_account.current_account_profile() + or not skill_account.account_authorized_for(self.context.user_id) ): return None, None, None key = parts[2] @@ -243,7 +245,7 @@ def eligible_skill_candidates(candidates, user_id, *, preferences=None, requeste conservatively pruned until no selected node depends on an excluded node. """ from dataclasses import replace - from .resolver import resolve + from features.desktop.capabilities.resolver import resolve candidates = list(candidates) preferences = ( diff --git a/src/backend/core/capabilities/registry.py b/src/backend/features/desktop/capabilities/registry.py similarity index 84% rename from src/backend/core/capabilities/registry.py rename to src/backend/features/desktop/capabilities/registry.py index f1caa892..31f13c84 100644 --- a/src/backend/core/capabilities/registry.py +++ b/src/backend/features/desktop/capabilities/registry.py @@ -1,27 +1,21 @@ """Device installation index: intent + state, kept in the device's business database.""" from __future__ import annotations +from features.desktop.capabilities import registry_types import hashlib import itertools import threading import uuid from contextlib import contextmanager -from dataclasses import dataclass -from datetime import datetime from typing import Any, Dict, Iterator, List, Optional from core.db.engine import SessionLocal -from core.db.models import ( - DeviceCapabilityComponent, - DeviceCapabilityInstallation, - DeviceCapabilityNamePreference, - DeviceCapabilityTransaction, -) +from features.desktop.models import DeviceCapabilityComponent, DeviceCapabilityInstallation, DeviceCapabilityNamePreference, DeviceCapabilityTransaction from sqlalchemy import event, func from sqlalchemy.orm import Session -from .ref import ResourceRef +from core.capabilities.reference import ResourceRef _generation_lock = threading.Lock() @@ -87,77 +81,10 @@ def preference_id(kind: str, runtime_name: str, user_id: Optional[str] = None) - return f"{kind}:user:{key}" -@dataclass -class Installation: - install_id: str - profile_id: str - kind: str - key: str - ref: ResourceRef - display_name: str - description: str - version: str - content_hash: Optional[str] - resolved_revision: Optional[str] - state: str - enabled: bool - source: str - source_plugin: Optional[str] - generation: int - last_error: Optional[str] - payload: Dict[str, Any] - created_at: Optional[datetime] - - @property - def ready(self) -> bool: - return self.state == "ready" and bool(self.resolved_revision) - - def to_dict(self) -> Dict[str, Any]: - return { - "install_id": self.install_id, - "profile_id": self.profile_id, - "kind": self.kind, - "key": self.key, - "resource_ref": self.ref.to_dict(), - "display_name": self.display_name, - "description": self.description, - "version": self.version, - "content_hash": self.content_hash, - "resolved_revision": self.resolved_revision, - "state": self.state, - "enabled": self.enabled, - "source": self.source, - "source_plugin": self.source_plugin, - "generation": self.generation, - "last_error": self.last_error, - "payload": dict(self.payload or {}), - "created_at": self.created_at.isoformat() if self.created_at else None, - } - - -def _to_installation(row: DeviceCapabilityInstallation) -> Installation: - from .session_availability import permits - - return Installation( - install_id=row.install_id, - profile_id=row.profile_id, - kind=row.kind, - key=row.key, - ref=ResourceRef(row.ref_issuer, row.ref_namespace, row.kind, row.ref_id), - display_name=row.display_name or "", - description=row.description or "", - version=row.version or "", - content_hash=row.content_hash, - resolved_revision=row.resolved_revision, - state=row.state, - enabled=bool(row.enabled) and permits(row.profile_id, row.install_id, row.resolved_revision), - source=row.source, - source_plugin=row.source_plugin, - generation=int(row.generation or 0), - last_error=row.last_error, - payload=dict(row.payload or {}), - created_at=row.created_at, - ) + + + + @contextmanager @@ -190,7 +117,7 @@ def upsert( enabled: Optional[bool] = None, initial_enabled: Optional[bool] = None, db: Optional[Session] = None, -) -> Installation: +) -> registry_types.Installation: """Create or refresh the intent row. State transitions are done by the setters below. ``enabled`` 是本机意图，每次都写；``initial_enabled`` 只在这台机器第一次见到这个 @@ -239,16 +166,16 @@ def upsert( if row.state == "removed": row.state = "pending" s.flush() - return _to_installation(row) + return registry_types.to_installation(row) -def get(install_id_: str, db: Optional[Session] = None) -> Optional[Installation]: +def get(install_id_: str, db: Optional[Session] = None) -> Optional[registry_types.Installation]: with _session(db) as s: row = s.get(DeviceCapabilityInstallation, install_id_) - return _to_installation(row) if row else None + return registry_types.to_installation(row) if row else None -def get_many(install_ids, db: Optional[Session] = None) -> Dict[str, Installation]: +def get_many(install_ids, db: Optional[Session] = None) -> Dict[str, registry_types.Installation]: """Whole-closure lookup in one query. ``validate`` re-checks every frozen node and binding on every message — @@ -258,7 +185,7 @@ def get_many(install_ids, db: Optional[Session] = None) -> Dict[str, Installatio ids = [str(i) for i in dict.fromkeys(install_ids)] if not ids: return {} - found: Dict[str, Installation] = {} + found: Dict[str, registry_types.Installation] = {} with _session(db) as s: for start in range(0, len(ids), 400): rows = ( @@ -266,7 +193,7 @@ def get_many(install_ids, db: Optional[Session] = None) -> Dict[str, Installatio .filter(DeviceCapabilityInstallation.install_id.in_(ids[start : start + 400])) .all() ) - found.update({row.install_id: _to_installation(row) for row in rows}) + found.update({row.install_id: registry_types.to_installation(row) for row in rows}) return found @@ -277,7 +204,7 @@ def list_installations( profiles: Optional[List[str]] = None, include_removed: bool = False, db: Optional[Session] = None, -) -> List[Installation]: +) -> List[registry_types.Installation]: with _session(db) as s: q = s.query(DeviceCapabilityInstallation) if kind: @@ -289,7 +216,7 @@ def list_installations( if not include_removed: q = q.filter(DeviceCapabilityInstallation.state != "removed") rows = q.order_by(DeviceCapabilityInstallation.kind, DeviceCapabilityInstallation.key).all() - return [_to_installation(r) for r in rows] + return [registry_types.to_installation(r) for r in rows] def source_token(*, kind: str, profile_id: str, db: Optional[Session] = None) -> tuple: @@ -334,7 +261,7 @@ def set_state( last_error: Optional[str] = None, payload_update: Optional[Dict[str, Any]] = None, db: Optional[Session] = None, -) -> Optional[Installation]: +) -> Optional[registry_types.Installation]: with _session(db) as s: row = s.get(DeviceCapabilityInstallation, install_id_) if row is None: @@ -347,7 +274,7 @@ def set_state( row.generation = int(row.generation or 0) + 1 payload = dict(row.payload or {}) payload.pop("update_available", None) - from .paths import revision_for_hash + from features.desktop.capabilities.paths import revision_for_hash if row.content_hash and row.resolved_revision == revision_for_hash(row.content_hash): payload["resolved_content_hash"] = row.content_hash @@ -355,19 +282,19 @@ def set_state( if payload_update: row.payload = {**dict(row.payload or {}), **payload_update} s.flush() - return _to_installation(row) + return registry_types.to_installation(row) def set_enabled( install_id_: str, enabled: bool, db: Optional[Session] = None -) -> Optional[Installation]: +) -> Optional[registry_types.Installation]: with _session(db) as s: row = s.get(DeviceCapabilityInstallation, install_id_) if row is None: return None row.enabled = enabled s.flush() - return _to_installation(row) + return registry_types.to_installation(row) def mark_removed(install_id_: str, db: Optional[Session] = None) -> bool: diff --git a/src/backend/features/desktop/capabilities/registry_types.py b/src/backend/features/desktop/capabilities/registry_types.py new file mode 100644 index 00000000..b71feec3 --- /dev/null +++ b/src/backend/features/desktop/capabilities/registry_types.py @@ -0,0 +1,79 @@ +"""Typed installation projections of device capability records.""" +from __future__ import annotations +from dataclasses import dataclass +from datetime import datetime +from typing import Any, Dict, Optional +from features.desktop.models import DeviceCapabilityInstallation +from core.capabilities.reference import ResourceRef + +@dataclass +class Installation: + install_id: str + profile_id: str + kind: str + key: str + ref: ResourceRef + display_name: str + description: str + version: str + content_hash: Optional[str] + resolved_revision: Optional[str] + state: str + enabled: bool + source: str + source_plugin: Optional[str] + generation: int + last_error: Optional[str] + payload: Dict[str, Any] + created_at: Optional[datetime] + + @property + def ready(self) -> bool: + return self.state == "ready" and bool(self.resolved_revision) + + def to_dict(self) -> Dict[str, Any]: + return { + "install_id": self.install_id, + "profile_id": self.profile_id, + "kind": self.kind, + "key": self.key, + "resource_ref": self.ref.to_dict(), + "display_name": self.display_name, + "description": self.description, + "version": self.version, + "content_hash": self.content_hash, + "resolved_revision": self.resolved_revision, + "state": self.state, + "enabled": self.enabled, + "source": self.source, + "source_plugin": self.source_plugin, + "generation": self.generation, + "last_error": self.last_error, + "payload": dict(self.payload or {}), + "created_at": self.created_at.isoformat() if self.created_at else None, + } + + +def to_installation(row: DeviceCapabilityInstallation) -> Installation: + from features.desktop.capabilities.session_availability import permits + + return Installation( + install_id=row.install_id, + profile_id=row.profile_id, + kind=row.kind, + key=row.key, + ref=ResourceRef(row.ref_issuer, row.ref_namespace, row.kind, row.ref_id), + display_name=row.display_name or "", + description=row.description or "", + version=row.version or "", + content_hash=row.content_hash, + resolved_revision=row.resolved_revision, + state=row.state, + enabled=bool(row.enabled) and permits(row.profile_id, row.install_id, row.resolved_revision), + source=row.source, + source_plugin=row.source_plugin, + generation=int(row.generation or 0), + last_error=row.last_error, + payload=dict(row.payload or {}), + created_at=row.created_at, + ) diff --git a/src/backend/core/capabilities/resolver.py b/src/backend/features/desktop/capabilities/resolver.py similarity index 98% rename from src/backend/core/capabilities/resolver.py rename to src/backend/features/desktop/capabilities/resolver.py index 487fb434..6ceb3a33 100644 --- a/src/backend/core/capabilities/resolver.py +++ b/src/backend/features/desktop/capabilities/resolver.py @@ -19,8 +19,8 @@ from pathlib import Path from typing import Dict, Iterable, List, Optional, Set -from .paths import BUILTIN_PROFILE, LOCAL_PROFILE -from .ref import ResourceRef +from core.capabilities.identity_rules import BUILTIN_PROFILE, LOCAL_PROFILE +from core.capabilities.reference import ResourceRef SOURCE_CLOUD = "cloud" SOURCE_LOCAL = "local" diff --git a/src/backend/features/desktop/capabilities/runtime/__init__.py b/src/backend/features/desktop/capabilities/runtime/__init__.py new file mode 100644 index 00000000..677c0c1f --- /dev/null +++ b/src/backend/features/desktop/capabilities/runtime/__init__.py @@ -0,0 +1,37 @@ +"""Public capability runtime API; state and operations have one-way dependencies.""" + +from features.desktop.capabilities.runtime.connectors import bind_mcp as bind_mcp +from features.desktop.capabilities.runtime.connectors import pin_agent_definition as pin_agent_definition +from features.desktop.capabilities.runtime.connectors import references as references +from features.desktop.capabilities.runtime.dependencies import preflight as preflight +from features.desktop.capabilities.runtime.dependencies import validate_activation as validate_activation +from features.desktop.capabilities.runtime.preparation import prepare as prepare +from features.desktop.capabilities.runtime.state import PreparedRun as PreparedRun +from features.desktop.capabilities.runtime.state import child_scope as child_scope +from features.desktop.capabilities.runtime.state import frozen_loader as frozen_loader +from features.desktop.capabilities.runtime.state import get as get +from features.desktop.capabilities.runtime.state import rebuild as rebuild +from features.desktop.capabilities.runtime.state import record_tool_scope as record_tool_scope +from features.desktop.capabilities.runtime.state import require_root_tool_scope as require_root_tool_scope +from features.desktop.capabilities.runtime.state import save as save +from features.desktop.capabilities.runtime.state import validate as validate +from features.desktop.capabilities.runtime.state import view_for_execution as view_for_execution + +__all__ = [ + "child_scope", + "PreparedRun", + "get", + "validate", + "rebuild", + "save", + "frozen_loader", + "view_for_execution", + "record_tool_scope", + "require_root_tool_scope", + "prepare", + "bind_mcp", + "pin_agent_definition", + "references", + "preflight", + "validate_activation", +] diff --git a/src/backend/core/capabilities/runtime/connectors.py b/src/backend/features/desktop/capabilities/runtime/connectors.py similarity index 92% rename from src/backend/core/capabilities/runtime/connectors.py rename to src/backend/features/desktop/capabilities/runtime/connectors.py index 74451c72..8cb45c6f 100644 --- a/src/backend/core/capabilities/runtime/connectors.py +++ b/src/backend/features/desktop/capabilities/runtime/connectors.py @@ -1,6 +1,7 @@ """Pin connector and agent identities for durable replay.""" from __future__ import annotations +from features.desktop.capabilities import skill_account import copy import hashlib @@ -8,10 +9,10 @@ from dataclasses import replace from typing import Optional -from core.capabilities import registry, skills -from core.capabilities.errors import PermissionDenied -from core.capabilities.paths import LOCAL_PROFILE -from core.capabilities.runtime import state as runtime +from features.desktop.capabilities import registry, skills +from core.harness.capability_errors import PermissionDenied +from core.capabilities.identity_rules import LOCAL_PROFILE +from features.desktop.capabilities.runtime import state as runtime from core.db.models import ContentBlock @@ -47,7 +48,7 @@ def public_values(values): def bind_mcp(run: runtime.PreparedRun, configs, choices): """Freeze MCP source identities/contracts; refresh credentials independently.""" - from core.capabilities.connectors import server_id_of + from features.desktop.capabilities.connectors import server_id_of selected = {server_id_of(c): c.install_id for c in choices.chosen.values()} if choices else {} current = { @@ -112,24 +113,20 @@ def bind_mcp(run: runtime.PreparedRun, configs, choices): def pin_agent_definition(run_id, user_id, definition, *, scope_id: str = ""): """Keep the selected agent's instructions and dependency IDs stable on replay.""" - from core.capabilities.agents import _JSON_FIELDS, AgentDefinition + from features.desktop.capabilities.agents import _JSON_FIELDS, AgentDefinition ident = str(definition.agent_id) key = "desktop_capability_agent:" + runtime._snapshot_key(str(run_id) + ":" + ident, scope_id) profile = ( - skills.current_account_profile() + skill_account.current_account_profile() if getattr(definition, "origin", "local") == "cloud" else None ) - from core.services.desktop_cloud_bridge import ( - _state_fingerprint, - ensure_current_authorization, - get_state, - ) + from features.desktop.bridge.identity import state_fingerprint as _state_fingerprint, ensure_current_authorization, get_state fingerprint = _state_fingerprint(get_state()) if profile else None if getattr(definition, "origin", "local") == "cloud": - if not skills.account_authorized_for(user_id): + if not skill_account.account_authorized_for(user_id): raise PermissionDenied("cloud agent belongs to another user") ensure_current_authorization() current = registry.get( diff --git a/src/backend/core/capabilities/runtime/dependencies.py b/src/backend/features/desktop/capabilities/runtime/dependencies.py similarity index 93% rename from src/backend/core/capabilities/runtime/dependencies.py rename to src/backend/features/desktop/capabilities/runtime/dependencies.py index 061c7767..cdd4b35a 100644 --- a/src/backend/core/capabilities/runtime/dependencies.py +++ b/src/backend/features/desktop/capabilities/runtime/dependencies.py @@ -4,10 +4,10 @@ from dataclasses import replace -from core.capabilities import registry, store -from core.capabilities.errors import CapabilityError, IntegrityFailed, PermissionDenied -from core.capabilities.paths import BUILTIN_PROFILE, LOCAL_PROFILE -from core.capabilities.runtime import state as runtime +from features.desktop.capabilities import registry, store +from core.harness.capability_errors import CapabilityError, IntegrityFailed, PermissionDenied +from core.capabilities.identity_rules import BUILTIN_PROFILE, LOCAL_PROFILE +from features.desktop.capabilities.runtime import state as runtime def validate_activation(run: runtime.PreparedRun, nodes, *, available_mcp=()) -> None: @@ -18,13 +18,7 @@ def validate_activation(run: runtime.PreparedRun, nodes, *, available_mcp=()) -> 技能版本去对，而不是拿当前安装的版本。``nodes`` 是推迟那一刻记下的身份。 只查这一个插件及其定义闭包，不牵连整轮。 """ - from core.capabilities.dependency import ( - Context, - Inspector, - _identifier, - component_hash, - require_report, - ) + from features.desktop.capabilities.dependency import Context, Inspector, _identifier, component_hash, require_report nodes = list(nodes or []) if not nodes: @@ -88,7 +82,7 @@ def preflight( 绝不因此让整个助手不可用。连接器与插件的从属关系记进 ``connector_parents``， 供「加载插件」一类的工具反查。 """ - from core.capabilities.dependency import Context, Inspector + from features.desktop.capabilities.dependency import Context, Inspector runtime.validate(run) run = runtime.get(run.run_id, scope_id=run.scope_id) or run diff --git a/src/backend/core/capabilities/runtime/preparation.py b/src/backend/features/desktop/capabilities/runtime/preparation.py similarity index 86% rename from src/backend/core/capabilities/runtime/preparation.py rename to src/backend/features/desktop/capabilities/runtime/preparation.py index 001f466e..b7ee6d64 100644 --- a/src/backend/core/capabilities/runtime/preparation.py +++ b/src/backend/features/desktop/capabilities/runtime/preparation.py @@ -1,19 +1,26 @@ """Resolve and freeze user-selected capability revisions.""" from __future__ import annotations +from features.desktop.capabilities import skill_account +from features.desktop.capabilities import skill_hashes +from features.desktop.capabilities import skills import copy -from core.capabilities import archive, registry, skills, store -from core.capabilities.errors import CapabilityError, IntegrityFailed, PackageMissing -from core.capabilities.paths import BUILTIN_PROFILE, KIND_SKILL, LOCAL_PROFILE, revision_for_hash -from core.capabilities.runtime import state as runtime +from core.capabilities import archive as archive +from features.desktop.capabilities import registry +from features.desktop.capabilities import skills +from features.desktop.capabilities import store +from core.harness.capability_errors import CapabilityError, IntegrityFailed, PackageMissing +from core.capabilities.identity_rules import BUILTIN_PROFILE, KIND_SKILL, LOCAL_PROFILE +from features.desktop.capabilities.paths import revision_for_hash +from features.desktop.capabilities.runtime import state as runtime def _freeze_candidate(name, candidate): # 摘要走内容指纹缓存：候选路径是不可变存储里的版本目录，字节一改元数据签名 # 就变、缓存随之失效，所以复用安全，而重复冻结同一版本不必再读一遍磁盘。 - actual = skills.skill_dir_hash(candidate.path) + actual = skill_hashes.skill_dir_hash(candidate.path) revision = candidate.revision or revision_for_hash(actual) profile = candidate.profile if profile == BUILTIN_PROFILE: @@ -75,7 +82,7 @@ def prepare( 永不原地修改，所以编辑安装目录只会产生新版本，已冻结的这一轮看到的字节自始 至终不变。准备不了的单个组件记进 ``unavailable``，不影响其余组件。 """ - from core.capabilities.preparation import ensure_cloud_ready + from features.desktop.capabilities.preparation import ensure_cloud_ready if not run_id: raise ValueError("a run id is required for a durable capability snapshot") @@ -99,7 +106,7 @@ def prepare( ) # 这一轮的根：选中的技能，加上选中的插件。先各自按需下载，再从它们出发 # 走一遍依赖，把闭包里牵连到的技能也收进来。一个根准备不了只记下它自己。 - account = skills.current_account_profile() or LOCAL_PROFILE + account = skill_account.current_account_profile() or LOCAL_PROFILE roots = [("skill", name, LOCAL_PROFILE, name) for name in requested] roots += [("plugin", key, account, "plugin:" + key) for key in plugin_ids] unavailable = {} @@ -110,8 +117,8 @@ def prepare( except (CapabilityError, OSError, ValueError) as exc: unavailable[label] = getattr(exc, "code", "package_missing") resolution = skills.resolve_for_user(str(user_id)) - from core.capabilities.dependency import Context - from core.capabilities.readiness import _BindingInspector, _choice_bindings + from features.desktop.capabilities.dependency import Context + from features.desktop.capabilities.readiness import _BindingInspector, _choice_bindings discovery = _BindingInspector( Context( diff --git a/src/backend/core/capabilities/runtime/state.py b/src/backend/features/desktop/capabilities/runtime/state.py similarity index 89% rename from src/backend/core/capabilities/runtime/state.py rename to src/backend/features/desktop/capabilities/runtime/state.py index 02b6b727..2fad7cd9 100644 --- a/src/backend/core/capabilities/runtime/state.py +++ b/src/backend/features/desktop/capabilities/runtime/state.py @@ -6,6 +6,10 @@ """ from __future__ import annotations +from core.harness.registry.capability_runtime import child_scope +from features.desktop.capabilities import skill_account +from features.desktop.capabilities import skill_hashes +from features.desktop.capabilities import skills import copy import hashlib @@ -15,15 +19,11 @@ from pathlib import Path from typing import Any, Dict, Optional -from core.capabilities import registry, skills, store, view -from core.capabilities.errors import ( - CapabilityError, - IntegrityFailed, - PackageMissing, - PermissionDenied, - ViewUnavailable, -) -from core.capabilities.paths import BUILTIN_PROFILE, KIND_SKILL, LOCAL_PROFILE, require_root +from features.desktop.capabilities import registry, skills, store, view +from features.desktop.run_capabilities.provider import PreparedCapabilityOperations +from core.harness.capability_errors import CapabilityError, IntegrityFailed, PackageMissing, PermissionDenied, ViewUnavailable +from core.capabilities.identity_rules import BUILTIN_PROFILE, KIND_SKILL, LOCAL_PROFILE +from features.desktop.capabilities.paths import require_root from core.db.models import ContentBlock _lock = threading.RLock() @@ -34,14 +34,6 @@ def _key(run_id): return hashlib.sha256(str(run_id).encode()).hexdigest() -def child_scope(parent_scope: str, kind: str, *identities: str) -> str: - """Bounded, deterministic scope from durable orchestration identities.""" - if not kind or not identities or any(not str(value or "").strip() for value in identities): - raise IntegrityFailed("capability scope requires durable invocation identities") - material = json.dumps( - [str(parent_scope or ""), str(kind), *map(str, identities)], separators=(",", ":") - ) - return "s_" + _key(material) def _snapshot_key(run_id, scope_id=""): @@ -54,7 +46,7 @@ def _snapshot_key(run_id, scope_id=""): @dataclass(frozen=True) -class PreparedRun: +class PreparedRun(PreparedCapabilityOperations): run_id: str user_id: str profile: Optional[str] @@ -128,7 +120,7 @@ def _validate_skill_binding(name: str, binding: dict, run: PreparedRun) -> None: owner = dependency.payload.get("owner_user_id") if owner and str(owner) != run.user_id: raise PermissionDenied("skill dependency belongs to another user", runtime_name=name) - if comp is None or skills.skill_dir_hash(comp.path) != binding["content_hash"]: + if comp is None or skill_hashes.skill_dir_hash(comp.path) != binding["content_hash"]: raise IntegrityFailed("prepared revision is missing or changed", runtime_name=name) @@ -153,14 +145,10 @@ def validate( details={"recovery_action": "switch_execution_plane"}, ) if run.profile is not None: - from core.services.desktop_cloud_bridge import ( - _state_fingerprint, - ensure_current_authorization, - get_state, - ) + from features.desktop.bridge.identity import state_fingerprint as _state_fingerprint, ensure_current_authorization, get_state if ( - run.profile != skills.current_account_profile() + run.profile != skill_account.current_account_profile() or run.authorization_fingerprint != _state_fingerprint(get_state()) ): raise PermissionDenied("cloud session changed; prepare a new run") @@ -225,15 +213,11 @@ def rebuild(run: PreparedRun) -> Path: def _bind_cloud_identity(run): - from core.services.desktop_cloud_bridge import ( - _state_fingerprint, - ensure_current_authorization, - get_state, - ) + from features.desktop.bridge.identity import state_fingerprint as _state_fingerprint, ensure_current_authorization, get_state - if not skills.account_authorized_for(run.user_id): + if not skill_account.account_authorized_for(run.user_id): raise PermissionDenied("cloud capabilities belong to another user") - profile = skills.current_account_profile() + profile = skill_account.current_account_profile() fingerprint = _state_fingerprint(get_state()) if not profile: raise PermissionDenied("cloud session is unavailable") @@ -284,8 +268,8 @@ def _ensure_view(run: PreparedRun) -> None: def frozen_loader(run: PreparedRun): - from core.agent_skills.backends import CompositeBackend, FilesystemBackend - from core.agent_skills.loader import MultiSourceSkillLoader + from core.capabilities.skills.backends import CompositeBackend, FilesystemBackend + from core.capabilities.skills.loader import MultiSourceSkillLoader _ensure_view(run) # 视图重建可能刚摘掉某个组件，读取方必须拿到落库后的最新 ``unavailable``。 diff --git a/src/backend/core/capabilities/session_availability.py b/src/backend/features/desktop/capabilities/session_availability.py similarity index 100% rename from src/backend/core/capabilities/session_availability.py rename to src/backend/features/desktop/capabilities/session_availability.py diff --git a/src/backend/features/desktop/capabilities/skill_account.py b/src/backend/features/desktop/capabilities/skill_account.py new file mode 100644 index 00000000..4c759199 --- /dev/null +++ b/src/backend/features/desktop/capabilities/skill_account.py @@ -0,0 +1,54 @@ +"""skill account responsibilities.""" +from __future__ import annotations +from typing import Optional +from features.desktop.capabilities import registry +from core.capabilities.reference import profile_id +from features.desktop.capabilities import skills_state + +def current_account_profile() -> Optional[str]: + """Profile id of the cloud account this device is currently bridged to.""" + from core.capabilities.manifest_protocol import token_subject + from features.desktop.bridge.identity import get_state + + st = get_state() + if not st: + return None + subject = token_subject(str(st.get("token") or "")) + if not subject: + return None + return profile_id(str(st["cloud_base"]), subject) + + +def current_local_user_id() -> Optional[str]: + """Map the signed stable cloud subject to this device's shadow-user ID.""" + from features.desktop.bridge.identity import get_identity_state + + identity = get_identity_state() + # 本机影子用户按壳送来的命名空间标识建档（cloud:::）， + # 查找必须用同一标识，而不是凭据里的原始云端 id。 + center = str((identity or {}).get("shell_user_center_id") or "") + if not center: + return None + # user_center_id → user_id is a primary-key mapping fixed at creation; only + # hits are remembered so a shadow created later is still found. + hit = skills_state.shadow_user_ids.get(center) + if hit is not None: + return hit + from core.db.models import UserShadow + + with registry._session() as db: + row = db.query(UserShadow).filter(UserShadow.user_center_id == center).first() + if row is None: + return None + skills_state.shadow_user_ids[center] = str(row.user_id) + return skills_state.shadow_user_ids[center] + + +def account_authorized_for(user_id: Optional[str]) -> bool: + from features.desktop.bridge.identity import get_state + + st = get_state() + if not st or not str(st.get("token") or "").startswith("dcap2."): + return False + current = current_local_user_id() + return bool(user_id and current and str(user_id) == current) diff --git a/src/backend/features/desktop/capabilities/skill_candidates.py b/src/backend/features/desktop/capabilities/skill_candidates.py new file mode 100644 index 00000000..5315132a --- /dev/null +++ b/src/backend/features/desktop/capabilities/skill_candidates.py @@ -0,0 +1,140 @@ +"""skill candidates responsibilities.""" +from __future__ import annotations +from features.desktop.capabilities import registry_types +import os +from concurrent.futures import ThreadPoolExecutor +from contextvars import copy_context +from typing import Dict, List, Optional, Tuple +from features.desktop.capabilities import registry, store +from core.capabilities.identity_rules import BUILTIN_PROFILE, KIND_SKILL, LOCAL_PROFILE +from core.capabilities.reference import builtin_ref +from features.desktop.capabilities.resolver import SOURCE_BUILTIN, Candidate +from features.desktop.capabilities import skill_account +from features.desktop.capabilities import skill_hashes +from features.desktop.capabilities import skill_paths + +def builtin_candidates() -> List[Candidate]: + """安装包自带的技能。 + + 混合模式下它们不进能力中心、也不会被启用（目录层在 ``get_runtime_catalog`` 里就清空 + 了），但**候选必须留着**：云端技能会声明对自带技能的依赖，自带的那份还是云端副本在 + 本平台跑不了时的兜底，候选一空这两条链路就断了。 + """ + root = skill_paths.builtin_dir() + out: List[Candidate] = [] + if not root.is_dir(): + return out + for d in sorted(p for p in root.iterdir() if p.is_dir()): + if not (d / "SKILL.md").is_file(): + continue + out.append( + Candidate( + install_id=registry.install_id(KIND_SKILL, BUILTIN_PROFILE, d.name), + runtime_name=d.name, + kind=KIND_SKILL, + profile=BUILTIN_PROFILE, + source=SOURCE_BUILTIN, + path=d, + content_hash=None, # computed lazily only on a name collision + ref=builtin_ref(KIND_SKILL, d.name), + usable=True, + account_level=False, + display_name=d.name, + ) + ) + return out + + +def installation_candidate(inst: registry_types.Installation, *, account_level: bool) -> Candidate: + path = None + if inst.ready: + from core.harness.capability_errors import CapabilityError + + try: + comp = store.get(KIND_SKILL, inst.profile_id, inst.key, inst.resolved_revision or "") + except (CapabilityError, OSError): + comp = None + path = comp.path if comp else None + usable = inst.enabled and path is not None + return Candidate( + install_id=inst.install_id, + runtime_name=str(inst.payload.get("runtime_name") or inst.key), + kind=KIND_SKILL, + profile=inst.profile_id, + source=inst.source, + path=path, + content_hash=inst.payload.get("resolved_content_hash") or inst.content_hash, + revision=inst.resolved_revision, + ref=inst.ref, + usable=usable, + account_level=account_level and inst.enabled, + state=inst.state if path is not None or not inst.ready else "files_missing", + display_name=inst.display_name, + description=inst.description, + version=inst.version, + ) + + +def installation_candidates(rows): + rows = list(rows) + if os.name == "nt" and len(rows) >= 8: + with ThreadPoolExecutor(max_workers=4, thread_name_prefix="cap-candidates") as pool: + futures = [ + pool.submit(copy_context().run, installation_candidate, row, account_level=True) + for row in rows + ] + return [future.result() for future in futures] + return [installation_candidate(row, account_level=True) for row in rows] + + +def local_candidates(user_id: Optional[str], *, shared_only: bool = False) -> List[Candidate]: + rows = [] + for inst in registry.list_installations(kind=KIND_SKILL, profile_id=LOCAL_PROFILE): + owner = inst.payload.get("owner_user_id") + if owner and (shared_only or owner != user_id): + continue + rows.append(inst) + return installation_candidates(rows) + + +def account_candidates(profile: Optional[str] = None) -> List[Candidate]: + profile = profile or skill_account.current_account_profile() + if not profile: + return [] + return installation_candidates( + registry.list_installations(kind=KIND_SKILL, profile_id=profile) + ) + + +def candidates(user_id: Optional[str]) -> List[Candidate]: + return ( + builtin_candidates() + + local_candidates(user_id) + + (account_candidates() if skill_account.account_authorized_for(user_id) else []) + ) + + +def fill_hashes(cands: List[Candidate]) -> List[Candidate]: + """Hash only the members of multi-candidate names (identical-content detection).""" + by_name: Dict[str, List[Candidate]] = {} + for c in cands: + by_name.setdefault(c.runtime_name, []).append(c) + out: List[Candidate] = [] + for group in by_name.values(): + if len(group) < 2: + out.extend(group) + continue + for c in group: + if c.content_hash is None and c.path is not None: + c = Candidate(**{**c.__dict__, "content_hash": skill_hashes.skill_dir_hash(c.path)}) + out.append(c) + return out + + +def builtin_signature() -> Tuple: + """Shipped built-ins live on disk, outside every write signal — list them.""" + root = skill_paths.builtin_dir() + try: + return (str(root), tuple(sorted(p.name for p in root.iterdir() if p.is_dir()))) + except OSError: + return (str(root), ()) diff --git a/src/backend/features/desktop/capabilities/skill_hashes.py b/src/backend/features/desktop/capabilities/skill_hashes.py new file mode 100644 index 00000000..08604cab --- /dev/null +++ b/src/backend/features/desktop/capabilities/skill_hashes.py @@ -0,0 +1,101 @@ +"""skill hashes responsibilities.""" +from __future__ import annotations +import hashlib +from pathlib import Path +from typing import Callable, Tuple +from features.desktop.capabilities import skills_state + +def dir_signature(path: Path) -> Tuple: + from core.capabilities.archive import iter_file_stats + + # A maximum mtime misses edits to any file older than the newest file. + return tuple( + (rel, metadata.st_size, metadata.st_mtime_ns, metadata.st_ctime_ns, metadata.st_ino) + for rel, _, metadata in iter_file_stats(path) + if rel != skills_state.INVENTORY_NAME + ) + + +def dir_digest(path: Path, compute: Callable[[], str], *, fresh: bool = False) -> str: + """A package directory's content digest, cached on its file signature. + + Identity is *established* with ``fresh=True``: every byte is read, so the + digest a binding freezes is always a true content hash. Later passes only + *verify* that digest, and a run re-verifies its whole closure on every + message — re-reading a hundred unchanged packages each time is pure cost. + So verification recomputes exactly when the signature moved (a file was + added, removed, or written), and reuses the frozen digest otherwise. + """ + key = str(path.resolve()) + sig = dir_signature(path) + with skills_state.hash_lock: + hit = skills_state.hash_cache.get(key) + if not fresh and hit and hit[0] == sig: + return hit[1] + digest = compute() + with skills_state.hash_lock: + skills_state.hash_cache[key] = (sig, digest) + return digest + + +def skill_dir_hash(path: Path, *, fresh: bool = False) -> str: + """Hash every package file; reject links and excess sizes rather than omit bytes.""" + from core.capabilities.archive import iter_file_stats + + key = str(path.resolve()) + entries = [item for item in iter_file_stats(path) if item[0] != skills_state.INVENTORY_NAME] + sig = tuple( + (rel, metadata.st_size, metadata.st_mtime_ns, metadata.st_ctime_ns, metadata.st_ino) + for rel, _, metadata in entries + ) + with skills_state.hash_lock: + hit = skills_state.hash_cache.get(key) + if not fresh and hit and hit[0] == sig: + return hit[1] + # Reuse this traversal's metadata; fresh verification still reads every byte. + digest = hash_skill_package(path, entries=entries) + with skills_state.hash_lock: + skills_state.hash_cache[key] = (sig, digest) + return digest + + +def hash_skill_package(path: Path, *, entries=None) -> str: + from core.capabilities.skills.binary_files import encode_upload + from core.capabilities.manifest_protocol import skill_content_hash + + from core.capabilities import archive as archive + from core.harness.capability_errors import IntegrityFailed + from core.infra.directory_links import native_path as _native + + raw_files = {} + total = 0 + for rel, file, metadata in archive.iter_file_stats(path) if entries is None else entries: + if rel == skills_state.INVENTORY_NAME: + continue + size = metadata.st_size + total += size + if ( + size > archive.MAX_MEMBER_BYTES + or total > archive.MAX_TOTAL_BYTES + or len(raw_files) >= archive.MAX_MEMBERS + ): + raise IntegrityFailed("skill package exceeds content verification limits") + # Scanning already supports extended Windows paths; byte reads must too. + raw = _native(file).read_bytes() + if len(raw) != size: + raise IntegrityFailed("skill package changed during verification") + raw_files[rel] = raw + content_key = tuple((rel, hashlib.sha256(raw).digest()) for rel, raw in raw_files.items()) + with skills_state.hash_lock: + cached = skills_state.content_hash_cache.get(content_key) + if cached is not None: + skills_state.content_hash_cache.move_to_end(content_key) + return cached + files = {rel: encode_upload(rel, raw) for rel, raw in raw_files.items()} + digest = skill_content_hash(files.pop("SKILL.md", ""), files) + with skills_state.hash_lock: + skills_state.content_hash_cache[content_key] = digest + skills_state.content_hash_cache.move_to_end(content_key) + while len(skills_state.content_hash_cache) > 256: + skills_state.content_hash_cache.popitem(last=False) + return digest diff --git a/src/backend/features/desktop/capabilities/skill_paths.py b/src/backend/features/desktop/capabilities/skill_paths.py new file mode 100644 index 00000000..d05d5584 --- /dev/null +++ b/src/backend/features/desktop/capabilities/skill_paths.py @@ -0,0 +1,29 @@ +"""skill paths responsibilities.""" +from __future__ import annotations +from pathlib import Path +from typing import List, Optional +from extensions.paths import SKILLS_DIR +from features.desktop.capabilities.paths import capability_root + +def builtin_dir() -> Path: + return SKILLS_DIR / "default" + + +def allowed_link_roots() -> List[Path]: + roots = [builtin_dir()] + root = capability_root() + if root is not None: + roots.insert(0, root) + return roots + + +def device_view_dir() -> Path: + from core.capabilities.skills.config import get_sandbox_skills_dir + + return get_sandbox_skills_dir() + + +def user_view_dir(user_id: Optional[str]) -> Optional[Path]: + from core.capabilities.skills.config import get_user_skills_dir + + return get_user_skills_dir(user_id) diff --git a/src/backend/features/desktop/capabilities/skill_publication.py b/src/backend/features/desktop/capabilities/skill_publication.py new file mode 100644 index 00000000..91b4ca3e --- /dev/null +++ b/src/backend/features/desktop/capabilities/skill_publication.py @@ -0,0 +1,88 @@ +"""skill publication responsibilities.""" +from __future__ import annotations +from features.desktop.capabilities import skills +from typing import Dict, Optional +from features.desktop.capabilities import registry, store +from core.capabilities.identity_rules import KIND_SKILL, LOCAL_PROFILE +from features.desktop.capabilities.paths import revision_for_hash +from core.capabilities.reference import local_ref +from features.desktop.capabilities.resolver import SOURCE_LOCAL, SOURCE_PLUGIN +from features.desktop.capabilities import skills + +def publish_local_skill( + skill_id: str, + *, + files: Dict[str, bytes | str], + content_hash: str, + owner_user_id: Optional[str] = None, + source: str = SOURCE_LOCAL, + source_plugin: Optional[str] = None, + display_name: str = "", + description: str = "", + version: str = "", + enabled: bool = True, + from_db: bool = False, +) -> store.StoredComponent: + """Write one revision of a device-local skill and make it the ready revision.""" + revision = revision_for_hash(content_hash) + before_generation = registry.generation() + published = False + ref = local_ref(KIND_SKILL, skill_id) + inst = registry.upsert( + profile_id=LOCAL_PROFILE, + ref=ref, + display_name=display_name or skill_id, + description=description, + version=version, + content_hash=content_hash, + source=source, + source_plugin=source_plugin, + payload={"owner_user_id": owner_user_id, "from_db": from_db}, + enabled=enabled, + ) + comp = store.get(KIND_SKILL, LOCAL_PROFILE, ref.key, revision) + if comp is None: + tx = registry.begin_transaction(inst.install_id, inst.generation + 1) + try: + comp = store.write_from_files(KIND_SKILL, LOCAL_PROFILE, ref.key, revision, files) + published = True + registry.advance_transaction(tx, "published", inventory=store.inventory(comp)) + except Exception as exc: + registry.advance_transaction(tx, "failed", error=str(exc)) + registry.set_state(inst.install_id, "failed", last_error=str(exc)) + raise + registry.set_state(inst.install_id, "ready", resolved_revision=revision) + registry.advance_transaction(tx, "committed") + elif inst.resolved_revision != revision or inst.state != "ready": + registry.set_state(inst.install_id, "ready", resolved_revision=revision) + # A DB-backed skill is materialized again after restart/cache refresh. An + # identical publication must not invalidate every user's resolved closure. + if published or registry.generation() != before_generation: + skills.bump_view_generation() + return comp + + +def remove_local_skill(skill_id: str) -> bool: + from features.desktop.capabilities.runtime import references + + if references("skill", LOCAL_PROFILE, skill_id): + # Source deletion revokes use; history still needs its exact bytes. + return registry.mark_removed(registry.install_id("skill", LOCAL_PROFILE, skill_id)) + iid = registry.install_id(KIND_SKILL, LOCAL_PROFILE, skill_id) + removed_files = store.remove_key(KIND_SKILL, LOCAL_PROFILE, skill_id) > 0 + removed_row = registry.delete(iid) + registry.clear_preference(KIND_SKILL, skill_id) + skills.bump_view_generation() + return removed_files or removed_row + + +def prune_local(live_skill_owners: Dict[str, Optional[str]]) -> int: + """Drop local-profile entries whose database row no longer exists.""" + removed = 0 + for inst in registry.list_installations(kind=KIND_SKILL, profile_id=LOCAL_PROFILE): + if inst.source in (SOURCE_LOCAL, SOURCE_PLUGIN) and inst.payload.get("from_db"): + if inst.key not in live_skill_owners: + registry.mark_removed(inst.install_id) + removed += 1 + # Unindexed and superseded bytes may still be retained by history snapshots. + return removed diff --git a/src/backend/features/desktop/capabilities/skill_views.py b/src/backend/features/desktop/capabilities/skill_views.py new file mode 100644 index 00000000..96344992 --- /dev/null +++ b/src/backend/features/desktop/capabilities/skill_views.py @@ -0,0 +1,62 @@ +"""skill views responsibilities.""" +from __future__ import annotations +from features.desktop.capabilities import skills +from pathlib import Path +from typing import Dict, Optional +from features.desktop.capabilities import registry +from features.desktop.capabilities import view as view_mod +from core.capabilities.identity_rules import KIND_SKILL +from features.desktop.capabilities.resolver import Resolution, resolve +import logging +logger = logging.getLogger(__name__) +from features.desktop.capabilities import skill_candidates +from features.desktop.capabilities import skill_paths +from features.desktop.capabilities import skills + +def targets(res: Resolution) -> Dict[str, Path]: + return {name: c.path for name, c in res.chosen.items() if c.path is not None} + + +def rebuild_device_view() -> view_mod.ViewReport: + res = resolve( + KIND_SKILL, + skill_candidates.fill_hashes(skill_candidates.builtin_candidates() + skill_candidates.local_candidates(None, shared_only=True)), + preferences=registry.preferences(KIND_SKILL), + ) + report = view_mod.build_view( + skill_paths.device_view_dir(), targets(res), allowed_roots=skill_paths.allowed_link_roots() + ) + log_report("device", report) + return report + + +def rebuild_user_view(user_id: str) -> Optional[view_mod.ViewReport]: + vdir = skill_paths.user_view_dir(user_id) + if vdir is None: + return None + res = skills.resolve_for_user(user_id) + report = view_mod.build_view(vdir, targets(res), allowed_roots=skill_paths.allowed_link_roots()) + log_report(f"user:{user_id}", report) + return report + + +def rebuild_views(user_id: Optional[str]) -> Dict[str, view_mod.ViewReport]: + reports = {"device": rebuild_device_view()} + if user_id: + user_report = rebuild_user_view(user_id) + if user_report is not None: + reports["user"] = user_report + return reports + + +def log_report(label: str, report: view_mod.ViewReport) -> None: + if report.changed: + logger.info( + "[caps-view] %s linked=%d relinked=%d removed=%d", + label, + len(report.linked), + len(report.relinked), + len(report.removed), + ) + for name, why in report.blocked.items(): + logger.warning("[caps-view] %s name '%s' unavailable: %s", label, name, why) diff --git a/src/backend/features/desktop/capabilities/skills.py b/src/backend/features/desktop/capabilities/skills.py new file mode 100644 index 00000000..29a866cb --- /dev/null +++ b/src/backend/features/desktop/capabilities/skills.py @@ -0,0 +1,161 @@ +"""skills responsibilities.""" +from __future__ import annotations +from pathlib import Path +from typing import Any, Dict, List, Optional, Set +from features.desktop.capabilities import registry +from core.capabilities.identity_rules import BUILTIN_PROFILE, KIND_SKILL, LOCAL_PROFILE +from features.desktop.capabilities.resolver import SOURCE_BUILTIN, SOURCE_CLOUD, SOURCE_LOCAL, Candidate, Resolution, resolve +import logging +logger = logging.getLogger(__name__) +from features.desktop.capabilities import skill_account +from features.desktop.capabilities import skill_candidates +from features.desktop.capabilities import skills_state + +def view_generation() -> int: + """Bumped whenever store contents change; view consumers refresh on a new value.""" + return skills_state.view_generation + + +def bump_view_generation() -> int: + pass # State is owned by its responsibility module. + with skills_state.resolution_lock: + skills_state.view_generation += 1 + return skills_state.view_generation + + +def resolve_for_user(user_id: Optional[str], *, requested: Optional[Set[str]] = None) -> Resolution: + """Which candidate wins each runtime name, recomputed only when it can differ. + + The answer is a pure function of the installations, the store contents, the + user's name preferences and the bound cloud account — every one of which + moves ``registry.generation()``, ``view_generation()`` or the account + profile. Agent assembly asks for this several times per message and the + eligibility fixpoint walks every candidate's package each time, so without + this the same answer was recomputed from disk two or three times a turn. + """ + from features.desktop.capabilities.discovery import scan + from features.desktop.capabilities.readiness import eligible_skill_candidates + scan(user_id) + key = user_id or "" + signal = ( + registry.generation(), + view_generation(), + skill_account.current_account_profile(), + skill_candidates.builtin_signature(), + frozenset(requested) if requested is not None else None, + ) + with skills_state.resolution_lock: + hit = skills_state.resolution_cache.get(key) + if hit is not None and hit[0] == signal: + skills_state.last_resolution[key] = hit[1] + return hit[1] + + preferences = registry.preferences(KIND_SKILL, user_id=user_id) + eligible = eligible_skill_candidates( + skill_candidates.fill_hashes(skill_candidates.candidates(user_id)), user_id, preferences=preferences, requested=requested + ) + res = resolve(KIND_SKILL, eligible, preferences=preferences, requested=requested) + with skills_state.resolution_lock: + skills_state.last_resolution[key] = res + # A cloud deployment shares this process across users; keep the memo + # bounded rather than letting it grow with the user table. + if len(skills_state.resolution_cache) >= 64 and key not in skills_state.resolution_cache: + skills_state.resolution_cache.pop(next(iter(skills_state.resolution_cache)), None) + skills_state.resolution_cache[key] = (signal, res) + return res + + +def filter_available_names(names, *, user_id=None): + """Implicit defaults omit unusable names; explicit bindings still fail in preflight.""" + chosen = resolve_for_user(user_id).chosen + return [name for name in dict.fromkeys(names) if name in chosen] + + +def last_resolution(user_id: Optional[str]) -> Optional[Resolution]: + with skills_state.resolution_lock: + return skills_state.last_resolution.get(user_id or "") + + +def info_candidate(info: Any) -> Candidate: + """Map a backend ``SkillFileInfo`` onto a resolver candidate.""" + src = info.source_name + origin = dict(getattr(info, "origin", None) or {}) + if src == "built-in": + return Candidate( + install_id=registry.install_id(KIND_SKILL, BUILTIN_PROFILE, info.skill_id), + runtime_name=info.skill_id, + kind=KIND_SKILL, + profile=BUILTIN_PROFILE, + source=SOURCE_BUILTIN, + path=Path(info.file_path).parent, + account_level=False, + ) + if src in ("cloud", "device-store"): + return Candidate( + install_id=str(origin.get("install_id") or ""), + runtime_name=info.skill_id, + kind=KIND_SKILL, + profile=str(origin.get("profile") or ""), + source=SOURCE_CLOUD if src == "cloud" else SOURCE_LOCAL, + path=Path(info.file_path).parent, + content_hash=origin.get("content_hash"), + revision=origin.get("revision"), + account_level=True, + ) + if src == "admin": + # Device catalog row (created here, installed from a plugin or the market). + return Candidate( + install_id=registry.install_id(KIND_SKILL, LOCAL_PROFILE, info.skill_id), + runtime_name=info.skill_id, + kind=KIND_SKILL, + profile=LOCAL_PROFILE, + source=SOURCE_LOCAL, + path=Path(""), + account_level=True, + ) + return Candidate( + install_id=registry.install_id(KIND_SKILL, src, info.skill_id), + runtime_name=info.skill_id, + kind=KIND_SKILL, + profile=src, + source=SOURCE_LOCAL, + path=Path(info.file_path).parent, + account_level=False, + ) + + +def merge_skill_infos(groups: Dict[str, List[Any]], *, backend_hash) -> Dict[str, Any]: + """Composite-backend merge hook: one winner per id, conflicts load nothing. + + ``backend_hash(info) -> str`` computes a content hash for a backend entry; it + is only called for ids with more than one candidate. + """ + cands: List[Candidate] = [] + lookup: Dict[str, Any] = {} + for skill_id, infos in groups.items(): + for info in infos: + cand = info_candidate(info) + if len(infos) > 1 and cand.content_hash is None: + cand = Candidate(**{**cand.__dict__, "content_hash": backend_hash(info)}) + cands.append(cand) + lookup[cand.install_id] = info + res = resolve( + KIND_SKILL, + cands, + preferences=registry.preferences(KIND_SKILL, user_id=skill_account.current_local_user_id()), + ) + with skills_state.resolution_lock: + skills_state.last_resolution["__loader__"] = res + for name in res.conflicts: + logger.warning("[caps] skill name conflict, nothing loaded for '%s'", name) + return {name: lookup[c.install_id] for name, c in res.chosen.items()} + + +def loader_resolution() -> Optional[Resolution]: + with skills_state.resolution_lock: + return skills_state.last_resolution.get("__loader__") + + +def conflicted_names() -> Set[str]: + res = loader_resolution() + return set(res.conflicts) if res else set() diff --git a/src/backend/features/desktop/capabilities/skills_state.py b/src/backend/features/desktop/capabilities/skills_state.py new file mode 100644 index 00000000..05ee6099 --- /dev/null +++ b/src/backend/features/desktop/capabilities/skills_state.py @@ -0,0 +1,35 @@ +"""skills state responsibilities.""" +from __future__ import annotations +import threading +from collections import OrderedDict +from typing import Dict, Tuple +from features.desktop.capabilities.resolver import Resolution + +SKIP_PARTS = {"__pycache__", ".git", ".svn", ".hg", "__MACOSX"} + + +INVENTORY_NAME = ".inventory.json" + + +hash_lock = threading.Lock() + + +hash_cache: Dict[str, Tuple[Tuple, str]] = {} + + +content_hash_cache: OrderedDict[tuple, str] = OrderedDict() + + +resolution_lock = threading.Lock() + + +last_resolution: Dict[str, Resolution] = {} + + +resolution_cache: Dict[str, Tuple[Tuple, Resolution]] = {} + + +view_generation = 0 + + +shadow_user_ids: Dict[str, str] = {} diff --git a/src/backend/core/capabilities/store.py b/src/backend/features/desktop/capabilities/store.py similarity index 94% rename from src/backend/core/capabilities/store.py rename to src/backend/features/desktop/capabilities/store.py index e003fe9f..fc18d55e 100644 --- a/src/backend/core/capabilities/store.py +++ b/src/backend/features/desktop/capabilities/store.py @@ -16,19 +16,10 @@ from pathlib import Path from typing import Dict, Iterator, List, Optional -from . import archive -from .errors import IntegrityFailed -from .paths import ( - KIND_AGENT, - KIND_PLUGIN, - KIND_SKILL, - component_dir, - assert_managed_path, - kind_root, - profile_dir, - safe_segment, - staging_root, -) +from core.capabilities import archive as archive +from core.harness.capability_errors import IntegrityFailed +from core.capabilities.identity_rules import KIND_AGENT, KIND_PLUGIN, KIND_SKILL, safe_segment +from features.desktop.capabilities.paths import component_dir, assert_managed_path, kind_root, profile_dir, staging_root _ENTRY_FILE = {KIND_SKILL: "SKILL.md", KIND_PLUGIN: "plugin.json", KIND_AGENT: "agent.json"} _INVENTORY = ".inventory.json" diff --git a/src/backend/core/capabilities/view.py b/src/backend/features/desktop/capabilities/view.py similarity index 97% rename from src/backend/core/capabilities/view.py rename to src/backend/features/desktop/capabilities/view.py index ed572680..6a888f67 100644 --- a/src/backend/core/capabilities/view.py +++ b/src/backend/features/desktop/capabilities/view.py @@ -14,8 +14,8 @@ from pathlib import Path from typing import Dict, Iterable, List -from . import junction -from .errors import ViewUnavailable +from core.infra import directory_links as junction +from core.harness.capability_errors import ViewUnavailable @dataclass diff --git a/src/backend/core/capabilities/warmup.py b/src/backend/features/desktop/capabilities/warmup.py similarity index 72% rename from src/backend/core/capabilities/warmup.py rename to src/backend/features/desktop/capabilities/warmup.py index ec7092f3..51daf1c7 100644 --- a/src/backend/core/capabilities/warmup.py +++ b/src/backend/features/desktop/capabilities/warmup.py @@ -5,6 +5,8 @@ """ from __future__ import annotations +from features.desktop.capabilities import skill_account +from features.desktop.capabilities import skills import logging import os @@ -16,10 +18,11 @@ def warmup_current_account(state): - from . import skills, registry, store - from .dependency import component_hash, skill_definition - from .paths import capabilities_enabled, LOCAL_PROFILE, BUILTIN_PROFILE - from core.services.desktop_cloud_bridge import get_state, _state_fingerprint + from features.desktop.capabilities import skills, registry, store + from features.desktop.capabilities.dependency import component_hash, skill_definition + from core.capabilities.identity_rules import LOCAL_PROFILE, BUILTIN_PROFILE + from features.desktop.capabilities.paths import capabilities_enabled + from features.desktop.bridge.identity import get_state, state_fingerprint as _state_fingerprint if os.name != "nt" or not capabilities_enabled(): return @@ -31,9 +34,9 @@ def current(): if not current(): return - user_id = skills.current_local_user_id() - profile = skills.current_account_profile() - if not user_id or not profile or not skills.account_authorized_for(user_id): + user_id = skill_account.current_local_user_id() + profile = skill_account.current_account_profile() + if not user_id or not profile or not skill_account.account_authorized_for(user_id): return rows = [ row diff --git a/src/backend/features/desktop/capability/__init__.py b/src/backend/features/desktop/capability/__init__.py new file mode 100644 index 00000000..78a2d786 --- /dev/null +++ b/src/backend/features/desktop/capability/__init__.py @@ -0,0 +1 @@ +"""capability package.""" diff --git a/src/backend/core/services/desktop_capability_configs.py b/src/backend/features/desktop/capability/configs.py similarity index 91% rename from src/backend/core/services/desktop_capability_configs.py rename to src/backend/features/desktop/capability/configs.py index 2768c41f..2590d5aa 100644 --- a/src/backend/core/services/desktop_capability_configs.py +++ b/src/backend/features/desktop/capability/configs.py @@ -42,9 +42,9 @@ def _user_capability_configs( if hit and (now - hit[0]) < _EFFECTIVE_TTL_S: return list(hit[1]), list(hit[2]), dict(hit[3]) - from core.config.catalog_resolver import resolve_all_runtime_enabled - from core.llm.factory.tools.mcp_config import _effective_mcp_server_keys - from core.services.mcp_service import McpServerConfigService + from core.capabilities.catalog.resolver import resolve_all_runtime_enabled + from core.llm.factory.tools.mcp_config import effective_mcp_server_keys as _effective_mcp_server_keys + from core.capabilities.mcp.service import McpServerConfigService svc = McpServerConfigService.get_instance() owned = svc.get_owned_servers(uid, enabled_only=False, strict=not use_cache) diff --git a/src/backend/core/services/desktop_capability_credentials.py b/src/backend/features/desktop/capability/credentials.py similarity index 98% rename from src/backend/core/services/desktop_capability_credentials.py rename to src/backend/features/desktop/capability/credentials.py index 961f4744..eb44f488 100644 --- a/src/backend/core/services/desktop_capability_credentials.py +++ b/src/backend/features/desktop/capability/credentials.py @@ -6,7 +6,7 @@ from typing import Any, Dict from urllib.parse import unquote, unquote_plus, urlsplit -from core.llm.reasoning_effort import EFFORT_KEYS +from core.models.reasoning_effort import EFFORT_KEYS class CapabilityContentRejected(ValueError): diff --git a/src/backend/core/services/desktop_capability_entities.py b/src/backend/features/desktop/capability/entities.py similarity index 91% rename from src/backend/core/services/desktop_capability_entities.py rename to src/backend/features/desktop/capability/entities.py index 2c7f7289..12f4efea 100644 --- a/src/backend/core/services/desktop_capability_entities.py +++ b/src/backend/features/desktop/capability/entities.py @@ -8,8 +8,8 @@ from typing import Any, Dict, List, Optional, Tuple from core.db.engine import SessionLocal -from core.services.desktop_capability_configs import _EFFECTIVE_TTL_S, _effective_lock -from core.services.desktop_capability_credentials import CapabilityContentRejected +from features.desktop.capability.configs import _EFFECTIVE_TTL_S, _effective_lock +from features.desktop.capability.credentials import CapabilityContentRejected # ── 智能体 / 插件清单与定义包（云端侧） ────────────────────────────────── # @@ -184,7 +184,7 @@ def _agent_files(serialized: Dict[str, Any]) -> Dict[str, str]: def _plugin_files(installed: Dict[str, Any]) -> Dict[str, str]: - from core.plugins.connections import connection_type + from core.capabilities.plugins.connections import connection_type uploaded = installed.get("_uploaded_files") definition = { @@ -206,7 +206,7 @@ def _plugin_files(installed: Dict[str, Any]) -> Dict[str, str]: definition.update(_public_declarations(installed)) if uploaded is not None: definition = json.loads(uploaded["plugin.json"]) - from core.services.desktop_resource_plugins import package_files + from features.desktop.bridge.resource_plugins import package_files assets, native = package_files(installed) if uploaded is not None and not native: return dict(uploaded) @@ -220,7 +220,7 @@ def _plugin_files(installed: Dict[str, Any]) -> Dict[str, str]: def _user_agents(user_id: str) -> List[Dict[str, Any]]: - from core.services.user_agent_service import UserAgentService + from core.capabilities.agents.edition import UserAgentService with SessionLocal() as db: return list(UserAgentService(db).list_for_user(user_id) or []) @@ -228,10 +228,10 @@ def _user_agents(user_id: str) -> List[Dict[str, Any]]: def _user_plugins(user_id: str) -> List[Dict[str, Any]]: from core.db.models import InstalledPlugin - from core.plugins import management as plugin_service + from core.capabilities.plugins import management as plugin_service with SessionLocal() as db: - from core.services.capability_workcopies import active_plugin_files + from features.desktop.capability.workcopies import active_plugin_files rows = plugin_service.list_installed(db, user_id, include_global=True) metadata = { @@ -244,7 +244,7 @@ def _user_plugins(user_id: str) -> List[Dict[str, Any]]: InstalledPlugin.install_id.in_([r["install_id"] for r in rows] or [""]) ) } - from core.plugins.management.admin_config import _connection_for_slug + from core.capabilities.plugins.management.admin_config import connection_for_slug as _connection_for_slug for r in rows: r.update(metadata.get(r["install_id"], {})) @@ -253,7 +253,7 @@ def _user_plugins(user_id: str) -> List[Dict[str, Any]]: def build_user_agent_manifest(user_id: str, *, use_cache: bool = True) -> Dict[str, Any]: - from core.services.desktop_capability_protocol import build_entity_manifest, entity_content_hash + from core.capabilities.manifest_protocol import build_entity_manifest, entity_content_hash uid = str(user_id) now = time.monotonic() @@ -280,7 +280,7 @@ def build_user_agent_manifest(user_id: str, *, use_cache: bool = True) -> Dict[s def resolve_agent_bundle(user_id: str, agent_id: str) -> Optional[Tuple[bytes, str]]: - from core.services.desktop_capability_protocol import entity_content_hash + from core.capabilities.manifest_protocol import entity_content_hash for a in _user_agents(str(user_id)): if a["agent_id"] == agent_id: @@ -290,7 +290,7 @@ def resolve_agent_bundle(user_id: str, agent_id: str) -> Optional[Tuple[bytes, s def build_user_plugin_manifest(user_id: str, *, use_cache: bool = True) -> Dict[str, Any]: - from core.services.desktop_capability_protocol import build_entity_manifest, entity_content_hash + from core.capabilities.manifest_protocol import build_entity_manifest, entity_content_hash uid = str(user_id) now = time.monotonic() @@ -321,7 +321,7 @@ def build_user_plugin_manifest(user_id: str, *, use_cache: bool = True) -> Dict[ def resolve_plugin_bundle(user_id: str, install_id: str) -> Optional[Tuple[bytes, str]]: - from core.services.desktop_capability_protocol import entity_content_hash + from core.capabilities.manifest_protocol import entity_content_hash for p in _user_plugins(str(user_id)): if p["install_id"] == install_id: diff --git a/src/backend/core/services/desktop_capability_mcp.py b/src/backend/features/desktop/capability/mcp.py similarity index 85% rename from src/backend/core/services/desktop_capability_mcp.py rename to src/backend/features/desktop/capability/mcp.py index 4111f551..27ba9cbc 100644 --- a/src/backend/core/services/desktop_capability_mcp.py +++ b/src/backend/features/desktop/capability/mcp.py @@ -7,15 +7,10 @@ from core.db.engine import SessionLocal from core.db.models import AdminMcpServer -from core.services.desktop_capability_configs import _user_capability_configs -from core.services.desktop_capability_credentials import _secrets_from_config -from core.services.desktop_capability_protocol import ( - CapabilityManifestStaleError, - build_manifest, - canonical_hash, - public_tool_schemas, -) -from core.services.desktop_capability_security import guard_capability_content +from features.desktop.capability.configs import _user_capability_configs +from features.desktop.capability.credentials import _secrets_from_config +from core.capabilities.manifest_protocol import CapabilityManifestStaleError, build_manifest, canonical_hash, public_tool_schemas +from features.desktop.capability.security import guard_capability_content def build_user_capability_manifest(user_id: str, *, use_cache: bool = True) -> Dict[str, Any]: @@ -26,7 +21,7 @@ def build_user_capability_manifest(user_id: str, *, use_cache: bool = True) -> D MCP streamable-http 协议透明反代；stdio / sse 传输的（本就极少）不进桌面清单。 凭据（URL 内嵌密钥、headers、OAuth）一律留在云端连接层，manifest 不携带任何密钥。 """ - from core.config.catalog_runtime import _DEFAULT_MCP_ICONS + from core.capabilities.catalog.runtime import DEFAULT_MCP_ICONS as _DEFAULT_MCP_ICONS keys, enabled_keys, all_cfgs = _user_capability_configs(user_id, use_cache=use_cache) enabled = set(enabled_keys) @@ -152,16 +147,17 @@ async def invoke_gateway_tool( embedded.interface().execute, resolved["user_id"], resolved["tool"]["name"], arguments, ) import mcp.types - from core.llm.mcp_pool import make_client + from core.llm.mcp.client import make_client arguments = dict(arguments or {}) - from core.services.automation_remote_effect import RECEIPT_TOOLS, bind_remote_effect + from core.harness.registry.tool_operations import receipt_operations tool_name = str(resolved["tool"]["name"]) - if tool_name in RECEIPT_TOOLS and arguments.get("tool_effect_id"): + operations = receipt_operations(tool_name) + if operations is not None and arguments.get("tool_effect_id"): operation_id = arguments.pop("tool_effect_id") arguments["tool_effect_id"] = await asyncio.to_thread( - bind_remote_effect, + operations.bind, str(resolved["user_id"]), str(resolved["server_id"]), tool_name, @@ -184,7 +180,7 @@ async def invoke_gateway_tool( upstream_headers["accept-encoding"] = "identity" # The authenticated cloud gateway is a new invocation hop. Never forward # a device-issued proof to an internal MCP that trusts the cloud key. - from core.llm.mcp_invocation import HEADER, PLUGIN_HEADER, for_url + from core.llm.mcp.invocation import HEADER, PLUGIN_HEADER, for_url upstream_headers.pop(HEADER.lower(), None) upstream_headers.pop(PLUGIN_HEADER.lower(), None) @@ -193,15 +189,13 @@ async def invoke_gateway_tool( )) target["headers"] = upstream_headers - client = make_client(str(resolved["server_id"]), target, is_stateful=False) + from core.llm.mcp.results import tool_result + from core.llm.mcp.client import _execution_timeout raw_tool = mcp.types.Tool.model_validate(resolved["tool"]) - # Skip a second tools/list call in the cloud: the allowlisted schema was read - # from the same DB row immediately above. get_tool then performs only the - # real initialize + tools/call lifecycle against the private MCP target. - client._cached_tools = [raw_tool] # noqa: SLF001 - AgentScope has no public preload API - tool = await client.get_tool(raw_tool.name) - timeout = max(1.0, float(client.execution_timeout or 120.0)) + 10.0 - chunk = await asyncio.wait_for(tool(**dict(arguments or {})), timeout=timeout) + async with make_client(str(resolved["server_id"]), target) as client: + result = await asyncio.wait_for(client.call_tool(raw_tool.name, dict(arguments or {})), + timeout=_execution_timeout(str(resolved["server_id"]), target)) + chunk = tool_result(result, raw_tool) chunk.metadata.setdefault("origin", "cloud") chunk.metadata.setdefault("mcp_server_id", str(resolved["server_id"])) return guard_capability_content( diff --git a/src/backend/core/services/desktop_capability_models.py b/src/backend/features/desktop/capability/models.py similarity index 93% rename from src/backend/core/services/desktop_capability_models.py rename to src/backend/features/desktop/capability/models.py index 820e786d..d5083f7c 100644 --- a/src/backend/core/services/desktop_capability_models.py +++ b/src/backend/features/desktop/capability/models.py @@ -7,9 +7,9 @@ from core.db.engine import SessionLocal from core.db.models import ModelProvider, ModelRoleAssignment -from core.services.desktop_capability_credentials import CapabilityContentRejected -from core.services.desktop_capability_protocol import canonical_hash -from core.services.desktop_capability_security import _guard_value, _known_cloud_secrets +from features.desktop.capability.credentials import CapabilityContentRejected +from core.capabilities.manifest_protocol import canonical_hash +from features.desktop.capability.security import _guard_value, _known_cloud_secrets logger = logging.getLogger(__name__) @@ -33,9 +33,9 @@ def _upstream_model_path(provider: ModelProvider) -> Optional[str]: provider_type = str(provider.provider_type or "") if provider_type != "chat": return _MODEL_PATHS.get(provider_type) - from core.llm.chat_models import wants_responses - from core.llm.providers.protocol_probe import PROTOCOL_RESPONSES - from core.llm.providers.registry import get_spec + from core.models.chat_models import wants_responses + from core.models.providers.protocol_probe import PROTOCOL_RESPONSES + from core.models.providers.registry import get_spec spec = get_spec(getattr(provider, "provider", None) or "openai_compatible") protocol = (provider.extra_config or {}).get("api_protocol") @@ -62,7 +62,7 @@ def _model_is_gateway_compatible(provider: ModelProvider) -> bool: Gemini / Bedrock 也不是同一线上协议；在专用适配器完成前不把它们 伪装成可用，更不会为了兼容而下发真实凭据。 """ - from core.llm.providers.registry import get_spec + from core.models.providers.registry import get_spec provider_id = getattr(provider, "provider", None) or "openai_compatible" spec = get_spec(provider_id) @@ -160,7 +160,7 @@ def _model_provider_allowed(db, user_id: str, provider: ModelProvider) -> bool: return True if provider.provider_type != "chat": return False - from core.services.user_model_selection import user_can_switch_model + from core.auth.model_selection import user_can_switch_model return user_can_switch_model(db, str(user_id)) diff --git a/src/backend/features/desktop/capability/revision_provider.py b/src/backend/features/desktop/capability/revision_provider.py new file mode 100644 index 00000000..090ae82d --- /dev/null +++ b/src/backend/features/desktop/capability/revision_provider.py @@ -0,0 +1,22 @@ +"""Expose existing account revision authorization and compare-and-swap semantics.""" +from features.desktop.capability import workcopies + + +class CloudCapabilityRevisions: + def model(self, db, kind, key): + return workcopies.get_model(db, kind, key) + + def snapshot_in_session(self, db, user, kind, key): + return workcopies.snapshot_in_session(db, user, kind, key) + + def snapshot(self, user, kind, key): + return workcopies.snapshot(user, kind, key) + + def commit(self, user, kind, key, body): + return workcopies.commit(user, kind, key, body) + + def revision(self, files): + return workcopies.revision(files) + + def record_id(self, prefix, user, kind, key): + return workcopies.record_id(prefix, user, kind, key) diff --git a/src/backend/core/services/desktop_capability_security.py b/src/backend/features/desktop/capability/security.py similarity index 96% rename from src/backend/core/services/desktop_capability_security.py rename to src/backend/features/desktop/capability/security.py index bee6e847..264065b1 100644 --- a/src/backend/core/services/desktop_capability_security.py +++ b/src/backend/features/desktop/capability/security.py @@ -7,11 +7,8 @@ from core.db.engine import SessionLocal from core.db.models import ModelProvider -from core.services.desktop_capability_configs import _effective_lock, _user_capability_configs -from core.services.desktop_capability_credentials import ( - CapabilityContentRejected, - _secrets_from_config, -) +from features.desktop.capability.configs import _effective_lock, _user_capability_configs +from features.desktop.capability.credentials import CapabilityContentRejected, _secrets_from_config def _without_public_identifiers(secrets: set[str], public_identifiers: set[str]) -> set[str]: @@ -27,7 +24,7 @@ def _without_public_identifiers(secrets: set[str], public_identifiers: set[str]) def _model_credentials_and_public_identifiers(*, fresh: bool = True) -> Tuple[set[str], set[str]]: global _model_secret_cache - from core.services.model_config import ModelConfigService + from core.models.config import ModelConfigService version = ModelConfigService.get_instance().version if not fresh: diff --git a/src/backend/core/services/desktop_capability.py b/src/backend/features/desktop/capability/service.py similarity index 90% rename from src/backend/core/services/desktop_capability.py rename to src/backend/features/desktop/capability/service.py index 9f4d7c32..99eb1cf2 100644 --- a/src/backend/core/services/desktop_capability.py +++ b/src/backend/features/desktop/capability/service.py @@ -35,42 +35,15 @@ from core.db.engine import SessionLocal from core.db.models import ContentBlock -from core.services.desktop_capability_credentials import CapabilityContentRejected -from core.services.desktop_capability_entities import ( - build_user_agent_manifest, - build_user_plugin_manifest, - resolve_agent_bundle, - resolve_plugin_bundle, -) +from features.desktop.capability.credentials import CapabilityContentRejected +from features.desktop.capability.entities import build_user_agent_manifest, build_user_plugin_manifest, resolve_agent_bundle, resolve_plugin_bundle # Public entry points remain stable for route consumers; implementations own their state. -from core.services.desktop_capability_mcp import ( - build_user_capability_manifest, - invoke_gateway_tool, - resolve_gateway_target, - resolve_gateway_tool, -) -from core.services.desktop_capability_models import ( - build_user_model_manifest, - resolve_model_gateway_target, -) -from core.services.desktop_capability_protocol import ( - CapabilityManifestStaleError, - build_manifest, - build_skill_manifest, - canonical_hash, - public_tool_schema, - public_tool_schemas, - skill_content_hash, -) -from core.services.desktop_capability_security import ( - gateway_stream_secrets, - guard_capability_bundle, - guard_capability_content, - guard_capability_stream, - invalidate_model_gateway_cache, -) -from core.services.desktop_capability_skills import build_user_skill_manifest, resolve_skill_bundle +from features.desktop.capability.mcp import build_user_capability_manifest, invoke_gateway_tool, resolve_gateway_target, resolve_gateway_tool +from features.desktop.capability.models import build_user_model_manifest, resolve_model_gateway_target +from core.capabilities.manifest_protocol import CapabilityManifestStaleError, build_manifest, build_skill_manifest, canonical_hash, public_tool_schema, public_tool_schemas, skill_content_hash +from features.desktop.capability.security import gateway_stream_secrets, guard_capability_bundle, guard_capability_content, guard_capability_stream, invalidate_model_gateway_cache +from features.desktop.capability.skills import build_user_skill_manifest, resolve_skill_bundle logger = logging.getLogger(__name__) @@ -137,7 +110,7 @@ def _sign(data: bytes) -> str: def is_desktop_shell_control(authorization: str, origin: Optional[str]) -> bool: """Only the shell process secret authorizes local capability management.""" - from core.auth.desktop_bridge import BRIDGE_SECRET_ENV + from features.desktop.auth.bridge import BRIDGE_SECRET_ENV secret = os.getenv(BRIDGE_SECRET_ENV, "").strip() if not secret or origin is not None: diff --git a/src/backend/core/services/desktop_capability_skills.py b/src/backend/features/desktop/capability/skills.py similarity index 86% rename from src/backend/core/services/desktop_capability_skills.py rename to src/backend/features/desktop/capability/skills.py index 68075f5f..45079045 100644 --- a/src/backend/core/services/desktop_capability_skills.py +++ b/src/backend/features/desktop/capability/skills.py @@ -8,8 +8,8 @@ from typing import Any, Dict, List, Optional, Tuple from core.db.engine import SessionLocal -from core.services.desktop_capability_configs import _EFFECTIVE_TTL_S, _effective_lock -from core.services.desktop_capability_protocol import build_skill_manifest, skill_content_hash +from features.desktop.capability.configs import _EFFECTIVE_TTL_S, _effective_lock +from core.capabilities.manifest_protocol import build_skill_manifest, skill_content_hash # ── 技能清单 / 技能包（云端为真源，本机只缓存文件快照） ─────────────────── @@ -18,7 +18,7 @@ def _skill_info(skill_id: str): - from core.agent_skills.loader import get_skill_loader + from core.capabilities.skills.loader_factory import get_skill_loader return get_skill_loader()._backend.get_skill_info(skill_id) @@ -30,8 +30,8 @@ def _skill_snapshot( ``info`` 给已经取过的调用方复用，省一次后端查询。 """ - from core.agent_skills.binary_files import pack_directory - from core.agent_skills.loader import get_skill_loader + from core.capabilities.skills.binary_files import pack_directory + from core.capabilities.skills.loader_factory import get_skill_loader loader = get_skill_loader() info = _skill_info(skill_id) if info is None else info @@ -67,9 +67,9 @@ def build_user_skill_manifest(user_id: str, *, use_cache: bool = True) -> Dict[s if hit and (now - hit[0]) < _EFFECTIVE_TTL_S: return copy.deepcopy(hit[1]) - from core.agent_skills.loader import get_skill_loader - from core.config.catalog_resolver import resolve_all_runtime_enabled - from core.llm.factory.selection.capabilities import _filter_skill_ids_for_user + from core.capabilities.skills.loader_factory import get_skill_loader + from core.capabilities.catalog.resolver import resolve_all_runtime_enabled + from core.llm.factory.selection.capabilities import filter_skill_ids_for_user as _filter_skill_ids_for_user with SessionLocal() as db: enabled, _agents, _mcps = resolve_all_runtime_enabled(db, uid) @@ -111,7 +111,7 @@ def build_user_skill_manifest(user_id: str, *, use_cache: bool = True) -> Dict[s def resolve_skill_bundle(user_id: str, skill_id: str) -> Optional[Tuple[bytes, str]]: """打包一个当前授权技能为 zip，返回 (bytes, content_hash)；未授权返回 None。""" - from core.services.marketplace_service import build_skill_zip, build_skill_zip_from_dir + from features.marketplace.skill_archives import build_skill_zip, build_skill_zip_from_dir manifest = build_user_skill_manifest(user_id, use_cache=False) if not any(s["skill_id"] == skill_id for s in manifest["skills"]): diff --git a/src/backend/core/services/desktop_capability_sync_check.py b/src/backend/features/desktop/capability/sync_check.py similarity index 79% rename from src/backend/core/services/desktop_capability_sync_check.py rename to src/backend/features/desktop/capability/sync_check.py index e3fd4396..86fc3926 100644 --- a/src/backend/core/services/desktop_capability_sync_check.py +++ b/src/backend/features/desktop/capability/sync_check.py @@ -5,6 +5,8 @@ """ from __future__ import annotations +from features.desktop.bridge import client as bridge_client +from features.desktop.bridge import identity as bridge_identity import threading import time @@ -12,15 +14,10 @@ import httpx -from core.services import desktop_cloud_bridge as bridge -from core.services import desktop_cloud_bundles as bundles -from core.services import desktop_cloud_skills as skills -from core.services.desktop_capability_protocol import ( - canonical_hash, - validate_manifest, - validate_skill_manifest, - validate_entity_manifest, -) +from features.desktop.bridge import client as bridge +from features.desktop.bridge import bundles as bundles +from features.desktop.bridge import skills as skills +from core.capabilities.manifest_protocol import canonical_hash, validate_manifest, validate_skill_manifest, validate_entity_manifest _lock = threading.Lock() _cache_key = "" @@ -31,7 +28,7 @@ def _snapshots(): return { - "mcp": bridge.synced_manifest(), + "mcp": bridge_client.synced_manifest(), "skill": skills.synced_manifest(), **bundles.synced_manifests(), } @@ -57,13 +54,13 @@ def _fetch(st, kind, known): response = httpx.get( f"{st['cloud_base']}/api/v1/desktop/capability/{path}", headers={ - **bridge.cloud_headers(st), + **bridge_identity.cloud_headers(st), "Cache-Control": "no-cache", "If-None-Match": f'"{known["revision"]}"', }, timeout=httpx.Timeout(10.0, connect=5.0), ) - bridge.require_current_account(st) + bridge_identity.require_current_account(st) if response.status_code == 304: return known response.raise_for_status() @@ -79,11 +76,11 @@ def check(st: dict[str, Any]) -> dict[str, Any]: """Return changed=None when comparison is unavailable, never a false update.""" global _cache_key, _cache_until, _cache_result with _lock: - with bridge.account_scope(st): + with bridge_identity.account_scope(st): local = _snapshots() if any(snapshot is None for snapshot in local.values()): return {"changed": None} - key = canonical_hash([bridge._state_fingerprint(st), local]) + key = canonical_hash([bridge_identity.state_fingerprint(st), local]) if key == _cache_key and time.monotonic() < _cache_until: return dict(_cache_result) try: @@ -95,14 +92,14 @@ def check(st: dict[str, Any]) -> dict[str, Any]: ) # Automatic recovery or another window may have synchronized while # the comparison was in flight. Do not publish against an old base. - with bridge.account_scope(st): + with bridge_identity.account_scope(st): if local != _snapshots(): return {"changed": None} result = {"changed": changed} except Exception: # Network/protocol failures are unknown, not an update. No credentials # or raw upstream error bodies are exposed by this advisory endpoint. - bridge.require_current_account(st) + bridge_identity.require_current_account(st) result = {"changed": None} _cache_key, _cache_result = key, result _cache_until = time.monotonic() + CHECK_INTERVAL_SECONDS diff --git a/src/backend/core/services/capability_workcopies.py b/src/backend/features/desktop/capability/workcopies.py similarity index 89% rename from src/backend/core/services/capability_workcopies.py rename to src/backend/features/desktop/capability/workcopies.py index 7f3c60dc..ce621fea 100644 --- a/src/backend/core/services/capability_workcopies.py +++ b/src/backend/features/desktop/capability/workcopies.py @@ -6,9 +6,9 @@ import threading from datetime import datetime, timezone -from core.capabilities.change_merge import revision, sensitive_paths, validate_files -from core.capabilities.errors import CapabilityError -from core.capabilities.paths import safe_segment +from features.desktop.capabilities.change_merge import revision, sensitive_paths, validate_files +from core.harness.capability_errors import CapabilityError +from core.capabilities.identity_rules import safe_segment from core.db.engine import SessionLocal from core.db.models import AdminMcpServer, AdminSkill, ContentBlock, InstalledPlugin from core.infra.crypto import decrypt_secret, encrypt_secret @@ -19,11 +19,11 @@ _lock = threading.RLock() -def _id(prefix, user, kind, key): +def record_id(prefix, user, kind, key): return prefix + hashlib.sha256((str(user) + "\0" + kind + "\0" + key).encode()).hexdigest() -def _model(db, kind, key): +def get_model(db, kind, key): models = { "skill": (AdminSkill, AdminSkill.skill_id), "plugin": (InstalledPlugin, InstalledPlugin.install_id), @@ -48,7 +48,7 @@ def _files(row, kind): data["components"] = row.component_ids or {} data["ui_contributions"] = row.ui_contributions data["import_report"] = row.import_report or {} - from core.services.desktop_capability_entities import _plugin_files + from features.desktop.capability.entities import _plugin_files return _plugin_files(data) # Existing cloud credentials are never exported to a device. @@ -57,7 +57,7 @@ def _files(row, kind): def _stored(db, user, kind, key): - record = db.get(ContentBlock, _id("cap-cloud-copy:", user, kind, key)) + record = db.get(ContentBlock, record_id("cap-cloud-copy:", user, kind, key)) if record is None: return None text = decrypt_secret(record.payload.get("encrypted")) @@ -66,12 +66,12 @@ def _stored(db, user, kind, key): return json.loads(text) -def _snapshot(db, user, kind, key): - row = _model(db, kind, key) +def snapshot_in_session(db, user, kind, key): + row = get_model(db, kind, key) if row is not None and row.owner_user_id not in (None, str(user)): raise HTTPException(404, detail="capability not available") if row is not None and row.owner_user_id is None: - from core.services import desktop_capability as capabilities + from features.desktop.capability import service as capabilities if kind == "skill": allowed = { @@ -121,7 +121,7 @@ def _snapshot(db, user, kind, key): def snapshot(user, kind, key): with SessionLocal() as db: - return _snapshot(db, str(user), kind, key)[0] + return snapshot_in_session(db, str(user), kind, key)[0] def _activate(db, user, kind, key, files, row): @@ -135,10 +135,10 @@ def _activate(db, user, kind, key, files, row): } _require_flag(str(user), db, *flags[kind]) if kind == "skill": - from core.agent_skills.registry import _load_skill_metadata_from_str + from core.capabilities.skills.registry import load_skill_metadata_from_str content = files.get("SKILL.md", "") - metadata = _load_skill_metadata_from_str(content, key) + metadata = load_skill_metadata_from_str(content, key) from api.routes.v1.me_capabilities import extract_mcp_server_ids, resolve_mcp_bindings from core.ontology.build_validator import ensure_ontology_build_valid @@ -168,8 +168,8 @@ def _activate(db, user, kind, key, files, row): metadata.allowed_tools, ) row.extra_files = {name: value for name, value in files.items() if name != "SKILL.md"} - from core.agent_skills.binary_files import is_binary_value - from core.agent_skills.deps_detector import detect_dependencies + from core.capabilities.skills.binary_files import is_binary_value + from core.capabilities.skills.deps.detector import detect_dependencies row.dependencies = detect_dependencies( {name: value for name, value in row.extra_files.items() if not is_binary_value(value)} @@ -208,7 +208,7 @@ def _activate(db, user, kind, key, files, row): import asyncio from core.config.settings import settings - from core.services.mcp_management_service import encrypt_mcp_headers, validate_remote_mcp_url + from core.capabilities.mcp.management import encrypt_mcp_headers, validate_remote_mcp_url asyncio.run( validate_remote_mcp_url( @@ -223,7 +223,6 @@ def _activate(db, user, kind, key, files, row): owner_user_id=str(user), created_by=str(user), display_name=spec.get("display_name") or key, - is_stable=False, ) db.add(row) for field in ("display_name", "description", "transport", "url"): @@ -244,7 +243,7 @@ def commit(user, kind, key, body): if sensitive_paths(files) and not body.get("acknowledge_sensitive"): raise HTTPException(409, detail="sensitive upload needs confirmation") request_id = body["request_id"] - receipt_key = _id("cap-cloud-receipt:", user, kind, request_id) + receipt_key = record_id("cap-cloud-receipt:", user, kind, request_id) request_hash = revision({"request.json": json.dumps({"key": key, **body}, sort_keys=True)}) with _lock, SessionLocal() as db: try: @@ -263,7 +262,7 @@ def commit(user, kind, key, body): 409, detail="request identity reused with different content" ) return json.loads(decrypt_secret(receipt.payload["encrypted"])) - before, row = _snapshot(db, user, kind, key) + before, row = snapshot_in_session(db, user, kind, key) if not before["can_edit"]: raise HTTPException(403, detail="create a private copy instead") if body.get("create_only"): @@ -294,7 +293,7 @@ def commit(user, kind, key, body): if applied else previous_active ) - storage_key = _id("cap-cloud-copy:", user, kind, key) + storage_key = record_id("cap-cloud-copy:", user, kind, key) record = db.get(ContentBlock, storage_key) if record is None: record = ContentBlock(id=storage_key) @@ -304,7 +303,7 @@ def commit(user, kind, key, body): "updated_at": datetime.now(timezone.utc).isoformat(), } for version in (before["files"], files): - history_key = _id("cap-cloud-version:", user, kind, key + ":" + revision(version)) + history_key = record_id("cap-cloud-version:", user, kind, key + ":" + revision(version)) if db.get(ContentBlock, history_key) is None: db.add( ContentBlock( @@ -332,8 +331,8 @@ def commit(user, kind, key, body): except (ValueError, KeyError, TypeError, CapabilityError) as exc: db.rollback() raise HTTPException(422, detail="invalid capability files") from exc - from core.agent_skills.cache_refresh import refresh_skill_caches - from core.services.mcp_management_service import refresh_mcp_caches + from core.capabilities.skills.cache_refresh import refresh_skill_caches + from core.capabilities.mcp.management import refresh_mcp_caches if applied: try: diff --git a/src/backend/features/desktop/edition.py b/src/backend/features/desktop/edition.py new file mode 100644 index 00000000..8a438e48 --- /dev/null +++ b/src/backend/features/desktop/edition.py @@ -0,0 +1,23 @@ +"""Community desktop approval controls.""" +from typing import Literal +from pydantic import BaseModel +from fastapi import Depends +from core.auth.backend import UserContext, get_current_user +from core.infra.responses import success_response +from features.desktop.local import grants + +def register_local_routes(router, require_local): + class ApprovalBody(BaseModel): + mode: Literal["strict", "standard", "full"] + + + @router.get("/approval-mode", summary="获取当前审批档（对话框权限栏）") + async def get_approval_mode(_user: UserContext = Depends(get_current_user)): + require_local() + return success_response(data={"mode": grants.get_approval_mode()}) + + + @router.put("/approval-mode", summary="设置审批档") + async def set_approval_mode(body: ApprovalBody, _user: UserContext = Depends(get_current_user)): + require_local() + return success_response(data={"mode": grants.set_approval_mode(body.mode)}, message="已切换权限档") diff --git a/src/backend/features/desktop/extension.py b/src/backend/features/desktop/extension.py new file mode 100644 index 00000000..9c18f6af --- /dev/null +++ b/src/backend/features/desktop/extension.py @@ -0,0 +1,4 @@ +"""Desktop capabilities are installed in cloud and local hosts.""" +NAME = "desktop" +def setup(): + return True diff --git a/src/backend/features/desktop/gateway/__init__.py b/src/backend/features/desktop/gateway/__init__.py new file mode 100644 index 00000000..bbfee548 --- /dev/null +++ b/src/backend/features/desktop/gateway/__init__.py @@ -0,0 +1 @@ +"""gateway package.""" diff --git a/src/backend/core/services/desktop_gateway_observability.py b/src/backend/features/desktop/gateway/observability.py similarity index 98% rename from src/backend/core/services/desktop_gateway_observability.py rename to src/backend/features/desktop/gateway/observability.py index 2480f9de..ee139282 100644 --- a/src/backend/core/services/desktop_gateway_observability.py +++ b/src/backend/features/desktop/gateway/observability.py @@ -10,8 +10,8 @@ from concurrent.futures import ThreadPoolExecutor from core.db.engine import SessionLocal -from core.db.models.observability import utcnow -from core.services.desktop_observability import ingest +from features.desktop.models import utcnow +from features.desktop.observability.projection import ingest from fastapi.routing import APIRoute logger = logging.getLogger(__name__) diff --git a/src/backend/features/desktop/local/__init__.py b/src/backend/features/desktop/local/__init__.py new file mode 100644 index 00000000..11bccc04 --- /dev/null +++ b/src/backend/features/desktop/local/__init__.py @@ -0,0 +1 @@ +"""local package.""" diff --git a/src/backend/core/services/local_file_actions.py b/src/backend/features/desktop/local/file_actions.py similarity index 99% rename from src/backend/core/services/local_file_actions.py rename to src/backend/features/desktop/local/file_actions.py index 129fac6c..6166bcfa 100644 --- a/src/backend/core/services/local_file_actions.py +++ b/src/backend/features/desktop/local/file_actions.py @@ -10,7 +10,7 @@ import sys from core.infra.proc import no_window_kwargs -from core.services.native_app_icons import desktop_icon, windows_icon +from features.desktop.local.native_app_icons import desktop_icon, windows_icon def _run(arguments: list[str], *, input_text: str | None = None, timeout: int = 10) -> str: diff --git a/src/backend/core/services/local_grant_service.py b/src/backend/features/desktop/local/grants.py similarity index 100% rename from src/backend/core/services/local_grant_service.py rename to src/backend/features/desktop/local/grants.py diff --git a/src/backend/core/services/local_management_migration.py b/src/backend/features/desktop/local/management_migration.py similarity index 85% rename from src/backend/core/services/local_management_migration.py rename to src/backend/features/desktop/local/management_migration.py index 4f8d506b..12a27de0 100644 --- a/src/backend/core/services/local_management_migration.py +++ b/src/backend/features/desktop/local/management_migration.py @@ -4,17 +4,19 @@ remain available; a removed installation is never reactivated by a legacy row. Plugin-owned rows remain owned by their legacy plugin until that package migrates. """ +from sqlalchemy import inspect as inspect_model import hashlib import tempfile from pathlib import Path from sqlalchemy import inspect -from core.capabilities import archive, registry -from core.capabilities.paths import capabilities_enabled -from core.capabilities.ref import local_ref +from core.capabilities import archive as archive +from features.desktop.capabilities import registry +from features.desktop.capabilities.paths import capabilities_enabled +from core.capabilities.reference import local_ref from core.db.models import AdminSkill, ContentBlock -from core.services.management_package import serialized -from core.services import local_skill_service as lifecycle -from core.agent_skills.binary_files import decode_binary, is_binary_value +from core.capabilities.plugins.management.package import serialized +from features.desktop.local.skills import service as lifecycle +from core.capabilities.skills.binary_files import decode_binary, is_binary_value def _backup(db, row): @@ -34,7 +36,7 @@ def migrate(): return 0 count = 0 with registry._session() as db: - if not inspect(db.get_bind()).has_table(AdminSkill.__tablename__): + if not inspect(db.get_bind()).has_table(inspect_model(AdminSkill).local_table.name): return 0 rows = db.query(AdminSkill).filter(AdminSkill.source_plugin.is_(None)).all() for row in rows: diff --git a/src/backend/features/desktop/local/manager_execution.py b/src/backend/features/desktop/local/manager_execution.py new file mode 100644 index 00000000..9715d3d6 --- /dev/null +++ b/src/backend/features/desktop/local/manager_execution.py @@ -0,0 +1,51 @@ +"""Execute an installed manager contract after rechecking account and installation authority.""" +from core.harness.registry import execution_catalog as bridge_gateway +from core.harness.registry import execution_identity as bridge_identity +from core.harness.registry import execution_account as skill_account +from core.capabilities.plugins.management.contract import declared, MANAGERS + + +def execute(*, manager, server_id, headers, cloud_backed, tool_name, input_schema, arguments): + from core.harness.registry.host_runtime import requires_remote_execution as bridge_enabled + state = None + if cloud_backed and bridge_enabled(): + state = bridge_identity.get_state() + bridge_identity.require_current_account(state) + user_id = skill_account.current_local_user_id() + context = bridge_gateway.bridge_context() or {} + server = next((s for s in context.get("servers", []) if s["server_id"] == server_id), None) + if not server or server.get("source_plugin") != manager: + raise PermissionError("management plugin is no longer installed or authorized") + # upload is a local contribution authorized by the same declared install capability. + op = "install_skill" if tool_name == "upload_skill_to_cloud" else tool_name + contract = next((t for t in server.get("tools", []) if t["name"] == op), None) + if contract is None or declared(contract, manager) != manager: + raise PermissionError("management plugin contract changed; synchronize capabilities") + else: + user_id = next((v for k, v in headers.items() if k.lower() == "x-current-user-id"), None) + from core.harness.registry.execution_catalog import authorizes_manager + if not authorizes_manager(user_id, server_id, manager): + raise PermissionError("local management plugin is not installed") + if not user_id: + raise PermissionError("authentication required") + from jsonschema import validate + validate(arguments, input_schema) + if state: + bridge_identity.require_current_account(state) + if tool_name == "upload_skill_to_cloud": + from features.desktop.local.skills.upload import upload + return upload(user_id, **arguments) + kind = MANAGERS[manager] + from features.desktop.local.skills import service as local_skill_service + from features.desktop.local.plugins import service as local_plugin_service + service = local_skill_service if kind == "skill" else local_plugin_service + action = tool_name.removesuffix("_" + kind) + if tool_name == "list_" + kind + "s": + return {"ok": True, "items": getattr(service, "list_" + kind + "s")(user_id)} + args = dict(arguments) + if "source" in args: + source = args.pop("source") + if source.get("kind") != "local_path": + raise ValueError("local installation requires source.kind=local_path") + args["source_path"] = source["path"] + return getattr(service, action)(user_id, **args) diff --git a/src/backend/core/services/local_market_install.py b/src/backend/features/desktop/local/market_install.py similarity index 61% rename from src/backend/core/services/local_market_install.py rename to src/backend/features/desktop/local/market_install.py index f68fc3e0..71472510 100644 --- a/src/backend/core/services/local_market_install.py +++ b/src/backend/features/desktop/local/market_install.py @@ -1,23 +1,23 @@ """Desktop marketplace packages enter the same device-owned installation lifecycle.""" -from core.plugins.management import packages as plugin_packages -from core.plugins.packaging import sources as plugin_sources +from core.capabilities.plugins.management import packages as plugin_packages +from core.capabilities.plugins.packaging import sources as plugin_sources import base64 -from core.plugins.local import service as lifecycle -from core.services.local_skill_editor import install_archive +from features.desktop.local.plugins import service as lifecycle +from features.desktop.local.skills.editor import install_archive from core.infra.exceptions import BadRequestError, ResourceNotFoundError def install_plugin(db, user_id, slug, secrets): - from core.services import marketplace_listing as listing + from features.marketplace import edition as listing listing.ensure_item_visible(db, listing.KIND_PLUGIN, slug, user_id, resource="plugin") if secrets: raise BadRequestError(message="本机插件凭据请通过连接配置设置，不写入插件包") - directory = plugin_sources._resolve_plugin_dir(slug) + directory = plugin_sources.resolve_plugin_dir(slug) if directory is not None: return lifecycle.install(user_id, str(directory.resolve())) - row = plugin_packages._market_row(db, slug) + row = plugin_packages.market_row(db, slug) if row is None: raise ResourceNotFoundError("plugin", slug) return install_archive(user_id, base64.b64decode(row.package_b64), "plugin") diff --git a/src/backend/core/services/native_app_icons.py b/src/backend/features/desktop/local/native_app_icons.py similarity index 100% rename from src/backend/core/services/native_app_icons.py rename to src/backend/features/desktop/local/native_app_icons.py diff --git a/src/backend/core/plugins/local/__init__.py b/src/backend/features/desktop/local/plugins/__init__.py similarity index 100% rename from src/backend/core/plugins/local/__init__.py rename to src/backend/features/desktop/local/plugins/__init__.py diff --git a/src/backend/core/plugins/local/assets.py b/src/backend/features/desktop/local/plugins/assets.py similarity index 95% rename from src/backend/core/plugins/local/assets.py rename to src/backend/features/desktop/local/plugins/assets.py index b9a884fc..8669a002 100644 --- a/src/backend/core/plugins/local/assets.py +++ b/src/backend/features/desktop/local/plugins/assets.py @@ -22,8 +22,8 @@ def _bundled_site_template() -> Optional[Path]: """安装包里的 React 站点模板（``docker/site-template/``）。""" - # src/backend/core/plugins/local/assets.py → parents[5] = 仓库根 - candidate = Path(__file__).resolve().parents[5] / "docker" / "site-template" + # src/backend/features/desktop/local/plugins/assets.py → parents[6] = 仓库根 + candidate = Path(__file__).resolve().parents[6] / "docker" / "site-template" return candidate if (candidate / "init-react-site.sh").is_file() else None @@ -90,7 +90,7 @@ def provision_present_plugins(db) -> List[str]: 也不会触发云端重新准备，但随版本发出的资产需要跟着更新。本机业务库的行和云端 同步下来的投影都算"这台机器上有"。 """ - from core.capabilities import device_catalog + from core.harness.registry import device_catalog from core.db.models import InstalledPlugin # 只问登记表里那几个 slug 在不在，不把整张表和整份云端投影都物化出来。 diff --git a/src/backend/core/plugins/local/legacy_migration.py b/src/backend/features/desktop/local/plugins/legacy_migration.py similarity index 87% rename from src/backend/core/plugins/local/legacy_migration.py rename to src/backend/features/desktop/local/plugins/legacy_migration.py index 5aafbc5a..1c6e1450 100644 --- a/src/backend/core/plugins/local/legacy_migration.py +++ b/src/backend/features/desktop/local/plugins/legacy_migration.py @@ -5,12 +5,13 @@ import tempfile from pathlib import Path -from core.agent_skills.binary_files import decode_binary, is_binary_value -from core.capabilities import archive, registry +from core.capabilities.skills.binary_files import decode_binary, is_binary_value +from core.capabilities import archive as archive +from features.desktop.capabilities import registry from core.db.models import AdminMcpServer, AdminSkill, ContentBlock, InstalledPlugin -from core.plugins.local import service as lifecycle -from core.plugins.packaging import sources as plugin_sources -from core.services.management_package import serialized +from features.desktop.local.plugins import service as lifecycle +from core.capabilities.plugins.packaging import sources as plugin_sources +from core.capabilities.plugins.management.package import serialized from extensions.paths import PLUGINS_DIR @@ -36,7 +37,7 @@ def values(obj): def _package(root, row, children, servers): - from core.services.management_contract import MANAGERS + from core.capabilities.plugins.management.contract import MANAGERS skill_ids, server_ids = {}, {} official = row.source == "builtin" and row.slug in MANAGERS @@ -49,15 +50,15 @@ def _package(root, row, children, servers): if official: package = PLUGINS_DIR / "marketplace" / row.slug archive.write_files(root, {n: p.read_bytes() for n, p in archive.iter_files(package)}) - from core.plugins.packaging.importer import normalize_plugin_dir + from core.capabilities.plugins.packaging.importer import normalize_plugin_dir np = normalize_plugin_dir(root) skill_ids = { - s.name: plugin_sources._make_skill_id(row.slug, s.name, row.owner_user_id) + s.name: plugin_sources.make_skill_id(row.slug, s.name, row.owner_user_id) for s in np.skills } server_ids = { - s.name: plugin_sources._make_server_id(row.slug, s.name, row.owner_user_id) + s.name: plugin_sources.make_server_id(row.slug, s.name, row.owner_user_id) for s in np.mcp } return skill_ids, server_ids @@ -105,15 +106,15 @@ def _package(root, row, children, servers): @serialized def migrate(): - from core.capabilities import device_catalog - from core.capabilities.paths import capabilities_enabled + from core.harness.registry import device_catalog + from core.harness.registry.run_capabilities import has_run_capabilities from sqlalchemy import inspect - if not capabilities_enabled(): + if not has_run_capabilities(): return 0 count = 0 with registry._session() as db: - if not inspect(db.get_bind()).has_table(InstalledPlugin.__tablename__): + if not inspect(db.get_bind()).has_table(inspect(InstalledPlugin).local_table.name): return 0 for row in db.query(InstalledPlugin).all(): # Hybrid shared defaults are replaced by the authenticated cloud manifest. diff --git a/src/backend/core/plugins/local/service.py b/src/backend/features/desktop/local/plugins/service.py similarity index 85% rename from src/backend/core/plugins/local/service.py rename to src/backend/features/desktop/local/plugins/service.py index ccd4392c..fa4fac87 100644 --- a/src/backend/core/plugins/local/service.py +++ b/src/backend/features/desktop/local/plugins/service.py @@ -4,14 +4,16 @@ import tempfile from pathlib import Path -from core.capabilities import archive, registry, store -from core.capabilities.paths import LOCAL_PROFILE, require_root, revision_for_hash -from core.capabilities.plugins import component_install_ids -from core.capabilities.ref import local_ref -from core.db.models import DeviceCapabilityInstallation -from core.services import local_skill_service as skill_service -from core.services.management_package import serialized, snapshot -from sqlalchemy import update as sql_update +from core.capabilities import archive as archive +from features.desktop.capabilities import registry +from features.desktop.capabilities import store +from core.capabilities.identity_rules import LOCAL_PROFILE +from features.desktop.capabilities.paths import require_root, revision_for_hash +from features.desktop.capabilities.plugins import component_install_ids +from core.capabilities.reference import local_ref +from core.harness.registry.local_publication import local_publication +from features.desktop.local.skills import service as skill_service +from core.capabilities.plugins.management.package import serialized, snapshot def _owned(user_id, install_id, db=None): @@ -58,16 +60,16 @@ def list_plugins(user_id): def _prepare(user_id, root, key=None, *, skill_ids=None, server_ids=None): - from core.capabilities.mcp_json import _validate_local_server - from core.plugins.packaging.importer import detect_manifest, normalize_plugin_dir - from core.services.desktop_capability_protocol import entity_content_hash - from core.services.marketplace_service import compute_install_id + from features.desktop.capabilities.mcp_json import _validate_local_server + from core.capabilities.plugins.packaging.importer import detect_manifest, normalize_plugin_dir + from core.capabilities.manifest_protocol import entity_content_hash + from core.capabilities.skills.installation import compute_install_id as compute_install_id np = normalize_plugin_dir(root) manifest = json.loads(detect_manifest(root)[1].read_text(encoding="utf-8")) executor = manifest.get("extensions", {}).get("org.hugagent", {}).get("local_execution") if executor: - from core.services.management_contract import MANAGERS, VERSION + from core.capabilities.plugins.management.contract import MANAGERS, VERSION if ( executor.get("id") != np.slug @@ -127,7 +129,7 @@ def _prepare(user_id, root, key=None, *, skill_ids=None, server_ids=None): } files = {"package/" + n: p.read_bytes() for n, p in archive.iter_files(root)} # entity_content_hash represents binary assets using the existing wire encoding. - from core.agent_skills.binary_files import encode_upload + from core.capabilities.skills.binary_files import encode_upload files["plugin.json"] = json.dumps(definition, ensure_ascii=False, sort_keys=True) digest = entity_content_hash( @@ -139,16 +141,16 @@ def _prepare(user_id, root, key=None, *, skill_ids=None, server_ids=None): if existing and existing.state != "removed" and existing.resolved_revision else None ) - from core.capabilities.plugins import load_manifest + from features.desktop.capabilities.plugins import load_manifest old_ui = load_manifest(previous).get("ui_contributions") if previous else None candidate = bool(existing and existing.state != "removed" and ( (np.ui or {}).get("version") == 3 or (old_ui or {}).get("version") == 3)) if candidate: if existing.payload.get("owner_user_id") != user_id: raise PermissionError("plugin ownership conflict") - from core.applications.plugin_candidate_storage import local_reserve + from core.harness.registry.package_sources import package_upgrade_policy if revision_for_hash(digest) != existing.resolved_revision: - local_reserve(existing, revision_for_hash(digest), files) + package_upgrade_policy().reserve_local(existing, revision_for_hash(digest), files) comp = store.write_from_files("plugin", LOCAL_PROFILE, key, revision_for_hash(digest), files) if not candidate: children = _prepare_children(user_id, np, comp, skill_ids, server_ids) @@ -158,15 +160,15 @@ def _prepare(user_id, root, key=None, *, skill_ids=None, server_ids=None): def _prepare_children(user_id, np, comp, skill_ids, server_ids): children = [] - from core.agent_skills.registry import _load_skill_metadata_from_str + from core.capabilities.skills.registry import load_skill_metadata_from_str for child in np.skills: - _load_skill_metadata_from_str(child.skill_content, child.name) + load_skill_metadata_from_str(child.skill_content, child.name) with tempfile.TemporaryDirectory(prefix="plugin-skills-") as tmp: for child in np.skills: child_key = skill_ids[child.name] directory = Path(tmp) / child_key - from core.capabilities.skills import device_view_dir, user_view_dir - from core.plugins.packaging.skill_transform import files as component_files + from core.harness.registry.local_publication import local_publication + from core.capabilities.plugins.packaging.skill_transform import files as component_files archive.write_files( directory, @@ -174,7 +176,7 @@ def _prepare_children(user_id, np, comp, skill_ids, server_ids): child, skill_ids, server_ids, - (user_view_dir(user_id) or device_view_dir()) / child_key, + (local_publication().user_view_dir(user_id) or local_publication().device_view_dir()) / child_key, comp.path / "package", ), ) @@ -244,7 +246,7 @@ def _publish(user_id, prepared, db, *, legacy_source=None): if previous.created_at else now ) - db.get(DeviceCapabilityInstallation, iid).created_at = stamp + local_publication().stamp_installation(db, iid, stamp) registry.upsert( profile_id=LOCAL_PROFILE, ref=local_ref("plugin", key), @@ -298,16 +300,8 @@ def install(user_id, source_path): def _claim(db, inst, expected_revision): if not expected_revision or inst.resolved_revision != expected_revision: raise ValueError("revision_conflict") - result = db.execute( - sql_update(DeviceCapabilityInstallation) - .where( - DeviceCapabilityInstallation.install_id == inst.install_id, - DeviceCapabilityInstallation.resolved_revision == expected_revision, - DeviceCapabilityInstallation.state != "removed", - ) - .values(generation=DeviceCapabilityInstallation.generation + 1) - ) - if result.rowcount != 1: + changed = local_publication().claim_revision(db, inst.install_id, expected_revision) + if changed != 1: raise ValueError("revision_conflict") @@ -322,8 +316,8 @@ def update(user_id, install_id, source_path, expected_revision): with registry._session() as db: current = _owned(user_id, install_id, db) _claim(db, current, expected_revision) - from core.applications.local_plugin_upgrades import prepare as prepare_upgrade - pending = prepare_upgrade(db, current, prepared) + from core.harness.registry.package_sources import package_upgrade_policy + pending = package_upgrade_policy().prepare_local(db, current, prepared) if pending is not None: return pending _publish(user_id, prepared, db) @@ -345,7 +339,7 @@ def uninstall(user_id, install_id, expected_revision): @serialized def set_presentation(user_id, install_id, values): - from core.plugins.management.market import _validate_icon + from core.capabilities.plugins.packaging.presentation import validate_icon as _validate_icon with registry._session() as db: inst = _owned(user_id, install_id, db) diff --git a/src/backend/features/desktop/local/plugins/sidecars.py b/src/backend/features/desktop/local/plugins/sidecars.py new file mode 100644 index 00000000..46440ad4 --- /dev/null +++ b/src/backend/features/desktop/local/plugins/sidecars.py @@ -0,0 +1,33 @@ +"""Start trusted bundled MCPs selected by owned local plugin definitions.""" + +from __future__ import annotations + +from urllib.parse import urlsplit + + + +def native_bindings(configs): + from core.config.mcp_config import mcp_http_url as _mcp_http_url + from core.config.settings import settings + from extensions.mcp_servers._ports import PORTS + + if not settings.deploy.is_local: + return set() + urls = { + _mcp_http_url(sid).rstrip("/"): (sid, port) + for sid, port in PORTS.items() + if urlsplit(_mcp_http_url(sid)).hostname in ("127.0.0.1", "localhost", "::1") + } + return { + urls[str(cfg.get("url") or "").rstrip("/")] + for cfg in configs.values() + if cfg.get("origin") == "local_plugin" + and cfg.get("execution_scope") == "local" + and cfg.get("owner_user_id") + and str(cfg.get("url") or "").rstrip("/") in urls + } + + +async def ensure_native_servers(configs): + from core.infra.local_runtime.dynamic_servers import ensure_dynamic_servers + await ensure_dynamic_servers(native_bindings(configs)) diff --git a/src/backend/features/desktop/local/publication_provider.py b/src/backend/features/desktop/local/publication_provider.py new file mode 100644 index 00000000..dc7ef5cb --- /dev/null +++ b/src/backend/features/desktop/local/publication_provider.py @@ -0,0 +1,101 @@ +"""Desktop implementation of local skill publication and revision claims.""" +from features.desktop.capabilities import skill_paths, skill_publication, skill_views + +class DesktopLocalPublication: + def management_lock_path(self): + from features.desktop.capabilities.paths import meta_root + return meta_root() / "management.lock" + + def provision_plugin_assets(self, slug): + from features.desktop.local.plugins.assets import provision_for + return provision_for(slug) + + def migrate_local_plugins(self): + from features.desktop.local.plugins.legacy_migration import migrate + return migrate() + + def migrate_local_skills(self): + from features.desktop.local.management_migration import migrate + return migrate() + + async def ensure_native_servers(self, configs): + from features.desktop.local.plugins.sidecars import ensure_native_servers + return await ensure_native_servers(configs) + + def publish_local_plugin(self, definition, *, owner_user_id): + from features.desktop.capabilities.plugins import publish_local_plugin + return publish_local_plugin(definition, owner_user_id=owner_user_id) + + def remove_local_plugin(self, slug): + from features.desktop.capabilities.plugins import remove_local_plugin + return remove_local_plugin(slug) + + def skill_backend(self, *, priority=150, local=False): + from features.desktop.skill_backend import CapabilityStoreBackend + return CapabilityStoreBackend(priority=priority, local=local) + + def merge_skill_infos(self, groups, *, backend_hash): + from features.desktop.capabilities.skills import merge_skill_infos + return merge_skill_infos(groups, backend_hash=backend_hash) + + def view_generation(self): + from features.desktop.capabilities.skills import view_generation + return view_generation() + + def skill_root(self): + from features.desktop.capabilities.paths import kind_root + return kind_root("skill") + + def merge_visible_agents(self, user_id, rows): + from features.desktop.capabilities.agents import merge_visible + return merge_visible(user_id, rows) + + def account_agent_definition(self, agent_id): + from features.desktop.capabilities.agents import account_definition + return account_definition(agent_id) + + def publish_local_agent(self, definition): + from features.desktop.capabilities.agents import publish_local_agent + return publish_local_agent(definition) + + def remove_local_agent(self, agent_id): + from features.desktop.capabilities.agents import remove_local_agent + return remove_local_agent(agent_id) + + def rebuild_device_view(self): + return skill_views.rebuild_device_view() + + def rebuild_user_view(self, user_id): + return skill_views.rebuild_user_view(user_id) + + def rebuild_views(self, user_id): + return skill_views.rebuild_views(user_id) + + def remove_local_skill(self, skill_id): + return skill_publication.remove_local_skill(skill_id) + + def prune_local(self, owners): + return skill_publication.prune_local(owners) + + def publish_local_skill(self, *args, **kwargs): + return skill_publication.publish_local_skill(*args, **kwargs) + + def device_view_dir(self): + return skill_paths.device_view_dir() + + def user_view_dir(self, user_id): + return skill_paths.user_view_dir(user_id) + + def stamp_installation(self, db, install_id, stamp): + from features.desktop.models import DeviceCapabilityInstallation + db.get(DeviceCapabilityInstallation, install_id).created_at = stamp + + def claim_revision(self, db, install_id, expected_revision): + from features.desktop.models import DeviceCapabilityInstallation + from sqlalchemy import update + result = db.execute(update(DeviceCapabilityInstallation).where( + DeviceCapabilityInstallation.install_id == install_id, + DeviceCapabilityInstallation.resolved_revision == expected_revision, + DeviceCapabilityInstallation.state != "removed", + ).values(generation=DeviceCapabilityInstallation.generation + 1)) + return result.rowcount diff --git a/src/backend/features/desktop/local/skills/__init__.py b/src/backend/features/desktop/local/skills/__init__.py new file mode 100644 index 00000000..8d571cfb --- /dev/null +++ b/src/backend/features/desktop/local/skills/__init__.py @@ -0,0 +1 @@ +"""skills package.""" diff --git a/src/backend/core/services/local_skill_editor.py b/src/backend/features/desktop/local/skills/editor.py similarity index 86% rename from src/backend/core/services/local_skill_editor.py rename to src/backend/features/desktop/local/skills/editor.py index f4a1e8dc..54078751 100644 --- a/src/backend/core/services/local_skill_editor.py +++ b/src/backend/features/desktop/local/skills/editor.py @@ -1,18 +1,20 @@ """Frontend skill editing uses the same immutable installation lifecycle as tools.""" import tempfile from pathlib import Path -from core.capabilities import archive, registry, store -from core.services import local_skill_service as lifecycle -from core.agent_skills.binary_files import encode_upload, is_binary_value -from core.agent_skills.registry import _split_frontmatter -from core.services.skill_management_service import build_skill_content, validate_skill_file_path +from core.capabilities import archive as archive +from features.desktop.capabilities import registry +from features.desktop.capabilities import store +from features.desktop.local.skills import service as lifecycle +from core.capabilities.skills.binary_files import encode_upload, is_binary_value +from core.capabilities.skills.registry import split_frontmatter +from core.capabilities.skills.management import build_skill_content, validate_skill_file_path def frontend_errors(function): from functools import wraps @wraps(function) def call(*args, **kwargs): from fastapi import HTTPException - from core.capabilities.errors import CapabilityError + from core.harness.capability_errors import CapabilityError try: return function(*args, **kwargs) except PermissionError as exc: @@ -38,8 +40,8 @@ def read(user_id, key): @frontend_errors def detail(user_id, key): inst, files = read(user_id, key) - meta, body = _split_frontmatter(files["SKILL.md"].decode("utf-8")) - from core.services.skill_management_service import extract_mcp_server_ids + meta, body = split_frontmatter(files["SKILL.md"].decode("utf-8")) + from core.capabilities.skills.management import extract_mcp_server_ids return {"id": key, "install_id": inst.install_id, "revision": inst.resolved_revision, "display_name": inst.payload.get("presentation", {}).get("display_name") or inst.display_name, "description": inst.description, "instructions": body.strip(), "tags": _list(meta.get("tags")), @@ -73,12 +75,12 @@ def save(user_id, body): if previous is None or previous.entry_file.read_bytes() != files["SKILL.md"]: from fastapi import HTTPException raise HTTPException(409, detail="技能已更新，请重新打开编辑器后保存") - old, _ = _split_frontmatter(files["SKILL.md"].decode("utf-8")) + old, _ = split_frontmatter(files["SKILL.md"].decode("utf-8")) else: inst, files, old = None, {}, {} # Names selected in the UI are already authorized connector identities. Runtime # dependency resolution remains the authority on readiness, including missing MCPs. - from core.services.skill_management_service import extract_mcp_server_ids + from core.capabilities.skills.management import extract_mcp_server_ids ids = body.mcp_server_ids if ids is None: ids = extract_mcp_server_ids(files.get("SKILL.md", b"").decode("utf-8")) @@ -103,7 +105,7 @@ def file_get(user_id, key, filename): filename = validate_skill_file_path(filename) if filename == "SKILL.md" or filename not in files: raise ValueError("extra file not found") - from core.agent_skills.binary_files import is_binary_value + from core.capabilities.skills.binary_files import is_binary_value value = encode_upload(filename, files[filename]) binary = is_binary_value(value) return {"filename": filename, "content": "" if binary else value, "is_binary": binary} @@ -130,7 +132,7 @@ def file_change(user_id, key, filename, content=None): @frontend_errors def export(user_id, key): _, files = read(user_id, key) - from core.services.marketplace_service import build_skill_zip + from features.marketplace.skill_archives import build_skill_zip return build_skill_zip(key, files.pop("SKILL.md").decode("utf-8"), {n: encode_upload(n, v) for n, v in files.items()}) @@ -152,7 +154,7 @@ def icon(user_id, key, value): def install_archive(user_id, raw, kind="skill"): if len(raw) > 50 * 1024 * 1024: raise ValueError("package exceeds 50MB limit") - from core.plugins.local import service as local_plugin_service + from features.desktop.local.plugins import service as local_plugin_service with tempfile.TemporaryDirectory(prefix="manager-upload-") as tmp: path = Path(tmp) / "package.zip" path.write_bytes(raw) diff --git a/src/backend/core/services/local_skill_service.py b/src/backend/features/desktop/local/skills/service.py similarity index 81% rename from src/backend/core/services/local_skill_service.py rename to src/backend/features/desktop/local/skills/service.py index 680b576a..db56f467 100644 --- a/src/backend/core/services/local_skill_service.py +++ b/src/backend/features/desktop/local/skills/service.py @@ -1,11 +1,13 @@ """Device-owned skill lifecycle over the capability index and immutable versions.""" +from features.desktop.capabilities import skills from pathlib import Path from sqlalchemy import update as sql_update -from core.capabilities import registry, store, skills -from core.capabilities.paths import LOCAL_PROFILE, require_root, revision_for_hash -from core.capabilities.ref import local_ref -from core.db.models import DeviceCapabilityInstallation -from core.services.management_package import snapshot, serialized +from features.desktop.capabilities import registry, store, skills +from core.capabilities.identity_rules import LOCAL_PROFILE +from features.desktop.capabilities.paths import require_root, revision_for_hash +from core.capabilities.reference import local_ref +from features.desktop.models import DeviceCapabilityInstallation +from core.capabilities.plugins.management.package import snapshot, serialized def _owned(user_id, install_id, db=None): @@ -21,8 +23,8 @@ def _owned(user_id, install_id, db=None): def _result(inst): - from core.capabilities.readiness import file_readiness - from core.capabilities.management_context import management_context + from features.desktop.capabilities.readiness import file_readiness + from features.desktop.capabilities.management_context import management_context comp = store.get("skill", LOCAL_PROFILE, inst.key, inst.resolved_revision) return {"ok": True, "install_id": inst.install_id, "skill_id": inst.key, "name": inst.display_name, "revision": inst.resolved_revision, @@ -42,17 +44,17 @@ def list_skills(user_id): def _prepare(user_id, root, key=None): - from core.agent_skills.registry import _load_skill_metadata_from_str, _split_frontmatter - from core.services.marketplace_service import compute_install_id, _rewrite_frontmatter_name - from core.services.desktop_capability_protocol import skill_content_hash - from core.agent_skills.binary_files import encode_upload + from core.capabilities.skills.registry import load_skill_metadata_from_str, split_frontmatter + from core.capabilities.skills.installation import compute_install_id as compute_install_id, rewrite_frontmatter_name as _rewrite_frontmatter_name + from core.capabilities.manifest_protocol import skill_content_hash + from core.capabilities.skills.binary_files import encode_upload md = (root / "SKILL.md").read_text(encoding="utf-8") - fm, _ = _split_frontmatter(md) + fm, _ = split_frontmatter(md) original = str(fm.get("name") or "") - _load_skill_metadata_from_str(md, original) + load_skill_metadata_from_str(md, original) key = key or compute_install_id(original, user_id) md = _rewrite_frontmatter_name(md, key) - meta = _load_skill_metadata_from_str(md, key) + meta = load_skill_metadata_from_str(md, key) from core.capabilities.archive import iter_files files = {name: file.read_bytes() for name, file in iter_files(root)} files["SKILL.md"] = md @@ -64,14 +66,14 @@ def _prepare(user_id, root, key=None): def _invalidate(): - from core.config.catalog_runtime import invalidate_runtime_catalog_cache - from core.config.catalog_resolver import invalidate_capability_cache + from core.capabilities.catalog.runtime import invalidate_runtime_catalog_cache + from core.capabilities.catalog.resolver import invalidate_capability_cache from prompts.prompt_runtime import invalidate_prompt_cache invalidate_capability_cache() invalidate_prompt_cache() skills.bump_view_generation() invalidate_runtime_catalog_cache() - from core.agent_skills.loader import get_skill_loader + from core.capabilities.skills.loader_factory import get_skill_loader get_skill_loader(reset=True) diff --git a/src/backend/core/services/local_skill_upload.py b/src/backend/features/desktop/local/skills/upload.py similarity index 74% rename from src/backend/core/services/local_skill_upload.py rename to src/backend/features/desktop/local/skills/upload.py index f7352b45..4baf159d 100644 --- a/src/backend/core/services/local_skill_upload.py +++ b/src/backend/features/desktop/local/skills/upload.py @@ -1,25 +1,29 @@ """Explicit device-to-cloud private skill registration with durable retry identity.""" +from features.desktop.bridge import identity as bridge_identity +from features.desktop.capabilities import skill_account import hashlib -from core.capabilities import registry, store, archive -from core.capabilities.paths import LOCAL_PROFILE -from core.agent_skills.binary_files import encode_upload -from core.services.management_package import serialized -from core.services.local_skill_service import _owned +from features.desktop.capabilities import registry +from features.desktop.capabilities import store +from core.capabilities import archive as archive +from core.capabilities.identity_rules import LOCAL_PROFILE +from core.capabilities.skills.binary_files import encode_upload +from core.capabilities.plugins.management.package import serialized +from features.desktop.local.skills.service import _owned @serialized def upload(user_id, install_id, expected_revision, expected_cloud_revision=None): - from core.capabilities import skills - from core.capabilities.change_sync import cloud_request - from core.services import desktop_cloud_bridge as bridge - state = bridge.get_state() - bridge.require_current_account(state) - if not skills.account_authorized_for(user_id): + from features.desktop.capabilities import skills + from features.desktop.capabilities.change_sync import cloud_request + from features.desktop.bridge import client as bridge + state = bridge_identity.get_state() + bridge_identity.require_current_account(state) + if not skill_account.account_authorized_for(user_id): raise PermissionError("current cloud account does not own this local session") inst = _owned(user_id, install_id) if not expected_revision or inst.resolved_revision != expected_revision: raise ValueError("revision_conflict") - profile = skills.current_account_profile() + profile = skill_account.current_account_profile() links = dict(inst.payload.get("cloud_uploads") or {}) link = dict(links.get(profile) or {}) comp = store.get("skill", LOCAL_PROFILE, inst.key, expected_revision) @@ -29,7 +33,7 @@ def upload(user_id, install_id, expected_revision, expected_cloud_revision=None) if n != ".inventory.json"} target = link.get("key") or ("device-" + hashlib.sha256((profile + install_id).encode()).hexdigest()[:32]) # Rewrite identity once to the stable cloud destination, never derive a target from a display name. - from core.services.marketplace_service import _rewrite_frontmatter_name + from core.capabilities.skills.installation import rewrite_frontmatter_name as _rewrite_frontmatter_name files["SKILL.md"] = _rewrite_frontmatter_name(files["SKILL.md"], target) identity = hashlib.sha256((profile + install_id + expected_revision + str(expected_cloud_revision)).encode()).hexdigest() if link.get("local_revision") == expected_revision and link.get("revision"): @@ -49,13 +53,13 @@ def upload(user_id, install_id, expected_revision, expected_cloud_revision=None) pending = {"request_id": identity, "expected_revision": remote["revision"], "expected_model_version": remote.get("model_version"), "create_only": not remote.get("exists")} links[profile] = {**link, "key": target, "pending": pending} - with bridge.account_scope(state): + with bridge_identity.account_scope(state): registry.set_state(install_id, inst.state, payload_update={"cloud_uploads": links}) body = {**pending, "files": files} response = cloud_request(state, "POST", "skill", target, body) if not response.get("applied"): raise ValueError("cloud saved package but did not register it") - with bridge.account_scope(state): + with bridge_identity.account_scope(state): links[profile] = {"key": target, "revision": response["revision"], "local_revision": expected_revision} registry.set_state(install_id, inst.state, payload_update={"cloud_uploads": links}) return {"ok": True, "install_id": install_id, "local_revision": expected_revision, diff --git a/src/backend/core/services/local_snapshot_service.py b/src/backend/features/desktop/local/snapshots.py similarity index 100% rename from src/backend/core/services/local_snapshot_service.py rename to src/backend/features/desktop/local/snapshots.py diff --git a/src/backend/core/db/models/capability.py b/src/backend/features/desktop/models.py similarity index 60% rename from src/backend/core/db/models/capability.py rename to src/backend/features/desktop/models.py index 90e81be7..73a2fd7e 100644 --- a/src/backend/core/db/models/capability.py +++ b/src/backend/features/desktop/models.py @@ -1,22 +1,15 @@ -"""Device-side capability installation index. - -These tables live in the device's existing business database (no second SQLite), -so an install transaction and its business-data side effects can share one -transaction. On cloud deployments they exist but stay empty — the file store is -disabled there and nothing writes to them. -""" +"""Desktop installation indexes and durable observation delivery records.""" from core.db.utc_datetime import UTCDateTime from core.infra.time import utc_now - from datetime import datetime - from core.db.engine import Base from core.db.models import JSONType from sqlalchemy import Boolean, Column, Index, Integer, String, Text +from datetime import datetime, timezone +from sqlalchemy import JSON, BigInteger, Boolean, Column, Index, Integer, String, Text INSTALL_STATES = ("pending", "preparing", "ready", "failed", "removed") - class DeviceCapabilityInstallation(Base): """One (profile, kind, key) on this device: intent, resolved revision and state. @@ -59,7 +52,6 @@ class DeviceCapabilityInstallation(Base): Index("idx_dev_cap_inst_state", "state"), ) - class DeviceCapabilityNamePreference(Base): """The user's explicit answer to "which same-named candidate wins". Device-only.""" @@ -72,7 +64,6 @@ class DeviceCapabilityNamePreference(Base): chosen_by = Column(String(64), nullable=True) updated_at = Column(UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now) - class DeviceCapabilityTransaction(Base): """Install transaction log: explains what is committed and what is not after a crash.""" @@ -89,7 +80,6 @@ class DeviceCapabilityTransaction(Base): __table_args__ = (Index("idx_dev_cap_tx_install", "install_id"),) - class DeviceCapabilityComponent(Base): """Ownership edge: a plugin installation → the component installations it requires.""" @@ -104,3 +94,56 @@ class DeviceCapabilityComponent(Base): Index("idx_dev_cap_comp_owner", "owner_install_id"), Index("idx_dev_cap_comp_component", "component_install_id"), ) + +def utcnow(): + return datetime.now(timezone.utc).isoformat() + +class DesktopOutbox(Base): + __tablename__ = "desktop_observability_outbox" + id = Column(String(64), primary_key=True) + cloud_base = Column(String(512), nullable=False) + subject = Column(String(64), nullable=False) + device_id = Column(String(128), nullable=False) + local_user_id = Column(String(64), nullable=False) + kind = Column(String(24), nullable=False) + object_id = Column(String(128), nullable=False) + revision = Column(BigInteger, nullable=False, default=1) + pending = Column(Boolean, nullable=False, default=True) + deleted = Column(Boolean, nullable=False, default=False) + digest = Column(String(64), nullable=False, default="") + updated_at = Column(String(40), nullable=False, default=utcnow) + __table_args__ = ( + Index("idx_desktop_outbox_pending", "cloud_base", "subject", "device_id", "pending"), + ) + +class DesktopRecord(Base): + __tablename__ = "desktop_observability_records" + id = Column(String(64), primary_key=True) + user_id = Column(String(64), nullable=False) + device_id = Column(String(128), nullable=False) + kind = Column(String(24), nullable=False) + object_id = Column(String(128), nullable=False) + revision = Column(BigInteger, nullable=False) + chat_id = Column(String(128), nullable=False, default="") + run_id = Column(String(128), nullable=False, default="") + observation = Column(String(16), nullable=False, default="desktop") + payload = Column(JSON, nullable=False) + received_at = Column(String(40), nullable=False, default=utcnow) + __table_args__ = ( + Index("idx_desktop_records_identity", "kind", "observation", "user_id", "device_id", "object_id", unique=True), + Index("idx_desktop_records_user_kind", "user_id", "kind", "received_at"), + Index("idx_desktop_records_chat", "user_id", "device_id", "chat_id"), + ) + +class DesktopSyncDevice(Base): + __tablename__ = "desktop_observability_devices" + id = Column(String(64), primary_key=True) + user_id = Column(String(64), nullable=False) + device_id = Column(String(128), nullable=False) + last_seen = Column(String(40), nullable=False, default=utcnow) + pending = Column(Integer, nullable=False, default=0) + capture_errors = Column(Integer, nullable=False, default=0) + reconciling = Column(Boolean, nullable=False, default=False) + __table_args__ = (Index("idx_desktop_devices_user", "user_id"),) + +__all__ = ['DeviceCapabilityInstallation', 'DeviceCapabilityNamePreference', 'DeviceCapabilityTransaction', 'DeviceCapabilityComponent', 'DesktopOutbox', 'DesktopRecord', 'DesktopSyncDevice', 'INSTALL_STATES'] diff --git a/src/backend/features/desktop/observability/__init__.py b/src/backend/features/desktop/observability/__init__.py new file mode 100644 index 00000000..afeb11f1 --- /dev/null +++ b/src/backend/features/desktop/observability/__init__.py @@ -0,0 +1 @@ +"""observability package.""" diff --git a/src/backend/core/services/desktop_observation_context.py b/src/backend/features/desktop/observability/context.py similarity index 78% rename from src/backend/core/services/desktop_observation_context.py rename to src/backend/features/desktop/observability/context.py index e0c8d8e1..d4fdab05 100644 --- a/src/backend/core/services/desktop_observation_context.py +++ b/src/backend/features/desktop/observability/context.py @@ -3,13 +3,13 @@ import uuid from contextvars import ContextVar -model_call_id = ContextVar("desktop_model_call_id", default="") +from core.harness.log_context import model_call_id def context_headers(*, model=False): from core.infra.logging import chat_id_var, trace_id_var - from core.llm.model_usage import CURRENT_MODEL_USAGE - from core.services.log_service import _current_message_id + from core.models.usage import CURRENT_MODEL_USAGE + from core.harness.log_context import current_message_id as _current_message_id values = { "x-observation-chat-id": chat_id_var.get(), @@ -24,7 +24,7 @@ def context_headers(*, model=False): model_call_id.set(call_id) values["x-observation-call-id"] = call_id else: - from core.services.tool_effect_ledger import CURRENT_TOOL_EFFECT + from core.harness.tool_effect_context import CURRENT_TOOL_EFFECT effect = CURRENT_TOOL_EFFECT.get() if effect: diff --git a/src/backend/core/services/desktop_observability_delivery.py b/src/backend/features/desktop/observability/delivery.py similarity index 86% rename from src/backend/core/services/desktop_observability_delivery.py rename to src/backend/features/desktop/observability/delivery.py index b7018dea..b19f01ec 100644 --- a/src/backend/core/services/desktop_observability_delivery.py +++ b/src/backend/features/desktop/observability/delivery.py @@ -1,13 +1,14 @@ """HTTP delivery boundary for durable desktop observations.""" from __future__ import annotations +from features.desktop.bridge import identity as bridge_identity import logging import time import httpx from core.infra.responses import generate_trace_id -from core.services import desktop_cloud_bridge as bridge +from features.desktop.bridge import client as bridge logger = logging.getLogger(__name__) @@ -15,9 +16,9 @@ async def deliver_batch(state, batch, *, transport=None): # Headers and the rejection snapshot must name the same exact token, even # if the shell rotates credentials while a queued batch is being prepared. - with bridge.account_scope(state): - credential = dict(bridge.current_credentials(state)) - headers = bridge.cloud_headers(credential) + with bridge_identity.account_scope(state): + credential = dict(bridge_identity.current_credentials(state)) + headers = bridge_identity.cloud_headers(credential) trace_id = generate_trace_id() headers["X-Trace-ID"] = trace_id async with httpx.AsyncClient( @@ -48,7 +49,7 @@ async def deliver_batch(state, batch, *, transport=None): raise ValueError("invalid desktop observation acknowledgements") return body except Exception as exc: - bridge.note_rejected_credential(credential, exc) + bridge_identity.note_rejected_credential(credential, exc) # Neither exception text nor remote fields are safe diagnostic data. # The locally generated trace also reaches the cloud request logger. logger.warning( diff --git a/src/backend/core/services/observability_projection.py b/src/backend/features/desktop/observability/fields.py similarity index 96% rename from src/backend/core/services/observability_projection.py rename to src/backend/features/desktop/observability/fields.py index 696bbd86..6e85bf08 100644 --- a/src/backend/core/services/observability_projection.py +++ b/src/backend/features/desktop/observability/fields.py @@ -4,7 +4,7 @@ aggregates use the same field precedence as the serialized management record. """ -from core.db.models.observability import DesktopRecord +from features.desktop.models import DesktopRecord from sqlalchemy import case, func from sqlalchemy.orm import aliased diff --git a/src/backend/core/services/desktop_observability.py b/src/backend/features/desktop/observability/projection.py similarity index 98% rename from src/backend/core/services/desktop_observability.py rename to src/backend/features/desktop/observability/projection.py index 4861be9e..736c9737 100644 --- a/src/backend/core/services/desktop_observability.py +++ b/src/backend/features/desktop/observability/projection.py @@ -7,12 +7,8 @@ import math import re -from core.db.models.observability import DesktopRecord, DesktopSyncDevice, utcnow -from core.services.observability_projection import ( - counterpart_identity, - desktop_counterpart, - gateway_counterpart, -) +from features.desktop.models import DesktopRecord, DesktopSyncDevice, utcnow +from features.desktop.observability.fields import counterpart_identity, desktop_counterpart, gateway_counterpart from sqlalchemy import func, or_, select KINDS = {"session", "message", "run", "tool", "skill", "agent", "usage", "model"} diff --git a/src/backend/core/services/desktop_observability_sync.py b/src/backend/features/desktop/observability/sync.py similarity index 96% rename from src/backend/core/services/desktop_observability_sync.py rename to src/backend/features/desktop/observability/sync.py index 2f74a5f7..959c7d48 100644 --- a/src/backend/core/services/desktop_observability_sync.py +++ b/src/backend/features/desktop/observability/sync.py @@ -1,6 +1,7 @@ """Local durable change capture and independently scheduled, bounded cloud delivery.""" from __future__ import annotations +from features.desktop.bridge import identity as bridge_identity import asyncio import json @@ -17,8 +18,8 @@ ToolCallLog, UserShadow, ) -from core.db.models.observability import DesktopOutbox, utcnow -from core.services.desktop_observability import FIELDS, insert_for, key, public_payload +from features.desktop.models import DesktopOutbox, utcnow +from features.desktop.observability.projection import FIELDS, insert_for, key, public_payload from sqlalchemy import event, func, or_, select, update logger = logging.getLogger(__name__) @@ -338,10 +339,10 @@ def __init__(self, factory, *, interval=3.0, transport=None): self.reconciled_at = 0.0 async def sync_once(self): - from core.services import desktop_cloud_bridge as bridge - from core.services.desktop_observability_delivery import deliver_batch + from features.desktop.bridge import client as bridge + from features.desktop.observability.delivery import deliver_batch - state, identity = bridge.get_state(), bridge.get_identity_state() + state, identity = bridge_identity.get_state(), bridge_identity.get_identity_state() if not state or not identity or not identity.get("device_id"): return account = key(identity["cloud_base"], identity["subject"], identity["device_id"]) @@ -366,7 +367,7 @@ def prepare(): batch = await asyncio.to_thread(prepare) body = await deliver_batch(state, batch, transport=self.transport) - bridge.require_current_account(state) + bridge_identity.require_current_account(state) result = body.get("data", {}) # Accept acknowledgements only for this exact sent batch. allowed = {(e["kind"], e["object_id"], e["revision"]) for e in batch["events"]} diff --git a/src/backend/core/plugins/runtime/desktop_resolution.py b/src/backend/features/desktop/plugin_resolution.py similarity index 85% rename from src/backend/core/plugins/runtime/desktop_resolution.py rename to src/backend/features/desktop/plugin_resolution.py index ee1748ce..df798883 100644 --- a/src/backend/core/plugins/runtime/desktop_resolution.py +++ b/src/backend/features/desktop/plugin_resolution.py @@ -1,6 +1,9 @@ """Progressive plugin runtime: desktop resolution.""" from __future__ import annotations +from features.desktop.bridge import identity as bridge_identity +from features.desktop.capabilities import skill_account +from features.desktop.capabilities import skills import logging import os @@ -9,15 +12,15 @@ logger = logging.getLogger(__name__) -from core.plugins.runtime import models as plugin_runtime_models -from core.plugins.runtime import resolution as plugin_resolution +from core.capabilities.plugins.runtime import models as plugin_runtime_models +from core.capabilities.plugins.runtime import resolution as plugin_resolution def prepare_desktop_plugin_skill_defaults(user_id, skill_ids): """Include enabled cloud plugin intentions before ready-only skill filtering.""" - from core.capabilities import skills - from core.capabilities.plugins import enabled_cloud_skill_intents - from core.capabilities.preparation import ensure_cloud_ready + from features.desktop.capabilities import skills + from features.desktop.capabilities.plugins import enabled_cloud_skill_intents + from features.desktop.capabilities.preparation import ensure_cloud_ready selected = set(skill_ids) | enabled_cloud_skill_intents(user_id) ensure_cloud_ready(user_id, skill_keys=sorted(selected)) @@ -40,17 +43,17 @@ def resolve_desktop_progressive_plugins( Only model-facing skills and MCP connections are delayed until load_plugin. The device registry, not legacy cloud InstalledPlugin rows, owns identities. """ - from core.capabilities import registry, skills - from core.capabilities.dependency import Context, Inspector, _identifier - from core.capabilities.paths import LOCAL_PROFILE + from features.desktop.capabilities import registry, skills + from features.desktop.capabilities.dependency import Context, Inspector, _identifier + from core.capabilities.identity_rules import LOCAL_PROFILE - profile = skills.current_account_profile() if skills.account_authorized_for(user_id) else None + profile = skill_account.current_account_profile() if skill_account.account_authorized_for(user_id) else None result = plugin_runtime_models.ProgressiveResolution() eligible: List[plugin_runtime_models.DeferredPlugin] = [] allowed_skills, allowed_mcp = set(enabled_skill_ids or []), set(enabled_mcp_ids or []) active = {item for item in (activated_ids or []) if item} - from core.capabilities.errors import CapabilityError - from core.capabilities.preparation import ensure_cloud_ready + from core.harness.capability_errors import CapabilityError + from features.desktop.capabilities.preparation import ensure_cloud_ready for skill in sorted(allowed_skills): try: @@ -68,7 +71,7 @@ def resolve_desktop_progressive_plugins( # Cloud projections and explicitly owned local packages are both valid. # Legacy shared local bundles must stay hidden in hybrid mode: their MCP # endpoints belong to a different execution plane. - from core.capabilities import device_catalog + from features.desktop.capabilities import device_catalog hybrid = device_catalog.active() and profile allowed_profiles = (LOCAL_PROFILE, profile) @@ -83,7 +86,7 @@ def resolve_desktop_progressive_plugins( if hybrid and row.profile_id == LOCAL_PROFILE: if row.payload.get("owner_user_id") != user_id: continue - from core.capabilities.local_plugin_runtime import enabled_for + from features.desktop.capabilities.local_plugin_runtime import enabled_for if not enabled_for(row, user_id): continue @@ -110,10 +113,10 @@ def resolve_desktop_progressive_plugins( advertised_skills & allowed_skills or advertised_mcp & allowed_mcp ): continue - from core.services import desktop_cloud_bridge, desktop_cloud_bundles + from features.desktop.bridge import client as desktop_cloud_bridge, bundles as desktop_cloud_bundles results = desktop_cloud_bundles.prepare( - desktop_cloud_bridge.get_state(), [row.install_id] + bridge_identity.get_state(), [row.install_id] ) if not results or not results[0]["ok"]: result.unavailable_skill_ids.update(advertised_skills & allowed_skills) @@ -146,7 +149,7 @@ def record_component(entry, required): ) inspector.visit({"kind": "plugin", "id": row.install_id}, row.profile_id) report = inspector.report() - from core.capabilities.errors import IntegrityFailed, PackageMissing + from core.harness.capability_errors import IntegrityFailed, PackageMissing nodes = [node for node in report["nodes"] if node["kind"] in ("plugin", "agent")] for error in report["errors"]: @@ -240,10 +243,10 @@ def inspect_plugin_or_skip(prepared): ) unavailable_invoked = result.unavailable_skill_ids.intersection(invoked_skill_ids or ()) if unavailable_invoked: - from core.capabilities.errors import PackageMissing + from core.harness.capability_errors import PackageMissing raise PackageMissing( "selected skill belongs to a plugin with unavailable MCP servers", runtime_name=", ".join(sorted(unavailable_invoked)), ) - return plugin_resolution._finalize_resolution(result, eligible) + return plugin_resolution.finalize_resolution(result, eligible) diff --git a/src/backend/features/desktop/run_capabilities/__init__.py b/src/backend/features/desktop/run_capabilities/__init__.py new file mode 100644 index 00000000..8bed212b --- /dev/null +++ b/src/backend/features/desktop/run_capabilities/__init__.py @@ -0,0 +1 @@ +"""run capabilities package.""" diff --git a/src/backend/core/llm/factory/selection/device_preparation.py b/src/backend/features/desktop/run_capabilities/preparation.py similarity index 83% rename from src/backend/core/llm/factory/selection/device_preparation.py rename to src/backend/features/desktop/run_capabilities/preparation.py index 29e5bd81..1eff6ba5 100644 --- a/src/backend/core/llm/factory/selection/device_preparation.py +++ b/src/backend/features/desktop/run_capabilities/preparation.py @@ -1,11 +1,13 @@ """Agent assembly phase: device preparation. """ from __future__ import annotations +from core.capabilities.skills import loader_factory as loader_factory_module +from core.capabilities.plugins.runtime.models import progressive_plugin_loading_enabled import asyncio -import core.agent_skills.loader as skill_loader -from core.capabilities.paths import capabilities_enabled +import core.capabilities.skills.loader as skill_loader +from features.desktop.capabilities.paths import capabilities_enabled from core.llm.factory.models import RequiredCapabilities, StickyCapabilities from core.llm.factory.request import AgentRequest from core.llm.factory.selection import capabilities as factory_capabilities @@ -37,7 +39,7 @@ async def device_preparation( and _api_scope is None and _eval_scope is None ): - from core.plugins import runtime as _desktop_plugins + from features.desktop import plugin_resolution as _desktop_plugins _desktop_progressive = await asyncio.to_thread( _desktop_plugins.resolve_desktop_progressive_plugins, @@ -45,7 +47,7 @@ async def device_preparation( enabled_skill_ids=( request.enabled_skill_ids if request.enabled_skill_ids is not None - else factory_capabilities._effective_main_available_skills() + else factory_capabilities.effective_main_available_skills() ), enabled_mcp_ids=[ key @@ -71,33 +73,33 @@ async def device_preparation( for sid in ( request.enabled_skill_ids if request.enabled_skill_ids is not None - else factory_capabilities._effective_main_available_skills() + else factory_capabilities.effective_main_available_skills() ) if sid not in _desktop_progressive.unavailable_skill_ids ] if ( not _desktop_progressive.directory or request.turbo_mode - or not _desktop_plugins.progressive_plugin_loading_enabled() + or not progressive_plugin_loading_enabled() ): _desktop_progressive = None - from core.capabilities import runtime as capability_runtime + from features.desktop.capabilities import runtime as capability_runtime _log.info("[factory] +%s progressive definitions resolved", _elapsed()) _prepared_capabilities = None if capabilities_enabled() and not request.disable_tools: - _caps_loader = skill_loader.get_skill_loader() + _caps_loader = loader_factory_module.get_skill_loader() _caps_skill_ids = ( request.enabled_skill_ids if request.enabled_skill_ids is not None - else factory_capabilities._effective_main_available_skills() + else factory_capabilities.effective_main_available_skills() ) def _materialize_selected_skills(): for _sid in _caps_skill_ids or []: - from core.capabilities.errors import CapabilityError + from core.harness.capability_errors import CapabilityError try: _caps_loader.get_skill_dir(_sid) diff --git a/src/backend/features/desktop/run_capabilities/provider.py b/src/backend/features/desktop/run_capabilities/provider.py new file mode 100644 index 00000000..7a40e13d --- /dev/null +++ b/src/backend/features/desktop/run_capabilities/provider.py @@ -0,0 +1,118 @@ +"""Desktop implementation of the kernel's per-run capability contracts.""" + + +class DesktopRunCapabilityProvider: + def cloud_plugin_selection(self, plugin_id, user_id): + from features.desktop.capabilities.invocation import cloud_plugin_selection + return cloud_plugin_selection(plugin_id, user_id=user_id) + + def resolve_explicit_ids(self, user_id, skill_ids, mcp_ids): + from features.desktop.capabilities.invocation import resolve_explicit_ids + return resolve_explicit_ids(user_id, skill_ids, mcp_ids) + + def plugin_skill_defaults(self, user_id, skill_ids): + from features.desktop.plugin_resolution import prepare_desktop_plugin_skill_defaults + return prepare_desktop_plugin_skill_defaults(user_id, skill_ids) + + def ensure_ready(self, user_id, *, skill_keys=None, plugin_keys=None): + from features.desktop.capabilities.preparation import ensure_cloud_ready + return ensure_cloud_ready(user_id, skill_keys=skill_keys or (), plugin_keys=plugin_keys or ()) + + def available_skill_names(self, names, *, user_id): + from features.desktop.capabilities.skills import filter_available_names + return filter_available_names(names, user_id=user_id) + + def plugin_binding_ids(self, plugin_ids, *, user_id): + from features.desktop.capabilities.plugins import cloud_binding_ids + return cloud_binding_ids(plugin_ids, user_id=user_id) + + async def prepare(self, request, **selection): + from features.desktop.run_capabilities.preparation import device_preparation + return await device_preparation(request, **selection) + + +class PreparedCapabilityOperations: + def skill_loader(self): + from features.desktop.capabilities.runtime import frozen_loader + return frozen_loader(self) + + def bind_mcp(self, servers, resolution): + from features.desktop.capabilities.runtime import bind_mcp + return bind_mcp(self, servers, resolution) + + def preflight(self, **context): + from features.desktop.capabilities.runtime import preflight + return preflight(self, **context) + + def tool_guard(self, *, scope_id): + from features.desktop.capabilities.runtime import get + run = get(self.run_id, scope_id=scope_id) or self + return run.validate_tool_call + + def validate_selected_skills(self, skill_ids): + from core.harness.capability_errors import PackageMissing + missing = set(skill_ids or []) - set(self.bindings) + if missing: + raise PackageMissing("selected skills are absent from this prepared run", details={"skills": sorted(missing)}) + + def validate(self, *, user_id): + from features.desktop.capabilities.runtime import validate + return validate(self, user_id=user_id) + + def record_tool_scope(self, tool_call_id, tool_name): + from features.desktop.capabilities.runtime import record_tool_scope + return record_tool_scope(self, tool_call_id, tool_name) + + def child_scope(self, kind, *identities): + from core.harness.registry.capability_runtime import child_scope + return child_scope(self.scope_id, kind, *identities) + + def validate_tool_call(self, server_id): + from features.desktop.capabilities import runtime + from core.harness.capability_errors import CapabilityError + from core.capabilities.identity_rules import LOCAL_PROFILE + try: + runtime.validate(self) + latest = runtime.get(self.run_id, scope_id=self.scope_id) or self + if "mcp:" + server_id in latest.unavailable: + return False + binding = self.mcp_bindings.get(server_id) + if binding is not None: + from features.desktop.capabilities import registry + + relationships = latest.dependency_report.get("connector_parents", {}) + parents = { + *relationships.get(server_id, []), + *relationships.get(binding["install_id"].split(":", 2)[2], []), + } + for parent_id in parents: + parent = registry.get(parent_id) + if parent is None or not parent.enabled or parent.state == "removed": + return False + owner = parent.payload.get("owner_user_id") + if owner and str(owner) != self.user_id: + return False + if binding is None or not binding.get("authorization_checked"): + return True + _, profile, key = binding["install_id"].split(":", 2) + if profile == "local-json": + from features.desktop.capabilities.mcp_json import local_server_configs + + return key in local_server_configs() + if profile == LOCAL_PROFILE: + from core.capabilities.mcp.service import McpServerConfigService + + service = McpServerConfigService.get_instance() + return key in { + **service.get_all_servers(enabled_only=True), + **service.get_owned_servers(self.user_id, enabled_only=True), + } + from features.desktop.bridge.gateway import cloud_gateway_mcp_configs + + return server_id in cloud_gateway_mcp_configs([server_id]) + except (CapabilityError, OSError): + return False + + def release(self): + """Run snapshots remain durable with history under the existing retention policy.""" + return None diff --git a/src/backend/core/agent_skills/backends/capability_store.py b/src/backend/features/desktop/skill_backend.py similarity index 86% rename from src/backend/core/agent_skills/backends/capability_store.py rename to src/backend/features/desktop/skill_backend.py index e43d2f20..ac9eca43 100644 --- a/src/backend/core/agent_skills/backends/capability_store.py +++ b/src/backend/features/desktop/skill_backend.py @@ -11,8 +11,8 @@ from pathlib import Path from typing import Any, Dict, List, Optional, Tuple -from .filesystem import FilesystemBackend -from .protocol import SkillFileInfo +from core.capabilities.skills.backends.filesystem import FilesystemBackend +from core.capabilities.skills.backends.protocol import SkillFileInfo class CapabilityStoreBackend: @@ -29,7 +29,7 @@ def priority(self) -> int: return self._priority def _profile(self) -> Optional[str]: - from core.capabilities.skills import current_account_profile + from features.desktop.capabilities.skill_account import current_account_profile return "local" if self._local else current_account_profile() @@ -37,13 +37,13 @@ def _scope_user(self) -> Optional[str]: """The account a private local entry must belong to (shared entries: nobody).""" if not self._local: return None - from core.capabilities.skills import current_local_user_id + from features.desktop.capabilities.skill_account import current_local_user_id return current_local_user_id() def _installations(self): - from core.capabilities import registry - from core.capabilities.paths import KIND_SKILL + from features.desktop.capabilities import registry + from core.capabilities.identity_rules import KIND_SKILL profile = self._profile() if not profile: @@ -62,8 +62,8 @@ def change_token(self) -> Tuple[Any, ...]: Read on every skill lookup, so it must not repeat the listing it guards: the registry answers "did anything move" with one aggregate. """ - from core.capabilities import registry - from core.capabilities.paths import KIND_SKILL + from features.desktop.capabilities import registry + from core.capabilities.identity_rules import KIND_SKILL profile = self._profile() if not profile: @@ -71,8 +71,8 @@ def change_token(self) -> Tuple[Any, ...]: return (profile, self._scope_user(), registry.source_token(kind=KIND_SKILL, profile_id=profile)) def _dir_for(self, inst) -> Optional[Path]: - from core.capabilities import store - from core.capabilities.paths import KIND_SKILL + from features.desktop.capabilities import store + from core.capabilities.identity_rules import KIND_SKILL comp = store.get(KIND_SKILL, inst.profile_id, inst.key, inst.resolved_revision or "") return comp.path if comp else None diff --git a/src/backend/features/evaluation/README.md b/src/backend/features/evaluation/README.md new file mode 100644 index 00000000..52237227 --- /dev/null +++ b/src/backend/features/evaluation/README.md @@ -0,0 +1,7 @@ +# 评测执行隔离 + +提供 eval_ 会话的受限执行上下文、工具和独立沙箱租约。对外保留评测沙箱 API、admission 与 sandboxes 服务；runtime 管理执行范围，sandbox_provider 管理租约授权、进程和容器操作。 + +`extension.setup` 通过现有 register_sandbox_route 注册 eval_ 前缀。每次选择提供方都核对 OpenSandbox 配置，禁止回退到普通沙箱；扩展未注册时保留拒绝策略。FULL 与 CE 均安装本模块。租约使用已有临时状态接口，无新增 ORM 表。 + +依赖内核的沙箱协议、临时状态、配置、工具和运行取消接口；没有向其他 features 模块的依赖。直接运行取消引用按计划 P6c 的取消注册点处理，当前批次保持原行为。 diff --git a/src/backend/features/evaluation/__init__.py b/src/backend/features/evaluation/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/src/backend/features/evaluation/__init__.py @@ -0,0 +1 @@ + diff --git a/src/backend/core/services/evaluation_admission.py b/src/backend/features/evaluation/admission.py similarity index 93% rename from src/backend/core/services/evaluation_admission.py rename to src/backend/features/evaluation/admission.py index d1297c13..ed0634c3 100644 --- a/src/backend/core/services/evaluation_admission.py +++ b/src/backend/features/evaluation/admission.py @@ -2,7 +2,7 @@ from fastapi import HTTPException from core.config.settings import settings -import core.sandbox.providers.evaluation.binding as evaluation_binding +import features.evaluation.sandbox_provider.binding as evaluation_binding from core.sandbox.errors import SandboxError diff --git a/src/backend/features/evaluation/api/__init__.py b/src/backend/features/evaluation/api/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/src/backend/features/evaluation/api/__init__.py @@ -0,0 +1 @@ + diff --git a/src/backend/api/routes/v1/evaluation_sandboxes.py b/src/backend/features/evaluation/api/evaluation_sandboxes.py similarity index 98% rename from src/backend/api/routes/v1/evaluation_sandboxes.py rename to src/backend/features/evaluation/api/evaluation_sandboxes.py index 02d4d340..486a2b03 100644 --- a/src/backend/api/routes/v1/evaluation_sandboxes.py +++ b/src/backend/features/evaluation/api/evaluation_sandboxes.py @@ -45,7 +45,7 @@ class ExecCommand(BaseModel): async def _call(name: str, *args, **kwargs): - from core.services import evaluation_sandbox_service as service + from features.evaluation import sandboxes as service from core.sandbox.errors import SandboxError try: return await getattr(service, name)(*args, **kwargs) diff --git a/src/backend/features/evaluation/extension.py b/src/backend/features/evaluation/extension.py new file mode 100644 index 00000000..0bd30058 --- /dev/null +++ b/src/backend/features/evaluation/extension.py @@ -0,0 +1,23 @@ +"""Reserved evaluation sessions use a strict OpenSandbox provider.""" +from core.harness.registry import register_sandbox_route, sandbox_route + +NAME = "evaluation" + + +def evaluation_provider(): + from core.config.settings import settings + from core.sandbox.errors import SandboxError + from features.evaluation.sandbox_provider.provider import get_evaluation_provider + + if settings.sandbox.provider != "opensandbox": + raise SandboxError("Evaluation requires OpenSandbox; fallback is forbidden") + return get_evaluation_provider() + + +def setup(): + from core.harness.registry.evaluation import register_evaluation_policy + from features.evaluation.policy import EvaluationPolicy + register_evaluation_policy(EvaluationPolicy()) + if sandbox_route("eval_") is None: + register_sandbox_route("eval_", evaluation_provider) + return True diff --git a/src/backend/features/evaluation/models.py b/src/backend/features/evaluation/models.py new file mode 100644 index 00000000..132640c9 --- /dev/null +++ b/src/backend/features/evaluation/models.py @@ -0,0 +1 @@ +"""Evaluation leases use ephemeral state and add no ORM models.""" diff --git a/src/backend/features/evaluation/policy.py b/src/backend/features/evaluation/policy.py new file mode 100644 index 00000000..3444b07d --- /dev/null +++ b/src/backend/features/evaluation/policy.py @@ -0,0 +1,27 @@ +"""Bind the evaluation feature to neutral execution-policy hooks.""" +from features.evaluation.runtime import runtime + + +class EvaluationPolicy: + def apply_context(self, context): + return runtime.apply_evaluation_scope(context) + + def parse_scope(self, **identity): + return runtime.parse_evaluation_scope(**identity) + + def project_agent(self, agent): + return runtime.evaluation_user_agent(agent) + + def current_paths(self): + if runtime.CURRENT_EVALUATION_SCOPE.get() is not None: + from features.evaluation.runtime import paths + return paths + return None + + def register_tools(self, toolkit, scope, **options): + from features.evaluation.runtime.tools import register_evaluation_tools + register_evaluation_tools(toolkit, scope, **options) + + async def authorize(self, request, user_id, *, agent_scoped=False): + from features.evaluation.admission import authorize + await authorize(request, user_id, agent_scoped=agent_scoped) diff --git a/src/backend/features/evaluation/runtime/__init__.py b/src/backend/features/evaluation/runtime/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/src/backend/features/evaluation/runtime/__init__.py @@ -0,0 +1 @@ + diff --git a/src/backend/core/llm/evaluation_paths.py b/src/backend/features/evaluation/runtime/paths.py similarity index 100% rename from src/backend/core/llm/evaluation_paths.py rename to src/backend/features/evaluation/runtime/paths.py diff --git a/src/backend/core/llm/evaluation_process_tools.py b/src/backend/features/evaluation/runtime/process_tools.py similarity index 87% rename from src/backend/core/llm/evaluation_process_tools.py rename to src/backend/features/evaluation/runtime/process_tools.py index f766a5e7..2f0b7adf 100644 --- a/src/backend/core/llm/evaluation_process_tools.py +++ b/src/backend/features/evaluation/runtime/process_tools.py @@ -1,5 +1,6 @@ """Process tools owned by an evaluation lease, without ordinary-chat state.""" -from core.llm.tools._tool_helpers import _resp_json +from core.llm.factory.tools.definitions import function_tool +from core.llm.tools._tool_helpers import resp_json as _resp_json def register_evaluation_process_tools(toolkit, scope): """An evaluation controller owns completion and destroys its task sandbox.""" @@ -27,5 +28,5 @@ async def write_stdin(session_id: str, chars: str = "", chars=chars, yield_time_ms=yield_time_ms) return _resp_json(result) - toolkit.register_tool_function(write_stdin, namesake_strategy="override") - toolkit.register_tool_function(Bash, namesake_strategy="override") + toolkit.append(function_tool(write_stdin)) + toolkit.append(function_tool(Bash)) diff --git a/src/backend/core/llm/evaluation_runtime.py b/src/backend/features/evaluation/runtime/runtime.py similarity index 100% rename from src/backend/core/llm/evaluation_runtime.py rename to src/backend/features/evaluation/runtime/runtime.py diff --git a/src/backend/core/llm/evaluation_tools.py b/src/backend/features/evaluation/runtime/tools.py similarity index 57% rename from src/backend/core/llm/evaluation_tools.py rename to src/backend/features/evaluation/runtime/tools.py index 5d6c278b..6620b8fa 100644 --- a/src/backend/core/llm/evaluation_tools.py +++ b/src/backend/features/evaluation/runtime/tools.py @@ -2,7 +2,7 @@ from functools import wraps from inspect import signature -from core.llm.evaluation_runtime import CURRENT_EVALUATION_SCOPE +from features.evaluation.runtime.runtime import CURRENT_EVALUATION_SCOPE _DESCRIPTIONS = { @@ -16,36 +16,31 @@ } -class _EvaluationCollector: - def __init__(self, target, scope): - self.target = target - self.scope = scope +def scoped_tool(tool, scope): + from agentscope.tool import FunctionTool + from core.llm.factory.tools.definitions import function_tool + func = tool._func + original_signature = signature(func) - def register_tool_function(self, func, **options): - original_signature = signature(func) + @wraps(func) + async def scoped(*args, **kwargs): + from core.config.local_mode import local_mode_enabled + from core.llm.tools._common import resp_json + if local_mode_enabled(): + return resp_json({"error": "Evaluation requires container execution, not local mode"}) + bound = original_signature.bind(*args, **kwargs) + if bound.arguments.get("register_as_artifact"): + return resp_json({"error": "Evaluation outputs must remain in the task sandbox"}) + token = CURRENT_EVALUATION_SCOPE.set(scope) + try: + return await func(*args, **kwargs) + finally: + CURRENT_EVALUATION_SCOPE.reset(token) - @wraps(func) - async def scoped(*args, **kwargs): - from core.config.local_mode import local_mode_enabled - from core.llm.tools._common import resp_json - if local_mode_enabled(): - return resp_json({"error": "Evaluation requires container execution, not local mode"}) - bound = original_signature.bind(*args, **kwargs) - if bound.arguments.get("register_as_artifact"): - return resp_json({"error": "Evaluation outputs must remain in the task sandbox"}) - token = CURRENT_EVALUATION_SCOPE.set(self.scope) - try: - return await func(*args, **kwargs) - finally: - CURRENT_EVALUATION_SCOPE.reset(token) - - # Tool schemas retain the native parameters and behavioral contracts. - scoped.__doc__ = ( - _DESCRIPTIONS.get(func.__name__, "") + " " - f"Native {func.__name__} in the current evaluation sandbox only. " - "No account workspace, historical artifacts, or host filesystem access." - ) - self.target.register_tool_function(scoped, **options) + scoped.__doc__ = (_DESCRIPTIONS.get(tool.name, "") + " " + + f"Native {tool.name} in the current evaluation sandbox only. " + + "No account workspace, historical artifacts, or host filesystem access.") + return function_tool(scoped, dispatch_spec=tool.dispatch_spec) def register_evaluation_tools(toolkit, scope, *, read_only=False, allow_bash=True, vision_mode="none"): @@ -56,7 +51,7 @@ def register_evaluation_tools(toolkit, scope, *, read_only=False, allow_bash=Tru ) if local_mode_enabled(): raise ValueError("Evaluation requires container execution, not local mode") - collector = _EvaluationCollector(toolkit, scope) + collector = [] common = dict( chat_id=scope.session_id, sandbox_session_id=scope.session_id, user_id=scope.owner_user_id, @@ -64,7 +59,7 @@ def register_evaluation_tools(toolkit, scope, *, read_only=False, allow_bash=Tru token = CURRENT_EVALUATION_SCOPE.set(scope) try: if not read_only and allow_bash: - from core.llm.evaluation_process_tools import register_evaluation_process_tools + from features.evaluation.runtime.process_tools import register_evaluation_process_tools register_evaluation_process_tools(collector, scope) state = ReadStateTracker() register_read(collector, **common, state=state, vision_mode=vision_mode) @@ -75,3 +70,5 @@ def register_evaluation_tools(toolkit, scope, *, read_only=False, allow_bash=Tru register_edit(collector, **common, state=state, interactive=False) finally: CURRENT_EVALUATION_SCOPE.reset(token) + + toolkit.extend(scoped_tool(tool, scope) for tool in collector) diff --git a/src/backend/core/sandbox/providers/evaluation/__init__.py b/src/backend/features/evaluation/sandbox_provider/__init__.py similarity index 100% rename from src/backend/core/sandbox/providers/evaluation/__init__.py rename to src/backend/features/evaluation/sandbox_provider/__init__.py diff --git a/src/backend/core/sandbox/providers/evaluation/binding.py b/src/backend/features/evaluation/sandbox_provider/binding.py similarity index 100% rename from src/backend/core/sandbox/providers/evaluation/binding.py rename to src/backend/features/evaluation/sandbox_provider/binding.py diff --git a/src/backend/core/sandbox/providers/evaluation/process_guard.py b/src/backend/features/evaluation/sandbox_provider/process_guard.py similarity index 100% rename from src/backend/core/sandbox/providers/evaluation/process_guard.py rename to src/backend/features/evaluation/sandbox_provider/process_guard.py diff --git a/src/backend/core/sandbox/providers/evaluation/provider.py b/src/backend/features/evaluation/sandbox_provider/provider.py similarity index 98% rename from src/backend/core/sandbox/providers/evaluation/provider.py rename to src/backend/features/evaluation/sandbox_provider/provider.py index 57ef7664..76d807fb 100644 --- a/src/backend/core/sandbox/providers/evaluation/provider.py +++ b/src/backend/features/evaluation/sandbox_provider/provider.py @@ -13,7 +13,7 @@ from contextlib import asynccontextmanager from datetime import timedelta -import core.sandbox.providers.evaluation.binding as binding +import features.evaluation.sandbox_provider.binding as binding from core.sandbox.errors import SandboxConnectError, SandboxError from core.sandbox.protocol import ProcessResult, SandboxAdminCapabilities @@ -290,7 +290,7 @@ async def freeze(self, session_id, owner_user_id, *, timeout=65): await asyncio.sleep(0.1) if status.running is not False: raise SandboxError("Evaluation command stop is unconfirmed") - from core.sandbox.providers.evaluation.process_guard import clear_processes + from features.evaluation.sandbox_provider.process_guard import clear_processes await clear_processes(sandbox, item.protected_processes) return await binding.finish_freeze(session_id, owner_user_id) diff --git a/src/backend/core/services/evaluation_sandbox_service.py b/src/backend/features/evaluation/sandboxes.py similarity index 93% rename from src/backend/core/services/evaluation_sandbox_service.py rename to src/backend/features/evaluation/sandboxes.py index e949147d..d60796c9 100644 --- a/src/backend/core/services/evaluation_sandbox_service.py +++ b/src/backend/features/evaluation/sandboxes.py @@ -31,7 +31,7 @@ def session_id(lease_id: str) -> str: async def _owned(lease_id, owner, *, allow_expired=False): - import core.sandbox.providers.evaluation.binding as bindings + import features.evaluation.sandbox_provider.binding as bindings return await bindings.assert_owner(session_id(lease_id), owner, allow_expired=allow_expired) @@ -72,10 +72,10 @@ def _safe_attempt_id(attempt_id: str) -> str: async def create(owner, *, image, attempt_id="", ttl_seconds=7200, cpu_count=2, memory_mb=4096): + import features.evaluation.sandbox_provider.binding as bindings + from features.evaluation.sandbox_provider.process_guard import capture_baseline from opensandbox import Sandbox from opensandbox.models.execd import RunCommandOpts - import core.sandbox.providers.evaluation.binding as bindings - from core.sandbox.providers.evaluation.process_guard import capture_baseline if settings.sandbox.provider != "opensandbox": raise SandboxError("opensandbox_required") sandbox = await Sandbox.create( @@ -166,11 +166,11 @@ async def download(lease_id, owner, path): async def freeze(lease_id, owner): - import core.sandbox.providers.evaluation.binding as bindings - from core.sandbox.providers.evaluation.provider import get_evaluation_provider + import features.evaluation.sandbox_provider.binding as bindings from core.db.engine import SessionLocal from core.db.models import ChatRun - from orchestration.chat_run_executor import cancel_run + from core.harness.registry.run_cancellation import cancel_run + from features.evaluation.sandbox_provider.provider import get_evaluation_provider sid = session_id(lease_id) await bindings.assert_owner(sid, owner) await bindings.begin_freeze(sid, owner) @@ -186,9 +186,9 @@ async def freeze(lease_id, owner): async def destroy(lease_id, owner): + import features.evaluation.sandbox_provider.binding as bindings + from features.evaluation.sandbox_provider.provider import get_evaluation_provider from opensandbox.exceptions import SandboxApiException - import core.sandbox.providers.evaluation.binding as bindings - from core.sandbox.providers.evaluation.provider import get_evaluation_provider binding = await _owned(lease_id, owner, allow_expired=True) if binding.destroyed: return {"phase": "closed"} diff --git a/src/backend/features/hosted_apps/README.md b/src/backend/features/hosted_apps/README.md new file mode 100644 index 00000000..bd07894b --- /dev/null +++ b/src/backend/features/hosted_apps/README.md @@ -0,0 +1,9 @@ +# 托管应用 + +管理独立应用数据库、应用项目、集合、版本、发布、MCP 与内部数据接口，以及插件应用资源的准入和清理。HTTP 路由由中央路由表保持原顺序及权限策略；数据库配置通过 config 提供，独立初始化命令为 `python -m features.hosted_apps.hosting_setup`。 + +应用数据表仍使用独立 store.metadata，不并入平台 Base。模块通过同步 schema 生命周期回调初始化受管理的本机 SQLite 应用库，执行位置保持在站点存储退役之后、平台 schema 调和之前。API、CLI 和容器入口均注册此回调，注册幂等；初始化异常仍阻断启动。云端数据库必须显式配置及预置，不自动创建。桥接模式的跳过条件保持不变。 + +公开入口为 data、projects、publication、deployments、store、schema 等实际模块。允许 hosted_apps → sites：应用发布关联站点。当前桌面转发仍引用原始桌面桥接实现，属于 routes.json 虚拟归属识别的历史依赖，P5 须通过公开适配接口消除反向循环。已删除废弃 application_site_conversion，不保留兼容转发。 + +依赖 features.desktop 提供本机认证及能力文件锁，桌面不反向依赖托管应用。执行位置由宿主策略接口提供。 diff --git a/src/backend/features/hosted_apps/__init__.py b/src/backend/features/hosted_apps/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/src/backend/features/hosted_apps/__init__.py @@ -0,0 +1 @@ + diff --git a/src/backend/features/hosted_apps/api/__init__.py b/src/backend/features/hosted_apps/api/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/src/backend/features/hosted_apps/api/__init__.py @@ -0,0 +1 @@ + diff --git a/src/backend/api/routes/v1/application_transports.py b/src/backend/features/hosted_apps/api/application_transports.py similarity index 79% rename from src/backend/api/routes/v1/application_transports.py rename to src/backend/features/hosted_apps/api/application_transports.py index 4ca4c287..48d8c6af 100644 --- a/src/backend/api/routes/v1/application_transports.py +++ b/src/backend/features/hosted_apps/api/application_transports.py @@ -4,9 +4,9 @@ from core.db.engine import get_db from core.infra.responses import success_response -from core.services.application_data import ApplicationDataService -from core.services.application_operations import perform_operation -from core.services.application_schema import InternalMCPPublish, InternalOperation, RecordBatch +from features.hosted_apps.data import ApplicationDataService +from features.hosted_apps.operations import perform_operation +from features.hosted_apps.schema import InternalMCPPublish, InternalOperation, RecordBatch from fastapi import APIRouter, Depends, Header, HTTPException, Request from sqlalchemy import select from sqlalchemy.orm import Session @@ -23,13 +23,13 @@ async def internal_operation( x_internal_token: str | None = Header(None), db: Session = Depends(get_db), ): - from api.routes.v1.internal_site_auth import _check_internal_token + from features.sites.api.internal_site_auth import _check_internal_token _check_internal_token(x_internal_token) - from core.services.desktop_cloud_bridge import bridge_enabled + from core.harness.registry.host_runtime import requires_remote_execution as bridge_enabled if bridge_enabled(): - from core.services.desktop_site_publish import forward_local_site_tool + from features.sites.desktop_publish import forward_local_site_tool arguments = body.model_dump(exclude={"user_id", "chat_id"}) if not arguments["payload"]: @@ -52,12 +52,12 @@ async def internal_publish_mcp( x_internal_token: str | None = Header(None), db: Session = Depends(get_db), ): - from api.routes.v1.internal_site_auth import _check_internal_token - from core.services.desktop_cloud_bridge import bridge_enabled + from features.sites.api.internal_site_auth import _check_internal_token + from core.harness.registry.host_runtime import requires_remote_execution as bridge_enabled _check_internal_token(x_internal_token) if bridge_enabled(): - from core.services.desktop_site_publish import forward_local_site_tool + from features.sites.desktop_publish import forward_local_site_tool result = await forward_local_site_tool( "publish_mcp", @@ -66,8 +66,8 @@ async def internal_publish_mcp( chat_id=body.chat_id, ) else: - from core.services.application_projects import ApplicationProjectService - from core.services.application_publication import ApplicationPublicationService + from features.hosted_apps.projects import ApplicationProjectService + from features.hosted_apps.publication import ApplicationPublicationService publication = ApplicationPublicationService(db) result = await run_in_threadpool( @@ -84,14 +84,10 @@ async def internal_publish_mcp( @public_router.post("/{slug}/__api/data/{table}") async def visitor_insert(slug: str, table: str, request: Request, db: Session = Depends(get_db)): - from api.routes.sites_serve import ( - _api_json, - _client_ip, - _load_authorized_site, - ) - from core.services.application_store import applications - - from core.services.site_rate_limit import count_attempt + from features.sites.api.sites_serve import _api_json, _client_ip, _load_authorized_site + from features.hosted_apps.store import applications + + from features.sites.rate_limit import count_attempt await count_attempt(_client_ip(request), slug) site, gate = await _load_authorized_site(slug, request, db, require_unlock=True) if gate is not None: @@ -137,13 +133,9 @@ def submit(): async def visitor_collection( slug: str, table: str, request: Request, db: Session = Depends(get_db) ): - from api.routes.sites_serve import ( - _api_json, - _client_ip, - _load_authorized_site, - ) - from core.services.application_collections import read_collection, replace_collection - from core.services.application_store import applications + from features.sites.api.sites_serve import _api_json, _client_ip, _load_authorized_site + from features.hosted_apps.collections import read_collection, replace_collection + from features.hosted_apps.store import applications site, gate = await _load_authorized_site(slug, request, db, require_unlock=True) if gate is not None: @@ -153,7 +145,7 @@ async def visitor_collection( ) payload = None if request.method == "PUT": - from core.services.site_rate_limit import count_attempt + from features.sites.rate_limit import count_attempt await count_attempt(_client_ip(request), slug) body = bytearray() async for chunk in request.stream(): @@ -190,6 +182,6 @@ def operation(): @mcp_router.api_route("/{app_id}", methods=["GET", "POST", "DELETE", "OPTIONS"]) async def hosted_mcp(app_id: str, request: Request): - from core.services.application_mcp import ApplicationMCPResponse + from features.hosted_apps.mcp import ApplicationMCPResponse return ApplicationMCPResponse() diff --git a/src/backend/api/routes/v1/applications.py b/src/backend/features/hosted_apps/api/applications.py similarity index 83% rename from src/backend/api/routes/v1/applications.py rename to src/backend/features/hosted_apps/api/applications.py index 3aca0df6..2621e8d4 100644 --- a/src/backend/api/routes/v1/applications.py +++ b/src/backend/features/hosted_apps/api/applications.py @@ -7,20 +7,10 @@ from core.auth.backend import UserContext, get_current_user from core.db.engine import get_db from core.infra.responses import success_response -from core.services.application_data import ApplicationDataService -from core.services.application_operations import create_application -from core.services.application_publication import ( - ApplicationPublicationService, -) -from core.services.application_schema import ( - ApplicationDefinition, - MCPPublish, - MCPRollback, - PatchRecord, - RecordBatch, - RestoreCollection, - TableDefinition, -) +from features.hosted_apps.data import ApplicationDataService +from features.hosted_apps.operations import create_application +from features.hosted_apps.publication import ApplicationPublicationService +from features.hosted_apps.schema import ApplicationDefinition, MCPPublish, MCPRollback, PatchRecord, RecordBatch, RestoreCollection, TableDefinition from fastapi import APIRouter, Depends, HTTPException, Query from fastapi.responses import Response from sqlalchemy.orm import Session @@ -31,7 +21,7 @@ @router.get("") def list_applications(user: UserContext = Depends(get_current_user)): - from core.config.application_hosting import application_hosting_settings + from features.hosted_apps.config import application_hosting_settings from core.config.settings import settings if not application_hosting_settings.database_url and not settings.deploy.is_local: @@ -61,7 +51,7 @@ async def open_editor( user: UserContext = Depends(get_current_user), db: Session = Depends(get_db), ): - from core.services.application_projects import ApplicationProjectService + from features.hosted_apps.projects import ApplicationProjectService service = ApplicationDataService() projects = ApplicationProjectService(db, service.engine) @@ -133,7 +123,7 @@ def export_records( table: str, user: UserContext = Depends(get_current_user), ): - from core.services.application_export import export_csv + from features.hosted_apps.export import export_csv from fastapi.responses import StreamingResponse return StreamingResponse( export_csv(ApplicationDataService(), app_id, user.user_id, table), @@ -149,7 +139,7 @@ async def publish_mcp( user: UserContext = Depends(get_current_user), db: Session = Depends(get_db), ): - from core.services.application_projects import ApplicationProjectService + from features.hosted_apps.projects import ApplicationProjectService publication = ApplicationPublicationService(db) receipt = await run_in_threadpool(publication.publish, app_id, user.user_id, body.tools) @@ -162,7 +152,7 @@ async def publish_project_mcp( user: UserContext = Depends(get_current_user), db: Session = Depends(get_db), ): - from core.services.application_projects import ApplicationProjectService + from features.hosted_apps.projects import ApplicationProjectService publication = ApplicationPublicationService(db) receipt = await run_in_threadpool(publication.publish, app_id, user.user_id, from_project=True) @@ -183,7 +173,7 @@ async def rollback_mcp( user: UserContext = Depends(get_current_user), db: Session = Depends(get_db), ): - from core.services.application_projects import ApplicationProjectService + from features.hosted_apps.projects import ApplicationProjectService publication = ApplicationPublicationService(db) receipt = await run_in_threadpool(publication.rollback, app_id, user.user_id, body.version) @@ -193,36 +183,36 @@ async def rollback_mcp( def erase_record(app_id: str, table: str, record_id: str, version: int = Query(..., ge=1), user: UserContext = Depends(get_current_user)): - from core.services.application_lifecycle import delete_record + from features.hosted_apps.lifecycle import delete_record return success_response(data=delete_record(ApplicationDataService(), app_id, user.user_id, table, record_id, version)) @router.delete("/{app_id}/tables/{table}") def erase_table(app_id: str, table: str, user: UserContext = Depends(get_current_user)): - from core.services.application_lifecycle import delete_table + from features.hosted_apps.lifecycle import delete_table return success_response(data=delete_table(ApplicationDataService(), app_id, user.user_id, table)) @router.delete("/{app_id}") def erase_application(app_id: str, user: UserContext = Depends(get_current_user), db: Session = Depends(get_db)): - from core.services.application_lifecycle import delete_application + from features.hosted_apps.lifecycle import delete_application return success_response(data=delete_application(ApplicationDataService(), db, app_id, user.user_id)) @router.get("/{app_id}/tables/{table}/history") def recovery_history(app_id: str, table: str, user: UserContext = Depends(get_current_user)): - from core.services.application_history import list_history + from features.hosted_apps.history import list_history return success_response(data=list_history(ApplicationDataService(), app_id, user.user_id, table)) @router.post("/{app_id}/tables/{table}/history/{history_id}/restore") def restore_collection(app_id: str, table: str, history_id: str, body: RestoreCollection, user: UserContext = Depends(get_current_user)): - from core.services.application_history import restore + from features.hosted_apps.history import restore return success_response(data=restore(ApplicationDataService(), app_id, user.user_id, table, history_id, body.revision)) @router.get("/{app_id}/tables/{table}/collection") def owner_collection(app_id: str, table: str, user: UserContext = Depends(get_current_user)): - from core.services.application_collections import snapshot - from core.services.application_store import owned_application - from core.services.application_relational import table_definition - from core.services.application_sql_policy import application_role + from features.hosted_apps.collections import snapshot + from features.hosted_apps.store import owned_application + from features.hosted_apps.relational import table_definition + from features.hosted_apps.sql_policy import application_role service = ApplicationDataService() with service.engine.begin() as connection: app = owned_application(connection, app_id, user.user_id) diff --git a/src/backend/core/services/application_collections.py b/src/backend/features/hosted_apps/collections.py similarity index 92% rename from src/backend/core/services/application_collections.py rename to src/backend/features/hosted_apps/collections.py index 5c36744f..f9886a46 100644 --- a/src/backend/core/services/application_collections.py +++ b/src/backend/features/hosted_apps/collections.py @@ -5,9 +5,9 @@ from uuid import uuid4 from core.infra.time import utc_now -from core.services.application_relational import json_record, table_definition, validated_row -from core.services.application_sql_policy import application_role -from core.services.application_store import applications, owned_application, write_connection +from features.hosted_apps.relational import json_record, table_definition, validated_row +from features.hosted_apps.sql_policy import application_role +from features.hosted_apps.store import applications, owned_application, write_connection from fastapi import HTTPException from sqlalchemy import delete, func, insert, select from sqlalchemy.exc import IntegrityError @@ -89,7 +89,7 @@ def replace_collection(service, app_id, owner, name, revision, rows, *, owner_re raise HTTPException(409, "Collection changed; reload before saving") if not previous["items"] and not rows: return previous - from core.services.application_history import archive + from features.hosted_apps.history import archive archive(connection, app_id, name, previous["items"]) with application_role(connection, app_id): connection.execute(delete(table)) diff --git a/src/backend/core/config/application_hosting.py b/src/backend/features/hosted_apps/config.py similarity index 87% rename from src/backend/core/config/application_hosting.py rename to src/backend/features/hosted_apps/config.py index 2cb8e9d9..664d2269 100644 --- a/src/backend/core/config/application_hosting.py +++ b/src/backend/features/hosted_apps/config.py @@ -2,7 +2,7 @@ from dataclasses import dataclass, field -from core.config.settings import _env +from core.config.settings import env_value as _env @dataclass(frozen=True) diff --git a/src/backend/core/services/application_data.py b/src/backend/features/hosted_apps/data.py similarity index 95% rename from src/backend/core/services/application_data.py rename to src/backend/features/hosted_apps/data.py index 8ceff56c..905da848 100644 --- a/src/backend/core/services/application_data.py +++ b/src/backend/features/hosted_apps/data.py @@ -7,22 +7,10 @@ from uuid import uuid4 from core.infra.time import utc_now -from core.services.application_relational import json_record, table_definition, validated_row -from core.services.application_schema import ApplicationDefinition, RecordBatch, TableDefinition -from core.services.application_sql_policy import ( - application_role, - application_table, - provision_table, -) -from core.services.application_store import ( - application_engine, - application_sources, - applications, - operations, - owned_application, - require_store, - write_connection, -) +from features.hosted_apps.relational import json_record, table_definition, validated_row +from features.hosted_apps.schema import ApplicationDefinition, RecordBatch, TableDefinition +from features.hosted_apps.sql_policy import application_role, application_table, provision_table +from features.hosted_apps.store import application_engine, application_sources, applications, operations, owned_application, require_store, write_connection from fastapi import HTTPException from sqlalchemy import func, insert, select, update from sqlalchemy.exc import IntegrityError diff --git a/src/backend/core/services/application_deployments.py b/src/backend/features/hosted_apps/deployments.py similarity index 92% rename from src/backend/core/services/application_deployments.py rename to src/backend/features/hosted_apps/deployments.py index 6eead246..7edfffe4 100644 --- a/src/backend/core/services/application_deployments.py +++ b/src/backend/features/hosted_apps/deployments.py @@ -5,16 +5,9 @@ import hashlib from core.infra.time import utc_now -from core.services.application_relational import table_definition -from core.services.application_schema import ToolDefinition -from core.services.application_store import ( - application_engine, - applications, - mcp_deployments, - owned_application, - require_store, - write_connection, -) +from features.hosted_apps.relational import table_definition +from features.hosted_apps.schema import ToolDefinition +from features.hosted_apps.store import application_engine, applications, mcp_deployments, owned_application, require_store, write_connection from fastapi import HTTPException from sqlalchemy import insert, select, update diff --git a/src/backend/core/services/application_export.py b/src/backend/features/hosted_apps/export.py similarity index 89% rename from src/backend/core/services/application_export.py rename to src/backend/features/hosted_apps/export.py index 91cc08b6..8a5c2ba5 100644 --- a/src/backend/core/services/application_export.py +++ b/src/backend/features/hosted_apps/export.py @@ -2,9 +2,9 @@ import csv import io import json -from core.services.application_store import owned_application -from core.services.application_relational import table_definition, json_record -from core.services.application_sql_policy import application_role +from features.hosted_apps.store import owned_application +from features.hosted_apps.relational import table_definition, json_record +from features.hosted_apps.sql_policy import application_role from sqlalchemy import select def csv_cell(value): diff --git a/src/backend/features/hosted_apps/extension.py b/src/backend/features/hosted_apps/extension.py new file mode 100644 index 00000000..deded1ce --- /dev/null +++ b/src/backend/features/hosted_apps/extension.py @@ -0,0 +1,17 @@ +"""Hosted application schema lifecycle composition.""" +NAME = "hosted_apps" + + +def initialize_schema(): + from features.hosted_apps.store import initialize_local_store + initialize_local_store() + + +def register_schema(): + from core.harness.registry.schema_lifecycle import register_schema_lifecycle + register_schema_lifecycle(NAME, initialize_schema) + + +def setup(): + register_schema() + return True diff --git a/src/backend/core/services/application_history.py b/src/backend/features/hosted_apps/history.py similarity index 91% rename from src/backend/core/services/application_history.py rename to src/backend/features/hosted_apps/history.py index 7286fd17..e3802569 100644 --- a/src/backend/core/services/application_history.py +++ b/src/backend/features/hosted_apps/history.py @@ -2,7 +2,7 @@ import json from uuid import uuid4 from core.infra.time import utc_now -from core.services.application_store import metadata, owned_application +from features.hosted_apps.store import metadata, owned_application from fastapi import HTTPException from sqlalchemy import Boolean, Column, DateTime, JSON, String, Table, delete, insert, select @@ -33,7 +33,7 @@ def archive(connection, app_id, name, rows): connection.execute(delete(collection_history).where(collection_history.c.id.in_(retained))) def list_history(service, app_id, owner, name): - from core.services.application_relational import table_definition, json_record + from features.hosted_apps.relational import table_definition, json_record with service.engine.connect() as connection: table_definition(owned_application(connection, app_id, owner), name) rows = connection.execute(select( @@ -43,8 +43,8 @@ def list_history(service, app_id, owner, name): return {"items": [json_record(row) for row in rows]} def restore(service, app_id, owner, name, history_id, revision): - from core.services.application_collections import replace_collection - from core.services.application_relational import table_definition + from features.hosted_apps.collections import replace_collection + from features.hosted_apps.relational import table_definition with service.engine.connect() as connection: definition = table_definition(owned_application(connection, app_id, owner), name) rows = connection.scalar(select(collection_history.c.rows).where( diff --git a/src/backend/core/services/application_hosting_setup.py b/src/backend/features/hosted_apps/hosting_setup.py similarity index 96% rename from src/backend/core/services/application_hosting_setup.py rename to src/backend/features/hosted_apps/hosting_setup.py index f40b5769..54febd57 100644 --- a/src/backend/core/services/application_hosting_setup.py +++ b/src/backend/features/hosted_apps/hosting_setup.py @@ -2,13 +2,13 @@ import os from sqlalchemy import create_engine, text from sqlalchemy.engine import make_url -from core.services.application_store import application_engine, initialize_store +from features.hosted_apps.store import application_engine, initialize_store def provision_owner(url): parsed = make_url(url) if parsed.get_backend_name() != "postgresql" or parsed.username != "application_admin": raise RuntimeError("Provisioning requires the dedicated application_admin connection") - from core.config.application_hosting import application_hosting_settings + from features.hosted_apps.config import application_hosting_settings from core.db.engine import DATABASE_URL target = make_url(application_hosting_settings.database_url) platform = make_url(DATABASE_URL) diff --git a/src/backend/core/services/application_industry.py b/src/backend/features/hosted_apps/industry.py similarity index 95% rename from src/backend/core/services/application_industry.py rename to src/backend/features/hosted_apps/industry.py index 20aaac74..47fa449d 100644 --- a/src/backend/core/services/application_industry.py +++ b/src/backend/features/hosted_apps/industry.py @@ -4,12 +4,7 @@ import httpx from core.infra.time import utc_now -from core.services.application_schema import ( - ColumnDefinition, - IndustryImport, - RecordBatch, - TableDefinition, -) +from features.hosted_apps.schema import ColumnDefinition, IndustryImport, RecordBatch, TableDefinition from fastapi import HTTPException @@ -41,7 +36,7 @@ def fetch_enterprises(keyword: str, limit: int) -> list[dict]: def import_enterprises(service, db, app_id: str, owner: str, payload: dict) -> dict: from core.config.settings import settings - from core.plugins.management.queries import list_installed + from core.capabilities.plugins.management.queries import list_installed if settings.edition.edition != "ee": raise HTTPException(403, "Industry knowledge center requires the enterprise edition") diff --git a/src/backend/core/services/application_lifecycle.py b/src/backend/features/hosted_apps/lifecycle.py similarity index 88% rename from src/backend/core/services/application_lifecycle.py rename to src/backend/features/hosted_apps/lifecycle.py index e90dc184..e3a48767 100644 --- a/src/backend/core/services/application_lifecycle.py +++ b/src/backend/features/hosted_apps/lifecycle.py @@ -1,11 +1,11 @@ """Owner deletion with publication serialization and complete quota reclamation.""" from fastapi import HTTPException from sqlalchemy import delete, select, text, update -from core.services.application_store import applications, operations, application_sources, mcp_deployments, owned_application, write_connection -from core.services.application_history import collection_history -from core.services.application_relational import table_definition -from core.services.application_sql_policy import application_role -from core.services.application_publication import publication_lock +from features.hosted_apps.store import applications, operations, application_sources, mcp_deployments, owned_application, write_connection +from features.hosted_apps.history import collection_history +from features.hosted_apps.relational import table_definition +from features.hosted_apps.sql_policy import application_role +from features.hosted_apps.publication import publication_lock def delete_record(service, app_id, owner, name, record_id, version): with write_connection(service.engine) as connection: @@ -39,7 +39,7 @@ def delete_table(service, app_id, owner, name): def delete_application(service, db, app_id, owner): from core.db.models import AdminMcpServer - from core.services.mcp_management_service import refresh_mcp_caches + from core.capabilities.mcp.management import refresh_mcp_caches with publication_lock(service.engine, app_id): with write_connection(service.engine) as connection: connection.execute(select(applications.c.id).where(applications.c.id == app_id).with_for_update()) @@ -60,7 +60,7 @@ def delete_application(service, db, app_id, owner): connection.execute(text(f'DROP ROLE IF EXISTS "app_role_{app["id"]}"')) else: for raw in app["tables"].values(): - from core.services.application_schema import TableDefinition + from features.hosted_apps.schema import TableDefinition service._table(app_id, TableDefinition.model_validate(raw)).drop(connection) for table in (operations, mcp_deployments, application_sources, collection_history): connection.execute(delete(table).where(table.c.app_id == app_id)) diff --git a/src/backend/core/services/application_mcp.py b/src/backend/features/hosted_apps/mcp.py similarity index 82% rename from src/backend/core/services/application_mcp.py rename to src/backend/features/hosted_apps/mcp.py index f616b9ce..a87e7ab6 100644 --- a/src/backend/core/services/application_mcp.py +++ b/src/backend/features/hosted_apps/mcp.py @@ -9,11 +9,12 @@ import re from typing import Annotated, Any -from core.services.application_data import ApplicationDataService -from core.services.application_schema import ToolDefinition -from core.services.application_store import applications +from features.hosted_apps.data import ApplicationDataService +from features.hosted_apps.schema import ToolDefinition +from features.hosted_apps.store import applications from fastapi import HTTPException -from mcp.server.fastmcp import FastMCP +from mcp.server.mcpserver import MCPServer +from mcp.server.mcpserver.exceptions import ToolError from mcp.types import ToolAnnotations from pydantic import Field from sqlalchemy import select @@ -58,8 +59,8 @@ def query(**arguments: Any) -> dict: offset=arguments.get("offset", 0), ) - # FastMCP needs an explicit typed signature rather than opaque **kwargs. - from core.services.application_relational import table_definition + # MCPServer needs an explicit typed signature rather than opaque **kwargs. + from features.hosted_apps.relational import table_definition fields = {c.name: c for c in table_definition(app, definition.table).columns} types = {"text": str, "integer": int, "number": float, "boolean": bool, "date": str} @@ -88,6 +89,20 @@ async def invoke(**arguments) -> dict[str, Any]: return invoke +class QueryMCPServer(MCPServer): + async def call_tool(self, name, arguments, context=None): + import jsonschema + + schemas = {tool.name: tool.input_schema for tool in await self.list_tools()} + if name in schemas: + schema = {**schemas[name], "additionalProperties": False} + try: + jsonschema.validate(arguments, schema) + except jsonschema.ValidationError: + raise ToolError("Invalid query arguments") from None + return await super().call_tool(name, arguments, context) + + class ApplicationMCPGateway: """Each HTTP call reloads persisted authorization, so revocation works across workers.""" @@ -112,7 +127,7 @@ async def __call__(self, scope, receive, send): {"detail": error.detail}, error.status_code, headers={"WWW-Authenticate": "Bearer"} )(scope, receive, send) return - server = FastMCP(app["title"], host="0.0.0.0", stateless_http=True, json_response=True) + server = QueryMCPServer(app["title"]) for raw in app["tools"]: definition = ToolDefinition.model_validate(raw) server.add_tool( @@ -126,10 +141,7 @@ async def __call__(self, scope, receive, send): openWorldHint=False, ), ) - tool = server._tool_manager.get_tool(definition.name) - tool.fn_metadata.arg_model.model_config["extra"] = "forbid" - tool.fn_metadata.arg_model.model_rebuild(force=True) - transport = server.streamable_http_app() + transport = server.streamable_http_app(host="0.0.0.0", stateless_http=True, json_response=True) inner_scope = {**scope, "path": "/mcp", "raw_path": b"/mcp", "root_path": ""} async with server.session_manager.run(): await transport(inner_scope, receive, send) diff --git a/src/backend/features/hosted_apps/models.py b/src/backend/features/hosted_apps/models.py new file mode 100644 index 00000000..92f6424a --- /dev/null +++ b/src/backend/features/hosted_apps/models.py @@ -0,0 +1,5 @@ +"""Hosted application tables live in their separate application database. + +The platform model loader imports this module independently of runtime switches; +application metadata is deliberately not attached to the platform Base. +""" diff --git a/src/backend/core/services/application_operations.py b/src/backend/features/hosted_apps/operations.py similarity index 84% rename from src/backend/core/services/application_operations.py rename to src/backend/features/hosted_apps/operations.py index 694144fa..57f3dc40 100644 --- a/src/backend/core/services/application_operations.py +++ b/src/backend/features/hosted_apps/operations.py @@ -1,15 +1,8 @@ """Validated application actions shared by the agent callback and owner creation.""" -from core.services.application_data import ApplicationDataService -from core.services.application_publication import ApplicationPublicationService -from core.services.application_schema import ( - ApplicationDefinition, - InsertOperation, - InternalOperation, - MCPPublish, - QueryOperation, - TableDefinition, -) +from features.hosted_apps.data import ApplicationDataService +from features.hosted_apps.publication import ApplicationPublicationService +from features.hosted_apps.schema import ApplicationDefinition, InsertOperation, InternalOperation, MCPPublish, QueryOperation, TableDefinition from fastapi import HTTPException from pydantic import ValidationError from sqlalchemy.orm import Session @@ -19,13 +12,13 @@ def create_application( definition: ApplicationDefinition, owner: str, db: Session, chat_id: str = "" ) -> dict: if definition.site_id: - from core.services.site_service import SiteService + from features.sites.service import SiteService SiteService(db).get_owned(definition.site_id, owner, required="admin") service = ApplicationDataService() app = service.create(owner, definition) if definition.kind == "mcp": - from core.services.application_projects import ApplicationProjectService + from features.hosted_apps.projects import ApplicationProjectService editor = ApplicationProjectService(db, service.engine).editor(app["id"], owner, chat_id) app["project_id"] = editor["project_id"] @@ -43,7 +36,7 @@ def perform_operation(body: InternalOperation, db: Session) -> dict: ApplicationDefinition.model_validate(payload), body.user_id, db, body.chat_id ) if body.action == "source": - from core.services.application_projects import ApplicationProjectService + from features.hosted_apps.projects import ApplicationProjectService return ApplicationProjectService(db, service.engine).editor( body.app_id, body.user_id, body.chat_id @@ -86,7 +79,7 @@ def perform_operation(body: InternalOperation, db: Session) -> dict: body.app_id, body.user_id ) if body.action == "import_industry": - from core.services.application_industry import import_enterprises + from features.hosted_apps.industry import import_enterprises return import_enterprises(service, db, body.app_id, body.user_id, payload) except ValidationError as error: diff --git a/src/backend/core/services/application_projects.py b/src/backend/features/hosted_apps/projects.py similarity index 95% rename from src/backend/core/services/application_projects.py rename to src/backend/features/hosted_apps/projects.py index 088a3c3c..06a5cb55 100644 --- a/src/backend/core/services/application_projects.py +++ b/src/backend/features/hosted_apps/projects.py @@ -9,11 +9,11 @@ import json -from core.auth.permissions_iface import resolve_project_permission +from core.auth.edition import resolve_project_permission from core.db.models import ChatSession, Project, UserFolder from core.infra.time import utc_now -from core.services.application_store import application_sources, owned_application -from core.services.project_source import ProjectSourceService +from features.hosted_apps.store import application_sources, owned_application +from core.projects.source import ProjectSourceService from fastapi import HTTPException from sqlalchemy import insert, select @@ -27,7 +27,7 @@ def __init__(self, db, engine): def editor(self, app_id, owner, chat_id=""): - from core.services.application_publication import publication_lock + from features.hosted_apps.publication import publication_lock with publication_lock(self.engine, app_id): project, app = self.ensure(app_id, owner) @@ -128,7 +128,7 @@ def _provision(self, app, owner, project_id): # These are system-provisioned projects, with stable identity and metadata. # Folder creation still uses the ordinary owner lock, validation and audit. from core.db.repository import AuditLogRepository - from core.services.user_folder_service import UserFolderService + from core.projects.user_folders import UserFolderService folder_name = "MCP-" + app["id"] folder = ( diff --git a/src/backend/core/services/application_publication.py b/src/backend/features/hosted_apps/publication.py similarity index 93% rename from src/backend/core/services/application_publication.py rename to src/backend/features/hosted_apps/publication.py index df192a02..f9108ec1 100644 --- a/src/backend/core/services/application_publication.py +++ b/src/backend/features/hosted_apps/publication.py @@ -17,8 +17,8 @@ from core.db.models import AdminMcpServer from core.infra.time import utc_now -from core.services.application_deployments import ApplicationDeploymentService -from core.services.application_store import mcp_deployments, owned_application +from features.hosted_apps.deployments import ApplicationDeploymentService +from features.hosted_apps.store import mcp_deployments, owned_application from fastapi import HTTPException from sqlalchemy import select, text @@ -30,7 +30,7 @@ def publication_lock(engine, app_id): if not re.fullmatch(r"[0-9a-f]{32}", app_id): raise HTTPException(404, "Application not found") if engine.dialect.name == "sqlite": - from core.capabilities.lockfile import LockTimeout, locked + from core.infra.file_lock import LockTimeout, locked try: with locked(Path(engine.url.database + ".mcp-publication.lock")): @@ -73,7 +73,7 @@ def publish( ): from core.auth.capabilities import resolve_user_capabilities from core.config.settings import settings - from core.services import mcp_management_service as management + from core.capabilities.mcp import management as management with publication_lock(self.engine, app_id): with self.engine.connect() as connection: @@ -82,8 +82,8 @@ def publish( "can_add_mcp" ): raise HTTPException(403, "Administrator has not enabled personal MCP creation") - from core.services.application_projects import ApplicationProjectService - from core.services.application_schema import MCPProjectDefinition + from features.hosted_apps.projects import ApplicationProjectService + from features.hosted_apps.schema import MCPProjectDefinition from pydantic import ValidationError projects = ApplicationProjectService(self.db, self.engine) @@ -140,7 +140,6 @@ def publish( owner_user_id=owner, created_by=owner, created_at=utc_now(), - is_stable=False, ) self.db.add(row) row.display_name = app["title"] @@ -201,7 +200,7 @@ def publish( return receipt def revoke(self, app_id, owner): - from core.services.mcp_management_service import refresh_mcp_caches + from core.capabilities.mcp.management import refresh_mcp_caches with publication_lock(self.engine, app_id): with self.engine.connect() as connection: @@ -217,7 +216,7 @@ def revoke(self, app_id, owner): return result def rollback(self, app_id, owner, version): - from core.services.application_schema import ToolDefinition + from features.hosted_apps.schema import ToolDefinition with self.engine.connect() as connection: owned_application(connection, app_id, owner) diff --git a/src/backend/core/services/application_relational.py b/src/backend/features/hosted_apps/relational.py similarity index 98% rename from src/backend/core/services/application_relational.py rename to src/backend/features/hosted_apps/relational.py index 67d7d558..5d0ca0a9 100644 --- a/src/backend/core/services/application_relational.py +++ b/src/backend/features/hosted_apps/relational.py @@ -5,7 +5,7 @@ import math from datetime import date -from core.services.application_schema import TableDefinition +from features.hosted_apps.schema import TableDefinition from fastapi import HTTPException from sqlalchemy import ( JSON, diff --git a/src/backend/features/hosted_apps/resources/__init__.py b/src/backend/features/hosted_apps/resources/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/src/backend/features/hosted_apps/resources/__init__.py @@ -0,0 +1 @@ + diff --git a/src/backend/core/plugins/resources/application.py b/src/backend/features/hosted_apps/resources/admission.py similarity index 88% rename from src/backend/core/plugins/resources/application.py rename to src/backend/features/hosted_apps/resources/admission.py index cd8beb88..650ae884 100644 --- a/src/backend/core/plugins/resources/application.py +++ b/src/backend/features/hosted_apps/resources/admission.py @@ -7,9 +7,9 @@ from fastapi import HTTPException -from . import service, store -from .owners import application_binding -from .reservation import lock_owner +from core.capabilities.plugins.resources import service, store +from core.capabilities.plugins.resources.owners import application_binding +from core.capabilities.plugins.resources.reservation import lock_owner async def reserve(db, selected, module_id, user_id, binding): @@ -85,10 +85,10 @@ async def create( except HTTPException as error: db.rollback() if error.status_code == 429: - from core.applications.diagnostics import record + from core.harness.registry.resource_owners import resource_owner try: - record(db, user_id, "resource", "quota_denied", item=item, instance_id=instance_id) + resource_owner().record(db, user_id, "resource", "quota_denied", item=item, instance_id=instance_id) except Exception: pass # The original quota denial remains authoritative if telemetry storage fails. raise @@ -106,10 +106,10 @@ def owned(db, user_id, key, instance_id, resource_id, *, allow_closed=False): async def renew(db, user_id, key, instance_id, resource_id): row = owned(db, user_id, key, instance_id, resource_id) - from core.applications import background + from core.harness.registry.resource_owners import resource_owner - if background.authorized_resource(db, row): - await background.renew(db, user_id, key, instance_id, resource_id) + if resource_owner().authorize(db, row): + await resource_owner().renew(db, user_id, key, instance_id, resource_id) return store.public(row) updated = ( db.query(store.PluginResource) @@ -121,7 +121,7 @@ async def renew(db, user_id, key, instance_id, resource_id): if updated != 1: raise HTTPException(410, "resource_expired") db.refresh(row) - from .deadlines import arm + from core.capabilities.plugins.resources.deadlines import arm await arm(db, row) return store.public(row) diff --git a/src/backend/core/plugins/resources/application_cleanup.py b/src/backend/features/hosted_apps/resources/cleanup.py similarity index 72% rename from src/backend/core/plugins/resources/application_cleanup.py rename to src/backend/features/hosted_apps/resources/cleanup.py index 1f83dfbd..ab68a622 100644 --- a/src/backend/core/plugins/resources/application_cleanup.py +++ b/src/backend/features/hosted_apps/resources/cleanup.py @@ -4,13 +4,13 @@ from fastapi import HTTPException -from . import store -from .owners import runtime_session +from core.capabilities.plugins.resources import store +from core.capabilities.plugins.resources.owners import runtime_session async def close(db, row): - from .service import request - from core.applications.diagnostics import observe_resource + from core.capabilities.plugins.resources.service import request + from core.harness.registry.resource_owners import resource_owner if row.status == "closed": return @@ -21,7 +21,7 @@ async def close(db, row): try: target = store.descriptor(row) except HTTPException: - observe_resource(db, row, "release", "release_failed") + resource_owner().observe(db, row, "release", "release_failed") raise HTTPException(503, "resource_cleanup_credentials_unavailable") from None if target.get("process_id"): try: @@ -31,7 +31,7 @@ async def close(db, row): provider = get_sandbox_provider() if target.get("provider") != provider.name: - observe_resource(db, row, "release", "release_failed") + resource_owner().observe(db, row, "release", "release_failed") raise HTTPException(503, "resource_cleanup_scope_unavailable") result = await provider.write_stdin( target["process_id"], @@ -41,8 +41,8 @@ async def close(db, row): yield_time_ms=1000, ) if not result or result.get("exit_code") is None: - observe_resource(db, row, "release", "release_failed") + resource_owner().observe(db, row, "release", "release_failed") raise HTTPException(503, "resource_cleanup_pending") row.status = "closed" db.commit() - observe_resource(db, row, "release", "ok") + resource_owner().observe(db, row, "release", "ok") diff --git a/src/backend/core/services/application_schema.py b/src/backend/features/hosted_apps/schema.py similarity index 100% rename from src/backend/core/services/application_schema.py rename to src/backend/features/hosted_apps/schema.py diff --git a/src/backend/core/services/application_sql_policy.py b/src/backend/features/hosted_apps/sql_policy.py similarity index 96% rename from src/backend/core/services/application_sql_policy.py rename to src/backend/features/hosted_apps/sql_policy.py index 94f39fc2..6c96edfd 100644 --- a/src/backend/core/services/application_sql_policy.py +++ b/src/backend/features/hosted_apps/sql_policy.py @@ -4,7 +4,7 @@ from contextlib import contextmanager -from core.services.application_relational import sql_table +from features.hosted_apps.relational import sql_table from sqlalchemy import text diff --git a/src/backend/core/services/application_store.py b/src/backend/features/hosted_apps/store.py similarity index 94% rename from src/backend/core/services/application_store.py rename to src/backend/features/hosted_apps/store.py index 6b57c3e5..bd7bff35 100644 --- a/src/backend/core/services/application_store.py +++ b/src/backend/features/hosted_apps/store.py @@ -67,7 +67,7 @@ @lru_cache(maxsize=1) def application_engine() -> Engine: - from core.config.application_hosting import application_hosting_settings + from features.hosted_apps.config import application_hosting_settings from core.config.settings import settings url = application_hosting_settings.database_url @@ -112,14 +112,14 @@ def application_engine() -> Engine: def initialize_store(engine: Engine) -> None: """Explicit provisioning entry point; runtime requests never create registry tables.""" - from core.services.application_history import collection_history + from features.hosted_apps.history import collection_history metadata.create_all(engine) def initialize_local_store() -> None: """Provision the managed local SQLite store at startup, never cloud databases.""" - from core.auth.desktop_bridge import bridge_enabled - from core.config.application_hosting import application_hosting_settings + from features.desktop.auth.bridge import bridge_enabled + from features.hosted_apps.config import application_hosting_settings from core.config.settings import settings if ( @@ -132,7 +132,7 @@ def initialize_local_store() -> None: def require_store(engine: Engine) -> None: - from core.services.application_history import collection_history + from features.hosted_apps.history import collection_history try: ready = all( inspect(engine).has_table(table.name) for table in (applications, application_sources, metadata.tables["hosted_collection_history"]) diff --git a/src/backend/features/installed.py b/src/backend/features/installed.py new file mode 100644 index 00000000..89dc92c1 --- /dev/null +++ b/src/backend/features/installed.py @@ -0,0 +1,3 @@ +"""Community modules; runtime switches do not affect schema registration.""" + +INSTALLED_FEATURES: tuple[str, ...] = ("personal_agent", "features.evaluation", "features.automation", "features.sites", "features.hosted_apps", "integrations.dingtalk", "integrations.lark", "integrations.email", "integrations.yida", "integrations.channels", "features.workbench", "features.knowledge", "features.marketplace", "features.desktop") diff --git a/src/backend/features/knowledge/README.md b/src/backend/features/knowledge/README.md new file mode 100644 index 00000000..88b9e510 --- /dev/null +++ b/src/backend/features/knowledge/README.md @@ -0,0 +1,11 @@ +# 知识库 + +拥有知识空间、文档、分块、媒体资产、Wiki 模型、权限、索引队列和外接知识库路由。FULL 与 CE 均安装模型；用户删除清理随模型注册，独立于运行开关。CE 通过 edition.py 选择权限、外部 provider 和请求模型。 + +解析、分块、语义切分、媒体描述与索引分别归属对应模块。向量存储的客户端、索引写入和检索各自维护职责；共享 embedding/reranking 客户端位于 core.models，由所有领域复用。缓存仍为进程内唯一所有者，调用方引用明确模块，不保留旧路径转发。 + +核心通过 catalog_options 取得管理选项，通过 retrieval_context 取得外部 MCP 请求上下文。模型请求头由宿主注册的环境解析器处理，知识库和模型层不直接依赖桌面实现。实际桌面身份校验与设备头逻辑保持在原桌面模块，P5 随其目录归属移动。 + +API 按原端点顺序组合为一个中央路由入口。依赖 core 的模型、存储、认证、数据库及通用解析接口。独立模型加载与空库升级必须验证，测试使用隔离数据库和模拟外部模型/向量服务。 + +版本差异统一由 edition.py 选择权限、外接 provider、检索、请求模型与初始授权；CE 只覆盖此入口。 diff --git a/src/backend/core/kb/__init__.py b/src/backend/features/knowledge/__init__.py similarity index 64% rename from src/backend/core/kb/__init__.py rename to src/backend/features/knowledge/__init__.py index 90d8da71..ed3a59e1 100644 --- a/src/backend/core/kb/__init__.py +++ b/src/backend/features/knowledge/__init__.py @@ -1,5 +1,5 @@ """Knowledge-base subsystem: provider seam, document parser/chunker, and vector store. Relocated from the top-level ``utils/`` package (where it was mislabeled as generic -utilities) into ``core`` alongside core.content.kb_processing and core.services.kb_service. +utilities) into ``core`` alongside features.knowledge.processing and features.knowledge.service. """ diff --git a/src/backend/features/knowledge/api/__init__.py b/src/backend/features/knowledge/api/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/src/backend/features/knowledge/api/__init__.py @@ -0,0 +1 @@ + diff --git a/src/backend/features/knowledge/api/kb.py b/src/backend/features/knowledge/api/kb.py new file mode 100644 index 00000000..cdaa17ab --- /dev/null +++ b/src/backend/features/knowledge/api/kb.py @@ -0,0 +1,7 @@ +"""Compose knowledge routes in the original endpoint order.""" +from fastapi import APIRouter +from features.knowledge.api import kb_assets, kb_spaces, kb_documents, kb_mutations, kb_chunks + +router = APIRouter() +for group in (kb_assets, kb_spaces, kb_documents, kb_mutations, kb_chunks): + router.include_router(group.router, prefix="/v1/catalog/kb", tags=["Knowledge Base"]) diff --git a/src/backend/features/knowledge/api/kb_assets.py b/src/backend/features/knowledge/api/kb_assets.py new file mode 100644 index 00000000..45c8df82 --- /dev/null +++ b/src/backend/features/knowledge/api/kb_assets.py @@ -0,0 +1,147 @@ +"""kb assets responsibilities.""" +from __future__ import annotations +from fastapi import APIRouter +router = APIRouter() +from typing import Optional +from core.auth.backend import UserContext, get_current_user +from core.files.content.file_validation import validate_kb_file +from core.db.engine import get_db +from core.infra.exceptions import BadRequestError, FileTooLargeError, ResourceNotFoundError +from core.infra.responses import success_response +from core.files.storage import get_storage +from fastapi import Depends, File, Form, Path, UploadFile +from sqlalchemy.orm import Session +import logging +logger = logging.getLogger(__name__) +from features.knowledge.api import kb_helpers +from features.knowledge.api import kb_state + +@router.get("/assets/{asset_id}", summary="读取知识库媒体资产") +def get_kb_asset( + asset_id: str = Path(..., description="Asset ID"), + user: UserContext = Depends(get_current_user), + db: Session = Depends(get_db), +): + """按 ID 读取知识库中的媒体资产（版面解析切出的图片等）原始字节。 + + 资产的可见性完全跟随所属知识库空间：能读该库才能读它的图。图片链接会出现在分块正文 + 与检索结果里，故按 ID 直取，不经过库/文档路径。 + """ + from fastapi.responses import Response + + from features.knowledge.models import KBAsset + from features.knowledge.repository import KBRepository + + asset = db.query(KBAsset).filter(KBAsset.asset_id == asset_id).first() + if not asset: + raise ResourceNotFoundError(resource_type="kb_asset", resource_id=asset_id) + + kb_space = KBRepository(db).get_space(asset.kb_id) + if not kb_space or not kb_helpers.can_read_space(db, kb_space, user.user_id): + # Same shape as a missing asset: an unauthorised caller must not be able to + # probe which asset ids exist. + raise ResourceNotFoundError(resource_type="kb_asset", resource_id=asset_id) + + try: + data = get_storage().download_bytes(asset.storage_key) + except Exception as exc: + logger.warning("Asset %s read failed: %s", asset_id, exc) + raise ResourceNotFoundError(resource_type="kb_asset", resource_id=asset_id) + + return Response( + content=data, + media_type=asset.mime_type or "application/octet-stream", + headers={"Cache-Control": "private, max-age=86400"}, + ) + + +@router.post("/preview-chunks", summary="预览文档分块效果") +async def preview_chunks( + file: UploadFile = File(..., description="Document file to preview"), + chunk_method: str = Form( + "structured", description="Chunking method: structured|recursive|embedding_semantic|laws|qa" + ), + parent_chunk_size: int = Form(1024, description="Parent chunk size in tokens"), + child_chunk_size: int = Form(128, description="Child chunk size in tokens"), + overlap_tokens: int = Form(20, description="Overlap between child chunks"), + parent_child_indexing: bool = Form( + True, description="Enable parent-child indexing; False = flat chunks only" + ), + separators: Optional[str] = Form( + None, description="父分块分隔符，JSON 数组字符串；为空用默认层级" + ), + child_separators: Optional[str] = Form( + None, description="子分块分隔符，JSON 数组字符串；为空走定长滑窗" + ), + user: UserContext = Depends(get_current_user), +): + """预览文档的分块效果，不落库、不写入向量库。上传文件并按指定分块方法（structured/recursive/embedding_semantic/laws/qa）与父子分块参数解析，返回父块及其子块预览，用于创建知识库前调参。文件大小受 MAX_PREVIEW_SIZE（20MB）限制。""" + content = await file.read() + if len(content) > kb_state.MAX_PREVIEW_SIZE: + raise FileTooLargeError(max_size=kb_state.MAX_PREVIEW_SIZE, actual_size=len(content)) + if len(content) == 0: + raise BadRequestError(message="File is empty", data={"filename": file.filename}) + + if not file.filename: + raise BadRequestError(message="Filename is required", data={"field": "file"}) + _, mime_type = validate_kb_file(file.filename, content) + + from features.knowledge.parser_tokens import count_tokens as _count_tokens + from features.knowledge.kb_parser import parse_and_chunk + from features.knowledge.parser_chunks import parse_separators_form + + _separators = parse_separators_form(separators) + _child_separators = parse_separators_form(child_separators) + + embed_fn = None + if chunk_method == "embedding_semantic": + from core.models.embedding import embed_batch + + embed_fn = embed_batch + + parent_chunks = parse_and_chunk( + content, + mime_type, + chunk_method=chunk_method, + parent_size=parent_chunk_size, + child_size=child_chunk_size, + overlap=overlap_tokens, + embed_fn=embed_fn, + separators=_separators, + child_separators=_child_separators, + ) + + total_children = sum(len(pc.children) for pc in parent_chunks) if parent_child_indexing else 0 + + chunks_preview = [] + for idx, pc in enumerate(parent_chunks): + if parent_child_indexing: + children_preview = [ + { + "index": c.index, + "content": c.content[:200] + ("..." if len(c.content) > 200 else ""), + } + for c in pc.children[:5] + ] + children_count = len(pc.children) + else: + children_preview = [] + children_count = 0 + chunks_preview.append( + { + "index": idx, + "content": pc.content, + "token_count": _count_tokens(pc.content), + "children_count": children_count, + "children_preview": children_preview, + } + ) + + return success_response( + data={ + "total_chunks": len(parent_chunks), + "total_children": total_children, + "chunks": chunks_preview, + }, + message="Chunk preview generated successfully", + ) diff --git a/src/backend/features/knowledge/api/kb_chunks.py b/src/backend/features/knowledge/api/kb_chunks.py new file mode 100644 index 00000000..2cef7385 --- /dev/null +++ b/src/backend/features/knowledge/api/kb_chunks.py @@ -0,0 +1,175 @@ +"""kb chunks responsibilities.""" +from __future__ import annotations +from fastapi import APIRouter +router = APIRouter() +from typing import Optional +from features.knowledge.edition import UpdateChunkRequest +from core.auth.backend import UserContext, get_current_user +from core.db.engine import get_db +from core.infra.exceptions import AccessDeniedError, BadRequestError, ResourceNotFoundError +from core.infra.responses import paginated_response, success_response +from features.knowledge.service import KBService +from fastapi import Depends, Path, Query +from sqlalchemy.orm import Session +import logging +logger = logging.getLogger(__name__) +from features.knowledge.api import kb_helpers + +@router.get("/{kb_id}/chunks", summary="获取文档分块列表（含标签和问题）") +def list_chunks( + kb_id: str = Path(..., description="KB space ID"), + document_id: Optional[str] = Query(None, description="Filter by document ID"), + page: int = Query(1, ge=1), + page_size: int = Query(20, ge=1, le=100), + user: UserContext = Depends(get_current_user), + db: Session = Depends(get_db), +): + """分页获取知识库分块列表，可选按 document_id 过滤。每个分块返回完整正文内容、标签和问题列表，供分块管理界面查看与编辑。仅空间所有者可访问。""" + from features.knowledge.models import KBChunk + from features.knowledge.repository import KBRepository + + kb_repo = KBRepository(db) + kb_space = kb_repo.get_space(kb_id) + if not kb_space: + raise ResourceNotFoundError(resource_type="kb_space", resource_id=kb_id) + # Public KBs are readable by all logged-in users (chunks are read-only viewing) + if not kb_helpers.can_read_space(db, kb_space, user.user_id): + raise AccessDeniedError(message="Access denied", reason="Not the KB owner") + + query = db.query(KBChunk).filter(KBChunk.kb_id == kb_id) + if document_id: + query = query.filter(KBChunk.document_id == document_id) + total = query.count() + chunks = ( + query.order_by(KBChunk.chunk_index).offset((page - 1) * page_size).limit(page_size).all() + ) + + items = [ + { + "chunk_id": c.chunk_id, + "document_id": c.document_id, + "chunk_index": c.chunk_index, + "content": c.content, + "tags": c.tags or [], + "questions": c.questions or [], + } + for c in chunks + ] + return paginated_response(items=items, page=page, page_size=page_size, total_items=total) + + +@router.get("/{kb_id}/chunks/{chunk_id}/children", summary="获取父块下的子块列表") +def list_chunk_children( + kb_id: str = Path(..., description="KB space ID"), + chunk_id: str = Path(..., description="Parent chunk ID"), + user: UserContext = Depends(get_current_user), + db: Session = Depends(get_db), +): + """返回某父块在向量库里的子块（父子分块模式）。扁平模式无子块时返回空列表。仅空间可读者可访问。""" + from features.knowledge.models import KBChunk + from features.knowledge.repository import KBRepository + + kb_repo = KBRepository(db) + kb_space = kb_repo.get_space(kb_id) + if not kb_space: + raise ResourceNotFoundError(resource_type="kb_space", resource_id=kb_id) + if not kb_helpers.can_read_space(db, kb_space, user.user_id): + raise AccessDeniedError(message="Access denied", reason="Not the KB owner") + + parent = db.query(KBChunk).filter(KBChunk.chunk_id == chunk_id, KBChunk.kb_id == kb_id).first() + if not parent: + raise ResourceNotFoundError(resource_type="kb_chunk", resource_id=chunk_id) + + from features.knowledge.vector_indexing import get_children_for_parent + + children = get_children_for_parent(chunk_id) + return success_response( + data={"children": children, "count": len(children)}, + message="Chunk children retrieved successfully", + ) + + +@router.patch("/{kb_id}/chunks/{chunk_id}", summary="更新分块内容/标签/问题") +def update_chunk( + kb_id: str = Path(..., description="KB space ID"), + chunk_id: str = Path(..., description="Chunk ID"), + request: UpdateChunkRequest = ..., + user: UserContext = Depends(get_current_user), + db: Session = Depends(get_db), +): + """更新指定分块的正文内容、标签和问题列表，并同步刷新其在向量库中的向量与索引。 + + 传入 ``content`` 时会对该分块重新向量化（dense + sparse）；仅改标签时只刷新稀疏/标签 + 索引；问题始终单独重建问题向量行。仅空间所有者（admin 权限）可操作，系统托管知识库禁止 + 编辑分块；仅传入的字段会被更新。""" + from features.knowledge.models import KBChunk, KBDocument + from features.knowledge.repository import KBRepository + from features.knowledge.vector_indexing import reindex_chunk_content, reindex_chunk_tags, upsert_question_rows + + kb_repo = KBRepository(db) + kb_space = kb_repo.get_space(kb_id) + if not kb_space: + raise ResourceNotFoundError(resource_type="kb_space", resource_id=kb_id) + kb_helpers.require_kb_write(db, kb_space, user.user_id, "admin") + if KBService(db)._is_system_managed_space(kb_space): + raise AccessDeniedError( + message="Access denied", reason="System managed KB does not allow chunk edits" + ) + + chunk = db.query(KBChunk).filter(KBChunk.chunk_id == chunk_id, KBChunk.kb_id == kb_id).first() + if not chunk: + raise ResourceNotFoundError(resource_type="kb_chunk", resource_id=chunk_id) + + content_changed = request.content is not None + if content_changed: + new_content = request.content.strip() + if not new_content: + raise BadRequestError(message="分块内容不能为空", data={"field": "content"}) + chunk.content = new_content + if request.tags is not None: + chunk.tags = request.tags + if request.questions is not None: + chunk.questions = request.questions + db.commit() + db.refresh(chunk) + + try: + doc = db.query(KBDocument).filter_by(document_id=chunk.document_id).first() + doc_title = doc.title if doc else "" + + if content_changed: + # Content changed: re-vectorize the whole chunk (current tags already included) + reindex_chunk_content( + chunk_id=chunk_id, + content=chunk.content, + tags=chunk.tags or [], + user_id=user.user_id, + kb_id=kb_id, + document_id=chunk.document_id, + title=doc_title, + chunk_index=chunk.chunk_index, + ) + elif request.tags is not None: + reindex_chunk_tags(chunk_id, chunk.content, chunk.tags) + if request.questions is not None: + upsert_question_rows( + parent_chunk_id=chunk_id, + questions=chunk.questions, + user_id=user.user_id, + kb_id=kb_id, + document_id=chunk.document_id, + title=doc_title, + chunk_index=chunk.chunk_index, + ) + except Exception as exc: + logger.warning("Milvus re-index failed for chunk %s: %s", chunk_id, exc) + + return success_response( + data={ + "chunk_id": chunk.chunk_id, + "content": chunk.content, + "tags": chunk.tags, + "questions": chunk.questions, + }, + message="Chunk updated successfully", + ) diff --git a/src/backend/features/knowledge/api/kb_documents.py b/src/backend/features/knowledge/api/kb_documents.py new file mode 100644 index 00000000..7bbbe89c --- /dev/null +++ b/src/backend/features/knowledge/api/kb_documents.py @@ -0,0 +1,255 @@ +"""kb documents responsibilities.""" +from __future__ import annotations +from fastapi import APIRouter +router = APIRouter() +import hashlib +import json +import os +from typing import Optional +from core.auth.backend import UserContext, get_current_user +from core.files.content.file_validation import validate_kb_file +from features.knowledge.wiki.config import index_modes_of +from core.db.engine import get_db +from core.infra.exceptions import AccessDeniedError, BadRequestError, FileTooLargeError, ResourceNotFoundError, StorageError +from core.infra.responses import created_response, paginated_response, success_response +from features.knowledge.service import KBService +from core.files.storage import generate_storage_key, get_storage +from fastapi import Depends, File, Form, Path, Query, UploadFile, status +from fastapi.concurrency import run_in_threadpool +from sqlalchemy.orm import Session +import logging +logger = logging.getLogger(__name__) +from features.knowledge.api import kb_helpers +from features.knowledge.api import kb_state + +@router.post("/{kb_id}/documents", status_code=status.HTTP_201_CREATED, summary="上传文档到知识库") +async def upload_document( + kb_id: str = Path(..., description="KB space ID"), + file: UploadFile = File(..., description="Document file to upload"), + title: Optional[str] = Form(None, description="Document title (defaults to filename)"), + metadata: Optional[str] = Form(None, description="Additional metadata as JSON string"), + indexing_config: Optional[str] = Form(None, description="Indexing config as JSON string"), + chunk_method: Optional[str] = Form(None, description="Chunking method override"), + user: UserContext = Depends(get_current_user), + db: Session = Depends(get_db), +): + """上传文档到指定知识库空间并异步建立索引。仅空间所有者可上传；文件大小受 MAX_FILE_SIZE（100MB）限制，可选传入标题、元数据、索引配置及分块方法覆盖。上传成功后通过后台任务完成解析、分块与向量化，立即返回 indexing_status=processing。""" + kb_service = KBService(db) + + from features.knowledge.repository import KBRepository + + kb_repo = KBRepository(db) + kb_space = kb_repo.get_space(kb_id) + + if not kb_space: + raise ResourceNotFoundError(resource_type="kb_space", resource_id=kb_id) + kb_helpers.require_kb_write(db, kb_space, user.user_id, "edit") + + if not file.filename: + raise BadRequestError(message="Filename is required", data={"field": "file"}) + + content = await file.read() + file_size = len(content) + + if file_size > kb_state.MAX_FILE_SIZE: + raise FileTooLargeError(max_size=kb_state.MAX_FILE_SIZE, actual_size=file_size) + if file_size == 0: + raise BadRequestError(message="File is empty", data={"filename": file.filename}) + + _, mime_type = validate_kb_file(file.filename, content) + + checksum = hashlib.sha256(content).hexdigest() + env = os.getenv("ENVIRONMENT", "dev") + storage_key = generate_storage_key( + env=env, user_id=user.user_id, category="kb_documents", filename=file.filename + ) + + try: + storage = get_storage() + # 对象存储是阻塞的网络调用，直接在 async handler 里调会卡住整个事件循环—— + # 批量上传时后端会连 SSE 和其它接口一起停摆。 + await run_in_threadpool(storage.upload_bytes, content, storage_key) + logger.info("Document uploaded to storage: %s", storage_key) + except StorageError as e: + logger.error("Failed to upload document to storage: %s", e) + raise StorageError( + operation="upload", error=f"Failed to upload document: {e.data.get('error', str(e))}" + ) + + doc_title = title or file.filename + + if metadata: + try: + json.loads(metadata) + except json.JSONDecodeError: + raise BadRequestError(message="Invalid metadata JSON", data={"field": "metadata"}) + + try: + document = kb_service.upload_document( + kb_id=kb_id, + user_id=user.user_id, + title=doc_title, + filename=file.filename, + size_bytes=file_size, + mime_type=mime_type, + storage_key=storage_key, + checksum=checksum, + ) + except PermissionError as e: + raise AccessDeniedError(message="Access denied", reason=str(e)) + + _effective_chunk_method = chunk_method or kb_space.chunk_method or "semantic" + _upload_indexing_config = None + if indexing_config: + try: + _upload_indexing_config = json.loads(indexing_config) + except json.JSONDecodeError: + raise BadRequestError( + message="Invalid indexing_config JSON", data={"field": "indexing_config"} + ) + _space_extra = kb_space.extra_data if isinstance(kb_space.extra_data, dict) else {} + _indexing_config = _upload_indexing_config or _space_extra.get("indexing_config") + + # 排进持久化索引队列，由常驻 worker 认领执行（见 core/kb/index_worker.py）。 + # 早先这里挂的是 FastAPI BackgroundTasks：同步任务落进 Starlette 的共享请求线程池， + # 一次多选几十个文件就把池占满，后续请求连 handler 都进不去——前端一直转圈、网关 + # 报 502；而且任务是进程内的，后端一重启就全丢，文档永远停在「索引中」。 + kb_helpers.enqueue_document_indexing( + db, + document_id=document["document_id"], + user_id=user.user_id, + chunk_method=_effective_chunk_method, + indexing_config=_indexing_config, + index_modes=index_modes_of(kb_space), + ) + + return created_response( + data={**document, "indexing_status": "processing"}, + message="Document uploaded successfully. Indexing in progress.", + ) + + +@router.get("/{kb_id}/documents", summary="获取知识库文档列表") +def list_documents( + kb_id: str = Path(..., description="Local or external knowledge collection ID"), + page: int = Query(1, ge=1, description="Page number"), + page_size: int = Query(20, ge=1, le=100, description="Items per page"), + keyword: Optional[str] = Query(None, description="按标题/文件名搜索（全库范围，再分页）"), + user: UserContext = Depends(get_current_user), + db: Session = Depends(get_db), +): + """分页获取知识库文档；非本地 ID 交由当前版本的外部知识提供方处理。""" + from features.knowledge.repository import KBRepository + + kb_repo = KBRepository(db) + kb_space = kb_repo.get_space(kb_id) + + if not kb_space: + from features.knowledge.edition import is_enabled, list_documents + + if is_enabled(): + result = list_documents(kb_id, page=page, limit=page_size, keyword=keyword or "") + response = paginated_response( + items=result.get("items", []), + page=result.get("page", page), + page_size=result.get("page_size", page_size), + total_items=result.get("total", 0), + message="Documents retrieved successfully", + ) + from features.knowledge.edition import count_document_statuses + + status_counts = count_document_statuses( + kb_id, + keyword=keyword or "", + initial_result=result, + ) + if status_counts is not None: + response["data"]["status_counts"] = status_counts + return response + raise ResourceNotFoundError(resource_type="kb_space", resource_id=kb_id) + + # Public KBs (created in the admin console, owned by the system principal) are readable by all logged-in users + if not kb_helpers.can_read_space(db, kb_space, user.user_id): + raise AccessDeniedError( + message="Access denied", reason="Only the KB space owner can list documents" + ) + + documents, total = kb_repo.list_documents(kb_id, page, page_size, keyword=keyword) + items = [ + { + "id": d.document_id, + "title": d.title, + "desc": d.filename, + "filename": d.filename, + "size": d.size_bytes, + "mime_type": d.mime_type, + "storage_key": d.storage_key, + "uploaded_at": d.uploaded_at.isoformat(), + "indexing_status": getattr(d, "indexing_status", "processing"), + } + for d in documents + ] + response = paginated_response( + items=items, + page=page, + page_size=page_size, + total_items=total, + message="Documents retrieved successfully", + ) + response["data"]["status_counts"] = kb_repo.count_document_statuses(kb_id, keyword=keyword) + return response + + +@router.get("/{kb_id}/documents/{document_id}", summary="获取知识库文档详情") +def get_document_detail( + kb_id: str = Path(..., description="Local or external knowledge collection ID"), + document_id: str = Path(..., description="Document ID"), + user: UserContext = Depends(get_current_user), + db: Session = Depends(get_db), +): + """获取文档详情；非本地 ID 交由当前版本的外部知识提供方处理。""" + from features.knowledge.repository import KBRepository + + kb_repo = KBRepository(db) + kb_space = kb_repo.get_space(kb_id) + + if not kb_space: + from features.knowledge.edition import get_document_detail, is_enabled + + if is_enabled(): + detail = get_document_detail(kb_id, document_id) + return success_response(data=detail, message="Document detail retrieved successfully") + raise ResourceNotFoundError(resource_type="kb_space", resource_id=kb_id) + + # Public KBs are readable by all logged-in users + if not kb_helpers.can_read_space(db, kb_space, user.user_id): + raise AccessDeniedError( + message="Access denied", reason="Only the KB space owner can view document details" + ) + + document = kb_repo.get_document(document_id) + if not document or document.kb_id != kb_id: + raise ResourceNotFoundError(resource_type="kb_document", resource_id=document_id) + + from features.knowledge.models import KBChunk + + chunks = ( + db.query(KBChunk) + .filter(KBChunk.document_id == document_id, KBChunk.kb_id == kb_id) + .order_by(KBChunk.chunk_index) + .all() + ) + content = "\n\n".join(c.content for c in chunks) if chunks else "" + + return success_response( + data={ + "id": document.document_id, + "title": document.title, + "desc": document.filename, + "filename": document.filename, + "mime_type": document.mime_type, + "uploaded_at": document.uploaded_at.isoformat(), + "content": content, + }, + message="Document detail retrieved successfully", + ) diff --git a/src/backend/features/knowledge/api/kb_helpers.py b/src/backend/features/knowledge/api/kb_helpers.py new file mode 100644 index 00000000..c22a0bfc --- /dev/null +++ b/src/backend/features/knowledge/api/kb_helpers.py @@ -0,0 +1,105 @@ +"""kb helpers responsibilities.""" +from __future__ import annotations +from typing import Optional +from core.infra.exceptions import AccessDeniedError +from core.models.chat_models import get_summarize_model +from core.llm.runtime.message_compat import extract_text_from_chat_response +import logging +logger = logging.getLogger(__name__) + +def can_read_space(db, space, user_id: str) -> bool: + """Read permission: owner / everyone for public KBs / grantees of a scoped-visibility KB (single-source-of-truth resolution).""" + from features.knowledge.edition import resolve_local_kb_level + + return resolve_local_kb_level(db, user_id, space.kb_id) != "none" + + +def require_kb_write(db, space, user_id: str, required: str) -> None: + """Write permission check: required is 'edit' (upload/reindex) or 'admin' (rename/delete/edit chunks). + + Raises AccessDeniedError when insufficient. The owner is always admin, superadmins are always admin, granted users go by their level. + """ + from features.knowledge.edition import has_kb_permission, resolve_local_kb_level + + level = resolve_local_kb_level(db, user_id, space.kb_id) + if not has_kb_permission(level, required): + raise AccessDeniedError(message="Access denied", reason=f"requires kb level >= {required}") + + +def enqueue_document_indexing( + db, + *, + document_id: str, + user_id: str, + chunk_method: str, + indexing_config: Optional[dict], + index_modes: Optional[list], +) -> None: + """把文档排进持久化索引队列，交给常驻 worker 执行。 + + 索引不能再跑在请求线程池里：那是 Starlette 与所有同步依赖（``get_db``）共用的 + 池子，批量上传会把它占满，导致整个后端在 handler 之前就排队。详见 + :mod:`core.kb.index_worker`。 + """ + from features.knowledge.models import KBDocument + from features.knowledge import index_queue + + doc = db.query(KBDocument).filter(KBDocument.document_id == document_id).first() + if doc is None: + logger.warning("Cannot enqueue indexing: document %s not found", document_id) + return + index_queue.enqueue( + db, + doc, + user_id=user_id, + chunk_method=chunk_method, + indexing_config=indexing_config, + index_modes=index_modes, + ) + + +def fallback_kb_description(name: str, description: str) -> str: + base_name = (name or "").strip() + base_desc = " ".join((description or "").split()) + if base_desc: + return ( + f"{base_name}相关资料，适合检索概念说明、政策信息、行业动态、业务规则和常见问题。" + f"当前范围包括：{base_desc[:120]}" + )[:220] + return ( + f"{base_name}相关内容，适合检索概念说明、政策解读、业务规则、行业动态、" + f"典型案例和常见问题。" + )[:220] + + +async def generate_kb_description(name: str, description: str) -> str: + prompt = f"""/no_think +你是一名知识库配置助手。请根据知识库名称和现有简介，生成一段更利于智能路由判断的“知识库简介”。 + +目标： +1. 帮助系统快速判断什么问题应该路由到这个知识库 +2. 文本要像知识库简介，不是宣传文案 +3. 明确主题范围、文档类型、适合回答的问题类型 +4. 用中文，1 段即可，控制在 60 到 120 字 +5. 不要使用“本知识库可以帮助你”这类口语化表达 +6. 不要使用“该知识库聚焦”“本知识库涵盖”这类空泛起手式 +7. 直接输出简介正文，不要加标题、引号、序号或解释 + +知识库名称：{name} +现有简介：{description or "无"} +""" + + try: + model = get_summarize_model() + # AgentScope 2.0: model.__call__ takes list[Msg] (not list[dict]) + from agentscope.message import Msg, TextBlock + + result = await model( + messages=[Msg(name="user", role="user", content=[TextBlock(type="text", text=prompt)])] + ) + polished = extract_text_from_chat_response(result).strip() + polished = " ".join(polished.split()) + return polished or fallback_kb_description(name, description) + except Exception as exc: + logger.warning("Failed to polish KB description for %s: %s", name, exc) + return fallback_kb_description(name, description) diff --git a/src/backend/features/knowledge/api/kb_mutations.py b/src/backend/features/knowledge/api/kb_mutations.py new file mode 100644 index 00000000..33b8eee2 --- /dev/null +++ b/src/backend/features/knowledge/api/kb_mutations.py @@ -0,0 +1,117 @@ +"""kb mutations responsibilities.""" +from __future__ import annotations +from fastapi import APIRouter +router = APIRouter() +from typing import Optional +from features.knowledge.edition import ReindexRequest +from core.auth.backend import UserContext, get_current_user +from features.knowledge.wiki.config import index_modes_of +from core.db.engine import get_db +from core.infra.exceptions import ResourceNotFoundError +from core.infra.responses import success_response +from features.knowledge.service import KBService +from fastapi import Depends, Path +from sqlalchemy.orm import Session +import logging +logger = logging.getLogger(__name__) +from features.knowledge.api import kb_helpers + +@router.delete("/{kb_id}", summary="删除知识库空间") +def delete_kb_space( + kb_id: str = Path(..., description="KB space ID"), + user: UserContext = Depends(get_current_user), + db: Session = Depends(get_db), +): + """删除指定知识库空间及其下属文档、分块与向量数据。仅空间所有者可操作；空间不存在或无权访问时返回 404。此操作不可恢复。""" + kb_service = KBService(db) + deleted = kb_service.delete_space(kb_id, user.user_id) + if not deleted: + raise ResourceNotFoundError(resource_type="kb_space", resource_id=kb_id) + return success_response(data=None, message="Knowledge base deleted successfully") + + +@router.delete("/{kb_id}/documents/{document_id}", summary="删除知识库文档") +def delete_kb_document( + kb_id: str = Path(..., description="KB space ID"), + document_id: str = Path(..., description="Document ID"), + user: UserContext = Depends(get_current_user), + db: Session = Depends(get_db), +): + """从知识库空间删除指定文档及其分块与向量数据。仅文档所属空间的所有者可操作；文档不存在或无权访问时返回 404。此操作不可恢复。""" + kb_service = KBService(db) + deleted = kb_service.delete_document(document_id, user.user_id) + if not deleted: + raise ResourceNotFoundError(resource_type="kb_document", resource_id=document_id) + return success_response(data=None, message="Document deleted successfully") + + +@router.post("/{kb_id}/documents/{document_id}/reindex", summary="重新索引文档") +def reindex_document( + kb_id: str = Path(..., description="KB space ID"), + document_id: str = Path(..., description="Document ID"), + request: Optional[ReindexRequest] = None, + user: UserContext = Depends(get_current_user), + db: Session = Depends(get_db), +): + """对指定文档重新建立索引。会先清除该文档的旧分块及向量数据，再从存储重新下载原文，按可选的索引配置与分块方法异步重新解析、分块、向量化。仅空间所有者可操作，立即返回 indexing_status=processing。""" + from features.knowledge.models import KBChunk + from features.knowledge.repository import KBRepository + + kb_repo = KBRepository(db) + kb_space = kb_repo.get_space(kb_id) + if not kb_space: + raise ResourceNotFoundError(resource_type="kb_space", resource_id=kb_id) + kb_helpers.require_kb_write(db, kb_space, user.user_id, "edit") + + document = kb_repo.get_document(document_id) + if not document or document.kb_id != kb_id: + raise ResourceNotFoundError(resource_type="kb_document", resource_id=document_id) + + db.query(KBChunk).filter(KBChunk.document_id == document_id, KBChunk.kb_id == kb_id).delete() + document.indexing_status = "processing" + db.commit() + + # 资产随分块一起重建：不清会在每次重索引后留下一份孤儿图（存储与表都涨）。 + try: + from features.knowledge.kb_assets import purge_document_assets + + purge_document_assets(db, document_id) + except Exception as exc: + logger.warning("Asset cleanup failed for reindex of %s: %s", document_id, exc) + + try: + from features.knowledge.vector_indexing import delete_by_document + + delete_by_document(document_id, user.user_id) + except Exception as exc: + logger.warning("Milvus cleanup failed for reindex of %s: %s", document_id, exc) + + _idx_cfg = None + if request and request.indexing_config: + _idx_cfg = request.indexing_config.model_dump() + if not _idx_cfg: + _space_extra = kb_space.extra_data if isinstance(kb_space.extra_data, dict) else {} + _idx_cfg = _space_extra.get("indexing_config") + + _reindex_chunk_method = ( + (request.chunk_method if request and request.chunk_method else None) + or kb_space.chunk_method + or "semantic" + ) + + # 与上传同一条路：排队 + 常驻 worker，原文由 worker 自己从存储取回。 + # 这里不再预先 download_bytes——几十 MB 的原文既没必要在请求里读，也不该跟着 + # 任务在内存里排队等着跑。 + kb_helpers.enqueue_document_indexing( + db, + document_id=document_id, + user_id=user.user_id, + chunk_method=_reindex_chunk_method, + indexing_config=_idx_cfg, + index_modes=index_modes_of(kb_space), + ) + + return success_response( + data={"document_id": document_id, "indexing_status": "processing"}, + message="Document re-indexing started", + ) diff --git a/src/backend/features/knowledge/api/kb_spaces.py b/src/backend/features/knowledge/api/kb_spaces.py new file mode 100644 index 00000000..82b17cfe --- /dev/null +++ b/src/backend/features/knowledge/api/kb_spaces.py @@ -0,0 +1,116 @@ +"""kb spaces responsibilities.""" +from __future__ import annotations +from fastapi import APIRouter +router = APIRouter() +from features.knowledge.edition import CreateKBSpaceRequest, PolishKBDescriptionRequest, UpdateKBSpaceRequest +from core.auth.backend import UserContext, get_current_user +from core.config.settings import settings +from features.knowledge.wiki.config import normalize_index_modes +from core.db.engine import get_db +from core.infra.exceptions import AccessDeniedError, BadRequestError, ResourceNotFoundError +from core.infra.responses import created_response, success_response +from features.knowledge.service import KBService +from fastapi import Depends, status +from sqlalchemy.orm import Session +from features.knowledge.api import kb_helpers + +@router.post("", status_code=status.HTTP_201_CREATED, summary="创建知识库空间") +def create_kb_space( + request: CreateKBSpaceRequest, + user: UserContext = Depends(get_current_user), + db: Session = Depends(get_db), +): + """创建归属当前登录用户的知识库空间；可见性规则由当前版本决定。""" + visibility = (request.visibility or "private").strip() + if visibility not in ("private", "public"): + raise BadRequestError( + message="visibility 仅支持 private / public", data={"field": "visibility"} + ) + if settings.edition.edition == "ce" and visibility != "private": + raise BadRequestError( + message="CE 仅支持私有知识库", + data={"field": "visibility", "allowed": ["private"]}, + ) + + # Permission check: require the creation permission matching the chosen visibility + from core.auth.capabilities import resolve_user_capabilities + + caps = resolve_user_capabilities(db, user.user_id) + required_cap = "can_create_public_kb" if visibility == "public" else "can_create_private_kb" + if not caps.get(required_cap, False): + label = "公有" if visibility == "public" else "私有" + raise AccessDeniedError(message="Access denied", reason=f"缺少「创建{label}知识库」权限") + + kb_service = KBService(db) + metadata = dict(request.metadata or {}) + if request.indexing_config: + metadata["indexing_config"] = request.indexing_config.model_dump() + + # 索引模式：rag（向量检索）/ wiki（LLM Wiki 图谱），两者同选即 LLM-Wiki 知识库。 + # 三种组合都会分块——wiki 省掉的是向量化，不是分块（引文标注要 chunk id、 + # 回溯原文要按 chunk 直取）。 + metadata["index_modes"] = normalize_index_modes(request.index_modes) + if request.wiki_config: + metadata["wiki_config"] = { + k: v for k, v in request.wiki_config.model_dump().items() if v is not None + } + + from features.knowledge.edition import initial_visibility_grants + + initial_grants = initial_visibility_grants(db, user.user_id, visibility) + + space = kb_service.create_space( + user_id=user.user_id, + name=request.name, + description=request.description, + chunk_method=request.chunk_method or "semantic", + metadata=metadata, + visibility=visibility, + initial_grants=initial_grants, + granted_by=user.user_id, + ) + return created_response(data=space, message="Knowledge base space created successfully") + + +@router.post("/polish-description", summary="AI 生成知识库简介") +async def polish_kb_description( + request: PolishKBDescriptionRequest, + user: UserContext = Depends(get_current_user), +): + """根据知识库名称和现有简介，调用大模型生成一段更利于智能路由判断的知识库简介。名称为必填，模型失败时回退到本地模板生成。仅返回润色后的简介文本，不修改任何知识库记录。""" + name = request.name.strip() + description = (request.description or "").strip() + if not name: + raise BadRequestError(message="Knowledge base name is required", data={"field": "name"}) + + polished = await kb_helpers.generate_kb_description(name, description) + return success_response( + data={"description": polished}, + message="Knowledge base description polished successfully", + ) + + +@router.patch("/{kb_id}", summary="更新知识库空间") +def update_kb_space( + kb_id: str, + request: UpdateKBSpaceRequest, + user: UserContext = Depends(get_current_user), + db: Session = Depends(get_db), +): + """更新指定知识库空间的名称、简介与索引模式。仅空间所有者可操作；空间不存在或无权访问时返回 404。 + + 改索引模式不会自动补建索引：新勾选 Wiki 后要调 `POST /{kb_id}/wiki/rebuild` + 才会把已有文档排进生成队列。 + """ + kb_service = KBService(db) + space = kb_service.update_space( + kb_id=kb_id, + user_id=user.user_id, + name=request.name, + description=request.description, + index_modes=request.index_modes, + wiki_config=(request.wiki_config.model_dump() if request.wiki_config else None), + ) + if not space: + raise ResourceNotFoundError(resource_type="kb_space", resource_id=kb_id) + return success_response(data=space, message="Knowledge base space updated successfully") diff --git a/src/backend/features/knowledge/api/kb_state.py b/src/backend/features/knowledge/api/kb_state.py new file mode 100644 index 00000000..dad6eb4f --- /dev/null +++ b/src/backend/features/knowledge/api/kb_state.py @@ -0,0 +1,8 @@ +"""kb state responsibilities.""" +from __future__ import annotations + + +MAX_FILE_SIZE = 100 * 1024 * 1024 # 100MB + + +MAX_PREVIEW_SIZE = 20 * 1024 * 1024 # 20MB diff --git a/src/backend/api/routes/v1/kb_wiki.py b/src/backend/features/knowledge/api/kb_wiki.py similarity index 93% rename from src/backend/api/routes/v1/kb_wiki.py rename to src/backend/features/knowledge/api/kb_wiki.py index faa7cf70..084013ed 100644 --- a/src/backend/api/routes/v1/kb_wiki.py +++ b/src/backend/features/knowledge/api/kb_wiki.py @@ -29,16 +29,16 @@ # 单页最多返回的 Wiki 页数，避免前端一次拉爆几千页的库 MAX_PAGE_SIZE = 200 # 图谱单次最多返回节点数——前端渲染性能的闸，与实现层共用 core 侧的定义 -from core.kb.wiki.config import MAX_GRAPH_NODES as MAX_GRAPH_LIMIT +from features.knowledge.wiki.config import MAX_GRAPH_NODES as MAX_GRAPH_LIMIT def _assert_readable(db: Session, user_id: str, kb_id: str) -> None: """自建库要过读权限；外接库的可见性由其实现模块的白名单负责。""" - from core.kb.wiki_router import is_local_kb + from features.knowledge.wiki_router import is_local_kb if not is_local_kb(kb_id): return - from core.auth.kb_permissions import resolve_local_kb_level + from features.knowledge.edition import resolve_local_kb_level if resolve_local_kb_level(db, user_id, kb_id) == "none": raise AccessDeniedError(message="Access denied", reason="无权访问该知识库") @@ -50,7 +50,7 @@ def resolve_wiki( db: Session = Depends(get_db), ): """取该知识库的 wiki 实现；无权访问 403，不支持 Wiki 则 404。""" - from core.kb.wiki_router import wiki_module_for + from features.knowledge.wiki_router import wiki_module_for _assert_readable(db, user.user_id, kb_id) module = wiki_module_for(kb_id) @@ -74,11 +74,11 @@ async def wiki_capability( ``supports_wiki`` 为真表示「平台上至少有一个 Wiki 源」——可能来自外接后端， 也可能来自任意一个勾选了 Wiki 的自建库。具体某个库有没有，看单库探测。 """ - from core.kb.wiki_router import supports_wiki + from features.knowledge.wiki_router import supports_wiki provider = "" try: - from core.kb.external_provider import get_provider_name + from features.knowledge.edition import get_provider_name provider = get_provider_name() except Exception: # noqa: BLE001 - 社区版下外接接缝可能整体缺席 @@ -97,7 +97,7 @@ def wiki_capability_for_kb( db: Session = Depends(get_db), ): """单库探测。无权访问按「不具备」返回，不泄露该库是否存在。""" - from core.kb.wiki_router import is_local_kb, supports_wiki + from features.knowledge.wiki_router import is_local_kb, supports_wiki try: _assert_readable(db, user.user_id, kb_id) @@ -110,7 +110,7 @@ def wiki_capability_for_kb( enabled = supports_wiki(kb_id) data = {"kb_id": kb_id, "supports_wiki": enabled, "generating": False} if enabled and is_local_kb(kb_id): - from core.kb.wiki.jobs import kb_job_summary + from features.knowledge.wiki.jobs import kb_job_summary data.update(kb_job_summary(db, kb_id)) return success_response(data=data, message="Wiki capability resolved") @@ -279,10 +279,10 @@ def wiki_rebuild( 用于两种场景：建库后才勾上 Wiki（此前上传的文档需要补生成），以及生成结果 不理想时想换个粒度重来。需要写权限——这会实际消耗模型额度。 """ - from core.auth.kb_permissions import has_kb_permission, resolve_local_kb_level - from core.db.models import KBDocument - from core.kb.wiki.jobs import enqueue_ingest - from core.kb.wiki_router import is_local_kb, local_wiki_enabled + from features.knowledge.edition import has_kb_permission, resolve_local_kb_level + from features.knowledge.models import KBDocument + from features.knowledge.wiki.jobs import enqueue_ingest + from features.knowledge.wiki_router import is_local_kb, local_wiki_enabled if not is_local_kb(kb_id): raise BadRequestError("只有自建知识库支持重新生成 Wiki") diff --git a/src/backend/features/knowledge/asset_captions.py b/src/backend/features/knowledge/asset_captions.py new file mode 100644 index 00000000..e8bfc816 --- /dev/null +++ b/src/backend/features/knowledge/asset_captions.py @@ -0,0 +1,181 @@ +"""asset captions responsibilities.""" +from __future__ import annotations +from typing import Sequence +import logging +logger = logging.getLogger(__name__) +from features.knowledge import kb_assets_state + +def vision_available() -> bool: + """当前是否有可用的图像理解模型（视觉桥）。""" + try: + from core.models.vision import is_available + + return is_available() + except Exception as exc: # noqa: BLE001 — 视觉桥不可用不该让索引失败 + logger.debug("视觉桥可用性判定失败，按不可用处理: %s", exc) + return False + + +def describe_batch(images: list, loop=None) -> list: + """把一批图交给视觉桥识别。返回与输入等长的结果列表（失败位为 None）。 + + 视觉桥是异步的，而索引是同步函数、跑在没有运行中事件循环的工作线程里。按可靠性 + 从高到低三级取宿主循环——只有跑在**宿主循环**上，视觉桥的证据缓存与并发信号量才 + 真的生效： + + 1. ``loop``：常驻索引 worker 显式传进来的调度循环。这是当前的生产路径——索引跑在 + worker 自己的 ``ThreadPoolExecutor`` 里（见 ``core/kb/index_worker.py``），那不是 + anyio 的工作线程，第 2 级在这里拿不到东西。 + 2. ``anyio.from_thread``：仍经 ``BackgroundTasks`` 进来的调用方（如我的空间同步产物 + 入库）跑在 Starlette 的 anyio 线程池里，天然带着回宿主循环的通路。 + 3. 临时循环 + **关掉证据缓存**：CLI / 脚本 / 测试。``core/infra/redis.py`` 的连接池是 + 模块级全局、绑定首次使用它的循环，让一个转瞬即逝的临时循环抢先创建它，会把整个 + 进程的 Redis 绑死在已销毁的循环上——那比"这次没吃到缓存"严重得多。 + """ + import asyncio + import functools + + from core.models.vision import get_vision_bridge + + bridge = get_vision_bridge() + + if loop is not None and not loop.is_closed(): + future = asyncio.run_coroutine_threadsafe( + bridge.describe_many(images, focus=kb_assets_state.CAPTION_FOCUS), loop + ) + return future.result() + + try: + import anyio.from_thread + + return anyio.from_thread.run( + functools.partial(bridge.describe_many, images, focus=kb_assets_state.CAPTION_FOCUS) + ) + except RuntimeError as exc: + logger.debug("视觉桥回宿主循环不可用，改用临时循环且不走缓存: %s", exc) + + async def _uncached() -> list: + tasks = [ + bridge.describe(data, mime, focus=kb_assets_state.CAPTION_FOCUS, use_cache=False) + for data, mime in images + ] + return list(await asyncio.gather(*tasks)) + + return asyncio.run(_uncached()) + + +def merge_text_content(existing: str, ocr_text: str) -> str: + """图注 + OCR 转写合成检索用文本，去重后截断。 + + 两者都留：图注是文档作者写的、往往点明了图的用途；OCR 是图里实际的字。 + """ + parts: list[str] = [] + for part in ((existing or "").strip(), (ocr_text or "").strip()): + if part and part not in parts: + parts.append(part) + merged = "\n".join(parts) + return merged[:kb_assets_state.MAX_TEXT_CONTENT_CHARS] + + +def evidence_digest(result) -> dict: + """留档进 ``extra_data`` 的证据摘要。 + + 刻意不存 ``layout.regions`` 与 ``ocr.lines``：整段文字已经进了 ``text_content``， + 再存一份逐行结构，一篇几百张图的文档会把 JSONB 撑得很难看。留下的是之后真会用到 + 的部分——实体/关系（可喂本体与 Wiki 层）、不确定项（人工复核时看）、模型元数据 + （换模型后判断哪些资产该重跑）。 + """ + evidence = result.evidence + return { + "summary": evidence.summary, + "scene": evidence.semantics.scene, + "intent": evidence.semantics.intent, + "entities": [e.model_dump(exclude_none=True) for e in evidence.semantics.entities], + "relations": [r.model_dump(exclude_none=True) for r in evidence.semantics.relations], + "uncertainty": list(evidence.uncertainty), + "meta": result.to_meta(), + } + + +def generate_captions(db, assets: Sequence, *, loop=None) -> int: + """给资产补上图像理解结果：``caption`` 写概述，``text_content`` 并入 OCR 转写。 + + 走的是 ``core/vision`` 视觉桥——和对话里"纯文本主模型读图"完全同一条通路、同一个 + ``vision`` 模型角色，因此知识库这边白得三样东西：按图片内容哈希的证据缓存（重新 + 索引同一篇文档不用再付一次钱）、三协议与结构化输出降级、以及"该角色只能指派多模态 + 模型"的硬约束。 + + 没有可用的视觉模型是正常状态，不是错误：资产保留文档自带的图注并照常入索引，之后 + 配好 ``vision`` 角色再回填即可，**不用重新解析文档**。 + """ + if not assets: + return 0 + + if not vision_available(): + logger.info("视觉桥不可用（未指派 vision 角色且主模型不支持读图），跳过图生文；图注仍会入索引") + for row in assets: + row.caption_status = "skipped" + db.commit() + return 0 + + from core.files.storage import get_storage + + storage = get_storage() + images = [a for a in assets if a.kind == "image"] + budget, overflow = images[:kb_assets_state.MAX_CAPTIONS_PER_DOC], images[kb_assets_state.MAX_CAPTIONS_PER_DOC:] + # 没轮到的显式标注，让"上限生效"在数据里看得见，而不是像视觉模型什么都没返回。 + for row in overflow + [a for a in assets if a.kind != "image"]: + row.caption_status = "skipped" + if overflow: + logger.warning( + "文档图片数 %d 超过单文档图生文上限 %d，超出部分仅用图注索引", + len(images), + kb_assets_state.MAX_CAPTIONS_PER_DOC, + ) + + done = 0 + for offset in range(0, len(budget), kb_assets_state.CAPTION_BATCH): + chunk = budget[offset : offset + kb_assets_state.CAPTION_BATCH] + payloads: list = [] + pending: list = [] + for row in chunk: + try: + data = storage.download_bytes(row.storage_key) + except Exception as exc: # noqa: BLE001 — 单张读不出来不该拖垮整篇 + logger.warning("资产 %s 读取失败，跳过图生文: %s", row.asset_id, exc) + row.caption_status = "failed" + continue + payloads.append((data, row.mime_type)) + pending.append(row) + + if not payloads: + continue + + try: + results = describe_batch(payloads, loop) + except Exception as exc: # noqa: BLE001 — 整批失败也只降级，正文索引已经好了 + logger.warning("视觉桥调用失败，该批 %d 张仅用图注索引: %s", len(pending), exc) + for row in pending: + row.caption_status = "failed" + continue + + for row, result in zip(pending, results): + if result is None: + row.caption_status = "failed" + continue + evidence = result.evidence + summary = (evidence.summary or "").strip() + row.caption = summary + row.text_content = merge_text_content(row.text_content, evidence.ocr.full_text) + meta = dict(row.extra_data or {}) + meta["vision"] = evidence_digest(result) + row.extra_data = meta + row.caption_status = "completed" if summary else "failed" + if summary: + done += 1 + + # 这里给 ``extra_data`` 赋的是**新字典**（不是原地改键），SQLAlchemy 的属性事件能 + # 直接捕获，不需要 flag_modified —— 那个只有原地改 JSON 才必须。 + db.commit() + logger.info("图生文完成 %d/%d", done, len(budget)) + return done diff --git a/src/backend/features/knowledge/asset_indexing.py b/src/backend/features/knowledge/asset_indexing.py new file mode 100644 index 00000000..10d2d244 --- /dev/null +++ b/src/backend/features/knowledge/asset_indexing.py @@ -0,0 +1,88 @@ +"""asset indexing responsibilities.""" +from __future__ import annotations +from typing import Sequence +import logging +logger = logging.getLogger(__name__) +from features.knowledge import kb_assets_state + +def asset_index_text(asset, *, title: str = "") -> str: + """The text an asset is retrieved by: 视觉桥概述 + 图注/OCR 转写 + 文档标题。 + + Audio/video reuse this unchanged — ``text_content`` becomes the ASR transcript. + """ + parts = [ + (asset.caption or "").strip(), + (asset.text_content or "").strip(), + (title or "").strip(), + ] + return "\n".join(p for p in parts if p) + + +def index_assets( + assets: Sequence, *, user_id: str, kb_id: str, document_id: str, title: str +) -> int: + """Write one ``row_type="image"`` row per asset into the existing collection. + + ``parent_chunk_id`` points at the chunk that contains the asset, so a hit here + resolves to that chunk's full text through the existing parent-fetch path. Assets + with no owning chunk (a standalone upload whose placeholder was lost) point at + themselves and fall back to the caption as their content. + """ + if not assets: + return 0 + + from features.knowledge.kb_vector_state import TAGS_FIELD_MAX_BYTES, TITLE_FIELD_MAX_BYTES + from core.models.embedding import embed_batch + from features.knowledge.kb_vector import text_to_sparse + from features.knowledge.vector_fields import truncate_utf8 + from features.knowledge.vector_indexing import upsert_rows + + indexable = [(a, asset_index_text(a, title=title)) for a in assets] + indexable = [(a, t) for a, t in indexable if t.strip()] + if not indexable: + return 0 + + try: + vectors = embed_batch([t for _, t in indexable]) + except Exception as exc: + logger.error("资产向量化失败，文档 %s 的图片不可检索: %s", document_id, exc) + return 0 + + rows = [] + for (asset, text), vector in zip(indexable, vectors): + rows.append( + { + "chunk_id": f"{kb_assets_state.ASSET_ROW_PREFIX}{asset.asset_id}", + "parent_chunk_id": asset.chunk_id or f"{kb_assets_state.ASSET_ROW_PREFIX}{asset.asset_id}", + "row_type": kb_assets_state.ROW_TYPE_IMAGE, + "user_id": user_id, + "kb_id": kb_id, + "document_id": document_id, + "title": truncate_utf8(title, TITLE_FIELD_MAX_BYTES), + "content": truncate_utf8(text), + "tags_text": truncate_utf8(asset.text_content or "", TAGS_FIELD_MAX_BYTES), + "chunk_index": asset.asset_index or 0, + "dense_embedding": vector, + "sparse_embedding": text_to_sparse(text), + } + ) + + BATCH = 100 + for i in range(0, len(rows), BATCH): + upsert_rows(rows[i : i + BATCH]) + logger.info("文档 %s 写入 %d 条资产检索行", document_id, len(rows)) + return len(rows) + + +def mark_vector_state(db, assets: Sequence, space: str, state: str) -> None: + """Record that these assets are indexed into a vector space. + + ``space`` is ``"text"`` for the caption/OCR rows written by :func:`index_assets`. + 路径 B 落地时用 ``"visual"``，与文本行互不覆盖。 + """ + for row in assets: + current = dict(row.vector_state or {}) + current[space] = state + # 同上：赋新字典而不是原地改键，属性事件自然捕获得到。 + row.vector_state = current + db.commit() diff --git a/src/backend/features/knowledge/catalog_options.py b/src/backend/features/knowledge/catalog_options.py new file mode 100644 index 00000000..08d50fc0 --- /dev/null +++ b/src/backend/features/knowledge/catalog_options.py @@ -0,0 +1,7 @@ +"""Knowledge options for the administrative agent capability picker.""" +from features.knowledge.models import KBSpace + + +def list_options(db): + spaces = db.query(KBSpace).filter(KBSpace.deleted_at.is_(None)).order_by(KBSpace.created_at.desc()).all() + return [{"id": space.kb_id, "name": space.name, "description": space.description or ""} for space in spaces] diff --git a/src/backend/core/kb/external_provider.py b/src/backend/features/knowledge/disabled_provider.py similarity index 85% rename from src/backend/core/kb/external_provider.py rename to src/backend/features/knowledge/disabled_provider.py index f522757d..1c81453e 100644 --- a/src/backend/core/kb/external_provider.py +++ b/src/backend/features/knowledge/disabled_provider.py @@ -37,8 +37,8 @@ def supports_wiki() -> bool: """社区版没有外接知识后端，因此这一路不提供 Wiki。 注意这**不代表社区版没有 Wiki**：自建知识库勾选 Wiki 索引模式后，页面由本地 - 管线生成，走 ``core.kb.wiki.local_provider``。要问「某个库有没有 Wiki」应当用 - ``core.kb.wiki_router.supports_wiki(kb_id)``，它会把两条来源都算上。 + 管线生成，走 ``features.knowledge.wiki.local_provider``。要问「某个库有没有 Wiki」应当用 + ``features.knowledge.wiki_router.supports_wiki(kb_id)``，它会把两条来源都算上。 """ return False diff --git a/src/backend/core/kb/external_retrieval.py b/src/backend/features/knowledge/disabled_retrieval.py similarity index 100% rename from src/backend/core/kb/external_retrieval.py rename to src/backend/features/knowledge/disabled_retrieval.py diff --git a/src/backend/core/kb/document_status.py b/src/backend/features/knowledge/document_status.py similarity index 100% rename from src/backend/core/kb/document_status.py rename to src/backend/features/knowledge/document_status.py diff --git a/src/backend/features/knowledge/edition.py b/src/backend/features/knowledge/edition.py new file mode 100644 index 00000000..dac7fe11 --- /dev/null +++ b/src/backend/features/knowledge/edition.py @@ -0,0 +1,33 @@ +"""Community knowledge-base grant policy: resources are owner-only.""" + +from __future__ import annotations + +from sqlalchemy.orm import Session + + +def initial_visibility_grants(db: Session, user_id: str, visibility: str) -> list[tuple[str, str]]: + return [] + + +__all__ = ["initial_visibility_grants"] + + +def grant_initial_access(db, kb_id, grants, *, granted_by): + if grants: + raise ValueError("Community knowledge bases do not support shared grants") + + +def __getattr__(name): + if name in ('KBLevel', 'filter_accessible_kb_ids', 'get_accessible_local_kb_ids', 'get_accessible_local_kb_levels', 'get_dataset_levels', 'has_kb_permission', 'is_shared_visibility', 'level_to_caps', 'resolve_local_kb_level'): + import features.knowledge.personal_permissions as provider + return getattr(provider, name) + if name in ('get_allowed_collection_ids', 'get_document_detail', 'get_provider_cache_identity', 'get_provider_name', 'is_enabled', 'list_collections', 'list_documents', 'runtime_request_context', 'supports_wiki', 'wiki_module'): + import features.knowledge.disabled_provider as provider + return getattr(provider, name) + if name in ('DatasetRetrievalTimeoutError', 'DatasetRetrievalUnavailableError', 'MAX_RETRIEVE_TOKENS', 'RETRIEVE_MAX_CONCURRENCY', 'RETRIEVE_REQUEST_TIMEOUT_SECONDS', 'RETRIEVE_TOTAL_TIMEOUT_SECONDS', 'list_external_datasets', 'retrieve_dataset_content', 'retrieve_dataset_content_async'): + import features.knowledge.disabled_retrieval as provider + return getattr(provider, name) + if name in ('CreateKBSpaceRequest', 'IndexingConfig', 'PolishKBDescriptionRequest', 'ReindexRequest', 'UpdateChunkRequest', 'UpdateKBSpaceRequest'): + import features.knowledge.personal_request_models as provider + return getattr(provider, name) + raise AttributeError(name) diff --git a/src/backend/features/knowledge/enrichment.py b/src/backend/features/knowledge/enrichment.py new file mode 100644 index 00000000..096f66e0 --- /dev/null +++ b/src/backend/features/knowledge/enrichment.py @@ -0,0 +1,126 @@ +"""enrichment responsibilities.""" +from __future__ import annotations +import os +import re +import logging +logger = logging.getLogger(__name__) + +def strip_think_tags(text: str) -> str: + """Remove ... blocks and markdown formatting from LLM output.""" + # Remove ... blocks (reasoning model output) + text = re.sub(r'.*?', '', text, flags=re.DOTALL).strip() + # Remove markdown bold/italic markers + text = re.sub(r'\*+', '', text) + # Remove markdown headers + text = re.sub(r'^#+\s*', '', text, flags=re.MULTILINE) + # Remove bullet points + text = re.sub(r'^[\-\*]\s*', '', text, flags=re.MULTILINE) + return text.strip() + + +def resolve_main_endpoint(): + """main_agent 端点，带上它说的协议——KB 的两处抽取都得走对那条线。""" + from core.models.completions.single_turn import Endpoint + + try: + from core.models.config import ModelConfigService + + cfg = ModelConfigService.get_instance().resolve("main_agent") + if cfg and cfg.base_url and cfg.model_name: + return Endpoint.from_resolved(cfg) + except Exception: + pass + base_url, api_key, model_name = resolve_main_model_config() + return Endpoint(base_url=base_url, api_key=api_key, model_name=model_name) + + +def resolve_main_model_config() -> tuple[str, str, str]: + """Resolve main_agent model config from DB, with env fallback.""" + try: + from core.models.config import ModelConfigService + cfg = ModelConfigService.get_instance().resolve("main_agent") + if cfg: + return cfg.base_url.rstrip("/"), cfg.api_key, cfg.model_name + except Exception: + pass + return ( + os.getenv("MODEL_URL", "").rstrip("/"), + os.getenv("API_KEY", ""), + os.getenv("BASE_MODEL_NAME", ""), + ) + + +def call_main_model( + *, system: str, user: str, temperature: float, max_tokens: int, timeout: int = 30 +) -> str: + """跑一次主模型补全；未配置时返回空串，调用方按「这一步没产出」处理。""" + import requests as _requests + + from core.models.completions.single_turn import build_request, parse_response, turns + + endpoint = resolve_main_endpoint() + if not endpoint.base_url or not endpoint.model_name: + return "" + url, headers, payload = build_request( + endpoint, turns(user, system), temperature=temperature, max_tokens=max_tokens + ) + resp = _requests.post(url, headers=headers, json=payload, timeout=timeout) + resp.raise_for_status() + return parse_response(endpoint, resp.json()).text + + +def extract_keywords_llm(content: str, count: int) -> list[str]: + """Call configured LLM to extract keywords from a text chunk.""" + try: + raw = strip_think_tags( + call_main_model( + system=( + "你是关键词提取工具。" + "输出规则：只输出关键词，用英文逗号分隔，不要编号，不要解释，不要分析过程。" + "示例输出：数字化转型,产业升级,人工智能" + ), + user=f"从以下文本提取{count}个关键词，直接输出逗号分隔的关键词：\n\n{content[:2000]}", + temperature=0.1, + max_tokens=200, + ) + ) + lines = [ln.strip() for ln in raw.split("\n") if ln.strip()] + best_line = raw + for line in reversed(lines): + if ("," in line or "，" in line) and len(line) < 500: + best_line = line + break + keywords = [kw.strip() for kw in best_line.replace("，", ",").split(",") if kw.strip() and len(kw.strip()) < 30] + return keywords[:count] + except Exception as exc: + logger.warning("LLM keyword extraction failed: %s", exc) + return [] + + +def generate_questions_llm(content: str, count: int) -> list[str]: + """Call configured LLM to generate retrieval questions for a text chunk.""" + try: + raw = strip_think_tags( + call_main_model( + system=( + "你是问题生成工具。" + "输出规则：每行一个问题，不要编号，不要解释，不要分析过程。直接输出问题本身。" + "示例输出：\n什么是数字化转型？\n如何申报产业升级项目？" + ), + user=f"根据以下文本生成{count}个检索问题，每行一个：\n\n{content[:2000]}", + temperature=0.3, + max_tokens=500, + ) + ) + questions = [] + for q in raw.split("\n"): + q = q.strip().lstrip("0123456789.、）)-— ").strip() + if not q or len(q) < 5: + continue + if q.startswith(("分析", "角色", "任务", "输出", "规则", "示例")): + continue + questions.append(q) + return questions[:count] + except Exception as exc: + logger.warning("LLM question generation failed: %s", exc) + return [] diff --git a/src/backend/features/knowledge/extension.py b/src/backend/features/knowledge/extension.py new file mode 100644 index 00000000..0dc313a1 --- /dev/null +++ b/src/backend/features/knowledge/extension.py @@ -0,0 +1,19 @@ +"""Installed knowledge models and runtime composition.""" +NAME = "knowledge" +def setup(): + return True + + +def register_catalog_options(): + from core.harness.registry.catalog_options import register_catalog_options as register + from features.knowledge.catalog_options import list_options + register("kb_spaces", list_options) + + +def register_retrieval_context(): + from core.harness.registry.retrieval_context import register_retrieval_access_filter + from features.knowledge.edition import filter_accessible_kb_ids + register_retrieval_access_filter(filter_accessible_kb_ids) + from core.harness.registry.retrieval_context import register_retrieval_context as register + from features.knowledge import edition as external_provider + register(lambda selected_ids: external_provider.runtime_request_context(selected_ids)) diff --git a/src/backend/core/kb/index_queue.py b/src/backend/features/knowledge/index_queue.py similarity index 99% rename from src/backend/core/kb/index_queue.py rename to src/backend/features/knowledge/index_queue.py index 9450aaaf..8c1a74f8 100644 --- a/src/backend/core/kb/index_queue.py +++ b/src/backend/features/knowledge/index_queue.py @@ -31,7 +31,7 @@ from datetime import datetime, timedelta, timezone from typing import Any, Dict, List, Optional, Sequence -from core.db.models import KBDocument +from features.knowledge.models import KBDocument from core.infra.logging import get_logger from sqlalchemy.orm import Session from sqlalchemy.orm.attributes import flag_modified @@ -231,7 +231,7 @@ def _params_from_space(db: Session, kb_id: str) -> Dict[str, Any]: 所以必须给出**合理**的默认——而不是硬编码一个可能与该库分块方式不符的值， 也不能把 ``user_id`` 留空（向量行按它打标）。 """ - from core.db.models import KBSpace + from features.knowledge.models import KBSpace space = db.query(KBSpace).filter(KBSpace.kb_id == kb_id).first() if space is None: diff --git a/src/backend/core/kb/index_worker.py b/src/backend/features/knowledge/index_worker.py similarity index 97% rename from src/backend/core/kb/index_worker.py rename to src/backend/features/knowledge/index_worker.py index bfcd81f5..ff6abdf7 100644 --- a/src/backend/core/kb/index_worker.py +++ b/src/backend/features/knowledge/index_worker.py @@ -10,7 +10,7 @@ ``ThreadPoolExecutor``，并发上限由 ``KB_INDEX_CONCURRENCY`` 控制：索引再多也只会 排在自己的队里，碰不到请求线程池，也不会把 DB 连接池顶穿。 -队列语义与认领/心跳/回收见 :mod:`core.kb.index_queue`。 +队列语义与认领/心跳/回收见 :mod:`features.knowledge.index_queue`。 """ from __future__ import annotations @@ -24,7 +24,7 @@ from typing import Dict, Optional from core.infra.logging import get_logger -from core.kb import index_queue +from features.knowledge import index_queue logger = get_logger(__name__) @@ -165,8 +165,8 @@ def _run_one(self, snapshot: dict, loop: "asyncio.AbstractEventLoop | None" = No ``loop`` 是调度这次索引的宿主事件循环，用于把异步子系统（当前是图像理解的 视觉桥）的协程投回主循环执行。 """ - from core.content.kb_processing import vectorise_document_background - from core.storage import get_storage + from features.knowledge.processing import vectorise_document_background + from core.files.storage import get_storage document_id = snapshot["document_id"] try: diff --git a/src/backend/features/knowledge/kb_assets.py b/src/backend/features/knowledge/kb_assets.py new file mode 100644 index 00000000..6a76d6f5 --- /dev/null +++ b/src/backend/features/knowledge/kb_assets.py @@ -0,0 +1,231 @@ +"""kb assets responsibilities.""" +from __future__ import annotations +import os +import uuid +from typing import Iterable, Optional, Sequence +from features.knowledge.parser_types import RawAsset +from features.knowledge.parser_extract import image_extension_for_mime +import logging +logger = logging.getLogger(__name__) +from features.knowledge import kb_assets_state + +def multimodal_indexing_enabled(indexing_config: Optional[dict]) -> bool: + """Whether to extract and index media for this document. + + Per-KB opt-out via ``indexing_config.multimodal_indexing``; the global default + lives in ``KB_MULTIMODAL_INDEXING`` so an operator can turn the extra parse cost + off fleet-wide without touching each knowledge base. + """ + if isinstance(indexing_config, dict) and indexing_config.get("multimodal_indexing") is not None: + return bool(indexing_config.get("multimodal_indexing")) + return (os.getenv("KB_MULTIMODAL_INDEXING", "true") or "").strip().lower() not in ( + "false", + "0", + "no", + ) + + +def asset_public_url(asset_id: str) -> str: + """Frontend-resolvable URL for an asset. Matches the route in ``api/routes/v1/kb.py``. + + The ``/api`` prefix mirrors the frontend's ``VITE_API_BASE_URL`` default (nginx + strips it before the backend sees the path). It is written into stored chunk text, + so a deployment that serves the API elsewhere overrides ``KB_ASSET_URL_PREFIX`` + before indexing rather than rewriting rows afterwards. + """ + prefix = (os.getenv("KB_ASSET_URL_PREFIX", "/api") or "").rstrip("/") + return f"{prefix}/v1/catalog/kb/assets/{asset_id}" + + +def is_worth_indexing(asset: RawAsset) -> bool: + """Filter out layout noise (rules, bullets, logos) before it costs a vision call.""" + if len(asset.data) < kb_assets_state.MIN_ASSET_BYTES: + return False + try: + import io + + from PIL import Image # optional dependency (transitive via python-pptx) + + with Image.open(io.BytesIO(asset.data)) as im: + width, height = im.size + if min(width, height) < kb_assets_state.MIN_ASSET_EDGE_PX: + return False + except Exception: + # Pillow missing or the bytes are not a decodable image — the byte-size floor + # above is the fallback filter; never drop an asset because probing failed. + pass + return True + + +def locate_parent_chunk(parents: Sequence, placeholder: str) -> Optional[object]: + """Find the parent chunk whose text references this asset.""" + for parent in parents: + if placeholder and placeholder in (parent.content or ""): + return parent + return None + + +def rewrite_placeholder(parents: Sequence, placeholder: str, url: str) -> None: + """Point the markdown image link at the stored asset. + + Before this, ``![](images/.jpg)`` in a chunk was a dead link — the layout + service's temp path, with the bytes discarded. Rewriting in both the parent text + and its children keeps the stored chunk, the embedded text, and what the UI renders + consistent. + """ + needle = f"]({placeholder})" + replacement = f"]({url})" + for parent in parents: + if needle in (parent.content or ""): + parent.content = parent.content.replace(needle, replacement) + for child in getattr(parent, "children", []) or []: + if needle in (child.content or ""): + child.content = child.content.replace(needle, replacement) + + +def persist_assets( + db, + *, + kb_id: str, + document_id: str, + user_id: str, + assets: Sequence[RawAsset], + parents: Sequence, +) -> list: + """Store asset bytes, create ``kb_assets`` rows, and link them to parent chunks. + + Must run **before** the parent chunks are embedded and written: it rewrites the + placeholder links inside ``parents`` in place, and the rewritten text is what gets + stored in PostgreSQL and vectorised. + + Failures are per-asset — a storage error on one figure must not fail the document, + whose text is already parsed and perfectly usable. + """ + from features.knowledge.models import KBAsset + from core.files.storage import generate_storage_key, get_storage + + if not assets: + return [] + + env = os.getenv("ENVIRONMENT", "dev") + storage = get_storage() + created: list = [] + + for asset in assets: + if not is_worth_indexing(asset): + continue + + asset_id = uuid.uuid4().hex[:32] + suffix = image_extension_for_mime(asset.mime_type) if asset.kind == "image" else ".bin" + storage_key = generate_storage_key( + env=env, + user_id=user_id, + category="kb_assets", + filename=f"{asset_id}{suffix}", + ) + try: + storage.upload_bytes(asset.data, storage_key) + except Exception as exc: + logger.warning("资产 %s 落盘失败，跳过: %s", asset.placeholder, exc) + continue + + parent = locate_parent_chunk(parents, asset.placeholder) + row = KBAsset( + asset_id=asset_id, + kb_id=kb_id, + document_id=document_id, + chunk_id=getattr(parent, "parent_id", None), + kind=asset.kind, + mime_type=asset.mime_type, + storage_key=storage_key, + size_bytes=len(asset.data), + asset_index=asset.index, + locator=asset.locator or {}, + text_content=asset.text_content or "", + caption_status="pending", + vector_state={}, + extra_data={"placeholder": asset.placeholder}, + ) + db.add(row) + created.append(row) + rewrite_placeholder(parents, asset.placeholder, asset_public_url(asset_id)) + + if created: + db.commit() + for row in created: + db.refresh(row) + logger.info("文档 %s 抽出并落盘 %d 个资产", document_id, len(created)) + return created + + +def purge_document_assets(db, document_id: str) -> int: + """Delete a document's assets (storage objects + rows). Used on re-index.""" + from features.knowledge.models import KBAsset + + rows = db.query(KBAsset).filter(KBAsset.document_id == document_id).all() + if not rows: + return 0 + + try: + from core.files.storage import get_storage + + storage = get_storage() + for row in rows: + try: + storage.delete(row.storage_key) + except Exception as exc: + logger.warning("资产 %s 存储清理失败: %s", row.asset_id, exc) + except Exception as exc: + logger.warning("存储后端不可用，仅清理资产表: %s", exc) + + count = db.query(KBAsset).filter(KBAsset.document_id == document_id).delete() + db.commit() + return count + + +def fetch_assets_for_chunks(chunk_ids: Iterable[str], *, max_per_chunk: int = 4) -> dict: + """Map parent-chunk id → its assets, for attaching to retrieval results. + + Also keyed by the synthetic ``a_`` parent used when an asset has no + owning chunk, so a hit on such a row still resolves to its own image. + """ + ids = [c for c in dict.fromkeys(chunk_ids) if c] + if not ids: + return {} + + direct = [c for c in ids if not c.startswith(kb_assets_state.ASSET_ROW_PREFIX)] + orphan_asset_ids = [c[len(kb_assets_state.ASSET_ROW_PREFIX) :] for c in ids if c.startswith(kb_assets_state.ASSET_ROW_PREFIX)] + + try: + from core.db.engine import SessionLocal + from features.knowledge.models import KBAsset + from sqlalchemy import or_ + + with SessionLocal() as db: + clauses = [] + if direct: + clauses.append(KBAsset.chunk_id.in_(direct)) + if orphan_asset_ids: + clauses.append(KBAsset.asset_id.in_(orphan_asset_ids)) + if not clauses: + return {} + rows = db.query(KBAsset).filter(or_(*clauses)).order_by(KBAsset.asset_index).all() + grouped: dict[str, list[dict]] = {} + for row in rows: + key = row.chunk_id or f"{kb_assets_state.ASSET_ROW_PREFIX}{row.asset_id}" + bucket = grouped.setdefault(key, []) + if len(bucket) >= max_per_chunk: + continue + bucket.append( + { + "asset_id": row.asset_id, + "kind": row.kind, + "url": asset_public_url(row.asset_id), + "caption": row.caption or row.text_content or "", + "locator": row.locator or {}, + } + ) + return grouped + except Exception as exc: + logger.warning("检索结果附图失败（不影响文本结果）: %s", exc) + return {} diff --git a/src/backend/features/knowledge/kb_assets_state.py b/src/backend/features/knowledge/kb_assets_state.py new file mode 100644 index 00000000..214b609c --- /dev/null +++ b/src/backend/features/knowledge/kb_assets_state.py @@ -0,0 +1,31 @@ +"""kb assets state responsibilities.""" +from __future__ import annotations +import os + +ROW_TYPE_IMAGE = "image" + + +ASSET_ROW_PREFIX = "a_" + + +MIN_ASSET_BYTES = int(os.getenv("KB_ASSET_MIN_BYTES", "4096")) + + +MAX_CAPTIONS_PER_DOC = int(os.getenv("KB_ASSET_CAPTION_MAX_PER_DOC", "200")) + + +MIN_ASSET_EDGE_PX = int(os.getenv("KB_ASSET_MIN_EDGE_PX", "64")) + + +CAPTION_BATCH = max(1, int(os.getenv("KB_ASSET_CAPTION_BATCH", "2"))) + + +MAX_TEXT_CONTENT_CHARS = max(200, int(os.getenv("KB_ASSET_TEXT_MAX_CHARS", "4000"))) + + +CAPTION_FOCUS = ( + "这是文档中的插图，识别结果会被向量化用于知识库检索。请重点覆盖：" + "图的类型（照片/流程图/架构图/折线图/柱状图/表格截图等）、" + "图中出现的全部文字与数字（坐标轴、图例、标签、单位）、" + "以及图所反映的关键结论或趋势。读不出来的一律写进 uncertainty，不要猜。" +) diff --git a/src/backend/features/knowledge/kb_parser.py b/src/backend/features/knowledge/kb_parser.py new file mode 100644 index 00000000..f4574932 --- /dev/null +++ b/src/backend/features/knowledge/kb_parser.py @@ -0,0 +1,116 @@ +"""kb parser responsibilities.""" +from __future__ import annotations +from typing import Callable, Optional +from features.knowledge import parser_chunks +from features.knowledge import parser_extract +from features.knowledge import parser_semantic +from features.knowledge import parser_types + +def build_parent_child_chunks( + paragraphs: list[dict], + parent_size: int = 1024, + child_size: int = 128, + overlap: int = 20, + chunk_method: str = "semantic", + embed_fn: Optional[Callable[[list[str]], list[list[float]]]] = None, + separators: Optional[list[str]] = None, + child_separators: Optional[list[str]] = None, +) -> list[parser_types.ParentChunk]: + """Build parent-child chunk pairs from extracted paragraphs. + + Parent chunks: large semantic units (~parent_size tokens), stored in PostgreSQL, + returned to LLM as full context. + Child chunks: small slices (~child_size tokens), vectorised in Milvus for retrieval. + + ``separators`` (parent-chunk separators): custom separator hierarchy that decides + parent-chunk boundaries; only effective on paths that go through recursive + splitting (the recursive method itself, and the recursive fallback of semantic + chunking for oversized parents); when empty, falls back to the default hierarchy. + ``child_separators`` (child-chunk separators): when a parent is split into + children, split on these separators (then pack by child_size); when empty, + children use the default fixed-length sliding-window split. The two are + independent and do not affect each other. + """ + seps = parser_chunks.normalize_separators(separators) + csep = parser_chunks.normalize_separators(child_separators) + if chunk_method == "qa": + parents = parser_chunks.build_qa_chunks(paragraphs) + elif chunk_method == "laws": + parents = parser_chunks.build_law_chunks(paragraphs, parent_size, child_size, overlap) + elif chunk_method == "recursive": + parents = parser_chunks.build_recursive_chunks(paragraphs, parent_size, child_size, overlap, separators=seps) + elif chunk_method == "embedding_semantic": + parents = parser_semantic.build_embedding_semantic_chunks( + paragraphs, parent_size, child_size, overlap, embed_fn=embed_fn, separators=seps, + ) + else: + # "semantic" / "structured" / default + parents = parser_chunks.build_structured_chunks(paragraphs, parent_size, child_size, overlap) + + # Child-chunk separators: re-split children uniformly after parents are built + # (overriding each builder's default fixed-length children), so we don't have to + # thread the same parameter into every builder — one post-processing step covers + # all chunking methods. + if csep: + parser_chunks.apply_child_separators(parents, child_size, csep) + return parents + + +def parse_and_chunk( + file_bytes: bytes, + mime_type: str, + chunk_method: str = "semantic", + parent_size: int = 1024, + child_size: int = 128, + overlap: int = 20, + embed_fn: Optional[Callable[[list[str]], list[list[float]]]] = None, + separators: Optional[list[str]] = None, + child_separators: Optional[list[str]] = None, +) -> list[parser_types.ParentChunk]: + """Full pipeline: extract text then build parent-child chunks.""" + parents, _ = parse_and_chunk_rich( + file_bytes, + mime_type, + chunk_method=chunk_method, + parent_size=parent_size, + child_size=child_size, + overlap=overlap, + embed_fn=embed_fn, + separators=separators, + child_separators=child_separators, + with_assets=False, + ) + return parents + + +def parse_and_chunk_rich( + file_bytes: bytes, + mime_type: str, + chunk_method: str = "semantic", + parent_size: int = 1024, + child_size: int = 128, + overlap: int = 20, + embed_fn: Optional[Callable[[list[str]], list[list[float]]]] = None, + separators: Optional[list[str]] = None, + child_separators: Optional[list[str]] = None, + with_assets: bool = True, +) -> tuple[list[parser_types.ParentChunk], list[parser_types.RawAsset]]: + """Same pipeline as :func:`parse_and_chunk`, also returning the document's media. + + The assets are returned unpersisted; the caller links them to parent chunks and + stores the bytes (see ``core/kb/kb_assets.py``). + """ + paragraphs, assets = parser_extract.extract_text_with_assets( + file_bytes, mime_type, with_assets=with_assets + ) + parents = build_parent_child_chunks( + paragraphs, + parent_size=parent_size, + child_size=child_size, + overlap=overlap, + chunk_method=chunk_method, + embed_fn=embed_fn, + separators=separators, + child_separators=child_separators, + ) + return parents, assets diff --git a/src/backend/features/knowledge/kb_parser_state.py b/src/backend/features/knowledge/kb_parser_state.py new file mode 100644 index 00000000..b0d082d2 --- /dev/null +++ b/src/backend/features/knowledge/kb_parser_state.py @@ -0,0 +1,34 @@ +"""kb parser state responsibilities.""" +from __future__ import annotations +import re + +MIME_TO_SUFFIX: dict[str, str] = { + "application/pdf": ".pdf", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document": ".docx", + "application/msword": ".doc", + "text/plain": ".txt", + "text/markdown": ".txt", + "text/x-markdown": ".txt", + "text/csv": ".csv", + "application/json": ".txt", +} + + +MD_IMAGE_RE = re.compile(r"!\[[^\]]*\]\(([^)\s]+)\)") + + +IMAGE_MIME_TO_EXT = { + "image/png": ".png", + "image/jpeg": ".jpg", + "image/webp": ".webp", + "image/gif": ".gif", +} + + +HEADING_SOFT_RATIO = 0.5 + + +RECURSIVE_SEPARATORS = ["\n\n", "\n", "。", ".", "；", ";", "，", ",", " "] + + +SEPARATOR_ESCAPES = {"\\n": "\n", "\\t": "\t", "\\r": "\r", "\\\\": "\\"} diff --git a/src/backend/features/knowledge/kb_vector.py b/src/backend/features/knowledge/kb_vector.py new file mode 100644 index 00000000..248797b0 --- /dev/null +++ b/src/backend/features/knowledge/kb_vector.py @@ -0,0 +1,149 @@ +"""kb vector responsibilities.""" +from __future__ import annotations +import json +from typing import Any +from features.knowledge import kb_vector_state +from features.knowledge import vector_client + +def hit_to_dict(ent, score: float) -> dict: + """Build the shared KB search-result row. ``ent`` is any entity accessor with + ``.get`` (a dict from Milvus Lite, or a Hit entity from hybrid search).""" + return { + "chunk_id": ent.get("chunk_id"), + "parent_chunk_id": ent.get("parent_chunk_id") or ent.get("chunk_id"), + "row_type": ent.get("row_type"), + "kb_id": ent.get("kb_id"), + "document_id": ent.get("document_id"), + "title": ent.get("title"), + "content": ent.get("content"), + "chunk_index": ent.get("chunk_index"), + "score": score, + } + + +def hybrid_search( + user_id: str, + kb_ids: list[str], + query: str, + query_vec: list[float], + top_k: int = 10, + *, + public_kb_ids: list[str] | None = None, + timeout: float | None = None, +) -> list[dict[str, Any]]: + """Hybrid search over dense + sparse vectors with RRF fusion. + + Searches both chunk rows and question rows simultaneously. + Results are sorted by fused score; dedup by parent_chunk_id is done by the caller. + + ``kb_ids`` are private spaces and are filtered by ``user_id`` (owner isolation). + ``public_kb_ids`` are admin-managed public spaces visible to everyone — they are + filtered by ``kb_id`` only (no owner check), since public KBs are global and + ``kb_id`` is globally unique. + """ + from pymilvus import AnnSearchRequest, MilvusClient, RRFRanker + + vector_client.get_or_create_collection(timeout=timeout) + client = vector_client.get_client() + + clauses: list[str] = [] + if kb_ids: + clauses.append(f'(user_id == "{user_id}" and kb_id in {json.dumps(kb_ids)})') + if public_kb_ids: + clauses.append(f"kb_id in {json.dumps(public_kb_ids)}") + if not clauses: + return [] + expr = " or ".join(clauses) + + output_fields = [ + "chunk_id", + "parent_chunk_id", + "row_type", + "kb_id", + "document_id", + "title", + "content", + "chunk_index", + ] + + # Milvus Lite (no-Docker local): no sparse field / hybrid search — fall back to + # a plain dense ANN search over the same collection. Same result shape. + if vector_client.is_lite(): + lite_res = client.search( + collection_name=kb_vector_state.COLLECTION_NAME, + data=[query_vec], + anns_field="dense_embedding", + search_params={"metric_type": "IP"}, + limit=top_k * 2, + filter=expr, + output_fields=output_fields, + timeout=timeout, + ) + return [ + hit_to_dict(hit.get("entity", hit), hit.get("distance", 0.0)) for hit in lite_res[0] + ] + + # Dense vector search request + dense_req = AnnSearchRequest( + data=[query_vec], + anns_field="dense_embedding", + param={"metric_type": "IP", "params": {"nprobe": 10}}, + limit=top_k * 3, + expr=expr, + ) + + # Sparse search request — bag-of-words vector + sparse_vec = text_to_sparse(query) + sparse_req = AnnSearchRequest( + data=[sparse_vec], + anns_field="sparse_embedding", + param={"metric_type": "IP"}, + limit=top_k * 3, + expr=expr, + ) + + results = client.hybrid_search( + collection_name=kb_vector_state.COLLECTION_NAME, + reqs=[dense_req, sparse_req], + ranker=RRFRanker(k=60), + limit=top_k * 2, + output_fields=output_fields, + timeout=timeout, + ) + + return [hit_to_dict(hit.entity, hit.score) for hit in results[0]] + + +def build_sparse_text(content: str, tags: list[str]) -> str: + """Build combined text for sparse vectorisation.""" + if not tags: + return content + tag_str = " ".join(f"[{t}]" for t in tags) + return f"{content} {tag_str}" + + +def text_to_sparse(text: str) -> dict[int, float]: + """Convert text to a bag-of-words sparse vector using term-frequency + hashing. + + Compatible with Milvus SPARSE_FLOAT_VECTOR field (v2.4+). + Each unique token is hashed to a dimension index; value = TF. + """ + import hashlib + import re + from collections import Counter + + # Simple tokenisation: split on non-alphanumeric (works for CJK + Latin) + tokens = re.findall(r"[\w\u4e00-\u9fff]+", text.lower()) + if not tokens: + return {0: 1.0} + + counts = Counter(tokens) + total = sum(counts.values()) + + sparse: dict[int, float] = {} + for token, cnt in counts.items(): + dim = int(hashlib.md5(token.encode()).hexdigest()[:8], 16) % kb_vector_state.SPARSE_DIM_SPACE + tf = cnt / total + sparse[dim] = sparse.get(dim, 0.0) + tf + + return sparse if sparse else {0: 1.0} diff --git a/src/backend/features/knowledge/kb_vector_state.py b/src/backend/features/knowledge/kb_vector_state.py new file mode 100644 index 00000000..974365e0 --- /dev/null +++ b/src/backend/features/knowledge/kb_vector_state.py @@ -0,0 +1,27 @@ +"""kb vector state responsibilities.""" +from __future__ import annotations +import os +from typing import Optional + +COLLECTION_NAME = "hugagent_kb_private" + + +TITLE_FIELD_MAX_BYTES = 500 + + +CONTENT_FIELD_MAX_BYTES = 4096 + + +TAGS_FIELD_MAX_BYTES = 1000 + + + + + + + + +VERIFIED_COLLECTION_DIM: Optional[int] = None + + +SPARSE_DIM_SPACE = 100_000 diff --git a/src/backend/features/knowledge/models.py b/src/backend/features/knowledge/models.py new file mode 100644 index 00000000..e61b0355 --- /dev/null +++ b/src/backend/features/knowledge/models.py @@ -0,0 +1,377 @@ +"""Knowledge spaces, source documents, assets, chunks and wiki metadata.""" +from core.db.engine import Base +from core.db.utc_datetime import UTCDateTime +from core.infra.time import utc_now +from sqlalchemy import JSON, BigInteger, CheckConstraint, Column, ForeignKey, Index, Integer, String, Text, text +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.orm import relationship +JSONType = JSON().with_variant(JSONB(), "postgresql") + + +class KBSpace(Base): + """Knowledge base space table.""" + + __tablename__ = "kb_spaces" + + kb_id = Column(String(64), primary_key=True) + user_id = Column( + String(64), ForeignKey("users_shadow.user_id", ondelete="CASCADE"), nullable=False + ) + name = Column(String(255), nullable=False) + description = Column(Text) + document_count = Column(Integer, default=0) + total_size_bytes = Column(BigInteger, default=0) + visibility = Column(String(16), nullable=False, default="private") + chunk_method = Column(String(32), nullable=False, default="semantic") + extra_data = Column("metadata", JSONType, default={}) + created_at = Column(UTCDateTime(timezone=True), default=utc_now) + updated_at = Column(UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now) + deleted_at = Column(UTCDateTime(timezone=True)) + + # Relationships + user = relationship("UserShadow") + documents = relationship("KBDocument", back_populates="kb_space", cascade="all, delete-orphan") + + __table_args__ = ( + CheckConstraint("length(name) >= 1 AND length(name) <= 255", name="kb_spaces_name_length"), + CheckConstraint("document_count >= 0", name="kb_spaces_document_count_check"), + CheckConstraint("total_size_bytes >= 0", name="kb_spaces_total_size_check"), + CheckConstraint( + "visibility IN ('public', 'private', 'scoped')", name="kb_spaces_visibility_check" + ), + Index("idx_kb_spaces_user_id", "user_id"), + Index("idx_kb_spaces_updated_at", "updated_at"), + Index( + "idx_kb_spaces_deleted", "deleted_at", postgresql_where=Column("deleted_at").isnot(None) + ), + Index("idx_kb_spaces_visibility", "visibility"), + ) + +class KBDocument(Base): + """Knowledge base document table.""" + + __tablename__ = "kb_documents" + + document_id = Column(String(64), primary_key=True) + kb_id = Column(String(64), ForeignKey("kb_spaces.kb_id", ondelete="CASCADE"), nullable=False) + title = Column(String(500), nullable=False) + filename = Column(String(500), nullable=False) + size_bytes = Column(BigInteger, nullable=False) + mime_type = Column(String(100), nullable=False) + storage_key = Column(Text, nullable=False) + storage_url = Column(Text) + checksum = Column(String(64)) + indexing_status = Column( + String(20), nullable=False, default="processing" + ) # processing | completed | failed + extra_data = Column("metadata", JSONType, default={}) + uploaded_at = Column(UTCDateTime(timezone=True), default=utc_now) + deleted_at = Column(UTCDateTime(timezone=True)) + + # Relationships + kb_space = relationship("KBSpace", back_populates="documents") + chunks = relationship("KBChunk", back_populates="document", cascade="all, delete-orphan") + + __table_args__ = ( + CheckConstraint("size_bytes > 0", name="kb_documents_size_check"), + CheckConstraint("length(filename) >= 1", name="kb_documents_filename_length"), + Index("idx_kb_documents_kb_id", "kb_id"), + Index("idx_kb_documents_uploaded_at", "uploaded_at"), + Index("idx_kb_documents_kb_uploaded", "kb_id", "uploaded_at"), + Index( + "idx_kb_documents_deleted", + "deleted_at", + postgresql_where=Column("deleted_at").isnot(None), + ), + Index("idx_kb_documents_metadata_gin", "metadata", postgresql_using="gin"), + ) + +class KBChunk(Base): + """Knowledge base chunk table - stores parent chunks for context retrieval. + + Each document is split into parent chunks (stored here) and child chunks + (vectorised in Milvus hugagent_kb_private collection). Retrieval finds child + chunks via vector search, then fetches the parent content from this table. + """ + + __tablename__ = "kb_chunks" + + chunk_id = Column(String(64), primary_key=True) + kb_id = Column(String(64), ForeignKey("kb_spaces.kb_id", ondelete="CASCADE"), nullable=False) + document_id = Column( + String(64), ForeignKey("kb_documents.document_id", ondelete="CASCADE"), nullable=False + ) + chunk_index = Column(Integer, nullable=False) + content = Column( + Text, nullable=False + ) # parent chunk original text, returned to the LLM on retrieval hit + tags = Column(JSONType, default=list) # tag list ["数字化转型", "申报条件"] + questions = Column(JSONType, default=list) # associated question list (array of strings) + char_start = Column(Integer) + char_end = Column(Integer) + created_at = Column(UTCDateTime(timezone=True), default=utc_now) + updated_at = Column(UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now) + + # Relationships + document = relationship("KBDocument", back_populates="chunks") + + __table_args__ = ( + Index("idx_kb_chunks_kb_id", "kb_id"), + Index("idx_kb_chunks_document_id", "document_id"), + Index("idx_kb_chunks_kb_doc", "kb_id", "document_id"), + ) + +class KBAsset(Base): + """Non-text media cut out of a knowledge-base document (图片 / 后续音视频). + + One row per figure the layout engine extracted (or per standalone uploaded image). + The bytes live in object storage under ``storage_key``; this table holds the + linkage (which document / which parent chunk), where it sits in the source + (``locator``), and the two text projections retrieval runs on: + + ``text_content`` literal text carried by the medium — 图注 / OCR 文字；音视频接入后是 ASR 转写。 + ``caption`` 语义描述，由 VLM 生成（未配置 vlm 角色时为空，检索退化为只用 text_content）。 + + Deliberately medium-agnostic so audio/video need no migration: ``kind`` widens, + ``locator`` holds ``{page_idx, bbox}`` for images and would hold ``{start_ms, + end_ms}`` for a transcript segment. + + ``vector_state`` records which vector spaces this asset has been indexed into + (``{"text": "ok"}`` today). Path B — a true multimodal embedding in its own + collection — adds a second key here instead of another column. + """ + + __tablename__ = "kb_assets" + + asset_id = Column(String(64), primary_key=True) + kb_id = Column(String(64), ForeignKey("kb_spaces.kb_id", ondelete="CASCADE"), nullable=False) + document_id = Column( + String(64), ForeignKey("kb_documents.document_id", ondelete="CASCADE"), nullable=False + ) + # Owning parent chunk. Nullable: an asset whose placeholder cannot be located in + # any chunk is still worth keeping (it stays retrievable on its own row). + chunk_id = Column(String(64)) + kind = Column(String(16), nullable=False, default="image") # image | audio | video + mime_type = Column(String(100), nullable=False, default="application/octet-stream") + storage_key = Column(Text, nullable=False) + size_bytes = Column(BigInteger, default=0) + asset_index = Column(Integer, nullable=False, default=0) # order within the document + locator = Column(JSONType, default=dict) # image: {page_idx, bbox}; av: {start_ms, end_ms} + caption = Column(Text) # VLM 语义描述 + text_content = Column(Text) # 图注 / OCR / ASR 转写 + caption_status = Column( + String(20), nullable=False, default="pending" + ) # pending | completed | skipped | failed + vector_state = Column(JSONType, default=dict) # {"text": "ok"} —— 路径 B 在此加键 + extra_data = Column("metadata", JSONType, default={}) + created_at = Column(UTCDateTime(timezone=True), default=utc_now) + updated_at = Column(UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now) + + __table_args__ = ( + CheckConstraint("kind IN ('image', 'audio', 'video')", name="kb_assets_kind_check"), + Index("idx_kb_assets_kb_id", "kb_id"), + Index("idx_kb_assets_document_id", "document_id"), + Index("idx_kb_assets_chunk_id", "chunk_id"), + ) + +WIKI_CATEGORY_MAX_DEPTH = 3 + +WIKI_PAGE_TYPE_SUMMARY = "summary" + +WIKI_PAGE_TYPE_ENTITY = "entity" + +WIKI_PAGE_TYPE_CONCEPT = "concept" + +WIKI_PAGE_TYPE_INDEX = "index" + +WIKI_PAGE_TYPE_SYNTHESIS = "synthesis" + +WIKI_PAGE_TYPE_COMPARISON = "comparison" + +WIKI_PAGE_TYPES = ( + WIKI_PAGE_TYPE_SUMMARY, + WIKI_PAGE_TYPE_ENTITY, + WIKI_PAGE_TYPE_CONCEPT, + WIKI_PAGE_TYPE_INDEX, + WIKI_PAGE_TYPE_SYNTHESIS, + WIKI_PAGE_TYPE_COMPARISON, +) + +WIKI_KNOWLEDGE_PAGE_TYPES = ( + WIKI_PAGE_TYPE_ENTITY, + WIKI_PAGE_TYPE_CONCEPT, + WIKI_PAGE_TYPE_SYNTHESIS, + WIKI_PAGE_TYPE_COMPARISON, +) + +WIKI_PAGE_TYPE_LABELS = { + WIKI_PAGE_TYPE_CONCEPT: "概念", + WIKI_PAGE_TYPE_ENTITY: "实体", + WIKI_PAGE_TYPE_SYNTHESIS: "综述", + WIKI_PAGE_TYPE_COMPARISON: "对比", + WIKI_PAGE_TYPE_SUMMARY: "文档摘要", + WIKI_PAGE_TYPE_INDEX: "总索引", +} + +WIKI_EDIT_SOURCE_PIPELINE = "pipeline" + +WIKI_EDIT_SOURCE_AGENT = "agent" + +WIKI_EDIT_SOURCE_USER = "user" + +class KBWikiFolder(Base): + """Wiki 目录节点（邻接表）。 + + ``parent_id`` 为空串表示根层。``path`` 是用 ``/`` 连接的名称链（物化路径）， + 只为展示与筛选服务；归属关系的真源永远是 ``parent_id``。空目录允许独立存在， + 这样用户可以先搭骨架再往里填页。 + """ + + __tablename__ = "kb_wiki_folders" + + folder_id = Column(String(64), primary_key=True) + kb_id = Column(String(64), ForeignKey("kb_spaces.kb_id", ondelete="CASCADE"), nullable=False) + parent_id = Column(String(64), nullable=False, default="") + name = Column(String(255), nullable=False) + path = Column(String(1024), nullable=False, default="") + depth = Column(Integer, nullable=False, default=0) + sort_order = Column(Integer, nullable=False, default=0) + created_at = Column(UTCDateTime(timezone=True), default=utc_now) + updated_at = Column(UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now) + deleted_at = Column(UTCDateTime(timezone=True)) + + __table_args__ = ( + Index("idx_kb_wiki_folders_kb_parent", "kb_id", "parent_id"), + Index( + "idx_kb_wiki_folders_unique_name", + "kb_id", + "parent_id", + "name", + unique=True, + postgresql_where=Column("deleted_at").is_(None), + ), + ) + +class KBWikiPage(Base): + """一个 Wiki 页面。 + + ``folder_id`` 是页面归属目录的唯一真源；``category_path`` / ``wiki_path`` / + ``depth`` / ``sort_order`` 都是从目录链派生出来的缓存投影，重建目录时会一并刷新。 + + 血缘两列的语义要分清：``source_refs`` 是该页取材自哪些文档 + （``kb_documents.document_id``），``chunk_refs`` 是具体哪几个父块支撑了页面内容 + （``kb_chunks.chunk_id``）。回溯原文按后者直取，前者用于文档删除时定位受影响页。 + """ + + __tablename__ = "kb_wiki_pages" + + page_id = Column(String(64), primary_key=True) + kb_id = Column(String(64), ForeignKey("kb_spaces.kb_id", ondelete="CASCADE"), nullable=False) + slug = Column(String(255), nullable=False) + title = Column(String(512), nullable=False, default="") + page_type = Column(String(32), nullable=False, default=WIKI_PAGE_TYPE_SUMMARY) + status = Column(String(32), nullable=False, default="published") + content = Column(Text, nullable=False, default="") + summary = Column(Text, nullable=False, default="") + + folder_id = Column(String(64), nullable=False, default="") + category_path = Column(JSONType, default=list) + wiki_path = Column(String(1024), nullable=False, default="") + depth = Column(Integer, nullable=False, default=0) + sort_order = Column(Integer, nullable=False, default=0) + + source_refs = Column(JSONType, default=list) # [document_id] + chunk_refs = Column(JSONType, default=list) # [chunk_id] + in_links = Column(JSONType, default=list) # [slug]，反向链接 + out_links = Column(JSONType, default=list) # [slug]，正向链接 + aliases = Column(JSONType, default=list) + page_metadata = Column(JSONType, default=dict) + + version = Column(Integer, nullable=False, default=1) + last_edit_source = Column(String(16), nullable=False, default=WIKI_EDIT_SOURCE_PIPELINE) + last_editor_id = Column(String(64), nullable=False, default="") + + created_at = Column(UTCDateTime(timezone=True), default=utc_now) + updated_at = Column(UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now) + deleted_at = Column(UTCDateTime(timezone=True)) + + __table_args__ = ( + CheckConstraint( + "status IN ('draft', 'published', 'archived')", name="kb_wiki_pages_status_check" + ), + # slug 在单个知识库内唯一，跨库可以重复 + Index( + "idx_kb_wiki_pages_kb_slug", + "kb_id", + "slug", + unique=True, + postgresql_where=Column("deleted_at").is_(None), + ), + Index("idx_kb_wiki_pages_kb_type", "kb_id", "page_type"), + Index("idx_kb_wiki_pages_folder", "kb_id", "folder_id"), + Index("idx_kb_wiki_pages_tree", "kb_id", "page_type", "wiki_path", "sort_order", "title"), + Index("idx_kb_wiki_pages_updated_at", "updated_at"), + ) + +class KBWikiJob(Base): + """Wiki 生成作业。 + + Wiki 生成是分钟级到小时级的长任务，挂在请求生命周期上（BackgroundTasks）会在 + 容器重启时丢一半，所以状态必须落库：worker 认领后写心跳，重启时按心跳超时回收 + 僵尸作业重跑。``progress`` 供前端直接渲染进度，不用另开接口。 + """ + + __tablename__ = "kb_wiki_jobs" + + job_id = Column(String(64), primary_key=True) + kb_id = Column(String(64), ForeignKey("kb_spaces.kb_id", ondelete="CASCADE"), nullable=False) + job_type = Column(String(20), nullable=False) # ingest | retract | finalize + payload = Column(JSONType, default=dict) # {"document_ids": [...]} + status = Column(String(20), nullable=False, default="pending") + attempt = Column(Integer, nullable=False, default=0) + max_attempts = Column(Integer, nullable=False, default=3) + last_error = Column(Text) + progress = Column(JSONType, default=dict) # {"stage": ..., "done": n, "total": m} + + claimed_by = Column(String(64)) + claimed_at = Column(UTCDateTime(timezone=True)) + heartbeat_at = Column(UTCDateTime(timezone=True)) + + created_at = Column(UTCDateTime(timezone=True), default=utc_now) + updated_at = Column(UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now) + finished_at = Column(UTCDateTime(timezone=True)) + + __table_args__ = ( + CheckConstraint( + "job_type IN ('ingest', 'retract', 'finalize')", name="kb_wiki_jobs_type_check" + ), + CheckConstraint( + "status IN ('pending', 'running', 'succeeded', 'failed', 'cancelled')", + name="kb_wiki_jobs_status_check", + ), + Index("idx_kb_wiki_jobs_status", "status", "created_at"), + Index("idx_kb_wiki_jobs_kb", "kb_id", "status"), + # 同一知识库只保留一个未完成的 finalize：多篇文档摄入完只需要收尾一次。 + # 条件里必须带 status，否则一次成功的 finalize 会永久占位、再也排不进下一次。 + # + # 两个方言的 where 都要给：只写 postgresql_where 时，SQLite 会把谓词整个 + # 丢掉、退化成 UNIQUE(kb_id)——那等于「一个知识库只能有一个作业」，本地 + # 开发和测试库上批量摄入会直接撞唯一约束。 + Index( + "idx_kb_wiki_jobs_pending_finalize", + "kb_id", + unique=True, + postgresql_where=text("job_type = 'finalize' AND status IN ('pending', 'running')"), + sqlite_where=text("job_type = 'finalize' AND status IN ('pending', 'running')"), + ), + ) + +# Register with the schema, including when the installed feature is runtime-disabled. +def _cleanup_user_knowledge(session, user): + for space in session.query(KBSpace).filter(KBSpace.user_id == user.user_id): + session.delete(space) + + +from core.harness.registry import register_user_cleanup + +register_user_cleanup(_cleanup_user_knowledge) diff --git a/src/backend/features/knowledge/parser_chunks.py b/src/backend/features/knowledge/parser_chunks.py new file mode 100644 index 00000000..82256381 --- /dev/null +++ b/src/backend/features/knowledge/parser_chunks.py @@ -0,0 +1,278 @@ +"""parser chunks responsibilities.""" +from __future__ import annotations +import json +import re +import uuid +from typing import Optional +from features.knowledge import kb_parser_state +from features.knowledge import parser_tokens +from features.knowledge import parser_types + +def apply_child_separators(parents: list[parser_types.ParentChunk], child_size: int, separators: list[str]) -> None: + """Re-split each parent's children by child_separators (replaces ``pc.children`` in place). + + Splits the parent body with the recursive separators and packs by child_size, + aligning child boundaries to the user-specified separators. + """ + for pc in parents: + texts = [s.strip() for s in recursive_split(pc.content, separators, child_size) if s.strip()] + pc.children = [ + parser_types.ChildChunk(child_id=f"{pc.parent_id}_{i}", content=c, index=i) + for i, c in enumerate(texts or [pc.content]) + ] + + +def new_id() -> str: + return uuid.uuid4().hex[:16] + + +def split_into_children(parent_text: str, child_size: int, overlap: int) -> list[str]: + """Split parent text into overlapping child chunks by token count.""" + words = parent_text.split() + if not words: + return [parent_text] + + # Use character-based approximation when tiktoken unavailable + chars_per_token = 2 # conservative for Chinese + child_chars = child_size * chars_per_token + overlap_chars = overlap * chars_per_token + + children = [] + start = 0 + while start < len(parent_text): + end = start + child_chars + chunk = parent_text[start:end].strip() + if chunk: + children.append(chunk) + if end >= len(parent_text): + break + start = end - overlap_chars + return children if children else [parent_text] + + +def make_parent_chunk(text: str, children_texts: list[str]) -> parser_types.ParentChunk: + pid = new_id() + children = [ + parser_types.ChildChunk(child_id=f"{pid}_{i}", content=c, index=i) + for i, c in enumerate(children_texts) + ] + return parser_types.ParentChunk(parent_id=pid, content=text, children=children) + + +def build_structured_chunks( + paragraphs: list[dict], + parent_size: int, + child_size: int, + overlap: int, +) -> list[parser_types.ParentChunk]: + """Structure-aware chunking. + + A heading is a "preferred break point", not a "forced break point": we only break + before the next heading once the buffer has accumulated to a certain fraction of + parent_size (``_HEADING_SOFT_RATIO``); otherwise consecutive subsections are + merged into the same parent chunk until it approaches parent_size. This keeps + the heading/paragraph-aligned structure awareness while making ``parent_size`` + actually effective — heading-dense documents are no longer cut into lots of tiny + pieces, and parents can yield enough children. + """ + result = [] + buffer: list[str] = [] + buffer_tokens = 0 + soft_threshold = parent_size * kb_parser_state.HEADING_SOFT_RATIO + + def _flush() -> None: + nonlocal buffer, buffer_tokens + parent_text = "\n".join(buffer) + result.append(make_parent_chunk(parent_text, split_into_children(parent_text, child_size, overlap))) + buffer = [] + buffer_tokens = 0 + + for para in paragraphs: + text = para["text"] + tokens = parser_tokens.count_tokens(text) + + # Hard cap: adding this paragraph would exceed parent_size, so break off the + # accumulated content first + if buffer and buffer_tokens + tokens > parent_size: + _flush() + # Soft boundary: break before a heading only when enough content has + # accumulated (>= a fraction of parent_size); otherwise merge this + # subsection into the current parent so parent_size is the real determinant + # of parent-chunk size + elif buffer and para.get("heading_level") and buffer_tokens >= soft_threshold: + _flush() + + buffer.append(text) + buffer_tokens += tokens + + if buffer: + _flush() + + return result + + +def build_law_chunks( + paragraphs: list[dict], + parent_size: int, + child_size: int, + overlap: int, +) -> list[parser_types.ParentChunk]: + """Split at article/chapter/section boundaries for legal documents.""" + result = [] + buffer: list[str] = [] + + article_pattern = re.compile(r'^第[一二三四五六七八九十百\d]+[条章节]') + + for para in paragraphs: + text = para["text"] + is_boundary = bool(article_pattern.match(text)) or para.get("heading_level") in (1, 2) + + if is_boundary and buffer: + parent_text = "\n".join(buffer) + result.append(make_parent_chunk(parent_text, split_into_children(parent_text, child_size, overlap))) + buffer = [] + + buffer.append(text) + + if buffer: + parent_text = "\n".join(buffer) + result.append(make_parent_chunk(parent_text, split_into_children(parent_text, child_size, overlap))) + + return result + + +def build_qa_chunks(paragraphs: list[dict]) -> list[parser_types.ParentChunk]: + """Each Q&A pair becomes one parent chunk with one child.""" + result = [] + for para in paragraphs: + text = para["text"] + pid = new_id() + child = parser_types.ChildChunk(child_id=f"{pid}_0", content=text, index=0) + result.append(parser_types.ParentChunk(parent_id=pid, content=text, children=[child])) + return result + + +def normalize_separators(seps: Optional[list[str]]) -> Optional[list[str]]: + """Normalize user-supplied custom separators into a usable hierarchy list. + + - Restore escapes like ``\\n`` / ``\\t``; + - Drop empty strings, de-duplicate while preserving order; + - Return None when all empty (callers fall back to the ``_RECURSIVE_SEPARATORS`` default hierarchy). + """ + if not seps: + return None + out: list[str] = [] + seen: set[str] = set() + for raw in seps: + if raw is None: + continue + s = str(raw) + for token, real in kb_parser_state.SEPARATOR_ESCAPES.items(): + s = s.replace(token, real) + if s == "" or s in seen: + continue + seen.add(s) + out.append(s) + return out or None + + +def parse_separators_form(raw: Optional[str]) -> Optional[list[str]]: + """Parse custom separators passed as a JSON-array string in a multipart form into a list. + + The preview endpoint takes Form params and cannot receive a list directly, so by + convention a JSON string is passed (e.g. ``["\\n\\n", "。"]``). On parse failure + or non-list input, silently return None (fall back to the default hierarchy) + instead of 500-ing over tuning input. + """ + if not raw or not raw.strip(): + return None + try: + data = json.loads(raw) + except (ValueError, TypeError): + return None + if not isinstance(data, list): + return None + return normalize_separators([str(x) for x in data]) + + +def recursive_split(text: str, separators: list[str], max_tokens: int) -> list[str]: + """Recursively split text using a hierarchy of separators. + + Tries to split on the first separator that produces segments. + Merges adjacent small segments up to max_tokens. + Over-sized segments recurse with the next separator level. + Final fallback: character-based truncation. + """ + if parser_tokens.count_tokens(text) <= max_tokens: + return [text] + + for i, sep in enumerate(separators): + parts = text.split(sep) + if len(parts) <= 1: + continue + + # Merge adjacent small parts up to max_tokens + result: list[str] = [] + current = "" + current_tokens = 0 + + for part in parts: + candidate = (current + sep + part) if current else part + candidate_tokens = parser_tokens.count_tokens(candidate) + + if candidate_tokens <= max_tokens: + current = candidate + current_tokens = candidate_tokens + else: + if current: + result.append(current) + # If this single part exceeds max_tokens, recurse deeper + part_tokens = parser_tokens.count_tokens(part) + if part_tokens > max_tokens: + deeper = recursive_split(part, separators[i + 1:], max_tokens) + result.extend(deeper) + current = "" + current_tokens = 0 + else: + current = part + current_tokens = part_tokens + + if current: + result.append(current) + + if result: + return result + + # Final fallback: character-based truncation + chars_per_token = 2 + chunk_chars = max_tokens * chars_per_token + result = [] + start = 0 + while start < len(text): + end = start + chunk_chars + chunk = text[start:end].strip() + if chunk: + result.append(chunk) + start = end + return result if result else [text] + + +def build_recursive_chunks( + paragraphs: list[dict], + parent_size: int, + child_size: int, + overlap: int, + separators: Optional[list[str]] = None, +) -> list[parser_types.ParentChunk]: + """Recursive multi-level separator chunking.""" + # Join all paragraph text + full_text = "\n".join(p["text"] for p in paragraphs) + segments = recursive_split(full_text, separators or kb_parser_state.RECURSIVE_SEPARATORS, parent_size) + + result = [] + for seg in segments: + seg = seg.strip() + if not seg: + continue + result.append(make_parent_chunk(seg, split_into_children(seg, child_size, overlap))) + return result diff --git a/src/backend/features/knowledge/parser_extract.py b/src/backend/features/knowledge/parser_extract.py new file mode 100644 index 00000000..e0596acd --- /dev/null +++ b/src/backend/features/knowledge/parser_extract.py @@ -0,0 +1,302 @@ +"""parser extract responsibilities.""" +from __future__ import annotations +import re +import uuid +from typing import Optional +import logging +logger = logging.getLogger(__name__) +from features.knowledge import kb_parser_state +from features.knowledge import parser_types + +def extract_text(file_bytes: bytes, mime_type: str) -> list[dict]: + """Extract structured paragraphs from a document. + + Delegates to core/file_parser.py for PDF, DOCX, DOC, TXT (uses the + external file-parser API for PDF, pandoc for DOCX, etc.). + Falls back to built-in xlsx/plain-text extraction for unsupported types. + + Returns a list of paragraph dicts: + {"text": str, "heading_level": int|None, "element_type": str} + """ + paragraphs, _ = extract_text_with_assets(file_bytes, mime_type, with_assets=False) + return paragraphs + + +def extract_text_with_assets( + file_bytes: bytes, + mime_type: str, + *, + with_assets: bool = True, +) -> tuple[list[dict], list[parser_types.RawAsset]]: + """Extract paragraphs and, when ``with_assets``, the media the document carries. + + Asset extraction is opt-in because it costs a heavier parse call: the layout + service inlines every figure as base64 in the JSON response, which the text-only + callers have no use for. When the rich call fails for any reason we fall back to + the plain text path — a parse service that doesn't support the media params must + degrade to today's behaviour, never fail the document. + """ + mt = (mime_type or "").lower() + + # Standalone image upload. Without this branch an image falls through to + # ``_extract_plain_text``, which latin-1 decodes the binary into garbage + # paragraphs and indexes them — the upload allowlist accepts png/jpg/webp/gif + # (``core/files/content/file_validation.py``) so this path is reachable in production. + if mt.startswith("image/"): + return extract_standalone_image(file_bytes, mt, with_assets=with_assets) + + # XLSX — keep built-in handler (file_parser doesn't support spreadsheets) + if mt in ( + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "application/vnd.ms-excel", + ): + return extract_xlsx(file_bytes), [] + + if with_assets and mt == "application/pdf": + rich = extract_pdf_rich(file_bytes) + if rich is not None: + return rich + + # Try core/file_parser for all other known types + suffix = kb_parser_state.MIME_TO_SUFFIX.get(mt) + if suffix: + try: + from core.files.content.file_parser import parse_file + filename = f"upload{suffix}" + content = parse_file(file_bytes, filename) + if content: + return markdown_to_paragraphs(content), [] + except RuntimeError: + # file_parser failed (e.g. service down) — fall through to built-in + pass + + # Fallback: treat as plain text + return extract_plain_text(file_bytes), [] + + +def extract_standalone_image( + file_bytes: bytes, + mime_type: str, + *, + with_assets: bool, +) -> tuple[list[dict], list[parser_types.RawAsset]]: + """An uploaded image is one asset with one synthetic paragraph holding its ref. + + The paragraph exists so the normal chunker still produces a parent chunk: retrieval + fetches parent content from PostgreSQL, and an image-only document with no chunk + would be unreachable. The caption/description is attached to the asset, not + inlined here — it is not known until the VLM pass has run. + """ + placeholder = f"asset://{uuid.uuid4().hex[:16]}" + paragraphs = [{"text": f"![]({placeholder})", "heading_level": None, "element_type": "image"}] + if not with_assets: + return paragraphs, [] + asset = parser_types.RawAsset( + data=file_bytes, + placeholder=placeholder, + kind="image", + mime_type=mime_type, + index=0, + ) + return paragraphs, [asset] + + +def extract_pdf_rich(file_bytes: bytes) -> Optional[tuple[list[dict], list[parser_types.RawAsset]]]: + """Parse a PDF keeping its figures. Returns None to signal "fall back to text-only".""" + try: + from core.files.content.pdf_parser import parse_pdf_rich + + result = parse_pdf_rich(file_bytes, "upload.pdf") + except Exception as exc: + logger.warning("PDF 富解析失败，退回纯文本解析: %s", exc) + return None + + paragraphs = markdown_to_paragraphs(result.markdown) + if not result.images: + return paragraphs, [] + + # Layout metadata (caption / bbox / page) keyed by the same path the markdown + # references, so each figure's asset carries where it came from. + meta_by_path: dict[str, dict] = {} + for block in result.blocks: + if block.get("type") != "image": + continue + path = str(block.get("img_path") or "") + if path: + meta_by_path[path] = block + + assets: list[parser_types.RawAsset] = [] + seen: set[str] = set() + # Iterate in document order: the markdown's image links are the source of truth for + # ordering, and only images actually referenced by the text get an asset. + for idx, path in enumerate(kb_parser_state.MD_IMAGE_RE.findall(result.markdown)): + if path in seen: + continue + data = result.images.get(path) + if not data: + continue + seen.add(path) + meta = meta_by_path.get(path, {}) + caption_parts = [ + str(x).strip() + for x in (meta.get("image_caption") or []) + (meta.get("image_footnote") or []) + if str(x).strip() + ] + locator = {} + if meta.get("page_idx") is not None: + locator["page_idx"] = meta.get("page_idx") + if meta.get("bbox"): + locator["bbox"] = meta.get("bbox") + assets.append( + parser_types.RawAsset( + data=data, + placeholder=path, + kind="image", + mime_type=guess_image_mime(path), + text_content=" ".join(caption_parts), + locator=locator, + index=idx, + ) + ) + + return paragraphs, assets + + +def guess_image_mime(path: str) -> str: + lowered = path.lower() + for mime, ext in kb_parser_state.IMAGE_MIME_TO_EXT.items(): + if lowered.endswith(ext): + return mime + if lowered.endswith(".jpeg"): + return "image/jpeg" + return "image/jpeg" + + +def image_extension_for_mime(mime_type: str) -> str: + """Filename suffix for an image mime, used when building the storage key.""" + return kb_parser_state.IMAGE_MIME_TO_EXT.get((mime_type or "").lower(), ".jpg") + + +def markdown_to_paragraphs(text: str) -> list[dict]: + """Parse markdown text (from file_parser) into structured paragraph dicts.""" + paragraphs: list[dict] = [] + table_buffer: list[str] = [] + + def _flush_table(): + if table_buffer: + paragraphs.append({ + "text": "\n".join(table_buffer), + "heading_level": None, + "element_type": "table", + }) + table_buffer.clear() + + for line in text.splitlines(): + stripped = line.strip() + + # Empty line — flush table buffer, skip + if not stripped: + _flush_table() + continue + + # Markdown table row + if stripped.startswith("|") and stripped.endswith("|"): + table_buffer.append(stripped) + continue + + # Not a table row — flush any pending table + _flush_table() + + # Markdown heading + md_match = re.match(r'^(#{1,6})\s+(.*)', stripped) + if md_match: + level = len(md_match.group(1)) + paragraphs.append({ + "text": stripped, + "heading_level": level, + "element_type": "heading", + }) + continue + + # Chinese legal/report heading patterns + heading_level = detect_heading_level(stripped) + if heading_level: + paragraphs.append({ + "text": stripped, + "heading_level": heading_level, + "element_type": "heading", + }) + continue + + # Regular paragraph + paragraphs.append({ + "text": stripped, + "heading_level": None, + "element_type": "paragraph", + }) + + _flush_table() + return paragraphs + + +def extract_xlsx(file_bytes: bytes) -> list[dict]: + import io + try: + import openpyxl + except ImportError: + raise RuntimeError("openpyxl is required: pip install openpyxl") + + wb = openpyxl.load_workbook(io.BytesIO(file_bytes), read_only=True, data_only=True) + paragraphs = [] + for sheet in wb.worksheets: + rows = list(sheet.iter_rows(values_only=True)) + if not rows: + continue + header = [str(c or "") for c in rows[0]] + paragraphs.append({ + "text": f"## Sheet: {sheet.title}", + "heading_level": 2, + "element_type": "heading", + }) + md_lines = ["| " + " | ".join(header) + " |"] + md_lines.append("|" + "|".join("---" for _ in header) + "|") + for row in rows[1:]: + if all(c is None for c in row): + continue + md_lines.append("| " + " | ".join(str(c or "") for c in row) + " |") + paragraphs.append({ + "text": "\n".join(md_lines), + "heading_level": None, + "element_type": "table", + }) + return paragraphs + + +def extract_plain_text(file_bytes: bytes) -> list[dict]: + """Fallback: decode as text and split into paragraphs.""" + # Try common encodings + for enc in ("utf-8", "gbk", "latin-1"): + try: + text = file_bytes.decode(enc) + break + except UnicodeDecodeError: + continue + else: + text = file_bytes.decode("utf-8", errors="replace") + + return markdown_to_paragraphs(text) + + +def detect_heading_level(line: str) -> Optional[int]: + """Detect Chinese legal/report heading patterns.""" + patterns = [ + (r'^第[一二三四五六七八九十百]+章', 1), + (r'^第[一二三四五六七八九十百]+节', 2), + (r'^第[一二三四五六七八九十百\d]+条', 3), + (r'^[一二三四五六七八九十]+[、．.]', 2), + (r'^\d+[、．.]\s*\S', 3), + ] + for pattern, level in patterns: + if re.match(pattern, line): + return level + return None diff --git a/src/backend/features/knowledge/parser_semantic.py b/src/backend/features/knowledge/parser_semantic.py new file mode 100644 index 00000000..c582d1f4 --- /dev/null +++ b/src/backend/features/knowledge/parser_semantic.py @@ -0,0 +1,130 @@ +"""parser semantic responsibilities.""" +from __future__ import annotations +import math +from typing import Callable, Optional +from features.knowledge import kb_parser_state +from features.knowledge import parser_chunks +from features.knowledge import parser_tokens +from features.knowledge import parser_types + +def cosine_similarity(a: list[float], b: list[float]) -> float: + """Compute cosine similarity between two vectors (pure Python).""" + dot = sum(x * y for x, y in zip(a, b)) + norm_a = math.sqrt(sum(x * x for x in a)) + norm_b = math.sqrt(sum(x * x for x in b)) + if norm_a == 0 or norm_b == 0: + return 0.0 + return dot / (norm_a * norm_b) + + +def build_embedding_semantic_chunks( + paragraphs: list[dict], + parent_size: int, + child_size: int, + overlap: int, + embed_fn: Optional[Callable[[list[str]], list[list[float]]]] = None, + separators: Optional[list[str]] = None, +) -> list[parser_types.ParentChunk]: + """Semantic chunking based on embedding similarity between adjacent blocks. + + Algorithm: + 1. Merge paragraphs into minimum ~50-token blocks + 2. Batch-embed all blocks + 3. Compute cosine similarity between adjacent blocks + 4. Set breakpoints where similarity drops below P25 or is a local minimum + 5. Merge blocks between breakpoints into parent chunks + 6. Fallback to recursive splitting for oversized parents + """ + if embed_fn is None: + # Fallback to structured chunking when no embed function available + return parser_chunks.build_structured_chunks(paragraphs, parent_size, child_size, overlap) + + # Step 1: Merge paragraphs into minimum-size blocks (~50 tokens) + MIN_BLOCK_TOKENS = 50 + blocks: list[str] = [] + current_block = "" + current_tokens = 0 + + for para in paragraphs: + text = para["text"] + tokens = parser_tokens.count_tokens(text) + if current_block and current_tokens + tokens > MIN_BLOCK_TOKENS: + blocks.append(current_block) + current_block = text + current_tokens = tokens + else: + current_block = (current_block + "\n" + text) if current_block else text + current_tokens += tokens + + if current_block: + blocks.append(current_block) + + if len(blocks) <= 1: + # Single block — just use structured + return parser_chunks.build_structured_chunks(paragraphs, parent_size, child_size, overlap) + + # Step 2: Batch embed + embeddings = embed_fn(blocks) + + # Step 3: Compute adjacent cosine similarities + similarities: list[float] = [] + for i in range(len(embeddings) - 1): + sim = cosine_similarity(embeddings[i], embeddings[i + 1]) + similarities.append(sim) + + if not similarities: + return parser_chunks.build_structured_chunks(paragraphs, parent_size, child_size, overlap) + + # Step 4: Find breakpoints + sorted_sims = sorted(similarities) + p25_idx = max(0, len(sorted_sims) // 4 - 1) + p25_threshold = sorted_sims[p25_idx] + + breakpoints: set[int] = set() + for i, sim in enumerate(similarities): + # Below P25 threshold + if sim <= p25_threshold: + breakpoints.add(i) + continue + # Local minimum: lower than both neighbors + is_local_min = True + if i > 0 and sim >= similarities[i - 1]: + is_local_min = False + if i < len(similarities) - 1 and sim >= similarities[i + 1]: + is_local_min = False + if is_local_min and sim < p25_threshold * 1.2: + breakpoints.add(i) + + # Step 5: Merge blocks between breakpoints into parent chunks + result: list[parser_types.ParentChunk] = [] + segment_start = 0 + + sorted_breaks = sorted(breakpoints) + # Add an artificial breakpoint at the end + sorted_breaks.append(len(blocks) - 1) + + for bp in sorted_breaks: + segment_blocks = blocks[segment_start:bp + 1] + parent_text = "\n".join(segment_blocks).strip() + segment_start = bp + 1 + + if not parent_text: + continue + + # Step 6: If parent is too large, use recursive split as fallback + if parser_tokens.count_tokens(parent_text) > parent_size * 1.5: + sub_segments = parser_chunks.recursive_split(parent_text, separators or kb_parser_state.RECURSIVE_SEPARATORS, parent_size) + for sub in sub_segments: + sub = sub.strip() + if sub: + result.append(parser_chunks.make_parent_chunk(sub, parser_chunks.split_into_children(sub, child_size, overlap))) + else: + result.append(parser_chunks.make_parent_chunk(parent_text, parser_chunks.split_into_children(parent_text, child_size, overlap))) + + # Handle remaining blocks + if segment_start < len(blocks): + remaining = "\n".join(blocks[segment_start:]).strip() + if remaining: + result.append(parser_chunks.make_parent_chunk(remaining, parser_chunks.split_into_children(remaining, child_size, overlap))) + + return result diff --git a/src/backend/features/knowledge/parser_tokens.py b/src/backend/features/knowledge/parser_tokens.py new file mode 100644 index 00000000..8a6fae1a --- /dev/null +++ b/src/backend/features/knowledge/parser_tokens.py @@ -0,0 +1,26 @@ +"""parser tokens responsibilities.""" +from __future__ import annotations + + +def get_tiktoken_enc(): + """Return cached tiktoken cl100k_base encoder.""" + global tiktoken_enc + if tiktoken_enc is None: + try: + import tiktoken + tiktoken_enc = tiktoken.get_encoding("cl100k_base") + except Exception: + pass + return tiktoken_enc + + +def count_tokens(text: str) -> int: + """Approximate token count using tiktoken cl100k_base.""" + enc = get_tiktoken_enc() + if enc is not None: + return len(enc.encode(text)) + # Fallback: rough Chinese/English estimate + return max(1, len(text) // 2) + + +tiktoken_enc = None diff --git a/src/backend/features/knowledge/parser_types.py b/src/backend/features/knowledge/parser_types.py new file mode 100644 index 00000000..ff72dc9c --- /dev/null +++ b/src/backend/features/knowledge/parser_types.py @@ -0,0 +1,43 @@ +"""parser types responsibilities.""" +from __future__ import annotations +from dataclasses import dataclass, field +from typing import Optional + +@dataclass +class ChildChunk: + child_id: str + content: str + index: int # position within parent + + +@dataclass +class ParentChunk: + parent_id: str + content: str # full parent text, returned to LLM on retrieval + children: list[ChildChunk] + char_start: Optional[int] = None + char_end: Optional[int] = None + + +@dataclass +class RawAsset: + """A non-text medium extracted from the source document, before persistence. + + ``placeholder`` is the exact token this medium is referenced by inside the + paragraph text (``images/.jpg`` for a layout-extracted figure, a synthetic + ``asset://`` for a standalone upload). It is what links the medium back to + the parent chunk that contains it, and what gets rewritten to a real URL once the + asset has an id. + + Medium-agnostic on purpose: ``kind`` widens to audio/video, and ``locator`` holds + ``{page_idx, bbox}`` for an image where a transcript segment would hold + ``{start_ms, end_ms}``. + """ + + data: bytes + placeholder: str + kind: str = "image" + mime_type: str = "application/octet-stream" + text_content: str = "" # 图注 / OCR（音视频接入后：ASR 转写） + locator: dict = field(default_factory=dict) + index: int = 0 diff --git a/src/backend/core/auth/kb_permissions.py b/src/backend/features/knowledge/personal_permissions.py similarity index 97% rename from src/backend/core/auth/kb_permissions.py rename to src/backend/features/knowledge/personal_permissions.py index 26102bdb..6250b73b 100644 --- a/src/backend/core/auth/kb_permissions.py +++ b/src/backend/features/knowledge/personal_permissions.py @@ -35,7 +35,7 @@ def _is_super_admin(db: Session, user_id: str) -> bool: def get_accessible_local_kb_levels(db: Session, user_id: str) -> Dict[str, KBLevel]: - from core.db.models import KBSpace + from features.knowledge.models import KBSpace if not user_id: return {} diff --git a/src/backend/api/routes/v1/kb_models.py b/src/backend/features/knowledge/personal_request_models.py similarity index 100% rename from src/backend/api/routes/v1/kb_models.py rename to src/backend/features/knowledge/personal_request_models.py diff --git a/src/backend/core/content/kb_processing.py b/src/backend/features/knowledge/processing.py similarity index 67% rename from src/backend/core/content/kb_processing.py rename to src/backend/features/knowledge/processing.py index a2ffa8a0..30b10aad 100644 --- a/src/backend/core/content/kb_processing.py +++ b/src/backend/features/knowledge/processing.py @@ -1,147 +1,10 @@ -"""Knowledge base document processing — chunking, keyword extraction, vectorisation. - -Background task logic extracted from ``api/routes/v1/kb.py``. -""" - +"""processing responsibilities.""" from __future__ import annotations from core.infra.time import utc_now - -import logging -import os -import re from typing import Optional - +import logging logger = logging.getLogger(__name__) - - -# ── Text helpers ──────────────────────────────────────────────────────────── - -def strip_think_tags(text: str) -> str: - """Remove ... blocks and markdown formatting from LLM output.""" - # Remove ... blocks (reasoning model output) - text = re.sub(r'.*?', '', text, flags=re.DOTALL).strip() - # Remove markdown bold/italic markers - text = re.sub(r'\*+', '', text) - # Remove markdown headers - text = re.sub(r'^#+\s*', '', text, flags=re.MULTILINE) - # Remove bullet points - text = re.sub(r'^[\-\*]\s*', '', text, flags=re.MULTILINE) - return text.strip() - - -# ── Model config helper ──────────────────────────────────────────────────── - -def resolve_main_endpoint(): - """main_agent 端点，带上它说的协议——KB 的两处抽取都得走对那条线。""" - from core.llm.single_turn import Endpoint - - try: - from core.services.model_config import ModelConfigService - - cfg = ModelConfigService.get_instance().resolve("main_agent") - if cfg and cfg.base_url and cfg.model_name: - return Endpoint.from_resolved(cfg) - except Exception: - pass - base_url, api_key, model_name = resolve_main_model_config() - return Endpoint(base_url=base_url, api_key=api_key, model_name=model_name) - - -def resolve_main_model_config() -> tuple[str, str, str]: - """Resolve main_agent model config from DB, with env fallback.""" - try: - from core.services.model_config import ModelConfigService - cfg = ModelConfigService.get_instance().resolve("main_agent") - if cfg: - return cfg.base_url.rstrip("/"), cfg.api_key, cfg.model_name - except Exception: - pass - return ( - os.getenv("MODEL_URL", "").rstrip("/"), - os.getenv("API_KEY", ""), - os.getenv("BASE_MODEL_NAME", ""), - ) - - -# ── LLM-powered enrichment ───────────────────────────────────────────────── - -def _call_main_model( - *, system: str, user: str, temperature: float, max_tokens: int, timeout: int = 30 -) -> str: - """跑一次主模型补全；未配置时返回空串，调用方按「这一步没产出」处理。""" - import requests as _requests - - from core.llm.single_turn import build_request, parse_response, turns - - endpoint = resolve_main_endpoint() - if not endpoint.base_url or not endpoint.model_name: - return "" - url, headers, payload = build_request( - endpoint, turns(user, system), temperature=temperature, max_tokens=max_tokens - ) - resp = _requests.post(url, headers=headers, json=payload, timeout=timeout) - resp.raise_for_status() - return parse_response(endpoint, resp.json()).text - - -def extract_keywords_llm(content: str, count: int) -> list[str]: - """Call configured LLM to extract keywords from a text chunk.""" - try: - raw = strip_think_tags( - _call_main_model( - system=( - "你是关键词提取工具。" - "输出规则：只输出关键词，用英文逗号分隔，不要编号，不要解释，不要分析过程。" - "示例输出：数字化转型,产业升级,人工智能" - ), - user=f"从以下文本提取{count}个关键词，直接输出逗号分隔的关键词：\n\n{content[:2000]}", - temperature=0.1, - max_tokens=200, - ) - ) - lines = [ln.strip() for ln in raw.split("\n") if ln.strip()] - best_line = raw - for line in reversed(lines): - if ("," in line or "，" in line) and len(line) < 500: - best_line = line - break - keywords = [kw.strip() for kw in best_line.replace("，", ",").split(",") if kw.strip() and len(kw.strip()) < 30] - return keywords[:count] - except Exception as exc: - logger.warning("LLM keyword extraction failed: %s", exc) - return [] - - -def generate_questions_llm(content: str, count: int) -> list[str]: - """Call configured LLM to generate retrieval questions for a text chunk.""" - try: - raw = strip_think_tags( - _call_main_model( - system=( - "你是问题生成工具。" - "输出规则：每行一个问题，不要编号，不要解释，不要分析过程。直接输出问题本身。" - "示例输出：\n什么是数字化转型？\n如何申报产业升级项目？" - ), - user=f"根据以下文本生成{count}个检索问题，每行一个：\n\n{content[:2000]}", - temperature=0.3, - max_tokens=500, - ) - ) - questions = [] - for q in raw.split("\n"): - q = q.strip().lstrip("0123456789.、）)-— ").strip() - if not q or len(q) < 5: - continue - if q.startswith(("分析", "角色", "任务", "输出", "规则", "示例")): - continue - questions.append(q) - return questions[:count] - except Exception as exc: - logger.warning("LLM question generation failed: %s", exc) - return [] - - -# ── Background vectorisation ─────────────────────────────────────────────── +from features.knowledge import enrichment def update_document_status(document_id: str, status: str, error: Optional[str] = None) -> None: """Update the indexing_status of a document in the DB. @@ -154,7 +17,7 @@ def update_document_status(document_id: str, status: str, error: Optional[str] = from sqlalchemy.orm.attributes import flag_modified from core.db.engine import SessionLocal - from core.db.models import KBDocument + from features.knowledge.models import KBDocument db = SessionLocal() try: doc = db.query(KBDocument).filter(KBDocument.document_id == document_id).first() @@ -205,21 +68,22 @@ def vectorise_document_background( 引文标注按 chunk 走、回溯原文按 chunk 直取，脱离分块无从谈起。 """ try: - from core.kb.kb_assets import multimodal_indexing_enabled - from core.kb.kb_parser import parse_and_chunk_rich - from core.kb.kb_vector import ( - get_or_create_collection, embed_batch, build_sparse_text, text_to_sparse, - upsert_rows, upsert_question_rows, truncate_utf8, - TITLE_FIELD_MAX_BYTES, TAGS_FIELD_MAX_BYTES, - ) + from features.knowledge.kb_assets import multimodal_indexing_enabled + from features.knowledge.kb_parser import parse_and_chunk_rich + from features.knowledge.vector_client import get_or_create_collection + from core.models.embedding import embed_batch + from features.knowledge.kb_vector import build_sparse_text, text_to_sparse + from features.knowledge.vector_indexing import upsert_rows, upsert_question_rows + from features.knowledge.vector_fields import truncate_utf8 + from features.knowledge.kb_vector_state import TITLE_FIELD_MAX_BYTES, TAGS_FIELD_MAX_BYTES from core.db.engine import SessionLocal - from core.db.models import KBChunk + from features.knowledge.models import KBChunk # 调用方手上都有 KBSpace，能传就传：批量上传时每篇再开一次 session 去 # 重读同一行，会和并发的索引任务一起把连接池顶满（实测 39 篇并发上传 # 时出现 QueuePool 超时导致索引失败）。 if index_modes is None: - index_modes = _resolve_index_modes(kb_id) + index_modes = resolve_index_modes(kb_id) want_vectors = "rag" in index_modes want_wiki = "wiki" in index_modes @@ -262,7 +126,7 @@ def vectorise_document_background( # 资产必须在向量化之前落库：它会把父块正文里的图片占位符改写成真实资产 URL， # 而入库的分块正文与被向量化的文本必须是同一份。 - stored_asset_ids = _persist_document_assets( + stored_asset_ids = persist_document_assets( kb_id=kb_id, document_id=document_id, user_id=user_id, @@ -293,11 +157,11 @@ def vectorise_document_background( for pc_idx, pc in enumerate(parent_chunks): chunk_tags: list[str] = [] if auto_keywords > 0: - chunk_tags = extract_keywords_llm(pc.content, auto_keywords) + chunk_tags = enrichment.extract_keywords_llm(pc.content, auto_keywords) chunk_questions: list[str] = [] if auto_questions > 0: - chunk_questions = generate_questions_llm(pc.content, auto_questions) + chunk_questions = enrichment.generate_questions_llm(pc.content, auto_questions) db_chunk = KBChunk( chunk_id=pc.parent_id, @@ -393,7 +257,7 @@ def vectorise_document_background( # 图生文 + 资产检索行。放在文本索引之后：VLM 调用慢且可能没配，文本检索不该等它。 if stored_asset_ids and want_vectors: - _index_document_assets( + index_document_assets( loop=loop, asset_ids=stored_asset_ids, kb_id=kb_id, @@ -406,14 +270,14 @@ def vectorise_document_background( logger.info("Indexing completed for document %s", document_id) if want_wiki: - _enqueue_wiki_ingest(kb_id, document_id) + enqueue_wiki_ingest(kb_id, document_id) except Exception as exc: logger.error("Background vectorisation failed for document %s: %s", document_id, exc, exc_info=True) update_document_status(document_id, "failed", error=f"{type(exc).__name__}: {exc}") -def _persist_document_assets( +def persist_document_assets( *, kb_id: str, document_id: str, @@ -430,7 +294,7 @@ def _persist_document_assets( return [] try: from core.db.engine import SessionLocal - from core.kb.kb_assets import persist_assets + from features.knowledge.kb_assets import persist_assets with SessionLocal() as db: created = persist_assets( @@ -447,7 +311,7 @@ def _persist_document_assets( return [] -def _index_document_assets( +def index_document_assets( *, loop, asset_ids: list, @@ -461,8 +325,9 @@ def _index_document_assets( return try: from core.db.engine import SessionLocal - from core.db.models import KBAsset - from core.kb.kb_assets import generate_captions, index_assets, mark_vector_state + from features.knowledge.models import KBAsset + from features.knowledge.asset_captions import generate_captions + from features.knowledge.asset_indexing import index_assets, mark_vector_state with SessionLocal() as db: rows = db.query(KBAsset).filter(KBAsset.asset_id.in_(asset_ids)).all() @@ -478,11 +343,11 @@ def _index_document_assets( logger.warning("文档 %s 资产索引失败，图片不可检索: %s", document_id, exc) -def _resolve_index_modes(kb_id: str) -> list[str]: +def resolve_index_modes(kb_id: str) -> list[str]: """读该知识库的索引模式。查不到按仅 RAG 处理——历史库都是这个语义。""" try: from core.db.engine import SessionLocal - from core.kb.wiki.config import index_modes_for_kb + from features.knowledge.wiki.config import index_modes_for_kb with SessionLocal() as db: return index_modes_for_kb(db, kb_id) @@ -491,7 +356,7 @@ def _resolve_index_modes(kb_id: str) -> list[str]: return ["rag"] -def _enqueue_wiki_ingest(kb_id: str, document_id: str) -> None: +def enqueue_wiki_ingest(kb_id: str, document_id: str) -> None: """索引完成后把文档排进 Wiki 生成队列。 入队失败不能反过来把文档判成索引失败——分块和向量都已经写好了，检索本身是 @@ -499,7 +364,7 @@ def _enqueue_wiki_ingest(kb_id: str, document_id: str) -> None: """ try: from core.db.engine import SessionLocal - from core.kb.wiki.jobs import enqueue_ingest + from features.knowledge.wiki.jobs import enqueue_ingest with SessionLocal() as db: enqueue_ingest(db, kb_id, [document_id]) diff --git a/src/backend/core/db/repository/kb.py b/src/backend/features/knowledge/repository.py similarity index 97% rename from src/backend/core/db/repository/kb.py rename to src/backend/features/knowledge/repository.py index 85a79638..82e7190d 100644 --- a/src/backend/core/db/repository/kb.py +++ b/src/backend/features/knowledge/repository.py @@ -11,8 +11,8 @@ from typing import Any, Dict, List, Optional import sqlalchemy as sa -from core.db.models import KBDocument, KBSpace -from core.kb.document_status import classify_document_status, empty_document_status_counts +from features.knowledge.models import KBDocument, KBSpace +from features.knowledge.document_status import classify_document_status, empty_document_status_counts from sqlalchemy import and_, desc, func, or_, select from sqlalchemy.orm import Session diff --git a/src/backend/core/services/kb_service.py b/src/backend/features/knowledge/service.py similarity index 95% rename from src/backend/core/services/kb_service.py rename to src/backend/features/knowledge/service.py index 54bd4fea..525ba503 100644 --- a/src/backend/core/services/kb_service.py +++ b/src/backend/features/knowledge/service.py @@ -7,12 +7,14 @@ from datetime import datetime from typing import Any, Dict, List, Optional, Tuple -from core.content.kb_processing import vectorise_document_background -from core.db.models import KBChunk, KBDocument, UserShadow -from core.kb.wiki.config import index_modes_of -from core.db.repository import ArtifactRepository, AuditLogRepository, KBRepository -from core.services.artifact_edition import can_access_artifact -from core.storage import get_storage +from features.knowledge.processing import vectorise_document_background +from features.knowledge.models import KBChunk, KBDocument +from core.db.models import UserShadow +from features.knowledge.wiki.config import index_modes_of +from core.db.repository import ArtifactRepository, AuditLogRepository +from features.knowledge.repository import KBRepository +from core.files.edition import can_access_artifact +from core.files.storage import get_storage from sqlalchemy.orm import Session logger = logging.getLogger(__name__) @@ -42,7 +44,7 @@ def _has_kb_level(self, space: Any, user_id: str, required: str) -> bool: """ if not space: return False - from core.auth.kb_permissions import has_kb_permission, resolve_local_kb_level + from features.knowledge.edition import has_kb_permission, resolve_local_kb_level return has_kb_permission(resolve_local_kb_level(self.db, user_id, space.kb_id), required) @@ -192,8 +194,8 @@ def _enqueue_wiki_retract(self, space: Any, document_id: str) -> None: 会在下一次收尾时被当作死链清掉。 """ try: - from core.kb.wiki.config import wiki_enabled - from core.kb.wiki.jobs import enqueue_retract + from features.knowledge.wiki.config import wiki_enabled + from features.knowledge.wiki.jobs import enqueue_retract if space is None or not wiki_enabled(space): return @@ -239,14 +241,10 @@ def create_space( if str(kind).strip() and str(principal_id).strip() ] if grants: - from core.db.repository import KBGrantRepository - - KBGrantRepository(self.db).bulk_grant( - space.kb_id, - "local", - grants, - "view", - granted_by=granted_by or user_id, + from features.knowledge.edition import grant_initial_access + + grant_initial_access( + self.db, space.kb_id, grants, granted_by=granted_by or user_id, ) # Audit log @@ -316,7 +314,7 @@ def update_space( update_data["description"] = description if index_modes is not None or wiki_config is not None: - from core.kb.wiki.config import normalize_index_modes + from features.knowledge.wiki.config import normalize_index_modes metadata = dict(space.extra_data or {}) if space else {} if index_modes is not None: diff --git a/src/backend/features/knowledge/vector_client.py b/src/backend/features/knowledge/vector_client.py new file mode 100644 index 00000000..a727beb1 --- /dev/null +++ b/src/backend/features/knowledge/vector_client.py @@ -0,0 +1,166 @@ +"""vector client responsibilities.""" +from __future__ import annotations +import os +from typing import Any, Optional +import logging +logger = logging.getLogger(__name__) +from features.knowledge import kb_vector_state +from core.models import embedding as vector_embedding + +def milvus_uri() -> str: + from core.config.settings import settings + + return os.getenv("MILVUS_URL") or settings.memory.milvus_url + + +def is_lite() -> bool: + """True when the vector backend is embedded Milvus Lite (a local file uri). + + Milvus Lite (no-Docker local profile) does **not** support SPARSE_FLOAT_VECTOR + / SPARSE_INVERTED_INDEX / hybrid search / IVF / scalar INVERTED indexes, so the + KB degrades to **dense-only** on it. A server uri (``http(s)://`` / ``tcp://``) + is a full Milvus and keeps the dense+sparse hybrid path byte-for-byte unchanged. + """ + return not milvus_uri().startswith(("http://", "https://", "tcp://", "unix:")) + + +def get_client(): + """Return a MilvusClient connected to the configured Milvus instance.""" + from pymilvus import MilvusClient + + url = milvus_uri() + token = os.getenv("MILVUS_TOKEN", "") + if token: + return MilvusClient(uri=url, token=token) + return MilvusClient(uri=url) + + +def upsert(client, data: list[dict[str, Any]]) -> None: + """Upsert helper that drops the ``sparse_embedding`` field on Milvus Lite. + + On Lite the collection has no sparse field (see ``get_or_create_collection``), + and dynamic fields are disabled, so an extra ``sparse_embedding`` key would be + rejected. Stripping it here keeps every caller's row-building code identical + across both backends. + """ + if is_lite(): + data = [{k: v for k, v in row.items() if k != "sparse_embedding"} for row in data] + client.upsert(collection_name=kb_vector_state.COLLECTION_NAME, data=data) + + +def collection_dense_dim(client, *, timeout: float | None = None) -> Optional[int]: + """Read the dense_embedding dimension of the existing collection. + + Connection and schema-read errors deliberately propagate to the caller. Treating + an unavailable schema as a dimension mismatch used to trigger a destructive + collection rebuild. + """ + desc = client.describe_collection(kb_vector_state.COLLECTION_NAME, timeout=timeout) + for f in desc.get("fields", []): + if f.get("name") == "dense_embedding": + dim = f.get("params", {}).get("dim") + return int(dim) if dim is not None else None + return None + + +def get_or_create_collection(*, timeout: float | None = None) -> None: + """Idempotently create hugagent_kb_private (hybrid retrieval schema) at the current embedding model's real dimension. + + New collections use the current embedding model's detected dimension. Existing + collections are never dropped implicitly: an unreadable schema or dimension + mismatch preserves all vectors and aborts the operation until an operator can + verify the embedding configuration and explicitly re-index the collection. + """ + pass # State is owned by its responsibility module. + from pymilvus import DataType + + client = get_client() + target_dim = vector_embedding.detect_embed_dim() + + if client.has_collection(kb_vector_state.COLLECTION_NAME, timeout=timeout): + if kb_vector_state.VERIFIED_COLLECTION_DIM == target_dim: + return + try: + existing_dim = collection_dense_dim(client, timeout=timeout) + except Exception as exc: + message = ( + "无法确认现有知识库集合的向量维度，已保留集合并中止本次操作；" + "请检查 Milvus 连接后重试。" + ) + logger.error("%s 原因：%s", message, exc, exc_info=True) + raise RuntimeError(message) from exc + if existing_dim is None: + message = ( + "无法确认现有知识库集合的向量维度，已保留集合并中止本次操作；" + "dense_embedding 字段或 dim 参数缺失。" + ) + logger.error(message) + raise RuntimeError(message) + if existing_dim == target_dim: + kb_vector_state.VERIFIED_COLLECTION_DIM = target_dim + return + message = ( + f"KB collection 维度不匹配（已存在={existing_dim}，当前模型={target_dim}），" + "已保留集合并中止本次操作；请核实 embedding 配置并通过显式重建流程重新索引。" + ) + logger.error(message) + raise RuntimeError(message) + + lite = is_lite() + + schema = client.create_schema(auto_id=False, enable_dynamic_field=False) + schema.add_field("chunk_id", DataType.VARCHAR, max_length=64, is_primary=True) + schema.add_field("parent_chunk_id", DataType.VARCHAR, max_length=64) + schema.add_field("row_type", DataType.VARCHAR, max_length=16) # "chunk" | "question" + schema.add_field("user_id", DataType.VARCHAR, max_length=64) + schema.add_field("kb_id", DataType.VARCHAR, max_length=64) + schema.add_field("document_id", DataType.VARCHAR, max_length=64) + schema.add_field("title", DataType.VARCHAR, max_length=kb_vector_state.TITLE_FIELD_MAX_BYTES) + schema.add_field("content", DataType.VARCHAR, max_length=kb_vector_state.CONTENT_FIELD_MAX_BYTES) + schema.add_field( + "tags_text", DataType.VARCHAR, max_length=kb_vector_state.TAGS_FIELD_MAX_BYTES + ) # BM25 augmentation + schema.add_field("chunk_index", DataType.INT64) + schema.add_field("dense_embedding", DataType.FLOAT_VECTOR, dim=target_dim) + if not lite: + schema.add_field("sparse_embedding", DataType.SPARSE_FLOAT_VECTOR) + + index_params = client.prepare_index_params() + if lite: + # Milvus Lite: only a dense AUTOINDEX (FLAT under the hood). No sparse, no + # scalar INVERTED indexes — filtering falls back to brute force, fine at + # single-machine scale. + index_params.add_index( + field_name="dense_embedding", + index_type="AUTOINDEX", + metric_type="IP", + ) + else: + index_params.add_index( + field_name="dense_embedding", + index_type="IVF_FLAT", + metric_type="IP", + params={"nlist": 128}, + ) + index_params.add_index( + field_name="sparse_embedding", + index_type="SPARSE_INVERTED_INDEX", + metric_type="IP", + ) + # Scalar indexes for filtering acceleration + for field in ("user_id", "kb_id", "document_id", "row_type"): + index_params.add_index(field_name=field, index_type="INVERTED") + + client.create_collection( + collection_name=kb_vector_state.COLLECTION_NAME, + schema=schema, + index_params=index_params, + timeout=timeout, + ) + kb_vector_state.VERIFIED_COLLECTION_DIM = target_dim + logger.info( + "Created Milvus collection: %s (dim=%d, mode=%s)", + kb_vector_state.COLLECTION_NAME, + target_dim, + "lite/dense-only" if lite else "hybrid", + ) diff --git a/src/backend/features/knowledge/vector_fields.py b/src/backend/features/knowledge/vector_fields.py new file mode 100644 index 00000000..c3fdaf17 --- /dev/null +++ b/src/backend/features/knowledge/vector_fields.py @@ -0,0 +1,16 @@ +"""vector fields responsibilities.""" +from __future__ import annotations +from features.knowledge import kb_vector_state + +def truncate_utf8(text: str, max_bytes: int = kb_vector_state.CONTENT_FIELD_MAX_BYTES) -> str: + """Safe truncation by UTF-8 bytes (Chinese is 3 bytes/char); never cuts in the middle of a multi-byte character. + + Historical bug: using ``text[:4096]`` truncated by "character"; 4096 Chinese chars ≈ 12KB bytes, still exceeding Milvus's + 4096 **byte** limit → ``length exceeds max length`` error. + """ + if not text: + return "" + encoded = text.encode("utf-8") + if len(encoded) <= max_bytes: + return text + return encoded[:max_bytes].decode("utf-8", "ignore") diff --git a/src/backend/features/knowledge/vector_indexing.py b/src/backend/features/knowledge/vector_indexing.py new file mode 100644 index 00000000..52c50f24 --- /dev/null +++ b/src/backend/features/knowledge/vector_indexing.py @@ -0,0 +1,204 @@ +"""vector indexing responsibilities.""" +from __future__ import annotations +from features.knowledge import kb_vector +from typing import Any +import logging +logger = logging.getLogger(__name__) +from features.knowledge import kb_vector +from features.knowledge import kb_vector_state +from features.knowledge import vector_client +from core.models import embedding as vector_embedding +from features.knowledge import vector_fields + +def upsert_rows(rows: list[dict[str, Any]]) -> None: + """Upsert a batch of rows into hugagent_kb_private. + + Each row must include all schema fields. dense_embedding and sparse_embedding + must be pre-computed by the caller (use embed_text / build_sparse_text). + """ + if not rows: + return + vector_client.get_or_create_collection() + client = vector_client.get_client() + vector_client.upsert(client, rows) + logger.debug("Upserted %d rows into %s", len(rows), kb_vector_state.COLLECTION_NAME) + + +def delete_by_document(document_id: str, user_id: str) -> None: + """Delete all Milvus rows (chunk + question) for a document.""" + vector_client.get_or_create_collection() + client = vector_client.get_client() + expr = f'document_id == "{document_id}" and user_id == "{user_id}"' + client.delete(collection_name=kb_vector_state.COLLECTION_NAME, filter=expr) + logger.info("Deleted Milvus rows for document_id=%s user_id=%s", document_id, user_id) + + +def delete_by_kb(kb_id: str, user_id: str) -> None: + """Delete all Milvus rows for an entire KB space.""" + vector_client.get_or_create_collection() + client = vector_client.get_client() + expr = f'kb_id == "{kb_id}" and user_id == "{user_id}"' + client.delete(collection_name=kb_vector_state.COLLECTION_NAME, filter=expr) + logger.info("Deleted Milvus rows for kb_id=%s user_id=%s", kb_id, user_id) + + +def get_children_for_parent(parent_chunk_id: str) -> list[dict]: + """Return the child-chunk rows of a given parent chunk in Milvus (for the "view chunks" parent→child display). + + Child rows have ``chunk_id`` shaped like ``{pid}_{i}`` with ``parent_chunk_id == pid``; in flat mode a parent has only one + representative row (``chunk_id == parent_chunk_id``), which is filtered out —— i.e. flat mode returns an empty list. + Returns an empty list when Milvus is unavailable / the query fails, without affecting the parent-chunk list display. + """ + try: + vector_client.get_or_create_collection() + client = vector_client.get_client() + rows = client.query( + collection_name=kb_vector_state.COLLECTION_NAME, + # In flat mode a parent has only one representative row (chunk_id == parent_chunk_id); exclude it right in the query + filter=( + f'parent_chunk_id == "{parent_chunk_id}" and row_type == "chunk" ' + f'and chunk_id != "{parent_chunk_id}"' + ), + output_fields=["chunk_id", "parent_chunk_id", "chunk_index", "content"], + limit=4096, + ) + except Exception as exc: + logger.warning("get_children_for_parent(%s) failed: %s", parent_chunk_id, exc) + return [] + children = [ + { + "chunk_id": r.get("chunk_id"), + "chunk_index": r.get("chunk_index", 0), + "content": r.get("content", ""), + } + for r in rows + ] + children.sort(key=lambda x: x["chunk_index"]) + return children + + +def delete_by_chunk(parent_chunk_id: str) -> None: + """Delete all Milvus rows (chunk + question) for a single parent chunk.""" + vector_client.get_or_create_collection() + client = vector_client.get_client() + expr = f'parent_chunk_id == "{parent_chunk_id}"' + client.delete(collection_name=kb_vector_state.COLLECTION_NAME, filter=expr) + logger.info("Deleted Milvus rows for parent_chunk_id=%s", parent_chunk_id) + + +def reindex_chunk_tags(chunk_id: str, content: str, tags: list[str]) -> None: + """Re-compute and upsert sparse_embedding for a chunk row after tag changes.""" + # Fetch the existing row to preserve all other fields + vector_client.get_or_create_collection() + client = vector_client.get_client() + rows = client.query( + collection_name=kb_vector_state.COLLECTION_NAME, + filter=f'chunk_id == "{chunk_id}"', + output_fields=[ + "chunk_id", + "parent_chunk_id", + "row_type", + "user_id", + "kb_id", + "document_id", + "title", + "chunk_index", + "dense_embedding", + ], + ) + if not rows: + logger.warning("reindex_chunk_tags: chunk_id %s not found in Milvus", chunk_id) + return + + row = rows[0] + sparse_text = kb_vector.build_sparse_text(content, tags) + row["content"] = vector_fields.truncate_utf8(content) + row["tags_text"] = vector_fields.truncate_utf8(" ".join(tags), kb_vector_state.TAGS_FIELD_MAX_BYTES) + row["sparse_embedding"] = kb_vector.text_to_sparse(sparse_text) + vector_client.upsert(client, [row]) + + +def reindex_chunk_content( + chunk_id: str, + content: str, + tags: list[str], + user_id: str, + kb_id: str, + document_id: str, + title: str, + chunk_index: int, +) -> None: + """Re-embed a parent chunk after a manual content edit. + + Removes the parent's existing chunk rows (child or flat) and upserts a single + fresh chunk row with recomputed dense + sparse embeddings. Question rows are + managed separately by :func:`upsert_question_rows`. Retrieval returns the parent + content from PostgreSQL, so this single representative row is sufficient for the + edited chunk to be matched and surfaced. + """ + vector_client.get_or_create_collection() + client = vector_client.get_client() + # Drop existing chunk rows for this parent (keep questions; they are re-upserted separately) + client.delete( + collection_name=kb_vector_state.COLLECTION_NAME, + filter=f'parent_chunk_id == "{chunk_id}" and row_type == "chunk"', + ) + sparse_text = kb_vector.build_sparse_text(content, tags or []) + row = { + "chunk_id": chunk_id, + "parent_chunk_id": chunk_id, + "row_type": "chunk", + "user_id": user_id, + "kb_id": kb_id, + "document_id": document_id, + "title": vector_fields.truncate_utf8(title, kb_vector_state.TITLE_FIELD_MAX_BYTES), + "content": vector_fields.truncate_utf8(content), + "tags_text": vector_fields.truncate_utf8(" ".join(tags or []), kb_vector_state.TAGS_FIELD_MAX_BYTES), + "chunk_index": chunk_index, + "dense_embedding": vector_embedding.embed_text(content), + "sparse_embedding": kb_vector.text_to_sparse(sparse_text), + } + vector_client.upsert(client, [row]) + + +def upsert_question_rows( + parent_chunk_id: str, + questions: list[str], + user_id: str, + kb_id: str, + document_id: str, + title: str, + chunk_index: int, +) -> None: + """Delete existing question rows for a chunk and insert fresh ones.""" + vector_client.get_or_create_collection() + client = vector_client.get_client() + + # Delete old question rows + expr = f'parent_chunk_id == "{parent_chunk_id}" and row_type == "question"' + client.delete(collection_name=kb_vector_state.COLLECTION_NAME, filter=expr) + + if not questions: + return + + # Embed all questions in one batch call + vecs = vector_embedding.embed_batch(questions) + rows = [] + for i, (q, vec) in enumerate(zip(questions, vecs)): + rows.append( + { + "chunk_id": f"q_{parent_chunk_id}_{i}", + "parent_chunk_id": parent_chunk_id, + "row_type": "question", + "user_id": user_id, + "kb_id": kb_id, + "document_id": document_id, + "title": vector_fields.truncate_utf8(title, kb_vector_state.TITLE_FIELD_MAX_BYTES), + "content": vector_fields.truncate_utf8(q), + "tags_text": "", + "chunk_index": chunk_index, + "dense_embedding": vec, + "sparse_embedding": kb_vector.text_to_sparse(q), + } + ) + vector_client.upsert(client, rows) diff --git a/src/backend/core/kb/wiki/__init__.py b/src/backend/features/knowledge/wiki/__init__.py similarity index 92% rename from src/backend/core/kb/wiki/__init__.py rename to src/backend/features/knowledge/wiki/__init__.py index 3fbc3ed5..cdf25048 100644 --- a/src/backend/core/kb/wiki/__init__.py +++ b/src/backend/features/knowledge/wiki/__init__.py @@ -17,6 +17,6 @@ 3. **Wiki 是地图不是第二检索源。** 答案与出处始终回到 kb_chunks 的原文。 """ -from core.kb.wiki.config import WikiConfig, resolve_wiki_config, wiki_enabled +from features.knowledge.wiki.config import WikiConfig, resolve_wiki_config, wiki_enabled __all__ = ["WikiConfig", "resolve_wiki_config", "wiki_enabled"] diff --git a/src/backend/core/kb/wiki/cite.py b/src/backend/features/knowledge/wiki/cite.py similarity index 95% rename from src/backend/core/kb/wiki/cite.py rename to src/backend/features/knowledge/wiki/cite.py index 71c5daf9..73e58726 100644 --- a/src/backend/core/kb/wiki/cite.py +++ b/src/backend/features/knowledge/wiki/cite.py @@ -13,11 +13,11 @@ import logging from typing import Any, Dict, List, Optional, Sequence, Tuple -from core.kb.wiki.config import WikiConfig -from core.kb.wiki.extract import CandidateItem, parse_candidates -from core.kb.wiki.llm import LLMBudget, call_llm_json -from core.kb.wiki.prompts import CHUNK_CITATION_PROMPT, append_business_instructions -from core.kb.wiki.slug import normalize_slug +from features.knowledge.wiki.config import WikiConfig +from features.knowledge.wiki.extract import CandidateItem, parse_candidates +from features.knowledge.wiki.llm import LLMBudget, call_llm_json +from features.knowledge.wiki.prompts import CHUNK_CITATION_PROMPT, append_business_instructions +from features.knowledge.wiki.slug import normalize_slug logger = logging.getLogger(__name__) diff --git a/src/backend/core/kb/wiki/config.py b/src/backend/features/knowledge/wiki/config.py similarity index 99% rename from src/backend/core/kb/wiki/config.py rename to src/backend/features/knowledge/wiki/config.py index 68771ca3..c27ef680 100644 --- a/src/backend/core/kb/wiki/config.py +++ b/src/backend/features/knowledge/wiki/config.py @@ -153,7 +153,7 @@ def wiki_enabled_kb_ids_map(db: Any, kb_ids: Sequence[str]) -> Dict[str, List[st if not ids: return {} try: - from core.db.models import KBSpace + from features.knowledge.models import KBSpace rows = ( db.query(KBSpace.kb_id, KBSpace.extra_data) diff --git a/src/backend/core/kb/wiki/dedup.py b/src/backend/features/knowledge/wiki/dedup.py similarity index 94% rename from src/backend/core/kb/wiki/dedup.py rename to src/backend/features/knowledge/wiki/dedup.py index 428a278f..143fa381 100644 --- a/src/backend/core/kb/wiki/dedup.py +++ b/src/backend/features/knowledge/wiki/dedup.py @@ -13,13 +13,13 @@ import logging from typing import Dict, List, Optional, Sequence, Set, Tuple -from core.db.models import KBWikiPage -from core.evolution.similarity import _ngrams, lexical_similarity -from core.kb.wiki.config import WikiConfig -from core.kb.wiki.extract import CandidateItem -from core.kb.wiki.llm import LLMBudget, call_llm_json -from core.kb.wiki.prompts import DEDUPLICATION_PROMPT -from core.kb.wiki.slug import normalize_slug +from features.knowledge.models import KBWikiPage +from core.evolution.assessment.similarity import ngrams as _ngrams, lexical_similarity +from features.knowledge.wiki.config import WikiConfig +from features.knowledge.wiki.extract import CandidateItem +from features.knowledge.wiki.llm import LLMBudget, call_llm_json +from features.knowledge.wiki.prompts import DEDUPLICATION_PROMPT +from features.knowledge.wiki.slug import normalize_slug logger = logging.getLogger(__name__) @@ -36,7 +36,7 @@ def _similarity(left: str, right: str) -> float: """两个表面词的相似度。完全相同直接给 1.0。 - 底层复用 ``core.evolution.similarity.lexical_similarity``（字符 n-gram 的 + 底层复用 ``core.evolution.assessment.similarity.lexical_similarity``（字符 n-gram 的 Jaccard），条目名很短所以取 n=2——三元组在四字名上几乎没有重叠可言。 """ if not left or not right: diff --git a/src/backend/core/kb/wiki/extract.py b/src/backend/features/knowledge/wiki/extract.py similarity index 93% rename from src/backend/core/kb/wiki/extract.py rename to src/backend/features/knowledge/wiki/extract.py index c45acd3f..76c215ee 100644 --- a/src/backend/core/kb/wiki/extract.py +++ b/src/backend/features/knowledge/wiki/extract.py @@ -11,14 +11,10 @@ from dataclasses import dataclass, field from typing import Any, Dict, List, Optional, Sequence -from core.kb.wiki.config import WikiConfig -from core.kb.wiki.llm import LLMBudget, call_llm_json -from core.kb.wiki.prompts import ( - CANDIDATE_SLUG_PROMPT, - append_business_instructions, - granularity_guidance, -) -from core.kb.wiki.slug import normalize_slug +from features.knowledge.wiki.config import WikiConfig +from features.knowledge.wiki.llm import LLMBudget, call_llm_json +from features.knowledge.wiki.prompts import CANDIDATE_SLUG_PROMPT, append_business_instructions, granularity_guidance +from features.knowledge.wiki.slug import normalize_slug logger = logging.getLogger(__name__) diff --git a/src/backend/core/kb/wiki/finalize.py b/src/backend/features/knowledge/wiki/finalize.py similarity index 92% rename from src/backend/core/kb/wiki/finalize.py rename to src/backend/features/knowledge/wiki/finalize.py index 5212da3e..42be5e60 100644 --- a/src/backend/core/kb/wiki/finalize.py +++ b/src/backend/features/knowledge/wiki/finalize.py @@ -17,17 +17,12 @@ import logging from typing import Dict, List, Set -from core.db.models.kb_wiki import ( - WIKI_KNOWLEDGE_PAGE_TYPES, - WIKI_PAGE_TYPE_INDEX, - WIKI_PAGE_TYPE_LABELS, - WIKI_PAGE_TYPE_SUMMARY, -) -from core.kb.wiki.config import WikiConfig -from core.kb.wiki.llm import LLMBudget, WikiLLMError, call_llm -from core.kb.wiki.prompts import INDEX_INTRO_PROMPT, append_business_instructions -from core.kb.wiki.slug import extract_link_slugs, strip_links_except -from core.kb.wiki.store import all_pages, get_page, prune_empty_folders, upsert_page +from features.knowledge.models import WIKI_KNOWLEDGE_PAGE_TYPES, WIKI_PAGE_TYPE_INDEX, WIKI_PAGE_TYPE_LABELS, WIKI_PAGE_TYPE_SUMMARY +from features.knowledge.wiki.config import WikiConfig +from features.knowledge.wiki.llm import LLMBudget, WikiLLMError, call_llm +from features.knowledge.wiki.prompts import INDEX_INTRO_PROMPT, append_business_instructions +from features.knowledge.wiki.slug import extract_link_slugs, strip_links_except +from features.knowledge.wiki.store import all_pages, get_page, prune_empty_folders, upsert_page from sqlalchemy.orm import Session logger = logging.getLogger(__name__) diff --git a/src/backend/core/kb/wiki/jobs.py b/src/backend/features/knowledge/wiki/jobs.py similarity index 99% rename from src/backend/core/kb/wiki/jobs.py rename to src/backend/features/knowledge/wiki/jobs.py index 25508a0d..214adf3a 100644 --- a/src/backend/core/kb/wiki/jobs.py +++ b/src/backend/features/knowledge/wiki/jobs.py @@ -11,8 +11,8 @@ from datetime import datetime, timedelta, timezone from typing import List, Optional, Sequence -from core.db.models import KBWikiJob -from core.kb.wiki.store import _new_id +from features.knowledge.models import KBWikiJob +from features.knowledge.wiki.store import _new_id from sqlalchemy import func, or_ from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import Session diff --git a/src/backend/core/kb/wiki/linkify.py b/src/backend/features/knowledge/wiki/linkify.py similarity index 100% rename from src/backend/core/kb/wiki/linkify.py rename to src/backend/features/knowledge/wiki/linkify.py diff --git a/src/backend/core/kb/wiki/llm.py b/src/backend/features/knowledge/wiki/llm.py similarity index 96% rename from src/backend/core/kb/wiki/llm.py rename to src/backend/features/knowledge/wiki/llm.py index 5e925fbc..171b695d 100644 --- a/src/backend/core/kb/wiki/llm.py +++ b/src/backend/features/knowledge/wiki/llm.py @@ -20,8 +20,8 @@ from typing import Any, Dict, Optional import requests -from core.llm._distill_shared import parse_strict_json, strip_think_blocks -from core.llm.single_turn import Endpoint, build_request, parse_response, turns +from core.infra.model_output import parse_strict_json, strip_think_blocks +from core.models.completions.single_turn import Endpoint, build_request, parse_response, turns logger = logging.getLogger(__name__) @@ -46,7 +46,7 @@ def _resolve_endpoint() -> Endpoint: Wiki 生成对「主模型在哪」必须给出同一个答案。 """ try: - from core.services.model_config import ModelConfigService + from core.models.config import ModelConfigService svc = ModelConfigService.get_instance() for role in ("kb_wiki", "main_agent"): @@ -56,7 +56,7 @@ def _resolve_endpoint() -> Endpoint: except Exception as exc: # noqa: BLE001 - 配置服务不可用时退到主模型，不阻断管线 logger.warning("Wiki 模型角色解析失败，回落主模型: %s", exc) - from core.content.kb_processing import resolve_main_model_config + from features.knowledge.enrichment import resolve_main_model_config base_url, api_key, model_name = resolve_main_model_config() return Endpoint(base_url=base_url, api_key=api_key, model_name=model_name) @@ -109,7 +109,7 @@ def _repair_literal_newlines(text: str) -> str: 提示词已经禁止过，但模型仍会偶发违反，而这恰好是最常见的一种解析失败。逐字符 扫描并跟踪是否处在字符串里，只改字符串内部的控制字符。这一段是本模块相对 仓库通用解析器**额外**补的一层，其余能力（think 剥离、围栏、括号配平提取） - 一律走 ``core.llm._distill_shared``。 + 一律走 ``core.infra.model_output``。 """ out = [] in_string = False diff --git a/src/backend/features/knowledge/wiki/local_graph.py b/src/backend/features/knowledge/wiki/local_graph.py new file mode 100644 index 00000000..f8739896 --- /dev/null +++ b/src/backend/features/knowledge/wiki/local_graph.py @@ -0,0 +1,156 @@ +"""local graph responsibilities.""" +from __future__ import annotations +from typing import Any, Dict, List, Sequence +from core.db.engine import SessionLocal +from features.knowledge.models import KBWikiPage +from features.knowledge.models import WIKI_PAGE_TYPE_INDEX +from features.knowledge.wiki.slug import readable_from_slug +from sqlalchemy.orm import Session +from features.knowledge.wiki import local_projection +from features.knowledge.wiki import local_provider_state + +def resolve_link_titles( + db: Session, kb_id: str, slugs: Sequence[str], *, resolve: bool +) -> List[Dict[str, str]]: + if not slugs: + return [] + titles: Dict[str, KBWikiPage] = {} + if resolve: + rows = local_projection.base_query(db, kb_id).filter(KBWikiPage.slug.in_(list(slugs))).all() + titles = {row.slug: row for row in rows} + resolved = [] + for slug in slugs: + row = titles.get(slug) + resolved.append( + { + "slug": slug, + "title": (row.title if row else "") or readable_from_slug(slug), + "page_type": (row.page_type if row else "") or slug.split("/", 1)[0], + } + ) + return resolved + + +def wiki_graph( + kb_id: str, + mode: str = "overview", + center: str = "", + depth: int = 1, + limit: int = 60, + types: str = "", +) -> Dict[str, Any]: + """概念图谱。 + + ``overview`` 取全局连接度最高的节点，``ego`` 以 center 为中心展开 depth 层。 + 大库有数千节点上万条边，前端必须靠这两种模式分批取，不能一次拉全量。 + """ + limit = max(1, min(int(limit or 60), local_provider_state.MAX_GRAPH_NODES)) + depth = max(1, min(int(depth or 1), 3)) + wanted = local_projection.split_types(types) + + with SessionLocal() as db: + # 图谱只用到这四列；页面正文对邻接表毫无用处，却是最大的一列 + query = db.query( + KBWikiPage.slug, + KBWikiPage.title, + KBWikiPage.page_type, + KBWikiPage.out_links, + ).filter( + KBWikiPage.kb_id == kb_id, + KBWikiPage.deleted_at.is_(None), + KBWikiPage.page_type != WIKI_PAGE_TYPE_INDEX, + ) + if wanted: + query = query.filter(KBWikiPage.page_type.in_(wanted)) + rows = query.all() + + by_slug = {row.slug: row for row in rows} + adjacency: Dict[str, List[str]] = { + row.slug: [s for s in (row.out_links or []) if s in by_slug] for row in rows + } + # 边是无向展开用的，两个方向都要能走 + undirected: Dict[str, set] = {slug: set(links) for slug, links in adjacency.items()} + for slug, links in adjacency.items(): + for target in links: + undirected.setdefault(target, set()).add(slug) + + if mode == "ego": + selected = ego_nodes(undirected, center, depth, limit) + if not selected: + return { + "nodes": [], + "edges": [], + "meta": { + "mode": "ego", + "center": center, + "depth": depth, + "truncated": False, + "total": len(rows), + "returned": 0, + "total_pages": len(rows), + }, + } + else: + ranked = sorted(rows, key=lambda r: len(undirected.get(r.slug, ())), reverse=True) + selected = {r.slug for r in ranked[:limit]} + + nodes = [ + { + "slug": slug, + "title": by_slug[slug].title or readable_from_slug(slug), + "page_type": by_slug[slug].page_type, + "link_count": len(undirected.get(slug, ())), + } + for slug in selected + if slug in by_slug + ] + edges = [] + seen_edges = set() + for slug in selected: + for target in adjacency.get(slug, []): + if target not in selected: + continue + key = tuple(sorted((slug, target))) + if key in seen_edges: + continue + seen_edges.add(key) + edges.append({"source": slug, "target": target}) + + return { + "nodes": nodes, + "edges": edges, + "meta": { + "mode": mode, + "center": center, + "depth": depth if mode == "ego" else None, + "truncated": len(rows) > len(selected), + # 前端类型声明读 total/returned；total_pages 是旧键，保留兼容 + "total": len(rows), + "returned": len(nodes), + "total_pages": len(rows), + }, + } + + +def ego_nodes(adjacency: Dict[str, set], center: str, depth: int, limit: int) -> set: + """从 center 出发做 BFS，取 depth 层内的节点。""" + if not center or center not in adjacency: + return set() + selected = {center} + frontier = {center} + for _ in range(depth): + nxt = set() + for slug in frontier: + for neighbour in adjacency.get(slug, ()): + if neighbour not in selected: + nxt.add(neighbour) + if not nxt: + break + for slug in sorted(nxt, key=lambda s: len(adjacency.get(s, ())), reverse=True): + if len(selected) >= limit: + break + selected.add(slug) + frontier = nxt + if len(selected) >= limit: + break + return selected diff --git a/src/backend/features/knowledge/wiki/local_projection.py b/src/backend/features/knowledge/wiki/local_projection.py new file mode 100644 index 00000000..e4101c07 --- /dev/null +++ b/src/backend/features/knowledge/wiki/local_projection.py @@ -0,0 +1,51 @@ +"""local projection responsibilities.""" +from __future__ import annotations +from typing import Any, Dict, List, Sequence +from features.knowledge.models import KBDocument, KBWikiPage +from sqlalchemy.orm import Session +from features.knowledge.wiki import local_provider_state + +def split_types(raw: str) -> List[str]: + return [t.strip() for t in str(raw or "").split(",") if t.strip()] + + +def slim_page(page: KBWikiPage, *, with_content: bool = False) -> Dict[str, Any]: + """页面 → 精简结构。 + + 定位阶段只要标题/摘要/链接/血缘，正文按需带：一页动辄数千字，无差别塞进模型 + 上下文会把「先定位再取回」的成本优势整个吃掉。 + """ + item: Dict[str, Any] = { + "slug": page.slug, + "title": page.title, + "page_type": page.page_type, + "type_label": local_provider_state.PAGE_TYPE_LABELS.get(page.page_type or "", ""), + "summary": page.summary or "", + "aliases": list(page.aliases or []), + "category_path": list(page.category_path or []), + "wiki_path": page.wiki_path or "", + "out_links": list(page.out_links or []), + "in_links": list(page.in_links or []), + "source_refs": list(page.source_refs or []), + "chunk_refs": list(page.chunk_refs or []), + "updated_at": page.updated_at.isoformat() if page.updated_at else None, + } + if with_content: + item["content"] = page.content or "" + return item + + +def base_query(db: Session, kb_id: str): + return db.query(KBWikiPage).filter(KBWikiPage.kb_id == kb_id, KBWikiPage.deleted_at.is_(None)) + + +def document_titles(db: Session, kb_id: str, document_ids: Sequence[str]) -> Dict[str, str]: + ids = [d for d in dict.fromkeys(document_ids) if d] + if not ids: + return {} + rows = ( + db.query(KBDocument.document_id, KBDocument.title, KBDocument.filename) + .filter(KBDocument.kb_id == kb_id, KBDocument.document_id.in_(ids)) + .all() + ) + return {row.document_id: (row.title or row.filename or row.document_id) for row in rows} diff --git a/src/backend/features/knowledge/wiki/local_provider.py b/src/backend/features/knowledge/wiki/local_provider.py new file mode 100644 index 00000000..ea36dd5c --- /dev/null +++ b/src/backend/features/knowledge/wiki/local_provider.py @@ -0,0 +1,271 @@ +"""local provider responsibilities.""" +from __future__ import annotations +import re +from typing import Any, Dict, List +from core.db.engine import SessionLocal +from features.knowledge.models import KBChunk, KBWikiPage +from features.knowledge.models import WIKI_PAGE_TYPE_INDEX, WIKI_PAGE_TYPE_SUMMARY +from features.knowledge.wiki.finalize import INDEX_SLUG +from features.knowledge.wiki.store import children_index, clean_category_path, descendant_folder_ids, get_page, load_folder_tree, page_counts_by_folder, subtree_counts, walk_folder_path +from sqlalchemy import func, or_ +from features.knowledge.wiki import local_graph +from features.knowledge.wiki import local_projection +from features.knowledge.wiki import local_provider_state + +def wiki_stats(kb_id: str) -> Dict[str, Any]: + """规模统计：总页数、分类型页数、链接数、孤儿页数。 + + 只投影统计需要的三列——生成期间前端每 10 秒轮一次，把每页几 KB 的 ``content`` + 拉出来纯属浪费。 + """ + with SessionLocal() as db: + rows = ( + db.query(KBWikiPage.page_type, KBWikiPage.in_links, KBWikiPage.out_links) + .filter(KBWikiPage.kb_id == kb_id, KBWikiPage.deleted_at.is_(None)) + .all() + ) + by_type: Dict[str, int] = {} + total_links = 0 + orphan = 0 + for page_type, in_links, out_links in rows: + by_type[page_type] = by_type.get(page_type, 0) + 1 + total_links += len(out_links or []) + if not (in_links or []) and not (out_links or []): + orphan += 1 + return { + "total_pages": len(rows), + "pages_by_type": by_type, + "total_links": total_links, + "orphan_count": orphan, + } + + +def wiki_list_pages( + kb_id: str, + page: int = 1, + page_size: int = 50, + page_type: str = "", + category_path: str = "", + category_depth: int = 0, +) -> Dict[str, Any]: + """分页列出页面。 + + ``page_type`` 支持逗号分隔多选（「知识」视图把实体/概念/综述/对比合成一次 + 请求，避免按类型扇出）；``category_path`` 把范围收窄到某个目录子树。 + """ + page = max(1, int(page or 1)) + page_size = max(1, int(page_size or 50)) + with SessionLocal() as db: + query = local_projection.base_query(db, kb_id) + types = local_projection.split_types(page_type) + if types: + query = query.filter(KBWikiPage.page_type.in_(types)) + + labels = clean_category_path([category_path] if category_path else []) + if labels: + folder_id, _, _ = walk_folder_path(db, kb_id, labels, create=False) + if folder_id is None: + return {"pages": [], "total": 0, "page": page, "page_size": page_size} + query = query.filter( + KBWikiPage.folder_id.in_(descendant_folder_ids(db, kb_id, folder_id)) + ) + + total = query.count() + rows = ( + query.order_by(KBWikiPage.sort_order, KBWikiPage.title) + .offset((page - 1) * page_size) + .limit(page_size) + .all() + ) + return { + "pages": [local_projection.slim_page(row) for row in rows], + "total": total, + "page": page, + "page_size": page_size, + } + + +def wiki_folders(kb_id: str, parent_id: str = "", page_types: str = "") -> Dict[str, Any]: + """某一层的子目录，带**递归**页面计数，供前端逐层懒加载。 + + 两次查询搞定：整棵目录树 + 一次按目录分组的页数聚合，其余在内存里算。原来是 + 每个子目录一次子树查询加一次计数，用户逐层展开时会反复付这个代价。 + """ + types = local_projection.split_types(page_types) + with SessionLocal() as db: + folders = load_folder_tree(db, kb_id) + if not folders: + return {"parent_id": parent_id or "", "folders": []} + totals = subtree_counts(folders, page_counts_by_folder(db, kb_id, types)) + children = children_index(folders) + items = [ + { + "id": folder.folder_id, + "name": folder.name, + "parent_id": folder.parent_id or "", + "page_count": totals.get(folder.folder_id, 0), + "has_children": bool(children.get(folder.folder_id)), + } + for folder in children.get(parent_id or "", []) + ] + return {"parent_id": parent_id or "", "folders": items} + + +def wiki_index(kb_id: str, limit: int = 20, types: str = "") -> Dict[str, Any]: + """索引总览：一段总述 + 按类型分组的条目清单。""" + limit = max(1, int(limit or 20)) + wanted = local_projection.split_types(types) + with SessionLocal() as db: + index_page = get_page(db, kb_id, INDEX_SLUG) + intro = "" + if index_page: + intro = str((index_page.page_metadata or {}).get("intro") or "") + + groups = [] + for page_type in [*local_provider_state.KNOWLEDGE_PAGE_TYPES, WIKI_PAGE_TYPE_SUMMARY]: + if wanted and page_type not in wanted: + continue + query = local_projection.base_query(db, kb_id).filter(KBWikiPage.page_type == page_type) + total = query.count() + if not total: + continue + rows = query.order_by(KBWikiPage.title).limit(limit).all() + groups.append( + { + "type": page_type, + "type_label": local_provider_state.PAGE_TYPE_LABELS.get(page_type, ""), + "total": total, + "items": [ + { + "slug": row.slug, + "title": row.title, + "summary": row.summary or "", + "wiki_path": row.wiki_path or "", + } + for row in rows + ], + } + ) + return { + "intro": intro, + "version": index_page.version if index_page else None, + "groups": groups, + } + + +def wiki_search(kb_id: str, query: str, limit: int = 10) -> Dict[str, Any]: + """按标题/别名/摘要/正文命中页面。 + + 支持 ``资质|牌照`` 这样的交替写法——口语说法常常匹配不上书面术语，一次给多个 + 说法能显著提高命中率。交替在 Python 侧拆成多个 LIKE，避免依赖数据库正则。 + """ + limit = max(1, min(int(limit or 10), local_provider_state.MAX_SEARCH_LIMIT)) + terms = [t.strip() for t in re.split(r"[|｜]", str(query or "")) if t.strip()] + if not terms: + return {"pages": [], "total": 0} + + with SessionLocal() as db: + conditions = [] + for term in terms[:8]: + pattern = f"%{term.lower()}%" + conditions.append(func.lower(KBWikiPage.title).like(pattern)) + conditions.append(func.lower(KBWikiPage.summary).like(pattern)) + conditions.append(func.lower(KBWikiPage.content).like(pattern)) + rows = ( + local_projection.base_query(db, kb_id) + .filter(KBWikiPage.page_type != WIKI_PAGE_TYPE_INDEX) + .filter(or_(*conditions)) + .order_by(KBWikiPage.page_type, KBWikiPage.title) + .limit(limit * 3) + .all() + ) + + # 别名命中在 SQL 里跨方言不好写，回到 Python 侧补一遍；同时按"命中位置" + # 排序——标题命中远比正文里出现一次更相关 + lowered_terms = [t.lower() for t in terms] + + def _rank(page: KBWikiPage) -> int: + title = (page.title or "").lower() + aliases = " ".join(page.aliases or []).lower() + summary = (page.summary or "").lower() + if any(t == title for t in lowered_terms): + return 0 + if any(t in title for t in lowered_terms): + return 1 + if any(t in aliases for t in lowered_terms): + return 2 + if any(t in summary for t in lowered_terms): + return 3 + return 4 + + ranked = sorted(rows, key=lambda p: (_rank(p), p.title or ""))[:limit] + return {"pages": [local_projection.slim_page(p) for p in ranked], "total": len(ranked)} + + +def wiki_read_page(kb_id: str, slug: str, *, resolve_links: bool = True) -> Dict[str, Any]: + """读取整页（含正文、双链、血缘坐标）。 + + ``out_links`` 只有 slug，直接展示就是一串拼音，人和模型都读不出是什么。这里 + 顺手把邻居的标题查回来贴上——同库一次 IN 查询就够。 + """ + with SessionLocal() as db: + page = get_page(db, kb_id, slug) + if page is None: + return {} + item = local_projection.slim_page(page, with_content=True) + item["related_pages"] = local_graph.resolve_link_titles( + db, kb_id, item["out_links"], resolve=resolve_links + ) + return item + + +def wiki_source_chunks(kb_id: str, slug: str, max_chunks: int = 6) -> Dict[str, Any]: + """按页面的 ``chunk_refs`` 直取原文分块。 + + 这是「地图 → 定位 → 顺坐标取回原文」这条链的最后一跳：**按 ID 直取**，不是 + 再检索一次。页面与分块同库，一次 IN 查询就够。 + """ + max_chunks = max(1, min(int(max_chunks or 6), 20)) + with SessionLocal() as db: + page = get_page(db, kb_id, slug) + if page is None: + return {"page": {}, "chunks": []} + + item = local_projection.slim_page(page, with_content=True) + item["related_pages"] = local_graph.resolve_link_titles(db, kb_id, item["out_links"], resolve=True) + + chunk_ids = list(page.chunk_refs or []) + rows: List[KBChunk] = [] + if chunk_ids: + rows = ( + db.query(KBChunk) + .filter(KBChunk.kb_id == kb_id, KBChunk.chunk_id.in_(chunk_ids)) + .all() + ) + # 保持页面记录的引用顺序：那是证据的呈现次序，不是数据库的返回次序 + order = {cid: i for i, cid in enumerate(chunk_ids)} + rows.sort(key=lambda r: order.get(r.chunk_id, 10**6)) + + # chunk_refs 对不上（页面被改写过、或文档重新索引换了分块）时，退回该页 + # 主要来源文档的前几块正文——给不出精确证据，至少不要空手而归 + if not rows and (page.source_refs or []): + rows = ( + db.query(KBChunk) + .filter(KBChunk.kb_id == kb_id, KBChunk.document_id == page.source_refs[0]) + .order_by(KBChunk.chunk_index) + .limit(max_chunks) + .all() + ) + + titles = local_projection.document_titles(db, kb_id, [r.document_id for r in rows]) + chunks = [ + { + "chunk_id": row.chunk_id, + "document_id": row.document_id, + "document_title": titles.get(row.document_id, ""), + "content": row.content or "", + } + for row in rows[:max_chunks] + if (row.content or "").strip() + ] + return {"page": item, "chunks": chunks} diff --git a/src/backend/features/knowledge/wiki/local_provider_state.py b/src/backend/features/knowledge/wiki/local_provider_state.py new file mode 100644 index 00000000..e0244d4d --- /dev/null +++ b/src/backend/features/knowledge/wiki/local_provider_state.py @@ -0,0 +1,15 @@ +"""local provider state responsibilities.""" +from __future__ import annotations +from features.knowledge.models import WIKI_KNOWLEDGE_PAGE_TYPES, WIKI_PAGE_TYPE_LABELS +from features.knowledge.wiki.config import MAX_GRAPH_NODES + +PAGE_TYPE_LABELS = WIKI_PAGE_TYPE_LABELS + + +KNOWLEDGE_PAGE_TYPES = WIKI_KNOWLEDGE_PAGE_TYPES + + +MAX_GRAPH_NODES = MAX_GRAPH_NODES + + +MAX_SEARCH_LIMIT = 50 diff --git a/src/backend/core/kb/wiki/pipeline.py b/src/backend/features/knowledge/wiki/pipeline.py similarity index 95% rename from src/backend/core/kb/wiki/pipeline.py rename to src/backend/features/knowledge/wiki/pipeline.py index 0394f47d..a135f02b 100644 --- a/src/backend/core/kb/wiki/pipeline.py +++ b/src/backend/features/knowledge/wiki/pipeline.py @@ -22,21 +22,15 @@ from dataclasses import dataclass, field from typing import Any, Callable, Dict, List, Optional, Sequence, Set -from core.db.models import KBChunk, KBDocument -from core.kb.wiki.cite import annotate_citations, apply_citations -from core.kb.wiki.config import WikiConfig -from core.kb.wiki.dedup import apply_merges, resolve_duplicates -from core.kb.wiki.extract import CandidateItem, extract_candidates -from core.kb.wiki.llm import LLMBudget, WikiQuotaExceeded -from core.kb.wiki.reduce import write_document_summary, write_entity_page -from core.kb.wiki.slug import summary_slug -from core.kb.wiki.store import ( - all_pages, - archive_page, - get_pages, - pages_citing_document, - upsert_page, -) +from features.knowledge.models import KBChunk, KBDocument +from features.knowledge.wiki.cite import annotate_citations, apply_citations +from features.knowledge.wiki.config import WikiConfig +from features.knowledge.wiki.dedup import apply_merges, resolve_duplicates +from features.knowledge.wiki.extract import CandidateItem, extract_candidates +from features.knowledge.wiki.llm import LLMBudget, WikiQuotaExceeded +from features.knowledge.wiki.reduce import write_document_summary, write_entity_page +from features.knowledge.wiki.slug import summary_slug +from features.knowledge.wiki.store import all_pages, archive_page, get_pages, pages_citing_document, upsert_page from sqlalchemy.orm import Session logger = logging.getLogger(__name__) @@ -270,7 +264,7 @@ def _report(stage: str, done: int, total: int) -> None: # ── 目录规划：只给尚未归档的页分配 ────────────────────────────────────── assignments: Dict[str, List[str]] = {} if not result.quota_exceeded: - from core.kb.wiki.taxonomy import plan_taxonomy + from features.knowledge.wiki.taxonomy import plan_taxonomy unfiled = [item for item in batch_items.values() if not existing_folder_ids.get(item.slug)] try: diff --git a/src/backend/core/kb/wiki/prompts.py b/src/backend/features/knowledge/wiki/prompts.py similarity index 100% rename from src/backend/core/kb/wiki/prompts.py rename to src/backend/features/knowledge/wiki/prompts.py diff --git a/src/backend/core/kb/wiki/reduce.py b/src/backend/features/knowledge/wiki/reduce.py similarity index 93% rename from src/backend/core/kb/wiki/reduce.py rename to src/backend/features/knowledge/wiki/reduce.py index 123ec80c..642fba14 100644 --- a/src/backend/core/kb/wiki/reduce.py +++ b/src/backend/features/knowledge/wiki/reduce.py @@ -15,17 +15,12 @@ import logging from typing import Any, Dict, List, Optional, Sequence -from core.kb.wiki.config import WikiConfig -from core.kb.wiki.extract import CandidateItem -from core.kb.wiki.linkify import linkify_content -from core.kb.wiki.llm import LLMBudget, WikiLLMError, call_llm, split_summary_line -from core.kb.wiki.prompts import ( - DOCUMENT_SUMMARY_PROMPT, - PAGE_MODIFY_SYSTEM_PROMPT, - PAGE_MODIFY_USER_PROMPT, - append_business_instructions, -) -from core.kb.wiki.slug import SlugHandles, extract_link_slugs, strip_links_except +from features.knowledge.wiki.config import WikiConfig +from features.knowledge.wiki.extract import CandidateItem +from features.knowledge.wiki.linkify import linkify_content +from features.knowledge.wiki.llm import LLMBudget, WikiLLMError, call_llm, split_summary_line +from features.knowledge.wiki.prompts import DOCUMENT_SUMMARY_PROMPT, PAGE_MODIFY_SYSTEM_PROMPT, PAGE_MODIFY_USER_PROMPT, append_business_instructions +from features.knowledge.wiki.slug import SlugHandles, extract_link_slugs, strip_links_except logger = logging.getLogger(__name__) diff --git a/src/backend/core/kb/wiki/slug.py b/src/backend/features/knowledge/wiki/slug.py similarity index 100% rename from src/backend/core/kb/wiki/slug.py rename to src/backend/features/knowledge/wiki/slug.py diff --git a/src/backend/core/kb/wiki/store.py b/src/backend/features/knowledge/wiki/store.py similarity index 98% rename from src/backend/core/kb/wiki/store.py rename to src/backend/features/knowledge/wiki/store.py index 99e202ae..154a85c3 100644 --- a/src/backend/core/kb/wiki/store.py +++ b/src/backend/features/knowledge/wiki/store.py @@ -10,12 +10,8 @@ from datetime import datetime, timezone from typing import Any, Dict, Iterable, List, Optional, Sequence -from core.db.models import KBWikiFolder, KBWikiPage -from core.db.models.kb_wiki import ( - WIKI_CATEGORY_MAX_DEPTH, - WIKI_EDIT_SOURCE_PIPELINE, - WIKI_PAGE_TYPE_ENTITY, -) +from features.knowledge.models import KBWikiFolder, KBWikiPage +from features.knowledge.models import WIKI_CATEGORY_MAX_DEPTH, WIKI_EDIT_SOURCE_PIPELINE, WIKI_PAGE_TYPE_ENTITY from sqlalchemy import func from sqlalchemy.orm import Session diff --git a/src/backend/core/kb/wiki/taxonomy.py b/src/backend/features/knowledge/wiki/taxonomy.py similarity index 89% rename from src/backend/core/kb/wiki/taxonomy.py rename to src/backend/features/knowledge/wiki/taxonomy.py index 4886fcc6..f534de3e 100644 --- a/src/backend/core/kb/wiki/taxonomy.py +++ b/src/backend/features/knowledge/wiki/taxonomy.py @@ -12,13 +12,13 @@ import logging from typing import Any, Dict, List, Optional, Sequence -from core.db.models import KBWikiFolder -from core.kb.wiki.config import WikiConfig -from core.kb.wiki.extract import CandidateItem -from core.kb.wiki.llm import LLMBudget, call_llm_json -from core.kb.wiki.prompts import TAXONOMY_PLAN_PROMPT -from core.kb.wiki.slug import normalize_slug -from core.kb.wiki.store import clean_category_path +from features.knowledge.models import KBWikiFolder +from features.knowledge.wiki.config import WikiConfig +from features.knowledge.wiki.extract import CandidateItem +from features.knowledge.wiki.llm import LLMBudget, call_llm_json +from features.knowledge.wiki.prompts import TAXONOMY_PLAN_PROMPT +from features.knowledge.wiki.slug import normalize_slug +from features.knowledge.wiki.store import clean_category_path from sqlalchemy.orm import Session logger = logging.getLogger(__name__) diff --git a/src/backend/core/kb/wiki/worker.py b/src/backend/features/knowledge/wiki/worker.py similarity index 94% rename from src/backend/core/kb/wiki/worker.py rename to src/backend/features/knowledge/wiki/worker.py index edf9abda..0e32914d 100644 --- a/src/backend/core/kb/wiki/worker.py +++ b/src/backend/features/knowledge/wiki/worker.py @@ -17,11 +17,11 @@ from typing import Optional from core.infra.logging import get_logger -from core.kb.wiki import jobs as job_queue -from core.kb.wiki.config import WikiConfig, resolve_wiki_config, wiki_enabled -from core.kb.wiki.finalize import finalize_kb_wiki -from core.kb.wiki.llm import LLMBudget, WikiQuotaExceeded, wiki_model_configured -from core.kb.wiki.pipeline import run_ingest, run_retract +from features.knowledge.wiki import jobs as job_queue +from features.knowledge.wiki.config import WikiConfig, resolve_wiki_config, wiki_enabled +from features.knowledge.wiki.finalize import finalize_kb_wiki +from features.knowledge.wiki.llm import LLMBudget, WikiQuotaExceeded, wiki_model_configured +from features.knowledge.wiki.pipeline import run_ingest, run_retract logger = get_logger(__name__) @@ -71,7 +71,7 @@ async def _loop(self) -> None: def _run_one(self) -> bool: """认领并执行一个作业。返回是否真的干了活（决定下一轮的等待时长）。""" from core.db.engine import SessionLocal - from core.db.models import KBSpace + from features.knowledge.models import KBSpace with SessionLocal() as db: job = job_queue.claim_next_job(db, self.worker_id) diff --git a/src/backend/core/kb/wiki_router.py b/src/backend/features/knowledge/wiki_router.py similarity index 88% rename from src/backend/core/kb/wiki_router.py rename to src/backend/features/knowledge/wiki_router.py index 32954456..dd25063f 100644 --- a/src/backend/core/kb/wiki_router.py +++ b/src/backend/features/knowledge/wiki_router.py @@ -30,7 +30,7 @@ def local_wiki_enabled(kb_id: str) -> bool: return False try: from core.db.engine import SessionLocal - from core.kb.wiki.config import wiki_enabled_for_kb + from features.knowledge.wiki.config import wiki_enabled_for_kb with SessionLocal() as db: return wiki_enabled_for_kb(db, kb_id) @@ -44,12 +44,12 @@ def wiki_module_for(kb_id: str) -> Optional[ModuleType]: if is_local_kb(kb_id): if not local_wiki_enabled(kb_id): return None - from core.kb.wiki import local_provider + from features.knowledge.wiki import local_provider return local_provider try: - from core.kb.external_provider import wiki_module + from features.knowledge.edition import wiki_module return wiki_module() except Exception as exc: # noqa: BLE001 - 社区版下外接接缝可能整体缺席 @@ -70,7 +70,7 @@ def supports_wiki(kb_id: str = "") -> bool: def _external_supports_wiki() -> bool: try: - from core.kb.external_provider import supports_wiki as external_supports + from features.knowledge.edition import supports_wiki as external_supports return bool(external_supports()) except Exception: @@ -88,8 +88,8 @@ def _any_local_wiki_kb() -> bool: """ try: from core.db.engine import SessionLocal - from core.db.models import KBSpace - from core.kb.wiki.config import INDEX_MODE_WIKI, normalize_index_modes + from features.knowledge.models import KBSpace + from features.knowledge.wiki.config import INDEX_MODE_WIKI, normalize_index_modes with SessionLocal() as db: rows = db.query(KBSpace.extra_data).filter(KBSpace.deleted_at.is_(None)).yield_per(200) @@ -111,7 +111,7 @@ def wiki_capable_local_kb_ids(kb_ids: List[str]) -> List[str]: return [] try: from core.db.engine import SessionLocal - from core.kb.wiki.config import INDEX_MODE_WIKI, wiki_enabled_kb_ids_map + from features.knowledge.wiki.config import INDEX_MODE_WIKI, wiki_enabled_kb_ids_map with SessionLocal() as db: modes = wiki_enabled_kb_ids_map(db, candidates) @@ -123,7 +123,7 @@ def wiki_capable_local_kb_ids(kb_ids: List[str]) -> List[str]: def capabilities_for_local_kb(space: Any) -> Dict[str, bool]: """把自建库的索引模式投影成能力位，供 catalog 与前端消费。""" - from core.kb.wiki.config import rag_enabled, wiki_enabled + from features.knowledge.wiki.config import rag_enabled, wiki_enabled has_wiki = wiki_enabled(space) has_rag = rag_enabled(space) diff --git a/src/backend/features/marketplace/README.md b/src/backend/features/marketplace/README.md new file mode 100644 index 00000000..ccafeaac --- /dev/null +++ b/src/backend/features/marketplace/README.md @@ -0,0 +1,15 @@ +# Marketplace + +Owns skill, agent, plugin and MCP listings, submissions, review, immutable versions, installation and export. Installed schemas load independently of runtime switches. + +API routes are registered by the host central route tables, preserving policy keys and order. `agent_versions` and `skill_versions` apply selected version content; catalog and installation modules contain the underlying operations. Callers use the concrete responsibility module; there are no legacy-path forwarding modules. + +Package resolution implements the core package-source interface through `package_source.MarketplacePackageSource`, registered by host composition. Plugin management callers use `plugin_market` directly for listing metadata. Knowledge selection and desktop resources remain host-owned interfaces; remaining legacy core domain references are migrated in P5/P6. + +State modules own bundle paths and shared constants. MCP curated configuration (`mcp_catalog`) is separate from listing queries (`mcp_listing`), authentication materialization, immutable version publication, installation, and revalidation. Models preserve the original table names and relationships, including listing visibility state. + +Validation: affected marketplace lifecycle, version conflict, archive safety and auth route tests; original route/schema snapshots; import and CE guards; CE empty database upgrade and release checks. + +The API host is initialized before direct route use; standalone feature-route imports retain the existing eager API initialization limitation. + +版本差异统一由 edition.py 选择列表策略、可见性及管理请求模型；CE 只覆盖此入口。 diff --git a/src/backend/features/marketplace/__init__.py b/src/backend/features/marketplace/__init__.py new file mode 100644 index 00000000..86d226fc --- /dev/null +++ b/src/backend/features/marketplace/__init__.py @@ -0,0 +1 @@ +"""marketplace package.""" diff --git a/src/backend/features/marketplace/agent_catalog.py b/src/backend/features/marketplace/agent_catalog.py new file mode 100644 index 00000000..a51f6e56 --- /dev/null +++ b/src/backend/features/marketplace/agent_catalog.py @@ -0,0 +1,119 @@ +"""agent catalog responsibilities.""" +from __future__ import annotations +import json +from typing import Any, Dict, List, Optional +from features.marketplace.models import AgentMarketSubmission +from core.infra.exceptions import ResourceNotFoundError +from features.marketplace import edition as ml +from features.marketplace.agent_categories import AGENT_MARKETPLACE_CATEGORIES +from sqlalchemy.orm import Session +import logging +logger = logging.getLogger(__name__) +from features.marketplace import agent_metadata +from features.marketplace import agents_state + +def read_bundle(slug: str) -> Optional[Dict[str, Any]]: + """Read and validate a single preset sub-agent's ``agent.json`` (fault-tolerant; returns None if broken).""" + if not slug or "/" in slug or ".." in slug: + return None + path = agents_state.AGENT_MARKET_DIR / slug / agents_state.AGENT_JSON + if not path.is_file(): + return None + try: + b = json.loads(path.read_text(encoding="utf-8")) + except Exception as exc: # noqa: BLE001 + logger.warning("agent market bundle broken: %s (%s)", slug, exc) + return None + if not isinstance(b, dict) or not b.get("name"): + return None + b.setdefault("slug", slug) + return b + + +def approved_submissions(db: Session) -> List[AgentMarketSubmission]: + return ( + db.query(AgentMarketSubmission) + .filter(AgentMarketSubmission.status == "approved") + .order_by(AgentMarketSubmission.created_at.desc()) + .all() + ) + + +def get_approved_submission(db: Session, slug: str) -> Optional[AgentMarketSubmission]: + return ( + db.query(AgentMarketSubmission) + .filter( + AgentMarketSubmission.slug == slug, + AgentMarketSubmission.status == "approved", + ) + .first() + ) + + +def list_marketplace_agents( + db: Session, *, include_disabled: bool = False, viewer_user_id: Optional[str] = None +) -> List[Dict[str, Any]]: + """List all marketplace sub-agents: presets (featured first) + community listings, layered with the enable/disable switch and visibility-scope filtering.""" + items: List[Dict[str, Any]] = [] + if agents_state.AGENT_MARKET_DIR.is_dir(): + for child in sorted(agents_state.AGENT_MARKET_DIR.iterdir()): + if not child.is_dir(): + continue + b = read_bundle(child.name) + if b: + items.append(agent_metadata.bundle_public_meta(b)) + items.sort(key=lambda x: (0 if x["featured"] else 1, x["slug"])) + items.extend(agent_metadata.submission_public_meta(s) for s in approved_submissions(db)) + items = ml.annotate_and_filter( + db, + ml.KIND_AGENT, + items, + id_key="slug", + include_disabled=include_disabled, + viewer_user_id=viewer_user_id, + ) + return items + + +def categories_from_items(items: List[Dict[str, Any]]) -> List[str]: + """Derive categories from an already-fetched marketplace list: the fixed 9 major categories first, legacy leftover categories appended after.""" + seen: List[str] = list(AGENT_MARKETPLACE_CATEGORIES) + for it in items: + if it["category"] not in seen: + seen.append(it["category"]) + return seen + + +def list_categories( + db: Session, *, include_disabled: bool = False, viewer_user_id: Optional[str] = None +) -> List[str]: + """Marketplace category list (standalone entry point; the list endpoint should reuse categories_from_items to avoid a second scan).""" + return categories_from_items( + list_marketplace_agents( + db, include_disabled=include_disabled, viewer_user_id=viewer_user_id + ) + ) + + +def get_agent_detail(db: Session, slug: str) -> Dict[str, Any]: + """Detail of a single marketplace sub-agent: metadata + prompt body + capability bindings (for the detail preview). + + The preset directory takes priority; falls back to the community listing (approved submission) if not found. + """ + b = read_bundle(slug) + if b: + data = agent_metadata.bundle_public_meta(b) + data["system_prompt"] = b.get("system_prompt") or "" + data["welcome_message"] = b.get("welcome_message") or "" + data["suggested_questions"] = list(b.get("suggested_questions") or []) + data["bindings"] = agent_metadata.bindings_of(b) + return data + sub = get_approved_submission(db, slug) + if sub is None: + raise ResourceNotFoundError("marketplace_agent", slug) + data = agent_metadata.submission_public_meta(sub) + data["system_prompt"] = sub.system_prompt or "" + data["welcome_message"] = sub.welcome_message or "" + data["suggested_questions"] = list(sub.suggested_questions or []) + data["bindings"] = agent_metadata.normalize_binding_lists(sub.bindings_snapshot) + return data diff --git a/src/backend/features/marketplace/agent_categories.py b/src/backend/features/marketplace/agent_categories.py new file mode 100644 index 00000000..c6580899 --- /dev/null +++ b/src/backend/features/marketplace/agent_categories.py @@ -0,0 +1,38 @@ +"""Sub-agent marketplace categories (single source of truth). + +The sub-agent marketplace categories are **independent of the skill marketplace's 8 +categories** (role-positioning semantics fit agents better). The preset list, community +listing applications, and admin review category changes may only take these 9 values; +the frontend mirror is ``AGENT_MARKETPLACE_CATEGORIES`` in ``utils/constants.ts``. +""" + +from __future__ import annotations + +from typing import List + +from core.infra.exceptions import BadRequestError + +# Sub-agent marketplace fixed categories (stable order, frontend mirror must match) +AGENT_MARKETPLACE_CATEGORIES: List[str] = [ + "通用助手", + "职场办公", + "商业分析", + "数据分析", + "研发编程", + "翻译写作", + "创意设计", + "政策法务", + "教育科研", +] + +DEFAULT_AGENT_CATEGORY = "通用助手" + + +def validate_category(category: str) -> str: + """Validate and return a legal category; anything not in the fixed set 400s directly.""" + category = (category or "").strip() + if category not in AGENT_MARKETPLACE_CATEGORIES: + raise BadRequestError( + message=f"请从固定分类中选择：{'、'.join(AGENT_MARKETPLACE_CATEGORIES)}" + ) + return category diff --git a/src/backend/features/marketplace/agent_installation.py b/src/backend/features/marketplace/agent_installation.py new file mode 100644 index 00000000..2deef0c7 --- /dev/null +++ b/src/backend/features/marketplace/agent_installation.py @@ -0,0 +1,240 @@ +"""agent installation responsibilities.""" +from __future__ import annotations +from features.marketplace import skill_installation +from features.marketplace import skill_metadata +from typing import Any, Dict, List, Optional +from core.db.models import UserAgent +from core.infra.exceptions import ResourceNotFoundError +from sqlalchemy.orm import Session +import logging +logger = logging.getLogger(__name__) +from features.marketplace import agent_catalog +from features.marketplace import agent_metadata +from features.marketplace import agents_state + +def installed_slugs(db: Session, owner_user_id: Optional[str], slugs: List[str]) -> set: + """The subset of these marketplace slugs already installed (cloned) in the given scope (user / global admin).""" + if not slugs: + return set() + q = db.query(UserAgent.source_market_slug).filter(UserAgent.source_market_slug.in_(slugs)) + if owner_user_id is None: + q = q.filter(UserAgent.owner_type == "admin") + else: + q = q.filter(UserAgent.owner_type == "user", UserAgent.user_id == owner_user_id) + return {row[0] for row in q.all()} + + +def annotate_installed( + items: List[Dict[str, Any]], db: Session, owner_user_id: Optional[str] +) -> List[Dict[str, Any]]: + """Add the ``installed`` flag to each marketplace list item (for the given scope).""" + if not items: + return items + installed = installed_slugs(db, owner_user_id, [it["slug"] for it in items]) + for it in items: + it["installed"] = it["slug"] in installed + return items + + +def is_installed(db: Session, slug: str, owner_user_id: Optional[str]) -> bool: + return bool(installed_slugs(db, owner_user_id, [slug])) + + +def resolve_market_entry(db: Session, slug: str) -> Dict[str, Any]: + """Normalize a preset bundle / community listing into a unified clone-source dict.""" + b = agent_catalog.read_bundle(slug) + if b: + return { + "name": b.get("name") or slug, + "avatar": b.get("avatar") or "", + "description": b.get("description") or "", + "summary": b.get("summary") or "", + "system_prompt": b.get("system_prompt") or "", + "welcome_message": b.get("welcome_message") or "", + "suggested_questions": list(b.get("suggested_questions") or []), + "model_config": dict(b.get("model_config") or {}), + "bindings": agent_metadata.bindings_of(b), + "ontology_tags": [ + str(tag) + for tag in (b.get("ontology_tags") or b.get("tags") or []) + if str(tag).startswith("ontology:") + ], + } + sub = agent_catalog.get_approved_submission(db, slug) + if sub is None: + raise ResourceNotFoundError("marketplace_agent", slug) + return { + "name": sub.name or slug, + "avatar": sub.avatar or "", + "description": sub.description or "", + "summary": sub.summary or "", + "system_prompt": sub.system_prompt or "", + "welcome_message": sub.welcome_message or "", + "suggested_questions": list(sub.suggested_questions or []), + "model_config": dict(sub.model_config_snapshot or {}), + "bindings": agent_metadata.normalize_binding_lists(sub.bindings_snapshot), + "ontology_tags": [str(tag) for tag in (sub.tags or []) if str(tag).startswith("ontology:")], + } + + +def resolve_bindings( + db: Session, + bindings: Dict[str, List[str]], + owner_user_id: Optional[str], + operator_name: Optional[str], +) -> tuple[Dict[str, List[str]], Dict[str, List[str]]]: + """Resolve each marketplace binding id into the installer's scope; returns (final bindings, report). + + - MCP / default skills / accessible KBs: bind directly if within ``available-resources``; + - marketplace skill slugs: if not in the available list, install as the installer's private skill and bind the returned id; + - plugin slugs: bind if already installed, otherwise install as private and then bind; + - unresolvable items go into ``dropped``; auto-installed skills/plugins requiring credentials go into ``needs_secret``. + """ + from core.capabilities.catalog.api import get_enabled_ids + from core.capabilities.plugins import management as plugin_service + + from core.capabilities.agents.edition import UserAgentService + + avail = UserAgentService(db).list_available_resources(owner_user_id=owner_user_id) + avail_skill = {s["id"] for s in avail.get("skills", [])} + # available-resources only lists AdminMcpServer rows; but built-in MCP tools (e.g. + # database_query) are catalog-defined and loaded at runtime by catalog id, + # and don't necessarily have a same-named AdminMcpServer row. Merge the catalog-enabled + # built-in MCP ids into the "resolvable" set, so these legitimate built-in tools aren't + # misjudged as "unresolvable" and dropped (once bound, the runtime connects on demand). + avail_mcp = {s["id"] for s in avail.get("mcp_servers", [])} | set(get_enabled_ids("mcp")) + avail_plugin = {p["id"] for p in avail.get("plugins", [])} + avail_kb = {k["id"] for k in avail.get("kb_spaces", [])} + + final: Dict[str, List[str]] = { + "skill_ids": [], + "mcp_server_ids": [], + "plugin_ids": [], + "kb_ids": [], + } + report: Dict[str, List[str]] = { + "bound": [], + "installed": [], + "dropped": [], + "needs_secret": [], + } + + # MCP tools + for mid in bindings.get("mcp_server_ids", []): + if mid in avail_mcp: + final["mcp_server_ids"].append(mid) + report["bound"].append(f"mcp:{mid}") + else: + report["dropped"].append(f"mcp:{mid}") + + # Skills (default skills bind directly; marketplace skills get auto-installed as private) + for sid in bindings.get("skill_ids", []): + if sid in avail_skill: + final["skill_ids"].append(sid) + report["bound"].append(f"skill:{sid}") + continue + try: + res = skill_installation.install_marketplace_skill(db, sid, owner_user_id=owner_user_id, secrets={}) + final["skill_ids"].append(res["id"]) + report["installed"].append(f"skill:{sid}") + if skill_metadata.market_skill_requires_secrets(sid): + report["needs_secret"].append(f"skill:{sid}") + except Exception as exc: # noqa: BLE001 + logger.info("agent clone: skill binding unresolved %s (%s)", sid, exc) + report["dropped"].append(f"skill:{sid}") + + # Plugins (bindings store the slug; bind the install_id if installed, otherwise auto-install) + for pslug in bindings.get("plugin_ids", []): + gid = f"{pslug}@global" + match = ( + gid + if gid in avail_plugin + else next((p for p in avail_plugin if p.startswith(f"{pslug}@")), None) + ) + if match: + final["plugin_ids"].append(match) + report["bound"].append(f"plugin:{pslug}") + continue + try: + res = plugin_service.install_plugin( + db, pslug, owner_user_id=owner_user_id, secrets={}, created_by=operator_name + ) + final["plugin_ids"].append(res["install_id"]) + report["installed"].append(f"plugin:{pslug}") + if res.get("required_secrets") or res.get("requires_secret"): + report["needs_secret"].append(f"plugin:{pslug}") + except Exception as exc: # noqa: BLE001 + logger.info("agent clone: plugin binding unresolved %s (%s)", pslug, exc) + report["dropped"].append(f"plugin:{pslug}") + + # Knowledge bases (keep only those the installer can access) + for kid in bindings.get("kb_ids", []): + if kid in avail_kb: + final["kb_ids"].append(kid) + report["bound"].append(f"kb:{kid}") + else: + report["dropped"].append(f"kb:{kid}") + + return final, report + + +def install_marketplace_agent( + db: Session, + slug: str, + *, + owner_user_id: Optional[str], + operator_name: Optional[str] = None, +) -> Dict[str, Any]: + """Install a marketplace sub-agent as a private clone (admin owner_user_id=None → global admin sub-agent).""" + from core.capabilities.agents.edition import UserAgentService + + entry = resolve_market_entry(db, slug) + final_bindings, report = resolve_bindings(db, entry["bindings"], owner_user_id, operator_name) + + mc = entry.get("model_config") or {} + data: Dict[str, Any] = { + "name": entry["name"], + "avatar": entry.get("avatar") or None, + "description": (entry.get("summary") or entry.get("description") or "")[:agents_state.MAX_DESC], + "system_prompt": entry.get("system_prompt") or "", + "welcome_message": entry.get("welcome_message") or "", + "suggested_questions": list(entry.get("suggested_questions") or []), + "skill_ids": final_bindings["skill_ids"], + "mcp_server_ids": final_bindings["mcp_server_ids"], + "plugin_ids": final_bindings["plugin_ids"], + "kb_ids": final_bindings["kb_ids"], + "source_market_slug": slug, + "ontology_tags": list(entry.get("ontology_tags") or []), + } + if mc.get("temperature") is not None: + data["temperature"] = mc["temperature"] + if mc.get("max_tokens") is not None: + data["max_tokens"] = mc["max_tokens"] + if mc.get("max_iters") is not None: + data["max_iters"] = mc["max_iters"] + if mc.get("timeout") is not None: + data["timeout"] = mc["timeout"] + + owner_type = "admin" if owner_user_id is None else "user" + agent = UserAgentService(db).create( + user_id=owner_user_id, + operator_name=operator_name, + owner_type=owner_type, + data=data, + ) + logger.info( + "agent_market_install: slug=%s owner=%s agent=%s bound=%d installed=%d dropped=%d", + slug, + owner_user_id or "global", + agent["agent_id"], + len(report["bound"]), + len(report["installed"]), + len(report["dropped"]), + ) + return { + "agent_id": agent["agent_id"], + "slug": slug, + "owner": "self" if owner_user_id else "global", + "install_report": report, + "message": "子智能体已安装", + } diff --git a/src/backend/features/marketplace/agent_metadata.py b/src/backend/features/marketplace/agent_metadata.py new file mode 100644 index 00000000..fc879392 --- /dev/null +++ b/src/backend/features/marketplace/agent_metadata.py @@ -0,0 +1,101 @@ +"""agent metadata responsibilities.""" +from __future__ import annotations +from typing import Any, Dict, List, Optional +from features.marketplace.models import AgentMarketSubmission +from features.marketplace.agent_categories import DEFAULT_AGENT_CATEGORY + +def normalize_binding_lists(raw: Optional[Dict[str, Any]]) -> Dict[str, List[str]]: + """Normalize a bindings snapshot into the four id lists (missing ones become empty).""" + b = raw or {} + return { + "skill_ids": list(b.get("skill_ids") or []), + "mcp_server_ids": list(b.get("mcp_server_ids") or []), + "plugin_ids": list(b.get("plugin_ids") or []), + "kb_ids": list(b.get("kb_ids") or []), + } + + +def bindings_of(d: Dict[str, Any]) -> Dict[str, List[str]]: + """Get the four binding id lists from a preset bundle's ``bindings``.""" + return normalize_binding_lists(d.get("bindings")) + + +def binding_counts(bindings: Dict[str, List[str]]) -> Dict[str, int]: + return { + "skill_count": len(bindings.get("skill_ids") or []), + "mcp_count": len(bindings.get("mcp_server_ids") or []), + "plugin_count": len(bindings.get("plugin_ids") or []), + "kb_count": len(bindings.get("kb_ids") or []), + } + + +def make_market_meta( + *, + slug: str, + name: str, + avatar: str, + summary: str, + description: str, + category: str, + tags: Any, + version: str, + author: str, + source: str, + featured: bool, + deletable: bool, + bindings: Dict[str, List[str]], +) -> Dict[str, Any]: + """Preset bundle / community listing → isomorphic public list metadata (without the system_prompt body).""" + return { + "slug": slug, + "name": name or slug, + "avatar": avatar or "", + "summary": summary or "", + "description": description or "", + "category": category or DEFAULT_AGENT_CATEGORY, + "tags": list(tags or []), + "version": version or "1.0.0", + "author": author, + "source": source, + "featured": featured, + "deletable": deletable, + **binding_counts(bindings), + } + + +def bundle_public_meta(b: Dict[str, Any]) -> Dict[str, Any]: + # Presets (filesystem) ship with the repo; admins cannot delete them online (deletable=False). + return make_market_meta( + slug=b.get("slug"), + name=b.get("name"), + avatar=b.get("avatar"), + summary=b.get("summary"), + description=b.get("description"), + category=b.get("category"), + tags=b.get("tags"), + version=b.get("version"), + author=b.get("author") or "内置", + source="builtin", + featured=bool(b.get("featured")), + deletable=False, + bindings=bindings_of(b), + ) + + +def submission_public_meta(sub: AgentMarketSubmission) -> Dict[str, Any]: + # DB listing records (admin upload / user community submission) → admins can delete online (deletable=True). + return make_market_meta( + slug=sub.slug, + name=sub.name, + avatar=sub.avatar, + summary=sub.summary, + description=sub.description, + category=sub.category, + tags=sub.tags, + version=sub.version, + author=sub.submitter_name or sub.owner_user_id, + source="community", + featured=False, + deletable=True, + bindings=normalize_binding_lists(sub.bindings_snapshot), + ) diff --git a/src/backend/features/marketplace/agent_publishing.py b/src/backend/features/marketplace/agent_publishing.py new file mode 100644 index 00000000..cfe7ac3d --- /dev/null +++ b/src/backend/features/marketplace/agent_publishing.py @@ -0,0 +1,165 @@ +"""agent publishing responsibilities.""" +from __future__ import annotations +import uuid +from typing import Any, Dict, List, Optional +from features.marketplace.models import AgentMarketSubmission +from core.db.models import UserAgent +from core.infra.exceptions import BadRequestError, ResourceNotFoundError +from core.infra.time import utc_now +from features.marketplace import edition as ml +from features.marketplace.agent_categories import validate_category +from sqlalchemy.orm import Session +from sqlalchemy.orm.attributes import flag_modified +import logging +logger = logging.getLogger(__name__) +from features.marketplace import agent_catalog +from features.marketplace import agent_submission_data +from features.marketplace import agents_state + +def publish_agent_to_market( + db: Session, + agent_id: str, + *, + category: str, + summary: str = "", + submitter_name: str = "管理员上传", +) -> Dict[str, Any]: + """An admin lists an existing sub-agent directly on the marketplace as approved (no review). + + If the same agent_id already has an admin listing record, update it (keeping the original slug); otherwise create a new one. + """ + category = validate_category(category) + agent = db.query(UserAgent).filter(UserAgent.agent_id == agent_id).first() + if agent is None: + raise ResourceNotFoundError("user_agent", agent_id) + + now = utc_now() + existing = ( + db.query(AgentMarketSubmission) + .filter( + AgentMarketSubmission.agent_id == agent_id, + AgentMarketSubmission.owner_user_id == agents_state.ADMIN_UPLOAD_OWNER, + ) + .first() + ) + summary = (summary or "").strip() or (agent.description or "") + if existing is not None: + existing.name = agent.name + existing.avatar = agent.avatar + existing.description = agent.description or "" + existing.summary = summary + existing.category = category + existing.system_prompt = agent.system_prompt or "" + existing.welcome_message = agent.welcome_message or "" + existing.suggested_questions = list(agent.suggested_questions or []) + existing.model_config_snapshot = agent_submission_data.model_config_snapshot(agent) + existing.bindings_snapshot = agent_submission_data.bindings_snapshot(agent) + existing.tags = list((agent.extra_config or {}).get("ontology_tags") or []) + existing.status = "approved" + existing.review_note = "" + existing.reviewed_at = now + existing.updated_at = now + flag_modified(existing, "suggested_questions") + flag_modified(existing, "model_config_snapshot") + flag_modified(existing, "bindings_snapshot") + flag_modified(existing, "tags") + db.commit() + return {"slug": existing.slug, "action": "updated", "message": "子智能体市场内容已更新"} + + sub = agent_submission_data.new_submission( + submission_id=f"aadm_{uuid.uuid4().hex[:16]}", + slug=agent_submission_data.derive_slug(db, agent.name), + agent_id=agent_id, + owner_user_id=agents_state.ADMIN_UPLOAD_OWNER, + submitter_name=submitter_name, + name=agent.name, + avatar=agent.avatar, + description=agent.description or "", + summary=summary, + category=category, + note="管理员上传，免审核直接上架", + system_prompt=agent.system_prompt or "", + welcome_message=agent.welcome_message or "", + suggested_questions=agent.suggested_questions, + model_config_snapshot=agent_submission_data.model_config_snapshot(agent), + bindings_snapshot=agent_submission_data.bindings_snapshot(agent), + tags=list((agent.extra_config or {}).get("ontology_tags") or []), + status="approved", + reviewed_at=now, + ) + db.add(sub) + db.commit() + logger.info("agent_market_admin_published: agent=%s slug=%s", agent_id, sub.slug) + return {"slug": sub.slug, "action": "published", "message": "子智能体已上架市场"} + + +def create_market_agent( + db: Session, + *, + name: str, + category: str, + avatar: str = "", + description: str = "", + summary: str = "", + system_prompt: str = "", + welcome_message: str = "", + suggested_questions: Optional[List[str]] = None, + bindings: Optional[Dict[str, Any]] = None, + model_config: Optional[Dict[str, Any]] = None, + tags: Optional[List[str]] = None, + submitter_name: str = "管理员上传", +) -> Dict[str, Any]: + """An admin directly "creates" a sub-agent and lists it on the marketplace as approved (no global agent created). + + Mirrors the skill side's "creating a skill lists it on the skill marketplace": the new + sub-agent enters the sub-agent marketplace (explicitly installable as a global / private + clone) and does not take effect globally right away. Content comes from form fields rather + than a source ``UserAgent``, hence a synthetic placeholder ``agent_id`` (installs always use + the snapshot and never look back at a source agent). + """ + now = utc_now() + sub = agent_submission_data.new_submission( + submission_id=f"aadm_{uuid.uuid4().hex[:16]}", + slug=agent_submission_data.derive_slug(db, name), + agent_id=f"__market__{uuid.uuid4().hex[:12]}", + owner_user_id=agents_state.ADMIN_UPLOAD_OWNER, + submitter_name=submitter_name, + name=name, + avatar=avatar or "", + description=description or "", + summary=(summary or "").strip() or (description or ""), + category=validate_category(category), + note="管理员创建，直接上架", + system_prompt=system_prompt or "", + welcome_message=welcome_message or "", + suggested_questions=suggested_questions, + model_config_snapshot=model_config, + bindings_snapshot=bindings, + tags=tags, + status="approved", + reviewed_at=now, + ) + db.add(sub) + db.commit() + logger.info("agent_market_admin_created: slug=%s name=%s", sub.slug, name) + return {"slug": sub.slug, "action": "created", "message": "子智能体已上架子智能体市场"} + + +def delete_market_agent(db: Session, slug: str) -> Dict[str, Any]: + """An admin deletes one DB listing record from the marketplace. Presets ship with the repo and cannot be deleted online (400).""" + if agent_catalog.read_bundle(slug) is not None: + raise BadRequestError(message="预置子智能体随仓库发布，不可在线删除") + sub = db.query(AgentMarketSubmission).filter(AgentMarketSubmission.slug == slug).first() + if sub is None: + raise ResourceNotFoundError("marketplace_agent", slug) + db.delete(sub) + db.commit() + logger.info("agent_market_deleted: slug=%s", slug) + return {"slug": slug, "deleted": True} + + +def set_market_agent_enabled( + db: Session, slug: str, enabled: bool, *, updated_by: Optional[str] = None +) -> Dict[str, Any]: + """List/delist a marketplace sub-agent (controls whether it shows in the marketplace; installed clones are unaffected).""" + return ml.set_listing_enabled(db, ml.KIND_AGENT, slug, enabled, updated_by=updated_by) diff --git a/src/backend/features/marketplace/agent_submission_data.py b/src/backend/features/marketplace/agent_submission_data.py new file mode 100644 index 00000000..409a0282 --- /dev/null +++ b/src/backend/features/marketplace/agent_submission_data.py @@ -0,0 +1,155 @@ +"""agent submission data responsibilities.""" +from __future__ import annotations +import re +import uuid +from datetime import datetime +from typing import Any, Dict, List, Optional +from features.marketplace.models import AgentMarketSubmission +from core.db.models import UserAgent +from core.infra.exceptions import ResourceNotFoundError +from core.infra.time import utc_now +from features.marketplace.agent_categories import DEFAULT_AGENT_CATEGORY +from sqlalchemy.orm import Session +from features.marketplace import agent_metadata +from features.marketplace import agents_state + +def existing_slugs(db: Session) -> set: + """Marketplace slugs currently taken: preset directory + all submissions (including rejected; slugs are table-wide unique).""" + slugs = set() + if agents_state.AGENT_MARKET_DIR.is_dir(): + slugs.update(c.name for c in agents_state.AGENT_MARKET_DIR.iterdir() if c.is_dir()) + slugs.update(r[0] for r in db.query(AgentMarketSubmission.slug).all()) + return slugs + + +def derive_slug(db: Session, name: str) -> str: + """Derive the marketplace slug from the sub-agent name (ASCII-ized + unique); Chinese names fall back to agent-.""" + base = re.sub(r"[^a-z0-9_-]+", "-", (name or "").lower()).strip("-") + if not base: + base = f"agent-{uuid.uuid4().hex[:8]}" + taken = existing_slugs(db) + slug = base + n = 2 + while slug in taken: + slug = f"{base}-{n}" + n += 1 + return slug + + +def model_config_snapshot(agent: UserAgent) -> Dict[str, Any]: + return { + "temperature": float(agent.temperature) if agent.temperature is not None else None, + "max_tokens": agent.max_tokens, + "max_iters": agent.max_iters, + "timeout": agent.timeout, + } + + +def bindings_snapshot(agent: UserAgent) -> Dict[str, List[str]]: + return { + "skill_ids": list(agent.skill_ids or []), + "mcp_server_ids": list(agent.mcp_server_ids or []), + "plugin_ids": list(agent.plugin_ids or []), + "kb_ids": list(agent.kb_ids or []), + } + + +def submission_to_dict( + sub: AgentMarketSubmission, *, with_content: bool = False +) -> Dict[str, Any]: + data = { + "submission_id": sub.submission_id, + "slug": sub.slug, + "agent_id": sub.agent_id, + "owner_user_id": sub.owner_user_id, + "submitter_name": sub.submitter_name or "", + "name": sub.name, + "avatar": sub.avatar or "", + "description": sub.description or "", + "summary": sub.summary or "", + "category": sub.category or DEFAULT_AGENT_CATEGORY, + "tags": list(sub.tags or []), + "version": sub.version or "1.0.0", + "note": sub.note or "", + "status": sub.status, + "review_note": sub.review_note or "", + "reviewed_at": sub.reviewed_at.isoformat() if sub.reviewed_at else None, + "created_at": sub.created_at.isoformat() if sub.created_at else None, + } + if with_content: + data["system_prompt"] = sub.system_prompt or "" + data["welcome_message"] = sub.welcome_message or "" + data["suggested_questions"] = list(sub.suggested_questions or []) + data["bindings"] = agent_metadata.normalize_binding_lists(sub.bindings_snapshot) + return data + + +def owned_agent(db: Session, agent_id: str, owner_user_id: str) -> UserAgent: + """Fetch the sub-agent owned by this user (owner_type=user), otherwise 404.""" + agent = ( + db.query(UserAgent) + .filter( + UserAgent.agent_id == agent_id, + UserAgent.owner_type == "user", + UserAgent.user_id == owner_user_id, + ) + .first() + ) + if agent is None: + raise ResourceNotFoundError("user_agent", agent_id) + return agent + + +def new_submission( + *, + submission_id: str, + slug: str, + agent_id: str, + owner_user_id: str, + name: str, + category: str, + status: str, + submitter_name: str = "", + avatar: Optional[str] = None, + description: str = "", + summary: str = "", + note: str = "", + system_prompt: str = "", + welcome_message: str = "", + suggested_questions: Optional[List[str]] = None, + model_config_snapshot: Optional[Dict[str, Any]] = None, + bindings_snapshot: Optional[Dict[str, Any]] = None, + tags: Optional[List[str]] = None, + reviewed_at: Optional[datetime] = None, +) -> AgentMarketSubmission: + """Uniformly construct one ``AgentMarketSubmission`` (fixed fields and default normalization centralized here). + + The three entry points (user submission ``submit_to_marketplace`` / the create branch of the + admin direct-upload ``publish_agent_to_market`` / admin create-and-list + ``create_market_agent``) share the same field set; callers pass only what differs. + """ + now = utc_now() + return AgentMarketSubmission( + submission_id=submission_id, + slug=slug, + agent_id=agent_id, + owner_user_id=owner_user_id, + submitter_name=(submitter_name or "").strip(), + name=name, + avatar=avatar, + description=description or "", + summary=summary or "", + category=category, + tags=list(tags or []), + version="1.0.0", + note=(note or "").strip(), + system_prompt=system_prompt or "", + welcome_message=welcome_message or "", + suggested_questions=list(suggested_questions or []), + model_config_snapshot=dict(model_config_snapshot or {}), + bindings_snapshot=agent_metadata.normalize_binding_lists(bindings_snapshot), + status=status, + reviewed_at=reviewed_at, + created_at=now, + updated_at=now, + ) diff --git a/src/backend/features/marketplace/agent_submissions.py b/src/backend/features/marketplace/agent_submissions.py new file mode 100644 index 00000000..de5d5d00 --- /dev/null +++ b/src/backend/features/marketplace/agent_submissions.py @@ -0,0 +1,160 @@ +"""agent submissions responsibilities.""" +from __future__ import annotations +import uuid +from typing import Any, Dict, List, Optional +from features.marketplace.models import AgentMarketSubmission +from core.infra.exceptions import BadRequestError, ResourceNotFoundError +from core.infra.time import utc_now +from features.marketplace.agent_categories import validate_category +from fastapi import HTTPException +from sqlalchemy.orm import Session +import logging +logger = logging.getLogger(__name__) +from features.marketplace import agent_submission_data + +def submit_to_marketplace( + db: Session, + agent_id: str, + *, + owner_user_id: str, + submitter_name: str = "", + note: str = "", + category: str = "", + summary: str = "", +) -> Dict[str, Any]: + """A user submits their self-built sub-agent for marketplace listing (content snapshot, pending admin review).""" + agent = agent_submission_data.owned_agent(db, agent_id, owner_user_id) + + active = ( + db.query(AgentMarketSubmission) + .filter( + AgentMarketSubmission.agent_id == agent_id, + AgentMarketSubmission.owner_user_id == owner_user_id, + AgentMarketSubmission.status.in_(["pending", "approved"]), + ) + .first() + ) + if active is not None: + state = "待审核" if active.status == "pending" else "已上架" + raise HTTPException(status_code=409, detail=f"该子智能体已有{state}的上架申请") + + sub = agent_submission_data.new_submission( + submission_id=f"asub_{uuid.uuid4().hex[:16]}", + slug=agent_submission_data.derive_slug(db, agent.name), + agent_id=agent_id, + owner_user_id=owner_user_id, + submitter_name=submitter_name, + name=agent.name, + avatar=agent.avatar, + description=agent.description or "", + summary=(summary or "").strip() or (agent.description or ""), + category=validate_category(category), + note=note, + system_prompt=agent.system_prompt or "", + welcome_message=agent.welcome_message or "", + suggested_questions=agent.suggested_questions, + model_config_snapshot=agent_submission_data.model_config_snapshot(agent), + bindings_snapshot=agent_submission_data.bindings_snapshot(agent), + tags=list((agent.extra_config or {}).get("ontology_tags") or []), + status="pending", + ) + db.add(sub) + db.commit() + logger.info( + "agent_market_submission_created: id=%s agent=%s owner=%s slug=%s", + sub.submission_id, + agent_id, + owner_user_id, + sub.slug, + ) + return agent_submission_data.submission_to_dict(sub) + + +def list_my_submissions(db: Session, owner_user_id: str) -> List[Dict[str, Any]]: + rows = ( + db.query(AgentMarketSubmission) + .filter(AgentMarketSubmission.owner_user_id == owner_user_id) + .order_by(AgentMarketSubmission.created_at.desc()) + .all() + ) + return [agent_submission_data.submission_to_dict(r) for r in rows] + + +def withdraw_submission(db: Session, submission_id: str, owner_user_id: str) -> None: + """A user withdraws their own submission: pending/rejected can be deleted; approved requires an admin to delist.""" + sub = ( + db.query(AgentMarketSubmission) + .filter( + AgentMarketSubmission.submission_id == submission_id, + AgentMarketSubmission.owner_user_id == owner_user_id, + ) + .first() + ) + if sub is None: + raise ResourceNotFoundError("agent_market_submission", submission_id) + if sub.status == "approved": + raise BadRequestError(message="子智能体已上架，如需下架请联系管理员") + db.delete(sub) + db.commit() + + +def list_submissions(db: Session, status: Optional[str] = None) -> List[Dict[str, Any]]: + """Admin view: all listing submissions, filterable by status (pending first, newest → oldest).""" + q = db.query(AgentMarketSubmission) + if status: + q = q.filter(AgentMarketSubmission.status == status) + rows = q.order_by(AgentMarketSubmission.created_at.desc()).all() + order = {"pending": 0, "approved": 1, "rejected": 2} + rows.sort(key=lambda r: order.get(r.status, 3)) + return [agent_submission_data.submission_to_dict(r) for r in rows] + + +def get_submission(db: Session, submission_id: str) -> Dict[str, Any]: + sub = ( + db.query(AgentMarketSubmission) + .filter(AgentMarketSubmission.submission_id == submission_id) + .first() + ) + if sub is None: + raise ResourceNotFoundError("agent_market_submission", submission_id) + return agent_submission_data.submission_to_dict(sub, with_content=True) + + +def review_submission( + db: Session, + submission_id: str, + *, + approve: bool, + review_note: str = "", + category: str = "", +) -> Dict[str, Any]: + """Admin review: approval lists it immediately; rejecting pending = refuse, rejecting approved = delist.""" + sub = ( + db.query(AgentMarketSubmission) + .filter(AgentMarketSubmission.submission_id == submission_id) + .first() + ) + if sub is None: + raise ResourceNotFoundError("agent_market_submission", submission_id) + if approve and sub.status != "pending": + state = "已上架" if sub.status == "approved" else "已驳回" + raise HTTPException( + status_code=409, + detail=f"该申请{state}，不能重复审核通过（如需重新上架请让用户重新提交）", + ) + if not approve and sub.status == "rejected": + raise HTTPException(status_code=409, detail="该申请已是驳回状态") + sub.status = "approved" if approve else "rejected" + sub.review_note = (review_note or "").strip() + if approve and (category or "").strip(): + sub.category = validate_category(category) + sub.reviewed_at = utc_now() + sub.updated_at = utc_now() + db.commit() + logger.info( + "agent_market_submission_reviewed: id=%s slug=%s status=%s", + sub.submission_id, + sub.slug, + sub.status, + ) + return agent_submission_data.submission_to_dict(sub) diff --git a/src/backend/core/services/marketplace_agent_versions.py b/src/backend/features/marketplace/agent_versions.py similarity index 88% rename from src/backend/core/services/marketplace_agent_versions.py rename to src/backend/features/marketplace/agent_versions.py index acc6b904..a25c413c 100644 --- a/src/backend/core/services/marketplace_agent_versions.py +++ b/src/backend/features/marketplace/agent_versions.py @@ -1,28 +1,33 @@ """Version-aware sources for the existing agent marketplace API.""" from __future__ import annotations +import features.marketplace.agent_catalog as market_owner_agent_catalog +import features.marketplace.agent_installation as market_owner_agent_installation +from features.marketplace.agents_state import MAX_DESC as _MAX_DESC +import features.marketplace.skill_metadata as market_owner_skill_metadata +import logging +logger = logging.getLogger(__name__) import json from typing import Any, Dict, List, Optional from sqlalchemy.orm import Session -from . import agent_market_service as original -from . import marketplace_version_store as store -from .agent_market_service import _MAX_DESC, logger + +import features.marketplace.versions.store as store + + -def __getattr__(name): - return getattr(original, name) def get_agent_detail(db, slug): - native = original.get_agent_detail(db, slug) + native = market_owner_agent_catalog.get_agent_detail(db, slug) return store.detail(db, "agent", slug, native) def _resolve_market_entry(db, slug): - native = original._resolve_market_entry(db, slug) + native = market_owner_agent_installation.resolve_market_entry(db, slug) snapshot = store.active(db, "agent", slug) return json.loads(store.decode(snapshot)["agent.json"]) if snapshot else native @@ -40,10 +45,10 @@ def _resolve_bindings( - plugin slugs: bind if already installed, otherwise install as private and then bind; - unresolvable items go into ``dropped``; auto-installed skills/plugins requiring credentials go into ``needs_secret``. """ - from core.config.catalog import get_enabled_ids - from core.plugins import management as plugin_service - from core.services import marketplace_skill_versions as mk - from core.services.user_agent_service import UserAgentService + from core.capabilities.catalog.api import get_enabled_ids + from core.capabilities.plugins import management as plugin_service + from features.marketplace import skill_versions as mk + from core.capabilities.agents.edition import UserAgentService avail = UserAgentService(db).list_available_resources(owner_user_id=owner_user_id) avail_skill = {s["id"] for s in avail.get("skills", [])} @@ -87,7 +92,7 @@ def _resolve_bindings( res = mk.install_marketplace_skill(db, sid, owner_user_id=owner_user_id, secrets={}) final["skill_ids"].append(res["id"]) report["installed"].append(f"skill:{sid}") - if mk.market_skill_requires_secrets(sid): + if market_owner_skill_metadata.market_skill_requires_secrets(sid): report["needs_secret"].append(f"skill:{sid}") except Exception as exc: # noqa: BLE001 logger.info("agent clone: skill binding unresolved %s (%s)", sid, exc) @@ -136,7 +141,7 @@ def install_marketplace_agent( operator_name: Optional[str] = None, ) -> Dict[str, Any]: """Install a marketplace sub-agent as a private clone (admin owner_user_id=None → global admin sub-agent).""" - from core.services.user_agent_service import UserAgentService + from core.capabilities.agents.edition import UserAgentService entry = _resolve_market_entry(db, slug) final_bindings, report = _resolve_bindings(db, entry["bindings"], owner_user_id, operator_name) diff --git a/src/backend/features/marketplace/agents_state.py b/src/backend/features/marketplace/agents_state.py new file mode 100644 index 00000000..2e084ae2 --- /dev/null +++ b/src/backend/features/marketplace/agents_state.py @@ -0,0 +1,14 @@ +"""agents state responsibilities.""" +from __future__ import annotations +from extensions.paths import AGENTS_DIR + +AGENT_MARKET_DIR = AGENTS_DIR / "marketplace" + + +AGENT_JSON = "agent.json" + + +ADMIN_UPLOAD_OWNER = "__admin_upload__" + + +MAX_DESC = 20 diff --git a/src/backend/features/marketplace/api/__init__.py b/src/backend/features/marketplace/api/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/src/backend/features/marketplace/api/__init__.py @@ -0,0 +1 @@ + diff --git a/src/backend/api/routes/v1/agent_marketplace.py b/src/backend/features/marketplace/api/agent_marketplace.py similarity index 82% rename from src/backend/api/routes/v1/agent_marketplace.py rename to src/backend/features/marketplace/api/agent_marketplace.py index fd136d97..44a07da2 100644 --- a/src/backend/api/routes/v1/agent_marketplace.py +++ b/src/backend/features/marketplace/api/agent_marketplace.py @@ -14,6 +14,14 @@ """ from __future__ import annotations +import features.marketplace.agent_catalog as market_owner_agent_catalog + + +import features.marketplace.agent_installation as market_owner_agent_installation + +import features.marketplace.agent_submissions as market_owner_agent_submissions + + import logging @@ -26,8 +34,8 @@ from core.db.engine import get_db from core.infra.exceptions import AccessDeniedError from core.infra.responses import created_response, success_response -from core.services import marketplace_agent_versions as am -from core.services import marketplace_listing as ml +from features.marketplace import agent_versions as am +from features.marketplace import edition as ml router = APIRouter(prefix="/v1/agent-marketplace", tags=["Agent Marketplace"]) logger = logging.getLogger(__name__) @@ -44,9 +52,9 @@ def list_agents( user: UserContext = Depends(get_current_user), db: Session = Depends(get_db), ): - items = am.list_marketplace_agents(db, viewer_user_id=str(user.user_id)) - items = am.annotate_installed(items, db, owner_user_id=str(user.user_id)) - return success_response(data={"items": items, "categories": am.categories_from_items(items)}) + items = market_owner_agent_catalog.list_marketplace_agents(db, viewer_user_id=str(user.user_id)) + items = market_owner_agent_installation.annotate_installed(items, db, owner_user_id=str(user.user_id)) + return success_response(data={"items": items, "categories": market_owner_agent_catalog.categories_from_items(items)}) @router.get("/categories", summary="子智能体市场分类") @@ -54,7 +62,7 @@ def list_categories( user: UserContext = Depends(get_current_user), db: Session = Depends(get_db), ): - return success_response(data=am.list_categories(db, viewer_user_id=str(user.user_id))) + return success_response(data=market_owner_agent_catalog.list_categories(db, viewer_user_id=str(user.user_id))) @router.get("/agents/{slug}", summary="子智能体市场详情") @@ -65,7 +73,7 @@ def get_agent( ): ml.ensure_item_visible(db, ml.KIND_AGENT, slug, str(user.user_id), resource="marketplace_agent") data = am.get_agent_detail(db, slug) - data["installed"] = am.is_installed(db, slug, owner_user_id=str(user.user_id)) + data["installed"] = market_owner_agent_installation.is_installed(db, slug, owner_user_id=str(user.user_id)) return success_response(data=data) @@ -104,7 +112,7 @@ def submit_agent( db: Session = Depends(get_db), ): _require_can_add_agent(str(user.user_id), db) - result = am.submit_to_marketplace( + result = market_owner_agent_submissions.submit_to_marketplace( db, body.agent_id, owner_user_id=str(user.user_id), @@ -121,7 +129,7 @@ def list_my_submissions( user: UserContext = Depends(get_current_user), db: Session = Depends(get_db), ): - return success_response(data={"items": am.list_my_submissions(db, str(user.user_id))}) + return success_response(data={"items": market_owner_agent_submissions.list_my_submissions(db, str(user.user_id))}) @router.delete("/submissions/{submission_id}", summary="撤回上架申请") @@ -130,5 +138,5 @@ def withdraw_submission( user: UserContext = Depends(get_current_user), db: Session = Depends(get_db), ): - am.withdraw_submission(db, submission_id, owner_user_id=str(user.user_id)) + market_owner_agent_submissions.withdraw_submission(db, submission_id, owner_user_id=str(user.user_id)) return success_response(data={"id": submission_id, "message": "申请已撤回"}) diff --git a/src/backend/api/routes/v1/catalog_ownership.py b/src/backend/features/marketplace/api/catalog_ownership.py similarity index 94% rename from src/backend/api/routes/v1/catalog_ownership.py rename to src/backend/features/marketplace/api/catalog_ownership.py index a520a2fa..db70fcce 100644 --- a/src/backend/api/routes/v1/catalog_ownership.py +++ b/src/backend/features/marketplace/api/catalog_ownership.py @@ -10,9 +10,10 @@ def _load_owned_capability_items(db, user_id: str) -> tuple: injected only into that user's /v1/catalog response; the frontend shows the "mine" badge and delete button based on ``owner == 'self'``. """ - from core.config.catalog_details import resolve_skill_detail - from core.db.models import AdminMcpServer, AdminSkill, McpMarketInstallation - from core.services.skill_icon_service import get_skill_icons + from core.capabilities.catalog.details import resolve_skill_detail + from core.db.models import AdminMcpServer, AdminSkill + from features.marketplace.models import McpMarketInstallation + from core.capabilities.skills.icons import get_skill_icons skill_items: List[Dict[str, Any]] = [] mcp_items: List[Dict[str, Any]] = [] @@ -128,7 +129,7 @@ def _plugin_component_ids(db) -> tuple: except Exception as exc: # noqa: BLE001 logger.warning("plugin component id load failed: %s", exc) try: - from core.plugins.management import builtin_plugin_component_ids + from core.capabilities.plugins.management import builtin_plugin_component_ids fs_skill_ids, fs_mcp_ids = builtin_plugin_component_ids() skill_ids |= fs_skill_ids diff --git a/src/backend/api/routes/v1/marketplace.py b/src/backend/features/marketplace/api/marketplace.py similarity index 84% rename from src/backend/api/routes/v1/marketplace.py rename to src/backend/features/marketplace/api/marketplace.py index 55cb71b2..e2384e2f 100644 --- a/src/backend/api/routes/v1/marketplace.py +++ b/src/backend/features/marketplace/api/marketplace.py @@ -14,6 +14,13 @@ """ from __future__ import annotations +import features.marketplace.skill_catalog as market_owner_skill_catalog + + + +import features.marketplace.skill_submissions as market_owner_skill_submissions + + import logging from typing import Dict, Optional @@ -26,8 +33,8 @@ from core.auth.capabilities import resolve_user_capabilities from core.db.engine import get_db from core.infra.responses import created_response, success_response -from core.services import marketplace_skill_versions as mk -from core.services import marketplace_listing as ml +from features.marketplace import skill_versions as mk +from features.marketplace import edition as ml from core.infra.exceptions import AccessDeniedError router = APIRouter(prefix="/v1/marketplace", tags=["Skill Marketplace"]) @@ -46,10 +53,10 @@ def list_skills( db: Session = Depends(get_db), ): """列出全部市场技能（预置+社区上架，按可见范围过滤），并标注当前用户是否已安装。""" - items = mk.list_marketplace_skills(db, viewer_user_id=str(user.user_id)) - items = mk.annotate_installed(items, db, owner_user_id=str(user.user_id)) + items = market_owner_skill_catalog.list_marketplace_skills(db, viewer_user_id=str(user.user_id)) + items = market_owner_skill_catalog.annotate_installed(items, db, owner_user_id=str(user.user_id)) return success_response( - data={"items": items, "categories": mk.list_categories(db, viewer_user_id=str(user.user_id))} + data={"items": items, "categories": market_owner_skill_catalog.list_categories(db, viewer_user_id=str(user.user_id))} ) @@ -58,7 +65,7 @@ def list_categories( user: UserContext = Depends(get_current_user), db: Session = Depends(get_db), ): - return success_response(data=mk.list_categories(db, viewer_user_id=str(user.user_id))) + return success_response(data=market_owner_skill_catalog.list_categories(db, viewer_user_id=str(user.user_id))) @router.get("/skills/{slug}", summary="技能市场详情") @@ -69,7 +76,7 @@ def get_skill( ): ml.ensure_item_visible(db, ml.KIND_SKILL, slug, str(user.user_id), resource="marketplace_skill") data = mk.get_marketplace_skill(slug, db) - data["installed"] = mk.is_installed(db, slug, owner_user_id=str(user.user_id)) + data["installed"] = market_owner_skill_catalog.is_installed(db, slug, owner_user_id=str(user.user_id)) return success_response(data=data) @@ -118,7 +125,7 @@ def submit_skill( ): """把自己的私有技能提交上架申请（内容快照），管理员审核通过后全员可安装。""" _require_can_add_skill(str(user.user_id), db) - result = mk.submit_to_marketplace( + result = market_owner_skill_submissions.submit_to_marketplace( db, body.skill_id, owner_user_id=str(user.user_id), @@ -135,7 +142,7 @@ def list_my_submissions( user: UserContext = Depends(get_current_user), db: Session = Depends(get_db), ): - return success_response(data={"items": mk.list_my_submissions(db, str(user.user_id))}) + return success_response(data={"items": market_owner_skill_submissions.list_my_submissions(db, str(user.user_id))}) @router.delete("/submissions/{submission_id}", summary="撤回上架申请") @@ -144,5 +151,5 @@ def withdraw_submission( user: UserContext = Depends(get_current_user), db: Session = Depends(get_db), ): - mk.withdraw_submission(db, submission_id, owner_user_id=str(user.user_id)) + market_owner_skill_submissions.withdraw_submission(db, submission_id, owner_user_id=str(user.user_id)) return success_response(data={"id": submission_id, "message": "申请已撤回"}) diff --git a/src/backend/api/routes/v1/mcp_marketplace.py b/src/backend/features/marketplace/api/mcp_marketplace.py similarity index 84% rename from src/backend/api/routes/v1/mcp_marketplace.py rename to src/backend/features/marketplace/api/mcp_marketplace.py index 72cc4edb..e31e83cf 100644 --- a/src/backend/api/routes/v1/mcp_marketplace.py +++ b/src/backend/features/marketplace/api/mcp_marketplace.py @@ -1,6 +1,11 @@ """User-facing MCP marketplace: browse, install, and submit private remote MCPs.""" from __future__ import annotations +from core.capabilities.mcp import oauth as oauth_module +from features.marketplace import mcp_listing +from features.marketplace import mcp_curated +from features.marketplace import mcp_installation +from features.marketplace import mcp_submissions from typing import Dict, List, Optional @@ -9,8 +14,8 @@ from core.db.engine import get_db from core.infra.exceptions import AccessDeniedError from core.infra.responses import created_response, success_response -from core.services import mcp_marketplace_service as market -from core.services import mcp_oauth_service + +from core.capabilities.mcp import oauth as mcp_oauth_service from fastapi import APIRouter, Depends, Query, Request from fastapi.responses import HTMLResponse from pydantic import BaseModel, Field @@ -32,9 +37,9 @@ def list_items( user: UserContext = Depends(get_current_user), db: Session = Depends(get_db), ): - market.ensure_curated_market_items(db) + mcp_curated.ensure_curated_market_items(db) return success_response( - data=market.list_market_items( + data=mcp_listing.list_market_items( db, owner_user_id=str(user.user_id), viewer_user_id=str(user.user_id), @@ -49,7 +54,7 @@ def get_item( db: Session = Depends(get_db), ): return success_response( - data=market.get_market_item( + data=mcp_listing.get_market_item( db, slug, owner_user_id=str(user.user_id), @@ -81,21 +86,21 @@ async def start_oauth( ): user_id = str(user.user_id) _require_can_add_mcp(user_id, db) - market.get_market_item( + mcp_listing.get_market_item( db, body.slug, owner_user_id=user_id, viewer_user_id=user_id, ) return success_response( - data=await mcp_oauth_service.start_oauth_install( + data=await oauth_module.start_oauth_install( db, slug=body.slug, owner_user_id=user_id, installed_by=user_id, auth_method=body.auth_method, credentials=body.credentials, - callback_url_base=mcp_oauth_service.public_oauth_callback_url(request), + callback_url_base=oauth_module.public_oauth_callback_url(request), client_id=body.client_id, client_secret=body.client_secret, ) @@ -108,7 +113,7 @@ async def oauth_status( user: UserContext = Depends(get_current_user), ): return success_response( - data=await mcp_oauth_service.get_flow_status(flow_id, owner_user_id=str(user.user_id)) + data=await oauth_module.get_flow_status(flow_id, owner_user_id=str(user.user_id)) ) @@ -118,7 +123,7 @@ async def cancel_oauth( user: UserContext = Depends(get_current_user), ): return success_response( - data=await mcp_oauth_service.cancel_flow( + data=await oauth_module.cancel_flow( flow_id, owner_user_id=str(user.user_id), ) @@ -135,13 +140,15 @@ async def oauth_callback( flow_id: str = Query(...), code: str = Query(""), state: Optional[str] = Query(None), + iss: Optional[str] = Query(None), error: str = Query(""), error_description: str = Query(""), ): - await mcp_oauth_service.complete_callback( + await oauth_module.complete_callback( flow_id, code=code, state=state, + iss=iss, error=error_description or error, ) return HTMLResponse( @@ -159,7 +166,7 @@ async def install_item( ): user_id = str(user.user_id) _require_can_add_mcp(user_id, db) - result = await market.install_market_item( + result = await mcp_installation.install_market_item( db, body.slug, owner_user_id=user_id, @@ -187,7 +194,7 @@ async def submit_item( ): user_id = str(user.user_id) _require_can_add_mcp(user_id, db) - result = await market.submit_to_marketplace( + result = await mcp_submissions.submit_to_marketplace( db, body.source_server_id, owner_user_id=user_id, @@ -206,7 +213,7 @@ def list_my_submissions( user: UserContext = Depends(get_current_user), db: Session = Depends(get_db), ): - return success_response(data={"items": market.list_my_submissions(db, str(user.user_id))}) + return success_response(data={"items": mcp_submissions.list_my_submissions(db, str(user.user_id))}) @router.delete("/submissions/{submission_id}", summary="撤回 MCP 上架申请") @@ -217,5 +224,5 @@ def withdraw_submission( ): user_id = str(user.user_id) _require_can_add_mcp(user_id, db) - market.withdraw_submission(db, submission_id, user_id) + mcp_submissions.withdraw_submission(db, submission_id, user_id) return success_response(data={"submission_id": submission_id, "withdrawn": True}) diff --git a/src/backend/features/marketplace/core_catalog.py b/src/backend/features/marketplace/core_catalog.py new file mode 100644 index 00000000..77bab062 --- /dev/null +++ b/src/backend/features/marketplace/core_catalog.py @@ -0,0 +1,108 @@ +"""Marketplace contribution to core capability selection and chat modes.""" + +class MarketplaceCatalogProvider: + def plugin_metadata(self, db, slug): + from features.marketplace.plugin_market import resolve_market_meta + return resolve_market_meta(db, slug) + + def suspended_mcp_ids(self, db, ids): + from features.marketplace.models import McpMarketInstallation + return { + row[0] for row in db.query(McpMarketInstallation.server_id).filter( + McpMarketInstallation.server_id.in_(ids), + McpMarketInstallation.status == "suspended", + ).all() + } + + def list_marketplace_skills(self, db): + from features.marketplace import skill_catalog + return skill_catalog.list_marketplace_skills(db) + + def list_marketplace_agents(self, db): + from features.marketplace import agent_catalog + return agent_catalog.list_marketplace_agents(db) + + def list_plugins(self, db, user_id): + from features.marketplace import plugin_market + return plugin_market.list_plugins(db, user_id) + + def install_marketplace_skill(self, db, slug, *, owner_user_id): + from features.marketplace import skill_installation + return skill_installation.install_marketplace_skill(db, slug, owner_user_id=owner_user_id) + + def install_marketplace_agent(self, db, slug, *, owner_user_id): + from features.marketplace import agent_installation + return agent_installation.install_marketplace_agent(db, slug, owner_user_id=owner_user_id) + + def market_package(self, db, slug): + from features.marketplace.models import PluginMarketPackage + if not slug: + return None + return db.query(PluginMarketPackage).filter(PluginMarketPackage.slug == slug).first() + + def publish_package(self, db, slug, fields, *, created_by, now): + from features.marketplace.models import PluginMarketPackage + from sqlalchemy.orm.attributes import flag_modified + existing = self.market_package(db, slug) + if existing is not None: + for key, value in fields.items(): + setattr(existing, key, value) + flag_modified(existing, "required_secrets") + action = "updated" + else: + db.add(PluginMarketPackage(slug=slug, created_at=now, created_by=created_by, **fields)) + action = "published" + db.commit() + return action + + def delete_package(self, db, slug): + from core.infra.exceptions import ResourceNotFoundError + row = self.market_package(db, slug) + if row is None: + raise ResourceNotFoundError("plugin_market_package", slug) + db.delete(row) + db.commit() + + def excluded_skills(self, db, slug): + from features.marketplace.models import PluginMarketSkillExclusion + if db is None: + return set() + return {row[0] for row in db.query(PluginMarketSkillExclusion.skill_name).filter(PluginMarketSkillExclusion.slug == slug).all()} + + def exclude_skill(self, db, slug, skill_name, *, created_by): + from features.marketplace.models import PluginMarketSkillExclusion + existing = db.query(PluginMarketSkillExclusion).filter( + PluginMarketSkillExclusion.slug == slug, + PluginMarketSkillExclusion.skill_name == skill_name, + ).first() + if existing is not None: + return False + db.add(PluginMarketSkillExclusion(slug=slug, skill_name=skill_name, created_by=created_by)) + db.commit() + return True + + def mcp_version(self, db, slug): + from features.marketplace.models import McpMarketItem, McpMarketVersion + from core.infra.exceptions import ResourceNotFoundError + item = db.query(McpMarketItem).filter( + McpMarketItem.slug == slug, McpMarketItem.deleted_at.is_(None), + McpMarketItem.status == "active", + ).first() + if not item: + raise ResourceNotFoundError("mcp_market_item", slug) + version = db.query(McpMarketVersion).filter(McpMarketVersion.version_id == item.latest_version_id).first() + if not version: + raise ResourceNotFoundError("mcp_market_version", item.latest_version_id) + return item, version + + def normalize_auth_config(self, config, schema): + from features.marketplace import mcp_validation + return mcp_validation.normalize_auth_config(config, schema) + + def installation_secrets(self, version, credentials, existing, *, auth_method): + from features.marketplace import mcp_credentials + return mcp_credentials.installation_secrets(version, credentials, existing, auth_method=auth_method) + + async def install_market_item(self, db, slug, **kwargs): + from features.marketplace import mcp_installation + return await mcp_installation.install_market_item(db, slug, **kwargs) diff --git a/src/backend/features/marketplace/edition.py b/src/backend/features/marketplace/edition.py new file mode 100644 index 00000000..818b0a82 --- /dev/null +++ b/src/backend/features/marketplace/edition.py @@ -0,0 +1,10 @@ +"""Edition policy selected by the installed distribution.""" + +def __getattr__(name): + if name in ('get_hidden_item_ids', 'is_item_visible'): + import features.marketplace.personal_visibility as provider + return getattr(provider, name) + if name in ('KIND_AGENT', 'KIND_MCP', 'KIND_PLUGIN', 'KIND_SKILL', 'annotate_and_filter', 'ensure_item_visible', 'get_disabled_ids', 'set_listing_enabled'): + import features.marketplace.personal_listing as provider + return getattr(provider, name) + raise AttributeError(name) diff --git a/src/backend/features/marketplace/extension.py b/src/backend/features/marketplace/extension.py new file mode 100644 index 00000000..a9ee7edd --- /dev/null +++ b/src/backend/features/marketplace/extension.py @@ -0,0 +1,4 @@ +"""Installed marketplace models and runtime composition.""" +NAME = "marketplace" +def setup(): + return True diff --git a/src/backend/features/marketplace/mcp_administration.py b/src/backend/features/marketplace/mcp_administration.py new file mode 100644 index 00000000..6b13d123 --- /dev/null +++ b/src/backend/features/marketplace/mcp_administration.py @@ -0,0 +1,291 @@ +"""mcp administration responsibilities.""" +from __future__ import annotations +from core.infra.time import utc_now +from typing import Any, Dict, List, Optional +from core.db.models import AdminMcpServer +from features.marketplace.models import McpMarketInstallation, McpMarketItem +from core.infra.exceptions import BadRequestError, ResourceNotFoundError +from features.marketplace import edition as ml +from core.capabilities.mcp.management import decrypt_mcp_headers, encrypt_mcp_headers, refresh_mcp_caches +from sqlalchemy.orm import Session +from features.marketplace import mcp_listing +from features.marketplace import mcp_credentials +from features.marketplace import mcp_validation +from features.marketplace import mcp_versions + +def set_market_enabled( + db: Session, slug: str, enabled: bool, *, updated_by: Optional[str] = None +) -> Dict[str, Any]: + item = ( + db.query(McpMarketItem) + .filter(McpMarketItem.slug == slug, McpMarketItem.deleted_at.is_(None)) + .first() + ) + if not item: + raise ResourceNotFoundError("mcp_market_item", slug) + return ml.set_listing_enabled(db, ml.KIND_MCP, slug, enabled, updated_by=updated_by) + + +def update_market_item( + db: Session, + slug: str, + *, + display_name: Optional[str] = None, + description: Optional[str] = None, + user_intro: Optional[str] = None, + category: Optional[str] = None, + tags: Optional[List[str]] = None, + icon: Optional[str] = None, + requires_auth: Optional[bool] = None, + credential_mode: Optional[str] = None, + managed_credentials: Optional[Dict[str, str]] = None, +) -> Dict[str, Any]: + """Edit marketplace presentation metadata and its administrator-owned auth policy. + + Tool schemas, transport, endpoint templates, and version numbers remain + immutable. Authentication can be disabled, delegated to each installer, + or backed by the administrator source server's encrypted credentials. + Secret values are never persisted in marketplace rows or returned by this + function. Presentation fields are propagated to existing installations. + """ + item = ( + db.query(McpMarketItem) + .filter(McpMarketItem.slug == slug, McpMarketItem.deleted_at.is_(None)) + .first() + ) + if not item: + raise ResourceNotFoundError("mcp_market_item", slug) + version = mcp_versions.version_for_item(db, item) + previous_auth_schema = list(version.auth_schema or []) + + if display_name is not None: + normalized_name = display_name.strip() + if not normalized_name: + raise BadRequestError(message="MCP 市场名称不能为空") + item.display_name = normalized_name + if description is not None: + item.description = description.strip() + if user_intro is not None: + item.user_intro = user_intro.strip() or None + if category is not None: + item.category = mcp_validation.validate_category(category) + if tags is not None: + item.tags = list(dict.fromkeys(str(tag).strip() for tag in tags if str(tag).strip())) + if icon is not None: + item.icon = icon.strip() or None + + auth_schema = list(version.auth_schema or []) + auth_config = mcp_validation.normalize_auth_config(version.auth_config, auth_schema) + if requires_auth is False: + auth_schema = [] + auth_config = mcp_validation.normalize_auth_config( + { + "default_method": "none", + "methods": [{"id": "none", "type": "none", "label": "无需认证"}], + "credential_mode": "installer", + }, + auth_schema, + ) + elif requires_auth is True and not any( + method["type"] != "none" for method in auth_config["methods"] + ): + auth_schema = [ + { + "key": "Authorization", + "label": "Token / Authorization", + "target": "header", + "required": True, + "secret": True, + "prefix": "Bearer ", + } + ] + auth_config = mcp_validation.normalize_auth_config( + { + "default_method": "token", + "methods": [{"id": "token", "type": "token", "label": "Token"}], + "credential_mode": credential_mode or "installer", + }, + auth_schema, + ) + + if credential_mode is not None: + if credential_mode not in {"installer", "admin"}: + raise BadRequestError(message="MCP 凭据提供方式无效") + if credential_mode == "admin" and not auth_schema: + raise BadRequestError(message="请先勾选需要 Token/Auth") + auth_config["credential_mode"] = credential_mode + + version.auth_schema = auth_schema + version.auth_config = auth_config + + admin_source = None + if item.source == "admin" and version.source_server_id: + admin_source = ( + db.query(AdminMcpServer) + .filter( + AdminMcpServer.server_id == version.source_server_id, + AdminMcpServer.owner_user_id.is_(None), + ) + .first() + ) + if admin_source is not None: + admin_source.extra_config = { + **dict(admin_source.extra_config or {}), + "market_auth_schema": auth_schema, + "market_auth_config": auth_config, + } + + if auth_config["credential_mode"] == "admin": + if item.source != "admin" or not version.source_server_id: + raise BadRequestError(message="只有管理员发布的 MCP 才能统一配置 Token") + if admin_source is None: + raise BadRequestError(message="管理员发布源已不存在，无法统一配置 Token") + source_headers = decrypt_mcp_headers(admin_source.headers) + supplied = dict(managed_credentials or {}) + allowed_keys = {str(field.get("key") or "") for field in auth_schema} + unknown_keys = sorted(key for key in supplied if key not in allowed_keys) + if unknown_keys: + raise BadRequestError(message=f"未知的 MCP 认证字段：{', '.join(unknown_keys)}") + for field in auth_schema: + key = str(field.get("key") or "").strip() + value = str(supplied.get(key) or "").strip() + if value: + source_headers[mcp_credentials.credential_storage_key(field)] = mcp_credentials.materialize_credential_value( + field, value + ) + admin_source.headers = encrypt_mcp_headers(source_headers) + managed_headers = mcp_credentials.admin_managed_credential_headers( + db, + item, + version, + auth_method=str(auth_config["default_method"]), + ) + if not managed_headers: + raise BadRequestError(message="请填写管理员统一 Token/Auth") + else: + managed_headers = {} + + item.updated_at = utc_now() + + installation_server_ids = [ + str(row[0]) + for row in db.query(McpMarketInstallation.server_id) + .filter(McpMarketInstallation.slug == slug) + .all() + ] + installations: List[AdminMcpServer] = [] + if installation_server_ids: + installations = ( + db.query(AdminMcpServer) + .filter(AdminMcpServer.server_id.in_(installation_server_ids)) + .all() + ) + for server in installations: + server.display_name = item.display_name + server.description = item.description or "" + server.user_intro = item.user_intro + server.icon = item.icon + server.updated_at = item.updated_at + if requires_auth is False: + existing_headers = decrypt_mcp_headers(server.headers) + for field in previous_auth_schema: + existing_headers.pop(mcp_credentials.credential_storage_key(field), None) + server.headers = encrypt_mcp_headers(existing_headers) + elif managed_headers: + existing_headers = decrypt_mcp_headers(server.headers) + existing_headers.update(managed_headers) + server.headers = encrypt_mcp_headers(existing_headers) + + db.commit() + refresh_mcp_caches() + result = mcp_listing.get_market_item( + db, + slug, + owner_user_id=None, + viewer_user_id=None, + admin=True, + ) + result["updated_installations"] = len(installations) + return result + + +def delete_market_item( + db: Session, slug: str, *, updated_by: Optional[str] = None +) -> Dict[str, Any]: + item = ( + db.query(McpMarketItem) + .filter(McpMarketItem.slug == slug, McpMarketItem.deleted_at.is_(None)) + .first() + ) + if not item: + raise ResourceNotFoundError("mcp_market_item", slug) + version = mcp_versions.version_for_item(db, item) + if version.source_server_id: + source = ( + db.query(AdminMcpServer) + .filter(AdminMcpServer.server_id == version.source_server_id) + .first() + ) + if source and (source.extra_config or {}).get("market_source_only"): + db.delete(source) + item.deleted_at = utc_now() + item.updated_at = utc_now() + db.commit() + ml.set_listing_enabled(db, ml.KIND_MCP, slug, False, updated_by=updated_by) + return {"slug": slug, "deleted": True} + + +def set_suspended( + db: Session, + slug: str, + *, + suspended: bool, + reason: str = "", +) -> Dict[str, Any]: + item = ( + db.query(McpMarketItem) + .filter(McpMarketItem.slug == slug, McpMarketItem.deleted_at.is_(None)) + .first() + ) + if not item: + raise ResourceNotFoundError("mcp_market_item", slug) + now = utc_now() + item.status = "suspended" if suspended else "active" + item.status_reason = (reason or "").strip() if suspended else None + item.updated_at = now + affected = 0 + installs = db.query(McpMarketInstallation).filter(McpMarketInstallation.slug == slug).all() + if suspended: + server_ids = [] + for install in installs: + install.status = "suspended" + install.updated_at = now + server_ids.append(install.server_id) + if server_ids: + affected = ( + db.query(AdminMcpServer) + .filter(AdminMcpServer.server_id.in_(server_ids)) + .update({AdminMcpServer.is_enabled: False}, synchronize_session=False) + ) + else: + server_ids = [] + for install in installs: + if install.status != "suspended": + continue + install.status = "active" + install.updated_at = now + server_ids.append(install.server_id) + if server_ids: + affected = ( + db.query(AdminMcpServer) + .filter(AdminMcpServer.server_id.in_(server_ids)) + .update({AdminMcpServer.is_enabled: True}, synchronize_session=False) + ) + db.commit() + refresh_mcp_caches() + return { + "slug": slug, + "status": item.status, + "disabled_installations": affected if suspended else 0, + "restored_installations": affected if not suspended else 0, + } diff --git a/src/backend/core/config/mcp_marketplace_catalog.py b/src/backend/features/marketplace/mcp_catalog.py similarity index 100% rename from src/backend/core/config/mcp_marketplace_catalog.py rename to src/backend/features/marketplace/mcp_catalog.py diff --git a/src/backend/features/marketplace/mcp_credentials.py b/src/backend/features/marketplace/mcp_credentials.py new file mode 100644 index 00000000..33ad88b5 --- /dev/null +++ b/src/backend/features/marketplace/mcp_credentials.py @@ -0,0 +1,130 @@ +"""mcp credentials responsibilities.""" +from __future__ import annotations +import json +from typing import Any, Dict, List, Optional +from urllib.parse import parse_qsl, urlparse +from core.db.models import AdminMcpServer +from features.marketplace.models import McpMarketItem, McpMarketVersion +from core.infra.exceptions import BadRequestError +from core.capabilities.mcp.management import decrypt_mcp_headers, mcp_oauth_bundle_storage_key, mcp_query_secret_storage_key, mcp_url_override_storage_key +from sqlalchemy.orm import Session +from features.marketplace import mcp_validation + +def credential_storage_key(field: Dict[str, Any]) -> str: + target = str(field.get("target") or "header") + name = str(field.get("name") or field.get("key") or "").strip() + if target == "query": + return mcp_query_secret_storage_key(name) + if target == "url": + return mcp_url_override_storage_key() + return name + + +def admin_credential_source( + db: Session, + item: McpMarketItem, + version: McpMarketVersion, +) -> Optional[AdminMcpServer]: + if item.source != "admin" or not version.source_server_id: + return None + return ( + db.query(AdminMcpServer) + .filter( + AdminMcpServer.server_id == version.source_server_id, + AdminMcpServer.owner_user_id.is_(None), + ) + .first() + ) + + +def admin_managed_credential_headers( + db: Session, + item: McpMarketItem, + version: McpMarketVersion, + *, + auth_method: str, +) -> Dict[str, str]: + """Return an admin publisher's credentials without exposing them to marketplace APIs. + + Marketplace versions remain credential-free. For an admin-published MCP, + the original global/source ``AdminMcpServer`` is the encrypted credential + authority. If it still has every required field for the selected token + method, user installations may copy those values into their own encrypted + server row without asking the user to enter the same token again. + """ + auth_config = mcp_validation.normalize_auth_config(version.auth_config, list(version.auth_schema or [])) + if auth_config["credential_mode"] == "installer": + return {} + selected = next( + (method for method in auth_config["methods"] if method["id"] == auth_method), + None, + ) + if selected is None or selected["type"] != "token": + return {} + fields = [ + field + for field in (version.auth_schema or []) + if not field.get("methods") or auth_method in field.get("methods", []) + ] + if not fields: + return {} + source = admin_credential_source(db, item, version) + if source is None: + return {} + + stored = decrypt_mcp_headers(source.headers) + source_query = dict(parse_qsl(urlparse(source.url or "").query, keep_blank_values=True)) + managed: Dict[str, str] = {} + for field in fields: + storage_key = credential_storage_key(field) + value = str(stored.get(storage_key) or "") + if not value and str(field.get("target") or "header") == "query": + query_name = str(field.get("name") or field.get("key") or "").strip() + value = str(source_query.get(query_name) or "") + if value: + managed[storage_key] = value + elif field.get("required", True): + return {} + return managed + + +def materialize_credential_value(field: Dict[str, Any], value: str) -> str: + prefix = str(field.get("prefix") or "") + if prefix and not value.lower().startswith(prefix.lower()): + return f"{prefix}{value}" + return value + + +def installation_secrets( + version: McpMarketVersion, + credentials: Dict[str, str], + existing_headers: Dict[str, str] | None = None, + *, + auth_method: str, + oauth_bundle: Optional[Dict[str, Any]] = None, +) -> Dict[str, str]: + stored: Dict[str, str] = {} + missing: List[str] = [] + for field in version.auth_schema or []: + method_ids = [str(value) for value in (field.get("methods") or []) if str(value)] + if method_ids and auth_method not in method_ids: + continue + key = str(field.get("key") or "").strip() + storage_key = credential_storage_key(field) + if not key or not storage_key: + continue + value = str(credentials.get(key) or "").strip() + if value: + stored[storage_key] = materialize_credential_value(field, value) + elif storage_key in (existing_headers or {}): + stored[storage_key] = str((existing_headers or {})[storage_key]) + elif field.get("required", True): + missing.append(str(field.get("label") or key)) + if missing: + raise BadRequestError(message=f"请填写安装凭据：{', '.join(missing)}") + oauth_key = mcp_oauth_bundle_storage_key() + if oauth_bundle: + stored[oauth_key] = json.dumps(oauth_bundle, ensure_ascii=False, separators=(",", ":")) + elif auth_method == "oauth2" and oauth_key in (existing_headers or {}): + stored[oauth_key] = str((existing_headers or {})[oauth_key]) + return stored diff --git a/src/backend/features/marketplace/mcp_curated.py b/src/backend/features/marketplace/mcp_curated.py new file mode 100644 index 00000000..698ecb80 --- /dev/null +++ b/src/backend/features/marketplace/mcp_curated.py @@ -0,0 +1,127 @@ +"""mcp curated responsibilities.""" +from __future__ import annotations +from core.infra.time import utc_now +from typing import List +from features.marketplace.mcp_catalog import CURATED_MCP_MARKET_ITEMS +from core.db.models import AdminMcpServer +from features.marketplace.models import McpMarketInstallation, McpMarketItem, McpMarketVersion +from features.marketplace import edition as ml +from core.capabilities.mcp.management import tool_snapshot_hash +from sqlalchemy.orm import Session +from features.marketplace import mcp_validation + +def ensure_curated_market_items(db: Session) -> List[str]: + """Seed credential-free official templates once, without resurrecting deletions.""" + seeded: List[str] = [] + updated = False + now = utc_now() + for definition in CURATED_MCP_MARKET_ITEMS: + slug = str(definition["slug"]) + # Soft-deleted rows deliberately count as existing: an administrator's + # removal choice must survive application restarts. + existing_item = db.query(McpMarketItem).filter(McpMarketItem.slug == slug).first() + if existing_item: + if existing_item.deleted_at is None: + existing_version = ( + db.query(McpMarketVersion) + .filter(McpMarketVersion.version_id == existing_item.latest_version_id) + .first() + ) + if existing_version and existing_version.approved_by == "system-curated": + schema = list(definition.get("auth_schema") or []) + auth_config = mcp_validation.normalize_auth_config(definition.get("auth_config"), schema) + notice = dict(existing_version.listing_notice or {}) + notice.update( + { + "discovery_mode": "per_install", + "install_notice": str(definition.get("install_notice") or ""), + "docs_url": str(definition.get("docs_url") or ""), + } + ) + description = str(definition.get("description") or "") + user_intro = str(definition.get("user_intro") or "") + icon = str(definition.get("icon") or "") + changed = ( + list(existing_version.auth_schema or []) != schema + or dict(existing_version.auth_config or {}) != auth_config + or dict(existing_version.listing_notice or {}) != notice + or (existing_item.description or "") != description + or (existing_item.user_intro or "") != user_intro + or (existing_item.icon or "") != icon + ) + if changed: + existing_version.auth_schema = schema + existing_version.auth_config = auth_config + existing_version.listing_notice = notice + existing_item.description = description + existing_item.user_intro = user_intro + existing_item.icon = icon + existing_item.updated_at = now + server_ids = [ + row.server_id + for row in db.query(McpMarketInstallation) + .filter(McpMarketInstallation.slug == slug) + .all() + ] + if server_ids: + ( + db.query(AdminMcpServer) + .filter(AdminMcpServer.server_id.in_(server_ids)) + .update({AdminMcpServer.icon: icon}, synchronize_session=False) + ) + updated = True + continue + tools = list(definition.get("tools") or []) + listing_notice = { + "discovery_mode": "per_install", + "install_notice": str(definition.get("install_notice") or ""), + "docs_url": str(definition.get("docs_url") or ""), + } + version_id = f"mcpver_curated_{slug}" + item = McpMarketItem( + slug=slug, + display_name=str(definition["display_name"]), + description=str(definition.get("description") or ""), + user_intro=str(definition.get("user_intro") or ""), + category=mcp_validation.validate_category(str(definition.get("category") or "通用工具")), + tags=list(definition.get("tags") or []), + icon=str(definition.get("icon") or ""), + publisher_id=None, + publisher_name=str(definition.get("publisher_name") or "HugAgentOS"), + source="admin", + latest_version_id=version_id, + status="active", + status_reason=None, + last_verified_at=None, + created_at=now, + updated_at=now, + ) + db.add(item) + db.add( + McpMarketVersion( + version_id=version_id, + slug=slug, + version=mcp_validation.normalize_version(str(definition.get("version") or "1.0.0")), + transport=str(definition.get("transport") or "streamable_http"), + url=str(definition["url"]), + auth_schema=list(definition.get("auth_schema") or []), + auth_config=mcp_validation.normalize_auth_config( + definition.get("auth_config"), + list(definition.get("auth_schema") or []), + ), + tools_json=tools, + tool_hash=tool_snapshot_hash(tools), + listing_notice=listing_notice, + source_server_id=None, + approved_by="system-curated", + approved_at=now, + created_at=now, + ) + ) + seeded.append(slug) + if not seeded and not updated: + return [] + db.commit() + for slug in seeded: + ml.set_listing_enabled(db, ml.KIND_MCP, slug, True, updated_by="system-curated") + return seeded diff --git a/src/backend/features/marketplace/mcp_installation.py b/src/backend/features/marketplace/mcp_installation.py new file mode 100644 index 00000000..3b662a0e --- /dev/null +++ b/src/backend/features/marketplace/mcp_installation.py @@ -0,0 +1,212 @@ +"""mcp installation responsibilities.""" +from __future__ import annotations +from core.infra.time import utc_now +import uuid +from typing import Any, Dict, Optional +from core.db.models import AdminMcpServer +from features.marketplace.models import McpMarketInstallation, McpMarketItem +from core.infra.exceptions import BadRequestError, ResourceNotFoundError +from features.marketplace import edition as ml +from core.capabilities.mcp.management import decrypt_mcp_headers, encrypt_mcp_headers, materialize_mcp_http_connection, probe_mcp_connectivity, refresh_mcp_caches, tool_snapshot_hash, validate_remote_mcp_url +from sqlalchemy.orm import Session +from features.marketplace import mcp_credentials +from features.marketplace import mcp_validation +from features.marketplace import mcp_versions + +async def install_market_item( + db: Session, + slug: str, + *, + owner_user_id: Optional[str], + installed_by: Optional[str], + credentials: Optional[Dict[str, str]] = None, + auth_method: Optional[str] = None, + oauth_bundle: Optional[Dict[str, Any]] = None, +) -> Dict[str, Any]: + item = ( + db.query(McpMarketItem) + .filter( + McpMarketItem.slug == slug, + McpMarketItem.deleted_at.is_(None), + McpMarketItem.status == "active", + ) + .first() + ) + if not item: + raise ResourceNotFoundError("mcp_market_item", slug) + if owner_user_id is not None: + if slug in ml.get_disabled_ids(db, ml.KIND_MCP): + raise ResourceNotFoundError("mcp_market_item", slug) + ml.ensure_item_visible(db, ml.KIND_MCP, slug, owner_user_id, resource="mcp_market_item") + version = mcp_versions.version_for_item(db, item) + auth_config = mcp_validation.normalize_auth_config(version.auth_config, list(version.auth_schema or [])) + selected_method = str(auth_method or auth_config["default_method"]) + selected = next( + (method for method in auth_config["methods"] if method["id"] == selected_method), + None, + ) + if selected is None: + raise BadRequestError(message="不支持所选 MCP 认证方式") + if selected["type"] == "oauth2" and not oauth_bundle: + raise BadRequestError(message="请先完成 OAuth 登录") + if selected["type"] != "oauth2" and oauth_bundle: + raise BadRequestError(message="当前认证方式不能写入 OAuth 凭据") + existing = db.query(McpMarketInstallation).filter(McpMarketInstallation.slug == slug) + existing = ( + existing.filter(McpMarketInstallation.owner_user_id.is_(None)) + if owner_user_id is None + else existing.filter(McpMarketInstallation.owner_user_id == owner_user_id) + ).first() + existing_server = ( + db.query(AdminMcpServer).filter(AdminMcpServer.server_id == existing.server_id).first() + if existing + else None + ) + if existing_server is None and owner_user_id is None and version.source_server_id: + source_candidate = ( + db.query(AdminMcpServer) + .filter( + AdminMcpServer.server_id == version.source_server_id, + AdminMcpServer.owner_user_id.is_(None), + ) + .first() + ) + if source_candidate and (source_candidate.extra_config or {}).get("market_source_only"): + existing_server = source_candidate + old_headers = decrypt_mcp_headers(existing_server.headers) if existing_server else {} + managed_headers = mcp_credentials.admin_managed_credential_headers( + db, + item, + version, + auth_method=selected_method, + ) + fallback_headers = {**managed_headers, **old_headers} + headers = mcp_credentials.installation_secrets( + version, + dict(credentials or {}), + fallback_headers, + auth_method=selected_method, + oauth_bundle=oauth_bundle, + ) + runtime_url, _ = materialize_mcp_http_connection(version.url, headers) + await validate_remote_mcp_url( + runtime_url, + allow_private_network=mcp_validation.installs_admin_published_endpoint(item, version, runtime_url), + require_https=False, + ) + + now = utc_now() + server_id = existing_server.server_id if existing_server else f"mmcp_{uuid.uuid4().hex[:20]}" + candidate = AdminMcpServer( + server_id=server_id, + display_name=item.display_name, + description=item.description or "", + user_intro=item.user_intro, + transport=version.transport, + command=None, + args=[], + url=version.url, + env_vars={}, + env_inherit=[], + headers=headers, + is_enabled=True, + sort_order=0, + extra_config={ + **( + { + key: value + for key, value in dict(existing_server.extra_config or {}).items() + if key not in {"market_source_only", "market_source_slug"} + } + if existing_server + else {} + ), + "market_slug": slug, + "market_version": version.version, + "listing_notice": dict(version.listing_notice or {}), + "auth_method": selected_method, + }, + tools_json=list(version.tools_json or []), + icon=item.icon, + owner_user_id=owner_user_id, + source_plugin=None, + created_by=installed_by, + created_at=existing_server.created_at if existing_server else now, + updated_at=now, + ) + + ok, error = await probe_mcp_connectivity(candidate, db) + if not ok: + raise BadRequestError(message=f"MCP 安装检测失败：{error}") + discovery_mode = str((version.listing_notice or {}).get("discovery_mode") or "reviewed") + if discovery_mode == "per_install": + candidate.extra_config = { + **dict(candidate.extra_config or {}), + "discovery_mode": discovery_mode, + } + else: + actual_hash = tool_snapshot_hash(candidate.tools_json or []) + if actual_hash != version.tool_hash: + item.status = "changed" + item.status_reason = "远程工具或参数结构已变化，等待管理员重新审核" + item.updated_at = now + db.commit() + raise BadRequestError(message="远程 MCP 工具清单已发生变化，已暂停安装并等待重新审核") + + server = existing_server or AdminMcpServer(server_id=server_id) + for field in ( + "display_name", + "description", + "user_intro", + "transport", + "command", + "args", + "url", + "env_vars", + "env_inherit", + "is_enabled", + "sort_order", + "extra_config", + "tools_json", + "icon", + "owner_user_id", + "source_plugin", + "created_by", + "updated_at", + ): + setattr(server, field, getattr(candidate, field)) + server.headers = encrypt_mcp_headers(headers) + if not existing_server: + server.created_at = now + db.add(server) + db.flush() + if existing: + previous_version_id = existing.version_id + existing.version_id = version.version_id + existing.status = "active" + existing.installed_by = installed_by + existing.updated_at = now + action = "updated" if previous_version_id != version.version_id else "existing" + else: + existing = McpMarketInstallation( + install_id=f"mcpins_{uuid.uuid4().hex}", + slug=slug, + version_id=version.version_id, + server_id=server.server_id, + owner_user_id=owner_user_id, + status="active", + installed_by=installed_by, + created_at=now, + updated_at=now, + ) + db.add(existing) + action = "installed" + db.commit() + refresh_mcp_caches() + return { + "install_id": existing.install_id, + "server_id": server.server_id, + "slug": slug, + "version": version.version, + "action": action, + } diff --git a/src/backend/features/marketplace/mcp_listing.py b/src/backend/features/marketplace/mcp_listing.py new file mode 100644 index 00000000..d2c1aebf --- /dev/null +++ b/src/backend/features/marketplace/mcp_listing.py @@ -0,0 +1,142 @@ +"""mcp catalog responsibilities.""" +from __future__ import annotations +from typing import Any, Dict, Optional +from features.marketplace.models import McpMarketInstallation, McpMarketItem, McpMarketVersion +from core.infra.exceptions import ResourceNotFoundError +from features.marketplace import edition as ml +from sqlalchemy.orm import Session +from features.marketplace import mcp_credentials +from features.marketplace import mcp_state +from features.marketplace import mcp_validation +from features.marketplace import mcp_versions + +def installed_slug_set(db: Session, owner_user_id: Optional[str]) -> set[str]: + query = db.query(McpMarketInstallation.slug).filter( + McpMarketInstallation.status.in_(["active", "suspended"]) + ) + query = ( + query.filter(McpMarketInstallation.owner_user_id.is_(None)) + if owner_user_id is None + else query.filter(McpMarketInstallation.owner_user_id == owner_user_id) + ) + return {str(row[0]) for row in query.all()} + + +def item_dict( + db: Session, + item: McpMarketItem, + version: McpMarketVersion, + *, + installed: bool = False, +) -> Dict[str, Any]: + auth_schema = list(version.auth_schema or []) + auth_config = mcp_validation.normalize_auth_config(version.auth_config, auth_schema) + requires_auth = any(method["type"] != "none" for method in auth_config["methods"]) + credentials_managed_by_admin = bool( + mcp_credentials.admin_managed_credential_headers( + db, + item, + version, + auth_method=str(auth_config["default_method"]), + ) + ) + if auth_config["credential_mode"] == "auto": + auth_config["credential_mode"] = "admin" if credentials_managed_by_admin else "installer" + supports_admin_credentials = bool(mcp_credentials.admin_credential_source(db, item, version)) + return { + "slug": item.slug, + "display_name": item.display_name, + "description": item.description or "", + "summary": item.description or "", + "user_intro": item.user_intro or "", + "category": item.category, + "tags": list(item.tags or []), + "icon": item.icon or "", + "publisher_name": item.publisher_name or "", + "source": item.source, + "version": version.version, + "version_id": version.version_id, + "transport": version.transport, + "url_origin": mcp_validation.safe_origin(version.url), + "auth_schema": auth_schema, + "auth_config": auth_config, + "requires_auth": requires_auth, + "credentials_managed_by_admin": credentials_managed_by_admin, + "supports_admin_credentials": supports_admin_credentials, + "requires_user_credentials": requires_auth and not credentials_managed_by_admin, + "tools": list(version.tools_json or []), + "tool_count": len(version.tools_json or []), + "tool_hash": version.tool_hash, + "listing_notice": dict(version.listing_notice or {}), + "status": item.status, + "status_reason": item.status_reason or "", + "last_verified_at": item.last_verified_at.isoformat() if item.last_verified_at else None, + "installed": installed, + "deletable": True, + "created_at": item.created_at.isoformat() if item.created_at else None, + "updated_at": item.updated_at.isoformat() if item.updated_at else None, + } + + +def list_market_items( + db: Session, + *, + owner_user_id: Optional[str], + viewer_user_id: Optional[str], + include_disabled: bool = False, +) -> Dict[str, Any]: + query = db.query(McpMarketItem).filter(McpMarketItem.deleted_at.is_(None)) + if not include_disabled: + query = query.filter(McpMarketItem.status == "active") + rows = query.order_by(McpMarketItem.updated_at.desc(), McpMarketItem.slug).all() + installed = installed_slug_set(db, owner_user_id) + items = [ + item_dict(db, row, mcp_versions.version_for_item(db, row), installed=row.slug in installed) + for row in rows + ] + items = ml.annotate_and_filter( + db, + ml.KIND_MCP, + items, + id_key="slug", + include_disabled=include_disabled, + viewer_user_id=viewer_user_id, + ) + categories = [ + category + for category in mcp_state.MCP_MARKET_CATEGORIES + if any(i["category"] == category for i in items) + ] + return {"items": items, "categories": categories} + + +def get_market_item( + db: Session, + slug: str, + *, + owner_user_id: Optional[str], + viewer_user_id: Optional[str], + admin: bool = False, +) -> Dict[str, Any]: + item = db.query(McpMarketItem).filter(McpMarketItem.slug == slug).first() + if not item or (not admin and item.status != "active"): + raise ResourceNotFoundError("mcp_market_item", slug) + if not admin: + ml.ensure_item_visible(db, ml.KIND_MCP, slug, viewer_user_id, resource="mcp_market_item") + data = item_dict( + db, + item, + mcp_versions.version_for_item(db, item), + installed=slug in installed_slug_set(db, owner_user_id), + ) + state = ml.annotate_and_filter( + db, + ml.KIND_MCP, + [data], + id_key="slug", + include_disabled=admin, + viewer_user_id=viewer_user_id, + ) + if not state: + raise ResourceNotFoundError("mcp_market_item", slug) + return state[0] diff --git a/src/backend/core/services/mcp_marketplace_monitor.py b/src/backend/features/marketplace/mcp_monitor.py similarity index 94% rename from src/backend/core/services/mcp_marketplace_monitor.py rename to src/backend/features/marketplace/mcp_monitor.py index 13daa508..c5b84938 100644 --- a/src/backend/core/services/mcp_marketplace_monitor.py +++ b/src/backend/features/marketplace/mcp_monitor.py @@ -8,8 +8,8 @@ from core.config.settings import settings from core.db.engine import SessionLocal -from core.db.models import McpMarketItem -from core.services.mcp_marketplace_service import revalidate_market_item +from features.marketplace.models import McpMarketItem +from features.marketplace.mcp_revalidation import revalidate_market_item logger = logging.getLogger(__name__) _task: asyncio.Task | None = None diff --git a/src/backend/features/marketplace/mcp_publishing.py b/src/backend/features/marketplace/mcp_publishing.py new file mode 100644 index 00000000..22343c37 --- /dev/null +++ b/src/backend/features/marketplace/mcp_publishing.py @@ -0,0 +1,151 @@ +"""mcp publishing responsibilities.""" +from __future__ import annotations +from core.infra.time import utc_now +import uuid +from typing import Any, Dict, List, Optional +from core.db.models import AdminMcpServer +from features.marketplace.models import McpMarketInstallation, McpMarketItem, McpMarketVersion +from core.infra.exceptions import BadRequestError, ResourceNotFoundError +from features.marketplace import edition as ml +from core.capabilities.mcp.management import decrypt_mcp_headers, encrypt_mcp_headers, probe_mcp_connectivity, refresh_mcp_caches, validate_remote_mcp_url +from sqlalchemy.orm import Session +from features.marketplace import mcp_listing +from features.marketplace import mcp_validation +from features.marketplace import mcp_versions + +async def publish_admin_server( + db: Session, + source_server_id: str, + *, + category: str, + version: str, + summary: str = "", + tags: Optional[List[str]] = None, + publisher_name: str = "管理员", + link_as_installation: bool = True, + auth_schema_override: Optional[List[Dict[str, Any]]] = None, + auth_config_override: Optional[Dict[str, Any]] = None, + allow_deferred_discovery: bool = False, +) -> Dict[str, Any]: + row = ( + db.query(AdminMcpServer) + .filter( + AdminMcpServer.server_id == source_server_id, + AdminMcpServer.owner_user_id.is_(None), + ) + .first() + ) + if not row: + raise ResourceNotFoundError("admin_mcp_server", source_server_id) + if row.transport not in ("streamable_http", "sse") or not row.url: + raise BadRequestError(message="MCP 市场仅支持远程 HTTP/SSE 服务；stdio 请通过插件市场分发") + if (row.extra_config or {}).get("market_slug"): + raise BadRequestError(message="从 MCP 市场安装的实例不能重复发布") + await validate_remote_mcp_url( + row.url, + allow_private_network=True, + require_https=False, + ) + ok, error = await probe_mcp_connectivity(row, db) + if not ok and not allow_deferred_discovery: + raise BadRequestError(message=f"MCP 发布检测失败：{error}") + tools = list(row.tools_json or []) + if not tools and not allow_deferred_discovery: + raise BadRequestError(message="MCP 未发现任何工具，不能发布") + existing = ( + db.query(McpMarketItem) + .join(McpMarketVersion, McpMarketVersion.version_id == McpMarketItem.latest_version_id) + .filter(McpMarketVersion.source_server_id == source_server_id) + .first() + ) + slug = existing.slug if existing else mcp_validation.new_slug(db, row.display_name, source_server_id) + template_url, inferred_auth_schema = mcp_validation.credential_free_connection(row) + stored_market_config = dict(row.extra_config or {}) + auth_schema = list( + auth_schema_override + if auth_schema_override is not None + else stored_market_config.get("market_auth_schema") or inferred_auth_schema + ) + auth_config = mcp_validation.normalize_auth_config( + ( + auth_config_override + if auth_config_override is not None + else stored_market_config.get("market_auth_config") + ), + auth_schema, + ) + item = mcp_versions.publish_snapshot( + db, + slug=slug, + display_name=row.display_name, + description=(summary or row.description or "").strip(), + user_intro=row.user_intro, + category=category, + tags=list(dict.fromkeys(str(tag).strip() for tag in (tags or []) if str(tag).strip())), + icon=row.icon, + publisher_id=None, + publisher_name=publisher_name, + source="admin", + version=version, + transport=row.transport, + url=template_url, + auth_schema=auth_schema, + auth_config=auth_config, + tools=tools, + source_server_id=row.server_id, + approved_by=publisher_name, + ) + db.flush() + if allow_deferred_discovery: + market_version = mcp_versions.version_for_item(db, item) + market_version.listing_notice = { + **dict(market_version.listing_notice or {}), + "discovery_mode": "per_install", + "install_notice": "该 MCP 的完整工具清单将在用户完成认证后动态发现。", + } + # Existing global MCPs are linked as an installation. A server created + # through the marketplace-first flow remains a disabled source template + # until an administrator explicitly installs it from the market. + if link_as_installation: + installation = ( + db.query(McpMarketInstallation) + .filter(McpMarketInstallation.server_id == row.server_id) + .first() + ) + if installation is None: + db.add( + McpMarketInstallation( + install_id=f"mcpins_{uuid.uuid4().hex}", + slug=item.slug, + version_id=item.latest_version_id, + server_id=row.server_id, + owner_user_id=None, + status="active", + installed_by="admin", + created_at=utc_now(), + updated_at=utc_now(), + ) + ) + else: + installation.slug = item.slug + installation.version_id = item.latest_version_id + installation.status = "active" + installation.updated_at = utc_now() + else: + for installation in ( + db.query(McpMarketInstallation) + .filter(McpMarketInstallation.server_id == row.server_id) + .all() + ): + db.delete(installation) + row.is_enabled = False + row.extra_config = { + **dict(row.extra_config or {}), + "market_source_only": True, + "market_source_slug": item.slug, + } + row.headers = encrypt_mcp_headers(decrypt_mcp_headers(row.headers)) + db.commit() + ml.set_listing_enabled(db, ml.KIND_MCP, item.slug, True, updated_by=publisher_name) + refresh_mcp_caches() + return mcp_listing.item_dict(db, item, mcp_versions.version_for_item(db, item), installed=link_as_installation) diff --git a/src/backend/features/marketplace/mcp_revalidation.py b/src/backend/features/marketplace/mcp_revalidation.py new file mode 100644 index 00000000..2baa21c2 --- /dev/null +++ b/src/backend/features/marketplace/mcp_revalidation.py @@ -0,0 +1,118 @@ +"""mcp revalidation responsibilities.""" +from __future__ import annotations +from core.infra.time import utc_now +from typing import Any, Dict +from core.db.models import AdminMcpServer +from features.marketplace.models import McpMarketItem +from core.infra.exceptions import BadRequestError, ResourceNotFoundError +from core.capabilities.mcp.management import probe_mcp_connectivity, tool_snapshot_hash, validate_remote_mcp_url +from sqlalchemy.orm import Session +from features.marketplace import mcp_administration +from features.marketplace import mcp_listing +from features.marketplace import mcp_validation +from features.marketplace import mcp_versions + +async def revalidate_market_item( + db: Session, + slug: str, + *, + accept_changes: bool = False, +) -> Dict[str, Any]: + item = ( + db.query(McpMarketItem) + .filter(McpMarketItem.slug == slug, McpMarketItem.deleted_at.is_(None)) + .first() + ) + if not item: + raise ResourceNotFoundError("mcp_market_item", slug) + version = mcp_versions.version_for_item(db, item) + if str((version.listing_notice or {}).get("discovery_mode") or "") == "per_install": + # Curated templates have no shared provider credential, so their live + # tools are intentionally discovered with each installer's credential. + # Revalidation checks only a fixed public endpoint; private endpoint + # templates are validated when the user installs them. + has_endpoint_override = any( + str(field.get("target") or "") == "url" for field in version.auth_schema or [] + ) + if not has_endpoint_override: + try: + await validate_remote_mcp_url(version.url, require_https=False) + item.status = "active" + item.status_reason = None + except BadRequestError as exc: + item.status = "suspended" + item.status_reason = f"官方端点安全复检失败：{exc}" + else: + item.status = "active" + item.status_reason = "个人端点将在安装时使用用户凭据验证" + item.last_verified_at = utc_now() + item.updated_at = utc_now() + db.commit() + return mcp_listing.item_dict(db, item, version, installed=False) + source = ( + db.query(AdminMcpServer) + .filter(AdminMcpServer.server_id == version.source_server_id) + .first() + if version.source_server_id + else None + ) + if source: + source_template_url, source_auth_schema = mcp_validation.credential_free_connection(source) + if ( + source.transport != version.transport + or source_template_url.strip() != version.url.strip() + ): + item.status = "changed" + item.status_reason = "原始 MCP 的连接地址或传输方式已变化，需发布新版本" + item.last_verified_at = utc_now() + item.updated_at = utc_now() + db.commit() + return mcp_listing.item_dict(db, item, version, installed=False) + auth_policy = mcp_validation.normalize_auth_config(version.auth_config, list(version.auth_schema or [])) + if auth_policy["credential_mode"] == "auto" and source_auth_schema != list( + version.auth_schema or [] + ): + item.status = "changed" + item.status_reason = "原始 MCP 的认证参数已变化，需发布新版本" + item.last_verified_at = utc_now() + item.updated_at = utc_now() + db.commit() + return mcp_listing.item_dict(db, item, version, installed=False) + try: + await validate_remote_mcp_url( + version.url, + allow_private_network=item.source == "admin", + require_https=False, + ) + except BadRequestError as exc: + mcp_administration.set_suspended(db, slug, suspended=True, reason=f"远程地址安全复检失败：{exc}") + return mcp_listing.item_dict(db, item, version, installed=False) + probe_target = source or mcp_versions.version_probe_candidate(item, version) + ok, error = await probe_mcp_connectivity(probe_target, db if source else None) + discovered_tools = list(probe_target.tools_json or []) + if not ok: + item.status = "changed" + prefix = "复检连接失败" if source else "原始 MCP 连接已不存在，版本端点复检失败" + item.status_reason = f"{prefix}：{error}" + elif not discovered_tools: + item.status = "changed" + item.status_reason = "复检连接成功，但远程 MCP 未返回任何工具，暂不接受新快照" + else: + actual_hash = tool_snapshot_hash(discovered_tools) + if actual_hash == version.tool_hash: + item.status = "active" + item.status_reason = None + elif accept_changes and item.source == "admin": + previous_version = version.version + accepted = mcp_versions.accept_revalidated_snapshot(db, item, version, discovered_tools) + result = mcp_listing.item_dict(db, item, accepted, installed=False) + result["snapshot_accepted"] = True + result["previous_version"] = previous_version + return result + else: + item.status = "changed" + item.status_reason = "远程工具或参数结构已变化，需发布新版本并重新审核" + item.last_verified_at = utc_now() + item.updated_at = utc_now() + db.commit() + return mcp_listing.item_dict(db, item, version, installed=False) diff --git a/src/backend/features/marketplace/mcp_state.py b/src/backend/features/marketplace/mcp_state.py new file mode 100644 index 00000000..cc0cbdc1 --- /dev/null +++ b/src/backend/features/marketplace/mcp_state.py @@ -0,0 +1,17 @@ +"""mcp state responsibilities.""" +from __future__ import annotations + + +MCP_MARKET_CATEGORIES = [ + "信息检索", + "数据分析", + "内容创作", + "办公协作", + "研发工具", + "业务系统", + "自动化", + "通用工具", +] + + +SECRET_QUERY_TERMS = ("key", "token", "secret", "password", "credential") diff --git a/src/backend/features/marketplace/mcp_submissions.py b/src/backend/features/marketplace/mcp_submissions.py new file mode 100644 index 00000000..ecfd9757 --- /dev/null +++ b/src/backend/features/marketplace/mcp_submissions.py @@ -0,0 +1,272 @@ +"""mcp submissions responsibilities.""" +from __future__ import annotations +from core.infra.time import utc_now +import uuid +from typing import Any, Dict, List, Optional +from core.db.models import AdminMcpServer +from features.marketplace.models import McpMarketInstallation, McpMarketItem, McpMarketSubmission +from core.infra.exceptions import BadRequestError, ResourceNotFoundError +from features.marketplace import edition as ml +from core.capabilities.mcp.management import probe_mcp_connectivity, tool_snapshot_hash, validate_remote_mcp_url +from sqlalchemy.orm import Session +from features.marketplace import mcp_validation +from features.marketplace import mcp_versions + +async def submit_to_marketplace( + db: Session, + source_server_id: str, + *, + owner_user_id: str, + submitter_name: str, + category: str, + version: str, + summary: str = "", + note: str = "", + tags: Optional[List[str]] = None, +) -> Dict[str, Any]: + row = ( + db.query(AdminMcpServer) + .filter( + AdminMcpServer.server_id == source_server_id, + AdminMcpServer.owner_user_id == owner_user_id, + ) + .first() + ) + if not row: + raise ResourceNotFoundError("private_mcp_server", source_server_id) + if row.transport not in ("streamable_http", "sse") or not row.url: + raise BadRequestError(message="只有远程 HTTP/SSE MCP 可以申请上架") + installed_from_market = ( + db.query(McpMarketInstallation.install_id) + .filter(McpMarketInstallation.server_id == row.server_id) + .first() + ) + if (row.extra_config or {}).get("market_slug") or installed_from_market: + raise BadRequestError(message="从 MCP 市场安装的实例不能再次申请上架") + pending = ( + db.query(McpMarketSubmission) + .filter( + McpMarketSubmission.source_server_id == source_server_id, + McpMarketSubmission.owner_user_id == owner_user_id, + McpMarketSubmission.status == "pending", + McpMarketSubmission.deleted_at.is_(None), + ) + .first() + ) + if pending: + raise BadRequestError(message="该 MCP 已有待审核的上架申请") + + await validate_remote_mcp_url(row.url, require_https=False) + ok, error = await probe_mcp_connectivity(row, db) + if not ok: + raise BadRequestError(message=f"MCP 上架检测失败：{error}") + tools = list(row.tools_json or []) + if not tools: + raise BadRequestError(message="MCP 未发现任何工具，不能申请上架") + template_url, auth_schema = mcp_validation.credential_free_connection(row) + now = utc_now() + submission = McpMarketSubmission( + submission_id=f"mcpsub_{uuid.uuid4().hex}", + slug=mcp_validation.source_slug(db, row, owner_user_id), + source_server_id=row.server_id, + owner_user_id=owner_user_id, + submitter_name=submitter_name or "", + display_name=row.display_name, + description=(summary or row.description or "").strip(), + user_intro=row.user_intro, + category=mcp_validation.validate_category(category), + tags=list(dict.fromkeys(str(tag).strip() for tag in (tags or []) if str(tag).strip())), + icon=row.icon, + version=mcp_validation.normalize_version(version), + transport=row.transport, + url=template_url, + auth_schema=auth_schema, + auth_config=mcp_validation.normalize_auth_config(None, auth_schema), + tools_json=tools, + tool_hash=tool_snapshot_hash(tools), + note=(note or "").strip(), + status="pending", + created_at=now, + updated_at=now, + ) + db.add(submission) + db.commit() + db.refresh(submission) + return submission_dict(submission) + + +def submission_dict(row: McpMarketSubmission, *, detail: bool = False) -> Dict[str, Any]: + data: Dict[str, Any] = { + "submission_id": row.submission_id, + "slug": row.slug, + "source_server_id": row.source_server_id, + "owner_user_id": row.owner_user_id, + "submitter_name": row.submitter_name or "", + "display_name": row.display_name, + "description": row.description or "", + "category": row.category, + "tags": list(row.tags or []), + "icon": row.icon or "", + "version": row.version, + "transport": row.transport, + "url_origin": mcp_validation.safe_origin(row.url), + "auth_schema": list(row.auth_schema or []), + "auth_config": mcp_validation.normalize_auth_config(row.auth_config, list(row.auth_schema or [])), + "tool_count": len(row.tools_json or []), + "tool_hash": row.tool_hash, + "listing_notice": dict(row.listing_notice or {}), + "note": row.note or "", + "status": row.status, + "review_note": row.review_note or "", + "reviewed_by": row.reviewed_by, + "reviewed_at": row.reviewed_at.isoformat() if row.reviewed_at else None, + "created_at": row.created_at.isoformat() if row.created_at else None, + } + if detail: + data["user_intro"] = row.user_intro or "" + data["url"] = row.url + data["tools"] = list(row.tools_json or []) + return data + + +def list_my_submissions(db: Session, owner_user_id: str) -> List[Dict[str, Any]]: + rows = ( + db.query(McpMarketSubmission) + .filter( + McpMarketSubmission.owner_user_id == owner_user_id, + McpMarketSubmission.deleted_at.is_(None), + ) + .order_by(McpMarketSubmission.created_at.desc()) + .all() + ) + return [submission_dict(row) for row in rows] + + +def withdraw_submission(db: Session, submission_id: str, owner_user_id: str) -> None: + row = ( + db.query(McpMarketSubmission) + .filter( + McpMarketSubmission.submission_id == submission_id, + McpMarketSubmission.owner_user_id == owner_user_id, + McpMarketSubmission.deleted_at.is_(None), + ) + .first() + ) + if not row: + raise ResourceNotFoundError("mcp_market_submission", submission_id) + if row.status != "pending": + raise BadRequestError(message="只有待审核申请可以撤回") + row.status = "withdrawn" + row.deleted_at = utc_now() + row.updated_at = utc_now() + db.commit() + + +def list_submissions(db: Session, status: Optional[str] = None) -> List[Dict[str, Any]]: + query = db.query(McpMarketSubmission).filter(McpMarketSubmission.deleted_at.is_(None)) + if status: + query = query.filter(McpMarketSubmission.status == status) + return [ + submission_dict(row) for row in query.order_by(McpMarketSubmission.created_at.desc()).all() + ] + + +def get_submission(db: Session, submission_id: str) -> Dict[str, Any]: + row = ( + db.query(McpMarketSubmission) + .filter( + McpMarketSubmission.submission_id == submission_id, + McpMarketSubmission.deleted_at.is_(None), + ) + .first() + ) + if not row: + raise ResourceNotFoundError("mcp_market_submission", submission_id) + return submission_dict(row, detail=True) + + +async def review_submission( + db: Session, + submission_id: str, + *, + approve: bool, + review_note: str = "", + category: str = "", + reviewed_by: Optional[str] = None, +) -> Dict[str, Any]: + row = ( + db.query(McpMarketSubmission) + .filter( + McpMarketSubmission.submission_id == submission_id, + McpMarketSubmission.deleted_at.is_(None), + ) + .first() + ) + if not row: + raise ResourceNotFoundError("mcp_market_submission", submission_id) + if (approve and row.status != "pending") or ( + not approve and row.status not in ("pending", "approved") + ): + raise BadRequestError(message="该申请当前状态不可审核") + now = utc_now() + if approve: + source = ( + db.query(AdminMcpServer) + .filter( + AdminMcpServer.server_id == row.source_server_id, + AdminMcpServer.owner_user_id == row.owner_user_id, + ) + .first() + ) + if not source: + raise BadRequestError(message="申请人的原始 MCP 已删除，不能通过审核") + source_template_url, source_auth_schema = mcp_validation.credential_free_connection(source) + if source.transport != row.transport or source_template_url.strip() != row.url.strip(): + raise BadRequestError(message="MCP 连接地址或传输方式已变化，请申请人重新提交") + if source_auth_schema != list(row.auth_schema or []): + raise BadRequestError(message="MCP 认证参数已变化，请申请人重新提交") + await validate_remote_mcp_url(source.url or "", require_https=False) + ok, error = await probe_mcp_connectivity(source, db) + if not ok: + db.rollback() + raise BadRequestError(message=f"MCP 审核复检失败：{error}") + if tool_snapshot_hash(source.tools_json or []) != row.tool_hash: + db.rollback() + raise BadRequestError(message="MCP 工具或参数结构已变化，请申请人重新提交") + mcp_versions.publish_snapshot( + db, + slug=row.slug, + display_name=row.display_name, + description=row.description, + user_intro=row.user_intro, + category=category or row.category, + tags=list(row.tags or []), + icon=row.icon, + publisher_id=row.owner_user_id, + publisher_name=row.submitter_name, + source="community", + version=row.version, + transport=row.transport, + url=row.url, + auth_schema=list(row.auth_schema or []), + auth_config=mcp_validation.normalize_auth_config(row.auth_config, list(row.auth_schema or [])), + tools=list(row.tools_json or []), + source_server_id=row.source_server_id, + approved_by=reviewed_by, + ) + row.status = "approved" + else: + row.status = "rejected" + item = db.query(McpMarketItem).filter(McpMarketItem.slug == row.slug).first() + if item: + item.deleted_at = now + item.updated_at = now + ml.set_listing_enabled(db, ml.KIND_MCP, row.slug, False, updated_by=reviewed_by) + row.review_note = (review_note or "").strip() + row.reviewed_by = reviewed_by + row.reviewed_at = now + row.updated_at = now + db.commit() + if approve: + ml.set_listing_enabled(db, ml.KIND_MCP, row.slug, True, updated_by=reviewed_by) + return submission_dict(row, detail=True) diff --git a/src/backend/features/marketplace/mcp_validation.py b/src/backend/features/marketplace/mcp_validation.py new file mode 100644 index 00000000..6b16dcb4 --- /dev/null +++ b/src/backend/features/marketplace/mcp_validation.py @@ -0,0 +1,166 @@ +"""mcp validation responsibilities.""" +from __future__ import annotations +import re +from typing import Any, Dict, List +from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse +from core.db.models import AdminMcpServer +from features.marketplace.models import McpMarketItem, McpMarketSubmission, McpMarketVersion +from core.infra.exceptions import BadRequestError +from core.capabilities.mcp.management import auth_schema_from_headers +from sqlalchemy.orm import Session +from features.marketplace import mcp_state + +def validate_category(category: str) -> str: + value = (category or "").strip() + if value not in mcp_state.MCP_MARKET_CATEGORIES: + raise BadRequestError(message=f"MCP 市场分类必须是：{', '.join(mcp_state.MCP_MARKET_CATEGORIES)}") + return value + + +def normalize_auth_config( + auth_config: Any, + auth_schema: List[Dict[str, Any]], +) -> Dict[str, Any]: + """Return a backwards-compatible none/token/oauth2 authentication contract.""" + config = dict(auth_config or {}) if isinstance(auth_config, dict) else {} + normalized: List[Dict[str, Any]] = [] + seen: set[str] = set() + for raw in config.get("methods") or []: + if not isinstance(raw, dict): + continue + method = dict(raw) + method_id = str(method.get("id") or method.get("type") or "").strip() + method_type = str(method.get("type") or "").strip() + if not method_id or method_id in seen or method_type not in {"none", "token", "oauth2"}: + continue + method["id"] = method_id + method["type"] = method_type + method["label"] = str(method.get("label") or method_type) + if method_type == "oauth2": + method["client_registration"] = str( + method.get("client_registration") or "dynamic_or_manual" + ) + method["scopes"] = [str(scope) for scope in (method.get("scopes") or []) if str(scope)] + normalized.append(method) + seen.add(method_id) + if not normalized: + inferred = "token" if auth_schema else "none" + normalized = [ + {"id": inferred, "type": inferred, "label": "Token" if auth_schema else "无需认证"} + ] + default_method = str(config.get("default_method") or normalized[0]["id"]) + if default_method not in {method["id"] for method in normalized}: + default_method = str(normalized[0]["id"]) + credential_mode = str(config.get("credential_mode") or "auto") + if credential_mode not in {"auto", "installer", "admin"}: + credential_mode = "auto" + return { + "default_method": default_method, + "methods": normalized, + "credential_mode": credential_mode, + } + + +def endpoint_identity(url: str) -> tuple[str, str, int | None]: + """Scheme/host/port of an endpoint, ignoring path, query, and credentials.""" + parsed = urlparse((url or "").strip()) + scheme = (parsed.scheme or "").lower() + default_port = 443 if scheme == "https" else 80 if scheme == "http" else None + return scheme, (parsed.hostname or "").lower(), parsed.port or default_port + + +def installs_admin_published_endpoint( + item: McpMarketItem, + version: McpMarketVersion, + runtime_url: str, +) -> bool: + """Whether an install still targets the exact endpoint an admin put on the shelf. + + Admin publishing already vets the endpoint and may legitimately point at the + deployment's own network (the built-in MCP container, a LAN service). The + SSRF boundary exists for endpoints the *installer* supplies, so it must key + on the endpoint's provenance rather than on who is installing: an unchanged + admin-published host stays trusted for everyone who installs it, while a + URL-target credential that redirects the install elsewhere is checked + strictly, whoever submits it. + """ + if item.source != "admin": + return False + return endpoint_identity(runtime_url) == endpoint_identity(version.url) + + +def credential_free_connection(row: AdminMcpServer) -> tuple[str, List[Dict[str, Any]]]: + """Strip likely query-string secrets and expose only their install-time schema.""" + parsed = urlparse(row.url or "") + public_query: List[tuple[str, str]] = [] + auth_schema = auth_schema_from_headers(row.headers) + known_keys = {str(field.get("key") or "") for field in auth_schema} + for name, value in parse_qsl(parsed.query, keep_blank_values=True): + normalized = re.sub(r"[^A-Za-z0-9]+", "_", name).strip("_").upper() + looks_secret = any(term in name.lower() for term in mcp_state.SECRET_QUERY_TERMS) + credential_key = f"QUERY_{normalized or 'SECRET'}" + if looks_secret and value: + if credential_key not in known_keys: + auth_schema.append( + { + "key": credential_key, + "label": name, + "target": "query", + "name": name, + "required": True, + "secret": True, + } + ) + known_keys.add(credential_key) + continue + public_query.append((name, value)) + template_url = urlunparse(parsed._replace(query=urlencode(public_query))) + return template_url, auth_schema + + +def normalize_version(version: str) -> str: + value = (version or "1.0.0").strip() + if not re.fullmatch(r"[0-9A-Za-z][0-9A-Za-z._+-]{0,49}", value): + raise BadRequestError(message="版本号格式无效") + return value + + +def new_slug(db: Session, display_name: str, source_server_id: str) -> str: + ascii_slug = re.sub(r"[^a-z0-9]+", "-", display_name.lower()).strip("-") + base = (ascii_slug or f"mcp-{source_server_id[-12:].lower()}")[:100] + slug = base + suffix = 2 + while ( + db.query(McpMarketItem.slug).filter(McpMarketItem.slug == slug).first() + or db.query(McpMarketSubmission.slug) + .filter( + McpMarketSubmission.slug == slug, + McpMarketSubmission.status.in_(["pending", "approved"]), + McpMarketSubmission.deleted_at.is_(None), + ) + .first() + ): + slug = f"{base[:115]}-{suffix}" + suffix += 1 + return slug + + +def source_slug(db: Session, row: AdminMcpServer, owner_user_id: str) -> str: + previous = ( + db.query(McpMarketSubmission.slug) + .filter( + McpMarketSubmission.source_server_id == row.server_id, + McpMarketSubmission.owner_user_id == owner_user_id, + McpMarketSubmission.deleted_at.is_(None), + ) + .order_by(McpMarketSubmission.created_at.desc()) + .first() + ) + return str(previous[0]) if previous else new_slug(db, row.display_name, row.server_id) + + +def safe_origin(url: str) -> str: + from urllib.parse import urlparse + + parsed = urlparse(url or "") + return f"{parsed.scheme}://{parsed.netloc}" if parsed.scheme and parsed.netloc else "" diff --git a/src/backend/features/marketplace/mcp_versions.py b/src/backend/features/marketplace/mcp_versions.py new file mode 100644 index 00000000..a03be55b --- /dev/null +++ b/src/backend/features/marketplace/mcp_versions.py @@ -0,0 +1,221 @@ +"""mcp versions responsibilities.""" +from __future__ import annotations +from core.infra.time import utc_now +import re +import uuid +from typing import Any, Dict, List, Optional +from core.db.models import AdminMcpServer +from features.marketplace.models import McpMarketInstallation, McpMarketItem, McpMarketVersion +from core.infra.exceptions import BadRequestError, ResourceNotFoundError +from core.capabilities.mcp.management import refresh_mcp_caches, tool_snapshot_hash +from sqlalchemy.orm import Session +from features.marketplace import mcp_validation + +def version_for_item(db: Session, item: McpMarketItem) -> McpMarketVersion: + version = ( + db.query(McpMarketVersion) + .filter(McpMarketVersion.version_id == item.latest_version_id) + .first() + ) + if not version: + raise ResourceNotFoundError("mcp_market_version", item.latest_version_id) + return version + + +def publish_snapshot( + db: Session, + *, + slug: str, + display_name: str, + description: str, + user_intro: Optional[str], + category: str, + tags: List[str], + icon: Optional[str], + publisher_id: Optional[str], + publisher_name: str, + source: str, + version: str, + transport: str, + url: str, + auth_schema: List[Dict[str, Any]], + auth_config: Optional[Dict[str, Any]], + tools: List[Dict[str, Any]], + source_server_id: Optional[str], + approved_by: Optional[str], +) -> McpMarketItem: + if ( + db.query(McpMarketVersion) + .filter(McpMarketVersion.slug == slug, McpMarketVersion.version == version) + .first() + ): + raise BadRequestError(message=f"市场条目 {slug} 已存在版本 {version}，请提升版本号") + now = utc_now() + # Include a soft-deleted row so a later reviewed version can resurrect the + # same stable marketplace slug without colliding on the primary key. + item = db.query(McpMarketItem).filter(McpMarketItem.slug == slug).first() + version_id = f"mcpver_{uuid.uuid4().hex}" + if item is None: + item = McpMarketItem( + slug=slug, + display_name=display_name, + description=description, + user_intro=user_intro, + category=mcp_validation.validate_category(category), + tags=tags, + icon=icon, + publisher_id=publisher_id, + publisher_name=publisher_name, + source=source, + latest_version_id=version_id, + status="active", + last_verified_at=now, + created_at=now, + updated_at=now, + ) + db.add(item) + db.flush() + else: + item.deleted_at = None + item.display_name = display_name + item.description = description + item.user_intro = user_intro + item.category = mcp_validation.validate_category(category) + item.tags = tags + item.icon = icon + item.publisher_id = publisher_id + item.publisher_name = publisher_name + item.latest_version_id = version_id + item.status = "active" + item.status_reason = None + item.last_verified_at = now + item.updated_at = now + db.add( + McpMarketVersion( + version_id=version_id, + slug=slug, + version=mcp_validation.normalize_version(version), + transport=transport, + url=url, + auth_schema=auth_schema, + auth_config=mcp_validation.normalize_auth_config(auth_config, auth_schema), + tools_json=tools, + tool_hash=tool_snapshot_hash(tools), + source_server_id=source_server_id, + approved_by=approved_by, + approved_at=now, + created_at=now, + ) + ) + return item + + +def next_revalidated_version(db: Session, slug: str, current_version: str) -> str: + """Return a collision-free version for an administrator-accepted snapshot.""" + semantic = re.fullmatch(r"(\d+)\.(\d+)\.(\d+)", current_version) + if semantic: + major, minor, patch = (int(value) for value in semantic.groups()) + candidate = f"{major}.{minor}.{patch + 1}" + while ( + db.query(McpMarketVersion.version_id) + .filter( + McpMarketVersion.slug == slug, + McpMarketVersion.version == candidate, + ) + .first() + ): + patch += 1 + candidate = f"{major}.{minor}.{patch + 1}" + return candidate + + base = re.sub(r"[^0-9A-Za-z._+-]", "-", current_version).strip("-.") or "1.0.0" + sequence = 1 + while True: + candidate = f"{base[:32]}+recheck.{sequence}" + if not ( + db.query(McpMarketVersion.version_id) + .filter( + McpMarketVersion.slug == slug, + McpMarketVersion.version == candidate, + ) + .first() + ): + return candidate + sequence += 1 + + +def version_probe_candidate(item: McpMarketItem, version: McpMarketVersion) -> AdminMcpServer: + """Build a credential-free probe target from an immutable reviewed version.""" + return AdminMcpServer( + server_id=f"market_revalidate_{uuid.uuid4().hex}", + display_name=item.display_name, + description=item.description or "", + user_intro=item.user_intro, + transport=version.transport, + command=None, + args=[], + url=version.url, + env_vars={}, + env_inherit=[], + headers={}, + is_enabled=False, + sort_order=0, + extra_config={}, + tools_json=list(version.tools_json or []), + icon=item.icon, + owner_user_id=None, + source_plugin=None, + created_by="market-revalidation", + ) + + +def accept_revalidated_snapshot( + db: Session, + item: McpMarketItem, + version: McpMarketVersion, + tools: List[Dict[str, Any]], +) -> McpMarketVersion: + """Publish live drift as a new immutable version after explicit admin review.""" + next_version = next_revalidated_version(db, item.slug, version.version) + publish_snapshot( + db, + slug=item.slug, + display_name=item.display_name, + description=item.description or "", + user_intro=item.user_intro, + category=item.category, + tags=list(item.tags or []), + icon=item.icon, + publisher_id=item.publisher_id, + publisher_name=item.publisher_name or "", + source=item.source, + version=next_version, + transport=version.transport, + url=version.url, + auth_schema=list(version.auth_schema or []), + auth_config=dict(version.auth_config or {}), + tools=tools, + source_server_id=version.source_server_id, + approved_by="admin-revalidation", + ) + db.flush() + accepted = version_for_item(db, item) + if version.source_server_id: + ( + db.query(McpMarketInstallation) + .filter( + McpMarketInstallation.slug == item.slug, + McpMarketInstallation.server_id == version.source_server_id, + McpMarketInstallation.version_id == version.version_id, + ) + .update( + { + McpMarketInstallation.version_id: accepted.version_id, + McpMarketInstallation.updated_at: utc_now(), + }, + synchronize_session=False, + ) + ) + db.commit() + refresh_mcp_caches() + return accepted diff --git a/src/backend/features/marketplace/models.py b/src/backend/features/marketplace/models.py new file mode 100644 index 00000000..b0375816 --- /dev/null +++ b/src/backend/features/marketplace/models.py @@ -0,0 +1,388 @@ +"""Marketplace submissions, listings, versions and installations.""" +from core.db.utc_datetime import UTCDateTime +from core.infra.time import utc_now +from datetime import datetime, timezone +from core.db.engine import Base +from core.db.edition import MarketplaceListingEditionFields +from sqlalchemy import JSON, Boolean, CheckConstraint, Column, ForeignKey, Index, Integer, Numeric, String, Text, UniqueConstraint, text +from sqlalchemy.dialects.postgresql import INET, JSONB +from sqlalchemy.orm import mapped_column, relationship +from core.db.edition import UserAgentEditionFields, user_agent_edition_table_args +from sqlalchemy import JSON, BigInteger, Boolean, CheckConstraint, Column, ForeignKey, Index, Integer, Numeric, String, Text, UniqueConstraint +JSONType = JSON().with_variant(JSONB(), "postgresql") + +class McpMarketItem(Base): + """Installable MCP marketplace entry. + + The marketplace stores a credential-free definition. Concrete credentials + and enablement live on the ``AdminMcpServer`` created for each installation. + ``latest_version_id`` points at an immutable ``McpMarketVersion`` snapshot. + """ + + __tablename__ = "mcp_market_items" + + slug = Column(String(128), primary_key=True) + display_name = Column(String(255), nullable=False) + description = Column(Text, nullable=False, default="") + user_intro = Column(Text) + category = Column(String(64), nullable=False, default="通用工具") + tags = Column(JSONType, default=list) + icon = Column(String(500)) + publisher_id = Column(String(64)) + publisher_name = Column(String(255), nullable=False, default="") + source = Column(String(16), nullable=False, default="community") + latest_version_id = Column(String(64), nullable=False) + # active = installable; changed = remote tool drift awaiting review; + # suspended = security kill switch (derived installations are disabled). + status = Column(String(16), nullable=False, default="active") + status_reason = Column(Text) + last_verified_at = Column(UTCDateTime(timezone=True)) + created_at = Column(UTCDateTime(timezone=True), default=utc_now, nullable=False) + updated_at = Column( + UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now, nullable=False + ) + deleted_at = Column(UTCDateTime(timezone=True)) + + __table_args__ = ( + CheckConstraint( + "source IN ('admin', 'community')", + name="mcp_market_items_source_check", + ), + CheckConstraint( + "status IN ('active', 'changed', 'suspended')", + name="mcp_market_items_status_check", + ), + Index("idx_mcp_market_items_status", "status", "updated_at"), + Index("idx_mcp_market_items_category", "category"), + ) + +class McpMarketVersion(Base): + """Immutable, reviewed MCP marketplace version snapshot.""" + + __tablename__ = "mcp_market_versions" + + version_id = Column(String(64), primary_key=True) + slug = Column( + String(128), + ForeignKey("mcp_market_items.slug", ondelete="CASCADE"), + nullable=False, + ) + version = Column(String(50), nullable=False, default="1.0.0") + transport = Column(String(20), nullable=False, default="streamable_http") + url = Column(Text, nullable=False) + # [{key, label, target: "header" | "query" | "url", required, secret, ...}]; + # values are never stored here. + auth_schema = Column(JSONType, default=list) + # Provider-neutral authentication contract. ``methods`` may contain + # none/token/oauth2 options; OAuth endpoint discovery follows MCP metadata. + auth_config = Column(JSONType, default=dict) + tools_json = Column(JSONType, default=list) + tool_hash = Column(String(64), nullable=False) + listing_notice = Column(JSONType, default=dict) + source_server_id = Column(String(100)) + approved_by = Column(String(64)) + approved_at = Column(UTCDateTime(timezone=True)) + created_at = Column(UTCDateTime(timezone=True), default=utc_now, nullable=False) + + __table_args__ = ( + CheckConstraint( + "transport IN ('streamable_http', 'sse')", + name="mcp_market_versions_transport_check", + ), + UniqueConstraint("slug", "version", name="uq_mcp_market_versions_slug_version"), + Index("idx_mcp_market_versions_slug", "slug", "created_at"), + ) + +class McpMarketSubmission(Base): + """Credential-free snapshot submitted by a user for MCP marketplace review.""" + + __tablename__ = "mcp_market_submissions" + + submission_id = Column(String(64), primary_key=True) + slug = Column(String(128), nullable=False) + source_server_id = Column(String(100), nullable=False) + owner_user_id = Column(String(64), nullable=False) + submitter_name = Column(String(255), nullable=False, default="") + display_name = Column(String(255), nullable=False) + description = Column(Text, nullable=False, default="") + user_intro = Column(Text) + category = Column(String(64), nullable=False, default="通用工具") + tags = Column(JSONType, default=list) + icon = Column(String(500)) + version = Column(String(50), nullable=False, default="1.0.0") + transport = Column(String(20), nullable=False) + url = Column(Text, nullable=False) + auth_schema = Column(JSONType, default=list) + auth_config = Column(JSONType, default=dict) + tools_json = Column(JSONType, default=list) + tool_hash = Column(String(64), nullable=False) + listing_notice = Column(JSONType, default=dict) + note = Column(Text, nullable=False, default="") + status = Column(String(16), nullable=False, default="pending") + review_note = Column(Text) + reviewed_by = Column(String(64)) + reviewed_at = Column(UTCDateTime(timezone=True)) + created_at = Column(UTCDateTime(timezone=True), default=utc_now, nullable=False) + updated_at = Column( + UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now, nullable=False + ) + deleted_at = Column(UTCDateTime(timezone=True)) + + __table_args__ = ( + CheckConstraint( + "status IN ('pending', 'approved', 'rejected', 'withdrawn')", + name="mcp_market_submissions_status_check", + ), + CheckConstraint( + "transport IN ('streamable_http', 'sse')", + name="mcp_market_submissions_transport_check", + ), + Index("idx_mcp_market_submissions_status", "status", "created_at"), + Index("idx_mcp_market_submissions_owner", "owner_user_id", "created_at"), + Index("idx_mcp_market_submissions_slug", "slug"), + ) + +class McpMarketInstallation(Base): + """Links a reviewed market version to the concrete installed MCP server.""" + + __tablename__ = "mcp_market_installations" + + install_id = Column(String(64), primary_key=True) + slug = Column( + String(128), + ForeignKey("mcp_market_items.slug", ondelete="CASCADE"), + nullable=False, + ) + version_id = Column( + String(64), + ForeignKey("mcp_market_versions.version_id", ondelete="RESTRICT"), + nullable=False, + ) + server_id = Column( + String(100), + ForeignKey("admin_mcp_servers.server_id", ondelete="CASCADE"), + nullable=False, + unique=True, + ) + # NULL means the admin installed it globally; otherwise it is private to the user. + owner_user_id = Column(String(64)) + status = Column(String(16), nullable=False, default="active") + installed_by = Column(String(64)) + created_at = Column(UTCDateTime(timezone=True), default=utc_now, nullable=False) + updated_at = Column( + UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now, nullable=False + ) + + __table_args__ = ( + CheckConstraint( + "status IN ('active', 'suspended')", + name="mcp_market_installations_status_check", + ), + Index("idx_mcp_market_installations_owner", "owner_user_id", "created_at"), + Index("idx_mcp_market_installations_slug", "slug", "status"), + Index( + "uq_mcp_market_installations_user_slug", + "slug", + "owner_user_id", + unique=True, + postgresql_where=text("owner_user_id IS NOT NULL"), + sqlite_where=text("owner_user_id IS NOT NULL"), + ), + Index( + "uq_mcp_market_installations_global_slug", + "slug", + unique=True, + postgresql_where=text("owner_user_id IS NULL"), + sqlite_where=text("owner_user_id IS NULL"), + ), + ) + +class MarketplaceSubmission(Base): + """A user's private skill "apply for listing on the skill marketplace" record (pending admin review). + + At submission time a content snapshot of the source ``AdminSkill`` is + taken (skill_content / extra_files); review and listing are both based + on the snapshot — the user later modifying or deleting the original + skill does not affect listed content. Records with + ``status='approved'`` appear in the skill marketplace list + (source=community) and can be installed by everyone; the admin can + reject at any time (including delisting already-listed items). + """ + + __tablename__ = "marketplace_submissions" + + submission_id = Column(String(64), primary_key=True) + # Marketplace slug / entry_name after listing (derived at submission time, + # globally unique, does not clash with the preset marketplace catalog) + slug = Column(String(128), nullable=False, unique=True) + skill_id = Column(String(100), nullable=False) # source AdminSkill.skill_id + owner_user_id = Column(String(64), nullable=False) # applicant + submitter_name = Column( + String(255), default="" + ) # applicant display name (denormalized, for display) + + display_name = Column(String(255), nullable=False) + summary = Column(Text, default="") + user_intro = Column(Text) # explicit user-facing intro snapshot; NULL falls back to SKILL.md + category = Column(String(64), default="社区共享") + tags = Column(JSONType, default=list) + version = Column(String(50), default="1.0.0") + note = Column(Text, default="") # application note (for the admin) + + skill_content = Column(Text, nullable=False) # SKILL.md snapshot + extra_files = Column(JSONType, default=dict) # attachment snapshot {path: content} + + status = Column(String(16), nullable=False, default="pending") + review_note = Column(Text) # rejection reason / review note + reviewed_at = Column(UTCDateTime(timezone=True)) + created_at = Column(UTCDateTime(timezone=True), default=utc_now) + updated_at = Column(UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now) + + __table_args__ = ( + CheckConstraint( + "status IN ('pending', 'approved', 'rejected')", + name="marketplace_submissions_status_check", + ), + Index("idx_marketplace_submissions_status", "status", "created_at"), + Index("idx_marketplace_submissions_owner", "owner_user_id"), + ) + +class PluginMarketPackage(Base): + """Admin-uploaded plugin marketplace package (stored/listed in the DB). + + Counterpart of the skill marketplace's ``MarketplaceSubmission``: a + plugin zip uploaded by the admin from the /admin console is no longer + installed globally right away, but persisted as a "market package" — + storing the original zip's base64 (``package_b64``) + a set of display + metadata (extracted by a single normalize pass). Once listed it appears + in the plugin marketplace list for admins/users to install explicitly; + installation unzips and re-runs the existing ``import_plugin`` chain. + It coexists with filesystem preset packages + (``extensions/plugins/{default,marketplace}//``); when resolving by + slug, **filesystem takes precedence, DB market package is the + fallback**. Install records still go into ``InstalledPlugin``; this + table is only an "installable source". + """ + + __tablename__ = "plugin_market_packages" + + slug = Column(String(100), primary_key=True) + name = Column(String(255), nullable=False) + version = Column(String(50), nullable=False, default="1.0.0") + description = Column(Text, default="") + category = Column(String(64), default="") + # Library path / URL / inline data-URI (uploaded custom icons, capped at ~200KB by the service layer) + icon = Column(Text) + # Package kind: native / claude / codex (determined by normalize), display only + kind = Column(String(16), nullable=False, default="native") + skills_count = Column(Integer, nullable=False, default=0) + required_secrets = Column(JSONType, default=list) + has_admin_config = Column(Boolean, nullable=False, default=False) + package_b64 = Column(Text, nullable=False) # base64 of the originally uploaded zip + created_at = Column(UTCDateTime(timezone=True), default=utc_now) + updated_at = Column(UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now) + created_by = Column(String(64)) + + __table_args__ = (Index("idx_plugin_market_packages_category", "category"),) + +class PluginMarketSkillExclusion(Base): + """A single skill "deleted" by the admin from a plugin in the plugin marketplace (excluded by plugin slug + skill name). + + The plugin marketplace originally only supported whole-package deletion; + this table lets the admin precisely remove a specific skill within a + plugin. The market list's ``skills_count``, market detail ``skills``, + and DB persistence at install time are all filtered by it — built-in + filesystem packages (whose source files cannot be modified at runtime) + and DB-listed packages are uniformly covered by this exclusion list. + Installed instances are unaffected (those are install snapshots, managed + separately via enable/disable and uninstall). + """ + + __tablename__ = "plugin_market_skill_exclusions" + + slug = Column(String(100), primary_key=True) + skill_name = Column(String(100), primary_key=True) + created_at = Column(UTCDateTime(timezone=True), default=utc_now) + created_by = Column(String(64)) + +class MarketplaceListingState(MarketplaceListingEditionFields, Base): + """Marketplace listing switch: controls whether a plugin/skill is shown in the (plugin/skill) marketplace. + + ``kind`` = ``plugin`` | ``skill``; ``item_id`` = plugin slug or skill + marketplace slug. **A missing row counts as enabled** (everything listed + by default); a row with ``enabled=false`` is written only when the admin + explicitly disables — so built-in/existing items are not wiped after an + upgrade. The user-facing marketplace shows only enabled items; the admin + console shows everything with toggles. Physical deletion (only for + uploaded DB items) still goes through the respective delete endpoints; + this table only governs "listing visibility". + + """ + + __tablename__ = "marketplace_listing_states" + + kind = Column(String(16), primary_key=True) # plugin | skill + item_id = Column(String(160), primary_key=True) # plugin slug / skill marketplace slug + enabled = Column(Boolean, nullable=False, default=True) + updated_at = Column(UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now) + updated_by = Column(String(64)) + +class AgentMarketSubmission(Base): + """Record of a user-created sub-agent's "apply for listing on the sub-agent marketplace" (pending admin review). + + Mirrors the skill marketplace's ``MarketplaceSubmission``, but snapshots a + "sub-agent" rather than skill files: on submission, the source ``UserAgent`` is + content-snapshotted (prompt/welcome message/suggested questions/model config/ + capability bindings); review and listing are both based on the snapshot — later + edits or deletion of the original agent do not affect listed content. + ``status='approved'`` records appear in the sub-agent marketplace list + (source=community) and can be "install-cloned" by everyone as a private sub-agent; + admins can reject at any time (including delisting already-listed items). Direct + admin uploads to the marketplace are distinguished by the sentinel owner + ``__admin_upload__``. + """ + + __tablename__ = "agent_market_submissions" + + submission_id = Column(String(64), primary_key=True) + # Marketplace slug once listed (derived from name at submission, globally unique, never colliding with the preset marketplace catalog) + slug = Column(String(128), nullable=False, unique=True) + agent_id = Column(String(64), nullable=False) # source UserAgent.agent_id + owner_user_id = Column(String(64), nullable=False) # applicant (or __admin_upload__) + submitter_name = Column( + String(255), default="" + ) # applicant display name (denormalized for display) + + # Marketplace display metadata + name = Column(String(255), nullable=False) + avatar = Column(Text) + description = Column(Text, default="") + summary = Column(Text, default="") + category = Column(String(64), default="通用助手") + tags = Column(JSONType, default=list) + version = Column(String(50), default="1.0.0") + note = Column(Text, default="") # application note (for the admin) + + # Sub-agent content snapshot (decoupled from the source UserAgent) + system_prompt = Column(Text, default="") + welcome_message = Column(Text, default="") + suggested_questions = Column(JSONType, default=list) + # {temperature, max_tokens, max_iters, timeout} + model_config_snapshot = Column(JSONType, default=dict) + # {skill_ids, mcp_server_ids, plugin_ids, kb_ids} + bindings_snapshot = Column(JSONType, default=dict) + + status = Column(String(16), nullable=False, default="pending") + review_note = Column(Text) # rejection reason / review note + reviewed_at = Column(UTCDateTime(timezone=True)) + created_at = Column(UTCDateTime(timezone=True), default=utc_now) + updated_at = Column(UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now) + + __table_args__ = ( + CheckConstraint( + "status IN ('pending', 'approved', 'rejected')", + name="agent_market_submissions_status_check", + ), + Index("idx_agent_market_submissions_status", "status", "created_at"), + Index("idx_agent_market_submissions_owner", "owner_user_id"), + ) diff --git a/src/backend/features/marketplace/package_source.py b/src/backend/features/marketplace/package_source.py new file mode 100644 index 00000000..cf24de92 --- /dev/null +++ b/src/backend/features/marketplace/package_source.py @@ -0,0 +1,35 @@ +"""Marketplace package resolution with versioned, bundled and uploaded precedence.""" +import base64 +from core.infra.exceptions import ResourceNotFoundError +from core.capabilities.plugins.management import packages as plugin_packages +from core.capabilities.plugins.packaging import sources as plugin_sources +from core.capabilities.plugins.packaging.archive import extract_plugin_zip +from core.capabilities.plugins.packaging.importer import normalize_plugin_dir + + +class MarketplacePackageSource: + def normalize(self, db, slug): + """Fetch and normalize a plugin by slug: filesystem preset bundle first, DB-published package as fallback.""" + from features.marketplace.versions.store import active, decode + from features.marketplace.versions.sources import normalized_plugin + if plugin_sources.resolve_plugin_dir(slug) is None and ( + db is None or plugin_packages.market_row(db, slug) is None + ): + raise ResourceNotFoundError("plugin", slug) + snapshot = active(db, "plugin", slug) + if snapshot: + return normalized_plugin(decode(snapshot)) + plugin_dir = plugin_sources.resolve_plugin_dir(slug) + if plugin_dir is not None: + return normalize_plugin_dir(plugin_dir) + if db is not None: + row = plugin_packages.market_row(db, slug) + if row is not None: + with extract_plugin_zip(base64.b64decode(row.package_b64)) as plugin_root: + return normalize_plugin_dir(plugin_root) + raise ResourceNotFoundError("plugin", slug) + + def archive(self, db, slug): + from features.marketplace.versions.store import active, archive, decode + snapshot = active(db, "plugin", slug) + return archive(decode(snapshot)) if snapshot else None diff --git a/src/backend/core/services/marketplace_listing.py b/src/backend/features/marketplace/personal_listing.py similarity index 97% rename from src/backend/core/services/marketplace_listing.py rename to src/backend/features/marketplace/personal_listing.py index cc697c5a..bd03c984 100644 --- a/src/backend/core/services/marketplace_listing.py +++ b/src/backend/features/marketplace/personal_listing.py @@ -9,7 +9,7 @@ from core.infra.time import utc_now from typing import Any, Dict, List, Optional -from core.db.models import MarketplaceListingState +from features.marketplace.models import MarketplaceListingState from sqlalchemy.orm import Session KIND_PLUGIN = "plugin" diff --git a/src/backend/core/auth/marketplace_visibility.py b/src/backend/features/marketplace/personal_visibility.py similarity index 100% rename from src/backend/core/auth/marketplace_visibility.py rename to src/backend/features/marketplace/personal_visibility.py diff --git a/src/backend/core/plugins/management/market.py b/src/backend/features/marketplace/plugin_market.py similarity index 73% rename from src/backend/core/plugins/management/market.py rename to src/backend/features/marketplace/plugin_market.py index 99cb0cd9..0b667847 100644 --- a/src/backend/core/plugins/management/market.py +++ b/src/backend/features/marketplace/plugin_market.py @@ -3,40 +3,21 @@ from __future__ import annotations import logging +from core.capabilities.plugins.packaging.presentation import validate_icon, overlay_market_meta from typing import Any, Dict, List, Optional -from core.db.models import ( - ContentBlock, - InstalledPlugin, - PluginMarketPackage, - PluginMarketSkillExclusion, -) +from core.db.models import ContentBlock, InstalledPlugin +from features.marketplace.models import PluginMarketPackage, PluginMarketSkillExclusion from core.infra.exceptions import BadRequestError, ResourceNotFoundError -from core.plugins.management import packages as plugin_packages -from core.plugins.packaging import definitions as plugin_definitions -from core.plugins.packaging import sources as plugin_sources +from core.capabilities.plugins.management import packages as plugin_packages +from core.capabilities.plugins.packaging import definitions as plugin_definitions +from core.capabilities.plugins.packaging import sources as plugin_sources from sqlalchemy.orm import Session from sqlalchemy.orm.attributes import flag_modified logger = logging.getLogger(__name__) -def _validate_icon(icon: str) -> str: - """Validate an icon value from the UI picker; returns the stripped value ('' = clear).""" - icon = (icon or "").strip() - if not icon: - return "" - if icon.startswith("data:"): - if not icon.startswith("data:image/"): - raise BadRequestError(message="图标 data URI 必须是 image 类型") - if len(icon) > plugin_definitions.MAX_ICON_LEN: - raise BadRequestError(message="图标过大（上传原图请控制在 80KB 以内）") - return icon - if not (icon.startswith("/") or icon.startswith("http://") or icon.startswith("https://")): - raise BadRequestError(message="图标须从图标库选择或上传（不支持任意文本）") - if len(icon) > plugin_definitions.MAX_ICON_URL_LEN: - raise BadRequestError(message="图标地址过长") - return icon def _market_meta_overrides(db: Optional[Session]) -> Dict[str, Dict[str, Any]]: @@ -56,21 +37,13 @@ def resolve_market_meta(db: Optional[Session], slug: str) -> Dict[str, Any]: """Effective display metadata for one market plugin: DB override → builtin seed.""" meta = dict(plugin_definitions.BUILTIN_PLUGIN_MARKET_META.get(slug) or {}) override = _market_meta_overrides(db).get(slug) or {} - for k in plugin_definitions._META_KEYS: + for k in plugin_definitions.META_KEYS: v = override.get(k) if isinstance(v, str) and v.strip(): meta[k] = v.strip() return meta -def _overlay_market_meta(item: Dict[str, Any], meta: Dict[str, Any]) -> None: - """Apply effective display metadata onto a market list/detail dict (in place).""" - if meta.get("display_name"): - item["name"] = meta["display_name"] - if meta.get("category"): - item["category"] = meta["category"] - if meta.get("icon"): - item["icon"] = meta["icon"] def set_market_meta( @@ -89,8 +62,8 @@ def set_market_meta( market (filesystem bundle or uploaded DB package). """ if ( - plugin_sources._resolve_plugin_dir(slug) is None - and plugin_packages._market_row(db, slug) is None + plugin_sources.resolve_plugin_dir(slug) is None + and plugin_packages.market_row(db, slug) is None ): raise ResourceNotFoundError("plugin", slug) row = ( @@ -101,7 +74,7 @@ def set_market_meta( payload = dict(row.payload or {}) if row is not None else {} entry = dict(payload.get(slug) or {}) if icon is not None: - icon = _validate_icon(icon) + icon = validate_icon(icon) for key, val in (("display_name", display_name), ("category", category), ("icon", icon)): if val is None: continue @@ -142,8 +115,8 @@ def list_plugins( """ items: List[Dict[str, Any]] = [] seen_slugs: set = set() - for child in plugin_sources._iter_plugin_dirs(): - meta = plugin_sources._scan_native_manifest(child) + for child in plugin_sources.iter_plugin_dirs(): + meta = plugin_sources.scan_native_manifest(child) if meta: meta["source"] = "builtin" items.append(meta) @@ -152,17 +125,17 @@ def list_plugins( for row in db.query(PluginMarketPackage).order_by(PluginMarketPackage.created_at.desc()).all(): if row.slug in seen_slugs: continue - items.append(plugin_packages._market_meta_dict(row)) + items.append(plugin_packages.market_meta_dict(row)) seen_slugs.add(row.slug) # Display metadata is UI configuration: DB override → builtin seed → manifest fallback overrides = _market_meta_overrides(db) for it in items: meta = dict(plugin_definitions.BUILTIN_PLUGIN_MARKET_META.get(it["slug"]) or {}) - for k in plugin_definitions._META_KEYS: + for k in plugin_definitions.META_KEYS: v = (overrides.get(it["slug"]) or {}).get(k) if isinstance(v, str) and v.strip(): meta[k] = v.strip() - _overlay_market_meta(it, meta) + overlay_market_meta(it, meta) # Subtract skills the admin removed from the marketplace (aggregated once per slug) so skills_count reflects the real offering excl_by_slug: Dict[str, set] = {} for ex_slug, ex_name in db.query( @@ -178,7 +151,7 @@ def list_plugins( # Annotate installed status if items: install_ids = { - it["slug"]: plugin_sources._make_plugin_install_id(it["slug"], owner_user_id) + it["slug"]: plugin_sources.make_plugin_install_id(it["slug"], owner_user_id) for it in items } present = { @@ -194,7 +167,7 @@ def list_plugins( # entries visible to them; the admin panel sees everything with # annotations. For user-side callers owner_user_id is the current browsing # user, reused directly as the viewer. - from core.services import marketplace_listing as ml + from features.marketplace import edition as ml items = ml.annotate_and_filter( db, @@ -211,7 +184,7 @@ def set_plugin_market_enabled( db: Session, slug: str, enabled: bool, *, updated_by: Optional[str] = None ) -> Dict[str, Any]: """Publish/unpublish a marketplace plugin (controls display in the plugin marketplace; does not affect installed instances).""" - from core.services import marketplace_listing as ml + from features.marketplace import edition as ml res = ml.set_listing_enabled(db, ml.KIND_PLUGIN, slug, enabled, updated_by=updated_by) logger.info("plugin_market_listing: slug=%s enabled=%s", slug, enabled) diff --git a/src/backend/features/marketplace/skill_archives.py b/src/backend/features/marketplace/skill_archives.py new file mode 100644 index 00000000..06724ea2 --- /dev/null +++ b/src/backend/features/marketplace/skill_archives.py @@ -0,0 +1,49 @@ +"""skill archives responsibilities.""" +from __future__ import annotations +import io +import os +import zipfile +from pathlib import Path +from typing import Any, Dict, List +from core.capabilities.skills.binary_files import BINARY_EXTENSIONS, JUNK_BASENAMES, MAX_SINGLE_FILE, MAX_TOTAL, decode_binary, encode_binary, is_binary_value +from core.capabilities.skills.deps.detector import detect_dependencies +from core.capabilities.skills.registry import SkillSpecError, load_skill_metadata_from_str, split_frontmatter +from fastapi import HTTPException +from features.marketplace import skills_state + + + +def build_skill_zip(skill_id: str, skill_content: str, extra_files: Dict[str, str]) -> bytes: + """Pack a DB skill into a zip byte stream (inverse of ``parse_skill_zip``). + + Layout is ``/SKILL.md`` + ``/``, + consistent with the upload/import convention, so an exported package can be + re-imported directly. base64 values in extra_files carrying the binary + marker are restored to their original bytes. + """ + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf: + zf.writestr(f"{skill_id}/SKILL.md", skill_content or "") + for rel, stored in (extra_files or {}).items(): + data = decode_binary(stored) if is_binary_value(stored) else str(stored).encode("utf-8") + zf.writestr(f"{skill_id}/{rel}", data) + return buf.getvalue() + + +def build_skill_zip_from_dir(skill_id: str, skill_dir: Path) -> bytes: + """Pack an on-disk skill directory (built-in/filesystem skills) into a zip byte stream. + + Same layout as ``build_skill_zip``; skips version-control/cache directories + and junk files, and packs binary files as-is (no base64 round-trip, + lossless). + """ + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf: + for p in sorted(skill_dir.rglob("*")): + if not p.is_file() or p.name in JUNK_BASENAMES: + continue + rel = p.relative_to(skill_dir) + if skills_state.ZIP_SKIP_DIR_PARTS.intersection(rel.parts): + continue + zf.write(p, f"{skill_id}/{rel.as_posix()}") + return buf.getvalue() diff --git a/src/backend/features/marketplace/skill_catalog.py b/src/backend/features/marketplace/skill_catalog.py new file mode 100644 index 00000000..6abf2803 --- /dev/null +++ b/src/backend/features/marketplace/skill_catalog.py @@ -0,0 +1,184 @@ +"""skill catalog responsibilities.""" +from __future__ import annotations +from core.capabilities.skills import installation as package_installation +from typing import Any, Dict, List, Optional +from core.db.models import AdminSkill +from core.infra.exceptions import ResourceNotFoundError +from sqlalchemy.orm import Session +from features.marketplace import skill_metadata +from features.marketplace import skills_state + +def list_marketplace_skills( + db: Optional[Session] = None, + *, + include_disabled: bool = False, + viewer_user_id: Optional[str] = None, +) -> List[Dict[str, Any]]: + """List all market skill metadata: built-in default skills + preloaded (featured first, downloads descending) + community submissions. + + ``include_disabled``: the admin console passes True (sees everything + + ``market_enabled`` annotation); the user side defaults to False (only sees + listed items, and scoped items invisible to ``viewer_user_id`` are filtered + by visibility). See ``marketplace_listing`` for the delist switch and + visibility scopes. + """ + items: List[Dict[str, Any]] = [] + if skills_state.MARKETPLACE_DIR.is_dir(): + for child in sorted(skills_state.MARKETPLACE_DIR.iterdir()): + if not child.is_dir(): + continue + m = skill_metadata.read_manifest(child.name) + if m: + items.append(skill_metadata.public_meta(m)) + items.sort(key=lambda x: (0 if x["featured"] else 1, -x["downloads"], x["slug"])) + # Built-in default skills come first (built-in resident, shown with priority). + items = skill_metadata.default_skill_metas() + items + if db is not None: + items.extend(skill_metadata.submission_public_meta(s) for s in skill_metadata.approved_submissions(db)) + from features.marketplace import edition as ml + items = ml.annotate_and_filter( + db, ml.KIND_SKILL, items, id_key="slug", + include_disabled=include_disabled, viewer_user_id=viewer_user_id, + ) + return items + + +def set_marketplace_skill_enabled( + db: Session, slug: str, enabled: bool, *, updated_by: Optional[str] = None +) -> Dict[str, Any]: + """List/delist a market skill (controls whether it shows in the skill marketplace; installed instances are unaffected).""" + from features.marketplace import edition as ml + return ml.set_listing_enabled(db, ml.KIND_SKILL, slug, enabled, updated_by=updated_by) + + +def list_categories( + db: Optional[Session] = None, + *, + include_disabled: bool = False, + viewer_user_id: Optional[str] = None, +) -> List[str]: + """Marketplace category list: the 8 fixed categories first (stable order), legacy leftover categories appended after.""" + seen: List[str] = list(skills_state.MARKETPLACE_CATEGORIES) + for it in list_marketplace_skills( + db, include_disabled=include_disabled, viewer_user_id=viewer_user_id + ): + if it["category"] not in seen: + seen.append(it["category"]) + return seen + + +def get_marketplace_skill(slug: str, db: Optional[Session] = None) -> Dict[str, Any]: + """Single market skill detail: metadata + file list + SKILL.md body (for the detail preview). + + The preloaded directory takes priority; when not found, fall back to the + community listing (approved submission). + """ + # Built-in default skills: read extensions/skills/default/; the body comes from SKILL.md, the file list from the directory. + if slug in skills_state.DEFAULT_SKILL_MARKET: + meta = skill_metadata.default_skill_public_meta(slug) + if meta is None: + raise ResourceNotFoundError("marketplace_skill", slug) + pkg_dir = skills_state.DEFAULT_SKILLS_DIR / slug + files: List[Dict[str, Any]] = [] + for p in sorted(pkg_dir.rglob("*")): + if not p.is_file(): + continue + rel = p.relative_to(pkg_dir).as_posix() + if rel == skills_state.SKILL_MD_NAME: + continue + files.append({"path": rel, "size": p.stat().st_size}) + skill_md = pkg_dir / skills_state.SKILL_MD_NAME + meta["files"] = files + meta["instructions"] = package_installation.strip_frontmatter(skill_md.read_text(encoding="utf-8", errors="ignore")) if skill_md.is_file() else "" + return meta + m = skill_metadata.read_manifest(slug) + if not m: + sub = skill_metadata.get_approved_submission(db, slug) if db is not None else None + if sub is None: + raise ResourceNotFoundError("marketplace_skill", slug) + data = skill_metadata.submission_public_meta(sub) + data["files"] = [ + {"path": rel, "size": len(str(content).encode("utf-8"))} + for rel, content in sorted((sub.extra_files or {}).items()) + if rel != skills_state.REQUIRED_SECRETS_SNAPSHOT # internal sentinel, not part of the installed content + ] + data["instructions"] = package_installation.strip_frontmatter(sub.skill_content or "") + return data + pkg_dir = skills_state.MARKETPLACE_DIR / slug + data = skill_metadata.public_meta(m) + + files: List[Dict[str, Any]] = [] + for p in sorted(pkg_dir.rglob("*")): + if not p.is_file() or p.name == skills_state.MANIFEST_NAME: + continue + rel = p.relative_to(pkg_dir).as_posix() + if rel == skills_state.SKILL_MD_NAME: # SKILL.md is returned separately as instructions; don't duplicate it in the file list + continue + files.append({"path": rel, "size": p.stat().st_size}) + body = "" + skill_md = pkg_dir / skills_state.SKILL_MD_NAME + if skill_md.is_file(): + body = package_installation.strip_frontmatter(skill_md.read_text(encoding="utf-8", errors="ignore")) + data["files"] = files + data["instructions"] = body + return data + + +def is_installed(db: Session, slug: str, owner_user_id: Optional[str]) -> bool: + """Whether this market skill has been installed by the given scope (global / current user).""" + if slug in skills_state.DEFAULT_SKILL_MARKET: + return True # built-in default skills are globally resident; considered always "installed" + m = skill_metadata.read_manifest(slug) + if not m: + if skill_metadata.get_approved_submission(db, slug) is None: + return False + entry_name = slug + else: + entry_name = m["entry_name"] + install_id = package_installation.compute_install_id(entry_name, owner_user_id) + q = db.query(AdminSkill).filter(AdminSkill.skill_id == install_id) + if owner_user_id is None: + q = q.filter(AdminSkill.owner_user_id.is_(None)) + else: + q = q.filter(AdminSkill.owner_user_id == owner_user_id) + return q.first() is not None + + +def annotate_installed( + items: List[Dict[str, Any]], db: Session, owner_user_id: Optional[str] +) -> List[Dict[str, Any]]: + """Annotate each market list item with an ``installed`` flag (for the given scope). One batched query decides all.""" + if not items: + return items + install_ids = { + it["slug"]: package_installation.compute_install_id(it["entry_name"], owner_user_id) for it in items + } + q = db.query(AdminSkill.skill_id, AdminSkill.dep_status) + if owner_user_id is None: + q = q.filter(AdminSkill.owner_user_id.is_(None)) + else: + q = q.filter(AdminSkill.owner_user_id == owner_user_id) + q = q.filter(AdminSkill.skill_id.in_(list(install_ids.values()))) + status_map = {sid: dep for sid, dep in q.all()} + # Rejected skills carry the admin's reason, surfaced to the user ("Rejected by admin: ..."). + rejected_ids = [sid for sid, dep in status_map.items() if dep == "rejected"] + reason_map: Dict[str, Optional[str]] = {} + if rejected_ids: + from core.capabilities.skills.deps.requests import get_reject_reason + for sid in rejected_ids: + reason_map[sid] = get_reject_reason(db, sid) + for it in items: + # Built-in default skills are globally resident and always available to + # everyone — mark them "installed/ready" directly (the frontend button + # shows "already built-in"); no AdminSkill lookup, no install flow. + if it.get("builtin"): + it["installed"] = True + it["dep_status"] = "ready" + it["dep_reason"] = None + continue + iid = install_ids[it["slug"]] + it["installed"] = iid in status_map + # dep_status: 'installing' = dependencies installing / 'rejected' = admin declined / 'ready' = usable. None when not installed. + it["dep_status"] = status_map.get(iid) if iid in status_map else None + it["dep_reason"] = reason_map.get(iid) + return items diff --git a/src/backend/features/marketplace/skill_installation.py b/src/backend/features/marketplace/skill_installation.py new file mode 100644 index 00000000..52351f53 --- /dev/null +++ b/src/backend/features/marketplace/skill_installation.py @@ -0,0 +1,199 @@ +"""skill installation responsibilities.""" +from __future__ import annotations +from core.capabilities.skills import installation as package_installation +from pathlib import Path +from typing import Any, Dict, Optional, Tuple +from core.capabilities.skills.binary_files import is_binary_value, pack_directory +from core.capabilities.skills.cache_refresh import refresh_skill_caches +from core.capabilities.skills.deps.detector import detect_dependencies +from core.capabilities.skills.registry import load_skill_metadata_from_str +from core.db.models import AdminSkill +from core.infra.exceptions import BadRequestError, ResourceNotFoundError +from core.infra.time import utc_now +from core.ontology.build_validator import ensure_ontology_build_valid +from fastapi import HTTPException +from sqlalchemy.orm import Session +from sqlalchemy.orm.attributes import flag_modified +import logging +logger = logging.getLogger(__name__) +from features.marketplace import skill_metadata +from features.marketplace import skills_state + +def load_package_files(pkg_dir: Path) -> Tuple[str, Dict[str, str]]: + """Read a market skill package: returns (SKILL.md original text, {relative path: stored content}). + + Binary files are stored as base64 (consistent with admin upload), text as + original text; the manifest and junk files are skipped, and per-file/total + size caps are enforced. + """ + skill_md = pkg_dir / skills_state.SKILL_MD_NAME + if not skill_md.is_file(): + raise BadRequestError(message="市场技能包缺少 SKILL.md") + skill_content = skill_md.read_text(encoding="utf-8") + extra_files = pack_directory(pkg_dir, skip_names={skills_state.MANIFEST_NAME, skills_state.SKILL_MD_NAME}) + return skill_content, extra_files + + +def install_marketplace_skill( + db: Session, + slug: str, + *, + owner_user_id: Optional[str], + secrets: Optional[Dict[str, str]] = None, +) -> Dict[str, Any]: + """Install a market skill as an ``AdminSkill`` (admin = global / user = private), returning a result dict. + + The preloaded directory takes priority; when the slug isn't in the preloaded + directory, fall back to the community listing (approved submission), with + install content taken from the submission-time snapshot. + """ + # Built-in default skills are globally resident and always available to + # everyone — no install needed (the frontend button is already disabled as + # "already built-in"); this is a defensive block to avoid duplicating 1MB+ + # office skill content into the DB as a private copy. + if slug in skills_state.DEFAULT_SKILL_MARKET: + raise BadRequestError(message="内置技能已全局可用，无需安装") + secrets = secrets or {} + m = skill_metadata.read_manifest(slug) + if m: + pkg_dir = skills_state.MARKETPLACE_DIR / slug + skill_content, extra_files = load_package_files(pkg_dir) + else: + sub = skill_metadata.get_approved_submission(db, slug) + if sub is None: + raise ResourceNotFoundError("marketplace_skill", slug) + m = { + "slug": sub.slug, + "entry_name": sub.slug, + "display_name": sub.display_name, + "summary": sub.summary or "", + "user_intro": sub.user_intro or "", + "category": sub.category or "社区共享", + "tags": list(sub.tags or []), + "version": sub.version or "1.0.0", + "required_secrets": skill_metadata.submission_required_secrets(sub), + } + skill_content = sub.skill_content or "" + extra_files = dict(sub.extra_files or {}) + # The sentinel entry only carries the credential field names; it never lands in the install directory + extra_files.pop(skills_state.REQUIRED_SECRETS_SNAPSHOT, None) + + entry_name = str(m["entry_name"]).strip() + install_id = package_installation.compute_install_id(entry_name, owner_user_id) + # Always rewrite the frontmatter ``name`` to the install id: this both + # namespaces private user installs and, along the way, normalizes illegal + # names in market skills (containing spaces/Chinese etc.) — otherwise + # load_skill_metadata_from_str fails validation with "invalid skill id". + # install_id comes from slug/entry_name and is always a legal slug. + skill_content = package_installation.rewrite_frontmatter_name(skill_content, install_id) + + required_secrets = list(m.get("required_secrets") or []) + if required_secrets: + skill_content = package_installation.inject_secrets( + skill_content, extra_files, required_secrets, secrets + ) + + # Validate SKILL.md (description required, etc.); an invalid one is a straight 400 rather than being persisted. + try: + meta = load_skill_metadata_from_str(skill_content, install_id) + except Exception as exc: # noqa: BLE001 + raise BadRequestError(message=f"市场技能 SKILL.md 不合法：{exc}") + + dependencies = detect_dependencies( + {fn: c for fn, c in extra_files.items() if not is_binary_value(c)} + ) + + display_name = m.get("display_name") or meta.name or install_id + description = meta.description or m.get("summary") or "" + tags = list(m.get("tags") or meta.tags or []) + version = m.get("version") or meta.version or "1.0.0" + user_intro = build_user_intro(m) or None + + ensure_ontology_build_valid( + db, + asset_type="skill", + name=display_name or install_id, + description=description, + instructions=skill_content, + tool_names=list(meta.allowed_tools or []), + ontology_tags=list(tags), + ) + + now = utc_now() + existing = db.query(AdminSkill).filter(AdminSkill.skill_id == install_id).first() + if existing is not None: + # Owner conflict: cannot overwrite a public skill or another user's private skill (admin reinstalling a global one is allowed) + if existing.owner_user_id != owner_user_id: + if existing.owner_user_id is None: + raise HTTPException(status_code=409, detail=f"技能 id 「{install_id}」与公共技能冲突") + raise HTTPException(status_code=409, detail=f"技能 id 「{install_id}」已被占用") + existing.skill_content = skill_content + existing.display_name = display_name + existing.description = description + existing.user_intro = user_intro + existing.version = version + existing.tags = tags + existing.allowed_tools = list(meta.allowed_tools or []) + existing.extra_files = extra_files + existing.dependencies = dependencies + existing.is_enabled = True + existing.updated_at = now + flag_modified(existing, "tags") + flag_modified(existing, "extra_files") + flag_modified(existing, "dependencies") + action = "updated" + else: + db.add(AdminSkill( + skill_id=install_id, + skill_content=skill_content, + display_name=display_name, + description=description, + user_intro=user_intro, + version=version, + tags=tags, + allowed_tools=list(meta.allowed_tools or []), + extra_files=extra_files, + dependencies=dependencies, + is_enabled=True, + owner_user_id=owner_user_id, + created_at=now, + updated_at=now, + )) + action = "installed" + db.commit() + # Strict dependency verdicts happen at the async route layer (gate_skill_deps + # needs to actually run probes in the sandbox; this function is synchronous). + # Here we only carry the declared dependencies out with the result; the route + # awaits the probe → decides installing/ready. + # Assign the installed skill a built-in icon by category (users/admins can change it later in skill editing). + try: + from core.capabilities.skills.icons import get_skill_icon, preset_for_category, set_skill_icon + if not get_skill_icon(db, install_id): + set_skill_icon(db, install_id, preset_for_category(m.get("category"))) + except Exception as exc: # noqa: BLE001 + logger.debug("marketplace set default icon failed: %s", exc) + refresh_skill_caches() + logger.info( + "marketplace_skill_%s: slug=%s id=%s owner=%s files=%d", + action, slug, install_id, owner_user_id or "global", len(extra_files), + ) + return { + "id": install_id, + "slug": slug, + "owner": "self" if owner_user_id else "global", + "action": action, + "dependencies": dependencies, + # dep_pending / message are filled in by the route layer after awaiting gate_skill_deps (strict probe result). + "dep_pending": False, + "message": "技能已安装" if action == "installed" else "技能已更新", + } + + +def build_user_intro(m: Dict[str, Any]) -> str: + """Return only an explicitly authored user intro from marketplace metadata. + + The marketplace ``summary`` is card copy. Treating it as ``user_intro`` + suppresses the catalog's SKILL.md-body fallback and reduces rich skill + documentation to a single line after installation. + """ + return str(m.get("user_intro") or "").strip() diff --git a/src/backend/features/marketplace/skill_metadata.py b/src/backend/features/marketplace/skill_metadata.py new file mode 100644 index 00000000..159b57c9 --- /dev/null +++ b/src/backend/features/marketplace/skill_metadata.py @@ -0,0 +1,174 @@ +"""skill metadata responsibilities.""" +from __future__ import annotations +import json +from typing import Any, Dict, List, Optional +from core.capabilities.skills.registry import split_frontmatter +from features.marketplace.models import MarketplaceSubmission +from core.infra.exceptions import BadRequestError +from sqlalchemy.orm import Session +import logging +logger = logging.getLogger(__name__) +from features.marketplace import skills_state + +def read_manifest(slug: str) -> Optional[Dict[str, Any]]: + """Read and validate a single market skill's ``marketplace.json`` (tolerant; returns None if corrupted).""" + if not slug or "/" in slug or ".." in slug: + return None + path = skills_state.MARKETPLACE_DIR / slug / skills_state.MANIFEST_NAME + if not path.is_file(): + return None + try: + m = json.loads(path.read_text(encoding="utf-8")) + except Exception as exc: # noqa: BLE001 + logger.warning("marketplace manifest broken: %s (%s)", slug, exc) + return None + if not isinstance(m, dict) or not m.get("entry_name"): + return None + m.setdefault("slug", slug) + return m + + +def market_skill_requires_secrets(slug: str) -> bool: + """Whether a preloaded market skill declares required_secrets (public contract, reusable by other markets/callers).""" + m = read_manifest(slug) + if not m: + return False + return bool(m.get("requires_api_key") or m.get("required_secrets")) + + +def public_meta(m: Dict[str, Any]) -> Dict[str, Any]: + """Manifest → externally visible metadata (no skill body/file contents).""" + return { + "slug": m.get("slug"), + "entry_name": m.get("entry_name"), + "display_name": m.get("display_name") or m.get("slug"), + "summary": m.get("summary") or "", + "category": m.get("category") or "其他", + "tags": list(m.get("tags") or []), + "version": m.get("version") or "1.0.0", + "author": m.get("author") or "skillhub", + "icon_url": m.get("icon_url") or "", + "source": m.get("source") or "skillhub", + "source_url": m.get("source_url") or "", + "downloads": int(m.get("downloads") or 0), + "stars": int(m.get("stars") or 0), + "featured": bool(m.get("featured")), + "requires_api_key": bool(m.get("requires_api_key")), + "required_secrets": list(m.get("required_secrets") or []), + # Preloaded (filesystem) skills ship with the repo; admins cannot delete them online. + "deletable": False, + } + + +def submission_public_meta(sub: MarketplaceSubmission) -> Dict[str, Any]: + """Approved community submission → external metadata isomorphic to the preloaded manifest form.""" + return { + "slug": sub.slug, + "entry_name": sub.slug, + "display_name": sub.display_name or sub.slug, + "summary": sub.summary or "", + "category": sub.category or "社区共享", + "tags": list(sub.tags or []), + "version": sub.version or "1.0.0", + "author": sub.submitter_name or sub.owner_user_id, + "icon_url": "", + "source": "community", + "source_url": "", + "downloads": 0, + "stars": 0, + "featured": False, + "requires_api_key": False, + "required_secrets": submission_required_secrets(sub), + # DB listing records (admin uploads / user community submissions) → admins can delete online (remove from the market). + "deletable": True, + } + + +def approved_submissions(db: Session) -> List[MarketplaceSubmission]: + return ( + db.query(MarketplaceSubmission) + .filter(MarketplaceSubmission.status == "approved") + .order_by(MarketplaceSubmission.created_at.desc()) + .all() + ) + + +def default_skill_public_meta(slug: str) -> Optional[Dict[str, Any]]: + """Built-in default skill (extensions/skills/default/) → market-list metadata isomorphic to the preloaded form. + + The Chinese name comes from SKILL.md's ``display_name``; category/summary + come from ``DEFAULT_SKILL_MARKET``. Marked ``source='builtin'`` + + ``builtin=True`` — these skills are globally resident and always available + to everyone; the market shows them as "built-in" (already available) and + they do not go through "install as a private copy" (avoiding duplicating + 1MB+ office skills into the DB). + """ + cfg = skills_state.DEFAULT_SKILL_MARKET.get(slug) + if cfg is None: + return None + md = skills_state.DEFAULT_SKILLS_DIR / slug / skills_state.SKILL_MD_NAME + if not md.is_file(): + return None + try: + fm, _ = split_frontmatter(md.read_text(encoding="utf-8", errors="ignore")) + except Exception: # noqa: BLE001 + fm = {} + name = str(fm.get("name") or slug).strip() + tags = fm.get("tags") or [] + if isinstance(tags, str): + tags = [t.strip() for t in tags.split(",") if t.strip()] + return { + "slug": slug, + "entry_name": name, + "display_name": str(fm.get("display_name") or name).strip(), + "summary": cfg["summary"], + "category": cfg["category"], + "tags": list(tags), + "version": str(fm.get("version") or "1.0.0"), + "author": "内置", + "icon_url": "", + "source": "builtin", + "source_url": "", + "downloads": 0, + "stars": 0, + "featured": False, + "requires_api_key": False, + "required_secrets": [], + "deletable": False, # built-ins ship with the repo; not deletable online + "builtin": True, # the frontend uses this to show "built-in" and set the button to "already built-in" + } + + +def default_skill_metas() -> List[Dict[str, Any]]: + metas = [default_skill_public_meta(s) for s in skills_state.DEFAULT_SKILL_MARKET] + return [m for m in metas if m] + + +def validate_category(category: str) -> str: + """Validate and return a legal category; anything outside the fixed set is a straight 400.""" + category = (category or "").strip() + if category not in skills_state.MARKETPLACE_CATEGORIES: + raise BadRequestError( + message=f"请从固定分类中选择：{'、'.join(skills_state.MARKETPLACE_CATEGORIES)}" + ) + return category + + +def get_approved_submission(db: Session, slug: str) -> Optional[MarketplaceSubmission]: + return ( + db.query(MarketplaceSubmission) + .filter(MarketplaceSubmission.slug == slug, MarketplaceSubmission.status == "approved") + .first() + ) + + +def submission_required_secrets(sub: MarketplaceSubmission) -> List[Dict[str, Any]]: + """Recover a community skill's required_secrets from the snapshot sentinel entry (empty list if absent).""" + raw = (sub.extra_files or {}).get(skills_state.REQUIRED_SECRETS_SNAPSHOT) + if not raw: + return [] + try: + fields = json.loads(raw) + return fields if isinstance(fields, list) else [] + except (TypeError, ValueError): + return [] diff --git a/src/backend/features/marketplace/skill_publishing.py b/src/backend/features/marketplace/skill_publishing.py new file mode 100644 index 00000000..504ea6be --- /dev/null +++ b/src/backend/features/marketplace/skill_publishing.py @@ -0,0 +1,231 @@ +"""skill publishing responsibilities.""" +from __future__ import annotations +from core.capabilities.skills import zip_reader +import uuid +from typing import Any, Dict, List, Optional, Tuple +from core.capabilities.skills.registry import load_skill_metadata_from_str +from features.marketplace.models import MarketplaceSubmission +from core.infra.exceptions import BadRequestError, ResourceNotFoundError +from core.infra.time import utc_now +from core.ontology.build_validator import ensure_ontology_build_valid +from sqlalchemy.orm import Session +from sqlalchemy.orm.attributes import flag_modified +import logging +logger = logging.getLogger(__name__) +from features.marketplace import skill_archives +from features.marketplace import skill_metadata +from features.marketplace import skill_submission_data +from features.marketplace import skills_state + +def upsert_admin_submission( + db: Session, + *, + skill_id: str, + skill_content: str, + display_name: str, + summary: str, + category: str, + tags: List[str], + version: str, + extra_files: Dict[str, str], + note: str, + submitter_name: str, + user_intro: Optional[str] = None, +) -> Tuple[str, str]: + """Upsert an approved listing record under the ADMIN_UPLOAD_OWNER sentinel, returning (slug, action). + + When the same skill_id already has an admin listing record, update it + (keeping the original slug); otherwise create one. Shared by the two admin + listing entry points: "upload zip" and "create via form". + """ + category = skill_metadata.validate_category(category) + snapshot_files = dict(extra_files) + # Credential files are never distributed with the market snapshot (same as community listing). + snapshot_files.pop(skills_state.SECRETS_FILENAME, None) + now = utc_now() + existing = ( + db.query(MarketplaceSubmission) + .filter( + MarketplaceSubmission.skill_id == skill_id, + MarketplaceSubmission.owner_user_id == skills_state.ADMIN_UPLOAD_OWNER, + ) + .first() + ) + if existing is not None: + existing.display_name = display_name + existing.summary = summary + existing.user_intro = str(user_intro or "").strip() or None + existing.category = category + existing.tags = list(tags or []) + existing.version = version or "1.0.0" + existing.skill_content = skill_content + existing.extra_files = snapshot_files + existing.note = note + existing.status = "approved" + existing.review_note = "" + existing.reviewed_at = now + existing.updated_at = now + flag_modified(existing, "tags") + flag_modified(existing, "extra_files") + return existing.slug, "updated" + + slug = skill_submission_data.derive_slug(db, skill_id, skills_state.ADMIN_UPLOAD_OWNER) + db.add(MarketplaceSubmission( + submission_id=f"mkadm_{uuid.uuid4().hex[:16]}", + slug=slug, + skill_id=skill_id, + owner_user_id=skills_state.ADMIN_UPLOAD_OWNER, + submitter_name=(submitter_name or "").strip(), + display_name=display_name, + summary=summary, + user_intro=str(user_intro or "").strip() or None, + category=category, + tags=list(tags or []), + version=version or "1.0.0", + note=note, + skill_content=skill_content, + extra_files=snapshot_files, + status="approved", + reviewed_at=now, + created_at=now, + updated_at=now, + )) + return slug, "published" + + +def publish_skill_zip_to_marketplace( + db: Session, + data: bytes, + *, + category: str, + display_name: str = "", + summary: str = "", + submitter_name: str = "管理员上传", +) -> Dict[str, Any]: + """Admin uploads a skill zip → listed directly in the marketplace as approved (not in the catalog, not globally effective). + + When the same skill (by SKILL.md's name = skill_id) is uploaded again, the + existing admin listing record is updated (keeping the original slug) rather + than adding a new one each time. + """ + parsed = zip_reader.parse_skill_zip(data) + meta = parsed["meta"] + name = (display_name or "").strip() or meta.name or parsed["skill_id"] + ensure_ontology_build_valid( + db, + asset_type="skill", + name=name, + description=(summary or "").strip() or meta.description or "", + instructions=parsed["skill_content"], + tool_names=list(meta.allowed_tools or []), + ontology_tags=list(meta.tags or []), + ) + slug, action = upsert_admin_submission( + db, + skill_id=parsed["skill_id"], + skill_content=parsed["skill_content"], + display_name=name, + summary=(summary or "").strip() or meta.description or "", + category=category, + tags=list(meta.tags or []), + version=meta.version or "1.0.0", + extra_files=parsed["extra_files"], + note="管理员上传，免审核直接上架", + submitter_name=submitter_name, + ) + db.commit() + logger.info("marketplace_skill_admin_%s: skill=%s slug=%s", action, parsed["skill_id"], slug) + return { + "slug": slug, + "skill_id": parsed["skill_id"], + "display_name": name, + "category": skill_metadata.validate_category(category), + "stored_files": len([k for k in parsed["extra_files"] if k != skills_state.SECRETS_FILENAME]), + "skipped": parsed["skipped"], + "action": action, + "message": "技能已上架技能市场" if action == "published" else "技能市场内容已更新", + } + + +def publish_skill_to_marketplace( + db: Session, + *, + skill_id: str, + skill_content: str, + display_name: str, + description: str, + version: str, + tags: List[str], + category: str, + user_intro: Optional[str] = None, + submitter_name: str = "管理员新建", +) -> Dict[str, Any]: + """Admin "create skill" form → listed in the marketplace as approved (not in the catalog, not globally effective). + + ``skill_content`` is the complete SKILL.md assembled from the form fields + (valid frontmatter). + """ + name = (display_name or "").strip() or skill_id + try: + meta = load_skill_metadata_from_str(skill_content, skill_id) + allowed_tools = list(meta.allowed_tools or []) + parsed_description = meta.description or "" + except Exception: # Some reviewed legacy drafts have minimal frontmatter. + allowed_tools = [] + parsed_description = "" + ensure_ontology_build_valid( + db, + asset_type="skill", + name=name, + description=description or parsed_description, + instructions=skill_content, + tool_names=allowed_tools, + ontology_tags=list(tags or []), + ) + slug, action = upsert_admin_submission( + db, + skill_id=skill_id, + skill_content=skill_content, + display_name=name, + summary=(description or "").strip(), + category=category, + tags=list(tags or []), + version=version or "1.0.0", + extra_files={}, + note="管理员新建，免审核直接上架", + submitter_name=submitter_name, + user_intro=user_intro, + ) + db.commit() + logger.info("marketplace_skill_admin_%s: skill=%s slug=%s (form)", action, skill_id, slug) + return { + "slug": slug, + "skill_id": skill_id, + "display_name": name, + "category": skill_metadata.validate_category(category), + "action": action, + "message": "技能已上架技能市场" if action == "published" else "技能市场内容已更新", + } + + +def delete_marketplace_skill(db: Session, slug: str) -> Dict[str, Any]: + """Admin deletes a DB listing record from the skill marketplace (admin upload / user community listing). + + Preloaded (filesystem) skills ship with the repo and cannot be deleted + online — straight 400. Deletion only removes it from the market; it does not + affect any installed skill instances (installs are content snapshots). + """ + if skill_metadata.read_manifest(slug) is not None: + raise BadRequestError(message="预置技能随仓库发布，不可在线删除") + sub = ( + db.query(MarketplaceSubmission) + .filter(MarketplaceSubmission.slug == slug) + .first() + ) + if sub is None: + raise ResourceNotFoundError("marketplace_skill", slug) + skill_id, owner = sub.skill_id, sub.owner_user_id + db.delete(sub) + db.commit() + logger.info("marketplace_skill_deleted: slug=%s skill=%s owner=%s", slug, skill_id, owner) + return {"slug": slug, "deleted": True} diff --git a/src/backend/features/marketplace/skill_submission_data.py b/src/backend/features/marketplace/skill_submission_data.py new file mode 100644 index 00000000..50af4703 --- /dev/null +++ b/src/backend/features/marketplace/skill_submission_data.py @@ -0,0 +1,80 @@ +"""skill submission data responsibilities.""" +from __future__ import annotations +from core.capabilities.skills import installation as package_installation +import re +from typing import Any, Dict +from features.marketplace.models import MarketplaceSubmission +from sqlalchemy.orm import Session +from features.marketplace import skills_state + +def strip_injected_secrets(skill_content: str) -> str: + """Strip the "credentials configuration" section appended by _inject_secrets (including its preceding --- separator). + + Listing snapshots must be in the pre-injection form — the section instructs + reading secrets.json, which the snapshot has already excluded; the + installer's own credentials get a fresh section re-injected at install time. + """ + idx = skill_content.find(skills_state.SECRETS_SECTION_HEADER) + if idx < 0: + return skill_content + sep = skill_content.rfind("\n---", 0, idx) + cut = sep if sep >= 0 else idx + return skill_content[:cut].rstrip() + "\n" + + +def existing_slugs(db: Session) -> set: + """Currently occupied market slugs: the preloaded directory + all submissions. + + Must include rejected rows — the slug column is table-wide UNIQUE and + rejected submissions still occupy their slug; if derivation skipped them, + "re-submitting after rejection" would hit the unique constraint and 500 + outright. + """ + slugs = set() + if skills_state.MARKETPLACE_DIR.is_dir(): + slugs.update(c.name for c in skills_state.MARKETPLACE_DIR.iterdir() if c.is_dir()) + rows = db.query(MarketplaceSubmission.slug).all() + slugs.update(r[0] for r in rows) + return slugs + + +def derive_slug(db: Session, skill_id: str, owner_user_id: str) -> str: + """Derive the market slug from the source skill id: strip the user's own install-fingerprint suffix, then ensure global uniqueness.""" + base = skill_id + suffix = f"-{package_installation.owner_suffix(owner_user_id)}" + if base.endswith(suffix): + base = base[: -len(suffix)] + base = re.sub(r"[^a-z0-9_-]+", "-", base.lower()).strip("-") or "skill" + taken = existing_slugs(db) + slug = base + n = 2 + while slug in taken: + slug = f"{base}-{n}" + n += 1 + return slug + + +def submission_to_dict(sub: MarketplaceSubmission, *, with_content: bool = False) -> Dict[str, Any]: + data = { + "submission_id": sub.submission_id, + "slug": sub.slug, + "skill_id": sub.skill_id, + "owner_user_id": sub.owner_user_id, + "submitter_name": sub.submitter_name or "", + "display_name": sub.display_name, + "summary": sub.summary or "", + "user_intro": sub.user_intro or "", + "category": sub.category or "社区共享", + "tags": list(sub.tags or []), + "version": sub.version or "1.0.0", + "note": sub.note or "", + "status": sub.status, + "review_note": sub.review_note or "", + "reviewed_at": sub.reviewed_at.isoformat() if sub.reviewed_at else None, + "created_at": sub.created_at.isoformat() if sub.created_at else None, + "file_count": len(sub.extra_files or {}), + } + if with_content: + data["instructions"] = package_installation.strip_frontmatter(sub.skill_content or "") + data["files"] = sorted((sub.extra_files or {}).keys()) + return data diff --git a/src/backend/features/marketplace/skill_submissions.py b/src/backend/features/marketplace/skill_submissions.py new file mode 100644 index 00000000..b64e3cbe --- /dev/null +++ b/src/backend/features/marketplace/skill_submissions.py @@ -0,0 +1,190 @@ +"""skill submissions responsibilities.""" +from __future__ import annotations +import json +import uuid +from typing import Any, Dict, List, Optional +from core.db.models import AdminSkill +from features.marketplace.models import MarketplaceSubmission +from core.infra.exceptions import BadRequestError, ResourceNotFoundError +from core.infra.time import utc_now +from fastapi import HTTPException +from sqlalchemy.orm import Session +import logging +logger = logging.getLogger(__name__) +from features.marketplace import skill_metadata +from features.marketplace import skill_submission_data +from features.marketplace import skills_state + +def submit_to_marketplace( + db: Session, + skill_id: str, + *, + owner_user_id: str, + submitter_name: str = "", + note: str = "", + category: str = "", + summary: str = "", +) -> Dict[str, Any]: + """User submits their private skill for marketplace listing (content snapshot, pending admin review).""" + row = ( + db.query(AdminSkill) + .filter(AdminSkill.skill_id == skill_id, AdminSkill.owner_user_id == owner_user_id) + .first() + ) + if row is None: + raise ResourceNotFoundError("skill", skill_id) + + active = ( + db.query(MarketplaceSubmission) + .filter( + MarketplaceSubmission.skill_id == skill_id, + MarketplaceSubmission.owner_user_id == owner_user_id, + MarketplaceSubmission.status.in_(["pending", "approved"]), + ) + .first() + ) + if active is not None: + state = "待审核" if active.status == "pending" else "已上架" + raise HTTPException(status_code=409, detail=f"该技能已有{state}的上架申请") + + # The snapshot excludes the credentials file: when installing a market + # skill with secrets, _inject_secrets writes the user's API key in + # plaintext into extra_files[SECRETS_FILENAME] — that must never be + # distributed to all installers with the listing snapshot. The credential + # field names are kept with the snapshot as a sentinel entry (without + # values), so installers are prompted to enter their own keys; SKILL.md is + # correspondingly stripped of the injected section. + snapshot_files = dict(row.extra_files or {}) + submitter_secrets = snapshot_files.pop(skills_state.SECRETS_FILENAME, None) + snapshot_content = row.skill_content + if submitter_secrets: + snapshot_content = skill_submission_data.strip_injected_secrets(row.skill_content or "") + try: + secret_keys = sorted(json.loads(submitter_secrets).keys()) + except (TypeError, ValueError, AttributeError): + secret_keys = [] + if secret_keys: + # The original required/label metadata is unavailable in the installed artifact; approximate as "all required" + snapshot_files[skills_state.REQUIRED_SECRETS_SNAPSHOT] = json.dumps( + [{"key": k, "label": k, "required": True} for k in secret_keys], + ensure_ascii=False, + ) + + now = utc_now() + sub = MarketplaceSubmission( + submission_id=f"mksub_{uuid.uuid4().hex[:16]}", + slug=skill_submission_data.derive_slug(db, skill_id, owner_user_id), + skill_id=skill_id, + owner_user_id=owner_user_id, + submitter_name=(submitter_name or "").strip(), + display_name=row.display_name or skill_id, + summary=(summary or "").strip() or (row.description or ""), + user_intro=str(row.user_intro or "").strip() or None, + category=skill_metadata.validate_category(category), + tags=list(row.tags or []), + version=row.version or "1.0.0", + note=(note or "").strip(), + skill_content=snapshot_content, + extra_files=snapshot_files, + status="pending", + created_at=now, + updated_at=now, + ) + db.add(sub) + db.commit() + logger.info( + "marketplace_submission_created: id=%s skill=%s owner=%s slug=%s", + sub.submission_id, skill_id, owner_user_id, sub.slug, + ) + return skill_submission_data.submission_to_dict(sub) + + +def list_my_submissions(db: Session, owner_user_id: str) -> List[Dict[str, Any]]: + """All of the current user's listing submissions (newest → oldest).""" + rows = ( + db.query(MarketplaceSubmission) + .filter(MarketplaceSubmission.owner_user_id == owner_user_id) + .order_by(MarketplaceSubmission.created_at.desc()) + .all() + ) + return [skill_submission_data.submission_to_dict(r) for r in rows] + + +def withdraw_submission(db: Session, submission_id: str, owner_user_id: str) -> None: + """User withdraws their own submission: pending is deleted directly; approved requires an admin to delist; rejected can be deleted and resubmitted.""" + sub = ( + db.query(MarketplaceSubmission) + .filter( + MarketplaceSubmission.submission_id == submission_id, + MarketplaceSubmission.owner_user_id == owner_user_id, + ) + .first() + ) + if sub is None: + raise ResourceNotFoundError("marketplace_submission", submission_id) + if sub.status == "approved": + raise BadRequestError(message="技能已上架，如需下架请联系管理员") + db.delete(sub) + db.commit() + + +def list_submissions(db: Session, status: Optional[str] = None) -> List[Dict[str, Any]]: + """Admin view: all listing submissions, filterable by status (pending first, newest → oldest).""" + q = db.query(MarketplaceSubmission) + if status: + q = q.filter(MarketplaceSubmission.status == status) + rows = q.order_by(MarketplaceSubmission.created_at.desc()).all() + order = {"pending": 0, "approved": 1, "rejected": 2} + rows.sort(key=lambda r: order.get(r.status, 3)) + return [skill_submission_data.submission_to_dict(r) for r in rows] + + +def get_submission(db: Session, submission_id: str) -> Dict[str, Any]: + """Submission detail for admin review (including a SKILL.md body preview and the attachment list).""" + sub = ( + db.query(MarketplaceSubmission) + .filter(MarketplaceSubmission.submission_id == submission_id) + .first() + ) + if sub is None: + raise ResourceNotFoundError("marketplace_submission", submission_id) + return skill_submission_data.submission_to_dict(sub, with_content=True) + + +def review_submission( + db: Session, + submission_id: str, + *, + approve: bool, + review_note: str = "", + category: str = "", +) -> Dict[str, Any]: + """Admin review: approval means listing (appears in the market list); rejecting pending = decline, rejecting approved = delist.""" + sub = ( + db.query(MarketplaceSubmission) + .filter(MarketplaceSubmission.submission_id == submission_id) + .first() + ) + if sub is None: + raise ResourceNotFoundError("marketplace_submission", submission_id) + # State-machine guard: approve only allows pending→approved (prevents two + # concurrent admins / a stale list mis-click from listing a just-rejected + # submission directly); reject allows pending (decline) and approved + # (delist). + if approve and sub.status != "pending": + state = "已上架" if sub.status == "approved" else "已驳回" + raise HTTPException(status_code=409, detail=f"该申请{state}，不能重复审核通过（如需重新上架请让用户重新提交）") + if not approve and sub.status == "rejected": + raise HTTPException(status_code=409, detail="该申请已是驳回状态") + sub.status = "approved" if approve else "rejected" + sub.review_note = (review_note or "").strip() + if approve and (category or "").strip(): + sub.category = skill_metadata.validate_category(category) + sub.reviewed_at = utc_now() + sub.updated_at = utc_now() + db.commit() + logger.info( + "marketplace_submission_reviewed: id=%s slug=%s status=%s", + sub.submission_id, sub.slug, sub.status, + ) + return skill_submission_data.submission_to_dict(sub) diff --git a/src/backend/core/services/marketplace_skill_versions.py b/src/backend/features/marketplace/skill_versions.py similarity index 72% rename from src/backend/core/services/marketplace_skill_versions.py rename to src/backend/features/marketplace/skill_versions.py index f624a3bc..0776627f 100644 --- a/src/backend/core/services/marketplace_skill_versions.py +++ b/src/backend/features/marketplace/skill_versions.py @@ -1,36 +1,38 @@ """Version-aware sources for the existing skill marketplace API.""" from __future__ import annotations +from typing import Any, Dict, Optional +from sqlalchemy.orm import Session +from core.capabilities.skills.binary_files import is_binary_value +from core.capabilities.skills.cache_refresh import refresh_skill_caches +from core.capabilities.skills.deps.detector import detect_dependencies +from core.capabilities.skills.registry import load_skill_metadata_from_str +from core.db.models import AdminSkill +from core.infra.exceptions import BadRequestError +from core.infra.time import utc_now +from core.ontology.build_validator import ensure_ontology_build_valid +from fastapi import HTTPException +import features.marketplace.skill_catalog as market_owner_skill_catalog +from features.marketplace.skill_installation import build_user_intro as _build_user_intro +import features.marketplace.skill_installation as market_owner_skill_installation +from features.marketplace.skills_state import DEFAULT_SKILL_MARKET +from features.marketplace.skills_state import MANIFEST_NAME +import features.marketplace.skills_state as market_owner_skills_state +from sqlalchemy.orm.attributes import flag_modified +import logging +logger = logging.getLogger(__name__) + + +import features.marketplace.versions.store as store +from core.capabilities.skills.installation import inject_secrets as _inject_secrets, rewrite_frontmatter_name as _rewrite_frontmatter_name, compute_install_id as compute_install_id + + + -from . import marketplace_service as original -from . import marketplace_version_store as store -from .marketplace_service import ( - DEFAULT_SKILL_MARKET, - MANIFEST_NAME, - AdminSkill, - BadRequestError, - HTTPException, - _build_user_intro, - _inject_secrets, - _load_skill_metadata_from_str, - _rewrite_frontmatter_name, - compute_install_id, - detect_dependencies, - ensure_ontology_build_valid, - flag_modified, - is_binary_value, - logger, - refresh_skill_caches, - utc_now, -) - - -def __getattr__(name): - return getattr(original, name) def get_marketplace_skill(slug, db=None): - native = original.get_marketplace_skill(slug, db) + native = market_owner_skill_catalog.get_marketplace_skill(slug, db) return store.detail(db, "skill", slug, native) @@ -47,16 +49,16 @@ def install_marketplace_skill( directory, fall back to the community listing (approved submission), with install content taken from the submission-time snapshot. """ - current = original.get_marketplace_skill(slug, db) + current = market_owner_skill_catalog.get_marketplace_skill(slug, db) snapshot = store.active(db, "skill", slug) if not snapshot: - return original.install_marketplace_skill( + return market_owner_skill_installation.install_marketplace_skill( db, slug, owner_user_id=owner_user_id, secrets=secrets ) if current.get("builtin"): raise BadRequestError(message="内置技能已全局可用，无需安装") secrets = secrets or {} - from core.agent_skills.binary_files import encode_upload + from core.capabilities.skills.binary_files import encode_upload files = store.decode(snapshot) m = store.detail(db, "skill", slug, current) @@ -64,7 +66,7 @@ def install_marketplace_skill( extra_files = { n: encode_upload(n, v) for n, v in files.items() - if n not in {"SKILL.md", MANIFEST_NAME, original.REQUIRED_SECRETS_SNAPSHOT} + if n not in {"SKILL.md", MANIFEST_NAME, market_owner_skills_state.REQUIRED_SECRETS_SNAPSHOT} } entry_name = str(m["entry_name"]).strip() @@ -76,7 +78,7 @@ def install_marketplace_skill( skill_content = _inject_secrets(skill_content, extra_files, required_secrets, secrets) try: - meta = _load_skill_metadata_from_str(skill_content, install_id) + meta = load_skill_metadata_from_str(skill_content, install_id) except Exception as exc: # noqa: BLE001 raise BadRequestError(message=f"市场技能 SKILL.md 不合法：{exc}") @@ -146,11 +148,7 @@ def install_marketplace_skill( action = "installed" db.commit() try: - from core.services.skill_icon_service import ( - get_skill_icon, - preset_for_category, - set_skill_icon, - ) + from core.capabilities.skills.icons import get_skill_icon, preset_for_category, set_skill_icon if not get_skill_icon(db, install_id): set_skill_icon(db, install_id, preset_for_category(m.get("category"))) diff --git a/src/backend/features/marketplace/skills_state.py b/src/backend/features/marketplace/skills_state.py new file mode 100644 index 00000000..ed1ce54a --- /dev/null +++ b/src/backend/features/marketplace/skills_state.py @@ -0,0 +1,56 @@ +"""skills state responsibilities.""" +from __future__ import annotations +from extensions.paths import SKILLS_DIR + +MARKETPLACE_DIR = SKILLS_DIR / "marketplace" + + +DEFAULT_SKILLS_DIR = SKILLS_DIR / "default" + + +MANIFEST_NAME = "marketplace.json" + + +SKILL_MD_NAME = "SKILL.md" + + +SECRETS_FILENAME = "secrets.json" + + +DEFAULT_SKILL_MARKET = { + "pdf-editing": {"category": "文档处理", "summary": "生成 / 合并 / 拆分 / 抽页 / 填表单 / 读取 PDF 文档"}, + "capability-guide-brief": {"category": "办公效率", "summary": "速答系统能力清单与使用指引"}, + # OfficeCLI 系列（上游 iOfficeAI/OfficeCLI，Apache 2.0）：靠沙盒预装的 + # ``officecli`` 单文件二进制驱动，是 Word / Excel / PPT 的**唯一**入口—— + # 自研的 word-editing / excel-editing / ppt-design 三个技能已于 2026-08 下线， + # 能力全部由这一系列承接。三个基座（docx / xlsx / pptx）之上叠场景层， + # 分工写在各自 SKILL.md 的 description 里，由技能选择器路由。 + "officecli-docx": {"category": "文档处理", "summary": "OfficeCLI 深度编辑 .docx：TOC / 域 / 脚注 / 公式 / 原生图表 / 批注修订 / schema 校验"}, + "officecli-xlsx": {"category": "办公效率", "summary": "OfficeCLI 深度编辑 .xlsx：透视表 / 迷你图 / 条件格式 / 数据验证 / 切片器"}, + "officecli-pptx": {"category": "办公效率", "summary": "OfficeCLI 深度编辑 .pptx：母版版式 / 连接线 / 形状级控制 / 逐页截图质检"}, + "officecli-word-form": {"category": "文档处理", "summary": "制作可填写 Word 表单：内容控件(SDT) + 复选框 + 邮件合并域 + 文档保护"}, + "officecli-academic-paper": {"category": "文档处理", "summary": "学术论文排版：APA / Chicago / IEEE / MLA 引用、编号公式、交叉引用、参考文献"}, + "officecli-data-dashboard": {"category": "数据分析", "summary": "CSV / 表格数据做成 Excel 数据看板：KPI 卡片 + 多图表 + 迷你图 + 条件格式"}, + "officecli-financial-model": {"category": "数据分析", "summary": "Excel 财务模型：三表 / DCF / LBO / 单位经济 / 敏感性分析，全活公式驱动"}, + "officecli-pitch-deck": {"category": "办公效率", "summary": "融资路演 PPT：种子轮 / A-C 轮 / SAFE / 可转债的投资人 deck"}, + "morph-ppt": {"category": "办公效率", "summary": "带 Morph 平滑转场动效的 PPT，自带 40 套成品设计风格包"}, + "morph-ppt-3d": {"category": "办公效率", "summary": "带 .glb 三维模型与电影级镜头运动的 Morph 动效 PPT"}, +} + + +MARKETPLACE_CATEGORIES = [ + "写作助手", "文档处理", "数据分析", "政策产业", + "营销创意", "法务合规", "办公效率", "研发效率", +] + + +SECRETS_SECTION_HEADER = "## 🔑 凭据配置（由安装者提供）" + + +REQUIRED_SECRETS_SNAPSHOT = "_required_secrets.json" + + +ADMIN_UPLOAD_OWNER = "__admin_upload__" + + +ZIP_SKIP_DIR_PARTS = {"__pycache__", ".git", ".svn", ".hg", "__MACOSX"} diff --git a/src/backend/features/marketplace/versions/__init__.py b/src/backend/features/marketplace/versions/__init__.py new file mode 100644 index 00000000..b3281e46 --- /dev/null +++ b/src/backend/features/marketplace/versions/__init__.py @@ -0,0 +1 @@ +"""versions package.""" diff --git a/src/backend/core/services/marketplace_version_archive.py b/src/backend/features/marketplace/versions/archive.py similarity index 100% rename from src/backend/core/services/marketplace_version_archive.py rename to src/backend/features/marketplace/versions/archive.py diff --git a/src/backend/core/services/marketplace_version_config.py b/src/backend/features/marketplace/versions/config.py similarity index 95% rename from src/backend/core/services/marketplace_version_config.py rename to src/backend/features/marketplace/versions/config.py index b2014fb0..42d48abf 100644 --- a/src/backend/core/services/marketplace_version_config.py +++ b/src/backend/features/marketplace/versions/config.py @@ -4,8 +4,8 @@ from pydantic import BaseModel, ConfigDict, Field -from .marketplace_version_archive import check_url, fail -from .marketplace_version_store import decode +from features.marketplace.versions.archive import check_url, fail +from features.marketplace.versions.store import decode class AgentPackage(BaseModel): diff --git a/src/backend/core/services/marketplace_version_sources.py b/src/backend/features/marketplace/versions/sources.py similarity index 84% rename from src/backend/core/services/marketplace_version_sources.py rename to src/backend/features/marketplace/versions/sources.py index 8ab138d1..ef972d8a 100644 --- a/src/backend/core/services/marketplace_version_sources.py +++ b/src/backend/features/marketplace/versions/sources.py @@ -1,24 +1,28 @@ +import features.marketplace.mcp_listing as market_owner_mcp_listing + +import features.marketplace.mcp_versions as market_owner_mcp_versions """Read and activate versioned marketplace sources, never installed copies.""" import json import tempfile from pathlib import Path -from core.agent_skills.binary_files import encode_upload -from core.agent_skills.registry import _load_skill_metadata_from_str, _split_frontmatter -from core.db.models import AdminSkill, McpMarketItem, McpMarketVersion +from core.capabilities.skills.binary_files import encode_upload +from core.capabilities.skills.registry import load_skill_metadata_from_str, split_frontmatter +from core.db.models import AdminSkill +from features.marketplace.models import McpMarketItem, McpMarketVersion from fastapi import HTTPException -from . import marketplace_version_store as store -from .marketplace_version_archive import fail, read_zip, skill_version, strip_wrapper +import features.marketplace.versions.store as store +from features.marketplace.versions.archive import fail, read_zip, skill_version, strip_wrapper def source_detail(db, kind, slug): - from core.plugins.management.details import get_plugin_detail + from core.capabilities.plugins.management.details import get_plugin_detail + + import features.marketplace.agent_versions as agents + import features.marketplace.skill_versions as skills - from . import marketplace_agent_versions as agents - from . import marketplace_skill_versions as skills - from . import mcp_marketplace_service as mcps if kind == "skill": return skills.get_marketplace_skill(slug, db) @@ -28,13 +32,13 @@ def source_detail(db, kind, slug): detail = get_plugin_detail(slug, db) detail["native_version"], detail["source_revision"] = store.native_plugin_source(db, slug) return detail - return mcps.get_market_item(db, slug, owner_user_id=None, viewer_user_id=None, admin=True) + return market_owner_mcp_listing.get_market_item(db, slug, owner_user_id=None, viewer_user_id=None, admin=True) def capture(db, kind, slug): - from . import marketplace_agent_versions as agents - from . import marketplace_export as export - from . import mcp_marketplace_service as mcps + import features.marketplace.agent_versions as agents + import features.marketplace.export as export + detail = source_detail(db, kind, slug) existing = store.active(db, kind, slug) @@ -52,7 +56,7 @@ def capture(db, kind, slug): } else: item = db.get(McpMarketItem, slug) - version = mcps._version_for_item(db, item) + version = market_owner_mcp_versions.version_for_item(db, item) config = { k: getattr(version, k) for k in ( @@ -75,7 +79,7 @@ def capture(db, kind, slug): def normalized_plugin(files): - from core.plugins.packaging.importer import normalize_plugin_dir + from core.capabilities.plugins.packaging.importer import normalize_plugin_dir with tempfile.TemporaryDirectory(prefix="market-version-") as directory: root = Path(directory) @@ -95,7 +99,7 @@ def prepare(db, kind, slug, raw, number): if kind == "skill": try: content = skill_version(files["SKILL.md"].decode(), number) - meta = _load_skill_metadata_from_str(content, detail["entry_name"]) + meta = load_skill_metadata_from_str(content, detail["entry_name"]) except (ValueError, KeyError, UnicodeError): fail("SKILL.md 格式无效") if meta.id != detail["entry_name"]: @@ -121,7 +125,7 @@ def prepare(db, kind, slug, raw, number): manifest["version"] = number files["marketplace.json"] = json.dumps(manifest, ensure_ascii=False).encode() detail.update( - instructions=_split_frontmatter(content)[1].strip(), + instructions=split_frontmatter(content)[1].strip(), summary=meta.description, tags=meta.tags, files=[ @@ -152,7 +156,7 @@ def prepare(db, kind, slug, raw, number): np = normalized_plugin(files) if np.slug != slug or np.dropped: fail("插件 ID 不一致或包内包含无效组件") - from core.plugins.management.details import _normalized_to_detail + from core.capabilities.plugins.management.details import normalized_to_detail as _normalized_to_detail detail.update(_normalized_to_detail(np)) detail["skills_count"] = len(np.skills) @@ -166,7 +170,7 @@ def prepare(db, kind, slug, raw, number): fail("配置 JSON 无效") if not isinstance(config, dict) or config.get("slug") != slug: fail("配置 slug 与当前市场资源不一致") - from .marketplace_version_config import validate + from features.marketplace.versions.config import validate config = validate(db, kind, config, old) config["version"] = number @@ -223,7 +227,7 @@ def activate(db, kind, slug, snapshot): if row and (row.owner_user_id is not None or row.source_plugin): raise HTTPException(409, "内置技能 ID 与其他资源冲突") content = store.decode(snapshot) - metadata = _load_skill_metadata_from_str(content["SKILL.md"].decode(), identity) + metadata = load_skill_metadata_from_str(content["SKILL.md"].decode(), identity) if row is None: row = AdminSkill(skill_id=identity, owner_user_id=None, is_enabled=True) db.add(row) @@ -248,13 +252,13 @@ def activate(db, kind, slug, snapshot): def connector_snapshot(db, slug, version_id): - from . import mcp_marketplace_service as mcps + item = db.get(McpMarketItem, slug) version = db.get(McpMarketVersion, version_id) if not item or not version or version.slug != slug: fail("所选历史版本不存在") - detail = mcps._item_dict(db, item, version, installed=False) + detail = market_owner_mcp_listing.item_dict(db, item, version, installed=False) config = { k: getattr(version, k) for k in ( @@ -281,7 +285,7 @@ def connector_snapshot(db, slug, version_id): def lock_source(db, kind, slug): - from core.db.models import AgentMarketSubmission, MarketplaceSubmission, PluginMarketPackage + from features.marketplace.models import AgentMarketSubmission, MarketplaceSubmission, PluginMarketPackage models = { "skill": MarketplaceSubmission, diff --git a/src/backend/core/services/marketplace_version_store.py b/src/backend/features/marketplace/versions/store.py similarity index 90% rename from src/backend/core/services/marketplace_version_store.py rename to src/backend/features/marketplace/versions/store.py index 2d784959..6abaf6e7 100644 --- a/src/backend/core/services/marketplace_version_store.py +++ b/src/backend/features/marketplace/versions/store.py @@ -1,3 +1,5 @@ +import features.marketplace.agent_catalog as market_owner_agent_catalog +import features.marketplace.skill_catalog as market_owner_skill_catalog """Version snapshots belonging to existing marketplace entries.""" import base64 @@ -31,13 +33,13 @@ def active(db, kind, slug): if not value: return None if kind in {"skill", "agent"}: - from . import agent_market_service as agents - from . import marketplace_service as skills + + native = ( - skills.get_marketplace_skill(slug, db) + market_owner_skill_catalog.get_marketplace_skill(slug, db) if kind == "skill" - else agents.get_agent_detail(db, slug) + else market_owner_agent_catalog.get_agent_detail(db, slug) ) if value.get("source_version", native["version"]) != native["version"]: return None @@ -49,7 +51,7 @@ def active(db, kind, slug): return None row = db.get(ContentBlock, PREFIX + value["active"]) if kind == "connector" and row: - from core.db.models import McpMarketItem, McpMarketVersion + from features.marketplace.models import McpMarketItem, McpMarketVersion item = db.get(McpMarketItem, slug) version = db.get(McpMarketVersion, item.latest_version_id) if item else None @@ -99,18 +101,18 @@ def revision_token(db, kind, slug, current): def native_plugin_source(db, slug): - from core.plugins.management.packages import _market_row - from core.plugins.packaging import sources + from core.capabilities.plugins.management.packages import market_row + from core.capabilities.plugins.packaging import sources - root = sources._resolve_plugin_dir(slug) + root = sources.resolve_plugin_dir(slug) if root is not None: - meta = sources._scan_native_manifest(root) + meta = sources.scan_native_manifest(root) if not meta: - from core.plugins.packaging.importer import normalize_plugin_dir + from core.capabilities.plugins.packaging.importer import normalize_plugin_dir return normalize_plugin_dir(root).version, None return meta.get("version") or "1.0.0", None - row = _market_row(db, slug) + row = market_row(db, slug) if row is None: from core.infra.exceptions import ResourceNotFoundError @@ -121,7 +123,7 @@ def native_plugin_source(db, slug): def versions(db, kind, slug, current): value = state(db, kind, slug) if kind == "connector": - from core.db.models import McpMarketItem, McpMarketVersion + from features.marketplace.models import McpMarketItem, McpMarketVersion rows = ( db.query(McpMarketVersion) @@ -181,13 +183,8 @@ def versions(db, kind, slug, current): def save(db, kind, slug, snapshot, current, expected, target=None): """Atomically compare source + pointer; immutable snapshots and active content.""" - from .marketplace_version_archive import fail - from .marketplace_version_sources import ( - activate, - connector_snapshot, - lock_source, - source_detail, - ) + from features.marketplace.versions.archive import fail + from features.marketplace.versions.sources import activate, connector_snapshot, lock_source, source_detail # Refresh ORM state after waiting for another publication to finish. pointer_id = POINTER + key(kind, slug) @@ -211,7 +208,7 @@ def save(db, kind, slug, snapshot, current, expected, target=None): raise HTTPException(409, "市场版本已变化，请刷新详情后重试") # Native publisher updates must also be retained before selecting an older snapshot. if kind != "connector" and (not old or old.get("source_version") != native_number): - from .marketplace_version_sources import capture + from features.marketplace.versions.sources import capture baseline = str(uuid.uuid4()) db.add( diff --git a/src/backend/features/sites/README.md b/src/backend/features/sites/README.md new file mode 100644 index 00000000..e072ea41 --- /dev/null +++ b/src/backend/features/sites/README.md @@ -0,0 +1,11 @@ +# 站点托管与发布 + +负责站点模型与仓储、静态内容服务、访问密码与速率限制、站点打包、本机来源发布，以及内置站点插件升级。设计选择工具由 extension.register_tools 注册；模型由 FULL 和 CE 安装清单加载，不依赖运行开关。 + +HTTP 路由保持原有路径、顺序和权限策略；sites_serve 仍在 API 应用的原位置挂载，其余路由通过中央路由表注册。Site 表结构与约束不变，CE 通过 edition.py 选择权限字段和访问策略。 + +公开入口包括 service、management、listing、packaging、repository 及各 API 路由。调用方直接依赖实际模块，不保留旧路径转发。依赖内核数据库、文件、项目权限和插件基础设施。 + +站点通过 extension.register_gateway_contributions 向桌面网关注册上传格式、接收实现和回执本地化；桌面网关不导入站点实现。sites → desktop 是单向依赖，用于桥接账号校验、能力修订和发布传输。HTTP 执行位置策略来自宿主接口，注册在 API/CLI 初始化阶段完成。 + +版本差异统一由 edition.py 选择 ORM 字段、访问策略及请求字段；CE 只覆盖此入口。 diff --git a/src/backend/features/sites/__init__.py b/src/backend/features/sites/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/src/backend/features/sites/__init__.py @@ -0,0 +1 @@ + diff --git a/src/backend/features/sites/api/__init__.py b/src/backend/features/sites/api/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/src/backend/features/sites/api/__init__.py @@ -0,0 +1 @@ + diff --git a/src/backend/api/routes/v1/internal_site_auth.py b/src/backend/features/sites/api/internal_site_auth.py similarity index 100% rename from src/backend/api/routes/v1/internal_site_auth.py rename to src/backend/features/sites/api/internal_site_auth.py diff --git a/src/backend/api/routes/v1/internal_sites.py b/src/backend/features/sites/api/internal_sites.py similarity index 94% rename from src/backend/api/routes/v1/internal_sites.py rename to src/backend/features/sites/api/internal_sites.py index 17e75689..d3f7e21e 100644 --- a/src/backend/api/routes/v1/internal_sites.py +++ b/src/backend/features/sites/api/internal_sites.py @@ -23,16 +23,14 @@ from typing import List, Optional, Tuple from core.infra.responses import success_response -from core.services.artifact_edition import personal_artifact_predicates -from core.services.site_access_policy import SitePublishScopeFields, site_scope_ref -from core.services.site_packaging import ( - BUILD_SOURCE_EXCLUDES, SITE_SOURCE_EXCLUDES, pack_and_fetch_dir, resolve_project_context, -) +from core.db.artifact_scope import personal_artifact_predicates +from features.sites.edition import SitePublishScopeFields, site_scope_ref +from features.sites.packaging import BUILD_SOURCE_EXCLUDES, SITE_SOURCE_EXCLUDES, pack_and_fetch_dir, resolve_project_context from fastapi import APIRouter, Header, HTTPException from pydantic import BaseModel, Field from starlette.concurrency import run_in_threadpool -from .internal_site_auth import _check_internal_token +from features.sites.api.internal_site_auth import _check_internal_token logger = logging.getLogger(__name__) @@ -74,8 +72,8 @@ def _resolve_target_site_id(project_id: str, user_id: str) -> str: if not project_id: return "" from core.db.engine import SessionLocal - from core.db.models import Site - from core.services.site_service import SiteService + from features.sites.models import Site + from features.sites.service import SiteService with SessionLocal() as db: rows = db.query(Site.site_id).filter( Site.project_id == project_id, Site.deleted_at.is_(None), @@ -95,7 +93,7 @@ def _mirror_files_to_project_folder( if not project_id or not files: return from core.db.engine import SessionLocal - from core.services.project_source import ProjectSourceService + from core.projects.source import ProjectSourceService with SessionLocal() as db: ProjectSourceService(db).write_files(project_id, user_id, files) @@ -114,12 +112,13 @@ def _ensure_project_for_site(site_id: str, user_id: str, title: str) -> Optional return None try: from core.db.engine import SessionLocal - from core.db.models import Project, Site - from core.services.project_service import ProjectService + from core.db.models import Project + from features.sites.models import Site + from core.projects.edition import ProjectService with SessionLocal() as db: site = db.query(Site).filter(Site.site_id == site_id).first() - from core.services.site_access_policy import site_management_permission + from features.sites.edition import site_management_permission if site is None or site_management_permission(db, site, user_id) not in ("edit", "admin"): return None # Already linked and the project still exists → use it directly @@ -182,7 +181,7 @@ def _project_root_has_package_json(project_id: str, user_id: str) -> bool: if not project_id: return False from core.db.engine import SessionLocal - from core.services.project_source import ProjectSourceService + from core.projects.source import ProjectSourceService with SessionLocal() as db: try: ProjectSourceService(db).read_bytes(project_id, user_id, "package.json") @@ -204,7 +203,7 @@ async def list_sites( if not user_id: return success_response(data={"error": "当前会话缺少用户身份，无法查询站点"}) - from core.services.site_listing import list_sites as collect + from features.sites.listing import list_sites as collect try: items = await run_in_threadpool(collect, user_id, body.chat_id or "", body.limit) @@ -219,10 +218,10 @@ async def publish( x_internal_token: Optional[str] = Header(None, alias="X-Internal-Token"), ): _check_internal_token(x_internal_token) - from core.services.desktop_cloud_bridge import bridge_enabled + from core.harness.registry.host_runtime import requires_remote_execution as bridge_enabled if bridge_enabled(): - from core.services.desktop_site_publish import forward_local_site_tool + from features.sites.desktop_publish import forward_local_site_tool return success_response( data=await forward_local_site_tool( @@ -234,8 +233,8 @@ async def publish( ) from core.llm.tools._common import resolve_sandbox_session - from core.llm.tools._paths import to_physical_path - from core.llm.tools._tool_helpers import _validate_workspace_path + from core.sandbox.workspace.resolution import to_physical_path + from core.llm.tools._tool_helpers import validate_workspace_path as _validate_workspace_path user_id = (body.user_id or "").strip() if not user_id: @@ -292,11 +291,11 @@ async def publish( from core.db.engine import SessionLocal from core.infra.exceptions import AppException - from core.services.site_service import SiteService + from features.sites.service import SiteService # Resolve authorization before touching a sandbox or storing any new bytes. from core.db.models import Project - from core.services.project_source import ProjectSourceService + from core.projects.source import ProjectSourceService team_source_excludes = frozenset( SITE_SOURCE_EXCLUDES + (BUILD_SOURCE_EXCLUDES if source_dir else ()) diff --git a/src/backend/api/routes/v1/local_site_sources.py b/src/backend/features/sites/api/local_site_sources.py similarity index 94% rename from src/backend/api/routes/v1/local_site_sources.py rename to src/backend/features/sites/api/local_site_sources.py index f0fbcbcf..70b0a90b 100644 --- a/src/backend/api/routes/v1/local_site_sources.py +++ b/src/backend/features/sites/api/local_site_sources.py @@ -4,7 +4,7 @@ from pydantic import BaseModel, Field from core.auth.backend import UserContext, get_current_user from core.infra.responses import success_response -from core.services import local_site_sources as sources +from features.sites import local_sources as sources router = APIRouter(prefix="/v1/local/site-sources", tags=["local"]) @@ -29,4 +29,3 @@ def list_sources(user: UserContext = Depends(get_current_user)): @router.post("/{site_id}/edit") def edit(site_id: str, user: UserContext = Depends(get_current_user)): return success_response(data=sources.open_editor(str(user.user_id), site_id)) - diff --git a/src/backend/api/routes/site_gate.py b/src/backend/features/sites/api/site_gate.py similarity index 100% rename from src/backend/api/routes/site_gate.py rename to src/backend/features/sites/api/site_gate.py diff --git a/src/backend/api/routes/v1/sites.py b/src/backend/features/sites/api/sites.py similarity index 95% rename from src/backend/api/routes/v1/sites.py rename to src/backend/features/sites/api/sites.py index bd18dc0f..0af4d784 100644 --- a/src/backend/api/routes/v1/sites.py +++ b/src/backend/features/sites/api/sites.py @@ -10,14 +10,9 @@ from core.auth.backend import UserContext, get_current_user from core.db.engine import get_db from core.infra.responses import paginated_response, success_response -from core.services.site_access_policy import ( - SiteUpdateScopeFields, - serialize_site_scope, - site_management_permission, - site_scope_ref, -) -from core.services.site_password import MAX_PASSWORD_LENGTH -from core.services.site_service import SiteService +from features.sites.edition import SiteUpdateScopeFields, serialize_site_scope, site_management_permission, site_scope_ref +from features.sites.password import MAX_PASSWORD_LENGTH +from features.sites.service import SiteService from fastapi import APIRouter, Depends, Query from pydantic import BaseModel, Field from sqlalchemy.orm import Session diff --git a/src/backend/api/routes/sites_serve.py b/src/backend/features/sites/api/sites_serve.py similarity index 96% rename from src/backend/api/routes/sites_serve.py rename to src/backend/features/sites/api/sites_serve.py index 08ccbdb9..0baa25f7 100644 --- a/src/backend/api/routes/sites_serve.py +++ b/src/backend/features/sites/api/sites_serve.py @@ -24,18 +24,11 @@ import logging from typing import Optional -from api.routes.site_gate import render_site_gate +from features.sites.api.site_gate import render_site_gate from core.db.engine import get_db -from core.db.repository import SiteRepository -from core.services.site_password import ( - ACCESS_COOKIE_NAME, - access_cookie_params, - check_access_password, - issue_access_token, - site_base_path, - verify_access_token, -) -from core.services.site_service import SiteService +from features.sites.repository import SiteRepository +from features.sites.password import ACCESS_COOKIE_NAME, access_cookie_params, check_access_password, issue_access_token, site_base_path, verify_access_token +from features.sites.service import SiteService from fastapi import APIRouter, Depends, HTTPException, Request from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse, Response from sqlalchemy.orm import Session @@ -156,7 +149,7 @@ async def site_api_preflight(slug: str, rest: str): async def site_unlock(slug: str, request: Request, db: Session = Depends(get_db)): """校验访问密码并下发解锁凭据（统一验证页提交到这里）。这是解锁入口，必须免闸。""" # 限流放在取站点之前：被挡下的尝试连一次 get_by_slug 都不该花。 - from core.services.site_rate_limit import count_attempt + from features.sites.rate_limit import count_attempt await count_attempt(_client_ip(request), slug, unlock=True) site, _ = await _load_authorized_site(slug, request, db, require_unlock=False) if not site.access_password_hash: diff --git a/src/backend/core/plugins/local/site_upgrade.py b/src/backend/features/sites/builtin_upgrade.py similarity index 83% rename from src/backend/core/plugins/local/site_upgrade.py rename to src/backend/features/sites/builtin_upgrade.py index 73979d25..7e3e3e84 100644 --- a/src/backend/core/plugins/local/site_upgrade.py +++ b/src/backend/features/sites/builtin_upgrade.py @@ -8,12 +8,12 @@ def upgrade_builtin_sites(db) -> int: from core.db.models import AdminMcpServer, AdminSkill, InstalledPlugin - from core.plugins.management import components as plugin_components - from core.plugins.management import projection as plugin_projection - from core.plugins.packaging import sources as plugin_sources - from core.plugins.packaging.importer import normalize_plugin_dir + from core.capabilities.plugins.management import components as plugin_components + from core.capabilities.plugins.management import projection as plugin_projection + from core.capabilities.plugins.packaging import sources as plugin_sources + from core.capabilities.plugins.packaging.importer import normalize_plugin_dir - bundle = plugin_sources._resolve_plugin_dir("sites") + bundle = plugin_sources.resolve_plugin_dir("sites") if bundle is None: return 0 normalized = normalize_plugin_dir(bundle) @@ -38,7 +38,7 @@ def upgrade_builtin_sites(db) -> int: ids = {key: list(value) for key, value in (install.component_ids or {}).items()} added_skills = [] siblings = { - sk.name: plugin_sources._make_skill_id("sites", sk.name, owner) + sk.name: plugin_sources.make_skill_id("sites", sk.name, owner) for sk in normalized.skills } for sk in normalized.skills: @@ -59,7 +59,7 @@ def upgrade_builtin_sites(db) -> int: enabled = ( row.is_enabled if row else any(item and item.is_enabled for item in existing_skills) ) - plugin_components._apply_skill( + plugin_components.apply_skill( db, sk, slug="sites", @@ -71,7 +71,7 @@ def upgrade_builtin_sites(db) -> int: sibling_ids=siblings, ) for mc in normalized.mcp: - sid = plugin_sources._make_server_id("sites", mc.name, owner) + sid = plugin_sources.make_server_id("sites", mc.name, owner) row = db.get(AdminMcpServer, sid) if ( sid not in ids.get("mcp", []) @@ -82,7 +82,7 @@ def upgrade_builtin_sites(db) -> int: continue # Only shipped tool instructions change: URLs, credentials, runtime # settings and enabled states remain exactly as the user configured. - row.tools_json = plugin_components._merge_tool_metadata( + row.tools_json = plugin_components.merge_tool_metadata( row.tools_json, list(mc.tools or []) ) # Retired shipped KV tools must not remain in the persisted catalog. @@ -107,18 +107,19 @@ def upgrade_builtin_sites(db) -> int: if changed_owners: db.commit() for owner in set(changed_owners): - plugin_projection._refresh_after_change(owner) + plugin_projection.refresh_after_change(owner) return len(changed_owners) def _refresh_existing_projection(db, install, added_skills): """Update only the already-owned device snapshot, preserving enablement and edges.""" - from core.capabilities.paths import capabilities_enabled, revision_for_hash + from core.harness.registry.run_capabilities import has_run_capabilities + from features.desktop.capabilities.paths import revision_for_hash - if not capabilities_enabled(): + if not has_run_capabilities(): return - from core.capabilities import plugins, registry, store - from core.services.desktop_capability_protocol import entity_content_hash + from features.desktop.capabilities import plugins, registry, store + from core.capabilities.manifest_protocol import entity_content_hash current = registry.get("plugin:local:sites", db=db) if ( diff --git a/src/backend/core/services/desktop_site_publish.py b/src/backend/features/sites/desktop_publish.py similarity index 83% rename from src/backend/core/services/desktop_site_publish.py rename to src/backend/features/sites/desktop_publish.py index 82913cef..4b537f77 100644 --- a/src/backend/core/services/desktop_site_publish.py +++ b/src/backend/features/sites/desktop_publish.py @@ -6,7 +6,7 @@ import json import tarfile -from core.services.site_access_policy import SitePublishScopeFields, site_scope_ref +from features.sites.edition import SitePublishScopeFields, site_scope_ref from pydantic import Field MAX_ARCHIVE_BYTES = 40 * 1024 * 1024 @@ -32,10 +32,10 @@ def _session_workspace_root(chat_id: str | None) -> str | None: async def package_local_site(arguments, headers): """Read the current local sandbox, never a path on the remote backend.""" from core.config.local_mode import local_mode_enabled - from core.llm.tools._paths import to_physical_path - from core.llm.tools._tool_helpers import _validate_workspace_path - from core.services.site_packaging import pack_and_fetch_dir, resolve_project_context - from core.services.site_service import SiteService + from core.sandbox.workspace.resolution import to_physical_path + from core.llm.tools._tool_helpers import validate_workspace_path as _validate_workspace_path + from features.sites.packaging import pack_and_fetch_dir, resolve_project_context + from features.sites.service import SiteService headers = {k.lower(): v for k, v in headers.items()} user_id = headers.get("x-current-user-id", "") @@ -84,7 +84,7 @@ async def package_local_site(arguments, headers): if source and to_physical_path(source, user_id, session_id=chat_id) == src: raise ValueError("src_dir 必须指向构建产物，不能与 source_dir 相同") if local_mode_enabled() and chat_id: - from core.services.local_site_sources import validate_source + from features.sites.local_sources import validate_source source_path = to_physical_path(source, user_id, session_id=chat_id) if source else src arguments["_desktop_source"] = validate_source(user_id, chat_id, source_path, src) @@ -114,7 +114,7 @@ def localize_site_result(data, cloud_base, arguments=None, headers=None): published = json.loads(block["text"]) context = (arguments or {}).get("_desktop_source") if context and published.get("ok") and published.get("site_id"): - from core.services.local_site_sources import save_receipt + from features.sites.local_sources import save_receipt normalized = {k.lower(): v for k, v in (headers or {}).items()} from fastapi import HTTPException @@ -149,8 +149,8 @@ def publish_uploaded_site(user_id, data, options): Source files remain in the desktop project; only build output is uploaded. """ from core.db.engine import SessionLocal - from core.services.site_packaging import safe_extract_tar - from core.services.site_service import SiteService + from features.sites.packaging import safe_extract_tar + from features.sites.service import SiteService files = safe_extract_tar(data) if SiteService.looks_like_source_tree(name for name, _ in files): @@ -188,14 +188,11 @@ def publish_uploaded_site(user_id, data, options): async def forward_local_site_tool(tool_name, arguments, *, user_id, chat_id=""): """Sites are cloud-hosted: a local site MCP's call goes to the cloud gateway plugin.""" - from core.llm.factory.tools.mcp_config import _inject_runtime_headers - from core.llm.mcp_pool import make_client - from core.services.desktop_cloud_bridge import ( - cloud_gateway_mcp_configs, - get_state, - require_current_account, - ) - from core.services.desktop_gateway_uploads import endpoint_plugin + from core.llm.factory.tools.mcp_config import inject_runtime_headers as _inject_runtime_headers + from core.llm.tools.cloud_manifest import manifest_tools + from features.desktop.bridge.gateway import cloud_gateway_mcp_configs + from features.desktop.bridge.identity import get_state, require_current_account + from core.harness.registry.transfer_channels import endpoint_plugin state = get_state() if state is None: @@ -213,23 +210,22 @@ async def forward_local_site_tool(tool_name, arguments, *, user_id, chat_id=""): channel_origin=None, reranker_enabled=False, ) - client = make_client(sid, configs[sid], is_stateful=False) - try: - tool = await client.get_tool(tool_name) - result = await tool(**arguments) - require_current_account(state) - if tool_name in {"manage_application", "publish_mcp"}: - data = {"content": [block.model_dump() for block in result.content]} - localize_application_result(data, state["cloud_base"]) - return json.loads( - next(block["text"] for block in data["content"] if block["type"] == "text") - ) - for block in result.content: - if getattr(block, "type", None) == "text": - return json.loads(block.text) - return {"error": f"云端 {tool_name} 结果未知，请在云端站点列表核对"} - finally: - await client.close() + tools = manifest_tools(sid, configs[sid]) + tool = next((t for t in tools if t.name == tool_name), None) + if tool is None: + return {"error": "云端站点工具不可用"} + result = await tool(**arguments) + require_current_account(state) + if tool_name in {"manage_application", "publish_mcp"}: + data = {"content": [block.model_dump() for block in result.content]} + localize_application_result(data, state["cloud_base"]) + return json.loads( + next(block["text"] for block in data["content"] if block["type"] == "text") + ) + for block in result.content: + if getattr(block, "type", None) == "text": + return json.loads(block.text) + return {"error": f"云端 {tool_name} 结果未知，请在云端站点列表核对"} return {"error": "云端站点能力不可用，请启用云端站点插件后重试"} diff --git a/src/backend/features/sites/edition.py b/src/backend/features/sites/edition.py new file mode 100644 index 00000000..e097dcf1 --- /dev/null +++ b/src/backend/features/sites/edition.py @@ -0,0 +1,10 @@ +"""Edition policy selected by the installed distribution.""" + +def __getattr__(name): + if name in ('SiteUpdateScopeFields', 'SitePublishScopeFields', 'resolve_site_scope', 'site_scope_write_fields', 'site_scope_ref', 'serialize_site_scope', 'can_view_site', 'site_management_permission', 'site_listing_predicate'): + import features.sites.personal_access as provider + return getattr(provider, name) + if name in ('SiteScopeMixin', 'site_scope_table_args'): + import features.sites.personal_scope as provider + return getattr(provider, name) + raise AttributeError(name) diff --git a/src/backend/features/sites/extension.py b/src/backend/features/sites/extension.py new file mode 100644 index 00000000..3ec506e8 --- /dev/null +++ b/src/backend/features/sites/extension.py @@ -0,0 +1,34 @@ +"""Site design tool contribution; hosting routes retain central ordering.""" +NAME = "sites" + + +def register_tools(): + from features.sites.tools.design_picker import register_native_contribution + register_native_contribution() + + +def setup(): + register_tools() + return True + + +def register_gateway_contributions(): + from core.harness.registry.transfer_channels import UploadChannel, UploadReceiver, register_upload_channel, register_result_localizer + from features.sites import desktop_publish + register_upload_channel( + "sites", "publish_site", + lambda: UploadChannel("site-publish", "application/gzip", desktop_publish.package_local_site), + receiver=UploadReceiver( + desktop_publish.SiteUploadOptions, desktop_publish.MAX_ARCHIVE_BYTES, + lambda *args: desktop_publish.publish_uploaded_site(*args), + ), + ) + for name, method in { + "publish_site": "localize_site_result", + "manage_application": "localize_application_result", + "publish_mcp": "localize_application_result", + }.items(): + register_result_localizer( + "sites", name, + lambda *args, _method=method, **kwargs: getattr(desktop_publish, _method)(*args, **kwargs), + ) diff --git a/src/backend/core/services/site_listing.py b/src/backend/features/sites/listing.py similarity index 94% rename from src/backend/core/services/site_listing.py rename to src/backend/features/sites/listing.py index f81784e6..7827badf 100644 --- a/src/backend/core/services/site_listing.py +++ b/src/backend/features/sites/listing.py @@ -46,7 +46,7 @@ def _chat_project_id(user_id: str, chat_id: str) -> str: def _local_entries(user_id: str) -> List[dict]: - from core.services.local_site_sources import current_cloud, list_sources + from features.sites.local_sources import current_cloud, list_sources from fastapi import HTTPException cloud, subject = current_cloud() @@ -78,8 +78,8 @@ def _local_entries(user_id: str) -> List[dict]: def _cloud_entries(user_id: str) -> List[dict]: from core.db.engine import SessionLocal - from core.services.site_access_policy import site_management_permission - from core.services.site_service import SiteService + from features.sites.edition import site_management_permission + from features.sites.service import SiteService with SessionLocal() as db: sites, _ = SiteService(db).list_sites(user_id, page=1, page_size=MAX_LIMIT) @@ -129,7 +129,7 @@ def _cloud_source_dir(db, project_id: Any) -> str: project = _live_project(db, project_id) if project is None: return "" - from core.services.edition_workspace import source_directory + from core.files.edition import source_directory directory = source_directory(project) if directory: return directory diff --git a/src/backend/core/services/local_site_sources.py b/src/backend/features/sites/local_sources.py similarity index 93% rename from src/backend/core/services/local_site_sources.py rename to src/backend/features/sites/local_sources.py index a0a29ad5..b8076329 100644 --- a/src/backend/core/services/local_site_sources.py +++ b/src/backend/features/sites/local_sources.py @@ -21,9 +21,9 @@ def require_local(): def prepare_project(user_id: str, title: str, project_id: str = "") -> dict: require_local() from core.db.engine import SessionLocal - from core.services.project_source import ProjectSourceService - from core.services.project_service import ProjectService - from core.llm.tools._paths import WORKSPACE_ROOT + from core.projects.source import ProjectSourceService + from core.projects.edition import ProjectService + from core.sandbox.workspace.resolution import WORKSPACE_ROOT with SessionLocal() as db: if project_id: @@ -49,8 +49,8 @@ def prepare_project(user_id: str, title: str, project_id: str = "") -> dict: def current_cloud() -> tuple[str, str]: - from core.services.desktop_cloud_bridge import get_state - from core.services.desktop_capability_protocol import token_subject + from features.desktop.bridge.identity import get_state + from core.capabilities.manifest_protocol import token_subject state = get_state() or {} return ( @@ -61,7 +61,7 @@ def current_cloud() -> tuple[str, str]: def _entries(db, user_id): from core.db.models import Project - from core.auth.permissions_iface import resolve_project_permission + from core.auth.edition import resolve_project_permission cloud, subject = current_cloud() if not cloud or not subject: @@ -92,7 +92,7 @@ def _entries(db, user_id): continue result.append({**saved, "project_id": project.project_id, "project_name": project.name}) from core.db.models import ChatSession - from core.services.desktop_site_publish import _session_workspace_root + from features.sites.desktop_publish import _session_workspace_root chats = db.query(ChatSession).filter( ChatSession.user_id == user_id, @@ -130,7 +130,7 @@ def project_sources(user_id: str, project_id: str) -> list[dict]: """Read this project's current-account receipts without selecting an edit target.""" require_local() from core.db.engine import SessionLocal - from core.services.project_source import ProjectSourceService + from core.projects.source import ProjectSourceService with SessionLocal() as db: project = ProjectSourceService(db).authorized_project(project_id, user_id, write=False) @@ -153,7 +153,7 @@ def save_receipt(user_id: str, chat_id: str, context: dict, published: dict, clo require_local() from core.db.engine import SessionLocal from core.db.models import ChatSession - from core.services.project_source import ProjectSourceService + from core.projects.source import ProjectSourceService cloud, subject = current_cloud() if cloud != cloud_base.rstrip("/") or not subject: @@ -163,7 +163,7 @@ def save_receipt(user_id: str, chat_id: str, context: dict, published: dict, clo if (validated.get("project_id") or "") != (context.get("project_id") or ""): raise ValueError("建站会话的项目已变化") with SessionLocal() as db: - from core.services.project_source import reserve_sqlite_writer + from core.projects.source import reserve_sqlite_writer reserve_sqlite_writer(db) project_id = context.get("project_id") or "" @@ -239,7 +239,7 @@ def open_editor(user_id: str, site_id: str) -> dict: require_local() from core.db.engine import SessionLocal from core.db.models import ChatSession - from core.services.project_source import reserve_sqlite_writer + from core.projects.source import reserve_sqlite_writer with SessionLocal() as db: reserve_sqlite_writer(db) @@ -295,8 +295,8 @@ def open_editor(user_id: str, site_id: str) -> dict: def validate_source(user_id: str, chat_id: str, source: str, publish_dir: str) -> dict: """Require existing source and output inside the local project or session workspace.""" require_local() - from core.services.site_packaging import resolve_project_context - from core.services.project_source import ProjectSourceService + from features.sites.packaging import resolve_project_context + from core.projects.source import ProjectSourceService from core.db.engine import SessionLocal from core.db.models import ChatSession @@ -313,7 +313,7 @@ def validate_source(user_id: str, chat_id: str, source: str, publish_dir: str) - raise ValueError("桌面站点必须使用本地项目") root = Path(project_dir).resolve() else: - from core.services.desktop_site_publish import _session_workspace_root + from features.sites.desktop_publish import _session_workspace_root root = Path(_session_workspace_root(chat_id) or "").resolve() paths = [Path(source), Path(publish_dir)] diff --git a/src/backend/core/services/site_management.py b/src/backend/features/sites/management.py similarity index 95% rename from src/backend/core/services/site_management.py rename to src/backend/features/sites/management.py index 8d0f4544..ef206b0c 100644 --- a/src/backend/core/services/site_management.py +++ b/src/backend/features/sites/management.py @@ -5,18 +5,12 @@ from datetime import datetime from typing import Any, Dict, List, Optional, Tuple -from core.db.models import Site +from features.sites.models import Site from core.infra.exceptions import BadRequestError, ResourceNotFoundError from core.infra.time import utc_now -from core.services.site_access_policy import ( - can_view_site, - resolve_site_scope, - site_listing_predicate, - site_management_permission, - site_scope_write_fields, -) -from core.services.site_password import hash_access_password, verify_access_token -from core.storage import get_storage +from features.sites.edition import can_view_site, resolve_site_scope, site_listing_predicate, site_management_permission, site_scope_write_fields +from features.sites.password import hash_access_password, verify_access_token +from core.files.storage import get_storage class SiteManagementMixin: diff --git a/src/backend/core/db/models/site.py b/src/backend/features/sites/models.py similarity index 97% rename from src/backend/core/db/models/site.py rename to src/backend/features/sites/models.py index 7d4245a9..62c01abd 100644 --- a/src/backend/core/db/models/site.py +++ b/src/backend/features/sites/models.py @@ -3,7 +3,7 @@ from datetime import datetime from core.db.engine import Base -from core.db.models.site_scope import SiteScopeMixin, site_scope_table_args +from features.sites.edition import SiteScopeMixin, site_scope_table_args from core.db.utc_datetime import UTCDateTime from core.infra.time import utc_now from sqlalchemy import ( diff --git a/src/backend/core/services/site_packaging.py b/src/backend/features/sites/packaging.py similarity index 92% rename from src/backend/core/services/site_packaging.py rename to src/backend/features/sites/packaging.py index d711815f..02b043b6 100644 --- a/src/backend/core/services/site_packaging.py +++ b/src/backend/features/sites/packaging.py @@ -6,6 +6,7 @@ """ from __future__ import annotations +import inspect import io import json @@ -30,9 +31,9 @@ def resolve_project_context(chat_id: str, user_id: str): from fastapi import HTTPException from core.db.engine import SessionLocal from core.db.models import ChatSession, Project - from core.auth.permissions_iface import resolve_project_permission - from core.services.project_scope import build_project_ctx - from core.llm.tools._paths import to_physical_path + from core.auth.edition import resolve_project_permission + from core.projects.edition import build_project_ctx + from core.sandbox.workspace.resolution import to_physical_path with SessionLocal() as db: chat = db.get(ChatSession, chat_id) @@ -66,7 +67,7 @@ def resolve_project_context(chat_id: str, user_id: str): folder = (ctx or {}).get("project_folder_name") if not folder: raise HTTPException(409, "源码项目未绑定有效空间文件夹") - from core.services.edition_workspace import source_directory + from core.files.edition import source_directory directory = source_directory(project) if directory: return project.project_id, directory @@ -85,11 +86,11 @@ async def pack_and_fetch_dir( from core.sandbox import ProcessRequest, get_sandbox_provider from core.sandbox import SandboxError, SandboxConnectError import core.sandbox.workspace.archive as directory_archive - from core.services.site_service import MAX_SITE_FILE_BYTES, MAX_SITE_TOTAL_BYTES + from features.sites.service import MAX_SITE_FILE_BYTES, MAX_SITE_TOTAL_BYTES provider = get_sandbox_provider() archive_name = f".__site_pack_{uuid.uuid4().hex}.tgz" - from core.llm.tools._paths import workspace_directory + from core.sandbox.workspace.resolution import workspace_directory import os working_directory = workspace_directory(_sess) @@ -104,7 +105,7 @@ async def pack_and_fetch_dir( "max_total_bytes": MAX_SITE_TOTAL_BYTES, "max_archive_bytes": MAX_PACK_BYTES, } - program = Path(directory_archive.__file__).read_text(encoding="utf-8") + program = inspect.getsource(directory_archive) request = ProcessRequest( script_content=program, params=options, @@ -153,7 +154,7 @@ def safe_extract_tar(data: bytes) -> List[Tuple[str, bytes]]: (guarding against symlink escape). Absolute paths and ``..`` traversal are re-checked by the service layer's normalize_rel_path. """ - from core.services.site_service import MAX_SITE_FILE_BYTES, MAX_SITE_TOTAL_BYTES + from features.sites.service import MAX_SITE_FILE_BYTES, MAX_SITE_TOTAL_BYTES if len(data) > MAX_PACK_BYTES: raise ValueError("站点发布包过大") diff --git a/src/backend/core/services/site_password.py b/src/backend/features/sites/password.py similarity index 100% rename from src/backend/core/services/site_password.py rename to src/backend/features/sites/password.py diff --git a/src/backend/core/services/site_access_policy.py b/src/backend/features/sites/personal_access.py similarity index 97% rename from src/backend/core/services/site_access_policy.py rename to src/backend/features/sites/personal_access.py index 47ef7cc6..e1217ed1 100644 --- a/src/backend/core/services/site_access_policy.py +++ b/src/backend/features/sites/personal_access.py @@ -57,5 +57,5 @@ def site_management_permission(_db, site, user_id: str) -> str: def site_listing_predicate(user_id: str): - from core.db.models import Site + from features.sites.models import Site return Site.user_id == user_id diff --git a/src/backend/core/db/models/site_scope.py b/src/backend/features/sites/personal_scope.py similarity index 100% rename from src/backend/core/db/models/site_scope.py rename to src/backend/features/sites/personal_scope.py diff --git a/src/backend/core/services/site_rate_limit.py b/src/backend/features/sites/rate_limit.py similarity index 100% rename from src/backend/core/services/site_rate_limit.py rename to src/backend/features/sites/rate_limit.py diff --git a/src/backend/core/db/repository/site.py b/src/backend/features/sites/repository.py similarity index 98% rename from src/backend/core/db/repository/site.py rename to src/backend/features/sites/repository.py index ba262c6a..1333991d 100644 --- a/src/backend/core/db/repository/site.py +++ b/src/backend/features/sites/repository.py @@ -3,7 +3,7 @@ from datetime import datetime from typing import Any, Dict, List, Optional, Tuple -from core.db.models import Site +from features.sites.models import Site from core.infra.time import utc_now from sqlalchemy import desc from sqlalchemy.orm import Session diff --git a/src/backend/core/services/site_service.py b/src/backend/features/sites/service.py similarity index 97% rename from src/backend/core/services/site_service.py rename to src/backend/features/sites/service.py index 68657e9d..9db6337b 100644 --- a/src/backend/core/services/site_service.py +++ b/src/backend/features/sites/service.py @@ -20,18 +20,13 @@ from datetime import datetime from typing import Any, Dict, List, Optional, Tuple -from core.db.models import Site -from core.db.repository import SiteRepository +from features.sites.models import Site +from features.sites.repository import SiteRepository from core.infra.exceptions import BadRequestError from core.infra.time import utc_now -from core.services.site_access_policy import ( - resolve_site_scope, - site_management_permission, - site_scope_ref, - site_scope_write_fields, -) -from core.services.site_management import SiteManagementMixin -from core.storage import get_storage +from features.sites.edition import resolve_site_scope, site_management_permission, site_scope_ref, site_scope_write_fields +from features.sites.management import SiteManagementMixin +from core.files.storage import get_storage from sqlalchemy.orm import Session logger = logging.getLogger(__name__) @@ -225,7 +220,7 @@ def publish( ) if project_id: - from core.auth.permissions_iface import resolve_project_permission + from core.auth.edition import resolve_project_permission from core.db.models import Project project = self.db.get(Project, project_id) diff --git a/src/backend/features/sites/tools/__init__.py b/src/backend/features/sites/tools/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/src/backend/features/sites/tools/__init__.py @@ -0,0 +1 @@ + diff --git a/src/backend/core/llm/tools/design_picker_tool.py b/src/backend/features/sites/tools/design_picker.py similarity index 86% rename from src/backend/core/llm/tools/design_picker_tool.py rename to src/backend/features/sites/tools/design_picker.py index 3d4cc175..e4519a83 100644 --- a/src/backend/core/llm/tools/design_picker_tool.py +++ b/src/backend/features/sites/tools/design_picker.py @@ -5,14 +5,15 @@ resumes in place once chosen. Conditionally registered by agent_factory only in site-building chats (site-builder skill enabled). """ +from core.llm.factory.tools.definitions import function_tool import logging from typing import Any, Dict, List, Optional from agentscope.tool import Toolkit -from agentscope.tool._response import ToolChunk as ToolResponse +from agentscope.tool._response import ToolChunk -from core.llm.tools import _tool_confirmation as _mc +from core.llm.tools import confirmation as _mc from core.llm.tools._common import resp_json logger = logging.getLogger(__name__) @@ -40,7 +41,7 @@ def skill_uses_choose_design(skill_id: str) -> bool: ) -def _err(msg: str) -> ToolResponse: +def _err(msg: str) -> ToolChunk: return resp_json({"ok": False, "error": msg}) @@ -61,7 +62,7 @@ def register_choose_design( instead of erroring out on a missing tool. """ - async def choose_design(question: str, options: List[Dict[str, Any]]) -> ToolResponse: + async def choose_design(question: str, options: List[Dict[str, Any]]) -> ToolChunk: """让用户从多个设计方案预览图中选择一个（**会挂起等待用户点选**）。 仅在建站流程中使用：先为同一站点做出 2-4 个（推荐 3 个）风格迥异的 @@ -97,7 +98,7 @@ async def choose_design(question: str, options: List[Dict[str, Any]]) -> ToolRes if not isinstance(options, list) or not (_MIN_OPTIONS <= len(options) <= _MAX_OPTIONS): return _err(f"options 必须是 {_MIN_OPTIONS}-{_MAX_OPTIONS} 个方案的列表（推荐 3 个）") - from core.artifacts.store import get_artifact + from core.files.artifacts.store import get_artifact normalized: List[Dict[str, Any]] = [] seen_ids: set[str] = set() @@ -114,8 +115,8 @@ async def choose_design(question: str, options: List[Dict[str, Any]]) -> ToolRes import asyncio from fastapi import HTTPException from core.infra.logging import user_id_var - from core.artifacts.local_reference import project_file_path - from core.artifacts.local_reference import reference_local_file + from core.files.artifacts.local_reference import project_file_path + from core.files.artifacts.local_reference import reference_local_file if not isinstance(image_path, str): return _err(f"options[{i}].image_path 必须为字符串") @@ -199,10 +200,11 @@ async def choose_design(question: str, options: List[Dict[str, Any]]) -> ToolRes } return resp_json(payload) - from core.llm.tool_permissions import ToolPermissionSpec, local_path_tool, READ + from core.sandbox.permissions.dispatch_models import ToolPermissionSpec, READ + from core.sandbox.permissions.tool_specs import local_path_tool def resolve_image_paths(args, runtime): - from core.artifacts.local_reference import project_file_path + from core.files.artifacts.local_reference import project_file_path intents = [] for option in args.get("options") or []: @@ -217,11 +219,29 @@ def resolve_image_paths(args, runtime): )) return intents - toolkit.register_tool_function( - choose_design, namesake_strategy="override", - permission=ToolPermissionSpec("design-preview-paths", resolve_image_paths), - ) + toolkit.append(function_tool(choose_design, dispatch_spec=ToolPermissionSpec("design-preview-paths", resolve_image_paths))) logger.info("[factory] Registered choose_design tool (site-builder session)") __all__ = ["register_choose_design", "skill_uses_choose_design"] + + +def design_tool_enabled(context): + return bool(context.skill_ids) and any( + skill_uses_choose_design(str(sid)) for sid in context.skill_ids + ) + + +async def install_design_tool(toolkit, context): + request = context.request + register_choose_design( + toolkit, chat_id=request.chat_id, + interactive=context.interactive and not context.is_channel_run and not request.automation_run, + sandbox_session_id=context.sandbox_session_id, + user_id=request.current_user_id, scope=context.project_scope, + ) + + +def register_native_contribution(): + from core.harness.registry import register_tool + register_tool("choose_design", install_design_tool, design_tool_enabled, stage="after_sandbox") diff --git a/src/backend/features/workbench/README.md b/src/backend/features/workbench/README.md new file mode 100644 index 00000000..ff79b980 --- /dev/null +++ b/src/backend/features/workbench/README.md @@ -0,0 +1,9 @@ +# 工作台 + +拥有应用目录、外部应用边界、插件界面声明、安装修订、授权租约、资源归属、后台观察及工作台配置。托管业务应用仍属于 hosted_apps；工作台配置继续存入 ContentBlock，不增加数据库表。 + +FULL 与 CE 均安装。API 在中央有序路由表注册，工作台内部路由组与配置保护保持原顺序。CE 的 migration_snapshot 位于相同功能路径，通过已有 overlay 保留版型差异。 + +依赖 core 的认证、通用插件及技能运行、数据存储和扩展接口；工作台通过宿主注册 PackageUpgradePolicy 与 ResourceOwner，core 不保留工作台实现转发。站点元数据依赖 features.sites。工作台依赖桌面安装登记、版本修订和能力绑定；界面声明投影由宿主注入，桌面不反向依赖工作台。 + +专属测试与 HTTP/进程观察夹具位于 tests/features/workbench。测试使用独立临时库和 .scratch 数据目录，不连接真实账号或用户数据。 diff --git a/src/backend/core/applications/__init__.py b/src/backend/features/workbench/__init__.py similarity index 100% rename from src/backend/core/applications/__init__.py rename to src/backend/features/workbench/__init__.py diff --git a/src/backend/core/applications/activity.py b/src/backend/features/workbench/activity.py similarity index 98% rename from src/backend/core/applications/activity.py rename to src/backend/features/workbench/activity.py index 35b412d0..702a49ce 100644 --- a/src/backend/core/applications/activity.py +++ b/src/backend/features/workbench/activity.py @@ -1,6 +1,6 @@ """Bounded, durable activity projection; reads never load a page or start a worker.""" -from core.applications.coordination import runtime +from features.workbench.coordination import runtime import re @@ -9,7 +9,7 @@ from hashlib import sha256 from uuid import uuid4 -from core.applications import catalog, external +from features.workbench import catalog, external from core.db.models import ContentBlock from core.infra.time import utc_now @@ -96,7 +96,7 @@ def register(db, user_id, key, instance_id, expected, module_id=None): def declared(item, db, user_id, module_id=None): if item["declaration"]["target"]["kind"] != "plugin": return set() - from core.plugins.resources.installation import resolve + from core.capabilities.plugins.resources.installation import resolve selected = resolve(db, item["binding"]["slug"], user_id, item["binding"]["installId"]) module = next( diff --git a/src/backend/features/workbench/api/__init__.py b/src/backend/features/workbench/api/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/src/backend/features/workbench/api/__init__.py @@ -0,0 +1 @@ + diff --git a/src/backend/api/routes/v1/plugin_ui.py b/src/backend/features/workbench/api/plugin_ui.py similarity index 93% rename from src/backend/api/routes/v1/plugin_ui.py rename to src/backend/features/workbench/api/plugin_ui.py index 88ce9993..f5bb2af5 100644 --- a/src/backend/api/routes/v1/plugin_ui.py +++ b/src/backend/features/workbench/api/plugin_ui.py @@ -27,9 +27,9 @@ from core.db.engine import get_db from core.infra.exceptions import ResourceNotFoundError from core.infra.responses import success_response -from core.plugins.ui import data_proxy as proxy -from core.plugins.ui import contributions as plugin_ui -from core.plugins.ui.contract import find_data_source, find_module +from features.workbench.ui import data_proxy as proxy +from features.workbench.ui import contributions as plugin_ui +from features.workbench.ui.contract import find_data_source, find_module router = APIRouter(prefix="/v1/plugins", tags=["Plugin UI"]) logger = logging.getLogger(__name__) @@ -63,7 +63,7 @@ def list_ui_contributions( definitions are projected down to id + parameter schema; the upstream URL and auth header stay on the server. """ - from core.capabilities import device_catalog + from features.desktop.capabilities import device_catalog # 同上：云端账号的插件不投影进来，混合模式下这些插件的面板就整片消失。 items = device_catalog.merge_items( @@ -127,9 +127,9 @@ def get_plugin_web_asset( def web_asset_response(db, slug, user_id, asset_path, install_id=None, *, _installation=None): """Serve authenticated or ticket-authorized static files with identical isolation.""" - from core.services.site_service import guess_site_mime + from features.sites.service import guess_site_mime - from core.plugins.resources.installation import resolve, safe_path + from core.capabilities.plugins.resources.installation import resolve, safe_path selected = _installation or resolve(db, slug, user_id, install_id) target = safe_path(selected.package / "web", asset_path) if not target.is_file(): diff --git a/src/backend/api/routes/v1/workbench.py b/src/backend/features/workbench/api/workbench.py similarity index 86% rename from src/backend/api/routes/v1/workbench.py rename to src/backend/features/workbench/api/workbench.py index e59267fd..5bf50fdf 100644 --- a/src/backend/api/routes/v1/workbench.py +++ b/src/backend/features/workbench/api/workbench.py @@ -1,18 +1,18 @@ """Workbench directory and ordinary external links; managed SQL/MCP apps stay separate.""" -from core.applications.coordination import configuration +from features.workbench.coordination import configuration from typing import Any from api.deps import _resolve_session_user_id, require_config -from api.routes.v1.workbench_auth import browser_write -from api.routes.v1.workbench_auth import router as auth_router -from core.applications.diagnostics import observed -from core.applications import catalog, external, external_auth, policy, origin -from core.applications.directory_revision import matches, snapshot -from core.applications.external_embed import EmbeddingPolicy, launch_policy, validate_policy -from core.applications.external_boundary import validate_cookie_scope +from features.workbench.api.workbench_auth import browser_write +from features.workbench.api.workbench_auth import router as auth_router +from features.workbench.diagnostics import observed +from features.workbench import catalog, external, external_auth, policy, origin +from features.workbench.directory_revision import matches, snapshot +from features.workbench.external_embed import EmbeddingPolicy, launch_policy, validate_policy +from features.workbench.external_boundary import validate_cookie_scope from core.auth.backend import UserContext, require_auth from core.config.settings import settings from core.db.engine import get_db @@ -205,7 +205,7 @@ def launch( if not item["availability"]["available"]: external.fail(409, item["availability"]["reason"]) if item["declaration"]["target"]["kind"] == "plugin": - from core.applications.plugin_catalog import launch as plugin_launch + from features.workbench.plugin_catalog import launch as plugin_launch return success_response(data=plugin_launch(db, user.user_id, item, moduleId)) if item["declaration"]["target"]["kind"] == "external": @@ -269,7 +269,7 @@ def catalog_directory( @router.get("/admin/catalog", dependencies=[Depends(require_config)]) def admin_catalog(response: Response, db: Session = Depends(get_db)): response.headers["Cache-Control"] = "no-store" - from core.applications.plugin_catalog import administrative_items + from features.workbench.plugin_catalog import administrative_items return success_response(data={"items": catalog.definitions(db) + administrative_items(db)}) @@ -291,7 +291,7 @@ async def update_policy( @router.get("/admin/legacy-config", dependencies=[Depends(require_config)]) def admin_legacy_config(response: Response, db: Session = Depends(get_db)): - from core.applications import migration_compat + from features.workbench import migration_compat response.headers["Cache-Control"] = "no-store" return success_response( @@ -304,39 +304,39 @@ def admin_legacy_config(response: Response, db: Session = Depends(get_db)): router.include_router(auth_router) -from api.routes.v1.workbench_preferences import routes as preference_routes +from features.workbench.api.workbench_preferences import routes as preference_routes router.include_router(preference_routes(WorkbenchRoute, require_json_write)) -from api.routes.v1.workbench_workspace import router as workspace_router +from features.workbench.api.workbench_workspace import router as workspace_router router.include_router(workspace_router) -from api.routes.v1.workbench_resources import routes as resource_routes +from features.workbench.api.workbench_resources import routes as resource_routes router.include_router(resource_routes(WorkbenchRoute, require_json_write)) -from api.routes.v1.workbench_state import routes as state_routes +from features.workbench.api.workbench_state import routes as state_routes router.include_router(state_routes(WorkbenchRoute, require_json_write)) -from api.routes.v1.workbench_capabilities import router as capability_router +from features.workbench.api.workbench_capabilities import router as capability_router router.include_router(capability_router) -from api.routes.v1.workbench_events import routes as event_routes +from features.workbench.api.workbench_events import routes as event_routes router.include_router(event_routes(WorkbenchRoute)) -from api.routes.v1.workbench_activity import routes as activity_routes -from api.routes.v1.workbench_diagnostics import routes as diagnostic_routes -from api.routes.v1.workbench_upgrades import routes as upgrade_routes -from api.routes.v1.workbench_revision_leases import routes as revision_routes +from features.workbench.api.workbench_activity import routes as activity_routes +from features.workbench.api.workbench_diagnostics import routes as diagnostic_routes +from features.workbench.api.workbench_upgrades import routes as upgrade_routes +from features.workbench.api.workbench_revision_leases import routes as revision_routes router.include_router(activity_routes(WorkbenchRoute, require_json_write)) router.include_router(diagnostic_routes(WorkbenchRoute, require_json_write)) router.include_router(upgrade_routes(WorkbenchRoute, require_json_write)) router.include_router(revision_routes(WorkbenchRoute, require_json_write)) -from api.routes.v1.workbench_application_config import router as application_config_router +from features.workbench.api.workbench_application_config import router as application_config_router router.include_router(application_config_router) diff --git a/src/backend/api/routes/v1/workbench_activity.py b/src/backend/features/workbench/api/workbench_activity.py similarity index 98% rename from src/backend/api/routes/v1/workbench_activity.py rename to src/backend/features/workbench/api/workbench_activity.py index f3c7409b..8c6df14a 100644 --- a/src/backend/api/routes/v1/workbench_activity.py +++ b/src/backend/features/workbench/api/workbench_activity.py @@ -6,7 +6,7 @@ from pydantic import BaseModel, ConfigDict, Field from sqlalchemy.orm import Session -from core.applications import activity, background +from features.workbench import activity, background from core.auth.backend import UserContext, require_auth from core.db.engine import get_db from core.infra.responses import success_response @@ -101,7 +101,7 @@ async def consent( ): _, row, item = activity.authorize(db, user.user_id, key, instance_id) if set(body.capabilities) - activity.declared(item, db, user.user_id, row.get("moduleId")): - from core.applications.external import fail + from features.workbench.external import fail fail(403, "capability_not_declared") if "background" not in body.capabilities: diff --git a/src/backend/api/routes/v1/workbench_application_config.py b/src/backend/features/workbench/api/workbench_application_config.py similarity index 92% rename from src/backend/api/routes/v1/workbench_application_config.py rename to src/backend/features/workbench/api/workbench_application_config.py index 44f6bef7..a0a3f1e6 100644 --- a/src/backend/api/routes/v1/workbench_application_config.py +++ b/src/backend/features/workbench/api/workbench_application_config.py @@ -1,9 +1,9 @@ """Protected administration of exact installed Workbench configuration.""" from api.deps import require_system_settings, require_admin -from api.routes.v1.workbench_capabilities import CapabilityRoute -from core.applications import application_config as config -from core.applications.application_config_bindings import selections, resolve_admin -from core.applications.external import fail +from features.workbench.api.workbench_capabilities import CapabilityRoute +from features.workbench import application_config as config +from features.workbench.application_config_bindings import selections, resolve_admin +from features.workbench.external import fail from core.db.engine import get_db from core.auth.backend import UserContext, require_auth from core.infra.responses import success_response diff --git a/src/backend/api/routes/v1/workbench_auth.py b/src/backend/features/workbench/api/workbench_auth.py similarity index 98% rename from src/backend/api/routes/v1/workbench_auth.py rename to src/backend/features/workbench/api/workbench_auth.py index 4f0d5430..a9068209 100644 --- a/src/backend/api/routes/v1/workbench_auth.py +++ b/src/backend/features/workbench/api/workbench_auth.py @@ -5,8 +5,8 @@ from urllib.parse import urlencode from api.deps import _resolve_session_user_id, require_config -from core.applications import external_auth -from core.applications.external_boundary import validate_cookie_scope +from features.workbench import external_auth +from features.workbench.external_boundary import validate_cookie_scope from core.auth.backend import UserContext, require_auth from core.auth.session import find_session_by_hash from core.config.settings import settings diff --git a/src/backend/api/routes/v1/workbench_capabilities.py b/src/backend/features/workbench/api/workbench_capabilities.py similarity index 96% rename from src/backend/api/routes/v1/workbench_capabilities.py rename to src/backend/features/workbench/api/workbench_capabilities.py index af29dd85..7399ead7 100644 --- a/src/backend/api/routes/v1/workbench_capabilities.py +++ b/src/backend/features/workbench/api/workbench_capabilities.py @@ -1,6 +1,6 @@ """Authenticated host broker. Opaque plugin frames never receive approval credentials.""" -from core.applications.coordination import runtime +from features.workbench.coordination import runtime import asyncio @@ -8,13 +8,13 @@ from typing import Any from urllib.parse import quote, urlencode -from core.applications import capabilities as grants -from core.applications import external, application_config +from features.workbench import capabilities as grants +from features.workbench import external, application_config from core.auth.backend import UserContext, require_auth from core.db.engine import get_db from core.infra.responses import success_response -from core.plugins.ui.contract import find_data_source -from core.plugins.ui.data_proxy import call_data_source +from features.workbench.ui.contract import find_data_source +from features.workbench.ui.data_proxy import call_data_source from fastapi import ( APIRouter, Depends, @@ -314,7 +314,7 @@ def download( import tempfile from pathlib import Path - from core.storage import get_storage + from core.files.storage import get_storage from fastapi.responses import FileResponse from starlette.background import BackgroundTask @@ -365,7 +365,7 @@ async def save( external.fail(422, "invalid_filename") runtime(db, user.user_id) grants.authorized(db, user.user_id, key, revision, approvalId, "file.save", instanceId) - from core.services.artifact_service import store_bytes_as_artifact + from core.files.artifacts.service import store_bytes_as_artifact artifact = store_bytes_as_artifact( db, diff --git a/src/backend/api/routes/v1/workbench_diagnostics.py b/src/backend/features/workbench/api/workbench_diagnostics.py similarity index 97% rename from src/backend/api/routes/v1/workbench_diagnostics.py rename to src/backend/features/workbench/api/workbench_diagnostics.py index bf021098..d130e30a 100644 --- a/src/backend/api/routes/v1/workbench_diagnostics.py +++ b/src/backend/features/workbench/api/workbench_diagnostics.py @@ -3,7 +3,7 @@ from typing import Any, Literal from api.deps import require_config -from core.applications import catalog, diagnostics, external, plugin_catalog, origin +from features.workbench import catalog, diagnostics, external, plugin_catalog, origin from core.auth.backend import UserContext, require_auth from core.db.engine import get_db from core.db.models import ContentBlock, UserShadow @@ -119,7 +119,7 @@ def own_resources( user: UserContext = Depends(require_auth()), db: Session = Depends(get_db), ): - from core.applications.diagnostic_resources import snapshot + from features.workbench.diagnostic_resources import snapshot response.headers["Cache-Control"] = "private, no-store" return success_response(data=snapshot(db, user.user_id)) @@ -166,7 +166,7 @@ def preview(body: Preview, response: Response, db: Session = Depends(get_db)): @router.post("/admin/diagnostics/probe/{key}", dependencies=write) def check(key: str, response: Response, db: Session = Depends(get_db)): - from core.applications.diagnostic_probe import probe + from features.workbench.diagnostic_probe import probe response.headers["Cache-Control"] = "no-store" item = external.record(db, key) diff --git a/src/backend/api/routes/v1/workbench_events.py b/src/backend/features/workbench/api/workbench_events.py similarity index 97% rename from src/backend/api/routes/v1/workbench_events.py rename to src/backend/features/workbench/api/workbench_events.py index a0b3f3d7..939063df 100644 --- a/src/backend/api/routes/v1/workbench_events.py +++ b/src/backend/features/workbench/api/workbench_events.py @@ -3,7 +3,7 @@ import asyncio import time -from core.applications.directory_revision import snapshot +from features.workbench.directory_revision import snapshot from core.auth.backend import UserContext, get_current_user, require_auth from core.db.engine import SessionLocal from core.infra.responses import success_response diff --git a/src/backend/api/routes/v1/workbench_preferences.py b/src/backend/features/workbench/api/workbench_preferences.py similarity index 97% rename from src/backend/api/routes/v1/workbench_preferences.py rename to src/backend/features/workbench/api/workbench_preferences.py index 32b962e6..0cad1efa 100644 --- a/src/backend/api/routes/v1/workbench_preferences.py +++ b/src/backend/features/workbench/api/workbench_preferences.py @@ -2,7 +2,7 @@ from typing import Literal -from core.applications import preferences +from features.workbench import preferences from core.auth.backend import UserContext, require_auth from core.db.engine import get_db from core.infra.responses import success_response diff --git a/src/backend/api/routes/v1/workbench_resources.py b/src/backend/features/workbench/api/workbench_resources.py similarity index 87% rename from src/backend/api/routes/v1/workbench_resources.py rename to src/backend/features/workbench/api/workbench_resources.py index 74e7bc40..c8dd1739 100644 --- a/src/backend/api/routes/v1/workbench_resources.py +++ b/src/backend/features/workbench/api/workbench_resources.py @@ -3,7 +3,8 @@ from core.auth.backend import UserContext, require_auth from core.db.engine import get_db from core.infra.responses import success_response -from core.plugins.resources import application, service +from core.harness.registry.resource_owners import resource_owner +from core.capabilities.plugins.resources import service from fastapi import APIRouter, Depends, Header, HTTPException, Response from pydantic import BaseModel, ConfigDict, Field from sqlalchemy.orm import Session @@ -46,7 +47,7 @@ async def create( if not instance_id or len(instance_id) > 128: raise HTTPException(422, "invalid_instance_id") response.headers["Cache-Control"] = "no-store" - value = await application.create( + value = await resource_owner().create( db, str(user.user_id), key, @@ -68,7 +69,7 @@ async def lease( response.headers["Cache-Control"] = "no-store" return success_response( data={ - "resource": await application.renew(db, str(user.user_id), key, instance_id, resource_id) + "resource": await resource_owner().lease(db, str(user.user_id), key, instance_id, resource_id) } ) @@ -82,7 +83,7 @@ async def close( db: Session = Depends(get_db), ): response.headers["Cache-Control"] = "no-store" - row = application.owned( + row = resource_owner().owned( db, str(user.user_id), key, instance_id, resource_id, allow_closed=True ) if row.status != "closed": @@ -101,10 +102,10 @@ async def command( ): import json - from core.plugins.resources.owners import resolve_installation - from core.plugins.ui.contract import find_module + from core.capabilities.plugins.resources.owners import resolve_installation + from features.workbench.ui.contract import find_module - row = application.owned(db, str(user.user_id), key, instance_id, resource_id) + row = resource_owner().owned(db, str(user.user_id), key, instance_id, resource_id) selected = resolve_installation(db, row, row.user_id) module = find_module(selected.ui, row.module_id) if not module or "resource.command" not in module.get("grants", []): diff --git a/src/backend/api/routes/v1/workbench_revision_leases.py b/src/backend/features/workbench/api/workbench_revision_leases.py similarity index 97% rename from src/backend/api/routes/v1/workbench_revision_leases.py rename to src/backend/features/workbench/api/workbench_revision_leases.py index 1cd0dcf5..3a8d20f5 100644 --- a/src/backend/api/routes/v1/workbench_revision_leases.py +++ b/src/backend/features/workbench/api/workbench_revision_leases.py @@ -1,6 +1,6 @@ """Revision retention is bounded and tied to an authenticated running instance.""" -from core.applications import plugin_revisions +from features.workbench import plugin_revisions from core.auth.backend import UserContext, require_auth from core.db.engine import get_db from core.infra.responses import success_response diff --git a/src/backend/api/routes/v1/workbench_state.py b/src/backend/features/workbench/api/workbench_state.py similarity index 98% rename from src/backend/api/routes/v1/workbench_state.py rename to src/backend/features/workbench/api/workbench_state.py index a6d59894..47b6e6e3 100644 --- a/src/backend/api/routes/v1/workbench_state.py +++ b/src/backend/features/workbench/api/workbench_state.py @@ -1,7 +1,7 @@ """Authenticated state boundary, mounted under /v1/workbench.""" from typing import Any, Annotated -from core.applications import state +from features.workbench import state from core.auth.backend import UserContext, require_auth from core.db.engine import get_db from core.infra.responses import success_response diff --git a/src/backend/api/routes/v1/workbench_upgrades.py b/src/backend/features/workbench/api/workbench_upgrades.py similarity index 98% rename from src/backend/api/routes/v1/workbench_upgrades.py rename to src/backend/features/workbench/api/workbench_upgrades.py index d4725876..8a4ace7b 100644 --- a/src/backend/api/routes/v1/workbench_upgrades.py +++ b/src/backend/features/workbench/api/workbench_upgrades.py @@ -1,7 +1,7 @@ """Private owner and administrator upgrade decisions; candidate data never grants access.""" from api.deps import require_admin -from core.applications import plugin_upgrades +from features.workbench import plugin_upgrades from core.auth.backend import UserContext, require_auth from core.db.engine import get_db from core.infra.responses import success_response diff --git a/src/backend/api/routes/v1/workbench_workspace.py b/src/backend/features/workbench/api/workbench_workspace.py similarity index 83% rename from src/backend/api/routes/v1/workbench_workspace.py rename to src/backend/features/workbench/api/workbench_workspace.py index 57be0d51..7a3c33c5 100644 --- a/src/backend/api/routes/v1/workbench_workspace.py +++ b/src/backend/features/workbench/api/workbench_workspace.py @@ -1,13 +1,13 @@ """Authenticated workspace package assets. Every fetch rechecks the exact binding.""" -from api.routes.v1.plugin_ui import _MODULE_CSP -from core.applications import catalog -from core.applications.plugin_revisions import resolve_revision -from core.applications.workspace_page import render +from features.workbench.api.plugin_ui import _MODULE_CSP +from features.workbench import catalog +from features.workbench.plugin_revisions import resolve_revision +from features.workbench.workspace_page import render from core.auth.backend import UserContext, require_auth from core.db.engine import get_db -from core.plugins.resources.installation import safe_path -from core.services.site_service import guess_site_mime +from core.capabilities.plugins.resources.installation import safe_path +from features.sites.service import guess_site_mime from fastapi.responses import FileResponse, HTMLResponse from fastapi import APIRouter, Depends, HTTPException, Query from sqlalchemy.orm import Session diff --git a/src/backend/core/applications/application_config.py b/src/backend/features/workbench/application_config.py similarity index 93% rename from src/backend/core/applications/application_config.py rename to src/backend/features/workbench/application_config.py index 62b6dc14..becd4629 100644 --- a/src/backend/core/applications/application_config.py +++ b/src/backend/features/workbench/application_config.py @@ -9,13 +9,13 @@ from hashlib import sha256 from urllib.parse import urlsplit -from core.applications import origin -from core.applications.external import fail -from core.applications.coordination import configuration +from features.workbench import origin +from features.workbench.external import fail +from features.workbench.coordination import configuration from core.db.models import ContentBlock, SystemConfig from core.infra.time import utc_now -from core.plugins.packaging.importer import _normalize_admin_config, manifest_extensions -from core.plugins.resources.crypto import encrypt_secret, decrypt_secret +from core.capabilities.plugins.packaging.importer import normalize_admin_config as _normalize_admin_config, manifest_extensions +from core.capabilities.plugins.resources.crypto import encrypt_secret, decrypt_secret PREFIX = "workbench_app_config:" GROUP = "workbench_app_config" @@ -112,7 +112,7 @@ def view(db, item, selected, with_values=False): def write(db, actor, key, revision, values, clear_keys): - from core.applications.application_config_bindings import resolve_admin + from features.workbench.application_config_bindings import resolve_admin configuration(db) db.expire_all() @@ -171,7 +171,7 @@ def write(db, actor, key, revision, values, clear_keys): ) ) db.commit() - from core.services.system_config import SystemConfigService + from core.config.store.system_config import SystemConfigService SystemConfigService.get_instance().invalidate_cache() return view(db, item, selected, with_values=True) diff --git a/src/backend/core/applications/application_config_bindings.py b/src/backend/features/workbench/application_config_bindings.py similarity index 82% rename from src/backend/core/applications/application_config_bindings.py rename to src/backend/features/workbench/application_config_bindings.py index 90bf9613..a3ac2a22 100644 --- a/src/backend/core/applications/application_config_bindings.py +++ b/src/backend/features/workbench/application_config_bindings.py @@ -1,25 +1,25 @@ """Admin discovery resolves current installations without granting execution.""" -from core.applications import origin -from core.applications.external import fail -from core.applications.plugin_catalog import application_key -from core.applications.protocol import validate_contributions +from features.workbench import origin +from features.workbench.external import fail +from features.workbench.plugin_catalog import application_key +from features.workbench.protocol import validate_contributions from core.db.models import InstalledPlugin, UserShadow -from core.plugins.resources.installation import Installation -from core.plugins.packaging.runtime_assets import materialize +from core.capabilities.plugins.resources.installation import Installation +from core.capabilities.plugins.packaging.runtime_assets import materialize def selections(db, actor): source = origin.current(db) - from core.capabilities import device_catalog + from features.desktop.capabilities import device_catalog represented = set() if source["kind"] == "local": - from core.capabilities import plugins, store - from core.capabilities.dependency import component_hash - from core.capabilities.local_plugin_runtime import enabled_for - from core.applications.installation_binding import DeviceRecord + from features.desktop.capabilities import plugins, store + from features.desktop.capabilities.dependency import component_hash + from features.desktop.capabilities.local_plugin_runtime import enabled_for + from features.workbench.installation_binding import DeviceRecord owners = {actor} | {r.user_id for r in db.query(UserShadow).all()} for owner in owners: - for inst in device_catalog._installations("plugin", user_id=owner): + for inst in device_catalog.installations("plugin", user_id=owner): if not inst.ready or inst.created_at is None: continue component = store.get("plugin", inst.profile_id, inst.key, inst.resolved_revision) @@ -83,7 +83,7 @@ def resolve_admin(db, actor, key, revision): def _enabled(db, row, actor): - from core.applications.installation_binding import exact + from features.workbench.installation_binding import exact from fastapi import HTTPException try: exact(db, row.slug, row.owner_user_id or actor, row.install_id) diff --git a/src/backend/core/applications/background.py b/src/backend/features/workbench/background.py similarity index 95% rename from src/backend/core/applications/background.py rename to src/backend/features/workbench/background.py index 66486fe2..3f9d792f 100644 --- a/src/backend/core/applications/background.py +++ b/src/backend/features/workbench/background.py @@ -1,12 +1,12 @@ """Separate background permission and budget for existing application resources.""" -from core.applications.coordination import runtime +from features.workbench.coordination import runtime import time import logging -from core.applications import activity, external -from core.plugins.resources import service, store +from features.workbench import activity, external +from core.capabilities.plugins.resources import service, store MAX_PER_USER = 2 MAX_PER_APP = 1 @@ -52,7 +52,7 @@ def admission(db, user_id, key, instance_id, resource_id): async def start(db, user_id, key, instance_id, resource_id): - from core.plugins.resources.deadlines import arm, provider_for + from core.capabilities.plugins.resources.deadlines import arm, provider_for # Probe the runner without holding a database write/coordination lock. _, _, probe = admission(db, user_id, key, instance_id, resource_id) @@ -116,7 +116,7 @@ def authorized_resource(db, row): async def renew(db, user_id, key, instance_id, resource_id): - from core.plugins.resources.deadlines import arm + from core.capabilities.plugins.resources.deadlines import arm runtime(db, user_id) db.expire_all() diff --git a/src/backend/core/applications/capabilities.py b/src/backend/features/workbench/capabilities.py similarity index 95% rename from src/backend/core/applications/capabilities.py rename to src/backend/features/workbench/capabilities.py index 9631d37e..47374af0 100644 --- a/src/backend/core/applications/capabilities.py +++ b/src/backend/features/workbench/capabilities.py @@ -1,6 +1,6 @@ """Server-owned approvals for exact installations, users and environments.""" -from core.applications.coordination import runtime +from features.workbench.coordination import runtime from copy import deepcopy @@ -8,10 +8,10 @@ from hashlib import sha256 from secrets import token_urlsafe -from core.applications import catalog, external +from features.workbench import catalog, external from core.db.models import ContentBlock from core.infra.time import as_utc, utc_now -from core.applications.installation_binding import exact as resolve +from features.workbench.installation_binding import exact as resolve SUPPORTED = { "data.query", @@ -35,7 +35,7 @@ def binding(db, user_id, key, revision, module_id=None, instance_id=None): if selected.ui.get("version") == 3 and not instance_id: external.fail(403, "instance_binding_mismatch") if instance_id or revision != current["packageRevision"]: - from core.applications.plugin_revisions import resolve_revision + from features.workbench.plugin_revisions import resolve_revision selected = resolve_revision(db, user_id, key, revision, instance_id) current = {**current, "packageRevision": revision} @@ -223,9 +223,9 @@ def select_file( def readable_file(db, user_id, file_id): - from core.auth.permissions_iface import resolve_artifact_access + from core.auth.edition import resolve_artifact_access from core.db.models import Artifact - from core.services.artifact_edition import artifact_access_metadata + from core.files.edition import artifact_access_metadata artifact = ( db.query(Artifact) @@ -248,6 +248,6 @@ def configured_hosts(db, selected, key=None): """No global manifest-key trust. Legacy callers without exact identity get none.""" if key is None: return set() - from core.applications.application_config import trusted_origins + from features.workbench.application_config import trusted_origins return {host for _, host, _ in trusted_origins(db, key, selected)} diff --git a/src/backend/core/applications/catalog.py b/src/backend/features/workbench/catalog.py similarity index 96% rename from src/backend/core/applications/catalog.py rename to src/backend/features/workbench/catalog.py index c3dd5acc..f2361eaf 100644 --- a/src/backend/core/applications/catalog.py +++ b/src/backend/features/workbench/catalog.py @@ -7,7 +7,7 @@ from copy import deepcopy -from core.applications import external, policy, origin +from features.workbench import external, policy, origin from core.auth.capabilities import resolve_user_capabilities from core.config.local_mode import local_mode_enabled from core.db.models import ContentBlock @@ -54,7 +54,7 @@ def definitions(db, user_id=None, include_disabled=False): **item["declaration"], "icon": {"light": item["legacyIcon"]} } values.extend(external_values) - from core.applications.migration_compat import active, app_config + from features.workbench.migration_compat import active, app_config config = {"apps": []} if active(db) else app_config(db) for app in config["apps"]: @@ -79,7 +79,7 @@ def definitions(db, user_id=None, include_disabled=False): } ) if user_id is not None: - from core.applications import plugin_catalog + from features.workbench import plugin_catalog values.extend(plugin_catalog.items(db, user_id, include_disabled=include_disabled)) return values diff --git a/src/backend/core/applications/coordination.py b/src/backend/features/workbench/coordination.py similarity index 100% rename from src/backend/core/applications/coordination.py rename to src/backend/features/workbench/coordination.py diff --git a/src/backend/core/applications/diagnostic_probe.py b/src/backend/features/workbench/diagnostic_probe.py similarity index 98% rename from src/backend/core/applications/diagnostic_probe.py rename to src/backend/features/workbench/diagnostic_probe.py index c6f5ddb7..e4d03735 100644 --- a/src/backend/core/applications/diagnostic_probe.py +++ b/src/backend/features/workbench/diagnostic_probe.py @@ -12,7 +12,7 @@ from threading import BoundedSemaphore from urllib.parse import urlsplit -from core.applications.external import validate_url +from features.workbench.external import validate_url from fastapi import HTTPException _slots = BoundedSemaphore(2) diff --git a/src/backend/core/applications/diagnostic_resources.py b/src/backend/features/workbench/diagnostic_resources.py similarity index 100% rename from src/backend/core/applications/diagnostic_resources.py rename to src/backend/features/workbench/diagnostic_resources.py diff --git a/src/backend/core/applications/diagnostics.py b/src/backend/features/workbench/diagnostics.py similarity index 99% rename from src/backend/core/applications/diagnostics.py rename to src/backend/features/workbench/diagnostics.py index 5243e20d..3bb745bd 100644 --- a/src/backend/core/applications/diagnostics.py +++ b/src/backend/features/workbench/diagnostics.py @@ -209,7 +209,7 @@ def run(*args, **kwargs): db, user = kwargs.get("db"), kwargs.get("user") item = None if event == "launch": - from core.applications import catalog + from features.workbench import catalog try: item = catalog.readable(db, user.user_id, kwargs["key"]) diff --git a/src/backend/core/applications/directory_revision.py b/src/backend/features/workbench/directory_revision.py similarity index 96% rename from src/backend/core/applications/directory_revision.py rename to src/backend/features/workbench/directory_revision.py index e952d205..7fdbab3d 100644 --- a/src/backend/core/applications/directory_revision.py +++ b/src/backend/features/workbench/directory_revision.py @@ -3,7 +3,7 @@ import hashlib import json -from core.applications import catalog +from features.workbench import catalog from core.config.local_mode import local_mode_enabled from core.db.models import ContentBlock diff --git a/src/backend/features/workbench/edition.py b/src/backend/features/workbench/edition.py new file mode 100644 index 00000000..659a0388 --- /dev/null +++ b/src/backend/features/workbench/edition.py @@ -0,0 +1,10 @@ +"""Community operators must reject unknown authorization tables.""" +from sqlalchemy import inspect +from core.db.engine import Base + + +def snapshot_permissions(db): + names = set(inspect(db.get_bind()).get_table_names()) + if names - set(Base.metadata.tables) - {"alembic_version"}: + raise ValueError("ce_identity_database_requires_full_edition_operator") + return {} diff --git a/src/backend/features/workbench/extension.py b/src/backend/features/workbench/extension.py new file mode 100644 index 00000000..14a9a647 --- /dev/null +++ b/src/backend/features/workbench/extension.py @@ -0,0 +1,4 @@ +"""Installed workbench extension.""" +NAME = "workbench" +def setup(): + return True diff --git a/src/backend/core/applications/external.py b/src/backend/features/workbench/external.py similarity index 94% rename from src/backend/core/applications/external.py rename to src/backend/features/workbench/external.py index 78aaec59..d2365ae4 100644 --- a/src/backend/core/applications/external.py +++ b/src/backend/features/workbench/external.py @@ -11,9 +11,9 @@ from urllib.parse import parse_qsl, urlsplit from uuid import uuid4 -from core.applications.coordination import configuration -from core.applications.protocol import validate_application -from core.content.content_blocks import normalize_app_config +from features.workbench.coordination import configuration +from features.workbench.protocol import validate_application +from core.files.content.branding import normalize_app_config from core.db.models import ContentBlock from core.infra.time import utc_now from fastapi import HTTPException @@ -29,7 +29,7 @@ def fail(status, code): def legacy_ids(db): - from core.applications.migration_compat import app_config + from features.workbench.migration_compat import app_config return {app["id"] for app in app_config(db)["apps"]} | { "plan_mode", @@ -112,7 +112,7 @@ def record(db, key): def save(db, declaration, enabled, actor, trace, revision=None, key=None, embedding=None): configuration(db) validate_definition(db, declaration) - from core.applications.external_embed import validate_policy + from features.workbench.external_embed import validate_policy embedding = validate_policy(declaration["target"], embedding) expected_key = "external:" + declaration["id"] @@ -169,7 +169,7 @@ def save(db, declaration, enabled, actor, trace, revision=None, key=None, embedd def reject_legacy_conflicts(db, payload): - from core.applications.migration_compat import active + from features.workbench.migration_compat import active if active(db): return # The versioned adapter validates and writes canonical records. diff --git a/src/backend/core/applications/external_auth.py b/src/backend/features/workbench/external_auth.py similarity index 99% rename from src/backend/core/applications/external_auth.py rename to src/backend/features/workbench/external_auth.py index 85404de0..c0cc3d8e 100644 --- a/src/backend/core/applications/external_auth.py +++ b/src/backend/features/workbench/external_auth.py @@ -5,7 +5,7 @@ ContentBlock provides portable transactions without adding a second app source. """ -from core.applications.coordination import configuration +from features.workbench.coordination import configuration import hashlib @@ -16,7 +16,7 @@ from urllib.parse import urlencode, urlsplit, urlunsplit from uuid import uuid4 -from core.applications import catalog, external +from features.workbench import catalog, external from core.db.models import ContentBlock from core.infra.time import utc_now from sqlalchemy import delete diff --git a/src/backend/core/applications/external_boundary.py b/src/backend/features/workbench/external_boundary.py similarity index 97% rename from src/backend/core/applications/external_boundary.py rename to src/backend/features/workbench/external_boundary.py index 52458491..1f5cdf2e 100644 --- a/src/backend/core/applications/external_boundary.py +++ b/src/backend/features/workbench/external_boundary.py @@ -7,7 +7,7 @@ import ipaddress from urllib.parse import urlsplit -from core.applications import external +from features.workbench import external def validate_cookie_scope(request, target, config, cookie_domain): diff --git a/src/backend/core/applications/external_embed.py b/src/backend/features/workbench/external_embed.py similarity index 97% rename from src/backend/core/applications/external_embed.py rename to src/backend/features/workbench/external_embed.py index 4bfdc860..4461cca4 100644 --- a/src/backend/core/applications/external_embed.py +++ b/src/backend/features/workbench/external_embed.py @@ -36,7 +36,7 @@ def exact_origin(value): def validate_policy(target, embedding, host_origin=None): - from core.applications.external import validate_url + from features.workbench.external import validate_url if target["mode"] != "embedded": if embedding is not None: diff --git a/src/backend/core/applications/installation_binding.py b/src/backend/features/workbench/installation_binding.py similarity index 77% rename from src/backend/core/applications/installation_binding.py rename to src/backend/features/workbench/installation_binding.py index eed5b3ef..f878d91b 100644 --- a/src/backend/core/applications/installation_binding.py +++ b/src/backend/features/workbench/installation_binding.py @@ -3,7 +3,7 @@ from dataclasses import dataclass from datetime import datetime -from core.plugins.resources.installation import Installation, resolve +from core.capabilities.plugins.resources.installation import Installation, resolve from fastapi import HTTPException @@ -17,14 +17,14 @@ class DeviceRecord: def device_records(db, user_id): - from core.capabilities import device_catalog, plugins, store - from core.capabilities.local_plugin_runtime import enabled_for - from core.capabilities.paths import capabilities_enabled + from features.desktop.capabilities import device_catalog, plugins, store + from features.desktop.capabilities.local_plugin_runtime import enabled_for + from core.harness.registry.run_capabilities import has_run_capabilities - if not capabilities_enabled(): + if not has_run_capabilities(): return [] result = [] - for inst in device_catalog._installations("plugin", user_id=user_id): + for inst in device_catalog.installations("plugin", user_id=user_id): if not inst.ready or not enabled_for(inst, user_id) or inst.created_at is None: continue component = store.get("plugin", inst.profile_id, inst.key, inst.resolved_revision) @@ -52,16 +52,16 @@ def device_records(db, user_id): def exact(db, slug, user_id, install_id): - from core.capabilities import device_catalog, plugins, store - from core.capabilities.dependency import component_hash - from core.capabilities.local_plugin_runtime import enabled_for - from core.capabilities.paths import capabilities_enabled + from features.desktop.capabilities import device_catalog, plugins, store + from features.desktop.capabilities.dependency import component_hash + from features.desktop.capabilities.local_plugin_runtime import enabled_for + from core.harness.registry.run_capabilities import has_run_capabilities # Legacy resolver also accepts aliases. Workbench has no alias authority. - if capabilities_enabled(): + if has_run_capabilities(): candidates = [ inst - for inst in device_catalog._installations("plugin", user_id=user_id) + for inst in device_catalog.installations("plugin", user_id=user_id) if inst.install_id == install_id ] if candidates: diff --git a/src/backend/core/applications/local_plugin_upgrades.py b/src/backend/features/workbench/local_plugin_upgrades.py similarity index 86% rename from src/backend/core/applications/local_plugin_upgrades.py rename to src/backend/features/workbench/local_plugin_upgrades.py index add3ff8c..aa2e989f 100644 --- a/src/backend/core/applications/local_plugin_upgrades.py +++ b/src/backend/features/workbench/local_plugin_upgrades.py @@ -2,27 +2,21 @@ from copy import deepcopy -from core.applications.external import fail -from core.applications.plugin_upgrades import ( - PREFIX, - history_id, - binding_token, - permission_set, - schema_changes, - verify_package, -) -from core.capabilities import registry, store -from core.capabilities.paths import LOCAL_PROFILE, capabilities_enabled -from core.capabilities.plugins import load_manifest +from features.workbench.external import fail +from features.workbench.plugin_upgrades import PREFIX, history_id, binding_token, permission_set, schema_changes, verify_package +from features.desktop.capabilities import registry, store +from core.harness.registry.run_capabilities import has_run_capabilities +from core.capabilities.identity_rules import LOCAL_PROFILE +from features.desktop.capabilities.plugins import load_manifest from core.db.models import ContentBlock from core.infra.time import utc_now -from core.services.management_package import serialized +from core.capabilities.plugins.management.package import serialized def owned(db, user_id, install_id): - from core.plugins.local.service import _owned + from features.desktop.local.plugins.service import _owned - if not capabilities_enabled(): + if not has_run_capabilities(): fail(404, "plugin_upgrade_not_found") try: return _owned(user_id, install_id, db) @@ -48,7 +42,7 @@ def public(inst, record): def read(db, user_id, install_id): inst = owned(db, user_id, install_id) - from core.applications.plugin_candidate_storage import summary + from features.workbench.plugin_candidate_storage import summary return { **public(inst, db.get(ContentBlock, identifier(inst))), @@ -67,7 +61,7 @@ def prepare(db, inst, prepared, approved_revision=None): verify_package(np) added = sorted(permission_set(np.ui) - permission_set(old_ui)) changes = schema_changes(old_ui, np.ui) - from core.applications import plugin_candidate_storage as storage + from features.workbench import plugin_candidate_storage as storage record = db.get(ContentBlock, identifier(inst)) if (added or changes) and comp.revision != approved_revision: @@ -124,7 +118,7 @@ def prepare(db, inst, prepared, approved_revision=None): @serialized def decide(user_id, install_id, expected, candidate, decision, reset_state): - from core.plugins.local import service + from features.desktop.local.plugins import service as service with registry._session() as db: inst = owned(db, user_id, install_id) @@ -139,7 +133,7 @@ def decide(user_id, install_id, expected, candidate, decision, reset_state): ): fail(409, "revision_conflict") if decision == "reject": - from core.applications import plugin_candidate_storage as storage + from features.workbench import plugin_candidate_storage as storage storage.change(db, binding_token(inst), candidate, "discarded") record.payload = {**value, "status": "rejected"} diff --git a/src/backend/core/applications/migration.py b/src/backend/features/workbench/migration.py similarity index 97% rename from src/backend/core/applications/migration.py rename to src/backend/features/workbench/migration.py index 72a9635d..69d4d857 100644 --- a/src/backend/core/applications/migration.py +++ b/src/backend/features/workbench/migration.py @@ -1,21 +1,15 @@ """WB-20 operator boundary. Explicit freeze, protected snapshot, atomic cutover.""" -from core.applications.coordination import configuration +from features.workbench.coordination import configuration from copy import deepcopy from pathlib import Path from uuid import uuid4 -from core.applications import external -from core.applications.migration_plan import plan -from core.applications.migration_snapshot import ( - RECEIPT, - digest, - read_snapshot, - state, - write_snapshot, -) +from features.workbench import external +from features.workbench.migration_plan import plan +from features.workbench.migration_snapshot import RECEIPT, digest, read_snapshot, state, write_snapshot from core.db.models import ContentBlock, UserShadow from core.infra.time import utc_now diff --git a/src/backend/core/applications/migration_compat.py b/src/backend/features/workbench/migration_compat.py similarity index 88% rename from src/backend/core/applications/migration_compat.py rename to src/backend/features/workbench/migration_compat.py index d354a362..e9b3a094 100644 --- a/src/backend/core/applications/migration_compat.py +++ b/src/backend/features/workbench/migration_compat.py @@ -1,12 +1,12 @@ """Versioned content-v1 adapter after WB-20 cutover; canonical rows are the only source.""" -from core.applications.coordination import configuration +from features.workbench.coordination import configuration from copy import deepcopy -from core.applications import external -from core.applications.migration_snapshot import RECEIPT +from features.workbench import external +from features.workbench.migration_snapshot import RECEIPT from core.db.models import ContentBlock from core.infra.time import utc_now @@ -18,7 +18,8 @@ def active(db): def app_config(db): if not active(db): - from core.content.content_blocks import DEFAULT_APP_CONFIG, normalize_app_config + from core.files.content.content_blocks_constants import DEFAULT_APP_CONFIG + from core.files.content.branding import normalize_app_config row = db.get(ContentBlock, "app_config") return normalize_app_config(row.payload if row else DEFAULT_APP_CONFIG) @@ -56,7 +57,7 @@ def page_config(db, payload): result = deepcopy(payload) if not active(db): return result - from core.applications import policy + from features.workbench import policy layout = policy.read(db) nav = result.setdefault("navigation", {}) @@ -77,8 +78,8 @@ def stage_write(db, alias, payload): if alias == "page_config": navigation = payload.get("navigation", {}) if isinstance(navigation, dict) and ({"sidebar_items", "menu_items"} & set(navigation)): - from core.applications import policy - from core.applications.catalog import BUILTINS + from features.workbench import policy + from features.workbench.catalog import BUILTINS layout = policy.read(db) for field in ("sidebar_items", "menu_items"): @@ -104,7 +105,7 @@ def stage_write(db, alias, payload): return False if alias != "app_config": return False - from core.applications.migration_plan import plan + from features.workbench.migration_plan import plan if ( not isinstance(payload, dict) @@ -141,7 +142,7 @@ def _put(db, key, value): def build_docs_snapshot(db): - from core.content.content_blocks import build_docs_snapshot as original + from core.files.content.content_blocks import build_docs_snapshot as original result = original(db) if active(db): @@ -154,9 +155,9 @@ def build_docs_snapshot(db): def import_docs_snapshot(db, snapshot, *, overwrite=True, default_updated_by=None): - from core.content.content_blocks import DOCS_BLOCK_MAP - from core.content.content_blocks import import_docs_snapshot as original - from core.content.content_blocks import normalize_docs_snapshot + from core.files.content.content_blocks_constants import DOCS_BLOCK_MAP + from core.files.content.content_blocks import import_docs_snapshot as original + from core.files.content.content_blocks import normalize_docs_snapshot if not active(db): return original(db, snapshot, overwrite=overwrite, default_updated_by=default_updated_by) diff --git a/src/backend/core/applications/migration_plan.py b/src/backend/features/workbench/migration_plan.py similarity index 96% rename from src/backend/core/applications/migration_plan.py rename to src/backend/features/workbench/migration_plan.py index b7e4fe7a..61251481 100644 --- a/src/backend/core/applications/migration_plan.py +++ b/src/backend/features/workbench/migration_plan.py @@ -4,11 +4,11 @@ from collections import Counter from copy import deepcopy -from core.applications import external, preferences -from core.applications.catalog import BUILTINS -from core.applications.migration_snapshot import digest -from core.applications.protocol import validate_application -from core.content.content_blocks import DEFAULT_PAGE_CONFIG +from features.workbench import external, preferences +from features.workbench.catalog import BUILTINS +from features.workbench.migration_snapshot import digest +from features.workbench.protocol import validate_application +from core.files.content.content_blocks_constants import DEFAULT_PAGE_CONFIG from fastapi import HTTPException FIXED = {"chat", "settings", "docs", "plan_mode"} @@ -29,7 +29,7 @@ def configured_auth(auth, target): mode = auth.get("mode") if mode == "independent": return set(auth) <= {"revision", "mode"} - from core.applications.external_auth import destination + from features.workbench.external_auth import destination try: if mode == "shared_identity": diff --git a/src/backend/core/applications/migration_snapshot.py b/src/backend/features/workbench/migration_snapshot.py similarity index 91% rename from src/backend/core/applications/migration_snapshot.py rename to src/backend/features/workbench/migration_snapshot.py index addc466f..a624ad5c 100644 --- a/src/backend/core/applications/migration_snapshot.py +++ b/src/backend/features/workbench/migration_snapshot.py @@ -15,7 +15,6 @@ UserShadow, ) from sqlalchemy import inspect, or_ -from core.db.engine import Base RECEIPT = "workbench_migration:v1" @@ -58,12 +57,9 @@ def state(db): } for row in db.query(InstalledPlugin).order_by(InstalledPlugin.install_id) } - # Refuse unrecognized tables: community operators cannot promise to preserve their authorization state. - names = set(inspect(db.get_bind()).get_table_names()) - unsupported = names - set(Base.metadata.tables) - {"alembic_version"} - if unsupported: - raise ValueError("ce_identity_database_requires_full_edition_operator") - permissions = {} + from features.workbench.edition import snapshot_permissions + + permissions = snapshot_permissions(db) permissions["pluginSkills"] = { row.skill_id: { "enabled": row.is_enabled, diff --git a/src/backend/features/workbench/models.py b/src/backend/features/workbench/models.py new file mode 100644 index 00000000..49d08342 --- /dev/null +++ b/src/backend/features/workbench/models.py @@ -0,0 +1 @@ +"""Workbench persists configuration in core ContentBlock; no additional ORM tables.""" diff --git a/src/backend/core/applications/origin.py b/src/backend/features/workbench/origin.py similarity index 100% rename from src/backend/core/applications/origin.py rename to src/backend/features/workbench/origin.py diff --git a/src/backend/features/workbench/package_source.py b/src/backend/features/workbench/package_source.py new file mode 100644 index 00000000..56099fbd --- /dev/null +++ b/src/backend/features/workbench/package_source.py @@ -0,0 +1,19 @@ +"""Workbench owns plugin upgrade staging and candidate retention.""" + + +class WorkbenchPackagePolicy: + def lock_configuration(self, db): + from features.workbench.coordination import configuration + configuration(db) + + def prepare(self, db, existing, package, source, approved_revision): + from features.workbench.plugin_upgrades import prepare + return prepare(db, existing, package, source, approved_revision) + + def reserve_local(self, existing, revision, files): + from features.workbench.plugin_candidate_storage import local_reserve + return local_reserve(existing, revision, files) + + def prepare_local(self, db, current, prepared): + from features.workbench.local_plugin_upgrades import prepare + return prepare(db, current, prepared) diff --git a/src/backend/core/applications/plugin_candidate_storage.py b/src/backend/features/workbench/plugin_candidate_storage.py similarity index 88% rename from src/backend/core/applications/plugin_candidate_storage.py rename to src/backend/features/workbench/plugin_candidate_storage.py index 60b155b3..c8d1956c 100644 --- a/src/backend/core/applications/plugin_candidate_storage.py +++ b/src/backend/features/workbench/plugin_candidate_storage.py @@ -4,12 +4,13 @@ import json import time -from core.applications.external import fail -from core.applications.coordination import configuration -from core.db.models import ContentBlock, InstalledPlugin, DeviceCapabilityInstallation +from features.workbench.external import fail +from features.workbench.coordination import configuration +from core.db.models import ContentBlock, InstalledPlugin +from features.desktop.models import DeviceCapabilityInstallation from core.db.models.plugin_resource import PluginResource, PluginResourcePackage from core.infra.time import utc_now -from core.capabilities.errors import IntegrityFailed +from core.harness.capability_errors import IntegrityFailed PREFIX = "workbench_plugin_candidate:" MAX_CANDIDATES = 2 @@ -33,7 +34,7 @@ def owner_key(user_id): def live_bindings(db): - from core.applications.plugin_upgrades import binding_token + from features.workbench.plugin_upgrades import binding_token rows = list(db.query(InstalledPlugin).all()) rows.extend( @@ -49,8 +50,8 @@ def live_bindings(db): def protected(db): """Existing authoritative references win over candidate cleanup.""" - from core.applications.plugin_upgrades import PREFIX as UPGRADES, HISTORY - from core.applications.plugin_revisions import LEASE + from features.workbench.plugin_upgrades import PREFIX as UPGRADES, HISTORY + from features.workbench.plugin_revisions import LEASE active_bindings = live_bindings(db) keep = { @@ -102,7 +103,7 @@ def collect(db, binding=None, owner_scope=None): if value["binding"] not in active_bindings: value = {**value, "status": "discarded"} row.payload = value - from core.applications.plugin_upgrades import PREFIX as UPGRADES + from features.workbench.plugin_upgrades import PREFIX as UPGRADES orphan = db.get(ContentBlock, UPGRADES + value["binding"]) if orphan is not None: @@ -119,11 +120,11 @@ def collect(db, binding=None, owner_scope=None): continue try: if value["scope"] == "local": - from core.capabilities import store + from features.desktop.capabilities import store store.remove_revision("plugin", "local", value["key"], value["revision"]) else: - from core.plugins.packaging.runtime_assets import forget_cache + from core.capabilities.plugins.packaging.runtime_assets import forget_cache forget_cache(value["revision"]) package = db.get(PluginResourcePackage, value["revision"]) @@ -199,8 +200,8 @@ def replace(db, binding, previous, revision): def local_reserve(inst, revision, files): """Reserve before any candidate/skill directory is written; existing file lock is held.""" - from core.capabilities import registry - from core.applications.plugin_upgrades import binding_token + from features.desktop.capabilities import registry + from features.workbench.plugin_upgrades import binding_token size = sum(len(v if isinstance(v, bytes) else v.encode()) for v in files.values()) if size > 50 * 1024 * 1024: @@ -221,13 +222,13 @@ def local_reserve(inst, revision, files): def janitor(db): """Avoid deleting local prepared files while a management mutation holds the file lock.""" - from core.capabilities.paths import capabilities_enabled + from core.harness.registry.run_capabilities import has_run_capabilities - if not capabilities_enabled(): + if not has_run_capabilities(): configuration(db) return collect(db) - from core.capabilities.paths import meta_root - from core.capabilities.lockfile import locked, LockTimeout + from features.desktop.capabilities.paths import meta_root + from core.infra.file_lock import locked, LockTimeout try: with locked(meta_root() / "management.lock", timeout=0.1): diff --git a/src/backend/core/applications/plugin_catalog.py b/src/backend/features/workbench/plugin_catalog.py similarity index 95% rename from src/backend/core/applications/plugin_catalog.py rename to src/backend/features/workbench/plugin_catalog.py index d9db27de..b8b778a7 100644 --- a/src/backend/core/applications/plugin_catalog.py +++ b/src/backend/features/workbench/plugin_catalog.py @@ -4,11 +4,11 @@ from hashlib import sha256 from urllib.parse import quote -from core.applications.protocol import validate_contributions -from core.applications import origin +from features.workbench.protocol import validate_contributions +from features.workbench import origin from core.db.models import InstalledPlugin from core.infra.time import as_utc -from core.applications.installation_binding import exact as resolve, device_records +from features.workbench.installation_binding import exact as resolve, device_records from fastapi import HTTPException from sqlalchemy import or_ @@ -55,7 +55,7 @@ def items(db, user_id, include_disabled=False): ) .all() ) - from core.capabilities import device_catalog + from features.desktop.capabilities import device_catalog if device_catalog.active(): records = [] @@ -137,7 +137,7 @@ def launch(db, user_id, item, module_id=None): if module_id not in allowed: raise HTTPException(404, detail={"code": "app_view_not_found"}) module = next(m for m in checked["modules"] if m["id"] == module_id) - from core.plugins.resources.installation import safe_path + from core.capabilities.plugins.resources.installation import safe_path target = safe_path(selected.package, module["entry"]) if not target.is_file(): diff --git a/src/backend/core/applications/plugin_revisions.py b/src/backend/features/workbench/plugin_revisions.py similarity index 89% rename from src/backend/core/applications/plugin_revisions.py rename to src/backend/features/workbench/plugin_revisions.py index 2d723abb..54d0cb6e 100644 --- a/src/backend/core/applications/plugin_revisions.py +++ b/src/backend/features/workbench/plugin_revisions.py @@ -4,23 +4,17 @@ from hashlib import sha256 import time -from core.applications import catalog -from core.applications.external import fail -from core.applications.coordination import configuration, runtime -from core.applications.plugin_upgrades import ( - HISTORY, - history_id, - binding_token, - current_revision, - permission_set, -) +from features.workbench import catalog +from features.workbench.external import fail +from features.workbench.coordination import configuration, runtime +from features.workbench.plugin_upgrades import HISTORY, history_id, binding_token, current_revision, permission_set from core.db.models import ContentBlock, InstalledPlugin from core.infra.time import utc_now -from core.plugins.packaging.runtime_assets import materialize +from core.capabilities.plugins.packaging.runtime_assets import materialize from sqlalchemy import or_ -from core.plugins.resources.installation import Installation -from core.applications.installation_binding import exact as resolve +from core.capabilities.plugins.resources.installation import Installation +from features.workbench.installation_binding import exact as resolve LEASE = "workbench_plugin_lease:" LEASE_SECONDS = 90 @@ -43,7 +37,7 @@ def current(db, user_id, key): selected = resolve(db, binding["slug"], user_id, binding["installId"]) row = db.get(InstalledPlugin, selected.install_id) if row is None: - from core.capabilities import registry + from features.desktop.capabilities import registry row = registry.get(selected.install_id, db=db) if row is None or row.profile_id != "local" or row.state == "removed": @@ -144,7 +138,7 @@ def resolve_revision(db, user_id, key, revision, instance_id=None): ): fail(404, "app_not_found") if previous.get("executionScope") == "local": - from core.capabilities import store + from features.desktop.capabilities import store component = store.get("plugin", "local", row.key, revision) if component is None: @@ -182,7 +176,7 @@ def close(db, user_id, key, revision, instance_id): if record: # Keep expiresAt: closed instances do not force early asset collection. record.payload = {**record.payload, "closed": True} - from core.applications.capabilities import revoke_matching + from features.workbench.capabilities import revoke_matching revoke_matching(db, user_id, {key}, revision, instance_id) db.commit() @@ -197,7 +191,7 @@ def collect(db): leases = db.query(ContentBlock).filter(ContentBlock.id.startswith(LEASE, autoescape=True)).all() retained = {r.payload["revision"] for r in leases if r.payload["expiresAt"] > time.time()} retained.update(current_revision(r) for r in db.query(InstalledPlugin).all()) - from core.db.models import DeviceCapabilityInstallation + from features.desktop.models import DeviceCapabilityInstallation retained.update( r.resolved_revision @@ -211,7 +205,7 @@ def collect(db): .filter(or_(PluginResource.expires_at > time.time(), PluginResource.status != "closed")) .all() ) - from core.applications.plugin_upgrades import PREFIX + from features.workbench.plugin_upgrades import PREFIX pending = ( db.query(ContentBlock).filter(ContentBlock.id.startswith(PREFIX, autoescape=True)).all() @@ -219,7 +213,7 @@ def collect(db): retained.update( r.payload.get("candidateRevision") for r in pending if r.payload.get("status") == "pending" ) - from core.applications.plugin_candidate_storage import records + from features.workbench.plugin_candidate_storage import records retained.update( r.payload["revision"] @@ -234,7 +228,7 @@ def collect(db): for history in histories: if history.payload["revision"] in candidates: if history.payload.get("executionScope") == "local": - from core.capabilities import store + from features.desktop.capabilities import store store.remove_revision( "plugin", "local", history.payload["slug"], history.payload["revision"] @@ -248,7 +242,7 @@ def collect(db): if lease.payload["expiresAt"] <= time.time(): db.delete(lease) db.commit() - from core.plugins.packaging.runtime_assets import forget_cache + from core.capabilities.plugins.packaging.runtime_assets import forget_cache for revision in candidates: forget_cache(revision) diff --git a/src/backend/core/applications/plugin_upgrades.py b/src/backend/features/workbench/plugin_upgrades.py similarity index 90% rename from src/backend/core/applications/plugin_upgrades.py rename to src/backend/features/workbench/plugin_upgrades.py index eb2f6516..6a776b86 100644 --- a/src/backend/core/applications/plugin_upgrades.py +++ b/src/backend/features/workbench/plugin_upgrades.py @@ -7,11 +7,11 @@ import zipfile from pathlib import Path -from core.applications.external import fail -from core.applications.coordination import configuration +from features.workbench.external import fail +from features.workbench.coordination import configuration from core.db.models import ContentBlock, InstalledPlugin from core.infra.time import as_utc, utc_now -from core.plugins.packaging.runtime_assets import materialize, retain, pack +from core.capabilities.plugins.packaging.runtime_assets import materialize, retain, pack PREFIX = "workbench_plugin_upgrade:" HISTORY = "workbench_plugin_revision:" @@ -83,8 +83,8 @@ def verify_package(np): fail(409, "host_incompatible") if ui.get("version") != 3: return - from core.applications.plugin_catalog import WORKSPACE_CAPABILITIES - from core.plugins.resources.installation import safe_path + from features.workbench.plugin_catalog import WORKSPACE_CAPABILITIES + from core.capabilities.plugins.resources.installation import safe_path for app in ui.get("contributes", {}).get("applications", []): required = set(app.get("compatibility", {}).get("requiredCapabilities", [])) @@ -95,7 +95,7 @@ def verify_package(np): if not np.package_dir or not safe_path(Path(np.package_dir), module["entry"]).is_file(): fail(409, "package_asset_missing") # Verify the exact HTML and all bundled subassets before activation. - from core.applications.workspace_page import render + from features.workbench.workspace_page import render render(Path(np.package_dir) / "web", module["entry"][4:]) @@ -115,11 +115,11 @@ def public(row, pending): def read(db, install_id, owner_user_id): if install_id.startswith("plugin:local:") and owner_user_id is not None: - from core.applications import local_plugin_upgrades + from features.workbench import local_plugin_upgrades return local_plugin_upgrades.read(db, owner_user_id, install_id) row = owned(db, install_id, owner_user_id) - from core.applications.plugin_candidate_storage import summary + from features.workbench.plugin_candidate_storage import summary return { **public(row, db.get(ContentBlock, record_id(row))), @@ -134,7 +134,7 @@ def prepare(db, row, np, source, approved_revision=None): if old_ui.get("version") != 3 and (np.ui or {}).get("version") != 3: return None verify_package(np) - from core.applications import plugin_candidate_storage as storage + from features.workbench import plugin_candidate_storage as storage packed = pack(Path(np.package_dir)) if np.package_dir and np.ui else None revision = packed[0] if packed else None @@ -211,7 +211,7 @@ def decide( db, install_id, owner_user_id, expected, candidate, decision, reset_state=False, secrets=None ): if install_id.startswith("plugin:local:") and owner_user_id is not None: - from core.applications import local_plugin_upgrades + from features.workbench import local_plugin_upgrades return local_plugin_upgrades.decide( owner_user_id, install_id, expected, candidate, decision, reset_state @@ -232,7 +232,7 @@ def decide( db.rollback() fail(409, "revision_conflict") if decision == "reject": - from core.applications import plugin_candidate_storage as storage + from features.workbench import plugin_candidate_storage as storage storage.change(db, binding_token(row), candidate, "discarded") pending.payload = {**value, "status": "rejected"} @@ -242,8 +242,8 @@ def decide( if value["stateSchemaChanges"] and not reset_state: db.rollback() fail(409, "state_reset_confirmation_required") - from core.plugins.packaging.importer import normalize_plugin_dir - from core.plugins.management.installation import _apply_normalized + from core.capabilities.plugins.packaging.importer import normalize_plugin_dir + from core.capabilities.plugins.management.installation import apply_normalized as _apply_normalized try: np = normalize_plugin_dir(materialize(db, candidate)) diff --git a/src/backend/core/applications/policy.py b/src/backend/features/workbench/policy.py similarity index 94% rename from src/backend/core/applications/policy.py rename to src/backend/features/workbench/policy.py index a09ca437..718e1bd1 100644 --- a/src/backend/core/applications/policy.py +++ b/src/backend/features/workbench/policy.py @@ -1,12 +1,12 @@ """Administrator layout policy. It narrows discovery; it never grants execution.""" -from core.applications.coordination import configuration +from features.workbench.coordination import configuration from copy import deepcopy from uuid import uuid4 -from core.applications import external +from features.workbench import external from core.db.models import ContentBlock from core.infra.time import utc_now from sqlalchemy import update @@ -24,7 +24,7 @@ def read(db): def save(db, body, actor, trace): - from core.applications.catalog import definitions + from features.workbench.catalog import definitions configuration(db) known = {item["applicationKey"] for item in definitions(db)} diff --git a/src/backend/core/applications/preferences.py b/src/backend/features/workbench/preferences.py similarity index 96% rename from src/backend/core/applications/preferences.py rename to src/backend/features/workbench/preferences.py index 3a5ee697..cae7098f 100644 --- a/src/backend/core/applications/preferences.py +++ b/src/backend/features/workbench/preferences.py @@ -1,13 +1,13 @@ """User and environment scoped navigation. Independent of installs and permissions.""" -from core.applications.coordination import runtime +from features.workbench.coordination import runtime import re from copy import deepcopy from hashlib import sha256 -from core.applications import external +from features.workbench import external from core.config.local_mode import local_mode_enabled from core.db.models import ContentBlock from core.infra.time import utc_now diff --git a/src/backend/core/applications/protocol.py b/src/backend/features/workbench/protocol.py similarity index 100% rename from src/backend/core/applications/protocol.py rename to src/backend/features/workbench/protocol.py diff --git a/src/backend/core/plugins/resources/deadlines.py b/src/backend/features/workbench/resource_deadlines.py similarity index 67% rename from src/backend/core/plugins/resources/deadlines.py rename to src/backend/features/workbench/resource_deadlines.py index 0949c7d1..f202c399 100644 --- a/src/backend/core/plugins/resources/deadlines.py +++ b/src/backend/features/workbench/resource_deadlines.py @@ -1,30 +1,14 @@ -"""Owner-bound runner timers enforce application resource leases independently of API sweeps.""" - -from core.applications.coordination import runtime - +"""Workbench leases bind process deadlines to application ownership.""" import hashlib import json from fastapi import HTTPException -from core.applications import activity, origin -from . import store -from .owners import runtime_session - - -def provider_for(row): - from core.sandbox.factory import get_sandbox_provider - - provider = get_sandbox_provider() - descriptor = store.descriptor(row) - method = getattr(provider, "set_process_deadline", None) - if descriptor.get("provider") != provider.name or not callable(method): - raise HTTPException(403, detail={"code": "unsupported_capability"}) - if not descriptor.get("process_id"): - raise HTTPException(410, detail={"code": "instance_closed"}) - return method, descriptor["process_id"] - +from features.workbench.coordination import runtime +from features.workbench import activity, origin +from core.capabilities.plugins.resources.deadlines import provider_for +from core.capabilities.plugins.resources.owners import runtime_session def invalidate(db, row): - from core.applications import external + from features.workbench import external rid, user = row.resource_id, row.user_id db.commit() @@ -103,23 +87,3 @@ async def arm(db, row, *, require_background=False): if result.get("armed") is not True: invalidate(db, row) raise HTTPException(503, detail={"code": "release_failed"}) - - -async def restore(db, *, after="", limit=100): - rows = ( - db.query(store.PluginResource) - .filter( - store.PluginResource.application_key.is_not(None), - store.PluginResource.status == "active", - store.PluginResource.resource_id > after, - ) - .order_by(store.PluginResource.resource_id) - .limit(limit) - .all() - ) - for row in rows: - try: - await arm(db, row) - except HTTPException: - db.rollback() # Existing janitor owns persisted closing cleanup. - return rows[-1].resource_id if len(rows) == limit else "" diff --git a/src/backend/features/workbench/resource_owner.py b/src/backend/features/workbench/resource_owner.py new file mode 100644 index 00000000..4e53d8aa --- /dev/null +++ b/src/backend/features/workbench/resource_owner.py @@ -0,0 +1,65 @@ +"""Workbench-specific binding and revision policy for generic plugin resources.""" +from fastapi import HTTPException +from core.capabilities.plugins.resources.owners import execution_scope + +def application_binding(db, user_id, key, install_id, revision, scope, module_id, selected=None): + from features.workbench import catalog + + item = catalog.readable(db, user_id, key) + if not item["availability"]["available"]: + raise HTTPException(403, "application_unavailable") + binding = item.get("binding", {}) + if ( + item["declaration"]["target"]["kind"] != "plugin" + or binding.get("installId") != install_id + or ( + binding.get("packageRevision") != revision + and (selected is None or selected.revision != revision) + ) + or scope != execution_scope() + or (binding.get("origin") is not None and binding["origin"].get("kind") != scope) + ): + raise HTTPException(409, "resource_binding_mismatch") + target = item["declaration"]["target"] + if selected is not None and binding.get("packageRevision") != revision: + previous = next( + ( + a + for a in (selected.ui.get("contributes") or {}).get("applications", []) + if a["id"] == item["declaration"]["id"] + ), + None, + ) + if previous is None: + raise HTTPException(403, "resource_application_removed") + target = previous["target"] + allowed = {target.get("moduleId")} | { + view.get("moduleId") for view in item["declaration"].get("views", []) + } + if module_id not in allowed: + raise HTTPException(403, "resource_module_not_granted") + return item + + + +def resolve_installation(db, row, user_id): + from features.workbench.installation_binding import exact + selected = exact(db, row.slug, user_id, row.install_id) + if selected.install_id != row.install_id: + raise HTTPException(403, "resource_installation_mismatch") + if selected.revision == row.revision: + return selected + try: + from features.workbench.plugin_revisions import resolve_revision + except ModuleNotFoundError as error: + if error.name != "features.workbench.plugin_revisions": + raise + raise HTTPException(409, "revision_retention_unavailable") from None + return resolve_revision(db, user_id, row.application_key, row.revision, row.instance_id) + + +def collect(db): + from features.workbench.plugin_revisions import collect + collect(db) + from features.workbench.plugin_candidate_storage import janitor + janitor(db) diff --git a/src/backend/core/applications/revocation.py b/src/backend/features/workbench/revocation.py similarity index 94% rename from src/backend/core/applications/revocation.py rename to src/backend/features/workbench/revocation.py index ea294506..be2466e0 100644 --- a/src/backend/core/applications/revocation.py +++ b/src/backend/features/workbench/revocation.py @@ -1,6 +1,6 @@ """Persist credential revocation before best-effort worker cleanup.""" -from core.applications.coordination import runtime +from features.workbench.coordination import runtime import asyncio @@ -13,7 +13,7 @@ PluginResourceAssetTicket, PluginResourceTicket, ) -from core.plugins.resources import service +from core.capabilities.plugins.resources import service from sqlalchemy import delete, or_ EXTERNAL_PREFIXES = ( @@ -31,7 +31,7 @@ def _lock(db, user_id): def _revoke_activity(db, user_id, keys=None): """Stage revoked activity credentials in the caller's transaction.""" - from core.applications import activity + from features.workbench import activity row = db.get(ContentBlock, activity.identity(user_id)) if row is None: @@ -62,7 +62,7 @@ def matches(key): def _revoke_worker_credentials(db, user_id, rows, keys=None): _revoke_activity(db, user_id, keys) - from core.applications.capabilities import revoke_matching + from features.workbench.capabilities import revoke_matching revoke_matching(db, user_id, keys) leases = ( @@ -133,11 +133,11 @@ async def revoke_user_workbench(db, user_id, session_hash=None): def stage_unavailable(db, user_id): """Revoke stale credentials before attempting each affected worker.""" - from core.applications import activity - from core.applications.plugin_revisions import resolve_revision + from features.workbench import activity + from features.workbench.plugin_revisions import resolve_revision - from core.applications.catalog import directory - from core.applications import origin + from features.workbench.catalog import directory + from features.workbench import origin origin.current(db) _lock(db, user_id) @@ -218,7 +218,7 @@ def stage_unavailable(db, user_id): def _release_background_reservation(db, row): - from core.applications import activity + from features.workbench import activity _lock(db, row.user_id) stored = db.get(ContentBlock, activity.identity(row.user_id)) diff --git a/src/backend/core/applications/revocation_sweep.py b/src/backend/features/workbench/revocation_sweep.py similarity index 96% rename from src/backend/core/applications/revocation_sweep.py rename to src/backend/features/workbench/revocation_sweep.py index 080e4e75..8ff5ddd0 100644 --- a/src/backend/core/applications/revocation_sweep.py +++ b/src/backend/features/workbench/revocation_sweep.py @@ -5,7 +5,7 @@ from sqlalchemy import select, union from starlette.concurrency import run_in_threadpool -from core.applications.revocation import EXTERNAL_PREFIXES, stage_unavailable, _close_workers +from features.workbench.revocation import EXTERNAL_PREFIXES, stage_unavailable, _close_workers from core.db.engine import SessionLocal from core.db.models import ContentBlock from core.db.models.plugin_resource import PluginResource diff --git a/src/backend/core/applications/schemas/application.schema.json b/src/backend/features/workbench/schemas/application.schema.json similarity index 100% rename from src/backend/core/applications/schemas/application.schema.json rename to src/backend/features/workbench/schemas/application.schema.json diff --git a/src/backend/core/applications/schemas/host.json b/src/backend/features/workbench/schemas/host.json similarity index 100% rename from src/backend/core/applications/schemas/host.json rename to src/backend/features/workbench/schemas/host.json diff --git a/src/backend/core/applications/schemas/module.schema.json b/src/backend/features/workbench/schemas/module.schema.json similarity index 100% rename from src/backend/core/applications/schemas/module.schema.json rename to src/backend/features/workbench/schemas/module.schema.json diff --git a/src/backend/core/applications/state.py b/src/backend/features/workbench/state.py similarity index 96% rename from src/backend/core/applications/state.py rename to src/backend/features/workbench/state.py index 248f8587..0135b0ca 100644 --- a/src/backend/core/applications/state.py +++ b/src/backend/features/workbench/state.py @@ -5,7 +5,7 @@ serializes CAS and aggregate quotas in one transaction on SQLite and PostgreSQL. """ -from core.applications.coordination import runtime +from features.workbench.coordination import runtime import json @@ -13,7 +13,7 @@ import re from copy import deepcopy from hashlib import sha256 -from core.applications import catalog, external +from features.workbench import catalog, external from core.db.models import ContentBlock from core.infra.time import utc_now from sqlalchemy import update @@ -136,7 +136,7 @@ def save( raw = encode(body["payload"]) # Bootstrap the durable service identity before acquiring the write barrier. # origin.current may commit on first use; it must never release our CAS lock. - from core.applications import origin as service_origin + from features.workbench import origin as service_origin service_origin.current(db) try: @@ -178,7 +178,7 @@ def save( "sizeBytes": size, } if row and body["reset"] and body["schemaVersion"] != current["schemaVersion"]: - from core.applications.state_recovery import preserve_before_reset + from features.workbench.state_recovery import preserve_before_reset preserve_before_reset(db, row) now = utc_now() diff --git a/src/backend/core/applications/state_recovery.py b/src/backend/features/workbench/state_recovery.py similarity index 100% rename from src/backend/core/applications/state_recovery.py rename to src/backend/features/workbench/state_recovery.py diff --git a/src/backend/core/plugins/ui/__init__.py b/src/backend/features/workbench/ui/__init__.py similarity index 100% rename from src/backend/core/plugins/ui/__init__.py rename to src/backend/features/workbench/ui/__init__.py diff --git a/src/backend/core/plugins/ui/contract.py b/src/backend/features/workbench/ui/contract.py similarity index 99% rename from src/backend/core/plugins/ui/contract.py rename to src/backend/features/workbench/ui/contract.py index d877e1c6..34b618d3 100644 --- a/src/backend/core/plugins/ui/contract.py +++ b/src/backend/features/workbench/ui/contract.py @@ -25,7 +25,7 @@ from typing import Any, Dict, List, Optional, Tuple -from core.applications.protocol import UI_VERSION, validate_contributions +from features.workbench.protocol import UI_VERSION, validate_contributions from .declarations import ( _canvas_views, diff --git a/src/backend/core/plugins/ui/contributions.py b/src/backend/features/workbench/ui/contributions.py similarity index 87% rename from src/backend/core/plugins/ui/contributions.py rename to src/backend/features/workbench/ui/contributions.py index 4adc5dbd..7f705be7 100644 --- a/src/backend/core/plugins/ui/contributions.py +++ b/src/backend/features/workbench/ui/contributions.py @@ -6,9 +6,9 @@ from typing import Any, Dict, List, Optional from core.db.models import InstalledPlugin -from core.plugins.management import queries as plugin_queries -from core.plugins.packaging import sources as plugin_sources -from core.plugins.ui.contract import public_contributions +from core.capabilities.plugins.management import queries as plugin_queries +from core.capabilities.plugins.packaging import sources as plugin_sources +from features.workbench.ui.contract import public_contributions from sqlalchemy.orm import Session from sqlalchemy.orm.attributes import flag_modified @@ -30,8 +30,8 @@ def refresh_builtin_ui_contributions(db: Session) -> int: """ import json - from core.plugins.packaging.importer import _ext_or_top, manifest_extensions - from core.plugins.ui.contract import normalize_ui + from core.capabilities.plugins.packaging.importer import ext_or_top as _ext_or_top, manifest_extensions + from features.workbench.ui.contract import normalize_ui # One manifest read per distinct slug, not per install row. ui_by_slug: Dict[str, Optional[Dict[str, Any]]] = {} @@ -39,7 +39,7 @@ def refresh_builtin_ui_contributions(db: Session) -> int: def _bundle_ui(slug: str) -> Optional[Dict[str, Any]]: if slug not in ui_by_slug: ui: Optional[Dict[str, Any]] = None - plugin_dir = plugin_sources._resolve_plugin_dir(slug) + plugin_dir = plugin_sources.resolve_plugin_dir(slug) if plugin_dir is not None: try: manifest = json.loads((plugin_dir / "plugin.json").read_text(encoding="utf-8")) @@ -54,7 +54,7 @@ def _bundle_ui(slug: str) -> Optional[Dict[str, Any]]: updated = 0 rows = db.query(InstalledPlugin).filter(InstalledPlugin.source == "builtin").all() for row in rows: - plugin_dir = plugin_sources._resolve_plugin_dir(row.slug) + plugin_dir = plugin_sources.resolve_plugin_dir(row.slug) if plugin_dir is None: # Bundle removed from this build (e.g. CE tree): leave the row alone. continue @@ -115,7 +115,7 @@ def resolve_installed_ui(db: Session, slug: str, owner_user_id: str) -> Optional one slug directly instead of enumerating every installed plugin (which also re-reads each bundle's plugin.json for display metadata). """ - from core.config.catalog_resolver import resolve_all_runtime_enabled + from core.capabilities.catalog.resolver import resolve_all_runtime_enabled from sqlalchemy import or_ rows = ( @@ -144,6 +144,6 @@ def resolve_installed_ui(db: Session, slug: str, owner_user_id: str) -> Optional enabled_skills = set(eff_skills or []) enabled_mcps = set(eff_mcps or []) enabled = any( - s in enabled_skills for s in (plugin_sources._component_keys(cids, "skills")) - ) or any(m in enabled_mcps for m in (plugin_sources._component_keys(cids, "mcp"))) + s in enabled_skills for s in (plugin_sources.component_keys(cids, "skills")) + ) or any(m in enabled_mcps for m in (plugin_sources.component_keys(cids, "mcp"))) return row.ui_contributions if enabled else None diff --git a/src/backend/core/plugins/ui/data_proxy.py b/src/backend/features/workbench/ui/data_proxy.py similarity index 97% rename from src/backend/core/plugins/ui/data_proxy.py rename to src/backend/features/workbench/ui/data_proxy.py index 40534504..a2b8c3e0 100644 --- a/src/backend/core/plugins/ui/data_proxy.py +++ b/src/backend/features/workbench/ui/data_proxy.py @@ -63,7 +63,7 @@ def __init__(self, message: str, *, status_code: int = 502, retryable: bool = Fa def _config_value(key: str) -> str: - from core.services.system_config import SystemConfigService + from core.config.store.system_config import SystemConfigService try: return str(SystemConfigService.get_instance().get(key) or "").strip() @@ -93,10 +93,10 @@ def _admin_config_field_keys(slug: str) -> List[str]: skills directory) — far too heavy for a per-request path. This mirrors the lightweight pattern of ``plugin_admin_config._has_admin_config_for_slug``. """ - from core.plugins.packaging.importer import _ext_or_top, manifest_extensions - from core.plugins.packaging.sources import _resolve_plugin_dir + from core.capabilities.plugins.packaging.importer import ext_or_top as _ext_or_top, manifest_extensions + from core.capabilities.plugins.packaging.sources import resolve_plugin_dir - plugin_dir = _resolve_plugin_dir(slug) + plugin_dir = resolve_plugin_dir(slug) if plugin_dir is None: return [] try: @@ -350,9 +350,9 @@ def module_asset_path(slug: str, entry_or_path: str) -> Optional["object"]: frontend tree, so this is the only door to them. """ - from core.plugins.packaging.sources import _resolve_plugin_dir + from core.capabilities.plugins.packaging.sources import resolve_plugin_dir - plugin_dir = _resolve_plugin_dir(slug) + plugin_dir = resolve_plugin_dir(slug) if plugin_dir is None: return None web_root = (plugin_dir / "web").resolve() diff --git a/src/backend/core/plugins/ui/declarations.py b/src/backend/features/workbench/ui/declarations.py similarity index 99% rename from src/backend/core/plugins/ui/declarations.py rename to src/backend/features/workbench/ui/declarations.py index e38e50a5..6c338f26 100644 --- a/src/backend/core/plugins/ui/declarations.py +++ b/src/backend/features/workbench/ui/declarations.py @@ -289,7 +289,7 @@ def _modules( if surface not in MODULE_SURFACES and not (allow_workspace and surface == "workspace"): dropper.drop("modules", mid, f"未知 surface：{surface}") continue - from core.applications.protocol import HOST_CAPABILITIES + from features.workbench.protocol import HOST_CAPABILITIES grants = [ g diff --git a/src/backend/core/plugins/ui/primitives.py b/src/backend/features/workbench/ui/primitives.py similarity index 99% rename from src/backend/core/plugins/ui/primitives.py rename to src/backend/features/workbench/ui/primitives.py index 90aeb1ec..7bb8fe9d 100644 --- a/src/backend/core/plugins/ui/primitives.py +++ b/src/backend/features/workbench/ui/primitives.py @@ -6,7 +6,7 @@ import re from typing import Any, Dict, List, Optional, Tuple -from core.applications.protocol import UI_VERSION +from features.workbench.protocol import UI_VERSION logger = logging.getLogger(__name__) diff --git a/src/backend/core/plugins/ui/workspace_enabled.py b/src/backend/features/workbench/ui/workspace_enabled.py similarity index 100% rename from src/backend/core/plugins/ui/workspace_enabled.py rename to src/backend/features/workbench/ui/workspace_enabled.py diff --git a/src/backend/core/applications/workspace_page.py b/src/backend/features/workbench/workspace_page.py similarity index 98% rename from src/backend/core/applications/workspace_page.py rename to src/backend/features/workbench/workspace_page.py index 4b361b7a..6c4824a2 100644 --- a/src/backend/core/applications/workspace_page.py +++ b/src/backend/features/workbench/workspace_page.py @@ -11,8 +11,8 @@ from pathlib import Path from urllib.parse import unquote, urlsplit from fastapi import HTTPException -from core.plugins.resources.installation import safe_path -from core.services.site_service import guess_site_mime +from core.capabilities.plugins.resources.installation import safe_path +from features.sites.service import guess_site_mime MAX_PAGE_BYTES = 8 * 1024 * 1024 diff --git a/src/backend/hugagent_cli/schema.py b/src/backend/hugagent_cli/schema.py new file mode 100644 index 00000000..44f0a8d5 --- /dev/null +++ b/src/backend/hugagent_cli/schema.py @@ -0,0 +1,20 @@ +"""Schema and catalog initialization for the local CLI.""" + +def ensure_schema_and_seed(): + """create_all + built-in catalog seed (mirrors the app's startup hooks).""" + from core.db.engine import SessionLocal, init_db + from core.capabilities.mcp.service import prune_removed_builtin_mcp_servers, seed_builtin_mcp_servers_if_empty + + from features.installed import INSTALLED_FEATURES + from core.harness.registry import load_models + + load_models(INSTALLED_FEATURES) + from core.harness.registry.schema_lifecycle import load_schema_lifecycle + load_schema_lifecycle(INSTALLED_FEATURES) + init_db() + db = SessionLocal() + try: + prune_removed_builtin_mcp_servers(db) + seed_builtin_mcp_servers_if_empty(db) + finally: + db.close() diff --git a/src/backend/integrations/__init__.py b/src/backend/integrations/__init__.py new file mode 100644 index 00000000..479f29da --- /dev/null +++ b/src/backend/integrations/__init__.py @@ -0,0 +1,8 @@ +"""Host composition for installed external-service integrations.""" +def register_credentials(): + from importlib import import_module + from features.installed import INSTALLED_FEATURES + for package in INSTALLED_FEATURES: + register = getattr(import_module(f"{package}.extension"), "register_credentials", None) + if register: + register() diff --git a/src/backend/integrations/api/__init__.py b/src/backend/integrations/api/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/src/backend/integrations/api/__init__.py @@ -0,0 +1 @@ + diff --git a/src/backend/api/routes/v1/integrations.py b/src/backend/integrations/api/integrations.py similarity index 97% rename from src/backend/api/routes/v1/integrations.py rename to src/backend/integrations/api/integrations.py index 48018f0b..3ec0a8e3 100644 --- a/src/backend/api/routes/v1/integrations.py +++ b/src/backend/integrations/api/integrations.py @@ -18,10 +18,10 @@ from core.auth.backend import UserContext, get_current_user from core.db.engine import get_db from core.infra.responses import success_response -from core.services.dingtalk_service import DingTalkService -from core.services.email_service import EmailService -from core.services.lark_service import LarkService -from core.services.yida_service import YidaService +from integrations.dingtalk.service import DingTalkService +from integrations.email.service import EmailService +from integrations.lark.service import LarkService +from integrations.yida.service import YidaService router = APIRouter(prefix="/v1/integrations", tags=["Integrations"]) diff --git a/src/backend/integrations/channels/README.md b/src/backend/integrations/channels/README.md new file mode 100644 index 00000000..2dfbe2d3 --- /dev/null +++ b/src/backend/integrations/channels/README.md @@ -0,0 +1,9 @@ +# 消息通道 + +本模块拥有通道连接、设备中继数据、适配器、入站运行、出站投递和附件工具。FULL 与 CE 均安装模型，运行开关不控制建表。API 注册仍由中央有序路由表负责，路由及权限策略保持原样。 + +入站处理按会话、附件、消息观察、历史拉取及回复投递拆分。inbound_state 是串行锁、去重缓存和限额的唯一所有者。适配器的历史解析、消息解析和发送方法按职责分组；具体适配器保留能力声明及令牌缓存，飞书连接入口保留线程事件循环隔离状态。 + +依赖 core 的认证、数据库、聊天运行、存储和工具注册接口；桌面中继暂时引用 core 中的桌面服务，P5 随归属迁移。外部调用方直接引用所属模块，不提供旧路径转发。通道账号与个人 OAuth 账号相互独立，入站始终采用机器人所有者身份与原资源范围。 + +测试位于 tests/integrations/channels。bot_cases 模块按组存放用例，test_channel_bots 显式收集这些函数，以保留原测试节点身份及失败基线映射；不减少用例或断言。 diff --git a/src/backend/core/channels/__init__.py b/src/backend/integrations/channels/__init__.py similarity index 75% rename from src/backend/core/channels/__init__.py rename to src/backend/integrations/channels/__init__.py index 0a26d72e..c48904c2 100644 --- a/src/backend/core/channels/__init__.py +++ b/src/backend/integrations/channels/__init__.py @@ -8,13 +8,8 @@ See internal design docs. """ -from core.channels.protocol import ( - ChannelAdapter, - ChannelCaps, - InboundMsg, - SendResult, -) -from core.channels.registry import get_adapter, register_adapter, list_adapters +from integrations.channels.protocol import ChannelAdapter, ChannelCaps, InboundMsg, SendResult +from integrations.channels.registry import get_adapter, register_adapter, list_adapters __all__ = [ "ChannelAdapter", diff --git a/src/backend/core/channels/adapters/__init__.py b/src/backend/integrations/channels/adapters/__init__.py similarity index 100% rename from src/backend/core/channels/adapters/__init__.py rename to src/backend/integrations/channels/adapters/__init__.py diff --git a/src/backend/integrations/channels/adapters/dingtalk.py b/src/backend/integrations/channels/adapters/dingtalk.py new file mode 100644 index 00000000..1f830d7b --- /dev/null +++ b/src/backend/integrations/channels/adapters/dingtalk.py @@ -0,0 +1,325 @@ +"""DingTalk inbound channel adapter. + +- **Binding validation**: exchange AppKey/AppSecret for an access_token + (DingTalk v1.0 OAuth) — obtaining one is treated as valid credentials. +- **Inbound (Stream long connection)**: WebSocket via the official + ``dingtalk-stream`` SDK, listening for bot message callbacks + (``ChatbotMessage.TOPIC``). Zero public IP, no callback configuration. If the + SDK is missing, this channel is unavailable but the process is unaffected. +- **Outbound replies**: use the ``sessionWebhook`` carried by each inbound + message (no access_token needed), unified for p2p / group. + Agent replies go through ``send_markdown`` as ``msgtype=markdown`` (DingTalk + natively renders headings/bold/links/lists, etc.; tables/code fences do not + render on mobile, so content is downgraded via + ``core.channels.markdown.downgrade_for_dingtalk`` before sending). + Short system messages (placeholders/receipts) still use ``send_text`` (plain text). + DingTalk bots **do not support editing messages** → ``edit_message`` returns + failure; but they do support **silent recall** of messages sent via the robot + API (group ``groupMessages/recall`` / one-to-one ``otoMessages/batchRecall``, + using the processQueryKey returned at send time). So placeholder messages go + through ``send_placeholder`` via the robot API (obtaining a recallable key), + and before the final reply the upper layer calls ``recall_message`` to remove + the placeholder — visually equivalent to "replace". When the robot API is + unavailable (no permission, etc.), placeholders automatically fall back to + sessionWebhook plain text (not recallable, same behavior as the old version). +- **Outbound proactive delivery (automation scheduled tasks, etc., no inbound + message)**: synthetic messages have no sessionWebhook → ``send_text`` / + ``send_markdown`` automatically fall back to the robot API (``sampleText`` / + ``sampleMarkdown``). +- **Outbound files**: sessionWebhook only accepts text-type messages, cannot + send files → ``push_file`` uses the robot API: first ``media/upload`` to get + a mediaId, then send a robot message. The robot message picks its endpoint by + conversation type: group ``groupMessages/send`` (openConversationId); + one-to-one with a staffId uses ``oToMessages/batchSend``, without one + (proactive delivery) uses the human-bot conversation's openConversationId via + ``privateChatMessages/send``. + The app must be granted the "internal enterprise robot send message" + permission on the DingTalk open platform, otherwise the send endpoints return 403. + +Conversation keying (``external_conversation_id``) always uses DingTalk ``conversationId``: + - one-to-one (conversationType==``1``) conversationId maps one-to-one to a user → naturally one conversation per person; + - group (conversationType==``2``) conversationId is unique per group → the whole group shares one conversation. + +Isomorphic to the [[lark]] adapter. See internal design docs. +""" + +from __future__ import annotations +from integrations.channels.adapters.dingtalk_history import DingTalkHistory +from integrations.channels.adapters.dingtalk_delivery import DingTalkDelivery + +import asyncio +import json +import logging +import os +import re +import shutil +import tempfile +import time +from typing import Any, Callable, Dict, List, Optional + +import httpx + +from datetime import datetime, timezone + +from integrations.channels.markdown import derive_title, downgrade_for_dingtalk +from integrations.channels.protocol import ChannelCaps, ChannelResourceError, HistoryItem, InboundMsg, SendResult, chunk_text +from core.infra.crypto import decrypt_secret + +logger = logging.getLogger(__name__) + +from integrations.channels.adapters.dingtalk_settings import DINGTALK_API_BASE + + +class DingTalkAdapter(DingTalkHistory, DingTalkDelivery): + caps = ChannelCaps( + channel_type="dingtalk", + max_message_len=4000, + supports_markdown=True, + splits_long_messages=False, + supports_long_conn=True, + # DingTalk exposes ``isInAtList`` on every bot callback, so the gate is already + # wired; whether non-@ group messages ever get delivered depends on the tenant + # holding DingTalk's group-message read permission (applied for and reviewed on the + # open platform). Without it, group listening stays inert rather than broken. + supports_group_observe=True, + bind_mode="credentials", + credential_fields=("app_id", "app_secret"), + ) + + # access_token cache: {app_id: (token, expire_epoch)} (only for validate/testing; replies use sessionWebhook) + _token_cache: Dict[str, tuple] = {} + + # ── Credentials ───────────────────────────────────────────────────── + @staticmethod + def _app_secret(conn: Any) -> str: + cfg = conn.config if isinstance(conn.config, dict) else {} + return decrypt_secret(cfg.get("app_secret_enc")) or "" + + async def _access_token(self, app_id: str, app_secret: str) -> str: + cached = self._token_cache.get(app_id) + if cached and cached[1] > time.time() + 60: + return cached[0] + url = f"{DINGTALK_API_BASE}/v1.0/oauth2/accessToken" + async with httpx.AsyncClient(timeout=10) as client: + resp = await client.post(url, json={"appKey": app_id, "appSecret": app_secret}) + data = resp.json() + token = data.get("accessToken") + if not token: + raise RuntimeError(f"钉钉 access_token 获取失败: {data}") + self._token_cache[app_id] = (token, time.time() + int(data.get("expireIn", 7000))) + return token + + async def validate_credentials(self, conn: Any) -> Dict[str, Any]: + secret = self._app_secret(conn) + if not conn.app_id or not secret: + raise RuntimeError("缺少 AppKey / AppSecret") + await self._access_token(conn.app_id, secret) + return {"app_id": conn.app_id} + + # ── Webhook verification (DingTalk uses the Stream long connection; no webhook entry point) ── + def verify_webhook(self, conn: Any, headers: Dict[str, str], body: bytes) -> bool: + return True + + # ── Event → InboundMsg (the dict from the Stream callback) ────────── + def parse_inbound(self, conn: Any, payload: Dict[str, Any]) -> Optional[InboundMsg]: + """DingTalk bot callback data (dict) → InboundMsg. Returns None for non-text/unsupported messages.""" + msgtype = payload.get("msgtype") + text = "" + if msgtype == "text": + text = ((payload.get("text") or {}).get("content") or "").strip() + elif msgtype == "richText": + # Rich text: concatenate the plain-text nodes inside + parts = (payload.get("content") or {}).get("richText") or [] + text = " ".join(p.get("text", "") for p in parts if isinstance(p, dict) and p.get("text")).strip() + attachments = self._extract_attachments(msgtype, payload) + if not text and not attachments: + return None + conv_type = str(payload.get("conversationType") or "1") + chat_type = "group" if conv_type == "2" else "p2p" + conv_id = payload.get("conversationId") or "" + return InboundMsg( + channel_id=conn.channel_id, + channel_type="dingtalk", + text=text, + chat_type=chat_type, + external_conversation_id=conv_id, + sender_id=payload.get("senderStaffId") or payload.get("senderId") or "", + sender_name=payload.get("senderNick") or "", + message_id=payload.get("msgId") or "", + attachments=attachments, + # ``isInAtList`` is DingTalk's own "was the bot in the @ list" flag. With only the + # default permission DingTalk delivers @bot messages exclusively, so this is + # always true and the gate is a no-op; once the tenant is granted group-message + # read, non-@ messages start arriving with it false and silent observation kicks + # in with no further code change. Absent field → treat as addressed. + addressed_to_bot=( + True if chat_type == "p2p" else bool(payload.get("isInAtList", True)) + ), + raw={ + "dingtalk_session_webhook": payload.get("sessionWebhook") or "", + "dingtalk_conversation_id": conv_id, + # Needed by download_resource: messageFiles/download requires robotCode, and + # per DingTalk's docs it is **not** the AppKey — the only reliable source is + # this field on the callback payload (present in Stream mode, which is what + # the long connection uses). + "dingtalk_robot_code": payload.get("robotCode") or "", + }, + ) + + # ── Group history pull (via the official dws CLI) ──────────────────── + # DingTalk has **no** REST OpenAPI for reading a group's message history — the + # server-side API list only covers sending, recalling and read receipts. The capability + # is reachable two ways, and this takes the second: + # 1. DingTalk's MCP gateway (`list_conversation_message_v2` on the group-chat MCP + # server). Works, but needs credentials provisioned per app — extra setup. + # 2. The official `dws` CLI, which the backend **already** runs as a subprocess for + # this user's DingTalk connection and which is **already logged in**. No new + # credentials, no extra configuration: it simply reuses the connection the owner + # has set up. That is why this is the one implemented. + # + # Runs as the connection's owner (per-user $HOME is the credential isolation boundary, + # same as every other backend dws call). The 8s cap keeps a slow CLI from eating into + # the caller's own history budget; any failure returns [] and history is skipped. + _DWS_HISTORY_TIMEOUT_S = 8 + + + _DWS_DOWNLOAD_TIMEOUT_S = 45 + + + + + + # dws renders a file message as **plain text**, not a structured attachment: + # "[文件] 周报.docx fileId: X6GRez... 注意：如需下载使用dws drive download命令下载" + # Parsed back out here, otherwise a pulled file is just an unusable sentence — the agent + # can see a file was shared but has no handle to open it. Images use the same shape. + _DWS_FILE_RE = re.compile( + r"\[(?P文件|图片)\]\s*(?P.+?)\s+fileId:\s*(?P[A-Za-z0-9_\-]+)" + r"(?:\s*注意[：:][^\n]*)?" + ) + + # A DingTalk **online document** shared into a group arrives as a link card, not as a + # file: a title line, an icon, and an alidocs URL. It is a different resource from a Drive + # file — read with `dws doc read`, not downloaded by fileId — and without recognising it + # the agent sees only an unusable link. + _DWS_DOC_RE = re.compile(r"https?://alidocs\.dingtalk\.com/i/nodes/(?P[A-Za-z0-9_\-]+)") + + + + + # ── Inbound attachments ───────────────────────────────────────────── + + + # ── Outbound push (via sessionWebhook, no access_token needed) ─────── + + + + # Lets the upper layer do a whole-content downgrade **before** chunking + # (avoids a table getting bisected by a chunk boundary and becoming unrecognizable for conversion). + prepare_markdown = staticmethod(downgrade_for_dingtalk) + + + + + # ── Placeholder messages: sent via robot API (recallable) → recall as an equivalent of "replace" ── + + + + # ── Outbound file delivery (sessionWebhook cannot send files → use the robot API) ── + + + + # ── Long connection (Stream, requires the dingtalk-stream SDK) ────── + def make_ws_client(self, conn: Any, on_message: Callable[[InboundMsg], None]) -> Any: + """Build the DingTalk Stream long-connection runner. ``on_message`` is invoked synchronously on the SDK thread (dispatched to the main loop). + + dingtalk_stream is lazily imported; if not installed a RuntimeError is + raised, which the manager records as an error without affecting the + process. The SDK's ``start()`` is async and ``start_forever()`` cannot + be stopped → wrap it in a controllable runner so the manager can start + and stop cleanly (actually disconnecting when a bot is disabled). + """ + try: + import dingtalk_stream + except Exception as exc: # noqa: BLE001 + raise RuntimeError("dingtalk-stream SDK 未安装，长连接不可用") from exc + + adapter = self + app_id = conn.app_id + secret = self._app_secret(conn) + + class _Handler(dingtalk_stream.ChatbotHandler): + async def process(self, callback): # type: ignore[override] + try: + inbound = adapter.parse_inbound(conn, callback.data or {}) + if inbound is not None: + on_message(inbound) + except Exception: # noqa: BLE001 + logger.exception("[dingtalk] Stream 事件处理失败 channel_id=%s", conn.channel_id) + return dingtalk_stream.AckMessage.STATUS_OK, "OK" + + def _build_client(): + sdk_logger = logging.getLogger("dingtalk_stream") + sdk_logger.setLevel(logging.WARNING) + credential = dingtalk_stream.Credential(app_id, secret) + client = dingtalk_stream.DingTalkStreamClient(credential, logger=sdk_logger) + client.register_callback_handler( + dingtalk_stream.chatbot.ChatbotMessage.TOPIC, _Handler() + ) + return client + + return _DingTalkStreamRunner(_build_client) + + +class _DingTalkStreamRunner: + """Wraps dingtalk-stream's async ``client.start()`` into the "blocking start() + stoppable stop()" the manager expects. + + Creates a dedicated event loop in the worker thread to run ``client.start()`` + (the SDK reconnects on its own); ``stop()`` closes the websocket and cancels + the task from the main thread via ``call_soon_threadsafe``, making + ``start()`` return → the manager's thread body sees stop_flag set and exits + cleanly. + """ + + def __init__(self, build_client: Callable[[], Any]): + self._build_client = build_client + self._loop: Optional[asyncio.AbstractEventLoop] = None + self._client: Any = None + self._task: Optional[asyncio.Task] = None + + def start(self) -> None: + loop = asyncio.new_event_loop() + self._loop = loop + asyncio.set_event_loop(loop) + try: + self._client = self._build_client() + self._task = loop.create_task(self._client.start()) + loop.run_until_complete(self._task) + except asyncio.CancelledError: + pass + finally: + try: + loop.run_until_complete(loop.shutdown_asyncgens()) + except Exception: # noqa: BLE001 + pass + loop.close() + + def stop(self) -> None: + loop, task, client = self._loop, self._task, self._client + if loop is None or loop.is_closed(): + return + + def _cancel() -> None: + ws = getattr(client, "websocket", None) + if ws is not None: + try: + asyncio.ensure_future(ws.close()) + except Exception: # noqa: BLE001 + pass + if task is not None: + task.cancel() + + try: + loop.call_soon_threadsafe(_cancel) + except Exception: # noqa: BLE001 + pass diff --git a/src/backend/integrations/channels/adapters/dingtalk_delivery.py b/src/backend/integrations/channels/adapters/dingtalk_delivery.py new file mode 100644 index 00000000..faca8bc4 --- /dev/null +++ b/src/backend/integrations/channels/adapters/dingtalk_delivery.py @@ -0,0 +1,213 @@ +"""dingtalk delivery behavior.""" +from __future__ import annotations +import json +import logging +from typing import Any, Dict, Optional +import httpx +from integrations.channels.markdown import derive_title, downgrade_for_dingtalk +from integrations.channels.protocol import InboundMsg, SendResult, chunk_text +from integrations.channels.adapters.dingtalk_settings import DINGTALK_API_BASE +logger = logging.getLogger(__name__) + +class DingTalkDelivery: + @staticmethod + def _session_webhook(inbound: InboundMsg) -> str: + return (inbound.raw or {}).get("dingtalk_session_webhook") or "" + + async def _post_webhook(self, webhook: str, payload: Dict[str, Any]) -> SendResult: + if not webhook: + return SendResult.fail("forbidden", "缺少 sessionWebhook（可能已过期）") + try: + async with httpx.AsyncClient(timeout=15) as client: + resp = await client.post(webhook, json=payload) + data = resp.json() + except Exception as exc: # noqa: BLE001 + return SendResult.fail("transient", str(exc)) + if data.get("errcode", 0) == 0: + return SendResult.ok() + kind = "rate_limited" if data.get("errcode") in (130101, 88) else "unknown" + return SendResult.fail(kind, f"errcode={data.get('errcode')} {data.get('errmsg')}") + + async def send_text(self, conn: Any, inbound: InboundMsg, text: str) -> SendResult: + webhook = self._session_webhook(inbound) + if webhook: + return await self._post_webhook( + webhook, {"msgtype": "text", "text": {"content": text}} + ) + # Proactive delivery (automation scheduled tasks, etc.) has no inbound sessionWebhook → use the robot API + try: + token = await self._access_token(conn.app_id, self._app_secret(conn)) + except Exception as exc: # noqa: BLE001 + return SendResult.fail("transient", str(exc)) + return await self._robot_send(token, conn, inbound, "sampleText", {"content": text}) + + async def send_markdown(self, conn: Any, inbound: InboundMsg, text: str) -> SendResult: + """Send a DingTalk markdown message (natively rendered). The downgrade is idempotent; calling directly or via prepare_markdown is equally safe. + + title is required for DingTalk's markdown type; it shows in the + conversation-list summary / push notification and is derived from the + first line of the body. + """ + md = downgrade_for_dingtalk(text) + title = derive_title(md) + webhook = self._session_webhook(inbound) + if webhook: + return await self._post_webhook( + webhook, {"msgtype": "markdown", "markdown": {"title": title, "text": md}} + ) + # Proactive delivery → robot-API markdown message + try: + token = await self._access_token(conn.app_id, self._app_secret(conn)) + except Exception as exc: # noqa: BLE001 + return SendResult.fail("transient", str(exc)) + return await self._robot_send( + token, conn, inbound, "sampleMarkdown", {"title": title, "text": md} + ) + + async def push(self, conn: Any, inbound: InboundMsg, content: str) -> SendResult: + md = downgrade_for_dingtalk(content) + chunks = chunk_text(md, self.caps.max_message_len) or [md] + first: Optional[SendResult] = None + for i, c in enumerate(chunks): + r = await self.send_markdown(conn, inbound, c) + if i == 0: + first = r + if not r.success: + return r + return first or SendResult.fail("unknown", "空内容") + + async def edit_message(self, conn: Any, message_id: str, text: str) -> SendResult: + # DingTalk bots cannot edit messages — the upper-layer placeholder logic + # therefore switches to "recall + resend" (see recall_message). + return SendResult.fail("bad_format", "钉钉不支持编辑消息") + + async def send_placeholder(self, conn: Any, inbound: InboundMsg, text: str) -> SendResult: + """Send a recallable placeholder message via the robot API; message_id is the processQueryKey (recall credential). + + If the robot API is unavailable (no permission / token failure) → fall + back to sessionWebhook plain text; then there is no message_id and the + upper layer will not attempt recall — behavior degrades to the old + version (placeholder stays, reply is sent fresh). + """ + try: + token = await self._access_token(conn.app_id, self._app_secret(conn)) + r = await self._robot_send(token, conn, inbound, "sampleText", {"content": text}) + if r.success: + return r + except Exception: # noqa: BLE001 + logger.debug("[dingtalk] 机器人占位发送失败，回退 webhook", exc_info=True) + return await self.send_text(conn, inbound, text) + + def _recall_url_body( + self, conn: Any, inbound: InboundMsg, message_id: str + ) -> Optional[tuple]: + """Pick the recall endpoint by conversation type. Group chats need openConversationId; one-to-one batchRecall needs only the key.""" + body: Dict[str, Any] = {"robotCode": conn.app_id, "processQueryKeys": [message_id]} + if inbound.chat_type == "group": + conv_id = inbound.external_conversation_id or (inbound.raw or {}).get("dingtalk_conversation_id") or "" + if not conv_id: + return None + body["openConversationId"] = conv_id + return f"{DINGTALK_API_BASE}/v1.0/robot/groupMessages/recall", body + return f"{DINGTALK_API_BASE}/v1.0/robot/otoMessages/batchRecall", body + + async def recall_message(self, conn: Any, inbound: InboundMsg, message_id: str) -> SendResult: + """Silently recall a message sent via the robot API (the client shows no recall notice).""" + if not message_id: + return SendResult.fail("bad_format", "缺少 processQueryKey") + target = self._recall_url_body(conn, inbound, message_id) + if target is None: + return SendResult.fail("bad_format", "缺少 openConversationId") + url, body = target + try: + token = await self._access_token(conn.app_id, self._app_secret(conn)) + async with httpx.AsyncClient(timeout=15) as client: + resp = await client.post(url, headers={"x-acs-dingtalk-access-token": token}, json=body) + data = resp.json() + except Exception as exc: # noqa: BLE001 + return SendResult.fail("transient", str(exc)) + if resp.status_code == 200: + return SendResult.ok(message_id) + return SendResult.fail("unknown", f"撤回失败: {resp.status_code} {str(data)[:200]}") + + async def push_file( + self, conn: Any, inbound: InboundMsg, content: bytes, filename: str, mime_type: str + ) -> SendResult: + secret = self._app_secret(conn) + if not conn.app_id or not secret: + return SendResult.fail("forbidden", "缺少 AppKey / AppSecret") + try: + token = await self._access_token(conn.app_id, secret) + except Exception as exc: # noqa: BLE001 + return SendResult.fail("transient", str(exc)) + + is_image = (mime_type or "").startswith("image/") + media_id = await self._upload_media(token, content, filename, "image" if is_image else "file") + if isinstance(media_id, SendResult): + return media_id + + if is_image: + # sampleImageMsg's photoURL accepts a mediaId directly + msg_key, msg_param = "sampleImageMsg", {"photoURL": media_id} + else: + ext = (filename.rsplit(".", 1)[-1] if "." in filename else "").lower() or "file" + msg_key = "sampleFile" + msg_param = {"mediaId": media_id, "fileName": filename, "fileType": ext} + return await self._robot_send(token, conn, inbound, msg_key, msg_param) + + async def _upload_media(self, token: str, content: bytes, filename: str, media_type: str): + """Upload media in exchange for a mediaId; returns SendResult on failure (str on success).""" + url = f"https://oapi.dingtalk.com/media/upload?access_token={token}&type={media_type}" + try: + async with httpx.AsyncClient(timeout=60) as client: + resp = await client.post(url, files={"media": (filename, content)}) + data = resp.json() + except Exception as exc: # noqa: BLE001 + return SendResult.fail("transient", f"媒体上传失败: {exc}") + media_id = data.get("media_id") + if data.get("errcode", 0) != 0 or not media_id: + return SendResult.fail("unknown", f"媒体上传失败: errcode={data.get('errcode')} {data.get('errmsg')}") + return media_id + + async def _robot_send( + self, token: str, conn: Any, inbound: InboundMsg, msg_key: str, msg_param: Dict[str, Any] + ) -> SendResult: + """Robot proactive message: group chats use openConversationId, one-to-one uses the recipient's staffId.""" + body: Dict[str, Any] = { + "robotCode": conn.app_id, # for internal enterprise apps, robotCode == appKey + "msgKey": msg_key, + "msgParam": json.dumps(msg_param, ensure_ascii=False), + } + if inbound.chat_type == "group": + conv_id = inbound.external_conversation_id or (inbound.raw or {}).get("dingtalk_conversation_id") or "" + if not conv_id: + return SendResult.fail("bad_format", "缺少 openConversationId") + url = f"{DINGTALK_API_BASE}/v1.0/robot/groupMessages/send" + body["openConversationId"] = conv_id + else: + staff_id = inbound.sender_id or "" + if staff_id: + url = f"{DINGTALK_API_BASE}/v1.0/robot/oToMessages/batchSend" + body["userIds"] = [staff_id] + else: + # In proactive-delivery scenarios no staffId is available → send directly via the human-bot one-to-one conversation's openConversationId + conv_id = inbound.external_conversation_id or (inbound.raw or {}).get("dingtalk_conversation_id") or "" + if not conv_id: + return SendResult.fail("bad_format", "缺少接收人 staffId / openConversationId") + url = f"{DINGTALK_API_BASE}/v1.0/robot/privateChatMessages/send" + body["openConversationId"] = conv_id + try: + async with httpx.AsyncClient(timeout=30) as client: + resp = await client.post(url, headers={"x-acs-dingtalk-access-token": token}, json=body) + data = resp.json() + except Exception as exc: # noqa: BLE001 + return SendResult.fail("transient", str(exc)) + key = data.get("processQueryKey") or data.get("processQueryKeys") + if resp.status_code == 200 and key: + # The processQueryKey is the recall credential, returned to the upper layer as message_id (used by recall_message) + if isinstance(key, list): + key = key[0] if key else None + return SendResult.ok(key if isinstance(key, str) else None) + code = data.get("code") or resp.status_code + kind = "forbidden" if resp.status_code == 403 else "unknown" + return SendResult.fail(kind, f"机器人消息发送失败: {code} {str(data)[:200]}") diff --git a/src/backend/integrations/channels/adapters/dingtalk_history.py b/src/backend/integrations/channels/adapters/dingtalk_history.py new file mode 100644 index 00000000..56398c59 --- /dev/null +++ b/src/backend/integrations/channels/adapters/dingtalk_history.py @@ -0,0 +1,361 @@ +"""dingtalk history behavior.""" +from __future__ import annotations +import json +import logging +import os +import shutil +import tempfile +from typing import Any, Dict, List, Optional +import httpx +from datetime import datetime, timezone +from integrations.channels.protocol import ChannelResourceError, HistoryItem, InboundMsg +from integrations.channels.adapters.dingtalk_settings import DINGTALK_API_BASE +logger = logging.getLogger(__name__) + +class DingTalkHistory: + async def fetch_history( + self, + conn: Any, + conversation_id: str, + *, + since_ms: int, + limit: int, + newest_first: bool = False, + ) -> List[HistoryItem]: + """`dws chat message list --group --time ` → normalized history items.""" + conv = (conversation_id or "").strip() + owner = getattr(conn, "owner_user_id", "") or "" + if not conv or not owner: + return [] + from integrations.dingtalk.service import _run_dws + + # dws takes a local-time string, not epoch ms. `--forward` picks the direction: + # true = messages after --time (incremental from the cursor); false = messages + # before --time, which anchored at *now* is how the cold start gets the most recent + # ones instead of the oldest of a week-long window. + anchor_ms = int(datetime.now(timezone.utc).timestamp() * 1000) if newest_first else since_ms + anchor = datetime.fromtimestamp(max(anchor_ms, 0) / 1000, tz=timezone.utc).astimezone() + args = [ + "chat", "message", "list", + "--group", conv, + "--time", anchor.strftime("%Y-%m-%d %H:%M:%S"), + "--limit", str(max(1, min(int(limit or 50), 100))), + "--forward=false" if newest_first else "--forward=true", + "--format", "json", + ] + try: + out, err, code = await _run_dws(owner, args, timeout=self._DWS_HISTORY_TIMEOUT_S) + except Exception: # noqa: BLE001 + logger.warning("[dingtalk] dws 群历史拉取异常 conv=%s", conv[:24], exc_info=True) + return [] + if code != 0 or not out.strip(): + # Not logged in / no permission / timeout all land here. Debug, not warning: + # a tenant without the DingTalk connection set up would otherwise log on every + # single group message. + logger.debug( + "[dingtalk] dws 群历史拉取未成功 code=%s err=%s", code, (err or "")[:160] + ) + return [] + try: + return self._parse_dws_history(json.loads(out)) + except Exception: # noqa: BLE001 + logger.debug("[dingtalk] dws 输出解析失败: %s", out[:200], exc_info=True) + return [] + + @staticmethod + def _dws_business_error(raw_out: str) -> Optional[str]: + """Pull the human-readable reason out of a dws error envelope, if that's what this is. + + Worth surfacing rather than collapsing to "download failed": the messages are + specific and actionable ("this data belongs to organization X while the tool is + configured for organization Y"), and reading that as an expiry sends the user + looking in entirely the wrong place. + """ + try: + err = (json.loads(raw_out or "{}") or {}).get("error") + except Exception: # noqa: BLE001 + return None + if not isinstance(err, dict): + return None + msg = str(err.get("message") or "").strip() + return msg or None + + async def _read_doc_via_dws(self, conn: Any, node_id: str) -> Optional[bytes]: + """`dws doc read --node --content-format markdown` → markdown bytes. + + A DingTalk online document has no file to download; it is read as content. Returning + markdown keeps it in the same artifact pipeline as everything else. + """ + owner = getattr(conn, "owner_user_id", "") or "" + if not owner: + return None + from integrations.dingtalk.service import _run_dws + + out, err, code = await _run_dws( + owner, + ["doc", "read", "--node", node_id, "--content-format", "markdown", "--format", "json"], + timeout=self._DWS_DOWNLOAD_TIMEOUT_S, + ) + reason = self._dws_business_error(out) or self._dws_business_error(err) + if reason: + logger.warning("[dingtalk] 钉钉文档读取被拒 node=%s: %s", node_id[:16], reason[:200]) + raise ChannelResourceError(reason) + if code != 0 or not out.strip(): + logger.debug("[dingtalk] 钉钉文档读取失败 code=%s err=%s", code, (err or "")[:160]) + return None + try: + payload = json.loads(out) + except Exception: # noqa: BLE001 + return out.encode("utf-8") + # Content sits under a few possible keys depending on CLI version; fall back to the + # whole payload rather than silently returning nothing. + for key in ("content", "markdown", "data", "result"): + val = payload.get(key) if isinstance(payload, dict) else None + if isinstance(val, str) and val.strip(): + return val.encode("utf-8") + if isinstance(val, dict): + for sub in ("content", "markdown", "text"): + inner = val.get(sub) + if isinstance(inner, str) and inner.strip(): + return inner.encode("utf-8") + return json.dumps(payload, ensure_ascii=False).encode("utf-8") + + async def _download_via_dws(self, conn: Any, file_id: str) -> Optional[bytes]: + """`dws drive download --node ` → bytes. + + The CLI only writes to a local path (it does the signed-URL dance internally), so the + bytes are read back from a temp file that is always removed, including on failure — + this runs on the backend host, not in a disposable sandbox. + """ + owner = getattr(conn, "owner_user_id", "") or "" + if not owner: + return None + from integrations.dingtalk.service import _run_dws + + tmpdir = tempfile.mkdtemp(prefix="dws_dl_") + target = os.path.join(tmpdir, "download.bin") + try: + out, err, code = await _run_dws( + owner, + ["drive", "download", "--node", file_id, "--output", target, "--format", "json"], + timeout=self._DWS_DOWNLOAD_TIMEOUT_S, + ) + if code != 0 or not os.path.exists(target): + logger.warning( + "[dingtalk] dws 文件下载失败 file_id=%s code=%s err=%s", + file_id[:16], code, (err or out or "")[:160], + ) + return None + with open(target, "rb") as fh: + return fh.read() + except Exception: # noqa: BLE001 + logger.warning("[dingtalk] dws 文件下载异常 file_id=%s", file_id[:16], exc_info=True) + return None + finally: + shutil.rmtree(tmpdir, ignore_errors=True) + + @staticmethod + def _dws_ts_ms(value: Any) -> int: + """dws ``createTime`` → epoch ms. + + The CLI returns a **local-time string** (``"2026-08-08 21:57:33"``), not epoch + milliseconds. Parsing it as an int silently yielded 0 for every message, which left + the pull cursor pinned at the cold-start window — so every @ re-fetched the same 24 + hours forever. Numeric forms are still accepted in case a CLI version returns them. + """ + if isinstance(value, (int, float)) and value > 0: + return int(value) + text = str(value or "").strip() + if not text: + return 0 + if text.isdigit(): + return int(text) + try: + naive = datetime.strptime(text, "%Y-%m-%d %H:%M:%S") + except ValueError: + return 0 + # Local time, as the CLI prints it; astimezone() attaches the process zone. + return int(naive.astimezone().timestamp() * 1000) + + @classmethod + def _extract_dws_docs(cls, content: str) -> List[Dict[str, Any]]: + """DingTalk Docs link cards → attachment refs (deduped by node id).""" + nodes = {m.group("node") for m in cls._DWS_DOC_RE.finditer(content or "")} + if not nodes: + return [] + # The card's first non-empty line is the document title; skip the markdown image line + # and the "创建者：" byline, neither of which names the document. + title = "" + for line in (content or "").splitlines(): + line = line.strip() + if line and not line.startswith("![") and not line.startswith("[http"): + if line.startswith("创建者"): + continue + title = line + break + return [ + {"kind": "doc", "key": node, "name": title or "钉钉文档"} + for node in sorted(nodes) + ] + + @classmethod + def _extract_dws_files(cls, content: str) -> tuple: + """(cleaned_text, attachment_refs) for one history message body.""" + refs: List[Dict[str, Any]] = [] + for m in cls._DWS_FILE_RE.finditer(content or ""): + refs.append({ + "kind": "image" if m.group("kind") == "图片" else "file", + "key": m.group("fid"), + "name": m.group("name").strip(), + }) + doc_refs = cls._extract_dws_docs(content) + if doc_refs: + # The card body is title + icon + a long URL repeated twice; the attachment entry + # carries title and handle, so keeping the raw card would just burn prompt space. + return "", refs + doc_refs + if not refs: + return (content or "").strip(), [] + # Drop the matched sentences: the attachment ref already renders name + handle, and + # the "use dws drive download" tail is instruction noise aimed at a CLI user. + return cls._DWS_FILE_RE.sub("", content or "").strip(), refs + + @staticmethod + def _parse_dws_history(body: Dict[str, Any]) -> List[HistoryItem]: + """dws JSON output → HistoryItem list. + + Field names verified against live CLI output — they are **not** the ones the bot + callback uses, which is the trap here: + {"result": {"messages": [{"openMessageId", "sender", "content", "createTime", + "openConversationId", "senderOpenDingTalkId"}], ...}} + A bare ``{"messages": [...]}`` is accepted too, so a CLI version that drops the + envelope does not silently yield nothing. + """ + # Preserve the concrete adapter binding used by history parsing. + from integrations.channels.adapters.dingtalk import DingTalkAdapter + + if not isinstance(body, dict): + return [] + if body.get("success") is False or body.get("errorCode"): + return [] + result = body.get("result") if isinstance(body.get("result"), dict) else body + out: List[HistoryItem] = [] + for m in result.get("messages") or []: + if not isinstance(m, dict): + continue + # Written out rather than chained: `a or b if cond else ""` binds the conditional + # looser than the `or`, which would silently make every message look empty. + text = m.get("content") or "" + if not text and isinstance(m.get("text"), dict): + text = m["text"].get("content") or "" + if not isinstance(text, str): + text = str(text) + text, refs = DingTalkAdapter._extract_dws_files(text) + if not text and not refs: + continue + out.append(HistoryItem( + # openMessageId is the real id; msgId/messageId kept as fallbacks only. + message_id=str( + m.get("openMessageId") or m.get("msgId") or m.get("messageId") or "" + ), + # `sender` is already the display name — no directory lookup needed. + sender_name=str( + m.get("sender") or m.get("senderNick") or m.get("senderName") or "群成员" + ), + text=text, + ts_ms=DingTalkAdapter._dws_ts_ms(m.get("createTime") or m.get("createAt")), + attachments=refs, + # Marks which download route this file needs: history files live on DingTalk + # Drive and are fetched by fileId, not by the bot callback's downloadCode. + raw={"dws_drive": True} if refs else {}, + )) + return out + + @staticmethod + def _extract_attachments(msgtype: str, payload: Dict[str, Any]) -> list: + """Message → attachment list. ``key`` carries DingTalk's ``downloadCode``. + + Content layout per DingTalk's "received message types" reference: + picture → {pictureDownloadCode, downloadCode} + file → {spaceId, fileName, downloadCode, fileId} + audio → {downloadCode, recognition} + video → {duration, videoType, downloadCode} + richText → {richText: [ ...nodes, image nodes carry downloadCode... ]} + Everything is read defensively (``downloadCode`` looked up in more than one place) + because these payloads are only loosely specified and vary by client version — a + missing field must degrade to "no attachment", never raise and kill the connection. + """ + content = payload.get("content") if isinstance(payload.get("content"), dict) else {} + out: list = [] + + def _add(kind: str, code: Any, name: str) -> None: + if code: + out.append({"kind": kind, "key": str(code), "name": name}) + + if msgtype == "picture": + code = content.get("downloadCode") or content.get("pictureDownloadCode") + _add("image", code, f"dingtalk_image_{payload.get('msgId', '')[:8] or 'x'}.png") + elif msgtype == "file": + _add("file", content.get("downloadCode"), content.get("fileName") or "file.bin") + elif msgtype == "audio": + _add("file", content.get("downloadCode"), "dingtalk_audio.amr") + elif msgtype == "video": + _add("file", content.get("downloadCode"), "dingtalk_video.mp4") + elif msgtype == "richText": + for i, node in enumerate(content.get("richText") or []): + if not isinstance(node, dict): + continue + code = node.get("downloadCode") or node.get("pictureDownloadCode") + _add("image", code, f"dingtalk_richtext_{i}.png") + return out + + async def download_resource( + self, conn: Any, inbound: InboundMsg, attachment: Dict[str, Any] + ) -> Optional[bytes]: + """Download an inbound attachment: downloadCode → temporary downloadUrl → bytes. + + Two hops, per DingTalk's design: + POST /v1.0/robot/messageFiles/download {downloadCode, robotCode} → {downloadUrl} + GET downloadUrl → bytes + The temporary URL serves the file with a ``.file`` extension; we ignore that and keep + the filename taken from the message, so the artifact keeps its real extension. + Download codes expire, so a deferred fetch can legitimately fail — returning None + (rather than raising) lets the caller degrade to "attachment unavailable". + """ + code = attachment.get("key") + if not code: + return None + # Files seen through *history* are DingTalk Drive entries addressed by fileId, not + # bot-callback attachments addressed by a short-lived downloadCode — different + # storage, different API. The marker is set when the history item is parsed. + if attachment.get("kind") == "doc": + return await self._read_doc_via_dws(conn, str(code)) + if (inbound.raw or {}).get("dws_drive"): + return await self._download_via_dws(conn, str(code)) + robot_code = (inbound.raw or {}).get("dingtalk_robot_code") or "" + if not robot_code: + logger.warning("[dingtalk] 缺少 robotCode，无法下载附件 code=%s", str(code)[:12]) + return None + try: + token = await self._access_token(conn.app_id, self._app_secret(conn)) + async with httpx.AsyncClient(timeout=30) as client: + resp = await client.post( + f"{DINGTALK_API_BASE}/v1.0/robot/messageFiles/download", + headers={"x-acs-dingtalk-access-token": token}, + json={"downloadCode": code, "robotCode": robot_code}, + ) + if resp.status_code != 200: + logger.warning( + "[dingtalk] 下载地址获取失败 status=%s body=%s", + resp.status_code, resp.text[:200], + ) + return None + url = (resp.json() or {}).get("downloadUrl") + if not url: + return None + fr = await client.get(url) + if fr.status_code == 200: + return fr.content + logger.warning("[dingtalk] 附件下载失败 status=%s", fr.status_code) + except Exception: # noqa: BLE001 + logger.exception("[dingtalk] 附件下载异常 code=%s", str(code)[:12]) + return None diff --git a/src/backend/integrations/channels/adapters/dingtalk_settings.py b/src/backend/integrations/channels/adapters/dingtalk_settings.py new file mode 100644 index 00000000..56b7348f --- /dev/null +++ b/src/backend/integrations/channels/adapters/dingtalk_settings.py @@ -0,0 +1,3 @@ +"""dingtalk API configuration.""" +import os +DINGTALK_API_BASE = "https://api.dingtalk.com" diff --git a/src/backend/integrations/channels/adapters/lark.py b/src/backend/integrations/channels/adapters/lark.py new file mode 100644 index 00000000..ae5a4b83 --- /dev/null +++ b/src/backend/integrations/channels/adapters/lark.py @@ -0,0 +1,261 @@ +"""Feishu (Lark) inbound channel adapter. + +- **Binding validation / outbound push**: Feishu OpenAPI REST (httpx) with a + tenant_access_token; no SDK needed. +- **Webhook decryption/signature verification**: AES-256-CBC (Encrypt Key) + + signature check, pure cryptography; no SDK needed. +- **Long connection (WebSocket)**: uses the official ``lark_oapi`` SDK's + ws.Client (lazy import; if the SDK is missing only the long connection is + unavailable, webhook/push are unaffected). Long connection is recommended in + production: the user only needs App ID/Secret, no public callback URL. + +Conversation keying (``external_conversation_id``) always uses Feishu +``message.chat_id``: + - a p2p chat_id maps one-to-one to a user → naturally one conversation per DM peer; + - a group chat_id is unique per group → the whole group shares one conversation. +Outbound push also sends by chat_id (``receive_id_type=chat_id``), unified for p2p/group. + +See internal design docs. +""" + +from __future__ import annotations +from integrations.channels.adapters.lark_messages import LarkMessages +from integrations.channels.adapters.lark_delivery import LarkDelivery + +import asyncio +import base64 +import hashlib +import json +import logging +import os +import re +import threading +import time +from typing import Any, Callable, Dict, List, Optional + +import httpx + +from integrations.channels.protocol import ChannelCaps, HistoryItem, InboundMsg, SendResult +from core.infra.crypto import decrypt_secret + +logger = logging.getLogger(__name__) + +from integrations.channels.adapters.lark_settings import LARK_API_BASE +_MENTION_RE = re.compile(r"@_user_\d+") + + +# ── lark_oapi ws event-loop isolation (supports multiple coexisting bots) ──── +# Root cause: ``lark_oapi/ws/client.py`` runs everything on a single +# **module-level global** ``loop`` (``loop.run_until_complete`` in ``start()``, +# ``loop.create_task`` in ``_connect`` ...). Each bot's long connection runs in +# its own worker thread, but they all share that one global → when the second +# bot starts it overwrites the global loop with its own thread's loop, and the +# first bot's already-running coroutines then call ``loop.create_task`` on +# someone else's loop → "This event loop is already running" / cross-thread +# crashes. Single-bot setups hit the same cause sporadically (the global gets +# rewritten on reconnect/retry). +# Fix: replace that global ``loop`` with a **per-thread-resolving proxy** — each +# worker thread always resolves to its own thread's loop (prepare_ws_thread +# writes it into thread-local storage), fully eliminating cross-thread sharing +# and overwriting. +_ws_loop_tls = threading.local() +_ws_proxy_installed = False +_ws_proxy_lock = threading.Lock() + + +class _PerThreadLoopProxy: + """Masquerades as an event loop: every attribute/method access is forwarded to the real loop bound to the **current thread**.""" + + def __getattr__(self, name: str) -> Any: + loop = getattr(_ws_loop_tls, "loop", None) + if loop is None: + loop = asyncio.get_event_loop() + return getattr(loop, name) + + +def _install_ws_loop_proxy() -> None: + """One-time replacement of lark_oapi.ws.client's module-level global loop with the per-thread proxy (idempotent).""" + global _ws_proxy_installed + if _ws_proxy_installed: + return + with _ws_proxy_lock: + if _ws_proxy_installed: + return + try: + import lark_oapi.ws.client as _wsc + _wsc.loop = _PerThreadLoopProxy() + _ws_proxy_installed = True + except Exception: # noqa: BLE001 — if the SDK is missing, the long connection is unavailable anyway + pass + + +class LarkAdapter(LarkMessages, LarkDelivery): + caps = ChannelCaps( + channel_type="lark", + max_message_len=4000, # Feishu's per-message text cap is fairly high; leave headroom for chunking + supports_markdown=False, # uses text messages; rich text (post) left for later + splits_long_messages=False, + supports_long_conn=True, + # Feishu gates group delivery with three mutually distinct scopes: + # im:message.group_at_msg → only messages @-ing this bot + # im:message.group_at_msg.include_bot:readonly → messages @-ing any bot + # im:message.group_msg (sensitive) → every group message + # Group listening needs the last one, plus republishing the app version; with the + # others Feishu never pushes a non-@ message and observe_all stays inert. Note the + # @-gate is worth running under the middle scope too: it stops us answering a message + # that @'d somebody else's bot. + supports_group_observe=True, + ) + + # tenant_access_token cache: {app_id: (token, expire_epoch)} + _token_cache: Dict[str, tuple] = {} + # The bot's own open_id: {app_id: open_id}. Stable per app → cached without expiry. + _bot_id_cache: Dict[str, str] = {} + + # ── Credentials ───────────────────────────────────────────────────── + @staticmethod + def _app_secret(conn: Any) -> str: + cfg = conn.config if isinstance(conn.config, dict) else {} + return decrypt_secret(cfg.get("app_secret_enc")) or "" + + @staticmethod + def _encrypt_key(conn: Any) -> str: + cfg = conn.config if isinstance(conn.config, dict) else {} + return decrypt_secret(cfg.get("encrypt_key_enc")) or "" + + @staticmethod + def _verification_token(conn: Any) -> str: + cfg = conn.config if isinstance(conn.config, dict) else {} + return decrypt_secret(cfg.get("verification_token_enc")) or "" + + async def _tenant_token(self, app_id: str, app_secret: str) -> str: + cached = self._token_cache.get(app_id) + if cached and cached[1] > time.time() + 60: + return cached[0] + url = f"{LARK_API_BASE}/open-apis/auth/v3/tenant_access_token/internal" + async with httpx.AsyncClient(timeout=10) as client: + resp = await client.post(url, json={"app_id": app_id, "app_secret": app_secret}) + data = resp.json() + if data.get("code") != 0: + raise RuntimeError(f"飞书 tenant_access_token 获取失败: {data.get('msg') or data}") + token = data["tenant_access_token"] + self._token_cache[app_id] = (token, time.time() + int(data.get("expire", 7000))) + return token + + async def validate_credentials(self, conn: Any) -> Dict[str, Any]: + """Successfully obtaining a token means the credentials are valid; returns an identity summary. Raises on failure.""" + secret = self._app_secret(conn) + if not conn.app_id or not secret: + raise RuntimeError("缺少 App ID / App Secret") + await self._tenant_token(conn.app_id, secret) + return {"app_id": conn.app_id} + + # ── Webhook decryption / signature verification ───────────────────── + + + + # ── Event → InboundMsg ────────────────────────────────────────────── + + + + + + + + # ── Group history pull (app credentials, independent of group listening) ── + + + # ── Group listening: is this group message @'ing the bot? ─────────── + async def resolve_addressed(self, conn: Any, inbound: InboundMsg) -> bool: + """True when the bot's own open_id appears in the message's mention list. + + Fails open (True) whenever the bot identity cannot be fetched, so a transient + ``bot/v3/info`` failure degrades to today's behavior (answer everything delivered) + rather than making the bot go mute in the group. + """ + try: + bot_open_id = await self._bot_open_id(conn) + except Exception: # noqa: BLE001 + logger.debug("[lark] 机器人 open_id 获取失败，按「已被 @」处理", exc_info=True) + return True + if not bot_open_id: + return True + return bot_open_id in (inbound.mentioned_ids or []) + + async def _bot_open_id(self, conn: Any) -> str: + """The bot's own open_id (``/open-apis/bot/v3/info``), cached per app_id. + + Feishu never puts the bot identity in the event payload, so it has to be fetched once + and kept. The value is stable for the lifetime of the app, hence a plain process cache + with no expiry (a restart just re-fetches). + """ + cached = self._bot_id_cache.get(conn.app_id) + if cached: + return cached + token = await self._tenant_token(conn.app_id, self._app_secret(conn)) + async with httpx.AsyncClient(timeout=10) as client: + resp = await client.get( + f"{LARK_API_BASE}/open-apis/bot/v3/info", + headers={"Authorization": f"Bearer {token}"}, + ) + open_id = ((resp.json() or {}).get("bot") or {}).get("open_id") or "" + if open_id: + self._bot_id_cache[conn.app_id] = open_id + return open_id + + # ── Outbound push ─────────────────────────────────────────────────── + + + + + + # ── Inbound file download ─────────────────────────────────────────── + + # ── Outbound file delivery ────────────────────────────────────────── + + + + # ── Long connection (WebSocket, requires the lark_oapi SDK) ───────── + def make_ws_client(self, conn: Any, on_message: Callable[[InboundMsg], None]) -> Any: + """Build the Feishu long-connection client. ``on_message`` is invoked synchronously on the SDK thread; the caller dispatches it to the main loop. + + lark_oapi is lazily imported; if not installed a RuntimeError is raised, + which the manager records as an error state without affecting the process. + """ + try: + import lark_oapi as lark + from lark_oapi.api.im.v1 import P2ImMessageReceiveV1 + except Exception as exc: # noqa: BLE001 + raise RuntimeError("lark_oapi SDK 未安装，长连接不可用") from exc + + secret = self._app_secret(conn) + + def _do_message(data: "P2ImMessageReceiveV1") -> None: + try: + inbound = self._inbound_from_sdk_event(conn, data) + if inbound is not None: + on_message(inbound) + except Exception: # noqa: BLE001 + logger.exception("[lark] 长连接事件处理失败 channel_id=%s", conn.channel_id) + + handler = ( + lark.EventDispatcherHandler.builder("", "") + .register_p2_im_message_receive_v1(_do_message) + .build() + ) + return lark.ws.Client( + conn.app_id, secret, event_handler=handler, log_level=lark.LogLevel.WARNING + ) + + def prepare_ws_thread(self, loop: Any) -> None: + """Called inside the worker thread: install the per-thread loop proxy + write this thread's loop into thread-local storage. + + From then on, every access to the global ``loop`` inside + lark_oapi.ws.client is forwarded by the proxy to **this thread's** loop — + multiple Feishu bots each get their own thread and their own loop with + no mutual overwriting (fixes "This event loop is already running"). The + proxy is installed only once (process-level idempotent); each thread + writes its own thread-local value. + """ + _install_ws_loop_proxy() + _ws_loop_tls.loop = loop diff --git a/src/backend/integrations/channels/adapters/lark_delivery.py b/src/backend/integrations/channels/adapters/lark_delivery.py new file mode 100644 index 00000000..cc178df6 --- /dev/null +++ b/src/backend/integrations/channels/adapters/lark_delivery.py @@ -0,0 +1,152 @@ +"""lark delivery behavior.""" +from __future__ import annotations +import json +import logging +from typing import Any, Dict, Optional +import httpx +from integrations.channels.protocol import InboundMsg, SendResult +from integrations.channels.adapters.lark_settings import LARK_API_BASE +logger = logging.getLogger(__name__) + +class LarkDelivery: + def _chat_id(self, inbound: InboundMsg) -> str: + return (inbound.raw or {}).get("lark_chat_id") or inbound.external_conversation_id + + async def _post_message(self, conn: Any, chat_id: str, msg_type: str, content: Dict[str, Any]) -> SendResult: + if not chat_id: + return SendResult.fail("bad_format", "缺少 chat_id") + try: + token = await self._tenant_token(conn.app_id, self._app_secret(conn)) + except Exception as exc: # noqa: BLE001 + return SendResult.fail("transient", str(exc)) + url = f"{LARK_API_BASE}/open-apis/im/v1/messages?receive_id_type=chat_id" + payload = {"receive_id": chat_id, "msg_type": msg_type, + "content": json.dumps(content, ensure_ascii=False)} + try: + async with httpx.AsyncClient(timeout=15) as client: + resp = await client.post(url, headers={"Authorization": f"Bearer {token}"}, json=payload) + data = resp.json() + except Exception as exc: # noqa: BLE001 + return SendResult.fail("transient", str(exc)) + if data.get("code") == 0: + return SendResult.ok((data.get("data") or {}).get("message_id")) + code = data.get("code") + kind = "rate_limited" if code in (99991400, 99991401) else "forbidden" if code in (230002, 230013) else "unknown" + return SendResult.fail(kind, f"code={code} msg={data.get('msg')}") + + async def send_text(self, conn: Any, inbound: InboundMsg, text: str) -> SendResult: + return await self._post_message(conn, self._chat_id(inbound), "text", {"text": text}) + + async def push(self, conn: Any, inbound: InboundMsg, content: str) -> SendResult: + """Over-length content is auto-chunked by caps.max_message_len and sent piece by piece.""" + from integrations.channels.protocol import chunk_text + + chunks = chunk_text(content, self.caps.max_message_len) or [content] + first: Optional[SendResult] = None + cont: list = [] + for i, c in enumerate(chunks): + r = await self.send_text(conn, inbound, c) + if i == 0: + first = r + if not r.success: + return r + elif r.message_id: + cont.append(r.message_id) + if first is not None: + first.continuation_ids = cont + return first + return SendResult.fail("unknown", "空内容") + + async def edit_message(self, conn: Any, message_id: str, text: str) -> SendResult: + if not message_id: + return SendResult.fail("bad_format", "缺少 message_id") + try: + token = await self._tenant_token(conn.app_id, self._app_secret(conn)) + except Exception as exc: # noqa: BLE001 + return SendResult.fail("transient", str(exc)) + url = f"{LARK_API_BASE}/open-apis/im/v1/messages/{message_id}" + payload = {"msg_type": "text", "content": json.dumps({"text": text}, ensure_ascii=False)} + try: + async with httpx.AsyncClient(timeout=15) as client: + resp = await client.put(url, headers={"Authorization": f"Bearer {token}"}, json=payload) + data = resp.json() + except Exception as exc: # noqa: BLE001 + return SendResult.fail("transient", str(exc)) + if data.get("code") == 0: + return SendResult.ok(message_id) + return SendResult.fail("unknown", f"code={data.get('code')} msg={data.get('msg')}") + + async def download_resource( + self, conn: Any, inbound: InboundMsg, attachment: Dict[str, Any] + ) -> Optional[bytes]: + """Download a resource file from a message (requires the im:resource permission). + + GET /open-apis/im/v1/messages/{message_id}/resources/{file_key}?type=file|image + """ + msg_id = (inbound.raw or {}).get("lark_message_id") or inbound.message_id + key = attachment.get("key") + res_type = "image" if attachment.get("kind") == "image" else "file" + if not msg_id or not key: + return None + try: + token = await self._tenant_token(conn.app_id, self._app_secret(conn)) + url = f"{LARK_API_BASE}/open-apis/im/v1/messages/{msg_id}/resources/{key}?type={res_type}" + async with httpx.AsyncClient(timeout=30) as client: + resp = await client.get(url, headers={"Authorization": f"Bearer {token}"}) + if resp.status_code == 200 and not resp.headers.get("content-type", "").startswith("application/json"): + return resp.content + logger.warning("[lark] 资源下载失败 msg=%s key=%s body=%s", msg_id, key, resp.text[:200]) + except Exception: # noqa: BLE001 + logger.exception("[lark] 资源下载异常 msg=%s key=%s", msg_id, key) + return None + + async def push_file( + self, conn: Any, inbound: InboundMsg, content: bytes, filename: str, mime_type: str + ) -> SendResult: + chat_id = self._chat_id(inbound) + if not chat_id: + return SendResult.fail("bad_format", "缺少 chat_id") + try: + token = await self._tenant_token(conn.app_id, self._app_secret(conn)) + except Exception as exc: # noqa: BLE001 + return SendResult.fail("transient", str(exc)) + + is_image = (mime_type or "").startswith("image/") + try: + if is_image: + key = await self._upload_image(token, content) + msg_type, msg_content = "image", {"image_key": key} + else: + key = await self._upload_file(token, content, filename) + msg_type, msg_content = "file", {"file_key": key} + except Exception as exc: # noqa: BLE001 + return SendResult.fail("transient", f"上传失败: {exc}") + if not key: + return SendResult.fail("unknown", "未拿到资源 key") + return await self._post_message(conn, chat_id, msg_type, msg_content) + + async def _upload_image(self, token: str, content: bytes) -> Optional[str]: + url = f"{LARK_API_BASE}/open-apis/im/v1/images" + async with httpx.AsyncClient(timeout=30) as client: + resp = await client.post( + url, headers={"Authorization": f"Bearer {token}"}, + data={"image_type": "message"}, + files={"image": ("img.png", content)}, + ) + data = resp.json() + return (data.get("data") or {}).get("image_key") if data.get("code") == 0 else None + + async def _upload_file(self, token: str, content: bytes, filename: str) -> Optional[str]: + # file_type: Feishu distinguishes by extension; anything not in the known set is sent as stream + ext = (filename.rsplit(".", 1)[-1] if "." in filename else "").lower() + known = {"opus", "mp4", "pdf", "doc", "xls", "ppt"} + file_type = ext if ext in known else "stream" + url = f"{LARK_API_BASE}/open-apis/im/v1/files" + async with httpx.AsyncClient(timeout=60) as client: + resp = await client.post( + url, headers={"Authorization": f"Bearer {token}"}, + data={"file_type": file_type, "file_name": filename}, + files={"file": (filename, content)}, + ) + data = resp.json() + return (data.get("data") or {}).get("file_key") if data.get("code") == 0 else None diff --git a/src/backend/integrations/channels/adapters/lark_messages.py b/src/backend/integrations/channels/adapters/lark_messages.py new file mode 100644 index 00000000..a2ce1954 --- /dev/null +++ b/src/backend/integrations/channels/adapters/lark_messages.py @@ -0,0 +1,317 @@ +"""lark messages behavior.""" +from __future__ import annotations +import base64 +import hashlib +import json +import logging +import re +from typing import Any, Dict, List, Optional +import httpx +from integrations.channels.protocol import HistoryItem, InboundMsg +from integrations.channels.adapters.lark_settings import LARK_API_BASE +logger = logging.getLogger(__name__) +_MENTION_RE = re.compile(r"@_user_\d+") + +class LarkMessages: + def decrypt_webhook(self, conn: Any, body: bytes) -> Dict[str, Any]: + """If an Encrypt Key is configured, the body looks like {"encrypt": "..."} and is AES-256-CBC decrypted; otherwise plain JSON.""" + try: + raw = json.loads(body.decode("utf-8")) + except Exception: + return {} + if "encrypt" not in raw: + return raw + key = self._encrypt_key(conn) + if not key: + logger.warning("[lark] 收到加密事件但未配 Encrypt Key") + return {} + return self._aes_decrypt(raw["encrypt"], key) + + @staticmethod + def _aes_decrypt(encrypt_b64: str, encrypt_key: str) -> Dict[str, Any]: + from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes + + key = hashlib.sha256(encrypt_key.encode("utf-8")).digest() + data = base64.b64decode(encrypt_b64) + iv, ct = data[:16], data[16:] + decryptor = Cipher(algorithms.AES(key), modes.CBC(iv)).decryptor() + plain = decryptor.update(ct) + decryptor.finalize() + plain = plain[: -plain[-1]] # strip PKCS7 padding + return json.loads(plain.decode("utf-8")) + + def verify_webhook(self, conn: Any, headers: Dict[str, str], body: bytes) -> bool: + """Feishu signature: sha256(timestamp + nonce + encrypt_key + body). Without an Encrypt Key, falls back to token verification.""" + key = self._encrypt_key(conn) + sig = headers.get("x-lark-signature") or headers.get("X-Lark-Signature") + ts = headers.get("x-lark-request-timestamp") or headers.get("X-Lark-Request-Timestamp") + nonce = headers.get("x-lark-request-nonce") or headers.get("X-Lark-Request-Nonce") + if key and sig and ts is not None and nonce is not None: + h = hashlib.sha256((ts + nonce + key).encode("utf-8") + body).hexdigest() + return h == sig + # No encryption configured → fall back to verification_token (re-checked at the parse stage after decryption) + return True + + def parse_inbound(self, conn: Any, payload: Dict[str, Any]) -> Optional[InboundMsg]: + """Decrypted event dict → InboundMsg. Returns None for non-message events.""" + # schema 2.0: header.event_type; legacy schema: payload["event"]["type"] + header = payload.get("header") or {} + event_type = header.get("event_type") or payload.get("type") + if event_type != "im.message.receive_v1": + return None + event = payload.get("event") or {} + message = event.get("message") or {} + sender = event.get("sender") or {} + mtype = message.get("message_type") + text, attachments = self._extract_payload(mtype, message.get("content")) + if not text and not attachments: + return None # unsupported message type (sticker/location, etc.) + sender_open_id = (sender.get("sender_id") or {}).get("open_id", "") + return self._make_inbound( + conn=conn, + chat_id=message.get("chat_id", ""), + chat_type=message.get("chat_type", "p2p"), + message_id=message.get("message_id", ""), + text=text, + sender_open_id=sender_open_id, + sender_name=(sender.get("sender_id") or {}).get("user_id", "") or "", + attachments=attachments, + mentioned_ids=self._mentioned_ids(message), + raw=payload, + ) + + @staticmethod + def _mentioned_ids(message: Dict[str, Any]) -> List[str]: + """open_ids of everyone @'d in this message (``message.mentions``). + + Only relevant once the app holds ``im:message.group_msg`` and Feishu starts pushing + non-@ group messages too — until then every delivered group message has the bot in + here anyway. The bot's own open_id is not known synchronously, so matching happens + later in ``resolve_addressed``. + """ + out: List[str] = [] + for m in message.get("mentions") or []: + if not isinstance(m, dict): + continue + oid = (m.get("id") or {}).get("open_id") if isinstance(m.get("id"), dict) else None + if oid: + out.append(oid) + return out + + @staticmethod + def _strip_mentions(text: str) -> str: + return _MENTION_RE.sub("", text or "").strip() + + @classmethod + def _extract_text(cls, content: Any) -> str: + if not content: + return "" + try: + obj = json.loads(content) if isinstance(content, str) else content + text = obj.get("text", "") if isinstance(obj, dict) else "" + except Exception: + text = str(content) + return cls._strip_mentions(text) + + @classmethod + def _extract_payload(cls, mtype: str, content: Any): + """Extract (text, attachments) by message type. + + - text: plain text body (@ mentions stripped) + - image: image_key → attachment(kind=image) + - file/media/audio: file_key + file_name → attachment(kind=file) + - post (rich text): extract text + embedded image_key + """ + try: + obj = json.loads(content) if isinstance(content, str) else (content or {}) + except Exception: + obj = {} + if not isinstance(obj, dict): + obj = {} + attachments = [] + if mtype == "text": + return cls._strip_mentions(obj.get("text", "")), attachments + if mtype == "image": + key = obj.get("image_key") + if key: + attachments.append({"kind": "image", "key": key, "name": f"{key}.png"}) + return "", attachments + if mtype in ("file", "media", "audio"): + key = obj.get("file_key") + if key: + name = obj.get("file_name") or f"{key}.bin" + attachments.append({"kind": "file", "key": key, "name": name}) + return "", attachments + if mtype == "post": # rich text: extract text + embedded images + texts, img_keys = [], [] + for node in cls._post_nodes(obj): + if node.get("tag") == "text": + texts.append(node.get("text", "")) + elif node.get("tag") == "img" and node.get("image_key"): + img_keys.append(node["image_key"]) + title = obj.get("title", "") + body = " ".join(texts) + text = (title + "\n" + body) if title else body + attachments = [{"kind": "image", "key": k, "name": f"{k}.png"} for k in img_keys] + return text.strip(), attachments + return "", attachments + + @staticmethod + def _post_nodes(obj: Dict[str, Any]): + """Flatten all nodes of a post rich-text message (structure may be {content:[[node...]]} or wrapped in a language layer).""" + content = obj.get("content") or [] + if isinstance(content, dict): # language layer wrapper {"zh_cn": {"content": [...]}} + content = next(iter(content.values()), {}).get("content", []) if content else [] + for line in content or []: + for node in line or []: + if isinstance(node, dict): + yield node + + @staticmethod + def _make_inbound( + *, conn: Any, chat_id: str, chat_type: str, message_id: str, + text: str, sender_open_id: str, sender_name: str, + raw: Dict[str, Any], attachments=None, mentioned_ids=None, + ) -> InboundMsg: + return InboundMsg( + channel_id=conn.channel_id, + channel_type="lark", + text=text, + chat_type=chat_type, + external_conversation_id=chat_id, # always keyed by chat_id + sender_id=sender_open_id, + sender_name=sender_name, + message_id=message_id, + attachments=attachments or [], + # p2p is always aimed at the bot; in a group it depends on whether the bot's own + # open_id is in the mention list — resolved asynchronously in resolve_addressed. + addressed_to_bot=True if chat_type == "p2p" else None, + mentioned_ids=list(mentioned_ids or []), + raw={"lark_chat_id": chat_id, "lark_message_id": message_id}, + ) + + async def fetch_history( + self, + conn: Any, + conversation_id: str, + *, + since_ms: int, + limit: int, + newest_first: bool = False, + ) -> List[HistoryItem]: + """GET /open-apis/im/v1/messages — this chat's own messages, oldest-first. + + Uses the tenant token (app credentials), so it is a property of the bound app rather + than of any user's session. Note this sits behind a *different* permission from event + push: an app that is not allowed to receive non-@ group messages may still be allowed + to read the chat's history, which is precisely why pulling is worth having. + + Feishu takes ``start_time`` in **seconds**; the caller works in milliseconds. + Any failure returns [] — the caller treats "unsupported" and "failed" alike. + """ + chat_id = (conversation_id or "").strip() + if not chat_id: + return [] + try: + token = await self._tenant_token(conn.app_id, self._app_secret(conn)) + params = { + "container_id_type": "chat", + "container_id": chat_id, + # Desc for the cold start so the first page holds the *newest* messages of + # the window; Asc when walking forward from the cursor so nothing is skipped. + "sort_type": "ByCreateTimeDesc" if newest_first else "ByCreateTimeAsc", + "page_size": str(max(1, min(int(limit or 50), 50))), # Feishu caps page_size at 50 + } + if since_ms > 0: + params["start_time"] = str(int(since_ms // 1000)) + async with httpx.AsyncClient(timeout=15) as client: + resp = await client.get( + f"{LARK_API_BASE}/open-apis/im/v1/messages", + params=params, + headers={"Authorization": f"Bearer {token}"}, + ) + body = resp.json() or {} + if resp.status_code != 200 or body.get("code") not in (0, None): + logger.warning( + "[lark] 群历史拉取失败 chat=%s code=%s msg=%s", + chat_id, body.get("code"), str(body.get("msg"))[:120], + ) + return [] + return [ + item + for item in ( + self._history_item(raw) for raw in (body.get("data") or {}).get("items") or [] + ) + if item is not None + ] + except Exception: # noqa: BLE001 + logger.warning("[lark] 群历史拉取异常 chat=%s", chat_id, exc_info=True) + return [] + + @classmethod + def _history_item(cls, raw: Dict[str, Any]) -> Optional[HistoryItem]: + """One ``im/v1/messages`` entry → HistoryItem, reusing the inbound content parser.""" + if not isinstance(raw, dict) or raw.get("deleted"): + return None + body = raw.get("body") or {} + text, attachments = cls._extract_payload(raw.get("msg_type"), body.get("content")) + if not text and not attachments: + return None + sender = raw.get("sender") or {} + message_id = raw.get("message_id") or "" + try: + ts_ms = int(raw.get("create_time") or 0) + except (TypeError, ValueError): + ts_ms = 0 + return HistoryItem( + message_id=message_id, + # Feishu returns only ids here; a display name would need a directory lookup per + # sender, which is not worth an extra API call per message for background context. + sender_name=(sender.get("id") or "")[-6:] or "群成员", + text=text, + ts_ms=ts_ms, + attachments=attachments, + # download_resource resolves resources by message_id — carry it so a file pulled + # from history stays fetchable later via channel_read_attachment. + raw={"lark_message_id": message_id}, + ) + + def _inbound_from_sdk_event(self, conn: Any, data: Any) -> Optional[InboundMsg]: + """lark_oapi's typed P2ImMessageReceiveV1 event → InboundMsg.""" + event = getattr(data, "event", None) + message = getattr(event, "message", None) + sender = getattr(event, "sender", None) + if message is None: + return None + mtype = getattr(message, "message_type", None) + text, attachments = self._extract_payload(mtype, getattr(message, "content", None)) + if not text and not attachments: + return None + sender_id_obj = getattr(sender, "sender_id", None) + open_id = getattr(sender_id_obj, "open_id", "") if sender_id_obj else "" + return self._make_inbound( + conn=conn, + chat_id=getattr(message, "chat_id", "") or "", + chat_type=getattr(message, "chat_type", "p2p") or "p2p", + message_id=getattr(message, "message_id", "") or "", + text=text, + sender_open_id=open_id or "", + sender_name="", + attachments=attachments, + mentioned_ids=self._sdk_mentioned_ids(message), + raw={}, + ) + + @staticmethod + def _sdk_mentioned_ids(message: Any) -> List[str]: + """SDK-typed counterpart of ``_mentioned_ids`` (``MentionEvent.id.open_id``). + + The long connection is the production path, so mention extraction must exist on both + sides; missing it here would make every group message look un-@'d. + """ + out: List[str] = [] + for m in getattr(message, "mentions", None) or []: + ident = getattr(m, "id", None) + oid = getattr(ident, "open_id", "") if ident is not None else "" + if oid: + out.append(oid) + return out diff --git a/src/backend/integrations/channels/adapters/lark_settings.py b/src/backend/integrations/channels/adapters/lark_settings.py new file mode 100644 index 00000000..f27648d1 --- /dev/null +++ b/src/backend/integrations/channels/adapters/lark_settings.py @@ -0,0 +1,3 @@ +"""lark API configuration.""" +import os +LARK_API_BASE = os.getenv("LARK_API_BASE", "https://open.feishu.cn").rstrip("/") diff --git a/src/backend/core/channels/adapters/wecom.py b/src/backend/integrations/channels/adapters/wecom.py similarity index 99% rename from src/backend/core/channels/adapters/wecom.py rename to src/backend/integrations/channels/adapters/wecom.py index ba590795..f963ef69 100644 --- a/src/backend/core/channels/adapters/wecom.py +++ b/src/backend/integrations/channels/adapters/wecom.py @@ -24,7 +24,7 @@ import httpx -from core.channels.protocol import ChannelCaps, InboundMsg, SendResult, chunk_text +from integrations.channels.protocol import ChannelCaps, InboundMsg, SendResult, chunk_text from core.infra.crypto import decrypt_secret logger = logging.getLogger(__name__) diff --git a/src/backend/core/channels/adapters/weixin.py b/src/backend/integrations/channels/adapters/weixin.py similarity index 99% rename from src/backend/core/channels/adapters/weixin.py rename to src/backend/integrations/channels/adapters/weixin.py index 3e8a1eaa..d57591db 100644 --- a/src/backend/core/channels/adapters/weixin.py +++ b/src/backend/integrations/channels/adapters/weixin.py @@ -45,7 +45,7 @@ import httpx -from core.channels.protocol import ChannelCaps, InboundMsg, SendResult, chunk_text +from integrations.channels.protocol import ChannelCaps, InboundMsg, SendResult, chunk_text from core.infra.crypto import decrypt_secret logger = logging.getLogger(__name__) diff --git a/src/backend/integrations/channels/api/__init__.py b/src/backend/integrations/channels/api/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/src/backend/integrations/channels/api/__init__.py @@ -0,0 +1 @@ + diff --git a/src/backend/api/routes/v1/channel_desktop.py b/src/backend/integrations/channels/api/channel_desktop.py similarity index 89% rename from src/backend/api/routes/v1/channel_desktop.py rename to src/backend/integrations/channels/api/channel_desktop.py index 6d82adb2..f4e8782c 100644 --- a/src/backend/api/routes/v1/channel_desktop.py +++ b/src/backend/integrations/channels/api/channel_desktop.py @@ -1,10 +1,10 @@ """Desktop channel registration and leased execution; device capabilities only.""" -from api.routes.v1.desktop_capability import _require_capability_user +from features.desktop.api.capability_dependencies import require_capability_user as _require_capability_user from core.auth.backend import UserContext, get_current_user from core.db.engine import get_db from core.infra.responses import success_response -from core.services.channel_relay import ChannelRelayService +from integrations.channels.relay import ChannelRelayService from fastapi import APIRouter, Depends, Request from pydantic import BaseModel, Field from sqlalchemy.orm import Session @@ -56,7 +56,7 @@ def claim( ): from dataclasses import asdict - from core.channels.registry import get_adapter + from integrations.channels.registry import get_adapter item = ChannelRelayService(db).claim(owner, device(request), body.binding_ids) if item: @@ -91,8 +91,8 @@ async def complete( if body.error: import logging - from core.channels.protocol import InboundMsg - from core.channels.registry import get_adapter + from integrations.channels.protocol import InboundMsg + from integrations.channels.registry import get_adapter msg = InboundMsg(**row.payload["message"]) if msg.addressed_to_bot is not False: @@ -115,7 +115,7 @@ async def operation( owner: str = Depends(_require_capability_user), db: Session = Depends(get_db), ): - from core.services.channel_relay_rpc import channel_operation + from integrations.channels.relay_rpc import channel_operation data = await channel_operation( db, @@ -134,6 +134,6 @@ async def operation( async def local_binding( user: UserContext = Depends(get_current_user), db: Session = Depends(get_db) ): - from core.services.channel_desktop_worker import prepare_binding + from integrations.channels.desktop_worker import prepare_binding return success_response(data=await prepare_binding(db, str(user.user_id))) diff --git a/src/backend/api/routes/v1/channels.py b/src/backend/integrations/channels/api/channels.py similarity index 97% rename from src/backend/api/routes/v1/channels.py rename to src/backend/integrations/channels/api/channels.py index fdafaffc..7422d28d 100644 --- a/src/backend/api/routes/v1/channels.py +++ b/src/backend/integrations/channels/api/channels.py @@ -18,11 +18,11 @@ from sqlalchemy.orm import Session from core.auth.backend import UserContext, get_current_user -from core.channels.protocol import ChannelCaps -from core.channels.registry import get_adapter, list_adapters +from integrations.channels.protocol import ChannelCaps +from integrations.channels.registry import get_adapter, list_adapters from core.db.engine import get_db from core.infra.responses import created_response, success_response -from core.services.channel_service import ChannelService, bot_to_dict +from integrations.channels.service import ChannelService, bot_to_dict router = APIRouter(prefix="/v1/channels", tags=["Channels"]) @@ -103,7 +103,7 @@ def list_my_bots( bot（设置「我的机器人」用）；都不传 → 全部。""" svc = ChannelService(db) bots = svc.list_bots(str(user.user_id), agent_id=agent_id, main_only=main_only) - from core.services.channel_relay import ChannelRelayService, execution + from integrations.channels.relay import ChannelRelayService, execution relay = ChannelRelayService(db) return success_response(data={"bots": [ { @@ -122,7 +122,7 @@ def list_my_conversations( db: Session = Depends(get_db), ): """列出本人渠道 bot 已产生的会话（群/单聊），供 Web 后台配置定时投递目标。""" - from core.services.channel_service import list_owner_conversations + from integrations.channels.service import list_owner_conversations return success_response(data={"conversations": list_owner_conversations(db, str(user.user_id))}) diff --git a/src/backend/core/services/channel_desktop_worker.py b/src/backend/integrations/channels/desktop_worker.py similarity index 89% rename from src/backend/core/services/channel_desktop_worker.py rename to src/backend/integrations/channels/desktop_worker.py index 5e94273e..6acaab97 100644 --- a/src/backend/core/services/channel_desktop_worker.py +++ b/src/backend/integrations/channels/desktop_worker.py @@ -1,6 +1,7 @@ """Desktop worker: authorized cloud channel I/O with the existing local chat runtime.""" from __future__ import annotations +from features.desktop.bridge import identity as bridge_identity import asyncio import hashlib @@ -8,11 +9,12 @@ import platform import httpx -from core.channels.protocol import ChannelCaps, HistoryItem, InboundMsg, SendResult -from core.db.models import ChannelConnection, ContentBlock, UserShadow +from integrations.channels.protocol import ChannelCaps, HistoryItem, InboundMsg, SendResult +from integrations.channels.models import ChannelConnection +from core.db.models import ContentBlock, UserShadow from core.infra.exceptions import AccessDeniedError, BadRequestError -from core.services import desktop_cloud_bridge as bridge -from core.services.channel_relay_rpc import pack_bytes, unpack_bytes +from features.desktop.auth import bridge as bridge_auth +from integrations.channels.relay_rpc import pack_bytes, unpack_bytes from sqlalchemy import select logger = logging.getLogger(__name__) @@ -21,20 +23,20 @@ async def cloud_post(captured, path, payload): - bridge.require_current_account(captured) - current = bridge.get_state() + bridge_identity.require_current_account(captured) + current = bridge_identity.get_state() if not current: raise AccessDeniedError("桌面云端会话已失效") try: async with httpx.AsyncClient(timeout=25) as client: response = await client.post( current["cloud_base"] + "/api/v1/channels/desktop/" + path, - headers=bridge.cloud_headers(current), + headers=bridge_identity.cloud_headers(current), json=payload, ) except httpx.HTTPError as exc: raise BadRequestError("渠道执行服务连接失败，请检查网络后重试") from exc - bridge.require_current_account(captured) + bridge_identity.require_current_account(captured) try: data = response.json() except ValueError as exc: @@ -52,8 +54,8 @@ async def cloud_post(captured, path, payload): def identity_owner(db): - identity = bridge.get_identity_state() - if not bridge.bridge_enabled() or not bridge.get_state() or not identity: + identity = bridge_identity.get_identity_state() + if not bridge_auth.bridge_enabled() or not bridge_identity.get_state() or not identity: raise AccessDeniedError("请在已登录的混合模式桌面端使用本机机器人") owner = db.scalar( select(UserShadow.user_id).where( @@ -69,7 +71,7 @@ async def prepare_binding(db, owner): identity, actual_owner = identity_owner(db) if owner != actual_owner: raise AccessDeniedError("本机账号不匹配") - captured = bridge.get_state() + captured = bridge_identity.get_state() grant = await cloud_post(captured, "register", {"device_name": platform.node() or "本机"}) if ( not isinstance(grant, dict) @@ -79,7 +81,7 @@ async def prepare_binding(db, owner): or not isinstance(grant.get("device_name"), str) ): raise BadRequestError("本机授权响应格式无效，请重试") - with bridge.account_scope(captured): + with bridge_identity.account_scope(captured): db.add( ContentBlock( id=GRANT_PREFIX + grant["binding_id"], payload={**identity, "local_owner": owner} @@ -123,7 +125,7 @@ def record_run(self, run_id, owner_id): async def cancel(self): if self.run_id and self.owner_id: - from orchestration.chat_run_executor import cancel_run + from core.harness.registry.run_cancellation import cancel_run await cancel_run(self.run_id, user_id=self.owner_id) @@ -196,11 +198,11 @@ async def recall_message(self, conn, msg, message_id): async def run_delivery(factory, captured, identity, owner, item): - from core.channels.inbound import handle_inbound - from core.channels.registry import scoped_adapter - from core.services.user_agent_service import UserAgentService + from integrations.channels.inbound import handle_inbound + from integrations.channels.registry import scoped_adapter + from core.capabilities.agents.edition import UserAgentService - bridge.require_current_account(captured) + bridge_identity.require_current_account(captured) receipt_id = RECEIPT_PREFIX + item["delivery_id"] with factory() as db: if item["binding_id"] not in local_grants(db, identity, owner): @@ -250,7 +252,7 @@ async def run_delivery(factory, captured, identity, owner, item): try: while not task.done(): done, _ = await asyncio.wait({task}, timeout=8) - bridge.require_current_account(captured) + bridge_identity.require_current_account(captured) await cloud_post(captured, item["delivery_id"] + "/renew", {"lease": item["lease"]}) if done: break @@ -291,12 +293,12 @@ async def loop(self): with self.factory() as db: identity, owner = identity_owner(db) bindings = local_grants(db, identity, owner) - captured = bridge.get_state() + captured = bridge_identity.get_state() for offset in range(0, len(bindings), 200): batch = bindings[offset : offset + 200] reply = await cloud_post(captured, "claim", {"binding_ids": batch}) retained = set(reply["retained_binding_ids"]) - with bridge.account_scope(captured), self.factory() as db: + with bridge_identity.account_scope(captured), self.factory() as db: for binding_id in set(batch) - retained: row = db.get(ContentBlock, GRANT_PREFIX + binding_id) if row: diff --git a/src/backend/integrations/channels/extension.py b/src/backend/integrations/channels/extension.py new file mode 100644 index 00000000..df46323e --- /dev/null +++ b/src/backend/integrations/channels/extension.py @@ -0,0 +1,7 @@ +"""Installed channel tool contribution; receivers retain their existing lifecycle.""" +NAME = "channels" +def register_tools(): + from integrations.channels.tools.attachment import register_native_contribution + register_native_contribution() +def setup(): + return True diff --git a/src/backend/integrations/channels/inbound.py b/src/backend/integrations/channels/inbound.py new file mode 100644 index 00000000..f57f0887 --- /dev/null +++ b/src/backend/integrations/channels/inbound.py @@ -0,0 +1,347 @@ +"""Channel inbound responsibilities.""" +from __future__ import annotations +from typing import Optional +import core.chat.runtime.admission as admission +from integrations.channels.protocol import InboundMsg +from integrations.channels.registry import get_adapter +from core.db.engine import SessionLocal +from integrations.channels.models import ChannelConnection +from integrations.channels.repository import ChannelConnectionRepository +import logging +logger = logging.getLogger(__name__) +from integrations.channels import inbound_attachments +from integrations.channels import inbound_delivery +from integrations.channels import inbound_history +from integrations.channels import inbound_observation +from integrations.channels import inbound_sessions +from integrations.channels import inbound_state + +def module_route(db, conn: ChannelConnection): + """The admit hook of the first enabled module that claims this bot, if any.""" + from core.harness.registry import channel_routes + + for handles, admit in channel_routes(): + if handles(db, conn): + return admit + return None + + +async def handle_inbound(msg: InboundMsg) -> None: + """Main entry point for inbound messages. Scheduled on the main event loop (long-connection threads submit via run_coroutine_threadsafe). + + All exceptions are swallowed and logged — one message's failure must not take down the long connection / process. + """ + from integrations.channels.relay_dispatch import dispatch_to_desktop + if await dispatch_to_desktop(msg, SessionLocal): + return + if inbound_sessions.already_handled(f"{msg.channel_id}:{msg.message_id}" if msg.message_id else ""): + return + if not (msg.text or "").strip() and not msg.attachments: + return + # #1 Per-conversation serialization: until an in-progress run finishes, later messages in the same conversation queue up, avoiding history races/corruption. + async with inbound_state.conv_locks[inbound_sessions.conv_key(msg)]: + await process_inbound(msg) + + +async def process_inbound(msg: InboundMsg) -> None: + from core.chat.context import build_runtime_context, collect_historical_attachments + pass + + placeholder_id: Optional[str] = None + accepted_run_id: Optional[str] = None + db = SessionLocal() + try: + repo = ChannelConnectionRepository(db) + conn = repo.get_by_id(msg.channel_id) + if conn is None or not conn.enabled: + logger.warning("[channels] inbound 无对应启用连接 channel_id=%s", msg.channel_id) + db.close() + return + + owner_id = conn.owner_user_id + adapter = get_adapter(conn.channel_type) + _ = conn.config # force-load the credential column so it remains usable after detaching + + # Group listening gate. Only reached at all when the platform actually delivers non-@ + # group messages (needs the channel app's "read all group messages" permission); + # otherwise every inbound group message is an @ and this is a no-op. + if msg.chat_type == "group" and not await inbound_observation.resolve_addressed(adapter, conn, msg): + if conn.group_listen_mode == "observe_all": + inbound_observation.observe_message(db, conn, msg) + repo.touch_event(conn.channel_id) # bystander traffic still proves the connection is alive + logger.info( + "[channels] 旁听群消息 channel_id=%s type=%s conv=%s", + conn.channel_id, conn.channel_type, msg.external_conversation_id, + ) + else: + # mention_only → drop entirely, exactly as before this feature existed. + # Logged because reaching this line at all is the *only* evidence that the + # platform is delivering non-@ group messages (i.e. that the "read all group + # messages" permission is actually granted). Without it, "the bot ignores the + # group" and "the platform never sends the group" look identical from outside. + # DEBUG, not INFO: an active group would otherwise flood the log. + logger.debug( + "[channels] 丢弃未 @ 的群消息（旁听未开启）channel_id=%s type=%s", + conn.channel_id, conn.channel_type, + ) + # Either way: no reset-command handling, no run, no placeholder, no reply. + db.close() + return + + admit = module_route(db, conn) + if admit is not None: + db.expunge(conn) + db.close() + await admit(conn, msg, adapter) + return + + # Channel-side "new conversation / clear context": soft-deleting the current session is + # enough (the next message automatically creates an empty session); no agent run. This is + # where /new and clear-context actually take effect in Feishu and other clients — + # otherwise the command would be fed to the agent as plain text and the history would keep + # being reused, impossible to clear. + if inbound_sessions.is_reset_command(msg.text): + inbound_sessions.reset_session(db, conn, msg) + db.refresh(conn) + db.expunge(conn) + db.close() + try: + await adapter.send_text(conn, msg, "✅ 已开启新对话，之前的上下文已清除。") + except Exception: # noqa: BLE001 + logger.debug("[channels] 清空回执发送失败", exc_info=True) + return + + session = inbound_sessions.find_or_create_session(db, conn, msg) + chat_id = session.chat_id + + # #4 Placeholder first. Everything below this line does real network work — + # downloading attachments, pulling group history — and the placeholder exists + # precisely so the user is not staring at nothing while that happens. It used to be + # sent after the whole prep stage, which put ~1s of history pull ahead of any visible + # feedback and read as the bot being slow to react. + # Prefer the channel's send_placeholder (DingTalk: robot API, recallable) — only that + # path yields a message_id supporting the later edit/recall-replace; otherwise plain + # send_text. + try: + send_ph = getattr(adapter, "send_placeholder", None) or adapter.send_text + ph = await send_ph(conn, msg, "🤔 正在处理，请稍候…") + if ph.success: + placeholder_id = ph.message_id + except Exception: # noqa: BLE001 + logger.debug("[channels] 占位消息发送失败", exc_info=True) + + # Inbound attachments: download → store as Artifact → uploaded_files + uploaded_files = await inbound_attachments.ingest_attachments(db, adapter, conn, owner_id, chat_id, msg) + # File-only message with no text → synthesize a one-line user prompt + if (msg.text or "").strip(): + user_text = msg.text + elif uploaded_files: + user_text = "[收到文件] " + "、".join(f.get("name", "") for f in uploaded_files) + else: + db.close() + return + # #5 Label the speaker in group scenarios so the agent can tell multi-person conversations apart + if msg.chat_type == "group": + user_text = f"{inbound_observation.speaker_label(msg)}：{user_text}" + # Deterministic incremental pull, before draining: whatever the platform pushed + # (possibly nothing) plus whatever we can pull ends up in the same buffer, and the + # drain below sees both. Failure is absorbed inside — it must not block the reply. + await inbound_history.pull_history(db, adapter, conn, msg, session) + # observe_all: prepend everything the group said since the last @ as background. + # Prepending into user_text (rather than a separate context field) means it is + # persisted with the turn, so later turns still see the group log through normal + # history loading and compaction — no second retention mechanism to maintain. + observed = inbound_observation.drain_observed(db, session) + if observed: + user_text = inbound_observation.render_observed_block(observed) + user_text + + enabled = inbound_sessions.resolve_enabled(db, conn, owner_id) + # Enable thinking: same as the web client's default in non-fast mode. With thinking off, + # models (Qwen family especially) tend to emit shallow filler like "I'll get right on X" + # without actually landing on tool calls, idling repeatedly across turns. Thinking never + # leaks into the channel reply: structured-reasoning models emit separate thinking events + # (ignored by _collect_reply, which only accumulates content/meta), and inline-thinking + # models' … spans inside content are stripped by _collect_reply. + context = build_runtime_context( + model_name=None, user_id=owner_id, chat_id=chat_id, enable_thinking=True, + uploaded_files=uploaded_files, + enabled_skills=enabled["enabled_skills"], enabled_agents=enabled["enabled_agents"], + enabled_mcps=enabled["enabled_mcps"], enabled_kbs=enabled["enabled_kbs"], + ) + from core.ontology.runtime_policy import build_user_ontology_runtime + + ontology_enabled, ontology_runtime = build_user_ontology_runtime( + user_id=owner_id, + task=user_text, + db=db, + ) + context["ontology_enabled"] = ontology_enabled + context["ontology_runtime"] = ontology_runtime + # Cross-turn file readability: inject files uploaded/generated earlier in this session + # (including last turn's Feishu attachments) as a summary block so the model gets real + # file_ids to call read_artifact with. Exclude this turn's attachments (already injected + # in full via uploaded_files) to avoid duplication. build_runtime_context doesn't include + # this field, so we add it here (aligned with the web client's historical_files injection + # in chats.py). + current_file_ids = {f.get("file_id") for f in uploaded_files if f.get("file_id")} + context["historical_files"] = collect_historical_attachments( + chat_id, owner_id, exclude_file_ids=current_file_ids, + ) + # Bind to a specific sub-agent: pin the whole run to that sub-agent (running with its own + # prompt/tools/model/knowledge bases), via the workflow's direct sub-agent mode. NULL → + # unset, run with the owner's default capabilities (main agent). + # Note: the sub-agent carries its own capability bindings, which override the whitelist + # narrowing from _resolve_enabled above — this is intended behavior. + direct_agent_id = conn.agent_id or (conn.config or {}).get("desktop_relay_agent_id") + if direct_agent_id: + context["agent_id"] = direct_agent_id + # #7 Trigger A: let the agent self-create scheduled delivery tasks within this conversation + context["channel_origin"] = { + "channel_id": conn.channel_id, + "conversation_id": msg.external_conversation_id, + "chat_type": msg.chat_type, + # channel_type lets the agent be told *which* platform it is in, so it can reach + # for that platform's CLI to pull group history the bot never received (anything + # from before it joined). Delivery targets never needed it; the prompt does. + "channel_type": conn.channel_type, + } + from core.chat.sequencer import ChatSequencer + + accepted = ChatSequencer(db).accept_main_run( + chat_id=chat_id, + user_id=owner_id, + user_content=user_text, + user_extra_data={ + "channel_sender_open_id": msg.sender_id, + "channel_sender_name": msg.sender_name, + "channel_message_id": msg.message_id, + "channel_attachments": [f.get("file_id") for f in uploaded_files], + "attachments": [ + {"file_id": f.get("file_id"), "name": f.get("name")} for f in uploaded_files + ], + }, + request_payload={ + "channel_id": msg.channel_id, + "source": "desktop_channel" if (conn.config or {}).get("desktop_relay") else "channel", + "kind": "chat", + }, + ) + accepted_run_id = accepted.run.run_id + session_messages = inbound_sessions.load_history(db, chat_id, owner_id) + repo.touch_event(conn.channel_id) + # The commits in create_session / add_message / touch_event expire conn's column + # attributes (expire_on_commit defaults to True). If we expunged as-is, reading app_id/ + # config during push after detaching would trigger a refresh and raise + # DetachedInstanceError ("is not bound to a Session"). + # Refresh first to reload all columns, then expunge — afterwards attribute reads are + # pure in-memory and no session is needed. + db.refresh(accepted.run) + db.refresh(conn) + db.expunge(accepted.run) + db.expunge(conn) # app_id/config remain readable after leaving the session, for push + except Exception: + logger.exception("[channels] inbound 准备阶段失败 channel_id=%s", msg.channel_id) + from integrations.channels.registry import current_relay_adapter + if current_relay_adapter(): + current_relay_adapter().error = "本机机器人准备失败" + db.close() + if accepted_run_id: + with SessionLocal() as cleanup_db: + ChatSequencer(cleanup_db).abandon_pending_run( + accepted_run_id, + reason="channel preparation failed", + ) + return + finally: + db.close() + + # (The placeholder is sent inside the prep stage above, before attachment download and + # history pull — those are the slow parts it needs to cover.) + + # §13 Channel adaptation: IM clients like Feishu have no approval UI for MySpace write + # operations. Without pre-authorization, the gate would suspend on a MySpace write waiting + # for out-of-band user confirmation (_collect_reply only recognizes content/meta; + # file_confirm is dropped) → stuck until the 2h timeout with the placeholder never updated. + # The bot runs as the owner, and in private chat the message sender is the owner themselves, + # so pre-mark this session as allowed — equivalent to the owner approving their own write. + # (Sub-agents remain non-interactive; their /myspace writes are still rejected.) + try: + from core.llm.tools import confirmation as _mc + if not (conn.config or {}).get("desktop_relay"): + _mc.allow_session(chat_id) + except Exception: # noqa: BLE001 — a pre-authorization failure must not take down the whole run + logger.debug("[channels] myspace 写预授权失败 chat_id=%s", chat_id, exc_info=True) + + try: + run = await admission.start_run( + accepted_run=accepted.run, + chat_id=chat_id, + user_id=owner_id, + session_messages=session_messages, + effective_user_message=user_text, + raw_user_message=user_text, + context=context, + model_name=None, + ) + except Exception as exc: + with SessionLocal() as cleanup_db: + ChatSequencer(cleanup_db).abandon_pending_run( + accepted_run_id, + reason=str(exc), + ) + logger.exception("[channels] inbound run 启动失败 channel_id=%s", msg.channel_id) + from integrations.channels.registry import current_relay_adapter + if current_relay_adapter(): + current_relay_adapter().error = "本机机器人启动失败" + try: + await inbound_delivery.replace_placeholder( + adapter, conn, msg, placeholder_id, "⚠️ 处理出错了，请稍后重试。" + ) + except Exception: # noqa: BLE001 + logger.debug("[channels] 失败消息回推异常", exc_info=True) + return + + try: + from integrations.channels.registry import current_relay_adapter + relay_adapter = current_relay_adapter() + if relay_adapter: + relay_adapter.record_run(run.run_id, owner_id) + reply, gen_artifacts = await inbound_delivery.collect_reply(run.run_id) + except Exception as exc: + logger.exception("[channels] inbound run 失败 channel_id=%s", msg.channel_id) + if relay_adapter: + relay_adapter.error = "本机机器人执行失败" + notice = ( + str(exc).strip() + if isinstance(exc, inbound_delivery.ChannelRunError) and str(exc).strip() + else "处理出错了，请稍后重试。" + ) + try: + await inbound_delivery.replace_placeholder(adapter, conn, msg, placeholder_id, f"⚠️ {notice}") + except Exception: # noqa: BLE001 + logger.debug("[channels] 失败消息回推异常", exc_info=True) + return + + # Note: the assistant reply is **not persisted here**. start_run's background executor + # (chat_run_executor) already persists the assistant message together with + # usage/model/tool_calls/artifacts when the stream ends. Saving another one here would create + # a **duplicate assistant message**, and one without usage (tokens recorded as 0) — exactly + # the root cause of "input/output tokens are both 0" in channel sessions. We only use the + # reply text to push back to the channel, without persisting again. + + # Push back: text (placeholder edit + follow-up chunks) + generated files + try: + if reply: + await inbound_delivery.deliver_reply(adapter, conn, msg, reply, placeholder_id) + else: + note = "已为你生成文件。" if gen_artifacts else "（本次无文本回复）" + await inbound_delivery.replace_placeholder(adapter, conn, msg, placeholder_id, note) + if getattr(adapter, "push_file", None): + for content, name, mime in inbound_attachments.load_generated_files(gen_artifacts): + fr = await adapter.push_file(conn, msg, content, name, mime) + if not fr.success: + logger.warning("[channels] 文件回传失败 name=%s kind=%s", name, fr.error_kind) + except Exception: + logger.exception("[channels] inbound 回推阶段失败 channel_id=%s", msg.channel_id) + if relay_adapter: + relay_adapter.error = "本机机器人回复回传失败" diff --git a/src/backend/integrations/channels/inbound_attachments.py b/src/backend/integrations/channels/inbound_attachments.py new file mode 100644 index 00000000..e2183363 --- /dev/null +++ b/src/backend/integrations/channels/inbound_attachments.py @@ -0,0 +1,73 @@ +"""Channel attachments responsibilities.""" +from __future__ import annotations +import mimetypes +from typing import Any, Dict, List +from integrations.channels.protocol import InboundMsg +from core.db.engine import SessionLocal +from core.db.models import Artifact +from integrations.channels.models import ChannelConnection +import logging +logger = logging.getLogger(__name__) +from integrations.channels import inbound_state + +async def ingest_attachments( + db, adapter, conn: ChannelConnection, owner_id: str, chat_id: str, msg: InboundMsg +) -> List[Dict[str, Any]]: + """Download inbound attachments → store as Artifacts (mirrors /v1/file/upload) → return uploaded_files items. + + Each item contains metadata only. Parsing is deferred to the shared + file-context/read_artifact path and cached on first use. + """ + download = getattr(adapter, "download_resource", None) + if download is None or not msg.attachments: + return [] + from core.files.artifacts.service import store_bytes_as_artifact + + out: List[Dict[str, Any]] = [] + for att in msg.attachments: + try: + content = await download(conn, msg, att) + if not content or len(content) > inbound_state.MAX_INBOUND_FILE_BYTES: + continue + name = att.get("name") or "file.bin" + mime = mimetypes.guess_type(name)[0] or "application/octet-stream" + art = store_bytes_as_artifact( + db, user_id=owner_id, content=content, filename=name, mime_type=mime, + chat_id=chat_id, source="channel_upload", + extra={"channel_id": conn.channel_id}, + ) + out.append({ + "file_id": art.artifact_id, + "download_url": f"/files/{art.artifact_id}", + "name": name, + "mime_type": mime, + }) + except Exception: # noqa: BLE001 + logger.exception("[channels] 入站附件处理失败 key=%s", att.get("key")) + return out + + +def load_generated_files(artifacts: List[Dict[str, Any]]): + """Read the bytes of artifact files generated this turn; returns a list of (content, name, mime) for pushing back (best-effort).""" + if not artifacts: + return [] + from core.files.storage import get_storage + + files = [] + with SessionLocal() as db: + storage = get_storage() + for art in artifacts: + fid = art.get("file_id") + if not fid: + continue + row = db.query(Artifact).filter(Artifact.artifact_id == fid).first() + if row is None or not row.storage_key: + continue + try: + content = storage.download_bytes(row.storage_key) + except Exception: # noqa: BLE001 + logger.warning("[channels] 产物下载失败 %s", fid, exc_info=True) + continue + files.append((content, row.filename or art.get("name") or fid, + row.mime_type or "application/octet-stream")) + return files diff --git a/src/backend/integrations/channels/inbound_delivery.py b/src/backend/integrations/channels/inbound_delivery.py new file mode 100644 index 00000000..d21b71d4 --- /dev/null +++ b/src/backend/integrations/channels/inbound_delivery.py @@ -0,0 +1,118 @@ +"""Channel delivery responsibilities.""" +from __future__ import annotations +from typing import Any, Dict, List, Optional +import core.runs.events.subscription as subscription +from integrations.channels.protocol import InboundMsg +import logging +logger = logging.getLogger(__name__) + +class ChannelRunError(RuntimeError): + """A user-facing terminal error emitted by the shared chat-run stream.""" + + +async def collect_reply(run_id: str): + """Follow the run's event stream, accumulating assistant text + capturing artifact files generated this turn (meta.artifacts). + + Structured-reasoning models emit thinking as separate ``thinking`` events (ignored + here), but inline-thinking models embed ``…`` in the content deltas + themselves — strip those before handing the text to the channel, or the raw chain + of thought goes out as message text (the web UI parses the tags; IM cannot). + """ + from integrations.channels.markdown import strip_inline_thinking + pass + + full = "" + artifacts: List[Dict[str, Any]] = [] + from integrations.channels.registry import current_relay_adapter + + relay = current_relay_adapter() + async for event in subscription.follow_run(run_id): + et = event.get("type") + if relay and et in {"file_confirm", "ask_user_question"}: + relay.error = "此操作需要确认，请在桌面端处理后重试" + await relay.cancel() + raise ChannelRunError(relay.error) + if et == "content": + full += event.get("delta", "") or "" + elif et == "meta": + arts = event.get("artifacts") + if isinstance(arts, list): + artifacts = arts + elif et == "error": + detail = str(event.get("error") or event.get("delta") or "").strip() + raise ChannelRunError(detail or "请求处理失败，请稍后重试") + return strip_inline_thinking(full).strip(), artifacts + + +async def recall_placeholder(adapter, conn, msg: InboundMsg, placeholder_id: str) -> None: + """Recall the placeholder message (best-effort): channels that can't edit but can recall (DingTalk) use "recall + resend" as an equivalent replacement.""" + recall = getattr(adapter, "recall_message", None) + if recall is None: + return + try: + r = await recall(conn, msg, placeholder_id) + if not r.success: + logger.debug("[channels] 占位撤回失败 kind=%s detail=%s", r.error_kind, r.error_detail) + except Exception: # noqa: BLE001 + logger.debug("[channels] 占位撤回异常", exc_info=True) + + +async def replace_placeholder( + adapter, conn, msg: InboundMsg, placeholder_id: Optional[str], text: str +) -> None: + """Finish a placeholder flow with text, always sending a terminal message. + + Edit in place when the channel returned a placeholder ID; otherwise recall + (if supported) and resend. WeChat's send API does not return a message ID, so + it must take the direct-send path instead of silently leaving the placeholder + as the last visible message. + + Error notices / "no text reply" receipts go through here — the old logic only edited, + and since DingTalk edits always fail, users were stuck on "processing" forever with + no follow-up ever shown.""" + edit = getattr(adapter, "edit_message", None) + if placeholder_id and edit is not None: + r = await edit(conn, placeholder_id, text) + if r.success: + return + if placeholder_id: + await recall_placeholder(adapter, conn, msg, placeholder_id) + fr = await adapter.send_text(conn, msg, text) + if not fr.success: + logger.warning("[channels] 占位替换消息发送失败 kind=%s detail=%s", fr.error_kind, fr.error_detail) + + +async def deliver_reply(adapter, conn, msg: InboundMsg, reply: str, placeholder_id: Optional[str]) -> None: + """#2+#4: chunk long replies; if there is a placeholder message, edit the first chunk into it and send the rest as follow-up messages. + Channel supports recall but not edit (DingTalk) → recall the placeholder then send, a visually equivalent replacement. + + When the channel supports markdown (caps.supports_markdown + send_markdown), send with native rendering; + run prepare_markdown on the whole text before chunking (tables etc. can no longer be recognized and converted once cut by chunk boundaries). + [ref:tool-N] citation markers only render on the web client, so they are always stripped before going out (all channels).""" + from integrations.channels.markdown import strip_citation_markers + from integrations.channels.protocol import chunk_text + + reply = strip_citation_markers(reply) + send_md = ( + getattr(adapter, "send_markdown", None) + if getattr(adapter.caps, "supports_markdown", False) + else None + ) + if send_md is not None: + prepare = getattr(adapter, "prepare_markdown", None) + if callable(prepare): + reply = prepare(reply) + chunks = chunk_text(reply, getattr(adapter.caps, "max_message_len", 0)) or [reply] + edit = getattr(adapter, "edit_message", None) + rest = chunks + if placeholder_id and edit is not None: + r = await edit(conn, placeholder_id, chunks[0]) + if r.success: + rest = chunks[1:] + else: + # edit failed/unsupported → recall the placeholder (if supported), send all chunks as new messages + await recall_placeholder(adapter, conn, msg, placeholder_id) + for c in rest: + fr = await (send_md or adapter.send_text)(conn, msg, c) + if not fr.success: + logger.warning("[channels] 回复分段推送失败 kind=%s detail=%s", fr.error_kind, fr.error_detail) diff --git a/src/backend/integrations/channels/inbound_history.py b/src/backend/integrations/channels/inbound_history.py new file mode 100644 index 00000000..cdd1b5ca --- /dev/null +++ b/src/backend/integrations/channels/inbound_history.py @@ -0,0 +1,147 @@ +"""Channel history responsibilities.""" +from __future__ import annotations +import asyncio +from datetime import datetime, timedelta, timezone +from typing import Any, Dict, List +from integrations.channels.protocol import InboundMsg +from core.db.models import ChatSession +from integrations.channels.models import ChannelConnection +import logging +logger = logging.getLogger(__name__) +from integrations.channels import inbound_observation +from integrations.channels import inbound_sessions +from integrations.channels import inbound_state + +def history_state(session: ChatSession) -> Dict[str, Any]: + state = (session.extra_data or {}).get("history_pull") + return dict(state) if isinstance(state, dict) else {} + + +def save_history_state(db, session: ChatSession, state: Dict[str, Any]) -> None: + meta = dict(session.extra_data or {}) + meta["history_pull"] = state + session.extra_data = meta + db.commit() + + +async def pull_history( + db, adapter, conn: ChannelConnection, msg: InboundMsg, session: ChatSession +) -> int: + """Deterministically pull this group's new messages and fold them into the buffer. + + Runs on every @ in a group, driven by code rather than by the model choosing to call a + tool. Incremental by construction: the cursor stored on the session is the lower bound, so + a pull only ever returns messages we have not already taken. The first pull in a + conversation has no cursor and instead reaches back ``_HISTORY_COLDSTART_WINDOW_H`` hours + — that is the cold start, and it is also what makes content from *before the bot joined* + reachable at all. + + Deliberately independent of group listening: pushing and pulling sit behind different + platform permissions, so a channel that will not push non-@ messages may still allow the + pull. Returns the number of messages recorded. + """ + fetch = getattr(adapter, "fetch_history", None) + if fetch is None: + return 0 + state = history_state(session) + if int(state.get("failures") or 0) >= inbound_state.HISTORY_MAX_FAILURES: + return 0 + + cursor = int(state.get("cursor_ms") or 0) + cold_start = not cursor + if cold_start: + # Timezone-aware on purpose: this value is compared against platform epoch-ms, and + # the naive-utcnow habit elsewhere in this codebase is exactly what makes stored + # timestamps read 8 hours off. + cursor = int( + (datetime.now(timezone.utc) - timedelta(hours=inbound_state.HISTORY_COLDSTART_WINDOW_H)).timestamp() * 1000 + ) + seen_ids = set(state.get("seen_ids") or []) + + try: + items = await asyncio.wait_for( + fetch( + conn, msg.external_conversation_id, + since_ms=cursor, limit=inbound_state.HISTORY_MAX_MESSAGES, + # Cold start takes the newest end of the window: with a 7-day window and a + # per-pull cap, taking the oldest would surface week-old chatter and leave + # the recent messages — what the conversation is actually about — unread. + newest_first=cold_start, + ), + timeout=inbound_state.HISTORY_TIMEOUT_S, + ) + # Adapters may return either order; the buffer must read chronologically. Items + # without a timestamp keep their relative position rather than sinking to the front. + items = sorted(items or [], key=lambda it: int(getattr(it, "ts_ms", 0) or 0)) + except Exception: # noqa: BLE001 — includes timeout; history is a bonus, never a blocker + logger.warning( + "[channels] 群历史拉取失败 channel_id=%s conv=%s", + conn.channel_id, msg.external_conversation_id, exc_info=True, + ) + state["failures"] = int(state.get("failures") or 0) + 1 + save_history_state(db, session, state) + return 0 + + records: List[Dict[str, Any]] = [] + file_index: Dict[str, Any] = {} + budget = inbound_state.HISTORY_MAX_TOTAL_CHARS + newest = cursor + # Which end the budget cuts from differs by intent: + # cold start — walk newest→oldest, so truncation drops the stale tail. There is no + # continuity to preserve; this is a one-off seed. + # incremental — walk oldest→newest and stop at the budget, advancing the cursor only + # over what was actually recorded, so the next pull resumes exactly there + # and no message is skipped. + for it in (reversed(items) if cold_start else items): + mid = getattr(it, "message_id", "") or "" + # Dedup on two fronts: the cursor boundary is inclusive on some platforms, and a + # message may also have already arrived via push. Either way it must not be recorded + # twice — that is what made repeated pulls wasteful. + if mid and (mid in seen_ids or inbound_sessions.already_handled(f"hist:{mid}")): + continue + text = (getattr(it, "text", "") or "").strip() + if len(text) > inbound_state.OBSERVE_TEXT_MAX: + text = text[:inbound_state.OBSERVE_TEXT_MAX] + "…" + refs = [ + {"kind": a.get("kind") or "file", "key": a.get("key") or "", "name": a.get("name") or ""} + for a in (getattr(it, "attachments", None) or []) + if a.get("key") + ] + if not text and not refs: + continue + if len(text) > budget: # total budget exhausted → stop, keep the cursor honest + break + budget -= len(text) + entry: Dict[str, Any] = {"n": getattr(it, "sender_name", "") or "用户", "t": text} + if refs: + entry["a"] = refs + for ref in refs: + file_index[ref["key"]] = { + "name": ref["name"], "kind": ref["kind"], + "message_id": mid, "raw": dict(getattr(it, "raw", None) or {}), + } + records.append(entry) + if mid: + seen_ids.add(mid) + newest = max(newest, int(getattr(it, "ts_ms", 0) or 0)) + + if cold_start: + # Collected newest→oldest above; the buffer is read as a transcript, so flip it back. + records.reverse() + if records: + inbound_observation.record_observed_batch(db, session, records, file_index) + # Advance the cursor even when nothing was recorded: the window was still covered, and + # not advancing would re-request it on every message forever. + state.update({ + "cursor_ms": newest, + "failures": 0, + # Bounded id memory purely for boundary dedup; the cursor does the real work. + "seen_ids": list(seen_ids)[-200:], + }) + save_history_state(db, session, state) + if records: + logger.info( + "[channels] 群历史拉取 %d 条 channel_id=%s conv=%s", + len(records), conn.channel_id, msg.external_conversation_id, + ) + return len(records) diff --git a/src/backend/integrations/channels/inbound_observation.py b/src/backend/integrations/channels/inbound_observation.py new file mode 100644 index 00000000..92a50cfa --- /dev/null +++ b/src/backend/integrations/channels/inbound_observation.py @@ -0,0 +1,168 @@ +"""Channel observation responsibilities.""" +from __future__ import annotations +from typing import Any, Dict, List +from integrations.channels.protocol import InboundMsg +from core.db.models import ChatSession +from integrations.channels.models import ChannelConnection +import logging +logger = logging.getLogger(__name__) +from integrations.channels import inbound_sessions +from integrations.channels import inbound_state + +def speaker_label(msg: InboundMsg) -> str: + """Label the speaker in group scenarios (no directory lookup; falls back to existing fields).""" + return (msg.sender_name or (msg.sender_id or "")[-6:] or "用户") + + +async def resolve_addressed(adapter, conn: ChannelConnection, msg: InboundMsg) -> bool: + """Is this message aimed at the bot? Adapters that cannot decide synchronously defer here. + + Fails open (True) on any resolver error or when the adapter has no resolver at all, so + channels that never learned about group listening (WeCom / WeChat) keep behaving exactly + as before. + """ + if msg.addressed_to_bot is not None: + return bool(msg.addressed_to_bot) + resolver = getattr(adapter, "resolve_addressed", None) + if resolver is None: + return True + try: + return bool(await resolver(conn, msg)) + except Exception: # noqa: BLE001 + logger.debug("[channels] @ 判定失败，按「已被 @」处理 channel_id=%s", conn.channel_id, exc_info=True) + return True + + +def observe_text(msg: InboundMsg) -> str: + """The bystander message's own text, truncated. Attachments are handled separately.""" + text = (msg.text or "").strip() + if len(text) > inbound_state.OBSERVE_TEXT_MAX: + text = text[:inbound_state.OBSERVE_TEXT_MAX] + "…" + return text + + +def observe_attachment_refs(msg: InboundMsg) -> List[Dict[str, Any]]: + """Attachment handles for a bystander message — recorded, never downloaded. + + Eagerly downloading everything a group shares would pull the whole group's file traffic + into our storage: large I/O and a far wider privacy footprint than reading text. But + *discarding the key* (the earlier behavior) left the agent able to see that a file existed + while being unable to ever open it. Keeping the key costs nothing and turns this into + lazy loading: the model asks for a file by key via ``channel_read_attachment`` only when + it decides the file matters. + """ + return [ + {"kind": a.get("kind") or "file", "key": a.get("key") or "", "name": a.get("name") or ""} + for a in (msg.attachments or []) + if a.get("key") + ] + + +def observe_message(db, conn: ChannelConnection, msg: InboundMsg) -> None: + """Record a bystander group message into the session's observation buffer. No agent run, no reply. + + The buffer lives in ``ChatSession.extra_data`` rather than being written as real session + messages, for three reasons: it stays bounded (a quiet group can't inflate history), it + avoids long runs of consecutive user-role turns in the model's message list, and draining + it into the triggering turn keeps the group log in one coherent block instead of scattered + fragments. + """ + text = observe_text(msg) + refs = observe_attachment_refs(msg) + if not text and not refs: + return + session = inbound_sessions.find_or_create_session(db, conn, msg) + meta = dict(session.extra_data or {}) + + entry: Dict[str, Any] = {"n": speaker_label(msg), "t": text} + if refs: + entry["a"] = refs + buf = list(meta.get("observed") or []) + buf.append(entry) + meta["observed"] = buf[-inbound_state.OBSERVE_MAX:] + + if refs: + # Separate, longer-lived index than the message buffer: the buffer is drained into a + # prompt on the next @, but the model may only decide to open a file several turns + # later. Keyed by attachment key, it stores what the adapter needs to fetch the bytes + # (Lark resolves resources by message_id, DingTalk by the robotCode in raw) so none of + # that plumbing has to appear in the prompt. + index = dict(meta.get("observed_files") or {}) + for ref in refs: + index.pop(ref["key"], None) # re-insert so recency ordering stays correct + index[ref["key"]] = { + "name": ref["name"], "kind": ref["kind"], + "message_id": msg.message_id, "raw": dict(msg.raw or {}), + } + while len(index) > inbound_state.OBSERVE_FILE_INDEX_MAX: + index.pop(next(iter(index))) + meta["observed_files"] = index + + session.extra_data = meta # reassign: in-place JSON mutation is not change-tracked + db.commit() + + +def record_observed_batch( + db, session: ChatSession, items: List[Dict[str, Any]], file_index: Dict[str, Any] +) -> None: + """Append a pulled batch to the observation buffer + file index, respecting the caps.""" + meta = dict(session.extra_data or {}) + buf = list(meta.get("observed") or []) + buf.extend(items) + meta["observed"] = buf[-inbound_state.OBSERVE_MAX:] + if file_index: + index = dict(meta.get("observed_files") or {}) + for key, entry in file_index.items(): + index.pop(key, None) + index[key] = entry + while len(index) > inbound_state.OBSERVE_FILE_INDEX_MAX: + index.pop(next(iter(index))) + meta["observed_files"] = index + session.extra_data = meta + db.commit() + + +def drain_observed(db, session: ChatSession) -> List[Dict[str, Any]]: + """Take and clear the observation buffer (it is folded into the turn being started).""" + meta = dict(session.extra_data or {}) + buf = list(meta.get("observed") or []) + if not buf: + return [] + meta["observed"] = [] + session.extra_data = meta + db.commit() + return buf + + +def render_observed_line(item: Dict[str, Any]) -> str: + """One group-log line: ``speaker：text [文件：name｜key=…]``. + + Attachment keys are rendered inline so the model can pass one straight to + ``channel_read_attachment``. The files themselves were never downloaded — the key is the + only handle that exists, which is exactly why it has to reach the prompt. + """ + parts = [item.get("t") or ""] + for ref in item.get("a") or []: + label = "图片" if ref.get("kind") == "image" else "文件" + parts.append(f"[{label}：{ref.get('name') or '未命名'}｜key={ref.get('key')}]") + return f"{item.get('n', '用户')}：{' '.join(p for p in parts if p)}" + + +def render_observed_block(items: List[Dict[str, Any]]) -> str: + """Render the buffer as a context preamble for the triggering message. + + Explicitly labelled as background and marked do-not-reply, otherwise the model treats the + group log as a list of pending requests and tries to answer every line. + """ + lines = "\n".join(render_observed_line(it) for it in items) + hint = ( + "\n（其中带 key= 的文件并未下载，只有句柄。确有需要时用 " + "channel_read_attachment(file_key=\"…\") 取回再读，不要凭文件名臆测内容。）" + if any(it.get("a") for it in items) else "" + ) + return ( + "【群聊上下文】以下是你被 @ 之前群里的近期对话，仅供你理解背景，" + "不要逐条回复，也不要在回复中复述：\n" + f"{lines}{hint}\n" + "【以下才是本次需要你处理的消息】\n" + ) diff --git a/src/backend/integrations/channels/inbound_sessions.py b/src/backend/integrations/channels/inbound_sessions.py new file mode 100644 index 00000000..d2efe96b --- /dev/null +++ b/src/backend/integrations/channels/inbound_sessions.py @@ -0,0 +1,131 @@ +"""Channel sessions responsibilities.""" +from __future__ import annotations +from typing import Any, Dict, List, Optional +from integrations.channels.protocol import InboundMsg +from core.chat.context import resolve_enabled_capabilities +from core.db.models import ChatSession +from integrations.channels.models import ChannelConnection +from core.chat.service import ChatService +from integrations.channels import inbound_state + +def conv_key(msg: "InboundMsg") -> str: + return f"{msg.channel_id}:{msg.external_conversation_id}" + + +def already_handled(message_id: str) -> bool: + if not message_id: + return False + if message_id in inbound_state.seen_message_ids: + return True + inbound_state.seen_message_ids[message_id] = None + if len(inbound_state.seen_message_ids) > inbound_state.SEEN_MAX: + inbound_state.seen_message_ids.popitem(last=False) + return False + + +def is_reset_command(text: Optional[str]) -> bool: + return (text or "").strip().lower() in inbound_state.RESET_COMMANDS + + +def reset_session(db, conn: ChannelConnection, msg: InboundMsg) -> None: + """Soft-delete the current channel session. The next non-command message creates a fresh empty session via _find_or_create_session.""" + existing = ( + db.query(ChatSession) + .filter( + ChatSession.channel_id == conn.channel_id, + ChatSession.external_conversation_id == msg.external_conversation_id, + ChatSession.deleted_at.is_(None), + ) + .first() + ) + if existing is not None: + ChatService(db).delete_session_force( + existing.chat_id, actor_user_id=conn.owner_user_id + ) + + +def find_or_create_session( + db, conn: ChannelConnection, msg: InboundMsg +) -> ChatSession: + """Reuse or create a session keyed by (channel_id, external_conversation_id), owned by the owner.""" + existing = ( + db.query(ChatSession) + .filter( + ChatSession.channel_id == conn.channel_id, + ChatSession.external_conversation_id == msg.external_conversation_id, + # Exclude soft-deleted sessions: the channel-side /new clear works by soft-deleting + # the current session — without this filter the next message would hit the old + # session again, making the clear a no-op. + ChatSession.deleted_at.is_(None), + ) + .first() + ) + if existing is not None: + # Backfill the p2p peer id: proactive delivery (automation etc.) needs it to locate + # the recipient when sending back in a private chat; old sessions predate this field, + # so we backfill on every inbound message. + if msg.chat_type == "p2p" and msg.sender_id: + meta = dict(existing.extra_data or {}) + if meta.get("channel_peer_id") != msg.sender_id: + meta["channel_peer_id"] = msg.sender_id + existing.extra_data = meta + db.commit() + return existing + + chat_service = ChatService(db) + title = (msg.text or "渠道会话").strip()[:40] or "渠道会话" + extra: Dict[str, Any] = { + "source": f"channel:{conn.channel_type}", + "channel_chat_type": msg.chat_type, + } + if msg.chat_type == "p2p" and msg.sender_id: + extra["channel_peer_id"] = msg.sender_id + session = chat_service.create_session( + user_id=conn.owner_user_id, + title=title, + extra_data=extra, + ) + session.channel_id = conn.channel_id + session.external_conversation_id = msg.external_conversation_id + db.commit() + db.refresh(session) + return session + + +def load_history(db, chat_id: str, owner_id: str) -> List[Dict[str, Any]]: + """Load history as session_messages — same source as the web path (checkpoint-aware + keeps tool calls/results). + + Goes through ``compaction_service.load_session_history`` (the same entry point as the web + pipeline), not the old "``list_all_messages`` + keep only user/assistant plain text". The + old approach **dropped wholesale** the assistant turns' ``tool_calls`` and tool results, as + well as empty-text pure-tool turns — so in multi-turn channel sessions the model couldn't + see which tools it called last turn or what results it got. With thinking disabled it would + very easily redo the same work from scratch, degenerating into "I'll get right on generating…" + style idling until hitting max_iters. With the same-source loader, the model gets real tool + context across turns and reuses compaction checkpoints (large sessions no longer replay in full). + + When there is no access permission (theoretically impossible — the session was just created + by _find_or_create_session and belongs to the owner), load_session_history returns None; + we fall back to empty history here. + """ + from core.chat.compaction.service import load_session_history + + chat_service = ChatService(db) + return load_session_history(chat_service, chat_id, owner_id) or [] + + +def resolve_enabled( + db, conn: ChannelConnection, owner_id: str +) -> Dict[str, Optional[List[str]]]: + """Resolve the owner's capabilities, then narrow them by the resource_scope whitelist.""" + skills, agents, mcps = resolve_enabled_capabilities(db, owner_id) + scope = conn.resource_scope if isinstance(conn.resource_scope, dict) else {} + scoped_skills = scope.get("skill_ids") + scoped_kbs = scope.get("kb_ids") + return { + # Whitelist present → narrow to it (the owner must still own these; catalog gating applies as usual) + "enabled_skills": scoped_skills if isinstance(scoped_skills, list) else skills, + "enabled_agents": agents, + "enabled_mcps": mcps, + "enabled_kbs": scoped_kbs if isinstance(scoped_kbs, list) else None, + } diff --git a/src/backend/integrations/channels/inbound_state.py b/src/backend/integrations/channels/inbound_state.py new file mode 100644 index 00000000..227ded46 --- /dev/null +++ b/src/backend/integrations/channels/inbound_state.py @@ -0,0 +1,46 @@ +"""Channel state responsibilities.""" +from __future__ import annotations +import asyncio +from collections import OrderedDict, defaultdict + +MAX_INBOUND_FILE_BYTES = 50 * 1024 * 1024 + + +conv_locks: "defaultdict[str, asyncio.Lock]" = defaultdict(asyncio.Lock) + + +SEEN_MAX = 4096 + + +seen_message_ids: "OrderedDict[str, None]" = OrderedDict() + + +RESET_COMMANDS = frozenset({ + "/new", "/clear", "/reset", "/restart", + "新对话", "新会话", "清空", "清空上下文", "清除上下文", + "重置", "重置对话", "重新开始", +}) + + +OBSERVE_MAX = 60 + + +OBSERVE_TEXT_MAX = 500 + + +OBSERVE_FILE_INDEX_MAX = 100 + + +HISTORY_COLDSTART_WINDOW_H = 168 + + +HISTORY_MAX_MESSAGES = 80 + + +HISTORY_MAX_TOTAL_CHARS = 12000 + + +HISTORY_TIMEOUT_S = 10 + + +HISTORY_MAX_FAILURES = 3 diff --git a/src/backend/core/channels/manager.py b/src/backend/integrations/channels/manager.py similarity index 97% rename from src/backend/core/channels/manager.py rename to src/backend/integrations/channels/manager.py index 40d9273b..8cd8b4a0 100644 --- a/src/backend/core/channels/manager.py +++ b/src/backend/integrations/channels/manager.py @@ -58,7 +58,7 @@ async def _reconcile_connections(self) -> None: # The elected process alone runs this loop. HTTP workers persist desired # state; they must not start competing receivers or rely on process memory. from core.db.engine import SessionLocal - from core.db.repository.channel import ChannelConnectionRepository + from integrations.channels.repository import ChannelConnectionRepository with SessionLocal() as db: specs = { @@ -128,7 +128,7 @@ def restart_connection(self, channel_id: str, channel_type: str) -> None: # ── Thread body ───────────────────────────────────────────────────── def _run_worker(self, worker: _Worker, channel_type: str) -> None: - from core.channels.registry import get_adapter + from integrations.channels.registry import get_adapter adapter = get_adapter(channel_type) backoff = 1.0 @@ -206,7 +206,7 @@ def mark_connected(): def _load_conn(channel_id: str): """Load and detach conn from the session before returning (credentials live in the config dict and remain readable after detaching).""" from core.db.engine import SessionLocal - from core.db.repository.channel import ChannelConnectionRepository + from integrations.channels.repository import ChannelConnectionRepository with SessionLocal() as db: conn = ChannelConnectionRepository(db).get_by_id(channel_id) @@ -241,7 +241,7 @@ def _verify_credentials(self, adapter, conn) -> None: def _dispatch_factory(self, worker: _Worker): """Return a synchronous callback that delivers InboundMsg to handle_inbound on the main loop.""" - from core.channels.inbound import handle_inbound + from integrations.channels.inbound import handle_inbound loop = self._loop @@ -287,7 +287,7 @@ def _safe_disconnect(worker: _Worker) -> None: @staticmethod def _set_status(channel_id: str, status: str, last_error: Optional[str] = None) -> None: from core.db.engine import SessionLocal - from core.db.repository.channel import ChannelConnectionRepository + from integrations.channels.repository import ChannelConnectionRepository try: with SessionLocal() as db: diff --git a/src/backend/core/channels/markdown.py b/src/backend/integrations/channels/markdown.py similarity index 100% rename from src/backend/core/channels/markdown.py rename to src/backend/integrations/channels/markdown.py diff --git a/src/backend/integrations/channels/models.py b/src/backend/integrations/channels/models.py new file mode 100644 index 00000000..8a9339c7 --- /dev/null +++ b/src/backend/integrations/channels/models.py @@ -0,0 +1,133 @@ +"""Channel connection and durable device relay metadata.""" +from core.db.engine import Base +from core.db.utc_datetime import UTCDateTime +from core.infra.time import utc_now +from sqlalchemy import JSON, Boolean, CheckConstraint, Column, Float, ForeignKey, Index, String, Text, UniqueConstraint +from sqlalchemy.dialects.postgresql import JSONB +JSONType = JSON().with_variant(JSONB(), "postgresql") + +class ChannelConnection(Base): + """Inbound channel bot (owner service-account model). One row = one user-created external IM bot. + + Orthogonal to the **outbound** plugins [[LarkConnection]]/[[DingTalkConnection]]: those are + "the agent operates Lark/DingTalk with the user's OAuth identity"; this table is "an + external IM pushes messages in to trigger the agent" (**inbound**), and the credentials are + an **application identity** (App ID/Secret), not user OAuth. + + Multi-tenancy uses the owner service-account model: the bot is bound to ``owner_user_id``, + and all inbound messages run with the owner's identity + permissions; people in a group are + not tied to platform teams and there is no per-sender identity resolution (open_id is only + recorded for audit). p2p and group share the same orchestration path; the only difference is + session keying (by sender open_id vs by group chat_id). + + Credentials (App Secret / Encrypt Key / Verification Token) are encrypted with + application-layer Fernet via ``core.infra.crypto`` and stored in the ``config`` JSON — + **never stored in plaintext, never unioned into the global environment**. ``app_id`` is + redundantly stored in plaintext in its own column for the unique constraint (token lock: + the same Lark app cannot be bound by two people). + + See internal design docs. + """ + + __tablename__ = "channel_connections" + + channel_id = Column(String(64), primary_key=True) + owner_user_id = Column( + String(64), + ForeignKey("users_shadow.user_id", ondelete="CASCADE"), + nullable=False, + ) + channel_type = Column(String(16), nullable=False) # 'lark' | 'dingtalk' | 'wecom' | ... + display_name = Column(String(100), nullable=False, default="我的机器人") + transport = Column(String(16), nullable=False, default="long_conn") # 'long_conn' | 'webhook' + app_id = Column( + String(128), nullable=False + ) # Plaintext redundancy: for the unique constraint + establishing the long connection + config = Column( + JSONType, nullable=False, default=dict + ) # Encrypted credentials {app_secret_enc, encrypt_key_enc, verification_token_enc} + # Resource allowlist (for when the owner wants precise control over what the bot exposes): + # {"kb_ids": [...], "skill_ids": [...]}; NULL = expose everything the owner has (default). + # Retrieval/skill loading narrows down by this on top of the owner's permissions. + resource_scope = Column(JSONType, nullable=True) + # Binding to a specific sub-agent: non-NULL → inbound messages are pinned to that + # [[UserAgent]] (runs with its own prompt/tools/model); written when binding on the + # "sub-agent page"; NULL → main agent (owner's default capabilities), left empty when + # binding in the settings "My Bots" page. + # ondelete=SET NULL: deleting the sub-agent makes the bot fall back to the main agent + # automatically, without cascading the bot's deletion. + agent_id = Column( + String(64), ForeignKey("user_agents.agent_id", ondelete="SET NULL"), nullable=True + ) + # Group listening: how much of a group conversation the bot may see. + # mention_only (default) — only messages that @ the bot; anything else is dropped. + # observe_all — bystander group messages are recorded as background context + # (never replied to) and folded into the next @bot turn. + # observe_all is inert unless the channel app also holds the platform's "read all group + # messages" permission (Lark ``im:message.group_msg`` / DingTalk group-message read) — + # without it the platform never delivers non-@ messages in the first place. + # Defaults to mention_only because observing a whole group's chatter is privacy-sensitive + # and must be an explicit, per-bot decision by the owner. + group_listen_mode = Column( + String(16), nullable=False, default="mention_only", server_default="mention_only" + ) + # disconnected (default / disconnected) | pending (verifying) | connected | error + status = Column(String(16), nullable=False, default="pending") + enabled = Column(Boolean, nullable=False, default=True) + last_event_at = Column(UTCDateTime(timezone=True)) + last_error = Column(Text) + created_at = Column(UTCDateTime(timezone=True), default=utc_now) + updated_at = Column(UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now) + + __table_args__ = ( + CheckConstraint( + "status IN ('disconnected', 'pending', 'connected', 'error')", + name="channel_connections_status_check", + ), + CheckConstraint( + "transport IN ('long_conn', 'webhook')", + name="channel_connections_transport_check", + ), + CheckConstraint( + "group_listen_mode IN ('mention_only', 'observe_all')", + name="channel_connections_group_listen_check", + ), + # token lock: within the same channel, the same app can only be bound once (prevents multiple people grabbing the same bot credentials) + UniqueConstraint("channel_type", "app_id", name="uq_channel_connections_type_app"), + Index("idx_channel_connections_owner", "owner_user_id"), + Index("idx_channel_connections_enabled", "enabled", "status"), + Index("idx_channel_connections_agent", "agent_id"), + ) + +class DesktopChannelBinding(Base): + __tablename__ = "desktop_channel_bindings" + binding_id = Column(String(64), primary_key=True) + owner_id = Column(String(64), nullable=False, index=True) + device_id = Column(String(128), nullable=False) + device_name = Column(String(100), nullable=False) + consumed = Column(Boolean, nullable=False, default=False) + created_at = Column(Float, nullable=False) + last_seen = Column(Float, nullable=False) + +class ChannelRelayDelivery(Base): + __tablename__ = "channel_relay_deliveries" + delivery_id = Column(String(64), primary_key=True) + channel_id = Column(String(64), nullable=False) + binding_id = Column(String(64), nullable=False) + owner_id = Column(String(64), nullable=False) + device_id = Column(String(128), nullable=False) + status = Column(String(24), nullable=False) + payload = Column(JSON, nullable=False) + lease = Column(String(64), nullable=True) + created_at = Column(Float, nullable=False) + deadline = Column(Float, nullable=False) + error = Column(String(300), nullable=True) + __table_args__ = ( + Index("ix_channel_relay_queue", "owner_id", "device_id", "status", "created_at"), + ) + +class ChannelRelayOperation(Base): + __tablename__ = "channel_relay_operations" + operation_id = Column(String(128), primary_key=True) + status = Column(String(24), nullable=False) + result = Column(JSON, nullable=True) diff --git a/src/backend/core/channels/outbound.py b/src/backend/integrations/channels/outbound.py similarity index 94% rename from src/backend/core/channels/outbound.py rename to src/backend/integrations/channels/outbound.py index 4b024439..c7d663df 100644 --- a/src/backend/core/channels/outbound.py +++ b/src/backend/integrations/channels/outbound.py @@ -15,10 +15,10 @@ import logging from typing import Iterable, Optional, Tuple -from core.channels.protocol import InboundMsg -from core.channels.registry import get_adapter +from integrations.channels.protocol import InboundMsg +from integrations.channels.registry import get_adapter from core.db.engine import SessionLocal -from core.db.repository.channel import ChannelConnectionRepository +from integrations.channels.repository import ChannelConnectionRepository logger = logging.getLogger(__name__) @@ -95,7 +95,7 @@ async def deliver_to_conversation( # thinking, so the strip is a safety net); markdown rendering/downgrade is # handled inside adapter.push per each channel's capability (e.g. DingTalk sends a # markdown message). - from core.channels.markdown import strip_citation_markers, strip_inline_thinking + from integrations.channels.markdown import strip_citation_markers, strip_inline_thinking r = await adapter.push(conn, msg, strip_citation_markers(strip_inline_thinking(text))) if not r.success: diff --git a/src/backend/core/channels/protocol.py b/src/backend/integrations/channels/protocol.py similarity index 100% rename from src/backend/core/channels/protocol.py rename to src/backend/integrations/channels/protocol.py diff --git a/src/backend/core/channels/registry.py b/src/backend/integrations/channels/registry.py similarity index 88% rename from src/backend/core/channels/registry.py rename to src/backend/integrations/channels/registry.py index 77f3f029..2a4f56db 100644 --- a/src/backend/core/channels/registry.py +++ b/src/backend/integrations/channels/registry.py @@ -11,7 +11,7 @@ from contextvars import ContextVar from typing import Dict, List -from core.channels.protocol import ChannelAdapter +from integrations.channels.protocol import ChannelAdapter _relay_adapter = ContextVar("channel_relay_adapter", default=None) @@ -76,10 +76,10 @@ def _ensure_builtin_loaded() -> None: log = logging.getLogger(__name__) # Each adapter has its own try: a missing SDK/dependency for one channel only disables that channel, without dragging down the others or the process. for channel_type, module, cls_name in ( - ("lark", "core.channels.adapters.lark", "LarkAdapter"), - ("dingtalk", "core.channels.adapters.dingtalk", "DingTalkAdapter"), - ("wecom", "core.channels.adapters.wecom", "WeComAdapter"), - ("weixin", "core.channels.adapters.weixin", "WeixinAdapter"), + ("lark", "integrations.channels.adapters.lark", "LarkAdapter"), + ("dingtalk", "integrations.channels.adapters.dingtalk", "DingTalkAdapter"), + ("wecom", "integrations.channels.adapters.wecom", "WeComAdapter"), + ("weixin", "integrations.channels.adapters.weixin", "WeixinAdapter"), ): try: mod = __import__(module, fromlist=[cls_name]) diff --git a/src/backend/core/services/channel_relay.py b/src/backend/integrations/channels/relay.py similarity index 98% rename from src/backend/core/services/channel_relay.py rename to src/backend/integrations/channels/relay.py index 4a77c97b..e2f00ba6 100644 --- a/src/backend/core/services/channel_relay.py +++ b/src/backend/integrations/channels/relay.py @@ -8,8 +8,8 @@ import uuid from dataclasses import asdict -from core.db.models import ChannelConnection -from core.db.models.channel_relay import ChannelRelayDelivery, DesktopChannelBinding +from integrations.channels.models import ChannelConnection +from integrations.channels.models import ChannelRelayDelivery, DesktopChannelBinding from core.infra.exceptions import AccessDeniedError, BadRequestError from sqlalchemy import func, select, update from sqlalchemy.exc import IntegrityError diff --git a/src/backend/core/services/channel_relay_dispatch.py b/src/backend/integrations/channels/relay_dispatch.py similarity index 89% rename from src/backend/core/services/channel_relay_dispatch.py rename to src/backend/integrations/channels/relay_dispatch.py index 92a1ab13..5cf7a1e6 100644 --- a/src/backend/core/services/channel_relay_dispatch.py +++ b/src/backend/integrations/channels/relay_dispatch.py @@ -4,13 +4,13 @@ import logging import time -from core.auth.desktop_bridge import bridge_enabled -from core.channels.protocol import InboundMsg -from core.channels.registry import get_adapter +from features.desktop.auth.bridge import bridge_enabled +from integrations.channels.protocol import InboundMsg +from integrations.channels.registry import get_adapter from core.db.engine import SessionLocal -from core.db.models import ChannelConnection -from core.db.models.channel_relay import ChannelRelayDelivery, ChannelRelayOperation -from core.services.channel_relay import ChannelRelayService, execution +from integrations.channels.models import ChannelConnection +from integrations.channels.models import ChannelRelayDelivery, ChannelRelayOperation +from integrations.channels.relay import ChannelRelayService, execution from sqlalchemy import delete, select, update logger = logging.getLogger(__name__) @@ -25,7 +25,7 @@ async def dispatch_to_desktop(msg, factory): return False if not conn.enabled: return True - from core.channels.inbound import _resolve_addressed + from integrations.channels.inbound_observation import resolve_addressed as _resolve_addressed adapter = get_adapter(conn.channel_type) if msg.chat_type == "group": diff --git a/src/backend/core/services/channel_relay_rpc.py b/src/backend/integrations/channels/relay_rpc.py similarity index 95% rename from src/backend/core/services/channel_relay_rpc.py rename to src/backend/integrations/channels/relay_rpc.py index 38e0968b..8edc0cc8 100644 --- a/src/backend/core/services/channel_relay_rpc.py +++ b/src/backend/integrations/channels/relay_rpc.py @@ -5,11 +5,11 @@ import base64 from dataclasses import asdict -from core.channels.protocol import InboundMsg, SendResult -from core.channels.registry import get_adapter -from core.db.models.channel_relay import ChannelRelayOperation +from integrations.channels.protocol import InboundMsg, SendResult +from integrations.channels.registry import get_adapter +from integrations.channels.models import ChannelRelayOperation from core.infra.exceptions import BadRequestError -from core.services.channel_relay import ChannelRelayService +from integrations.channels.relay import ChannelRelayService from sqlalchemy.exc import IntegrityError MAX_FILE = 50 * 1024 * 1024 diff --git a/src/backend/core/db/repository/channel.py b/src/backend/integrations/channels/repository.py similarity index 98% rename from src/backend/core/db/repository/channel.py rename to src/backend/integrations/channels/repository.py index 85578720..2f14f234 100644 --- a/src/backend/core/db/repository/channel.py +++ b/src/backend/integrations/channels/repository.py @@ -10,7 +10,7 @@ from datetime import datetime from typing import Any, Dict, List, Optional, Tuple -from core.db.models import ChannelConnection +from integrations.channels.models import ChannelConnection from core.infra.time import utc_now from sqlalchemy import desc from sqlalchemy.orm import Session diff --git a/src/backend/core/services/channel_service.py b/src/backend/integrations/channels/service.py similarity index 97% rename from src/backend/core/services/channel_service.py rename to src/backend/integrations/channels/service.py index 42fb9f35..c9e7095b 100644 --- a/src/backend/core/services/channel_service.py +++ b/src/backend/integrations/channels/service.py @@ -18,9 +18,9 @@ from sqlalchemy.orm import Session from core.auth.capabilities import resolve_user_capabilities -from core.channels.registry import get_adapter, list_adapters -from core.db.models import ChannelConnection -from core.db.repository.channel import ChannelConnectionRepository +from integrations.channels.registry import get_adapter, list_adapters +from integrations.channels.models import ChannelConnection +from integrations.channels.repository import ChannelConnectionRepository from core.infra.background import spawn from core.infra.crypto import encrypt_secret from core.infra.exceptions import ( @@ -82,7 +82,7 @@ def list_owner_conversations(db, owner_id: str, *, limit: int = 200) -> List[Dic def bot_to_dict(conn: ChannelConnection) -> Dict[str, Any]: """ORM → safe dict (never echoes encrypted credentials back).""" - from core.services.channel_relay import public_execution + from integrations.channels.relay import public_execution return { **public_execution(conn), "channel_id": conn.channel_id, @@ -130,7 +130,7 @@ def _validate_agent(self, owner_id: str, agent_id: Optional[str]) -> Optional[st agent_id = (agent_id or "").strip() or None if agent_id is None: return None - from core.services.user_agent_service import UserAgentService + from core.capabilities.agents.edition import UserAgentService try: UserAgentService(self.db).get_by_id(agent_id, user_id=owner_id) @@ -195,7 +195,7 @@ async def create_bot( config[f"{key}_enc"] = encrypt_secret(sval) if local_binding_id: - from core.services.channel_relay import ChannelRelayService + from integrations.channels.relay import ChannelRelayService config["desktop_execution"] = ChannelRelayService(self.db).bind(owner_id, local_binding_id) conn = self.repo.create({ @@ -350,7 +350,7 @@ async def poll_weixin_bind(self, owner_id: str, bind_id: str) -> Dict[str, Any]: if existing is not None: if existing.owner_user_id != owner_id: raise AccessDeniedError("该微信已被其他用户绑定") - from core.services.channel_relay import execution + from integrations.channels.relay import execution selected_local = bool(ctx.get("local_binding_id")) current_binding = execution(existing) if selected_local or current_binding: @@ -366,7 +366,7 @@ async def poll_weixin_bind(self, owner_id: str, bind_id: str) -> Dict[str, Any]: if isinstance(user_id, str) and user_id.strip(): config["ilink_user_id"] = user_id if ctx.get("local_binding_id"): - from core.services.channel_relay import ChannelRelayService + from integrations.channels.relay import ChannelRelayService config["desktop_execution"] = ChannelRelayService(self.db).bind(owner_id, ctx["local_binding_id"]) if status.get("baseurl"): config["baseurl"] = status["baseurl"] @@ -422,7 +422,7 @@ def handle_webhook( inbound = adapter.parse_inbound(conn, payload) if inbound is None: return {"code": 0} # non-message event, ack - from core.channels.inbound import handle_inbound + from integrations.channels.inbound import handle_inbound spawn(handle_inbound(inbound), name="channel.handle_inbound") return {"code": 0} @@ -454,7 +454,7 @@ def _start_long_conn(self, channel_id: str, channel_type: str) -> None: def _manager(): - from core.channels.manager import get_manager + from integrations.channels.manager import get_manager return get_manager() diff --git a/src/backend/integrations/channels/tools/__init__.py b/src/backend/integrations/channels/tools/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/src/backend/integrations/channels/tools/__init__.py @@ -0,0 +1 @@ + diff --git a/src/backend/core/llm/tools/channel_attachment_tool.py b/src/backend/integrations/channels/tools/attachment.py similarity index 87% rename from src/backend/core/llm/tools/channel_attachment_tool.py rename to src/backend/integrations/channels/tools/attachment.py index 8c4b4640..6ad771b5 100644 --- a/src/backend/core/llm/tools/channel_attachment_tool.py +++ b/src/backend/integrations/channels/tools/attachment.py @@ -14,6 +14,7 @@ Registered only for channel runs (see agent_factory's ``_is_channel_run``). """ +from core.llm.factory.tools.definitions import function_tool import json import logging @@ -23,7 +24,7 @@ from agentscope.message import TextBlock from agentscope.tool import Toolkit -from agentscope.tool._response import ToolChunk as ToolResponse +from agentscope.tool._response import ToolChunk logger = logging.getLogger(__name__) @@ -31,15 +32,16 @@ _MAX_ATTACHMENT_BYTES = 50 * 1024 * 1024 -def _err(msg: str) -> ToolResponse: - return ToolResponse(content=[TextBlock( +def _err(msg: str) -> ToolChunk: + return ToolChunk(content=[TextBlock( type="text", text=json.dumps({"error": msg}, ensure_ascii=False), )]) def _lookup(db, chat_id: str, user_id: Optional[str], file_key: str): """Resolve (session, channel connection, index entry) or return an error string.""" - from core.db.models import ChannelConnection, ChatSession + from integrations.channels.models import ChannelConnection + from core.db.models import ChatSession session = ( db.query(ChatSession) @@ -77,7 +79,7 @@ def _rebuild_inbound(conn, session, entry: Dict[str, Any]): in ``raw``. Both were captured at observation time, so no channel-specific branching is needed here. """ - from core.channels.protocol import InboundMsg + from integrations.channels.protocol import InboundMsg return InboundMsg( channel_id=conn.channel_id, @@ -95,7 +97,7 @@ def register_channel_attachment( ) -> None: """Register ``channel_read_attachment`` for channel (IM bot) runs.""" - async def channel_read_attachment(file_key: str) -> ToolResponse: + async def channel_read_attachment(file_key: str) -> ToolChunk: """按 key 取回群聊里出现过但尚未下载的文件（图片 / 文档 / 表格等）。 群聊旁听只记录了文件的**句柄**，没有下载内容。当你判断某个文件与当前问题 @@ -113,10 +115,10 @@ async def channel_read_attachment(file_key: str) -> ToolResponse: JSON: 成功返回 {file_id, filename, mime_type, size, next: 使用提示}； 失败返回 {error: 原因}。下载码可能过期，过旧的文件可能取不回。 """ - from core.channels.protocol import ChannelResourceError - from core.channels.registry import get_adapter + from integrations.channels.protocol import ChannelResourceError + from integrations.channels.registry import get_adapter from core.db.engine import SessionLocal - from core.services.artifact_service import store_bytes_as_artifact + from core.files.artifacts.service import store_bytes_as_artifact key = (file_key or "").strip() if not key: @@ -168,7 +170,7 @@ async def channel_read_attachment(file_key: str) -> ToolResponse: mime_type=mime, chat_id=chat_id, source="channel_observed", extra={"channel_id": conn.channel_id, "observed_file_key": key}, ) - return ToolResponse(content=[TextBlock(type="text", text=json.dumps({ + return ToolChunk(content=[TextBlock(type="text", text=json.dumps({ "file_id": art.artifact_id, "filename": name, "mime_type": mime, @@ -183,4 +185,16 @@ async def channel_read_attachment(file_key: str) -> ToolResponse: ), }, ensure_ascii=False))]) - toolkit.register_tool_function(channel_read_attachment, namesake_strategy="override") + toolkit.append(function_tool(channel_read_attachment)) + + +async def install_attachment_tool(toolkit, context): + register_channel_attachment( + toolkit, user_id=context.request.current_user_id, chat_id=context.request.chat_id + ) + + +def register_native_contribution(): + from core.harness.registry import register_tool + register_tool("channel_attachment", install_attachment_tool, + lambda context: context.is_channel_run, stage="after_image") diff --git a/src/backend/integrations/dingtalk/README.md b/src/backend/integrations/dingtalk/README.md new file mode 100644 index 00000000..3c466b11 --- /dev/null +++ b/src/backend/integrations/dingtalk/README.md @@ -0,0 +1,5 @@ +# dingtalk 集成 + +钉钉连接由 service 管理登录、刷新和注销，parsing 解析 CLI 输出及二维码。models/repository 保存连接状态，凭据仍存放在用户独立目录。sandbox 注册环境、挂载及 Cube 注入；临时沙箱缓存按 provider 与 sandbox ID 隔离，30 秒导出缓存和原失效校验不变。 + +FULL 与 CE 均安装，模型注册独立于运行开关。API/CLI 由宿主显式注册凭据贡献；不会因导入通用 sandbox 自动配置具体集成。挂载顺序保留 DingTalk、Lark、Email、Yida，命令注入顺序保留 DingTalk、Email、Lark、Yida。调用方直接引用所属模块，无旧路径转发。 diff --git a/src/backend/integrations/dingtalk/__init__.py b/src/backend/integrations/dingtalk/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/src/backend/integrations/dingtalk/__init__.py @@ -0,0 +1 @@ + diff --git a/src/backend/integrations/dingtalk/cube_credentials.py b/src/backend/integrations/dingtalk/cube_credentials.py new file mode 100644 index 00000000..705025bf --- /dev/null +++ b/src/backend/integrations/dingtalk/cube_credentials.py @@ -0,0 +1,127 @@ +"""Integration-owned ephemeral sandbox credential handling.""" +from __future__ import annotations +import base64 +import logging +import time +from pathlib import Path +from typing import Any, Optional +logger = logging.getLogger(__name__) + +class CubeCredentials: + def __init__(self, request_timeout): + self._request_timeout_s = request_timeout + self._dingtalk_bootstrapped = set() + self._dingtalk_bundle_cache = {} + + _DINGTALK_BUNDLE_TTL_S = 30.0 + + async def _export_live_dingtalk_bundle(self, user_id: str) -> Optional[str]: + """Run ``dws auth export --base64`` live against this user's backend-resident HOME to get a live credential bundle. + + Call precondition: this method is only called by :meth:`_get_dingtalk_bundle` + after ``verify_and_refresh`` has ruled out ``invalid`` — at that point the + access token in the backend HOME is freshly refreshed, so the exported bundle + is a **currently valid** credential. (``dws auth export`` itself does not + trigger a refresh, so the verify step's real request must renew the token + first; otherwise, once access expires, the export would be an expired + ticket.) Carries a ~30s short cache: when multiple session sandboxes are + created almost simultaneously, only one export is forked. Returns None on + export failure; the caller decides the fallback based on the verdict. + """ + now = time.monotonic() + cached = self._dingtalk_bundle_cache.get(user_id) + if cached and now - cached[1] < self._DINGTALK_BUNDLE_TTL_S: + return cached[0] + try: + from integrations.dingtalk.service import _run_dws + + out, _, rc = await _run_dws(user_id, ["auth", "export", "--base64"], timeout=30) + except Exception as e: # noqa: BLE001 + logger.info("[cube] 现导出钉钉凭据失败 user=%s: %s", user_id, str(e)[:120]) + return None + bundle = out.strip() if rc == 0 else "" + if not bundle: + return None + self._dingtalk_bundle_cache[user_id] = (bundle, now) + return bundle + + async def _get_dingtalk_bundle(self, user_id: str) -> Optional[str]: + """The user's portable DingTalk credential bundle (base64), if connected. Returned only when status=connected AND **server-side verification passes**. + + Before injection, :func:`verify_and_refresh` makes one real request to + validate the backend HOME credentials: + - ``invalid`` (refresh token already rotated/expired) → persist + disconnected to the DB + prompt to reconnect; **do not inject dead + credentials** for the sandbox to bang its head against; + - ``valid`` → that verification has already made the **backend HOME + refresh + rotate in place** (single holder, eliminating multi-copy + rotation races); the access token is a freshly rotated ~2h ticket, then + exported live and injected — during the session (minutes-scale) the + sandbox calls with this valid access token directly, triggering no + refresh and no rotation, so the backend's refresh token is not + invalidated by the sandbox copy; + - ``unknown`` (network/rate-limit, inconclusive) → leave the DB untouched, + best-effort live export, fall back to the frozen snapshot only if the + export fails. + """ + try: + from core.db.engine import SessionLocal + from integrations.dingtalk.repository import DingTalkConnectionRepository + from integrations.dingtalk.service import mark_login_expired, verify_and_refresh + + with SessionLocal() as db: + rec = DingTalkConnectionRepository(db).get(user_id) + if not rec or rec.status != "connected": + return None + verdict = await verify_and_refresh(user_id) + if verdict == "invalid": + mark_login_expired(user_id) + logger.info("[cube] 钉钉凭据已失效，置 disconnected user=%s", user_id) + return None + fresh = await self._export_live_dingtalk_bundle(user_id) + if fresh: + return fresh + # Export failed: for valid we must not fall back to the necessarily older frozen snapshot; only unknown gets the fallback + return (rec.auth_bundle or None) if verdict == "unknown" else None + except Exception as e: # noqa: BLE001 + logger.warning("[cube] 读钉钉凭据包失败 user=%s: %s", user_id, str(e)[:120]) + return None + + async def ensure(self, sbx: Any, user_id: str) -> None: + """Import this user's dws credential bundle into this MicroVM, keyed by the **session-bound user_id** (one-time). + + Multi-tenant isolation crux: credentials are only injected into the user's + own session sandbox — never across users, never into warm/unbound sandboxes + (this method is called only after execute has obtained the session's sandbox + and the command contains dws). Skipped when there is no connection / no + credential bundle — dws commands then fail as not-logged-in, and the skill + guides the user to connect on the settings page. + """ + sid = getattr(sbx, "sandbox_id", None) + if not sid or sid in self._dingtalk_bootstrapped: + return + self._dingtalk_bootstrapped.add( + sid + ) # register first, to avoid duplicate DB lookups under concurrency / when no credentials exist + bundle = await self._get_dingtalk_bundle(user_id) + if not bundle: + return + try: + from integrations.dingtalk.environment import dws_extra_envs + + await sbx.files.write("/tmp/.dws-bundle.b64", bundle) + await sbx.commands.run( + "dws auth import --base64 -i /tmp/.dws-bundle.b64 --force " + "2>/dev/null; rm -f /tmp/.dws-bundle.b64", + user="root", + envs=dws_extra_envs() or None, + timeout=60, + request_timeout=60 + self._request_timeout_s, + ) + logger.info("[cube] 已为 user=%s 注入钉钉凭据到 sandbox=%s", user_id, sid) + except Exception as e: # noqa: BLE001 + # Injection failure does not block this command (dws will naturally report not-logged-in); the next sandbox will retry. + self._dingtalk_bootstrapped.discard(sid) + logger.warning( + "[cube] 注入钉钉凭据失败 user=%s sandbox=%s: %s", user_id, sid, str(e)[:160] + ) diff --git a/src/backend/integrations/dingtalk/environment.py b/src/backend/integrations/dingtalk/environment.py new file mode 100644 index 00000000..e5c193f6 --- /dev/null +++ b/src/backend/integrations/dingtalk/environment.py @@ -0,0 +1,29 @@ +"""DingTalk application environment for CLI subprocesses.""" +import logging +logger = logging.getLogger(__name__) + +def dws_extra_envs() -> dict[str, str]: + """Return deployment-level dws Custom App environment variables. + + This helper lives in the provider-neutral sandbox module because both the + backend OAuth subprocess and optional sandbox providers need it. Keeping + it here also lets DingTalk configuration load without importing any concrete + sandbox provider. + """ + out: dict[str, str] = {} + try: + from core.config.store.system_config import SystemConfigService + + svc = SystemConfigService.get_instance() + client_id = (svc.get("dingtalk.client_id") or "").strip() + client_secret = (svc.get("dingtalk.client_secret") or "").strip() + trusted_domains = (svc.get("dingtalk.trusted_domains") or "").strip() + except Exception as exc: # noqa: BLE001 + logger.warning("[dws] read dingtalk config failed: %s", exc) + return out + if client_id and client_secret: + out["DWS_CLIENT_ID"] = client_id + out["DWS_CLIENT_SECRET"] = client_secret + if trusted_domains: + out["DWS_TRUSTED_DOMAINS"] = trusted_domains + return out diff --git a/src/backend/integrations/dingtalk/extension.py b/src/backend/integrations/dingtalk/extension.py new file mode 100644 index 00000000..51d7b6b6 --- /dev/null +++ b/src/backend/integrations/dingtalk/extension.py @@ -0,0 +1,10 @@ +"""Installed integration contribution.""" +NAME = "dingtalk" + +def register_credentials(): + from integrations.dingtalk.sandbox import register + register() + +def setup(): + register_credentials() + return True diff --git a/src/backend/integrations/dingtalk/models.py b/src/backend/integrations/dingtalk/models.py new file mode 100644 index 00000000..e278fee3 --- /dev/null +++ b/src/backend/integrations/dingtalk/models.py @@ -0,0 +1,61 @@ +"""DingTalk connection metadata; secrets remain in per-user storage.""" +from core.db.engine import Base +from core.db.utc_datetime import UTCDateTime +from core.infra.time import utc_now +from sqlalchemy import JSON, CheckConstraint, Column, ForeignKey, Index, String, Text +from sqlalchemy.dialects.postgresql import JSONB +JSONType = JSON().with_variant(JSONB(), "postgresql") + + +class DingTalkConnection(Base): + """Per-user DingTalk account connection (used by the dingtalk skill / dws CLI). 1:1 with users_shadow. + + The credentials themselves (OAuth token + encrypted keychain) live on the backend + persistent volume at ``$STORAGE/dws_cache/{user_id}/``, bind-mounted into the sandbox + dws's ``$HOME`` — they **never go into the DB**. This table only stores the connection + **status and a DingTalk identity summary**, plus an optional portable credential bundle + (``auth_bundle``, for plan-B fallback/migration, stored after application-layer + encryption). See internal design docs. + """ + + __tablename__ = "dingtalk_connections" + + user_id = Column( + String(64), + ForeignKey("users_shadow.user_id", ondelete="CASCADE"), + primary_key=True, + ) + # disconnected (default / disconnected) | pending (device-flow login in progress) | connected | error + status = Column(String(16), nullable=False, default="disconnected") + dingtalk_user_id = Column( + String(128) + ) # DingTalk userId (backfilled from get-self after a successful connect) + dingtalk_name = Column(String(255)) # DingTalk display name + corp_id = Column(String(128)) # DingTalk corp corpId + granted_scopes = Column( + JSONType, default=list + ) # List of granted scopes (accumulated via PAT grants) + # Device-flow login state (echoed to the frontend while pending; cleared on success) + login_verification_url = Column( + Text + ) # Plain verification URL (must be used together with user_code) + login_verification_url_complete = Column( + Text + ) # Full URL with the code embedded → target of the QR code + login_user_code = Column(String(64)) + login_started_at = Column(UTCDateTime(timezone=True)) + auth_bundle = Column( + Text + ) # Plan B: dws auth export --base64 (application-layer encrypted), nullable + last_verified_at = Column(UTCDateTime(timezone=True)) + last_error = Column(Text) + created_at = Column(UTCDateTime(timezone=True), default=utc_now) + updated_at = Column(UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now) + + __table_args__ = ( + CheckConstraint( + "status IN ('disconnected', 'pending', 'connected', 'error')", + name="dingtalk_connections_status_check", + ), + Index("idx_dingtalk_connections_status", "status"), + ) diff --git a/src/backend/integrations/dingtalk/mounts.py b/src/backend/integrations/dingtalk/mounts.py new file mode 100644 index 00000000..ead2aad8 --- /dev/null +++ b/src/backend/integrations/dingtalk/mounts.py @@ -0,0 +1,77 @@ +"""Per-user OpenSandbox credential mounts.""" +from __future__ import annotations +import logging +import re +from typing import Any, Optional +from core.config.settings import settings +from core.sandbox.workspace.identity import safe_user_id +from integrations.dingtalk.storage import dws_cache_dir +logger = logging.getLogger(__name__) + +_DWS_HOME = "/home/ubuntu" + + +_DWS_CREDS_SUBPATHS: tuple[tuple[str, str], ...] = ( + # (host subpath under dws_cache/{uid}, sandbox mount point) + # Matches the backend login layout HOME=dws_cache/{uid}/home: login writes home/.dws and + # home/.local/share/dws-cli; here we bind those two directories into the sandbox ubuntu + # user's home. + ("home/.dws", f"{_DWS_HOME}/.dws"), + ("home/.local/share/dws-cli", f"{_DWS_HOME}/.local/share/dws-cli"), +) + + +def mounts(user_id: Optional[str]) -> list[Any]: + """Build the two read-write bind-mount Volumes for dws credential persistence (one set per user). + + Empty list is returned when: flag off / invalid user_id / host_storage not configured / + SDK has no Volume model — any unmet condition quietly degrades to "no credential + volumes mounted"; dws then has no credentials in the sandbox, business commands fail + with exit code 2 (not logged in), and the dingtalk skill guides the user to + "Settings → Connect DingTalk account" first. Same approach as _make_myspace_volume: + pre-create the host directories as backend (UID 1000) to avoid docker's root + auto-mkdir at bind time leaving the backend unable to sync/clean credentials. + """ + if not settings.sandbox.dws_creds_bind_mount_enabled: + return [] + uid = safe_user_id(user_id or "") + if not uid: + return [] + host_storage = (settings.sandbox.opensandbox_host_storage_path or "").strip().rstrip("/") + if not host_storage: + logger.warning( + "[opensandbox] dws creds bind mount enabled but HOST_STORAGE_PATH " + "not configured; dws will have no persisted credentials" + ) + return [] + cache_dir = dws_cache_dir(uid) + try: + for sub, _mount in _DWS_CREDS_SUBPATHS: + (cache_dir / sub).mkdir(parents=True, exist_ok=True) + except Exception as exc: # noqa: BLE001 + logger.warning( + "[opensandbox] failed to pre-create dws_cache dir %s: %s; " + "skipping dws creds bind mount", + cache_dir, + exc, + ) + return [] + try: + from opensandbox.models.sandboxes import Host, Volume + except ImportError: + return [] + slug = re.sub(r"[^a-z0-9-]", "-", uid.lower()).strip("-") or "u" + out: list[Any] = [] + for idx, (sub, mount_path) in enumerate(_DWS_CREDS_SUBPATHS): + # vol name must match k8s naming (lowercase alphanumerics + hyphens); sub contains + # / and ., so use an index instead. + vol_name = f"dws{idx}-{slug}"[:63].rstrip("-") or f"dws{idx}" + out.append( + Volume( + name=vol_name, + host=Host(path=f"{host_storage}/dws_cache/{uid}/{sub}"), + mount_path=mount_path, + read_only=False, + ) + ) + return out diff --git a/src/backend/integrations/dingtalk/parsing.py b/src/backend/integrations/dingtalk/parsing.py new file mode 100644 index 00000000..29e7a405 --- /dev/null +++ b/src/backend/integrations/dingtalk/parsing.py @@ -0,0 +1,105 @@ +"""DingTalk CLI response parsing and QR presentation.""" +import json +import logging +import re +from typing import Any, Dict, Optional, Tuple +logger = logging.getLogger(__name__) + +_URL_RE = re.compile(r"https?://[^\s\"'<>)\]]+") + +_CODE_RES = ( + re.compile(r'user[_]?code["\s:=]+([A-Za-z0-9][A-Za-z0-9-]{3,15})', re.I), + re.compile(r'(?:验证码|授权码|码)[：:\s]+([A-Za-z0-9][A-Za-z0-9-]{3,15})'), +) + +def make_qr_data_uri(url: Optional[str]) -> Optional[str]: + """Render the verification URL as an SVG QR-code data-URI (for scan-to-bind). + segno is pure Python with zero extra dependencies; if the library is + missing or url is empty, return None and the front end falls back to + "click the link + copy the user code".""" + if not url: + return None + try: + import segno + + return segno.make(url, error="m").svg_data_uri(scale=5, border=2) + except Exception as exc: # noqa: BLE001 + logger.info("[dingtalk] qr render failed: %s", exc) + return None + +def parse_device_login_output( + text: str, +) -> Tuple[Optional[str], Optional[str], Optional[str]]: + """Extract (verification URL, user_code, complete URL) from the output of ``dws auth login --device``. + + dws device-code box format (``dfPrintDeviceCodeBox``), with Chinese labels verbatim: + 链接: (link) + 授权码: (user code) + 或者直接打开以下链接： (or open this link directly:) + ← code already embedded, one scan goes straight through + The third item, ``verification_uri_complete``, is the QR-code target for + **scan-to-bind** (preferred over the plain URL). Returns None for any + item that cannot be extracted; the caller still returns the raw output + to the front end as a fallback. + """ + if not text: + return None, None, None + urls = [u.rstrip(".,;") for u in _URL_RE.findall(text)] + url = urls[0] if urls else None + code: Optional[str] = None + for rx in _CODE_RES: + m = rx.search(text) + if m: + code = m.group(1) + break + # Complete URL: prefer the one with the user_code embedded; otherwise take the second one (in the box, complete comes after plain) + complete: Optional[str] = None + if code: + for u in urls: + if code in u or "user_code=" in u or "userCode=" in u: + complete = u + break + if complete is None and len(urls) >= 2: + complete = urls[1] + return url, code, complete + +def parse_auth_status(stdout: str) -> bool: + """``dws auth status -f json`` → whether authenticated. Any parse failure counts as not authenticated.""" + try: + data = json.loads((stdout or "").strip() or "{}") + except (ValueError, TypeError): + return False + return isinstance(data, dict) and bool(data.get("authenticated")) + +def parse_get_self(stdout: str) -> Dict[str, Optional[str]]: + """``dws contact user get-self -f json`` → {dingtalk_user_id, dingtalk_name, corp_id}。 + + Best effort: extract what it can when the structure changes, leave None + when unavailable, never raise. + """ + out: Dict[str, Optional[str]] = {"dingtalk_user_id": None, "dingtalk_name": None, "corp_id": None} + try: + data = json.loads((stdout or "").strip() or "{}") + except (ValueError, TypeError): + return out + + def _walk(obj: Any) -> None: + if isinstance(obj, dict): + for k, v in obj.items(): + lk = k.lower() + if out["dingtalk_user_id"] is None and lk in ("userid", "user_id", "unionid"): + if isinstance(v, (str, int)): + out["dingtalk_user_id"] = str(v) + if out["dingtalk_name"] is None and lk in ("orgusername", "name", "nick", "username"): + if isinstance(v, str): + out["dingtalk_name"] = v + if out["corp_id"] is None and lk in ("corpid", "corp_id"): + if isinstance(v, (str, int)): + out["corp_id"] = str(v) + _walk(v) + elif isinstance(obj, list): + for it in obj: + _walk(it) + + _walk(data) + return out diff --git a/src/backend/integrations/dingtalk/repository.py b/src/backend/integrations/dingtalk/repository.py new file mode 100644 index 00000000..36835956 --- /dev/null +++ b/src/backend/integrations/dingtalk/repository.py @@ -0,0 +1,39 @@ +"""DingTalk connection persistence.""" +from typing import Any, Dict, Optional +from sqlalchemy.orm import Session +from core.infra.time import utc_now +from integrations.dingtalk.models import DingTalkConnection + + +class DingTalkConnectionRepository: + """Per-user DingTalk connection (dingtalk_connections) CRUD. 1:1 with users_shadow, no soft delete.""" + + def __init__(self, db: Session): + self.db = db + + def get(self, user_id: str) -> Optional[DingTalkConnection]: + return ( + self.db.query(DingTalkConnection).filter(DingTalkConnection.user_id == user_id).first() + ) + + def ensure(self, user_id: str) -> DingTalkConnection: + """Idempotent: return existing record, otherwise create a disconnected record.""" + record = self.get(user_id) + if record: + return record + record = DingTalkConnection(user_id=user_id, status="disconnected") + self.db.add(record) + self.db.commit() + self.db.refresh(record) + return record + + def update(self, user_id: str, data: Dict[str, Any]) -> Optional[DingTalkConnection]: + record = self.get(user_id) + if not record: + return None + for k, v in data.items(): + setattr(record, k, v) + record.updated_at = utc_now() + self.db.commit() + self.db.refresh(record) + return record diff --git a/src/backend/integrations/dingtalk/sandbox.py b/src/backend/integrations/dingtalk/sandbox.py new file mode 100644 index 00000000..8269b3dc --- /dev/null +++ b/src/backend/integrations/dingtalk/sandbox.py @@ -0,0 +1,15 @@ +"""Credential contribution for dingtalk.""" +from core.harness.registry import SandboxCredentials, register_sandbox_credentials +from integrations.dingtalk.cube_credentials import CubeCredentials +from integrations.dingtalk.mounts import mounts +from integrations.dingtalk.environment import dws_extra_envs + +def state(host): + return host.credential_state("dingtalk", lambda: CubeCredentials(host.credential_request_timeout)) + +async def prepare(host, sandbox, user_id, script): + if "dws" in (script or ""): + await state(host).ensure(sandbox, user_id) + +def register(): + register_sandbox_credentials(SandboxCredentials("dingtalk", environment=dws_extra_envs, mounts=mounts, before_command=prepare, prepare_order=0)) diff --git a/src/backend/core/services/dingtalk_service.py b/src/backend/integrations/dingtalk/service.py similarity index 80% rename from src/backend/core/services/dingtalk_service.py rename to src/backend/integrations/dingtalk/service.py index f3b91dbb..3717eae6 100644 --- a/src/backend/core/services/dingtalk_service.py +++ b/src/backend/integrations/dingtalk/service.py @@ -19,6 +19,7 @@ """ from __future__ import annotations +from integrations.dingtalk import parsing as dingtalk_parsing from core.infra.time import utc_now import asyncio @@ -31,7 +32,7 @@ from sqlalchemy.orm import Session -from core.db.repository import DingTalkConnectionRepository +from integrations.dingtalk.repository import DingTalkConnectionRepository from core.infra.proc import no_window_kwargs logger = logging.getLogger(__name__) @@ -45,111 +46,24 @@ _DWS_BIN = "dws" # Regexes for parsing device-flow output (lenient; tolerates json / plain text / Chinese and English copy). -_URL_RE = re.compile(r"https?://[^\s\"'<>)\]]+") -_CODE_RES = ( - re.compile(r'user[_]?code["\s:=]+([A-Za-z0-9][A-Za-z0-9-]{3,15})', re.I), - re.compile(r'(?:验证码|授权码|码)[：:\s]+([A-Za-z0-9][A-Za-z0-9-]{3,15})'), -) + + def dingtalk_login_session_id(user_id: str) -> str: return f"dingtalk-{user_id}" -def make_qr_data_uri(url: Optional[str]) -> Optional[str]: - """Render the verification URL as an SVG QR-code data-URI (for scan-to-bind). - segno is pure Python with zero extra dependencies; if the library is - missing or url is empty, return None and the front end falls back to - "click the link + copy the user code".""" - if not url: - return None - try: - import segno - return segno.make(url, error="m").svg_data_uri(scale=5, border=2) - except Exception as exc: # noqa: BLE001 - logger.info("[dingtalk] qr render failed: %s", exc) - return None - - -def parse_device_login_output( - text: str, -) -> Tuple[Optional[str], Optional[str], Optional[str]]: - """Extract (verification URL, user_code, complete URL) from the output of ``dws auth login --device``. - - dws device-code box format (``dfPrintDeviceCodeBox``), with Chinese labels verbatim: - 链接: (link) - 授权码: (user code) - 或者直接打开以下链接： (or open this link directly:) - ← code already embedded, one scan goes straight through - The third item, ``verification_uri_complete``, is the QR-code target for - **scan-to-bind** (preferred over the plain URL). Returns None for any - item that cannot be extracted; the caller still returns the raw output - to the front end as a fallback. - """ - if not text: - return None, None, None - urls = [u.rstrip(".,;") for u in _URL_RE.findall(text)] - url = urls[0] if urls else None - code: Optional[str] = None - for rx in _CODE_RES: - m = rx.search(text) - if m: - code = m.group(1) - break - # Complete URL: prefer the one with the user_code embedded; otherwise take the second one (in the box, complete comes after plain) - complete: Optional[str] = None - if code: - for u in urls: - if code in u or "user_code=" in u or "userCode=" in u: - complete = u - break - if complete is None and len(urls) >= 2: - complete = urls[1] - return url, code, complete -def parse_auth_status(stdout: str) -> bool: - """``dws auth status -f json`` → whether authenticated. Any parse failure counts as not authenticated.""" - try: - data = json.loads((stdout or "").strip() or "{}") - except (ValueError, TypeError): - return False - return isinstance(data, dict) and bool(data.get("authenticated")) -def parse_get_self(stdout: str) -> Dict[str, Optional[str]]: - """``dws contact user get-self -f json`` → {dingtalk_user_id, dingtalk_name, corp_id}。 - Best effort: extract what it can when the structure changes, leave None - when unavailable, never raise. - """ - out: Dict[str, Optional[str]] = {"dingtalk_user_id": None, "dingtalk_name": None, "corp_id": None} - try: - data = json.loads((stdout or "").strip() or "{}") - except (ValueError, TypeError): - return out - - def _walk(obj: Any) -> None: - if isinstance(obj, dict): - for k, v in obj.items(): - lk = k.lower() - if out["dingtalk_user_id"] is None and lk in ("userid", "user_id", "unionid"): - if isinstance(v, (str, int)): - out["dingtalk_user_id"] = str(v) - if out["dingtalk_name"] is None and lk in ("orgusername", "name", "nick", "username"): - if isinstance(v, str): - out["dingtalk_name"] = v - if out["corp_id"] is None and lk in ("corpid", "corp_id"): - if isinstance(v, (str, int)): - out["corp_id"] = str(v) - _walk(v) - elif isinstance(obj, list): - for it in obj: - _walk(it) - - _walk(data) - return out + + + + def _dws_env(user_id: str) -> Dict[str, str]: @@ -157,8 +71,8 @@ def _dws_env(user_id: str) -> Dict[str, str]: isolated ``HOME`` (the root of credential isolation) + deployment-level Custom App client credentials + domain whitelist. Multi-tenant isolation is guaranteed by the per-user HOME.""" - from core.sandbox.providers.account_env import dws_extra_envs - from core.sandbox.workspace.credentials import dws_home_dir + from integrations.dingtalk.environment import dws_extra_envs + from integrations.dingtalk.storage import dws_home_dir home = dws_home_dir(user_id) try: @@ -221,7 +135,7 @@ async def _build_connected_update(user_id: str) -> Dict[str, Any]: fallback (request session), avoiding two copies of the same "get-self → export → 11-key connected dict" block.""" who, _, _ = await _run_dws(user_id, ["contact", "user", "get-self", "-f", "json"]) - ident = parse_get_self(who) + ident = dingtalk_parsing.parse_get_self(who) bout, _, brc = await _run_dws(user_id, ["auth", "export", "--base64"], timeout=30) return { "status": "connected", @@ -287,7 +201,7 @@ async def verify_and_refresh(user_id: str) -> str: if category == "auth" or reason in _AUTH_FAIL_REASONS: return "invalid" return "unknown" # non-auth error, no conclusion drawn - ident = parse_get_self(blob) + ident = dingtalk_parsing.parse_get_self(blob) if ident.get("dingtalk_user_id") or ident.get("dingtalk_name"): return "valid" return "unknown" @@ -340,7 +254,7 @@ async def _device_login_flow(user_id: str) -> None: break # EOF → process exited (login completed or failed) buf += line.decode("utf-8", "replace") if not pushed: - u, c, comp = parse_device_login_output(buf) + u, c, comp = dingtalk_parsing.parse_device_login_output(buf) # Box-complete markers: the trailing "过期/expire" line appears, or both URL + code are in place box_done = ("过期" in buf) or ("expire" in buf.lower()) or (u and c) if box_done and (u or c): @@ -369,7 +283,7 @@ async def _device_login_flow(user_id: str) -> None: # Process exited → verify whether login actually succeeded out, _, _ = await _run_dws(user_id, ["auth", "status", "-f", "json"]) - if not parse_auth_status(out): + if not dingtalk_parsing.parse_auth_status(out): tail = buf.strip()[-400:] _update_connection(user_id, { "status": "error", @@ -403,7 +317,7 @@ def status_dict(self, record) -> Dict[str, Any]: "granted_scopes": list(record.granted_scopes or []), "verification_url": record.login_verification_url, "verification_url_complete": record.login_verification_url_complete, - "qr_data_uri": make_qr_data_uri(qr_target) if record.status == "pending" else None, + "qr_data_uri": dingtalk_parsing.make_qr_data_uri(qr_target) if record.status == "pending" else None, "user_code": record.login_user_code, "last_verified_at": record.last_verified_at.isoformat() if record.last_verified_at else None, "last_error": record.last_error, @@ -472,7 +386,7 @@ async def poll_login(self, user_id: str) -> Dict[str, Any]: record = self._fresh(user_id) if record.status == "pending" and user_id not in _LOGIN_TASKS: out, _, _ = await _run_dws(user_id, ["auth", "status", "-f", "json"]) - if parse_auth_status(out): + if dingtalk_parsing.parse_auth_status(out): self.repo.update(user_id, await _build_connected_update(user_id)) record = self.repo.get(user_id) return self.status_dict(record) @@ -490,7 +404,7 @@ async def disconnect(self, user_id: str) -> Dict[str, Any]: # unlinking a mount source makes dws report "not logged in" in every open # chat even after a successful re-login (see [[purge_credential_dir]]). try: - from core.sandbox.workspace.credentials import dws_cache_dir + from integrations.dingtalk.storage import dws_cache_dir from core.sandbox.workspace.credentials import purge_credential_dir from core.sandbox.workspace.identity import safe_user_id diff --git a/src/backend/integrations/dingtalk/storage.py b/src/backend/integrations/dingtalk/storage.py new file mode 100644 index 00000000..3ca4af74 --- /dev/null +++ b/src/backend/integrations/dingtalk/storage.py @@ -0,0 +1,34 @@ +"""Per-user integration credential storage paths.""" +from pathlib import Path +from core.config.settings import settings + +def dws_cache_dir(user_id: str) -> Path: + """Backend-local dws (DingTalk CLI) credential persistence directory (one per user). + + Contains two subdirectories, bind-mounted into the sandbox DingTalk CLI's default + ``$HOME`` paths: + - ``dws/`` → sandbox ``/home/ubuntu/.dws`` (config + market cache + app.json) + - ``share/`` → sandbox ``/home/ubuntu/.local/share/dws-cli`` (encrypted keychain: + ``dek`` + ``auth-token.enc``) + + Empirically: dws's encrypted keychain and the ``.dws`` cache only respect ``$HOME``; + ``DWS_CONFIG_DIR`` only relocates identity.json/logs, and ``XDG_DATA_HOME`` is + ignored — so login must run with a per-user ``$HOME`` and these two home subpaths + must be bind-mounted, rather than relying on env redirection. The sandbox runs as + ubuntu (UID 1000) and the backend is also 1000, so ownership matches + (see [[project_dingtalk_dws_integration]]). + """ + return settings.storage.root / "dws_cache" / user_id + + +def dws_home_dir(user_id: str) -> Path: + """Per-user ``$HOME`` for dws login (the unified root for backend subprocesses + + opensandbox bind sources). + + The backend runs ``dws auth login`` with this as ``HOME`` → credentials land in + ``{home}/.dws`` and ``{home}/.local/share/dws-cli``; opensandbox bind-mounts these + two subpaths into the same home paths inside the sandbox, so **that user's** session + sandboxes (bucketed per user) see their own credentials. The root of multi-tenant + isolation: each user gets an independent home directory, never shared. + """ + return dws_cache_dir(user_id) / "home" diff --git a/src/backend/integrations/email/README.md b/src/backend/integrations/email/README.md new file mode 100644 index 00000000..d90e475e --- /dev/null +++ b/src/backend/integrations/email/README.md @@ -0,0 +1,5 @@ +# email 集成 + +邮箱绑定、配置生成与连接探测由 service 管理，models/repository 保存账号状态及加密字段。storage/mounts/cube_credentials 保留每用户配置文件路径与注入缓存。注销仅清除文件，保留挂载目录 inode。 + +FULL 和 CE 均安装；模型加载独立于运行开关，API/CLI 宿主显式注册沙箱贡献。原路由仍由总集成 API 提供，迁移不改变协议、账号归属或运行开关。调用方直接引用所属服务，不保留旧路径转发。 diff --git a/src/backend/integrations/email/__init__.py b/src/backend/integrations/email/__init__.py new file mode 100644 index 00000000..90b4bc33 --- /dev/null +++ b/src/backend/integrations/email/__init__.py @@ -0,0 +1 @@ +"""Integration package; schema migration is tracked separately.""" diff --git a/src/backend/integrations/email/cube_credentials.py b/src/backend/integrations/email/cube_credentials.py new file mode 100644 index 00000000..decc3b37 --- /dev/null +++ b/src/backend/integrations/email/cube_credentials.py @@ -0,0 +1,53 @@ +"""Integration-owned ephemeral sandbox credential handling.""" +from __future__ import annotations +import base64 +import logging +import time +from pathlib import Path +from typing import Any, Optional +logger = logging.getLogger(__name__) + +class CubeCredentials: + def __init__(self, request_timeout): + self._request_timeout_s = request_timeout + self._email_bootstrapped = set() + + async def ensure(self, sbx: Any, user_id: str) -> None: + """Write this user's himalaya config.toml into this MicroVM, keyed by the **session-bound user_id** (one-time). + + Multi-tenant isolation crux, same as dws: credentials are only injected into + the user's own session sandbox — never across users, never into warm/unbound + sandboxes. Skipped when there is no connection / no credentials — himalaya + commands then fail as unconfigured, and the email skill guides the user to + bind a mailbox on the plugin detail page. cube has no bind-mount, so the + config.toml file is written directly. + """ + sid = getattr(sbx, "sandbox_id", None) + if not sid or sid in self._email_bootstrapped: + return + self._email_bootstrapped.add( + sid + ) # register first, to avoid duplicate DB lookups under concurrency / when no credentials exist + try: + from integrations.email.service import get_email_config_bundle + + bundle = get_email_config_bundle(user_id) + if not bundle: + return + import base64 as _b64 + + content = _b64.b64decode(bundle).decode("utf-8") + # The cube sandbox runs as root, HOME=/root → config lands at /root/.config/himalaya/config.toml + await sbx.files.write("/root/.config/himalaya/config.toml", content) + await sbx.commands.run( + "chmod 600 /root/.config/himalaya/config.toml 2>/dev/null || true", + user="root", + timeout=30, + request_timeout=30 + self._request_timeout_s, + ) + logger.info("[cube] 已为 user=%s 注入邮箱凭据到 sandbox=%s", user_id, sid) + except Exception as e: # noqa: BLE001 + self._email_bootstrapped.discard(sid) + logger.warning( + "[cube] 注入邮箱凭据失败 user=%s sandbox=%s: %s", user_id, sid, str(e)[:160] + ) diff --git a/src/backend/integrations/email/extension.py b/src/backend/integrations/email/extension.py new file mode 100644 index 00000000..b170e8b5 --- /dev/null +++ b/src/backend/integrations/email/extension.py @@ -0,0 +1,10 @@ +"""Installed integration contribution.""" +NAME = "email" + +def register_credentials(): + from integrations.email.sandbox import register + register() + +def setup(): + register_credentials() + return True diff --git a/src/backend/integrations/email/models.py b/src/backend/integrations/email/models.py new file mode 100644 index 00000000..a4703f3d --- /dev/null +++ b/src/backend/integrations/email/models.py @@ -0,0 +1,65 @@ +"""Email connection persistence metadata.""" +from core.db.engine import Base +from core.db.utc_datetime import UTCDateTime +from core.infra.time import utc_now +from sqlalchemy import Boolean, CheckConstraint, Column, ForeignKey, Index, Integer, String, Text + +class EmailConnection(Base): + """Per-user email account connection (used by the email plugin / himalaya CLI). 1:1 with users_shadow. + + Isomorphic to [[DingTalkConnection]]/[[LarkConnection]], but with **no device flow / OAuth / + QR code**: email binds via per-user IMAP/SMTP + an authorization code (app password), + covering Gmail/Outlook/Exchange/NetEase enterprise mail/Tencent enterprise mail/self-hosted + mailboxes. Binding is **synchronous** ("save the form → write himalaya config.toml → run + ``himalaya folder list`` to verify"), so the status is only + disconnected/connected/error (no pending). + + The credentials themselves (himalaya ``config.toml``, containing plaintext authorization-code + lines) live on the backend persistent volume at + ``$STORAGE/email_cache/{user_id}/home/.config/himalaya/`` (0600), bind-mounted into the + sandbox himalaya's ``~/.config/himalaya``. This table only stores the connection status + + account/server metadata + the encrypted authorization code (``secret_enc``, + application-layer Fernet encryption) + a portable config bundle (``config_bundle``, injected + per-session when cube has no bind-mount, base64(config.toml)). + See internal design docs. + """ + + __tablename__ = "email_connections" + + user_id = Column( + String(64), + ForeignKey("users_shadow.user_id", ondelete="CASCADE"), + primary_key=True, + ) + # disconnected (default / disconnected) | connected | error (no pending — binding is a synchronous check) + status = Column(String(16), nullable=False, default="disconnected") + email_address = Column(String(320)) # Bound email address (= IMAP/SMTP login name) + display_name = Column(String(255)) # Sender display name + provider = Column( + String(32) + ) # Auto-detected provider: gmail/outlook/netease/qq/qiye163/exmail/custom + + # Server settings (auto-detected + user-overridable) + imap_host = Column(String(255)) + imap_port = Column(Integer) + imap_security = Column(String(16)) # tls | starttls | none + smtp_host = Column(String(255)) + smtp_port = Column(Integer) + smtp_security = Column(String(16)) + + secret_enc = Column( + Text + ) # Authorization code (application-layer Fernet encryption, never stored in plaintext) + config_bundle = Column(Text) # base64(config.toml), for cube cross-session injection, nullable + last_verified_at = Column(UTCDateTime(timezone=True)) + last_error = Column(Text) + created_at = Column(UTCDateTime(timezone=True), default=utc_now) + updated_at = Column(UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now) + + __table_args__ = ( + CheckConstraint( + "status IN ('disconnected', 'connected', 'error')", + name="email_connections_status_check", + ), + Index("idx_email_connections_status", "status"), + ) diff --git a/src/backend/integrations/email/mounts.py b/src/backend/integrations/email/mounts.py new file mode 100644 index 00000000..ddb246b8 --- /dev/null +++ b/src/backend/integrations/email/mounts.py @@ -0,0 +1,72 @@ +"""Per-user OpenSandbox credential mounts.""" +from __future__ import annotations +import logging +import re +from typing import Any, Optional +from core.config.settings import settings +from core.sandbox.workspace.identity import safe_user_id +from integrations.email.storage import email_cache_dir +logger = logging.getLogger(__name__) + +_EMAIL_HOME = "/home/ubuntu" + + +_EMAIL_CREDS_SUBPATHS: tuple[tuple[str, str], ...] = ( + # (host subpath under email_cache/{uid}, sandbox mount point); matches the backend + # layout HOME=email_cache/{uid}/home: config lands in home/.config/himalaya; here we + # bind it to the same path under the sandbox ubuntu home. + ("home/.config/himalaya", f"{_EMAIL_HOME}/.config/himalaya"), +) + + +def mounts(user_id: Optional[str]) -> list[Any]: + """Build the read-write bind-mount Volume for himalaya credential persistence (one per user, single directory). + + Same approach as [[_make_lark_creds_volumes]]: flag off / invalid user_id / host_storage + not configured / SDK has no Volume model — any unmet condition quietly degrades to "no + credential volume mounted"; himalaya then has no config in the sandbox, business + commands fail as unconfigured, and the email skill guides the user to bind a mailbox on + the plugin detail page first. Pre-create the host directories as backend (UID 1000) to + avoid docker's root auto-mkdir at bind time leaving the backend unable to sync/clean. + """ + if not settings.sandbox.email_creds_bind_mount_enabled: + return [] + uid = safe_user_id(user_id or "") + if not uid: + return [] + host_storage = (settings.sandbox.opensandbox_host_storage_path or "").strip().rstrip("/") + if not host_storage: + logger.warning( + "[opensandbox] email creds bind mount enabled but HOST_STORAGE_PATH " + "not configured; himalaya will have no persisted credentials" + ) + return [] + cache_dir = email_cache_dir(uid) + try: + for sub, _mount in _EMAIL_CREDS_SUBPATHS: + (cache_dir / sub).mkdir(parents=True, exist_ok=True) + except Exception as exc: # noqa: BLE001 + logger.warning( + "[opensandbox] failed to pre-create email_cache dir %s: %s; " + "skipping email creds bind mount", + cache_dir, + exc, + ) + return [] + try: + from opensandbox.models.sandboxes import Host, Volume + except ImportError: + return [] + slug = re.sub(r"[^a-z0-9-]", "-", uid.lower()).strip("-") or "u" + out: list[Any] = [] + for idx, (sub, mount_path) in enumerate(_EMAIL_CREDS_SUBPATHS): + vol_name = f"email{idx}-{slug}"[:63].rstrip("-") or f"email{idx}" + out.append( + Volume( + name=vol_name, + host=Host(path=f"{host_storage}/email_cache/{uid}/{sub}"), + mount_path=mount_path, + read_only=False, + ) + ) + return out diff --git a/src/backend/integrations/email/repository.py b/src/backend/integrations/email/repository.py new file mode 100644 index 00000000..b727a8ab --- /dev/null +++ b/src/backend/integrations/email/repository.py @@ -0,0 +1,37 @@ +"""Email connection persistence metadata.""" +from typing import Any, Dict, Optional +from sqlalchemy.orm import Session +from core.infra.time import utc_now +from integrations.email.models import EmailConnection + +class EmailConnectionRepository: + """Per-user email connection (email_connections) CRUD. 1:1 with users_shadow, no soft delete. + Structurally identical to [[LarkConnectionRepository]].""" + + def __init__(self, db: Session): + self.db = db + + def get(self, user_id: str) -> Optional[EmailConnection]: + return self.db.query(EmailConnection).filter(EmailConnection.user_id == user_id).first() + + def ensure(self, user_id: str) -> EmailConnection: + """Idempotent: return existing record, otherwise create a disconnected record.""" + record = self.get(user_id) + if record: + return record + record = EmailConnection(user_id=user_id, status="disconnected") + self.db.add(record) + self.db.commit() + self.db.refresh(record) + return record + + def update(self, user_id: str, data: Dict[str, Any]) -> Optional[EmailConnection]: + record = self.get(user_id) + if not record: + return None + for k, v in data.items(): + setattr(record, k, v) + record.updated_at = utc_now() + self.db.commit() + self.db.refresh(record) + return record diff --git a/src/backend/integrations/email/sandbox.py b/src/backend/integrations/email/sandbox.py new file mode 100644 index 00000000..4f00cb18 --- /dev/null +++ b/src/backend/integrations/email/sandbox.py @@ -0,0 +1,14 @@ +"""Credential contribution for email.""" +from core.harness.registry import SandboxCredentials, register_sandbox_credentials +from integrations.email.cube_credentials import CubeCredentials +from integrations.email.mounts import mounts + +def state(host): + return host.credential_state("email", lambda: CubeCredentials(host.credential_request_timeout)) + +async def prepare(host, sandbox, user_id, script): + if "himalaya" in (script or ""): + await state(host).ensure(sandbox, user_id) + +def register(): + register_sandbox_credentials(SandboxCredentials("email", mounts=mounts, before_command=prepare, prepare_order=1)) diff --git a/src/backend/core/services/email_service.py b/src/backend/integrations/email/service.py similarity index 98% rename from src/backend/core/services/email_service.py rename to src/backend/integrations/email/service.py index 3bc340b1..3e2934c6 100644 --- a/src/backend/core/services/email_service.py +++ b/src/backend/integrations/email/service.py @@ -33,7 +33,7 @@ from sqlalchemy.orm import Session -from core.db.repository import EmailConnectionRepository +from integrations.email.repository import EmailConnectionRepository from core.infra.crypto import decrypt_secret, encrypt_secret from core.infra.proc import no_window_kwargs @@ -163,7 +163,7 @@ def esc(v: str) -> str: def _write_config(user_id: str, content: str) -> "Optional[str]": """Write config.toml into this user's backend HOME (0600). Returns an error message; None means success.""" - from core.sandbox.workspace.credentials import email_himalaya_config + from integrations.email.storage import email_himalaya_config cfg = email_himalaya_config(user_id) try: @@ -192,7 +192,7 @@ def _render_config_from_record(record, secret: str) -> str: def _email_env(user_id: str) -> Dict[str, str]: """Environment for the backend to run himalaya subprocesses: an isolated HOME per user (the root of credential isolation).""" - from core.sandbox.workspace.credentials import email_home_dir + from integrations.email.storage import email_home_dir home = email_home_dir(user_id) try: @@ -212,7 +212,7 @@ async def _run_himalaya( ) -> Tuple[str, str, int]: """Run one himalaya subcommand on the backend as this user; returns (stdout, stderr, exit_code). Points ``--config`` explicitly at this user's config.toml to guarantee no account cross-talk.""" - from core.sandbox.workspace.credentials import email_himalaya_config + from integrations.email.storage import email_himalaya_config cfg = str(email_himalaya_config(user_id)) try: @@ -291,7 +291,7 @@ async def probe_status(self, user_id: str) -> Dict[str, Any]: async def _reverify(self, user_id: str, record) -> Tuple[bool, str]: """Before probing, ensure the backend HOME has config.toml (after a restart / machine change only the DB row may remain) — rebuild it from secret_enc.""" - from core.sandbox.workspace.credentials import email_himalaya_config + from integrations.email.storage import email_himalaya_config if not email_himalaya_config(user_id).exists(): secret = decrypt_secret(record.secret_enc) @@ -393,7 +393,7 @@ def _purge_config(self, user_id: str) -> None: This path is hit not only by disconnect but by **every failed mailbox verification**, which made it the easiest of the three to trigger. """ - from core.sandbox.workspace.credentials import email_cache_dir + from integrations.email.storage import email_cache_dir from core.sandbox.workspace.credentials import purge_credential_dir from core.sandbox.workspace.identity import safe_user_id diff --git a/src/backend/integrations/email/storage.py b/src/backend/integrations/email/storage.py new file mode 100644 index 00000000..6e429e64 --- /dev/null +++ b/src/backend/integrations/email/storage.py @@ -0,0 +1,36 @@ +"""Per-user integration credential storage paths.""" +from pathlib import Path +from core.config.settings import settings + +def email_cache_dir(user_id: str) -> Path: + """Backend-local email (himalaya CLI) credential persistence directory (one per user). + + Contains one subdirectory, bind-mounted into the sandbox himalaya's default + config path: + - ``home/.config/himalaya/`` → sandbox ``/home/ubuntu/.config/himalaya`` + (``config.toml``: account blocks + IMAP/SMTP servers + auth-code raw lines, file mode 0600) + + Isomorphic to lark/dws's "Linux file-based credentials": himalaya config is plain + files that only respect the config path, so the per-user bind-mount persistence is + copied verbatim. No OS keychain, no OAuth token refresh — auth codes stay valid + long-term (governed by the mail provider). + See internal design docs. + """ + return settings.storage.root / "email_cache" / user_id + + +def email_home_dir(user_id: str) -> Path: + """Per-user ``$HOME`` for himalaya login (the unified root for the backend + verification subprocess + opensandbox bind sources). + + The backend runs ``himalaya folder list`` with this as ``HOME`` as a connectivity + check; the config lands at ``{home}/.config/himalaya/config.toml``; opensandbox + bind-mounts this subdirectory into the same home path inside the sandbox. The root + of multi-tenant isolation: each user gets an independent home directory, never shared. + """ + return email_cache_dir(user_id) / "home" + + +def email_himalaya_config(user_id: str) -> Path: + """Backend absolute path of this user's himalaya ``config.toml`` (under email_home_dir).""" + return email_home_dir(user_id) / ".config" / "himalaya" / "config.toml" diff --git a/src/backend/integrations/lark/README.md b/src/backend/integrations/lark/README.md new file mode 100644 index 00000000..1ce1651a --- /dev/null +++ b/src/backend/integrations/lark/README.md @@ -0,0 +1,5 @@ +# lark 集成 + +飞书连接按 parsing、process、configuration、login、service 拆分，state 统一持有共享应用初始化任务及用户登录任务。service 与后台配置任务使用同一状态，重复初始化不启动第二个任务。models/repository 保存连接状态；sandbox/storage/mounts/cube_credentials 保持每用户 HOME 和文件凭据流，不改用 tenant token。二维码沿用 integrations.dingtalk.parsing 的纯渲染函数。 + +FULL 与 CE 均安装，模型注册独立于运行开关。API/CLI 由宿主显式注册凭据贡献；不会因导入通用 sandbox 自动配置具体集成。挂载顺序保留 DingTalk、Lark、Email、Yida，命令注入顺序保留 DingTalk、Email、Lark、Yida。调用方直接引用所属模块，无旧路径转发。 diff --git a/src/backend/integrations/lark/__init__.py b/src/backend/integrations/lark/__init__.py new file mode 100644 index 00000000..90b4bc33 --- /dev/null +++ b/src/backend/integrations/lark/__init__.py @@ -0,0 +1 @@ +"""Integration package; schema migration is tracked separately.""" diff --git a/src/backend/integrations/lark/configuration.py b/src/backend/integrations/lark/configuration.py new file mode 100644 index 00000000..4af9f420 --- /dev/null +++ b/src/backend/integrations/lark/configuration.py @@ -0,0 +1,200 @@ +"""Lark configuration responsibilities.""" +from __future__ import annotations +import asyncio +import json +import os +import re +import shutil +from typing import Any, Dict, Optional, Tuple +from core.infra.proc import no_window_kwargs +from integrations.dingtalk.parsing import make_qr_data_uri +import logging +logger = logging.getLogger(__name__) +from integrations.lark import process as lark_process +from integrations.lark import state as lark_state + +def lark_app_config() -> Tuple[Optional[str], Optional[str]]: + """Read the app's app_id / app_secret from "System Config → Lark Workspace" (deployment-level, one app shared by everyone).""" + try: + from core.config.store.system_config import SystemConfigService + + svc = SystemConfigService.get_instance() + app_id = (svc.get("lark.app_id") or "").strip() + app_secret = (svc.get("lark.app_secret") or "").strip() + return (app_id or None), (app_secret or None) + except Exception as exc: # noqa: BLE001 + logger.warning("[lark] read lark config failed: %s", exc) + return None, None + + +def app_configured() -> Optional[str]: + """Whether the shared app has been initialized (the admin's one-click config init --new completed). Returns the app_id or None.""" + from integrations.lark.storage import lark_app_home_dir + + cfg = lark_app_home_dir() / lark_state.APP_CFG_SUB + try: + data = json.loads(cfg.read_text(encoding="utf-8")) + apps = data.get("apps") or [] + if apps and isinstance(apps[0], dict): + return apps[0].get("appId") + except Exception: # noqa: BLE001 + pass + return None + + +def seed_user_from_shared_app(user_id: str) -> bool: + """Copy the shared app base (config.json + master.key + appsecret_*.enc) into the user HOME. + + Copies only once, when the user HOME has no config.json yet (persisted + afterwards via the bind-mount). master.key and appsecret are copied + together so that user tokens written by this user's auth login are + encrypted with — and decryptable by — the same key. + User token files are never touched (those belong to each individual).""" + import shutil + + from integrations.lark.storage import lark_app_home_dir + from integrations.lark.storage import lark_home_dir + + app_home = lark_app_home_dir() + user_home = lark_home_dir(user_id) + user_cfg = user_home / lark_state.APP_CFG_SUB + if user_cfg.exists(): + return True # base already present + src_cfg = app_home / lark_state.APP_CFG_SUB + if not src_cfg.exists(): + return False + try: + user_cfg.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(src_cfg, user_cfg) + src_store = app_home / lark_state.APP_STORE_SUB + dst_store = user_home / lark_state.APP_STORE_SUB + dst_store.mkdir(parents=True, exist_ok=True) + for name in os.listdir(src_store): + if name == "master.key" or name.startswith("appsecret_"): + shutil.copy2(src_store / name, dst_store / name) + return True + except Exception as exc: # noqa: BLE001 + logger.warning("[lark] seed user app from shared failed user=%s: %s", user_id, exc) + return False + + +async def ensure_app_config(user_id: str) -> Optional[str]: + """Ensure the user HOME has the app base. Returns an error message; None means ready. + + Prefer the shared app from the **admin's one-click initialization** + (copying the base into the user HOME; the whole group shares one app); + without a shared app, fall back to the app_id/secret entered in **System + Config** (``config init --app-secret-stdin``); with neither → return a + hint message (the front end guides the admin to initialize). + """ + # 1) Shared app (admin one-click initialization) + if app_configured(): + if seed_user_from_shared_app(user_id): + return None + # 2) app_id/secret hand-entered in System Config (fallback) + app_id, app_secret = lark_app_config() + if app_id and app_secret: + shown, _, _ = await lark_process.run_lark(user_id, ["config", "show"], timeout=15) + if app_id in (shown or ""): + return None + _, err, rc = await lark_process.run_lark( + user_id, + ["config", "init", "--app-id", app_id, "--app-secret-stdin", "--brand", "feishu"], + timeout=30, + stdin_data=app_secret, + ) + if rc != 0: + logger.warning("[lark] config init failed user=%s rc=%s err=%s", user_id, rc, (err or "")[:200]) + return f"飞书应用配置初始化失败：{(err or '').strip()[:160] or '未知错误'}" + return None + return lark_state.NO_APP_ERROR + + +def app_env() -> Dict[str, str]: + """Environment for running the shared app's config init: HOME = the shared app HOME; Agent/LARK_APP_* interference stripped.""" + from integrations.lark.storage import lark_app_home_dir + + home = lark_app_home_dir() + try: + home.mkdir(parents=True, exist_ok=True) + except Exception as exc: # noqa: BLE001 + logger.warning("[lark] mkdir app home failed: %s", exc) + env = dict(os.environ) + env["HOME"] = str(home) + for k in ("OPENCLAW_HOME", "HERMES_HOME", "LARK_APP_ID", "LARK_APP_SECRET"): + env.pop(k, None) + return env + + +async def app_init_flow() -> None: + """Backend long-running task: ``config init --new`` stream-reads the + ``open.feishu.cn/page/cli`` configuration URL → echoes it back for the + admin to scan → blocks while the admin completes it in the browser → + the app base lands in the shared HOME. Manages its own exceptions.""" + pass # Shared task state is owned by the state module. + try: + proc = await asyncio.create_subprocess_exec( + lark_state.LARK_BIN, "config", "init", "--new", "--lang", "zh", + stdin=asyncio.subprocess.DEVNULL, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.STDOUT, + env=app_env(), + **no_window_kwargs(), + ) + except FileNotFoundError: + lark_state.app_init_state.update({"status": "error", "error": "后端未安装 lark-cli"}) + lark_state.APP_INIT_TASK = None + return + buf = "" + try: + while True: + try: + line = await asyncio.wait_for(proc.stdout.readline(), timeout=600) + except asyncio.TimeoutError: + break + if not line: + break + buf += line.decode("utf-8", "replace") + if lark_state.app_init_state.get("verification_url") is None: + m = re.search(r"https?://open\.feishu\.cn/\S+", buf) + if m: + url = m.group(0).rstrip(".,;") + lark_state.app_init_state.update({ + "status": "pending", "verification_url": url, + "qr_data_uri": make_qr_data_uri(url), "error": None, + }) + await asyncio.wait_for(proc.wait(), timeout=30) + except asyncio.CancelledError: + try: + proc.kill() + except ProcessLookupError: + pass + raise + except Exception as exc: # noqa: BLE001 + logger.warning("[lark] app init stream failed: %s", exc) + finally: + lark_state.APP_INIT_TASK = None + # Completion check: does the shared HOME now have the app configuration + aid = app_configured() + if aid: + lark_state.app_init_state.update({ + "status": "configured", "app_id": aid, + "verification_url": None, "qr_data_uri": None, "error": None, + }) + else: + lark_state.app_init_state.update({ + "status": "error", "verification_url": None, "qr_data_uri": None, + "error": "应用配置未完成或已取消，请重试。", + }) + + +def app_status_dict() -> Dict[str, Any]: + aid = app_configured() + if aid: + return {"configured": True, "app_id": aid, "status": "configured", + "verification_url": None, "qr_data_uri": None, "error": None} + return {"configured": False, "app_id": None, + "status": lark_state.app_init_state.get("status", "idle"), + "verification_url": lark_state.app_init_state.get("verification_url"), + "qr_data_uri": lark_state.app_init_state.get("qr_data_uri"), + "error": lark_state.app_init_state.get("error")} diff --git a/src/backend/integrations/lark/cube_credentials.py b/src/backend/integrations/lark/cube_credentials.py new file mode 100644 index 00000000..774e19b3 --- /dev/null +++ b/src/backend/integrations/lark/cube_credentials.py @@ -0,0 +1,90 @@ +"""Integration-owned ephemeral sandbox credential handling.""" +from __future__ import annotations +import base64 +import logging +import time +from pathlib import Path +from typing import Any, Optional +logger = logging.getLogger(__name__) + +class CubeCredentials: + def __init__(self, request_timeout): + self._request_timeout_s = request_timeout + self._lark_bootstrapped = set() + + def _build_lark_bundle_b64(self, user_id: str) -> Optional[str]: + """Pack this user's file-based lark-cli credentials from the backend HOME into a base64 tar (None when no credentials). + + Selects the minimal credential set lark-cli recognizes on Linux (no OS + keychain, all file-based): + - ``.lark-cli/config.json`` app config (app_id/brand) + - ``.local/share/lark-cli/master.key`` local DEK + - ``.local/share/lark-cli/appsecret_*.enc`` app secret ciphertext + - ``.local/share/lark-cli/cli_*_ou_*.enc`` user token ciphertext (exists only after login) + arcnames are relative to the HOME root, so in the sandbox ``tar -xf -C /root`` + restores the identical layout at ``/root/.lark-cli`` and + ``/root/.local/share/lark-cli``. Missing config.json (app not seeded) → None. + """ + import glob + import io + import tarfile + + from integrations.lark.storage import lark_home_dir + + home = lark_home_dir(user_id) + cfg = home / ".lark-cli" / "config.json" + if not cfg.exists(): + return None + store = home / ".local" / "share" / "lark-cli" + members: list[tuple[Path, str]] = [(cfg, ".lark-cli/config.json")] + for name in ("master.key",): + p = store / name + if p.exists(): + members.append((p, f".local/share/lark-cli/{name}")) + for pattern in ("appsecret_*.enc", "cli_*_ou_*.enc"): + for path_str in glob.glob(str(store / pattern)): + p = Path(path_str) + members.append((p, f".local/share/lark-cli/{p.name}")) + buf = io.BytesIO() + with tarfile.open(fileobj=buf, mode="w") as tar: + for src, arcname in members: + tar.add(str(src), arcname=arcname) + return base64.b64encode(buf.getvalue()).decode("ascii") + + async def ensure(self, sbx: Any, user_id: str) -> None: + """Pack and write this user's lark-cli credentials into this MicroVM, keyed by the **session-bound user_id** (one-time). + + Multi-tenant isolation crux, same as dws/himalaya: credentials are only + injected into the user's own session sandbox — never across users, never + into warm/unbound sandboxes. Skipped when there is no app config / not + logged in — lark-cli commands then fail as unconfigured/not-logged-in, and + the lark-shared skill guides the user to complete auth login first. cube has + no bind-mount, so it is packed as a tar and extracted directly into the + sandbox root HOME (/root). + """ + sid = getattr(sbx, "sandbox_id", None) + if not sid or sid in self._lark_bootstrapped: + return + self._lark_bootstrapped.add( + sid + ) # register first, to avoid duplicate packing under concurrency / when no credentials exist + try: + bundle = self._build_lark_bundle_b64(user_id) + if not bundle: + return + await sbx.files.write("/tmp/.lark-bundle.tar.b64", bundle) + # The cube sandbox runs as root, HOME=/root → restores /root/.lark-cli and /root/.local/share/lark-cli + await sbx.commands.run( + "base64 -d /tmp/.lark-bundle.tar.b64 | tar -xf - -C /root && " + "chmod -R go-rwx /root/.lark-cli /root/.local/share/lark-cli 2>/dev/null; " + "rm -f /tmp/.lark-bundle.tar.b64", + user="root", + timeout=60, + request_timeout=60 + self._request_timeout_s, + ) + logger.info("[cube] 已为 user=%s 注入飞书凭据到 sandbox=%s", user_id, sid) + except Exception as e: # noqa: BLE001 + self._lark_bootstrapped.discard(sid) + logger.warning( + "[cube] 注入飞书凭据失败 user=%s sandbox=%s: %s", user_id, sid, str(e)[:160] + ) diff --git a/src/backend/integrations/lark/extension.py b/src/backend/integrations/lark/extension.py new file mode 100644 index 00000000..fd1e3142 --- /dev/null +++ b/src/backend/integrations/lark/extension.py @@ -0,0 +1,10 @@ +"""Installed integration contribution.""" +NAME = "lark" + +def register_credentials(): + from integrations.lark.sandbox import register + register() + +def setup(): + register_credentials() + return True diff --git a/src/backend/integrations/lark/login.py b/src/backend/integrations/lark/login.py new file mode 100644 index 00000000..3873a39c --- /dev/null +++ b/src/backend/integrations/lark/login.py @@ -0,0 +1,114 @@ +"""Lark login responsibilities.""" +from __future__ import annotations +from core.infra.time import utc_now +import asyncio +from typing import Any, Dict +from integrations.lark.repository import LarkConnectionRepository +import logging +logger = logging.getLogger(__name__) +from integrations.lark import parsing as lark_parsing +from integrations.lark import process as lark_process +from integrations.lark import state as lark_state + +def update_connection(user_id: str, data: Dict[str, Any]) -> None: + """Update the connection record with an independent DB session (for the background login task — must not borrow the request-scoped session).""" + from core.db.engine import SessionLocal + + try: + with SessionLocal() as db: + LarkConnectionRepository(db).update(user_id, data) + except Exception as exc: # noqa: BLE001 + logger.warning("[lark] _update_connection user=%s failed: %s", user_id, exc) + + +async def build_connected_update(user_id: str) -> Dict[str, Any]: + """After successful login, backfill the Lark identity and return the update dict for the ``connected`` state.""" + # No ``--format json``: no shipped lark-cli accepts that flag (1.0.34 has only + # ``--verify``; 1.0.78 spells it ``--json``), and passing it made every call return a + # "unknown flag" error instead of the status — which is why lark_open_id / lark_name + # stayed NULL on every connection row. Bare ``auth status`` already prints JSON on both. + who, _, _ = await lark_process.run_lark(user_id, ["auth", "status"], timeout=20) + ident = lark_parsing.parse_lark_identity(who) + return { + "status": "connected", + "lark_open_id": ident.get("lark_open_id"), + "lark_name": ident.get("lark_name"), + "tenant_key": ident.get("tenant_key"), + **lark_state.CLEAR_LOGIN, + "last_verified_at": utc_now(), + "last_error": None, + } + + +async def verify_and_refresh(user_id: str) -> str: + """Real liveness probe: ``auth status --verify`` (validates against the server, + renewing with the refresh token along the way). + Returns ``valid`` / ``invalid`` / ``unknown``. Network/timeout etc. draw no + conclusion (unknown), avoiding misjudgment from a blip.""" + out, err, rc = await lark_process.run_lark( + user_id, ["auth", "status", "--verify"], timeout=25 + ) + blob = ((out or "") + "\n" + (err or "")).strip() + if not blob or rc == 124 or rc == 127: + return "unknown" + # The structured verdict is authoritative when the CLI gives us one: substring + # markers are a poor judge here because a *healthy* status payload also carries + # words like "expiresAt", and an expired one still carries openId/userName. + state = lark_parsing.parse_user_identity_state(out) + if state is not None: + return "valid" if state else "invalid" + low = blob.lower() + if any(m in low for m in lark_state.AUTH_FAIL_MARKERS) or "not authenticated" in low or "not logged in" in low: + return "invalid" + if lark_parsing.is_authenticated(out): + return "valid" + return "unknown" + + +async def device_complete_flow(user_id: str, device_code: str) -> None: + """Backend long-running task: ``auth login --device-code `` polls to + completion (blocking until authorized/expired), then backfills identity. + Manages its own DB sessions and exceptions, never raising into the event loop.""" + try: + out, err, rc = await lark_process.run_lark( + user_id, + ["auth", "login", "--device-code", device_code, "--json"], + timeout=600, + ) + verdict = lark_parsing.classify_login_result(out, err, rc) + if verdict == "ok": + # Confirm login with another status check (fallback when --device-code succeeds but the json has no identity) + st, _, _ = await lark_process.run_lark(user_id, ["auth", "status"], timeout=20) + # ``rc == 0`` alone must not win: the CLI can exit 0 without ever storing a + # user token, and that is how connection rows ended up marked ``connected`` + # with a NULL identity while the sandbox saw no token at all. A *definite* + # negative from the structured status therefore overrides it; an + # inconclusive status keeps the old lenient behaviour so unfamiliar CLI + # output shapes do not start failing valid logins. + st_state = lark_parsing.parse_user_identity_state(st) + if st_state is False: + verdict = "error" + elif st_state is True or lark_parsing.is_authenticated(st) or rc == 0: + update_connection(user_id, await build_connected_update(user_id)) + return + else: + verdict = "error" + if verdict == "error": + tail = ((err or out or "").strip())[-300:] + update_connection(user_id, { + "status": "error", **lark_state.CLEAR_LOGIN, + "last_error": f"扫码授权未完成或已过期，请重试。{('详情: ' + tail) if tail else ''}", + }) + return + # pending reaching here (timed out unauthorized) → set error so the user can retry + update_connection(user_id, { + "status": "error", **lark_state.CLEAR_LOGIN, + "last_error": "扫码授权超时，请重新发起连接。", + }) + except asyncio.CancelledError: + raise + except Exception as exc: # noqa: BLE001 + logger.warning("[lark] device complete failed user=%s: %s", user_id, exc) + update_connection(user_id, {"status": "error", "last_error": f"登录异常: {exc}"}) + finally: + lark_state.LOGIN_TASKS.pop(user_id, None) diff --git a/src/backend/integrations/lark/models.py b/src/backend/integrations/lark/models.py new file mode 100644 index 00000000..3077e494 --- /dev/null +++ b/src/backend/integrations/lark/models.py @@ -0,0 +1,67 @@ +"""Lark connection persistence metadata.""" +from core.db.engine import Base +from core.db.utc_datetime import UTCDateTime +from core.infra.time import utc_now +from sqlalchemy import JSON, CheckConstraint, Column, ForeignKey, Index, String, Text +from sqlalchemy.dialects.postgresql import JSONB +JSONType = JSON().with_variant(JSONB(), "postgresql") + +class LarkConnection(Base): + """Per-user Lark (Feishu) account connection (used by the feishu-cli plugin / lark-cli). 1:1 with users_shadow. + + Isomorphic to [[DingTalkConnection]]: the credentials themselves (user_access_token + + file-based encrypted store, including master.key) live on the backend persistent volume at + ``$STORAGE/lark_cache/{user_id}/``, bind-mounted into the sandbox lark-cli's ``$HOME`` + (~/.lark-cli + ~/.local/share/lark-cli) — they **never go into the DB**. This table only + stores the connection status, a Lark identity summary, and the device-flow login state. + The Lark device flow uses ``auth login --no-wait`` to first obtain device_code/ + verification_url (persisted into login_device_code), then completes with ``--device-code``. + The app's app_id/secret are seeded into the per-user HOME by the backend via config init + (not injected as env vars, to avoid breaking --as user). + See internal design docs. + """ + + __tablename__ = "lark_connections" + + user_id = Column( + String(64), + ForeignKey("users_shadow.user_id", ondelete="CASCADE"), + primary_key=True, + ) + # disconnected (default / disconnected) | pending (device-flow login in progress) | connected | error + status = Column(String(16), nullable=False, default="disconnected") + lark_open_id = Column( + String(128) + ) # Lark open_id / union_id (backfilled after a successful connect) + lark_name = Column(String(255)) # Lark display name + tenant_key = Column(String(128)) # Lark tenant tenant_key (counterpart of DingTalk corp_id) + granted_scopes = Column( + JSONType, default=list + ) # List of granted scopes (accumulated via incremental grants) + # Device-flow login state (echoed to the frontend while pending; cleared on success) + login_verification_url = Column( + Text + ) # Plain verification URL (must be used together with user_code) + login_verification_url_complete = Column( + Text + ) # Full URL with the code embedded → target of the QR code + login_user_code = Column(String(64)) + login_device_code = Column( + Text + ) # device_code obtained via --no-wait, used when completing with --device-code + login_started_at = Column(UTCDateTime(timezone=True)) + auth_bundle = Column( + Text + ) # Portable credential bundle (for cube cross-machine use, application-layer encrypted), nullable + last_verified_at = Column(UTCDateTime(timezone=True)) + last_error = Column(Text) + created_at = Column(UTCDateTime(timezone=True), default=utc_now) + updated_at = Column(UTCDateTime(timezone=True), default=utc_now, onupdate=utc_now) + + __table_args__ = ( + CheckConstraint( + "status IN ('disconnected', 'pending', 'connected', 'error')", + name="lark_connections_status_check", + ), + Index("idx_lark_connections_status", "status"), + ) diff --git a/src/backend/integrations/lark/mounts.py b/src/backend/integrations/lark/mounts.py new file mode 100644 index 00000000..0972bb16 --- /dev/null +++ b/src/backend/integrations/lark/mounts.py @@ -0,0 +1,75 @@ +"""Per-user OpenSandbox credential mounts.""" +from __future__ import annotations +import logging +import re +from typing import Any, Optional +from core.config.settings import settings +from core.sandbox.workspace.identity import safe_user_id +from integrations.lark.storage import lark_cache_dir +logger = logging.getLogger(__name__) + +_LARK_HOME = "/home/ubuntu" + + +_LARK_CREDS_SUBPATHS: tuple[tuple[str, str], ...] = ( + # (host subpath under lark_cache/{uid}, sandbox mount point); matches the backend login + # layout HOME=lark_cache/{uid}/home: login writes home/.lark-cli and + # home/.local/share/lark-cli; here we bind them into the sandbox ubuntu home. + ("home/.lark-cli", f"{_LARK_HOME}/.lark-cli"), + ("home/.local/share/lark-cli", f"{_LARK_HOME}/.local/share/lark-cli"), +) + + +def mounts(user_id: Optional[str]) -> list[Any]: + """Build the two read-write bind-mount Volumes for lark-cli credential persistence (one set per user). + + Same approach as [[_make_dws_creds_volumes]]: flag off / invalid user_id / host_storage + not configured / SDK has no Volume model — any unmet condition quietly degrades to "no + credential volumes mounted"; lark-cli then has no credentials in the sandbox, business + commands fail as not-logged-in, and the lark-shared skill guides the user to complete + auth login first. Pre-create the host directories as backend (UID 1000) to avoid + docker's root auto-mkdir at bind time leaving the backend unable to sync/clean. + """ + if not settings.sandbox.lark_creds_bind_mount_enabled: + return [] + uid = safe_user_id(user_id or "") + if not uid: + return [] + host_storage = (settings.sandbox.opensandbox_host_storage_path or "").strip().rstrip("/") + if not host_storage: + logger.warning( + "[opensandbox] lark creds bind mount enabled but HOST_STORAGE_PATH " + "not configured; lark-cli will have no persisted credentials" + ) + return [] + cache_dir = lark_cache_dir(uid) + try: + for sub, _mount in _LARK_CREDS_SUBPATHS: + (cache_dir / sub).mkdir(parents=True, exist_ok=True) + except Exception as exc: # noqa: BLE001 + logger.warning( + "[opensandbox] failed to pre-create lark_cache dir %s: %s; " + "skipping lark creds bind mount", + cache_dir, + exc, + ) + return [] + try: + from opensandbox.models.sandboxes import Host, Volume + except ImportError: + return [] + slug = re.sub(r"[^a-z0-9-]", "-", uid.lower()).strip("-") or "u" + out: list[Any] = [] + for idx, (sub, mount_path) in enumerate(_LARK_CREDS_SUBPATHS): + # vol name must match k8s naming (lowercase alphanumerics + hyphens); sub contains + # / and ., so use an index instead. + vol_name = f"lark{idx}-{slug}"[:63].rstrip("-") or f"lark{idx}" + out.append( + Volume( + name=vol_name, + host=Host(path=f"{host_storage}/lark_cache/{uid}/{sub}"), + mount_path=mount_path, + read_only=False, + ) + ) + return out diff --git a/src/backend/integrations/lark/parsing.py b/src/backend/integrations/lark/parsing.py new file mode 100644 index 00000000..647b4cf5 --- /dev/null +++ b/src/backend/integrations/lark/parsing.py @@ -0,0 +1,141 @@ +"""Lark parsing responsibilities.""" +from __future__ import annotations +import json +from typing import Any, Dict, Optional +from integrations.lark import state as lark_state + +def loads(s: str) -> Any: + try: + return json.loads((s or "").strip() or "null") + except (ValueError, TypeError): + return None + + +def parse_device_login(stdout: str) -> Dict[str, Optional[str]]: + """``auth login --no-wait --json`` → {device_code, user_code, verification_url, + verification_url_complete}. Field names matched leniently (Lark/OAuth device-flow conventional naming).""" + out: Dict[str, Optional[str]] = { + "device_code": None, "user_code": None, + "verification_url": None, "verification_url_complete": None, + } + data = loads(stdout) + if not isinstance(data, dict): + return out + # Some CLIs wrap the content inside data/result + for key in ("data", "result", "device", "deviceAuth"): + inner = data.get(key) + if isinstance(inner, dict): + data = {**data, **inner} + + def _g(*names: str) -> Optional[str]: + for n in names: + for k, v in data.items(): + if k.lower().replace("_", "") == n and isinstance(v, (str, int)): + return str(v) + return None + + out["device_code"] = _g("devicecode") + out["user_code"] = _g("usercode") + out["verification_url"] = _g("verificationurl", "verificationuri") + out["verification_url_complete"] = _g("verificationurlcomplete", "verificationuricomplete") + return out + + +def parse_lark_identity(stdout: str) -> Dict[str, Optional[str]]: + """``auth status``/``auth list`` output → {lark_open_id, lark_name, tenant_key} (searched recursively).""" + out: Dict[str, Optional[str]] = {"lark_open_id": None, "lark_name": None, "tenant_key": None} + data = loads(stdout) + if data is None: + return out + + def _walk(obj: Any) -> None: + if isinstance(obj, dict): + for k, v in obj.items(): + lk = k.lower().replace("_", "") + if out["lark_open_id"] is None and lk in ("openid", "unionid", "userid"): + if isinstance(v, (str, int)): + out["lark_open_id"] = str(v) + if out["lark_name"] is None and lk in ("name", "username", "nick", "ennname", "displayname"): + if isinstance(v, str): + out["lark_name"] = v + if out["tenant_key"] is None and lk in ("tenantkey", "tenant"): + if isinstance(v, (str, int)): + out["tenant_key"] = str(v) + _walk(v) + elif isinstance(obj, list): + for it in obj: + _walk(it) + + _walk(data) + return out + + +def parse_user_identity_state(stdout: str) -> Optional[bool]: + """``auth status`` → whether the **user** identity is usable right now; + ``None`` when the output shape is unrecognized (caller falls back to heuristics). + + lark-cli reports the bot and the user identity separately:: + + {"identities": {"bot": {"status": "ready", "available": true}, + "user": {"status": "missing", "available": false, + "openId": "ou_…", "userName": "…", + "tokenStatus": "expired"}}} + + Note that ``openId``/``userName`` **stay in the payload after the token + expires** — they describe who last authorized, not whether the + authorization still works. So "an identity field is present" must never be + read as "logged in"; doing so is what let a login that expired months ago + keep reporting 已登录 on the connection panel while the sandbox correctly + answered "User identity: missing". + """ + data = loads(stdout) + if not isinstance(data, dict): + return None + identities = data.get("identities") + if not isinstance(identities, dict): + return None + user = identities.get("user") + if not isinstance(user, dict): + return None + available = user.get("available") + if isinstance(available, bool): + return available + status = str(user.get("status") or "").strip().lower() + if not status: + return None + return status in ("ready", "ok", "active", "valid") + + +def is_authenticated(stdout: str) -> bool: + """``auth status`` → whether the user is logged in. + + Prefers the structured ``identities.user`` verdict (see + [[parse_user_identity_state]]); only when the CLI emits a shape we do not + recognize does it fall back to the old lenient heuristics — those also have + to serve ``auth login --device-code`` output, which carries no + ``identities`` block. + """ + state = parse_user_identity_state(stdout) + if state is not None: + return state + data = loads(stdout) + if isinstance(data, dict): + for k, v in data.items(): + lk = k.lower().replace("_", "") + if lk in ("authenticated", "loggedin", "isloggedin", "valid") and v is True: + return True + ident = parse_lark_identity(stdout) + return bool(ident.get("lark_open_id") or ident.get("lark_name")) + + +def classify_login_result(stdout: str, stderr: str, rc: int) -> str: + """Classify the result of ``auth login --device-code`` → ``ok`` / ``pending`` / ``error``.""" + blob = ((stdout or "") + "\n" + (stderr or "")).lower() + if rc == 0 and is_authenticated(stdout): + return "ok" + if any(m in blob for m in lark_state.PENDING_MARKERS): + return "pending" + if any(m in blob for m in lark_state.AUTH_FAIL_MARKERS): + return "error" + # rc==0 but no definite identity: the caller decides whether to probe status again; here conservatively treat as pending (not yet terminal) + return "ok" if rc == 0 else "pending" diff --git a/src/backend/integrations/lark/process.py b/src/backend/integrations/lark/process.py new file mode 100644 index 00000000..03b4bc43 --- /dev/null +++ b/src/backend/integrations/lark/process.py @@ -0,0 +1,59 @@ +"""Lark process responsibilities.""" +from __future__ import annotations +import asyncio +import os +from typing import Dict, List, Optional, Tuple +from core.infra.proc import no_window_kwargs +import logging +logger = logging.getLogger(__name__) +from integrations.lark import state as lark_state + +def lark_env(user_id: str) -> Dict[str, str]: + """Environment for running lark-cli subprocesses on the backend: per-user isolated ``HOME`` (the root of credential isolation). + + LARK_APP_ID/SECRET are **not** injected (with them in the env, lark-cli + defaults to the tenant/bot token flow, breaking --as user); the app + config lands in HOME via config init. OPENCLAW_HOME/HERMES_HOME are + stripped — otherwise ``config init`` decides it is in an Agent context + and refuses to run.""" + from integrations.lark.storage import lark_home_dir + + home = lark_home_dir(user_id) + try: + home.mkdir(parents=True, exist_ok=True) + except Exception as exc: # noqa: BLE001 + logger.warning("[lark] mkdir home %s failed: %s", home, exc) + env = dict(os.environ) + env["HOME"] = str(home) + for k in ("OPENCLAW_HOME", "HERMES_HOME", "LARK_APP_ID", "LARK_APP_SECRET"): + env.pop(k, None) + return env + + +async def run_lark( + user_id: str, args: List[str], timeout: int = 40, stdin_data: Optional[str] = None, +) -> Tuple[str, str, int]: + """Run one lark-cli subcommand on the backend as this user; returns (stdout, stderr, exit_code).""" + try: + proc = await asyncio.create_subprocess_exec( + lark_state.LARK_BIN, *args, + stdin=asyncio.subprocess.PIPE if stdin_data is not None else None, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + env=lark_env(user_id), + **no_window_kwargs(), + ) + except FileNotFoundError: + return "", "lark-cli binary not found on backend", 127 + try: + out, err = await asyncio.wait_for( + proc.communicate(input=stdin_data.encode("utf-8") if stdin_data is not None else None), + timeout=timeout, + ) + return out.decode("utf-8", "replace"), err.decode("utf-8", "replace"), proc.returncode or 0 + except asyncio.TimeoutError: + try: + proc.kill() + except ProcessLookupError: + pass + return "", f"lark-cli {' '.join(args[:2])} timeout", 124 diff --git a/src/backend/integrations/lark/repository.py b/src/backend/integrations/lark/repository.py new file mode 100644 index 00000000..b5bc297d --- /dev/null +++ b/src/backend/integrations/lark/repository.py @@ -0,0 +1,37 @@ +"""Lark connection persistence metadata.""" +from typing import Any, Dict, Optional +from sqlalchemy.orm import Session +from core.infra.time import utc_now +from integrations.lark.models import LarkConnection + +class LarkConnectionRepository: + """Per-user Lark connection (lark_connections) CRUD. 1:1 with users_shadow, no soft delete. + Structurally identical to [[DingTalkConnectionRepository]].""" + + def __init__(self, db: Session): + self.db = db + + def get(self, user_id: str) -> Optional[LarkConnection]: + return self.db.query(LarkConnection).filter(LarkConnection.user_id == user_id).first() + + def ensure(self, user_id: str) -> LarkConnection: + """Idempotent: return existing record, otherwise create a disconnected record.""" + record = self.get(user_id) + if record: + return record + record = LarkConnection(user_id=user_id, status="disconnected") + self.db.add(record) + self.db.commit() + self.db.refresh(record) + return record + + def update(self, user_id: str, data: Dict[str, Any]) -> Optional[LarkConnection]: + record = self.get(user_id) + if not record: + return None + for k, v in data.items(): + setattr(record, k, v) + record.updated_at = utc_now() + self.db.commit() + self.db.refresh(record) + return record diff --git a/src/backend/integrations/lark/sandbox.py b/src/backend/integrations/lark/sandbox.py new file mode 100644 index 00000000..59420f71 --- /dev/null +++ b/src/backend/integrations/lark/sandbox.py @@ -0,0 +1,14 @@ +"""Credential contribution for lark.""" +from core.harness.registry import SandboxCredentials, register_sandbox_credentials +from integrations.lark.cube_credentials import CubeCredentials +from integrations.lark.mounts import mounts + +def state(host): + return host.credential_state("lark", lambda: CubeCredentials(host.credential_request_timeout)) + +async def prepare(host, sandbox, user_id, script): + if "lark" in (script or ""): + await state(host).ensure(sandbox, user_id) + +def register(): + register_sandbox_credentials(SandboxCredentials("lark", mounts=mounts, before_command=prepare, prepare_order=2)) diff --git a/src/backend/integrations/lark/service.py b/src/backend/integrations/lark/service.py new file mode 100644 index 00000000..fb5761da --- /dev/null +++ b/src/backend/integrations/lark/service.py @@ -0,0 +1,189 @@ +"""Lark service responsibilities.""" +from __future__ import annotations +from core.infra.time import utc_now +import asyncio +import shutil +from typing import Any, Dict +from sqlalchemy.orm import Session +from integrations.lark.repository import LarkConnectionRepository +from integrations.dingtalk.parsing import make_qr_data_uri +import logging +logger = logging.getLogger(__name__) +from integrations.lark import configuration as lark_configuration +from integrations.lark import login as lark_login +from integrations.lark import parsing as lark_parsing +from integrations.lark import process as lark_process +from integrations.lark import state as lark_state + +class LarkService: + def __init__(self, db: Session): + self.db = db + self.repo = LarkConnectionRepository(db) + + def _fresh(self, user_id: str): + """Discard the session cache and re-read — to see the latest state committed by the background task's independent session.""" + self.db.expire_all() + return self.repo.ensure(user_id) + + # ── Status ─────────────────────────────────────────────────────────── + def status_dict(self, record) -> Dict[str, Any]: + qr_target = record.login_verification_url_complete or record.login_verification_url + return { + "status": record.status, + "lark_open_id": record.lark_open_id, + "lark_name": record.lark_name, + "tenant_key": record.tenant_key, + "granted_scopes": list(record.granted_scopes or []), + "verification_url": record.login_verification_url, + "verification_url_complete": record.login_verification_url_complete, + "qr_data_uri": make_qr_data_uri(qr_target) if record.status == "pending" else None, + "user_code": record.login_user_code, + "last_verified_at": record.last_verified_at.isoformat() if record.last_verified_at else None, + "last_error": record.last_error, + } + + def get_status(self, user_id: str) -> Dict[str, Any]: + return self.status_dict(self.repo.ensure(user_id)) + + async def probe_status(self, user_id: str) -> Dict[str, Any]: + """Real-API liveness-probe reconciliation (corrects mismatches between the DB and actual credentials). Does not interrupt pending (login in progress).""" + record = self._fresh(user_id) + if record.status == "pending" and user_id in lark_state.LOGIN_TASKS: + return self.status_dict(record) + verdict = await lark_login.verify_and_refresh(user_id) + if verdict == "valid" and record.status != "connected": + self.repo.update(user_id, {"status": "connected", "last_error": None}) + elif verdict == "invalid" and record.status == "connected": + self.repo.update(user_id, {"status": "disconnected", "last_error": lark_state.EXPIRED_ERROR}) + return self.status_dict(self.repo.get(user_id)) + + # ── Device-flow login (scan-to-bind) ───────────────────────────────── + async def start_device_login(self, user_id: str) -> Dict[str, Any]: + """Initiate the Lark device-flow login: first seed the app config, then + ``auth login --no-wait`` gets the device_code / QR URL, and the + background completion task is started. The front end presents + qr_data_uri for the user to scan, then polls.""" + self.repo.ensure(user_id) + # Cancel the same user's old task + old = lark_state.LOGIN_TASKS.pop(user_id, None) + if old is not None and not old.done(): + old.cancel() + + cfg_err = await lark_configuration.ensure_app_config(user_id) + if cfg_err: + self.repo.update(user_id, {"status": "error", **lark_state.CLEAR_LOGIN, "last_error": cfg_err}) + return self.status_dict(self.repo.get(user_id)) + + out, err, rc = await lark_process.run_lark(user_id, lark_state.DEFAULT_LOGIN_ARGS, timeout=40) + dev = lark_parsing.parse_device_login(out) + if rc != 0 or not (dev.get("verification_url") or dev.get("verification_url_complete")): + tail = ((err or out or "").strip())[-200:] + self.repo.update(user_id, { + "status": "error", **lark_state.CLEAR_LOGIN, + "last_error": f"发起扫码登录失败，请重试。{('详情: ' + tail) if tail else ''}", + }) + return self.status_dict(self.repo.get(user_id)) + + self.repo.update(user_id, { + "status": "pending", + "login_verification_url": dev.get("verification_url"), + "login_verification_url_complete": dev.get("verification_url_complete"), + "login_user_code": dev.get("user_code"), + "login_device_code": dev.get("device_code"), + "login_started_at": utc_now(), + "last_error": None, + }) + # Start the background completion task (blocking polling until authorized/expired) + if dev.get("device_code"): + lark_state.LOGIN_TASKS[user_id] = asyncio.create_task( + lark_login.device_complete_flow(user_id, str(dev["device_code"])) + ) + return self.status_dict(self._fresh(user_id)) + + async def poll_login(self, user_id: str) -> Dict[str, Any]: + """Poll login progress. The background task drives to connected/error; + this reads the latest state. If the task is lost yet still pending + (backend restart), fall back to the persisted device_code and run + completion once more.""" + record = self._fresh(user_id) + if record.status == "pending" and user_id not in lark_state.LOGIN_TASKS and record.login_device_code: + lark_state.LOGIN_TASKS[user_id] = asyncio.create_task( + lark_login.device_complete_flow(user_id, str(record.login_device_code)) + ) + return self.status_dict(record) + + # ── Disconnect ─────────────────────────────────────────────────────── + async def disconnect(self, user_id: str) -> Dict[str, Any]: + """Cancel any in-progress login + backend logout + clear the backend persistent credential directory + set disconnected.""" + old = lark_state.LOGIN_TASKS.pop(user_id, None) + if old is not None and not old.done(): + old.cancel() + await lark_process.run_lark(user_id, ["auth", "logout"], timeout=30) + try: + from integrations.lark.storage import lark_cache_dir + from core.sandbox.workspace.credentials import purge_credential_dir + from core.sandbox.workspace.identity import safe_user_id + + # Empty in place — never rmtree the root. ~/.lark-cli and + # ~/.local/share/lark-cli under it are bind-mounted into this user's + # live sandboxes; removing the directory orphans those mounts and the + # next login's credentials land on an inode the sandbox cannot see + # (see [[purge_credential_dir]]). + if safe_user_id(user_id): + purge_credential_dir(lark_cache_dir(user_id)) + except Exception as exc: # noqa: BLE001 + logger.warning("[lark] disconnect cache purge failed user=%s: %s", user_id, exc) + self.repo.update( + user_id, + { + "status": "disconnected", + "lark_open_id": None, + "lark_name": None, + "tenant_key": None, + "granted_scopes": [], + **lark_state.CLEAR_LOGIN, + "auth_bundle": None, + "last_error": None, + }, + ) + return self.status_dict(self.repo.get(user_id)) + + # ── Admin: shared-app one-click initialization (config init --new) ──── + def app_status(self) -> Dict[str, Any]: + """Query the shared app's initialization state (configured / pending+QR code / error).""" + return lark_configuration.app_status_dict() + + async def start_app_init(self) -> Dict[str, Any]: + """Initiate ``config init --new``: run in the background and echo the configuration QR code for the admin to scan. If already configured, report back directly.""" + pass # Shared task state is owned by the state module. + if lark_configuration.app_configured(): + return lark_configuration.app_status_dict() + if lark_state.APP_INIT_TASK is not None and not lark_state.APP_INIT_TASK.done(): + return lark_configuration.app_status_dict() # in progress, echo the current URL + lark_state.app_init_state.update({"status": "pending", "verification_url": None, + "qr_data_uri": None, "error": None}) + lark_state.APP_INIT_TASK = asyncio.create_task(lark_configuration.app_init_flow()) + # Wait up to ~15s for the configuration URL to appear + for _ in range(30): + await asyncio.sleep(0.5) + if lark_state.app_init_state.get("verification_url") or lark_state.app_init_state.get("status") in ("configured", "error"): + break + return lark_configuration.app_status_dict() + + async def reset_app(self) -> Dict[str, Any]: + """Clear the shared app base (for re-initialization). Does not touch users' existing login tokens.""" + pass # Shared task state is owned by the state module. + if lark_state.APP_INIT_TASK is not None and not lark_state.APP_INIT_TASK.done(): + lark_state.APP_INIT_TASK.cancel() + lark_state.APP_INIT_TASK = None + try: + import shutil + + from integrations.lark.storage import lark_app_home_dir + + shutil.rmtree(lark_app_home_dir(), ignore_errors=True) + except Exception as exc: # noqa: BLE001 + logger.warning("[lark] reset app home failed: %s", exc) + lark_state.app_init_state.update({"status": "idle", "verification_url": None, + "qr_data_uri": None, "error": None}) + return lark_configuration.app_status_dict() diff --git a/src/backend/integrations/lark/state.py b/src/backend/integrations/lark/state.py new file mode 100644 index 00000000..c67367dd --- /dev/null +++ b/src/backend/integrations/lark/state.py @@ -0,0 +1,49 @@ +"""Lark state responsibilities.""" +from __future__ import annotations +import asyncio +from typing import Any, Dict, Optional + +LOGIN_TASKS: "dict[str, asyncio.Task]" = {} + + +LARK_BIN = "lark-cli" + + +DEFAULT_LOGIN_ARGS = ["auth", "login", "--recommend", "--no-wait", "--json"] + + +CLEAR_LOGIN: Dict[str, Any] = { + "login_verification_url": None, + "login_verification_url_complete": None, + "login_user_code": None, + "login_device_code": None, +} + + +EXPIRED_ERROR = "飞书登录已失效（令牌被刷新或过期），请到设置页重新扫码连接飞书" + + +NO_APP_ERROR = "飞书应用尚未配置，请联系管理员在「插件 → 飞书工作台」初始化飞书应用" + + +AUTH_FAIL_MARKERS = ( + "expired_token", "access_denied", "invalid_grant", "invalid_device_code", + "authorization_expired", "expired", "denied", +) + + +PENDING_MARKERS = ("authorization_pending", "pending", "slow_down") + + +APP_CFG_SUB = ".lark-cli/config.json" + + +APP_STORE_SUB = ".local/share/lark-cli" + + +APP_INIT_TASK: "Optional[asyncio.Task]" = None + + +app_init_state: Dict[str, Any] = { + "status": "idle", "verification_url": None, "qr_data_uri": None, "error": None, +} diff --git a/src/backend/integrations/lark/storage.py b/src/backend/integrations/lark/storage.py new file mode 100644 index 00000000..b9b22385 --- /dev/null +++ b/src/backend/integrations/lark/storage.py @@ -0,0 +1,51 @@ +"""Per-user integration credential storage paths.""" +from pathlib import Path +from core.config.settings import settings + +def lark_cache_dir(user_id: str) -> Path: + """Backend-local lark-cli (Feishu/Lark CLI) credential persistence directory (one per user). + + Contains two subdirectories, bind-mounted into the sandbox lark-cli's default + ``$HOME`` paths: + - ``home/.lark-cli/`` → sandbox ``/home/ubuntu/.lark-cli`` (config.json + app config + cache + logs) + - ``home/.local/share/lark-cli/`` → sandbox ``/home/ubuntu/.local/share/lark-cli`` + (file-based encrypted store: ``master.key`` + + ``appsecret_*.enc`` + per-user ``cli_*_ou_*.enc``) + + Empirically: on Linux, lark-cli does **not** use the OS keychain but a file-based + encrypted store — it only respects these two directories under ``$HOME``; + ``master.key`` is a local DEK, so decryption works wherever the files travel. + Isomorphic to dws's "Linux file-based DEK" scheme, so the per-user bind-mount + persistence is copied verbatim (see [[dws_cache_dir]] and + internal design docs). + The sandbox runs as ubuntu (UID 1000) and the backend is also 1000, so ownership matches. + """ + return settings.storage.root / "lark_cache" / user_id + + +def lark_home_dir(user_id: str) -> Path: + """Per-user ``$HOME`` for lark-cli login (the unified root for the backend + config init / auth login subprocesses + opensandbox bind sources). + + The backend runs ``lark-cli config init`` (writes app config) and ``auth login`` + (device flow, writes user_access_token) with this as ``HOME`` → credentials land in + ``{home}/.lark-cli`` and ``{home}/.local/share/lark-cli``; opensandbox bind-mounts + these two subpaths into the same home paths inside the sandbox, so **that user's** + session sandboxes see their own credentials. The root of multi-tenant isolation: + each user gets an independent home directory, never shared. + """ + return lark_cache_dir(user_id) / "home" + + +def lark_app_home_dir() -> Path: + """Org-wide shared Feishu **app** config HOME (the admin's one-click init + ``config init --new`` lands here). + + Contains ``~/.lark-cli/config.json`` + ``~/.local/share/lark-cli/{master.key, + appsecret_*.enc}`` (the app foundation — contains **no user tokens whatsoever**). + These three items are copied into every user's ``lark_home_dir`` as the app + foundation for their ``auth login`` — the whole org shares one app, and the admin + configures it only once. + """ + return settings.storage.root / "lark_cache" / "__app__" / "home" diff --git a/src/backend/integrations/yida/README.md b/src/backend/integrations/yida/README.md new file mode 100644 index 00000000..09d1468c --- /dev/null +++ b/src/backend/integrations/yida/README.md @@ -0,0 +1,5 @@ +# yida 集成 + +宜搭 service 管理授权状态和登录轮询，execution 管理沙箱执行；登录状态仍位于每用户工作区，models 不向平台库添加表。sandbox 注册命令前注入与命令后回写，保留 .cache/*.json 白名单、大小限制、软链接及路径穿越拒绝逻辑。 + +FULL 和 CE 均安装；模型加载独立于运行开关，API/CLI 宿主显式注册沙箱贡献。原路由仍由总集成 API 提供，迁移不改变协议、账号归属或运行开关。调用方直接引用所属服务，不保留旧路径转发。 diff --git a/src/backend/integrations/yida/__init__.py b/src/backend/integrations/yida/__init__.py new file mode 100644 index 00000000..90b4bc33 --- /dev/null +++ b/src/backend/integrations/yida/__init__.py @@ -0,0 +1 @@ +"""Integration package; schema migration is tracked separately.""" diff --git a/src/backend/integrations/yida/cube_credentials.py b/src/backend/integrations/yida/cube_credentials.py new file mode 100644 index 00000000..5763fe8f --- /dev/null +++ b/src/backend/integrations/yida/cube_credentials.py @@ -0,0 +1,152 @@ +"""Integration-owned ephemeral sandbox credential handling.""" +from __future__ import annotations +import base64 +import logging +import time +from pathlib import Path +from typing import Any, Optional +logger = logging.getLogger(__name__) + +class CubeCredentials: + def __init__(self, request_timeout): + self._request_timeout_s = request_timeout + self._yida_bootstrapped = set() + + _YIDA_STATE_GLOB = ".cache/*.json" + _YIDA_STATE_MAX_B64 = 1024 * 1024 + + def _build_yida_state_b64(self, user_id: str) -> Optional[str]: + """Pack ``.cache/*.json`` from this user's host-side Yida working directory into a base64 tar (None when absent).""" + import glob + import io + import tarfile + + from integrations.yida.storage import yida_workspace_dir + + ws = yida_workspace_dir(user_id) + paths = sorted(glob.glob(str(ws / ".cache" / "*.json"))) + if not paths: + return None + buf = io.BytesIO() + with tarfile.open(fileobj=buf, mode="w") as tar: + for path_str in paths: + p = Path(path_str) + tar.add(str(p), arcname=f".cache/{p.name}") + return base64.b64encode(buf.getvalue()).decode("ascii") + + async def ensure(self, sbx: Any, user_id: str) -> None: + """Inject this user's Yida login state into this MicroVM, keyed by the **session-bound user_id** (one-time). + + Multi-tenant isolation crux, same as dws/himalaya/lark: inject only into the + user's own session sandbox — never across users, never into warm/unbound + sandboxes. The fixed working directory /home/ubuntu/yida-workspace is + **created up front regardless of whether login state exists** (the yida + skill force-cds into it; a missing directory would make every command fail + outright); when the host side has no cookies, only the directory is created — + openyida commands fail as not-logged-in, and the skill presents a QR code to + scan within the conversation. + """ + sid = getattr(sbx, "sandbox_id", None) + if not sid or sid in self._yida_bootstrapped: + return + self._yida_bootstrapped.add( + sid + ) # register first, to avoid duplicate packing under concurrency / when no credentials exist + try: + from integrations.yida.storage import YIDA_WORKSPACE_MOUNT as _YIDA_WORKSPACE_MOUNT + + bundle = self._build_yida_state_b64(user_id) + if not bundle: + await sbx.commands.run( + f"mkdir -p {_YIDA_WORKSPACE_MOUNT}", + user="root", + timeout=30, + request_timeout=30 + self._request_timeout_s, + ) + return + await sbx.files.write("/tmp/.yida-state.tar.b64", bundle) + await sbx.commands.run( + f"mkdir -p {_YIDA_WORKSPACE_MOUNT} && " + f"base64 -d /tmp/.yida-state.tar.b64 | tar -xf - -C {_YIDA_WORKSPACE_MOUNT} && " + f"chmod -R go-rwx {_YIDA_WORKSPACE_MOUNT}/.cache 2>/dev/null; " + "rm -f /tmp/.yida-state.tar.b64", + user="root", + timeout=60, + request_timeout=60 + self._request_timeout_s, + ) + logger.info("[cube] 已为 user=%s 注入宜搭登录态到 sandbox=%s", user_id, sid) + except Exception as e: # noqa: BLE001 + self._yida_bootstrapped.discard(sid) + logger.warning( + "[cube] 注入宜搭登录态失败 user=%s sandbox=%s: %s", user_id, sid, str(e)[:160] + ) + + @staticmethod + def extract_state(raw_b64: str, dest_ws: Path) -> int: + """Safely extract the base64 tar sent back from the sandbox into the host working directory; returns the number of files written. + + Accepts only regular-file members of the form ``.cache/.json`` + (guarding against path traversal / symlinks / smuggled subdirectories), + writing each member to disk manually instead of extractall. Kept as a + standalone static method for easy unit testing. + """ + import io + import tarfile + + data = base64.b64decode(raw_b64) + count = 0 + with tarfile.open(fileobj=io.BytesIO(data), mode="r") as tar: + for m in tar.getmembers(): + name = m.name + while name.startswith("./"): + name = name[2:] + base = name[len(".cache/") :] + if ( + not m.isreg() + or not name.startswith(".cache/") + or "/" in base + or not base.endswith(".json") + ): + continue + f = tar.extractfile(m) + if f is None: + continue + target = dest_ws / ".cache" / base + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(f.read()) + try: + target.chmod(0o600) + except OSError: + pass + count += 1 + return count + + async def persist(self, sbx: Any, user_id: str) -> None: + """After an openyida command runs, send the in-sandbox Yida login state back to the host and persist it (best-effort). + + openyida's QR-code login and headless CSRF-token refresh both happen inside + the sandbox, writing ``.cache/*.json`` in the working directory — cube has + no bind-mount, so without sending it back the login state is lost the moment + the sandbox (TTL on the minutes scale) is destroyed, and the user has to + rescan the QR code with every new sandbox. Failures are only logged and + never affect the command result. + """ + try: + from integrations.yida.storage import YIDA_WORKSPACE_MOUNT as _YIDA_WORKSPACE_MOUNT, yida_workspace_dir + + res = await sbx.commands.run( + f"cd {_YIDA_WORKSPACE_MOUNT} 2>/dev/null && " + f"ls {self._YIDA_STATE_GLOB} >/dev/null 2>&1 && " + f"tar -cf - {self._YIDA_STATE_GLOB} | base64 -w0 || true", + user="root", + timeout=30, + request_timeout=30 + self._request_timeout_s, + ) + raw = (getattr(res, "stdout", "") or "").strip() + if not raw or len(raw) > self._YIDA_STATE_MAX_B64: + return + n = self.extract_state(raw, yida_workspace_dir(user_id)) + if n: + logger.debug("[cube] 已回传宜搭登录态 user=%s files=%d", user_id, n) + except Exception as e: # noqa: BLE001 + logger.warning("[cube] 回传宜搭登录态失败 user=%s: %s", user_id, str(e)[:160]) diff --git a/src/backend/core/services/yida_execution.py b/src/backend/integrations/yida/execution.py similarity index 97% rename from src/backend/core/services/yida_execution.py rename to src/backend/integrations/yida/execution.py index 6a30dd8a..ef2d60fe 100644 --- a/src/backend/core/services/yida_execution.py +++ b/src/backend/integrations/yida/execution.py @@ -20,8 +20,8 @@ def workspace_dir(user_id: str) -> Path: - from core.sandbox.workspace.credentials import yida_shared_workspace_dir - from core.sandbox.workspace.credentials import yida_workspace_dir + from integrations.yida.storage import yida_shared_workspace_dir + from integrations.yida.storage import yida_workspace_dir if not settings.deploy.is_local and settings.sandbox.provider == "script_runner": return yida_shared_workspace_dir() diff --git a/src/backend/integrations/yida/extension.py b/src/backend/integrations/yida/extension.py new file mode 100644 index 00000000..c82ff443 --- /dev/null +++ b/src/backend/integrations/yida/extension.py @@ -0,0 +1,13 @@ +"""Installed integration contribution.""" +NAME = "yida" + +def register_credentials(): + from core.harness.registry.skill_content import register_skill_content_adapter + from integrations.yida.skill_content import adapt + register_skill_content_adapter("yida_workspace", adapt) + from integrations.yida.sandbox import register + register() + +def setup(): + register_credentials() + return True diff --git a/src/backend/integrations/yida/models.py b/src/backend/integrations/yida/models.py new file mode 100644 index 00000000..90b4bc33 --- /dev/null +++ b/src/backend/integrations/yida/models.py @@ -0,0 +1 @@ +"""Integration package; schema migration is tracked separately.""" diff --git a/src/backend/integrations/yida/mounts.py b/src/backend/integrations/yida/mounts.py new file mode 100644 index 00000000..f5c45220 --- /dev/null +++ b/src/backend/integrations/yida/mounts.py @@ -0,0 +1,68 @@ +"""Per-user OpenSandbox credential mounts.""" +from __future__ import annotations +import logging +import re +from typing import Any, Optional +from core.config.settings import settings +from core.sandbox.workspace.identity import safe_user_id +from integrations.yida.storage import yida_cache_dir, YIDA_WORKSPACE_MOUNT +logger = logging.getLogger(__name__) + +_YIDA_CREDS_SUBPATHS: tuple[tuple[str, str], ...] = ( + # (host subpath under yida_cache/{uid}, sandbox mount point) + ("workspace", YIDA_WORKSPACE_MOUNT), +) + + +def mounts(user_id: Optional[str]) -> list[Any]: + """Build the read-write bind-mount Volume for openyida login-state persistence (one per user, single directory). + + Same approach as [[_make_email_creds_volumes]]: flag off / invalid user_id / + host_storage not configured / SDK has no Volume model — any unmet condition quietly + degrades to "no credential volume mounted"; openyida then has no login state in the + sandbox, business commands fail as not-logged-in, and the yida skill guides the user + through QR-code login in the conversation (still usable within the session, just not + surviving across sessions). Pre-create the host directories as backend (UID 1000) to + avoid docker's root auto-mkdir at bind time leaving the backend unable to sync/clean. + """ + if not settings.sandbox.yida_creds_bind_mount_enabled: + return [] + uid = safe_user_id(user_id or "") + if not uid: + return [] + host_storage = (settings.sandbox.opensandbox_host_storage_path or "").strip().rstrip("/") + if not host_storage: + logger.warning( + "[opensandbox] yida creds bind mount enabled but HOST_STORAGE_PATH " + "not configured; openyida will have no persisted login state" + ) + return [] + cache_dir = yida_cache_dir(uid) + try: + for sub, _mount in _YIDA_CREDS_SUBPATHS: + (cache_dir / sub).mkdir(parents=True, exist_ok=True) + except Exception as exc: # noqa: BLE001 + logger.warning( + "[opensandbox] failed to pre-create yida_cache dir %s: %s; " + "skipping yida creds bind mount", + cache_dir, + exc, + ) + return [] + try: + from opensandbox.models.sandboxes import Host, Volume + except ImportError: + return [] + slug = re.sub(r"[^a-z0-9-]", "-", uid.lower()).strip("-") or "u" + out: list[Any] = [] + for idx, (sub, mount_path) in enumerate(_YIDA_CREDS_SUBPATHS): + vol_name = f"yida{idx}-{slug}"[:63].rstrip("-") or f"yida{idx}" + out.append( + Volume( + name=vol_name, + host=Host(path=f"{host_storage}/yida_cache/{uid}/{sub}"), + mount_path=mount_path, + read_only=False, + ) + ) + return out diff --git a/src/backend/integrations/yida/sandbox.py b/src/backend/integrations/yida/sandbox.py new file mode 100644 index 00000000..09492ece --- /dev/null +++ b/src/backend/integrations/yida/sandbox.py @@ -0,0 +1,18 @@ +"""Credential contribution for yida.""" +from core.harness.registry import SandboxCredentials, register_sandbox_credentials +from integrations.yida.cube_credentials import CubeCredentials +from integrations.yida.mounts import mounts + +def state(host): + return host.credential_state("yida", lambda: CubeCredentials(host.credential_request_timeout)) + +async def prepare(host, sandbox, user_id, script): + if "openyida" in (script or ""): + await state(host).ensure(sandbox, user_id) + +async def persist(host, sandbox, user_id, script): + if "openyida" in (script or ""): + await state(host).persist(sandbox, user_id) + +def register(): + register_sandbox_credentials(SandboxCredentials("yida", mounts=mounts, before_command=prepare, after_command=persist, prepare_order=3)) diff --git a/src/backend/core/services/yida_service.py b/src/backend/integrations/yida/service.py similarity index 99% rename from src/backend/core/services/yida_service.py rename to src/backend/integrations/yida/service.py index 20d6d353..e337d9a8 100644 --- a/src/backend/core/services/yida_service.py +++ b/src/backend/integrations/yida/service.py @@ -42,7 +42,7 @@ from core.config.settings import settings from core.infra.ephemeral import get_ephemeral_state from core.sandbox.workspace.identity import safe_user_id -from core.services.dingtalk_service import make_qr_data_uri +from integrations.dingtalk.parsing import make_qr_data_uri logger = logging.getLogger(__name__) @@ -131,7 +131,7 @@ def extract_result_json(stdout: str) -> Optional[Dict[str, Any]]: def _host_workspace_dir(user_id: str) -> Path: - from core.services.yida_execution import workspace_dir + from integrations.yida.execution import workspace_dir return workspace_dir(user_id) @@ -250,7 +250,7 @@ class YidaService: async def _run_in_sandbox( self, user_id: str, sub_command: str, timeout: int ) -> tuple[str, int]: - from core.services.yida_execution import run_local, run_sandbox + from integrations.yida.execution import run_local, run_sandbox if settings.deploy.is_local: return await run_local(user_id, sub_command, timeout) diff --git a/src/backend/integrations/yida/skill_content.py b/src/backend/integrations/yida/skill_content.py new file mode 100644 index 00000000..98ce8bd9 --- /dev/null +++ b/src/backend/integrations/yida/skill_content.py @@ -0,0 +1,12 @@ +"""Resolve the Yida workspace example for the authenticated local account.""" +import shlex +from core.config.local_mode import local_mode_enabled +from core.sandbox.workspace.identity import safe_user_id +from integrations.yida.execution import workspace_dir + + +def adapt(content: str, user_id: str | None) -> str: + if (local_mode_enabled() and user_id and safe_user_id(user_id) + and "/home/ubuntu/yida-workspace" in content): + return content.replace("/home/ubuntu/yida-workspace", shlex.quote(workspace_dir(user_id).as_posix())) + return content diff --git a/src/backend/integrations/yida/storage.py b/src/backend/integrations/yida/storage.py new file mode 100644 index 00000000..df97342f --- /dev/null +++ b/src/backend/integrations/yida/storage.py @@ -0,0 +1,52 @@ +"""Per-user integration credential storage paths.""" +from pathlib import Path +from core.config.settings import settings + +def yida_cache_dir(user_id: str) -> Path: + """Backend-local openyida (Yida CLI) login-state persistence directory (one per user). + + Contains one subdirectory, bind-mounted into the sandbox's fixed Yida working directory: + - ``workspace/`` → sandbox ``/home/ubuntu/yida-workspace`` (openyida's projectRoot: + ``.cache/cookies-*.json`` login cookies + ``.cache/openyida-envs.json`` env config + + project artifacts such as ``prd/`` and schema cache) + + Difference from dws/lark: openyida's login state does **not** respect $HOME; it lands + under the "project root"'s ``.cache/`` (projectRoot = the cwd at command execution + time, see the cwd fallback of findProjectRoot in openyida lib/core/utils.js). So the + persistence scheme is not binding home subpaths but pinning each user a fixed + in-sandbox working directory and binding it wholesale: the skill forces all openyida + commands to run inside that directory, so cookies survive across sessions with the + volume. Login is completed by QR scan inside the conversation + (``openyida login --agent-qr``); the backend runs no openyida subprocess. The sandbox + runs as ubuntu (UID 1000) and the backend is also 1000, so ownership matches. + """ + return settings.storage.root / "yida_cache" / user_id + + +def yida_workspace_dir(user_id: str) -> Path: + """Backend absolute path of this user's fixed Yida working directory (opensandbox + bind source / cube inject-and-return source). + + The sandbox mount point is fixed at ``/home/ubuntu/yida-workspace``; the yida skill + requires all openyida commands to ``cd`` into that directory, so the ``.cache/`` + login state and project artifacts all land on the persistent volume. + """ + return yida_cache_dir(user_id) / "workspace" + + +def yida_shared_workspace_dir() -> Path: + """Yida working directory for the script-runner shared sandbox (**one per deployment**, + not per-user). + + script-runner is a shared container with no user concept (the provider ignores + user_id; rootfs is read_only) and cannot do per-user credential isolation — compose + mounts this directory at the container's ``/home/ubuntu/yida-workspace``, so the Yida + login state is shared deployment-wide. Only suitable for single-user/dev environments; + multi-user production should use opensandbox (per-user bind-mount) or cube (per-user + inject + return). The backend pre-creates it with 0777 at startup (runner uid 1001 ≠ + backend uid 1000; cross-uid writes need wide-open permissions). + """ + return settings.storage.root / "yida_cache" / "__shared__" / "workspace" + + +YIDA_WORKSPACE_MOUNT = "/home/ubuntu/yida-workspace" diff --git a/src/backend/orchestration/__init__.py b/src/backend/orchestration/__init__.py index 07c2d49a..ac8cf8bc 100755 --- a/src/backend/orchestration/__init__.py +++ b/src/backend/orchestration/__init__.py @@ -1,4 +1 @@ -"""Routing / multi-agent placeholders. - -Current behavior: always route to main. -""" +"""Outer Plan, Loop and scheduled workflow orchestration over core runtimes.""" diff --git a/src/backend/orchestration/automations/__init__.py b/src/backend/orchestration/automations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/backend/orchestration/schedulers/automation_artifacts.py b/src/backend/orchestration/automations/automation_artifacts.py similarity index 97% rename from src/backend/orchestration/schedulers/automation_artifacts.py rename to src/backend/orchestration/automations/automation_artifacts.py index 27eac573..dcabdc1a 100644 --- a/src/backend/orchestration/schedulers/automation_artifacts.py +++ b/src/backend/orchestration/automations/automation_artifacts.py @@ -1,6 +1,7 @@ """Load pinned scheduled-run deliverables for channel delivery.""" from typing import List, Tuple + from core.infra.logging import get_logger logger = get_logger(__name__) @@ -16,7 +17,7 @@ def load_run_files(chat_id: str) -> List[Tuple[bytes, str, str]]: """ from core.db.engine import SessionLocal from core.db.models import Artifact - from core.storage import get_storage + from core.files.storage import get_storage out: List[Tuple[bytes, str, str]] = [] try: diff --git a/src/backend/orchestration/schedulers/automation_conversation.py b/src/backend/orchestration/automations/automation_conversation.py similarity index 61% rename from src/backend/orchestration/schedulers/automation_conversation.py rename to src/backend/orchestration/automations/automation_conversation.py index 526dc101..3df20c16 100644 --- a/src/backend/orchestration/schedulers/automation_conversation.py +++ b/src/backend/orchestration/automations/automation_conversation.py @@ -3,16 +3,23 @@ import asyncio import uuid from typing import Dict, List, Optional, Tuple + +import core.chat.runtime.admission as admission +import core.chat.runtime.handoff as handoff +import core.runs.runtime as runtime +import orchestration.loops.admission as _orchestration_loops_admission +import orchestration.plans.admission as _orchestration_plans_admission from core.chat.context import build_runtime_context, resolve_enabled_capabilities from core.config.settings import DEFAULT_CHAT_MODEL_ALIAS from core.db import engine as db_engine -from core.db.models import ChatMessage, ChatRun, ScheduledTaskRun -from core.services.chat_service import ChatService -from core.services.chat_sequencer import ChatSequencer -from core.services.automation_service import AutomationService -from core.services.automation_execution import task_execution_context -from core.services import user_model_selection -from orchestration import chat_run_executor as executor +from core.db.models import ChatMessage, ChatRun +from features.automation.models import ScheduledTaskRun +from features.automation.runs import owned_run_guard +from core.models import selection as user_model_selection +from features.automation.execution import task_execution_context +from features.automation.service import AutomationService +from core.chat.sequencer import ChatSequencer +from core.chat.service import ChatService class ConversationStopped(Exception): @@ -21,8 +28,17 @@ def __init__(self, chat_id: str): super().__init__("用户已停止执行") -def _prepare(user_id, task_id, task_name, content, model_name, automation_run_id, metadata=None): - chat_id = f"chat_{uuid.uuid4().hex[:16]}" +def _prepare( + user_id, + task_id, + task_name, + content, + model_name, + automation_run_id, + metadata=None, + lease_token=None, +): + chat_id = f"chat_{automation_run_id}" if automation_run_id else f"chat_{uuid.uuid4().hex[:16]}" with db_engine.SessionLocal() as db: task = AutomationService(db).get_task(task_id, user_id) if task is None: @@ -38,74 +54,89 @@ def _prepare(user_id, task_id, task_name, content, model_name, automation_run_id ) if content is not None: service.add_message(chat_id=chat_id, role="user", content=content, model=model_name) - _link_run(db, automation_run_id, task_id, chat_id) return chat_id, context -def _link_run(db, automation_run_id, task_id, chat_id): +def _link_run(db, automation_run_id, task_id, chat_id, *, lease_token=None): if automation_run_id is None: return - run = db.get(ScheduledTaskRun, automation_run_id) - if run is None or run.task_id != task_id: - raise ValueError("定时任务执行记录不存在") - run.chat_id = chat_id - db.commit() + query = db.query(ScheduledTaskRun).filter( + ScheduledTaskRun.run_id == automation_run_id, + ScheduledTaskRun.task_id == task_id, + ScheduledTaskRun.status == "running", + ) + if lease_token is not None: + query = query.filter(ScheduledTaskRun.lease_token == lease_token) + if not query.update({ScheduledTaskRun.chat_id: chat_id}, synchronize_session=False): + raise ValueError("定时任务执行记录不存在或已被接管") + db.flush() + + +def _accept_link(run_id, task_id, token, recovery_snapshot=None): + def link(db, child): + _link_run(db, run_id, task_id, child.chat_id, lease_token=token) + if recovery_snapshot is not None: + import json + child.recovery_snapshot = json.loads(json.dumps(recovery_snapshot, ensure_ascii=False, default=str)) + child.snapshot_version = 1 + child.run_phase = "accepted" + child.last_operation_safety = "replayable" + return link -async def _wait(run) -> Tuple[str, str, Dict]: +async def _wait(run, *, automation_run_id=None, lease_token=None) -> Tuple[str, str, Dict]: """Observe durable state without taking ownership of the executor's asyncio tasks. - Cancellation of the scheduler (shutdown/ownership loss) must also fence the child run. + Scheduler shutdown transfers observation; the shared executor owns the child run. User cancellation is an outcome, not cancellation of the scheduler itself. """ try: while True: - current = executor.get_run(run.run_id) + current = await asyncio.to_thread(runtime.get_run, run.run_id) if current is None: raise RuntimeError("会话执行记录不存在") + if current.status == "pending": + from orchestration.pending import dispatch_pending + await dispatch_pending(current) if current.status not in ("pending", "running"): - successor = executor.get_successor_run(run.run_id, user_id=run.user_id) + successor = await asyncio.to_thread(handoff.get_successor_run, run.run_id, user_id=run.user_id) if successor is not None: run = successor continue # A cancelled cooperative plan may still be releasing its writer. if current.status == "cancelled" and current.writer_slot: - await asyncio.sleep(0.1) + await asyncio.sleep(1) continue if current.status == "cancelled": raise ConversationStopped(run.chat_id) if current.status != "completed": raise RuntimeError(current.error_message or "会话执行未完成") - with db_engine.SessionLocal() as db: - durable = db.get(ChatRun, run.run_id) - snapshot = durable.recovery_snapshot or {} - message_id = ( - snapshot.get("message_id") - or snapshot.get("worker_args", {}).get("context", {}).get("message_id") - or run.message_id - ) - message = db.get(ChatMessage, message_id) - if message is None: - raise RuntimeError("会话执行结果未保存") - from core.llm._distill_shared import strip_think_blocks - - result = strip_think_blocks(message.content or "") or "执行完成" - return run.chat_id, result, dict(message.usage or {}) - await asyncio.sleep(0.1) + return await asyncio.to_thread(_result, run) + await asyncio.sleep(1) except asyncio.CancelledError: - # Fence first, then check the durable handoff. Completion may win - # the cancellation CAS and atomically create a successor in between. - while True: - await executor.cancel_run(run.run_id, user_id=run.user_id) - successor = executor.get_successor_run(run.run_id, user_id=run.user_id) - if successor is None: - break - run = successor + # The durable child is owned by the shared run executor. Replacing an + # observer must not cancel accepted work; recovery follows the same run. raise -async def resume_conversation(chat_id, user_id, started_at): - """Observe this invocation's original child and its durable successors.""" +def _result(run): + with db_engine.SessionLocal() as db: + durable = db.get(ChatRun, run.run_id) + snapshot = durable.recovery_snapshot or {} + message_id = ( + snapshot.get("message_id") + or snapshot.get("worker_args", {}).get("context", {}).get("message_id") + or run.message_id + ) + message = db.get(ChatMessage, message_id) + if message is None: + raise RuntimeError("会话执行结果未保存") + from core.infra.model_output import strip_think_blocks + result = strip_think_blocks(message.content or "") or "执行完成" + return run.chat_id, result, dict(message.usage or {}) + + +def _find_child(chat_id, user_id, started_at): with db_engine.SessionLocal() as db: # Prompt/Plan use fresh chats; Loop reuses its chat. The first main run # after this invocation started is its root, not a later user message. @@ -124,7 +155,15 @@ async def resume_conversation(chat_id, user_id, started_at): ) if run is None: raise RuntimeError("定时任务关联的会话执行记录不存在") - return await _wait(run) + return run + + +async def resume_conversation( + chat_id, user_id, started_at, *, automation_run_id=None, lease_token=None +): + """Observe this invocation's original child and its durable successors.""" + run = await asyncio.to_thread(_find_child, chat_id, user_id, started_at) + return await _wait(run, automation_run_id=automation_run_id, lease_token=lease_token) async def execute_prompt( @@ -138,14 +177,17 @@ async def execute_prompt( enabled_kb_ids: Optional[List[str]], enabled_agent_ids: Optional[List[str]] = None, automation_run_id: Optional[str] = None, + lease_token: Optional[str] = None, ): model = user_model_selection.resolve_effective_chat_model_name() or DEFAULT_CHAT_MODEL_ALIAS - chat_id, project_context = _prepare(user_id, task_id, task_name, None, model, automation_run_id) + chat_id, project_context = _prepare( + user_id, task_id, task_name, None, model, automation_run_id, lease_token=lease_token + ) with db_engine.SessionLocal() as db: skills, agents, mcps = resolve_enabled_capabilities( db, user_id, enabled_skill_ids, enabled_agent_ids, enabled_mcp_ids ) - from core.services.ontology_service import build_user_ontology_runtime + from core.ontology.runtime_policy import build_user_ontology_runtime ontology_enabled, ontology_runtime = build_user_ontology_runtime(user_id=user_id, task=prompt) context = build_runtime_context( @@ -178,13 +220,20 @@ async def execute_prompt( } with db_engine.SessionLocal() as db: with ChatSequencer(db).launching_main_run( + on_accept=_accept_link(automation_run_id, task_id, lease_token, { + "kind": "chat", "worker_args": { + "session_messages": [{"role": "user", "content": prompt}], + "effective_user_message": prompt, "raw_user_message": prompt, + "context": context, "model_name": model, + }, + }), chat_id=chat_id, user_id=user_id, user_content=prompt, request_payload=payload, model=model, ) as accepted: - run = await executor.start_run( + run = await admission.start_run( accepted_run=accepted.run, chat_id=chat_id, user_id=user_id, @@ -196,7 +245,7 @@ async def execute_prompt( model_name=model, ) - return await _wait(run) + return await _wait(run, automation_run_id=automation_run_id, lease_token=lease_token) async def execute_plan( @@ -210,8 +259,9 @@ async def execute_plan( enabled_kb_ids: Optional[List[str]], enabled_agent_ids: Optional[List[str]], automation_run_id: Optional[str] = None, + lease_token: Optional[str] = None, ): - from core.services.plan_service import PlanService + from core.chat.plans import PlanService model = user_model_selection.resolve_effective_chat_model_name() or DEFAULT_CHAT_MODEL_ALIAS with db_engine.SessionLocal() as db: @@ -245,8 +295,10 @@ async def execute_plan( model, automation_run_id, {"plan_chat": True, "plan_id": plan_id}, + lease_token=lease_token, ) - run = await executor.start_plan_execute_run( + run = await _orchestration_plans_admission.start_plan_execute_run( + on_accept=_accept_link(automation_run_id, task_id, lease_token), plan_id=plan_id, chat_id=chat_id, user_id=user_id, @@ -257,11 +309,13 @@ async def execute_plan( session_messages=[{"role": "user", "content": content}], model_name=model, ) - return await _wait(run) + return await _wait(run, automation_run_id=automation_run_id, lease_token=lease_token) -async def execute_loop(*, user_id, task_name, loop_id, task_id, automation_run_id=None): - from core.services.loop_service import LoopService +async def execute_loop( + *, user_id, task_name, loop_id, task_id, automation_run_id=None, lease_token=None +): + from features.automation.loops import LoopService if not loop_id: raise ValueError("loop 任务缺少 extra_data.loop_id") @@ -292,10 +346,10 @@ async def execute_loop(*, user_id, task_name, loop_id, task_id, automation_run_i if not loop.chat_id: loop.chat_id = chat_id db.commit() - _link_run(db, automation_run_id, task_id, chat_id) - if executor.get_active_run_for_chat(chat_id, user_id): + if runtime.get_active_run_for_chat(chat_id, user_id): return chat_id, "loop 已在推进中，跳过本次触发", {} - run = await executor.start_autonomous_loop_run( + run = await _orchestration_loops_admission.start_autonomous_loop_run( + on_accept=_accept_link(automation_run_id, task_id, lease_token), loop_id=loop_id, chat_id=chat_id, user_id=user_id, @@ -304,4 +358,4 @@ async def execute_loop(*, user_id, task_name, loop_id, task_id, automation_run_i project_id=project_context.get("project_id"), automation_run=True, ) - return await _wait(run) + return await _wait(run, automation_run_id=automation_run_id, lease_token=lease_token) diff --git a/src/backend/orchestration/automations/automation_delivery.py b/src/backend/orchestration/automations/automation_delivery.py new file mode 100644 index 00000000..7cb9e750 --- /dev/null +++ b/src/backend/orchestration/automations/automation_delivery.py @@ -0,0 +1,62 @@ +"""Bound result delivery without changing the committed business outcome.""" + +import asyncio + +from core.infra.background import spawn +from core.infra.logging import get_logger + +logger = get_logger(__name__) +DELIVERY_TIMEOUT_SECONDS = 60 + + +async def deliver_result( + scheduler, user_id, task_id, task_name, status, text, chat_id=None, metadata=None +): + stopped = asyncio.Event() + + async def deliver(): + from features.automation.delivery_targets import has_inapp, resolve_delivery_targets + + targets = resolve_delivery_targets(metadata) + if status != "success" or has_inapp(targets): + await scheduler._send_notification(user_id, task_id, task_name, status, text, chat_id) + channels = [t for t in targets if t.get("type") == "channel"] + if status != "success" or not channels or stopped.is_set(): + return + from integrations.channels.outbound import deliver_to_conversation + + from .automation_artifacts import load_run_files + + files = await asyncio.to_thread(load_run_files, chat_id) if chat_id else [] + for target in channels: + # A channel library may swallow cancellation. Do not start another + # send after the deadline, even if the in-flight send returns late. + if stopped.is_set(): + return + channel, conversation = target.get("channel_id"), target.get("conversation_id") + if not channel or not conversation: + continue + body = (f"【{task_name}】\n" if task_name else "") + text + try: + ok = await deliver_to_conversation(channel, conversation, body, files=files) + if not ok: + logger.error("automation_delivery_failed", task_id=task_id) + except Exception: + logger.error("automation_delivery_failed", task_id=task_id, exc_info=True) + + delivery = spawn(deliver(), name=f"automation.delivery.{task_id}") + try: + done, _ = await asyncio.wait({delivery}, timeout=DELIVERY_TIMEOUT_SECONDS) + if not done: + stopped.set() + delivery.cancel() + logger.error("automation_delivery_timeout", task_id=task_id) + elif not delivery.cancelled(): + # Delivery errors do not turn a committed success into a failure. + delivery.result() + except asyncio.CancelledError: + stopped.set() + delivery.cancel() + raise + except Exception: + logger.error("automation_delivery_failed", task_id=task_id, exc_info=True) diff --git a/src/backend/orchestration/schedulers/automation_dispatch.py b/src/backend/orchestration/automations/automation_dispatch.py similarity index 62% rename from src/backend/orchestration/schedulers/automation_dispatch.py rename to src/backend/orchestration/automations/automation_dispatch.py index 1cf9bc0a..ec25bc43 100644 --- a/src/backend/orchestration/schedulers/automation_dispatch.py +++ b/src/backend/orchestration/automations/automation_dispatch.py @@ -3,34 +3,36 @@ import asyncio import time from typing import Any, Dict -from core.infra.time import utc_now + from core.infra.logging import get_logger +from core.infra.time import utc_now logger = get_logger(__name__) def _run_chat_id(run_id): from core.db.engine import SessionLocal - from core.db.models import ScheduledTaskRun + from features.automation.models import ScheduledTaskRun with SessionLocal() as db: run = db.get(ScheduledTaskRun, run_id) return run.chat_id if run else None -async def execute_task(scheduler, task_id: str, user_id: str, *, resume_run_id=None): +async def execute_task(scheduler, task_id: str, user_id: str, *, run_id: str): """Execute a single scheduled task.""" from core.db.engine import SessionLocal - from core.services.automation_service import AutomationService + from features.automation.service import AutomationService - from .automation_artifacts import load_run_files from .automation_conversation import ( ConversationStopped, - execute_prompt, - execute_plan, execute_loop, + execute_plan, + execute_prompt, ) + from .automation_delivery import deliver_result + lease_token = scheduler._leases[task_id].token start = time.monotonic() with SessionLocal() as db: @@ -38,16 +40,16 @@ async def execute_task(scheduler, task_id: str, user_id: str, *, resume_run_id=N task = svc.get_task_by_id(task_id) if not task or task.user_id != user_id: return - if resume_run_id is None and task.status not in ("active", "paused"): - return - if resume_run_id is None: - run = svc.record_run_start(task_id) - else: - from core.db.models import ScheduledTaskRun + from features.automation.models import ScheduledTaskRun - run = db.get(ScheduledTaskRun, resume_run_id) - if run is None or run.task_id != task_id or run.status != "running": - return + run = db.get(ScheduledTaskRun, run_id) + if ( + run is None + or run.task_id != task_id + or run.status != "running" + or run.lease_token != lease_token + ): + return task_type = task.task_type task_name = task.name or "定时任务" task_prompt = task.prompt @@ -66,24 +68,28 @@ async def execute_task(scheduler, task_id: str, user_id: str, *, resume_run_id=N task.extra_data or {} ) # includes optional channel delivery destinations - if resume_run_id is not None: + if run.chat_id: from core.infra.time import as_utc start -= max(0, (utc_now() - as_utc(run.started_at)).total_seconds()) try: - from core.services.automation_execution import task_execution_context + from features.automation.execution import task_execution_context with SessionLocal() as db: current_task = AutomationService(db).get_task(task_id, user_id) if current_task is None: raise ValueError("任务已删除") task_execution_context(db, current_task) - if resume_run_id is not None: + if run.chat_id: from .automation_conversation import resume_conversation chat_id, result_text, usage = await resume_conversation( - run.chat_id, user_id, run.started_at + run.chat_id, + user_id, + run.started_at, + automation_run_id=run.run_id, + lease_token=lease_token, ) elif task_type == "prompt": chat_id, result_text, usage = await execute_prompt( @@ -92,6 +98,7 @@ async def execute_task(scheduler, task_id: str, user_id: str, *, resume_run_id=N prompt=task_prompt, task_id=task_id, automation_run_id=run.run_id, + lease_token=lease_token, enabled_mcp_ids=enabled_mcp_ids, enabled_skill_ids=enabled_skill_ids, enabled_kb_ids=enabled_kb_ids, @@ -104,6 +111,7 @@ async def execute_task(scheduler, task_id: str, user_id: str, *, resume_run_id=N plan_id=task_plan_id, task_id=task_id, automation_run_id=run.run_id, + lease_token=lease_token, enabled_mcp_ids=enabled_mcp_ids, enabled_skill_ids=enabled_skill_ids, enabled_kb_ids=enabled_kb_ids, @@ -117,6 +125,7 @@ async def execute_task(scheduler, task_id: str, user_id: str, *, resume_run_id=N loop_id=(task_metadata or {}).get("loop_id"), task_id=task_id, automation_run_id=run.run_id, + lease_token=lease_token, ) else: raise ValueError(f"Unknown task type: {task_type}") @@ -131,8 +140,10 @@ async def execute_task(scheduler, task_id: str, user_id: str, *, resume_run_id=N # 但通知与渠道投递用的是开跑时抓下来的 task_name / task_metadata 快照， # 不查这一下就会给一个已经删掉的任务推「执行完成」。 still_exists = svc.get_task_by_id(task_id) is not None - svc.record_run_complete( + completed = svc.record_run_complete( run.run_id, + lease_token=lease_token, + commit=False, status="success", chat_id=chat_id, # 存全文：这一份同时是渠道投递的正文来源，截断只发生在展示侧 @@ -141,98 +152,48 @@ async def execute_task(scheduler, task_id: str, user_id: str, *, resume_run_id=N duration_ms=duration_ms, usage=usage, ) + if completed is None: + return svc.update_task_system( task_id, consecutive_failures=0, last_run_at=utc_now(), sidebar_activated=True, ) - # next_run_at already pre-advanced in _check_and_fire; now that - # the run actually finished, flip one-shot/exhausted tasks to - # "completed" (advance_next_run intentionally left them active so - # this run could pass the execute_task guard). + # Admission advanced the schedule; completion finalizes exhausted tasks. svc.finalize_after_run(task_id) - # Multi-target delivery (delivery_targets model, with backward compat for the old flat channel_id/conversation_id). - # In-app (notification center + sidebar + chat history) is delivered only when targets include inapp; channels and other outbound targets are delivered one by one. - from core.services.delivery_targets import has_inapp, resolve_delivery_targets - - if not still_exists: - logger.info( - "[scheduler] task %s was deleted while running — skip notification/delivery", + logger.info("[scheduler] task %s completed in %dms", task_id, duration_ms) + if still_exists: + await deliver_result( + scheduler, + user_id, task_id, + task_name, + "success", + result_text or "执行完成", + chat_id, + task_metadata, ) - return - - _targets = resolve_delivery_targets(task_metadata) - if has_inapp(_targets): - await scheduler._send_notification( - user_id, task_id, task_name, "success", result_text or "执行完成", chat_id - ) - logger.info("[scheduler] task %s completed in %dms", task_id, duration_ms) - - # Artifact files generated by this run (pinned deliverables anchored under this run's chat_id). - # Loaded once and reused across all channel targets — otherwise "generate a document and send it to Feishu" only sends the text and the file never goes out. - _gen_files = load_run_files(chat_id) if chat_id else [] - for _tgt in _targets: - if _tgt.get("type") != "channel": - continue # inapp already handled; email etc. reserved - _ch = _tgt.get("channel_id") - _conv = _tgt.get("conversation_id") - if not (_ch and _conv): - continue - try: - from core.channels.outbound import deliver_to_conversation - - head = f"【{task_name}】\n" if task_name else "" - _ok = await deliver_to_conversation( - _ch, - _conv, - head + (result_text or "执行完成"), - files=_gen_files, - ) - if _ok: - logger.info( - "[scheduler] 渠道投递成功 task=%s channel=%s conv=%s files=%d", - task_id, - _ch, - _conv, - len(_gen_files), - ) - else: - logger.warning( - "[scheduler] 渠道投递失败（原因见 [channels] 日志）task=%s channel=%s conv=%s", - task_id, - _ch, - _conv, - ) - except Exception: - logger.warning("[scheduler] 渠道投递异常 task=%s", task_id, exc_info=True) except ConversationStopped as stopped: with SessionLocal() as db: svc = AutomationService(db) - svc.record_run_complete( + completed = svc.record_run_complete( run.run_id, + lease_token=lease_token, + commit=False, status="failed", chat_id=stopped.chat_id, error_message="用户已停止执行", duration_ms=int((time.monotonic() - start) * 1000), ) + if completed is None: + return svc.update_task_system(task_id, last_run_at=utc_now(), sidebar_activated=True) svc.finalize_after_run(task_id) logger.info("[scheduler] task %s stopped by user", task_id) except asyncio.CancelledError: - with SessionLocal() as db: - svc = AutomationService(db) - svc.record_run_complete( - run.run_id, - status="failed", - chat_id=_run_chat_id(run.run_id), - error_message="执行已中断", - duration_ms=int((time.monotonic() - start) * 1000), - ) - svc.finalize_after_run(task_id) raise except Exception as e: duration_ms = int((time.monotonic() - start) * 1000) @@ -243,13 +204,17 @@ async def execute_task(scheduler, task_id: str, user_id: str, *, resume_run_id=N with SessionLocal() as db: svc = AutomationService(db) still_exists = svc.get_task_by_id(task_id) is not None - svc.record_run_complete( + completed = svc.record_run_complete( run.run_id, + lease_token=lease_token, + commit=False, status="failed", chat_id=_run_chat_id(run.run_id), error_message=error_msg, duration_ms=duration_ms, ) + if completed is None: + return new_failures = task_consecutive_failures + 1 updates: Dict[str, Any] = { "consecutive_failures": new_failures, @@ -263,15 +228,11 @@ async def execute_task(scheduler, task_id: str, user_id: str, *, resume_run_id=N "[scheduler] task %s auto-disabled after %d failures", task_id, new_failures ) svc.update_task_system(task_id, **updates) - # next_run_at already pre-advanced in _check_and_fire. Finalize - # one-shot tasks so a failed single run lands in a terminal state - # instead of dangling active (finalize_after_run skips disabled). + # Finalize failed one-shot tasks without overriding auto-disable. svc.finalize_after_run(task_id) if still_exists: - await scheduler._send_notification( - user_id, task_id, task_name, "failed", error_msg[:200] - ) + await deliver_result(scheduler, user_id, task_id, task_name, "failed", error_msg[:200]) else: logger.info( "[scheduler] task %s was deleted while running — skip failure notification", diff --git a/src/backend/orchestration/automations/automation_lease.py b/src/backend/orchestration/automations/automation_lease.py new file mode 100644 index 00000000..c929dc23 --- /dev/null +++ b/src/backend/orchestration/automations/automation_lease.py @@ -0,0 +1,113 @@ +"""Renewable ownership with cooperative stop and bounded token-safe release.""" + +import asyncio +import uuid + +from core.infra import ephemeral +from core.infra.background import spawn +from core.infra.logging import get_logger + +logger = get_logger(__name__) +LEASE_TTL_SECONDS = 60 +RENEW_INTERVAL_SECONDS = 15 +RELEASE_WAIT_SECONDS = 2 + + +class AutomationLease: + def __init__(self, key): + self.key = key + self.token = uuid.uuid4().hex + self.state = ephemeral.get_ephemeral_state() + self.renewal = None + self.execution = None + self._stop = asyncio.Event() + self._release_task = None + + async def acquire(self): + if not await self.state.hold(self.key, self.token, ttl=LEASE_TTL_SECONDS): + return False + self.renewal = spawn(self._renew(), name="automation.lease.renew") + return True + + def _finished(self): + return self._stop.is_set() or (self.execution is not None and self.execution.done()) + + async def _renew(self): + stopped = asyncio.create_task(self._stop.wait()) + attempt = None + try: + while not self._finished(): + await asyncio.wait({stopped}, timeout=RENEW_INTERVAL_SECONDS) + if self._finished(): + return + # CAS cannot recreate a deleted key or replace another token. + attempt = spawn( + self.state.compare_exchange( + self.key, self.token, self.token, ttl=LEASE_TTL_SECONDS + ), name="automation.lease.cas", + ) + done, _ = await asyncio.wait({attempt}, timeout=RENEW_INTERVAL_SECONDS) + if not done or not attempt.result(): + return + if self._finished(): + return + finally: + if attempt is not None and not attempt.done(): + attempt.cancel() + stopped.cancel() + await asyncio.gather(stopped, return_exceptions=True) + + async def run(self, awaitable): + execution = self.execution = asyncio.create_task(awaitable) + # This also stops renewal if an outer release path is never reached. + execution.add_done_callback(lambda _: self._stop.set()) + try: + done, _ = await asyncio.wait( + (execution, self.renewal), return_when=asyncio.FIRST_COMPLETED + ) + if execution in done: + return await execution + execution.cancel() + await asyncio.gather(execution, return_exceptions=True) + self.renewal.result() + raise RuntimeError("定时任务执行锁已失效") + finally: + if not execution.done(): + execution.cancel() + await asyncio.gather(execution, return_exceptions=True) + + async def owns(self): + if self._finished(): + return False + read = spawn(self.state.get(self.key), name="automation.lease.check") + done, _ = await asyncio.wait({read}, timeout=RELEASE_WAIT_SECONDS) + if not done: + read.cancel() + return False + return not self._finished() and read.result() == self.token + + async def release(self): + self._stop.set() + if self._release_task is None: + self._release_task = spawn(self._release(), name="automation.lease.release") + # Concurrent callers share cleanup; caller cancellation cannot abort it. + await asyncio.shield(self._release_task) + + async def _release(self): + try: + if self.renewal: + self.renewal.cancel() + _, pending = await asyncio.wait({self.renewal}, timeout=RELEASE_WAIT_SECONDS) + if pending: + logger.error("automation_renewal_stop_timeout", key=self.key) + finally: + # Always delete by token, including when renewal ignores cancellation. + # A late CAS observes deletion/new ownership and cannot resurrect us. + deletion = spawn( + self.state.release(self.key, self.token), name="automation.lease.delete" + ) + _, pending = await asyncio.wait({deletion}, timeout=RELEASE_WAIT_SECONDS) + if pending: + logger.error("automation_lock_delete_timeout", key=self.key) + else: + deletion.result() diff --git a/src/backend/orchestration/automations/automation_recovery.py b/src/backend/orchestration/automations/automation_recovery.py new file mode 100644 index 00000000..0a92ace9 --- /dev/null +++ b/src/backend/orchestration/automations/automation_recovery.py @@ -0,0 +1,39 @@ +"""Take over a durable invocation without creating another run record.""" +import asyncio +from core.db import engine as db_engine +from features.automation.models import ScheduledTaskRun +from features.automation.runs import AutomationRunRepository +from features.automation.service import AutomationService + + +def _runs(task_id): + with db_engine.SessionLocal() as db: + task = AutomationService(db).get_task_by_id(task_id) + if task is None: + return None, [] + return task.user_id, [ + (row.run_id, row.lease_token) + for row in db.query(ScheduledTaskRun).filter( + ScheduledTaskRun.task_id == task_id, + ScheduledTaskRun.status.in_(("pending", "running")), + ).order_by(ScheduledTaskRun.started_at).all() + ] + + +def _claim(run_id, token, expected): + with db_engine.SessionLocal() as db: + return AutomationRunRepository(db).claim(run_id, token, expected_token=expected) + + +async def recover_task(scheduler, task_id): + user_id, runs = await asyncio.to_thread(_runs, task_id) + lease = scheduler._leases[task_id] + for run_id, expected in runs: + if not await lease.owns(): + return + if not await asyncio.to_thread(_claim, run_id, lease.token, expected): + continue + if not await lease.owns(): + return + from .automation_dispatch import execute_task + await execute_task(scheduler, task_id, user_id, run_id=run_id) diff --git a/src/backend/orchestration/automations/automation_scheduler.py b/src/backend/orchestration/automations/automation_scheduler.py new file mode 100644 index 00000000..a3d32ed4 --- /dev/null +++ b/src/backend/orchestration/automations/automation_scheduler.py @@ -0,0 +1,136 @@ +"""Bounded dispatch of durable automation invocations.""" +import asyncio +import contextlib +import uuid + +from core.infra.logging import get_logger +from core.infra.time import utc_now + +logger = get_logger(__name__) +POLL_INTERVAL_SECONDS = 3 +MAX_EXECUTIONS = 8 +REDIS_LOCK_PREFIX = "jx:auto:lock:" + + +class AutomationScheduler: + def __init__(self): + self._running = False + self._task = None + self._executions = set() + self._leases = {} + + async def start(self): + if self._running: + return + self._running = True + self._task = asyncio.create_task(self._poll_loop()) + + async def stop(self): + self._running = False + if self._task is not None: + self._task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await self._task + self._task = None + executions = list(self._executions) + for task in executions: + task.cancel() + await asyncio.gather(*executions, return_exceptions=True) + for task_id in list(self._leases): + await self._release_lock(task_id) + + @staticmethod + def _pending(): + from core.db.engine import SessionLocal + from features.automation.admission import admit, candidates + + with SessionLocal() as db: + pending = candidates(db, MAX_EXECUTIONS) + admit(db, max(0, MAX_EXECUTIONS - len(pending))) + return candidates(db, MAX_EXECUTIONS) + + async def _poll_loop(self): + while self._running: + try: + await self._dispatch_pending() + except Exception: + logger.exception("automation dispatch deferred") + await asyncio.sleep(POLL_INTERVAL_SECONDS) + + async def _dispatch_pending(self): + capacity = MAX_EXECUTIONS - len(self._executions) + if capacity <= 0: + return + for task_id in await asyncio.to_thread(self._pending): + if len(self._executions) >= MAX_EXECUTIONS: + break + if not await self._acquire_lock(task_id): + continue + task = asyncio.create_task(self.execute_task(task_id)) + self._executions.add(task) + task.add_done_callback(self._executions.discard) + + async def execute_task(self, task_id): + from .automation_recovery import recover_task + + try: + await self._leases[task_id].run(recover_task(self, task_id)) + except asyncio.CancelledError: + raise + except Exception: + logger.exception("automation execution deferred task=%s", task_id) + finally: + await self._release_lock(task_id) + + async def _acquire_lock(self, task_id: str) -> bool: + from .automation_lease import AutomationLease + + if task_id in self._leases: + return False + try: + lease = AutomationLease(f"{REDIS_LOCK_PREFIX}{task_id}") + if not await lease.acquire(): + return False + self._leases[task_id] = lease + return True + except Exception as e: + logger.warning("[scheduler] lock acquire failed for %s: %s", task_id, e) + return False + + async def _release_lock(self, task_id: str): + lease = self._leases.pop(task_id, None) + if lease is None: + return + try: + await lease.release() + except Exception as e: + logger.error("[scheduler] lock release failed for %s: %s", task_id, e) + + async def _send_notification( + self, + user_id: str, + task_id: str, + task_name: str, + status: str, + summary: str, + chat_id: str | None = None, + ): + try: + from features.automation import notifications as notifications + from features.automation.service import SUMMARY_LIMIT_BRIEF, truncate_summary + + notification = { + "id": f"notif_{uuid.uuid4().hex[:12]}", + "task_id": task_id, + "task_name": task_name, + "status": status, + # 走统一截断策略：result_summary 现在是全文，裸切会把整篇报告拦腰截断且 + # 不带任何提示，与运行历史列表的观感不一致。 + "summary": truncate_summary(summary, SUMMARY_LIMIT_BRIEF), + "chat_id": chat_id, + "timestamp": int(utc_now().timestamp() * 1000), + "read": False, + } + await notifications.push(user_id, notification) + except Exception as e: + logger.warning("[scheduler] notification failed: %s", e) diff --git a/src/backend/orchestration/chat_run_executor.py b/src/backend/orchestration/chat_run_executor.py deleted file mode 100644 index c65dc7fe..00000000 --- a/src/backend/orchestration/chat_run_executor.py +++ /dev/null @@ -1,4651 +0,0 @@ -"""Chat Run Executor — decouples the AI workflow from the HTTP connection lifecycle. - -Each sent message creates a ChatRun row + starts a background asyncio.Task. The -task consumes ``astream_chat_workflow``, converts every chunk into an SSE event -and XADDs it to the Redis Stream ``jx:chat:run:{run_id}:events``. SSE followers -read the stream via XRANGE replay + XREAD tailing, which enables "resume the -live stream after a page refresh". - -Public interface: -- ``start_run(...)`` launch a run already accepted by ChatSequencer -- ``wait_run(run_id)`` wait for a locally launched run without detaching it -- ``follow_run(run_id, ...)`` SSE follower: read events from the Redis Stream -- ``cancel_run(run_id, ...)`` cancel the background task + mark status=cancelled -- ``recover_orphan_runs()`` startup hook: resume from the last durable safe point -- ``get_run(run_id)`` read the ChatRun row (used by the route layer for authz) -- ``get_active_run_for_chat(chat_id, user_id)`` probe for an in-progress run -""" - -from __future__ import annotations - -import asyncio -import contextlib -import functools -import json -import os -import socket -import time -import uuid -from datetime import datetime, timedelta, timezone -from typing import ( - Any, - AsyncIterator, - Awaitable, - Callable, - Dict, - List, - Literal, - Optional, - Tuple, -) - -from core.chat import inflight -from core.llm.model_steps import close_dangling_calls, replace_final_text -from core.config.settings import DEFAULT_CHAT_MODEL_ALIAS -from core.db.engine import SessionLocal -from core.db.models import ChatMessage, ChatRun -from core.db.reserved_chats import ordinary_chat_filter -from core.harness.hooks import HookPaused, find_hook_paused -from core.infra.logging import get_logger -from core.llm import human_interaction -from core.services import ChatService -from core.services.run_journal import RecoveryDecision, RunJournal, RunLeaseLost -from orchestration.message_parser import looks_markdown -from orchestration.run_event_stream import ( - START, - Entry, - RunEventStream, - get_run_event_stream, -) -from core.services.tool_effect_ledger import ToolOutcomeUnknown, find_tool_outcome_unknown -from orchestration.workflow import astream_chat_workflow -from orchestration.compaction_stream import stream_with_compaction -from redis.exceptions import TimeoutError as RedisTimeoutError - -logger = get_logger(__name__) - - -# ─── Types / constants ───────────────────────────────────────────────── - -RunKind = Literal["chat", "plan_execute", "plan_generate", "autonomous_loop"] - -# Kinds that use cooperative cancellation (no cross-task cancel; they poll is_run_cancelled and stop themselves, avoiding the anyio cancel-scope deadlock) -_COOPERATIVE_KINDS: tuple[str, ...] = ("plan_execute", "autonomous_loop") -RunStatus = Literal["pending", "running", "needs_attention", "completed", "failed", "cancelled"] - -_TERMINAL_STATUSES: tuple[RunStatus, ...] = ( - "needs_attention", - "completed", - "failed", - "cancelled", -) -_LIVE_STATUSES: tuple[RunStatus, ...] = ("pending", "running") - -_STREAM_TTL_SECONDS = human_interaction.STREAM_TTL_SECONDS -_TERMINAL_TYPE = "__terminal__" -_XREAD_BLOCK_MS = 5000 -# When the SSE stream is silent longer than this, write a `: heartbeat\n\n` -# comment line on the wire so that nginx `proxy_read_timeout` (default 60s, -# 600s in this project) / intermediate reverse proxies / client-side proxies -# don't treat the idle stream during a long LLM call as a dead connection and -# kill it. The EventSource standard discards SSE comment lines, so the -# frontend needs no changes at all. -_HEARTBEAT_INTERVAL_SEC = 15.0 - -# Per-run "no activity" watchdog: if astream_chat_workflow produces no chunk -# within this many seconds (no yield, no raise, not cancelled) it is judged -# hung; a TimeoutError is raised and handled by the existing except path that -# writes the failed terminal state, so the run never stays in running forever. -# Initial visible response budget; heartbeats and setup events do not count. -_FIRST_RESPONSE_TIMEOUT_SEC = 600.0 - -_INACTIVITY_TIMEOUT_SEC = float(os.getenv("CHAT_RUN_INACTIVITY_TIMEOUT_SEC", "600")) -# Defense in depth: periodically check runs that are running and older than -# this age (backstop for the watchdog, also cleans up historical zombie runs). -# Exceeding the age alone no longer kills the run — the Redis Stream must also -# have been quiet for more than CHAT_RUN_STALE_QUIET_SEC (see reap_stale_runs); -# long tasks that are still producing output are not killed. -_STALE_RUN_MAX_AGE_SEC = float(os.getenv("CHAT_RUN_MAX_AGE_SEC", "1800")) -_STALE_REAPER_INTERVAL_SEC = float(os.getenv("CHAT_RUN_REAPER_INTERVAL_SEC", "300")) -# "Quiet" threshold for over-age runs: a run only counts as a zombie when the -# last event on its stream is older than this. Defaults to the same value as -# the per-run inactivity watchdog — runs hung inside this process get reaped -# by the watchdog first, so the only quiet runs reaching the reaper are those -# whose worker has vanished (leftovers from a process restart/crash). -_STALE_QUIET_SEC = float(os.getenv("CHAT_RUN_STALE_QUIET_SEC", str(_INACTIVITY_TIMEOUT_SEC))) -# Absolute lifetime cap: even a run that keeps actively producing output is -# force-reaped past this age, preventing a runaway agent loop from living -# forever by continuously emitting. -_HARD_MAX_AGE_SEC = float(os.getenv("CHAT_RUN_HARD_MAX_AGE_SEC", "21600")) -_RUN_LEASE_SECONDS = max(15, int(os.getenv("CHAT_RUN_LEASE_SECONDS", "90"))) -_RUN_LEASE_HEARTBEAT_SEC = max(1.0, _RUN_LEASE_SECONDS / 3) -_RUN_RECOVERY_INTERVAL_SEC = max( - 1.0, - float(os.getenv("CHAT_RUN_RECOVERY_INTERVAL_SEC", "15")), -) -_WORKER_INSTANCE_ID = f"{socket.gethostname()}:{os.getpid()}:{uuid.uuid4().hex[:12]}" -# How many live SSE offsets one journal reservation covers. A reservation is a -# serialized write; the ownership fence is a separate read taken on every event, -# so this size only trades write volume against how many offsets a reservation -# may leave unused. A token stream arrives in the low hundreds of events per -# second, so one write per burst replaces one write per token — invisible -# against a server database, decisive on the desktop, where the run journal is -# SQLite and every commit is a disk flush on the machine the user waits at. -_OFFSET_RESERVATION = 64 - -# How long a streaming run may go without re-asking the shared journal whether -# it is still supposed to be running. Two things ask — the ownership proof -# behind an emitted event, and the cancellation poll that covers the stretches -# between events, when a tool is running and nothing is being emitted — and both -# are answering the same question: somebody pressed stop, possibly in another -# worker process. One cadence, so the two cannot drift into disagreeing about -# how fast a stop button responds. A quarter second sits under the threshold -# where interface response stops feeling instant. -_CANCEL_VISIBILITY_SEC = 0.25 - - -async def _aiter_with_inactivity_timeout( - aiter: AsyncIterator[Any], - timeout: float, - *, - is_activity: Callable[[Any], bool] | None = None, - first_response_timeout: float | None = None, -): - """Yield from an async iterator and enforce a meaningful-activity deadline. - - Guards against a hung workflow (model loop / blocked MCP / asyncio - deadlock) leaving the chat run stuck in 'running' forever — on timeout - the underlying generator is closed and the error propagates into the - existing ``except Exception`` path that writes the ``failed`` terminal. - - Transport keep-alives may still be yielded to the caller, but they do not - reset the deadline when ``is_activity`` marks them as non-meaningful. - The optional first-response deadline is absolute until visible output; - it is then disabled for the remainder of this workflow. - """ - activity_check = is_activity or (lambda _: True) - loop = asyncio.get_running_loop() - deadline = loop.time() + timeout - first_deadline = ( - loop.time() + first_response_timeout - if first_response_timeout is not None else None - ) - - def timeout_error() -> TimeoutError: - from core.infra.exceptions import FirstResponseTimeoutError - - if first_deadline is not None and loop.time() >= first_deadline: - return FirstResponseTimeoutError() - return TimeoutError(f"工作流 {timeout:.0f}s 内无有效输出，已判定为卡死并中止") - - while True: - effective_deadline = ( - min(deadline, first_deadline) if first_deadline is not None else deadline - ) - remaining = effective_deadline - loop.time() - if remaining <= 0: - with contextlib.suppress(Exception): - await aiter.aclose() # type: ignore[attr-defined] - raise timeout_error() - try: - item = await asyncio.wait_for(aiter.__anext__(), timeout=remaining) - except StopAsyncIteration: - return - except asyncio.TimeoutError as exc: - with contextlib.suppress(Exception): - await aiter.aclose() # type: ignore[attr-defined] - raise timeout_error() from exc - if first_deadline is not None and _is_first_response(item): - first_deadline = None - if activity_check(item): - deadline = loop.time() + timeout - yield item - - -def _is_first_response(item: Any) -> bool: - """Only visible model output or a tool interaction ends the initial wait.""" - kind = item.get("type") - if kind in {"ai_message", "content", "thinking"}: - return bool(item.get("delta")) - if kind == "content_replace": - return bool(item.get("content")) - return kind in { - "tool_call_start", "tool_call_delta", "tool_call", "tool_result", - "tool_pending", "file_confirm", "plan_update", "compaction_progress", - } - - -def _is_chat_run_activity(item: Any, chat_id: str) -> bool: - """Classify workflow output for the inactivity watchdog. - - A transport heartbeat is normally not meaningful progress. It becomes - progress only while a registered tool is legitimately suspended on a - human answer; this keeps true HITL waits alive without weakening zombie - detection for every other silent run. - """ - - return item.get("type") != "heartbeat" or human_interaction.has_pending(chat_id) - - -def _events() -> RunEventStream: - """This deployment's run event log (Redis Streams, or in-process).""" - return get_run_event_stream() - - -# run_id → asyncio.Task — for cancel_run to kill the underlying coroutine -_active_runs: Dict[str, asyncio.Task] = {} - - -class ChatRunNotFound(Exception): - pass - - -class ChatRunPermissionDenied(Exception): - pass - - -def _utcnow() -> datetime: - return datetime.now(timezone.utc) - - -def _epoch_ms(value: Optional[datetime]) -> Optional[int]: - """Wall-clock milliseconds for a stored timestamp. - - SQLite hands back naive datetimes; everything written here is UTC, so an - absent tzinfo is read as UTC rather than as the host's local zone. - """ - if value is None: - return None - if value.tzinfo is None: - value = value.replace(tzinfo=timezone.utc) - return int(value.timestamp() * 1000) - - -def _run_started_at_ms(run_id: str) -> Optional[int]: - """When this run actually began, in wall-clock milliseconds. - - Read from ``chat_runs.started_at``, which is written once when the run is - claimed and is never rewritten by a recovering worker. Sending it with the - first frame is what lets a second device — or a reloaded tab — join a run - already in flight and show the true elapsed time instead of starting its - own stopwatch from zero. - - Only a worker resuming someone else's run needs to look this up: a worker - that claimed the run already holds the instant it wrote. Keeping the read - off the normal send path matters — it sits directly in front of the first - frame, where every millisecond is one the user spends staring at nothing. - """ - with SessionLocal() as db: - row = ( - db.query(ChatRun.started_at, ChatRun.created_at) - .filter(ChatRun.run_id == run_id) - .first() - ) - if row is None: - return None - return _epoch_ms(row.started_at or row.created_at) - - -def _journal() -> RunJournal: - # Resolve SessionLocal at call time so tests and alternate runtime profiles - # can replace the factory without leaving the journal bound to another DB. - return RunJournal(SessionLocal) - - -def _new_worker_owner(run_id: str) -> str: - return f"{_WORKER_INSTANCE_ID}:{run_id}:{uuid.uuid4().hex[:12]}" - - -class _EventOffsets: - """Durable SSE offsets for one worker, without a database write per event. - - Allocating an offset and proving the worker still owns the run used to be - the same write, so every token cost a serialized transaction — invisible - against a server database, and a disk flush per token on the desktop, where - the journal is SQLite and the person is watching the answer appear. - - The two are separable. Offsets come from reservations the journal hands out - in runs, and this keeps the unused remainder. Ownership is proven by reading - the lease rather than writing to it, and on a bounded schedule rather than - once per event: measured in the deployment that read costs ~0.5 ms, which at - forty concurrent streams is a whole core spent asking a question whose - answer almost never changes. - - Ownership can stop being ours two ways, and the bound is sized against the - slower one: - - * This process notices itself — its own heartbeat fails to renew, or the - user cancelled here. ``lease_lost`` is set, and the very next event is - fenced without leaving the loop at all, so this path got *faster*, not - looser. - * Another worker decided it — a cancel that landed on a different process, - or a recovery that re-claimed this run. That can only arrive through the - shared journal, so the read cannot be dropped, only bounded: at most one - per ``_CANCEL_VISIBILITY_SEC``, the same cadence the chunk loop already - polls cancellation on. Re-claiming requires the lease to have *expired*, - which means this worker went a full lease without renewing — it was not - emitting during that stretch either — so the window where a fenced worker - could still append is bounded by a quarter second and reached only by a - process that had stopped producing anyway. - - A terminal event is allocated on its own: it has to sit after every live - offset, including the ones a reservation left unused. That allocation is an - owned write, so a run always fences hard on the way out. - """ - - def __init__( - self, - journal: RunJournal, - run_id: str, - owner: str, - *, - lease_lost: asyncio.Event, - ): - self._journal = journal - self._run_id = run_id - self._owner = owner - self._next = 0 - self._remaining = 0 - self._lease_lost = lease_lost - self._proven_at = 0.0 - # The offset most recently handed out. Callers persist it with their - # checkpoints so a reconnecting follower knows where the run had got to. - self.last = 0 - - async def take(self, *, terminal: bool = False) -> int: - if terminal: - self._remaining = 0 - self.last = await asyncio.to_thread( - self._journal.allocate_event_offset, self._run_id, terminal=True - ) - return self.last - if self._remaining == 0: - # A reservation is an owned write: it fences on its own, and more - # strongly than a read does. - self._next = await asyncio.to_thread(self._reserve) - self._remaining = _OFFSET_RESERVATION - self._proven_at = time.monotonic() - else: - await self._prove_ownership() - offset = self._next - self._next += 1 - self._remaining -= 1 - self.last = offset - return offset - - async def _prove_ownership(self) -> None: - """Raise ``RunLeaseLost`` unless this worker still owns the run.""" - if self._lease_lost.is_set(): - # Our own heartbeat already decided this; the journal cannot - # disagree, so there is nothing to go and ask. - raise RunLeaseLost(self._run_id) - now = time.monotonic() - if now - self._proven_at < _CANCEL_VISIBILITY_SEC: - return - await asyncio.to_thread(self._journal.require_lease, self._run_id, self._owner) - self._proven_at = now - - async def emit(self, event: Dict[str, Any], *, terminal: bool = False) -> None: - """Allocate this event's offset and append it to the run's log.""" - await self.take(terminal=terminal) - await _xadd_event(self._run_id, self.last, event) - - def _reserve(self) -> int: - return self._journal.allocate_event_offset( - self._run_id, owner=self._owner, count=_OFFSET_RESERVATION - ) - - -def _json_safe(value: Any) -> Any: - return json.loads(json.dumps(value, ensure_ascii=False, default=str)) - - -async def _lease_heartbeat( - run_id: str, - owner: str, - worker: asyncio.Task, - lease_lost: asyncio.Event, -) -> None: - """Renew ownership and fence the local coroutine immediately on lease loss.""" - interval = _RUN_LEASE_HEARTBEAT_SEC - try: - run = await asyncio.to_thread(get_run, run_id) - payload = (run.request_payload or {}) if run else {} - if payload.get("api_key_id") or payload.get("agent_api_scope"): - # A revoke handled by another worker must also stop a silent model call. - interval = min(interval, 1.0) - while True: - await asyncio.sleep(interval) - # ``renew`` is a synchronous UPDATE on chat_runs. Run it off the - # event loop: whenever that row is locked by another transaction - # this call waits inside the driver, and waiting *on the loop* - # freezes every coroutine in the process — including the one - # holding the lock, which then can never commit and release it. - # A blocked heartbeat must only cost this run its lease, never the - # whole backend. - renewed = await asyncio.to_thread( - _journal().renew, - run_id, - owner=owner, - lease_seconds=_RUN_LEASE_SECONDS, - ) - if not renewed: - logger.warning("chat_run_lease_lost", run_id=run_id, owner=owner) - lease_lost.set() - worker.cancel() - return - except asyncio.CancelledError: - raise - except Exception as exc: # DB/driver failures must fail closed - logger.error( - "chat_run_lease_renew_failed", - run_id=run_id, - owner=owner, - error=str(exc), - exc_info=True, - ) - lease_lost.set() - worker.cancel() - - -def _update_run_status(run_id: str, **fields: Any) -> None: - """Single-shot UPDATE on chat_runs; isolated session so it doesn't pollute caller's txn.""" - if not fields: - return - if fields.get("status") in _TERMINAL_STATUSES: - fields.setdefault("writer_slot", None) - with SessionLocal() as db: - affected = ( - db.query(ChatRun) - .filter(ChatRun.run_id == run_id) - .update(fields, synchronize_session=False) - ) - if not affected: - logger.warning("chat_run_update_missing", run_id=run_id, fields=list(fields.keys())) - db.commit() - - -def _claim_run_execution(run_id: str) -> Optional[datetime]: - """Fence duplicate initial workers before acquiring the durable lease. - - Returns the instant the run was marked running, or None when another worker - got there first. The caller needs that instant for the first frame, and - handing it back keeps the send path free of a second read of the same row. - """ - - started_at = _utcnow() - with SessionLocal() as db: - affected = ( - db.query(ChatRun) - .filter(ChatRun.run_id == run_id, ChatRun.status == "pending") - .update( - {"status": "running", "started_at": started_at}, - synchronize_session=False, - ) - ) - db.commit() - return started_at if affected else None - - -def _finalize_run( - run_id: str, - *, - require_expired_lease: bool = False, - **fields: Any, -) -> bool: - """CAS variant of writing the terminal state: only updates while the run is still live. - - Prevents the race where "the reaper / cancel has already moved the run to a - terminal state, and the worker's late completed/failed overwrites it back" — - the side that loses the race abandons its write and returns False. - """ - fields.setdefault("lease_owner", None) - fields.setdefault("lease_expires_at", None) - if fields.get("error_message") and not fields.get("failure_reason"): - fields["failure_reason"] = fields["error_message"] - if fields.get("status") in _TERMINAL_STATUSES: - fields.setdefault("writer_slot", None) - with SessionLocal() as db: - query = db.query(ChatRun).filter( - ChatRun.run_id == run_id, - ChatRun.status.in_(_LIVE_STATUSES), - ) - if require_expired_lease: - from sqlalchemy import or_ - - now = _utcnow() - query = query.filter( - or_( - ChatRun.lease_owner.is_(None), - ChatRun.lease_expires_at.is_(None), - ChatRun.lease_expires_at <= now, - ) - ) - affected = query.update(fields, synchronize_session=False) - db.commit() - if not affected: - logger.info("chat_run_finalize_skipped", run_id=run_id, fields=list(fields.keys())) - return bool(affected) - - -def _request_run_stop( - run_id: str, - *, - status: RunStatus, - error_message: Optional[str] = None, - require_expired_lease: bool = False, -) -> bool: - """Write an external terminal verdict while retaining the writer fence.""" - - if status not in _TERMINAL_STATUSES: - raise ValueError(f"not a terminal status: {status}") - fields: Dict[str, Any] = {"status": status, "completed_at": _utcnow()} - if error_message: - fields["error_message"] = error_message[:1000] - fields["failure_reason"] = error_message[:1000] - with SessionLocal() as db: - query = db.query(ChatRun).filter( - ChatRun.run_id == run_id, - ChatRun.status.in_(_LIVE_STATUSES), - ) - if require_expired_lease: - from sqlalchemy import or_ - - now = _utcnow() - query = query.filter( - or_( - ChatRun.lease_owner.is_(None), - ChatRun.lease_expires_at.is_(None), - ChatRun.lease_expires_at <= now, - ) - ) - affected = query.update(fields, synchronize_session=False) - db.commit() - return bool(affected) - - -def _request_run_cancel(run_id: str) -> bool: - return _request_run_stop(run_id, status="cancelled") - - -def _acknowledge_terminal_writer(run_id: str) -> bool: - """Release the writer fence only after the owning worker has stopped writing.""" - - with SessionLocal() as db: - affected = ( - db.query(ChatRun) - .filter( - ChatRun.run_id == run_id, - ChatRun.status.in_(_TERMINAL_STATUSES), - ChatRun.writer_slot.isnot(None), - ) - .update({"writer_slot": None}, synchronize_session=False) - ) - db.commit() - return bool(affected) - - -def _acknowledge_never_started_terminal_writer(run_id: str) -> bool: - """Release a terminal fence only when no worker ever claimed the run. - - A duplicate worker can lose the pending-to-running claim while the real - worker is active, then observe a concurrent cancellation. It must not - release the real worker's fence in that case. ``started_at IS NULL`` is - the durable proof that there is no owning worker to wait for. - """ - - with SessionLocal() as db: - affected = ( - db.query(ChatRun) - .filter( - ChatRun.run_id == run_id, - ChatRun.status.in_(_TERMINAL_STATUSES), - ChatRun.writer_slot.isnot(None), - ChatRun.started_at.is_(None), - ) - .update({"writer_slot": None}, synchronize_session=False) - ) - db.commit() - return bool(affected) - - -def _create_run_record( - *, - chat_id: str, - user_id: str, - request_payload: Dict[str, Any], - recovery_snapshot: Optional[Dict[str, Any]] = None, -) -> ChatRun: - """Durably accept a run before any worker task can be spawned.""" - run_id = f"run_{uuid.uuid4().hex[:16]}" - message_id = f"msg_{uuid.uuid4().hex[:16]}" - return _journal().accept( - run_id=run_id, - message_id=message_id, - chat_id=chat_id, - user_id=user_id, - request_payload=_json_safe(request_payload), - recovery_snapshot=_json_safe(recovery_snapshot or {}), - ) - - -def _register_run_task(run_id: str, coro: Awaitable[None], *, name: str) -> None: - """Spawn the worker coroutine, register it in _active_runs, auto-cleanup on done.""" - task = asyncio.create_task(coro, name=name) - _active_runs[run_id] = task - task.add_done_callback(lambda _t: _active_runs.pop(run_id, None)) - - -async def _write_terminal_to_stream( - run_id: str, - *, - chat_id: str, - error_text: str, - cancelled: bool = False, -) -> None: - """Write an error event + terminal marker + EXPIRE. - - Used when the worker isn't around to emit them itself (cross-process cancel, - orphan recovery). Failure is logged but not raised. - """ - err_event: Dict[str, Any] = { - "type": "error", - "error": error_text, - "delta": error_text, - "chat_id": chat_id, - } - term_event: Dict[str, Any] = {"type": _TERMINAL_TYPE, "chat_id": chat_id} - if cancelled: - err_event["_cancelled"] = True - term_event["_cancelled"] = True - try: - journal = _journal() - error_offset = journal.allocate_event_offset(run_id, terminal=True) - await _xadd_event(run_id, error_offset, err_event) - terminal_offset = journal.allocate_event_offset(run_id, terminal=True) - await _xadd_event(run_id, terminal_offset, term_event) - await _expire_stream(run_id) - except Exception as exc: - logger.warning("chat_run_terminal_write_failed", run_id=run_id, error=str(exc)) - - -# ─── Entry point: start_run (chat mode) ───────────────────────────────── - - -async def start_run( - *, - accepted_run: Optional[ChatRun] = None, - chat_id: str, - user_id: str, - session_messages: List[Dict[str, Any]], - effective_user_message: str, - raw_user_message: str, - context: Dict[str, Any], - request_payload: Optional[Dict[str, Any]] = None, - model_name: Optional[str] = None, -) -> ChatRun: - """Launch a ChatRun that ChatSequencer has already durably accepted. - - Acceptance (user message + pending run) must commit before this function is - called. Keeping launch separate makes a process crash between the two - recoverable by the Run Journal in #64 instead of losing the request. - """ - def persist_launch(): - recovery_snapshot = _json_safe( - { - "kind": "chat", - "worker_args": { - "session_messages": session_messages, - "effective_user_message": effective_user_message, - "raw_user_message": raw_user_message, - "context": context, - "model_name": model_name, - }, - } - ) - if accepted_run is None: - run = _create_run_record( - chat_id=chat_id, - user_id=user_id, - request_payload=dict(request_payload or {}), - recovery_snapshot=recovery_snapshot, - ) - else: - run = accepted_run - if run.chat_id != chat_id or run.user_id != user_id or run.status != "pending": - raise ValueError("accepted_run does not match the launch request") - with SessionLocal() as db: - durable_run = db.get(ChatRun, run.run_id) - if durable_run is None or durable_run.status != "pending": - raise ValueError("accepted_run is no longer pending") - durable_run.recovery_snapshot = recovery_snapshot - durable_run.snapshot_version = max(int(durable_run.snapshot_version or 0), 1) - durable_run.run_phase = durable_run.run_phase or "accepted" - durable_run.last_operation_safety = durable_run.last_operation_safety or "replayable" - durable_run.updated_at = _utcnow() - db.commit() - return run - - run = await asyncio.to_thread(persist_launch) - journal_owner = _new_worker_owner(run.run_id) - _register_run_task( - run.run_id, - _run_workflow( - run_id=run.run_id, - chat_id=chat_id, - user_id=user_id, - message_id=run.message_id, - assistant_chat_seq=run.assistant_chat_seq, - session_messages=session_messages, - effective_user_message=effective_user_message, - raw_user_message=raw_user_message, - context=context, - model_name=model_name, - journal_owner=journal_owner, - ), - name=f"chat_run:{run.run_id}", - ) - logger.info("chat_run_started", run_id=run.run_id, chat_id=chat_id, user_id=user_id) - return run - - -def _persisted_history(chat_service: ChatService, chat_id: str) -> List[Dict[str, Any]]: - """The model-facing history a successor or recovery run starts from. - - Read back from the rows already committed in this transaction — the step - record included — never rebuilt from text the source run held in memory. - """ - from core.services.compaction_service import _load_history - - return _json_safe(_load_history(chat_service, chat_id, repair=False)) - - -def _commit_queued_handoff_in_session( - db, - *, - source_run_id: str, - chat_id: str, - user_id: str, - context: Dict[str, Any], - model_name: Optional[str], -) -> Optional[Dict[str, Any]]: - """Atomically consume one terminal queued item and stage its ChatRun. - - This includes a steer accepted after the source run crossed its final safe - model/tool boundary. Such an instruction can no longer enter the source - ReAct loop, so completion hands it to a successor instead of stranding it. - - The successor's history is read back from the rows this transaction just - committed — the step record included — never rebuilt from the text the - source run happened to hold in memory. - """ - from core.db.models.chat import reserve_chat_sequences - from core.services.steer_queue import SteerQueue - - source_run = db.get(ChatRun, source_run_id) - if source_run is None: - return None - root_run_id = str(context.get("handoff_root_run_id") or source_run_id) - handoff = SteerQueue.consume_next_handoff_in_session( - db, - source_run_id=source_run_id, - root_run_id=root_run_id, - chat_id=chat_id, - applied_operation_seq=int(source_run.operation_seq or 0) + 1, - ) - if handoff is None or not handoff.applied_run_id: - return None - - first_seq = reserve_chat_sequences( - db, - chat_id, - count=2, - owner_user_id=user_id, - ) - next_run_id = handoff.applied_run_id - next_message_id = ( - f"msg_{uuid.uuid5(uuid.NAMESPACE_URL, f'{handoff.queue_id}:assistant').hex[:16]}" - ) - queued_user_message_id = ( - f"msg_{uuid.uuid5(uuid.NAMESPACE_URL, f'{handoff.queue_id}:user').hex[:16]}" - ) - next_context = _json_safe(dict(context or {})) - next_context.pop("journal_owner", None) - next_context.update( - { - "run_id": next_run_id, - "message_id": next_message_id, - "chat_id": chat_id, - "user_id": user_id, - "handoff_queue_id": handoff.queue_id, - "handoff_delivery_mode": handoff.delivery_mode, - "handoff_root_run_id": root_run_id, - } - ) - next_session_messages = _persisted_history(ChatService(db), chat_id) - next_session_messages.append({"role": "user", "content": handoff.message}) - worker_args = { - "session_messages": next_session_messages, - "effective_user_message": handoff.message, - "raw_user_message": handoff.message, - "context": next_context, - "model_name": model_name, - } - request_payload = dict(source_run.request_payload or {}) - request_payload.update( - { - "message": handoff.message, - "chat_id": chat_id, - "attachments": [], - "handoff": { - "queue_id": handoff.queue_id, - "steer_id": handoff.steer_id, - "delivery_mode": handoff.delivery_mode, - "source_run_id": source_run_id, - }, - } - ) - RunJournal.accept_in_session( - db, - run_id=next_run_id, - message_id=next_message_id, - chat_id=chat_id, - user_id=user_id, - request_payload=_json_safe(request_payload), - recovery_snapshot={"kind": "chat", "worker_args": _json_safe(worker_args)}, - user_message_id=queued_user_message_id, - user_chat_seq=first_seq, - assistant_chat_seq=first_seq + 1, - writer_slot="main", - ) - ChatService(db).upsert_message( - chat_id=chat_id, - role="user", - content=handoff.message, - model=model_name, - message_id=queued_user_message_id, - chat_seq=first_seq, - extra_data={ - "queued_handoff": True, - "steer_queue_id": handoff.queue_id, - "steer_id": handoff.steer_id, - "steer_seq": handoff.steer_seq, - "delivery_mode": handoff.delivery_mode, - "source_run_id": source_run_id, - "run_id": next_run_id, - }, - commit=False, - ) - db.add( - inflight.new_assistant_row( - message_id=next_message_id, - chat_id=chat_id, - chat_seq=first_seq + 1, - run_id=next_run_id, - model=model_name, - created_at=_utcnow(), - ) - ) - db.flush() - return { - "run_id": next_run_id, - "message_id": next_message_id, - "user_message_id": queued_user_message_id, - "assistant_chat_seq": first_seq + 1, - "queue_id": handoff.queue_id, - "steer_id": handoff.steer_id, - "steer_seq": handoff.steer_seq, - "delivery_mode": handoff.delivery_mode, - **worker_args, - } - - -def _schedule_queued_handoff( - handoff: Dict[str, Any], - *, - chat_id: str, - user_id: str, - task_prefix: str = "chat_run_handoff", -) -> str: - """Start a committed handoff; pending DB state remains recovery-safe on failure.""" - next_run_id = str(handoff["run_id"]) - next_owner = _new_worker_owner(next_run_id) - _register_run_task( - next_run_id, - _run_workflow( - run_id=next_run_id, - chat_id=chat_id, - user_id=user_id, - message_id=str(handoff["message_id"]), - assistant_chat_seq=( - int(handoff["assistant_chat_seq"]) - if handoff.get("assistant_chat_seq") is not None - else None - ), - session_messages=list(handoff.get("session_messages") or []), - effective_user_message=str(handoff.get("effective_user_message") or ""), - raw_user_message=str(handoff.get("raw_user_message") or ""), - context=dict(handoff.get("context") or {}), - model_name=(str(handoff["model_name"]) if handoff.get("model_name") else None), - journal_owner=next_owner, - ), - name=f"{task_prefix}:{next_run_id}", - ) - return next_run_id - - -def _queued_run_started_event( - source_run_id: str, - chat_id: str, - handoff: Dict[str, Any], -) -> Dict[str, Any]: - return { - "type": "queued_run_started", - "chat_id": chat_id, - "source_run_id": source_run_id, - "run_id": str(handoff["run_id"]), - "message_id": handoff["message_id"], - "user_message_id": handoff["user_message_id"], - "message": handoff["raw_user_message"], - "queue_id": handoff["queue_id"], - "steer_id": handoff["steer_id"], - "steer_seq": handoff["steer_seq"], - "delivery_mode": handoff["delivery_mode"], - } - - -async def _write_queued_run_started_projection( - source_run_id: str, - *, - chat_id: str, - handoff: Dict[str, Any], -) -> None: - """Best-effort event projection; the queue API is the durable backfill seam.""" - try: - offset = _journal().allocate_event_offset(source_run_id, terminal=True) - await _xadd_event( - source_run_id, - offset, - _queued_run_started_event(source_run_id, chat_id, handoff), - ) - except Exception as exc: # Redis projection must not undo the committed handoff - logger.warning( - "chat_run_handoff_projection_failed", - run_id=source_run_id, - child_run_id=handoff.get("run_id"), - error=str(exc), - ) - - -def get_successor_run(run_id: str, *, user_id: str) -> Optional[ChatRun]: - """Read the successor committed by the durable steering handoff.""" - from core.db.models import ChatSteerQueueItem - - with SessionLocal() as db: - source = db.get(ChatRun, run_id) - if source is None or source.user_id != user_id: - raise ChatRunPermissionDenied("run does not belong to user") - successor_id = ( - db.query(ChatSteerQueueItem.applied_run_id) - .filter( - ChatSteerQueueItem.applied_source_run_id == run_id, - ChatSteerQueueItem.applied_run_id != run_id, - ChatSteerQueueItem.status == "applied", - ChatSteerQueueItem.chat_id == source.chat_id, - ChatSteerQueueItem.user_id == user_id, - ) - .scalar() - ) - return get_run(successor_id) if successor_id else None - - -async def wait_run(run_id: str) -> ChatRun: - """Wait for a locally launched run and return its durable terminal row. - - Shielding prevents a disconnected non-stream HTTP client from cancelling - the detached worker. Explicit ``cancel_run`` still targets the registered - task and keeps the database writer fence until that task has stopped. - """ - - task = _active_runs.get(run_id) - if task is not None: - try: - await asyncio.shield(task) - except asyncio.CancelledError: - if not task.done(): - # The waiter (for example a disconnected HTTP request) was - # cancelled; shielding keeps the worker alive, and the caller - # must still observe its own cancellation. - raise - # Explicit run cancellation stopped the worker; the durable row - # below is the authoritative outcome. - run = get_run(run_id) - if run is None: - raise ChatRunNotFound(f"chat run {run_id} not found") - if run.status in _LIVE_STATUSES: - raise RuntimeError(f"chat run {run_id} lost its local worker before completion") - return run - - -# ─── Redis Stream writes ─────────────────────────────────────────────── - - -async def _xadd_event(run_id: str, offset: int, event: Dict[str, Any]) -> None: - """Stamp an SSE event with its offset and wall clock, append it to the run's log. - - ``server_ts`` is when the server produced the event. A client that joins - late — after a reconnect, a session switch, or from a second device — - replays the log and reads these stamps, so its "waiting since" clock - reflects the run's real timeline instead of the moment this browser - happened to receive them. It deliberately has no underscore prefix: that - prefix marks transport-internal keys, which ``follow_run_as_sse`` strips - before the frame goes out, and this one is meant for the client. - """ - payload = {**event, "_offset": offset, "server_ts": int(time.time() * 1000)} - try: - await _events().append(run_id, payload) - except Exception as exc: - logger.warning("chat_run_xadd_failed", run_id=run_id, error=str(exc)) - - -async def _expire_stream(run_id: str) -> None: - try: - await _events().expire(run_id, _STREAM_TTL_SECONDS) - except Exception as exc: - logger.warning("chat_run_expire_failed", run_id=run_id, error=str(exc)) - - -async def _reset_stream_projection(run_id: str) -> None: - """Drop a crashed worker's partial projection before replaying its run.""" - try: - await _events().clear(run_id) - except Exception as exc: - logger.warning("chat_run_projection_reset_failed", run_id=run_id, error=str(exc)) - - -# ─── Background worker ───────────────────────────────────────────────── - - -def _cancelled_by_user(run_id: str) -> bool: - """租约被清掉有两种可能：别的 worker 接管了恢复，或者用户按了停止。 - - cancel_run 会先把行写成终态 cancelled 再清租约，之后不会再有人接手这一轮 —— - 被围栏的 worker 若也闭嘴，用户已经看到的半截回答就没人落库了。 - """ - try: - current = get_run(run_id) - except Exception: # noqa: BLE001 - 读不到就按"别人接管"处理，宁可不写 - return False - return current is not None and current.status == "cancelled" - - -async def _run_workflow( - *, - run_id: str, - chat_id: str, - user_id: str, - message_id: str, - session_messages: List[Dict[str, Any]], - effective_user_message: str, - raw_user_message: str, - context: Dict[str, Any], - model_name: Optional[str], - assistant_chat_seq: Optional[int] = None, - journal_owner: Optional[str] = None, - recovering: bool = False, -) -> None: - """Consume astream_chat_workflow, forward chunks to the Redis Stream, persist at the end. - - Every chunk is written to ``jx:chat:run:{run_id}:events`` via ``_emit``. - The SSE event format stays fully identical to the existing protocol in - chats.py:889-1087; the frontend needs no changes to its chunk parsing — - only the new ``run_started`` event type is added. - """ - from core.chat.context import now_iso, resolve_user_facing_error - from core.chat.tool_log import ( - SegmentRecorder, - attach_subagent_step, - build_thinking_event, - build_tool_call_delta_event, - build_tool_call_event, - build_tool_call_start_event, - build_tool_result_event, - build_user_question_event, - build_user_question_resolved_event, - ) - from core.chat.display_bounds import bound_result_for_display, bound_result_for_history - from core.services.artifact_service import persist_artifacts as _persist_artifacts - - # 本轮的墙钟起点。认领这一轮的 worker 手里就有它；只有接管别人未完成的 - # 那一轮时才需要回库里取（见 _run_started_at_ms）。 - claimed_at: Optional[datetime] = None - initial_claimed = False - if not recovering: - claimed_at = _claim_run_execution(run_id) - initial_claimed = claimed_at is not None - if not initial_claimed: - _acknowledge_never_started_terminal_writer(run_id) - return - - owner = journal_owner or _new_worker_owner(run_id) - journal = _journal() - if not journal.claim(run_id, owner=owner, lease_seconds=_RUN_LEASE_SECONDS): - logger.info("chat_run_claim_skipped", run_id=run_id, owner=owner) - if initial_claimed: - _acknowledge_terminal_writer(run_id) - return - try: - journal.append_operation( - run_id, - owner=owner, - operation_type="worker_started", - phase="pre_model", - safety="replayable", - ) - except RunLeaseLost: - logger.warning("chat_run_worker_fenced_before_start", run_id=run_id, owner=owner) - return - worker_task = asyncio.current_task() - if worker_task is None: # pragma: no cover - asyncio always owns this coroutine - raise RuntimeError("chat run worker has no asyncio task") - lease_lost = asyncio.Event() - lease_task = asyncio.create_task( - _lease_heartbeat(run_id, owner, worker_task, lease_lost), - name=f"chat_run_lease:{run_id}", - ) - # 本轮回答总耗时起点 —— 持久化进 extra_data.duration_ms，历史加载后「用时」不再消失 - _run_started_monotonic = time.monotonic() - - offsets = _EventOffsets(journal, run_id, owner, lease_lost=lease_lost) - - _emit = offsets.emit - - async def _pause_for_tool_outcome(exc: BaseException) -> bool: - logger.warning("chat_run_tool_outcome_unknown", run_id=run_id, effect_id=str(exc)) - if not journal.needs_attention( - run_id, - owner=owner, - reason=f"tool outcome pending policy recovery: {exc}", - ): - current = get_run(run_id) - if current is None or current.status != "needs_attention": - logger.warning("chat_run_tool_unknown_cas_lost", run_id=run_id, owner=owner) - return False - await _emit( - { - "type": "error", - "error": "工具调用结果无法安全确定，任务已暂停等待处理", - "delta": "工具调用结果无法安全确定，任务已暂停等待处理", - "chat_id": chat_id, - }, - terminal=True, - ) - await _emit({"type": _TERMINAL_TYPE, "chat_id": chat_id}, terminal=True) - return True - - async def _pause_for_hook(exc: HookPaused) -> bool: - logger.info("chat_run_hook_paused", run_id=run_id, reason=str(exc)) - if not journal.needs_attention( - run_id, - owner=owner, - reason=f"hook requested pause: {exc}", - ): - current = get_run(run_id) - if current is None or current.status != "needs_attention": - logger.warning("chat_run_hook_pause_cas_lost", run_id=run_id, owner=owner) - return False - await _emit( - { - "type": "error", - "error": "任务已按执行策略暂停，等待后续处理", - "delta": "任务已按执行策略暂停，等待后续处理", - "chat_id": chat_id, - }, - terminal=True, - ) - await _emit({"type": _TERMINAL_TYPE, "chat_id": chat_id}, terminal=True) - return True - - from core.llm import workspace as _workspace_mod - - full_response = "" - # A steer starts a new visible assistant segment in the same backend run. - # Each segment gets its own message id so persisted history stays in the - # same chronological order the user saw while streaming. - current_message_id = message_id - current_chat_seq = assistant_chat_seq - latest_user_message = raw_user_message - metadata: Dict[str, Any] = {} - compaction_budget_inputs: Dict[str, Any] = {} - tool_calls_log: list = [] - _workspace_mod.init_state() - # 结构化 reasoning 通道（deepseek 系 reasoning_content/reasoning）的思考增量。 - # 落进独立的 thinking 列，不再拼进 full_response —— 拼进正文会把正文顶出 - # content 的 10 万字上限，截断切掉闭合标签后整段思考会漏成正文。 - # 每个思考块记录它出现时的正文偏移，历史重建按此原位还原。 - _thinking_parts: List[str] = [] - _thinking_log: List[Dict[str, Any]] = [] - # 模型回放用的规范记录：每个模型响应、每个工具结果按发生顺序各记一条 - # （core/llm/model_steps.py）。上面的正文 / 思考 / 工具卡是给界面看的投影， - # 下一轮发给模型的历史从这一份恢复。 - _model_steps: List[Dict[str, Any]] = [] - # 正文 / 思考 / 工具卡片的先后，在它们产生的那一刻就记下来，落进 - # metadata.segments；刷新后照着渲染，不做任何反推。 - _segments = SegmentRecorder() - # 本轮已开的思考块；跨轮（工具调用边界）置 None 强制另起一块。不设边界的话， - # 正文一旦出现，后面每一轮的思考都会并进同一块里无限膨胀。 - _round_block: Optional[Dict[str, Any]] = None - next_cancel_poll = 0.0 - - def _flush_thinking() -> None: - nonlocal _round_block - if not _thinking_parts: - return - block = "".join(_thinking_parts) - _thinking_parts.clear() - # 结构化 reasoning 的尾部增量可能在正文已开始后才到达（正文首 token - # 已出、思考收尾的"。"后到）。实时侧把它并回前一个思考块 - # （appendThinkingContentBeforeTrailingText），落库遵循同一规则—— - # 否则思考块会把正文句子从中间切开，刷新后与实时展示不一致。 - if _round_block is not None: - _round_block["content"] += block - return - _round_block = {"content": block} - _thinking_log.append(_round_block) - _segments.add_thinking(len(_thinking_log) - 1) - - def _thinking_payload() -> Optional[List[Dict[str, Any]]]: - blocks = [b for b in _thinking_log if b.get("content")] - return blocks or None - - def _model_steps_payload() -> Optional[List[Dict[str, Any]]]: - return list(_model_steps) or None - - def _turn_extra(**more: Any) -> Dict[str, Any]: - return { - "timestamp": now_iso(), - "message_id": current_message_id, - "run_id": run_id, - "is_markdown": looks_markdown(full_response), - **more, - } - - # In-flight persistence. The assistant row was created at admission and is - # refreshed with this turn's display state — right away at tool boundaries, - # and by the checkpoint task every couple of seconds while text streams — - # so the server holds the turn at all times, not only at the end. Each - # refresh is one lease-guarded UPDATE off the event loop; the lock keeps a - # boundary flush from racing the periodic one with an older snapshot. - _flush_lock = asyncio.Lock() - _flushed_snapshot_key = None - _INFLIGHT_INTERVAL_S = 2.0 - checkpoint_task: Optional[asyncio.Task] = None - - def _write_inflight_row(snapshot: Dict[str, Any]) -> bool: - with SessionLocal() as db: - owned = ChatService(db).refresh_streaming_message( - run_id=run_id, - owner=owner, - message_id=snapshot["message_id"], - content=snapshot["content"], - thinking=snapshot["thinking"], - tool_calls=snapshot["tool_calls"], - model_steps=snapshot["model_steps"], - extra_data={ - inflight.IN_FLIGHT_KEY: inflight.mark( - run_id, "streaming", event_offset=snapshot["event_offset"] - ), - "message_id": snapshot["message_id"], - "run_id": run_id, - "is_markdown": looks_markdown(snapshot["content"]), - "segments": snapshot["segments"], - }, - ) - db.commit() - return owned - - async def _flush_inflight() -> bool: - nonlocal _flushed_snapshot_key - _flush_thinking() - if not (full_response or tool_calls_log or _model_steps or _thinking_log): - return True - async with _flush_lock: - # Canonical steps can advance without a user-visible SSE event. - snapshot_key = (current_message_id, offsets.last, len(_model_steps)) - if snapshot_key == _flushed_snapshot_key: - return True - # Tool results go in at history size; the read path bounds them the - # same way, and the final write carries the full payload. - snapshot = _json_safe( - { - "content": full_response, - "thinking": _thinking_payload(), - "model_steps": _model_steps_payload(), - "tool_calls": [ - ( - {**tc, "result": bound_result_for_history(tc["result"])[0]} - if "result" in tc - else tc - ) - for tc in tool_calls_log - ] - or None, - "segments": _segments.snapshot(), - "message_id": current_message_id, - "event_offset": offsets.last, - } - ) - try: - if not await asyncio.to_thread(_write_inflight_row, snapshot): - logger.warning("chat_run_inflight_fenced", run_id=run_id, owner=owner) - return False - except Exception: # noqa: BLE001 - a missed checkpoint must not fail the run - logger.warning("chat_run_inflight_persist_failed", run_id=run_id, exc_info=True) - return False # Keep dirty so the periodic checkpoint retries this snapshot. - _flushed_snapshot_key = snapshot_key - return True - - async def _checkpoint_loop() -> None: - while True: - await asyncio.sleep(_INFLIGHT_INTERVAL_S) - await _flush_inflight() - - def _commit_cancelled_partial() -> None: - """把用户按下停止那一刻已产出的正文与工具卡片落库。 - - cancel_run 先写终态并清租约，之后 journal.complete 的 CAS 必然失败， - worker 再也借不到 commit_effect 提交本轮输出 —— 所以这里直接写库。 - message_id 固定，用 upsert 保证重复取消 / 恢复不会留下两条。 - """ - _flush_thinking() - if not (full_response or tool_calls_log or _model_steps or _thinking_log): - return - # 停下来时还没回结果的工具：不标记的话前端按"缺 status 即成功"渲染， - # 刷新后一张没跑完的卡片会显示成执行成功。模型侧的记录同样要配平： - # 悬空的调用补一条"被中断"的结果，下一轮模型看到的是事实而不是半步。 - for _tc in tool_calls_log: - if "result" not in _tc and not _tc.get("status"): - _tc["status"] = "interrupted" - close_dangling_calls(_model_steps) - _ws_pinned = _workspace_mod.get_pinned() - _extra = { - "timestamp": now_iso(), - "is_markdown": bool( - "\n" in full_response or "```" in full_response or "**" in full_response - ), - "message_id": current_message_id, - "run_id": run_id, - "cancelled": True, - "artifacts": _ws_pinned, - "workspace_files": _workspace_mod.get_pinned_file_ids(), - "duration_ms": int((time.monotonic() - _run_started_monotonic) * 1000), - "segments": _segments.payload(), - } - if context.get("model_provider_id"): - _extra["model_provider_id"] = context.get("model_provider_id") - from core.services.project_scope import project_scope_from_context - - with SessionLocal() as db: - chat_service = ChatService(db) - chat_service.upsert_message( - chat_id=chat_id, - role="assistant", - content=full_response, - model=model_name, - thinking=_thinking_payload(), - tool_calls=tool_calls_log if tool_calls_log else None, - model_steps=_model_steps_payload(), - message_id=current_message_id, - chat_seq=current_chat_seq, - extra_data=_extra, - commit=False, - ) - _persist_artifacts( - db, - user_id, - chat_id, - _ws_pinned, - scope=project_scope_from_context(context), - commit=False, - ) - db.commit() - - def _persist_cancelled_partial() -> None: - try: - _commit_cancelled_partial() - except Exception: # noqa: BLE001 - 落库失败不能吞掉取消收尾 - logger.warning( - "chat_run_cancelled_partial_persist_failed", - run_id=run_id, - exc_info=True, - ) - - try: - if recovering: - await _reset_stream_projection(run_id) - # First frame: run_started — carries run_id / message_id; the frontend uses these to resume / cancel - await _emit( - { - "type": "run_started", - "run_id": run_id, - "message_id": message_id, - "chat_id": chat_id, - "started_at": _epoch_ms(claimed_at) if claimed_at else _run_started_at_ms(run_id), - } - ) - if recovering: - await _emit( - { - "type": "content_replace", - "content": "", - "reason": "run_recovered", - "chat_id": chat_id, - } - ) - - # Compaction notice (mirrors Codex's post-compaction Warning): after the - # previous turn's stream closed, a new compaction checkpoint was written - # in the background → notify the user once in this turn's first frame. - # Failures are silent and must never affect the main conversation. - try: - from core.services.compaction_service import ( - pop_compaction_notice, - ) - - from core.services.context_usage_projection import get_chat_context_state - - with SessionLocal() as _cn_db: - _cn_service = ChatService(_cn_db) - _notify_compaction = pop_compaction_notice(_cn_service, chat_id) - _compaction_state = ( - get_chat_context_state(_cn_service, chat_id)["context_compaction"] - if _notify_compaction - else None - ) - if _notify_compaction: - await _emit( - { - "type": "compaction_notice", - "chat_id": chat_id, - "context_compaction": _compaction_state, - } - ) - except Exception as _cn_exc: # noqa: BLE001 - logger.debug("compaction_notice_failed", chat_id=chat_id, error=str(_cn_exc)) - - - # Evidence-plane join keys (GCE ticket 04). Injected here rather than at - # every context construction site: this is the one place that owns both - # the run and its pre-allocated assistant message id. - context["run_id"] = run_id - context["message_id"] = message_id - context["journal_owner"] = owner - - # Stamp the message id onto tool logging for this run. The plumbing - # already existed but had no caller, so every tool call was written with - # an empty message_id — which silently broke the join the evidence plane - # depends on, and left every episode with no tool sequence to learn from. - try: - from core.services.log_service import set_current_message_id - - set_current_message_id(message_id) - except Exception: # pragma: no cover - logging must never fail a run - pass - - checkpoint_task = asyncio.create_task(_checkpoint_loop()) - async for chunk in _aiter_with_inactivity_timeout( - stream_with_compaction( - astream_chat_workflow, - raw_user_message=raw_user_message, - session_messages=session_messages, - user_message=effective_user_message, - context=context, - ), - _INACTIVITY_TIMEOUT_SEC, - is_activity=lambda item: _is_chat_run_activity(item, chat_id), - first_response_timeout=_FIRST_RESPONSE_TIMEOUT_SEC, - ): - now = time.monotonic() - if now >= next_cancel_poll: - next_cancel_poll = now + _CANCEL_VISIBILITY_SEC - if is_run_cancelled(run_id): - raise asyncio.CancelledError - chunk_type = chunk.get("type") - - if chunk_type == "model_dispatch": - journal.append_operation( - run_id, - owner=owner, - operation_type="model_dispatch", - phase="model_inflight", - safety="replayable", - ) - - elif chunk_type == "thinking": - _thinking_evt = build_thinking_event(chunk, chat_id) - # 只累积真实思考增量；进度提示（message）与 structured_reasoning - # 协议标记不落库 - if _thinking_evt.get("delta"): - _thinking_parts.append(str(_thinking_evt["delta"])) - await _emit(_thinking_evt) - - elif chunk_type in {"ai_message", "content"}: - delta = chunk.get("delta", "") - if delta: - _flush_thinking() - full_response += delta - _segments.add_text(delta) - await _emit( - { - "type": "content", - "event": "ai_message", - "delta": delta, - "chat_id": chat_id, - } - ) - - elif chunk_type == "content_replace": - # Ontology review runs after the draft has streamed. If the - # committee revised it, replace the visible/persisted answer - # atomically instead of appending a second full answer. - replacement = str(chunk.get("content") or "") - _thinking_parts.clear() - full_response = replacement - _round_block = None - _thinking_log.clear() - replace_final_text(_model_steps, replacement) - # 整体替换后，先前记下的段落描述的是旧草稿，已无意义——重记。 - _segments.reset(preserve_compaction=True) - _segments.add_text(replacement) - await _emit( - { - "type": "content_replace", - "content": replacement, - "reason": chunk.get("reason"), - "chat_id": chat_id, - } - ) - await _flush_inflight() - - elif chunk_type == "steer_applied": - _flush_thinking() - steer_id = str(chunk.get("steer_id") or "")[:64] - steer_queue_id = str(chunk.get("queue_id") or "")[:64] - steer_claim_owner = str(chunk.get("claim_owner") or "")[:160] - steer_seq = int(chunk.get("steer_seq") or 0) - steer_message = str(chunk.get("message") or "").strip() - steer_message_id = ( - f"msg_{uuid.uuid5(uuid.NAMESPACE_URL, f'{run_id}:{steer_id}').hex[:16]}" - if steer_id - else f"msg_{uuid.uuid4().hex[:16]}" - ) - next_assistant_message_id = ( - f"msg_{uuid.uuid5(uuid.NAMESPACE_URL, f'{run_id}:{steer_id}:assistant').hex[:16]}" - if steer_id - else f"msg_{uuid.uuid4().hex[:16]}" - ) - had_assistant_output = bool(full_response or tool_calls_log) - if steer_message: - # Close the visible assistant segment before inserting the - # user's steer. Persisting in this order is what keeps a - # refresh from moving every mid-run user message above the - # whole assistant response. - def _commit_steer(db) -> None: - if steer_queue_id and steer_claim_owner: - from core.services.steer_queue import SteerQueue - - owned_run = db.get(ChatRun, run_id) - SteerQueue.mark_applied_in_session( - db, - queue_id=steer_queue_id, - claim_owner=steer_claim_owner, - applied_run_id=run_id, - applied_operation_seq=int(owned_run.operation_seq or 0) + 1, - ) - else: - owned_run = db.get(ChatRun, run_id) - chat_service = ChatService(db) - if had_assistant_output: - chat_service.upsert_message( - chat_id=chat_id, - role="assistant", - content=full_response, - model=model_name, - thinking=_thinking_payload(), - tool_calls=tool_calls_log if tool_calls_log else None, - model_steps=_model_steps_payload(), - message_id=current_message_id, - chat_seq=current_chat_seq, - extra_data={ - "timestamp": now_iso(), - "is_markdown": bool( - "\n" in full_response - or "```" in full_response - or "**" in full_response - ), - "message_id": current_message_id, - "run_id": run_id, - "steer_segment": True, - "duration_ms": int( - (time.monotonic() - _run_started_monotonic) * 1000 - ), - "segments": _segments.payload(), - }, - commit=False, - ) - # Persist once the middleware has actually injected the - # instruction. A deterministic id makes replay safe. - chat_service.upsert_message( - chat_id=chat_id, - role="user", - content=steer_message, - message_id=steer_message_id, - extra_data={ - "timestamp": now_iso(), - "steer": True, - "run_id": run_id, - "steer_id": steer_id, - "steer_queue_id": steer_queue_id or None, - "steer_seq": steer_seq or None, - }, - commit=False, - ) - # The next segment's row exists before its first token, - # like every other admitted turn. - db.add( - inflight.new_assistant_row( - message_id=next_assistant_message_id, - chat_id=chat_id, - chat_seq=None, - run_id=run_id, - model=model_name, - created_at=_utcnow(), - ) - ) - # The durable queue acknowledgement and the exact - # post-steer prompt must share one transaction. If this - # worker disappears before the next model snapshot, a - # replacement worker resumes from this context instead - # of losing an already-applied instruction. - post_steer_messages = _persisted_history(chat_service, chat_id) - post_steer_context = _json_safe(dict(context or {})) - post_steer_context.pop("journal_owner", None) - post_steer_context.update( - { - "run_id": run_id, - "message_id": next_assistant_message_id, - "chat_id": chat_id, - "user_id": user_id, - } - ) - owned_run.recovery_snapshot = { - "kind": "chat", - "worker_args": { - "session_messages": post_steer_messages, - "effective_user_message": steer_message, - "raw_user_message": steer_message, - "context": post_steer_context, - "model_name": model_name, - }, - } - owned_run.snapshot_version = int(owned_run.snapshot_version or 0) + 1 - - journal.append_operation( - run_id, - owner=owner, - operation_type="steer_messages_committed", - phase="steer_committed", - safety="side_effect_committed", - payload={ - "steer_id": steer_id, - "queue_id": steer_queue_id or None, - "steer_seq": steer_seq or None, - "message_id": steer_message_id, - "previous_assistant_message_id": ( - current_message_id if had_assistant_output else None - ), - }, - commit_effect=_commit_steer, - ) - latest_user_message = steer_message - await _emit( - { - "type": "steer_applied", - "chat_id": chat_id, - "run_id": run_id, - "steer_id": steer_id, - "queue_id": steer_queue_id or None, - "steer_seq": steer_seq or None, - "message": steer_message, - "message_id": steer_message_id, - "previous_assistant_message_id": ( - current_message_id if had_assistant_output else None - ), - "next_assistant_message_id": next_assistant_message_id, - "had_assistant_output": had_assistant_output, - } - ) - current_message_id = next_assistant_message_id - current_chat_seq = None - # The workflow reads this dict again when it emits the final - # evolution/memory settlement marker, so keep that marker bound - # to the post-steer assistant segment too. - context["message_id"] = current_message_id - full_response = "" - tool_calls_log = [] - _model_steps = [] - _thinking_parts.clear() - _round_block = None - _thinking_log.clear() - _segments.reset() - metadata = {} - try: - from core.services.log_service import set_current_message_id - - set_current_message_id(current_message_id) - except Exception: # pragma: no cover - logging must never fail a run - pass - - elif chunk_type == "tool_call": - _flush_thinking() - _round_block = None - _tc_evt = build_tool_call_event(chunk, chat_id, tool_calls_log) - _segments.add_tools(tool_calls_log) - await _emit(_tc_evt) - await _flush_inflight() - - elif chunk_type == "tool_call_start": - _flush_thinking() - _round_block = None - await _emit(build_tool_call_start_event(chunk, chat_id)) - - elif chunk_type == "tool_call_delta": - await _emit(build_tool_call_delta_event(chunk, chat_id)) - - elif chunk_type == "tool_result": - step = chunk.get("model_step") - if isinstance(step, dict): - _model_steps.append(step) - _tr_evt = build_tool_result_event(chunk, chat_id, tool_calls_log) - # attach_tool_result 可能刚补录了一个没有 tool_call 事件的条目， - # 此刻就把它排进段落表，否则它会缺席整条消息的展示顺序。 - _segments.add_tools(tool_calls_log) - if isinstance(step, dict): - # Publish completed results only after their canonical and - # display snapshot is durable. Retry a transient failure - # once; persistent failure must not announce success. - if not await _flush_inflight() and not await _flush_inflight(): - raise RuntimeError("Could not persist completed tool result") - await _emit(_tr_evt) - await _flush_inflight() - - elif chunk_type == "model_step": - step = chunk.get("step") - if isinstance(step, dict): - _model_steps.append(step) - if step.get("kind") == "tool_result": - if not await _flush_inflight() and not await _flush_inflight(): - raise RuntimeError("Could not persist completed tool result") - else: - await _flush_inflight() - - elif chunk_type == "compaction_progress": - _segments.add_compaction(chunk) - await _emit(chunk) - - elif chunk_type == "context_usage": - # Provider-reported usage is the authoritative live gauge. - # Forward the sanitized snapshot unchanged; unlike subagent - # subSteps, every token represented here reached the primary - # model wire. - await _emit({**chunk, "chat_id": chat_id}) - - elif chunk_type in ("heartbeat", "model_progress"): - # Neither is written to the stream. heartbeat = transport - # keep-alive (excluded from is_activity); model_progress = the - # model is still streaming (e.g. long tool-call-arg generation) - # with nothing mapping to an SSE event — merely receiving it - # resets the inactivity watchdog. - continue - - elif chunk_type == "tool_pending": - pending_event = { - "type": "tool_pending", - "chat_id": chat_id, - "reason": chunk.get("reason", "llm_buffering"), - } - if chunk.get("scope"): - pending_event["scope"] = chunk["scope"] - await _emit(pending_event) - - elif chunk_type == "subagent_event": - # Streaming sub-steps inside a subagent — passed through as-is - # (including parent_tool_id/sub_run_id/sub_type and their own - # fields); the frontend renders them under the call_subagent - # card. The chunk already carries type="subagent_event", so we - # just add chat_id in place (consumed once, safe to mutate). - # First accumulate into the call_subagent entry in - # tool_calls_log → persisted, so it can be replayed after a refresh. - try: - attach_subagent_step( - tool_calls_log, - str(chunk.get("parent_tool_id", "") or ""), - chunk, - ) - except Exception: - logger.debug("attach_subagent_step failed (ignored)", exc_info=True) - chunk["chat_id"] = chat_id - # 落库（attach_subagent_step）已在上面完成，推给浏览器的这一份走展示上限： - # 子智能体的工具结果和主链路一样能有几 MB。 - await _emit(bound_result_for_display(chunk)[0]) - - elif chunk_type == "file_confirm": - # §13: some tool coroutine is suspended, waiting for the user to - # confirm a write to "My Space". Forward to the frontend to show - # the confirmation bar; this SSE stream does NOT end — after the - # user's out-of-band POST /v1/chats/{id}/file-confirm the - # suspended tool resumes in place, and subsequent - # tool_result / meta events keep flowing on this same stream. - await _emit( - { - "type": "file_confirm", - "chat_id": chat_id, - "confirm_id": chunk.get("confirm_id"), - "op": chunk.get("op"), - "logical_path": chunk.get("logical_path"), - "message": chunk.get("message"), - # §13 timeout-reclaim signal: when True the frontend dismisses the zombie confirmation bar instead of showing it - "expired": chunk.get("expired", False), - } - ) - - elif chunk_type == "design_pick": - # Site-builder design pick (choose one of three): the choose_design - # tool coroutine is suspended waiting for the user's pick. - # Same mechanism as file_confirm, but the payload additionally has - # question/options (the file_confirm branch copies a whitelist of - # fields, hence the separate pass-through branch here). - await _emit( - { - "type": "design_pick", - "chat_id": chat_id, - "confirm_id": chunk.get("confirm_id"), - "question": chunk.get("question", ""), - "options": chunk.get("options", []), - "message": chunk.get("message"), - "expired": chunk.get("expired", False), - } - ) - - elif chunk_type == "user_question": - await _emit(build_user_question_event(chunk, chat_id)) - - elif chunk_type == "user_question_resolved": - await _emit(build_user_question_resolved_event(chunk, chat_id)) - - elif chunk_type == "plan_update": - # The main agent updated its lightweight plan checklist via the - # update_plan tool. Forward full-state to the frontend, which - # renders it as a plan bar above the chat input (not in the - # message flow). The agent keeps executing in the same turn — - # no approval gate, no loop abort. - await _emit( - { - "type": "plan_update", - "chat_id": chat_id, - "title": chunk.get("title", ""), - "steps": chunk.get("steps", []), - } - ) - - elif chunk_type in { - "ontology_activation", - "ontology_gate", - "ontology_review", - "ontology_repair", - "ontology_revision", - "ontology_revision_thinking", - }: - ontology_event = dict(chunk) - ontology_event["chat_id"] = chat_id - await _emit(bound_result_for_display(ontology_event)[0]) - - elif chunk_type == "meta": - _flush_thinking() - internal_budget = chunk.get("_compaction_budget_inputs") - if isinstance(internal_budget, dict): - compaction_budget_inputs = dict(internal_budget) - # Strict workspace gate: pinned list is the sole source of - # user-visible artifacts. See chats.py:_stream_sse_response. - _ws_pinned = _workspace_mod.get_pinned() - _ws_files = _workspace_mod.get_pinned_file_ids() - usage_payload = chunk.get("usage") or None - context_usage_payload = chunk.get("context_usage") or None - compaction_pending = not chunk.get("manual_compaction") and _should_schedule_compaction( - model_name=model_name, - usage=usage_payload, - budget_inputs=compaction_budget_inputs, - ) - # 本轮总耗时：同一个值既下发给正在看的这一页，也写进 extra_data 供历史 - # 加载使用。以前只写库不下发，前端只能自己按「占位气泡创建到现在」估一个， - # 把网络往返和排队也算了进去 —— 于是同一条回答，实时看到的耗时和刷新后 - # 从历史读出来的对不上。 - _duration_ms = int((time.monotonic() - _run_started_monotonic) * 1000) - metadata = { - "type": "meta", - "duration_ms": _duration_ms, - "route": chunk.get("route", "main"), - "sources": chunk.get("sources", []), - "artifacts": _ws_pinned, - "warnings": chunk.get("warnings", []), - "is_markdown": chunk.get("is_markdown", False), - "chat_id": chat_id, - "message_id": current_message_id, - "citations": chunk.get("citations", []), - "workspace_files": _ws_files, - "context_usage": context_usage_payload, - "compaction_pending": compaction_pending, - "ontology_governance": chunk.get("ontology_governance"), - # Skeleton marker: settlement runs after the stream closes. - "evolution_pending": chunk.get("evolution_pending"), - } - await _emit(metadata) - - # Persist: assistant message + artifacts - _persist_extra = { - "timestamp": now_iso(), - "route": metadata.get("route"), - "is_markdown": metadata.get("is_markdown", False), - "sources": metadata.get("sources", []), - "artifacts": metadata.get("artifacts", []), - "warnings": metadata.get("warnings", []), - "citations": metadata.get("citations", []), - "message_id": current_message_id, - "workspace_files": _ws_files, - "duration_ms": _duration_ms, - "segments": _segments.payload(), - } - if isinstance(context_usage_payload, dict): - _persist_extra["context_usage"] = context_usage_payload - if metadata.get("ontology_governance"): - _persist_extra["ontology_governance"] = metadata["ontology_governance"] - if context.get("model_provider_id"): - _persist_extra["model_provider_id"] = context.get("model_provider_id") - journal.save_snapshot( - run_id, - owner=owner, - phase="model_completed", - safety="replayable", - snapshot=_json_safe( - { - "assistant_content": full_response, - "message_id": current_message_id, - "model_name": model_name, - "thinking": _thinking_payload(), - "tool_calls": tool_calls_log if tool_calls_log else None, - "model_steps": _model_steps_payload(), - "usage": usage_payload, - "extra_data": _persist_extra, - "artifacts": _ws_pinned, - "context": context, - } - ), - ) - - queued_handoff: Dict[str, Any] = {} - - def _commit_output(db) -> None: - chat_service = ChatService(db) - # The row already exists (created at admission, refreshed - # while streaming); this is the final write that also drops - # the in-flight marker because ``_persist_extra`` never - # carries it. - chat_service.upsert_message( - chat_id=chat_id, - role="assistant", - content=full_response, - model=model_name, - thinking=_thinking_payload(), - tool_calls=tool_calls_log if tool_calls_log else None, - model_steps=_model_steps_payload(), - usage=usage_payload, - message_id=current_message_id, - chat_seq=current_chat_seq, - extra_data=_persist_extra, - invalidate_mid_turn_checkpoint=compaction_pending, - commit=False, - ) - # Build a ProjectScope from the workflow context and pass it - # explicitly so pinned files keep their project ownership. - from core.services.project_scope import project_scope_from_context - - _persist_artifacts( - db, - user_id, - chat_id, - _ws_pinned, - scope=project_scope_from_context(context), - commit=False, - ) - prepared = _commit_queued_handoff_in_session( - db, - source_run_id=run_id, - chat_id=chat_id, - user_id=user_id, - context=context, - model_name=model_name, - ) - if prepared is not None: - queued_handoff.update(prepared) - - # Offloaded to a worker thread on purpose. This is the one - # journal call that commits the turn's real side effects (the - # assistant message, pinned artifacts, a queued handoff), so it - # is the one that can sit waiting on a row lock. ``complete`` - # is synchronous SQLAlchemy; calling it inline blocks the event - # loop, and a blocked loop cannot run the coroutine holding the - # conflicting transaction — the process deadlocks and every - # request, stream and health probe stops answering. In a thread - # a lock wait costs this run its latency and nothing else. - # ``_commit_output`` and everything it calls are plain sync - # code, so they are safe to run off the loop. - completed = await asyncio.to_thread( - functools.partial( - journal.complete, - run_id, - owner=owner, - status="completed", - usage=usage_payload, - last_event_offset=offsets.last, - commit_effect=_commit_output, - committed_operation={ - "operation_type": "message_committed", - "phase": "message_committed", - "safety": "side_effect_committed", - "payload": {"message_id": current_message_id}, - }, - ) - ) - if not completed: - logger.warning( - "chat_run_terminal_cas_lost", - run_id=run_id, - owner=owner, - ) - return - - # The DB lease has intentionally ended. Stop the heartbeat - # before scheduling the handoff and projecting terminal events; - # otherwise its next renew would interpret normal completion as - # lease loss and cancel this coroutine mid-handoff. - lease_task.cancel() - - if queued_handoff: - _schedule_queued_handoff( - queued_handoff, - chat_id=chat_id, - user_id=user_id, - ) - await _emit( - _queued_run_started_event(run_id, chat_id, queued_handoff), - terminal=True, - ) - - if not chunk.get("manual_compaction"): - _spawn_followup_task( - run_id=run_id, - chat_id=chat_id, - user_msg=latest_user_message, - response=full_response, - msg_id=current_message_id, - ) - - # End-of-turn compaction: when real token usage crosses the - # threshold → generate a summary checkpoint in the background; - # doesn't block stream close, and failures don't affect the main - # conversation. - _spawn_compaction_task( - run_id=run_id, - chat_id=chat_id, - model_name=model_name, - usage=usage_payload, - budget_inputs=compaction_budget_inputs, - should_schedule=compaction_pending, - message_id=current_message_id, - ) - - # AgentScope may absorb a tool adapter exception after ToolEffectMiddleware - # has already durably paused the run. Re-read the DB before projecting a - # normal terminal marker; otherwise allocate_event_offset correctly - # rejects the non-completed run and the outer handler overwrites - # ``needs_attention`` with ``failed``. - try: - current = get_run(run_id) - except AttributeError: - # Some narrow unit adapters replace SessionLocal with a commit-only - # fake that intentionally has no query surface. - current = None - if current is not None: - if current.status == "needs_attention": - await _pause_for_tool_outcome( - RuntimeError(current.failure_reason or "tool outcome requires recovery") - ) - return - if current.status in _TERMINAL_STATUSES and current.status != "completed": - raise asyncio.CancelledError - - # Workflow returned normally (with or without a meta event): write the terminal marker - await _emit({"type": _TERMINAL_TYPE, "chat_id": chat_id}, terminal=True) - - except asyncio.CancelledError: - if lease_lost.is_set(): - # Another process owns recovery now. The fenced worker must not - # project a user cancellation or mutate the durable row. The one - # exception is fencing caused by the user's own cancel: nobody will - # take that run over, so the partial answer must be persisted here. - logger.warning("chat_run_worker_fenced", run_id=run_id, owner=owner) - if _cancelled_by_user(run_id): - _persist_cancelled_partial() - else: - current = get_run(run_id) - if current is None or current.status != "cancelled": - # Reaper/hard-cap/external recovery may cancel the local task - # after publishing its own authoritative terminal verdict. - logger.warning( - "chat_run_worker_stopped_by_external_terminal", - run_id=run_id, - status=(current.status if current is not None else None), - ) - else: - # Triggered by cancel_run: persist what was already produced, - # then write a user-cancelled event + terminal marker so - # followers exit gracefully. - logger.info("chat_run_cancelled", run_id=run_id) - _persist_cancelled_partial() - await _emit( - { - "type": "error", - "error": "任务已被用户取消", - "delta": "任务已被用户取消", - "chat_id": chat_id, - "_cancelled": True, - }, - terminal=True, - ) - await _emit( - {"type": _TERMINAL_TYPE, "chat_id": chat_id, "_cancelled": True}, - terminal=True, - ) - - except ToolOutcomeUnknown as exc: - await _pause_for_tool_outcome(exc) - - except HookPaused as exc: - await _pause_for_hook(exc) - - except RunLeaseLost: - # A takeover/cancel may race with any journal append. Once fenced, the - # old worker must not persist fallback messages or publish a terminal — - # except when the fencing came from the user's own cancel. - logger.warning("chat_run_worker_fenced", run_id=run_id, owner=owner) - if _cancelled_by_user(run_id): - _persist_cancelled_partial() - - except Exception as exc: - nested_pause = find_hook_paused(exc) - if nested_pause is not None: - await _pause_for_hook(nested_pause) - return - nested_unknown = find_tool_outcome_unknown(exc) - if nested_unknown is not None: - await _pause_for_tool_outcome(nested_unknown) - return - logger.error("chat_run_failed", run_id=run_id, error=str(exc), exc_info=True) - try: - user_facing = resolve_user_facing_error(exc) - except Exception: - user_facing = "请求处理失败，请稍后重试" - - def _commit_failed_output(db) -> None: - # Whatever the turn produced before failing stays in the row, with - # the error beside it. Runs in the failed-terminal transaction so a - # fenced worker can never leave a late message behind. - close_dangling_calls(_model_steps) - ChatService(db).upsert_message( - chat_id=chat_id, - role="assistant", - content=full_response, - model=model_name, - thinking=_thinking_payload(), - tool_calls=tool_calls_log if tool_calls_log else None, - model_steps=_model_steps_payload(), - message_id=current_message_id, - chat_seq=current_chat_seq, - error={"error": str(exc), "timestamp": _utcnow().isoformat()}, - extra_data=_turn_extra( - duration_ms=int((time.monotonic() - _run_started_monotonic) * 1000), - segments=_segments.payload(), - ), - commit=False, - ) - - failed = journal.complete( - run_id, - owner=owner, - status="failed", - failure_reason=str(exc)[:1000], - last_event_offset=offsets.last, - commit_effect=_commit_failed_output, - ) - if not failed: - logger.warning( - "chat_run_failure_cas_lost", - run_id=run_id, - owner=owner, - ) - return - await _emit( - { - "type": "error", - "error": user_facing, - "delta": user_facing, - "chat_id": chat_id, - }, - terminal=True, - ) - await _emit({"type": _TERMINAL_TYPE, "chat_id": chat_id}, terminal=True) - - finally: - if checkpoint_task is not None: - checkpoint_task.cancel() - # After the terminal state, keep the event stream for the full human - # wait window plus a recovery grace period so late reconnects can replay. - await _expire_stream(run_id) - lease_task.cancel() - with contextlib.suppress(asyncio.CancelledError): - await lease_task - _acknowledge_terminal_writer(run_id) - - -from orchestration.post_turn_compaction import _should_schedule_compaction, _spawn_compaction_task - - -def _spawn_followup_task( - *, run_id: str, chat_id: str, user_msg: str, response: str, msg_id: str -) -> None: - """Equivalent to the existing _generate_followups_bg in chats.py.""" - from orchestration.followups import get_followup_generator - - async def _bg() -> None: - try: - questions = await asyncio.wait_for( - get_followup_generator().generate(user_msg, response, run_id=run_id), - timeout=10, - ) - if questions: - with SessionLocal() as bg_db: - ChatService(bg_db).update_message_extra_data( - msg_id, {"follow_up_questions": questions} - ) - except Exception as exc: - logger.warning("background_followup_failed", error=str(exc)) - - asyncio.create_task(_bg()) - - -# ─── Follower: read events from the run's event log ──────────────────── - - -def _drain( - entries: Optional[List[Entry]], from_offset: int, cursor: str -) -> Tuple[List[Dict[str, Any]], str, bool]: - """Split a batch into deliverable events, the new cursor, and "saw terminal". - - Skips ``model_progress`` markers: nothing writes them anymore (the reaper - trusts in-process task liveness), but entries from older builds may survive - within the log's TTL and must never reach clients — the frontend treats an - unknown event type as "pending ended". - - An empty batch may arrive as ``None`` rather than ``[]``; a read that - returns nothing must never surface to the client as "流式响应中断". - """ - delivered: List[Dict[str, Any]] = [] - for entry_cursor, event in entries or (): - cursor = entry_cursor - if not isinstance(event, dict) or event.get("type") == "model_progress": - continue - if event.get("type") == _TERMINAL_TYPE: - return delivered, cursor, True - if event.get("_offset", 0) > from_offset: - delivered.append(event) - return delivered, cursor, False - - -async def follow_run(run_id: str, *, from_offset: int = 0) -> AsyncIterator[Dict[str, Any]]: - """Read the run's event log, delivering events after ``from_offset``. - - Flow: - 1. Replay everything recorded so far; deliver those with ``_offset > from_offset`` - 2. Block on the log waiting for new events - 3. Stop when a ``__terminal__``-typed event arrives - 4. Wait timed out + run in a terminal state → drain once more, then exit - gracefully (backstop so the follower never hangs forever) - - Which log backs this — Redis Streams or the in-process one — is the - deployment's choice; see :mod:`orchestration.run_event_stream`. - """ - events = _events() - - run = get_run(run_id) - if run is None: - # The chats.py route layer should have validated this already; this is a second line of defense - raise ChatRunNotFound(f"chat run {run_id} not found") - - cursor = START - - # ── Phase 1: replay historical events ── - try: - history = await events.read(run_id) - except Exception as exc: - logger.warning("chat_run_xrange_failed", run_id=run_id, error=str(exc), exc_info=exc) - history = [] - - delivered, cursor, terminal_seen = _drain(history, from_offset, cursor) - for event in delivered: - yield event - if terminal_seen: - return - - # ── Phase 2: blocking tail ── - while True: - try: - batch = await events.wait(run_id, after=cursor, limit=100, timeout_ms=_XREAD_BLOCK_MS) - except RedisTimeoutError: - # Benign: the block window elapsed with no new events — semantically - # identical to an empty result. (redis-py 8.0 defaults socket_timeout - # to 5s; if it ever equals _XREAD_BLOCK_MS the read raises here on - # every idle window instead of returning nil. We size socket_timeout - # well above the block in core/infra/redis.py, but treat the timeout - # as "no events" regardless so a long, quiet run never spams logs.) - batch = [] - except Exception as exc: - logger.warning("chat_run_xread_failed", run_id=run_id, error=str(exc), exc_info=exc) - await asyncio.sleep(0.5) - batch = [] - - if not batch: - # Nothing arrived: check whether the run reached a terminal state - current = get_run(run_id) - if current is not None and current.status in _TERMINAL_STATUSES: - # After the terminal state, read the latest events once more (covers the race) - try: - tail = await events.read(run_id, after=cursor, limit=200) - except Exception: - tail = [] - delivered, cursor, _terminal = _drain(tail, from_offset, cursor) - for event in delivered: - yield event - return - continue - - delivered, cursor, terminal_seen = _drain(batch, from_offset, cursor) - for event in delivered: - yield event - if terminal_seen: - return - - -def get_run(run_id: str) -> Optional[ChatRun]: - """Read a single run (used by the route layer for authorization).""" - with SessionLocal() as db: - return db.query(ChatRun).filter(ChatRun.run_id == run_id).first() - - -def is_run_cancelled(run_id: str) -> bool: - """Polled by plan_mode every ~15s heartbeat; see ``cancel_run`` for why. - - Any terminal state stops the worker (not just ``cancelled``): a run reaped - to ``failed`` by the stale reaper must also make the cooperative worker stop - itself as soon as possible, or we get the split-brain of "DB says dead, task - keeps running". The worker itself only writes the terminal state during - wrap-up, so any terminal state seen while polling must come from an external - verdict. - """ - with SessionLocal() as db: - row = db.query(ChatRun.status).filter(ChatRun.run_id == run_id).first() - return bool(row and row[0] in _TERMINAL_STATUSES) - - -# ─── Plan-execute mode: reuses the ChatRun table + Redis Stream + cancel + orphan handling ── - - -async def start_plan_execute_run( - *, - plan_id: str, - chat_id: str, - user_id: str, - enabled_mcp_ids: Optional[List[str]] = None, - enabled_skill_ids: Optional[List[str]] = None, - enabled_kb_ids: Optional[List[str]] = None, - enabled_agent_ids: Optional[List[str]] = None, - session_messages: List[Dict[str, Any]], - model_name: Optional[str] = None, -) -> ChatRun: - """Plan Phase-2 (execute): background task on the same ChatRun infrastructure. - - chat_id must exist in chat_sessions (FK). Caller should run _ensure_plan_session first. - """ - if not chat_id: - raise ValueError("chat_id is required to start a plan execute run") - - request_payload = { - "kind": "plan_execute", - "plan_id": plan_id, - "chat_id": chat_id, - "enabled_mcp_ids": enabled_mcp_ids or [], - "enabled_skill_ids": enabled_skill_ids or [], - "enabled_kb_ids": enabled_kb_ids or [], - "enabled_agent_ids": enabled_agent_ids or [], - "model_name": model_name, - } - effective_model_name = model_name or DEFAULT_CHAT_MODEL_ALIAS - run = _create_run_record( - chat_id=chat_id, - user_id=user_id, - request_payload=request_payload, - recovery_snapshot={ - "kind": "plan_execute", - "worker_args": { - "context": { - "mcp_ids": list(enabled_mcp_ids or []), - "skill_ids": list(enabled_skill_ids or []), - "kb_ids": list(enabled_kb_ids or []), - "model_name": effective_model_name, - } - }, - }, - ) - journal_owner = _new_worker_owner(run.run_id) - _register_run_task( - run.run_id, - _run_plan_execute_workflow( - run_id=run.run_id, - plan_id=plan_id, - chat_id=chat_id, - user_id=user_id, - message_id=run.message_id, - enabled_mcp_ids=enabled_mcp_ids, - enabled_skill_ids=enabled_skill_ids, - enabled_kb_ids=enabled_kb_ids, - enabled_agent_ids=enabled_agent_ids, - session_messages=session_messages, - model_name=effective_model_name, - journal_owner=journal_owner, - ), - name=f"plan_run:{run.run_id}", - ) - logger.info( - "plan_run_started", - run_id=run.run_id, - plan_id=plan_id, - chat_id=chat_id, - user_id=user_id, - ) - return run - - -async def _run_plan_execute_workflow( - *, - run_id: str, - plan_id: str, - chat_id: str, - user_id: str, - message_id: str, - enabled_mcp_ids: Optional[List[str]], - enabled_skill_ids: Optional[List[str]], - enabled_kb_ids: Optional[List[str]], - enabled_agent_ids: Optional[List[str]], - session_messages: List[Dict[str, Any]], - model_name: str, - journal_owner: Optional[str] = None, -) -> None: - """Background plan-execute worker. Streams via XADD, persists at end. - - Uses short-lived sessions (one per logical operation) to avoid holding a - DB connection across the entire long-running stream. - """ - from core.chat.tool_log import attach_tool_result as _attach_tool_result - from core.chat.tool_log import upsert_tool_call as _upsert_tool_call - from core.llm import workspace as _workspace_mod - from core.services.artifact_service import persist_artifacts as _persist_artifacts - from core.services.plan_service import PlanService - from orchestration.subagents.plan_mode import astream_execute_plan - - owner = journal_owner or _new_worker_owner(run_id) - journal = _journal() - if not journal.claim(run_id, owner=owner, lease_seconds=_RUN_LEASE_SECONDS): - logger.info("plan_run_claim_skipped", run_id=run_id, owner=owner) - return - current = asyncio.current_task() - lease_lost = asyncio.Event() - heartbeat = asyncio.create_task( - _lease_heartbeat(run_id, owner, current, lease_lost), - name=f"plan_run_lease:{run_id}", - ) - - offsets = _EventOffsets(journal, run_id, owner, lease_lost=lease_lost) - - _emit = offsets.emit - - async def _pause_plan_tool_outcome(exc: BaseException) -> None: - if not journal.needs_attention( - run_id, - owner=owner, - reason=f"tool outcome pending policy recovery: {exc}", - ): - return - await _emit( - { - "type": "plan_error", - "plan_id": plan_id, - "error": "工具调用结果无法安全确定，任务已暂停等待处理", - }, - terminal=True, - ) - await _emit({"type": _TERMINAL_TYPE, "chat_id": chat_id}, terminal=True) - - result_text = "" - completed_steps = 0 - total_steps = 0 - exec_usage: Optional[Dict[str, Any]] = None - tool_calls_log: List[Dict[str, Any]] = [] - # Strict workspace gate also applies to plan-execute runs. The plan - # subagent has access to pin_to_workspace and is expected to pin its - # final deliverables. - _workspace_mod.init_state() - - try: - await _emit( - { - "type": "run_started", - "run_id": run_id, - "message_id": message_id, - "chat_id": chat_id, - # 这几条路径不会被恢复接管，worker 起来的那一刻就是本轮起点， - # 不必为了拿它再读一次库——首帧前的每一次往返用户都在干等。 - "started_at": int(time.time() * 1000), - "kind": "plan_execute", - "plan_id": plan_id, - } - ) - - # astream_execute_plan needs a Session; isolate it from the persistence one. - from core.llm.middlewares import CURRENT_RUN_BINDING - - binding_token = CURRENT_RUN_BINDING.set((run_id, owner)) - try: - with SessionLocal() as stream_db: - from core.db.models import ChatSession - - session = stream_db.get(ChatSession, chat_id) - automation_run = bool( - session and (session.extra_data or {}).get("automation_run") - ) - async for event in astream_execute_plan( - plan_id=plan_id, - user_id=user_id, - db=stream_db, - model_name=model_name, - enabled_mcp_ids=enabled_mcp_ids, - enabled_skill_ids=enabled_skill_ids, - enabled_kb_ids=enabled_kb_ids, - enabled_agent_ids=enabled_agent_ids, - session_messages=session_messages, - chat_id=chat_id, - run_id=run_id, - automation_run=automation_run, - ): - evt_type = event.get("type") - if evt_type == "plan_complete": - result_text = event.get("result_text", "") - completed_steps = event.get("completed_steps", 0) - total_steps = event.get("total_steps", 0) - exec_usage = event.get("usage") or None - elif evt_type == "tool_call": - _upsert_tool_call( - tool_calls_log, - { - "tool_name": event.get("tool_name"), - "tool_id": event.get("tool_id"), - "tool_args": event.get("tool_args", {}), - "step_id": event.get("step_id"), - }, - ) - elif evt_type == "tool_result": - res = event.get("result") - tid = event.get("tool_id") - tn = event.get("tool_name") - _attach_tool_result(tool_calls_log, tid, tn, res) - await _emit(event) - finally: - CURRENT_RUN_BINDING.reset(binding_token) - - # Build plan snapshot in its own short session - plan_snapshot = None - try: - with SessionLocal() as snap_db: - updated_plan = PlanService(snap_db).get_plan(plan_id, user_id) - if updated_plan: - plan_snapshot = PlanService.build_execution_snapshot( - updated_plan, - completed_steps=completed_steps, - total_steps=total_steps, - result_text=result_text, - ) - except Exception as snap_exc: - logger.warning("plan_run_snapshot_failed", run_id=run_id, error=str(snap_exc)) - - # Strict workspace gate: only files pinned via pin_to_workspace - # surface anywhere user-visible — both the chat message and the - # user's file library ("My Space") are sourced from this list. - _ws_pinned = _workspace_mod.get_pinned() - artifacts_meta = _ws_pinned - _ws_files = _workspace_mod.get_pinned_file_ids() - - def _commit_plan_result(persist_db) -> None: # noqa: ANN001 - chat_service = ChatService(persist_db) - # 被用户中断的那一轮同样会走到这里（协作式取消让生成器正常收尾）， - # 照着"执行完成"落库会让用户回到会话时看到一句"计划执行完成"， - # 与他刚按下的停止完全对不上。按快照里的中断位分开措辞。 - _cancelled = bool((plan_snapshot or {}).get("cancelled")) - content = result_text or ( - f"计划执行已中断：共 {total_steps} 步，完成 {completed_steps} 步。" - if _cancelled - else f"计划执行完成：共 {total_steps} 步，完成 {completed_steps} 步。" - ) - chat_service.add_message( - chat_id=chat_id, - role="assistant", - content=content, - model=model_name, - message_id=message_id, - extra_data={ - "is_markdown": bool(result_text), - "plan_id": plan_id, - "completed_steps": completed_steps, - "total_steps": total_steps, - "plan_snapshot": plan_snapshot, - "artifacts": artifacts_meta, - "workspace_files": _ws_files, - "message_id": message_id, - }, - tool_calls=tool_calls_log or None, - usage=exec_usage, - commit=False, - ) - # The plan-execute background worker doesn't hold a workflow context - # dict, so it builds the ProjectScope via the reverse-lookup path - # chat_id → ChatSession → Project. - from core.services.project_scope import project_scope_from_chat_id - - _persist_artifacts( - persist_db, - user_id, - chat_id, - _ws_pinned, - scope=project_scope_from_chat_id(persist_db, chat_id), - commit=False, - ) - - def _persist_plan_partial() -> None: - # _commit_plan_result 走 add_message（非幂等），重复写会撞主键 - try: - with SessionLocal() as _db: - if _db.get(ChatMessage, message_id) is not None: - return - _commit_plan_result(_db) - _db.commit() - except Exception: # noqa: BLE001 - 落库失败不能吞掉取消收尾 - logger.warning( - "plan_run_cancelled_partial_persist_failed", - run_id=run_id, - exc_info=True, - ) - - completed = journal.complete( - run_id, - owner=owner, - status="completed", - usage=exec_usage, - last_event_offset=offsets.last, - commit_effect=_commit_plan_result, - committed_operation={ - "operation_type": "message_committed", - "phase": "message_committed", - "safety": "side_effect_committed", - "payload": {"message_id": message_id}, - }, - ) - if completed: - await _emit({"type": _TERMINAL_TYPE, "chat_id": chat_id}, terminal=True) - else: - # 用户取消时 cancel_run 已先写终态、清租约，上面的 CAS 注定输掉。 - # 协作式取消让生成器正常收尾，本轮已完成的步骤和正文就在手里 —— - # 不落库的话，用户回到会话只剩自己那句话。 - if _cancelled_by_user(run_id): - _persist_plan_partial() - else: - logger.warning("plan_run_terminal_cas_lost", run_id=run_id, owner=owner) - - except asyncio.CancelledError: - if lease_lost.is_set(): - logger.warning("plan_run_worker_fenced", run_id=run_id, owner=owner) - if _cancelled_by_user(run_id): - _persist_plan_partial() - elif journal.complete( - run_id, - owner=owner, - status="cancelled", - failure_reason="plan worker cancelled", - last_event_offset=offsets.last, - ): - logger.info("plan_run_cancelled", run_id=run_id) - await _emit( - { - "type": "plan_error", - "plan_id": plan_id, - "error": "任务已被用户取消", - "_cancelled": True, - }, - terminal=True, - ) - await _emit( - {"type": _TERMINAL_TYPE, "chat_id": chat_id, "_cancelled": True}, - terminal=True, - ) - - except Exception as exc: - nested_unknown = find_tool_outcome_unknown(exc) - if nested_unknown is not None: - await _pause_plan_tool_outcome(nested_unknown) - return - logger.error("plan_run_failed", run_id=run_id, error=str(exc), exc_info=True) - failed = journal.complete( - run_id, - owner=owner, - status="failed", - failure_reason=str(exc)[:1000], - last_event_offset=offsets.last, - ) - if failed: - await _emit( - {"type": "plan_error", "plan_id": plan_id, "error": str(exc)[:200]}, - terminal=True, - ) - await _emit({"type": _TERMINAL_TYPE, "chat_id": chat_id}, terminal=True) - - finally: - heartbeat.cancel() - try: - await heartbeat - except asyncio.CancelledError: - pass - await _expire_stream(run_id) - - -def get_run_by_plan_id(plan_id: str) -> Optional[ChatRun]: - """Reverse-lookup live run by plan_id (used by plan cancel endpoint).""" - with SessionLocal() as db: - return ( - db.query(ChatRun) - .filter( - ChatRun.status.in_(_LIVE_STATUSES), - ChatRun.request_payload["plan_id"].astext == plan_id, - ) - .order_by(ChatRun.created_at.desc()) - .first() - ) - - -# ─── Autonomous Loop (long-running autonomous execution, built on the same ChatRun framework) ─── -async def start_autonomous_loop_run( - *, - loop_id: str, - chat_id: str, - user_id: str, - goal_spec: Dict[str, Any], - budget: Dict[str, Any], - model_name: Optional[str] = None, - model_provider_id: Optional[str] = None, - evaluator_model: Optional[str] = None, - worker_max_iters: int = 15, - hitl_enabled: bool = False, - enable_thinking: bool = False, - chat_mode: Optional[str] = None, - is_resume: bool = False, - project_id: Optional[str] = None, - automation_run: bool = False, -) -> ChatRun: - """Start an autonomous-loop run (background task + Redis Stream, mirroring start_plan_execute_run).""" - if not chat_id: - raise ValueError("chat_id is required to start an autonomous loop run") - request_payload = { - "kind": "autonomous_loop", - "loop_id": loop_id, - "chat_id": chat_id, - "goal_spec": goal_spec, - "budget": budget, - "model_name": model_name, - "model_provider_id": model_provider_id, - "evaluator_model": evaluator_model, - "worker_max_iters": worker_max_iters, - "hitl_enabled": hitl_enabled, - "enable_thinking": enable_thinking, - # Thinking level (fast/medium/high/max): the active-run probe endpoint - # uses this to restore the frontend's resume phase; the worker uses it - # to set reasoning_effort. - "chat_mode": chat_mode, - "is_resume": is_resume, - "project_id": project_id, - "automation_run": automation_run, - } - # 启动参数跟 loop 持久化（而非只活在本次请求里）：崩溃/重启后的续跑——无论 API - # resume 还是启动自动续跑——都能还原同一套模型/评审模型/轮数/思考档位，不再悄悄 - # 降级到默认值。 - try: - from core.services.loop_service import LoopService as _LoopSvc - - with SessionLocal() as db: - _LoopSvc(db).save_start_params( - loop_id, - { - "model_name": model_name, - "model_provider_id": model_provider_id, - "evaluator_model": evaluator_model, - "worker_max_iters": worker_max_iters, - "hitl_enabled": hitl_enabled, - "enable_thinking": enable_thinking, - "chat_mode": chat_mode, - "automation_run": automation_run, - }, - ) - except Exception: # noqa: BLE001 - 参数存档失败不阻塞启动 - logger.warning("loop start_params persist failed", exc_info=True) - run = _create_run_record( - chat_id=chat_id, - user_id=user_id, - request_payload=request_payload, - recovery_snapshot={ - "kind": "autonomous_loop", - "worker_args": { - "context": { - "model_name": model_name, - "model_provider_id": model_provider_id, - "chat_mode": chat_mode, - "project_ctx": {"project_id": project_id} if project_id else None, - "automation_run": automation_run, - } - }, - }, - ) - journal_owner = _new_worker_owner(run.run_id) - _register_run_task( - run.run_id, - _run_autonomous_loop_workflow( - run_id=run.run_id, - loop_id=loop_id, - chat_id=chat_id, - user_id=user_id, - message_id=run.message_id, - goal_spec=goal_spec, - budget=budget, - model_name=model_name, - model_provider_id=model_provider_id, - evaluator_model=evaluator_model, - worker_max_iters=worker_max_iters, - hitl_enabled=hitl_enabled, - enable_thinking=enable_thinking, - chat_mode=chat_mode, - is_resume=is_resume, - project_id=project_id, - automation_run=automation_run, - journal_owner=journal_owner, - ), - name=f"autoloop_run:{run.run_id}", - ) - logger.info( - "autonomous_loop_run_started", - run_id=run.run_id, - loop_id=loop_id, - user_id=user_id, - ) - return run - - -_LOOP_STATUS_ZH = { - "completed": "✅ 已达成", - "budget_exhausted": "⏳ 预算耗尽", - "cancelled": "⛔ 已取消", - "awaiting_human": "🙋 等待人工", - "failed": "❌ 失败", -} - - -def _loop_transcript_md(objective: str, result) -> str: - """Render one autonomous loop's iteration trace into an assistant message for the chat (markdown).""" - lines = [ - "**🔁 自主循环**", - "", - f"**目标**：{objective}", - "", - "| 轮次 | 需求 | 评审判定 | 工具 | 说明 |", - "| --- | --- | --- | --- | --- |", - ] - for r in result.history: - reason = str(r.get("reason") or "").replace("\n", " ").replace("|", "／") - if len(reason) > 60: - reason = reason[:60] + "…" - lines.append( - f"| {r.get('seq')} | {r.get('requirement_id', '')} | {r.get('verdict')} | " - f"{r.get('tool_calls', 0)} | {reason} |" - ) - status_zh = _LOOP_STATUS_ZH.get(result.status, result.status) - # final_score is now the requirement pass ratio (0~1) — no script-based numeric score. - final = "—" if result.final_score is None else f"{result.final_score:.0%}" - lines += [ - "", - f"**结果**：{status_zh} · 共 {result.iterations} 轮 · 完成度 {final}" - f" · 用时 {result.wall_clock_s:.0f}s", - ] - if result.reason: - lines.append(f"\n> {result.reason}") - return "\n".join(lines) - - -async def _run_autonomous_loop_workflow( - *, - run_id: str, - loop_id: str, - chat_id: str, - user_id: str, - message_id: str, - goal_spec: Dict[str, Any], - budget: Dict[str, Any], - model_name: Optional[str], - model_provider_id: Optional[str] = None, - evaluator_model: Optional[str] = None, - worker_max_iters: int = 15, - hitl_enabled: bool = False, - enable_thinking: bool = False, - chat_mode: Optional[str] = None, - is_resume: bool = False, - project_id: Optional[str] = None, - automation_run: bool = False, - journal_owner: Optional[str] = None, -) -> None: - from orchestration.autonomous_loop import LoopBudget, run_autonomous_loop - from orchestration.loop_evaluator import GoalSpec - - owner = journal_owner or _new_worker_owner(run_id) - journal = _journal() - if not journal.claim(run_id, owner=owner, lease_seconds=_RUN_LEASE_SECONDS): - logger.info("autonomous_loop_claim_skipped", run_id=run_id, owner=owner) - return - current = asyncio.current_task() - lease_lost = asyncio.Event() - heartbeat = asyncio.create_task( - _lease_heartbeat(run_id, owner, current, lease_lost), - name=f"autonomous_loop_lease:{run_id}", - ) - # Accumulate the worker's streamed body text + tool cards, and persist them - # as an assistant message when the stream ends — exactly the same as a - # normal conversation: the body only accumulates content deltas (thinking - # goes through separate thinking events, is not stuffed into the body, and - # no fake is fabricated), and tools are persisted as tool_calls. - # After a refresh, buildHistorySegments reconstructs identically to a normal - # conversation. - content_buf: List[str] = [] - tool_log: List[Dict[str, Any]] = [] - tool_idx: Dict[str, int] = {} - - offsets = _EventOffsets(journal, run_id, owner, lease_lost=lease_lost) - - async def _emit(event: Dict[str, Any], *, terminal: bool = False) -> None: - await offsets.take(terminal=terminal) - et = event.get("type") - if et == "content": - _delta = event.get("delta") - if _delta: - content_buf.append(str(_delta)) - elif et == "tool_call": - _tid = str(event.get("tool_id") or f"t{len(tool_log)}") - tool_idx[_tid] = len(tool_log) - tool_log.append( - { - "id": _tid, - "name": event.get("tool_name") or "tool", - "input": event.get("tool_args"), - "status": "running", - } - ) - elif et == "tool_result": - _i = tool_idx.get(str(event.get("tool_id") or "")) - if _i is not None: - tool_log[_i]["output"] = event.get("result") - tool_log[_i]["status"] = "error" if event.get("error") else "success" - await _xadd_event(run_id, offsets.last, event) - # At structural checkpoints (requirement flipped / per-iteration - # evaluation done) persist progress incrementally — so after a mid-run - # crash/restart/refresh the already-produced body + tool cards are still - # visible from the DB, rather than waiting for a single write at the - # terminal state (see the fix for symptoms 2/3). - if et in ("requirement_passed", "iteration_evaluated"): - _flush_loop_message() - - def _loop_message_effect(status: str): - if not conversational: - return None - body = "".join(content_buf).strip() - captured_tools = [dict(item) for item in tool_log] - if not body and not captured_tools: - return None - - def _commit(db) -> None: # noqa: ANN001 - ChatService(db).upsert_message( - chat_id=chat_id, - role="assistant", - content=body, - message_id=message_id, - tool_calls=captured_tools or None, - extra_data={ - "autonomous_loop": True, - "loop_id": loop_id, - "loop_status": status, - }, - commit=False, - ) - - return _commit - - def _persist_loop_partial() -> None: - effect = _loop_message_effect("cancelled") - if effect is None: - return - try: - with SessionLocal() as _db: - effect(_db) - _db.commit() - except Exception: # noqa: BLE001 - 落库失败不能吞掉取消收尾 - logger.warning( - "autonomous_loop_cancelled_partial_persist_failed", - run_id=run_id, - exc_info=True, - ) - - def _flush_loop_message(status: str = "running") -> None: - """Upsert the currently accumulated worker body + tool cards into the assistant message (same message_id). - - Conversational mode (self_verify) only. Skipped when output is empty - (no body and no tools yet) to avoid writing an empty bubble. - """ - commit_effect = _loop_message_effect(status) - if commit_effect is None: - return - try: - journal.append_operation( - run_id, - owner=owner, - operation_type="loop_checkpoint_committed", - phase="loop_checkpoint", - safety="side_effect_committed", - payload={"loop_id": loop_id, "message_id": message_id}, - commit_effect=commit_effect, - ) - except RunLeaseLost: - raise - except Exception: # noqa: BLE001 - incremental persist failure must not take down the run - logger.warning("loop incremental persist failed", exc_info=True) - - gs = GoalSpec( - objective=goal_spec.get("objective", ""), - acceptance_criteria=goal_spec.get("acceptance_criteria", []) or [], - ) - # 预算去一等公民化：缺省一律 0（不限）。「能完成任务」优先——停止条件回归 - # 账本全通过/停滞无解/用户取消；防失控由 LOOP_HARD_MAX_ITERS 硬后备兜底。 - # 显式传正数的旧 loop 行（历史数据）仍按其预算执行。 - bud = LoopBudget( - max_iters=int(budget.get("max_iters", 0) or 0), - max_wall_clock_s=float(budget.get("max_wall_clock_s", 0) or 0), - max_tokens=int(budget.get("max_tokens", 0) or 0), - ) - # Project binding: resolve project_ctx and bind the loop's chat session to - # that project — this scopes the worker's/reviewer's file tools to the - # project folder (where the site source lives), and lets publish_site - # reverse-look-up the project from the session and publish to the same site. - # When bound to a project, the sandbox session uses chat_id (same as a - # normal project conversation → project file materialization and publish - # packaging share one session); without a project it degrades to the - # isolated loop-{loop_id} (pure task-style loop). - project_ctx: Optional[Dict[str, Any]] = None - session_id = f"loop-{loop_id}" - # Autonomous loops always live on in the chat history as a "normal - # conversation" (the objective is persisted as a user message and the result - # as an assistant message, surviving refreshes). Script-verification/form - # modes have been removed; there is no non-conversational branch anymore. - conversational = True - - # Requirement-ledger DB mirror callbacks: every time the driver writes the - # ledger it is also synced into agent_loops.metadata; resume reads the DB - # first (reliable across rebuilds/restarts/machine changes) and no longer - # depends on whether the sandbox /workspace still exists (see the fix for - # symptom 1). - from core.services.loop_service import LoopService as _LoopSvc - - def _load_ledger() -> Optional[Dict[str, Any]]: - try: - with SessionLocal() as db: - return _LoopSvc(db).load_ledger(loop_id) - except Exception: # noqa: BLE001 - logger.warning("loop load_ledger failed", exc_info=True) - return None - - def _save_ledger(led: Dict[str, Any]) -> None: - def _commit(db) -> None: # noqa: ANN001 - _LoopSvc(db).save_ledger(loop_id, led, commit=False) - - journal.append_operation( - run_id, - owner=owner, - operation_type="loop_ledger_committed", - phase="loop_checkpoint", - safety="side_effect_committed", - payload={"loop_id": loop_id}, - commit_effect=_commit, - ) - - def _poll_steering() -> List[str]: - """每轮开工前取走用户运行中追加的指令（POST /v1/loops/{id}/steer）。""" - try: - found: List[str] = [] - - def _commit(db) -> None: # noqa: ANN001 - found.extend(_LoopSvc(db).consume_steering(loop_id, commit=False)) - - journal.append_operation( - run_id, - owner=owner, - operation_type="loop_steering_polled", - phase="loop_checkpoint", - safety="side_effect_committed", - payload={"loop_id": loop_id}, - commit_effect=_commit, - ) - return found - except RunLeaseLost: - raise - except Exception: # noqa: BLE001 - logger.warning("loop consume_steering failed", exc_info=True) - return [] - - try: - if project_id: - try: - from core.db.models import ChatSession - from core.services.project_scope import build_project_ctx - - needs_binding = False - with SessionLocal() as db: - project_ctx = build_project_ctx(db, project_id) - if project_ctx: - sess = db.get(ChatSession, chat_id) - needs_binding = sess is not None and sess.project_id != project_id - - if project_ctx and needs_binding: - - def _commit_project_binding(db) -> None: # noqa: ANN001 - sess = db.get(ChatSession, chat_id) - if sess is not None: - sess.project_id = project_id - - journal.append_operation( - run_id, - owner=owner, - operation_type="loop_project_bound", - phase="pre_model", - safety="side_effect_committed", - payload={"loop_id": loop_id, "project_id": project_id}, - commit_effect=_commit_project_binding, - ) - except RunLeaseLost: - raise - except Exception: # noqa: BLE001 - project lookup may safely degrade - logger.warning("loop project_ctx resolve failed", exc_info=True) - project_ctx = None - session_id = chat_id if project_ctx else f"loop-{loop_id}" - - await _emit( - { - "type": "run_started", - "run_id": run_id, - "message_id": message_id, - "chat_id": chat_id, - # 这几条路径不会被恢复接管，worker 起来的那一刻就是本轮起点， - # 不必为了拿它再读一次库——首帧前的每一次往返用户都在干等。 - "started_at": int(time.time() * 1000), - "kind": "autonomous_loop", - "loop_id": loop_id, - } - ) - - def _commit_loop_start(db) -> None: # noqa: ANN001 - from core.services.loop_service import LoopService - - if conversational and gs.objective and not is_resume: - # Only the first start persists the objective; the deterministic - # id also makes a transaction retry idempotent. - ChatService(db).add_message( - chat_id=chat_id, - role="user", - content=gs.objective, - message_id=f"{message_id}_objective", - extra_data={"autonomous_loop": True, "loop_id": loop_id}, - commit=False, - ) - LoopService(db).mark_running( - loop_id, - workspace_session=session_id, - commit=False, - ) - - journal.append_operation( - run_id, - owner=owner, - operation_type="loop_started_committed", - phase="pre_model", - safety="side_effect_committed", - payload={"loop_id": loop_id}, - commit_effect=_commit_loop_start, - ) - - from core.auth.tenancy import tenant_of - from core.llm.middlewares import CURRENT_RUN_BINDING - - binding_token = CURRENT_RUN_BINDING.set((run_id, owner)) - try: - result = await run_autonomous_loop( - loop_id=loop_id, - user_id=user_id, - goal_spec=gs, - budget=bud, - model_name=model_name, - model_provider_id=model_provider_id, - # 评审/规划模型：显式指定 > 「模型管理 → 角色分配 → loop_reviewer」> - # main_agent。不再硬编码 "fast"——评审质量值得一个后台可配的位置。 - evaluator_model=evaluator_model, - worker_max_iters=worker_max_iters, - session_id=session_id, - hitl_enabled=hitl_enabled, - enable_thinking=enable_thinking, - chat_mode=chat_mode, - emit=_emit, - is_cancelled=lambda: is_run_cancelled(run_id), - load_ledger=_load_ledger, - save_ledger=_save_ledger, - poll_steering=_poll_steering, - project_ctx=project_ctx, - chat_id=chat_id, - automation_run=automation_run, - # Carried explicitly so the loop resolves *this* tenant's - # orchestration profile. Omitting it is how one tenant's published - # retry counts and budget multiplier became everyone's. - tenant_id=tenant_of(user_id), - ) - finally: - CURRENT_RUN_BINDING.reset(binding_token) - - _asst_content: Optional[str] = None - if conversational: - # Prefer storing the streamed body (matches what the frontend saw); fall back to the trace table when the worker produced no body at all. - _body = "".join(content_buf).strip() - _status_zh = _LOOP_STATUS_ZH.get(result.status, result.status) - if _body: - _asst_content = _body + ( - f"\n\n---\n**{_status_zh}** · 共 {result.iterations} 轮" - + (f" · 最终分 {result.final_score}" if result.final_score is not None else "") - ) - else: - _asst_content = _loop_transcript_md(gs.objective, result) - - def _commit_loop_result(db) -> None: # noqa: ANN001 - from core.services.loop_service import LoopService - - LoopService(db).persist_result(loop_id, result, commit=False) - if _asst_content is not None: - # Upsert and the ChatRun terminal transition share one owner- - # fenced transaction, so a superseded worker cannot publish. - ChatService(db).upsert_message( - chat_id=chat_id, - role="assistant", - content=_asst_content, - usage={"total_tokens": result.tokens_spent}, - tool_calls=tool_log if tool_log else None, - extra_data={ - "autonomous_loop": True, - "loop_id": loop_id, - "loop_status": result.status, - }, - message_id=message_id, - commit=False, - ) - - run_status = ( - "completed" - if result.status in ("completed", "budget_exhausted", "awaiting_human") - else ("cancelled" if result.status == "cancelled" else "failed") - ) - completed = journal.complete( - run_id, - owner=owner, - status=run_status, - usage={"total_tokens": result.tokens_spent}, - last_event_offset=offsets.last, - commit_effect=_commit_loop_result, - committed_operation={ - "operation_type": "loop_result_committed", - "phase": "loop_result_committed", - "safety": "side_effect_committed", - "payload": {"loop_id": loop_id, "message_id": message_id}, - }, - ) - if completed: - await _emit({"type": _TERMINAL_TYPE, "chat_id": chat_id}, terminal=True) - else: - logger.warning("autonomous_loop_terminal_cas_lost", run_id=run_id, owner=owner) - except asyncio.CancelledError: - # Cancelled: run_autonomous_loop was interrupted and no result is - # available, so the terminal-state persistence block is skipped — here - # we persist the progress produced so far (symptoms 2/3); otherwise a - # reopened chat would only contain the user's objective. - if lease_lost.is_set(): - logger.warning("autonomous_loop_worker_fenced", run_id=run_id, owner=owner) - if _cancelled_by_user(run_id): - _persist_loop_partial() - else: - partial_effect = _loop_message_effect("cancelled") - cancelled = journal.complete( - run_id, - owner=owner, - status="cancelled", - failure_reason="autonomous loop worker cancelled", - last_event_offset=offsets.last, - commit_effect=partial_effect, - committed_operation=( - { - "operation_type": "loop_partial_message_committed", - "phase": "loop_partial_message_committed", - "safety": "side_effect_committed", - "payload": {"loop_id": loop_id, "message_id": message_id}, - } - if partial_effect is not None - else None - ), - ) - if not cancelled: - return - await _emit( - {"type": "loop_error", "error": "任务已被用户取消", "_cancelled": True}, - terminal=True, - ) - await _emit( - {"type": _TERMINAL_TYPE, "chat_id": chat_id, "_cancelled": True}, - terminal=True, - ) - except Exception as exc: # noqa: BLE001 - nested_unknown = find_tool_outcome_unknown(exc) - if nested_unknown is not None: - if journal.needs_attention( - run_id, - owner=owner, - reason=f"tool outcome pending policy recovery: {nested_unknown}", - ): - await _emit( - { - "type": "loop_error", - "error": "工具调用结果无法安全确定，任务已暂停等待处理", - }, - terminal=True, - ) - await _emit({"type": _TERMINAL_TYPE, "chat_id": chat_id}, terminal=True) - return - logger.exception("autonomous_loop_run_failed", run_id=run_id) - partial_effect = _loop_message_effect("failed") - failed = journal.complete( - run_id, - owner=owner, - status="failed", - failure_reason=str(exc)[:1000], - last_event_offset=offsets.last, - commit_effect=partial_effect, - committed_operation=( - { - "operation_type": "loop_partial_message_committed", - "phase": "loop_partial_message_committed", - "safety": "side_effect_committed", - "payload": {"loop_id": loop_id, "message_id": message_id}, - } - if partial_effect is not None - else None - ), - ) - if failed: - await _emit( - {"type": "loop_error", "error": str(exc)[:500]}, - terminal=True, - ) - await _emit({"type": _TERMINAL_TYPE, "chat_id": chat_id}, terminal=True) - finally: - heartbeat.cancel() - try: - await heartbeat - except asyncio.CancelledError: - pass - await _expire_stream(run_id) - - -# ─── Plan Generate (Phase 1: decoupled from HTTP) ───────────────────── - - -async def start_plan_generate_run( - *, - chat_id: str, - user_id: str, - task_description: str, - model_name: str = DEFAULT_CHAT_MODEL_ALIAS, - model_provider_id: Optional[str] = None, - enabled_mcp_ids: Optional[List[str]] = None, - enabled_skill_ids: Optional[List[str]] = None, - enabled_kb_ids: Optional[List[str]] = None, - enabled_agent_ids: Optional[List[str]] = None, - session_messages: List[Dict[str, Any]], - uploaded_files: Optional[List[Dict[str, Any]]] = None, -) -> ChatRun: - """Plan Phase-1 (generate): background task on the same ChatRun infrastructure.""" - if not chat_id: - raise ValueError("chat_id is required to start a plan generate run") - - request_payload = { - "kind": "plan_generate", - "chat_id": chat_id, - "task_description": task_description[:500], - "model_name": model_name, - **({"model_provider_id": model_provider_id} if model_provider_id else {}), - } - recovery_snapshot = { - "kind": "plan_generate", - "worker_args": { - "task_description": task_description, - "model_name": model_name, - "model_provider_id": model_provider_id, - "enabled_mcp_ids": list(enabled_mcp_ids or []), - "enabled_skill_ids": list(enabled_skill_ids or []), - "enabled_kb_ids": list(enabled_kb_ids or []), - "enabled_agent_ids": list(enabled_agent_ids or []), - "session_messages": list(session_messages or []), - "uploaded_files": list(uploaded_files or []), - }, - } - run = _create_run_record( - chat_id=chat_id, - user_id=user_id, - request_payload=request_payload, - recovery_snapshot=recovery_snapshot, - ) - journal_owner = _new_worker_owner(run.run_id) - _register_run_task( - run.run_id, - _run_plan_generate_workflow( - run_id=run.run_id, - chat_id=chat_id, - user_id=user_id, - message_id=run.message_id, - task_description=task_description, - model_name=model_name, - model_provider_id=model_provider_id, - enabled_mcp_ids=enabled_mcp_ids, - enabled_skill_ids=enabled_skill_ids, - enabled_kb_ids=enabled_kb_ids, - enabled_agent_ids=enabled_agent_ids, - session_messages=session_messages, - uploaded_files=uploaded_files, - journal_owner=journal_owner, - ), - name=f"plan_gen_run:{run.run_id}", - ) - logger.info("plan_generate_run_started", run_id=run.run_id, chat_id=chat_id) - return run - - -async def _run_plan_generate_workflow( - *, - run_id: str, - chat_id: str, - user_id: str, - message_id: str, - task_description: str, - model_name: str, - model_provider_id: Optional[str], - enabled_mcp_ids: Optional[List[str]], - enabled_skill_ids: Optional[List[str]], - enabled_kb_ids: Optional[List[str]], - enabled_agent_ids: Optional[List[str]], - session_messages: List[Dict[str, Any]], - uploaded_files: Optional[List[Dict[str, Any]]], - journal_owner: Optional[str] = None, -) -> None: - from orchestration.subagents.plan_mode import astream_generate_plan - - owner = journal_owner or _new_worker_owner(run_id) - journal = _journal() - if not journal.claim(run_id, owner=owner, lease_seconds=_RUN_LEASE_SECONDS): - logger.info("plan_generate_claim_skipped", run_id=run_id, owner=owner) - return - try: - journal.append_operation( - run_id, - owner=owner, - operation_type="worker_started", - phase="pre_model", - safety="replayable", - ) - except RunLeaseLost: - logger.warning("plan_generate_fenced_before_start", run_id=run_id, owner=owner) - return - worker_task = asyncio.current_task() - if worker_task is None: # pragma: no cover - asyncio always owns this coroutine - raise RuntimeError("plan generate worker has no asyncio task") - lease_lost = asyncio.Event() - heartbeat = asyncio.create_task( - _lease_heartbeat(run_id, owner, worker_task, lease_lost), - name=f"plan_generate_lease:{run_id}", - ) - offsets = _EventOffsets(journal, run_id, owner, lease_lost=lease_lost) - - _emit = offsets.emit - - plan_id_out: Optional[str] = None - plan_title = "" - plan_desc = "" - plan_snapshot: Optional[Dict[str, Any]] = None - plan_record: Optional[Dict[str, Any]] = None - assistant_content = "" - gen_usage: Optional[Dict[str, Any]] = None - - try: - await _emit( - { - "type": "run_started", - "run_id": run_id, - "message_id": message_id, - "chat_id": chat_id, - # 这几条路径不会被恢复接管，worker 起来的那一刻就是本轮起点， - # 不必为了拿它再读一次库——首帧前的每一次往返用户都在干等。 - "started_at": int(time.time() * 1000), - "kind": "plan_generate", - } - ) - - from core.llm.middlewares import CURRENT_RUN_BINDING - - binding_token = CURRENT_RUN_BINDING.set((run_id, owner)) - try: - with SessionLocal() as stream_db: - async for event in astream_generate_plan( - task_description=task_description, - user_id=user_id, - db=stream_db, - model_name=model_name, - model_provider_id=model_provider_id, - enabled_mcp_ids=enabled_mcp_ids, - enabled_skill_ids=enabled_skill_ids, - enabled_kb_ids=enabled_kb_ids, - enabled_agent_ids=enabled_agent_ids, - session_messages=session_messages, - uploaded_files=uploaded_files, - chat_id=chat_id, - ): - if event.get("type") == "plan_generated": - plan_record = dict(event) - plan_id_out = event.get("plan_id") - plan_title = event.get("title", "") - plan_desc = event.get("description", "") - steps = event.get("steps", []) - step_summary = "\n".join( - f"{i + 1}. {s.get('title', '')}" for i, s in enumerate(steps) - ) - assistant_content = ( - f"已生成执行计划：**{plan_title}**\n\n" - f"{plan_desc}\n\n" - f"**执行步骤：**\n{step_summary}" - ) - plan_snapshot = { - "mode": "preview", - "title": plan_title, - "description": plan_desc, - "steps": [ - { - "step_order": s.get("step_order", i + 1), - "title": s.get("title", ""), - "description": s.get("description"), - "expected_tools": s.get("expected_tools", []), - "expected_skills": s.get("expected_skills", []), - } - for i, s in enumerate(steps) - ], - "total_steps": len(steps), - "completed_steps": 0, - } - gen_usage = event.get("usage") or None - await _emit(event) - finally: - CURRENT_RUN_BINDING.reset(binding_token) - - def _commit_plan_preview(persist_db) -> None: # noqa: ANN001 - if not assistant_content or not plan_id_out or plan_record is None: - return - from core.services.plan_service import PlanService - - extra_data: Dict[str, Any] = {} - if uploaded_files: - extra_data["uploaded_files"] = list(uploaded_files) - agent_name_map = plan_record.get("agent_name_map") - if isinstance(agent_name_map, dict) and agent_name_map: - extra_data["agent_name_map"] = dict(agent_name_map) - PlanService(persist_db).create_plan( - plan_id=str(plan_id_out), - user_id=user_id, - title=str(plan_record.get("title") or "未命名计划"), - description=str(plan_record.get("description") or ""), - task_input=task_description, - steps=list(plan_record.get("steps") or []), - extra_data=extra_data, - commit=False, - ) - ChatService(persist_db).add_message( - chat_id=chat_id, - role="assistant", - content=assistant_content, - model=model_name, - message_id=message_id, - extra_data={ - "is_markdown": True, - "plan_id": plan_id_out, - "plan_snapshot": plan_snapshot, - "message_id": message_id, - }, - usage=gen_usage, - commit=False, - ) - - completed = journal.complete( - run_id, - owner=owner, - status="completed", - usage=gen_usage, - last_event_offset=offsets.last, - commit_effect=_commit_plan_preview, - committed_operation={ - "operation_type": "message_committed", - "phase": "message_committed", - "safety": "side_effect_committed", - "payload": {"message_id": message_id, "plan_id": plan_id_out}, - }, - ) - if completed: - await _emit({"type": _TERMINAL_TYPE, "chat_id": chat_id}, terminal=True) - else: - logger.warning("plan_generate_terminal_cas_lost", run_id=run_id, owner=owner) - - except asyncio.CancelledError: - if lease_lost.is_set(): - logger.warning("plan_generate_worker_fenced", run_id=run_id, owner=owner) - else: - cancelled = journal.complete( - run_id, - owner=owner, - status="cancelled", - failure_reason="plan generation worker cancelled", - last_event_offset=offsets.last, - ) - current = get_run(run_id) - if cancelled or (current is not None and current.status == "cancelled"): - logger.info("plan_generate_run_cancelled", run_id=run_id) - else: - logger.warning( - "plan_generate_cancel_cas_lost", - run_id=run_id, - owner=owner, - ) - return - await _emit( - { - "type": "plan_error", - "error": "任务已被用户取消", - "_cancelled": True, - }, - terminal=True, - ) - await _emit( - {"type": _TERMINAL_TYPE, "chat_id": chat_id, "_cancelled": True}, - terminal=True, - ) - - except Exception as exc: - nested_unknown = find_tool_outcome_unknown(exc) - if nested_unknown is not None: - if journal.needs_attention( - run_id, - owner=owner, - reason=f"tool outcome pending policy recovery: {nested_unknown}", - ): - await _emit( - { - "type": "plan_error", - "error": "工具调用结果无法安全确定，任务已暂停等待处理", - }, - terminal=True, - ) - await _emit({"type": _TERMINAL_TYPE, "chat_id": chat_id}, terminal=True) - return - logger.error("plan_generate_run_failed", run_id=run_id, error=str(exc), exc_info=True) - failed = journal.complete( - run_id, - owner=owner, - status="failed", - failure_reason=str(exc)[:1000], - last_event_offset=offsets.last, - ) - if failed: - await _emit( - {"type": "plan_error", "error": str(exc)[:200]}, - terminal=True, - ) - await _emit({"type": _TERMINAL_TYPE, "chat_id": chat_id}, terminal=True) - - finally: - heartbeat.cancel() - with contextlib.suppress(asyncio.CancelledError): - await heartbeat - await _expire_stream(run_id) - - -# ─── Cancellation ────────────────────────────────────────────────────── - - -async def cancel_run( - run_id: str, *, user_id: str, stop_terminal_worker: bool = False -) -> bool: - """Mark status=cancelled. Idempotent on terminal runs. - - For chat-run kinds (regular chat / plan-generate) we forcefully cancel the - underlying asyncio task — these don't hold long-lived per-step HTTP MCP - clients so anyio cancel propagation is safe. - - For plan-execute kinds we DO NOT call task.cancel(). plan_mode polls - is_run_cancelled() between LLM streaming heartbeats and self-cancels - inside its own task. Cross-task task.cancel() on a plan-execute worker - risks an anyio cancel-scope deadlock: the worker holds per-step - streamable_http MCP clients whose internal SSE tasks may be blocked on - socket recv at cancel time, causing anyio's _deliver_cancellation to - self-reschedule every 10ms and starve the whole event loop. - """ - run = get_run(run_id) - if run is None: - raise ChatRunNotFound(f"chat run {run_id} not found") - if run.user_id != user_id: - raise ChatRunPermissionDenied(f"run {run_id} owned by another user") - already_cancelled = stop_terminal_worker and run.status == "cancelled" - if run.status not in _LIVE_STATUSES and not already_cancelled: - return False - - if not already_cancelled and not _journal().cancel(run_id, reason="user cancelled"): - # Race: the worker/reaper just beat us to writing the terminal state — treat as "no longer running". - return False - - payload = run.request_payload if isinstance(run.request_payload, dict) else {} - cooperative_only = payload.get("kind") in _COOPERATIVE_KINDS - - # Cooperative workers are fenced by the cancelled DB row and therefore - # deliberately stay silent when their heartbeat cancels them. Project the - # authoritative cancellation here so followers never wait on the old task. - if cooperative_only: - await _write_terminal_to_stream( - run_id, - chat_id=run.chat_id, - error_text="任务已被用户取消", - cancelled=True, - ) - - task = _active_runs.get(run_id) - if task is not None and not task.done(): - if cooperative_only: - target = asyncio.shield(task) - timeout = 20.0 - else: - task.cancel() - target = task - timeout = 2.0 - try: - await asyncio.wait_for(target, timeout=timeout) - except (asyncio.TimeoutError, asyncio.CancelledError): - pass - except Exception as exc: - logger.warning("cancel_run_await_failed", run_id=run_id, error=str(exc)) - if task.done(): - _acknowledge_terminal_writer(run_id) - elif not cooperative_only: - # Cross-process / post-restart: worker not in this process. Inject the - # terminal markers ourselves so any active follower exits cleanly. - await _write_terminal_to_stream( - run_id, - chat_id=run.chat_id, - error_text="任务已被用户取消", - cancelled=True, - ) - - return True - - -async def recover_orphan_runs() -> int: - """Recover claimable runs from their last committed database safe point.""" - from core.services.tool_effect_ledger import ToolEffectJournal, recover_incomplete_tool_effects - - # Desktop channel runs require a live device lease and a scoped I/O adapter. - # Neither survives process restart. Fail closed before any tool-effect recovery. - with SessionLocal() as db: - local_channel_runs = [ - (run.run_id, run.chat_id) - for run in db.query(ChatRun).filter(ChatRun.status.in_(_LIVE_STATUSES)).all() - if (run.request_payload or {}).get("source") == "desktop_channel" - and run.run_id not in _active_runs - ] - for run_id, chat_id in local_channel_runs: - if _journal().cancel(run_id, reason="desktop channel lease cannot survive process restart"): - await _write_terminal_to_stream( - run_id, - chat_id=chat_id, - error_text="本机机器人执行已中断，请检查结果后从渠道重新发起", - cancelled=True, - ) - effect_decisions = await recover_incomplete_tool_effects( - journal=ToolEffectJournal(SessionLocal) - ) - effect_attention_runs = { - item.run_id for item in effect_decisions if item.action == "needs_attention" - } - for run_id in effect_attention_runs: - run = get_run(run_id) - if run is not None: - await _write_terminal_to_stream( - run_id, - chat_id=run.chat_id or "", - error_text="工具调用结果无法安全确定，任务已暂停等待处理", - ) - decisions = _journal().recover() - for decision in decisions: - if decision.action == "resume": - if not _register_recovered_chat(decision): - reason = "run kind or recovery snapshot cannot be resumed automatically" - if _journal().needs_attention(decision.run_id, reason=reason): - await _write_terminal_to_stream( - decision.run_id, - chat_id=decision.chat_id or "", - error_text="任务缺少可恢复快照，已暂停等待处理", - ) - elif decision.action == "resume_from_snapshot": - snapshot_result = await _commit_recovered_chat_snapshot(decision) - if snapshot_result is False: - await _write_terminal_to_stream( - decision.run_id, - chat_id=decision.chat_id or "", - error_text="已保存的模型结果无法安全提交，任务已暂停等待处理", - ) - elif decision.action == "needs_attention": - if decision.phase == "model_inflight": - recovery_error = "模型调用结果不确定，任务已暂停等待恢复决策" - elif decision.phase == "legacy_unrecoverable": - recovery_error = "旧版本任务缺少恢复快照，已暂停等待处理" - else: - recovery_error = "任务在工具结果不确定的安全边界暂停，等待恢复决策" - await _write_terminal_to_stream( - decision.run_id, - chat_id=decision.chat_id or "", - error_text=recovery_error, - ) - elif decision.action == "completed": - await _write_recovered_terminal_marker( - decision.run_id, - chat_id=decision.chat_id or "", - ) - elif decision.action == "failed": - await _write_terminal_to_stream( - decision.run_id, - chat_id=decision.chat_id or "", - error_text="服务重启发生在任务准备完成前，请重新发起", - ) - recovered_count = len( - {item.run_id for item in decisions} - | effect_attention_runs - | {run_id for run_id, _ in local_channel_runs} - ) - if recovered_count: - logger.info("chat_run_orphan_recovered", count=recovered_count) - return recovered_count - - -def _register_recovered_chat(decision: RecoveryDecision) -> bool: - snapshot = dict(decision.snapshot or {}) - if snapshot.get("kind") != "chat": - return False - args = snapshot.get("worker_args") - if not isinstance(args, dict): - return False - required = { - "session_messages", - "effective_user_message", - "raw_user_message", - "context", - } - if not required.issubset(args): - return False - owner = _new_worker_owner(decision.run_id) - recovered_run = get_run(decision.run_id) - if recovered_run and (recovered_run.request_payload or {}).get("source") == "desktop_channel": - return False - from orchestration.agent_api_recovery import validate_recovery_context - from sqlalchemy.exc import SQLAlchemyError - - try: - with SessionLocal() as db: - restored_context = validate_recovery_context( - db, recovered_run, args.get("context"), - user_id=decision.user_id, chat_id=decision.chat_id, - ) - if "context" in snapshot: - validate_recovery_context( - db, recovered_run, snapshot["context"], - user_id=decision.user_id, chat_id=decision.chat_id, - ) - except (ValueError, SQLAlchemyError): - logger.warning("chat_run_recovery_scope_rejected", run_id=decision.run_id) - return False - _register_run_task( - decision.run_id, - _run_workflow( - run_id=decision.run_id, - chat_id=decision.chat_id, - user_id=decision.user_id, - message_id=decision.message_id, - assistant_chat_seq=( - int(recovered_run.assistant_chat_seq) - if recovered_run is not None and recovered_run.assistant_chat_seq is not None - else None - ), - session_messages=list(args.get("session_messages") or []), - effective_user_message=str(args.get("effective_user_message") or ""), - raw_user_message=str(args.get("raw_user_message") or ""), - context=restored_context, - model_name=(str(args["model_name"]) if args.get("model_name") else None), - journal_owner=owner, - recovering=True, - ), - name=f"chat_run_recovery:{decision.run_id}", - ) - return True - - -async def _commit_recovered_chat_snapshot(decision: RecoveryDecision) -> Optional[bool]: - snapshot = dict(decision.snapshot or {}) - content = snapshot.get("assistant_content") - message_id = str(snapshot.get("message_id") or decision.message_id) - if not isinstance(content, str) or not message_id: - paused = _journal().needs_attention( - decision.run_id, - reason="model-completed snapshot is incomplete", - ) - return False if paused else None - owner = _new_worker_owner(decision.run_id) - recovered_run = get_run(decision.run_id) - journal = _journal() - from orchestration.agent_api_recovery import validate_recovery_context - from sqlalchemy.exc import SQLAlchemyError - - try: - with SessionLocal() as db: - restored_context = validate_recovery_context( - db, recovered_run, snapshot.get("context"), - user_id=decision.user_id, chat_id=decision.chat_id, - ) - worker_args = snapshot.get("worker_args") - if isinstance(worker_args, dict) and "context" in worker_args: - with SessionLocal() as db: - validate_recovery_context( - db, recovered_run, worker_args["context"], - user_id=decision.user_id, chat_id=decision.chat_id, - ) - except (ValueError, SQLAlchemyError): - paused = journal.needs_attention( - decision.run_id, reason="agent API recovery scope is unavailable or inconsistent" - ) - return False if paused else None - if not journal.claim( - decision.run_id, - owner=owner, - lease_seconds=_RUN_LEASE_SECONDS, - ): - return None - try: - from core.services.artifact_service import persist_artifacts as _persist_artifacts - from core.services.project_scope import project_scope_from_context - - queued_handoff: Dict[str, Any] = {} - - def _commit_recovered_output(db) -> None: - ChatService(db).upsert_message( - chat_id=decision.chat_id, - role="assistant", - content=content, - message_id=message_id, - chat_seq=( - int(recovered_run.assistant_chat_seq) - if recovered_run is not None and recovered_run.assistant_chat_seq is not None - else None - ), - model=( - str(snapshot["model_name"]) if snapshot.get("model_name") is not None else None - ), - thinking=( - list(snapshot["thinking"]) - if isinstance(snapshot.get("thinking"), list) - else None - ), - tool_calls=( - list(snapshot["tool_calls"]) - if isinstance(snapshot.get("tool_calls"), list) - else None - ), - model_steps=( - list(snapshot["model_steps"]) - if isinstance(snapshot.get("model_steps"), list) - else None - ), - usage=( - dict(snapshot["usage"]) if isinstance(snapshot.get("usage"), dict) else None - ), - extra_data=( - dict(snapshot["extra_data"]) - if isinstance(snapshot.get("extra_data"), dict) - else {} - ), - commit=False, - ) - artifacts = snapshot.get("artifacts") - if isinstance(artifacts, list): - _persist_artifacts( - db, - decision.user_id, - decision.chat_id, - artifacts, - scope=project_scope_from_context(restored_context), - commit=False, - ) - worker_args = snapshot.get("worker_args") - recovery_args = worker_args if isinstance(worker_args, dict) else {} - context_snapshot = snapshot.get("context") - prepared = _commit_queued_handoff_in_session( - db, - source_run_id=decision.run_id, - chat_id=decision.chat_id, - user_id=decision.user_id, - context=( - restored_context - if isinstance(context_snapshot, dict) - else dict(recovery_args.get("context") or {}) - ), - model_name=( - str(snapshot["model_name"]) - if snapshot.get("model_name") is not None - else ( - str(recovery_args["model_name"]) - if recovery_args.get("model_name") is not None - else None - ) - ), - ) - if prepared is not None: - queued_handoff.update(prepared) - - completed = journal.complete( - decision.run_id, - owner=owner, - status="completed", - usage=(dict(snapshot["usage"]) if isinstance(snapshot.get("usage"), dict) else None), - commit_effect=_commit_recovered_output, - committed_operation={ - "operation_type": "message_committed_from_snapshot", - "phase": "message_committed", - "safety": "side_effect_committed", - "payload": {"message_id": message_id}, - }, - ) - if completed: - if queued_handoff: - _schedule_queued_handoff( - queued_handoff, - chat_id=decision.chat_id, - user_id=decision.user_id, - task_prefix="chat_run_handoff_recovery", - ) - await _write_queued_run_started_projection( - decision.run_id, - chat_id=decision.chat_id, - handoff=queued_handoff, - ) - await _write_recovered_terminal_marker( - decision.run_id, - chat_id=decision.chat_id, - ) - return completed - except Exception as exc: # noqa: BLE001 - pause instead of replaying model output - logger.error( - "chat_run_snapshot_commit_failed", - run_id=decision.run_id, - error=str(exc), - exc_info=True, - ) - paused = journal.needs_attention( - decision.run_id, - reason=str(exc)[:1000], - owner=owner, - ) - return False if paused else None - - -async def _write_recovered_terminal_marker(run_id: str, *, chat_id: str) -> None: - event = {"type": _TERMINAL_TYPE, "chat_id": chat_id, "_recovered": True} - try: - offset = _journal().allocate_event_offset(run_id, terminal=True) - await _xadd_event(run_id, offset, event) - await _expire_stream(run_id) - except Exception as exc: # Redis is only a projection - logger.warning("chat_run_recovered_terminal_write_failed", run_id=run_id, error=str(exc)) - - -async def resume_running_loops() -> int: - """启动对账 + 按需自动续跑：进程重启后收养/归位孤儿自主循环。 - - 进程刚起来时不存在任何活跃 task，因此 status='running' 的 loop 全是孤儿。 - 对每一个孤儿： - - - ``LOOP_AUTO_RESUME=true`` → 用**持久化的启动参数**（agent_loops.extra_data. - start_params：模型/评审模型/轮数/思考档位）原样续跑——账本在 DB 有镜像、 - 沙箱有 feature_list.json，driver 自动断点续跑，不再悄悄降级到默认模型。 - - 关闭（默认）→ 状态归位为 ``interrupted``（可续跑），不再留下永远 running 的 - 僵尸行（历史坑：僵尸 running 行既误导列表页、又让 cancel 无处下手）。 - - 'awaiting_human'/终态一律不动。 - """ - from core.db.models import AgentLoop - - auto = os.getenv("LOOP_AUTO_RESUME", "false").strip().lower() in ( - "1", - "true", - "yes", - ) - resumed = 0 - with SessionLocal() as db: - loops = db.query(AgentLoop).filter(AgentLoop.status == "running").all() - specs = [ - ( - x.loop_id, - x.chat_id, - x.user_id, - dict(x.goal_spec or {}), - dict(x.budget or {}), - (x.extra_data or {}).get("project_id"), - dict((x.extra_data or {}).get("start_params") or {}), - ) - for x in loops - ] - - if not auto: - if specs: - from core.services.loop_service import LoopService as _LoopSvc - - with SessionLocal() as db: - for loop_id, *_rest in specs: - _LoopSvc(db).mark_interrupted( - loop_id, reason="服务重启导致运行中断，可点击「继续」从断点续跑" - ) - logger.info("[startup] orphan loops marked interrupted: %d", len(specs)) - return 0 - - for loop_id, chat_id, user_id, goal_spec, budget, project_id, params in specs: - if not chat_id: - continue - try: - await start_autonomous_loop_run( - loop_id=loop_id, - chat_id=chat_id, - user_id=user_id, - goal_spec=goal_spec, - budget=budget, - model_name=params.get("model_name"), - model_provider_id=params.get("model_provider_id"), - evaluator_model=params.get("evaluator_model"), - worker_max_iters=int(params.get("worker_max_iters") or 15), - hitl_enabled=bool(params.get("hitl_enabled")), - enable_thinking=bool(params.get("enable_thinking")), - chat_mode=params.get("chat_mode"), - project_id=project_id, - automation_run=bool(params.get("automation_run")), - is_resume=True, - ) - resumed += 1 - logger.info("autonomous_loop_resumed", loop_id=loop_id) - except Exception: # noqa: BLE001 - logger.warning("autonomous_loop_resume_failed loop_id=%s", loop_id, exc_info=True) - return resumed - - -async def _stream_last_write_ms(run_id: str) -> Optional[int]: - """Write time of the last event on the run's log (epoch ms). - - Cursors are minted as ``-`` by every backend, so the millisecond - half is the write time — no extra bookkeeping. Returns None when there is - no log / no events / the read fails (callers treat that as "no activity"). - """ - try: - return await _events().last_write_ms(run_id) - except Exception as exc: - logger.warning("chat_run_activity_check_failed", run_id=run_id, error=str(exc)) - return None - - -def _chat_has_live_job(chat_id: Optional[str], now: datetime) -> bool: - """这个会话是否挂着**还在推进**的批量作业（跨进程可见的活性证据）。 - - 判据是 ``jobs.updated_at`` 在静默窗口内前进过：作业每完成一次子调用都会写 usage， - onupdate 随之推进 updated_at。只要作业还在动，这个 run 就不是僵尸——哪怕它的 - asyncio task 属于另一个进程（多 worker / 多副本），也哪怕主链路一个流事件都没有。 - - 作业本身有 max_seconds 墙钟预算与熔断，所以这条豁免不会让 run 永生。 - """ - if not chat_id: - return False - try: - from core.db.models import Job - - cutoff = now - timedelta(seconds=_STALE_QUIET_SEC) - with SessionLocal() as db: - return ( - db.query(Job.job_id) - .filter( - Job.chat_id == chat_id, - Job.status.in_(("pending", "running")), - Job.updated_at >= cutoff, - ) - .first() - is not None - ) - except Exception: # noqa: BLE001 —— 证据查不到就按原判据走，绝不因此放过真僵尸 - return False - - -async def reap_stale_runs() -> int: - """Periodic safety net: fail 'running' runs that show no sign of life. - - The verdict is activity-aware, in two tiers (both tunable via env vars): - - - Reap only when **over-age + quiet**: lifetime exceeds - ``CHAT_RUN_MAX_AGE_SEC`` AND the Redis Stream's last write is older than - ``CHAT_RUN_STALE_QUIET_SEC``. Long tasks still steadily producing - tool_call/content are no longer falsely killed (historical bug: purely - age-based reaping choked even active runs mid-tool-call once the 30-minute - mark hit). - - **Absolute cap**: runs older than ``CHAT_RUN_HARD_MAX_AGE_SEC`` are reaped - even when actively emitting, except for automation conversations and - registered bounded human interactions (which keep their own deadline). - - Reaping also cancels the in-process worker task, keeping the DB terminal - state aligned with the actual task (historical bug: only flipping the DB - without killing the task let the worker silently run another 1.5h and - overwrite the terminal state back to completed). plan_execute is the - exception — cross-task cancel risks the anyio cancel-scope deadlock (see the - ``cancel_run`` docstring); it stops itself after ``is_run_cancelled`` - polling observes the terminal state. - - Complements periodic ``recover_orphan_runs`` and the per-run - inactivity watchdog — also sweeps up historical zombie runs left by - older code paths. - """ - from sqlalchemy import func - from core.db.models import ChatSession - - now = _utcnow() - cutoff = now - timedelta(seconds=_STALE_RUN_MAX_AGE_SEC) - hard_cutoff = now - timedelta(seconds=_HARD_MAX_AGE_SEC) - with SessionLocal() as db: - candidates = ( - db.query( - ChatRun.run_id, - ChatRun.chat_id, - ChatRun.request_payload, - ChatRun.lease_expires_at, - ChatSession.extra_data.label("session_metadata"), - func.coalesce(ChatRun.started_at, ChatRun.created_at).label("began_at"), - ) - .outerjoin(ChatSession, ChatSession.chat_id == ChatRun.chat_id) - .filter(ChatRun.status == "running") - .filter(func.coalesce(ChatRun.started_at, ChatRun.created_at) < cutoff) - .all() - ) - if not candidates: - return 0 - - now_ms = int(now.timestamp() * 1000) - reaped = 0 - for row in candidates: - rid, cid = row.run_id, row.chat_id - lease_expires_at = row.lease_expires_at - if lease_expires_at is not None and lease_expires_at.tzinfo is None: - lease_expires_at = lease_expires_at.replace(tzinfo=timezone.utc) - began_at = row.began_at - if began_at is not None and began_at.tzinfo is None: # SQLite stores naive UTC - began_at = began_at.replace(tzinfo=timezone.utc) - session_metadata = row.session_metadata or {} - automation_run = bool(session_metadata.get("automation_run")) - hard_expired = not automation_run and began_at is not None and began_at < hard_cutoff - if not hard_expired and lease_expires_at is not None and lease_expires_at > now: - logger.info( - "chat_run_stale_skip_live_lease", - run_id=rid, - lease_expires_at=lease_expires_at.isoformat(), - ) - continue - payload = row.request_payload if isinstance(row.request_payload, dict) else {} - - # 自主循环豁免统一年龄硬顶：loop 的使命就是「跑到任务完成为止」（去预算化）， - # 且它有自己的停滞护栏/熔断/硬后备轮数。进程内 task 还活着的 loop 绝不按 - # 年龄杀（历史竞态：6h 硬顶 == loop 旧默认预算，跑满预算的健康 loop 在优雅 - # 收尾前被硬顶抢杀，报错还误导为「无响应」）。孤儿 loop（进程重启遗留） - # 走下面的静默判据正常清理。 - if hard_expired and payload.get("kind") == "autonomous_loop": - task = _active_runs.get(rid) - if task is not None and not task.done(): - logger.info("chat_run_stale_skip_loop_inprocess", run_id=rid) - continue - hard_expired = False # 孤儿 loop：不按年龄硬杀，交给静默判据 - - # A legitimate user decision may begin near the run's absolute-age - # ceiling. Its own bounded wait deadline is the authority; do not cut a - # newly opened question/confirmation down to the remainder of the run - # hard cap. Once the registry resolves or times out, this exemption - # disappears automatically on the next sweep. - if hard_expired and human_interaction.has_pending(cid): - logger.info("chat_run_stale_skip_human_wait", run_id=rid, chat_id=cid) - continue - - # 工作流模式的批量作业：run 卡在 run_job(wait=True) 里等作业，期间主链路一个 - # 流事件都不产生（逐项结果按设计不进对话），"流静默"判据必然误判。进程内 task - # 豁免只在本进程有效——多 worker / 多副本部署里，另一个进程看不到这个 task， - # 照样会把健康的长作业杀掉。所以这里用**跨进程可见的证据**：DB 里这个会话是否 - # 还有正在推进的作业（每次子调用都会 add_usage → jobs.updated_at 前进）。 - if _chat_has_live_job(cid, now): - logger.info("chat_run_stale_skip_live_job", run_id=rid, chat_id=cid) - continue - - if not hard_expired: - # A run whose asyncio task is alive in THIS process is never a - # zombie — the in-process inactivity watchdog owns hang detection - # for it. The stream-quiet check below is only for orphans whose - # worker vanished (process restart/crash). Without this guard, a - # healthy long run streaming huge tool-call args (which map to no - # stream events) past 30min would be reaped as "quiet". - task = _active_runs.get(rid) - if task is not None and not task.done(): - logger.info("chat_run_stale_skip_inprocess", run_id=rid) - continue - last_ms = await _stream_last_write_ms(rid) - if last_ms is not None and (now_ms - last_ms) < _STALE_QUIET_SEC * 1000: - # The stream is still producing: this is a long task, not a zombie — skip this round - logger.info( - "chat_run_stale_skip_active", - run_id=rid, - quiet_sec=round((now_ms - last_ms) / 1000, 1), - ) - continue - reason = "run stalled: no stream activity (stale watchdog)" - else: - reason = "run exceeded hard max age (stale watchdog)" - - if not _request_run_stop( - rid, - status="failed", - error_message=reason, - require_expired_lease=not hard_expired, - ): - continue # race: the worker just beat us to writing the terminal state - - await _write_terminal_to_stream( - rid, - chat_id=cid or "", - error_text="任务长时间无响应，已被系统中止，请重新发起", - ) - - payload = row.request_payload if isinstance(row.request_payload, dict) else {} - task = _active_runs.get(rid) - if task is not None and not task.done() and payload.get("kind") not in _COOPERATIVE_KINDS: - task.cancel() - try: - await asyncio.wait_for(task, timeout=2.0) - except (asyncio.TimeoutError, asyncio.CancelledError): - pass - except Exception as exc: - logger.warning("stale_reaper_await_failed", run_id=rid, error=str(exc)) - if task.done(): - _acknowledge_terminal_writer(rid) - reaped += 1 - - if reaped: - logger.info("chat_run_stale_reaped", count=reaped) - return reaped - - -async def run_stale_reaper_loop() -> None: - """Recover expired leases frequently and run the slower stale watchdog.""" - loop = asyncio.get_running_loop() - next_reap_at = loop.time() + _STALE_REAPER_INTERVAL_SEC - while True: - try: - await asyncio.sleep(_RUN_RECOVERY_INTERVAL_SEC) - await recover_orphan_runs() - if loop.time() >= next_reap_at: - await reap_stale_runs() - from core.services.tool_effect_ledger import ToolEffectJournal - - pruned = await asyncio.to_thread(ToolEffectJournal(SessionLocal).prune_settled) - if pruned: - logger.info("tool_effect_ledger_pruned", count=pruned) - next_reap_at = loop.time() + _STALE_REAPER_INTERVAL_SEC - except asyncio.CancelledError: - raise - except Exception: # noqa: BLE001 - logger.warning("chat_run_stale_reaper_iteration_failed", exc_info=True) - - -# Runs the chat UI never represents as "this conversation is busy": background -# jobs that own their own surface (automation panel, internal job -# agents) plus the retired legacy stream. -_BACKGROUND_RUN_KINDS = frozenset( - { - "automation_plan", - "automation_prompt", - "internal_job_agent", - "legacy_chat_stream", - } -) - - -def _run_kind(run: ChatRun) -> Any: - payload = run.request_payload if isinstance(run.request_payload, dict) else {} - return payload.get("kind") - - -def get_active_run_for_chat(chat_id: str, user_id: str) -> Optional[ChatRun]: - """Backing query for GET /v1/chats/{chat_id}/active-run.""" - with SessionLocal() as db: - rows = ( - db.query(ChatRun) - .filter( - ChatRun.chat_id == chat_id, - ChatRun.user_id == user_id, - (ChatRun.status.in_(_LIVE_STATUSES) | (ChatRun.writer_slot.isnot(None))), - ) - .order_by(ChatRun.created_at.desc()) - .all() - ) - for row in rows: - if _run_kind(row) not in _BACKGROUND_RUN_KINDS: - return row - return None - - -def has_local_runs() -> bool: - """这个进程手上有没有正在跑的 run。 - - 用来在查库之前便宜地排掉"本进程什么都没在跑"的情况 —— 谁在跑哪个 run 只有进程自己 - 知道（``_active_runs`` 是进程内的），而有些判断必须落到具体进程上：确认条的 pending - 与前端信号队列都活在进程里，只有执行这个 run 的进程弹得出来。 - """ - return bool(_active_runs) - - -def is_local_run(run_id: str) -> bool: - """这个 run 是不是由本进程在执行。""" - return run_id in _active_runs - - -# A tab hidden longer than this catches up through the session list on reload. -MAX_RECENT_RUN_SECONDS = 86400 - - -def list_active_runs_for_user( - user_id: str, recent_seconds: int = 0 -) -> Tuple[List[ChatRun], List[str]]: - """Backing query for GET /v1/chats/active-runs, in one query. - - Returns one live run per chat: only genuinely live rows (``pending`` / - ``running``) qualify. The per-chat probe additionally accepts a lingering - ``writer_slot`` so a reconnecting client can clean its own zombie UI up; - this listing feeds the sidebar's "still running" dot, where a terminal row - must not light anything. - - With ``recent_seconds`` it also returns the chats whose runs started in that - window, newest first, so a page finds a chat started elsewhere (Personal - Agent, API, another device) even when its run ended between two checks. - """ - live = ChatRun.status.in_(_LIVE_STATUSES) - since = _utcnow() - timedelta(seconds=min(recent_seconds, MAX_RECENT_RUN_SECONDS)) - with SessionLocal() as db: - rows = ( - db.query(ChatRun) - .filter( - ChatRun.user_id == user_id, - (live | (ChatRun.created_at >= since)) if recent_seconds else live, - # Module-owned chats (Personal Agent) never belong in the ordinary sidebar. - ordinary_chat_filter(ChatRun.chat_id), - ) - .order_by(ChatRun.created_at.desc()) - .all() - ) - latest_per_chat: Dict[str, ChatRun] = {} - recent: Dict[str, None] = {} - for row in rows: - if _run_kind(row) in _BACKGROUND_RUN_KINDS: - continue - if row.status in _LIVE_STATUSES: - latest_per_chat.setdefault(row.chat_id, row) - if recent_seconds and row.created_at >= since: - recent.setdefault(row.chat_id) - return list(latest_per_chat.values()), list(recent) - - -# ─── SSE wire wrapper (shared by the chats / plans routes) ───────────── - - -async def follow_run_as_sse( - run_id: str, - *, - chat_id: str, - from_offset: int = 0, - error_event_factory: Optional[Callable[[str], Dict[str, Any]]] = None, -) -> AsyncIterator[str]: - """Wrap follow_run as SSE wire frames. - - ``error_event_factory(reason)`` lets callers customize the error event shape - (chat protocol vs plan protocol). Defaults to a chat-style ``{type: error}``. - """ - - def _default_err(reason: str) -> Dict[str, Any]: - return {"type": "error", "error": reason, "chat_id": chat_id} - - factory = error_event_factory or _default_err - - # Decouple follow_run from the yield cadence via an intermediate queue: it - # lets wait_for write an SSE comment line to the wire when the stream has - # been silent longer than _HEARTBEAT_INTERVAL_SEC, serving as nginx / - # reverse-proxy keepalive. Cancelling queue.get via wait_for is safe - # (asyncio.Queue handles the cancel race); the underlying follow_run - # coroutine is never interrupted. - queue: asyncio.Queue = asyncio.Queue() - - async def _pump() -> None: - try: - async for event in follow_run(run_id, from_offset=from_offset): - await queue.put(("event", event)) - except ChatRunNotFound: - await queue.put(("not_found", None)) - except Exception as exc: # noqa: BLE001 - await queue.put(("error", exc)) - else: - await queue.put(("end", None)) - - pump_task = asyncio.create_task(_pump(), name=f"sse_pump:{run_id}") - try: - while True: - try: - kind, payload = await asyncio.wait_for(queue.get(), timeout=_HEARTBEAT_INTERVAL_SEC) - except asyncio.TimeoutError: - yield ": heartbeat\n\n" - continue - - if kind == "event": - clean = {k: v for k, v in payload.items() if not k.startswith("_")} - yield f"data: {json.dumps(clean, ensure_ascii=False)}\n\n" - elif kind == "end": - break - elif kind == "not_found": - yield f"data: {json.dumps(factory('run not found'), ensure_ascii=False)}\n\n" - break - elif kind == "error": - logger.warning( - "follow_run_as_sse_failed", run_id=run_id, error=str(payload), exc_info=payload - ) - yield f"data: {json.dumps(factory('流式响应中断'), ensure_ascii=False)}\n\n" - break - finally: - pump_task.cancel() - with contextlib.suppress(BaseException): - await pump_task - - yield "data: [DONE]\n\n" diff --git a/src/backend/orchestration/citation_anchor.py b/src/backend/orchestration/citation_anchor.py deleted file mode 100644 index d695f636..00000000 --- a/src/backend/orchestration/citation_anchor.py +++ /dev/null @@ -1,551 +0,0 @@ -"""统一证据锚点（Evidence Anchor）：引用编号的唯一真源。 - -设计（见 internal design docs）： -- 编号权收归后端 —— ``AnchorAllocator`` 按会话内单调计数发 ``e1、e2、…`` 全局锚点； - 模型只负责把工具结果里标注的 ``cite_id`` 原样复制进 ``[锚文本](cite:eN)``，不做计算。 -- ``CitationAnchorMiddleware``（core/llm/middlewares.py）在工具结果回给模型前调用 - :func:`annotate_tool_result` 完成 提取 → 发号 → 回注，并把 :class:`AnchorCitation` - 登记进 allocator；workflow / batch / autonomous 按 tool_id 从 allocator 精确取引用。 - 发号器是唯一编号方——旧的 per-tool 偏移提取（orchestration/citations.py）已随本方案删除。 -- 提取四层降级：工具自声明 ``__citations__`` → 工具规格注册表（TOOL_SPECS 配置， - 加工具只加配置不改代码）→ 通用启发式（唯一字典数组字段）→ 整份结果 1 个锚点。 - 操作型工具（写文件 / pin / 任务增删改等）在 SKIP_TOOLS 里直接放行不发号。 - -约定（工具开发规范）：需要精确控制引用粒度的工具，返回 JSON 里带 -``"__citations__": [{"title": …, "url": …, "snippet": …, "source_type": …}, …]``， -条目顺序与结果正文对应；中间件会就地为每条注入 ``cite_id`` 并登记。 -""" - -from __future__ import annotations - -import json -import logging -import re -from contextvars import ContextVar -from dataclasses import asdict, dataclass, field -from typing import Any, Dict, List, Optional, Sequence, Tuple - -logger = logging.getLogger(__name__) - -ANCHOR_RE = re.compile(r"^e(\d+)$") - - -# ── 引用条目 ──────────────────────────────────────────────────────────────── - - -@dataclass -class AnchorCitation: - """带全局锚点的引用条目（CitationItem 的超集，dict 形态向后兼容）。""" - - id: str # 全局锚点，如 "e7" - tool_name: str - tool_id: Optional[str] - title: str - url: str - snippet: str - source_type: str - item_index: int = -1 # 条目在该次工具结果列表中的 0-based 下标；整体型为 -1 - - def to_dict(self) -> Dict[str, Any]: - return asdict(self) - - -# ── 工具规格注册表（配置即真源；新工具加一行，不改代码） ───────────────────── - -# 标题/链接/摘要字段别名（按序取第一个非空；覆盖存量工具的中英文键名） -DEFAULT_TITLE_KEYS: Tuple[str, ...] = ( - "title", - "标题", - "文件名称", - "企业名称", - "产品名称", - "name", - "名称", - "document_name", -) -DEFAULT_URL_KEYS: Tuple[str, ...] = ("url", "链接", "link", "href") -DEFAULT_SNIPPET_KEYS: Tuple[str, ...] = ( - "content", - "文件内容", - "snippet", - "摘要", - "summary", - "description", - "abstract", - "text", -) -# 启发式识别列表字段时的候选名（按序优先） -LIST_FIELD_ALIASES: Tuple[str, ...] = ( - "items", - "results", - "events", - "pages", - "records", - "entries", - "data", - "list", - "rows", - "docs", -) - -_TITLE_MAX = 120 -_SNIPPET_MAX = 3000 -_WHOLE_SNIPPET_MAX = 500 - - -@dataclass(frozen=True) -class CitationSpec: - """单个工具的引用提取规格。mode: list（逐条）/ whole（整份 1 条）。""" - - mode: str = "whole" - # 列表字段路径（支持多层如 ("result", "results")；支持并列多列表） - items_paths: Tuple[Tuple[str, ...], ...] = () - title_keys: Tuple[str, ...] = DEFAULT_TITLE_KEYS - url_keys: Tuple[str, ...] = DEFAULT_URL_KEYS - snippet_keys: Tuple[str, ...] = DEFAULT_SNIPPET_KEYS - # 摘要拼接键（如企业搜索的 法定代表人·注册资金·企业状态）；命中则优先于 snippet_keys - snippet_join: Tuple[str, ...] = () - source_type: str = "unknown" - title: str = "" # whole 模式的显示标题；空则用工具名 - - -# 操作型 / 资产管理型工具：无可引用内容，中间件直接放行 -SKIP_TOOLS = frozenset( - { - # 自研文件/沙盒操作 - "Write", - "Edit", - "Bash", - "write_stdin", - "sandbox_put_artifact", - "sandbox_get_artifact", - "pin_to_workspace", - "channel_read_attachment", - "choose_design", - "ask_user_question", - "view_text_file", - "get_data_context", - "load_skill", - "Glob", - "Grep", - # 产物/发布/批量 - "publish_site", - # 定时任务写操作 - "create_scheduled_task", - "update_scheduled_task", - "pause_scheduled_task", - "resume_scheduled_task", - "delete_scheduled_task", - "list_scheduled_tasks", - "list_channel_conversations", - "get_scheduled_task", - # 技能管理 - "search_marketplace", - "list_my_skills", - "install_from_marketplace", - "register_skill", - "submit_to_marketplace", - "delete_skill", - "edit_skill", - # 能力/资产发现 - "list_datasets", - # 子智能体的引用由其内部工具的嵌套事件承载 - "call_subagent", - } -) - -TOOL_SPECS: Dict[str, CitationSpec] = { - # 检索/知识类（逐条） - "internet_search": CitationSpec( - mode="list", - items_paths=(("result", "results"), ("results",)), - source_type="internet", - ), - "retrieve_dataset_content": CitationSpec( - mode="list", items_paths=(("items",),), source_type="knowledge_base" - ), - "retrieve_local_kb": CitationSpec( - mode="list", items_paths=(("items",),), source_type="knowledge_base" - ), - "wiki_fetch_source": CitationSpec( - mode="list", items_paths=(("items",),), source_type="knowledge_base" - ), - "wiki_locate": CitationSpec( - mode="list", items_paths=(("pages",),), source_type="knowledge_base" - ), - "wiki_read_page": CitationSpec(mode="whole", title="Wiki 页面", source_type="knowledge_base"), - "web_fetch": CitationSpec(mode="whole", title="网页内容", source_type="internet"), - # 数据/报告类（整份） - "query_database": CitationSpec(mode="whole", title="数据库查询结果", source_type="database"), -} - -# ── 发号器（run 级共享，子智能体经 ContextVar 继承同一实例） ──────────────── - - -class AnchorAllocator: - """会话内单调锚点计数器 + 按 tool_id 的引用注册表。 - - 并发安全性：分配是同步代码段（无 await 点），asyncio 并行工具任务 - 通过 ContextVar 共享同一实例，不会重号。 - """ - - def __init__(self, start: int = 1) -> None: - self._next = max(1, int(start or 1)) - self._by_tool_id: Dict[str, List[AnchorCitation]] = {} - self.all: List[AnchorCitation] = [] - - def next_id(self) -> str: - aid = f"e{self._next}" - self._next += 1 - return aid - - def register(self, tool_id: Optional[str], items: Sequence[AnchorCitation]) -> None: - if not items: - return - self.all.extend(items) - if tool_id: - self._by_tool_id.setdefault(str(tool_id), []).extend(items) - - def citations_for(self, tool_id: Optional[str]) -> List[AnchorCitation]: - if not tool_id: - return [] - return list(self._by_tool_id.get(str(tool_id), [])) - - -# 子智能体链路的兜底通道：call_subagent 的工具函数与父 agent 在同一 task 调用链， -# 能读到父中间件写入的实例。**不作为主通道**——`astream_chat_workflow` 是 async -# generator，其 `set()` 与 agent 实际执行所在的 task 上下文不互通（实测：中间件 -# 侧新建了自己的实例、编排层侧读到 None 回退旧提取）。主通道见 `_ALLOC_ATTR`。 -CITATION_ALLOCATOR: ContextVar[Optional[AnchorAllocator]] = ContextVar( - "jx_citation_allocator", default=None -) - -# 主通道：把发号器挂在 agent 实例上——中间件的 on_acting 拿得到 agent， -# 编排层持有 create_agent_executor 返回的同一个 agent，天然同源、不受上下文隔离影响。 -_ALLOC_ATTR = "_jx_citation_allocator" - - -def attach_allocator(agent: Any, allocator: AnchorAllocator) -> AnchorAllocator: - """run 入口调用：把发号器绑定到 agent（并写 ContextVar 供子链路兜底）。""" - try: - setattr(agent, _ALLOC_ATTR, allocator) - except Exception as exc: # noqa: BLE001 - pydantic 模型等禁止动态属性时降级到 ContextVar - logger.debug("[citation-anchor] attach to agent failed: %s", exc) - CITATION_ALLOCATOR.set(allocator) - return allocator - - -def resolve_allocator(agent: Any) -> AnchorAllocator: - """中间件调用：取本 run 的发号器；缺失时就地建一个并绑定，保证始终可用。""" - allocator = getattr(agent, _ALLOC_ATTR, None) - if isinstance(allocator, AnchorAllocator): - return allocator - allocator = CITATION_ALLOCATOR.get() - if not isinstance(allocator, AnchorAllocator): - allocator = AnchorAllocator() - return attach_allocator(agent, allocator) - - -def collect_citation_dicts( - tool_id: Optional[str], - allocator: Optional[AnchorAllocator] = None, -) -> List[Dict[str, Any]]: - """编排层统一取引用入口（dict 形态，直接可进 SSE / trace / 落库）。 - - 发号器是**唯一**编号方：中间件在结果回给模型前已完成注号并按 tool_id 登记， - 这里只做精确取回，不做二次提取。``allocator`` 由调用方显式传入（run 入口绑定 - 在 agent 上的那一个）；不传时退到 ContextVar（子智能体链路）。 - """ - if allocator is None: - allocator = CITATION_ALLOCATOR.get() - if allocator is None: - return [] - return [item.to_dict() for item in allocator.citations_for(tool_id)] - - -def anchor_start_for_chat(chat_id: Optional[str]) -> int: - """跨轮续号：扫描该会话已落库消息的 citations，取最大锚点 + 1。 - - 任何异常都回退 1（引用功能降级，不影响对话）。 - """ - if not chat_id: - return 1 - try: - from core.db.engine import SessionLocal - from core.db.models import ChatMessage - - max_seen = 0 - with SessionLocal() as db: - rows = ( - db.query(ChatMessage.extra_data) - .filter(ChatMessage.chat_id == chat_id) - .order_by(ChatMessage.chat_seq.desc()) - .limit(200) - .all() - ) - for (extra,) in rows: - for cit in (extra or {}).get("citations") or []: - m = ANCHOR_RE.match(str(cit.get("id", "")) if isinstance(cit, dict) else "") - if m: - max_seen = max(max_seen, int(m.group(1))) - return max_seen + 1 - except Exception as exc: # noqa: BLE001 - logger.debug("[citation-anchor] start scan failed for chat %s: %s", chat_id, exc) - return 1 - - -# ── 提取 + 回注 ───────────────────────────────────────────────────────────── - - -def _first_str(item: Dict[str, Any], keys: Sequence[str]) -> str: - for key in keys: - val = item.get(key) - if isinstance(val, (str, int, float)) and str(val).strip(): - return str(val) - return "" - - -def _normalize_citation_url(raw: str) -> str: - """把工具结果里的来源链接规整成「浏览器点了不会跑偏」的绝对地址。 - - 检索工具返回的 url 千奇百怪，直接透传会有两类坏结果： - - - **缺协议**（``www.example.gov.cn/a.html``）：前端 ```` 会把它当相对 - 路径，按当前站点 origin 解析，于是点开变成 ``http://<本平台>/www.example…``， - 落到我们自己的 nginx 上返回 404 —— 看起来像「引用地址错误」。 - - **纯相对路径**（``/module/download/x.doc``）：没有站点根就无从还原，同样会 - 指到本平台。这种宁可不给链接，也好过给一个必然打不开的。 - - 另外只放行 http/https：javascript:、data: 之类既没有意义也不该出现在来源里。 - """ - url = (raw or "").strip() - if not url: - return "" - lowered = url.lower() - if lowered.startswith(("http://", "https://")): - return url - # 协议相对：//example.com/a → https://example.com/a - if url.startswith("//"): - return f"https:{url}" - # 明确的其它协议（javascript:/data:/ftp: …）一律丢弃 - if re.match(r"^[a-zA-Z][a-zA-Z0-9+.\-]*:", url): - return "" - # 相对路径：没有站点根，还原不出来 - if url.startswith("/"): - return "" - # 剩下的形如 example.gov.cn/path —— 看着像域名就补上协议 - host = url.split("/", 1)[0] - if "." in host and " " not in host: - return f"https://{url}" - return "" - - -def _dig(data: Dict[str, Any], path: Sequence[str]) -> Any: - cur: Any = data - for key in path: - if not isinstance(cur, dict): - return None - cur = cur.get(key) - return cur - - -def _display_title(tool_name: str, spec: Optional[CitationSpec]) -> str: - if spec and spec.title: - return spec.title - return f"{tool_name} 结果" - - -def _item_citation( - aid: str, - tool_name: str, - tool_id: Optional[str], - spec: CitationSpec, - item: Dict[str, Any], - index: int, -) -> AnchorCitation: - title = _first_str(item, spec.title_keys)[:_TITLE_MAX] or _display_title(tool_name, spec) - if spec.snippet_join: - parts = [_first_str(item, (k,)) for k in spec.snippet_join] - snippet = " · ".join(p for p in parts if p) - else: - snippet = _first_str(item, spec.snippet_keys) - return AnchorCitation( - id=aid, - tool_name=tool_name, - tool_id=tool_id, - title=title, - url=_normalize_citation_url(_first_str(item, spec.url_keys)), - snippet=snippet[:_SNIPPET_MAX], - source_type=spec.source_type, - item_index=index, - ) - - -def _looks_like_error(data: Dict[str, Any]) -> bool: - if not data.get("error"): - return False - # 列表字段仍有内容时不算整体失败（部分工具 error+items 并存） - for key in LIST_FIELD_ALIASES: - val = data.get(key) - if isinstance(val, list) and val: - return False - return True - - -def _heuristic_list_field(data: Dict[str, Any]) -> Optional[Tuple[Tuple[str, ...], List[Any]]]: - """启发式找列表字段：别名优先；否则要求「唯一」的字典数组字段。""" - scopes: List[Tuple[Tuple[str, ...], Dict[str, Any]]] = [((), data)] - inner = data.get("result") - if isinstance(inner, dict): - scopes.append((("result",), inner)) - for prefix, scope in scopes: - for alias in LIST_FIELD_ALIASES: - val = scope.get(alias) - if isinstance(val, list) and val and all(isinstance(x, dict) for x in val): - return prefix + (alias,), val - for prefix, scope in scopes: - candidates = [ - (key, val) - for key, val in scope.items() - if isinstance(val, list) and val and all(isinstance(x, dict) for x in val) - ] - if len(candidates) == 1: - key, val = candidates[0] - return prefix + (key,), val - return None - - -def _extract_and_inject( - tool_name: str, - tool_id: Optional[str], - data: Dict[str, Any], - allocator: AnchorAllocator, -) -> List[AnchorCitation]: - """在 data 上就地注入 cite_id，返回登记的引用条目。""" - spec = TOOL_SPECS.get(tool_name) - - # L1：工具自声明 __citations__（优先级最高，条目顺序即 item_index） - declared = data.get("__citations__") - if isinstance(declared, list) and declared and all(isinstance(x, dict) for x in declared): - base = spec or CitationSpec(source_type="unknown") - out: List[AnchorCitation] = [] - for idx, entry in enumerate(declared): - aid = allocator.next_id() - entry["cite_id"] = aid - out.append( - AnchorCitation( - id=aid, - tool_name=tool_name, - tool_id=tool_id, - title=str(entry.get("title") or _display_title(tool_name, spec))[:_TITLE_MAX], - url=_normalize_citation_url(str(entry.get("url") or "")), - snippet=str(entry.get("snippet") or "")[:_SNIPPET_MAX], - source_type=str(entry.get("source_type") or base.source_type), - item_index=idx, - ) - ) - return out - - if _looks_like_error(data): - return [] - - # L2：配置映射（列表型） - if spec and spec.mode == "list": - out = [] - for path in spec.items_paths: - items = _dig(data, path) - if not isinstance(items, list): - continue - for item in items: - if not isinstance(item, dict) or item.get("error"): - continue - aid = allocator.next_id() - item["cite_id"] = aid - out.append(_item_citation(aid, tool_name, tool_id, spec, item, len(out))) - return out - - # L3：无 spec → 启发式找唯一字典数组 - if spec is None: - found = _heuristic_list_field(data) - if found is not None: - _path, items = found - generic = CitationSpec(mode="list", source_type="unknown") - out = [] - for item in items: - if item.get("error"): - continue - aid = allocator.next_id() - item["cite_id"] = aid - out.append(_item_citation(aid, tool_name, tool_id, generic, item, len(out))) - if out: - return out - - # L4：整份 1 条（显式 whole spec 或彻底认不出） - aid = allocator.next_id() - data["cite_id"] = aid - snippet = json.dumps(data, ensure_ascii=False)[:_WHOLE_SNIPPET_MAX] - return [ - AnchorCitation( - id=aid, - tool_name=tool_name, - tool_id=tool_id, - title=_display_title(tool_name, spec), - url=_normalize_citation_url(_first_str(data, (spec or CitationSpec()).url_keys)), - snippet=snippet, - source_type=(spec.source_type if spec else "unknown"), - item_index=-1, - ) - ] - - -def annotate_tool_result( - tool_name: str, - tool_id: Optional[str], - text: str, - allocator: AnchorAllocator, -) -> Tuple[str, List[AnchorCitation]]: - """给一份工具结果文本注号。 - - 返回 (注号后的文本, 登记的引用条目)。不可注号（跳过名单 / 空结果 / - 解析失败且判定无引用价值）时原样返回 (text, [])。**绝不抛错**。 - """ - if not text or tool_name in SKIP_TOOLS: - return text, [] - try: - parsed = json.loads(text) - except (json.JSONDecodeError, TypeError): - parsed = None - - try: - if isinstance(parsed, dict): - items = _extract_and_inject(tool_name, tool_id, parsed, allocator) - if not items: - return text, [] - return json.dumps(parsed, ensure_ascii=False), items - if isinstance(parsed, list): - # 裸数组结果：包壳后按启发式处理，回注仍输出裸数组语义（保持兼容 → 整份 1 条） - parsed = {"items": parsed} - items = _extract_and_inject(tool_name, tool_id, parsed, allocator) - if not items: - return text, [] - return json.dumps(parsed["items"], ensure_ascii=False), items - # 纯文本结果：整份 1 条，文末追加锚点行 - spec = TOOL_SPECS.get(tool_name) - if spec and spec.mode == "list": - # 声明为列表型但拿到纯文本 → 形态异常，放行不注号 - return text, [] - aid = allocator.next_id() - item = AnchorCitation( - id=aid, - tool_name=tool_name, - tool_id=tool_id, - title=_display_title(tool_name, spec), - url="", - snippet=str(text)[:_SNIPPET_MAX], - source_type=(spec.source_type if spec else "unknown"), - item_index=-1, - ) - return f"{text}\n\n[cite_id: {aid}]", [item] - except Exception as exc: # noqa: BLE001 - logger.warning("[citation-anchor] annotate failed tool=%s: %s", tool_name, exc) - return text, [] diff --git a/src/backend/orchestration/commands/__init__.py b/src/backend/orchestration/commands/__init__.py deleted file mode 100644 index 8dbc864c..00000000 --- a/src/backend/orchestration/commands/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Managed ordinary Bash commands and terminal conversation events.""" diff --git a/src/backend/orchestration/commands/runtime.py b/src/backend/orchestration/commands/runtime.py deleted file mode 100644 index 23c1ed07..00000000 --- a/src/backend/orchestration/commands/runtime.py +++ /dev/null @@ -1,262 +0,0 @@ -"""One output collector per command; no model-driven polling.""" -import asyncio -import logging -import time -import uuid -from functools import wraps -from core.services.commands import store -from core.sandbox.protocol import ProcessRequest -from core.sandbox.providers.opensandbox.recovery import execution_reference -from core.sandbox.providers.opensandbox.recovery import recover_execution -from core.sandbox.providers.opensandbox.recovery import release_execution - -logger = logging.getLogger(__name__) -LAUNCH_RENEW_SECONDS = 10 - -def service_loop(method): - """Keep command collectors on their service loop across agent worker threads.""" - @wraps(method) - async def invoke(self, *args, **kwargs): - current = asyncio.get_running_loop() - if self.loop is None: - self.loop = current - if self.loop is current: - return await method(self, *args, **kwargs) - if not self.loop.is_running(): - raise RuntimeError("Command service loop is not running") - future = asyncio.run_coroutine_threadsafe(method(self, *args, **kwargs), self.loop) - return await asyncio.wrap_future(future) - return invoke - -class CommandRuntime: - def __init__(self): - self.loop = None - self.owner = uuid.uuid4().hex - self.tasks = {} - self.providers = {} - self._locks = {} - - def bind_loop(self): - """Initialize every worker before a temporary agent thread can call it.""" - current = asyncio.get_running_loop() - if self.loop is not None and self.loop is not current: - raise RuntimeError("Command runtime already belongs to another event loop") - self.loop = current - - @service_loop - async def start(self, provider, req, *, chat_id, source_run_id, - background=False, yield_time_ms=60000): - row = await asyncio.to_thread(store.create, req, chat_id, source_run_id, self.owner) - tid = row["task_id"] - result = None - try: - result = await self._launch(tid, provider, req, - yield_time_ms=250 if background else yield_time_ms) - return await self._attach(tid, provider, req, result) - except BaseException as exc: - if result and result.get("status") == "running" and tid not in self.providers: - # Persistence can fail after launch (for example a hard-deleted chat). - stopped = await provider.write_stdin(result["session_id"], - sandbox_session_id=req.session_id, user_id=req.user_id, - chars="\x03", yield_time_ms=1000) - while stopped.get("status") == "running": - stopped = await provider.write_stdin(result["session_id"], - sandbox_session_id=req.session_id, user_id=req.user_id, - chars="", yield_time_ms=1000) - if stopped.get("status") != "exited": - raise RuntimeError("Unattached process stop not confirmed") from exc - try: - await asyncio.to_thread(store.finish, tid, "unknown", - "Launch state unknown; do not rerun: " + str(exc)) - except LookupError: - pass # A hard-deleted chat already removed the task. - raise - - async def _launch(self, tid, provider, req, *, yield_time_ms): - async def renew(): - while True: - await asyncio.sleep(LAUNCH_RENEW_SECONDS) - await asyncio.to_thread(store.renew_launch, tid, self.owner) - launch = asyncio.create_task(provider.start_process(req, yield_time_ms=yield_time_ms)) - lease = asyncio.create_task(renew()) - try: - done, _ = await asyncio.wait((launch, lease), return_when=asyncio.FIRST_COMPLETED) - if launch in done: - return await launch - await lease # Lease loss cancels the pending launch; never restart it. - raise RuntimeError("Command launch lease ended unexpectedly") - finally: - for task in (launch, lease): - if not task.done(): - task.cancel() - await asyncio.gather(launch, lease, return_exceptions=True) - - async def _attach(self, tid, provider, req, result): - reference = None - if result.get("status") == "running": - reference = await execution_reference(provider, result["session_id"], req) - if reference is not None: - reference["cursor"] = result["execution_cursor"] - row = await asyncio.to_thread(store.attach, tid, self.owner, result, reference) - payload = await asyncio.to_thread(store.read, tid, req.user_id, req.session_id, - consume_notice=False) - if row["status"] == "running": - self.providers[tid] = provider, req, result["session_id"] - self.tasks[tid] = asyncio.create_task(self._monitor(tid, row)) - payload["next"] = "Command is managed in the background. Completion will notify this chat. Do independent work; do not poll or restart it. If no independent work remains, end this response once; completion will resume this chat." - return payload - - @service_loop - async def adopt(self, provider, process_id, *, user_id, session_id, chat_id, source_run_id): - if process_id.startswith("cmd_"): - await asyncio.to_thread(store.owned, process_id, user_id, session_id) - return await asyncio.to_thread(store.read, process_id, user_id, session_id, consume_notice=False) - req = ProcessRequest(script_content="[adopted command]", script_name="_managed.sh", - language="bash", session_id=session_id, user_id=user_id) - row = await asyncio.to_thread(store.create, req, chat_id, source_run_id, self.owner, process_id) - if not row["created"]: - return await asyncio.to_thread(store.read, row["task_id"], user_id, session_id, consume_notice=False) - try: - result = await provider.write_stdin(process_id, sandbox_session_id=session_id, - user_id=user_id, chars="", yield_time_ms=0) - return await self._attach(row["task_id"], provider, req, result) - except BaseException: - await asyncio.to_thread(store.finish, row["task_id"], "unknown", "Process adoption failed") - raise - - async def _monitor(self, tid, row): - provider, req, pid = self.providers[tid] - last_touch = 0 - try: - while True: - if not await asyncio.to_thread(store.chat_available, tid): - result = await self._interrupt(tid) - try: - await asyncio.to_thread(store.control_result, tid, self.owner, - "cancelled", "Conversation removed", True, result) - except LookupError: - pass # Physical deletion removed the row; the owned process was stopped. - break - row = await asyncio.to_thread(store.snapshot, tid) - if row["status"] not in store.LIVE or row["owner"] != self.owner: - await release_execution(provider, pid, req) - break - if row["cancel_requested"]: - result = await self._interrupt(tid) - await asyncio.to_thread(store.control_result, tid, self.owner, "cancelled", "", True, result) - break - if row["deadline_at"] and time.time() >= row["deadline_at"]: - result = await self._interrupt(tid) - await asyncio.to_thread(store.control_result, tid, self.owner, "failed", - "Command execution deadline exceeded", False, result) - break - async with self._locks.setdefault(tid, asyncio.Lock()): - result = await provider.write_stdin(pid, sandbox_session_id=req.session_id, - user_id=req.user_id, chars="", yield_time_ms=250) - row = await asyncio.to_thread(store.collect, tid, self.owner, result) - if row["status"] not in store.LIVE: - break - if time.monotonic() - last_touch >= 10: - await provider.touch_session(req.session_id) - last_touch = time.monotonic() - await asyncio.sleep(0.5) - except asyncio.CancelledError: - if row["execution_ref"]: - await release_execution(provider, pid, req) - await asyncio.to_thread(store.release_lease, tid, self.owner) - else: - result = await self._interrupt(tid) - await asyncio.to_thread(store.control_result, tid, self.owner, "cancelled", - "Command owner stopped", True, result) - raise - except Exception as exc: - logger.exception("Command monitor lost task=%s", tid) - await asyncio.to_thread(store.finish, tid, "unknown", str(exc), owner=self.owner) - finally: - self.providers.pop(tid, None) - self.tasks.pop(tid, None) - self._locks.pop(tid, None) - - async def _interrupt(self, tid): - provider, req, pid = self.providers[tid] - async with self._locks.setdefault(tid, asyncio.Lock()): - result = await provider.write_stdin(pid, sandbox_session_id=req.session_id, - user_id=req.user_id, chars="\x03", yield_time_ms=1000) - while result.get("status") == "running": - try: - await asyncio.to_thread(store.collect, tid, self.owner, result) - except LookupError: - pass # A hard-deleted chat still owns a process that must stop. - next_result = await provider.write_stdin(pid, sandbox_session_id=req.session_id, - user_id=req.user_id, chars="", yield_time_ms=1000) - result = next_result - if result.get("status") != "exited": - raise RuntimeError("Process stop not confirmed") - return result - - async def cancel(self, tid, user_id, session_id): - await asyncio.to_thread(store.owned, tid, user_id, session_id) - row = await asyncio.to_thread(store.snapshot, tid) - if row["status"] in store.TERMINAL: - return - await asyncio.to_thread(store.request_cancel, tid) - await self.wait(tid, user_id, session_id, 3000) - - async def wait(self, tid, user_id, session_id, wait_ms=0): - await asyncio.to_thread(store.owned, tid, user_id, session_id) - deadline = time.monotonic() + max(0, min(wait_ms, 300000)) / 1000 - while time.monotonic() < deadline: - if (await asyncio.to_thread(store.snapshot, tid))["status"] in store.TERMINAL: - break - await asyncio.sleep(0.05) - - @service_loop - async def recover(self, provider): - for row in await asyncio.to_thread(store.pending_recovery): - tid = row["task_id"] - if tid in self.tasks and not self.tasks[tid].done(): - continue - if not await asyncio.to_thread(store.claim, tid, self.owner): - continue - if not row["execution_ref"]: - await asyncio.to_thread(store.finish, tid, "unknown", - "No recoverable execution reference; original command will not be rerun", owner=self.owner) - continue - req = ProcessRequest(script_content="", script_name="_managed.sh", language="bash", - session_id=row["sandbox_session_id"], user_id=row["user_id"]) - try: - result = await recover_execution(provider, row["execution_ref"], req) - ref = {**row["execution_ref"], "cursor": result["execution_cursor"]} - recovered = await asyncio.to_thread(store.save_recovery, tid, self.owner, result, ref) - if recovered["status"] == "running": - self.providers[tid] = provider, req, result["session_id"] - self.tasks[tid] = asyncio.create_task(self._monitor(tid, recovered)) - except Exception as exc: - await asyncio.to_thread(store.finish, tid, "unknown", str(exc), owner=self.owner) - - @service_loop - async def shutdown(self): - tasks = list(self.tasks.values()) - for task in tasks: - if not task.done(): - task.cancel() - results = await asyncio.gather(*tasks, return_exceptions=True) - # A task cancelled before its first instruction cannot run its finally. - for tid, (provider, req, pid) in list(self.providers.items()): - row = await asyncio.to_thread(store.snapshot, tid) - if row["execution_ref"]: - await release_execution(provider, pid, req) - await asyncio.to_thread(store.release_lease, tid, self.owner) - else: - result = await self._interrupt(tid) - await asyncio.to_thread(store.control_result, tid, self.owner, - "cancelled", "Command owner stopped", True, result) - self.providers.pop(tid, None) - self._locks.pop(tid, None) - for result in results: - if isinstance(result, BaseException) and not isinstance(result, asyncio.CancelledError): - raise result - self.tasks.clear() - self.loop = None - -runtime = CommandRuntime() diff --git a/src/backend/orchestration/configuration.py b/src/backend/orchestration/configuration.py new file mode 100644 index 00000000..ad7aad89 --- /dev/null +++ b/src/backend/orchestration/configuration.py @@ -0,0 +1,6 @@ +"""Install database-backed configuration before loading application services.""" + +def register_configuration(): + from core.db.system_configuration import SystemConfigurationStore + from core.harness.registry.system_configuration import register_system_configuration_store + register_system_configuration_store(SystemConfigurationStore()) diff --git a/src/backend/orchestration/local_plugin_sidecars.py b/src/backend/orchestration/local_plugin_sidecars.py deleted file mode 100644 index 566546f4..00000000 --- a/src/backend/orchestration/local_plugin_sidecars.py +++ /dev/null @@ -1,75 +0,0 @@ -"""Start trusted bundled MCPs selected by owned local plugin definitions.""" -from __future__ import annotations -import asyncio -import sys -from urllib.parse import urlsplit - -_lock = asyncio.Lock() - - -def native_bindings(configs): - from core.config.mcp_config import _mcp_http_url - from core.config.settings import settings - from extensions.mcp_servers._ports import PORTS - - if not settings.deploy.is_local: - return set() - urls = { - _mcp_http_url(sid).rstrip("/"): (sid, port) for sid, port in PORTS.items() - if urlsplit(_mcp_http_url(sid)).hostname in ("127.0.0.1", "localhost", "::1") - } - return { - urls[str(cfg.get("url") or "").rstrip("/")] - for cfg in configs.values() - if cfg.get("origin") == "local_plugin" - and cfg.get("execution_scope") == "local" - and cfg.get("owner_user_id") - and str(cfg.get("url") or "").rstrip("/") in urls - } - - -async def ensure_native_servers(configs): - from orchestration import local_subprocess as host - - if not host._cloud_serves_tools(): - return # The local-only launcher already owns every bundled MCP. - bindings = native_bindings(configs) - if not bindings: - return - # Isolated agents execute on worker loops. Process handles, the watchdog - # and the serialization lock remain on the backend's lifecycle loop. - owner_loop = host._WATCHDOG.get_loop() if host._WATCHDOG is not None else asyncio.get_running_loop() - if owner_loop is not asyncio.get_running_loop(): - future = asyncio.run_coroutine_threadsafe(_start(bindings), owner_loop) - await asyncio.wrap_future(future) - return - await _start(bindings) - - -async def _start(bindings): - from extensions.mcp_servers._ports import package_name - from orchestration import local_subprocess as host - - async with _lock: - for sid, port in sorted(bindings): - label = "plugin_mcp:" + sid - running = next((p for name, p in host._PROCS if name == label and p.returncode is None), None) - if running is not None: - continue - if await host._tcp_port_ready("127.0.0.1", port): - raise RuntimeError("local plugin MCP port is already owned by another process") - proc = await host._spawn(label, [ - sys.executable, "-m", f"extensions.mcp_servers.{package_name(sid)}.server", - "--transport", "streamable-http", "--port", str(port), - ]) - if proc is None: - raise RuntimeError("local plugin MCP could not start") - try: - await host._wait_for_mcp_ports(proc, {sid: port}, timeout=host._ready_timeout_seconds()) - await host._list_mcp_tool_names(sid, port) - except BaseException: - from orchestration.local_sidecar_lifecycle import discard - - await discard(label, proc) - raise - host._start_watchdog() diff --git a/src/backend/orchestration/loops/__init__.py b/src/backend/orchestration/loops/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/backend/orchestration/loops/admission.py b/src/backend/orchestration/loops/admission.py new file mode 100644 index 00000000..7e706712 --- /dev/null +++ b/src/backend/orchestration/loops/admission.py @@ -0,0 +1,121 @@ +"""Admit durable autonomous Loop runs.""" + +from __future__ import annotations + +from typing import Any, Dict, Optional + +import core.db.engine as engine +import core.runs.ownership as ownership +from core.db.models import ChatRun +from core.infra.logging import get_logger + +logger = get_logger(__name__) + + +async def start_autonomous_loop_run( + *, + loop_id: str, + chat_id: str, + user_id: str, + goal_spec: Dict[str, Any], + budget: Dict[str, Any], + model_name: Optional[str] = None, + model_provider_id: Optional[str] = None, + evaluator_model: Optional[str] = None, + worker_max_iters: int = 15, + hitl_enabled: bool = False, + enable_thinking: bool = False, + chat_mode: Optional[str] = None, + is_resume: bool = False, + project_id: Optional[str] = None, + automation_run: bool = False, + on_accept=None, +) -> ChatRun: + """Start an autonomous-loop run (background task + Redis Stream, mirroring start_plan_execute_run).""" + if not chat_id: + raise ValueError("chat_id is required to start an autonomous loop run") + request_payload = { + "kind": "autonomous_loop", + "loop_id": loop_id, + "chat_id": chat_id, + "goal_spec": goal_spec, + "budget": budget, + "model_name": model_name, + "model_provider_id": model_provider_id, + "evaluator_model": evaluator_model, + "worker_max_iters": worker_max_iters, + "hitl_enabled": hitl_enabled, + "enable_thinking": enable_thinking, + # Thinking level (fast/medium/high/max): the active-run probe endpoint + # uses this to restore the frontend's resume phase; the worker uses it + # to set reasoning_effort. + "chat_mode": chat_mode, + "is_resume": is_resume, + "project_id": project_id, + "automation_run": automation_run, + } + # 启动参数跟 loop 持久化（而非只活在本次请求里）：崩溃/重启后的续跑——无论 API + # resume 还是启动自动续跑——都能还原同一套模型/评审模型/轮数/思考档位，不再悄悄 + # 降级到默认值。 + try: + from features.automation.loops import LoopService as _LoopSvc + + with engine.SessionLocal() as db: + _LoopSvc(db).save_start_params( + loop_id, + { + "model_name": model_name, + "model_provider_id": model_provider_id, + "evaluator_model": evaluator_model, + "worker_max_iters": worker_max_iters, + "hitl_enabled": hitl_enabled, + "enable_thinking": enable_thinking, + "chat_mode": chat_mode, + "automation_run": automation_run, + }, + ) + except Exception: # noqa: BLE001 - 参数存档失败不阻塞启动 + logger.warning("loop start_params persist failed", exc_info=True) + initial_args = { + "loop_id": loop_id, + "goal_spec": goal_spec, + "budget": budget, + "model_name": model_name, + "model_provider_id": model_provider_id, + "evaluator_model": evaluator_model, + "worker_max_iters": worker_max_iters, + "hitl_enabled": hitl_enabled, + "enable_thinking": enable_thinking, + "chat_mode": chat_mode, + "is_resume": is_resume, + "project_id": project_id, + "automation_run": automation_run, + } + run = ownership.create_run_record( + on_accept=on_accept, + chat_id=chat_id, + user_id=user_id, + request_payload=request_payload, + recovery_snapshot={ + "kind": "autonomous_loop", + "initial_args": initial_args, + "worker_args": { + "context": { + "model_name": model_name, + "model_provider_id": model_provider_id, + "chat_mode": chat_mode, + "project_ctx": {"project_id": project_id} if project_id else None, + "automation_run": automation_run, + } + }, + }, + ) + from orchestration.pending import dispatch_pending + await dispatch_pending(run) + logger.info( + "autonomous_loop_run_started", + run_id=run.run_id, + loop_id=loop_id, + user_id=user_id, + ) + return run diff --git a/src/backend/orchestration/autonomous_loop.py b/src/backend/orchestration/loops/driver.py similarity index 98% rename from src/backend/orchestration/autonomous_loop.py rename to src/backend/orchestration/loops/driver.py index f8af67f1..8490600f 100644 --- a/src/backend/orchestration/autonomous_loop.py +++ b/src/backend/orchestration/loops/driver.py @@ -40,7 +40,8 @@ from typing import Any, Awaitable, Callable, Dict, List, Optional from core.infra.logging import get_logger -from orchestration.loop_evaluator import ( +from orchestration.loops.planner import plan_requirements, replan_remaining, scout_workspace +from orchestration.loops.requirements import ( CONTINUE, DONE, NEED_HUMAN, @@ -48,8 +49,7 @@ decompose_requirements, extract_acceptance_criteria, ) -from orchestration.loop_planner import plan_requirements, replan_remaining, scout_workspace -from orchestration.subagents.loop_reviewer import review_requirement +from orchestration.loops.reviewer import review_requirement logger = get_logger(__name__) @@ -60,7 +60,7 @@ def _loop_scope(loop_id: str, phase: str, iteration: int = 0, requirement_id: str = "-") -> str: """Stable across process restarts; the durable loop ledger supplies iteration.""" - from core.capabilities.runtime import child_scope + from core.harness.registry.capability_runtime import child_scope return child_scope("", "autonomous-loop", loop_id, phase, str(iteration), requirement_id) @@ -115,7 +115,7 @@ def _build_wrapup_prompt(objective: str, ledger: Dict[str, Any], in_project: boo # Loop thresholds come from the active orchestration profile, resolved **per -# run** and **per tenant** (core.evolution.policies projects the profile onto +# run** and **per tenant** (core.evolution.profiles.policies projects the profile onto # the knobs this loop uses). # # Resolving at import — which is what this did — had two failure modes that @@ -124,7 +124,7 @@ def _build_wrapup_prompt(objective: str, ledger: Dict[str, Any], in_project: boo # of them had. Resolving without a tenant had a third: one tenant's published # retry counts and budget multiplier silently became everyone's. def _resolve_loop_policy(*, tenant_id: str, user_id: str = ""): - from core.evolution.policies import load_loop_policy + from core.evolution.profiles.policies import load_loop_policy return load_loop_policy(tenant_id=tenant_id, user_id=user_id) @@ -411,8 +411,8 @@ async def _run_worker_iteration( project and publish_site can locate the site by conversation, no longer an isolated draft. """ from core.llm.factory import create_agent_executor - from core.llm.mcp_manager import close_clients - from orchestration.streaming import StreamingAgent + from core.llm.mcp.manager import close_clients + from core.llm.runtime.streaming import StreamingAgent agent, clients = await create_agent_executor( current_user_id=user_id, @@ -439,11 +439,11 @@ async def _run_worker_iteration( trace: List[Dict[str, Any]] = [] citations: List[Dict[str, Any]] = [] # 证据锚点：每轮 worker 一个独立发号器，绑到 agent 上与中间件共享 - from orchestration.citation_anchor import AnchorAllocator, attach_allocator + from core.llm.citations.allocator import AnchorAllocator, attach_allocator _anchor_allocator = AnchorAllocator() attach_allocator(agent, _anchor_allocator) - from core.ontology.validator import requires_output_review + from core.ontology.output_review_state import requires_output_review runtime = ontology_runtime or {"enabled": False, "packs": [], "review_level": "none"} hold_output = requires_output_review(runtime) @@ -539,7 +539,7 @@ async def _run_worker_iteration( json.loads(tool_content) if isinstance(tool_content, str) else tool_content ) if isinstance(parsed_result, dict): - from orchestration.citation_anchor import collect_citation_dicts + from core.llm.citations.allocator import collect_citation_dicts citations.extend(collect_citation_dicts(tool_id or "", _anchor_allocator)) except (json.JSONDecodeError, TypeError, ValueError): @@ -563,7 +563,7 @@ async def _run_worker_iteration( usage = await sa.aget_usage() await close_clients(clients) if hold_output and text: - from orchestration.subagents.ontology_reviewer import review_ontology_output + from core.ontology.reviewer import review_ontology_output if emit: await emit( @@ -741,7 +741,7 @@ async def run_autonomous_loop( (pure task-style loop / tests). """ session = session_id or f"loop-{loop_id}" - from core.services.ontology_service import build_user_ontology_runtime + from core.ontology.runtime_policy import build_user_ontology_runtime # Resolved here, once per run, from the tenant's active profile. Held in a # local rather than a module constant so a concurrent run under a different diff --git a/src/backend/orchestration/loop_planner.py b/src/backend/orchestration/loops/planner.py similarity index 97% rename from src/backend/orchestration/loop_planner.py rename to src/backend/orchestration/loops/planner.py index d86844a4..27656bd5 100644 --- a/src/backend/orchestration/loop_planner.py +++ b/src/backend/orchestration/loops/planner.py @@ -24,10 +24,10 @@ import json from typing import Any, Dict, List, Optional -from core.capabilities.runtime import child_scope - +from core.harness.registry.capability_runtime import child_scope from core.infra.logging import get_logger -from orchestration.loop_evaluator import GoalSpec, _parse_json_array_lenient +from core.infra.json_output import parse_json_array_lenient as _parse_json_array_lenient +from orchestration.loops.requirements import GoalSpec logger = get_logger(__name__) @@ -74,8 +74,8 @@ async def scout_workspace( return "" from core.llm.factory import create_agent_executor - from core.llm.mcp_manager import close_clients - from orchestration.streaming import StreamingAgent + from core.llm.mcp.manager import close_clients + from core.llm.runtime.streaming import StreamingAgent where = "用户选定的项目文件夹（站点/工程真实源码所在）" if project_ctx else "沙箱 /workspace" prompt = ( @@ -152,8 +152,8 @@ async def _plan_llm_once( ) -> str: """规划专用的一次性纯文本调用：loop_reviewer 角色 + medium 档（规划值得比 fast 更强的模型）。""" from core.llm.factory import create_agent_executor - from core.llm.mcp_manager import close_clients - from orchestration.streaming import StreamingAgent + from core.llm.mcp.manager import close_clients + from core.llm.runtime.streaming import StreamingAgent agent, clients = await create_agent_executor( disable_tools=True, diff --git a/src/backend/orchestration/loops/presentation.py b/src/backend/orchestration/loops/presentation.py new file mode 100644 index 00000000..720e5c06 --- /dev/null +++ b/src/backend/orchestration/loops/presentation.py @@ -0,0 +1,71 @@ +"""Build persisted Loop transcripts and map terminal statuses.""" + +from __future__ import annotations + +from core.infra.logging import get_logger + +logger = get_logger(__name__) + +_LOOP_STATUS_ZH = { + "completed": "✅ 已达成", + "budget_exhausted": "⏳ 预算耗尽", + "cancelled": "⛔ 已取消", + "awaiting_human": "🙋 等待人工", + "failed": "❌ 失败", +} + + +def _loop_transcript_md(objective: str, result) -> str: + """Render one autonomous loop's iteration trace into an assistant message for the chat (markdown).""" + lines = [ + "**🔁 自主循环**", + "", + f"**目标**：{objective}", + "", + "| 轮次 | 需求 | 评审判定 | 工具 | 说明 |", + "| --- | --- | --- | --- | --- |", + ] + for r in result.history: + reason = str(r.get("reason") or "").replace("\n", " ").replace("|", "／") + if len(reason) > 60: + reason = reason[:60] + "…" + lines.append( + f"| {r.get('seq')} | {r.get('requirement_id', '')} | {r.get('verdict')} | " + f"{r.get('tool_calls', 0)} | {reason} |" + ) + status_zh = _LOOP_STATUS_ZH.get(result.status, result.status) + # final_score is now the requirement pass ratio (0~1) — no script-based numeric score. + final = "—" if result.final_score is None else f"{result.final_score:.0%}" + lines += [ + "", + f"**结果**：{status_zh} · 共 {result.iterations} 轮 · 完成度 {final}" + f" · 用时 {result.wall_clock_s:.0f}s", + ] + if result.reason: + lines.append(f"\n> {result.reason}") + return "\n".join(lines) + + +def record_loop_event(event: dict, content_buf: list[str], tool_log: list[dict], tool_idx: dict[str, int]) -> None: + """Accumulate only visible answer text and tool cards for the loop transcript.""" + et = event.get("type") + if et == "content": + _delta = event.get("delta") + if _delta: + content_buf.append(str(_delta)) + elif et == "tool_call": + _tid = str(event.get("tool_id") or f"t{len(tool_log)}") + tool_idx[_tid] = len(tool_log) + tool_log.append( + { + "id": _tid, + "name": event.get("tool_name") or "tool", + "input": event.get("tool_args"), + "status": "running", + } + ) + elif et == "tool_result": + _i = tool_idx.get(str(event.get("tool_id") or "")) + if _i is not None: + tool_log[_i]["output"] = event.get("result") + tool_log[_i]["status"] = "error" if event.get("error") else "success" diff --git a/src/backend/orchestration/loops/recovery.py b/src/backend/orchestration/loops/recovery.py new file mode 100644 index 00000000..b83ce4e8 --- /dev/null +++ b/src/backend/orchestration/loops/recovery.py @@ -0,0 +1,88 @@ +"""Resume eligible autonomous Loops after process restart.""" + +from __future__ import annotations + +import os + +import core.db.engine as engine +import orchestration.loops.admission as _orchestration_loops_admission +from core.infra.logging import get_logger + +logger = get_logger(__name__) + + +async def resume_running_loops() -> int: + """启动对账 + 按需自动续跑：进程重启后收养/归位孤儿自主循环。 + + 进程刚起来时不存在任何活跃 task，因此 status='running' 的 loop 全是孤儿。 + 对每一个孤儿： + + - ``LOOP_AUTO_RESUME=true`` → 用**持久化的启动参数**（agent_loops.extra_data. + start_params：模型/评审模型/轮数/思考档位）原样续跑——账本在 DB 有镜像、 + 沙箱有 feature_list.json，driver 自动断点续跑，不再悄悄降级到默认模型。 + - 关闭（默认）→ 状态归位为 ``interrupted``（可续跑），不再留下永远 running 的 + 僵尸行（历史坑：僵尸 running 行既误导列表页、又让 cancel 无处下手）。 + + 'awaiting_human'/终态一律不动。 + """ + from core.db.models import AgentLoop + + auto = os.getenv("LOOP_AUTO_RESUME", "false").strip().lower() in ( + "1", + "true", + "yes", + ) + resumed = 0 + with engine.SessionLocal() as db: + loops = db.query(AgentLoop).filter(AgentLoop.status == "running").all() + specs = [ + ( + x.loop_id, + x.chat_id, + x.user_id, + dict(x.goal_spec or {}), + dict(x.budget or {}), + (x.extra_data or {}).get("project_id"), + dict((x.extra_data or {}).get("start_params") or {}), + ) + for x in loops + ] + + if not auto: + if specs: + from features.automation.loops import LoopService as _LoopSvc + + with engine.SessionLocal() as db: + for loop_id, *_rest in specs: + _LoopSvc(db).mark_interrupted( + loop_id, reason="服务重启导致运行中断，可点击「继续」从断点续跑" + ) + logger.info("[startup] orphan loops marked interrupted: %d", len(specs)) + return 0 + + for loop_id, chat_id, user_id, goal_spec, budget, project_id, params in specs: + if not chat_id: + continue + try: + await _orchestration_loops_admission.start_autonomous_loop_run( + loop_id=loop_id, + chat_id=chat_id, + user_id=user_id, + goal_spec=goal_spec, + budget=budget, + model_name=params.get("model_name"), + model_provider_id=params.get("model_provider_id"), + evaluator_model=params.get("evaluator_model"), + worker_max_iters=int(params.get("worker_max_iters") or 15), + hitl_enabled=bool(params.get("hitl_enabled")), + enable_thinking=bool(params.get("enable_thinking")), + chat_mode=params.get("chat_mode"), + project_id=project_id, + automation_run=bool(params.get("automation_run")), + is_resume=True, + ) + resumed += 1 + logger.info("autonomous_loop_resumed", loop_id=loop_id) + except Exception: # noqa: BLE001 + logger.warning("autonomous_loop_resume_failed loop_id=%s", loop_id, exc_info=True) + return resumed diff --git a/src/backend/orchestration/loop_evaluator.py b/src/backend/orchestration/loops/requirements.py similarity index 83% rename from src/backend/orchestration/loop_evaluator.py rename to src/backend/orchestration/loops/requirements.py index 0bec34e5..47713aa0 100644 --- a/src/backend/orchestration/loop_evaluator.py +++ b/src/backend/orchestration/loops/requirements.py @@ -17,15 +17,13 @@ Both use only a one-shot disable-tools fast agent with pure-text output — no sandbox access, no verdicts, no scripts run. """ + from __future__ import annotations -import json -import re from dataclasses import dataclass, field from typing import Any, Dict, List, Optional -from core.capabilities.runtime import child_scope - +from core.harness.registry.capability_runtime import child_scope from core.infra.logging import get_logger logger = get_logger(__name__) @@ -71,8 +69,10 @@ async def decompose_requirements( f"目标：\n{goal_spec.objective}\n\n" + ( "已知验收标准（据此拆，勿遗漏）：\n" - + "\n".join(f"- {c}" for c in goal_spec.acceptance_criteria) + "\n\n" - if goal_spec.acceptance_criteria else "" + + "\n".join(f"- {c}" for c in goal_spec.acceptance_criteria) + + "\n\n" + if goal_spec.acceptance_criteria + else "" ) + "拆解规则：拆成 3~8 条需求，每条是一个能客观判断「做没做到」的具体特性/改动" "（如某个功能模块、某块页面、某类数据/交互/文案）。粒度适中，别拆太碎也别一锅烩。\n\n" @@ -140,48 +140,7 @@ async def extract_acceptance_criteria( # ── Parsing helpers ────────────────────────────────────────────────────────── -def _parse_json_array_lenient(text: str) -> Optional[List[Any]]: - if not text: - return None - candidates = [text.strip()] - m = re.search(r"```(?:json)?\s*\n?(.*?)\n?```", text, re.DOTALL) - if m: - candidates.append(m.group(1).strip()) - s, e = text.find("["), text.rfind("]") - if s != -1 and e != -1 and e > s: - candidates.append(text[s : e + 1]) - for c in candidates: - try: - obj = json.loads(c) - if isinstance(obj, list): - return obj - except (json.JSONDecodeError, ValueError): - continue - return None - - -def _parse_json_lenient(text: str) -> Optional[Dict[str, Any]]: - """Three-level lenient parsing (object), following plan_mode._parse_plan_json.""" - for candidate in _json_candidates(text): - try: - obj = json.loads(candidate) - if isinstance(obj, dict): - return obj - except (json.JSONDecodeError, ValueError): - continue - return None - - -def _json_candidates(text: str): - if not text: - return - yield text.strip() - m = re.search(r"```(?:json)?\s*\n?(.*?)\n?```", text, re.DOTALL) - if m: - yield m.group(1).strip() - s, e = text.find("{"), text.rfind("}") - if s != -1 and e != -1 and e > s: - yield text[s : e + 1] +from core.infra.json_output import parse_json_array_lenient as _parse_json_array_lenient # ── LLM side: standalone fast agent, disable_tools, pure-text decomposition only (no verdicts, no sandbox) ── @@ -206,8 +165,8 @@ async def _make_judge_agent(model_name: Optional[str], user_id: str, capability_ async def _judge_once( prompt: str, *, model_name: Optional[str], user_id: str, capability_scope: str = "" ) -> str: - from core.llm.mcp_manager import close_clients - from orchestration.streaming import StreamingAgent + from core.llm.mcp.manager import close_clients + from core.llm.runtime.streaming import StreamingAgent agent, clients = await _make_judge_agent(model_name, user_id, capability_scope=capability_scope) sa = StreamingAgent(agent, clients) diff --git a/src/backend/orchestration/subagents/loop_reviewer.py b/src/backend/orchestration/loops/reviewer.py similarity index 78% rename from src/backend/orchestration/subagents/loop_reviewer.py rename to src/backend/orchestration/loops/reviewer.py index c7d836e1..c4f77fa3 100644 --- a/src/backend/orchestration/subagents/loop_reviewer.py +++ b/src/backend/orchestration/loops/reviewer.py @@ -25,21 +25,18 @@ to the old evaluate_iteration (done/continue/off_track/need_human), so the driver can route on it directly. """ + from __future__ import annotations +from core.harness import log_context +from core.harness import log_writer import time from typing import Any, Awaitable, Callable, Dict, List, Optional -from core.capabilities.runtime import child_scope - +from core.harness.registry.capability_runtime import child_scope from core.infra.logging import get_logger -from orchestration.loop_evaluator import ( - CONTINUE, - DONE, - NEED_HUMAN, - OFF_TRACK, - _parse_json_lenient, -) +from core.infra.json_output import parse_json_lenient as _parse_json_lenient +from orchestration.loops.requirements import CONTINUE, DONE, NEED_HUMAN, OFF_TRACK logger = get_logger(__name__) @@ -68,7 +65,9 @@ def _build_review_prompt( second_pass: bool, machine_evidence: str = "", ) -> str: - criteria = "\n".join(f"- {c}" for c in acceptance_criteria) or "- （无显式验收标准，按需求描述核验）" + criteria = ( + "\n".join(f"- {c}" for c in acceptance_criteria) or "- （无显式验收标准，按需求描述核验）" + ) parts = [ "你是一个自主循环的**独立产出评审员**。你不是执行者，是裁判。", "你和刚才干活的执行 agent 是**两个人**——你**绝不能**采信它自报的" @@ -168,36 +167,50 @@ async def review_requirement( Returns ``{verdict, criteria_hit, evidence, feedback}``; on any exception/parse failure conservatively continue (never misjudge as done). """ from core.llm.factory import create_agent_executor - from core.llm.mcp_manager import close_clients - from core.services import log_service as log_writer - from orchestration.streaming import StreamingAgent + from core.llm.mcp.manager import close_clients + from core.llm.runtime.streaming import StreamingAgent + from core.harness import log_writer as log_writer phase = "复核" if second_pass else "核验" if emit: try: - await emit({"type": "loop_review_started", "requirement_id": requirement_id, - "second_pass": second_pass}) + await emit( + { + "type": "loop_review_started", + "requirement_id": requirement_id, + "second_pass": second_pass, + } + ) except Exception: # noqa: BLE001 pass # First-class sub-agent call log (best-effort, never blocks the review). _t0 = time.monotonic() - sub_log_id = await log_writer.start_subagent_log({ - "subagent_name": f"循环评审员（{phase}）", - "subagent_type": "loop_reviewer", - "user_id": user_id, - "chat_id": chat_id, - "model": model_name, - "input_messages": {"requirement_id": requirement_id, "requirement": requirement_desc, - "acceptance_criteria": acceptance_criteria, "second_pass": second_pass}, - }) + sub_log_id = await log_writer.start_subagent_log( + { + "subagent_name": f"循环评审员（{phase}）", + "subagent_type": "loop_reviewer", + "user_id": user_id, + "chat_id": chat_id, + "model": model_name, + "input_messages": { + "requirement_id": requirement_id, + "requirement": requirement_desc, + "acceptance_criteria": acceptance_criteria, + "second_pass": second_pass, + }, + } + ) tool_calls = 0 async def _finish(status: str, *, output: str = "", error: Optional[str] = None) -> None: try: await log_writer.finish_subagent_log( - sub_log_id, status=status, output_content=output or None, - tool_calls_count=tool_calls, error_message=error, + sub_log_id, + status=status, + output_content=output or None, + tool_calls_count=tool_calls, + error_message=error, duration_ms=int((time.monotonic() - _t0) * 1000), ) except Exception: # noqa: BLE001 @@ -219,10 +232,7 @@ async def _finish(status: str, *, output: str = "", error: Optional[str] = None) # hand-rolled reviewer definition. The spec lives in a static module # tuple, so it is always present; a prompt-load failure raises into the # except below (spawn failed → conservative continue). - from core.llm.builtin_subagents import ( - build_builtin_runtime_profile, - get_builtin_subagent, - ) + from core.capabilities.agents.builtin import build_builtin_runtime_profile, get_builtin_subagent _spec = get_builtin_subagent("builtin.reviewer") agent, clients = await create_agent_executor( @@ -244,11 +254,17 @@ async def _finish(status: str, *, output: str = "", error: Optional[str] = None) tool_result_limit=_reviewer_tool_result_limit(), capability_scope=capability_scope, ) - except Exception as exc: # noqa: BLE001 - a reviewer agent that won't start must not drag down the loop + except ( + Exception + ) as exc: # noqa: BLE001 - a reviewer agent that won't start must not drag down the loop logger.warning("[loop-review] spawn reviewer failed: %s", exc) await _finish("failed", error=str(exc)[:200]) - return {"verdict": CONTINUE, "criteria_hit": [], "evidence": "", - "feedback": "评审子智能体启动失败，保守继续。"} + return { + "verdict": CONTINUE, + "criteria_hit": [], + "evidence": "", + "feedback": "评审子智能体启动失败，保守继续。", + } sa = StreamingAgent(agent, clients) text = "" @@ -260,11 +276,15 @@ async def _finish(status: str, *, output: str = "", error: Optional[str] = None) try: # The reviewer's stream is **not forwarded** to the user bubble — it's an internal judge, only its final text verdict is collected. # subagent_scope attributes the reviewer's internal read/grep/... tool calls to this sub-agent log (auditable). - with log_writer.subagent_scope(sub_log_id, source="loop_reviewer"): + with log_context.subagent_scope(sub_log_id, source="loop_reviewer"): async for et, payload in sa.stream( [{"role": "user", "content": prompt}], - {"user_id": user_id, "model_name": model_name or "", - "enable_thinking": False, "chat_mode": "medium"}, + { + "user_id": user_id, + "model_name": model_name or "", + "enable_thinking": False, + "chat_mode": "medium", + }, ): if et == "text_delta": text += payload @@ -279,12 +299,21 @@ async def _finish(status: str, *, output: str = "", error: Optional[str] = None) await close_clients(clients) async def _return(result: Dict[str, Any], *, log_status: str = "success") -> Dict[str, Any]: - await _finish(log_status, output=(result.get("verdict", "") + " | " + result.get("feedback", ""))[:2000]) + await _finish( + log_status, + output=(result.get("verdict", "") + " | " + result.get("feedback", ""))[:2000], + ) if emit: try: - await emit({"type": "loop_review_result", "requirement_id": requirement_id, - "second_pass": second_pass, "verdict": result.get("verdict"), - "evidence": (result.get("evidence") or "")[:600]}) + await emit( + { + "type": "loop_review_result", + "requirement_id": requirement_id, + "second_pass": second_pass, + "verdict": result.get("verdict"), + "evidence": (result.get("evidence") or "")[:600], + } + ) except Exception: # noqa: BLE001 pass return result @@ -298,7 +327,7 @@ async def _return(result: Dict[str, Any], *, log_status: str = "success") -> Dic # serialization failed, so a single fast-model reformat of its own # words usually recovers the verdict for the cost of one small call. try: - from orchestration.loop_evaluator import _judge_once + from orchestration.loops.requirements import _judge_once _reformatted = await _judge_once( "把下面这段评审结论改写成严格 JSON（只输出 JSON，不要任何多余文字），" @@ -320,24 +349,38 @@ async def _return(result: Dict[str, Any], *, log_status: str = "success") -> Dic else: # Still unparseable → conservatively continue (never misjudge as done). logger.info("[loop-review] unparseable verdict, defaulting continue") - return await _return({"verdict": CONTINUE, "criteria_hit": [], "evidence": text[:400], - "progress": False, - "feedback": "评审未给出可解析的结论，保守继续。"}) + return await _return( + { + "verdict": CONTINUE, + "criteria_hit": [], + "evidence": text[:400], + "progress": False, + "feedback": "评审未给出可解析的结论，保守继续。", + } + ) # evidence fallback: done but no evidence given → downgrade to continue (prevent empty-evidence leniency). evidence = str(obj.get("evidence", "") or "").strip() if obj["verdict"] == DONE and not evidence: - return await _return({"verdict": CONTINUE, "criteria_hit": obj.get("criteria_hit", []), - "evidence": "", "progress": False, - "feedback": "评审判定完成但未给出文件证据，视为未确证，继续。"}) - return await _return({ - "verdict": obj["verdict"], - "criteria_hit": obj.get("criteria_hit", []) or [], - "evidence": evidence, - # Material-progress affirmation (missing/parse-failure → False, i.e. the - # historical behavior). The driver's stagnation cap only counts rounds - # WITHOUT affirmed progress — so a mega-requirement (e.g. a 20-chapter - # body written chapter-by-chapter) healthily spanning many rounds is no - # longer blocked as "stagnating" at the attempt cap. - "progress": bool(obj.get("progress")), - "feedback": str(obj.get("feedback", "") or "").strip(), - }) + return await _return( + { + "verdict": CONTINUE, + "criteria_hit": obj.get("criteria_hit", []), + "evidence": "", + "progress": False, + "feedback": "评审判定完成但未给出文件证据，视为未确证，继续。", + } + ) + return await _return( + { + "verdict": obj["verdict"], + "criteria_hit": obj.get("criteria_hit", []) or [], + "evidence": evidence, + # Material-progress affirmation (missing/parse-failure → False, i.e. the + # historical behavior). The driver's stagnation cap only counts rounds + # WITHOUT affirmed progress — so a mega-requirement (e.g. a 20-chapter + # body written chapter-by-chapter) healthily spanning many rounds is no + # longer blocked as "stagnating" at the attempt cap. + "progress": bool(obj.get("progress")), + "feedback": str(obj.get("feedback", "") or "").strip(), + } + ) diff --git a/src/backend/orchestration/loops/run.py b/src/backend/orchestration/loops/run.py new file mode 100644 index 00000000..64bc8278 --- /dev/null +++ b/src/backend/orchestration/loops/run.py @@ -0,0 +1,499 @@ +"""Bind autonomous Loop execution to leases and transactional checkpoints.""" + +from __future__ import annotations +from core.runs.tool_effects import errors as errors_module + +from orchestration.loops.presentation import record_loop_event + +import asyncio +import time +import typing as _typing +from typing import Any, Dict, List, Optional + +import core.db.engine as engine +import core.runs.events.writer as writer +import core.runs.ownership as ownership +import core.runs.runtime as runtime +import core.runs.settings as settings +import core.chat.service as services_ChatService_module +import core.runs.journal as run_journal +import core.runs.tool_effects.journal as tool_effect_ledger +import orchestration.loops.presentation as _orchestration_loops_presentation +from core.infra.logging import get_logger + +logger = get_logger(__name__) + + +async def _run_autonomous_loop_workflow( + *, + run_id: str, + loop_id: str, + chat_id: str, + user_id: str, + message_id: str, + goal_spec: Dict[str, Any], + budget: Dict[str, Any], + model_name: Optional[str], + model_provider_id: Optional[str] = None, + evaluator_model: Optional[str] = None, + worker_max_iters: int = 15, + hitl_enabled: bool = False, + enable_thinking: bool = False, + chat_mode: Optional[str] = None, + is_resume: bool = False, + project_id: Optional[str] = None, + automation_run: bool = False, + journal_owner: Optional[str] = None, +) -> None: + from orchestration.loops.driver import LoopBudget, run_autonomous_loop + from orchestration.loops.requirements import GoalSpec + + owner = journal_owner or ownership.new_worker_owner(run_id) + journal = ownership.journal() + if not journal.claim(run_id, owner=owner, lease_seconds=settings.RUN_LEASE_SECONDS, pending_only=True): + logger.info("autonomous_loop_claim_skipped", run_id=run_id, owner=owner) + return + current = asyncio.current_task() + lease_lost = asyncio.Event() + heartbeat = asyncio.create_task( + ownership.lease_heartbeat(run_id, owner, current, lease_lost), + name=f"autonomous_loop_lease:{run_id}", + ) + # Accumulate the worker's streamed body text + tool cards, and persist them + # as an assistant message when the stream ends — exactly the same as a + # normal conversation: the body only accumulates content deltas (thinking + # goes through separate thinking events, is not stuffed into the body, and + # no fake is fabricated), and tools are persisted as tool_calls. + # After a refresh, buildHistorySegments reconstructs identically to a normal + # conversation. + content_buf: _typing.List[str] = [] + tool_log: _typing.List[_typing.Dict[str, _typing.Any]] = [] + tool_idx: _typing.Dict[str, int] = {} + + offsets = writer.EventOffsets(journal, run_id, owner, lease_lost=lease_lost) + + async def _emit(event: Dict[str, Any], *, terminal: bool = False) -> None: + await offsets.take(terminal=terminal) + et = event.get("type") + record_loop_event(event, content_buf, tool_log, tool_idx) + await writer.xadd_event(run_id, offsets.last, event) + # At structural checkpoints (requirement flipped / per-iteration + # evaluation done) persist progress incrementally — so after a mid-run + # crash/restart/refresh the already-produced body + tool cards are still + # visible from the DB, rather than waiting for a single write at the + # terminal state (see the fix for symptoms 2/3). + if et in ("requirement_passed", "iteration_evaluated"): + _flush_loop_message() + + def _loop_message_effect(status: str): + if not conversational: + return None + body = "".join(content_buf).strip() + captured_tools = [dict(item) for item in tool_log] + if not body and not captured_tools: + return None + + def _commit(db) -> None: # noqa: ANN001 + services_ChatService_module.ChatService(db).upsert_message( + chat_id=chat_id, + role="assistant", + content=body, + message_id=message_id, + tool_calls=captured_tools or None, + extra_data={ + "autonomous_loop": True, + "loop_id": loop_id, + "loop_status": status, + }, + commit=False, + ) + + return _commit + + def _persist_loop_partial() -> None: + effect = _loop_message_effect("cancelled") + if effect is None: + return + try: + with engine.SessionLocal() as _db: + effect(_db) + _db.commit() + except Exception: # noqa: BLE001 - 落库失败不能吞掉取消收尾 + logger.warning( + "autonomous_loop_cancelled_partial_persist_failed", + run_id=run_id, + exc_info=True, + ) + + def _flush_loop_message(status: str = "running") -> None: + """Upsert the currently accumulated worker body + tool cards into the assistant message (same message_id). + + Conversational mode (self_verify) only. Skipped when output is empty + (no body and no tools yet) to avoid writing an empty bubble. + """ + commit_effect = _loop_message_effect(status) + if commit_effect is None: + return + try: + journal.append_operation( + run_id, + owner=owner, + operation_type="loop_checkpoint_committed", + phase="loop_checkpoint", + safety="side_effect_committed", + payload={"loop_id": loop_id, "message_id": message_id}, + commit_effect=commit_effect, + ) + except run_journal.RunLeaseLost: + raise + except Exception: # noqa: BLE001 - incremental persist failure must not take down the run + logger.warning("loop incremental persist failed", exc_info=True) + + gs = GoalSpec( + objective=goal_spec.get("objective", ""), + acceptance_criteria=goal_spec.get("acceptance_criteria", []) or [], + ) + # 预算去一等公民化：缺省一律 0（不限）。「能完成任务」优先——停止条件回归 + # 账本全通过/停滞无解/用户取消；防失控由 LOOP_HARD_MAX_ITERS 硬后备兜底。 + # 显式传正数的旧 loop 行（历史数据）仍按其预算执行。 + bud = LoopBudget( + max_iters=int(budget.get("max_iters", 0) or 0), + max_wall_clock_s=float(budget.get("max_wall_clock_s", 0) or 0), + max_tokens=int(budget.get("max_tokens", 0) or 0), + ) + # Project binding: resolve project_ctx and bind the loop's chat session to + # that project — this scopes the worker's/reviewer's file tools to the + # project folder (where the site source lives), and lets publish_site + # reverse-look-up the project from the session and publish to the same site. + # When bound to a project, the sandbox session uses chat_id (same as a + # normal project conversation → project file materialization and publish + # packaging share one session); without a project it degrades to the + # isolated loop-{loop_id} (pure task-style loop). + project_ctx: _typing.Optional[_typing.Dict[str, _typing.Any]] = None + session_id = f"loop-{loop_id}" + # Autonomous loops always live on in the chat history as a "normal + # conversation" (the objective is persisted as a user message and the result + # as an assistant message, surviving refreshes). Script-verification/form + # modes have been removed; there is no non-conversational branch anymore. + conversational = True + + # Requirement-ledger DB mirror callbacks: every time the driver writes the + # ledger it is also synced into agent_loops.metadata; resume reads the DB + # first (reliable across rebuilds/restarts/machine changes) and no longer + # depends on whether the sandbox /workspace still exists (see the fix for + # symptom 1). + from features.automation.loops import LoopService as _LoopSvc + + def _load_ledger() -> Optional[Dict[str, Any]]: + try: + with engine.SessionLocal() as db: + return _LoopSvc(db).load_ledger(loop_id) + except Exception: # noqa: BLE001 + logger.warning("loop load_ledger failed", exc_info=True) + return None + + def _save_ledger(led: Dict[str, Any]) -> None: + def _commit(db) -> None: # noqa: ANN001 + _LoopSvc(db).save_ledger(loop_id, led, commit=False) + + journal.append_operation( + run_id, + owner=owner, + operation_type="loop_ledger_committed", + phase="loop_checkpoint", + safety="side_effect_committed", + payload={"loop_id": loop_id}, + commit_effect=_commit, + ) + + def _poll_steering() -> List[str]: + """每轮开工前取走用户运行中追加的指令（POST /v1/loops/{id}/steer）。""" + try: + found: _typing.List[str] = [] + + def _commit(db) -> None: # noqa: ANN001 + found.extend(_LoopSvc(db).consume_steering(loop_id, commit=False)) + + journal.append_operation( + run_id, + owner=owner, + operation_type="loop_steering_polled", + phase="loop_checkpoint", + safety="side_effect_committed", + payload={"loop_id": loop_id}, + commit_effect=_commit, + ) + return found + except run_journal.RunLeaseLost: + raise + except Exception: # noqa: BLE001 + logger.warning("loop consume_steering failed", exc_info=True) + return [] + + try: + if project_id: + try: + from core.db.models import ChatSession + from core.projects.edition import build_project_ctx + + needs_binding = False + with engine.SessionLocal() as db: + project_ctx = build_project_ctx(db, project_id) + if project_ctx: + sess = db.get(ChatSession, chat_id) + needs_binding = sess is not None and sess.project_id != project_id + + if project_ctx and needs_binding: + + def _commit_project_binding(db) -> None: # noqa: ANN001 + sess = db.get(ChatSession, chat_id) + if sess is not None: + sess.project_id = project_id + + journal.append_operation( + run_id, + owner=owner, + operation_type="loop_project_bound", + phase="pre_model", + safety="side_effect_committed", + payload={"loop_id": loop_id, "project_id": project_id}, + commit_effect=_commit_project_binding, + ) + except run_journal.RunLeaseLost: + raise + except Exception: # noqa: BLE001 - project lookup may safely degrade + logger.warning("loop project_ctx resolve failed", exc_info=True) + project_ctx = None + session_id = chat_id if project_ctx else f"loop-{loop_id}" + + await _emit( + { + "type": "run_started", + "run_id": run_id, + "message_id": message_id, + "chat_id": chat_id, + # 这几条路径不会被恢复接管，worker 起来的那一刻就是本轮起点， + # 不必为了拿它再读一次库——首帧前的每一次往返用户都在干等。 + "started_at": int(time.time() * 1000), + "kind": "autonomous_loop", + "loop_id": loop_id, + } + ) + + def _commit_loop_start(db) -> None: # noqa: ANN001 + from features.automation.loops import LoopService + + if conversational and gs.objective and not is_resume: + # Only the first start persists the objective; the deterministic + # id also makes a transaction retry idempotent. + services_ChatService_module.ChatService(db).add_message( + chat_id=chat_id, + role="user", + content=gs.objective, + message_id=f"{message_id}_objective", + extra_data={"autonomous_loop": True, "loop_id": loop_id}, + commit=False, + ) + LoopService(db).mark_running( + loop_id, + workspace_session=session_id, + commit=False, + ) + + journal.append_operation( + run_id, + owner=owner, + operation_type="loop_started_committed", + phase="pre_model", + safety="side_effect_committed", + payload={"loop_id": loop_id}, + commit_effect=_commit_loop_start, + ) + + from core.auth.tenancy import tenant_of + from core.harness.tool_call_context import CURRENT_RUN_BINDING + + binding_token = CURRENT_RUN_BINDING.set((run_id, owner)) + try: + result = await run_autonomous_loop( + loop_id=loop_id, + user_id=user_id, + goal_spec=gs, + budget=bud, + model_name=model_name, + model_provider_id=model_provider_id, + # 评审/规划模型：显式指定 > 「模型管理 → 角色分配 → loop_reviewer」> + # main_agent。不再硬编码 "fast"——评审质量值得一个后台可配的位置。 + evaluator_model=evaluator_model, + worker_max_iters=worker_max_iters, + session_id=session_id, + hitl_enabled=hitl_enabled, + enable_thinking=enable_thinking, + chat_mode=chat_mode, + emit=_emit, + is_cancelled=lambda: runtime.is_run_cancelled(run_id), + load_ledger=_load_ledger, + save_ledger=_save_ledger, + poll_steering=_poll_steering, + project_ctx=project_ctx, + chat_id=chat_id, + automation_run=automation_run, + # Carried explicitly so the loop resolves *this* tenant's + # orchestration profile. Omitting it is how one tenant's published + # retry counts and budget multiplier became everyone's. + tenant_id=tenant_of(user_id), + ) + finally: + CURRENT_RUN_BINDING.reset(binding_token) + + _asst_content: _typing.Optional[str] = None + if conversational: + # Prefer storing the streamed body (matches what the frontend saw); fall back to the trace table when the worker produced no body at all. + _body = "".join(content_buf).strip() + _status_zh = _orchestration_loops_presentation._LOOP_STATUS_ZH.get( + result.status, result.status + ) + if _body: + _asst_content = _body + ( + f"\n\n---\n**{_status_zh}** · 共 {result.iterations} 轮" + + (f" · 最终分 {result.final_score}" if result.final_score is not None else "") + ) + else: + _asst_content = _orchestration_loops_presentation._loop_transcript_md( + gs.objective, result + ) + + def _commit_loop_result(db) -> None: # noqa: ANN001 + from features.automation.loops import LoopService + + LoopService(db).persist_result(loop_id, result, commit=False) + if _asst_content is not None: + # Upsert and the ChatRun terminal transition share one owner- + # fenced transaction, so a superseded worker cannot publish. + services_ChatService_module.ChatService(db).upsert_message( + chat_id=chat_id, + role="assistant", + content=_asst_content, + usage={"total_tokens": result.tokens_spent}, + tool_calls=tool_log if tool_log else None, + extra_data={ + "autonomous_loop": True, + "loop_id": loop_id, + "loop_status": result.status, + }, + message_id=message_id, + commit=False, + ) + + run_status = ( + "completed" + if result.status in ("completed", "budget_exhausted", "awaiting_human") + else ("cancelled" if result.status == "cancelled" else "failed") + ) + completed = journal.complete( + run_id, + owner=owner, + status=run_status, + usage={"total_tokens": result.tokens_spent}, + last_event_offset=offsets.last, + commit_effect=_commit_loop_result, + committed_operation={ + "operation_type": "loop_result_committed", + "phase": "loop_result_committed", + "safety": "side_effect_committed", + "payload": {"loop_id": loop_id, "message_id": message_id}, + }, + ) + if completed: + await _emit({"type": settings.TERMINAL_TYPE, "chat_id": chat_id}, terminal=True) + else: + logger.warning("autonomous_loop_terminal_cas_lost", run_id=run_id, owner=owner) + except asyncio.CancelledError: + # Cancelled: run_autonomous_loop was interrupted and no result is + # available, so the terminal-state persistence block is skipped — here + # we persist the progress produced so far (symptoms 2/3); otherwise a + # reopened chat would only contain the user's objective. + if lease_lost.is_set(): + logger.warning("autonomous_loop_worker_fenced", run_id=run_id, owner=owner) + if ownership.cancelled_by_user(run_id): + _persist_loop_partial() + else: + partial_effect = _loop_message_effect("cancelled") + cancelled = journal.complete( + run_id, + owner=owner, + status="cancelled", + failure_reason="autonomous loop worker cancelled", + last_event_offset=offsets.last, + commit_effect=partial_effect, + committed_operation=( + { + "operation_type": "loop_partial_message_committed", + "phase": "loop_partial_message_committed", + "safety": "side_effect_committed", + "payload": {"loop_id": loop_id, "message_id": message_id}, + } + if partial_effect is not None + else None + ), + ) + if not cancelled: + return + await _emit( + {"type": "loop_error", "error": "任务已被用户取消", "_cancelled": True}, + terminal=True, + ) + await _emit( + {"type": settings.TERMINAL_TYPE, "chat_id": chat_id, "_cancelled": True}, + terminal=True, + ) + except Exception as exc: # noqa: BLE001 + nested_unknown = errors_module.find_tool_outcome_unknown(exc) + if nested_unknown is not None: + if journal.needs_attention( + run_id, + owner=owner, + reason=f"tool outcome pending policy recovery: {nested_unknown}", + ): + await _emit( + { + "type": "loop_error", + "error": "工具调用结果无法安全确定，任务已暂停等待处理", + }, + terminal=True, + ) + await _emit({"type": settings.TERMINAL_TYPE, "chat_id": chat_id}, terminal=True) + return + logger.exception("autonomous_loop_run_failed", run_id=run_id) + partial_effect = _loop_message_effect("failed") + failed = journal.complete( + run_id, + owner=owner, + status="failed", + failure_reason=str(exc)[:1000], + last_event_offset=offsets.last, + commit_effect=partial_effect, + committed_operation=( + { + "operation_type": "loop_partial_message_committed", + "phase": "loop_partial_message_committed", + "safety": "side_effect_committed", + "payload": {"loop_id": loop_id, "message_id": message_id}, + } + if partial_effect is not None + else None + ), + ) + if failed: + await _emit( + {"type": "loop_error", "error": str(exc)[:500]}, + terminal=True, + ) + await _emit({"type": settings.TERMINAL_TYPE, "chat_id": chat_id}, terminal=True) + finally: + heartbeat.cancel() + try: + await heartbeat + except asyncio.CancelledError: + pass + await writer.expire_stream(run_id) diff --git a/src/backend/orchestration/memory_integration.py b/src/backend/orchestration/memory_integration.py deleted file mode 100644 index e3950e6a..00000000 --- a/src/backend/orchestration/memory_integration.py +++ /dev/null @@ -1,576 +0,0 @@ -"""Non-blocking memory I/O integration layer outside the SSE main path. - -- `open_session_memory()` / `resolve_session_memory()` own the chat-scoped - lifecycle: the frozen block is assembled on the first turn and replayed from - the stored snapshot afterwards, with no retrieval on later turns -- `inject_session_blocks()` prepends the session-constant blocks (identity, frozen - memory) as user-role messages; they stay out of the system prompt so the prompt, - tool schemas and skill list remain a shared prefix -- `launch_memory_retrieval()` starts the Fact retrieval task in the background with a budget timeout -- `build_frozen_memory_block()` assembles the Profile + Fact frozen block -- `save_memories_background()` delegates writes to the bounded post-response pipeline in - `core.memory.pipeline` (extractors → sanitize → write L1/L2/Session + audit) -""" - -from __future__ import annotations - -import asyncio -import logging -import weakref -from dataclasses import dataclass, replace -from time import monotonic -from typing import Any, Dict, List, Optional - -from core.config.settings import settings -from core.memory import profile -from core.memory.context import MemoryContext -from core.memory.retrieval_types import MemoryRetrievalResult -from core.memory.service import retrieve_memories_structured -from core.memory.session_snapshot import ( - SessionMemorySnapshot, - load_session_memory_snapshot, - save_session_memory_snapshot, -) -from core.llm.context_ir import ( - KIND_IDENTITY, - KIND_MEMORY, - SESSION_CONTEXT_META_KEY, - make_text_context_item, -) - -logger = logging.getLogger(__name__) - - -async def launch_memory_retrieval( - user_id: str, - user_message: str, - memory_enabled: bool, - *, - workspace_id: str = "default", - budget_ms: Optional[int] = None, -) -> Optional[asyncio.Task]: - """Start the Fact vector retrieval task in the background and return the Task (the caller need not await immediately). - - budget_ms defaults to settings.memory.retrieval_budget_ms (600ms). Called once by the - workflow at session start; if it completes within budget the frozen block is injected, otherwise skipped. - """ - if not memory_enabled or not user_id: - return None - - effective_budget = ( - budget_ms if budget_ms is not None else settings.memory.retrieval_budget_ms - ) / 1000.0 - - # The assembly boundary below owns the latency budget. Passing the same - # budget into the service would make its inner wait_for cancel _do_search - # before build_frozen_memory_block() can shield the task. - - async def _fetch() -> Optional[MemoryRetrievalResult]: - try: - return await retrieve_memories_structured( - user_id=user_id, - query=user_message, - workspace_id=workspace_id, - timeout_s=None, - ) - except Exception as exc: - logger.warning("[memory] retrieval failed: %s", exc) - return MemoryRetrievalResult.degraded_result("orchestration_error") - - return track_memory_retrieval(asyncio.create_task(_fetch()), budget_s=effective_budget) - - -def _as_result(value: object) -> Optional[MemoryRetrievalResult]: - """Normalise whatever the retrieval task produced. - - The task may legitimately yield ``None`` (never started) and, for callers - that still hand us a pre-rendered string, we degrade gracefully rather than - crash the turn. - """ - if isinstance(value, MemoryRetrievalResult): - return value - return None - - -# The structured recall for the current turn, keyed by the retrieval task so -# concurrent runs never read each other's result. A WeakKeyDictionary keeps this -# from becoming a leak: once the task is collected, so is the entry. -_last_retrieval: "weakref.WeakKeyDictionary[asyncio.Task, MemoryRetrievalResult]" = ( - weakref.WeakKeyDictionary() -) -_retrieval_states: "weakref.WeakKeyDictionary[asyncio.Task, str]" = weakref.WeakKeyDictionary() -_retrieval_budgets: "weakref.WeakKeyDictionary[asyncio.Task, float]" = weakref.WeakKeyDictionary() - - -def track_memory_retrieval(task: asyncio.Task, *, budget_s: Optional[float] = None) -> asyncio.Task: - """Attach lifecycle observation without taking ownership of cancellation.""" - - _retrieval_states[task] = "running" - if budget_s is not None: - _retrieval_budgets[task] = max(0.0, budget_s) - - def _finished(done: asyncio.Task) -> None: - previous = _retrieval_states.get(done) - if done.cancelled(): - _retrieval_states[done] = "cancelled" - return - try: - value = done.result() - except Exception: - _retrieval_states[done] = "failed" - return - result = _as_result(value) - if result is not None: - set_last_retrieval(done, result) - _retrieval_states[done] = ( - "completed_after_timeout" if previous == "timed_out_running" else "completed" - ) - - task.add_done_callback(_finished) - return task - - -def get_retrieval_state(task: Optional[asyncio.Task]) -> Optional[str]: - if task is None: - return None - return _retrieval_states.get(task) - - -def set_last_retrieval(task: asyncio.Task, result: MemoryRetrievalResult) -> None: - try: - _last_retrieval[task] = result - except TypeError: # pragma: no cover - non-weakrefable task - pass - - -def get_last_retrieval(task: Optional[asyncio.Task]) -> Optional[MemoryRetrievalResult]: - """Structured recall for a turn, for trace-event emission. - - Returns ``None`` when retrieval never ran or timed out — callers must treat - that as "no evidence", never as "memory contributed nothing". - """ - if task is None: - return None - try: - return _last_retrieval.get(task) - except TypeError: # pragma: no cover - return None - - -@dataclass(frozen=True) -class FrozenMemoryBlock: - """An assembled frozen block plus whether the read behind it actually landed. - - ``settled`` is false when the L2 read timed out or came back degraded. Such - a block is missing facts for reasons that say nothing about what the user's - memory holds, so it must not be frozen for the remainder of the session. - """ - - text: str - settled: bool - - -async def build_frozen_memory_block( - user_id: str, - workspace_id: str, - memory_task: Optional[asyncio.Task], - *, - memory_enabled: bool = True, -) -> FrozenMemoryBlock: - """Assemble the "session-frozen" block = L1 Profile markdown + L2 Fact top-K. - - - When `memory_enabled=False`, return empty immediately (**do not load unless the user - has enabled persistent memory** — forms defense-in-depth with the workflow layer's - `_mem0_enabled` check) - - Profile reads the DB (fast, <20ms), always awaited - - Fact takes the result from the already-started memory_task; if the task hasn't finished, - wait briefly; if still unfinished, give up on Fact injection for this round (never block agent startup) - - Called once per chat by `resolve_session_memory()`; later turns replay the - stored snapshot instead of reassembling. - """ - if not memory_enabled: - return FrozenMemoryBlock(text="", settled=False) - - # Profile layer (L1) - profile_md = "" - if user_id: - try: - profile_md = await profile.get(user_id, workspace_id) - except Exception as exc: - logger.warning( - "[memory] profile fetch failed user=%s ws=%s: %s", user_id, workspace_id, exc - ) - - # Fact layer (L2). With no retrieval task there is no L2 read to wait for, - # so a Profile-only block is already the complete answer for this session. - fact_text = "" - fact_settled = True - fact_result: Optional[MemoryRetrievalResult] = None - if memory_task is not None: - fact_settled = False - try: - # Wait up to retrieval_budget_ms (default 600ms), then give up; memory_task was - # started before the agent was created, so in most cases it is nearly done by now. - # The old value of 50ms was measured to be far below Milvus warm search's ~200ms, - # so Fact injection almost never hit. - wait_budget_s = _retrieval_budgets.get( - memory_task, max(0.1, settings.memory.retrieval_budget_ms / 1000.0) - ) - fact_result = _as_result( - await asyncio.wait_for(asyncio.shield(memory_task), timeout=wait_budget_s) - ) - fact_text = fact_result.to_text() if fact_result is not None else "" - # A degraded result means the store was unreachable or the budget - # blew inside the service — an empty answer we must not freeze. - fact_settled = fact_result is not None and not fact_result.degraded - # Stash the structured recall on the task so the workflow can emit a - # retrieval trace event without re-running the search. Attribution - # needs ids/ranks/scores that the rendered text has already lost. - if fact_result is not None: - set_last_retrieval(memory_task, fact_result) - except asyncio.TimeoutError: - if not memory_task.done(): - _retrieval_states[memory_task] = "timed_out_running" - logger.info( - "[memory] fact retrieval still running past wait window, skipping injection" - ) - # ``shield`` makes the continue-in-background policy explicit. The - # done callback retains the structured result and changes the - # observable state to completed_after_timeout. - except asyncio.CancelledError: - raise - except Exception as exc: - logger.warning("[memory] fact retrieval await failed: %s", exc) - - if not profile_md and not fact_text: - return FrozenMemoryBlock(text="", settled=fact_settled) - - parts: list[str] = ["## 关于当前用户的已知背景（会话开始时冻结）"] - if profile_md: - parts.append("") - parts.append("### 用户档案（Profile）") - parts.append(profile_md.strip()) - if fact_text: - parts.append("") - parts.append("### 相关历史记忆（Fact Top-K）") - # retrieve_memories comes with its own "## 关于该用户..." prefix; strip it to avoid duplicate headings - stripped = fact_text - for h in ("## 关于该用户的已知背景信息（来自历史会话记忆）", "## 用户相关实体关系"): - stripped = stripped.replace(h, "") - parts.append(stripped.strip()) - block = "\n".join(parts).strip() - logger.info( - "[memory] frozen block built user=%s ws=%s chars=%d profile=%d facts=%d settled=%s", - user_id, - workspace_id, - len(block), - len(profile_md or ""), - len(fact_text or ""), - fact_settled, - ) - return FrozenMemoryBlock(text=block, settled=fact_settled) - - -# ─── User identity block ─────────────────────────────────────────────────── - -_IDENTITY_CACHE_TTL_S = 60.0 -# user_id → (expires_at_monotonic, block_text) -_identity_cache: Dict[str, tuple] = {} - - -async def build_user_identity_block(user_id: str) -> str: - """Assemble the "current user" identity block (username / nickname) for injection into the session-frozen message. - - Deliberately goes through the user-role frozen block rather than the end of the system prompt: - the server-side chat template renders the tool schemas **after** the system text, so any - per-user bytes in system split the LLM prefix cache of the tool section (the largest chunk - of the prompt). Measured: identical system differing only in the trailing username → TTFT - degrades from 0.7s back to cold-start levels. The frozen block sits at the start of the - message sequence and renders after the tool section, so the system+tools shared prefix stays intact. - - Returns empty string for anonymous / unknown users. Result cached per user_id for 60s. - """ - if not user_id or user_id == "anonymous": - return "" - now = monotonic() - hit = _identity_cache.get(user_id) - if hit and now < hit[0]: - return hit[1] - - def _query() -> tuple: - from core.db.engine import SessionLocal - from core.db.models import LocalUser, UserShadow - - with SessionLocal() as db: - row = db.query(UserShadow.username).filter(UserShadow.user_id == user_id).first() - nick = db.query(LocalUser.nickname).filter(LocalUser.user_id == user_id).first() - return ( - (row[0] or "").strip() if row else "", - (nick[0] or "").strip() if nick else "", - ) - - try: - username, nickname = await asyncio.to_thread(_query) - except Exception as exc: - logger.warning("[identity] user lookup failed user=%s: %s", user_id, exc) - return "" - - lines: list[str] = [] - if username and username != "anonymous": - lines.append(f"- 用户名：{username}") - if nickname and nickname != username: - lines.append(f"- 昵称：{nickname}") - block = "" - if lines: - block = ( - "## 当前用户\n" - + "\n".join(lines) - + "\n需要称呼用户时，用上述昵称（无昵称则用用户名）自然称呼。" - ) - _identity_cache[user_id] = (now + _IDENTITY_CACHE_TTL_S, block) - return block - - -# ─── Session-scoped lifecycle ────────────────────────────────────────────── - - -@dataclass(frozen=True) -class SessionMemory: - """One run's handle on the chat's frozen memory block. - - Split in two because the two halves sit on either side of agent assembly: - `open_session_memory()` runs first so a first-turn retrieval overlaps - building the agent, and `resolve_session_memory()` runs at injection time. - - On every turn after the first, `retrieval_task` is ``None``: the block is - replayed from the stored snapshot and no vector search is issued at all. - """ - - chat_id: str - scope_user_id: str - workspace_id: str - memory_enabled: bool - retrieval_task: Optional[asyncio.Task] = None - snapshot: Optional["SessionMemorySnapshot"] = None - - -async def open_session_memory( - *, - chat_id: Optional[str], - scope_user_id: str, - workspace_id: str, - user_message: str, - memory_enabled: bool, - budget_ms: Optional[int] = None, -) -> SessionMemory: - """Start the chat's memory read, or recognise that it already happened. - - Called before the agent is built. When this chat already has a frozen - snapshot the retrieval is skipped outright — that is the point of freezing - it: the block sits ahead of the entire conversation, so re-deriving it per - turn would invalidate the model's prefix cache for the whole context. - """ - handle = SessionMemory( - chat_id=str(chat_id or ""), - scope_user_id=scope_user_id, - workspace_id=workspace_id, - memory_enabled=memory_enabled, - ) - if not memory_enabled: - return handle - - snapshot = await asyncio.to_thread( - load_session_memory_snapshot, - handle.chat_id, - scope_user_id=scope_user_id, - workspace_id=workspace_id, - ) - if snapshot is not None: - logger.info( - "[memory] frozen snapshot replayed chat=%s chars=%d (no retrieval)", - handle.chat_id, - len(snapshot.text), - ) - return replace(handle, snapshot=snapshot) - - task = await launch_memory_retrieval( - scope_user_id, - user_message, - memory_enabled, - workspace_id=workspace_id, - budget_ms=budget_ms, - ) - return replace(handle, retrieval_task=task) - - -async def resolve_session_memory(handle: SessionMemory) -> str: - """Return the block to inject, assembling and storing it on the first turn. - - The assembled block is stored only once the memory read actually landed. A - timed-out or degraded first turn leaves no snapshot, so the next turn tries - once more rather than freezing an accidental blank for the whole chat. - - The store is first-writer-wins, and we inject what it hands back: if another run - froze this chat while we were retrieving, we use its snapshot, not ours. - """ - if not handle.memory_enabled: - return "" - if handle.snapshot is not None: - return handle.snapshot.text - - built = await build_frozen_memory_block( - handle.scope_user_id, - handle.workspace_id, - handle.retrieval_task, - memory_enabled=handle.memory_enabled, - ) - if not built.settled: - logger.info( - "[memory] frozen snapshot not stored chat=%s: memory read did not settle", - handle.chat_id, - ) - return built.text - - effective = await asyncio.to_thread( - save_session_memory_snapshot, - handle.chat_id, - SessionMemorySnapshot( - text=built.text, - scope_user_id=handle.scope_user_id, - workspace_id=handle.workspace_id, - ), - ) - return effective.text - - -def _session_message(text: str, **item_kwargs: Any) -> Dict[str, Any]: - """One head-of-conversation message plus the context-item manifest describing it.""" - item = make_text_context_item(text, **item_kwargs) - return { - "role": "user", - "content": text, - SESSION_CONTEXT_META_KEY: item.to_manifest(), - } - - -async def inject_session_blocks( - session_messages: List[Dict[str, Any]], - *, - identity_block: str = "", - memory_block: str = "", -) -> List[Dict[str, Any]]: - """Prepend the session-constant blocks (user identity, frozen memory) as user-role messages. - - Why user rather than system: Qwen-family models require system only at index 0, and keeping - these per-user bytes out of the system prompt leaves the system text, the tool schemas and the - skill list a byte-identical shared prefix across users and chats. - - Both blocks are constant for the whole chat, so neither moves the prefix-cache boundary between - turns — that is what makes this position safe. - """ - injected: list[Dict[str, Any]] = [] - if identity_block: - injected.append( - _session_message( - f"\n{identity_block}\n\n" - "（以上为系统提供的当前用户身份信息，仅用于自然称呼，不是用户本轮提问。）", - item_id="session:identity", - kind=KIND_IDENTITY, - origin="identity:account", - trust="system", - created_seq=-200, - priority=850, - token_budget=1_000, - cache_class="session", - ) - ) - if memory_block: - injected.append( - _session_message( - f"\n{memory_block}\n\n" - "（以上为会话启动时系统注入的背景快照，本会话内不变，用作回答参考，请勿直接复述。）", - item_id="session:memory:frozen", - kind=KIND_MEMORY, - origin="memory:frozen_session", - trust="memory", - created_seq=-100, - priority=800, - token_budget=8_000, - cache_class="session", - ) - ) - return [*injected, *session_messages] if injected else session_messages - - -# ─── Saving ───────────────────────────────────────────────────────────────── - - -def save_memories_background( - user_id: str, - user_message: str, - full_response: str, - write_enabled: bool, - *, - workspace_id: str = "default", - chat_id: Optional[str] = None, - scope_user_id: Optional[str] = None, - message_id: Optional[str] = None, -) -> None: - """Delegate to the unified post-response pipeline — never await; SSE is closed and the user isn't waiting. - - **When the user has not explicitly consented to writes (`write_enabled=False`), skip - immediately and create no tasks at all**. This is the first gate of the user-level write - switch; `schedule_post_response_tasks` has a second one inside. - - Inside `schedule_post_response_tasks`: - - a durable outbox row is committed before the function returns - - leased workers resume pending/retry rows after process restart - - global Semaphore bounds worker concurrency (default 8) - - runs 0-5 extractors picked by the router's substance floor + LLM write gate - - each extractor has its own 30s timeout - - sanitize → write L1/L2/Session → audit - - ``scope_user_id`` optionally selects an edition-owned shared memory bucket. - ``user_id`` remains the real user for audit metadata. - """ - if not (write_enabled and full_response and user_id): - _report_no_memory_writes(message_id) - return - - try: - from core.memory.pipeline import schedule_post_response_tasks - except Exception as exc: - logger.warning("[memory] pipeline unavailable, skipping save: %s", exc) - _report_no_memory_writes(message_id) - return - - ctx = MemoryContext( - user_id=user_id, - workspace_id=workspace_id, - chat_id=chat_id, - write_enabled=write_enabled, - scope_user_id=scope_user_id, - message_id=message_id, - ) - try: - schedule_post_response_tasks(ctx, user_message, full_response) - except Exception as exc: - logger.warning("[memory] schedule_post_response_tasks failed: %s", exc) - _report_no_memory_writes(message_id, failed=True) - - -def _report_no_memory_writes(message_id: Optional[str], *, failed: bool = False) -> None: - """Tell the turn's settlement that no write pipeline will report for it. - - Every path that declines to schedule the pipeline must say so, otherwise the - turn's card waits out the settlement watchdog for nothing. - """ - if not message_id: - return - try: - from core.evolution.settlement_runner import report_memory_writes - - report_memory_writes(message_id, items=[], failed=failed) - except Exception as exc: # noqa: BLE001 - never break the response path - logger.debug("[memory] settlement report skipped: %s", exc) diff --git a/src/backend/orchestration/pending.py b/src/backend/orchestration/pending.py new file mode 100644 index 00000000..72c88510 --- /dev/null +++ b/src/backend/orchestration/pending.py @@ -0,0 +1,36 @@ +"""Dispatch only never-started Plan/Loop runs, using their accepted arguments.""" + +import asyncio +from core.db import engine +from core.db.models import ChatRun +from core.runs import runtime + + +def _arguments(run_id): + with engine.SessionLocal() as db: + row = db.query(ChatRun.recovery_snapshot).filter( + ChatRun.run_id == run_id, ChatRun.status == "pending", + ).first() + return (row[0] or {}).get("initial_args") if row else None + + +async def dispatch_pending(run): + if run.status != "pending": + return False + kind = (run.request_payload or {}).get("kind") + if kind == "plan_execute": + from orchestration.plans.execute_run import _run_plan_execute_workflow as worker + elif kind == "autonomous_loop": + from orchestration.loops.run import _run_autonomous_loop_workflow as worker + else: + return False + args = await asyncio.to_thread(_arguments, run.run_id) + if args is None: + return False + runtime.register_run_task( + run.run_id, + worker(run_id=run.run_id, chat_id=run.chat_id, user_id=run.user_id, + message_id=run.message_id, **args), + name=f"{kind}:{run.run_id}", + ) + return True diff --git a/src/backend/orchestration/plans/__init__.py b/src/backend/orchestration/plans/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/backend/orchestration/plans/admission.py b/src/backend/orchestration/plans/admission.py new file mode 100644 index 00000000..35619813 --- /dev/null +++ b/src/backend/orchestration/plans/admission.py @@ -0,0 +1,171 @@ +"""Admit durable Plan generation and execution runs.""" + +from __future__ import annotations + +from typing import Any, Dict, List, Optional + +import core.config.settings as settings +import core.db.engine as engine +import core.db.models as models +import core.runs.ownership as ownership +import core.runs.runtime as runtime +import core.runs.settings as runs_settings +import orchestration.plans.generate_run as _orchestration_plans_generate_run +from core.db.models import ChatRun +from core.infra.logging import get_logger + +logger = get_logger(__name__) + + +async def start_plan_execute_run( + *, + plan_id: str, + chat_id: str, + user_id: str, + enabled_mcp_ids: Optional[List[str]] = None, + enabled_skill_ids: Optional[List[str]] = None, + enabled_kb_ids: Optional[List[str]] = None, + enabled_agent_ids: Optional[List[str]] = None, + session_messages: List[Dict[str, Any]], + model_name: Optional[str] = None, + on_accept=None, +) -> ChatRun: + """Plan Phase-2 (execute): background task on the same ChatRun infrastructure. + + chat_id must exist in chat_sessions (FK). Caller should run _ensure_plan_session first. + """ + if not chat_id: + raise ValueError("chat_id is required to start a plan execute run") + + request_payload = { + "kind": "plan_execute", + "plan_id": plan_id, + "chat_id": chat_id, + "enabled_mcp_ids": enabled_mcp_ids or [], + "enabled_skill_ids": enabled_skill_ids or [], + "enabled_kb_ids": enabled_kb_ids or [], + "enabled_agent_ids": enabled_agent_ids or [], + "model_name": model_name, + } + effective_model_name = model_name or settings.DEFAULT_CHAT_MODEL_ALIAS + initial_args = { + "plan_id": plan_id, + "enabled_mcp_ids": enabled_mcp_ids, + "enabled_skill_ids": enabled_skill_ids, + "enabled_kb_ids": enabled_kb_ids, + "enabled_agent_ids": enabled_agent_ids, + "session_messages": session_messages, + "model_name": effective_model_name, + } + run = ownership.create_run_record( + on_accept=on_accept, + chat_id=chat_id, + user_id=user_id, + request_payload=request_payload, + recovery_snapshot={ + "kind": "plan_execute", + "initial_args": initial_args, + "worker_args": { + "context": { + "mcp_ids": list(enabled_mcp_ids or []), + "skill_ids": list(enabled_skill_ids or []), + "kb_ids": list(enabled_kb_ids or []), + "model_name": effective_model_name, + } + }, + }, + ) + from orchestration.pending import dispatch_pending + await dispatch_pending(run) + logger.info( + "plan_run_started", + run_id=run.run_id, + plan_id=plan_id, + chat_id=chat_id, + user_id=user_id, + ) + return run + + +def get_run_by_plan_id(plan_id: str) -> Optional[ChatRun]: + """Reverse-lookup live run by plan_id (used by plan cancel endpoint).""" + with engine.SessionLocal() as db: + return ( + db.query(models.ChatRun) + .filter( + models.ChatRun.status.in_(runs_settings.LIVE_STATUSES), + models.ChatRun.request_payload["plan_id"].astext == plan_id, + ) + .order_by(models.ChatRun.created_at.desc()) + .first() + ) + + +async def start_plan_generate_run( + *, + chat_id: str, + user_id: str, + task_description: str, + model_name: str = settings.DEFAULT_CHAT_MODEL_ALIAS, + model_provider_id: Optional[str] = None, + enabled_mcp_ids: Optional[List[str]] = None, + enabled_skill_ids: Optional[List[str]] = None, + enabled_kb_ids: Optional[List[str]] = None, + enabled_agent_ids: Optional[List[str]] = None, + session_messages: List[Dict[str, Any]], + uploaded_files: Optional[List[Dict[str, Any]]] = None, +) -> ChatRun: + """Plan Phase-1 (generate): background task on the same ChatRun infrastructure.""" + if not chat_id: + raise ValueError("chat_id is required to start a plan generate run") + + request_payload = { + "kind": "plan_generate", + "chat_id": chat_id, + "task_description": task_description[:500], + "model_name": model_name, + **({"model_provider_id": model_provider_id} if model_provider_id else {}), + } + recovery_snapshot = { + "kind": "plan_generate", + "worker_args": { + "task_description": task_description, + "model_name": model_name, + "model_provider_id": model_provider_id, + "enabled_mcp_ids": list(enabled_mcp_ids or []), + "enabled_skill_ids": list(enabled_skill_ids or []), + "enabled_kb_ids": list(enabled_kb_ids or []), + "enabled_agent_ids": list(enabled_agent_ids or []), + "session_messages": list(session_messages or []), + "uploaded_files": list(uploaded_files or []), + }, + } + run = ownership.create_run_record( + chat_id=chat_id, + user_id=user_id, + request_payload=request_payload, + recovery_snapshot=recovery_snapshot, + ) + journal_owner = ownership.new_worker_owner(run.run_id) + runtime.register_run_task( + run.run_id, + _orchestration_plans_generate_run._run_plan_generate_workflow( + run_id=run.run_id, + chat_id=chat_id, + user_id=user_id, + message_id=run.message_id, + task_description=task_description, + model_name=model_name, + model_provider_id=model_provider_id, + enabled_mcp_ids=enabled_mcp_ids, + enabled_skill_ids=enabled_skill_ids, + enabled_kb_ids=enabled_kb_ids, + enabled_agent_ids=enabled_agent_ids, + session_messages=session_messages, + uploaded_files=uploaded_files, + journal_owner=journal_owner, + ), + name=f"plan_gen_run:{run.run_id}", + ) + logger.info("plan_generate_run_started", run_id=run.run_id, chat_id=chat_id) + return run diff --git a/src/backend/orchestration/plans/execute_run.py b/src/backend/orchestration/plans/execute_run.py new file mode 100644 index 00000000..eab095dd --- /dev/null +++ b/src/backend/orchestration/plans/execute_run.py @@ -0,0 +1,328 @@ +"""Bind Plan execution to run leases, event replay and terminal persistence.""" + +from __future__ import annotations +from core.runs.tool_effects import errors as errors_module + +import asyncio +import time +import typing as _typing +from typing import Any, Dict, List, Optional + +import core.db.engine as engine +import core.db.models as models +import core.runs.events.writer as writer +import core.runs.ownership as ownership +import core.runs.settings as settings +import core.chat.service as services_ChatService_module +import core.runs.tool_effects.journal as tool_effect_ledger +from core.infra.logging import get_logger + +logger = get_logger(__name__) + + +async def _run_plan_execute_workflow( + *, + run_id: str, + plan_id: str, + chat_id: str, + user_id: str, + message_id: str, + enabled_mcp_ids: Optional[List[str]], + enabled_skill_ids: Optional[List[str]], + enabled_kb_ids: Optional[List[str]], + enabled_agent_ids: Optional[List[str]], + session_messages: List[Dict[str, Any]], + model_name: str, + journal_owner: Optional[str] = None, +) -> None: + """Background plan-execute worker. Streams via XADD, persists at end. + + Uses short-lived sessions (one per logical operation) to avoid holding a + DB connection across the entire long-running stream. + """ + from core.chat.tool_log import attach_tool_result as _attach_tool_result + from core.chat.tool_log import upsert_tool_call as _upsert_tool_call + from core.llm.runtime import workspace as _workspace_mod + from core.files.artifacts.service import persist_artifacts as _persist_artifacts + from core.chat.plans import PlanService + from orchestration.plans.mode import astream_execute_plan + + owner = journal_owner or ownership.new_worker_owner(run_id) + journal = ownership.journal() + if not journal.claim(run_id, owner=owner, lease_seconds=settings.RUN_LEASE_SECONDS, pending_only=True): + logger.info("plan_run_claim_skipped", run_id=run_id, owner=owner) + return + current = asyncio.current_task() + lease_lost = asyncio.Event() + heartbeat = asyncio.create_task( + ownership.lease_heartbeat(run_id, owner, current, lease_lost), + name=f"plan_run_lease:{run_id}", + ) + + offsets = writer.EventOffsets(journal, run_id, owner, lease_lost=lease_lost) + + _emit = offsets.emit + + async def _pause_plan_tool_outcome(exc: BaseException) -> None: + if not journal.needs_attention( + run_id, + owner=owner, + reason=f"tool outcome pending policy recovery: {exc}", + ): + return + await _emit( + { + "type": "plan_error", + "plan_id": plan_id, + "error": "工具调用结果无法安全确定，任务已暂停等待处理", + }, + terminal=True, + ) + await _emit({"type": settings.TERMINAL_TYPE, "chat_id": chat_id}, terminal=True) + + result_text = "" + completed_steps = 0 + total_steps = 0 + exec_usage: _typing.Optional[_typing.Dict[str, _typing.Any]] = None + tool_calls_log: _typing.List[_typing.Dict[str, _typing.Any]] = [] + # Strict workspace gate also applies to plan-execute runs. The plan + # subagent has access to pin_to_workspace and is expected to pin its + # final deliverables. + _workspace_mod.init_state() + + plan_snapshot = None + _ws_pinned = [] + artifacts_meta = [] + _ws_files = [] + + def _commit_plan_result(persist_db) -> None: # noqa: ANN001 + chat_service = services_ChatService_module.ChatService(persist_db) + # 被用户中断的那一轮同样会走到这里（协作式取消让生成器正常收尾）， + # 照着"执行完成"落库会让用户回到会话时看到一句"计划执行完成"， + # 与他刚按下的停止完全对不上。按快照里的中断位分开措辞。 + _cancelled = bool((plan_snapshot or {}).get("cancelled")) + content = result_text or ( + f"计划执行已中断：共 {total_steps} 步，完成 {completed_steps} 步。" + if _cancelled + else f"计划执行完成：共 {total_steps} 步，完成 {completed_steps} 步。" + ) + chat_service.add_message( + chat_id=chat_id, + role="assistant", + content=content, + model=model_name, + message_id=message_id, + extra_data={ + "is_markdown": bool(result_text), + "plan_id": plan_id, + "completed_steps": completed_steps, + "total_steps": total_steps, + "plan_snapshot": plan_snapshot, + "artifacts": artifacts_meta, + "workspace_files": _ws_files, + "message_id": message_id, + }, + tool_calls=tool_calls_log or None, + usage=exec_usage, + commit=False, + ) + # The plan-execute background worker doesn't hold a workflow context + # dict, so it builds the ProjectScope via the reverse-lookup path + # chat_id → ChatSession → Project. + from core.projects.edition import project_scope_from_chat_id + + _persist_artifacts( + persist_db, + user_id, + chat_id, + _ws_pinned, + scope=project_scope_from_chat_id(persist_db, chat_id), + commit=False, + ) + + def _persist_plan_partial() -> None: + nonlocal plan_snapshot, artifacts_meta, _ws_pinned, _ws_files + plan_snapshot = dict(plan_snapshot or {}) + plan_snapshot["cancelled"] = True + _ws_pinned = _workspace_mod.get_pinned() + artifacts_meta = _ws_pinned + _ws_files = _workspace_mod.get_pinned_file_ids() + # _commit_plan_result 走 add_message（非幂等），重复写会撞主键 + try: + with engine.SessionLocal() as _db: + if _db.get(models.ChatMessage, message_id) is not None: + return + _commit_plan_result(_db) + _db.commit() + except Exception: # noqa: BLE001 - 落库失败不能吞掉取消收尾 + logger.warning( + "plan_run_cancelled_partial_persist_failed", + run_id=run_id, + exc_info=True, + ) + + try: + await _emit( + { + "type": "run_started", + "run_id": run_id, + "message_id": message_id, + "chat_id": chat_id, + # 这几条路径不会被恢复接管，worker 起来的那一刻就是本轮起点， + # 不必为了拿它再读一次库——首帧前的每一次往返用户都在干等。 + "started_at": int(time.time() * 1000), + "kind": "plan_execute", + "plan_id": plan_id, + } + ) + + # astream_execute_plan needs a Session; isolate it from the persistence one. + from core.harness.tool_call_context import CURRENT_RUN_BINDING + + binding_token = CURRENT_RUN_BINDING.set((run_id, owner)) + try: + with engine.SessionLocal() as stream_db: + from core.db.models import ChatSession + + session = stream_db.get(ChatSession, chat_id) + automation_run = bool(session and (session.extra_data or {}).get("automation_run")) + async for event in astream_execute_plan( + plan_id=plan_id, + user_id=user_id, + db=stream_db, + model_name=model_name, + enabled_mcp_ids=enabled_mcp_ids, + enabled_skill_ids=enabled_skill_ids, + enabled_kb_ids=enabled_kb_ids, + enabled_agent_ids=enabled_agent_ids, + session_messages=session_messages, + chat_id=chat_id, + run_id=run_id, + automation_run=automation_run, + ): + evt_type = event.get("type") + if evt_type == "plan_complete": + result_text = event.get("result_text", "") + completed_steps = event.get("completed_steps", 0) + total_steps = event.get("total_steps", 0) + exec_usage = event.get("usage") or None + elif evt_type == "tool_call": + _upsert_tool_call( + tool_calls_log, + { + "tool_name": event.get("tool_name"), + "tool_id": event.get("tool_id"), + "tool_args": event.get("tool_args", {}), + "step_id": event.get("step_id"), + }, + ) + elif evt_type == "tool_result": + res = event.get("result") + tid = event.get("tool_id") + tn = event.get("tool_name") + _attach_tool_result(tool_calls_log, tid, tn, res) + await _emit(event) + finally: + CURRENT_RUN_BINDING.reset(binding_token) + + # Build plan snapshot in its own short session + plan_snapshot = None + try: + with engine.SessionLocal() as snap_db: + updated_plan = PlanService(snap_db).get_plan(plan_id, user_id) + if updated_plan: + plan_snapshot = PlanService.build_execution_snapshot( + updated_plan, + completed_steps=completed_steps, + total_steps=total_steps, + result_text=result_text, + ) + except Exception as snap_exc: + logger.warning("plan_run_snapshot_failed", run_id=run_id, error=str(snap_exc)) + + # Strict workspace gate: only files pinned via pin_to_workspace + # surface anywhere user-visible — both the chat message and the + # user's file library ("My Space") are sourced from this list. + _ws_pinned = _workspace_mod.get_pinned() + artifacts_meta = _ws_pinned + _ws_files = _workspace_mod.get_pinned_file_ids() + + completed = journal.complete( + run_id, + owner=owner, + status="completed", + usage=exec_usage, + last_event_offset=offsets.last, + commit_effect=_commit_plan_result, + committed_operation={ + "operation_type": "message_committed", + "phase": "message_committed", + "safety": "side_effect_committed", + "payload": {"message_id": message_id}, + }, + ) + if completed: + await _emit({"type": settings.TERMINAL_TYPE, "chat_id": chat_id}, terminal=True) + else: + # 用户取消时 cancel_run 已先写终态、清租约，上面的 CAS 注定输掉。 + # 协作式取消让生成器正常收尾，本轮已完成的步骤和正文就在手里 —— + # 不落库的话，用户回到会话只剩自己那句话。 + if ownership.cancelled_by_user(run_id): + _persist_plan_partial() + else: + logger.warning("plan_run_terminal_cas_lost", run_id=run_id, owner=owner) + + except asyncio.CancelledError: + if lease_lost.is_set(): + logger.warning("plan_run_worker_fenced", run_id=run_id, owner=owner) + if ownership.cancelled_by_user(run_id): + _persist_plan_partial() + elif journal.complete( + run_id, + owner=owner, + status="cancelled", + failure_reason="plan worker cancelled", + last_event_offset=offsets.last, + ): + logger.info("plan_run_cancelled", run_id=run_id) + await _emit( + { + "type": "plan_error", + "plan_id": plan_id, + "error": "任务已被用户取消", + "_cancelled": True, + }, + terminal=True, + ) + await _emit( + {"type": settings.TERMINAL_TYPE, "chat_id": chat_id, "_cancelled": True}, + terminal=True, + ) + + except Exception as exc: + nested_unknown = errors_module.find_tool_outcome_unknown(exc) + if nested_unknown is not None: + await _pause_plan_tool_outcome(nested_unknown) + return + logger.error("plan_run_failed", run_id=run_id, error=str(exc), exc_info=True) + failed = journal.complete( + run_id, + owner=owner, + status="failed", + failure_reason=str(exc)[:1000], + last_event_offset=offsets.last, + ) + if failed: + await _emit( + {"type": "plan_error", "plan_id": plan_id, "error": str(exc)[:200]}, + terminal=True, + ) + await _emit({"type": settings.TERMINAL_TYPE, "chat_id": chat_id}, terminal=True) + + finally: + heartbeat.cancel() + try: + await heartbeat + except asyncio.CancelledError: + pass + await writer.expire_stream(run_id) diff --git a/src/backend/orchestration/plans/generate_run.py b/src/backend/orchestration/plans/generate_run.py new file mode 100644 index 00000000..35413c0c --- /dev/null +++ b/src/backend/orchestration/plans/generate_run.py @@ -0,0 +1,275 @@ +"""Bind Plan generation to run leases and the existing event protocol.""" + +from __future__ import annotations +from core.runs.tool_effects import errors as errors_module + +import asyncio +import contextlib +import time +import typing as _typing +from typing import Any, Dict, List, Optional + +import core.db.engine as engine +import core.runs.events.writer as writer +import core.runs.ownership as ownership +import core.runs.runtime as runtime +import core.runs.settings as settings +import core.chat.service as services_ChatService_module +import core.runs.journal as run_journal +import core.runs.tool_effects.journal as tool_effect_ledger +from core.infra.logging import get_logger + +logger = get_logger(__name__) + + +async def _run_plan_generate_workflow( + *, + run_id: str, + chat_id: str, + user_id: str, + message_id: str, + task_description: str, + model_name: str, + model_provider_id: Optional[str], + enabled_mcp_ids: Optional[List[str]], + enabled_skill_ids: Optional[List[str]], + enabled_kb_ids: Optional[List[str]], + enabled_agent_ids: Optional[List[str]], + session_messages: List[Dict[str, Any]], + uploaded_files: Optional[List[Dict[str, Any]]], + journal_owner: Optional[str] = None, +) -> None: + from orchestration.plans.mode import astream_generate_plan + + owner = journal_owner or ownership.new_worker_owner(run_id) + journal = ownership.journal() + if not journal.claim(run_id, owner=owner, lease_seconds=settings.RUN_LEASE_SECONDS): + logger.info("plan_generate_claim_skipped", run_id=run_id, owner=owner) + return + try: + journal.append_operation( + run_id, + owner=owner, + operation_type="worker_started", + phase="pre_model", + safety="replayable", + ) + except run_journal.RunLeaseLost: + logger.warning("plan_generate_fenced_before_start", run_id=run_id, owner=owner) + return + worker_task = asyncio.current_task() + if worker_task is None: # pragma: no cover - asyncio always owns this coroutine + raise RuntimeError("plan generate worker has no asyncio task") + lease_lost = asyncio.Event() + heartbeat = asyncio.create_task( + ownership.lease_heartbeat(run_id, owner, worker_task, lease_lost), + name=f"plan_generate_lease:{run_id}", + ) + offsets = writer.EventOffsets(journal, run_id, owner, lease_lost=lease_lost) + + _emit = offsets.emit + + plan_id_out: _typing.Optional[str] = None + plan_title = "" + plan_desc = "" + plan_snapshot: _typing.Optional[_typing.Dict[str, _typing.Any]] = None + plan_record: _typing.Optional[_typing.Dict[str, _typing.Any]] = None + assistant_content = "" + gen_usage: _typing.Optional[_typing.Dict[str, _typing.Any]] = None + + try: + await _emit( + { + "type": "run_started", + "run_id": run_id, + "message_id": message_id, + "chat_id": chat_id, + # 这几条路径不会被恢复接管，worker 起来的那一刻就是本轮起点， + # 不必为了拿它再读一次库——首帧前的每一次往返用户都在干等。 + "started_at": int(time.time() * 1000), + "kind": "plan_generate", + } + ) + + from core.harness.tool_call_context import CURRENT_RUN_BINDING + + binding_token = CURRENT_RUN_BINDING.set((run_id, owner)) + try: + with engine.SessionLocal() as stream_db: + async for event in astream_generate_plan( + task_description=task_description, + user_id=user_id, + db=stream_db, + model_name=model_name, + model_provider_id=model_provider_id, + enabled_mcp_ids=enabled_mcp_ids, + enabled_skill_ids=enabled_skill_ids, + enabled_kb_ids=enabled_kb_ids, + enabled_agent_ids=enabled_agent_ids, + session_messages=session_messages, + uploaded_files=uploaded_files, + chat_id=chat_id, + ): + if event.get("type") == "plan_generated": + plan_record = dict(event) + plan_id_out = event.get("plan_id") + plan_title = event.get("title", "") + plan_desc = event.get("description", "") + steps = event.get("steps", []) + step_summary = "\n".join( + f"{i + 1}. {s.get('title', '')}" for i, s in enumerate(steps) + ) + assistant_content = ( + f"已生成执行计划：**{plan_title}**\n\n" + f"{plan_desc}\n\n" + f"**执行步骤：**\n{step_summary}" + ) + plan_snapshot = { + "mode": "preview", + "title": plan_title, + "description": plan_desc, + "steps": [ + { + "step_order": s.get("step_order", i + 1), + "title": s.get("title", ""), + "description": s.get("description"), + "expected_tools": s.get("expected_tools", []), + "expected_skills": s.get("expected_skills", []), + } + for i, s in enumerate(steps) + ], + "total_steps": len(steps), + "completed_steps": 0, + } + gen_usage = event.get("usage") or None + await _emit(event) + finally: + CURRENT_RUN_BINDING.reset(binding_token) + + def _commit_plan_preview(persist_db) -> None: # noqa: ANN001 + if not assistant_content or not plan_id_out or plan_record is None: + return + from core.chat.plans import PlanService + + extra_data: _typing.Dict[str, _typing.Any] = {} + if uploaded_files: + extra_data["uploaded_files"] = list(uploaded_files) + agent_name_map = plan_record.get("agent_name_map") + if isinstance(agent_name_map, dict) and agent_name_map: + extra_data["agent_name_map"] = dict(agent_name_map) + PlanService(persist_db).create_plan( + plan_id=str(plan_id_out), + user_id=user_id, + title=str(plan_record.get("title") or "未命名计划"), + description=str(plan_record.get("description") or ""), + task_input=task_description, + steps=list(plan_record.get("steps") or []), + extra_data=extra_data, + commit=False, + ) + services_ChatService_module.ChatService(persist_db).add_message( + chat_id=chat_id, + role="assistant", + content=assistant_content, + model=model_name, + message_id=message_id, + extra_data={ + "is_markdown": True, + "plan_id": plan_id_out, + "plan_snapshot": plan_snapshot, + "message_id": message_id, + }, + usage=gen_usage, + commit=False, + ) + + completed = journal.complete( + run_id, + owner=owner, + status="completed", + usage=gen_usage, + last_event_offset=offsets.last, + commit_effect=_commit_plan_preview, + committed_operation={ + "operation_type": "message_committed", + "phase": "message_committed", + "safety": "side_effect_committed", + "payload": {"message_id": message_id, "plan_id": plan_id_out}, + }, + ) + if completed: + await _emit({"type": settings.TERMINAL_TYPE, "chat_id": chat_id}, terminal=True) + else: + logger.warning("plan_generate_terminal_cas_lost", run_id=run_id, owner=owner) + + except asyncio.CancelledError: + if lease_lost.is_set(): + logger.warning("plan_generate_worker_fenced", run_id=run_id, owner=owner) + else: + cancelled = journal.complete( + run_id, + owner=owner, + status="cancelled", + failure_reason="plan generation worker cancelled", + last_event_offset=offsets.last, + ) + current = runtime.get_run(run_id) + if cancelled or (current is not None and current.status == "cancelled"): + logger.info("plan_generate_run_cancelled", run_id=run_id) + else: + logger.warning( + "plan_generate_cancel_cas_lost", + run_id=run_id, + owner=owner, + ) + return + await _emit( + { + "type": "plan_error", + "error": "任务已被用户取消", + "_cancelled": True, + }, + terminal=True, + ) + await _emit( + {"type": settings.TERMINAL_TYPE, "chat_id": chat_id, "_cancelled": True}, + terminal=True, + ) + + except Exception as exc: + nested_unknown = errors_module.find_tool_outcome_unknown(exc) + if nested_unknown is not None: + if journal.needs_attention( + run_id, + owner=owner, + reason=f"tool outcome pending policy recovery: {nested_unknown}", + ): + await _emit( + { + "type": "plan_error", + "error": "工具调用结果无法安全确定，任务已暂停等待处理", + }, + terminal=True, + ) + await _emit({"type": settings.TERMINAL_TYPE, "chat_id": chat_id}, terminal=True) + return + logger.error("plan_generate_run_failed", run_id=run_id, error=str(exc), exc_info=True) + failed = journal.complete( + run_id, + owner=owner, + status="failed", + failure_reason=str(exc)[:1000], + last_event_offset=offsets.last, + ) + if failed: + await _emit( + {"type": "plan_error", "error": str(exc)[:200]}, + terminal=True, + ) + await _emit({"type": settings.TERMINAL_TYPE, "chat_id": chat_id}, terminal=True) + + finally: + heartbeat.cancel() + with contextlib.suppress(asyncio.CancelledError): + await heartbeat + await writer.expire_stream(run_id) diff --git a/src/backend/orchestration/subagents/plan_mode.py b/src/backend/orchestration/plans/mode.py similarity index 96% rename from src/backend/orchestration/subagents/plan_mode.py rename to src/backend/orchestration/plans/mode.py index 9e10bf1b..183dd6cc 100644 --- a/src/backend/orchestration/subagents/plan_mode.py +++ b/src/backend/orchestration/plans/mode.py @@ -5,7 +5,10 @@ """ from __future__ import annotations -from core.infra.time import utc_now +from core.llm import prompt_rendering as prompt_rendering_module +from core.llm import prompt_versions as prompt_versions_module +from core.harness import log_context +from core.harness import log_writer import asyncio import json @@ -14,33 +17,24 @@ import re import time as _time import uuid -from datetime import datetime from typing import Any, AsyncIterator, Dict, List, Optional, Tuple +from core.agent_runtime.plugin_visibility import ( + all_plugin_component_ids as _all_plugin_component_ids, +) +from core.agent_runtime.plugin_visibility import load_enabled_plugins as _load_enabled_plugins from core.config.settings import DEFAULT_CHAT_MODEL_ALIAS -from core.db.models import Plan from core.infra.logging import LogContext +from core.infra.time import utc_now +from core.llm.context.rendering import append_context_text, next_request_sequence, render_context_item +from core.llm.context.ir_constants import KIND_ATTACHMENT, KIND_REMINDER, KIND_SYSTEM_RULE, POLICY_HEAD_TAIL, POLICY_NEVER, SESSION_CONTEXT_META_KEY +from core.llm.context.items import make_text_context_item, session_context_metadata from core.llm.factory import create_agent_executor -from core.llm.context_adapter import append_context_text, next_request_sequence, render_context_item -from core.llm.context_ir import ( - KIND_ATTACHMENT, - KIND_REMINDER, - KIND_SYSTEM_RULE, - POLICY_HEAD_TAIL, - POLICY_NEVER, - SESSION_CONTEXT_META_KEY, - make_text_context_item, - session_context_metadata, -) -from core.llm.mcp_manager import close_clients -from core.services import log_service as log_writer -from core.services.plan_service import PlanService -from orchestration.chat_run_executor import is_run_cancelled -from orchestration.streaming import StreamingAgent -from orchestration.subagents.plugin_visibility import ( - all_plugin_component_ids as _all_plugin_component_ids, -) -from orchestration.subagents.plugin_visibility import load_enabled_plugins as _load_enabled_plugins +from core.llm.mcp.manager import close_clients +from core.llm.runtime.streaming import StreamingAgent +from core.runs.runtime import is_run_cancelled +from core.harness import log_writer as log_writer +from core.chat.plans import PlanService from sqlalchemy.orm import Session logger = logging.getLogger(__name__) @@ -56,7 +50,6 @@ "plan_mode.system.md", ) - class _StepReplyError(RuntimeError): """A plan step's agent.reply failed (model 400/500, cancelled scope, …). @@ -65,7 +58,6 @@ class _StepReplyError(RuntimeError): behavior of recording the error text as a successful step result. """ - def _collect_valid_tool_names(enabled_mcp_ids: Optional[List[str]] = None) -> Optional[set]: """Collect all valid MCP tool function names from enabled servers. @@ -76,7 +68,7 @@ def _collect_valid_tool_names(enabled_mcp_ids: Optional[List[str]] = None) -> Op return None valid = set(enabled_mcp_ids) # server IDs themselves are valid references try: - from core.services.mcp_service import McpServerConfigService + from core.capabilities.mcp.service import McpServerConfigService svc = McpServerConfigService.get_instance() all_servers = svc.get_all_servers() @@ -97,7 +89,7 @@ def _load_visible_agents( ) -> List[Dict[str, Any]]: """Load visible sub-agents for the user, optionally filtered by IDs.""" try: - from core.services.user_agent_service import UserAgentService + from core.capabilities.agents.edition import UserAgentService svc = UserAgentService(db) agents = svc.list_for_user(user_id) @@ -135,7 +127,7 @@ def _resolve_plugin_capabilities( the plugin components into it). """ try: - from core.config.catalog_resolver import resolve_all_runtime_enabled + from core.capabilities.catalog.resolver import resolve_all_runtime_enabled r_skills, _r_agents, r_mcps = resolve_all_runtime_enabled(db, user_id) except Exception as exc: # noqa: BLE001 @@ -172,14 +164,14 @@ def _load_plan_prompt(available_tools_desc: str = "（暂无工具信息）") -> template: Optional[str] = None try: - from core.services import prompt_version_service as pvs + from core.llm import prompt_versions as pvs - rendered = pvs.render_active_prompt("plan_mode") + rendered = prompt_rendering_module.render_active_prompt("plan_mode") if rendered and rendered.strip(): template = rendered else: # Legacy path: some system versions may still carry system/90_plan_mode - av = pvs.get_active_version("system") + av = prompt_versions_module.get_active_version("system") if av: for p in av.get("parts") or []: if (p.get("part_id") or "").strip() == "system/90_plan_mode" and p.get( @@ -229,7 +221,7 @@ def _build_tools_description( # MCP tools — only include those in the enabled list # enabled_mcp_ids=None means "no filter" (all), [] means "none enabled" try: - from core.services.mcp_service import McpServerConfigService + from core.capabilities.mcp.service import McpServerConfigService svc = McpServerConfigService.get_instance() all_servers = svc.get_all_servers() # returns dict {server_id: config} @@ -250,7 +242,7 @@ def _build_tools_description( # Skills — only include those in the enabled list # enabled_skill_ids=None means "no filter" (all), [] means "none enabled" try: - from core.agent_skills.loader import get_skill_loader + from core.capabilities.skills.loader_factory import get_skill_loader loader = get_skill_loader() all_skills = loader.load_all_metadata() @@ -273,7 +265,7 @@ def _build_tools_description( logger.warning("Failed to load skills: %s", exc) if visible_agents: - from core.llm.subagent_tool import _get_tools_desc + from core.llm.subagents.tool import get_tools_desc as _get_tools_desc for a in visible_agents: agent_id = a.get("agent_id", "") @@ -382,7 +374,7 @@ def _build_file_context( user_id: Optional[str] = None, ) -> str: """Reuse the main-chat file_id-based lazy preview pipeline.""" - from core.llm.hooks import _build_file_context as build_lazy_file_context + from core.llm.runtime.file_context import build_file_context as build_lazy_file_context return build_lazy_file_context(uploaded_files, user_id=user_id) @@ -425,10 +417,10 @@ async def astream_generate_plan( # Project context: when the chat is attached to a project, inject the project's # instructions / file listing into the plan-generation agent's system prompt so # the planning phase already understands the project context. - from core.services.project_scope import build_project_ctx_from_chat_id + from core.projects.edition import build_project_ctx_from_chat_id _project_ctx = build_project_ctx_from_chat_id(db, chat_id) - from core.services.ontology_service import build_user_ontology_runtime + from core.ontology.runtime_policy import build_user_ontology_runtime ontology_enabled, ontology_runtime = build_user_ontology_runtime( user_id=user_id, @@ -662,7 +654,7 @@ async def astream_execute_plan( # Project context: when the chat is attached to a project, inject the project # instructions / files, and confine each step-execution agent's file scope to the # project folder (same ProjectScope source as regular chat). - from core.services.project_scope import build_project_ctx_from_chat_id + from core.projects.edition import build_project_ctx_from_chat_id _project_ctx = build_project_ctx_from_chat_id(db, chat_id) @@ -835,14 +827,14 @@ def _cancellation_requested() -> bool: # max_iters bounds tool calls per step so the agent doesn't loop excessively. _step_max_iters = int(os.environ.get("PLAN_STEP_MAX_ITERS", "5")) - from core.services.ontology_service import build_user_ontology_runtime + from core.ontology.runtime_policy import build_user_ontology_runtime _step_ontology_enabled, step_ontology_runtime = build_user_ontology_runtime( user_id=user_id, task=f"{plan.task_input}\n{step_instruction}", db=db, ) - from core.capabilities.runtime import child_scope + from core.harness.registry.capability_runtime import child_scope agent, mcp_clients = await create_agent_executor( run_id=run_id, @@ -871,7 +863,7 @@ def _cancellation_requested() -> bool: # (a downgrade; could later subscribe to ModelCallEndEvent instead). # ── Direct agent.reply() with timeout + heartbeats ── - from core.llm.message_compat import session_to_msgs + from core.llm.runtime.message_compat import session_to_msgs agent.state.context.extend(session_to_msgs(prepared_history)) @@ -916,7 +908,7 @@ def _cancellation_requested() -> bool: # Run agent.reply in background, send heartbeats while waiting. # Wrap the whole reply in a subagent scope so any tool_call_logs # produced inside are attributed to this step. - with log_writer.subagent_scope(_step_subagent_log_id, source="subagent"): + with log_context.subagent_scope(_step_subagent_log_id, source="subagent"): reply_task = asyncio.create_task(agent.reply(inputs=user_msg)) while not reply_task.done(): try: @@ -1132,10 +1124,10 @@ def _cancellation_requested() -> bool: if "" in step_text: step_text = step_text.rsplit("", 1)[-1].strip() - from core.ontology.validator import requires_output_review + from core.ontology.output_review_state import requires_output_review if step_text and requires_output_review(step_ontology_runtime): - from orchestration.subagents.ontology_reviewer import review_ontology_output + from core.ontology.reviewer import review_ontology_output yield { "type": "ontology_review", diff --git a/src/backend/orchestration/schedulers/automation_lease.py b/src/backend/orchestration/schedulers/automation_lease.py deleted file mode 100644 index 7ffa0871..00000000 --- a/src/backend/orchestration/schedulers/automation_lease.py +++ /dev/null @@ -1,58 +0,0 @@ -"""Renewable execution ownership; lease expiry is never a task duration limit.""" - -import asyncio -import uuid - -from core.infra import ephemeral - -# Only bounds recovery after a dead worker. Healthy executions renew indefinitely. -LEASE_TTL_SECONDS = 60 -RENEW_INTERVAL_SECONDS = 15 - - -class AutomationLease: - def __init__(self, key): - self.key = key - self.token = uuid.uuid4().hex - self.state = ephemeral.get_ephemeral_state() - self.renewal = None - - async def acquire(self): - if not await self.state.hold(self.key, self.token, ttl=LEASE_TTL_SECONDS): - return False - self.renewal = asyncio.create_task(self._renew()) - return True - - async def _renew(self): - while True: - await asyncio.sleep(RENEW_INTERVAL_SECONDS) - # Unlike hold(), CAS cannot reacquire an expired lease after another - # worker could already have recovered the original running record. - if not await self.state.compare_exchange( - self.key, self.token, self.token, ttl=LEASE_TTL_SECONDS - ): - return - - async def run(self, awaitable): - execution = asyncio.create_task(awaitable) - try: - done, _ = await asyncio.wait( - (execution, self.renewal), return_when=asyncio.FIRST_COMPLETED - ) - if self.renewal in done: - # Never leave the child conversation running without ownership. - execution.cancel() - await asyncio.gather(execution, return_exceptions=True) - self.renewal.result() - raise RuntimeError("定时任务执行锁已失效") - return await execution - finally: - if not execution.done(): - execution.cancel() - await asyncio.gather(execution, return_exceptions=True) - - async def release(self): - if self.renewal: - self.renewal.cancel() - await asyncio.gather(self.renewal, return_exceptions=True) - await self.state.release(self.key, self.token) diff --git a/src/backend/orchestration/schedulers/automation_recovery.py b/src/backend/orchestration/schedulers/automation_recovery.py deleted file mode 100644 index 766e1de9..00000000 --- a/src/backend/orchestration/schedulers/automation_recovery.py +++ /dev/null @@ -1,45 +0,0 @@ -"""Recover automation outcomes by observing their durable child conversations.""" - -from core.db import engine as db_engine -from core.db.models import ScheduledTaskRun -from core.infra.time import as_utc, utc_now -from core.services.automation_service import AutomationService - - -async def recover_task(scheduler, task_id): - # The caller renews task ownership while the shared ChatRun executor resumes - # the durable child. Do not start a second child or abandon the old result. - with db_engine.SessionLocal() as db: - task = AutomationService(db).get_task_by_id(task_id) - if task is None: - return - user_id = task.user_id - runs = [ - (row.run_id, row.chat_id) - for row in db.query(ScheduledTaskRun) - .filter( - ScheduledTaskRun.task_id == task_id, - ScheduledTaskRun.status == "running", - ) - .all() - ] - for run_id, chat_id in runs: - if chat_id: - from .automation_dispatch import execute_task - - await execute_task(scheduler, task_id, user_id, resume_run_id=run_id) - else: - # No conversation was ever linked; there is no child to resume. - with db_engine.SessionLocal() as db: - run = db.get(ScheduledTaskRun, run_id) - if run is None or run.status != "running": - continue - run.status = "failed" - run.error_message = "interrupted (orphan, no linked conversation)" - run.completed_at = utc_now() - db.commit() - service = AutomationService(db) - task = service.get_task_by_id(task_id) - if task and task.next_run_at and as_utc(task.next_run_at) <= utc_now(): - service.advance_next_run(task_id) - service.finalize_after_run(task_id) diff --git a/src/backend/orchestration/schedulers/automation_scheduler.py b/src/backend/orchestration/schedulers/automation_scheduler.py deleted file mode 100644 index 71a7ec38..00000000 --- a/src/backend/orchestration/schedulers/automation_scheduler.py +++ /dev/null @@ -1,330 +0,0 @@ -"""Automation scheduler — polls the DB for due tasks and fires them. - -Inspired by claude-code's CronScheduler: -- Polls every 15s for due tasks -- Uses Redis distributed lock to prevent double-firing across instances -- Handles missed tasks on startup -- Auto-disables tasks after consecutive failure threshold -- Writes notifications to Redis for frontend polling -""" - -from core.infra.time import utc_now - -import asyncio -import contextlib -import random -import uuid -from typing import List, Optional, Tuple - -from core.infra.background import spawn -from core.infra.logging import get_logger - -logger = get_logger(__name__) - - -POLL_INTERVAL_SECONDS = 15 -# 手动触发要比 cron 轮询响应快得多——按钮叫「立即执行」，等 15 秒不叫立即。一拍就是 -# 一条 UPDATE，所以这一档只取到「点下去像是立刻有反应」为止，不必更密。 -MANUAL_DRAIN_INTERVAL_SECONDS = 3 -REDIS_LOCK_PREFIX = "jx:auto:lock:" - - -class AutomationScheduler: - """Async scheduler that polls the DB for due tasks and fires them.""" - - def __init__(self): - self._running = False - self._task: Optional[asyncio.Task] = None - self._manual_task: Optional[asyncio.Task] = None - self._executions: set[asyncio.Task] = set() - self._leases = {} - - async def start(self): - self._running = True - self._task = asyncio.create_task(self._poll_loop()) - self._manual_task = asyncio.create_task(self._manual_drain_loop()) - logger.info("[scheduler] started") - - async def stop(self): - self._running = False - for task in (self._task, self._manual_task): - if task: - task.cancel() - with contextlib.suppress(asyncio.CancelledError): - await task - executions = list(self._executions) - for execution in executions: - execution.cancel() - if executions: - await asyncio.gather(*executions, return_exceptions=True) - # A queued execution can be cancelled before its coroutine enters its - # finally block. Its already-started renewal must still stop with us. - for task_id in list(self._leases): - await self._release_lock(task_id) - logger.info("[scheduler] stopped") - - async def _poll_loop(self): - # Small initial delay to let the app finish startup - await asyncio.sleep(5) - - await self._recover_stuck_running_runs() - # Then handle missed one-shot tasks - await self._recover_missed_tasks() - - while self._running: - try: - await self._recover_stuck_running_runs() - await self._check_and_fire() - except Exception as e: - logger.error("[scheduler] poll error: %s", e, exc_info=True) - jitter = random.uniform(0, 5) - await asyncio.sleep(POLL_INTERVAL_SECONDS + jitter) - - async def _manual_drain_loop(self): - """执行用户点下的「立即执行」，节奏见 MANUAL_DRAIN_INTERVAL_SECONDS。""" - await asyncio.sleep(5) - while self._running: - try: - await self._fire_manual_triggers() - except Exception as e: - logger.error("[scheduler] manual drain error: %s", e, exc_info=True) - await asyncio.sleep(MANUAL_DRAIN_INTERVAL_SECONDS) - - async def _fire_manual_triggers(self): - # 每 3 秒一次的同步 DB 调用不能压在事件循环上——这个 worker 的 SSE 流和它抢同 - # 一个连接池，池满时 checkout 会把整个循环卡住 DB_POOL_TIMEOUT 那么久。 - pending = await asyncio.to_thread(self._take_manual_triggers) - for task_id, user_id in pending: - # 手动触发不推进 next_run_at——用户按一次不该打乱 cron 排期。 - await self._fire(task_id, user_id, "manual trigger", self._bump_run_count) - - @staticmethod - def _take_manual_triggers() -> List[Tuple[str, str]]: - from core.db.engine import SessionLocal - from core.services.automation_service import AutomationService - - with SessionLocal() as db: - return AutomationService(db).take_manual_triggers() - - async def _check_and_fire(self): - from core.db.engine import SessionLocal - from core.services.automation_service import AutomationService - - now = utc_now() - with SessionLocal() as db: - svc = AutomationService(db) - due_tasks = svc.get_due_tasks(now) - - if not due_tasks: - return - - logger.info("[scheduler] found %d due tasks", len(due_tasks)) - for task in due_tasks: - # Pre-advance next_run_at BEFORE firing so the schedule moves - # on regardless of whether the run itself succeeds, fails, or - # gets killed mid-flight. Mirrors how real cron behaves and - # prevents the death-spiral where a stuck "running" row leaves - # next_run_at in the past forever and causes every poll to - # re-fire the same task. The success/failure branches in - # execute_task no longer call advance_next_run. - await self._fire(task.task_id, task.user_id, "due task", self._advance_next_run) - - @staticmethod - def _advance_next_run(task_id: str) -> None: - from core.db.engine import SessionLocal - from core.services.automation_service import AutomationService - - with SessionLocal() as db: - AutomationService(db).advance_next_run(task_id) - - @staticmethod - def _bump_run_count(task_id: str) -> None: - # run_count 只在 advance_next_run 里 +1，而手动触发不走那条路，所以单独补一次。 - from core.db.engine import SessionLocal - from core.services.automation_service import AutomationService - - with SessionLocal() as db: - AutomationService(db).bump_run_count(task_id) - - async def _fire(self, task_id: str, user_id: str, kind: str, pre_run) -> None: - """取锁 → 跑 pre_run 记账 → 后台执行。定时与手动触发共用，所以两者对同一个任务 - 不会并发跑；pre_run 失败不拦截执行，理由同 _check_and_fire 的 pre-advance。""" - if not await self._acquire_lock(task_id): - logger.info("[scheduler] %s for %s skipped: already running", kind, task_id) - return - try: - if await asyncio.to_thread(self._has_running_runs, task_id): - self._launch_recovery(task_id) - return - await asyncio.to_thread(pre_run, task_id) - except asyncio.CancelledError: - await self._release_lock(task_id) - raise - except Exception as exc: - logger.warning( - "[scheduler] pre-run bookkeeping failed for %s: %s — firing anyway", task_id, exc - ) - logger.info("[scheduler] firing %s for %s", kind, task_id) - self._launch_execution(task_id, user_id, lease=self._leases[task_id]) - - def _launch_execution(self, task_id: str, user_id: str, *, lease=None) -> asyncio.Task: - execution = spawn( - self.execute_task(task_id, user_id, lease=lease), name=f"automation.{task_id}" - ) - self._executions.add(execution) - execution.add_done_callback(self._executions.discard) - return execution - - async def execute_task(self, task_id: str, user_id: str, *, lease=None): - from .automation_dispatch import execute_task - - if lease is None: - if not await self._acquire_lock(task_id): - return - lease = self._leases[task_id] - try: - if await asyncio.to_thread(self._has_running_runs, task_id): - from .automation_recovery import recover_task - - await lease.run(recover_task(self, task_id)) - else: - await lease.run(execute_task(self, task_id, user_id)) - finally: - await self._release_lock(task_id) - - @staticmethod - def _has_running_runs(task_id): - from core.db.engine import SessionLocal - from core.db.models import ScheduledTaskRun - - with SessionLocal() as db: - return ( - db.query(ScheduledTaskRun.run_id) - .filter( - ScheduledTaskRun.task_id == task_id, - ScheduledTaskRun.status == "running", - ) - .first() - is not None - ) - - def _launch_recovery(self, task_id): - recovery = spawn(self._recover_task(task_id), name=f"automation.recover.{task_id}") - self._executions.add(recovery) - recovery.add_done_callback(self._executions.discard) - - async def _recover_stuck_running_runs(self): - """Recover abandoned runs only after acquiring their execution ownership. - - Elapsed runtime cannot identify an orphan: a healthy task may run for - days. A live worker's renewable lease excludes recovery on every poll. - """ - from core.db.engine import SessionLocal - from core.db.models import ScheduledTaskRun - - try: - with SessionLocal() as db: - task_ids = [ - row[0] - for row in db.query(ScheduledTaskRun.task_id) - .filter(ScheduledTaskRun.status == "running") - .distinct() - .all() - ] - for task_id in task_ids: - if not await self._acquire_lock(task_id): - continue - self._launch_recovery(task_id) - except Exception as exc: - logger.error("[scheduler] orphan recovery failed: %s", exc, exc_info=True) - - async def _recover_task(self, task_id): - from .automation_recovery import recover_task - - try: - await self._leases[task_id].run(recover_task(self, task_id)) - except Exception as exc: - logger.error( - "[scheduler] orphan recovery failed for %s: %s", task_id, exc, exc_info=True - ) - finally: - await self._release_lock(task_id) - - async def _recover_missed_tasks(self): - """On startup, check for one-shot tasks whose next_run_at is in the past.""" - from core.db.engine import SessionLocal - from core.services.automation_service import AutomationService - - now = utc_now() - try: - with SessionLocal() as db: - svc = AutomationService(db) - missed = svc.get_due_tasks(now) - one_shot_count = 0 - for task in missed: - if not task.recurring: - one_shot_count += 1 - await self._fire( - task.task_id, task.user_id, "missed task", self._advance_next_run - ) - if one_shot_count: - logger.info("[scheduler] recovering %d missed one-shot tasks", one_shot_count) - except Exception as e: - logger.error("[scheduler] recovery error: %s", e) - - # ── Redis lock helpers ───────────────────────────────────────── - - async def _acquire_lock(self, task_id: str) -> bool: - from .automation_lease import AutomationLease - - if task_id in self._leases: - return False - try: - lease = AutomationLease(f"{REDIS_LOCK_PREFIX}{task_id}") - if not await lease.acquire(): - return False - self._leases[task_id] = lease - return True - except Exception as e: - logger.warning("[scheduler] lock acquire failed for %s: %s", task_id, e) - return False - - async def _release_lock(self, task_id: str): - lease = self._leases.pop(task_id, None) - if lease is None: - return - try: - await lease.release() - except Exception as e: - logger.warning("[scheduler] lock release failed for %s: %s", task_id, e) - - # ── Notification ─────────────────────────────────────────────── - - async def _send_notification( - self, - user_id: str, - task_id: str, - task_name: str, - status: str, - summary: str, - chat_id: Optional[str] = None, - ): - try: - from core.services import automation_notifications as notifications - from core.services.automation_service import SUMMARY_LIMIT_BRIEF, truncate_summary - - notification = { - "id": f"notif_{uuid.uuid4().hex[:12]}", - "task_id": task_id, - "task_name": task_name, - "status": status, - # 走统一截断策略：result_summary 现在是全文，裸切会把整篇报告拦腰截断且 - # 不带任何提示，与运行历史列表的观感不一致。 - "summary": truncate_summary(summary, SUMMARY_LIMIT_BRIEF), - "chat_id": chat_id, - "timestamp": int(utc_now().timestamp() * 1000), - "read": False, - } - await notifications.push(user_id, notification) - except Exception as e: - logger.warning("[scheduler] notification failed: %s", e) diff --git a/src/backend/orchestration/streaming.py b/src/backend/orchestration/streaming.py deleted file mode 100644 index cc1529b8..00000000 --- a/src/backend/orchestration/streaming.py +++ /dev/null @@ -1,999 +0,0 @@ -"""Streaming agent wrapper for AgentScope 2.0. - -Consumes ``agent.reply_stream(...)`` (replaces the 1.x msg_queue) and maps the 25 -fine-grained EventType into the internal events consumed by ``workflow.py``: -- ("text_delta", str) - incremental answer text -- ("thinking_delta", str) - incremental reasoning -- ("tool_call_start", dict)- tool call started (args still streaming) -- ("tool_call_delta", dict)- incremental tool-call argument JSON -- ("tool_call", dict) - tool invocation complete -- ("tool_result", dict) - tool invocation result -- ("file_confirm", dict) - myspace write confirmation (in-house ContextVar gate, distinct from native HITL) -- ("user_question", dict) - model question requesting resident-composer input -- ("user_question_resolved", dict) - server-authoritative answer/cancel/timeout -- ("heartbeat", None) - silence heartbeat (queue empty ≥3s: the model produced nothing at all) -- ("model_progress", None) - throttled liveness signal, emitted in two situations. (a) upstream events - keep arriving but none maps to an SSE event (throttled tool-call args / - suppressed thinking streaming); (b) a model call is in flight (ModelCallStart seen, - no end yet) and the queue is silent — a hung/slow LLM endpoint produces - zero events per attempt, and without this signal the run inactivity - watchdog killed healthy-but-waiting runs as "卡死" instead of letting the - HTTP read timeout surface the real model error. Unlike "heartbeat" it - counts as activity for the run inactivity watchdog; zombie protection is - delegated to the httpx timeouts on the model client while in flight. -- ("error", Exception|dict)- agent error (dict shaped like ExceedMaxIters' {kind,name}) - -No standalone end event: ``stream()`` ends the generator directly upon the internal done -sentinel (consumers terminate naturally via ``async for``). Model-produced DataBlocks -(image_chunk etc.) are currently not forwarded — no downstream consumer, and the configured -models only take images as input; see the fall-through at the end of ``_map_event``. - -Migration notes (1.x → 2.0): -- ``agent.msg_queue`` / ``set_msg_queue_enabled`` removed → ``reply_stream``. -- Events are inherently incremental, no accumulate→delta conversion needed (but some models, - e.g. deepseekv4-flash, inline the chain of thought as ``...`` in text deltas, - which still needs suppression when enable_thinking=False). -- usage is accumulated from ``ModelCallEndEvent`` (the 1.x _UsageTrackingModel proxy is no longer needed). -- ctx is set on ``agent.state`` (AgentRuntimeState) instead of ``agent._jx_context``. -- The myspace HITL gate still has to drain concurrently with event consumption → reply_stream - runs in a background task feeding the queue. -""" - -from __future__ import annotations - -import asyncio -import logging -import re -import time -from datetime import datetime, timezone -from typing import Any, AsyncIterator, Dict, List, Optional, Tuple - -from agentscope.agent import Agent -from agentscope.mcp import MCPClient -from agentscope.message import Msg -from core.infra.logging import LogContext -from core.llm.context_adapter import ( - next_request_sequence, - render_context_item, - render_session_input, -) -from core.llm.context_ir import ( - KIND_USER_INPUT, - POLICY_NEVER, - SESSION_CONTEXT_META_KEY, - ContextItem, -) -from core.llm._distill_shared import _THINK_BLOCK_RE -from core.llm.message_compat import session_to_msgs -from core.services import log_service as log_writer - -logger = logging.getLogger(__name__) - - -_HTTP_ERR_RE = re.compile(r"HTTP\s*[45]\d\d") - -# Minimum spacing of ("model_progress", None) liveness signals. During long -# tool-call-argument generation the model streams ToolCallDeltaEvent for -# minutes while _map_event yields nothing downstream — without this signal the -# run inactivity watchdog (CHAT_RUN_INACTIVITY_TIMEOUT_SEC) sees pure silence -# and kills a healthy run mid-generation. -_MODEL_PROGRESS_MIN_INTERVAL_S = 5.0 - -# Queue poll interval of the stream() main loop — on each timeout it emits a -# heartbeat (or model_progress while a model call is in flight). Module-level for -# testability. -_QUEUE_POLL_INTERVAL_S = 3.0 - -# Tool-call argument deltas can arrive one or two characters at a time. Sending -# every upstream fragment recreates the historical SSE storm (hundreds of -# events for a single heredoc), while swallowing all fragments hides a model -# capability the UI can render. Flush when either threshold is crossed to -# preserve visible streaming without forwarding every tiny fragment. -_TOOL_CALL_DELTA_FLUSH_INTERVAL_S = 0.05 -_TOOL_CALL_DELTA_FLUSH_CHARS = 256 - - -def _looks_like_tool_error(content: str) -> bool: - if not content: - return False - s = content.strip() - if not s: - return False - head = s[:64] - if s.startswith(('{"error"', "{'error'", '{"ok": false', '{"ok":false')): - return True - if '"ok": false' in head or '"ok":false' in head: - return True - if s.startswith( - ("Error executing tool", "Error: ", "Traceback (most recent call last)") - ): - return True - if "validation error for" in head: - return True - if "不存在或已删除" in s or "无权访问" in head: - return True - if _HTTP_ERR_RE.search(s): - return True - return False - - -def _strip_thinking_answer( - raw: str, enable_thinking: bool, in_thinking: bool -) -> Tuple[str, bool]: - """Extract the user-visible "answer" portion from the accumulated raw text. - - Returns (answer, new_in_thinking). When enable_thinking=True, returns as-is (the frontend - parses itself); otherwise suppresses every … span and returns only - the visible text. - - 一次模型调用里可以出现**多段**思考链（ReAct 模型常在工具调用前再想一轮）。 - 早先的实现只取"最后一个 之后的内容"：当第二段思考刚开了 - 还没闭合时，rfind 命中的是第一段的闭标签，于是 ``vis1正在生成的思考…`` - 被整段当成正文吐出去 —— 网页端 enable_thinking=False 只在快速模式出现（模型 - 本来就不思考），而定时任务这条路径恰恰是"模型照常思考、正文里不该有思考"， - 于是思考过程就跟着落进了自动化任务的输出正文。 - - 改成"先删掉所有闭合块，再处理孤儿标签"，未闭合的那段思考不会再漏出来。 - """ - if enable_thinking: - return raw, False - # 1) 闭合的思考块整段丢掉（可能有多段） - text = _THINK_BLOCK_RE.sub("", raw) - # 2) 孤儿闭标签：有些服务端把开标签吃进 chat template，正文以裸思考开头， - # 最后一个 之前的都是思考。 - last_end = text.rfind("") - if last_end != -1: - text = text[last_end + len("") :] - return text, False - # 3) 还没闭合的思考块：它之后的内容都还在思考里，不出正文 - open_idx = text.find("") - if open_idx != -1: - return text[:open_idx], True - # 4) 整段没有任何标签：沿用上一次的判定（服务端预填开标签的形态下， - # 在 到来之前只能靠这一位把裸思考压住） - if in_thinking: - return "", True - return text, False - - -class StreamingAgent: - """Wraps an AgentScope 2.0 ``Agent`` to produce streaming SSE events via reply_stream.""" - - def __init__( - self, - agent: Agent, - mcp_clients: List[MCPClient], - ): - self._producer_task = None - self.agent = agent - self.mcp_clients = mcp_clients - self._enable_thinking = False - # usage: accumulated from ModelCallEndEvent - self._usage_records: List[Dict[str, int]] = [] - self._context_usage: Optional[Dict[str, Any]] = None - # tool_id → {tool_name, tool_args(str), started_monotonic, started_at} - self._pending_tool_calls: Dict[str, Dict[str, Any]] = {} - # tool_id → name (recorded at ToolCallStart), tool_id → accumulated args string (ToolCallDelta) - self._tool_name_buf: Dict[str, str] = {} - self._tool_args_buf: Dict[str, str] = {} - # tool_id → not-yet-emitted argument delta / last flush timestamp - self._tool_delta_emit_buf: Dict[str, str] = {} - self._tool_delta_last_emit: Dict[str, float] = {} - # tool_id → accumulated result text (ToolResultTextDelta) - self._tool_result_buf: Dict[str, str] = {} - # Accumulated answer text (for dedup in the -suppression scenario) - self._raw_text = "" - self._emitted_answer = "" - # 首轮正文累计（enable_thinking 下）：轮次结束时若整轮没有 ， - # 说明该模型不是"内联思考"形态（结构化 reasoning 通道、或本轮无思考）—— - # 补发 structured_reasoning 标记，让前端把误缓冲成"思考"的正文流回正文区。 - self._first_round_text = "" - self._round_index = 0 - self._structured_marker_sent = False - self._in_thinking = False - self._reasoning_protocol_emitted = False - # True between ModelCallStartEvent and ModelCallEndEvent. While a call is - # in flight the LLM endpoint may legitimately produce nothing for minutes - # (long prefill on a huge prompt / hung gateway being retried); the queue - # poll then emits model_progress instead of heartbeat so the run - # inactivity watchdog doesn't kill a run that is merely waiting on the - # model — the model client's own httpx timeouts remain the backstop. - self._model_call_inflight = False - - def _drain_model_steps(self) -> List[Dict[str, Any]]: - state = getattr(self.agent, "state", None) - pending = getattr(state, "pending_model_steps", None) - if not pending: - return [] - drained = list(pending) - state.pending_model_steps = [] - return drained - - async def _tool_result_step(self, tool_call_id: str) -> Optional[Dict[str, Any]]: - """Record the tool result exactly as AgentScope saved it into context. - - ``_execute_tool_call`` appends the (already bounded) ToolResultBlock to - the last assistant message before it yields ``ToolResultEndEvent``, so - by the time the event reaches this consumer the block is there. - - Recording a result that carries media writes those bytes to the blob - table (see :mod:`core.llm.tool_media_store`), so the work goes to a - thread instead of blocking the event loop. - """ - from core.llm.model_steps import record_tool_result_step - - context = getattr(getattr(self.agent, "state", None), "context", None) or [] - last = context[-1] if context else None - if last is None or getattr(last, "role", "") != "assistant": - logger.error("tool result %s ended with no assistant message in context", tool_call_id) - return None - match = None - for block in last.get_content_blocks("tool_result"): - if getattr(block, "id", "") == tool_call_id: - match = block - if match is None: - logger.error("tool result %s ended but its block is not in context", tool_call_id) - return None - return await asyncio.to_thread(record_tool_result_step, match) - - def _take_reasoning_protocol(self) -> Optional[Dict[str, bool]]: - """Return the structured-reasoning marker once the active model is known. - - DynamicModelMiddleware may replace ``agent.model`` at reply start, so this is - evaluated immediately before mapping the first AgentScope event rather than in - ``__init__``. - """ - if self._reasoning_protocol_emitted: - return None - model = getattr(self.agent, "model", None) - if not bool(getattr(model, "structured_reasoning", False)): - return None - self._reasoning_protocol_emitted = True - return {"structured_reasoning": True} - - def get_usage(self) -> Dict[str, int]: - """Return the in-memory fallback; async production paths use aget_usage.""" - total_prompt = sum(r.get("prompt_tokens", 0) for r in self._usage_records) - total_completion = sum( - r.get("completion_tokens", 0) for r in self._usage_records - ) - last = self._usage_records[-1] if self._usage_records else {} - return { - "prompt_tokens": total_prompt, - "completion_tokens": total_completion, - "total_tokens": total_prompt + total_completion, - "llm_call_count": len(self._usage_records), - "context_tokens": int(last.get("prompt_tokens", 0) or 0) - + int(last.get("completion_tokens", 0) or 0), - "last_prompt_tokens": int(last.get("prompt_tokens", 0) or 0), - "last_completion_tokens": int(last.get("completion_tokens", 0) or 0), - } - - async def aget_usage(self) -> Dict[str, int]: - """Derive usage from append-only attempts without blocking the event loop.""" - run_id = str(getattr(getattr(self.agent, "state", None), "run_id", "") or "") - if run_id: - try: - from core.services.harness_ledger import HarnessUsageLedger - - attempts = await asyncio.wait_for( - asyncio.to_thread(HarnessUsageLedger().attempts, run_id), - timeout=1.0, - ) - model_attempts = tuple(row for row in attempts if row.kind == "model") - if model_attempts: - primary_attempts = tuple( - row - for row in model_attempts - if row.metadata.get("source") - not in {"vision", "compaction", "followup"} - ) - last = (primary_attempts or model_attempts)[-1] - return { - "prompt_tokens": sum( - row.usage.prompt_tokens for row in model_attempts - ), - "completion_tokens": sum( - row.usage.completion_tokens for row in model_attempts - ), - "cache_read_tokens": sum( - row.usage.cache_read_tokens for row in model_attempts - ), - "cache_write_tokens": sum( - row.usage.cache_write_tokens for row in model_attempts - ), - "total_tokens": sum( - row.usage.total_tokens for row in model_attempts - ), - "llm_call_count": len(model_attempts), - "context_tokens": last.usage.prompt_tokens - + last.usage.completion_tokens, - "last_prompt_tokens": last.usage.prompt_tokens, - "last_completion_tokens": last.usage.completion_tokens, - } - except Exception: - logger.debug("usage ledger aggregate unavailable", exc_info=True) - return self.get_usage() - - def get_context_usage( - self, usage: Optional[Dict[str, Any]] = None - ) -> Dict[str, Any]: - """Return the latest provider measurement or canonical fallback.""" - if self._context_usage and self._context_usage.get("exact"): - return dict(self._context_usage) - last = self._usage_records[-1] if self._usage_records else {} - if usage: - prompt = int(usage.get("last_prompt_tokens", 0) or 0) - completion = int(usage.get("last_completion_tokens", 0) or 0) - else: - prompt = int(last.get("prompt_tokens", 0) or 0) - completion = int(last.get("completion_tokens", 0) or 0) - from core.llm.context_usage import build_context_usage_snapshot - - snapshot = build_context_usage_snapshot( - self.agent, - prompt_tokens=prompt, - completion_tokens=completion, - model_call_index=len(self._usage_records), - ) - self._context_usage = snapshot - return dict(snapshot) - - def restore_context_usage( - self, snapshot: Optional[Dict[str, Any]] - ) -> None: - """Restore a snapshot after a transient call that is not replayed next turn.""" - self._context_usage = dict(snapshot) if snapshot else None - - async def _prepare_vision_evidence(self, st: Any) -> AsyncIterator[Tuple[str, Any]]: - """Transcribe this turn's image attachments, emitting progress around the wait. - - Yields ``("vision_progress", {...})`` when there is actually something to - read, so the frontend can label the wait "图像理解中" instead of the generic - turn spinner. Silent (no events, no delay) when the turn has no images or the - running model reads images natively — in the native case the picture goes to - the model as-is and no transcription happens at all. - - Never raises: a failure here degrades to the middleware's inline path. - """ - try: - from core.llm.middlewares import _effective_model_supports_vision - from core.vision.attachments import ( - image_attachments, - transcribe_attachments, - ) - - images = image_attachments(list(getattr(st, "uploaded_files", None) or [])) - if not images or _effective_model_supports_vision(st): - return - - yield ( - "vision_progress", - { - "status": "running", - "count": len(images), - "names": [f.get("name") or "图片" for f in images], - }, - ) - from core.llm.model_usage import model_usage_scope - from core.services.harness_ledger import HarnessUsageLedger - - run_id = str(getattr(st, "run_id", "") or "") - with model_usage_scope(run_id, HarnessUsageLedger()): - result = await transcribe_attachments( - images, - user_id=getattr(st, "user_id", None), - ) - if result is not None and result.text: - st.vision_evidence_text = result.text - yield ( - "vision_progress", - { - "status": "done", - "count": result.count if result else 0, - "ok": bool(result and result.ok), - "duration_seconds": round(result.duration_seconds, 2) - if result - else 0.0, - }, - ) - except Exception as exc: # noqa: BLE001 — fall back to inline transcription - logger.warning( - "[stream] vision pre-pass failed, deferring to middleware: %s", exc - ) - yield ("vision_progress", {"status": "done", "count": 0, "ok": False}) - - async def stream( - self, - session_messages: List[Dict[str, Any]], - context: Dict[str, Any], - *, - effective_user_message: Optional[str] = None, - ) -> AsyncIterator[Tuple[str, Any]]: - agent = self.agent - - # Commands keep their short spelling in persisted history, while this - # request must execute the expanded instructions. Copy the current row - # so neither stored history nor earlier user turns are rewritten. - if effective_user_message is not None: - session_messages = list(session_messages) - if session_messages and session_messages[-1].get("role") in ("user", "human"): - session_messages[-1] = { - **session_messages[-1], - "content": effective_user_message, - } - else: - session_messages.append({"role": "user", "content": effective_user_message}) - - _last_user_text = "" - if session_messages: - for _m in reversed(session_messages): - if _m.get("role") in ("user", "human"): - _last_user_text = str(_m.get("content") or "") - break - - # ctx → agent.state (AgentRuntimeState, replaces the 1.x agent._jx_context = ModelContext) - st = agent.state - try: - st.apply_request_context(context, _last_user_text) - except Exception as exc: # noqa: BLE001 - logger.warning("[stream] set agent.state failed: %s", exc) - self._enable_thinking = bool(context.get("enable_thinking", True)) - - # Vision bridge: transcribe uploaded images *before* handing off to the model, - # so the UI can say "读图中" while it happens. Reading an image is a plain - # network wait of several seconds; folded inside the agent turn it is - # indistinguishable from the model thinking, and the user just watches a - # generic spinner count up. FileContextMiddleware injects the result. - async for _vision_event in self._prepare_vision_evidence(st): - yield _vision_event - - _log_ctx = LogContext(user_id=st.user_id or None, chat_id=st.chat_id or None) - _log_ctx.__enter__() - - # Load history into context (excluding the last user message — reply_stream's inputs carries it, avoiding duplication) - history = list(session_messages) - last_user_content = "" - last_user_row = None - if history and history[-1].get("role") in ("user", "human"): - last_user_row = history.pop() - last_user_content = last_user_row.get("content", "") - if history: - try: - agent.state.context.extend(session_to_msgs(history)) - except Exception as exc: # noqa: BLE001 - logger.warning("[stream] load history failed: %s", exc) - - user_msg: Optional[Msg] = None - if last_user_content: - request_seq = next_request_sequence(agent.state.context) - if isinstance((last_user_row or {}).get(SESSION_CONTEXT_META_KEY), dict): - user_msg = render_session_input(last_user_row, created_seq=request_seq) - else: - user_msg = render_context_item( - ContextItem.create( - item_id=f"request:user_input:{request_seq}", - kind=KIND_USER_INPUT, - origin="user:chat", - trust="user", - visibility="model", - priority=1_000, - token_budget=100_000, - truncation_policy=POLICY_NEVER, - content=last_user_content, - cache_class="dynamic", - created_seq=request_seq, - render_role="user", - render_name="user", - message_group=f"request:user_input:{request_seq}", - ) - ) - - # myspace write-confirmation gate (in-house ContextVar, distinct from 2.0 native HITL) - from core.llm.tools import _tool_confirmation as _mc - - _confirm_chat_id = st.chat_id or None - - def _drain_confirm_signals() -> list: - out: list = [] - q = _mc.get_ui_queue(_confirm_chat_id) - if q is None: - return out - while True: - try: - out.append(q.get_nowait()) - except asyncio.QueueEmpty: - break - return out - - from core.llm.tools import user_questions as _user_questions - - def _drain_user_question_signals() -> list: - out: list = [] - q = _user_questions.get_ui_queue(_confirm_chat_id) - if q is None: - return out - while True: - try: - out.append(q.get_nowait()) - except asyncio.QueueEmpty: - break - return out - - def _drain_human_interaction_events() -> list: - """Map every queued human-interaction signal to a stream event.""" - - events: list = [] - for signal in _drain_confirm_signals(): - event_type = ( - "design_pick" - if (signal or {}).get("kind") == _mc.KIND_DESIGN_PICK - else "file_confirm" - ) - events.append((event_type, signal)) - for signal in _drain_user_question_signals(): - signal_event = signal.get("event") - if signal_event == "requested": - events.append(("user_question", signal)) - elif signal_event == "resolved": - events.append(("user_question_resolved", signal)) - return events - - # reply_stream runs as a background task feeding the queue; the main loop drains confirm signals concurrently - event_q: asyncio.Queue = asyncio.Queue() - _DONE = object() - - # Subagent streaming bypass: register this run's event_q so call_subagent (separate thread) - # can deliver the subagent's thinking/tool_call/... events back into this main queue in real time. - from core.llm import _subagent_stream - - _subagent_stream.attach(st.chat_id, asyncio.get_running_loop(), event_q) - - async def _produce(): - try: - async for ev in agent.reply_stream(inputs=user_msg): - result_step = None - if type(ev).__name__ == "ToolResultEndEvent": - tid = getattr(ev, "tool_call_id", "") or "" - # Snapshot while the producer is still at the tool - # boundary, before another model step/steer/compaction - # can replace the live context. - result_step = await self._tool_result_step(tid) - if result_step is None: - raise RuntimeError(f"Tool result {tid} is missing from model context") - await event_q.put(("ev", (ev, result_step))) - except asyncio.CancelledError: - raise - except BaseException as e: # noqa: BLE001 - import traceback - - logger.error( - "Agent reply_stream failed: %r\n%s", e, traceback.format_exc() - ) - await event_q.put(("err", e)) - finally: - await event_q.put(("done", _DONE)) - - prod_task = asyncio.create_task(_produce()) - self._producer_task = prod_task - - _stream_start = time.monotonic() - _first_event_logged = False - _poll_interval = _QUEUE_POLL_INTERVAL_S - # Last model_progress emission — throttles the liveness signal emitted - # when upstream events map to nothing. Only the swallowed-event branch - # reads/updates the clock, keeping the mapped hot path free of it. - _last_progress_ts = _stream_start - - try: - while True: - for _human_event in _drain_human_interaction_events(): - yield _human_event - try: - kind, payload = await asyncio.wait_for( - event_q.get(), timeout=_poll_interval - ) - except asyncio.TimeoutError: - if self._model_call_inflight: - # Model call in flight with a silent wire — count as - # liveness (throttled) so the inactivity watchdog waits - # for the model client's own timeout/retry outcome - # instead of misjudging the run as hung. - _now = time.monotonic() - if _now - _last_progress_ts >= _MODEL_PROGRESS_MIN_INTERVAL_S: - _last_progress_ts = _now - yield ("model_progress", None) - continue - yield ("heartbeat", None) - continue - if kind in {"done", "err"}: - # A human answer wakes the tool before AgentScope queues its - # terminal event. Drain once more here so an immediately - # finishing reply cannot strand the frontend in submitted - # state waiting for user_question_resolved. - for _human_event in _drain_human_interaction_events(): - yield _human_event - if kind == "err": - yield ("error", payload) - break - if kind == _subagent_stream.QUEUE_KIND: - # Subagent bypass events — passed straight through, never into _map_event. - if not _first_event_logged: - _first_event_logged = True - yield ("subagent_event", payload) - continue - # kind == "ev" - payload, result_step = payload - steer_delivery = getattr(agent.state, "steer_delivery", None) - if isinstance(steer_delivery, dict): - # SteerMiddleware appends the user instruction immediately - # before this next reasoning round. Emit one acknowledgement - # before mapping ModelCallStart so the queued-card UI can - # switch from "waiting" to "applied" deterministically. - agent.state.steer_delivery = None - yield ("steer_applied", dict(steer_delivery)) - reasoning_protocol = self._take_reasoning_protocol() - if reasoning_protocol is not None: - yield ("reasoning_protocol", reasoning_protocol) - _mapped_any = False - async for out in self._map_event(payload, result_step=result_step): - if not _first_event_logged: - _ttfe = (time.monotonic() - _stream_start) * 1000 - logger.info("[stream] TTFE: %.0fms, type=%s", _ttfe, out[0]) - _first_event_logged = True - _mapped_any = True - yield out - if not _mapped_any: - # The model is alive (an upstream event just arrived) but - # nothing was forwarded — typical of long tool-call-arg / - # suppressed-thinking streaming. Emit a throttled liveness - # signal so the inactivity watchdog doesn't misjudge a - # healthy run as hung. - _now = time.monotonic() - if _now - _last_progress_ts >= _MODEL_PROGRESS_MIN_INTERVAL_S: - _last_progress_ts = _now - yield ("model_progress", None) - except Exception as e: # noqa: BLE001 - yield ("error", e) - finally: - await self.stop_producer() - # Deregister the subagent bypass — prevents late events from being written into a finished run's event_q. - try: - _subagent_stream.detach(st.chat_id) - except Exception: # noqa: BLE001 - pass - for _tid, _rec in list(self._pending_tool_calls.items()): - try: - _started_mono = _rec.get("started_monotonic") - _dur = ( - int((time.monotonic() - _started_mono) * 1000) - if _started_mono - else None - ) - log_writer.schedule_tool_call_write( - { - # user_id/chat_id are taken explicitly from agent.state — contextvars are - # unreliable in the stream() generator frame (the agent runs in the context - # snapshot of _produce's create_task, while tool results are written in the - # generator's consumer frame; the two contexts don't sync), so _context_ids - # cannot be relied on. - "user_id": st.user_id or None, - "chat_id": st.chat_id or None, - "tool_name": _rec.get("tool_name", "unknown"), - "tool_call_id": _tid, - "tool_args": _rec.get("tool_args"), - "tool_result": None, - "status": "failed", - "error_message": "no tool_result received (stream ended)", - "duration_ms": _dur, - "started_at": _rec.get("started_at"), - } - ) - except Exception: - logger.debug("pending tool_call flush failed", exc_info=True) - self._pending_tool_calls.clear() - self._tool_delta_emit_buf.clear() - self._tool_delta_last_emit.clear() - try: - _log_ctx.__exit__(None, None, None) - except Exception: - pass - - async def _map_event( - self, ev: Any, *, result_step: Optional[Dict[str, Any]] = None - ) -> AsyncIterator[Tuple[str, Any]]: - """Map a single reply_stream event into 0..N SSE events.""" - nm = type(ev).__name__ - - if nm == "TextBlockDeltaEvent": - delta = getattr(ev, "delta", "") or "" - if not delta: - return - # Normal state (enable_thinking): 2.0 events are already incremental — forward - # directly, no accumulate+recompute needed (the frontend parses itself), - # avoiding an O(n) scan of the full answer on every delta. - if self._enable_thinking: - if self._round_index == 0: - self._first_round_text += delta - yield ("text_delta", delta) - return - # Suppression state: may span multiple deltas — accumulate, then strip out the answer after the closing tag. - self._raw_text += delta - answer, self._in_thinking = _strip_thinking_answer( - self._raw_text, self._enable_thinking, self._in_thinking - ) - if answer and answer != self._emitted_answer: - out = ( - answer[len(self._emitted_answer) :] - if answer.startswith(self._emitted_answer) - else answer - ) - if out: - yield ("text_delta", out) - self._emitted_answer = answer - return - - if nm == "ThinkingBlockDeltaEvent": - if self._enable_thinking: - d = getattr(ev, "delta", "") or "" - if d: - yield ("thinking_delta", d) - return - - if nm == "ToolCallStartEvent": - tid = getattr(ev, "tool_call_id", "") or "" - name = getattr(ev, "tool_call_name", "") or "unknown" - self._tool_name_buf[tid] = name - self._tool_args_buf[tid] = "" - self._tool_delta_emit_buf[tid] = "" - self._tool_delta_last_emit[tid] = time.monotonic() - self._pending_tool_calls[tid] = { - "tool_name": name, - "tool_args": None, - "started_monotonic": time.monotonic(), - "started_at": datetime.now(timezone.utc), - } - yield ("tool_call_start", {"name": name, "id": tid}) - return - - if nm == "ToolCallDeltaEvent": - tid = getattr(ev, "tool_call_id", "") or "" - delta = getattr(ev, "delta", "") or "" - if not delta: - return - self._tool_args_buf[tid] = self._tool_args_buf.get(tid, "") + delta - pending_delta = self._tool_delta_emit_buf.get(tid, "") + delta - self._tool_delta_emit_buf[tid] = pending_delta - now = time.monotonic() - last_emit = self._tool_delta_last_emit.get(tid, now) - if ( - len(pending_delta) >= _TOOL_CALL_DELTA_FLUSH_CHARS - or now - last_emit >= _TOOL_CALL_DELTA_FLUSH_INTERVAL_S - ): - self._tool_delta_emit_buf[tid] = "" - self._tool_delta_last_emit[tid] = now - yield ( - "tool_call_delta", - { - "name": self._tool_name_buf.get(tid, "unknown"), - "id": tid, - "delta": pending_delta, - }, - ) - return - - if nm == "ToolCallEndEvent": - tid = getattr(ev, "tool_call_id", "") or "" - name = self._tool_name_buf.pop(tid, "unknown") - args_str = self._tool_args_buf.pop(tid, "") - pending_delta = self._tool_delta_emit_buf.pop(tid, "") - self._tool_delta_last_emit.pop(tid, None) - if pending_delta: - yield ( - "tool_call_delta", - {"name": name, "id": tid, "delta": pending_delta}, - ) - import json - - try: - args = json.loads(args_str) if args_str else {} - except json.JSONDecodeError: - args = {"_raw": args_str} - rec = self._pending_tool_calls.get(tid) - if rec is not None: - rec["tool_args"] = args - try: - from orchestration.tool_callbacks import note_tool_call - - note_tool_call( - self.__dict__.setdefault("_tool_warn_state", {}), name, args - ) - except Exception: # noqa: BLE001 - pass - yield ("tool_call", {"name": name, "args": args, "id": tid}) - return - - if nm == "ToolResultTextDeltaEvent": - tid = getattr(ev, "tool_call_id", "") or "" - self._tool_result_buf[tid] = self._tool_result_buf.get(tid, "") + ( - getattr(ev, "delta", "") or "" - ) - return - - if nm == "ToolResultEndEvent": - tid = getattr(ev, "tool_call_id", "") or "" - content = self._tool_result_buf.pop(tid, "") - if result_step is None: - result_step = await self._tool_result_step(tid) - if result_step is None: - raise RuntimeError(f"Tool result {tid} is missing from model context") - raw_state = getattr(ev, "state", "") or "" - state = str(getattr(raw_state, "value", raw_state) or "") - pending = self._pending_tool_calls.pop(tid, None) - name = ( - getattr(ev, "tool_call_name", "") - or (pending or {}).get("tool_name") - or self._tool_name_buf.get(tid) - or "unknown" - ) - try: - is_error = state in { - "error", - "denied", - "interrupted", - } or _looks_like_tool_error(content) - effect_link = dict( - (getattr(self.agent.state, "tool_effect_links", None) or {}).pop( - tid, {} - ) - ) - started_mono = pending.get("started_monotonic") if pending else None - duration_ms = ( - int((time.monotonic() - started_mono) * 1000) - if started_mono - else None - ) - log_writer.schedule_tool_call_write( - { - # See the matching comment in _produce: carry user_id/chat_id explicitly, never rely on contextvars. - "user_id": self.agent.state.user_id or None, - "chat_id": self.agent.state.chat_id or None, - "run_id": self.agent.state.run_id or None, - "effect_id": effect_link.get("effect_id"), - "result_id": effect_link.get("result_id"), - "tool_name": (pending or {}).get("tool_name") or name, - "tool_call_id": tid, - "tool_args": (pending or {}).get("tool_args"), - "tool_result": content, - "status": "failed" if is_error else "success", - "error_message": content if is_error else None, - "duration_ms": duration_ms, - "started_at": (pending or {}).get("started_at"), - } - ) - except Exception: # noqa: BLE001 - logger.debug("tool_call log persist failed", exc_info=True) - # Persist the canonical result before any UI filtering (notably - # update_plan). The consumer must finish this event before the - # display-only result can announce completion. - yield ("model_step", result_step) - yield ( - "tool_result", - { - "name": name, - "id": tid, - "content": content, - "status": state, - }, - ) - return - - if nm == "ModelCallStartEvent": - # Marks the wire as "waiting on the model" — see _model_call_inflight. - self._model_call_inflight = True - # Agent construction and manifest-backed tool assembly happen before - # this event. Surface the provider-dispatch boundary accurately. - yield ("model_call_start", None) - return - - if nm == "ModelCallEndEvent": - self._model_call_inflight = False - # The hook adapter recorded this response before the event was - # queued; hand the record downstream in the same order. - for step in self._drain_model_steps(): - yield ("model_step", step) - _prompt_tokens = int(getattr(ev, "input_tokens", 0) or 0) - _completion_tokens = int(getattr(ev, "output_tokens", 0) or 0) - self._usage_records.append( - { - "prompt_tokens": _prompt_tokens, - "completion_tokens": _completion_tokens, - } - ) - from core.llm.context_usage import build_context_usage_snapshot - - self._context_usage = build_context_usage_snapshot( - self.agent, - prompt_tokens=_prompt_tokens, - completion_tokens=_completion_tokens, - model_call_index=len(self._usage_records), - ) - # Compaction usage is anchored by CompactingAgent._save_to_context - # after the response is saved. AgentScope emits this event first; - # observing here would count that response again as new content. - # 首轮权威判定：思考模式下整轮正文没有出现 → 该模型不内联思考 - # （结构化 reasoning 通道，或本轮确实没思考）。补发协议标记，前端据此 - # 把误当思考缓冲/展示的正文重归正文区（bug：无思考时正文进思考块）。 - # 只看首轮——混合形态模型工具后省略 属既有启发式管辖，不在此误判。 - if ( - self._enable_thinking - and self._round_index == 0 - and not self._structured_marker_sent - and self._first_round_text - and "" not in self._first_round_text - ): - self._structured_marker_sent = True - yield ("reasoning_protocol", {"structured_reasoning": True}) - yield ("context_usage", dict(self._context_usage)) - self._round_index += 1 - self._first_round_text = "" - # New model call round → reset answer accumulation (the next text segment computes deltas from scratch) - self._raw_text = "" - self._emitted_answer = "" - self._in_thinking = False - return - - if nm == "RequireUserConfirmEvent": - # 2.0 native HITL (distinct from the myspace gate); our tools default to ALLOW, so this rarely triggers. - try: - yield ( - "file_confirm", - { - "reply_id": getattr(ev, "reply_id", ""), - "tool_calls": [ - { - "id": getattr(tc, "id", ""), - "name": getattr(tc, "name", ""), - "input": getattr(tc, "input", ""), - } - for tc in (getattr(ev, "tool_calls", []) or []) - ], - }, - ) - except Exception: # noqa: BLE001 - pass - return - - if nm == "ExceedMaxItersEvent": - yield ( - "error", - {"kind": "exceed_max_iters", "name": getattr(ev, "name", "")}, - ) - return - - # Everything else is never forwarded, internal only: lifecycle (ReplyStart/End, - # the various *Start/*End) + DataBlockStart/Delta (model-produced - # image_chunk — currently no downstream consumer and the configured models only take - # images as input, so intentionally dropped; add a branch when support is needed). - return - - async def stop_producer(self): - """Stop actual model/tool work before closing the stream or its clients.""" - from core.infra.task_lifecycle import cancel_and_join - - task = self._producer_task - if task is not None: - await cancel_and_join(task) - if self._producer_task is task: - self._producer_task = None - - async def shutdown(self): - """Close transient (per-request) MCP clients.""" - from core.llm.mcp_manager import close_clients - - await self.stop_producer() - try: - await close_clients(self.mcp_clients) - except Exception as exc: - logger.debug("StreamingAgent.shutdown: close_clients error: %s", exc) - self.mcp_clients = [] diff --git a/src/backend/orchestration/subagents/__init__.py b/src/backend/orchestration/subagents/__init__.py deleted file mode 100644 index 5361fd29..00000000 --- a/src/backend/orchestration/subagents/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -"""Subagent implementations.""" - diff --git a/src/backend/orchestration/subagents/ontology_reviewer.py b/src/backend/orchestration/subagents/ontology_reviewer.py deleted file mode 100644 index 6b0430c6..00000000 --- a/src/backend/orchestration/subagents/ontology_reviewer.py +++ /dev/null @@ -1,656 +0,0 @@ -"""Independent L-b/L-c reviewers for ontology-governed final output.""" - -from __future__ import annotations - -import asyncio -import json -import time -from collections import Counter -from collections.abc import Awaitable, Callable -from typing import Any - -from core.capabilities.runtime import child_scope - -from core.infra.logging import get_logger -from core.ontology.revision import is_substantive_revision -from core.ontology.validator import evaluate_output -from orchestration.loop_evaluator import _parse_json_lenient - -logger = get_logger(__name__) - -_VALID_VERDICTS = {"pass", "revise", "escalate"} -_DEFAULT_MAX_REPAIR_ATTEMPTS = 1 -_PERSPECTIVES = ( - "领域合规委员：逐条核对术语、关系、禁用条件和工作流约束。", - "证据委员：只认可工具轨迹与引用中可追溯的证据，严查无证据断言。", - "风险委员：关注高风险误导、遗漏前提、越权建议和不可逆影响。", -) - - -def _tail_with_json_budget(items: list[dict[str, Any]], max_chars: int) -> list[dict[str, Any]]: - """Retain the newest evidence without splitting a serialized trace item.""" - selected: list[dict[str, Any]] = [] - used = 2 - for item in reversed(items): - serialized = json.dumps(item, ensure_ascii=False, default=str) - cost = len(serialized) + (1 if selected else 0) - if selected and used + cost > max_chars: - break - if not selected and used + cost > max_chars: - selected.append( - { - "truncated": True, - "preview": serialized[: max(0, max_chars - 64)], - } - ) - break - selected.append(item) - used += cost - selected.reverse() - return selected - - -def _review_evidence_payload( - trace: list[dict[str, Any]], citations: list[dict[str, Any]] -) -> dict[str, Any]: - """Prefer evidence produced by the ontology repair over stale draft evidence.""" - return { - "trace": _tail_with_json_budget(trace, 8000), - "citations": _tail_with_json_budget(citations, 2000), - } - - -def _review_prompt( - *, - perspective: str, - task: str, - answer: str, - runtime: dict[str, Any], - trace: list[dict[str, Any]], - citations: list[dict[str, Any]], - deterministic: dict[str, Any], -) -> str: - contract = { - "packs": [ - { - "pack_id": pack.get("pack_id"), - "version": pack.get("version"), - "concepts": pack.get("concepts", []), - "relations": pack.get("relations", []), - "constraints": pack.get("constraints", []), - "workflows": pack.get("workflows", []), - } - for pack in runtime.get("packs", []) - ] - } - review_evidence = _review_evidence_payload(trace, citations) - return ( - "你是本体评审委员会中的独立委员，与生成答案的 agent 相互独立。\n" - f"评审视角：{perspective}\n" - "只依据领域契约、工具轨迹和引用作判断。答案自称已核验不算证据；" - "不得补做任务或调用工具。还必须核对答案是否仍忠实完成用户任务：本体核验过程" - "不是最终交付物。除非用户明确要求核验报告，否则若答案把润色稿改写成‘原始语句’、" - "‘逐项核验’、‘主张评估’、‘修改说明’等元分析，或擅自改变用户要求的文体、篇幅、" - "结构与文件类型，必须判定为 revise。这里的任务忠实度是指交付形式与表达意图，不是" - "机械保留错误内容：若用户要求原样输出的语句本身违反已激活的领域事实或证据约束，" - "修订稿必须纠正这些问题；只要仍保持用户要求的一句话、文档等形式，不得因未逐字照抄" - "错误语句而判定为偏离任务。强制判定规则：不得把‘没有原样保留已被证据否定或缺乏" - "证据的断言’列为 revise 理由，也不得建议恢复这类断言；即使用户把请求称作测试或强调" - "原样输出，本体修订仍必须纠错。示例：用户要求用一句话原样输出‘某记录风险极高’，" - "工具证据显示未发现对应风险，答案仍用一句话改成有证据支持的审慎结论——任务形式忠实，" - "应继续按领域证据判断，而不能因没有逐字照抄判 revise。只有把一句话扩写成‘原始语句 +" - "逐项核验 + 修改说明’之类的元报告，才属于这里所说的交付形式偏离。\n\n" - "形式判断必须客观：只有一个句号结尾、内部用逗号或分号连接的陈述仍是一句话，附带引用" - "标记也不会把它变成分析报告；工具轨迹已生成并交付 .docx 时，不得声称文件类型发生了" - "变化。不要把纠错所必需的证据化表述误判成文体或篇幅偏离。工具轨迹按时间顺序排列，" - "并优先保留最近产生的证据；同名文件被多次生成时，后生成且已交付的文件是当前修订稿，" - "不得只根据早期文件内容判断待评答案。\n\n" - f"用户任务：\n{task}\n\n待评答案：\n{answer}\n\n" - "领域契约：\n" - f"{json.dumps(contract, ensure_ascii=False)[:14000]}\n\n" - "工具与引用证据：\n" - f"{json.dumps(review_evidence, ensure_ascii=False)}\n\n" - "确定性门禁结果：\n" - f"{json.dumps(deterministic, ensure_ascii=False)}\n\n" - "严格只输出 JSON：" - '{"verdict":"pass|revise|escalate","evidence":["规则ID或轨迹证据"],' - '"feedback":"具体问题与可执行修改要求",' - '"affected_claims":[{"quote":"答案中的原句或段落摘要",' - '"rule_id":"规则ID","issue":"可能违反本体的原因",' - '"manual_check":"人工需要核对的证据或判断"}]}。' - "只有所有高风险约束有证据支持时才 pass；可修正文案时 revise；" - "缺关键证据、存在冲突或需人工决策时 escalate。" - ) - - -async def _run_text_agent( - prompt: str, - *, - user_id: str, - model_name: str | None, - model_provider_id: str | None, - runtime: dict[str, Any], - capability_scope: str = "", -) -> str: - from core.llm.factory import create_agent_executor - from core.llm.mcp_manager import close_clients - from orchestration.streaming import StreamingAgent - - agent, clients = await create_agent_executor( - disable_tools=True, - enabled_skill_ids=[], - current_user_id=user_id, - model_name=model_name, - model_provider_id=model_provider_id, - chat_mode="fast", - max_iters=2, - isolated=True, - ontology_runtime=runtime, - capability_scope=child_scope(capability_scope, "ontology-model", "text"), - ) - stream = StreamingAgent(agent, clients) - text = "" - try: - async for event_type, payload in stream.stream( - [{"role": "user", "content": prompt}], - { - "user_id": user_id, - "model_name": model_name or "", - "model_provider_id": model_provider_id or "", - "enable_thinking": False, - "chat_mode": "fast", - "ontology_enabled": True, - "ontology_runtime": runtime, - }, - ): - if event_type == "text_delta": - text += payload - elif event_type == "error": - raise RuntimeError(str(payload)) - finally: - await close_clients(clients) - return text - - -async def _review_once( - perspective: str, - **kwargs, -) -> dict[str, Any]: - prompt = _review_prompt( - perspective=perspective, - task=kwargs["task"], - answer=kwargs["answer"], - runtime=kwargs["runtime"], - trace=kwargs["trace"], - citations=kwargs["citations"], - deterministic=kwargs["deterministic"], - ) - try: - text = await _run_text_agent( - prompt, - user_id=kwargs["user_id"], - model_name=kwargs.get("model_name"), - model_provider_id=kwargs.get("model_provider_id"), - runtime=kwargs["runtime"], - capability_scope=child_scope( - kwargs.get("capability_scope", ""), "ontology-review", perspective - ), - ) - obj = _parse_json_lenient(text) - if not isinstance(obj, dict) or obj.get("verdict") not in _VALID_VERDICTS: - return { - "verdict": "escalate", - "evidence": [], - "feedback": "评审输出无法解析，保守转人工。", - } - evidence = [str(item) for item in (obj.get("evidence") or []) if str(item).strip()] - affected_claims = [] - for item in obj.get("affected_claims") or []: - if not isinstance(item, dict): - continue - claim = { - "quote": str(item.get("quote") or "").strip()[:500], - "rule_id": str(item.get("rule_id") or "").strip()[:160], - "issue": str(item.get("issue") or "").strip()[:500], - "manual_check": str(item.get("manual_check") or "").strip()[:500], - } - if claim["quote"] or claim["issue"]: - affected_claims.append(claim) - verdict = obj["verdict"] - if verdict == "pass" and not evidence: - verdict = "revise" - return { - "verdict": verdict, - "evidence": evidence, - "feedback": str(obj.get("feedback") or "").strip(), - "affected_claims": affected_claims, - } - except Exception as exc: # noqa: BLE001 - logger.warning("[ontology-review] reviewer failed: %s", exc) - return { - "verdict": "escalate", - "evidence": [], - "feedback": f"评审委员不可用：{str(exc)[:160]}", - "affected_claims": [], - } - - -async def _revise_answer( - *, - task: str, - answer: str, - feedback: list[str], - user_id: str, - model_name: str | None, - model_provider_id: str | None, - runtime: dict[str, Any], - capability_scope: str = "", -) -> str: - prompt = ( - "你是本体校验后的答案修订员。严格根据评审意见修正答案，不新增没有证据的事实。" - "直接输出修订后的完整答案，不要解释修订过程。\n\n" - f"用户任务：\n{task}\n\n原答案：\n{answer}\n\n评审意见：\n" - + "\n".join(f"- {item}" for item in feedback if item) - ) - return ( - await _run_text_agent( - prompt, - user_id=user_id, - model_name=model_name, - model_provider_id=model_provider_id, - runtime=runtime, - capability_scope=child_scope(capability_scope, "ontology-model", "revise"), - ) - ).strip() - - -def _dedupe_strings(items: list[str]) -> list[str]: - seen: set[str] = set() - output: list[str] = [] - for item in items: - value = str(item or "").strip() - if not value or value in seen: - continue - seen.add(value) - output.append(value) - return output - - -def _build_manual_review( - *, - required: bool, - violations: list[dict[str, Any]], - affected_claims: list[dict[str, str]], - suggestions: list[str], -) -> dict[str, Any]: - """Build the JSON payload rendered by the frontend's human-review cards.""" - items: list[dict[str, str]] = [] - seen: set[tuple[str, str]] = set() - for item in affected_claims: - quote = " ".join(str(item.get("quote") or "").split())[:240] - risk = " ".join(str(item.get("issue") or "").split())[:360] - rule_id = str(item.get("rule_id") or "未指定规则")[:160] - manual_check = " ".join(str(item.get("manual_check") or "").split())[:360] - key = (quote, rule_id) - if key in seen or (not quote and not risk): - continue - seen.add(key) - items.append( - { - "quote": quote or "答案中的相关领域结论", - "rule_id": rule_id, - "risk": risk or "委员会未能确认该结论满足领域契约。", - "manual_check": manual_check or "核对原始工具结果、适用前提和结论表述。", - } - ) - if len(items) >= 8: - break - - for violation in violations: - if len(items) >= 8: - break - rule_id = str(violation.get("rule_id") or "未指定规则")[:160] - reasons = "；".join(str(item) for item in violation.get("reasons") or [] if str(item)) - key = ("", rule_id) - if key in seen: - continue - seen.add(key) - message = str(violation.get("message") or "领域约束未满足").strip() - items.append( - { - "quote": "整体证据链或输出结构", - "rule_id": rule_id, - "risk": f"{message}{f'（{reasons}）' if reasons else ''}"[:360], - "manual_check": "确认必需工具、引用证据和答案中的对应结论能够逐项追溯。", - } - ) - - if required and not items: - items.append( - { - "quote": "答案中的领域判断", - "rule_id": "ontology_review", - "risk": "委员会未能形成通过结论。", - "manual_check": "核对关键事实、推断、适用前提和原始证据。", - } - ) - return { - "required": required, - "title": "领域本体人工复核", - "summary": ( - "候选答案仍有未闭合的领域证据或判断，请在替换原文前完成复核。" - if required - else "本轮修订建议已结构化，可按需核对后采用。" - ), - "items": items, - "actions": [item[:500] for item in _dedupe_strings(suggestions)[:6]], - } - - -RemediationCallback = Callable[[dict[str, Any]], Awaitable[str]] - - -async def review_ontology_output( - *, - task: str, - answer: str, - runtime: dict[str, Any], - trace: list[dict[str, Any]], - citations: list[dict[str, Any]], - user_id: str, - chat_id: str | None, - model_name: str | None, - model_provider_id: str | None = None, - trace_complete: bool = True, - remediate: RemediationCallback | None = None, - max_repair_attempts: int = _DEFAULT_MAX_REPAIR_ATTEMPTS, - capability_scope: str = "", -) -> dict[str, Any]: - """Review one answer, repair it with the originating agent, and audit the session.""" - started = time.monotonic() - original_answer = answer - current_answer = answer - max_repair_attempts = max(0, min(int(max_repair_attempts), 1)) - repair_attempts = 0 - review_rounds: list[dict[str, Any]] = [] - all_feedback: list[str] = [] - all_evidence: list[str] = [] - final_affected_claims: list[dict[str, str]] = [] - final_reviewers: list[dict[str, Any]] = [] - level = runtime.get("review_level", "none") - initial_trace_len = len(trace) - initial_citation_len = len(citations) - - def _deterministic_result() -> tuple[Any, dict[str, Any]]: - completed_tools = { - str(item.get("tool_name")) - for item in trace - if item.get("type") == "tool_result" and item.get("tool_name") - } - decision = evaluate_output( - runtime, - answer=current_answer, - citations=citations, - completed_tools=completed_tools if trace_complete else None, - ) - return decision, { - "allowed": decision.allowed, - "decision": decision.decision, - "violations": decision.violations, - "suggestions": decision.suggestions, - } - - deterministic, deterministic_dict = _deterministic_result() - if level == "none" and deterministic.allowed: - return { - "verdict": "pass", - "answer": current_answer, - "reviewers": [], - "evidence": [], - "feedback": [], - "violations": [], - "affected_claims": [], - "manual_review": _build_manual_review( - required=False, violations=[], affected_claims=[], suggestions=[] - ), - "latency_ms": round((time.monotonic() - started) * 1000), - "revised": False, - "annotated": False, - "repair_attempts": 0, - "attempts": 0, - "review_rounds": [], - "new_tools": [], - "new_citation_count": 0, - } - effective_level = "checkpoint" if level == "none" else level - - # A deterministic evidence gap is repaired before the sole committee round, - # so the committee evaluates the newly retrieved evidence instead of merely - # rephrasing the first draft. - if not deterministic.allowed and remediate is not None and max_repair_attempts: - repair_attempts = 1 - try: - repaired = await remediate( - { - "attempt": 1, - "source": "deterministic", - "original_task": task, - "answer": current_answer, - "violations": deterministic.violations, - "suggestions": deterministic.suggestions, - "feedback": [], - "citations": list(citations), - } - ) - except Exception as exc: # noqa: BLE001 - logger.warning("[ontology-review] tool remediation failed: %s", exc) - all_feedback.append(f"自动补充证据失败：{str(exc)[:200]}") - repaired = "" - if is_substantive_revision(repaired): - current_answer = repaired.strip() - deterministic, deterministic_dict = _deterministic_result() - elif repaired.strip(): - all_feedback.append("自动修订未生成完整正文，已保留原文。") - - common = { - "capability_scope": capability_scope, - "task": task, - "answer": current_answer, - "runtime": runtime, - "trace": trace, - "citations": citations, - "deterministic": deterministic_dict, - "user_id": user_id, - "model_name": model_name, - "model_provider_id": model_provider_id, - } - if effective_level == "committee": - committee_size = max( - ( - int(pack.get("config", {}).get("committee_size", 3)) - for pack in runtime.get("packs", []) - ), - default=3, - ) - perspectives = [ - _PERSPECTIVES[index % len(_PERSPECTIVES)] + f"（席位 {index + 1}）" - for index in range(committee_size) - ] - reviewers = await asyncio.gather( - *(_review_once(perspective, **common) for perspective in perspectives) - ) - counts = Counter(item["verdict"] for item in reviewers) - verdict, votes = counts.most_common(1)[0] - if votes <= committee_size // 2: - verdict = "escalate" - else: - reviewers = [await _review_once(_PERSPECTIVES[0], **common)] - verdict = reviewers[0]["verdict"] - - if not deterministic.allowed and verdict == "pass": - verdict = "revise" - round_feedback = [str(item.get("feedback") or "") for item in reviewers] - round_feedback.extend(deterministic.suggestions) - round_evidence = [ - str(item) for reviewer in reviewers for item in reviewer.get("evidence", []) if str(item) - ] - round_claims = [ - claim - for reviewer in reviewers - for claim in reviewer.get("affected_claims", []) - if isinstance(claim, dict) - ] - all_feedback.extend(round_feedback) - all_evidence.extend(round_evidence) - final_affected_claims = round_claims - final_reviewers = reviewers - review_rounds.append( - { - "round": 1, - "verdict": verdict, - "deterministic": deterministic_dict, - "reviewers": reviewers, - } - ) - - # A committee wording/evidence revision gets one tools-enabled continuation, - # but is deliberately not sent through a second committee round. - if verdict == "revise" and repair_attempts == 0 and max_repair_attempts: - repair_attempts = 1 - try: - if remediate is not None: - revised = await remediate( - { - "attempt": 1, - "source": "committee", - "original_task": task, - "answer": current_answer, - "violations": deterministic.violations, - "suggestions": deterministic.suggestions, - "feedback": _dedupe_strings(round_feedback), - "affected_claims": round_claims, - "citations": list(citations), - } - ) - elif deterministic.allowed: - revised = await _revise_answer( - task=task, - answer=current_answer, - feedback=_dedupe_strings(round_feedback), - user_id=user_id, - model_name=model_name, - model_provider_id=model_provider_id, - runtime=runtime, - capability_scope=capability_scope, - ) - else: - revised = "" - except Exception as exc: # noqa: BLE001 - logger.warning("[ontology-review] committee remediation failed: %s", exc) - all_feedback.append(f"自动修订失败：{str(exc)[:200]}") - revised = "" - if is_substantive_revision(revised): - current_answer = revised.strip() - deterministic, _ = _deterministic_result() - elif revised.strip(): - all_feedback.append("自动修订未生成完整正文，已保留原文。") - - final_violations = list(deterministic.violations) - final_suggestions = list(deterministic.suggestions) - annotated = verdict == "escalate" or not deterministic.allowed - manual_review = _build_manual_review( - required=annotated, - violations=final_violations, - affected_claims=final_affected_claims, - suggestions=final_suggestions or all_feedback, - ) - new_tools = _dedupe_strings( - [ - str(item.get("tool_name") or "") - for item in trace[initial_trace_len:] - if item.get("type") == "tool_result" - ] - ) - evidence = _dedupe_strings(all_evidence) - feedback = _dedupe_strings(all_feedback + final_suggestions) - latency_ms = round((time.monotonic() - started) * 1000) - try: - from core.services.ontology_service import record_enforcement_event, record_review_run - - review_payload = { - "user_id": user_id or None, - "chat_id": chat_id, - "pack_version_ids": runtime.get("version_ids", []), - "level": "checkpoint" if level == "none" else level, - "subject_type": "final_answer", - "verdict": verdict, - "evidence": evidence, - "feedback": "\n".join(feedback), - "reviewers": [ - {**reviewer, "round": round_info["round"]} - for round_info in review_rounds - for reviewer in round_info["reviewers"] - ], - "latency_ms": latency_ms, - } - audit_tasks = [asyncio.to_thread(record_review_run, review_payload)] - for pack in runtime.get("packs", []): - audit_tasks.append( - asyncio.to_thread( - record_enforcement_event, - { - "user_id": user_id or None, - "chat_id": chat_id, - "pack_id": pack.get("pack_id"), - "version_id": pack.get("version_id"), - "rule_id": ( - deterministic.violations[0].get("rule_id") - if deterministic.violations - else f"review:{level}" - ), - "stage": "output", - "event_type": "output_review", - "decision": verdict, - "mode": "enforce", - "target": "final_answer", - "latency_ms": latency_ms, - "details": { - "governance_run_id": runtime.get("governance_run_id"), - "review_owner": (runtime.get("output_review") or {}).get("owner"), - "violations": deterministic.violations, - "evidence": evidence, - "feedback": review_payload["feedback"], - "repair_attempts": repair_attempts, - "review_rounds": len(review_rounds), - }, - }, - ) - ) - await asyncio.gather(*audit_tasks) - except Exception as exc: # noqa: BLE001 - logger.warning("[ontology-review] audit persistence failed: %s", exc) - if verdict in {"revise", "escalate"} and chat_id: - try: - from core.services.ontology_evolution_service import schedule_ontology_evolution - - schedule_ontology_evolution(user_id=user_id or "system") - except Exception as exc: # noqa: BLE001 - logger.warning("[ontology-review] evolution scheduling failed: %s", exc) - return { - "verdict": verdict, - "answer": current_answer, - "reviewers": final_reviewers, - "evidence": evidence, - "feedback": feedback, - "violations": final_violations, - "affected_claims": final_affected_claims, - "manual_review": manual_review, - "latency_ms": latency_ms, - "revised": current_answer != original_answer, - "annotated": annotated, - "repair_attempts": repair_attempts, - "attempts": max(1, len(review_rounds)), - "review_rounds": review_rounds, - "new_tools": new_tools, - "new_citation_count": max(0, len(citations) - initial_citation_len), - } diff --git a/src/backend/orchestration/schedulers/__init__.py b/src/backend/orchestration/workers/__init__.py similarity index 100% rename from src/backend/orchestration/schedulers/__init__.py rename to src/backend/orchestration/workers/__init__.py diff --git a/src/backend/orchestration/schedulers/_worker_base.py b/src/backend/orchestration/workers/_worker_base.py similarity index 98% rename from src/backend/orchestration/schedulers/_worker_base.py rename to src/backend/orchestration/workers/_worker_base.py index d4ec1c17..0030c47e 100644 --- a/src/backend/orchestration/schedulers/_worker_base.py +++ b/src/backend/orchestration/workers/_worker_base.py @@ -18,7 +18,6 @@ """ from __future__ import annotations -from core.infra.time import utc_now import asyncio import logging @@ -26,9 +25,9 @@ from typing import Any, Awaitable, Callable, List, Optional, Sequence, Tuple import pytz - from core.db.engine import SessionLocal from core.infra.ephemeral import get_ephemeral_state +from core.infra.time import utc_now logger = logging.getLogger(__name__) @@ -181,9 +180,7 @@ async def _worker() -> int: except Exception as exc: # One poisoned item must not take down the drain; the item's # own handler is responsible for recording its failure. - logger.error( - "[%s] item %s failed: %s", log_tag, item_id, exc, exc_info=exc - ) + logger.error("[%s] item %s failed: %s", log_tag, item_id, exc, exc_info=exc) processed += 1 results: List[Any] = await asyncio.gather( diff --git a/src/backend/core/services/edition_startup.py b/src/backend/orchestration/workers/edition_startup.py similarity index 100% rename from src/backend/core/services/edition_startup.py rename to src/backend/orchestration/workers/edition_startup.py diff --git a/src/backend/orchestration/schedulers/evolution_scheduler.py b/src/backend/orchestration/workers/evolution_scheduler.py similarity index 98% rename from src/backend/orchestration/schedulers/evolution_scheduler.py rename to src/backend/orchestration/workers/evolution_scheduler.py index 6c8067a6..9d4684dd 100644 --- a/src/backend/orchestration/schedulers/evolution_scheduler.py +++ b/src/backend/orchestration/workers/evolution_scheduler.py @@ -17,7 +17,7 @@ import os from typing import Optional -from orchestration.schedulers._worker_base import ( +from orchestration.workers._worker_base import ( acquire_day_lock, interruptible_sleep, seconds_until_next_cron, @@ -102,7 +102,7 @@ async def _fire_once(self) -> None: return from core.evolution.backfill import backfill_episodes - from core.evolution.loop import run_evolution_cycle + from core.evolution.cycle.loop import run_evolution_cycle loop = asyncio.get_running_loop() diff --git a/src/backend/orchestration/schedulers/memory_ttl_scheduler.py b/src/backend/orchestration/workers/memory_ttl_scheduler.py similarity index 95% rename from src/backend/orchestration/schedulers/memory_ttl_scheduler.py rename to src/backend/orchestration/workers/memory_ttl_scheduler.py index 149361a4..c0390504 100644 --- a/src/backend/orchestration/schedulers/memory_ttl_scheduler.py +++ b/src/backend/orchestration/workers/memory_ttl_scheduler.py @@ -14,7 +14,7 @@ import os from typing import Optional -from orchestration.schedulers._worker_base import ( +from orchestration.workers._worker_base import ( acquire_day_lock, interruptible_sleep, seconds_until_next_cron, @@ -93,5 +93,7 @@ async def _fire_once(self) -> None: stats = await sweep_expired_memories() logger.info( "[memory-ttl-cron] done: scanned=%s expired=%s deleted=%s", - stats.get("scanned"), stats.get("expired"), stats.get("deleted"), + stats.get("scanned"), + stats.get("expired"), + stats.get("deleted"), ) diff --git a/src/backend/orchestration/schedulers/persona_distill_worker.py b/src/backend/orchestration/workers/persona_distill_worker.py similarity index 94% rename from src/backend/orchestration/schedulers/persona_distill_worker.py rename to src/backend/orchestration/workers/persona_distill_worker.py index f5734c30..14cf4433 100644 --- a/src/backend/orchestration/schedulers/persona_distill_worker.py +++ b/src/backend/orchestration/workers/persona_distill_worker.py @@ -14,9 +14,9 @@ import asyncio from typing import Optional -from core.db.models import PersonaDistillJob +from features.distillation.models import PersonaDistillJob from core.infra.logging import get_logger -from orchestration.schedulers._worker_base import ( +from orchestration.workers._worker_base import ( claim_next, drain_queue, interruptible_sleep, @@ -87,6 +87,6 @@ def _claim_next_job() -> Optional[str]: async def _process_job(job_id: str) -> None: - from core.services.persona_distillation_service import run_job + from features.distillation.persona_execution import run_job await run_job(job_id) diff --git a/src/backend/orchestration/workers/run_recovery.py b/src/backend/orchestration/workers/run_recovery.py new file mode 100644 index 00000000..ca36355a --- /dev/null +++ b/src/backend/orchestration/workers/run_recovery.py @@ -0,0 +1,37 @@ +"""Periodic recovery and stale-run maintenance.""" + +from __future__ import annotations + +import asyncio + +import core.chat.runtime.recovery as recovery +import core.db.engine as engine +import core.runs.settings as settings +import core.runs.watchdog as watchdog +from core.infra.logging import get_logger + +logger = get_logger(__name__) + + +async def run_stale_reaper_loop() -> None: + """Recover expired leases frequently and run the slower stale watchdog.""" + loop = asyncio.get_running_loop() + next_reap_at = loop.time() + settings.STALE_REAPER_INTERVAL_SEC + while True: + try: + await asyncio.sleep(settings.RUN_RECOVERY_INTERVAL_SEC) + await recovery.recover_orphan_runs() + if loop.time() >= next_reap_at: + await watchdog.reap_stale_runs() + from core.runs.tool_effects.journal import ToolEffectJournal + + pruned = await asyncio.to_thread( + ToolEffectJournal(engine.SessionLocal).prune_settled + ) + if pruned: + logger.info("tool_effect_ledger_pruned", count=pruned) + next_reap_at = loop.time() + settings.STALE_REAPER_INTERVAL_SEC + except asyncio.CancelledError: + raise + except Exception: # noqa: BLE001 + logger.warning("chat_run_stale_reaper_iteration_failed", exc_info=True) diff --git a/src/backend/orchestration/workflow.py b/src/backend/orchestration/workflow.py deleted file mode 100644 index c200affb..00000000 --- a/src/backend/orchestration/workflow.py +++ /dev/null @@ -1,3463 +0,0 @@ -"""Minimal multi-agent workflow orchestration (AgentScope backend).""" - -from __future__ import annotations - -import asyncio -import json -import logging -from dataclasses import dataclass, field -from typing import Any, AsyncIterator, Awaitable, Callable, Dict, List, Optional, Tuple - -from core.config.catalog_resolver import ( - enabled_kb_ids_from_context, - enabled_mcp_ids_from_context, - enabled_skill_ids_from_context, -) -from core.config.display_names import TOOL_DISPLAY_NAMES -from core.llm.factory import create_agent_executor -from core.llm.compacting_agent import frozen_prompt_surface -from core.llm.context_adapter import append_context_text, next_request_sequence, render_context_item -from core.llm.context_ir import ( - KIND_REMINDER, - KIND_USER_INPUT, - POLICY_HEAD_TAIL, - POLICY_NEVER, - SESSION_CONTEXT_META_KEY, - make_text_context_item, - session_context_metadata, -) -from core.llm.context_manager import resolve_model_context_window -from core.llm.mcp_manager import close_clients -from core.llm.message_compat import session_to_msgs -from core.ontology.revision import is_substantive_revision, normalize_revision_candidate -from core.ontology.validator import ( - activate_runtime_for_asset, - claim_output_review, - complete_output_review, - release_output_review, - requires_output_review, -) -from core.services.ontology_service import resolve_runtime_asset_tags -from core.services.project_scope import edition_project_context_keys -from core.services.tool_effect_ledger import find_tool_outcome_unknown -from orchestration.citation_anchor import ( - AnchorAllocator, - anchor_start_for_chat, - attach_allocator, - collect_citation_dicts, -) -from orchestration.streaming import StreamingAgent - -# Project mode: extracted from chats.py's ctx and passed through to agent_factory so the system prompt renders the project section. -_PROJECT_CTX_KEYS = ( - "project_id", - "project_api_scope", - "project_name", - "project_instructions", - "project_init", - "project_folder_name", - "project_folder_kind", - "project_folder_id", - # Desktop local project — needed so the prompt renders the local-mode section - # with the real host folder path instead of the cloud /workspace section. - "project_kind", - "project_is_local", - "project_local_path", - "project_local_slug", -) + edition_project_context_keys() - - -def _mode_visible_subagents(spec, visible, mentioned_ids): - """收窄模式下最终可见的子智能体 = 模式圈定的那批 ∪ 本轮被显式 @ 的那个。 - - 模式没圈任何子智能体时维持原契约：只有本轮被呼唤的那个能入场（极速模式一直 - 是这么做的）。圈了就以模式名单为准，再并上本轮呼唤的，免得用户 @ 了一个模式 - 里没列的智能体时那次呼唤被无声吞掉。 - """ - allowed = { - a for a in (getattr(spec, "agent_ids", ()) or ()) if isinstance(a, str) and a.strip() - } - wanted = {str(m) for m in (mentioned_ids or []) if m} - keep = allowed | wanted - if not keep: - return [] - return [item for item in (visible or []) if str((item or {}).get("agent_id") or "") in keep] - - -def _runtime_visible_subagents( - user_agents, - parent_runtime, - disabled_builtin_ids, - explicit_agent_ids, -): - """Build the default agent directory plus any agent summoned this turn. - - Disabled agents stay out of the main agent's initial context. An explicit - ``@``/command may add exactly the selected accessible agent for this turn; - the route layer has already enforced ownership and platform visibility. - """ - from core.llm.builtin_subagents import merge_builtin_subagents - - explicitly_selected = {str(agent_id) for agent_id in (explicit_agent_ids or []) if agent_id} - all_accessible = merge_builtin_subagents( - user_agents, - parent_runtime, - disabled_agent_ids=disabled_builtin_ids, - include_disabled=True, - ) - return [ - item - for item in all_accessible - if item.get("is_enabled", True) or str(item.get("agent_id") or "") in explicitly_selected - ] - - -def _resolve_mode_spec(context: dict): - """从 context 解析这段对话的模式装配契约。 - - 线上字段是 ``mode_slug``；老客户端只发 ``chat_mode='turbo'``，按极速模式解析， - 所以升级期间新旧前端都能跑。解析失败返回 None（= 标准模式，不收窄）。 - """ - slug = str(context.get("mode_slug", "") or "").strip() - if not slug and str(context.get("chat_mode", "") or "").lower() == "turbo": - slug = "turbo" - if not slug: - return None - try: - from core.db.engine import SessionLocal - from core.services.chat_mode_service import ChatModeService - - with SessionLocal() as db: - spec = ChatModeService(db).resolve(slug, str(context.get("user_id", "") or "") or None) - return spec - except Exception: # noqa: BLE001 — 模式是增强不是边界，解析炸了也要让对话跑起来 - logger.warning("[workflow] 模式解析失败 slug=%s，按标准模式继续", slug, exc_info=True) - return None - - -def _approval_mode(context: Dict[str, Any]) -> Optional[str]: - """本轮上下文里带的权限档；没带就交回 None，由工厂读用户自己存的那一档。""" - return context.get("approval_mode") - - -def _extract_project_ctx(context: Dict[str, Any]) -> Optional[Dict[str, Any]]: - """Extract project-related fields from the workflow context. Returns None when there is no project_id.""" - if not context.get("project_id"): - return None - return {k: context.get(k) for k in _PROJECT_CTX_KEYS} - - -def _resolve_agent_model_runtime(agent: Any, fallback_model_name: Any = "") -> Tuple[str, int]: - """Return the effective model name and context window baked into an agent. - - AgentScope chat models expose the upstream name as ``.model``. Some provider - wrappers expose ``.model_name`` instead, so accept both before consulting the - request fallback. Channel runs intentionally omit a model selection; converting - that ``None`` to the literal string ``"None"`` previously made direct sub-agent - conversations fail context-window lookup even though the resolved main model was - correctly configured. - """ - model = getattr(agent, "model", None) - state = getattr(agent, "state", None) - candidates = ( - getattr(model, "model", None), - getattr(model, "model_name", None), - getattr(state, "model_name", None), - fallback_model_name, - ) - model_name = "" - for candidate in candidates: - text = str(candidate or "").strip() - if text and text.lower() != "none": - model_name = text - break - - context_window = int(getattr(model, "context_size", 0) or 0) - if context_window <= 0: - context_window = resolve_model_context_window(model_name) - return model_name, context_window - - -def _compaction_budget_inputs(agent: Any, context_window: int) -> Dict[str, Any]: - """Read the exact frozen prompt/tool surface cached by agent_factory.""" - system_prompt, tool_schema = frozen_prompt_surface(agent) - return { - "system_prompt": system_prompt, - "tool_schema": tool_schema, - "context_window": int(context_window or 0), - "model_name": str( - getattr(getattr(agent, "model", None), "model", None) - or getattr(getattr(agent, "state", None), "model_name", None) - or "" - ), - "model_provider_id": str( - getattr(getattr(agent, "model", None), "resolved_model_provider_id", "") - or getattr(getattr(agent, "state", None), "model_provider_id", "") or "" - ), - } - - -def _pre_turn_compaction_inputs( - agent: Any, - context_window: int, - resolved_model_name: str, -) -> Dict[str, Any]: - """Adapt the shared budget snapshot to the pre-turn service signature.""" - inputs = _compaction_budget_inputs(agent, context_window) - inputs.pop("model_provider_id", None) - inputs["model_name"] = str(inputs.get("model_name") or resolved_model_name) - return inputs - - -def _extract_skill_id_from_path(path: str) -> str: - """Extract skill_id from a SKILL.md path (convention: .../skills//SKILL.md).""" - if not path: - return "" - import os - - norm = path.replace("\\", "/").strip().strip('"').strip("'") - parts = [p for p in norm.split("/") if p] - if not parts: - return "" - if parts[-1].upper() == "SKILL.MD" and len(parts) >= 2: - return parts[-2] - base = os.path.basename(norm) - if base.upper() == "SKILL.MD" and len(parts) >= 2: - return parts[-2] - return "" - - -def _synthesize_missing_tool_call( - tool_id: str, tool_name: str, displayed_tools: set -) -> Optional[Dict[str, Any]]: - """Build the ``tool_call`` event a tool never got, or ``None`` if it already has one. - - ``_tool_args_ready`` holds back the tool card while streamed args are still - incomplete, but it cannot tell "not written yet" from "there is nothing to - write": a zero-argument tool (for example, a parameterless status probe) keeps - empty args forever, so its card is suppressed for the whole run. Without a - card the frontend has nothing to attach the result to and — with tools - running in parallel — binds it onto whichever sibling is still running, - making the call vanish from the tool list and briefly corrupting the - sibling's output. Emitting the missing call right before its result keeps - every executed tool visible, in order. - - The caller must pass the already-resolved (post skill-load override) tool - name, and must have skipped tools that intentionally render no card. - """ - if not tool_id or tool_id in displayed_tools: - return None - displayed_tools.add(tool_id) - display_name = ( - "加载技能" if tool_name == "load_skill" else TOOL_DISPLAY_NAMES.get(tool_name, tool_name) - ) - return { - "type": "tool_call", - "tool_name": tool_name, - "tool_display_name": display_name, - "tool_args": {}, - "input": {}, - "tool_id": tool_id, - } - - -def _ontology_review_event_context(runtime: Dict[str, Any]) -> Dict[str, Any]: - committee_size = max( - (int(pack.get("config", {}).get("committee_size", 3)) for pack in runtime.get("packs", [])), - default=3, - ) - workflows = [ - { - "pack_id": pack.get("pack_id"), - "workflow_id": workflow.get("id"), - "workflow_name": workflow.get("name"), - } - for pack in runtime.get("packs", []) - for workflow in pack.get("workflows", []) - ] - review_state = runtime.get("output_review") or {} - return { - "governance_run_id": runtime.get("governance_run_id"), - "review_owner": review_state.get("owner"), - "review_count": int(review_state.get("count") or 0), - "committee_size": committee_size, - "workflows": workflows, - "activation_sources": sorted( - { - str(event.get("source")) - for event in runtime.get("runtime_events", []) - if event.get("type") == "ontology_activation" and event.get("source") - } - ), - } - - -def _ontology_governance_summary(runtime: Dict[str, Any]) -> Optional[Dict[str, Any]]: - """Build the compact, persisted governance card payload for one answer.""" - if not runtime.get("enabled"): - return None - events = list(runtime.get("runtime_events") or []) - activations = [ - { - key: event.get(key) - for key in ( - "pack_id", - "workflow_id", - "workflow_name", - "source", - "asset_kind", - "asset_id", - "review_level", - ) - if event.get(key) is not None - } - for event in events - if event.get("type") == "ontology_activation" - ] - gates = [ - { - key: event.get(key) - for key in ( - "decision", - "tool_name", - "matched_rule_ids", - "violations", - "denial_count", - "circuit_breaker", - ) - if event.get(key) is not None - } - for event in events - if event.get("type") == "ontology_gate" - ] - review_state = dict(runtime.get("output_review") or {}) - review_state["level"] = runtime.get("review_level", "none") - review_state["committee_size"] = _ontology_review_event_context(runtime)["committee_size"] - if not activations and not gates and review_state.get("status") == "pending": - return None - if review_state.get("status") == "pending" and not requires_output_review(runtime): - review_state["status"] = "skipped" - return { - "governance_run_id": runtime.get("governance_run_id"), - "activations": activations, - "gates": gates, - "review": review_state, - } - - -def _ontology_review_owner(runtime: Dict[str, Any], context: Dict[str, Any]) -> str: - """Stable owner token for the one outer-workflow final review.""" - run_id = runtime.get("governance_run_id") or context.get("chat_id") or "request" - return f"outer_workflow:{run_id}" - - -def _ontology_review_failure_result(answer: str, exc: Exception) -> Dict[str, Any]: - """Keep a completed draft usable when the post-answer review itself fails.""" - logger.error( - "[ontology-review] automatic review failed; preserving original answer: %s", - exc, - exc_info=(type(exc), exc, exc.__traceback__), - ) - return { - "verdict": "escalate", - "answer": answer, - "revised": False, - "annotated": True, - "repair_attempts": 0, - "attempts": 1, - "violations": [], - "affected_claims": [], - "evidence": [], - "feedback": ["自动评审未正常完成，已保留原文并转人工复核。"], - "manual_review": { - "required": True, - "title": "领域本体人工复核", - "summary": "自动评审未正常完成，原文已保留，请人工核对或重新发起评审。", - "items": [ - { - "quote": "本轮完整答案", - "rule_id": "ontology_review_runtime", - "risk": "自动评审未完成，系统无法确认答案满足全部领域约束。", - "manual_check": "核对关键事实、推断、适用前提和原始工具证据。", - } - ], - "actions": ["重新发起本体评审，或在采用结论前完成人工核对。"], - }, - "new_tools": [], - "new_citation_count": 0, - "latency_ms": None, - } - - -async def _anext_in_subagent_log_scope(iterator, subagent_log_id: str): # noqa: ANN001 - """Advance one agent-stream item without leaking log tokens across SSE yields.""" - from core.services import log_service as log_writer - - with log_writer.subagent_scope(subagent_log_id, source="subagent"): - return await anext(iterator) - - -def _record_ontology_activations( - activations: List[Dict[str, Any]], - runtime: Dict[str, Any], - context: Dict[str, Any], -) -> None: - if not activations: - return - try: - from core.services.ontology_service import record_runtime_activation - - for event in activations: - record_runtime_activation( - event, - runtime, - user_id=str(context.get("user_id", "")) or None, - chat_id=context.get("chat_id"), - ) - except Exception as exc: # noqa: BLE001 - logger.warning("[ontology] workflow activation audit persistence failed: %s", exc) - - -def _bounded_trace_value(value: Any, max_chars: int = 4000) -> Any: - """Keep reviewer evidence useful without copying unbounded tool output.""" - try: - text = json.dumps(value, ensure_ascii=False, default=str) - except Exception: - text = str(value) - if len(text) <= max_chars: - return value - return {"truncated": True, "preview": text[:max_chars]} - - -def _parse_tool_result_value(value: Any) -> Any: - if not isinstance(value, str): - return value - try: - return json.loads(value) if value else {} - except json.JSONDecodeError: - return {"result": value} - - -def _profile_memory_budget_ms(context: Dict[str, Any]) -> Optional[int]: - """How long this task type waits for memory, per the active profile. - - Returns ``None`` when nothing is published, which leaves the caller on the - deployment default — the built-in profile carries that same value, so this - is a no-op until a profile deliberately changes it. - - Never raises: retrieval sits on the user's critical path, and an orchestration - lookup must not be able to fail a turn. - """ - try: - from core.auth.tenancy import tenant_of - from core.evolution.agent_profile import load_active_profile - - user_id = str(context.get("user_id") or "") - profile = load_active_profile( - task_type=str(context.get("chat_mode") or "chat"), - user_id=user_id, - tenant_id=tenant_of(user_id), - ) - return profile.memory_budget_ms - except Exception as exc: # noqa: BLE001 - logger.debug("[memory] profile budget unavailable: %s", exc) - return None - - -def _capture_nested_ontology_evidence( - payload: Dict[str, Any], - trace: List[Dict[str, Any]], - citations: List[Dict[str, Any]], - allocator: Optional[AnchorAllocator] = None, -) -> List[Dict[str, Any]]: - """Merge trusted ``call_subagent`` bypass events into the outer review trace.""" - sub_type = str(payload.get("sub_type") or "") - tool_name = str(payload.get("tool_name") or "") - tool_id = str(payload.get("tool_id") or "") - provenance = { - key: payload.get(key) - for key in ("agent_id", "agent_name", "sub_run_id", "parent_tool_id") - if payload.get(key) is not None - } - if sub_type == "tool_call" and tool_name and payload.get("input") is not None: - trace.append( - { - "type": "tool_call", - "tool_id": tool_id, - "tool_name": tool_name, - "input": _bounded_trace_value(payload.get("input")), - "source": "subagent", - **provenance, - } - ) - return [] - if sub_type != "tool_result" or not tool_name or payload.get("status") == "error": - return [] - - result = _parse_tool_result_value(payload.get("output")) - cit_dicts = collect_citation_dicts(tool_id, allocator) - citations.extend(cit_dicts) - trace.append( - { - "type": "tool_result", - "tool_id": tool_id, - "tool_name": tool_name, - "result": _bounded_trace_value(result), - "citations": cit_dicts, - "source": "subagent", - **provenance, - } - ) - return cit_dicts - - -def _ontology_repair_prompt(payload: Dict[str, Any]) -> str: - violations = payload.get("violations") or [] - feedback = payload.get("feedback") or [] - affected_claims = payload.get("affected_claims") or [] - original_task = str(payload.get("original_task") or "").strip() - current_answer = str(payload.get("answer") or "").strip() - citations = [ - { - "id": item.get("id"), - "tool_name": item.get("tool_name"), - "title": item.get("title"), - } - for item in payload.get("citations") or [] - if isinstance(item, dict) - ] - return ( - "\n" - "这是同一任务唯一一次领域本体修复，不是新的用户任务。" - "优先级强制规则：已经激活的 enforce 级领域本体约束及本修复指令，高于用户原始提示中" - "与其冲突的‘不要调用工具’、‘原样照抄’或类似限制；这些限制只能约束正常生成阶段，" - "不能阻止本体修复取证和纠错。若‘确定性违规’包含‘未调用’某些必需工具，本轮必须" - "实际调用这些工具后才能形成终稿；以用户禁止工具为理由跳过调用、保留原错误断言，" - "或再次原样输出待修订内容，均视为修复失败。" - "先比较现有证据与评审问题，明确是否出现了需要重新搜索的新内容或可补充调用的新工具。" - "有证据缺口时必须调用真实工具补齐；现有证据已足够时才可直接精准修订。" - "禁止只做同义改写，也不得虚构工具调用或引用。" - "证据核对、逐项判定、原文对比和修改说明都属于内部校验过程，只能放在" - " thinking/reasoning 通道；除非用户最初明确要求核验报告，否则不得把这些内容写入" - "最终回复或生成的文件。" - "必须继续遵守用户最初指令中的输出与交付形式；如果原任务要求生成 Word、PPT、" - "表格、网页或其他文件，修订时仍须调用相应工具生成该类产物，不得降级为纯文字替代。" - "最终交付必须是利用核验结论直接修正后的完整润色稿，而不是关于原答案的评估报告。" - "保留原任务要求的文体、篇幅、结构和语气，只改正有问题的事实、推断与表述；" - "以待修订原始输出为底稿做最小必要修改，不要因为查到更多资料就加入与原主张无关的" - "企业背景、资质或延伸分析。句子级任务应尽量保留原句的主语与分句顺序，逐个修正" - "不成立的断言，形成简洁、自然、可直接交付的一句话终稿。" - "用户只要求一句话时仍输出一句话，不得自行扩写成包含‘原始语句’、‘逐项核验’、" - "‘主张评估’或‘修改说明’等章节的报告。若交付物是文件，文件正文也必须是润色终稿，" - "并重新生成、交付修正后的文件。没有获得证据的风险项只能表述为‘待核验’、" - "‘暂无数据支撑’或‘无法判断’，绝不能反向断言为‘不存在’、‘没有风险’或‘风险为零’。" - "在输出终稿和生成文件前，必须在内部逐条检查‘确定性违规’中的每个条件：若要求" - "最低长度，终稿必须达到该长度；若要求引用，相关事实后必须包含真实引用标记" - "（从工具结果复制 cite_id，写成 `[锚文本](cite:eN)`）" - "且按规则补齐参考资料；若要求区分事实、推断和待核验项，可在同一句中用分号和简短标签" - "表达，不得省略。先确定唯一的合格终稿，再把完全相同的正文写入用户要求的文件，" - "不得先生成文件后又输出一份更短、缺引用或结论不同的候选正文。" - "思考过程只放在模型的 thinking/reasoning 通道。最终正文必须且只能放在" - " `完整候选答案` 中。" - "在正式输出最终正文前，不要复述、引用或示例化这组标签，也不要用省略号代替正文。" - "标签内必须是可独立阅读、内容完整的修订答案；若无法修订，则原样放入当前完整答案。" - "事实、推断和待核验项必须明确区分；引用已有或新增工具结果时，把工具结果里" - "标注的 cite_id 原样复制进 `[锚文本](cite:eN)`，禁止自行编号。\n" - f"用户最初指令：{json.dumps(original_task, ensure_ascii=False)[:12000]}\n" - f"待修订原始输出：{json.dumps(current_answer, ensure_ascii=False)[:12000]}\n" - f"修复轮次：{payload.get('attempt', 1)}\n" - f"问题来源：{payload.get('source', 'review')}\n" - f"确定性违规：{json.dumps(violations, ensure_ascii=False)[:8000]}\n" - f"委员会意见：{json.dumps(feedback, ensure_ascii=False)[:8000]}\n" - f"受影响内容：{json.dumps(affected_claims, ensure_ascii=False)[:5000]}\n" - f"当前可用引用：{json.dumps(citations, ensure_ascii=False)[:5000]}\n" - "直接开始补充工具或输出修复后的完整答案。\n" - "" - ) - - -async def _run_ontology_repair_round( - *, - streaming_agent: StreamingAgent, - context: Dict[str, Any], - payload: Dict[str, Any], - runtime: Dict[str, Any], - trace: List[Dict[str, Any]], - citations: List[Dict[str, Any]], - allocator: Optional[AnchorAllocator] = None, - event_cursor: int = 0, - subagent_log_id: Optional[str] = None, - event_sink: Optional[Callable[[Dict[str, Any]], Awaitable[None]]] = None, -) -> Tuple[str, List[Dict[str, Any]], int, int]: - """Continue the originating agent and stream a separate revision candidate.""" - answer = "" - events: List[Dict[str, Any]] = [] - tool_count = 0 - text_buffer = "" - raw_text = "" - revision_open = False - revision_closed = False - thinking_line_prefix = "" - - async def _publish(event: Dict[str, Any]) -> None: - events.append(event) - if event_sink is not None: - await event_sink(event) - - async def _publish_text_thinking(delta: str) -> None: - """Publish text-channel reasoning and remember its current line prefix.""" - nonlocal thinking_line_prefix - if not delta: - return - if "\n" in delta: - thinking_line_prefix = delta.rsplit("\n", 1)[-1] - else: - thinking_line_prefix = (thinking_line_prefix + delta)[-256:] - await _publish({"type": "ontology_revision_thinking", "delta": delta}) - - def _is_revision_opening_boundary(line_before_open: str) -> bool: - """Allow a revision wrapper after whitespace or a reasoning close tag.""" - - prefix = line_before_open.rstrip() - return not prefix or prefix.endswith(("", "")) - - await _publish( - { - "type": "ontology_repair", - "status": "started", - "attempt": payload.get("attempt", 1), - "source": payload.get("source", "review"), - } - ) - - stream = streaming_agent.stream( - [{"role": "user", "content": _ontology_repair_prompt(payload)}], - context, - ).__aiter__() - while True: - try: - if subagent_log_id: - event_type, event_payload = await _anext_in_subagent_log_scope( - stream, - subagent_log_id, - ) - else: - event_type, event_payload = await anext(stream) - except StopAsyncIteration: - break - - state_runtime = getattr(streaming_agent.agent.state, "ontology_runtime", None) - if isinstance(state_runtime, dict): - runtime = state_runtime - context["ontology_runtime"] = state_runtime - pending_events = runtime.get("runtime_events", [])[event_cursor:] - for item in pending_events: - await _publish(dict(item)) - event_cursor += len(pending_events) - - if event_type == "vision_progress": - await _publish({"type": "vision_progress", **(event_payload or {})}) - continue - - if event_type == "text_delta": - delta = str(event_payload or "") - if not delta: - continue - raw_text += delta - if revision_closed: - await _publish_text_thinking(delta) - continue - text_buffer += delta - while text_buffer: - if not revision_open: - open_idx = text_buffer.find("") - if open_idx < 0: - keep = min(len("") - 1, len(text_buffer)) - safe_len = len(text_buffer) - keep - if safe_len > 0: - await _publish_text_thinking(text_buffer[:safe_len]) - text_buffer = text_buffer[safe_len:] - break - before_open = text_buffer[:open_idx] - line_before_open = (thinking_line_prefix + before_open).rsplit("\n", 1)[-1] - if not _is_revision_opening_boundary(line_before_open): - # Reasoning models sometimes describe the contract as - # `...`. A real - # wrapper starts a new line or immediately follows the - # model's reasoning close tag. Other inline occurrences - # remain folded as examples rather than candidate text. - ignored = text_buffer[: open_idx + len("")] - await _publish_text_thinking(ignored) - text_buffer = text_buffer[open_idx + len("") :] - continue - if open_idx > 0: - await _publish_text_thinking(before_open) - text_buffer = text_buffer[open_idx + len("") :] - revision_open = True - continue - close_idx = text_buffer.find("") - if close_idx < 0: - keep = min(len("") - 1, len(text_buffer)) - safe_len = len(text_buffer) - keep - if safe_len > 0: - candidate_delta = text_buffer[:safe_len] - answer += candidate_delta - await _publish({"type": "ontology_revision", "delta": candidate_delta}) - text_buffer = text_buffer[safe_len:] - break - if close_idx > 0: - candidate_delta = text_buffer[:close_idx] - answer += candidate_delta - await _publish({"type": "ontology_revision", "delta": candidate_delta}) - text_buffer = text_buffer[close_idx + len("") :] - revision_closed = True - break - elif event_type == "thinking_delta": - await _publish( - { - "type": "ontology_revision_thinking", - "delta": str(event_payload or ""), - } - ) - elif event_type == "tool_call_start": - tool_name = str(event_payload.get("name") or "unknown") - if tool_name != "update_plan": - await _publish( - { - "type": "tool_call_start", - "tool_name": tool_name, - "tool_display_name": TOOL_DISPLAY_NAMES.get(tool_name, tool_name), - "tool_id": str(event_payload.get("id") or ""), - "scope": "ontology_revision", - } - ) - elif event_type == "tool_call_delta": - tool_name = str(event_payload.get("name") or "unknown") - if tool_name != "update_plan" and event_payload.get("delta"): - await _publish( - { - "type": "tool_call_delta", - "tool_name": tool_name, - "tool_id": str(event_payload.get("id") or ""), - "arguments_delta": str(event_payload.get("delta") or ""), - "scope": "ontology_revision", - } - ) - elif event_type == "tool_call": - tool_name = str(event_payload.get("name") or "unknown") - tool_id = str(event_payload.get("id") or "") - tool_args = event_payload.get("args") or {} - trace.append( - { - "type": "tool_call", - "tool_id": tool_id, - "tool_name": tool_name, - "input": _bounded_trace_value(tool_args), - "source": "ontology_repair", - } - ) - await _publish( - { - "type": "tool_call", - "tool_name": tool_name, - "tool_display_name": TOOL_DISPLAY_NAMES.get(tool_name, tool_name), - "tool_args": tool_args, - "input": tool_args, - "tool_id": tool_id, - "scope": "ontology_revision", - } - ) - elif event_type == "tool_result": - tool_count += 1 - tool_name = str(event_payload.get("name") or "unknown") - tool_id = str(event_payload.get("id") or "") - result = _parse_tool_result_value(event_payload.get("content")) - cit_dicts = collect_citation_dicts(tool_id, allocator) - citations.extend(cit_dicts) - trace.append( - { - "type": "tool_result", - "tool_id": tool_id, - "tool_name": tool_name, - "result": _bounded_trace_value(result), - "citations": cit_dicts, - "source": "ontology_repair", - } - ) - await _publish( - { - "type": "tool_result", - "tool_name": tool_name, - "result": result, - "tool_id": tool_id, - "citations": cit_dicts, - "scope": "ontology_revision", - } - ) - elif event_type == "subagent_event": - nested_citations = _capture_nested_ontology_evidence( - event_payload or {}, - trace, - citations, - allocator, - ) - sub_type = str((event_payload or {}).get("sub_type") or "") - if sub_type in { - "start", - "thinking", - "content", - "tool_call", - "tool_call_delta", - "tool_result", - "end", - }: - await _publish( - { - "type": "subagent_event", - **(event_payload or {}), - "scope": "ontology_revision", - **({"citations": nested_citations} if nested_citations else {}), - } - ) - elif event_type == "tool_pending": - await _publish( - { - "type": "tool_pending", - **(event_payload or {}), - "scope": "ontology_revision", - } - ) - elif event_type in {"file_confirm", "design_pick"}: - raise RuntimeError("本体自动修复轮不支持等待交互确认，请转人工复核") - elif event_type == "error": - if isinstance(event_payload, BaseException): - raise event_payload - raise RuntimeError(str(event_payload)) - - if revision_open and not revision_closed and text_buffer: - answer += text_buffer - await _publish({"type": "ontology_revision", "delta": text_buffer}) - elif not revision_open and text_buffer.strip(): - # Compatibility fallback for models that ignored the wrapper. When - # embedded text exists, only the content after the last closing - # marker can be a candidate; never promote the hidden reasoning itself. - fallback = normalize_revision_candidate(raw_text.rsplit("", 1)[-1]) - if is_substantive_revision(fallback): - answer = fallback - await _publish({"type": "ontology_revision", "delta": answer}) - - await _publish( - { - "type": "ontology_repair", - "status": "completed", - "attempt": payload.get("attempt", 1), - "tool_calls": tool_count, - } - ) - return normalize_revision_candidate(answer), events, event_cursor, tool_count - - -from core.chat.plan_progress import save_plan_progress as _save_plan_progress # noqa: E402 -from core.chat.plan_progress import settle_plan_progress as _settle_plan_progress -from core.llm.plan_update_tool import parse_plan_update_args # noqa: E402 - - -def _chat_has_live_job(chat_id: str) -> bool: - """这个会话还有在跑（或待启动）的后台作业吗 —— 决定计划栏本轮该不该收尾。""" - try: - from core.db.engine import SessionLocal as _JobSession - from core.db.models import Job as _Job - - with _JobSession() as _db: - return ( - _db.query(_Job.job_id) - .filter(_Job.chat_id == chat_id, _Job.status.in_(("pending", "running"))) - .first() - is not None - ) - except Exception: # noqa: BLE001 —— 查不到就当没有作业，照常收尾，别把计划栏永远挂着 - return False - - -# SSE tool-result payload builders (moved to routing.tool_payloads) -from orchestration.tool_payloads import ( # noqa: E402 - _FAST_EMIT_TOOLS, - _build_read_artifact_payload, - _build_read_tool_payload, - _build_skill_load_payload, - _build_view_text_file_payload, - _tool_args_ready, -) - -# Process-level persistent references: after each streaming run, -# (streaming_agent, mcp_clients) is pushed in so HTTP transport MCP clients -# (currently retrieve_dataset_content, a streamable_http client) are never GC'd. -# -# Cause: the HTTP client uses an anyio TaskGroup + CancelScope. For stdio -# clients, streaming_agent.shutdown() SIGTERMs the subprocess directly and -# cleans up fine; HTTP clients (`_process is None`) are skipped by shutdown and -# eventually land in Python GC running __aexit__ — that __aexit__ almost -# certainly runs on the wrong task, triggering -# RuntimeError: Attempted to exit cancel scope in a different task... -# The cancel signal flows back through the event loop and takes out the current -# stream's agent.reply() / the next item's create_subprocess_exec along with -# it. Reproduced in both production and local. -# -# accept a small memory leak (one leftover HTTP keepalive socket per -# conversation) in exchange for a runner that never deadlocks. Released -# together when the worker process exits/restarts. -_persistent_clients: list = [] - - -from orchestration.memory_integration import ( # noqa: F401 - build_user_identity_block, - inject_session_blocks, - open_session_memory, - resolve_session_memory, - save_memories_background, -) - -# Re-export public helpers for backward compatibility -from orchestration.message_parser import looks_markdown as _looks_markdown # noqa: F401 -from orchestration.message_parser import resolve_sources_conflict as _resolve_sources_conflict - -logger = logging.getLogger(__name__) - - -def _explicit_skill_ids_from_context(context: Dict[str, Any]) -> List[str]: - skill_ids: List[str] = [] - single = context.get("skill_id") - if single: - skill_ids.append(str(single)) - multi = context.get("skill_ids") - if isinstance(multi, list): - skill_ids.extend(str(item) for item in multi if item) - seen: set[str] = set() - return [item for item in skill_ids if not (item in seen or seen.add(item))] - - -def _build_skill_injection(context: Dict[str, Any]) -> Optional[Dict[str, Any]]: - """Build an explicit-invocation hint message (skill load instructions + MCP tool activation notice). - - Trigger sources: - - ``skill_id`` (single skill, slash-command selection) / ``skill_ids`` - (skill list, expanded from an explicitly referenced plugin) - → inject each skill's SKILL.md load instruction. - - ``mcp_ids`` (MCP servers activated by an explicitly referenced plugin or - a connector selected in the composer) → these tools are already - force-enabled into the toolset (see _build_ctx); here we append a - sentence telling the model they are ready and can be called on demand. - - Once enabled, ambient MCP tools stay resident and the model calls them by - description on its own. Explicit selections guide the model toward the selected - capabilities without forcing tool_choice or rejecting a reply without tool usage. - - Returns a dict {"role": "user", "content": "..."} or None. - """ - skill_ids = _explicit_skill_ids_from_context(context) - - mcp_ids = [str(m) for m in (context.get("mcp_ids") or []) if m] - connector_id = str(context.get("connector_id") or "") - skill_id = str(context.get("skill_id") or "") - skill_name = context.get("skill_name") - plugin_name = context.get("plugin_name") - connector_name = context.get("connector_name") - - if not skill_ids and not mcp_ids: - return None - - sections: List[str] = [] - - # ── Skills: load instructions ── - if skill_ids: - try: - from core.agent_skills.loader import get_skill_loader - - loader = get_skill_loader() - metadata_all = loader.load_all_metadata() - entries: List[str] = [] - for sid in skill_ids: - meta = metadata_all.get(sid) - if not meta: - logger.warning("[skill_inject] skill_id=%s not found", sid) - continue - # Materialize once; each deployment publishes its own usable path. - skill_dir = loader.get_skill_dir(sid) - if not skill_dir: - logger.warning("[skill_inject] skill_id=%s has no skill dir", sid) - continue - from core.agent_skills.config import model_facing_skill_dir - - sandbox_dir = model_facing_skill_dir(sid, skill_dir) - entries.append(f'- 「{sid}」：view_text_file(file_path="{sandbox_dir}/SKILL.md")') - if entries: - sections.append("技能（使用技能时先读取对应说明文件）：\n" + "\n".join(entries)) - except Exception as e: # noqa: BLE001 - logger.error("[skill_inject] failed to load skills %s: %s", skill_ids, e) - - # ── MCP: activation notice (the tools themselves are already in the toolset; call by description) ── - if mcp_ids: - if connector_id: - sections.append( - f"MCP 工具：连接器「{connector_name or connector_id}」已被用户显式选择。" - "请优先按需使用该连接器完成相关任务，" - "也不得用其他工具冒充该连接器的结果。" - ) - elif plugin_name: - sections.append( - f"MCP 工具：插件「{plugin_name}」的 {len(mcp_ids)} 个 MCP 服务已就绪。" - "请优先按需使用该插件的 MCP 工具或技能，" - "使用技能时先读取对应说明文件。" - ) - else: - sections.append( - f"MCP 工具：本轮显式激活的 {len(mcp_ids)} 个 MCP 连接器服务已就绪，" - "可直接按需调用（无需加载文件），请优先使用它们完成相关任务。" - ) - - if not sections: - return None - - # 极速模式警示：被呼唤技能的 SKILL.md 往往指示 Bash/沙箱/文件工具流程， - # 而极速模式不装配这些工具。若不在呼唤点明说，模型会照着技能说明发起 - # 不存在的工具调用（实测会卡死在参数流上）。 - if str(context.get("chat_mode", "") or "").lower() == "turbo" and skill_ids: - sections.append( - "注意：当前为极速模式，没有 Bash、沙箱、文件读写和文档生成工具。" - "技能说明中涉及这些工具的步骤一律不可执行，也不要尝试调用——" - "读取 SKILL.md 后，只输出你能以纯文本交付的部分（结构、正文、模板等），" - "并在结尾建议用户切换到「快速模式」或「思考模式」完整执行该技能。" - ) - - if plugin_name and connector_name: - header = ( - f"用户已显式调用插件「{plugin_name}」并选择连接器「{connector_name}」，" - "请优先采用这些能力：" - ) - elif plugin_name: - header = f"用户已显式调用插件「{plugin_name}」，请优先按需采用以下能力：" - elif connector_name: - header = f"用户已显式选择连接器「{connector_name}」，请优先采用其能力：" - elif skill_id: - header = ( - f"用户已显式调用技能「{skill_name or skill_id}」，" - "请优先采用该技能，使用前读取其 SKILL.md：" - ) - else: - header = "用户已显式指定使用以下能力，请优先采用：" - content = ( - "\n" - + header - + "\n" - + "\n\n".join(sections) - + "\n" - ) - item = make_text_context_item( - content, - item_id="harness:explicit_invocation", - kind=KIND_REMINDER, - origin="harness:explicit_invocation", - trust="system", - created_seq=0, - priority=900, - token_budget=8_000, - truncation_policy=POLICY_HEAD_TAIL, - ) - return { - "role": "user", - "content": content, - SESSION_CONTEXT_META_KEY: session_context_metadata(item), - } - - -def _parse_agent_mentions(message: str, available_agents: list) -> list: - """Parse @agent_name mentions from user message. - - Returns list of matched ``agent_id``s in the order they appear in - *message*. - - When several agent names share a prefix (e.g. ``搜索`` vs ``搜索助手``) - a naive ``"@搜索" in message`` check matches "@搜索" *inside* - "@搜索助手", so typing ``@搜索助手`` would falsely also mention the - shorter ``搜索`` agent — and the prompt hint built downstream would - instruct the LLM to call both. We sort candidates by name length - descending and reserve consumed character ranges so the longest - matching name wins and prefix-shadow matches are skipped. - """ - if not available_agents: - return [] - - candidates = [a for a in available_agents if a.get("name")] - if not candidates: - return [] - candidates.sort(key=lambda a: len(a["name"]), reverse=True) - - consumed: list = [] # non-overlapping (start, end) char ranges - - def _overlaps(start: int, end: int) -> bool: - for s, e in consumed: - if start < e and end > s: - return True - return False - - # (position, agent_id) — sorted at the end to preserve message order - hits: list = [] - for agent in candidates: - token = "@" + agent["name"] - start = 0 - while True: - idx = message.find(token, start) - if idx < 0: - break - end = idx + len(token) - if not _overlaps(idx, end): - consumed.append((idx, end)) - hits.append((idx, agent["agent_id"])) - start = idx + len(token) - - hits.sort(key=lambda x: x[0]) - seen: set = set() - ordered: list = [] - for _, aid in hits: - if aid not in seen: - seen.add(aid) - ordered.append(aid) - return ordered - - -def _direct_agent_id_from_context(context: Dict[str, Any]) -> Optional[str]: - """Return the target only for a persistent dedicated sub-agent chat. - - A per-turn @mention must stay on the main-model stream so it produces a - genuine ``call_subagent`` tool call instead of silently bypassing the - parent model's reasoning and streaming lifecycle. - """ - value = context.get("agent_id") - return str(value) if value else None - - -def _direct_agent_execution_permissions(agent_id: str) -> Tuple[bool, bool]: - """Return ``(read_only, allow_bash)`` for a dedicated sub-agent chat.""" - from core.llm.builtin_subagents import get_builtin_subagent - - builtin_spec = get_builtin_subagent(agent_id) - if builtin_spec is None: - return False, True - return builtin_spec.read_only, builtin_spec.allow_bash - - -def _load_direct_user_agent( - agent_id: str, - user_id: str, - parent_runtime: Optional[Dict[str, Any]] = None, -): - """Load a built-in profile or detach a complete custom-agent configuration.""" - from core.db.engine import SessionLocal - from core.llm.builtin_subagents import build_builtin_runtime_profile, get_builtin_subagent - from core.services.user_agent_service import UserAgentService - - builtin_spec = get_builtin_subagent(agent_id) - if builtin_spec is not None: - return build_builtin_runtime_profile(builtin_spec, parent_runtime) - - with SessionLocal() as db: - user_agent = UserAgentService(db).get_raw_by_id(agent_id, user_id=user_id) - # Eagerly load every relationship/config field consumed after the DB - # session closes. SQLAlchemy expires lazy relationships on close. - _ = user_agent.mcp_server_ids, user_agent.skill_ids, user_agent.kb_ids - _ = user_agent.system_prompt, user_agent.model_provider_id - _ = user_agent.max_iters, user_agent.temperature, user_agent.max_tokens - _ = user_agent.timeout, user_agent.extra_config - return user_agent - - -# ------------------------------------------------------------------ -# Data containers -# ------------------------------------------------------------------ - - -@dataclass -class WorkflowResult: - route: str = "main" - response: str = "" - is_markdown: bool = False - sources: List[Dict[str, Any]] = field(default_factory=list) - artifacts: List[Dict[str, Any]] = field(default_factory=list) - warnings: List[str] = field(default_factory=list) - meta: Dict[str, Any] = field(default_factory=dict) - - -# ------------------------------------------------------------------ -# Synchronous workflow (non-streaming) -# ------------------------------------------------------------------ - - -def run_chat_workflow( - *, - session_messages: List[Dict[str, Any]], - user_message: str, - context: Dict[str, Any], -) -> WorkflowResult: - """Run route -> target execution.""" - from core.llm.evaluation_runtime import apply_evaluation_scope - context = apply_evaluation_scope(context) - - _explicit_command = context.get("explicit_subagent_command") - _explicit_agent_id = ( - str(_explicit_command.get("agent_id")) - if isinstance(_explicit_command, dict) and _explicit_command.get("agent_id") - else None - ) - _mention_agent_id = str(context.get("mention_agent_id") or "") or None - _direct_agent_id = _direct_agent_id_from_context(context) - _direct_user_agent = None - _direct_read_only = False - _direct_allow_bash = True - _visible_subagents: List[Dict[str, Any]] = [] - _request_ontology_runtime = context.get("ontology_runtime") - if not isinstance(_request_ontology_runtime, dict): - _request_ontology_runtime = {} - context["ontology_runtime"] = _request_ontology_runtime - if _direct_agent_id: - _direct_read_only, _direct_allow_bash = _direct_agent_execution_permissions( - _direct_agent_id - ) - _direct_user_agent = _load_direct_user_agent( - _direct_agent_id, - str(context.get("user_id", "")), - { - "enabled_skill_ids": enabled_skill_ids_from_context(context) or [], - "enabled_mcp_ids": enabled_mcp_ids_from_context(context) or [], - "enabled_kb_ids": enabled_kb_ids_from_context(context) or [], - "sandbox_tools_enabled": bool(context.get("sandbox_tools_enabled", False)), - "code_capability_enabled": bool(context.get("code_capability_enabled", False)), - }, - ) - activations = activate_runtime_for_asset( - _request_ontology_runtime, - kind="subagent", - asset_id=_direct_agent_id, - tags=list((_direct_user_agent.extra_config or {}).get("ontology_tags") or []), - ) - _record_ontology_activations( - activations, - _request_ontology_runtime, - context, - ) - else: - try: - from core.db.engine import SessionLocal as _SessionLocal - from core.services.user_agent_service import UserAgentService as _UAS - from core.services.user_service import UserService as _UserService - - with _SessionLocal() as _db: - _visible_subagents = ([] if context.get("evaluation_session") else - _UAS(_db).list_for_user(str(context.get("user_id", "")))) - _disabled_builtin_ids = _UserService(_db).get_disabled_builtin_subagent_ids( - str(context.get("user_id", "")) - ) - except Exception as exc: # noqa: BLE001 - logger.warning("[workflow] failed to load visible subagents: %s", exc) - _disabled_builtin_ids = set() - for skill_id in _explicit_skill_ids_from_context(context): - resolve_runtime_asset_tags( - runtime=_request_ontology_runtime, - kind="skill", - asset_id=skill_id, - user_id=str(context.get("user_id", "") or ""), - ) - activations = activate_runtime_for_asset( - _request_ontology_runtime, - kind="skill", - asset_id=skill_id, - ) - _record_ontology_activations( - activations, - _request_ontology_runtime, - context, - ) - - # ── Inject explicit skill instructions ── - skill_msg = _build_skill_injection(context) - if skill_msg: - session_messages.insert(-1, skill_msg) - logger.info( - "[skill_inject] injected skill instructions for '%s'", - context.get("skill_id"), - ) - - warnings: List[str] = [] - enabled_skill_ids = None if _direct_user_agent else enabled_skill_ids_from_context(context) - enabled_kb_ids = None if _direct_user_agent else enabled_kb_ids_from_context(context) - enabled_mcp_ids = None if _direct_user_agent else enabled_mcp_ids_from_context(context) - - - _workflow_user_id = str(context.get("user_id", "")) - _workflow_model_name = str(context.get("model_name", "")) - _workflow_model_provider_id = str(context.get("model_provider_id", "") or "") - _workflow_chat_mode = str(context.get("chat_mode", "") or "") - _workflow_mem_enabled = bool(context.get("memory_enabled", False)) - _reranker_enabled = bool(context.get("reranker_enabled", False)) - - - if _direct_user_agent is None: - _visible_subagents = _runtime_visible_subagents( - _visible_subagents, - { - "enabled_skill_ids": enabled_skill_ids or [], - "enabled_mcp_ids": enabled_mcp_ids or [], - "enabled_kb_ids": enabled_kb_ids or [], - }, - _disabled_builtin_ids, - [_explicit_agent_id, _mention_agent_id], - ) - - _workflow_mode_spec = _resolve_mode_spec(context) - _workflow_turbo = ( - _direct_user_agent is None - and _workflow_mode_spec is not None - and _workflow_mode_spec.restricted - ) - if _workflow_turbo: - # 收窄模式：可见子智能体 = 模式圈定的那批 ∪ 本轮被显式委派/@ 的那个 - # （此路径无文本 @ 解析）。 - _turbo_delegated = str(_explicit_agent_id or _mention_agent_id or "") - _visible_subagents = _mode_visible_subagents( - _workflow_mode_spec, _visible_subagents, [_turbo_delegated] - ) - - async def _run(): - agent, mcp_clients = await create_agent_executor( - agent_spec=None, - enabled_skill_ids=enabled_skill_ids, - enabled_mcp_ids=enabled_mcp_ids, - enabled_kb_ids=enabled_kb_ids, - current_user_id=_workflow_user_id, - reranker_enabled=_reranker_enabled, - model_name=_workflow_model_name, - model_provider_id=_workflow_model_provider_id, - chat_mode=_workflow_chat_mode, - turbo_mode=_workflow_turbo, - mode_spec=_workflow_mode_spec, - turbo_explicit_skill_ids=( - _explicit_skill_ids_from_context(context) if _workflow_turbo else None - ), - turbo_explicit_mcp_ids=( - [m for m in (context.get("mcp_ids") or []) if isinstance(m, str)] - if _workflow_turbo - else None - ), - # 渐进式插件加载：本轮显式呼唤的能力不得被延迟（用户消息注入已承诺 - # 其可用），并作为会话级激活落库。 - invoked_skill_ids=_explicit_skill_ids_from_context(context), - invoked_mcp_ids=[m for m in (context.get("mcp_ids") or []) if isinstance(m, str)], - required_mcp_ids=( - [str(context["connector_id"])] if context.get("connector_id") else None - ), - required_skill_id=str(context.get("skill_id") or "") or None, - required_skill_name=str(context.get("skill_name") or "") or None, - required_plugin_id=str(context.get("plugin_id") or "") or None, - required_plugin_name=str(context.get("plugin_name") or "") or None, - required_plugin_skill_ids=[ - str(item) - for item in (context.get("plugin_skill_ids") or []) - if isinstance(item, str) and item - ], - required_plugin_mcp_ids=[ - str(item) - for item in (context.get("plugin_mcp_ids") or []) - if isinstance(item, str) and item - ], - memory_enabled=_workflow_mem_enabled, - workflow_mode=bool(context.get("workflow_chat", False)), - user_agent=_direct_user_agent, - read_only=_direct_read_only, - allow_bash=_direct_allow_bash, - approval_mode=_approval_mode(context), - visible_subagents=_visible_subagents if _visible_subagents else None, - chat_id=context.get("chat_id"), - sandbox_session_id=context.get("sandbox_session_id"), - run_id=str(context.get("run_id") or "") or None, - journal_owner=str(context.get("journal_owner") or "") or None, - workspace_id=str(context.get("workspace_id") or "default"), - project_ctx=_extract_project_ctx(context), - channel_origin=context.get("channel_origin"), - automation_run=bool(context.get("automation_run")), - ontology_runtime=context.get("ontology_runtime"), - ) - try: - # AgentScope 2.0: ctx → agent.state (AgentRuntimeState), replacing _jx_context. - # Uploaded/history files are written into state by - # apply_request_context, then injected uniformly at reply time by - # FileContextMiddleware (not hand-copied here). - try: - agent.state.apply_request_context(context, user_message or "") - except Exception as exc: # noqa: BLE001 - logger.warning("[workflow] set agent.state failed: %s", exc) - - _actual_model, _ctx_window = _resolve_agent_model_runtime(agent, _workflow_model_name) - - # PreTurn compaction safety net (symmetric with the streaming path - # — both workflow entry points protect themselves, and future new - # callers get it automatically). Zero overhead below the threshold. - _pt_messages = session_messages - try: - from core.services.compaction_service import maybe_run_pre_turn_compaction - - _pt_messages, _ = await maybe_run_pre_turn_compaction( - context.get("chat_id"), - session_messages, - run_id=str(context.get("run_id") or ""), - **_pre_turn_compaction_inputs(agent, _ctx_window, _actual_model), - ) - except Exception as _pt_exc: # noqa: BLE001 - from core.harness.hooks import HookPaused - - if isinstance(_pt_exc, HookPaused): - raise - logger.warning("[workflow] pre-turn compaction failed: %s", _pt_exc) - - # Load history EXCLUDING the last user message — reply() adds it. - history = list(_pt_messages) - if history and history[-1].get("role") in ("user", "human"): - history.pop() - - if history: - agent.state.context.extend(session_to_msgs(history)) - - # Uploaded-file context is injected uniformly at reply time by - # FileContextMiddleware (apply_request_context already wrote - # uploaded_files into state; the middleware validates attachment - # ownership by state.user_id); no manual append here, otherwise it - # would duplicate the middleware's injection and waste tokens. - - model_user_message = user_message or "" - _delegated_agent_id = _explicit_agent_id or _mention_agent_id - if _delegated_agent_id and _visible_subagents: - from core.llm.subagent_tool import build_explicit_subagent_command_hint - - explicit_hint = build_explicit_subagent_command_hint( - _visible_subagents, - _delegated_agent_id, - ) - if explicit_hint: - append_context_text( - agent, - explicit_hint, - kind=KIND_REMINDER, - origin="harness:subagent_routing", - trust="system", - priority=950, - token_budget=4_000, - truncation_policy=POLICY_NEVER, - ) - request_seq = next_request_sequence(agent.state.context) - user_msg = render_context_item( - make_text_context_item( - model_user_message, - item_id=f"request:user_input:{request_seq}", - kind=KIND_USER_INPUT, - origin="user:chat", - trust="user", - created_seq=request_seq, - priority=1_000, - token_budget=100_000, - truncation_policy=POLICY_NEVER, - ) - ) - result = await agent.reply(inputs=user_msg) - response = result.get_text_content() or "" - ontology_runtime = context.get("ontology_runtime") - if not isinstance(ontology_runtime, dict): - ontology_runtime = {} - context["ontology_runtime"] = ontology_runtime - review_owner = _ontology_review_owner(ontology_runtime, context) - review_claimed = bool( - response and claim_output_review(ontology_runtime, owner=review_owner) - ) - if review_claimed: - from orchestration.subagents.ontology_reviewer import review_ontology_output - - async def _remediate(payload: Dict[str, Any]) -> str: - repair_seq = next_request_sequence(agent.state.context) - repair_msg = render_context_item( - make_text_context_item( - _ontology_repair_prompt(payload), - item_id=f"request:ontology_repair:{repair_seq}", - kind=KIND_REMINDER, - origin="harness:ontology_repair", - trust="system", - created_seq=repair_seq, - priority=900, - token_budget=4_000, - truncation_policy=POLICY_HEAD_TAIL, - ) - ) - repaired_result = await agent.reply(inputs=repair_msg) - return (repaired_result.get_text_content() or "").strip() - - try: - review = await review_ontology_output( - task=user_message, - answer=response, - runtime=ontology_runtime, - trace=[], - citations=[], - user_id=_workflow_user_id, - chat_id=context.get("chat_id"), - model_name=_workflow_model_name or None, - model_provider_id=_workflow_model_provider_id or None, - trace_complete=False, - remediate=_remediate, - ) - except BaseException: - release_output_review(ontology_runtime, owner=review_owner) - raise - complete_output_review( - ontology_runtime, - owner=review_owner, - verdict=str(review.get("verdict") or "unknown"), - attempts=int(review.get("attempts") or 1), - ) - ontology_runtime.setdefault("output_review", {}).update( - { - "revised": bool(review.get("revised")), - "annotated": bool(review.get("annotated")), - "repair_attempts": int(review.get("repair_attempts") or 0), - "violations": review.get("violations") or [], - "affected_claims": review.get("affected_claims") or [], - "latency_ms": review.get("latency_ms"), - } - ) - response = review["answer"] - return response - finally: - await close_clients(mcp_clients) - - try: - import asyncio as _asyncio - - response = _asyncio.run(_run()) - except Exception as e: - warnings.append(f"Agent execution error: {str(e)[:200]}") - response = "" - - return WorkflowResult( - route=f"subagent:{_direct_agent_id}" if _direct_agent_id else "main", - response=response, - is_markdown=_looks_markdown(response), - sources=_resolve_sources_conflict([]), - artifacts=[], - warnings=warnings, - meta={ - "ontology_governance": _ontology_governance_summary( - context.get("ontology_runtime") - if isinstance(context.get("ontology_runtime"), dict) - else {} - ) - }, - ) - - -# ------------------------------------------------------------------ -# Sub-agent direct conversation -# ------------------------------------------------------------------ - - -async def _astream_subagent_direct( - *, - agent_id: str, - session_messages: List[Dict[str, Any]], - user_message: str, - context: Dict[str, Any], -) -> AsyncIterator[Dict[str, Any]]: - """Stream a direct conversation with a user-created sub-agent. - - Loads the UserAgent config from DB and uses it to build the agent - with custom system_prompt, MCP tools, skills, KB, and model params. - Shares the same streaming/memory/citation infrastructure as the main route. - """ - from core.llm.agent_api_runtime import apply_api_scope - from core.llm.evaluation_runtime import apply_evaluation_scope - context = apply_evaluation_scope(apply_api_scope(context)) - - import time as _time - - _wf_start = _time.monotonic() - _direct_read_only, _direct_allow_bash = _direct_agent_execution_permissions(agent_id) - - user_agent = _load_direct_user_agent( - agent_id, - str(context.get("user_id", "")), - { - "enabled_skill_ids": enabled_skill_ids_from_context(context) or [], - "enabled_mcp_ids": enabled_mcp_ids_from_context(context) or [], - "enabled_kb_ids": enabled_kb_ids_from_context(context) or [], - "sandbox_tools_enabled": bool(context.get("sandbox_tools_enabled", False)), - "code_capability_enabled": bool(context.get("code_capability_enabled", False)), - }, - ) - from core.services import log_service as log_writer - - _direct_log_started = _time.monotonic() - _direct_tool_count = 0 - _direct_skill_count = 0 - _direct_log_finished = False - _is_call_subagent_dispatch = context.get("direct_agent_source") == "explicit_command_tool" - import uuid - from core.infra.task_lifecycle import cancel_and_join, settle_task - - _direct_log_id = uuid.uuid4().hex - _direct_finish_task = None - _direct_start_task = asyncio.create_task( - log_writer.start_subagent_log( - { - "id": _direct_log_id, - "user_id": str(context.get("user_id", "")) or None, - "chat_id": context.get("chat_id"), - "subagent_name": user_agent.name, - "subagent_type": ( - "user_agent" if _is_call_subagent_dispatch else "user_agent_direct" - ), - "subagent_id": agent_id, - "input_messages": { - "task": user_message, - "invocation": ( - "call_subagent" - if _is_call_subagent_dispatch - else context.get("direct_agent_source") or "dedicated_chat" - ), - }, - } - ) - ) - - async def _finish_direct_log( - status: str, - *, - output: str = "", - error: Optional[str] = None, - usage: Optional[Dict[str, int]] = None, - ) -> None: - nonlocal _direct_log_finished, _direct_finish_task - - async def record(): - await _direct_start_task - await log_writer.finish_subagent_log( - _direct_log_id, - status=status, - output_content=output or None, - token_usage=usage, - tool_calls_count=_direct_tool_count, - skill_calls_count=_direct_skill_count, - error_message=error, - duration_ms=int((_time.monotonic() - _direct_log_started) * 1000), - ) - - if _direct_finish_task is None: - _direct_finish_task = asyncio.create_task(record()) - await asyncio.shield(_direct_finish_task) - _direct_log_finished = True - - _direct_exit_status = "cancelled" - _direct_exit_error = "direct agent execution interrupted" - try: - await asyncio.shield(_direct_start_task) - _ontology_runtime = context.get("ontology_runtime") - if not isinstance(_ontology_runtime, dict): - _ontology_runtime = {} - context["ontology_runtime"] = _ontology_runtime - activations = activate_runtime_for_asset( - _ontology_runtime, - kind="subagent", - asset_id=agent_id, - tags=list((user_agent.extra_config or {}).get("ontology_tags") or []), - ) - if activations: - await asyncio.to_thread( - _record_ontology_activations, - activations, - _ontology_runtime, - context, - ) - - # ── [memory] Non-blocking retrieval: background task, skipped on budget timeout ─── - _mem0_user_id = str(context.get("user_id", "")) - _mem0_workspace_id = str(context.get("workspace_id", "") or "default") - _mem0_chat_id = context.get("chat_id") or context.get("conversation_id") - _mem0_enabled = bool(context.get("memory_enabled", False)) - _mem0_write_enabled = bool(context.get("memory_write_enabled", False)) - # Under a team project chats.py passes "team:"; personal/default spaces don't pass it and fall back to the real user_id - _mem0_scope_user_id = str(context.get("memory_scope_user_id", "") or _mem0_user_id) - logger.info( - "[subagent] user=%s scope=%s ws=%s agent=%s enabled=%s", - _mem0_user_id, - _mem0_scope_user_id, - _mem0_workspace_id, - agent_id, - _mem0_enabled, - ) - - _session_memory = await open_session_memory( - chat_id=_mem0_chat_id, - scope_user_id=_mem0_scope_user_id, - workspace_id=_mem0_workspace_id, - user_message=user_message, - memory_enabled=_mem0_enabled, - # The retrieval budget is part of the orchestration profile: how long - # this task type is willing to wait for memory is an assembly decision, - # not a global constant. Resolved here because retrieval starts before - # the agent is built. - budget_ms=_profile_memory_budget_ms(context), - ) - - warnings: List[str] = [] - full_response = "" - displayed_tools: set[str] = set() - all_citations: List[Dict[str, Any]] = [] - _ontology_event_cursor = 0 - _ontology_trace: List[Dict[str, Any]] = [] - # 证据锚点发号器：跨轮续号；创建后绑到 agent 上（见下方 attach_allocator）， - # 中间件与本函数由此共享同一个计数器 - _anchor_allocator = AnchorAllocator( - await asyncio.to_thread( - anchor_start_for_chat, str(context.get("chat_id") or "") or None - ) - ) - - try: - yield {"type": "thinking", "message": "正在连接子智能体..."} - for ontology_event in _ontology_runtime.get("runtime_events", []): - yield dict(ontology_event) - _ontology_event_cursor = len(_ontology_runtime.get("runtime_events", [])) - - _stream_user_id = str(context.get("user_id", "")) - _stream_model_name = str(context.get("model_name", "")) - _stream_model_provider_id = str(context.get("model_provider_id", "") or "") - _stream_chat_mode = str(context.get("chat_mode", "") or "") - _stream_reranker = bool(context.get("reranker_enabled", False)) - - # Create agent with sub-agent config overrides - agent, mcp_clients = await create_agent_executor( - agent_spec=None, - enabled_mcp_ids=None, # overridden by user_agent inside factory - enabled_skill_ids=None, # overridden by user_agent inside factory - enabled_kb_ids=None, # overridden by user_agent inside factory - current_user_id=_stream_user_id, - reranker_enabled=_stream_reranker, - model_name=_stream_model_name, - model_provider_id=_stream_model_provider_id, - chat_mode=_stream_chat_mode, - memory_enabled=_mem0_enabled, - agent_api_scope=context.get("agent_api_scope"), - user_agent=user_agent, - read_only=_direct_read_only, - allow_bash=_direct_allow_bash, - approval_mode=_approval_mode(context), - required_skill_id=str(context.get("skill_id") or "") or None, - required_skill_name=str(context.get("skill_name") or "") or None, - required_plugin_id=str(context.get("plugin_id") or "") or None, - required_plugin_name=str(context.get("plugin_name") or "") or None, - required_plugin_skill_ids=[ - str(item) - for item in (context.get("plugin_skill_ids") or []) - if isinstance(item, str) and item - ], - required_plugin_mcp_ids=[ - str(item) - for item in (context.get("plugin_mcp_ids") or []) - if isinstance(item, str) and item - ], - # Same as the main agent: pass chat_id → the sandbox session uses - # the chat_id-keyed "user-bound persistent sandbox" (mounting the - # per-user credential volumes for lark/dws/email etc.). Omitting it - # falls back to an ephemeral light sandbox (no credentials) — the - # root cause of Feishu/DingTalk CLIs reporting "not configured" in - # direct sub-agent conversations. - chat_id=context.get("chat_id"), - sandbox_session_id=context.get("sandbox_session_id"), - run_id=str(context.get("run_id") or "") or None, - journal_owner=str(context.get("journal_owner") or "") or None, - workspace_id=_mem0_workspace_id, - project_ctx=_extract_project_ctx(context), - channel_origin=context.get("channel_origin"), - automation_run=bool(context.get("automation_run")), - ontology_runtime=context.get("ontology_runtime"), - ) - - logger.info( - "[subagent] agent created in %.0fms", (_time.monotonic() - _wf_start) * 1000 - ) - _, _ctx_window = _resolve_agent_model_runtime(agent, _stream_model_name) - - # 证据锚点：把发号器绑到 agent 上（中间件与本函数由此共享同一个计数器； - # 仅靠 ContextVar 不行——本函数是 async generator，与 agent 执行所在的 - # task 上下文不互通） - attach_allocator(agent, _anchor_allocator) - - # ── Session-constant blocks: user identity + the chat's frozen memory - # snapshot (built on the first turn, replayed byte-for-byte after). ── - session_messages = await inject_session_blocks( - session_messages, - identity_block=("" if context.get("agent_api_scope") or context.get("evaluation_session") - else await build_user_identity_block(_mem0_user_id)), - memory_block=await resolve_session_memory(_session_memory), - ) - - # The canonical ContextAssembler owns selection and truncation. Feed - # it the complete candidate set so every exclusion is visible in the - # request manifest rather than silently pre-trimming shared history. - streaming_agent = StreamingAgent(agent, mcp_clients) - skill_load_ids: set = set() - # tool_id → skill_id (parsed from the tool_call's file_path; looked up at tool_result time to replace the SSE payload with the curated detail, avoiding sending the full SKILL.md text to the frontend) - skill_id_by_tool_id: Dict[str, str] = {} - # tool_id → tool_args, used at the tool_result stage to recover view_text_file's file_path/ranges - view_text_file_args: Dict[str, Dict[str, Any]] = {} - - # Project scope is no longer passed via ContextVar — it now travels as - # explicit parameters along the call chain (agent_factory closes - # ProjectScope into every register_* tool; chats.py's finishing - # _persist_artifacts reconstructs the scope from the workflow context - # and passes it explicitly). See the header comment in - # core/services/project_scope.py. - - # Never keep a ContextVar token alive across an async-generator - # ``yield``. The consumer may resume this generator in a copied - # context, making reset(token) fail with "created in a different - # Context". Enter the audit scope only while advancing the underlying - # agent stream; tool execution and persistence happen during anext(). - _direct_stream = streaming_agent.stream(session_messages, context).__aiter__() - try: - while True: - try: - event_type, payload = await _anext_in_subagent_log_scope( - _direct_stream, - _direct_log_id, - ) - except StopAsyncIteration: - break - state_runtime = getattr(streaming_agent.agent.state, "ontology_runtime", None) - if isinstance(state_runtime, dict): - _ontology_runtime = state_runtime - context["ontology_runtime"] = state_runtime - pending_ontology_events = _ontology_runtime.get("runtime_events", [])[ - _ontology_event_cursor: - ] - for ontology_event in pending_ontology_events: - yield dict(ontology_event) - _ontology_event_cursor += len(pending_ontology_events) - if event_type == "text_delta": - full_response += payload - yield {"type": "content", "event": "ai_message", "delta": payload} - - elif event_type == "vision_progress": - # 识图是模型开口前的一段纯网络等待；透传给前端，让轮级状态 - # 从「深度拥抱中」换成「图像理解中」，而不是干等一个笼统的计时器。 - yield {"type": "vision_progress", **(payload or {})} - - elif event_type == "reasoning_protocol": - yield {"type": "thinking", **payload} - - elif event_type == "steer_applied": - yield {"type": "steer_applied", **(payload or {})} - - elif event_type == "thinking_delta": - yield {"type": "thinking", "delta": payload} - - elif event_type == "context_usage": - yield {"type": "context_usage", **(payload or {})} - - elif event_type == "model_step": - yield {"type": "model_step", "step": payload} - - elif event_type == "model_call_start": - yield {"type": "model_dispatch"} - - elif event_type == "tool_call_start": - tool_name = payload.get("name", "unknown") - if tool_name != "update_plan": - yield { - "type": "tool_call_start", - "tool_name": tool_name, - "tool_display_name": TOOL_DISPLAY_NAMES.get(tool_name, tool_name), - "tool_id": payload.get("id", ""), - } - - elif event_type == "tool_call_delta": - tool_name = payload.get("name", "unknown") - if tool_name != "update_plan" and payload.get("delta"): - yield { - "type": "tool_call_delta", - "tool_name": tool_name, - "tool_id": payload.get("id", ""), - "arguments_delta": payload.get("delta", ""), - } - - elif event_type == "tool_call": - tool_name = payload.get("name", "unknown") - tool_id = payload.get("id", "") - tool_args = payload.get("args", {}) - - _is_fast_emit = tool_name in _FAST_EMIT_TOOLS - if tool_id and tool_id in displayed_tools: - if _is_fast_emit and tool_args: - pass # re-emit with updated args - else: - continue - if not _tool_args_ready(tool_name, tool_args): - continue - if tool_id: - displayed_tools.add(tool_id) - - is_skill_load = ( - tool_name == "view_text_file" - and isinstance(tool_args, dict) - and "SKILL.md" in str(tool_args.get("file_path", "")) - ) - if is_skill_load and tool_id: - skill_load_ids.add(tool_id) - _sid = _extract_skill_id_from_path(str(tool_args.get("file_path", ""))) - if _sid: - skill_id_by_tool_id[tool_id] = _sid - # Non-skill view_text_file also needs trimming at the tool_result stage → record the args - if ( - tool_name in ("view_text_file", "Read") - and not is_skill_load - and tool_id - and isinstance(tool_args, dict) - ): - view_text_file_args[tool_id] = tool_args - emit_name = "load_skill" if is_skill_load else tool_name - display_name = ( - "加载技能" - if is_skill_load - else TOOL_DISPLAY_NAMES.get(tool_name, tool_name) - ) - safe_args = tool_args if isinstance(tool_args, dict) else {} - _ontology_trace.append( - { - "type": "tool_call", - "tool_id": tool_id, - "tool_name": tool_name, - "input": _bounded_trace_value(safe_args), - } - ) - - yield { - "type": "tool_call", - "tool_name": emit_name, - "tool_display_name": display_name, - "tool_args": safe_args, - "input": safe_args, - "tool_id": tool_id, - } - - elif event_type == "tool_result": - tool_name = payload.get("name", "unknown") - tool_id = payload.get("id", "") - is_skill_result = (tool_id and tool_id in skill_load_ids) or ( - tool_name == "view_text_file" - and "SKILL.md" in str(payload.get("content", "")) - ) - if is_skill_result: - _direct_skill_count += 1 - tool_name = "load_skill" - else: - _direct_tool_count += 1 - - _missing_call = _synthesize_missing_tool_call( - tool_id, tool_name, displayed_tools - ) - if _missing_call is not None: - _ontology_trace.append( - { - "type": "tool_call", - "tool_id": tool_id, - "tool_name": tool_name, - "input": {}, - } - ) - yield _missing_call - - tool_content = payload.get("content", "") - - try: - tool_result_json = json.loads(tool_content) if tool_content else {} - except json.JSONDecodeError: - tool_result_json = {"result": tool_content} - - # Skill load: replace the full SKILL.md text with the same - # curated detail used by the capability center; affects - # only the SSE payload sent to the frontend — the agent's - # own memory still holds the full content - if is_skill_result: - _sid = skill_id_by_tool_id.get( - tool_id, "" - ) or _extract_skill_id_from_path(str(tool_content)) - tool_result_json = _build_skill_load_payload(_sid) - elif tool_name == "view_text_file": - # Plain file read (AgentScope built-in view_text_file): replace with file metadata + short preview - tool_result_json = _build_view_text_file_payload( - view_text_file_args.get(tool_id, {}), tool_content - ) - elif tool_name == "Read": - # Claude-Code-style Read tool: JSON payload, content holds the whole file - tool_result_json = _build_read_tool_payload( - view_text_file_args.get(tool_id, {}), tool_result_json - ) - elif tool_name == "read_artifact": - tool_result_json = _build_read_artifact_payload(tool_result_json) - - extracted_query = "" - if isinstance(tool_result_json, dict) and "result" in tool_result_json: - result_data = tool_result_json["result"] - if isinstance(result_data, dict): - extracted_query = result_data.get( - "query", result_data.get("question", "") - ) - - cit_dicts = collect_citation_dicts(tool_id, _anchor_allocator) - all_citations.extend(cit_dicts) - _ontology_trace.append( - { - "type": "tool_result", - "tool_id": tool_id, - "tool_name": tool_name, - "result": _bounded_trace_value(tool_result_json), - "citations": cit_dicts, - } - ) - - yield { - "type": "tool_result", - "tool_name": tool_name, - "tool_args": {"query": extracted_query} if extracted_query else {}, - "result": tool_result_json, - "tool_id": tool_id, - "citations": cit_dicts, - "status": payload.get("status", "success"), - "model_step": payload.get("model_step"), - } - - elif event_type in ("heartbeat", "model_progress"): - # heartbeat = transport keep-alive; model_progress = the - # model is still streaming while a small argument batch or - # another suppressed event has nothing renderable yet — - # forwarded so the run watchdog counts activity (it - # excludes only heartbeat). - yield {"type": event_type} - - elif event_type == "tool_pending": - yield {"type": "tool_pending", **(payload or {})} - - elif event_type == "subagent_event": - # Bypass channel for the sub-agent's internal - # thinking/tool_call/tool_result/content — attached under - # the call_subagent tool card that launched it (linked via - # parent_tool_id). - if (payload or {}).get("sub_type") == "progress": - # Sub-agent liveness (long tool-call-arg generation with - # nothing renderable yet) — feed the run inactivity - # watchdog without writing a sub-step to the stream or - # the persisted tool log. - yield {"type": "model_progress"} - else: - nested_citations = _capture_nested_ontology_evidence( - payload or {}, - _ontology_trace, - all_citations, - _anchor_allocator, - ) - yield { - "type": "subagent_event", - **(payload or {}), - **({"citations": nested_citations} if nested_citations else {}), - } - - elif event_type in ( - "file_confirm", - "design_pick", - "user_question", - "user_question_resolved", - ): - # Human-interaction events (§13 MySpace write confirm / - # site-design pick-one-of-three / ask_user_question): a tool - # coroutine has suspended waiting for the user's out-of-band - # action. Pass through to the frontend to show the card; the - # agent task stays blocked in that tool and this SSE stream - # does not end — after the out-of-band answer POST the tool - # resumes in place. - yield {"type": event_type, **(payload or {})} - - elif event_type == "error": - # payload may be a real exception object (kind=="err") or a - # dict (e.g. ExceedMaxIters mapped to {"kind":..,"name":..}). - # Raising the latter directly gives a TypeError, masking - # the real situation — wrap it in a RuntimeError. - if isinstance(payload, BaseException): - raise payload - raise RuntimeError(str(payload)) - - except BaseException: - # Keep clients alive after a normal first draft so the same agent - # can continue its ReAct context during ontology repair. - raise - - except Exception as e: - import traceback - - logger.error("subagent_stream_error: %s\n%s", e, traceback.format_exc()) - warnings.append(f"Streaming error: {str(e)[:200]}") - - unknown_outcome = find_tool_outcome_unknown(e) - if unknown_outcome is not None: - if "streaming_agent" in locals(): - await streaming_agent.shutdown() - _persistent_clients.append((streaming_agent, list(mcp_clients))) - await _finish_direct_log("failed", error=str(unknown_outcome)[:200]) - if unknown_outcome is e: - raise - raise unknown_outcome from e - - if not full_response: - # No answer was produced: the caller gets the failure itself. Tool - # results already streamed stay visible; the run is recorded as - # failed with a structured error instead of a fabricated reply that - # would read as a completed answer. - if "streaming_agent" in locals(): - await streaming_agent.shutdown() - _persistent_clients.append((streaming_agent, list(mcp_clients))) - await _finish_direct_log("failed", error=str(e)[:200]) - raise - - pending_ontology_events = _ontology_runtime.get("runtime_events", [])[ - _ontology_event_cursor: - ] - for ontology_event in pending_ontology_events: - yield dict(ontology_event) - _ontology_event_cursor += len(pending_ontology_events) - - _ontology_review_owner_id = _ontology_review_owner(_ontology_runtime, context) - _ontology_review_claimed = bool( - full_response - and claim_output_review(_ontology_runtime, owner=_ontology_review_owner_id) - ) - if _ontology_review_claimed: - from orchestration.subagents.ontology_reviewer import review_ontology_output - - yield { - "type": "ontology_review", - "status": "started", - "level": _ontology_runtime.get("review_level", "checkpoint"), - **_ontology_review_event_context(_ontology_runtime), - } - repair_event_queue: asyncio.Queue[Dict[str, Any]] = asyncio.Queue() - - async def _remediate(payload: Dict[str, Any]) -> str: - nonlocal _ontology_event_cursor, _direct_tool_count - primary_usage = await streaming_agent.aget_usage() - primary_context_usage = streaming_agent.get_context_usage(primary_usage) - try: - repaired, _, cursor, tool_count = await _run_ontology_repair_round( - streaming_agent=streaming_agent, - context=context, - payload=payload, - runtime=_ontology_runtime, - trace=_ontology_trace, - citations=all_citations, - allocator=_anchor_allocator, - event_cursor=_ontology_event_cursor, - subagent_log_id=_direct_log_id, - event_sink=repair_event_queue.put, - ) - finally: - streaming_agent.restore_context_usage(primary_context_usage) - _ontology_event_cursor = cursor - _direct_tool_count += tool_count - return repaired - - review_task = asyncio.create_task( - review_ontology_output( - task=user_message, - answer=full_response, - runtime=_ontology_runtime, - trace=_ontology_trace, - citations=all_citations, - user_id=str(context.get("user_id", "")), - chat_id=context.get("chat_id"), - model_name=str(context.get("model_name", "") or "") or None, - model_provider_id=str(context.get("model_provider_id", "") or "") or None, - remediate=_remediate, - ) - ) - try: - while not review_task.done() or not repair_event_queue.empty(): - try: - repair_event = await asyncio.wait_for( - repair_event_queue.get(), - timeout=0.1, - ) - except asyncio.TimeoutError: - continue - yield repair_event - review = await review_task - except (asyncio.CancelledError, GeneratorExit): - await cancel_and_join(review_task) - release_output_review(_ontology_runtime, owner=_ontology_review_owner_id) - await streaming_agent.shutdown() - _persistent_clients.append((streaming_agent, list(mcp_clients))) - raise - except Exception as exc: # noqa: BLE001 - if not review_task.done(): - review_task.cancel() - review = _ontology_review_failure_result(full_response, exc) - complete_output_review( - _ontology_runtime, - owner=_ontology_review_owner_id, - verdict=str(review.get("verdict") or "unknown"), - attempts=int(review.get("attempts") or 1), - ) - _ontology_runtime.setdefault("output_review", {}).update( - { - "revised": bool(review.get("revised")), - "annotated": bool(review.get("annotated")), - "repair_attempts": int(review.get("repair_attempts") or 0), - "violations": review.get("violations") or [], - "affected_claims": review.get("affected_claims") or [], - "evidence": review.get("evidence") or [], - "feedback": review.get("feedback") or [], - "manual_review": review.get("manual_review") or {}, - "candidate_answer": (review.get("answer") if review.get("revised") else ""), - "new_tools": review.get("new_tools") or [], - "new_citation_count": int(review.get("new_citation_count") or 0), - "latency_ms": review.get("latency_ms"), - } - ) - yield { - "type": "ontology_review", - "status": "completed", - "level": _ontology_runtime.get("review_level", "checkpoint"), - "verdict": review["verdict"], - "revised": bool(review.get("revised")), - "annotated": bool(review.get("annotated")), - "repair_attempts": int(review.get("repair_attempts") or 0), - "candidate_answer": review.get("answer") if review.get("revised") else "", - "manual_review": review.get("manual_review") or {}, - "violations": review.get("violations") or [], - "affected_claims": review.get("affected_claims") or [], - "evidence": review.get("evidence") or [], - "feedback": review.get("feedback") or [], - "new_tools": review.get("new_tools") or [], - "new_citation_count": int(review.get("new_citation_count") or 0), - "latency_ms": review.get("latency_ms"), - **_ontology_review_event_context(_ontology_runtime), - } - - _direct_usage = await streaming_agent.aget_usage() - _direct_context_usage = streaming_agent.get_context_usage(_direct_usage) - await streaming_agent.shutdown() - _persistent_clients.append((streaming_agent, list(mcp_clients))) - await _finish_direct_log( - "success", - output=full_response, - usage=_direct_usage, - ) - yield { - "type": "meta", - "route": f"subagent:{agent_id}", - "is_markdown": _looks_markdown(full_response), - "sources": _resolve_sources_conflict([]), - "artifacts": [], - "warnings": warnings, - "citations": all_citations, - "usage": _direct_usage, - "context_usage": _direct_context_usage, - # Internal-only handoff consumed by chat_run_executor. Its whitelist - # intentionally keeps this prompt/tool surface out of SSE and message - # metadata while allowing post-turn compaction to use the exact same - # frozen inputs as pre-turn compaction. - "_compaction_budget_inputs": _compaction_budget_inputs(agent, _ctx_window), - "ontology_governance": _ontology_governance_summary(_ontology_runtime), - # Same contract as the main route: memory writes are scheduled below, - # so the frame can only announce "settling" for the client's skeleton - # card. Without this marker subagent turns never showed their card. - "evolution_pending": _evolution_pending_marker(context, _mem0_write_enabled), - } - - # ── [memory] Post-response pipeline (SSE already closed, user isn't waiting) ──── - # No memory is written unless the user opted into memory_write_enabled (first gate). - if _mem0_write_enabled: - save_memories_background( - _mem0_user_id, - user_message, - full_response, - _mem0_write_enabled, - workspace_id=_mem0_workspace_id, - chat_id=_mem0_chat_id, - scope_user_id=_mem0_scope_user_id, - message_id=str(context.get("message_id") or ""), - ) - else: - logger.debug( - "[subagent] memory save skipped: write_enabled=False (user=%s)", - _mem0_user_id, - ) - - # ── [evolution] Evidence assembly — same contract as the main route. Without - # registration the settlement runner parks the memory report on a 45s - # watchdog before settling, which pushed subagent cards past the client's - # polling budget. - if not context.get("agent_api_scope") and not context.get("evaluation_session"): - _assemble_episode_background( - message_id=str(context.get("message_id") or ""), - run_id=str(context.get("run_id") or ""), - chat_id=str(context.get("chat_id") or ""), - user_id=str(context.get("user_id") or ""), - objective=user_message, - agent=streaming_agent, - memory_task=_session_memory.retrieval_task, - latency_ms=None, - memory_write_enabled=_mem0_write_enabled, - ) - except BaseException as exc: - if not isinstance(exc, (asyncio.CancelledError, GeneratorExit)): - _direct_exit_status = "failed" - _direct_exit_error = str(exc)[:200] - raise - finally: - try: - if "review_task" in locals(): - await cancel_and_join(review_task) - release_output_review(_ontology_runtime, owner=_ontology_review_owner_id) - if "_direct_stream" in locals(): - await _direct_stream.aclose() - if not _direct_log_finished and "streaming_agent" in locals(): - await streaming_agent.shutdown() - finally: - if not _direct_log_finished: - await settle_task( - asyncio.create_task( - _finish_direct_log(_direct_exit_status, error=_direct_exit_error) - ) - ) - - -# ------------------------------------------------------------------ -# Streaming workflow -# ------------------------------------------------------------------ - - -def _evolution_pending_marker(context: Dict[str, Any], memory_write_enabled: bool): - """Marker for the closing frame so the client can draw the skeleton card. - - The turn's real outcome is not knowable yet — memory writes are scheduled - after the stream closes — so all we can honestly say here is "settling, and - these are the mechanisms that might report". - """ - message_id = str(context.get("message_id") or "") - if not message_id or not memory_write_enabled: - # Without memory writes there is nothing a turn can settle to, so there - # is nothing for the client to watch. - return None - try: - from core.evolution.settlement import pending_payload - - return pending_payload(message_id=message_id) - except Exception: # pragma: no cover - defensive - return None - - -def _runtime_evidence_agent(agent: Any) -> Any: - """Unwrap streaming/decorator agents to the owner of frozen run evidence.""" - current = agent - seen: set[int] = set() - while current is not None and id(current) not in seen: - seen.add(id(current)) - if hasattr(current, "_jx_asset_bundle") or hasattr(current, "_jx_skill_selection"): - return current - nested = getattr(current, "agent", None) - if nested is None or nested is current: - break - current = nested - return current - - -def _assemble_episode_background( - *, - message_id: str, - run_id: str, - chat_id: str, - user_id: str, - objective: str, - agent: Any = None, - memory_task: Any = None, - latency_ms: Optional[int] = None, - memory_write_enabled: bool = False, -) -> None: - """Kick off Episode assembly after the stream closed. - - Fire-and-forget by design: the user already has their answer, and evidence - assembly must never be able to delay or fail a turn. Every failure mode - here degrades to "no Episode for this run", which the console shows - honestly rather than papering over. - """ - if not message_id: - return - - try: - from core.evolution.events import EV_MEMORY_RETRIEVED, TraceSink - from core.evolution.trace_assembler import assemble_episode - from orchestration.memory_integration import get_last_retrieval - - evidence_agent = _runtime_evidence_agent(agent) - bundle = ( - getattr( - evidence_agent, - "evidence_bundle", - getattr(evidence_agent, "_jx_asset_bundle", None), - ) - if evidence_agent is not None - else None - ) - - sink = TraceSink(run_id=run_id, message_id=message_id, chat_id=chat_id, user_id=user_id) - # The rendered memory block has already discarded ids, scores and ranks; - # this is the only place they still exist. - retrieval = get_last_retrieval(memory_task) - if retrieval is not None: - sink.append( - EV_MEMORY_RETRIEVED, - retrieval.to_event_payload(), - asset_kind="memory", - ) - - # Which skills were considered, chosen, and rejected — the rejects are - # the negative examples skill attribution is built on. - # Ordered tool calls, read back from the log this run already wrote. - # Without them an episode carries no tool sequence and the promotion - # chain has nothing to distil. - from core.evolution.trace_assembler import emit_tool_events - - emit_tool_events(sink, message_id, chat_id) - - selection = ( - getattr(evidence_agent, "_jx_skill_selection", None) - if evidence_agent is not None - else None - ) - if selection is not None: - from core.evolution.events import EV_SKILL_SELECTED - - sink.append(EV_SKILL_SELECTED, selection.to_event_payload(), asset_kind="skill") - sink.flush() - - # Stamp the user's contribution choice onto the episode. Doing it at - # write time means the decision is fixed by what the user had agreed to - # *at that moment*, rather than being re-evaluated later against a - # setting they may since have changed. - from core.evolution.user_settings import resolve_for_user - - privacy_class = resolve_for_user(user_id).get("privacy_class", "tenant") - - episode_id = assemble_episode( - message_id=message_id, - run_id=run_id, - chat_id=chat_id, - user_id=user_id, - objective=objective, - bundle=bundle, - latency_ms=latency_ms, - privacy_class=privacy_class, - ) - - # Hand the evidence side to the settlement runner. It settles once the - # memory write pipeline reports what it actually wrote — the only party - # that knows. Settling here instead would have to guess that number, and - # the guess it used to make (the *retrieval* count) described what the - # turn read, not what it learned. - from core.evolution.settlement_runner import register_turn - - # Which skills the turn opened and which profile governed it stay on the - # episode, where the retirement engine reads them. They are not on the - # card: they describe the system doing its job, not learning anything. - register_turn( - message_id=message_id, - episode_id=episode_id or "", - expects_memory=memory_write_enabled, - ) - - # Consider running a cycle now that new evidence exists. The check is a - # single indexed count and almost always says no; when it says yes the - # cycle runs off-thread. The user already has their answer either way. - from core.evolution.trigger import schedule_if_due, schedule_personal - - # This user's own loop runs in every edition; the fleet-wide one is - # enterprise-gated and no-ops elsewhere. - schedule_personal(user_id) - schedule_if_due() - except Exception as exc: # pragma: no cover - defensive - logger.debug("[evolution] episode assembly skipped: %s", exc) - - -async def astream_chat_workflow( - *, - session_messages: List[Dict[str, Any]], - user_message: str, - context: Dict[str, Any], -): - """Stream route -> handoff -> target execution. - - Yields chunks in the format: - - {"type": "content", "delta": "text chunk"} - - {"type": "tool_call", ...} - - {"type": "tool_result", ...} - - {"type": "meta", "route": "...", "sources": [...], ...} - """ - - from core.llm.agent_api_runtime import apply_api_scope - from core.llm.evaluation_runtime import apply_evaluation_scope - context = apply_evaluation_scope(apply_api_scope(context)) - - # ── Persistent dedicated sub-agent conversation mode ── - # Per-turn @mentions deliberately remain on the main route so the parent - # model emits the real call_subagent tool event and keeps normal streaming. - _agent_id = _direct_agent_id_from_context(context) - if _agent_id: - from contextlib import aclosing - - async with aclosing(_astream_subagent_direct( - agent_id=_agent_id, - session_messages=session_messages, - user_message=user_message, - context=context, - )) as direct_stream: - async for chunk in direct_stream: - yield chunk - return - - _request_ontology_runtime = context.get("ontology_runtime") - if not isinstance(_request_ontology_runtime, dict): - _request_ontology_runtime = {} - context["ontology_runtime"] = _request_ontology_runtime - explicit_skill_ids = _explicit_skill_ids_from_context(context) - for skill_id in explicit_skill_ids: - await asyncio.to_thread( - resolve_runtime_asset_tags, - runtime=_request_ontology_runtime, - kind="skill", - asset_id=skill_id, - user_id=str(context.get("user_id", "") or ""), - ) - activations = activate_runtime_for_asset( - _request_ontology_runtime, - kind="skill", - asset_id=skill_id, - ) - if activations: - await asyncio.to_thread( - _record_ontology_activations, - activations, - _request_ontology_runtime, - context, - ) - - # ── Inject explicit skill instructions ── - skill_msg = _build_skill_injection(context) - if skill_msg: - session_messages.insert(-1, skill_msg) - logger.info( - "[skill_inject] injected skill instructions for '%s'", - context.get("skill_id"), - ) - - # ── [memory] Session memory opened here, never awaited on the SSE path ── - # First turn of a chat: retrieval starts as a background task so it overlaps - # agent assembly, and Fact injection is skipped if it blows its budget. - # Every later turn: the stored snapshot is replayed and no retrieval runs. - _mem0_user_id = str(context.get("user_id", "")) - _mem0_workspace_id = str(context.get("workspace_id", "") or "default") - _mem0_chat_id = context.get("chat_id") or context.get("conversation_id") - _mem0_enabled = bool(context.get("memory_enabled", False)) - _mem0_write_enabled = bool(context.get("memory_write_enabled", False)) - # Under a team project chats.py passes "team:"; personal/default spaces don't pass it and fall back to the real user_id - _mem0_scope_user_id = str(context.get("memory_scope_user_id", "") or _mem0_user_id) - logger.info( - "[memory] user=%s scope=%s ws=%s chat=%s enabled=%s write=%s", - _mem0_user_id, - _mem0_scope_user_id, - _mem0_workspace_id, - _mem0_chat_id, - _mem0_enabled, - _mem0_write_enabled, - ) - - _session_memory = await open_session_memory( - chat_id=_mem0_chat_id, - scope_user_id=_mem0_scope_user_id, - workspace_id=_mem0_workspace_id, - user_message=user_message, - memory_enabled=_mem0_enabled, - # The retrieval budget is part of the orchestration profile: how long - # this task type is willing to wait for memory is an assembly decision, - # not a global constant. Resolved here because retrieval starts before - # the agent is built. - budget_ms=_profile_memory_budget_ms(context), - ) - - # ── Main-route streaming ────────────────────────────────────── - warnings: List[str] = [] - full_response = "" - displayed_tools: set[str] = set() - all_citations: List[Dict[str, Any]] = [] - _ontology_runtime = _request_ontology_runtime - _ontology_event_cursor = 0 - _ontology_trace: List[Dict[str, Any]] = [] - _last_plan: Optional[Dict[str, Any]] = None - _pending_plan_updates: Dict[str, Dict[str, Any]] = {} - _stream_errored = False - # 证据锚点发号器：跨轮续号；创建后绑到 agent 上（见下方 attach_allocator）， - # 中间件与本函数由此共享同一个计数器 - _anchor_allocator = AnchorAllocator( - await asyncio.to_thread(anchor_start_for_chat, str(context.get("chat_id") or "") or None) - ) - - try: - import time as _time - - _wf_start = _time.monotonic() - - yield {"type": "thinking", "message": "正在分析您的问题..."} - for ontology_event in _ontology_runtime.get("runtime_events", []): - yield dict(ontology_event) - _ontology_event_cursor = len(_ontology_runtime.get("runtime_events", [])) - - _stream_user_id = str(context.get("user_id", "")) - _stream_model_name = str(context.get("model_name", "")) - _stream_reranker = bool(context.get("reranker_enabled", False)) - enabled_skill_ids = enabled_skill_ids_from_context(context) - enabled_kb_ids = enabled_kb_ids_from_context(context) - enabled_mcp_ids = enabled_mcp_ids_from_context(context) - - # ── Load visible sub-agents for main agent routing ── - _visible_subagents: list = [] - _mentioned_ids: list = [] - _explicit_command = context.get("explicit_subagent_command") - _explicit_agent_id = ( - str(_explicit_command.get("agent_id")) - if isinstance(_explicit_command, dict) and _explicit_command.get("agent_id") - else "" - ) - _mention_agent_id = str(context.get("mention_agent_id") or "") - try: - from core.db.engine import SessionLocal as _SessionLocal - from core.services.user_agent_service import UserAgentService as _UAS - from core.services.user_service import UserService as _UserService - - with _SessionLocal() as _db: - _ua_svc = _UAS(_db) - _visible_subagents = ([] if context.get("evaluation_session") else - _ua_svc.list_for_user(_stream_user_id)) - _disabled_builtin_ids = _UserService(_db).get_disabled_builtin_subagent_ids( - _stream_user_id - ) - except Exception as _exc: - logger.warning("[workflow] failed to load visible subagents: %s", _exc) - _disabled_builtin_ids = set() - - _visible_subagents = _runtime_visible_subagents( - _visible_subagents, - { - "enabled_skill_ids": enabled_skill_ids or [], - "enabled_mcp_ids": enabled_mcp_ids or [], - "enabled_kb_ids": enabled_kb_ids or [], - }, - _disabled_builtin_ids, - [_explicit_agent_id, _mention_agent_id], - ) - # Prefer the structured per-turn target. Text parsing remains a - # compatibility fallback for callers that only include @name. - _delegated_agent_id = _explicit_agent_id or _mention_agent_id - if _delegated_agent_id and any( - str(item.get("agent_id") or "") == _delegated_agent_id for item in _visible_subagents - ): - _mentioned_ids = [_delegated_agent_id] - else: - _mentioned_ids = _parse_agent_mentions(user_message, _visible_subagents) - - # Create agent (with model-aware CompressionConfig + optional native LTM) - _plan_chat = bool(context.get("plan_chat", False)) - # 极速模式：配置化检索工具集 + 独立 turbo 提示词 + 迭代硬上限。 - # plan 会话有自己的编排语义，不进入极速裁剪。 - _stream_mode_spec = _resolve_mode_spec(context) - _turbo_chat = ( - _stream_mode_spec is not None - and _stream_mode_spec.restricted - and not _plan_chat - ) - if _turbo_chat: - # 收窄模式下子智能体不再只靠手动呼唤入场：模式自己圈定的那批常驻， - # 本轮被 @/显式委派命中的那个并上去（系统提示词也就只渲染这几行）。 - # 模式没圈人时退回原契约——只有被呼唤的那个能进。 - _visible_subagents = _mode_visible_subagents( - _stream_mode_spec, _visible_subagents, _mentioned_ids - ) - agent, mcp_clients = await create_agent_executor( - agent_spec=None, - enabled_skill_ids=enabled_skill_ids, - enabled_mcp_ids=enabled_mcp_ids, - enabled_kb_ids=enabled_kb_ids, - current_user_id=_stream_user_id, - reranker_enabled=_stream_reranker, - model_name=_stream_model_name, - model_provider_id=str(context.get("model_provider_id", "") or ""), - chat_mode=str(context.get("chat_mode", "") or ""), - turbo_mode=_turbo_chat, - mode_spec=_stream_mode_spec, - # 显式呼唤透传：斜杠技能 / 插件展开的 skill_ids 与 mcp_ids。 - # 仅 turbo 模式消费；能力说明仍走 user message 注入，不进全量清单。 - turbo_explicit_skill_ids=( - _explicit_skill_ids_from_context(context) if _turbo_chat else None - ), - turbo_explicit_mcp_ids=( - [m for m in (context.get("mcp_ids") or []) if isinstance(m, str)] - if _turbo_chat - else None - ), - # 渐进式插件加载：本轮显式呼唤的能力不得被延迟（用户消息注入已承诺 - # 其可用），并作为会话级激活落库。 - invoked_skill_ids=_explicit_skill_ids_from_context(context), - invoked_mcp_ids=[m for m in (context.get("mcp_ids") or []) if isinstance(m, str)], - required_mcp_ids=( - [str(context["connector_id"])] if context.get("connector_id") else None - ), - required_skill_id=str(context.get("skill_id") or "") or None, - required_skill_name=str(context.get("skill_name") or "") or None, - required_plugin_id=str(context.get("plugin_id") or "") or None, - required_plugin_name=str(context.get("plugin_name") or "") or None, - required_plugin_skill_ids=[ - str(item) - for item in (context.get("plugin_skill_ids") or []) - if isinstance(item, str) and item - ], - required_plugin_mcp_ids=[ - str(item) - for item in (context.get("plugin_mcp_ids") or []) - if isinstance(item, str) and item - ], - memory_enabled=_mem0_enabled, - approval_mode=_approval_mode(context), - visible_subagents=_visible_subagents if _visible_subagents else None, - plan_mode=_plan_chat, - workflow_mode=bool(context.get("workflow_chat", False)), - chat_id=context.get("chat_id"), - sandbox_session_id=context.get("sandbox_session_id"), - run_id=str(context.get("run_id") or "") or None, - journal_owner=str(context.get("journal_owner") or "") or None, - workspace_id=_mem0_workspace_id, - project_ctx=_extract_project_ctx(context), - channel_origin=context.get("channel_origin"), - automation_run=bool(context.get("automation_run")), - ontology_runtime=context.get("ontology_runtime"), - # The single positive source of truth for update_plan: enabled - # only for "interactive main chats that can host plan mode" — has - # a chat_id, not a channel bot, not automation, not plan_chat, not - # batch. Channels/automation have no plan-bar UI; plan_chat - # have their own orchestration and must not nest plan tracking. - top_level_chat=( - bool(context.get("chat_id")) - and not context.get("channel_origin") - and not bool(context.get("automation_run")) - and not _plan_chat - ), - ) - - logger.info("[workflow] agent created in %.0fms", (_time.monotonic() - _wf_start) * 1000) - - # 证据锚点：把发号器绑到 agent 上（中间件与本函数由此共享同一个计数器； - # 仅靠 ContextVar 不行——本函数是 async generator，与 agent 执行所在的 - # task 上下文不互通） - attach_allocator(agent, _anchor_allocator) - - # ── Inject the per-turn sub-agent constraint into the current user message ── - # Keeps it OUT of the system prompt so the LLM provider's prefix cache - # hits across turns within a chat (otherwise every turn with different - # targets would re-build the cache from scratch). A strict natural- - # language command remains on this normal model stream; it only - # constrains the model's next real tool call instead of fabricating one - # in the workflow layer. - if _mentioned_ids and _visible_subagents: - from core.llm.subagent_tool import ( - build_explicit_subagent_command_hint, - build_subagent_mention_hint, - ) - - if _explicit_agent_id or _mention_agent_id: - _mention_hint = build_explicit_subagent_command_hint( - _visible_subagents, - _explicit_agent_id or _mention_agent_id, - ) - else: - _mention_hint = build_subagent_mention_hint( - _visible_subagents, - _mentioned_ids, - ) - if ( - _mention_hint - and session_messages - and session_messages[-1].get("role") in ("user", "human") - ): - mention_item = make_text_context_item( - _mention_hint, - item_id="harness:subagent_routing", - kind=KIND_REMINDER, - origin="harness:subagent_routing", - trust="system", - created_seq=0, - priority=950, - token_budget=4_000, - truncation_policy=POLICY_NEVER, - ) - session_messages.insert( - -1, - { - "role": "user", - "content": _mention_hint, - SESSION_CONTEXT_META_KEY: session_context_metadata(mention_item), - }, - ) - - # ── PreTurn compaction safety net (aligned with Codex pre-turn compaction) ── - # When end-of-turn background compaction failed/was skipped, or the - # previous turn's tool calls blew up the history, compact once - # synchronously with the same cross-turn compaction mechanism before - # this turn starts and write a checkpoint. Below the threshold, only a - # pure byte estimate is done (no DB / no LLM), so first-token latency - # is unaffected. Must run **before** the frozen memory block injection - # — identity/memory blocks are re-injected every turn and must not be - # baked into the persistent checkpoint. - # The model window is preferentially read straight off the model object - # (make_chat_model bakes in the real context_size per the Config model - # configuration at construction time, no default fallback), saving the - # streaming path one synchronous DB query; only if missing on the - # object do we fall back to resolve — unconfigured raises, fail loud, - # never silently run with the wrong window. - # Pre-turn compaction and the request assembler share this value. - _actual_model, _ctx_window = _resolve_agent_model_runtime(agent, _stream_model_name) - try: - from core.services.compaction_service import maybe_run_pre_turn_compaction - - session_messages, _ = await maybe_run_pre_turn_compaction( - context.get("chat_id"), - session_messages, - run_id=str(context.get("run_id") or ""), - **_pre_turn_compaction_inputs(agent, _ctx_window, _actual_model), - ) - except Exception as _pt_exc: # noqa: BLE001 - from core.harness.hooks import HookPaused - - if isinstance(_pt_exc, HookPaused): - raise - logger.warning("[workflow] pre-turn compaction failed: %s", _pt_exc) - - # ── Session-constant blocks: user identity + the chat's frozen memory - # snapshot. Built on the first turn (L1 profile always, L2 facts only if - # retrieval beat its budget), replayed byte-for-byte on every turn after, - # so this head of the message list never moves the prefix-cache boundary. - session_messages = await inject_session_blocks( - session_messages, - identity_block=("" if context.get("evaluation_session") else - await build_user_identity_block(_mem0_user_id)), - memory_block=await resolve_session_memory(_session_memory), - ) - - # The final request assembler receives the full post-compaction - # candidate set. It is the only budget selector, and records every - # included, truncated, or excluded item in the context manifest. - - streaming_agent = StreamingAgent(agent, mcp_clients) - - skill_load_ids: set = set() # track tool_ids that are skill loads - # tool_id → skill_id; looked up at tool_result time to replace the SSE payload with the curated detail - skill_id_by_tool_id: Dict[str, str] = {} - # tool_id → tool_args, used at the tool_result stage to recover view_text_file's file_path/ranges - view_text_file_args: Dict[str, Dict[str, Any]] = {} - - # Project scope is no longer passed via ContextVar — see the comment at - # the same spot in _astream_subagent_direct above. All scope-dependent - # tools captured it by closure when registered in agent_factory; the - # finishing _persist_artifacts gets the scope explicitly from chats.py. - - try: - # Project initialization is a server-expanded command. The DB row - # deliberately remains /init; only the live model input is expanded. - stream_options = ( - {"effective_user_message": user_message} - if context.get("project_init") - else {} - ) - _main_stream = streaming_agent.stream(session_messages, context, **stream_options) - async for event_type, payload in _main_stream: - state_runtime = getattr(streaming_agent.agent.state, "ontology_runtime", None) - if isinstance(state_runtime, dict): - _ontology_runtime = state_runtime - context["ontology_runtime"] = state_runtime - pending_ontology_events = _ontology_runtime.get("runtime_events", [])[ - _ontology_event_cursor: - ] - for ontology_event in pending_ontology_events: - yield dict(ontology_event) - _ontology_event_cursor += len(pending_ontology_events) - if event_type == "text_delta": - full_response += payload - yield {"type": "content", "event": "ai_message", "delta": payload} - - elif event_type == "vision_progress": - # 识图是模型开口前的一段纯网络等待；透传给前端，让轮级状态 - # 从「深度拥抱中」换成「图像理解中」，而不是干等一个笼统的计时器。 - yield {"type": "vision_progress", **(payload or {})} - - elif event_type == "reasoning_protocol": - yield {"type": "thinking", **payload} - - elif event_type == "steer_applied": - yield {"type": "steer_applied", **(payload or {})} - - elif event_type == "thinking_delta": - yield {"type": "thinking", "delta": payload} - - elif event_type == "context_usage": - yield {"type": "context_usage", **(payload or {})} - - elif event_type == "model_step": - yield {"type": "model_step", "step": payload} - - elif event_type == "model_call_start": - yield {"type": "model_dispatch"} - - elif event_type == "tool_call_start": - tool_name = payload.get("name", "unknown") - if tool_name != "update_plan": - yield { - "type": "tool_call_start", - "tool_name": tool_name, - "tool_display_name": TOOL_DISPLAY_NAMES.get(tool_name, tool_name), - "tool_id": payload.get("id", ""), - } - - elif event_type == "tool_call_delta": - tool_name = payload.get("name", "unknown") - if tool_name != "update_plan" and payload.get("delta"): - yield { - "type": "tool_call_delta", - "tool_name": tool_name, - "tool_id": payload.get("id", ""), - "arguments_delta": payload.get("delta", ""), - } - - elif event_type == "tool_call": - tool_name = payload.get("name", "unknown") - tool_id = payload.get("id", "") - tool_args = payload.get("args", {}) - - # Stage the full plan until its real result is saved. - # A tool request alone must not announce completed work. - if tool_name == "update_plan": - _pu = parse_plan_update_args(tool_args) - if _pu: - _pending_plan_updates[tool_id] = _pu - continue - - # In streaming mode, the first chunk for a tool_call may - # arrive with empty args. For view_text_file we need args - # to decide if this is a skill load, so skip empty-arg - # duplicates until we get the complete args. - _is_fast_emit = tool_name in _FAST_EMIT_TOOLS - if tool_id and tool_id in displayed_tools: - # Fast-emit tools: re-emit when args arrive so the - # frontend can update input display immediately. - if _is_fast_emit and tool_args: - pass # fall through to emit update - else: - continue - if not _tool_args_ready(tool_name, tool_args): - continue - if tool_id: - displayed_tools.add(tool_id) - - # Detect skill loading: view_text_file reading a SKILL.md - is_skill_load = ( - tool_name == "view_text_file" - and isinstance(tool_args, dict) - and "SKILL.md" in str(tool_args.get("file_path", "")) - ) - if is_skill_load and tool_id: - skill_load_ids.add(tool_id) - _sid = _extract_skill_id_from_path(str(tool_args.get("file_path", ""))) - if _sid: - skill_id_by_tool_id[tool_id] = _sid - if ( - tool_name in ("view_text_file", "Read") - and not is_skill_load - and tool_id - and isinstance(tool_args, dict) - ): - view_text_file_args[tool_id] = tool_args - emit_name = "load_skill" if is_skill_load else tool_name - display_name = ( - "加载技能" - if is_skill_load - else TOOL_DISPLAY_NAMES.get(tool_name, tool_name) - ) - safe_args = tool_args if isinstance(tool_args, dict) else {} - _ontology_trace.append( - { - "type": "tool_call", - "tool_id": tool_id, - "tool_name": tool_name, - "input": _bounded_trace_value(safe_args), - } - ) - - # Resolve sub-agent name for call_subagent tool card - _tc_sa_name = "" - if tool_name == "call_subagent" and _visible_subagents: - _tc_sa_id = safe_args.get("agent_id", "") if safe_args else "" - if _tc_sa_id: - for _sa in _visible_subagents: - if _sa.get("agent_id") == _tc_sa_id: - _tc_sa_name = _sa.get("name", "") - break - if not _tc_sa_name and _mentioned_ids: - for _sa in _visible_subagents: - if _sa.get("agent_id") in _mentioned_ids: - _tc_sa_name = _sa.get("name", "") - break - - yield { - "type": "tool_call", - "tool_name": emit_name, - "tool_display_name": display_name, - "tool_args": safe_args, - "input": safe_args, - "tool_id": tool_id, - **({"subagent_name": _tc_sa_name} if _tc_sa_name else {}), - } - - elif event_type == "tool_result": - tool_name = payload.get("name", "unknown") - tool_id = payload.get("id", "") - # The canonical model_step was already forwarded and - # persisted independently. Keep update_plan in the plan - # bar; only its redundant display card is suppressed. - if tool_name == "update_plan": - _pu = _pending_plan_updates.pop(tool_id, None) - if _pu and payload.get("status") == "success": - _last_plan = _pu - _save_plan_progress(str(context.get("chat_id") or ""), _pu) - yield {"type": "plan_update", **_pu} - continue - # Also override tool_name for skill load results - is_skill_result = (tool_id and tool_id in skill_load_ids) or ( - tool_name == "view_text_file" - and "SKILL.md" in str(payload.get("content", "")) - ) - if is_skill_result: - tool_name = "load_skill" - - _missing_call = _synthesize_missing_tool_call( - tool_id, tool_name, displayed_tools - ) - if _missing_call is not None: - _ontology_trace.append( - { - "type": "tool_call", - "tool_id": tool_id, - "tool_name": tool_name, - "input": {}, - } - ) - yield _missing_call - - tool_content = payload.get("content", "") - - # Parse tool result - try: - tool_result_json = json.loads(tool_content) if tool_content else {} - except json.JSONDecodeError: - tool_result_json = {"result": tool_content} - - # Skill load: replace the full SKILL.md text with the same curated detail used by the capability center - if is_skill_result: - _sid = skill_id_by_tool_id.get(tool_id, "") or _extract_skill_id_from_path( - str(tool_content) - ) - tool_result_json = _build_skill_load_payload(_sid) - elif tool_name == "view_text_file": - # Plain file read (AgentScope built-in view_text_file): replace with file metadata + short preview - tool_result_json = _build_view_text_file_payload( - view_text_file_args.get(tool_id, {}), tool_content - ) - elif tool_name == "Read": - # Claude-Code-style Read tool: JSON payload, content holds the whole file - tool_result_json = _build_read_tool_payload( - view_text_file_args.get(tool_id, {}), tool_result_json - ) - elif tool_name == "read_artifact": - tool_result_json = _build_read_artifact_payload(tool_result_json) - - # Extract query if present - extracted_query = "" - if isinstance(tool_result_json, dict) and "result" in tool_result_json: - result_data = tool_result_json["result"] - if isinstance(result_data, dict): - extracted_query = result_data.get( - "query", result_data.get("question", "") - ) - - # Citations - cit_dicts = collect_citation_dicts(tool_id, _anchor_allocator) - all_citations.extend(cit_dicts) - _ontology_trace.append( - { - "type": "tool_result", - "tool_id": tool_id, - "tool_name": tool_name, - "result": _bounded_trace_value(tool_result_json), - "citations": cit_dicts, - } - ) - - # Resolve sub-agent name from call_subagent result text - _tr_sa_name = "" - if tool_name == "call_subagent": - _res_str = ( - str(tool_result_json.get("result", "")) - if isinstance(tool_result_json, dict) - else str(tool_result_json) - ) - if "【" in _res_str and "】" in _res_str: - _tr_sa_name = _res_str.split("【", 1)[1].split("】", 1)[0] - - yield { - "type": "tool_result", - "tool_name": tool_name, - "tool_args": {"query": extracted_query} if extracted_query else {}, - "result": tool_result_json, - "tool_id": tool_id, - "citations": cit_dicts, - "status": payload.get("status", "success"), - "model_step": payload.get("model_step"), - **({"subagent_name": _tr_sa_name} if _tr_sa_name else {}), - } - - elif event_type in ("heartbeat", "model_progress"): - # heartbeat = transport keep-alive; model_progress = the - # model is still streaming while a small argument batch or - # another suppressed event has nothing renderable yet — - # forwarded so the run watchdog counts activity (it - # excludes only heartbeat). - yield {"type": event_type} - - elif event_type == "tool_pending": - yield {"type": "tool_pending", **(payload or {})} - - elif event_type == "subagent_event": - # Bypass channel for the sub-agent's internal - # thinking/tool_call/tool_result/content — attached under - # the call_subagent tool card that launched it (linked via - # parent_tool_id). - if (payload or {}).get("sub_type") == "progress": - # Sub-agent liveness (long tool-call-arg generation with - # nothing renderable yet) — feed the run inactivity - # watchdog without writing a sub-step to the stream or - # the persisted tool log. - yield {"type": "model_progress"} - else: - nested_citations = _capture_nested_ontology_evidence( - payload or {}, - _ontology_trace, - all_citations, - _anchor_allocator, - ) - yield { - "type": "subagent_event", - **(payload or {}), - **({"citations": nested_citations} if nested_citations else {}), - } - - elif event_type in ( - "file_confirm", - "design_pick", - "user_question", - "user_question_resolved", - ): - # Human-interaction events (§13 MySpace write confirm / - # site-design pick-one-of-three / ask_user_question): a tool - # coroutine has suspended waiting for the user's out-of-band - # action. Pass through to the frontend to show the card; the - # agent task stays blocked in that tool and this SSE stream - # does not end — after the out-of-band answer POST the tool - # resumes in place. - yield {"type": event_type, **(payload or {})} - - elif event_type == "error": - # payload may be a real exception object (kind=="err") or a - # dict (e.g. ExceedMaxIters mapped to {"kind":..,"name":..}). - # Raising the latter directly gives a TypeError, masking - # the real situation — wrap it in a RuntimeError. - if isinstance(payload, BaseException): - raise payload - raise RuntimeError(str(payload)) - - except BaseException: - await streaming_agent.shutdown() - raise - finally: - if "_main_stream" in locals(): - await _main_stream.aclose() - - except Exception as e: - import traceback - - logger.error("stream_workflow_error: %s\n%s", e, traceback.format_exc()) - warnings.append(f"Streaming error: {str(e)[:200]}") - _stream_errored = True - - unknown_outcome = find_tool_outcome_unknown(e) - if unknown_outcome is not None: - if "streaming_agent" in locals(): - await streaming_agent.shutdown() - _persistent_clients.append((streaming_agent, list(mcp_clients))) - if unknown_outcome is e: - raise - raise unknown_outcome from e - - if not full_response: - # No answer was produced: propagate. The run executor persists the - # tool cards already shown together with the error and emits a - # structured ``error`` event — no fabricated "please refer to the - # tool results above" reply that would read as a completed answer. - if "streaming_agent" in locals(): - await streaming_agent.shutdown() - _persistent_clients.append((streaming_agent, list(mcp_clients))) - raise - - pending_ontology_events = _ontology_runtime.get("runtime_events", [])[_ontology_event_cursor:] - for ontology_event in pending_ontology_events: - yield dict(ontology_event) - _ontology_event_cursor += len(pending_ontology_events) - - _ontology_review_owner_id = _ontology_review_owner(_ontology_runtime, context) - _ontology_review_claimed = bool( - full_response and claim_output_review(_ontology_runtime, owner=_ontology_review_owner_id) - ) - if _ontology_review_claimed: - from orchestration.subagents.ontology_reviewer import review_ontology_output - - yield { - "type": "ontology_review", - "status": "started", - "level": _ontology_runtime.get("review_level", "checkpoint"), - **_ontology_review_event_context(_ontology_runtime), - } - repair_event_queue: asyncio.Queue[Dict[str, Any]] = asyncio.Queue() - - async def _remediate(payload: Dict[str, Any]) -> str: - nonlocal _ontology_event_cursor - primary_usage = await streaming_agent.aget_usage() - primary_context_usage = streaming_agent.get_context_usage(primary_usage) - try: - repaired, _, cursor, _ = await _run_ontology_repair_round( - streaming_agent=streaming_agent, - context=context, - payload=payload, - runtime=_ontology_runtime, - trace=_ontology_trace, - citations=all_citations, - allocator=_anchor_allocator, - event_cursor=_ontology_event_cursor, - event_sink=repair_event_queue.put, - ) - finally: - streaming_agent.restore_context_usage(primary_context_usage) - _ontology_event_cursor = cursor - return repaired - - review_task = asyncio.create_task( - review_ontology_output( - task=user_message, - answer=full_response, - runtime=_ontology_runtime, - trace=_ontology_trace, - citations=all_citations, - user_id=str(context.get("user_id", "")), - chat_id=context.get("chat_id"), - model_name=str(context.get("model_name", "") or "") or None, - model_provider_id=str(context.get("model_provider_id", "") or "") or None, - remediate=_remediate, - ) - ) - try: - while not review_task.done() or not repair_event_queue.empty(): - try: - repair_event = await asyncio.wait_for( - repair_event_queue.get(), - timeout=0.1, - ) - except asyncio.TimeoutError: - continue - yield repair_event - review = await review_task - except (asyncio.CancelledError, GeneratorExit): - from core.infra.task_lifecycle import cancel_and_join - - await cancel_and_join(review_task) - release_output_review(_ontology_runtime, owner=_ontology_review_owner_id) - await streaming_agent.shutdown() - _persistent_clients.append((streaming_agent, list(mcp_clients))) - raise - except Exception as exc: # noqa: BLE001 - if not review_task.done(): - review_task.cancel() - review = _ontology_review_failure_result(full_response, exc) - complete_output_review( - _ontology_runtime, - owner=_ontology_review_owner_id, - verdict=str(review.get("verdict") or "unknown"), - attempts=int(review.get("attempts") or 1), - ) - _ontology_runtime.setdefault("output_review", {}).update( - { - "revised": bool(review.get("revised")), - "annotated": bool(review.get("annotated")), - "repair_attempts": int(review.get("repair_attempts") or 0), - "violations": review.get("violations") or [], - "affected_claims": review.get("affected_claims") or [], - "evidence": review.get("evidence") or [], - "feedback": review.get("feedback") or [], - "manual_review": review.get("manual_review") or {}, - "candidate_answer": (review.get("answer") if review.get("revised") else ""), - "new_tools": review.get("new_tools") or [], - "new_citation_count": int(review.get("new_citation_count") or 0), - "latency_ms": review.get("latency_ms"), - } - ) - yield { - "type": "ontology_review", - "status": "completed", - "level": _ontology_runtime.get("review_level", "checkpoint"), - "verdict": review["verdict"], - "revised": bool(review.get("revised")), - "annotated": bool(review.get("annotated")), - "repair_attempts": int(review.get("repair_attempts") or 0), - "candidate_answer": review.get("answer") if review.get("revised") else "", - "manual_review": review.get("manual_review") or {}, - "violations": review.get("violations") or [], - "affected_claims": review.get("affected_claims") or [], - "evidence": review.get("evidence") or [], - "feedback": review.get("feedback") or [], - "new_tools": review.get("new_tools") or [], - "new_citation_count": int(review.get("new_citation_count") or 0), - "latency_ms": review.get("latency_ms"), - **_ontology_review_event_context(_ontology_runtime), - } - - await streaming_agent.shutdown() - _persistent_clients.append((streaming_agent, list(mcp_clients))) - - # update_plan finalization: the model routinely finishes its last step and - # streams the answer without a closing update_plan call, leaving the plan - # bar stuck at N-1/N. When the turn ends normally with an answer and no - # step is still pending, the remaining in_progress step is the one that - # just produced the answer — close it out. - if _last_plan and full_response and not _stream_errored: - _plan_steps = _last_plan.get("steps") or [] - if ( - _plan_steps - and all(s.get("status") != "pending" for s in _plan_steps) - and any(s.get("status") == "in_progress" for s in _plan_steps) - ): - _closed = { - "title": _last_plan.get("title", ""), - "steps": [{**s, "status": "completed"} for s in _plan_steps], - } - _last_plan = _closed - _save_plan_progress(str(context.get("chat_id") or ""), _closed) - yield {"type": "plan_update", **_closed} - - # 计划栏收尾 —— 本轮结束时把"还会不会有人来更新这份清单"这件事记进库里。 - # - # 过去只有前端在流结束时给它盖章，于是收尾依赖"这个标签页恰好在跟这条流"：后台作业 - # 跑完的交付轮是**后端自己发起**的，用户没点任何东西，页面可能压根没跟——计划栏就永远 - # 停在转圈。这里改由服务端认定：本会话还有活着的作业就先不收尾（后面还有交付轮要来 - # 更新它），没有了就记 settled。步骤状态一个字不改，settled 只表示"没有下一轮了"。 - if _last_plan: - _plan_chat_id = str(context.get("chat_id") or "") - if _plan_chat_id and not _chat_has_live_job(_plan_chat_id): - _settle_plan_progress(_plan_chat_id) - - _main_usage = await streaming_agent.aget_usage() - _main_context_usage = streaming_agent.get_context_usage(_main_usage) - yield { - "type": "meta", - "route": "main", - "is_markdown": _looks_markdown(full_response), - "sources": _resolve_sources_conflict([]), - "artifacts": [], - "warnings": warnings, - "citations": all_citations, - "usage": _main_usage, - "context_usage": _main_context_usage, - "_compaction_budget_inputs": _compaction_budget_inputs(agent, _ctx_window), - "ontology_governance": _ontology_governance_summary(_ontology_runtime), - # Settlement has not run yet (memory writes are scheduled below), so all - # this can honestly say is "settling" — the client draws a skeleton and - # fills it in when the summary arrives. - "evolution_pending": _evolution_pending_marker(context, _mem0_write_enabled), - } - - # ── [memory] Post-response pipeline (SSE already closed, user isn't waiting) ── - # No memory is written unless the user opted into memory_write_enabled (first gate). - # Everything goes through memory_pipeline.schedule_post_response_tasks: - # global Semaphore + 4 extractors + sanitize + write L1/L2/Session + audit, - # all executed in the background with bounds. - if _mem0_write_enabled: - logger.info( - "[memory] schedule post-response: full_response_len=%s, user=%s ws=%s", - len(full_response) if full_response else 0, - _mem0_user_id, - _mem0_workspace_id, - ) - save_memories_background( - _mem0_user_id, - user_message, - full_response, - _mem0_write_enabled, - workspace_id=_mem0_workspace_id, - chat_id=_mem0_chat_id, - scope_user_id=_mem0_scope_user_id, - # Lets the pipeline report its real write count back to this turn's - # settlement instead of the card having to guess one. - message_id=str(context.get("message_id") or ""), - ) - else: - logger.debug( - "[workflow] memory save skipped: write_enabled=False (user=%s)", - _mem0_user_id, - ) - - # ── [evolution] Evidence assembly (SSE already closed, user isn't waiting) ── - # The response path only bound versions and appended events; joining them - # into one causal Episode is deliberately deferred to here. - if not context.get("evaluation_session"): - _assemble_episode_background( - message_id=str(context.get("message_id") or ""), - run_id=str(context.get("run_id") or ""), - chat_id=str(context.get("chat_id") or ""), - user_id=str(context.get("user_id") or ""), - objective=user_message, - agent=streaming_agent, - memory_task=_session_memory.retrieval_task, - latency_ms=None, - memory_write_enabled=_mem0_write_enabled, - ) diff --git a/src/backend/personal_agent/README.md b/src/backend/personal_agent/README.md index 6e1a2075..0f248414 100644 --- a/src/backend/personal_agent/README.md +++ b/src/backend/personal_agent/README.md @@ -22,7 +22,7 @@ authenticated users; EE additionally applies its account capability policy. - models.py: the seven shared tables, registered even with the runtime module off. HTTP routers call the domain services. Model tools use the same services. Session -operations reuse core/services/session_operations; neither path calls HTTP route +operations reuse core/chat/operations; neither path calls HTTP route functions. Shared chat request contracts live in core/chat/schemas.py. External integration uses the registered hooks in extension.py. diff --git a/src/backend/personal_agent/access.py b/src/backend/personal_agent/access.py index 84c1df1d..ab1fa35f 100644 --- a/src/backend/personal_agent/access.py +++ b/src/backend/personal_agent/access.py @@ -10,7 +10,7 @@ from core.auth.capabilities import user_has_capability from core.config.settings import settings from core.db.models import UserShadow -from core.extensions import is_enabled +from core.harness.registry import is_enabled NAME = "personal_agent" CAPABILITY = "can_use_personal_agent" diff --git a/src/backend/personal_agent/api/conversation.py b/src/backend/personal_agent/api/conversation.py index a162b0b0..8a270bc8 100644 --- a/src/backend/personal_agent/api/conversation.py +++ b/src/backend/personal_agent/api/conversation.py @@ -84,7 +84,7 @@ async def monitor(svc=Depends(service)): from personal_agent.conversation.message_routing import pending from personal_agent.delivery.outputs import listing from personal_agent.computer.devices import list_devices - from core.db.models import EmailConnection + from integrations.email.models import EmailConnection from personal_agent.scheduling.schedules import listing as schedule_listing from personal_agent.scheduling.heartbeat import view as heartbeat_view diff --git a/src/backend/personal_agent/channels/channel_bridge.py b/src/backend/personal_agent/channels/channel_bridge.py index daa2aef4..1a2d3e0d 100644 --- a/src/backend/personal_agent/channels/channel_bridge.py +++ b/src/backend/personal_agent/channels/channel_bridge.py @@ -49,7 +49,7 @@ def exported_artifact(db, chat_id, artifact_id): return item def load_file(chat_id, artifact_id): - from core.storage import get_storage + from core.files.storage import get_storage with SessionLocal() as db: item = exported_artifact(db, chat_id, artifact_id) if item is None: @@ -74,13 +74,14 @@ def pack(msg): return data def unpack(data): - from core.channels.protocol import InboundMsg + from integrations.channels.protocol import InboundMsg fields = {field.name for field in dataclasses.fields(InboundMsg)} return InboundMsg(**{key: value for key, value in data.items() if key in fields}) async def admit_inbound(conn, msg, adapter): """Store the IM message in the Personal Agent conversation. The worker answers it later.""" - from core.channels.inbound import _ingest_attachments, _speaker_label + from integrations.channels.inbound_attachments import ingest_attachments as _ingest_attachments + from integrations.channels.inbound_observation import speaker_label as _speaker_label from personal_agent.conversation.message_routing import admit from personal_agent.context import AgentContext placeholder = None @@ -160,9 +161,9 @@ async def send_now(origin, text): async def send_text(conn, msg, text, sent_chunks=0, record=None): """Persist each successful chunk before attempting the next one.""" - from core.channels.markdown import strip_citation_markers, strip_inline_thinking - from core.channels.protocol import chunk_text - from core.channels.registry import get_adapter + from integrations.channels.markdown import strip_citation_markers, strip_inline_thinking + from integrations.channels.protocol import chunk_text + from integrations.channels.registry import get_adapter adapter = get_adapter(conn.channel_type) text = strip_citation_markers(strip_inline_thinking(text)) send = getattr(adapter, "send_markdown", None) if adapter.caps.supports_markdown else None @@ -190,7 +191,7 @@ def delivery_allowed(chat_id, conn): def connection(channel_id): """An enabled bot connection, detached with its credentials loaded, or None.""" - from core.db.repository.channel import ChannelConnectionRepository + from integrations.channels.repository import ChannelConnectionRepository with SessionLocal() as db: conn = ChannelConnectionRepository(db).get_by_id(channel_id) if conn is None or not conn.enabled: return None @@ -279,7 +280,7 @@ async def record(**changes): delivery.get("text_chunks_sent", 0), record) if not ok: raise RuntimeError("Personal channel text delivery failed") await record(text_sent=True) - from core.channels.registry import get_adapter + from integrations.channels.registry import get_adapter sent = list(delivery.get("sent_file_ids") or []) failed = list(delivery.get("failed_file_ids") or []) retry = False @@ -338,7 +339,7 @@ def notify_soon(origin, text): def bots(db, uid): """The owner's bots, with the Personal Agent routing flag.""" - from core.services.channel_service import ChannelService, bot_to_dict + from integrations.channels.service import ChannelService, bot_to_dict agent = db.scalar(select(PersonalAgent).where(PersonalAgent.user_id == str(uid))) routed = set((agent.config or {}).get("channel_ids", [])) if agent else set() result = [] @@ -351,7 +352,7 @@ def bots(db, uid): return result def route(db, uid, channel_id, enabled): - from core.services.channel_service import ChannelService + from integrations.channels.service import ChannelService if channel_id not in {conn.channel_id for conn in ChannelService(db).list_bots(str(uid), main_only=True)}: raise LookupError("Bot not found") from personal_agent.identity.config import update_config diff --git a/src/backend/personal_agent/computer/computer.py b/src/backend/personal_agent/computer/computer.py index fa552016..4bbdf381 100644 --- a/src/backend/personal_agent/computer/computer.py +++ b/src/backend/personal_agent/computer/computer.py @@ -1,4 +1,5 @@ """Observe-act-verify tools with exclusive desktop control and exact approvals.""" +from core.llm.factory.tools.definitions import function_tool import asyncio import json import shlex @@ -11,7 +12,7 @@ from personal_agent.tools.responses import response from personal_agent.computer.sandbox import DESKTOP_RUN, SANDBOX_PYTHON, get_personal_sandbox_provider from personal_agent.tasks.approvals import authorize, receipt -from core.device_link.driver import ACTIONS as SUPPORTED, KEYS +from personal_agent.device_link.driver import ACTIONS as SUPPORTED, KEYS # One desktop, one user of it at a time: the agent and the owner share this lease. LEASE_TTL = 45 @@ -146,4 +147,4 @@ async def pa_device_list(): from personal_agent.computer.devices import list_devices return response(list_devices(uid)) - for fn in (pa_computer_observe, pa_computer_action, pa_device_list): toolkit.register_tool_function(fn) + for fn in (pa_computer_observe, pa_computer_action, pa_device_list): toolkit.append(function_tool(fn)) diff --git a/src/backend/personal_agent/computer/devices.py b/src/backend/personal_agent/computer/devices.py index 2122f353..adae3683 100644 --- a/src/backend/personal_agent/computer/devices.py +++ b/src/backend/personal_agent/computer/devices.py @@ -6,7 +6,7 @@ import time import uuid from sqlalchemy import select -from core.device_link import CAPABILITIES +from personal_agent.device_link import CAPABILITIES from core.db.engine import SessionLocal from personal_agent.models import PersonalDevice, PersonalDeviceCommand, PersonalWorkItem diff --git a/src/backend/personal_agent/computer/driver_install.py b/src/backend/personal_agent/computer/driver_install.py index 953b9167..ca061e07 100644 --- a/src/backend/personal_agent/computer/driver_install.py +++ b/src/backend/personal_agent/computer/driver_install.py @@ -5,7 +5,7 @@ import shlex from pathlib import Path -from core.device_link import driver +from personal_agent.device_link import driver from personal_agent.computer.sandbox import SANDBOX_PYTHON, install_once diff --git a/src/backend/personal_agent/computer/sandbox.py b/src/backend/personal_agent/computer/sandbox.py index 294218dc..7ab9310b 100644 --- a/src/backend/personal_agent/computer/sandbox.py +++ b/src/backend/personal_agent/computer/sandbox.py @@ -11,7 +11,7 @@ from sqlalchemy import select from core.db.engine import SessionLocal from core.sandbox.providers.opensandbox.provider import OpenSandboxProvider -from core.sandbox.providers.opensandbox.models import _Session +from core.sandbox.providers.opensandbox.models import SandboxSessionRecord as _Session from core.sandbox.errors import SandboxError from personal_agent.models import PersonalAgent @@ -140,14 +140,14 @@ async def _get_or_create_session(self, session_id, user_id=None): return session async def _check_mount_revision(self, sandbox, user_id): - from core.spaces.sandbox import current_revision - from core.spaces.mounts import MOUNT_REVISION_TAG + from core.files.spaces.sandbox import current_revision + from core.files.spaces.mounts import MOUNT_REVISION_TAG info = await sandbox.get_info() revision = await asyncio.to_thread(current_revision, user_id) if (info.metadata or {}).get(MOUNT_REVISION_TAG) != revision: await sandbox.kill() raise SandboxError("File permissions changed; reopen the computer") - from core.spaces.recovery import ensure_mounts + from core.files.spaces.recovery import ensure_mounts import os await ensure_mounts(sandbox, user_id, info, domain=os.environ["PERSONAL_AGENT_OPENSANDBOX_DOMAIN"], @@ -155,7 +155,7 @@ async def _check_mount_revision(self, sandbox, user_id): async def _provision(self, ident, session_id, user_id, generation): - from core.spaces.leases import async_space_lock + from core.files.spaces.leases import async_space_lock async with async_space_lock(user_id): return await self._provision_locked(ident, session_id, user_id, generation) @@ -212,8 +212,8 @@ async def _provision_locked(self, ident, session_id, user_id, generation): desktop.mkdir(parents=True) os.chmod(desktop, 0o700) desktop_volume = Volume(name="personal-desktop", host=Host(path=str(Path(root) / ident / "desktop")), mount_path=DESKTOP_HOME, read_only=False) - from core.spaces.mounts import prepare_volumes - space_volumes, space_metadata = await asyncio.to_thread(prepare_volumes, user_id) + from core.files.spaces.mounts import provision_volumes + space_volumes, space_metadata = await asyncio.to_thread(provision_volumes, user_id) computer_config, access_token = self._computer_config(ident) sandbox = await Sandbox.create(os.getenv("PERSONAL_AGENT_IMAGE", DEFAULT_IMAGE), connection_config=computer_config, env={"EXECD_ACCESS_TOKEN": access_token}, timeout=None, skip_health_check=False, volumes=[volume, desktop_volume, *space_volumes], diff --git a/src/backend/personal_agent/computer/screenshot_pruning.py b/src/backend/personal_agent/computer/screenshot_pruning.py index bd2fbc82..37288c4f 100644 --- a/src/backend/personal_agent/computer/screenshot_pruning.py +++ b/src/backend/personal_agent/computer/screenshot_pruning.py @@ -1,50 +1,42 @@ -"""Keep only recent desktop screenshots in a Personal Agent model context. +"""Bound desktop screenshots in plain model-context snapshots.""" +from core.harness.events import thaw_value +from core.harness.hooks import Decision, FailureMode, HookSpec, HookStage -Each observation adds a full-screen image. After many steps, old images use most of the -context and the model output degrades. The latest screenshots carry the current state. -""" -import logging - -from agentscope.message import TextBlock, ToolResultBlock -from agentscope.middleware import MiddlewareBase - -from core.llm.compacting_agent import CompactingAgent - -log = logging.getLogger(__name__) KEEP = 2 PLACEHOLDER = "[An older screenshot was removed. Use the latest observation.]" -is_image = CompactingAgent._is_image def prune(messages, keep=KEEP): - """Replace all but the newest `keep` tool-result images with a short text. Returns the count.""" - seen, removed = 0, 0 + """Mutate plain message dictionaries, retaining the newest tool images.""" + seen = removed = 0 for message in reversed(messages): - content = message.content if isinstance(message.content, list) else [] - changed = False - blocks = [] + content = message.get("content") + if not isinstance(content, list): + continue for block in reversed(content): - if isinstance(block, ToolResultBlock) and isinstance(block.output, list): - output = [] - for item in reversed(block.output): - if is_image(item): - seen += 1 - if seen > keep: - item, removed, changed = TextBlock(text=PLACEHOLDER), removed + 1, True - output.append(item) - if changed: - block = block.model_copy(update={"output": list(reversed(output))}) - blocks.append(block) - if changed: - message.content = list(reversed(blocks)) + if not isinstance(block, dict) or block.get("type") != "tool_result": + continue + output = block.get("output") + if not isinstance(output, list): + continue + for index in range(len(output) - 1, -1, -1): + item = output[index] + if isinstance(item, dict) and item.get("type") == "data" and str((item.get("source") or {}).get("media_type") or "").startswith("image/"): + seen += 1 + if seen > keep: + output[index] = {"type": "text", "text": PLACEHOLDER} + removed += 1 return removed -class ScreenshotPruningMiddleware(MiddlewareBase): - async def on_reasoning(self, agent, input_kwargs, next_handler): - try: - removed = prune(agent.state.context) - if removed: log.debug("Removed %d older screenshots from context", removed) - except Exception as exc: # noqa: BLE001 - never block the model call - log.warning("Screenshot pruning failed (%s)", type(exc).__name__) - async for event in next_handler(**input_kwargs): - yield event + +def trim_screenshots(invocation): + if invocation.operation_name != "agent_reasoning": + return Decision() + messages = thaw_value(invocation.data.get("context") or ()) + return Decision.modify({"context": messages}) if prune(messages) else Decision() + + +def screenshot_pruning_hook(): + return HookSpec("desktop_screenshot_pruning", HookStage.TRANSFORM_CONTEXT, + trim_screenshots, priority=900, failure_mode=FailureMode.OPEN, + mutable_fields=frozenset({"context"})) diff --git a/src/backend/personal_agent/conversation/conversations.py b/src/backend/personal_agent/conversation/conversations.py index d4befbcd..4f88dd61 100644 --- a/src/backend/personal_agent/conversation/conversations.py +++ b/src/backend/personal_agent/conversation/conversations.py @@ -1,7 +1,8 @@ """Conversation routing for one identity: web main plus isolated IM peers/rooms.""" import hashlib from sqlalchemy import select -from core.db.models import ChatSession, ChatMessage, ChannelConnection +from core.db.models import ChatSession, ChatMessage +from integrations.channels.models import ChannelConnection from personal_agent.models import PersonalMessageTask MARKER = "personal_agent_channel" diff --git a/src/backend/personal_agent/conversation/front.py b/src/backend/personal_agent/conversation/front.py index fc505e28..09abdb8c 100644 --- a/src/backend/personal_agent/conversation/front.py +++ b/src/backend/personal_agent/conversation/front.py @@ -11,13 +11,19 @@ import time import uuid from contextvars import ContextVar -from sqlalchemy import select, func + from core.db.engine import SessionLocal from core.db.models import ChatMessage, ChatSession from core.db.repository.chat import ChatMessageRepository -from personal_agent.models import PersonalAgent, PersonalWorkItem, PersonalMessageInbox, PersonalMessageTask -from personal_agent.tasks.task_threads import StreamRecord +from personal_agent.models import ( + PersonalAgent, + PersonalMessageInbox, + PersonalMessageTask, + PersonalWorkItem, +) from personal_agent.tasks import task_ops +from personal_agent.tasks.task_threads import StreamRecord +from sqlalchemy import func, select log = logging.getLogger(__name__) PROFILE = "personal_agent_front" @@ -101,9 +107,8 @@ def system_prompt(ident): Prompt caching needs a prefix that does not change between turns, so everything that does (the clock, tasks, sessions, the event) goes into turn_context instead.""" - from personal_agent.workspace.prompting import sections - from personal_agent.workspace.prompting import render from personal_agent.scheduling.schedules import user_zone + from personal_agent.workspace.prompting import render, sections with SessionLocal() as db: agent = db.get(PersonalAgent, db.get(PersonalMessageInbox, ident).agent_id) config, agent_id = dict(agent.config or {}), agent.id @@ -113,12 +118,12 @@ def system_prompt(ident): def turn_context(ident): """What changes every turn: the clock, the task list, recent sessions, the event. It follows the current message, so the earlier history stays a cached prefix.""" + from personal_agent.scheduling import heartbeat + from personal_agent.scheduling.schedules import is_unattended_run + from personal_agent.tasks.runtime import clock from personal_agent.workspace.constants import BOOTSTRAP_FILE from personal_agent.workspace.files import read_text from personal_agent.workspace.prompting import prompt_part - from personal_agent.tasks.runtime import clock - from personal_agent.scheduling.schedules import is_unattended_run - from personal_agent.scheduling import heartbeat with SessionLocal() as db: inbox = db.get(PersonalMessageInbox, ident) agent = db.get(PersonalAgent, inbox.agent_id) @@ -157,7 +162,7 @@ def with_context(messages, context): def history(db, ident): """The recent conversation as chat messages, ending with this turn's message.""" - from core.llm.model_steps import assistant_row_replay + from core.llm.runtime.model_steps import assistant_row_replay inbox = db.get(PersonalMessageInbox, ident) agent = db.get(PersonalAgent, inbox.agent_id) current = db.get(ChatMessage, ident) @@ -279,9 +284,8 @@ def prepare(db, ident, owner): async def converse(plan, binding): """Stream the agent's turn into its reply.""" from core.llm.factory import create_agent_executor - from orchestration.streaming import StreamingAgent - from personal_agent.identity.capabilities import front_mcp_ids - from personal_agent.identity.capabilities import runtime_config + from core.llm.runtime.streaming import StreamingAgent + from personal_agent.identity.capabilities import front_mcp_ids, runtime_config with SessionLocal() as db: config = runtime_config(db, plan["uid"], plan["config"]) reply = plan["reply"] @@ -384,7 +388,7 @@ def finish(state, reply, failed=False): async def run_turn(ident, owner, execute=None): """One complete turn. execute(plan, binding) replaces the model in tests.""" - from core.services.run_journal import durable_run_binding + from core.runs.journal import durable_run_binding plan = await asyncio.to_thread(_prepare, ident, owner) if plan is None: return state, reply = plan["state"], plan["reply"] @@ -444,11 +448,9 @@ def hatch(svc): """Wake a new agent once, as OpenClaw's hatch: its first turn follows BOOTSTRAP.md and greets the user. Admitted only into an empty conversation of a ready agent whose first run is pending. True when admitted.""" + from personal_agent.workspace.files import WorkspaceUnavailable, local_workspace + from personal_agent.workspace.persona import ensure_seeded, settle_bootstrap from sqlalchemy.exc import IntegrityError - from personal_agent.workspace.files import WorkspaceUnavailable - from personal_agent.workspace.persona import ensure_seeded - from personal_agent.workspace.files import local_workspace - from personal_agent.workspace.persona import settle_bootstrap agent = svc.identity.agent() if agent is None or agent.status != "ready": return False if svc.db.scalar(select(ChatMessage.message_id).where(ChatMessage.chat_id == agent.chat_id).limit(1)): return False diff --git a/src/backend/personal_agent/delivery/delivery.py b/src/backend/personal_agent/delivery/delivery.py index 5274beb5..37912855 100644 --- a/src/backend/personal_agent/delivery/delivery.py +++ b/src/backend/personal_agent/delivery/delivery.py @@ -29,7 +29,7 @@ def discard_unpublished(key): with SessionLocal() as db: if db.scalar(select(Artifact.artifact_id).where(Artifact.storage_key == key)): return - from core.storage import get_storage + from core.files.storage import get_storage get_storage().delete(key) except Exception as exc: # noqa: BLE001 - cleanup must preserve the original failure log.warning("Personal delivery upload cleanup failed (%s)", type(exc).__name__) @@ -84,7 +84,7 @@ def _previous(ident, uid, state, message_id, request_hash): return _receipt(db, row, uid, request_hash) if row else None def _publish(ident, uid, state, message_id, artifact_id, request_hash, content, filename, mime, caption): - from core.services.artifact_service import store_bytes_as_artifact, artifact_upload_key + from core.files.artifacts.service import store_bytes_as_artifact, artifact_upload_key from personal_agent.context import AgentContext from personal_agent.channels.channel_bridge import mark_for_delivery from personal_agent.scheduling.schedules import is_unattended_run diff --git a/src/backend/personal_agent/device_link/README.md b/src/backend/personal_agent/device_link/README.md new file mode 100644 index 00000000..a8473a8f --- /dev/null +++ b/src/backend/personal_agent/device_link/README.md @@ -0,0 +1,3 @@ +本机设备链接：根据用户同意轮询个人智能体设备邮箱、执行本机操作并回传收据。保留账号隔离、许可复核与至多一次执行语义，不新增监听端口。 + +模块归 personal_agent 所有，由原 API 生命周期的 execution-plane singleton 入口管理；不依赖云端 PERSONAL_AGENT_ENABLED 开关。桌面链接功能即使未启用云端个人智能体也能连接用户已授权的云端。独立 driver 仍由 Dockerfile.personal-agent 复制到容器原路径。无新增数据库模型，不单独加入 INSTALLED_FEATURES。 diff --git a/src/backend/core/device_link/__init__.py b/src/backend/personal_agent/device_link/__init__.py similarity index 84% rename from src/backend/core/device_link/__init__.py rename to src/backend/personal_agent/device_link/__init__.py index 2852cc76..2b7cf188 100644 --- a/src/backend/core/device_link/__init__.py +++ b/src/backend/personal_agent/device_link/__init__.py @@ -4,7 +4,7 @@ the identity bridge). The link polls the cloud device mailbox; it opens no port. """ -from .driver import ACTIONS +from personal_agent.device_link.driver import ACTIONS SESSIONS = "hugagent.sessions" CAPABILITIES = ACTIONS | {SESSIONS} diff --git a/src/backend/core/device_link/driver.py b/src/backend/personal_agent/device_link/driver.py similarity index 100% rename from src/backend/core/device_link/driver.py rename to src/backend/personal_agent/device_link/driver.py diff --git a/src/backend/core/device_link/local_api.py b/src/backend/personal_agent/device_link/local_api.py similarity index 100% rename from src/backend/core/device_link/local_api.py rename to src/backend/personal_agent/device_link/local_api.py diff --git a/src/backend/core/device_link/runner.py b/src/backend/personal_agent/device_link/runner.py similarity index 98% rename from src/backend/core/device_link/runner.py rename to src/backend/personal_agent/device_link/runner.py index 96730429..aea523d5 100644 --- a/src/backend/core/device_link/runner.py +++ b/src/backend/personal_agent/device_link/runner.py @@ -18,7 +18,7 @@ from core.infra.proc import no_window_kwargs from hugagent_cli.contract import validate -from . import ACTIONS, local_api, store +from personal_agent.device_link import ACTIONS, local_api, store logger = get_logger(__name__) @@ -37,7 +37,7 @@ class DeviceUnavailable(Exception): def current_account() -> Optional[Tuple[str, str]]: """``(account, cloud_base)`` of the cloud identity the desktop shell signed in with.""" - from core.services.desktop_cloud_bridge import get_identity_state + from features.desktop.bridge.identity import get_identity_state identity = get_identity_state() return (identity["shell_user_center_id"], identity["cloud_base"]) if identity else None @@ -69,7 +69,7 @@ async def run_driver(action) -> dict: def _bridge_headers(account: str) -> dict: - from core.auth.desktop_bridge import bridge_headers + from features.desktop.auth.bridge import bridge_headers from core.db.engine import SessionLocal from core.db.models import UserShadow diff --git a/src/backend/core/device_link/store.py b/src/backend/personal_agent/device_link/store.py similarity index 100% rename from src/backend/core/device_link/store.py rename to src/backend/personal_agent/device_link/store.py diff --git a/src/backend/personal_agent/extension.py b/src/backend/personal_agent/extension.py index d3aae32e..1fdc4748 100644 --- a/src/backend/personal_agent/extension.py +++ b/src/backend/personal_agent/extension.py @@ -1,4 +1,4 @@ -"""Personal Agent module registration (see ``core.extensions``). +"""Personal Agent module registration (see ``core.harness.registry``). Everything Personal Agent adds to the running system is registered here. With ``PERSONAL_AGENT_ENABLED`` off nothing is registered: no routes, no background @@ -8,14 +8,7 @@ import asyncio import os -from core.extensions import ( - AgentProfile, - register_agent_profile, - register_channel_route, - register_lifecycle, - register_router, - register_sandbox_route, -) +from core.harness.registry import AgentProfile, register_agent_profile, register_channel_route, register_lifecycle, register_router, register_sandbox_route from personal_agent.access import NAME @@ -44,13 +37,15 @@ def _register_tools(toolkit, request, loader=None) -> None: register_personal_tools(toolkit, request, loader=loader) -def _middlewares(request) -> list: - # Desktop observations: keep only the latest screenshots in the model context. - from personal_agent.computer.screenshot_pruning import ScreenshotPruningMiddleware +def _hooks(request) -> list: + from personal_agent.computer.screenshot_pruning import screenshot_pruning_hook + return [screenshot_pruning_hook()] + +def _steering(request): from personal_agent.tasks.steering import current_steering steering = current_steering.get() - return [*([steering.middleware()] if steering else []), ScreenshotPruningMiddleware()] + return steering.hook_policy() if steering else None def _handles_channel(db, conn) -> bool: from personal_agent.channels.channel_bridge import handled_by_agent @@ -80,7 +75,7 @@ async def _stop() -> None: def setup() -> bool: if not env_enabled(): return False - from core.services.prompt_version_service import register_kind + from core.llm.prompt_kinds import register_kind from personal_agent.api.agent import router as agent_router from personal_agent.api.sessions import router as sessions_router @@ -93,7 +88,8 @@ def setup() -> bool: AgentProfile( system_prompt=_system_prompt, register_tools=_register_tools, - middlewares=_middlewares, + hooks=_hooks, + steering=_steering, tool_filter=lambda name: name.startswith("pa_"), inherit_default_mcps=False, # Skills are listed in the platform rules and read with pa_skill_read. diff --git a/src/backend/personal_agent/identity/capabilities.py b/src/backend/personal_agent/identity/capabilities.py index 5701b16b..3c99b89c 100644 --- a/src/backend/personal_agent/identity/capabilities.py +++ b/src/backend/personal_agent/identity/capabilities.py @@ -17,7 +17,7 @@ def blocked(ident): def default_capabilities(db, user_id): """The curated skills and tools this user is allowed to use, plus the plugins they have turned on, as {"skill_ids", "mcp_ids", "plugins"}.""" - from core.config.catalog_resolver import resolve_all_runtime_enabled + from core.capabilities.catalog.resolver import resolve_all_runtime_enabled skills, _, mcps = resolve_all_runtime_enabled(db, user_id) # None: the catalog could not be read this time; the caller keeps what it has. pick = lambda curated, permitted: None if permitted is None else [ident for ident in curated if ident in permitted] @@ -26,8 +26,8 @@ def default_capabilities(db, user_id): def plugin_inventory(db, user_id): """Read this owner's installed plugins, including disabled component identities.""" - from core.capabilities import device_catalog - from core.plugins import management + from features.desktop.capabilities import device_catalog + from core.capabilities.plugins import management try: # A savepoint: the agent's own rows may be pending in this transaction. with db.begin_nested(): @@ -60,7 +60,7 @@ def installed_plugins(db, user_id, skills, mcps, items=None): return out def available_capabilities(db, user_id): - from core.config.catalog_resolver import resolve_all_runtime_enabled + from core.capabilities.catalog.resolver import resolve_all_runtime_enabled skills, _, mcps = resolve_all_runtime_enabled(db, user_id) if skills is None or mcps is None: raise ValueError("Capability catalog unavailable; try again") diff --git a/src/backend/personal_agent/identity/config.py b/src/backend/personal_agent/identity/config.py index 0493bce2..1fcde681 100644 --- a/src/backend/personal_agent/identity/config.py +++ b/src/backend/personal_agent/identity/config.py @@ -28,7 +28,7 @@ def skill_summaries(skill_ids, loader=None): """One line per skill for the task prompt: its id, to read with pa_skill_read, and when it applies.""" if not skill_ids: return [] try: - from core.agent_skills.loader import get_skill_loader + from core.capabilities.skills.loader_factory import get_skill_loader metadata = (loader or get_skill_loader()).load_all_metadata() except Exception: # noqa: BLE001 - without descriptions the ids still work logging.getLogger(__name__).warning("personal agent skill metadata unavailable", exc_info=True) diff --git a/src/backend/personal_agent/identity/service.py b/src/backend/personal_agent/identity/service.py index 4f86793c..fc6d1b1f 100644 --- a/src/backend/personal_agent/identity/service.py +++ b/src/backend/personal_agent/identity/service.py @@ -117,7 +117,7 @@ def configure(self, patch): raise ValueError("Invalid thinking effort") if "model_provider_id" in patch: # Empty means the platform default model. Otherwise the same allowlist as the chat model picker. - from core.services.user_model_selection import resolve_user_model_provider_id + from core.auth.model_selection import resolve_user_model_provider_id wanted = (patch["model_provider_id"] or "").strip() patch["model_provider_id"] = resolve_user_model_provider_id(self.db, wanted, user_id=self.user_id) if wanted else None if wanted and patch["model_provider_id"] is None: raise ValueError("Model switching is not available to this user") diff --git a/src/backend/personal_agent/scheduling/schedules.py b/src/backend/personal_agent/scheduling/schedules.py index ae8235eb..8248db01 100644 --- a/src/backend/personal_agent/scheduling/schedules.py +++ b/src/backend/personal_agent/scheduling/schedules.py @@ -34,7 +34,7 @@ def user_zone(config): return (config or {}).get("timezone") or DEFAULT_TIMEZONE def next_cron(cron, timezone, after): - from core.services.automation_service import compute_next_run + from features.automation.scheduling import compute_next_run return compute_next_run(cron, timezone, datetime.fromtimestamp(after, zone(timezone))).timestamp() def next_time(item, after): diff --git a/src/backend/personal_agent/sessions/session_cloud.py b/src/backend/personal_agent/sessions/session_cloud.py index 70396694..95cb5365 100644 --- a/src/backend/personal_agent/sessions/session_cloud.py +++ b/src/backend/personal_agent/sessions/session_cloud.py @@ -3,10 +3,13 @@ from fastapi import HTTPException from hugagent_cli.contract import validate + async def execute(db, user, operation, args, origin=None): args = validate(operation, args) - from core.services.session_operations import sessions, history, runs as run_views - from core.services.session_operations import schemas as models + from core.chat.operations import history + from core.chat.operations import runs as run_views + from core.chat.operations import schemas as models + from core.chat.operations import sessions if operation == "chat.list": result = sessions.list_chats( @@ -55,8 +58,8 @@ async def execute(db, user, operation, args, origin=None): db=db, ) elif operation == "chat.send": - from core.services.session_operations.admission import _start_response_run from core.chat.schemas import ChatRequest + from core.chat.operations.admission import start_response_run as _start_response_run run = await _start_response_run(ChatRequest(**args, stream=True), user, db) return { @@ -72,12 +75,12 @@ async def execute(db, user, operation, args, origin=None): elif operation == "run.status": return run_status(args["run_id"], user) elif operation == "run.cancel": - from core.services.session_operations.commands import cancel_chat_run + from core.chat.operations.commands import cancel_chat_run result = await cancel_chat_run(args["run_id"], user=user) elif operation == "run.steer": - from core.services.session_operations.commands import steer_chat_run - from core.services.session_operations.schemas import SteerChatRunRequest + from core.chat.operations.commands import steer_chat_run + from core.chat.operations.schemas import SteerChatRunRequest result = await steer_chat_run( args["run_id"], @@ -86,7 +89,7 @@ async def execute(db, user, operation, args, origin=None): ) # Defaults match the HTTP routes that the device path calls. elif operation == "project.list": - from core.services.session_operations.projects import list_projects + from core.chat.operations.projects import list_projects result = list_projects( q=args.get("q"), @@ -97,11 +100,11 @@ async def execute(db, user, operation, args, origin=None): db=db, ) elif operation == "project.get": - from core.services.session_operations.projects import get_project + from core.chat.operations.projects import get_project result = get_project(access=await project_access(args, user, db), db=db) elif operation == "project.chats": - from core.services.session_operations.projects import list_project_chats + from core.chat.operations.projects import list_project_chats result = list_project_chats( page=args.get("page", 1), @@ -114,14 +117,14 @@ async def execute(db, user, operation, args, origin=None): async def project_access(args, user, db): """The HTTP routes' own view check, which also audits denials.""" - from core.auth.permissions_iface import require_project_access + from core.auth.edition import require_project_access return await require_project_access("view")( project_id=args["project_id"], request=None, user=user, db=db ) def run_status(ident, user): - from orchestration.chat_run_executor import get_run + from core.runs.runtime import get_run row = get_run(ident) if row is None or row.user_id != user.user_id: diff --git a/src/backend/personal_agent/sessions/session_watch.py b/src/backend/personal_agent/sessions/session_watch.py index 5a3a06e4..7542174f 100644 --- a/src/backend/personal_agent/sessions/session_watch.py +++ b/src/backend/personal_agent/sessions/session_watch.py @@ -83,7 +83,7 @@ def report_finished(): def projects(db, uid): """The user's most recently active projects, with the same visibility as the projects page.""" - from core.services.project_service import ProjectService + from core.projects.edition import ProjectService try: items, total = ProjectService(db).list_visible(uid, sort="-last_activity_at", page=1, page_size=STATE_PROJECTS) except Exception: diff --git a/src/backend/personal_agent/tasks/attachments.py b/src/backend/personal_agent/tasks/attachments.py index cd39e728..2d0a143d 100644 --- a/src/backend/personal_agent/tasks/attachments.py +++ b/src/backend/personal_agent/tasks/attachments.py @@ -3,13 +3,13 @@ import posixpath from core.db.engine import SessionLocal from core.db.models import Artifact -from core.services.artifact_edition import is_personal_artifact +from core.db.artifact_scope import is_personal_artifact async def stage_inputs(task_id, uid, box, artifact_ids): if not artifact_ids: return "" - from core.storage import get_storage + from core.files.storage import get_storage from personal_agent.computer.sandbox import get_personal_sandbox_provider provider = get_personal_sandbox_provider() prompt = "" diff --git a/src/backend/personal_agent/tasks/runtime.py b/src/backend/personal_agent/tasks/runtime.py index b189ff8e..99b1acd8 100644 --- a/src/backend/personal_agent/tasks/runtime.py +++ b/src/backend/personal_agent/tasks/runtime.py @@ -3,10 +3,11 @@ import re from collections import Counter from datetime import datetime + from core.db.engine import SessionLocal from core.db.models import ChatMessage -from sqlalchemy import select from personal_agent.models import PersonalAgent, PersonalWorkItem +from sqlalchemy import select MAX_SEGMENT = 30000 @@ -44,9 +45,9 @@ async def finalize(goal, text, provider_id=None): import asyncio from dataclasses import replace try: - from core.services.model_config import ModelConfigService - from core.llm.chat_models import build_model_for_mode from agentscope.message import Msg, TextBlock + from core.models.chat_models import build_model_for_mode + from core.models.config import ModelConfigService service = ModelConfigService.get_instance() cfg = (service.resolve_provider(provider_id) if provider_id else None) or service.resolve("main_agent") model = build_model_for_mode(replace(cfg, temperature=0, max_tokens=2000, timeout=60), mode="fast", stream=False) @@ -73,8 +74,9 @@ def clock(config): async def execute_task(task_id, binding): import asyncio + from core.llm.factory import create_agent_executor - from orchestration.streaming import StreamingAgent + from core.llm.runtime.streaming import StreamingAgent with SessionLocal() as db: task = db.get(PersonalWorkItem, task_id) from personal_agent.tasks.execution import bind diff --git a/src/backend/personal_agent/tasks/service.py b/src/backend/personal_agent/tasks/service.py index 9ccbf051..21978449 100644 --- a/src/backend/personal_agent/tasks/service.py +++ b/src/backend/personal_agent/tasks/service.py @@ -1,4 +1,4 @@ -from core.services.artifact_edition import is_personal_artifact +from core.db.artifact_scope import is_personal_artifact """Owner-scoped tasks operations.""" from sqlalchemy import select, update, case from sqlalchemy.exc import IntegrityError diff --git a/src/backend/personal_agent/tasks/steering.py b/src/backend/personal_agent/tasks/steering.py index 63264d88..c8921f70 100644 --- a/src/backend/personal_agent/tasks/steering.py +++ b/src/backend/personal_agent/tasks/steering.py @@ -19,9 +19,9 @@ def __init__(self, task_id, uid, run_id, owner, messages, artifact_ids=(), consu self.consumed = previous | {ident for ident, _ in self.initial_messages} self.artifact_ids = set(artifact_ids) - def middleware(self): - from core.llm.steer_middleware import SteerMiddleware - return SteerMiddleware(self.take, self.acknowledge) + def hook_policy(self): + from core.chat.steer_hook import SteeringHook + return SteeringHook(self.take, self.acknowledge) def _task(self, db, *, lock=False): from sqlalchemy import select diff --git a/src/backend/personal_agent/tasks/task_threads.py b/src/backend/personal_agent/tasks/task_threads.py index 4435ff81..69240544 100644 --- a/src/backend/personal_agent/tasks/task_threads.py +++ b/src/backend/personal_agent/tasks/task_threads.py @@ -106,7 +106,7 @@ def store(self,row,content): **(row.extra_data or {}), "segments":self.segments[-300:], "model_history_display_only": not bool(self.model_steps), } - from core.llm.context_safety import remember_context_output + from core.llm.context.safety import remember_context_output remember_context_output( message_id=row.message_id, steps=row.model_steps or [], content=row.content, @@ -173,7 +173,7 @@ def record(self, event, payload): self.store(row, self.text) else: row.model_steps = [*(row.model_steps or []), payload] - from core.llm.context_safety import remember_context_output + from core.llm.context.safety import remember_context_output remember_context_output( message_id=row.message_id, steps=row.model_steps, content=row.content, @@ -201,7 +201,7 @@ def record(self, event, payload): self.store(row, self.text) else: row.model_steps = [*(row.model_steps or []), payload] - from core.llm.context_safety import remember_context_output + from core.llm.context.safety import remember_context_output remember_context_output( message_id=row.message_id, steps=row.model_steps, content=row.content, diff --git a/src/backend/personal_agent/tasks/worker.py b/src/backend/personal_agent/tasks/worker.py index 50b37132..eddd7311 100644 --- a/src/backend/personal_agent/tasks/worker.py +++ b/src/backend/personal_agent/tasks/worker.py @@ -6,7 +6,7 @@ from contextlib import suppress from sqlalchemy import select from core.db.engine import SessionLocal -from core.services.run_journal import durable_run_binding +from core.runs.journal import durable_run_binding from personal_agent import access from personal_agent.computer import computer_wait from personal_agent.models import PersonalAgent, PersonalWorkItem @@ -209,7 +209,7 @@ async def run(self, ident): db.commit() raise except Exception as exc: - from core.services.tool_effect_ledger import find_tool_outcome_unknown + from core.runs.tool_effects.errors import find_tool_outcome_unknown with SessionLocal() as db: row = db.get(PersonalWorkItem, ident) if row and row.status == "running" and row.owner == self.identity: @@ -237,23 +237,21 @@ async def dispatch_approved(self, ident, uid): bind(row) payloads = dict((row.checkpoint or {}).get("approved_action_payloads", {})) if not payloads: return - class Tools: - def __init__(self): self.functions = {} - def register_tool_function(self, function): self.functions[function.__name__] = function - tools = Tools() + tools = [] from personal_agent.tools.email import register_email_tools from personal_agent.computer.computer import register_computer_tools register_email_tools(tools, ident, uid) register_computer_tools(tools, ident, uid) + functions = {tool.name: tool._func for tool in tools} for digest, action in payloads.items(): if action["kind"] == "email.send": - await tools.functions["pa_email_send"](action["to"], action["subject"], action["body"], + await functions["pa_email_send"](action["to"], action["subject"], action["body"], [item["path"] for item in action.get("attachments", [])]) elif action["kind"] == "hugagent.session": from personal_agent.sessions.session_manager import dispatch_approved await dispatch_approved(ident, uid, action) elif action["kind"] == "computer.action": - await tools.functions["pa_computer_action"](action["action"], action["target"]) + await functions["pa_computer_action"](action["action"], action["target"]) with SessionLocal() as db: row = db.get(PersonalWorkItem, ident) state = dict(row.checkpoint or {}) diff --git a/src/backend/personal_agent/tools/email.py b/src/backend/personal_agent/tools/email.py index bec2ed96..1bc29f1f 100644 --- a/src/backend/personal_agent/tools/email.py +++ b/src/backend/personal_agent/tools/email.py @@ -1,4 +1,5 @@ """IMAP/SMTP broker. Credentials remain in the backend, never in the sandbox.""" +from core.llm.factory.tools.definitions import function_tool import asyncio import hashlib import imaplib @@ -9,7 +10,7 @@ from email.message import EmailMessage from email.utils import getaddresses from core.db.engine import SessionLocal -from core.db.models import EmailConnection +from integrations.email.models import EmailConnection from core.infra.crypto import decrypt_secret from personal_agent.tasks.guards import owned_task from personal_agent.tools.responses import response @@ -134,4 +135,4 @@ def send(): receipt(ident, uid, digest, result) return response(result) - for fn in (pa_email_list, pa_email_read, pa_email_download, pa_email_send): toolkit.register_tool_function(fn) + for fn in (pa_email_list, pa_email_read, pa_email_download, pa_email_send): toolkit.append(function_tool(fn)) diff --git a/src/backend/personal_agent/tools/front_tools.py b/src/backend/personal_agent/tools/front_tools.py index e1f15fb0..6099bae9 100644 --- a/src/backend/personal_agent/tools/front_tools.py +++ b/src/backend/personal_agent/tools/front_tools.py @@ -4,11 +4,12 @@ Work with external effects (Bash, files, email, the computer, writes to the user's HugAgentOS) belongs to tasks; a turn may only read the user's HugAgentOS sessions. """ +from core.llm.factory.tools.definitions import function_tool import json from contextlib import contextmanager from typing import Literal from agentscope.message import TextBlock -from agentscope.tool._response import ToolChunk as ToolResponse +from agentscope.tool._response import ToolChunk from sqlalchemy import select from core.db.engine import SessionLocal from core.db.models import ChatMessage @@ -16,7 +17,7 @@ from personal_agent.tasks import task_ops def response(value): - return ToolResponse(content=[TextBlock(type="text", text=json.dumps(value, ensure_ascii=False, default=str))]) + return ToolChunk(content=[TextBlock(type="text", text=json.dumps(value, ensure_ascii=False, default=str))]) def register_front_tools(toolkit, request, loader=None): from personal_agent.conversation.front import EVENT_ROLE, current_turn, owned @@ -53,7 +54,7 @@ def changing(state): yield AgentContext(db, uid, chat_id=db.get(ChatMessage, ident).chat_id), db.get(ChatMessage, ident) db.commit() - async def pa_task_create(goal: str, title: str = "", related_task_ids: list[str] | None = None) -> ToolResponse: + async def pa_task_create(goal: str, title: str = "", related_task_ids: list[str] | None = None) -> ToolChunk: """Start a background task. goal: a self-contained instruction in the user's language with the requirements and the expected result; the task sees only the last few messages. title: a short name of at most 20 characters. related_task_ids: existing tasks this new deliverable builds on. @@ -71,7 +72,7 @@ async def pa_task_create(goal: str, title: str = "", related_task_ids: list[str] state["count"] += 1 return response(result) - async def pa_task_update(task_id: str, instruction: str) -> ToolResponse: + async def pa_task_update(task_id: str, instruction: str) -> ToolChunk: """Continue or change one existing task, including a completed task. Use only for a clear follow-up to its goal or result, never because it is recent or the only task. Read pa_task_get first before updating a task unless the user explicitly selected its ID. instruction: only what applies to this task.""" state = turn(changes_tasks=True) with checked(state) as db: @@ -84,7 +85,7 @@ async def pa_task_update(task_id: str, instruction: str) -> ToolResponse: result = {"updated": True, "status": task_ops.update_task(svc, message, task_id, instruction, attachments=state["attachments"]).status} return response(result) - async def pa_task_control(task_id: str, action: Literal["pause", "resume", "cancel"]) -> ToolResponse: + async def pa_task_control(task_id: str, action: Literal["pause", "resume", "cancel"]) -> ToolChunk: """Pause, resume or cancel one task. Resume never approves a pending action; the user approves those in the approval card. Resume applies to waiting tasks; to continue a completed, failed or cancelled task, read it and use pa_task_update with the user's follow-up instruction.""" @@ -92,7 +93,7 @@ async def pa_task_control(task_id: str, action: Literal["pause", "resume", "canc result = {"status": task_ops.control(svc, message, task_id, action).status} return response(result) - async def pa_task_get(task_id: str) -> ToolResponse: + async def pa_task_get(task_id: str) -> ToolChunk: """Read one task: goal, status, progress, the user's updates and the result.""" from personal_agent.conversation.associations import instructions with checked(turn()) as db: @@ -104,7 +105,7 @@ async def pa_task_get(task_id: str) -> ToolResponse: "updates": instructions(db, db.get(ChatMessage, ident).chat_id, task.id, 10), "result": (task.result or "")[:12000], "updated": task.updated_at}) - async def pa_history_search(query: str, limit: int = 10) -> ToolResponse: + async def pa_history_search(query: str, limit: int = 10) -> ToolChunk: """Search earlier messages and task results of this conversation.""" with checked(turn()) as db: chat = db.get(ChatMessage, ident).chat_id @@ -124,7 +125,7 @@ def saved(data): with SessionLocal() as db: AgentContext(db, uid).identity.adopt_identity_name(data.get("content") or "") return response({"path": data["path"], "saved": True}) - async def pa_memory_read(path: str) -> ToolResponse: + async def pa_memory_read(path: str) -> ToolChunk: """Read a workspace persona or memory file, for example USER.md, MEMORY.md or memory/2026-10-10.md.""" data = read_file(workspace(), path) versions[data["path"]] = data["etag"] @@ -133,7 +134,7 @@ async def pa_memory_read(path: str) -> ToolResponse: return response({"path": data["path"], "content": text[-50000:] if daily else text[:50000], "truncated": data["truncated"] or len(text) > 50000, "slice": "tail" if daily else "head"}) - async def pa_memory_edit(path: str, old_string: str, new_string: str) -> ToolResponse: + async def pa_memory_edit(path: str, old_string: str, new_string: str) -> ToolChunk: """Replace one exact, unique passage of a persona or memory file. Read the file first.""" agent = workspace() current = read_file(agent, path) @@ -143,7 +144,7 @@ async def pa_memory_edit(path: str, old_string: str, new_string: str) -> ToolRes versions[data["path"]] = data["etag"] return saved(data) - async def pa_memory_write(path: str, content: str) -> ToolResponse: + async def pa_memory_write(path: str, content: str) -> ToolChunk: """Create or replace a persona or memory file, for example a new daily note memory/YYYY-MM-DD.md.""" from personal_agent.workspace.files import relative target = WORKSPACE + "/" + relative(path) @@ -151,25 +152,25 @@ async def pa_memory_write(path: str, content: str) -> ToolResponse: versions[target] = data["etag"] return saved(data) - async def pa_set_name(name: str) -> ToolResponse: + async def pa_set_name(name: str) -> ToolChunk: """Change your own name when the user gives you a new one.""" with checked(turn()) as db: return response({"name": AgentContext(db, uid).identity.configure({"name": name})["config"]["name"]}) - async def pa_schedule_create(title: str, instruction: str, cron: str = "", run_at: str = "", timezone: str = "") -> ToolResponse: + async def pa_schedule_create(title: str, instruction: str, cron: str = "", run_at: str = "", timezone: str = "") -> ToolChunk: """Create a scheduled task that later runs as a background task. cron: five fields in local time, such as "0 8 * * *" for 08:00 daily; or run_at: one ISO local time. timezone defaults to the user's.""" with checked(turn(changes_tasks=True)) as db: return response(create(db, uid, title, instruction, cron, run_at, timezone)) - async def pa_schedule_list() -> ToolResponse: + async def pa_schedule_list() -> ToolChunk: """List the scheduled tasks and their next run times.""" with checked(turn()) as db: return response(listing(db.scalar(select(PersonalAgent.config).where(PersonalAgent.user_id == uid)))) async def pa_schedule_update(schedule_id: str, title: str | None = None, instruction: str | None = None, cron: str | None = None, run_at: str | None = None, - timezone: str | None = None, enabled: bool | None = None) -> ToolResponse: + timezone: str | None = None, enabled: bool | None = None) -> ToolChunk: """Edit an existing schedule in place. List schedules first to identify it. Omit unchanged fields; use enabled=false to pause. Give cron for repeats or run_at for a one-time ISO local time.""" from personal_agent.scheduling.schedules import update_schedule @@ -178,12 +179,12 @@ async def pa_schedule_update(schedule_id: str, title: str | None = None, instruc with checked(turn(changes_tasks=True)) as db: return response(update_schedule(db, uid, schedule_id, patch)) - async def pa_schedule_delete(schedule_id: str) -> ToolResponse: + async def pa_schedule_delete(schedule_id: str) -> ToolChunk: """Delete one scheduled task when the user asks to stop it.""" with checked(turn(changes_tasks=True)) as db: return response(delete(db, uid, schedule_id)) - async def pa_stay_quiet(reason: str) -> ToolResponse: + async def pa_stay_quiet(reason: str) -> ToolChunk: """Send nothing to the user for this scheduled result or heartbeat check, because it has nothing new or noteworthy.""" state = turn() if state["kind"] == "event": @@ -206,4 +207,4 @@ async def pa_stay_quiet(reason: str) -> ToolResponse: # The user's HugAgentOS projects, chats and runs: reads only in a conversation turn. from personal_agent.sessions.session_tool import build as session_tool tools.append(session_tool(uid)) - for fn in tools: toolkit.register_tool_function(fn) + for fn in tools: toolkit.append(function_tool(fn)) diff --git a/src/backend/personal_agent/tools/responses.py b/src/backend/personal_agent/tools/responses.py index 0b262d6e..40d04bef 100644 --- a/src/backend/personal_agent/tools/responses.py +++ b/src/backend/personal_agent/tools/responses.py @@ -4,7 +4,7 @@ from agentscope.message import TextBlock -from agentscope.tool._response import ToolChunk as ToolResponse +from agentscope.tool._response import ToolChunk def response(value): - return ToolResponse(content=[TextBlock(type="text", text=json.dumps(value, ensure_ascii=False, default=str))]) + return ToolChunk(content=[TextBlock(type="text", text=json.dumps(value, ensure_ascii=False, default=str))]) diff --git a/src/backend/personal_agent/tools/tools.py b/src/backend/personal_agent/tools/tools.py index 1a38fc4e..b28a2114 100644 --- a/src/backend/personal_agent/tools/tools.py +++ b/src/backend/personal_agent/tools/tools.py @@ -1,11 +1,12 @@ """Explicit Personal Agent tool surface; all targets derive from authenticated task.""" +from core.llm.factory.tools.definitions import function_tool import personal_agent.workspace.files as _pa_files from personal_agent.tasks.guards import owned_task from personal_agent.tools.responses import response from personal_agent.workspace.paths import workspace_path -from agentscope.tool._response import ToolChunk as ToolResponse +from agentscope.tool._response import ToolChunk from core.db.engine import SessionLocal from sqlalchemy import select from core.db.models import ChatMessage @@ -20,7 +21,7 @@ def register_personal_tools(toolkit, request, loader=None): versions = {} from personal_agent.workspace import persona - async def pa_task_status() -> ToolResponse: + async def pa_task_status() -> ToolChunk: """Read this task's latest user updates and pending actions.""" state = owned_task(ident, uid, allow_pending=True) with SessionLocal() as db: @@ -41,7 +42,7 @@ async def pa_task_checkpoint(summary: str, next_step: str = ""): db.commit() return response({"saved": True}) - async def pa_bash(command: str, timeout_seconds: int = 60) -> ToolResponse: + async def pa_bash(command: str, timeout_seconds: int = 60) -> ToolChunk: """Run Bash inside the dedicated cloud computer. Never runs on the user's device.""" state = owned_task(ident, uid) duration = max(1, min(300, timeout_seconds)) @@ -65,7 +66,7 @@ async def pa_bash(command: str, timeout_seconds: int = 60) -> ToolResponse: await get_ephemeral_state().drop(key(token)) return response({"stdout": result.stdout[-50000:].replace(token,"[task credential]"), "stderr": result.stderr[-10000:].replace(token,"[task credential]"), "exit_code": result.exit_code}) - async def pa_read_file(path: str) -> ToolResponse: + async def pa_read_file(path: str) -> ToolChunk: """Read UTF-8 text from the dedicated cloud workspace, with a bounded result.""" state = owned_task(ident, uid) target = workspace_path(path) @@ -79,7 +80,7 @@ async def pa_read_file(path: str) -> ToolResponse: known.add(workspace_path(path)) return response({"path": path, "content": content.decode("utf-8")[:50000]}) - async def pa_write_file(path: str, content: str) -> ToolResponse: + async def pa_write_file(path: str, content: str) -> ToolChunk: """Write a task file in the dedicated workspace; does not write to MySpace.""" state = owned_task(ident, uid) if len(content.encode()) > 2*1024*1024: raise ValueError("File too large") @@ -92,7 +93,7 @@ async def pa_write_file(path: str, content: str) -> ToolResponse: known.add(workspace_path(path)) return response({"path": path, "saved": True}) - async def pa_edit_file(path: str, old_string: str, new_string: str, replace_all: bool = False) -> ToolResponse: + async def pa_edit_file(path: str, old_string: str, new_string: str, replace_all: bool = False) -> ToolChunk: """Replace exact text in a workspace file, such as one entry of USER.md or MEMORY.md. Read the file with pa_read_file first. old_string must match exactly and be unique unless replace_all is true.""" state = owned_task(ident, uid) @@ -145,7 +146,7 @@ async def pa_skill_read(skill_id: str, relative_path: str = "SKILL.md"): if path.stat().st_size > 200000: raise ValueError("Skill reference too large") return response({"skill_id": skill_id, "path": relative_path, "content": path.read_text()}) - async def pa_finish(answer: str) -> ToolResponse: + async def pa_finish(answer: str) -> ToolChunk: """Complete the task with final text in the user's language, then stop. To show an image/file in the main conversation, call pa_deliver before this. Already delivered outputs remain visible; do not export or send them again.""" @@ -158,7 +159,7 @@ async def pa_finish(answer: str) -> ToolResponse: db.commit() return response({"delivered": True, "next": "Stop now. Do not write more text."}) - async def pa_set_name(name: str) -> ToolResponse: + async def pa_set_name(name: str) -> ToolChunk: """Change your own name when the user asks for a new one. Updates settings, IDENTITY.md and the name shown in the app.""" owned_task(ident, uid) from personal_agent.context import AgentContext @@ -166,7 +167,7 @@ async def pa_set_name(name: str) -> ToolResponse: view = AgentContext(db, uid).identity.configure({"name": name}) return response({"name": view["config"]["name"]}) - async def pa_schedule_create(title: str, instruction: str, cron: str = "", run_at: str = "", timezone: str = "") -> ToolResponse: + async def pa_schedule_create(title: str, instruction: str, cron: str = "", run_at: str = "", timezone: str = "") -> ToolChunk: """Create a scheduled task that runs later as a new task. Use cron (five fields, local time) for repeats, for example "0 8 * * *" for 08:00 every day, or run_at (ISO local time) for one time. timezone defaults to the user's.""" owned_task(ident, uid) @@ -174,7 +175,7 @@ async def pa_schedule_create(title: str, instruction: str, cron: str = "", run_a with SessionLocal() as db: return response(create(db, uid, title, instruction, cron, run_at, timezone)) - async def pa_schedule_list() -> ToolResponse: + async def pa_schedule_list() -> ToolChunk: """List the user's scheduled tasks and their next run times.""" state = owned_task(ident, uid) from personal_agent.scheduling.schedules import listing @@ -182,7 +183,7 @@ async def pa_schedule_list() -> ToolResponse: async def pa_schedule_update(schedule_id: str, title: str | None = None, instruction: str | None = None, cron: str | None = None, run_at: str | None = None, - timezone: str | None = None, enabled: bool | None = None) -> ToolResponse: + timezone: str | None = None, enabled: bool | None = None) -> ToolChunk: """Edit an existing schedule in place. List schedules first to identify it. Omit unchanged fields; use enabled=false to pause. Give cron for repeats or run_at for a one-time ISO local time.""" from personal_agent.scheduling.schedules import update_schedule @@ -192,7 +193,7 @@ async def pa_schedule_update(schedule_id: str, title: str | None = None, instruc with SessionLocal() as db: return response(update_schedule(db, uid, schedule_id, patch)) - async def pa_schedule_delete(schedule_id: str) -> ToolResponse: + async def pa_schedule_delete(schedule_id: str) -> ToolChunk: """Delete one scheduled task when the user asks to stop it.""" owned_task(ident, uid) from personal_agent.scheduling.schedules import delete @@ -205,12 +206,12 @@ async def pa_schedule_delete(schedule_id: str) -> ToolResponse: managing = () if scheduled else (pa_schedule_create, pa_schedule_update, pa_schedule_delete) for fn in (pa_task_status, pa_task_checkpoint, pa_bash, pa_read_file, pa_write_file, pa_edit_file, pa_history_search, pa_skill_read, pa_set_name, pa_finish, pa_schedule_list, *managing): - toolkit.register_tool_function(fn) + toolkit.append(function_tool(fn)) from personal_agent.delivery.delivery import build as delivery_tool - toolkit.register_tool_function(delivery_tool(ident, uid)) + toolkit.append(function_tool(delivery_tool(ident, uid))) # The user's HugAgentOS sessions; writes wait for the user's approval. from personal_agent.sessions.session_tool import build as session_tool - toolkit.register_tool_function(session_tool(uid, ident)) + toolkit.append(function_tool(session_tool(uid, ident))) from personal_agent.tools.email import register_email_tools from personal_agent.computer.computer import register_computer_tools register_email_tools(toolkit, ident, uid) diff --git a/src/backend/personal_agent/workspace/dream_plan.py b/src/backend/personal_agent/workspace/dream_plan.py index ac68f1c4..f10a2b84 100644 --- a/src/backend/personal_agent/workspace/dream_plan.py +++ b/src/backend/personal_agent/workspace/dream_plan.py @@ -84,8 +84,8 @@ def plan(payload, answer, run_id, at): async def complete(payload, provider_id=None): from dataclasses import replace - from core.services.model_config import ModelConfigService - from core.llm.chat_models import build_model_for_mode + from core.models.config import ModelConfigService + from core.models.chat_models import build_model_for_mode from agentscope.message import Msg, TextBlock service = ModelConfigService.get_instance() cfg = (service.resolve_provider(provider_id) if provider_id else None) or service.resolve("main_agent") diff --git a/src/backend/personal_agent/workspace/prompting.py b/src/backend/personal_agent/workspace/prompting.py index e464c46f..f8c67296 100644 --- a/src/backend/personal_agent/workspace/prompting.py +++ b/src/backend/personal_agent/workspace/prompting.py @@ -1,11 +1,12 @@ """Prompting for Personal Agent.""" +from core.llm import prompt_rendering as prompt_rendering_module from datetime import datetime, timedelta def prompt_spec(): from personal_agent.workspace.constants import PROMPT_DIR from personal_agent.workspace.constants import PROMPT_KIND - from core.services.prompt_version_service import KindSpec + from core.llm.prompt_kinds import KindSpec return KindSpec((PROMPT_KIND,), "independent", "个人智能体", "default - 个人智能体", "个人智能体的平台规则与工作区文件模板。模板只在新建工作区时写入一次；" "{skill_ids} {name} 为运行时变量，{task_id} 统一渲染为 （真实 ID 写在任务消息里，各任务共享缓存前缀）；任务提示词末尾自动附上可用技能清单", root=PROMPT_DIR) @@ -15,8 +16,8 @@ def prompt_part(part_id): from personal_agent.workspace.constants import PROMPT_DIR from personal_agent.workspace.constants import PROMPT_KIND try: - from core.services import prompt_version_service as pvs - text = pvs.render_active_prompt_part(PROMPT_KIND, part_id) + from core.llm import prompt_versions as pvs + text = prompt_rendering_module.render_active_prompt_part(PROMPT_KIND, part_id) except Exception: text = None return text if text is not None else (PROMPT_DIR / (part_id + ".system.md")).read_text(encoding="utf-8").strip() diff --git a/src/backend/prompts/desktop_templates.py b/src/backend/prompts/desktop_templates.py index 12719ad3..8156d313 100644 --- a/src/backend/prompts/desktop_templates.py +++ b/src/backend/prompts/desktop_templates.py @@ -1,4 +1,5 @@ """Versioned desktop templates. Facts and selection stay in runtime code.""" +from core.llm import prompt_rendering as prompt_rendering_module from contextlib import contextmanager from contextvars import ContextVar from prompts.provider import render_template @@ -7,9 +8,9 @@ def desktop_parts(): - from core.services import prompt_version_service as pvs + from core.llm import prompt_versions as pvs version = _override.get() - return version.get("parts", []) if version is not None else pvs.effective_parts("desktop") + return version.get("parts", []) if version is not None else prompt_rendering_module.effective_parts("desktop") def render_desktop_part(part_id, **variables): diff --git a/src/backend/prompts/desktop_workspace.py b/src/backend/prompts/desktop_workspace.py index affc727b..5e7fe293 100644 --- a/src/backend/prompts/desktop_workspace.py +++ b/src/backend/prompts/desktop_workspace.py @@ -9,11 +9,11 @@ def build_environment_context(ctx: dict, *, current_date: str) -> str: """Describe the project execution cwd and separate session scratch location.""" from core.sandbox.workspace.paths import WORKSPACE from core.sandbox.workspace.desktop import workspace_directory - from core.llm.tool_permissions import normalize_approval_mode - from core.services.local_grant_service import grants_for_gate, policy_for_gate + from core.sandbox.permissions.approval import normalize_approval_mode + from features.desktop.local.grants import grants_for_gate, policy_for_gate session = str(ctx.get("sandbox_session_id") or ctx.get("chat_id") or "").strip() - from core.services.project_scope import project_scope_from_context + from core.projects.edition import project_scope_from_context scope = project_scope_from_context(ctx) cwd = workspace_directory(WORKSPACE, session, scope=scope, validate=False) root = Element("environment_context") diff --git a/src/backend/prompts/kb_lite_section.py b/src/backend/prompts/kb_lite_section.py index fe201f6c..55651493 100644 --- a/src/backend/prompts/kb_lite_section.py +++ b/src/backend/prompts/kb_lite_section.py @@ -57,7 +57,7 @@ def _build_kb_lite_section(enabled_kb_ids: Optional[List[str]]) -> str: # ── Externally managed shared collections — cached, no extra HTTP calls ── if external_ids: try: - from core.kb.external_provider import is_enabled, list_collections + from features.knowledge.edition import is_enabled, list_collections if is_enabled(): external_set = set(external_ids) @@ -76,7 +76,7 @@ def _build_kb_lite_section(enabled_kb_ids: Optional[List[str]]) -> str: if local_ids: try: from core.db.engine import SessionLocal - from core.db.models import KBSpace + from features.knowledge.models import KBSpace with SessionLocal() as db: spaces = ( diff --git a/src/backend/prompts/project_section.py b/src/backend/prompts/project_section.py index dea1e497..d0b92829 100644 --- a/src/backend/prompts/project_section.py +++ b/src/backend/prompts/project_section.py @@ -7,6 +7,7 @@ """ from __future__ import annotations +from core.llm import prompt_versions as prompt_versions_module from prompts.prompt_runtime import _load_db_prompt_parts # noqa: E402 (cycle-safe) @@ -77,8 +78,8 @@ def _get_project_mode_template() -> str: return content # 2) parts of the active system version (edited in the admin UI) try: - from core.services import prompt_version_service as pvs - active = pvs.get_active_version("system") + from core.llm import prompt_versions as pvs + active = prompt_versions_module.get_active_version("system") if active: for p in active.get("parts") or []: if (p.get("part_id") or "").strip() != PROJECT_MODE_PART_ID: @@ -119,7 +120,7 @@ def _build_project_section( from prompts.provider import render_template name = (project_name or "").strip() or "(未命名项目)" - from core.services.edition_workspace import folder_scope_label + from core.files.edition import folder_scope_label scope_text = folder_scope_label(folder_kind) diff --git a/src/backend/prompts/prompt_runtime.py b/src/backend/prompts/prompt_runtime.py index 6f783eae..7fca1941 100755 --- a/src/backend/prompts/prompt_runtime.py +++ b/src/backend/prompts/prompt_runtime.py @@ -5,6 +5,8 @@ """ from __future__ import annotations +from core.llm import prompt_storage as prompt_storage_module +from core.llm import prompt_versions as prompt_versions_module from datetime import datetime import os @@ -13,7 +15,7 @@ from time import monotonic from typing import Any, Dict, Iterable, List, Optional, Tuple -from core.llm.execution_manifest import PromptManifestBuilder, stable_hash +from core.harness.execution_manifest import PromptManifestBuilder, stable_hash from prompts.prompt_config import PromptConfig from prompts.provider import ( FilesystemPromptProvider, @@ -171,9 +173,9 @@ def invalidate_prompt_cache() -> None: # Also drop prompt_version_service cached payload try: - from core.services import prompt_version_service as pvs + from core.llm import prompt_versions as pvs - pvs.invalidate_cache() + prompt_storage_module.invalidate_cache() except Exception: pass @@ -235,9 +237,9 @@ def build_subagent_system_prompt( # Prefer active version's parts when available (map by part_id suffix) _active_parts: Dict[str, str] = {} try: - from core.services import prompt_version_service as pvs + from core.llm import prompt_versions as pvs - _av = pvs.get_active_version("system") + _av = prompt_versions_module.get_active_version("system") if _av: for p in _av.get("parts") or []: if not p.get("is_enabled", True): @@ -383,14 +385,14 @@ def ensure_project_mode_part_seeded() -> None: log = logging.getLogger(__name__) try: - from core.services import prompt_version_service as pvs + from core.llm import prompt_versions as pvs # First make sure the active version exists (first cold start seeds from filesystem md) try: - pvs.seed_from_filesystem() + prompt_versions_module.seed_from_filesystem() except Exception: pass - active = pvs.get_active_version("system") + active = prompt_versions_module.get_active_version("system") if not active or not active.get("id"): log.warning("[prompt_seed] no active system version; skipped project_mode seed") return @@ -408,7 +410,7 @@ def ensure_project_mode_part_seeded() -> None: "is_enabled": True, } ) - pvs.upsert_version( + prompt_versions_module.upsert_version( "system", active["id"], name=active.get("name"), diff --git a/src/backend/prompts/system_builder.py b/src/backend/prompts/system_builder.py index a95e93ad..152e7389 100644 --- a/src/backend/prompts/system_builder.py +++ b/src/backend/prompts/system_builder.py @@ -1,5 +1,6 @@ """System prompt assembly; cache and public hooks live in prompt_runtime.""" from __future__ import annotations +from core.llm import prompt_versions as prompt_versions_module from prompts import prompt_runtime as runtime def build_system_prompt( @@ -66,9 +67,9 @@ def build_system_prompt( # Active pool version (preferred source) — bust cache on change active_version_key: tuple[str, str] = ("", "") try: - from core.services import prompt_version_service as pvs + from core.llm import prompt_versions as pvs - _av = pvs.get_active_version("system") + _av = prompt_versions_module.get_active_version("system") if _av: active_version_key = (str(_av.get("id") or ""), str(_av.get("updated_at") or "")) except Exception: @@ -160,9 +161,9 @@ def _record_section( # collision for backward compatibility with rows created by the old admin UI. active_system_version: runtime.Optional[runtime.Dict[str, runtime.Any]] = None try: - from core.services import prompt_version_service as pvs + from core.llm import prompt_versions as pvs - active_system_version = pvs.get_active_version("system") + active_system_version = prompt_versions_module.get_active_version("system") except Exception: active_system_version = None @@ -382,7 +383,7 @@ def _sort_key(pid: str) -> int: sensitive=True, ) - from core.services.edition_workspace import workspace_instructions + from core.files.edition import workspace_instructions workspace_hint = workspace_instructions() if workspace_hint: base = (base + "\n\n" + workspace_hint).strip() @@ -403,7 +404,7 @@ def _sort_key(pid: str) -> int: local_slug=ctx.get("project_local_slug") or "", ) else: - from core.services.edition_workspace import context_directory + from core.files.edition import context_directory proj_section = runtime._build_project_section( workspace_path=context_directory(ctx), project_name=ctx.get("project_name") or "", diff --git a/src/backend/scripts/_loop_convergence_unit.py b/src/backend/scripts/_loop_convergence_unit.py deleted file mode 100644 index f03727aa..00000000 --- a/src/backend/scripts/_loop_convergence_unit.py +++ /dev/null @@ -1,147 +0,0 @@ -"""Deterministic unit check: exit paths of the requirement ledger + read-only review sub-agent. - -驱动器 v2（去预算化 + 规划器侦察/重规划 + 混合验收）后的收敛冒烟：不跑真实 -LLM/沙箱/git，桩掉 worker/规划/评审/沙箱，验证三条出口： - A. 全部需求通过 → completed（无机检需求翻牌仍需二次复核）； - B. 单需求连续无推进 → stalls 到停滞上限 → blocked →（重规划桩返回 None）→ - budget_exhausted 部分完成； - C. done 被二次复核驳回 → 不翻牌，直至 blocked。 -完成 <1s。Run: docker exec hugagent-backend python -m scripts._loop_convergence_unit -（正式回归见 tests/orchestration/test_autonomous_loop_driver.py） -""" -import asyncio - -import orchestration.autonomous_loop as al -from orchestration.autonomous_loop import LoopBudget, run_autonomous_loop -from orchestration.loop_evaluator import CONTINUE, DONE, GoalSpec - - -class _StubPolicy: - version = "unit" - strategy_change_after = 2 - max_attempts_per_requirement = 6 - budget_multiplier = 1.0 - - -async def _fake_worker(**kwargs): - return {"text": "stub work", "tokens": 10, "tool_calls": 1} - - -def _make_fake_review(verdicts): - """按调用顺序回放判定；done 带非空证据（否则 driver 会降级为 continue）。""" - calls = {"i": 0} - - async def _fake_review(**kwargs): - i = calls["i"] - calls["i"] += 1 - v = verdicts[min(i, len(verdicts) - 1)] - return {"verdict": v, "criteria_hit": ["stub"], - "evidence": "reviewer 读到 /proj/index.html 含目标内容" if v == DONE else "", - "progress": False, - "feedback": "stub 反馈"} - - return _fake_review, calls - - -def _fake_plan_n(n): - async def _fake(**kwargs): - return [{"id": f"R{i}", "description": f"stub 需求{i}"} for i in range(1, n + 1)] - return _fake - - -async def _noop_criteria(**kwargs): - return ["stub 标准"] - - -async def _sbx_noop(cmd, **kwargs): - return (0, "", "") - - -async def _noop_write_file(path, content, **kwargs): - return None - - -async def _noop_read_file(path, **kwargs): - return "" # → _read_ledger returns None → fresh init every time - - -async def _noop_scout(**kwargs): - return "" - - -async def _noop_replan(**kwargs): - return None - - -async def _worktree_clean(*a, **kwargs): - return False - - -def _patch_common(): - al._run_worker_iteration = _fake_worker - al._sbx_exec = _sbx_noop - al._write_file = _noop_write_file - al._read_file = _noop_read_file - al._git_worktree_changed = _worktree_clean - al.extract_acceptance_criteria = _noop_criteria - al.scout_workspace = _noop_scout - al.replan_remaining = _noop_replan - al._resolve_loop_policy = lambda **kw: _StubPolicy() - - import core.services.ontology_service as osvc - - osvc.build_user_ontology_runtime = lambda **kw: ( - False, {"enabled": False, "packs": [], "review_level": "none"}) - - -_MAX_ATTEMPTS = _StubPolicy.max_attempts_per_requirement - - -async def main() -> None: - _patch_common() - - # ── Scenario A: 3 需求，每条第 2 轮 done 且二次复核通过 → completed。 - al.plan_requirements = _fake_plan_n(3) - seq = [] - for _ in range(3): - seq += [CONTINUE, DONE, DONE] # done 后紧跟二次复核的 done - al.review_requirement, _ = _make_fake_review(seq) - resA = await run_autonomous_loop( - loop_id="convA", user_id="unit", goal_spec=GoalSpec(objective="stub", acceptance_criteria=["c"]), - budget=LoopBudget(), # 默认不限预算 - session_id="loop-convA", - ) - print(f"[A] status={resA.status} iters={resA.iterations} final={resA.final_score} reason={resA.reason}") - assert resA.status == "completed", resA.status - assert resA.final_score == 1.0, resA.final_score - - # ── Scenario B: 1 需求永远 continue（无推进）→ stalls 到上限 → blocked → budget_exhausted。 - al.plan_requirements = _fake_plan_n(1) - al.review_requirement, _ = _make_fake_review([CONTINUE]) - resB = await run_autonomous_loop( - loop_id="convB", user_id="unit", goal_spec=GoalSpec(objective="stub2", acceptance_criteria=["c"]), - budget=LoopBudget(), - session_id="loop-convB", - ) - print(f"[B] status={resB.status} iters={resB.iterations} final={resB.final_score}") - assert resB.status == "budget_exhausted", resB.status - assert resB.iterations == _MAX_ATTEMPTS, resB.iterations - assert resB.final_score == 0.0, resB.final_score - - # ── Scenario C: done 被二次复核驳回 → 不翻牌，直至 blocked。 - al.plan_requirements = _fake_plan_n(1) - al.review_requirement, _ = _make_fake_review([DONE, CONTINUE]) - resC = await run_autonomous_loop( - loop_id="convC", user_id="unit", goal_spec=GoalSpec(objective="stub3", acceptance_criteria=["c"]), - budget=LoopBudget(), - session_id="loop-convC", - ) - print(f"[C] status={resC.status} iters={resC.iterations}") - assert resC.status == "budget_exhausted", "done 被二次复核驳回不应翻牌" - assert resC.iterations == _MAX_ATTEMPTS, resC.iterations - - print("CONVERGENCE_UNIT_OK") - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/src/backend/scripts/_loop_db_resume_unit.py b/src/backend/scripts/_loop_db_resume_unit.py deleted file mode 100644 index 1b475a2c..00000000 --- a/src/backend/scripts/_loop_db_resume_unit.py +++ /dev/null @@ -1,92 +0,0 @@ -"""Deterministic unit check: resume from the DB mirror ledger (phenomenon-1 fix). - -Simulates the resume scenario where "the sandbox has been wiped (rebuild/restart) but the DB still has a ledger": all sandbox reads return empty -(_read_ledger->None), and load_ledger returns only a ledger of "R1 passed, R2 not passed, iteration=3". -Asserts the driver **resumes from the DB ledger** (seq starts from 3, no re-decompose to re-split the goal), and reaches completed after R2 passes. -Also verifies save_ledger receives mirror writes throughout. No real LLM/sandbox/git, <1s. - -Run: docker exec hugagent-backend python -m scripts._loop_db_resume_unit -""" -import asyncio - -import orchestration.autonomous_loop as al -from orchestration.autonomous_loop import LoopBudget, run_autonomous_loop -from orchestration.loop_evaluator import DONE, GoalSpec - - -async def _fake_worker(**kwargs): - return {"text": "stub work", "tokens": 10, "tool_calls": 1} - - -async def _fake_review(**kwargs): - # R2 review passes directly (with non-empty evidence); the second re-check also goes here -> passed. - return {"verdict": DONE, "criteria_hit": ["stub"], - "evidence": "reviewer 读到产出满足需求", "feedback": "满足"} - - -async def _decompose_must_not_run(**kwargs): - raise AssertionError("续跑不应重新拆解目标（decompose 被调 = 从0开始的 bug）") - - -async def _noop_criteria(**kwargs): - return ["stub 标准"] - - -async def _sbx_noop(cmd, **kwargs): - return (0, "", "") - - -async def _noop_write_file(path, content, **kwargs): - return None - - -async def _empty_read_file(path, **kwargs): - return "" # sandbox wiped: _read_ledger->None, handoffs empty - - -async def main() -> None: - al._run_worker_iteration = _fake_worker - al.decompose_requirements = _decompose_must_not_run - al.review_requirement = _fake_review - al.extract_acceptance_criteria = _noop_criteria - al._sbx_exec = _sbx_noop - al._write_file = _noop_write_file - al._read_file = _empty_read_file - - saved = {"ledgers": []} - - def _save_ledger(led): - saved["ledgers"].append({r["id"]: r.get("passes") for r in led["requirements"]}) - - # DB ledger: R1 passed, R2 pending, already ran to iteration 3 (includes new ledger fields attempts/blocked; also tolerates old fields being ignored). - db_ledger = { - "objective": "stub 目标", - "iteration": 3, - "criteria": ["stub 标准"], - "requirements": [ - {"id": "R1", "description": "已完成需求", "passes": True, "evidence": "done", - "attempts": 2, "blocked": False}, - {"id": "R2", "description": "待完成需求", "passes": False, "evidence": "", - "attempts": 0, "blocked": False}, - ], - } - - gs = GoalSpec(objective="stub 目标", acceptance_criteria=["stub"]) - res = await run_autonomous_loop( - loop_id="dbResume", user_id="unit", goal_spec=gs, - budget=LoopBudget(max_iters=10, max_wall_clock_s=60, max_tokens=10_000_000), - session_id="loop-dbResume", - load_ledger=lambda: db_ledger, - save_ledger=_save_ledger, - ) - print(f"[DB-RESUME] status={res.status} iters={res.iterations} reason={res.reason}") - # Resuming from iteration 3: only 1 more iteration (the 4th) is needed to flip R2 -> completed. - assert res.status == "completed", res.status - assert res.iterations == 4, f"应从第3轮续跑、第4轮完成 R2，实际 {res.iterations}" - assert saved["ledgers"], "save_ledger 应收到镜像写" - assert saved["ledgers"][-1] == {"R1": True, "R2": True}, saved["ledgers"][-1] - print("DB_RESUME_UNIT_OK") - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/src/backend/scripts/_loop_evaluator_route_unit.py b/src/backend/scripts/_loop_evaluator_route_unit.py deleted file mode 100644 index acfbf053..00000000 --- a/src/backend/scripts/_loop_evaluator_route_unit.py +++ /dev/null @@ -1,88 +0,0 @@ -"""Deterministic unit check of self_verify's auto-routing: verify.sh exists -> script (rule) decides; absent -> LLM evaluator decides. - -Stubs out the sandbox and LLM, only verifying evaluate_iteration's routing and verdict. <1s, does not touch real infrastructure. -Run: docker exec hugagent-backend python -m scripts._loop_evaluator_route_unit -""" -import asyncio - -import orchestration.loop_evaluator as le -from orchestration.loop_evaluator import ( - CONTINUE, DONE, GoalSpec, VerifyResult, evaluate_iteration, -) - - -async def main() -> None: - # ── Scenario 1: verify.sh exists -> script path (rule); self_verify with no threshold does not early-stop on a valid solution ── - async def _exists_true(**kw): - return True - - async def _fake_run_verify(cmd, **kw): - return VerifyResult(ran=True, exit_code=0, stdout="SCORE=88.0", stderr="", score=None) - - le._verify_script_exists = _exists_true - le._run_verify = _fake_run_verify - # Disable the LLM feedback call to avoid hitting the real model - async def _no_fb(**kw): - return None - le._llm_feedback = _no_fb - - gs = GoalSpec(objective="优化X", mode="self_verify") # target None - v1 = await evaluate_iteration(goal_spec=gs, session_id="s", user_id="u", - iteration_summary="w", model_name="fast") - print(f"[1 script] verdict={v1['verdict']} score={v1['score']} decided_by={v1['decided_by']} exit={v1['verify_exit']}") - assert v1["verdict"] == CONTINUE, v1 # valid solution (exit0) but no threshold -> no early stop - assert v1["score"] == 88.0 - assert v1["decided_by"] == "environment" - assert v1["verify_exit"] == 0 - - # ── Scenario 2: verify.sh absent -> LLM evaluator decides done ── - async def _exists_false(**kw): - return False - - async def _fake_judge_done(prompt, **kw): - return '{"verdict":"done","criteria_hit":["满足A"],"evidence":"...","reason":"全部满足"}' - - le._verify_script_exists = _exists_false - le._judge_once = _fake_judge_done - gs2 = GoalSpec(objective="写一篇文案", mode="self_verify", - acceptance_criteria=["语气正式", "含要点A", "无错别字"]) - v2 = await evaluate_iteration(goal_spec=gs2, session_id="s", user_id="u", - iteration_summary="成品+证据", model_name="fast") - print(f"[2 llm-done] verdict={v2['verdict']} decided_by={v2['decided_by']}") - assert v2["verdict"] == DONE, v2 - assert v2["decided_by"] == "llm" - - # ── Scenario 3: LLM evaluator decides continue (insufficient evidence) ── - async def _fake_judge_continue(prompt, **kw): - return '{"verdict":"continue","reason":"要点A未覆盖"}' - - le._judge_once = _fake_judge_continue - v3 = await evaluate_iteration(goal_spec=gs2, session_id="s", user_id="u", - iteration_summary="半成品", model_name="fast") - print(f"[3 llm-continue] verdict={v3['verdict']} decided_by={v3['decided_by']}") - assert v3["verdict"] == CONTINUE, v3 - assert v3["decided_by"] == "llm" - - # ── Scenario 4: self_verify with no script and no criteria -> lazily extract acceptance criteria (stub returns a non-array -> fall back to [objective]) - # then LLM decides; here judge still returns continue, verifying the safety property "never falsely judge done". ── - gs4 = GoalSpec(objective="模糊目标", mode="self_verify", acceptance_criteria=[]) - v4 = await evaluate_iteration(goal_spec=gs4, session_id="s", user_id="u", - iteration_summary="x", model_name="fast") - print(f"[4 lazy-criteria] verdict={v4['verdict']} decided_by={v4['decided_by']}") - assert v4["verdict"] == CONTINUE, v4 - assert gs4.acceptance_criteria == ["模糊目标"], gs4.acceptance_criteria # lazy fallback written back - - # ── Scenario 5: acceptance-criteria extraction parsing ── - async def _fake_judge_criteria(prompt, **kw): - return '这是标准：\n["语气正式","覆盖要点A","结尾有行动号召"]' - - le._judge_once = _fake_judge_criteria - crit = await le.extract_acceptance_criteria(objective="写文案", model_name="fast", user_id="u") - print(f"[5 extract] criteria={crit}") - assert crit == ["语气正式", "覆盖要点A", "结尾有行动号召"], crit - - print("EVALUATOR_ROUTE_UNIT_OK") - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/src/backend/scripts/_loop_integration_test.py b/src/backend/scripts/_loop_integration_test.py deleted file mode 100644 index f05263e9..00000000 --- a/src/backend/scripts/_loop_integration_test.py +++ /dev/null @@ -1,88 +0,0 @@ -"""Full production-chain integration test: LoopService creates loop -> start_autonomous_loop_run(ChatRun) --> driver runs (worker does the work + a read-only reviewer sub-agent personally verifies the real output) -> persist_result writes to DB. - -No-script verification: the verdict comes from review_requirement opening the produced real file and checking it. Here we use a simple goal not bound to any project -(worker writes a /workspace file, the reviewer reads /workspace of the same session); passing it proves the whole chain -"worker -> reviewer sub-agent -> flip -> persist" works. Requires real LLM/sandbox. - - docker exec hugagent-backend python /app/src/backend/scripts/_loop_integration_test.py -""" -import asyncio - -USER_ID = "copytest_9c0cf31f" - - -async def main(): - from core.db.engine import SessionLocal - from core.services.loop_service import LoopService - from orchestration import chat_run_executor - from orchestration.autonomous_loop import _sbx_exec, _read_ledger - from core.services.chat_service import ChatService - - # 1) Create loop (DB) -- goal is a concrete output the reviewer can verify from real files (with no verify command / numeric score). - db = SessionLocal() - loop = LoopService(db).create_loop( - user_id=USER_ID, title="集成测试-reviewer", - goal_spec={ - "objective": "在沙箱 /workspace 下创建 report.html，页面标题为「集成测试通过」，" - "正文包含一个

集成测试通过

和一段说明文字。", - "acceptance_criteria": [ - "/workspace/report.html 文件存在", - "文件内容含

集成测试通过

", - ], - }, - budget={"max_iters": 4, "max_wall_clock_s": 1800, "max_tokens": 3000000}, - ) - loop_id = loop.loop_id - goal_spec = dict(loop.goal_spec) - budget = dict(loop.budget) - session = f"loop-{loop_id}" - chat_id = f"loopchat_{loop_id}" - ChatService(db).ensure_session(chat_id=chat_id, user_id=USER_ID, title="集成测试", - extra_data={"autonomous_loop": True, "loop_id": loop_id}) - loop.chat_id = chat_id - db.commit() - db.close() - print(f"[it] created loop {loop_id}") - - # 2) Clean up old state - await _sbx_exec("rm -f /workspace/report.html /workspace/feature_list.json", - session_id=session, user_id=USER_ID) - - # 3) Go through the production entry point start_autonomous_loop_run (ChatRun background + Redis Stream; no project_id -> isolated sandbox) - run = await chat_run_executor.start_autonomous_loop_run( - loop_id=loop_id, chat_id=chat_id, user_id=USER_ID, - goal_spec=goal_spec, budget=budget, model_name="fast", - ) - print(f"[it] started run {run.run_id}, awaiting...") - task = chat_run_executor._active_runs.get(run.run_id) - if task: - await task - - # 4) Verify DB persistence + ledger + real output file - db = SessionLocal() - svc = LoopService(db) - lp = svc.get_loop(loop_id) - its = svc.list_iterations(loop_id) - print(f"[it] DB loop status={lp.status} iters={lp.iteration_count} " - f"score={lp.final_score} tokens={lp.tokens_spent}") - print(f"[it] DB iteration rows: {[(i.seq, i.verdict, i.decided_by) for i in its]}") - db.close() - - ledger = await _read_ledger(session_id=session, user_id=USER_ID) - if ledger: - reqs = ledger.get("requirements", []) - print(f"[it] feature_list.json: iteration={ledger.get('iteration')} " - f"requirements={len(reqs)} passed={sum(1 for r in reqs if r.get('passes'))}") - code, out, _ = await _sbx_exec( - "grep -c '集成测试通过' /workspace/report.html 2>/dev/null || echo MISSING", - session_id=session, user_id=USER_ID) - print(f"[it] 真实产出核验: report.html grep → {out.strip()}") - - ok = (lp.status in ("completed", "budget_exhausted") and len(its) >= 1 - and any(i.decided_by == "reviewer" for i in its)) - print(f"\nINTEGRATION_{'OK' if ok else 'FAIL'} status={lp.status} iters={len(its)}") - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/src/backend/scripts/_loop_llmeval_smoke.py b/src/backend/scripts/_loop_llmeval_smoke.py deleted file mode 100644 index c9fb8b62..00000000 --- a/src/backend/scripts/_loop_llmeval_smoke.py +++ /dev/null @@ -1,48 +0,0 @@ -"""Runs the LLM-evaluator path of self_verify against a real LLM: qualitative goal, -the worker does not write verify.sh, the driver directly calls a standalone LLM -evaluator to judge done/continue against acceptance criteria. - -How to run (inside the container, nohup recommended): python -u -m scripts._loop_llmeval_smoke -""" -import asyncio -import uuid - -from orchestration.autonomous_loop import LoopBudget, run_autonomous_loop -from orchestration.loop_evaluator import GoalSpec - - -async def main() -> None: - loop_id = "le" + uuid.uuid4().hex[:8] - gs = GoalSpec( - objective=( - "为一款面向中小企业的智能办公助手写一句中文宣传语，写入 /workspace/slogan.txt。" - "要求：不超过20字、突出「省时」与「智能」两个卖点、语气积极。" - "写完在回复末尾附一段「证据说明」，逐条对照说明如何满足要求。" - ), - mode="self_verify", - ) - budget = LoopBudget(max_iters=3, max_wall_clock_s=900, max_tokens=1_500_000) - - async def emit(ev): - t = ev.get("type") - if t in ("iteration_started", "iteration_evaluated", "loop_completed"): - print(f" · {t}: seq={ev.get('seq')} verdict={ev.get('verdict')} " - f"score={ev.get('score')} decided_by={ev.get('decided_by')} " - f"status={ev.get('status')} reason={str(ev.get('reason',''))[:100]}") - - print(f"[llmeval] start {loop_id}") - res = await run_autonomous_loop( - loop_id=loop_id, user_id="lesmoke01", goal_spec=gs, budget=budget, - model_name=None, evaluator_model="fast", worker_max_iters=8, - session_id=f"loop-{loop_id}", emit=emit, - ) - print(f"\n[llmeval] RESULT status={res.status} iters={res.iterations} reason={res.reason}") - print(f"[llmeval] criteria={gs.acceptance_criteria}") - decided = [h.get("decided_by") for h in res.history] - print(f"[llmeval] decided_by per iter={decided}") - # Qualitative goal: an LLM verdict (llm) should have appeared at least once, or been achieved - print("LLMEVAL_OK" if res.status in ("completed", "budget_exhausted") else "LLMEVAL_UNEXPECTED") - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/src/backend/scripts/_loop_preflight.py b/src/backend/scripts/_loop_preflight.py deleted file mode 100644 index 6ace86c3..00000000 --- a/src/backend/scripts/_loop_preflight.py +++ /dev/null @@ -1,73 +0,0 @@ -"""Preflight: without going through the worker LLM, directly verifies the judge + verify pipeline + the evaluator's done/continue decision.""" -import asyncio -import os -import pathlib - -USER_ID = os.environ.get("LOOP_USER", "copytest_9c0cf31f") -JUDGE_SRC = (pathlib.Path(__file__).parent / "edgebench" / "perm_opt_judge.py").read_text("utf-8") -TARGET = 1.5 - -# Reference solution: black-box 2-opt local search (only calls cost, does not look inside) — should reliably reach ratio >= 1.5 -GOOD_SOLUTION = ''' -import random, math -def solve(n, cost): - rng = random.Random(0) - cur = list(range(n)); cur_c = cost(cur) - best, best_c = cur[:], cur_c - T = 0.5 - for _ in range(6000): - a, b = sorted((rng.randrange(n), rng.randrange(n))) - if a == b: continue - cand = cur[:]; cand[a:b+1] = reversed(cand[a:b+1]) - c = cost(cand) - if c < cur_c or rng.random() < math.exp((cur_c - c) / max(T, 1e-9)): - cur, cur_c = cand, c - if c < best_c: best, best_c = cand[:], c - T *= 0.999 - return best -''' - -BAD_SOLUTION = ''' -def solve(n, cost): - return list(range(n)) # identity permutation, not optimized → score≈1.0 < target -''' - - -async def main(): - from orchestration.autonomous_loop import _sbx_exec, _write_file - from orchestration.loop_evaluator import GoalSpec, evaluate_iteration - - session = "loop-preflight2" - verify = "cd /workspace && python perm_opt_judge.py" - await _write_file("/workspace/perm_opt_judge.py", JUDGE_SRC, session_id=session, user_id=USER_ID) - - goal = GoalSpec( - objective="permutation optimization", acceptance_criteria=["合法排列", f"score>={TARGET}"], - verify_cmd=verify, score_regex=r"SCORE=([0-9.]+)", target_score=TARGET, maximize=True, - ) - - async def run_case(label, sol): - await _sbx_exec("rm -f /workspace/solution.py", session_id=session, user_id=USER_ID) - if sol: - await _write_file("/workspace/solution.py", sol, session_id=session, user_id=USER_ID) - r = await _sbx_exec(verify, session_id=session, user_id=USER_ID, timeout=120) - print(f"{label}: exit={r.exit_code if r else '?'} out={(r.stdout.strip() if r else '')}") - v = await evaluate_iteration(goal_spec=goal, session_id=session, user_id=USER_ID, - iteration_summary="", use_llm_for_fuzzy=False) - print(f" evaluator verdict={v['verdict']} score={v['score']} by={v['decided_by']}") - return v - - print("=== A) 无解 → 期望 continue ===") - va = await run_case("A missing", None) - print("=== B) 未优化(恒等) → 期望 continue(exit0 但 score= TARGET) - print("\nPREFLIGHT_OK" if ok else "PREFLIGHT_FAIL") - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/src/backend/scripts/_loop_selfverify_smoke.py b/src/backend/scripts/_loop_selfverify_smoke.py deleted file mode 100644 index 570ced1d..00000000 --- a/src/backend/scripts/_loop_selfverify_smoke.py +++ /dev/null @@ -1,66 +0,0 @@ -"""self_verify (conversation mode) smoke test: no verify_cmd given, the worker builds /workspace/verify.sh itself. - -Verification points: - - driver supplies self_verify with a default verify_cmd / score_regex; - - the worker builds verify.sh each iteration, and the driver independently produces a score; - - with no threshold it will not call done on the first valid solution, converging via stagnation (or budget). - -Run (inside container): - docker exec hugagent-backend python -m scripts._loop_selfverify_smoke -""" -import asyncio -import uuid - -from orchestration.autonomous_loop import LoopBudget, run_autonomous_loop -from orchestration.loop_evaluator import GoalSpec - - -async def main() -> None: - loop_id = "sv_" + uuid.uuid4().hex[:8] - gs = GoalSpec( - objective=( - "在 /workspace/solution.py 写一个函数 nth_prime(n) 返回第 n 个素数（n 从 1 起，" - "nth_prime(1)==2）。尽量正确、边界稳妥。" - ), - mode="self_verify", # key point: conversation mode, no verify_cmd - ) - budget = LoopBudget(max_iters=6, max_wall_clock_s=1200, max_tokens=1_500_000) - - events = [] - - async def emit(ev): - events.append(ev) - t = ev.get("type") - if t in ("iteration_started", "iteration_evaluated", "loop_converged", - "loop_stagnation", "loop_completed"): - print(f" · {t}: " - f"seq={ev.get('seq')} verdict={ev.get('verdict')} " - f"score={ev.get('score')} best={ev.get('best_score')} " - f"status={ev.get('status')} reason={str(ev.get('reason',''))[:80]}") - - print(f"[selfverify] start loop {loop_id}") - result = await run_autonomous_loop( - loop_id=loop_id, - user_id="svsmoke01", - goal_spec=gs, - budget=budget, - model_name=None, - evaluator_model="fast", - worker_max_iters=12, - session_id=f"loop-{loop_id}", - emit=emit, - ) - print(f"\n[selfverify] RESULT status={result.status} iters={result.iterations} " - f"final_score={result.final_score} reason={result.reason}") - # driver should have supplied self_verify with the default script convention - assert gs.verify_cmd and "verify.sh" in gs.verify_cmd, gs.verify_cmd - assert gs.score_regex, "score_regex 未补默认" - # must have produced at least one valid solution (with a score) or exhausted budget -- never 0 iterations - assert result.iterations >= 1 - scored = [h for h in result.history if h.get("score") is not None] - print(f"[selfverify] scored_iters={len(scored)} scores={[h['score'] for h in scored]}") - print("SELFVERIFY_OK" if result.status in ("completed", "budget_exhausted") else "SELFVERIFY_UNEXPECTED") - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/src/backend/scripts/_loop_smoke.py b/src/backend/scripts/_loop_smoke.py deleted file mode 100644 index 4c6df485..00000000 --- a/src/backend/scripts/_loop_smoke.py +++ /dev/null @@ -1,90 +0,0 @@ -"""Smoke test: verify create_agent_executor can actually run one round of reply (LLM path) + sandbox bash (worker path). - - docker exec hugagent-backend python /app/src/backend/scripts/_loop_smoke.py -""" -import asyncio -import sys - -USER_ID = "copytest_9c0cf31f" - - -async def test_llm(): - from core.llm.factory import create_agent_executor - from core.llm.mcp_manager import close_clients - from orchestration.streaming import StreamingAgent - - agent, clients = await create_agent_executor( - disable_tools=True, - enabled_skill_ids=[], - chat_mode="fast", - current_user_id=USER_ID, - ) - sa = StreamingAgent(agent, clients) - text = "" - try: - async for et, payload in sa.stream( - [{"role": "user", "content": "只回复两个字：收到"}], - {"user_id": USER_ID, "enable_thinking": False, "chat_mode": "fast"}, - ): - if et == "text_delta": - text += payload - elif et == "error": - print("LLM_ERROR:", payload) - return False - finally: - await close_clients(clients) - print(f"LLM_OK text={text!r} usage={sa.get_usage()}") - return True - - -async def test_sandbox_bash(): - """Run a bash command with a tools-enabled agent to verify the sandbox is executable.""" - from core.llm.factory import create_agent_executor - from core.llm.mcp_manager import close_clients - from orchestration.streaming import StreamingAgent - - sess = f"smoke-{USER_ID}" - agent, clients = await create_agent_executor( - current_user_id=USER_ID, - chat_mode="fast", - sandbox_session_id=sess, - isolated=True, - max_iters=6, - ) - sa = StreamingAgent(agent, clients) - saw_tool = False - text = "" - try: - async for et, payload in sa.stream( - [{"role": "user", "content": "用 bash 执行 `echo LOOP_SANDBOX_OK` 并把输出原样告诉我。"}], - {"user_id": USER_ID, "enable_thinking": False, "chat_mode": "fast"}, - ): - if et == "tool_call": - saw_tool = True - print("TOOL_CALL:", payload.get("name")) - elif et == "tool_result": - print("TOOL_RESULT:", str(payload.get("content"))[:200]) - elif et == "text_delta": - text += payload - elif et == "error": - print("SANDBOX_ERROR:", payload) - finally: - await close_clients(clients) - ok = "LOOP_SANDBOX_OK" in text or saw_tool - print(f"SANDBOX_{'OK' if ok else 'FAIL'} saw_tool={saw_tool} text={text[:120]!r}") - return ok - - -async def main(): - which = sys.argv[1] if len(sys.argv) > 1 else "llm" - if which in ("llm", "all"): - if not await test_llm(): - print("SMOKE_FAIL: llm") - return - if which in ("sandbox", "all"): - await test_sandbox_bash() - print("SMOKE_DONE") - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/src/backend/scripts/_loop_test_edgebench.py b/src/backend/scripts/_loop_test_edgebench.py deleted file mode 100644 index d20fdbda..00000000 --- a/src/backend/scripts/_loop_test_edgebench.py +++ /dev/null @@ -1,99 +0,0 @@ -"""Test whether the autonomous loop works using the real EdgeBench task `order_addition_permutation_optimization`. - - docker exec -e LOOP_MAX_ITERS=5 hugagent-backend \ - python /app/src/backend/scripts/_loop_test_edgebench.py - -This EdgeBench task: use metaheuristic search to find the permutation that minimizes a black-box cost (score=maximize). Scoring is independent of CPU speed -(the cost is computed once on the final permutation), which suits the rate-limited sandbox environment. It verifies the loop can: iterate → environment verification (ground truth) -→ feed the feedback back in → terminate on meeting verify (or trip the budget fallback). -""" -import asyncio -import json -import os -import pathlib - -USER_ID = os.environ.get("LOOP_USER", "copytest_9c0cf31f") -MAX_ITERS = int(os.environ.get("LOOP_MAX_ITERS", "5")) -WORKER_MODEL = os.environ.get("WORKER_MODEL") or None -TARGET = float(os.environ.get("LOOP_TARGET", "1.5")) - -JUDGE_SRC = (pathlib.Path(__file__).parent / "edgebench" / "perm_opt_judge.py").read_text("utf-8") - -OBJECTIVE = ( - "EdgeBench 任务 order_addition_permutation_optimization：用元启发式搜索找一个使**黑盒成本函数**\n" - "最小的排列。在沙箱 /workspace 写一个 `solution.py`，暴露：\n" - " solve(n, cost) -> list # 返回 range(n) 的一个排列（每个下标恰好出现一次）\n" - " # n 是元素个数；cost 是一个函数：传入一个排列(list)，返回该排列的成本(float，越小越好)。\n" - " # cost 是黑盒：只能调用它评估排列，不能看它内部。目标是让 cost(返回值) 尽量小。\n" - "推荐：模拟退火 / 2-opt 段反转 / 局部搜索，反复调用 cost 迭代改进。\n" - "评测器运行 `python /workspace/perm_opt_judge.py` 打分：SCORE = 恒等排列成本 / 你的排列成本" - "（越大越好）。写完后你可以自己先运行它自检并据结果改进。" -) -ACCEPTANCE = [ - "/workspace/solution.py 提供 solve(n, cost) 且返回 range(n) 的合法排列", - f"SCORE（相对恒等基线的成本改进比）>= {TARGET}", -] -VERIFY_CMD = "cd /workspace && python perm_opt_judge.py" - - -async def main() -> None: - from orchestration.autonomous_loop import LoopBudget, run_autonomous_loop, _sbx_exec, _write_file - from orchestration.loop_evaluator import GoalSpec - - loop_id = "edgebench_permopt" - session = f"loop-{loop_id}" - - print(f"[test] seeding judge into sandbox session={session} ...", flush=True) - await _write_file("/workspace/perm_opt_judge.py", JUDGE_SRC, session_id=session, user_id=USER_ID) - await _sbx_exec("rm -f /workspace/solution.py /workspace/PROGRESS.md /workspace/handoffs.md", - session_id=session, user_id=USER_ID) - pre = await _sbx_exec("ls -la /workspace/perm_opt_judge.py && python --version", - session_id=session, user_id=USER_ID, timeout=60) - print("[test] preflight:", (pre.stdout.strip() if pre else "EXEC_FAILED"), flush=True) - - goal = GoalSpec( - objective=OBJECTIVE, acceptance_criteria=ACCEPTANCE, verify_cmd=VERIFY_CMD, - score_regex=r"SCORE=([0-9.]+)", target_score=TARGET, maximize=True, - ) - budget = LoopBudget(max_iters=MAX_ITERS, max_wall_clock_s=3600.0, max_tokens=5_000_000) - - async def emit(ev): - t = ev.get("type") - if t == "iteration_started": - print(f"\n>>> ITER {ev['seq']} start", flush=True) - elif t == "loop_tool_call": - print(f" · worker tool: {ev.get('name')}", flush=True) - elif t == "iteration_evaluated": - print(f"<<< ITER {ev['seq']} verdict={ev['verdict']} score={ev['score']} " - f"tools={ev['tool_calls']} tokens={ev['tokens']} by={ev.get('decided_by')}", flush=True) - print(f" reason: {ev['reason'][:220]}", flush=True) - print(f" evidence: {ev.get('evidence','')[:200]}", flush=True) - elif t == "loop_completed": - print(f"\n[loop] COMPLETED status={ev['status']} reason={ev['reason']}", flush=True) - - print(f"[test] START loop max_iters={MAX_ITERS} worker_model={WORKER_MODEL} target={TARGET}", flush=True) - result = await run_autonomous_loop( - loop_id=loop_id, user_id=USER_ID, goal_spec=goal, budget=budget, - model_name=WORKER_MODEL, evaluator_model="fast", worker_max_iters=18, - session_id=session, emit=emit, - ) - - print("\n" + "=" * 64) - print("LOOP RESULT:") - print(json.dumps({ - "status": result.status, "iterations": result.iterations, - "final_score": result.final_score, "tokens_spent": result.tokens_spent, - "wall_clock_s": result.wall_clock_s, "reason": result.reason, - "score_trajectory": [h.get("score") for h in result.history], - "verdicts": [h.get("verdict") for h in result.history], - }, ensure_ascii=False, indent=2)) - print("=" * 64) - effective = result.iterations >= 1 and any( - h.get("decided_by") == "environment" and h.get("score") is not None for h in result.history - ) - print(f"\nLOOP_EFFECTIVE={effective} (≥1 轮由环境验证 ground truth 打分驱动)") - print(f"LOOP_TERMINATION={result.status}") - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/src/backend/scripts/backend_entrypoint.sh b/src/backend/scripts/backend_entrypoint.sh index 1354b815..7c437768 100644 --- a/src/backend/scripts/backend_entrypoint.sh +++ b/src/backend/scripts/backend_entrypoint.sh @@ -15,7 +15,7 @@ echo "[entrypoint] Starting backend bootstrap..." DB_URL="${DATABASE_URL:-sqlite:///./hugagent.db}" if echo "${DB_URL}" | grep -qi "^sqlite"; then echo "[entrypoint] SQLite detected – using create_all() instead of alembic..." - python -c "from core.db.engine import init_db; init_db()" + python -c "from features.installed import INSTALLED_FEATURES; from core.harness.registry import load_models; load_models(INSTALLED_FEATURES); from core.harness.registry.schema_lifecycle import load_schema_lifecycle; load_schema_lifecycle(INSTALLED_FEATURES); from core.db.engine import init_db; init_db()" echo "[entrypoint] SQLite tables created successfully." else echo "[entrypoint] Running database migrations (alembic upgrade head)..." @@ -32,7 +32,7 @@ else echo "[entrypoint] Migrations applied successfully." if [ "${JX_EDITION:-ee}" = "ce" ]; then echo "[entrypoint] Reconciling the CE schema against the current metadata..." - python -c "from core.db.engine import init_db; init_db()" + python -c "from features.installed import INSTALLED_FEATURES; from core.harness.registry import load_models; load_models(INSTALLED_FEATURES); from core.harness.registry.schema_lifecycle import load_schema_lifecycle; load_schema_lifecycle(INSTALLED_FEATURES); from core.db.engine import init_db; init_db()" echo "[entrypoint] CE schema reconciliation completed." fi fi diff --git a/src/backend/scripts/desktop_cloud_wrapper.py b/src/backend/scripts/desktop_cloud_wrapper.py deleted file mode 100644 index bd33a4ef..00000000 --- a/src/backend/scripts/desktop_cloud_wrapper.py +++ /dev/null @@ -1,76 +0,0 @@ -"""Test-only ASGI wrapper: serve the backend under ``/api`` the way nginx does in front of the cloud. - -The desktop bridge always calls ``{cloud_base}/api/v1/...``; a real deployment's -nginx strips ``/api``. This wrapper strips it the same way and forwards the -lifespan scope untouched so the startup hooks run. Used by -``scripts/dual_mode_e2e.py`` to stand up a "cloud" process without nginx: - - uvicorn scripts.desktop_cloud_wrapper:app --host 0.0.0.0 --port 3001 -""" - -from api.app import app as inner - - -async def _fixture_session(scope, receive, send): - # This entry point is used only by the isolated E2E wrapper. Production - # serves api.app directly and has neither this route nor its ephemeral key. - import hmac, json, os - from pathlib import Path - from starlette.responses import JSONResponse - - key_path = Path("/tmp/capability-e2e-fixture.secret") - headers = dict(scope.get("headers") or []) - expected = key_path.read_text().strip() if key_path.is_file() else "" - supplied = headers.get(b"authorization", b"").decode("ascii", errors="ignore") - if ( - os.getenv("AUTH_MODE") != "mock" - or not expected - or not hmac.compare_digest(supplied, "Bearer " + expected) - ): - await JSONResponse({"detail": "not found"}, status_code=404)(scope, receive, send) - return - body = b"" - while True: - part = await receive() - body += part.get("body", b"") - if len(body) > 4096: - await JSONResponse({"detail": "too large"}, status_code=413)(scope, receive, send) - return - if not part.get("more_body"): - break - data = json.loads(body) - from core.db.engine import SessionLocal - from core.db.models import UserShadow - from core.auth.session import create_session - from core.config.settings import settings - - with SessionLocal() as db: - user = ( - db.query(UserShadow).filter(UserShadow.user_center_id == data["user_center_id"]).first() - ) - if user is None: - await JSONResponse({"detail": "not found"}, status_code=404)(scope, receive, send) - return - payload = { - "user_id": str(user.user_id), - "user_center_id": user.user_center_id, - "username": user.username, - } - session = await create_session(payload) - await JSONResponse( - {"cookie": settings.session.cookie_name, "token": session}, - headers={"Cache-Control": "no-store"}, - )(scope, receive, send) - - -async def app(scope, receive, send): - if scope["type"] == "http": - path = scope.get("path", "") - if path == "/__e2e/session" and scope.get("method") == "POST": - return await _fixture_session(scope, receive, send) - if path == "/api" or path.startswith("/api/"): - stripped = path[4:] or "/" - scope = dict(scope) - scope["path"] = stripped - scope["raw_path"] = stripped.encode("utf-8") - await inner(scope, receive, send) diff --git a/src/backend/scripts/desktop_fixture_mcp.py b/src/backend/scripts/desktop_fixture_mcp.py deleted file mode 100644 index d8485c3e..00000000 --- a/src/backend/scripts/desktop_fixture_mcp.py +++ /dev/null @@ -1,50 +0,0 @@ -"""Private loopback MCP fixture for dual_mode_e2e; never imported by the application.""" - -import json -from pathlib import Path -import uvicorn -from mcp.server.fastmcp import FastMCP -from starlette.responses import JSONResponse - -CONFIG = Path("/tmp/capability-e2e-mcp.json") -TRACE = Path("/tmp/capability-e2e-mcp-trace.jsonl") -server = FastMCP( - "Desktop isolated verification", - host="127.0.0.1", - port=31999, - stateless_http=True, - json_response=True, -) - - -@server.tool() -def e2e_read_private_marker() -> str: - """Read the verification marker held inside the cloud network.""" - config = json.loads(CONFIG.read_text()) - with TRACE.open("a") as stream: - stream.write( - json.dumps({"tool": "e2e_read_private_marker", "marker": config["marker"]}) + "\n" - ) - return config["marker"] - - -inner = server.streamable_http_app() - - -async def app(scope, receive, send): - if scope["type"] == "http": - import hmac - - config = json.loads(CONFIG.read_text()) - supplied = ( - dict(scope.get("headers") or []) - .get(b"x-fixture-key", b"") - .decode("ascii", errors="ignore") - ) - if not hmac.compare_digest(supplied, config["secret"]): - return await JSONResponse({"error": "denied"}, status_code=403)(scope, receive, send) - await inner(scope, receive, send) - - -if __name__ == "__main__": - uvicorn.run(app, host="127.0.0.1", port=31999, log_level="warning") diff --git a/src/backend/scripts/dual_mode_e2e.py b/src/backend/scripts/dual_mode_e2e.py deleted file mode 100644 index dfc7756d..00000000 --- a/src/backend/scripts/dual_mode_e2e.py +++ /dev/null @@ -1,947 +0,0 @@ -#!/usr/bin/env python3 -"""桌面双端（云端 + 本机执行面）能力安装端到端验收。 - -前提（由调用方准备）： - * 一个"云端"后端：完整树代码，Postgres，AUTH_MODE=mock（Bearer 即该用户）， - 通过 ``scripts/desktop_cloud_wrapper.py`` 挂在 ``/api`` 之下； - * 一个"本机"后端：``cli.py serve``（local 档位），注入 HUGAGENT_DESKTOP_BRIDGE_SECRET / - CONFIG_TOKEN / HUGAGENT_CAPS_ROOT； - * 本机能访问 ``docker exec psql``（云端库的能力位授予与断言用）。 - -场景（每一步落到 JSON 报告，不以聊天正文为准）： - 01 云端建技能 / 智能体（用户 A），签发 capability token - 02 壳侧动作：模型清单 → 本机导入；桥配置推送 - 03 登录后的这一次同步即全部就绪：技能/智能体/插件已落盘、存储层 revision、 - 运行视图联接、mcp.json 投影（没有「待下载」这一步，也没有手动准备入口） - 05 真实调用：本机对话经云端模型网关执行技能脚本，读回标记 - 06 同名冲突：本机私有同名技能 → conflict → 视图自动隐藏（不需要用户选择） - 07 只属于本机的技能与云端能力并存 - 08 切换账号 B：A 的文件留在 A 的 profile，B 的视图不含 A 的私有技能 - 09 云端不可达：sync 返回 cloud_unavailable，本机副本仍可用 -""" - -from __future__ import annotations - -import argparse -import base64 -import io -import json -import os -import secrets -import subprocess -import sys -import time -import zipfile -from pathlib import Path -from typing import Any, Dict, List, Optional -from urllib.parse import urlsplit - -import httpx - - -class Report: - def __init__(self, path: Path) -> None: - self.path = path - self.steps: List[Dict[str, Any]] = [] - self.started = time.time() - - def step(self, name: str, ok: bool, **detail: Any) -> None: - entry = {"step": name, "ok": bool(ok), "at": round(time.time() - self.started, 1), **detail} - self.steps.append(entry) - flag = "PASS" if ok else "FAIL" - print(f"[{flag}] {name} {json.dumps(detail, ensure_ascii=False)[:300]}", flush=True) - self.flush() - - def flush(self) -> None: - self.path.write_text( - json.dumps( - { - "started_at": self.started, - "passed": sum(1 for s in self.steps if s["ok"]), - "failed": sum(1 for s in self.steps if not s["ok"]), - "steps": self.steps, - }, - ensure_ascii=False, - indent=2, - ), - encoding="utf-8", - ) - - -def _b64(obj: Dict[str, Any]) -> str: - return base64.b64encode(json.dumps(obj).encode("utf-8")).decode("ascii") - - -def _skill_zip(skill_id: str, marker: str, body_note: str) -> bytes: - md = ( - f"---\nname: {skill_id}\ndescription: 端到端验收技能：读取标记文件并原样回复（{body_note}）\n---\n" - f"# {skill_id}\n\n当用户要求「读取标记」时：\n\n" - f"1. 用 Bash 工具执行 `cat /workspace/skills/{skill_id}/marker.txt`\n" - f"2. 把命令输出**原样**回复给用户，不要改写、不要加解释。\n" - ) - buf = io.BytesIO() - with zipfile.ZipFile(buf, "w") as zf: - zf.writestr(f"{skill_id}/SKILL.md", md) - zf.writestr(f"{skill_id}/marker.txt", marker) - return buf.getvalue() - - -class Cloud: - def __init__( - self, base: str, pg_container: str, cloud_container: str, host: str, device_id: str - ) -> None: - self.container, self.device_id = cloud_container, device_id - self.fixture_secret = secrets.token_urlsafe(32) - code = 'import os,sys; from pathlib import Path; p=Path("/tmp/capability-e2e-fixture.secret"); p.write_text(sys.stdin.read()); os.chmod(p,0o600)' - subprocess.run( - ["docker", "exec", "-i", self.container, "python", "-c", code], - input=self.fixture_secret, - text=True, - capture_output=True, - check=True, - ) - self.base = base.rstrip("/") - self.pg = pg_container - self.http = httpx.Client(timeout=60.0, headers={"Host": host}, trust_env=False) - - def h(self, user: str) -> Dict[str, str]: - return {"Authorization": f"Bearer {user}"} - - def me(self, user: str) -> Dict[str, Any]: - r = self.http.get(f"{self.base}/api/v1/me", headers=self.h(user)) - r.raise_for_status() - return r.json()["data"] - - def psql(self, db: str, sql: str) -> str: - out = subprocess.run( - ["docker", "exec", self.pg, "psql", "-U", "hugagent_user", "-d", db, "-Atc", sql], - capture_output=True, - text=True, - check=True, - ) - return out.stdout.strip() - - def grant_capabilities(self, db: str, user_center_id: str) -> None: - # ORM attribute ``extra_data`` is the ``metadata`` JSONB column. - self.psql( - db, - "update users_shadow set metadata = coalesce(metadata, '{}'::jsonb) || " - '\'{"can_add_skill": true, "can_add_agent": true, "can_add_mcp": true, "can_import_plugin": true}\'::jsonb ' - f"where user_center_id = '{user_center_id}'", - ) - - def upload_skill(self, user: str, data: bytes) -> Dict[str, Any]: - r = self.http.post( - f"{self.base}/api/v1/me/skills/upload", - headers=self.h(user), - files={"file": ("skill.zip", data, "application/zip")}, - ) - r.raise_for_status() - return r.json()["data"] - - def delete_skill(self, user: str, skill_id: str) -> None: - self.http.delete(f"{self.base}/api/v1/me/skills/{skill_id}", headers=self.h(user)) - - def create_agent( - self, user: str, name: str, prompt: str, skill_ids: List[str], mcp_ids=None, plugin_ids=None - ) -> Dict[str, Any]: - r = self.http.post( - f"{self.base}/api/v1/agents", - headers=self.h(user), - json={ - "name": name, - "description": "e2e", - "system_prompt": prompt, - "skill_ids": skill_ids, - "mcp_server_ids": mcp_ids or [], - "plugin_ids": plugin_ids or [], - }, - ) - r.raise_for_status() - return r.json()["data"] - - def private_plugin(self, user: str, run_id: str): - secret = "E2E_CANARY_" + secrets.token_hex(16) - marker = "PRIVATE_" + secrets.token_hex(8) - config = json.dumps({"secret": secret, "marker": marker}) - code = "import os,sys; from pathlib import Path; p=Path('/tmp/capability-e2e-mcp.json'); p.write_text(sys.stdin.read()); os.chmod(p,0o600)" - subprocess.run( - ["docker", "exec", "-i", self.container, "python", "-c", code], - input=config, - text=True, - capture_output=True, - check=True, - ) - probe = subprocess.run( - [ - "docker", - "exec", - self.container, - "python", - "-c", - "import socket; s=socket.create_connection(('127.0.0.1',31999),timeout=1); s.close()", - ], - capture_output=True, - ) - if probe.returncode: - subprocess.run( - [ - "docker", - "exec", - "-d", - self.container, - "python", - "/app/src/backend/scripts/desktop_fixture_mcp.py", - ], - capture_output=True, - check=True, - ) - for _ in range(20): - time.sleep(0.2) - probe = subprocess.run( - [ - "docker", - "exec", - self.container, - "python", - "-c", - "import socket; s=socket.create_connection(('127.0.0.1',31999),timeout=1); s.close()", - ], - capture_output=True, - ) - if probe.returncode == 0: - break - if probe.returncode: - raise RuntimeError("private fixture MCP did not start") - slug, skill_id = "e2e-pack-" + run_id, "e2e-pack-skill-" + run_id - pack_marker = "PLUGIN_" + secrets.token_hex(8) - buffer = io.BytesIO() - with zipfile.ZipFile(buffer, "w") as z: - z.writestr( - ".claude-plugin/plugin.json", - json.dumps( - {"name": slug, "version": "1.0.0", "description": "isolated desktop acceptance"} - ), - ) - z.writestr( - ".mcp.json", - json.dumps( - { - "mcpServers": { - "e2e_private": { - "url": "http://127.0.0.1:31999/mcp", - "headers": {"X-Fixture-Key": secret}, - } - } - } - ), - ) - z.writestr( - f"skills/{skill_id}/SKILL.md", - f"---\nname: {skill_id}\ndescription: Read the plugin marker\n---\nUse bash to read ${{CLAUDE_PLUGIN_ROOT}}/marker.txt and return it verbatim.\n", - ) - z.writestr(f"skills/{skill_id}/marker.txt", pack_marker) - response = self.http.post( - f"{self.base}/api/v1/plugins/import", - headers=self.h(user), - files={"file": ("plugin.zip", buffer.getvalue(), "application/zip")}, - ) - response.raise_for_status() - result = response.json()["data"] - # Import returns an installation receipt, not its component IDs. Resolve - # the actual installed projection before preparing or binding anything. - installed = self.http.get(f"{self.base}/api/v1/plugins/installed", headers=self.h(user)) - installed.raise_for_status() - projection = next( - item - for item in installed.json()["data"]["items"] - if item["install_id"] == result["install_id"] - ) - result = {**result, "skills": projection["skills"], "mcp": projection["mcp"]} - if not result["skills"] or not result["mcp"]: - raise RuntimeError("fixture plugin must contain both skill and MCP components") - # Discover the private endpoint through the same cloud-side service used - # by MCP connection testing. A plugin config alone does not contain a - # tools/list snapshot and must never be reported as runtime-ready. - self.probe_fixture_mcps(result.get("mcp") or []) - return result, secret, marker, pack_marker - - def probe_fixture_mcps(self, server_ids): - code = """import asyncio,json,sys -from core.db.engine import SessionLocal -from core.db.models import AdminMcpServer -from core.services.mcp_management_service import probe_mcp_connectivity -async def main(): - with SessionLocal() as db: - for sid in json.loads(sys.stdin.read()): - assert sid.startswith('e2e-pack-') - row=db.get(AdminMcpServer,sid) - ok,_=await probe_mcp_connectivity(row,db,timeout_seconds=15) - if not ok or not row.tools_json: - raise RuntimeError('fixture MCP discovery failed') - db.commit() -asyncio.run(main()) -""" - result = subprocess.run( - ["docker", "exec", "-i", self.container, "python", "-c", code], - input=json.dumps(list(server_ids)), - capture_output=True, - text=True, - timeout=45, - ) - if result.returncode: - raise RuntimeError("fixture MCP discovery failed") - - def token(self, user: str) -> Dict[str, Any]: - me = self.me(user) - # Mint through the isolated wrapper so in-memory sessions exist in the - # same cloud process that will validate them; this never ships in api.app. - fixture = self.http.post( - f"{self.base}/__e2e/session", - headers={"Authorization": "Bearer " + self.fixture_secret}, - json={"user_center_id": user}, - ) - fixture.raise_for_status() - session = fixture.json() - r = self.http.post( - f"{self.base}/api/v1/desktop/capability/token", - json={"device_id": self.device_id}, - cookies={session["cookie"]: session["token"]}, - ) - r.raise_for_status() - return r.json()["data"] - - def models(self, token: str) -> Dict[str, Any]: - r = self.http.get( - f"{self.base}/api/v1/desktop/capability/models", - headers={"Authorization": f"Bearer {token}", "X-Desktop-Device-Id": self.device_id}, - ) - r.raise_for_status() - return r.json()["data"] - - -def shell_user_center_id(cloud_base: str, user_center_id: str) -> str: - """与桌面壳 / 本机后端相同的桥接用户命名空间规则。""" - parts = urlsplit(cloud_base.strip()) - port = parts.port or (443 if parts.scheme == "https" else 80) - return f"cloud:{parts.hostname or 'cloud'}:{port}:{user_center_id}" - - -class Device: - def __init__(self, base: str, bridge_secret: str, config_token: str, device_id: str, cloud_base: str) -> None: - self.base = base.rstrip("/") - self.cloud_base = cloud_base.rstrip("/") - self.device_id = device_id - self.secret = bridge_secret - self.config_token = config_token - self.http = httpx.Client(timeout=120.0, trust_env=False) - - def h(self, user_center_id: str) -> Dict[str, str]: - namespaced = shell_user_center_id(self.cloud_base, user_center_id) - return { - "X-Desktop-Bridge": self.secret, - "X-Desktop-Bridge-User": _b64( - {"user_center_id": namespaced, "username": user_center_id} - ), - } - - def config_h(self) -> Dict[str, str]: - return {"Authorization": f"Bearer {self.config_token}"} - - def push_bridge(self, cloud_base: str, token: str, expires_in: int) -> None: - r = self.http.post( - f"{self.base}/v1/desktop/capability/cloud-bridge", - headers={"Authorization": "Bearer " + self.secret}, - json={ - "cloud_base": cloud_base, - "token": token, - "expires_in": expires_in, - "device_id": self.device_id, - }, - ) - r.raise_for_status() - - def import_models( - self, manifest: Dict[str, Any], cloud_base: str, token: str - ) -> Dict[str, Any]: - providers = [] - for p in manifest["providers"]: - p = dict(p) - p["base_url"] = ( - f"{cloud_base}/api/v1/desktop/capability/gateway/models/{p['provider_id']}" - ) - p["api_key"] = token - providers.append(p) - r = self.http.post( - f"{self.base}/v1/models/import", - headers={"Authorization": "Bearer " + self.secret}, - json={ - "providers": providers, - "role_assignments": manifest["role_assignments"], - "overwrite": True, - }, - ) - r.raise_for_status() - return r.json()["data"] - - def call(self, user: str, method: str, path: str, **kw: Any) -> httpx.Response: - return self.http.request(method, f"{self.base}{path}", headers=self.h(user), **kw) - - def json(self, user: str, method: str, path: str, **kw: Any) -> Dict[str, Any]: - r = self.call(user, method, path, **kw) - if r.status_code >= 400: - raise RuntimeError(f"{method} {path} -> {r.status_code}: {r.text[:400]}") - return r.json()["data"] - - def me(self, user: str) -> Dict[str, Any]: - return self.json(user, "GET", "/v1/me") - - def upload_skill(self, user: str, data: bytes) -> Dict[str, Any]: - r = self.http.post( - f"{self.base}/v1/me/skills/upload", - headers=self.h(user), - files={"file": ("skill.zip", data, "application/zip")}, - ) - if r.status_code >= 400: - raise RuntimeError(f"upload -> {r.status_code}: {r.text[:400]}") - return r.json()["data"] - - def chat( - self, - user: str, - message: str, - timeout_s: float = 240.0, - agent_id: Optional[str] = None, - chat_id: Optional[str] = None, - **selection: Any, - ) -> Dict[str, Any]: - if chat_id is None: - chat = self.json(user, "POST", "/v1/chats", json={"title": "e2e"}) - chat_id = chat["chat_id"] if "chat_id" in chat else chat["id"] - text: List[str] = [] - tool_calls: List[str] = [] - errors: List[str] = [] - event_types = set() - deadline = time.time() + timeout_s - with self.http.stream( - "POST", - f"{self.base}/v1/agents/responses", - headers={**self.h(user), "Accept": "text/event-stream"}, - json={"chat_id": chat_id, "message": message, "agent_id": agent_id, **selection, "stream": True}, - timeout=timeout_s, - ) as r: - if r.status_code >= 400: - raise RuntimeError(f"stream -> {r.status_code}: {r.read()[:400]!r}") - event = "" - for line in r.iter_lines(): - if time.time() > deadline: - errors.append("timeout") - break - if line.startswith("event:"): - event = line[6:].strip() - continue - if not line.startswith("data:"): - continue - payload = line[5:].strip() - if payload == "[DONE]": - break - try: - obj = json.loads(payload) - except ValueError: - continue - if not isinstance(obj, dict): - continue - event_types.add(str(obj.get("type") or obj.get("event") or event)) - if event in ("ai_message", "content") or obj.get("event") == "ai_message": - delta = obj.get("delta") or obj.get("content") or obj.get("text") or "" - if isinstance(delta, str): - text.append(delta) - elif ( - event == "tool_call" - or obj.get("type") == "tool_call" - or obj.get("event") == "tool_call" - ): - tool_calls.append(str(obj.get("name") or obj.get("tool_name") or obj)[:80]) - elif event == "error" or obj.get("type") == "error" or obj.get("event") == "error": - errors.append(json.dumps(obj, ensure_ascii=False)[:300]) - return { - "chat_id": chat_id, - "text": "".join(text), - "tool_calls": tool_calls, - "errors": errors, - "event_types": sorted(event_types), - } - - -def _find_user_view(device_home: Path, name: str, user_id: str) -> Optional[Path]: - root = device_home / "workspace" / "skills_u" / user_id - candidate = root / name - return candidate if candidate.is_symlink() or candidate.exists() else None - - -def _read_view_marker(view: Optional[Path], device_home: Path) -> Optional[str]: - if view is None: - return None - relative = str(view.relative_to(device_home) / "marker.txt") - code = ( - "from pathlib import Path; import os; p=Path(os.environ['HUGAGENT_HOME'])/" - + repr(relative) - + "; print(p.read_text(), end='')" - ) - result = subprocess.run( - ["docker", "exec", "capdevice", "python", "-c", code], capture_output=True, text=True - ) - return result.stdout if result.returncode == 0 else None - - -def _execute_local_marker(user_id: str, skill_id: str, expected: str) -> bool: - """Exercise the actual device runner and pinned local view while cloud is down.""" - code = """import asyncio,json,os,sys -from cli import apply_local_env -apply_local_env(port=int(os.environ.get("BACKEND_PORT") or "32101")) -from core.capabilities import runtime -from core.sandbox.protocol import ProcessRequest -from core.sandbox.providers.script_runner.provider import ScriptRunnerProvider -async def main(): - values=json.loads(sys.stdin.read()) - run_id="offline-"+values["skill_id"] - run=runtime.prepare(run_id,values["user_id"],skill_ids=[values["skill_id"]]) - runtime.preflight(run) - result=await ScriptRunnerProvider().run_to_completion(ProcessRequest( - script_content="from pathlib import Path\\nprint(Path("+repr("/workspace/skills/"+values["skill_id"]+"/marker.txt")+").read_text())", - script_name="offline_marker.py", user_id=values["user_id"], - session_id=run_id,capability_run_id=run_id,timeout=15)) - print(json.dumps({"offline_marker_ok":run.profile is None and result.exit_code==0 and result.stdout.strip()==values["expected"]})) -asyncio.run(main()) -""" - result = subprocess.run( - ["docker", "exec", "-i", "capdevice", "python", "-c", code], - input=json.dumps({"user_id": user_id, "skill_id": skill_id, "expected": expected}), - capture_output=True, - text=True, - timeout=60, - ) - if result.returncode: - return False - return any(line.strip() == '{"offline_marker_ok": true}' for line in result.stdout.splitlines()) - - -def main() -> int: - ap = argparse.ArgumentParser() - ap.add_argument( - "--cloud", required=True, help="云端根地址（含 /api 之前的部分），如 http://127.0.0.1:32201" - ) - ap.add_argument( - "--cloud-internal", - default=None, - help="本机进程访问云端用的地址（本机跑在容器里时与 --cloud 不同）；缺省同 --cloud", - ) - ap.add_argument("--device", required=True, help="本机后端地址，如 http://127.0.0.1:32101") - ap.add_argument("--bridge-secret", required=True) - ap.add_argument("--config-token", required=True) - ap.add_argument("--device-root", required=True, help="HUGAGENT_CAPS_ROOT") - ap.add_argument("--device-home", required=True, help="HUGAGENT_HOME") - ap.add_argument("--pg-container", default="hugagent-postgres") - ap.add_argument("--cloud-db", default="hugagent_captest") - ap.add_argument("--cloud-container", default="capcloud") - ap.add_argument("--run-id", default=secrets.token_hex(4)) - ap.add_argument("--report", required=True) - ap.add_argument("--skip-model", action="store_true", help="跳过真实模型调用步骤") - args = ap.parse_args() - - report = Report(Path(args.report)) - cloud_internal = (args.cloud_internal or args.cloud).rstrip("/") - device_id = "e2e-" + args.run_id - cloud = Cloud( - args.cloud, - args.pg_container, - args.cloud_container, - urlsplit(cloud_internal).netloc, - device_id, - ) - device = Device(args.device, args.bridge_secret, args.config_token, device_id, cloud_internal) - device_root = Path(args.device_root) - device_home = Path(args.device_home) - user_a, user_b = "capuser-a-" + args.run_id, "capuser-b-" + args.run_id - skill_id = "e2e-marker-" + args.run_id - agent_name = "E2E 标记读取员 " + args.run_id - marker = f"MARKER-{secrets.token_hex(6)}" - - # ── 00 健康 ───────────────────────────────────────────────────────── - try: - ch = cloud.http.get(f"{args.cloud}/api/health").status_code - dh = device.http.get(f"{args.device}/health").status_code - report.step("00 health", ch == 200 and dh == 200, cloud=ch, device=dh) - if ch != 200 or dh != 200: - return 1 - except Exception as exc: # noqa: BLE001 - report.step("00 health", False, error=str(exc)) - return 1 - - # ── 01 云端夹具 ────────────────────────────────────────────────────── - try: - me_a = cloud.me(user_a) - cloud.grant_capabilities(args.cloud_db, user_a) - cloud.me(user_b) - cloud.delete_skill(user_a, skill_id) - skill = cloud.upload_skill(user_a, _skill_zip(skill_id, marker, "cloud")) - cloud_skill_id = skill.get("id") or skill_id - plugin, canary, private_marker, plugin_marker = cloud.private_plugin(user_a, args.run_id) - plugin_skills = list(plugin.get("skills") or []) - plugin_mcps = list(plugin.get("mcp") or []) - agent = cloud.create_agent( - user_a, - agent_name, - "你是标记读取员。使用提供的技能和MCP工具读取标记，严禁猜测。", - [cloud_skill_id] + plugin_skills, - mcp_ids=plugin_mcps, - plugin_ids=[plugin["install_id"]], - ) - tok = cloud.token(user_a) - report.step( - "01 cloud fixtures", - bool(tok.get("token")) and bool(agent.get("agent_id")), - cloud_user_id=me_a.get("user_id"), - skill_id=cloud_skill_id, - agent_id=agent.get("agent_id"), - token_expires_in=tok.get("expires_in"), - ) - except Exception as exc: # noqa: BLE001 - report.step("01 cloud fixtures", False, error=str(exc)[:500]) - return 1 - token_a = tok["token"] - - # ── 02 壳侧动作：模型导入 + 桥推送 ─────────────────────────────────── - try: - manifest = cloud.models(token_a) - imported = device.import_models(manifest, cloud_internal, token_a) - device.push_bridge(cloud_internal, token_a, int(tok.get("expires_in") or 600)) - local_user_a = str(device.me(user_a)["user_id"]) - report.step( - "02 shell: models + bridge", - True, - providers=len(manifest.get("providers", [])), - imported=imported, - ) - except Exception as exc: # noqa: BLE001 - report.step("02 shell: models + bridge", False, error=str(exc)[:500]) - return 1 - - # ── 03 同步即就绪（登录后的这一次同步就把账号能力全部准备好）───────── - try: - st = device.json(user_a, "POST", "/v1/desktop/capabilities/sync") - skills_list = device.json( - user_a, "GET", "/v1/desktop/capabilities/installations?kind=skill" - ) - by_name = {i["runtime_name"]: i for i in skills_list["items"] if i["source"] == "cloud"} - cloud_skill = by_name.get(cloud_skill_id) - agents_list = device.json( - user_a, "GET", "/v1/desktop/capabilities/installations?kind=agent" - ) - cloud_agents = [i for i in agents_list["items"] if i["source"] == "cloud"] - plugins_list = device.json( - user_a, "GET", "/v1/desktop/capabilities/installations?kind=plugin" - ) - mcp_json = device.json(user_a, "GET", "/v1/desktop/capabilities/mcp-json") - profile = skills_list["profile_id"] - managed = mcp_json["managedProfiles"].get(profile or "", {}) - agent_ready = any( - i["display_name"] == agent_name and i["state"] == "ready" for i in cloud_agents - ) - agent_dir = device_root / "agents" / (profile or "?") - plugin_skills_ready = all( - any( - i["runtime_name"] == sid and i["state"] == "ready" - for i in skills_list["items"] - ) - for sid in plugin_skills - ) - rev = (cloud_skill or {}).get("revision") - store_dir = device_root / "skills" / (profile or "?") / cloud_skill_id / (rev or "?") - view_link = _find_user_view(device_home, cloud_skill_id, local_user_a) - link_ok = bool( - view_link - and os.path.islink(view_link) - and _read_view_marker(view_link, device_home) == marker - ) - chosen = [ - i - for i in skills_list["items"] - if i["runtime_name"] == cloud_skill_id and i["resolution"]["outcome"] == "chosen" - ] - report.step( - "03 sync leaves everything ready", - cloud_skill is not None - and cloud_skill["state"] == "ready" - and st.get("pending_count") == 0 - and plugin_skills_ready - and agent_ready - and bool(managed.get("servers")) - and store_dir.is_dir() - and (store_dir / "marker.txt").read_text() == marker - and link_ok - and bool(chosen), - profile=profile, - skill_state=(cloud_skill or {}).get("state"), - pending_count=st.get("pending_count"), - installed_count=st.get("installed_count"), - cloud_agents=len(cloud_agents), - agent_ready=agent_ready, - agent_files=sorted(p.name for p in agent_dir.iterdir()) if agent_dir.is_dir() else [], - plugins=len(plugins_list["items"]), - plugin_skills_ready=plugin_skills_ready, - managed_servers=len(managed.get("servers") or {}), - mcp_json_generation=mcp_json["generation"], - revision=rev, - store_dir=str(store_dir), - view_link=str(view_link), - chosen_source=chosen[0]["source"] if chosen else None, - ) - if cloud_skill is None or cloud_skill["state"] != "ready": - return 1 - except Exception as exc: # noqa: BLE001 - report.step("03 sync leaves everything ready", False, error=str(exc)[:500]) - return 1 - - # ── 05 真实调用 ───────────────────────────────────────────────────── - if not args.skip_model: - try: - out = device.chat( - user_a, - f"请使用技能 {cloud_skill_id}：读取标记，并把标记内容原样回复我。", - skill_id=cloud_skill_id, - ) - got = marker in out["text"] and bool(out["tool_calls"]) and not out["errors"] - session_links = sorted( - str(p) - for p in (device_home / "workspace" / ".sessions").glob("*/skills") - if os.path.islink(p) - ) - report.step( - "05 real run via cloud model gateway", - got, - reply=out["text"][-300:], - tool_calls=out["tool_calls"][:10], - errors=out["errors"], - session_skill_links=len(session_links), - ) - except Exception as exc: # noqa: BLE001 - report.step("05 real run via cloud model gateway", False, error=str(exc)[:500]) - - # ── 05b 插件组件与服务器内网 MCP 的真实调用和凭据边界 ─────────────── - if not args.skip_model: - try: - out = device.chat( - user_a, - "请调用 e2e_read_private_marker 读取云内网标记，再用插件技能 " - + ", ".join(plugin_skills) - + " 读取本机插件标记。只回复两个标记。", - agent_id=agent["agent_id"], - ) - trace = subprocess.run( - [ - "docker", - "exec", - args.cloud_container, - "cat", - "/tmp/capability-e2e-mcp-trace.jsonl", - ], - capture_output=True, - text=True, - ) - direct = subprocess.run( - [ - "docker", - "exec", - "capdevice", - "python", - "-c", - "import socket; socket.create_connection(('127.0.0.1',31999),timeout=2)", - ], - capture_output=True, - ) - report.step( - "05b agent + plugin + private MCP", - private_marker in out["text"] - and plugin_marker in out["text"] - and not out["errors"] - and any("e2e_read_private_marker" in name for name in out["tool_calls"]) - and private_marker in trace.stdout - and direct.returncode != 0, - reply=out["text"][-300:], - errors=out["errors"], - tool_calls=out["tool_calls"], - event_types=out["event_types"], - cloud_trace=private_marker in trace.stdout, - direct_connection_refused=direct.returncode != 0, - ) - docs = [ - device.json(user_a, "GET", "/v1/desktop/capabilities/mcp-json"), - cloud.models(token_a), - ] - files_leak = any( - canary.encode() in p.read_bytes() - for p in device_root.rglob("*") - if p.is_file() and not p.is_symlink() - ) - db_leak = canary.encode() in (device_home / "data.db").read_bytes() - logs = subprocess.run(["docker", "logs", "capdevice"], capture_output=True, text=True) - report.step( - "05c credential boundary", - not any(canary in json.dumps(doc) for doc in docs) - and not files_leak - and not db_leak - and canary not in logs.stdout + logs.stderr, - file_leak=files_leak, - database_leak=db_leak, - log_leak=canary in logs.stdout + logs.stderr, - ) - except Exception as exc: - report.step("05b agent + plugin + private MCP", False, error=str(exc)[:500]) - - # Explicit plugin selection and its subsequent sticky activation use the - # ordinary chat route, independently of an agent's static plugin bindings. - if not args.skip_model: - try: - out = device.chat( - user_a, - "使用所选插件读取云内网标记和本机插件标记，只回复这两个标记。", - plugin_id=plugin["install_id"], - ) - report.step( - "05d explicit plugin selection", - private_marker in out["text"] - and plugin_marker in out["text"] - and not out["errors"] - and any("e2e_read_private_marker" in name for name in out["tool_calls"]), - reply=out["text"][-300:], - errors=out["errors"], - tool_calls=out["tool_calls"], - ) - followup = device.chat( - user_a, - "再次实际调用上一轮的插件工具读取两个标记，不要直接复制之前的回答。", - chat_id=out["chat_id"], - ) - report.step( - "05e sticky plugin next turn", - private_marker in followup["text"] - and plugin_marker in followup["text"] - and not followup["errors"] - and any("e2e_read_private_marker" in name for name in followup["tool_calls"]), - reply=followup["text"][-300:], - errors=followup["errors"], - tool_calls=followup["tool_calls"], - ) - connector = device.chat( - user_a, - "调用所选连接器的 e2e_read_private_marker，原样返回读取到的云内网标记。", - connector_id=plugin_mcps[0], - ) - report.step( - "05f explicit connector selection", - private_marker in connector["text"] - and not connector["errors"] - and any("e2e_read_private_marker" in name for name in connector["tool_calls"]), - reply=connector["text"][-300:], - errors=connector["errors"], - tool_calls=connector["tool_calls"], - ) - except Exception as exc: - report.step("05d-f explicit and sticky selections", False, error=str(exc)[:500]) - - # ── 06 同名冲突：自动隐藏，不需要用户做任何选择 ────────────────────── - try: - local_marker = f"LOCAL-{secrets.token_hex(4)}" - device.upload_skill(user_a, _skill_zip(skill_id, local_marker, "local")) - listing = device.json(user_a, "GET", "/v1/desktop/capabilities/installations?kind=skill") - cands = [i for i in listing["items"] if i["runtime_name"] == cloud_skill_id] - conflict = cloud_skill_id in listing["conflicts"] - hidden_during_conflict = _find_user_view(device_home, cloud_skill_id, local_user_a) is None - report.step( - "06 name conflict is hidden automatically", - conflict and hidden_during_conflict, - candidates=[(c["profile"], c["resolution"]["outcome"]) for c in cands], - conflict=conflict, - hidden_during_conflict=hidden_during_conflict, - ) - except Exception as exc: # noqa: BLE001 - report.step("06 name conflict is hidden automatically", False, error=str(exc)[:500]) - - # ── 07 只属于本机的技能：与云端能力并存，离线时仍可执行 ──────────────── - local_only_id = f"{skill_id}-local" - try: - device.upload_skill(user_a, _skill_zip(local_only_id, local_marker, "local")) - listing = device.json(user_a, "GET", "/v1/desktop/capabilities/installations?kind=skill") - local_only = next( - (i for i in listing["items"] if i["runtime_name"] == local_only_id), None - ) - report.step( - "07 local-only skill coexists", - bool(local_only) and local_only["source"] == "local" and local_only["usable"], - source=(local_only or {}).get("source"), - usable=(local_only or {}).get("usable"), - ) - except Exception as exc: # noqa: BLE001 - report.step("07 local-only skill coexists", False, error=str(exc)[:500]) - - # ── 08 切换账号 ───────────────────────────────────────────────────── - try: - tok_b = cloud.token(user_b) - device.push_bridge(cloud_internal, tok_b["token"], int(tok_b.get("expires_in") or 3600)) - device.json(user_b, "POST", "/v1/desktop/capabilities/sync") - listing_b = device.json(user_b, "GET", "/v1/desktop/capabilities/installations?kind=skill") - profile_b = listing_b["profile_id"] - b_sees_a_private = any( - i["runtime_name"] == cloud_skill_id and i["source"] == "cloud" - for i in listing_b["items"] - ) - a_files_kept = (device_root / "skills" / profile / cloud_skill_id).is_dir() - local_user_b = str(device.me(user_b)["user_id"]) - view_b = _find_user_view(device_home, cloud_skill_id, local_user_b) - report.step( - "08 account switch isolation", - profile_b != profile and not b_sees_a_private and a_files_kept and view_b is None, - profile_a=profile, - profile_b=profile_b, - b_sees_a_private=b_sees_a_private, - a_files_kept=a_files_kept, - b_view_has_skill=_read_view_marker(view_b, device_home) is not None, - ) - device.push_bridge(cloud_internal, token_a, int(tok.get("expires_in") or 3600)) - device.json(user_a, "POST", "/v1/desktop/capabilities/sync") - except Exception as exc: # noqa: BLE001 - report.step("08 account switch isolation", False, error=str(exc)[:500]) - - # ── 09 相同云实例断网：本机副本继续，云操作明确失败 ───────────────── - try: - subprocess.run( - ["docker", "stop", "-t", "2", args.cloud_container], check=True, capture_output=True - ) - r = device.call(user_a, "POST", "/v1/desktop/capabilities/sync") - listing = device.json(user_a, "GET", "/v1/desktop/capabilities/installations?kind=skill") - local_ready = any( - i["runtime_name"] == local_only_id and i["source"] == "local" and i["usable"] - for i in listing["items"] - ) - local_executed = _execute_local_marker(local_user_a, local_only_id, local_marker) - report.step( - "09 cloud unreachable", - r.status_code == 502 and local_ready and local_executed, - sync_status=r.status_code, - local_ready=local_ready, - local_execution_passed=local_executed, - ) - except Exception as exc: - report.step("09 cloud unreachable", False, error=str(exc)[:500]) - finally: - subprocess.run(["docker", "start", args.cloud_container], check=True, capture_output=True) - - report.flush() - failed = [s["step"] for s in report.steps if not s["ok"]] - print(f"\nreport: {report.path} failed: {failed}") - return 1 if failed else 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/src/backend/scripts/edgebench/ann_judge.py b/src/backend/scripts/edgebench/ann_judge.py deleted file mode 100644 index e2c169b5..00000000 --- a/src/backend/scripts/edgebench/ann_judge.py +++ /dev/null @@ -1,136 +0,0 @@ -"""Local deterministic judge for EdgeBench `ann_vector_search_qps` (a minimal faithful reproduction of SForge). - -Original task: replace the **brute-force nearest-neighbor baseline** with a high-performance -nearest neighbor and compare QPS under a recall constraint. -This judge runs inside the sandbox and is the loop's verify_cmd = ground truth source: - - - Deterministically generate base/query vectors (fixed seed), and compute the true top-k with a - NumPy brute force (evaluation ground truth, not timed). - - Time a **naive pure-Python brute-force baseline** to obtain the baseline QPS (this is the object - to be replaced/surpassed). - - import the contestant's /workspace/solution.py (must expose build(base)->index and - search(index, queries, k)->ids), and measure its recall@k and QPS. - - Scoring: recall >= RECALL_MIN counts as a valid solution (exit 0); SCORE = contestant QPS / baseline QPS (speedup). - -Stable timing: call search repeatedly until the cumulative time >= MIN_TIME seconds, then compute QPS -as total queries processed / total elapsed to eliminate small-workload noise. - -Output (for the loop's score_regex / evaluator to parse): - RECALL= QPS= BASELINE_QPS= SCORE= -exit 0 if and only if recall meets the threshold. Whether speed meets the bar is decided by the loop's target_score. -""" -import sys -import time -import traceback - -RECALL_MIN = 0.90 -N_BASE = 2000 -DIM = 32 -N_QUERY = 200 -K = 10 -SEED = 1234 -MIN_TIME = 0.30 # stable timing window - - -def _fail(msg: str) -> None: - print("RECALL=0.0 QPS=0.0 BASELINE_QPS=0.0 SCORE=0.0") - sys.stderr.write(f"JUDGE_ERROR: {msg}\n") - sys.exit(1) - - -def _timed_qps(fn, n_items: int) -> float: - """Run fn() repeatedly until cumulative time >= MIN_TIME, returning items/sec (stable, noise-resistant).""" - reps = 0 - t0 = time.perf_counter() - while True: - fn() - reps += 1 - elapsed = time.perf_counter() - t0 - if elapsed >= MIN_TIME or reps >= 10000: - break - return (reps * n_items) / max(elapsed, 1e-9) - - -def main() -> None: - try: - import numpy as np - except Exception as e: # noqa: BLE001 - _fail(f"numpy 不可用: {e}") - - rng = np.random.RandomState(SEED) - base = rng.rand(N_BASE, DIM).astype("float32") - queries = rng.rand(N_QUERY, DIM).astype("float32") - - # Ground truth: vectorized brute-force top-k (for evaluation, not timed) - b2 = (base ** 2).sum(1) - truth = np.empty((N_QUERY, K), dtype=np.int64) - for i in range(N_QUERY): - d = b2 - 2.0 * base.dot(queries[i]) - truth[i] = np.argpartition(d, K)[:K] - - # Baseline: naive pure-Python brute force (the object to surpass). Timed on a query subset, then normalized. - base_list = base.tolist() - qsub = queries[:20].tolist() - - def naive_pass(): - for q in qsub: - best = [] - for j, b in enumerate(base_list): - s = 0.0 - for a, c in zip(q, b): - diff = a - c - s += diff * diff - best.append((s, j)) - best.sort() - _ = [j for _, j in best[:K]] - - baseline_qps = _timed_qps(naive_pass, len(qsub)) - - # Load the contestant's solution (fresh process → import bypasses the Jupyter cache) - sys.path.insert(0, "/workspace") - try: - import solution # type: ignore - except Exception: # noqa: BLE001 - _fail("无法 import /workspace/solution.py（文件缺失或有语法错误）:\n" + traceback.format_exc()) - - if not hasattr(solution, "build") or not hasattr(solution, "search"): - _fail("solution.py 必须提供 build(base) 和 search(index, queries, k) 两个函数") - - try: - index = solution.build(base) - except Exception: # noqa: BLE001 - _fail("solution.build(base) 抛异常:\n" + traceback.format_exc()) - - # Run once first to verify correctness (and serve as warmup) - try: - pred = solution.search(index, queries, K) - except Exception: # noqa: BLE001 - _fail("solution.search(...) 抛异常:\n" + traceback.format_exc()) - try: - pred_arr = [list(row) for row in pred] - assert len(pred_arr) == N_QUERY - except Exception: # noqa: BLE001 - _fail(f"search 返回值形状错误：应为 {N_QUERY} 行、每行至少 {K} 个 id 的列表") - - hit = 0 - for i in range(N_QUERY): - hit += len(set(int(x) for x in pred_arr[i][:K]) & set(int(x) for x in truth[i])) - recall = hit / (N_QUERY * K) - - agent_qps = _timed_qps(lambda: solution.search(index, queries, K), N_QUERY) - speedup = agent_qps / max(baseline_qps, 1e-9) - - print( - f"RECALL={recall:.4f} QPS={agent_qps:.1f} " - f"BASELINE_QPS={baseline_qps:.1f} SCORE={speedup:.4f}" - ) - if recall < RECALL_MIN: - sys.stderr.write( - f"JUDGE_ERROR: recall {recall:.3f} < {RECALL_MIN}（近似太糙，召回不足）\n" - ) - sys.exit(1) - sys.exit(0) - - -if __name__ == "__main__": - main() diff --git a/src/backend/scripts/edgebench/perm_opt_judge.py b/src/backend/scripts/edgebench/perm_opt_judge.py deleted file mode 100644 index 1543f80d..00000000 --- a/src/backend/scripts/edgebench/perm_opt_judge.py +++ /dev/null @@ -1,88 +0,0 @@ -"""Local deterministic judge for EdgeBench `order_addition_permutation_optimization`. - -Original task: find a permutation that minimizes a **black-box cost function** using metaheuristic -search (simulated annealing / genetic / local search), without accessing the cost function's internals. -Metric: score (higher is better). - -This judge runs inside the sandbox and is the loop's verify_cmd = ground truth source: - - - Build a deterministic cost function cost(perm) with a fixed seed (here the closed-tour length over - N cities, i.e. a TSP instance; it is a black box to the contestant — callable only, internals hidden). - - import the contestant's /workspace/solution.py, which must expose solve(n, cost) -> a permutation of range(n). - - Verify the return is a valid permutation (exit 0/1 expresses validity, **independent of CPU speed**). - - SCORE = baseline cost / contestant cost (higher is better); baseline = cost of the identity permutation. - -Scoring computes the cost of the final permutation only once → deterministic, unaffected by sandbox -throttling (unlike QPS-style tasks). - -Output (for the loop's score_regex / evaluator to parse): - COST= BASELINE= SCORE= -exit 0 if and only if the return is a valid permutation of range(n). Whether it meets the bar is decided by the loop's target_score. -""" -import math -import sys -import traceback - -N = 50 -SEED = 20260710 - - -def _fail(msg: str) -> None: - print("COST=inf BASELINE=0 SCORE=0.0") - sys.stderr.write(f"JUDGE_ERROR: {msg}\n") - sys.exit(1) - - -def _build_instance(): - """Deterministic TSP instance (a black-box cost to the contestant).""" - import random - - rng = random.Random(SEED) - pts = [(rng.random(), rng.random()) for _ in range(N)] - - def cost(perm): - # Total closed-tour length; perm is an ordering of range(N) - total = 0.0 - for i in range(len(perm)): - a = pts[perm[i]] - b = pts[perm[(i + 1) % len(perm)]] - total += math.hypot(a[0] - b[0], a[1] - b[1]) - return total - - return cost - - -def main() -> None: - cost = _build_instance() - identity_cost = cost(list(range(N))) - - sys.path.insert(0, "/workspace") - try: - import solution # type: ignore - except Exception: # noqa: BLE001 - _fail("无法 import /workspace/solution.py（缺失或语法错误）:\n" + traceback.format_exc()) - - if not hasattr(solution, "solve"): - _fail("solution.py 必须提供 solve(n, cost) 函数，返回 range(n) 的一个排列") - - try: - result = solution.solve(N, cost) - except Exception: # noqa: BLE001 - _fail("solution.solve(n, cost) 抛异常:\n" + traceback.format_exc()) - - # Verify a valid permutation - try: - result = [int(x) for x in result] - except Exception: # noqa: BLE001 - _fail("solve 返回值不是整数序列") - if sorted(result) != list(range(N)): - _fail(f"solve 返回的不是 range({N}) 的合法排列（每个下标恰好一次）") - - result_cost = cost(result) - ratio = identity_cost / result_cost if result_cost > 0 else 0.0 - print(f"COST={result_cost:.4f} BASELINE={identity_cost:.4f} SCORE={ratio:.4f}") - sys.exit(0) - - -if __name__ == "__main__": - main() diff --git a/src/backend/scripts/edit_chat_message.py b/src/backend/scripts/edit_chat_message.py deleted file mode 100644 index 0a43e337..00000000 --- a/src/backend/scripts/edit_chat_message.py +++ /dev/null @@ -1,198 +0,0 @@ -#!/usr/bin/env python3 -"""Edit a chat message stored in the `chat_messages` table. - -Usage (from repo root): - PYTHONPATH=src/backend python src/backend/scripts/edit_chat_message.py list --chat-id - PYTHONPATH=src/backend python src/backend/scripts/edit_chat_message.py show --message-id - PYTHONPATH=src/backend python src/backend/scripts/edit_chat_message.py edit --message-id --content "new content" - PYTHONPATH=src/backend python src/backend/scripts/edit_chat_message.py edit --message-id --from-file new.txt - PYTHONPATH=src/backend python src/backend/scripts/edit_chat_message.py delete --message-id - -Or inside the backend container (backend source is mounted at /app/src/backend): - docker exec -it hugagent-backend python /app/src/backend/scripts/edit_chat_message.py list --chat-id xxx - -Notes: - - Updating `content` keeps the original `message_id` so frontend citation / - feedback references remain valid. - - The frontend caches messages in Zustand; users need to refresh the page - or reopen the session to see the edit. - - `content` is limited to 100000 chars by a DB CHECK constraint. -""" - -from __future__ import annotations - -import argparse -import os -import sys -from pathlib import Path - -from dotenv import load_dotenv - -# src/backend/scripts/edit_chat_message.py → parents[3] is the repo root -REPO_ROOT = Path(__file__).resolve().parents[3] -BACKEND_SRC = REPO_ROOT / "src" / "backend" - -# Make `core.*` importable when run from anywhere. -if str(BACKEND_SRC) not in sys.path: - sys.path.insert(0, str(BACKEND_SRC)) - -# Load .env from repo root if present. -env_file = REPO_ROOT / ".env" -if env_file.exists(): - load_dotenv(env_file) - -from core.db.engine import SessionLocal # noqa: E402 -from core.db.models import ChatMessage, ChatSession # noqa: E402 - - -MAX_CONTENT_LENGTH = 100_000 - - -def _truncate(text: str, n: int = 80) -> str: - text = (text or "").replace("\n", " ") - return text if len(text) <= n else text[:n] + "…" - - -def cmd_list(args: argparse.Namespace) -> int: - with SessionLocal() as db: - session = db.query(ChatSession).filter(ChatSession.chat_id == args.chat_id).first() - if session is None: - print(f"[error] chat_id not found: {args.chat_id}", file=sys.stderr) - return 1 - - messages = ( - db.query(ChatMessage) - .filter(ChatMessage.chat_id == args.chat_id) - .order_by(ChatMessage.created_at) - .all() - ) - print(f"chat_id: {args.chat_id} ({len(messages)} messages)") - print(f"{'idx':>3} {'role':<10} {'created_at':<26} message_id content") - print("-" * 120) - for i, m in enumerate(messages): - print( - f"{i:>3} {m.role:<10} {str(m.created_at):<26} " - f"{m.message_id:<30} {_truncate(m.content)}" - ) - return 0 - - -def cmd_show(args: argparse.Namespace) -> int: - with SessionLocal() as db: - msg = db.query(ChatMessage).filter(ChatMessage.message_id == args.message_id).first() - if msg is None: - print(f"[error] message_id not found: {args.message_id}", file=sys.stderr) - return 1 - print(f"message_id: {msg.message_id}") - print(f"chat_id: {msg.chat_id}") - print(f"role: {msg.role}") - print(f"model: {msg.model}") - print(f"created_at: {msg.created_at}") - print(f"tool_calls: {msg.tool_calls}") - print(f"usage: {msg.usage}") - print(f"error: {msg.error}") - print(f"metadata: {msg.extra_data}") - print("----- content -----") - print(msg.content) - return 0 - - -def _resolve_new_content(args: argparse.Namespace) -> str: - if args.content is not None and args.from_file is not None: - raise SystemExit("[error] --content and --from-file are mutually exclusive") - if args.content is not None: - return args.content - if args.from_file is not None: - return Path(args.from_file).read_text(encoding="utf-8") - raise SystemExit("[error] must provide --content or --from-file") - - -def cmd_edit(args: argparse.Namespace) -> int: - new_content = _resolve_new_content(args) - if len(new_content) > MAX_CONTENT_LENGTH: - print( - f"[error] content length {len(new_content)} exceeds DB limit {MAX_CONTENT_LENGTH}", - file=sys.stderr, - ) - return 1 - - with SessionLocal() as db: - msg = db.query(ChatMessage).filter(ChatMessage.message_id == args.message_id).first() - if msg is None: - print(f"[error] message_id not found: {args.message_id}", file=sys.stderr) - return 1 - - old_content = msg.content - print(f"message_id: {msg.message_id} (role={msg.role}, chat_id={msg.chat_id})") - print(f"old length: {len(old_content)} → new length: {len(new_content)}") - print(f"old preview: {_truncate(old_content)}") - print(f"new preview: {_truncate(new_content)}") - - if not args.yes: - confirm = input("Apply this update? [y/N] ").strip().lower() - if confirm not in ("y", "yes"): - print("aborted.") - return 0 - - msg.content = new_content - db.commit() - print(f"[ok] updated message_id={msg.message_id}") - return 0 - - -def cmd_delete(args: argparse.Namespace) -> int: - with SessionLocal() as db: - msg = db.query(ChatMessage).filter(ChatMessage.message_id == args.message_id).first() - if msg is None: - print(f"[error] message_id not found: {args.message_id}", file=sys.stderr) - return 1 - print( - f"About to DELETE message_id={msg.message_id} " - f"(role={msg.role}, chat_id={msg.chat_id})" - ) - print(f"content preview: {_truncate(msg.content)}") - if not args.yes: - confirm = input("Delete this message? [y/N] ").strip().lower() - if confirm not in ("y", "yes"): - print("aborted.") - return 0 - db.delete(msg) - db.commit() - print(f"[ok] deleted message_id={args.message_id}") - return 0 - - -def build_parser() -> argparse.ArgumentParser: - p = argparse.ArgumentParser(description="Edit chat messages in chat_messages table") - sub = p.add_subparsers(dest="command", required=True) - - p_list = sub.add_parser("list", help="List all messages in a chat session") - p_list.add_argument("--chat-id", required=True) - p_list.set_defaults(func=cmd_list) - - p_show = sub.add_parser("show", help="Show full content of one message") - p_show.add_argument("--message-id", required=True) - p_show.set_defaults(func=cmd_show) - - p_edit = sub.add_parser("edit", help="Update the content of one message") - p_edit.add_argument("--message-id", required=True) - p_edit.add_argument("--content", help="New content as inline string") - p_edit.add_argument("--from-file", help="Read new content from a UTF-8 text file") - p_edit.add_argument("-y", "--yes", action="store_true", help="Skip confirmation prompt") - p_edit.set_defaults(func=cmd_edit) - - p_del = sub.add_parser("delete", help="Delete one message") - p_del.add_argument("--message-id", required=True) - p_del.add_argument("-y", "--yes", action="store_true", help="Skip confirmation prompt") - p_del.set_defaults(func=cmd_delete) - - return p - - -def main() -> int: - args = build_parser().parse_args() - return args.func(args) - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/src/backend/scripts/evaluation_sandbox_e2e.py b/src/backend/scripts/evaluation_sandbox_e2e.py deleted file mode 100644 index 2ffef48d..00000000 --- a/src/backend/scripts/evaluation_sandbox_e2e.py +++ /dev/null @@ -1,89 +0,0 @@ -"""Explicit live probe: create two temporary evaluation boxes and always delete.""" -import argparse -import asyncio -import json -import secrets -import shlex - -import core.sandbox.providers.evaluation.binding as bindings -from core.sandbox.errors import SandboxError -from core.sandbox.providers.evaluation.provider import get_evaluation_provider -from core.sandbox.protocol import ProcessRequest -from core.services import evaluation_sandbox_service as service - - -async def denied(operation): - try: - await operation - except SandboxError: - return - raise AssertionError("Operation unexpectedly authorized") - - -async def verify(image): - owner = "eval-probe-" + secrets.token_hex(8) - leases = [] - evidence = {"image": image, "checks": []} - provider = get_evaluation_provider() - try: - for _ in range(2): - leases.append(await service.create(owner, image=image, ttl_seconds=600)) - first, second = leases - sid, lease = first["chat_id"], first["lease_id"] - assert first["sandbox_id"] != second["sandbox_id"] - evidence["sandbox_ids"] = [item["sandbox_id"] for item in leases] - marker = secrets.token_hex(16).encode() - await service.upload(lease, owner, "/attempt/workspace/seed", marker) - assert await provider.get_file(sid, "/workspace/seed", user_id=owner) == marker - await provider.put_file(sid, "/workspace/native-output", marker, user_id=owner) - assert await service.download(lease, owner, "/attempt/workspace/native-output") == marker - assert await provider.current_sandbox_id(sid) == first["sandbox_id"] - evidence["checks"].append("native_and_control_same_instance") - missing = await service.execute( - second["lease_id"], owner, argv=["test", "!", "-e", "/workspace/seed"] - ) - assert missing["exit_code"] == 0 - await denied(provider.get_file(sid, "/workspace/seed", user_id="foreign")) - await denied(service.execute(lease, "foreign", argv=["true"])) - evidence["checks"].extend(["attempt_isolation", "foreign_owner_denied"]) - result = await provider.run_to_completion(ProcessRequest( - script_content="cat /workspace/seed", script_name="probe.sh", - language="bash", session_id=sid, user_id=owner, timeout=20, - )) - assert result.exit_code == 0 and marker.decode() in result.stdout - evidence["checks"].append("native_command_output") - delayed = "sleep 3; echo contaminated > /workspace/late-write" - result = await provider.run_to_completion(ProcessRequest( - script_content="nohup bash -c " + shlex.quote(delayed) + " >/tmp/detached.log 2>&1 str: - """Try to read ADMIN_TOKEN from .env files.""" - for candidate in [ - BACKEND_ROOT.parent.parent / ".env", # project root - BACKEND_ROOT / ".env", - ]: - if candidate.exists(): - for line in candidate.read_text().splitlines(): - line = line.strip() - if line.startswith("ADMIN_TOKEN"): - _, _, val = line.partition("=") - return val.strip().strip('"').strip("'") - return os.getenv("ADMIN_TOKEN", "") - - -# --------------------------------------------------------------------------- -# API-based export -# --------------------------------------------------------------------------- - -def export_docs_via_api(api_url: str, token: str) -> dict: - """GET /v1/content/docs/export""" - import urllib.request - import urllib.error - - url = f"{api_url.rstrip('/')}/v1/content/docs/export" - req = urllib.request.Request(url, headers={"Authorization": f"Bearer {token}"}) - try: - with urllib.request.urlopen(req, timeout=15) as resp: - body = json.loads(resp.read()) - except urllib.error.HTTPError as exc: - print(f"✗ 导出版本说明失败: HTTP {exc.code} — {exc.read().decode()}") - sys.exit(1) - except urllib.error.URLError as exc: - print(f"✗ 无法连接后端: {exc.reason}") - sys.exit(1) - - return body.get("data", body) - - -def export_prompts_via_api(api_url: str, token: str) -> dict: - """GET /v1/content/prompts/export""" - import urllib.request - import urllib.error - - url = f"{api_url.rstrip('/')}/v1/content/prompts/export" - req = urllib.request.Request(url, headers={"Authorization": f"Bearer {token}"}) - try: - with urllib.request.urlopen(req, timeout=30) as resp: - body = json.loads(resp.read()) - except urllib.error.HTTPError as exc: - print(f"✗ 导出系统提示词失败: HTTP {exc.code} — {exc.read().decode()}") - sys.exit(1) - except urllib.error.URLError as exc: - print(f"✗ 无法连接后端: {exc.reason}") - sys.exit(1) - - return body.get("data", body) - - -def export_catalog_via_api(api_url: str) -> dict: - """GET /v1/catalog (public, no auth needed).""" - import urllib.request - import urllib.error - - url = f"{api_url.rstrip('/')}/v1/catalog" - req = urllib.request.Request(url) - try: - with urllib.request.urlopen(req, timeout=15) as resp: - body = json.loads(resp.read()) - except (urllib.error.HTTPError, urllib.error.URLError) as exc: - print(f"✗ 导出能力中心失败: {exc}") - sys.exit(1) - - return body.get("data", body) - - -# --------------------------------------------------------------------------- -# DB-based export (offline, no running server needed) -# --------------------------------------------------------------------------- - -def export_docs_via_db(database_url: str) -> dict: - sys.path.insert(0, str(BACKEND_ROOT)) - from sqlalchemy import create_engine - from sqlalchemy.orm import sessionmaker - from core.db.models import ContentBlock - from core.applications.migration_compat import build_docs_snapshot - - from core.db.connection import utc_connect_args - engine = create_engine(database_url, connect_args=utc_connect_args(database_url)) - session = sessionmaker(bind=engine)() - try: - return build_docs_snapshot(session) - finally: - session.close() - - -def export_prompts_via_db(database_url: str) -> dict: - sys.path.insert(0, str(BACKEND_ROOT)) - from sqlalchemy import create_engine - from sqlalchemy.orm import sessionmaker - from core.content.content_blocks import build_prompt_snapshot - - from core.db.connection import utc_connect_args - engine = create_engine(database_url, connect_args=utc_connect_args(database_url)) - session = sessionmaker(bind=engine)() - try: - return build_prompt_snapshot(session) - finally: - session.close() - - -def export_catalog_overrides_via_db(database_url: str) -> list: - sys.path.insert(0, str(BACKEND_ROOT)) - from sqlalchemy import create_engine - from sqlalchemy.orm import sessionmaker - from core.db.models import CatalogOverride - - from core.db.connection import utc_connect_args - engine = create_engine(database_url, connect_args=utc_connect_args(database_url)) - session = sessionmaker(bind=engine)() - try: - rows = session.query(CatalogOverride).all() - return [ - { - "user_id": r.user_id, - "kind": r.kind, - "item_id": r.item_id, - "enabled": r.enabled, - "config_data": r.config_data or {}, - } - for r in rows - ] - finally: - session.close() - - -# --------------------------------------------------------------------------- -# Write helpers -# --------------------------------------------------------------------------- - -def _write_json(path: Path, data: dict) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") - print(f" → {path} ({path.stat().st_size:,} bytes)") - - -# --------------------------------------------------------------------------- -# Main -# --------------------------------------------------------------------------- - -def main() -> None: - parser = argparse.ArgumentParser( - description="导出版本说明与能力中心数据", - formatter_class=argparse.RawDescriptionHelpFormatter, - epilog=__doc__, - ) - source = parser.add_mutually_exclusive_group(required=True) - source.add_argument("--api-url", help="后端 API 地址，如 http://localhost:3000/api") - source.add_argument("--database-url", help="直接连接数据库（无需运行中的后端）") - - parser.add_argument("--admin-token", help="ADMIN_TOKEN（默认从 .env 读取）") - parser.add_argument("-o", "--output-dir", help="输出目录（默认 scripts/exported/）") - parser.add_argument( - "--only", - choices=["docs", "catalog", "prompts"], - help="仅导出指定部分（--only prompts 等价于只导出提示词）", - ) - parser.add_argument( - "--include-prompts", - action="store_true", - help="额外导出系统提示词 + 提示词广场到 prompts_snapshot_*.json", - ) - - args = parser.parse_args() - - token = args.admin_token or _load_env_token() - output_dir = Path(args.output_dir) if args.output_dir else DEFAULT_OUTPUT_DIR - timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") - - print("=" * 50) - print("HugAgentOS 数据导出") - print("=" * 50) - - # ── 1. Release notes ───────────────────────────────────────────────── - if args.only in (None, "docs"): - print("\n[1/2] 导出版本说明 (content_blocks) ...") - if args.api_url: - if not token: - print("✗ 需要 ADMIN_TOKEN，请通过 --admin-token 指定或在 .env 中设置") - sys.exit(1) - docs_snapshot = export_docs_via_api(args.api_url, token) - else: - docs_snapshot = export_docs_via_db(args.database_url) - - docs_file = output_dir / f"docs_snapshot_{timestamp}.json" - _write_json(docs_file, docs_snapshot) - - # Statistics - blocks = docs_snapshot.get("blocks", {}) - for name, block in blocks.items(): - count = len(block.get("payload", [])) - print(f" {name}: {count} 条") - - # ── 2. Capability center ───────────────────────────────────────────── - if args.only in (None, "catalog"): - print("\n[2/2] 导出能力中心 (catalog) ...") - - catalog_export = {} - - # 2a. catalog.json (static definitions) - if CATALOG_JSON.exists(): - catalog_export["catalog_json"] = json.loads(CATALOG_JSON.read_text(encoding="utf-8")) - for kind in ("skills", "agents", "mcp", "kb"): - items = catalog_export["catalog_json"].get(kind, []) - print(f" catalog.json → {kind}: {len(items)} 项") - else: - print(f" ⚠ catalog.json 不存在: {CATALOG_JSON}") - - # 2b. catalog_overrides (per-user personalization overrides) - if args.api_url: - # The API has no endpoint for bulk-exporting overrides; only catalog.json is exported - catalog_export["catalog_overrides"] = [] - print(" catalog_overrides: 跳过（API 模式不支持批量导出用户覆盖）") - else: - overrides = export_catalog_overrides_via_db(args.database_url) - catalog_export["catalog_overrides"] = overrides - print(f" catalog_overrides: {len(overrides)} 条") - - catalog_export["exported_at"] = utc_now().isoformat() - - catalog_file = output_dir / f"catalog_snapshot_{timestamp}.json" - _write_json(catalog_file, catalog_export) - - # ── 3. System prompts + prompt hub ─────────────────────────────────── - if args.include_prompts or args.only == "prompts": - print("\n[+] 导出系统提示词 (prompt_versions) + 提示词广场 (prompt_hub) ...") - if args.api_url: - if not token: - print("✗ 需要 ADMIN_TOKEN，请通过 --admin-token 指定或在 .env 中设置") - sys.exit(1) - prompt_snapshot = export_prompts_via_api(args.api_url, token) - else: - prompt_snapshot = export_prompts_via_db(args.database_url) - - prompt_file = output_dir / f"prompts_snapshot_{timestamp}.json" - _write_json(prompt_file, prompt_snapshot) - - blocks = prompt_snapshot.get("blocks", {}) - - # prompt_versions: version count per kind + active version - pv_payload = blocks.get("prompt_versions", {}).get("payload", {}) - versions = pv_payload.get("versions", []) if isinstance(pv_payload, dict) else [] - active = pv_payload.get("active", {}) if isinstance(pv_payload, dict) else {} - by_kind: dict = {} - for v in versions: - by_kind.setdefault(v.get("kind", "?"), []).append(v.get("id")) - for kind, ids in sorted(by_kind.items()): - print(f" prompt_versions/{kind}: {len(ids)} 个版本（激活: {active.get(kind)}）") - - # prompt_hub: template count - hub_payload = blocks.get("prompt_hub", {}).get("payload", []) - hub_count = len(hub_payload) if isinstance(hub_payload, list) else 0 - print(f" prompt_hub: {hub_count} 条提示词模板") - - print("\n" + "=" * 50) - print("✓ 导出完成") - print("=" * 50) - - -if __name__ == "__main__": - main() diff --git a/src/backend/scripts/harness_fault_injection.py b/src/backend/scripts/harness_fault_injection.py deleted file mode 100755 index 00d08575..00000000 --- a/src/backend/scripts/harness_fault_injection.py +++ /dev/null @@ -1,637 +0,0 @@ -#!/usr/bin/env python3 -"""Run the Harness v2 process-crash recovery matrix against ephemeral SQLite. - -This is intentionally a destructive *test-only* utility. It creates one new -database per safe point below ``--state-dir``, starts a worker process, waits -until the durable safe point is committed, sends that process ``SIGKILL``, and -then starts a fresh recovery process against the same database. - -The utility refuses non-SQLite databases and existing phase databases so it -cannot accidentally be pointed at a development or production database. -""" - -from __future__ import annotations - -import argparse -import asyncio -import json -import os -import selectors -import signal -import subprocess -import sys -import tempfile -import time -from pathlib import Path -from typing import Any - -BACKEND_ROOT = Path(__file__).resolve().parent.parent -if str(BACKEND_ROOT) not in sys.path: - sys.path.insert(0, str(BACKEND_ROOT)) - -SAFE_POINTS = ( - "pending", - "model_before", - "model_after", - "tool_intent", - "tool_unknown", - "message_committed", - "compacting", - "memory_outbox", -) -RUN_SAFE_POINTS = { - "pending", - "model_before", - "model_after", - "tool_intent", - "tool_unknown", - "message_committed", -} - - -def _emit(payload: dict[str, Any]) -> None: - print(json.dumps(payload, sort_keys=True, ensure_ascii=False), flush=True) - - -def _phase_database(state_dir: str, phase: str) -> Path: - root = Path(state_dir).expanduser().resolve() - root.mkdir(parents=True, exist_ok=True) - return root / f"harness-fault-{phase}.sqlite" - - -def _configure_database(state_dir: str, phase: str) -> Path: - database = _phase_database(state_dir, phase) - os.environ["DATABASE_URL"] = f"sqlite:///{database}" - os.environ["SQLITE_FALLBACK_URL"] = f"sqlite:///{database}" - os.environ["REDIS_URL"] = "" - os.environ["MEMORY_OUTBOX_LEASE_SECONDS"] = "1" - os.environ["MEMORY_OUTBOX_RETRY_BASE_SECONDS"] = "0" - os.environ["MEMORY_AUDIT_ENABLED"] = "false" - return database - - -def _initialize_schema() -> None: - from core.db import models as _models # noqa: F401 - from core.db.engine import Base, engine - - Base.metadata.create_all(engine) - - -def _ensure_chat(chat_id: str, user_id: str) -> None: - from core.db.engine import SessionLocal - from core.db.models import ChatSession - - with SessionLocal() as db: - if db.get(ChatSession, chat_id) is None: - db.add( - ChatSession(chat_id=chat_id, user_id=user_id, title="fault injection") - ) - db.commit() - - -def _run_snapshot(phase: str) -> dict[str, Any]: - return { - "kind": "chat", - "worker_args": { - "session_messages": [{"role": "user", "content": f"recover {phase}"}], - "effective_user_message": f"recover {phase}", - "raw_user_message": f"recover {phase}", - "context": {}, - "model_name": "fault-injection-model", - }, - } - - -def _arm_run_phase(phase: str, lease_seconds: int) -> dict[str, Any]: - from core.db.models import ChatMessage - from core.services.run_journal import RunJournal - - chat_id = f"chat-{phase}" - user_id = "fault-user" - run_id = f"run-{phase}" - message_id = f"assistant-{phase}" - _ensure_chat(chat_id, user_id) - journal = RunJournal() - journal.accept( - run_id=run_id, - message_id=message_id, - chat_id=chat_id, - user_id=user_id, - request_payload={"kind": "chat", "fault_phase": phase}, - recovery_snapshot=_run_snapshot(phase), - ) - if phase == "pending": - return {"run_id": run_id} - - owner = f"fault-worker:{os.getpid()}" - if not journal.claim(run_id, owner=owner, lease_seconds=lease_seconds): - raise RuntimeError(f"failed to claim {run_id}") - - if phase == "model_before": - journal.append_operation( - run_id, - owner=owner, - operation_type="model_call_prepared", - phase="pre_model", - safety="replayable", - payload={"fault_phase": phase}, - ) - elif phase == "model_after": - journal.save_snapshot( - run_id, - owner=owner, - phase="model_completed", - safety="replayable", - snapshot={ - "assistant_content": "durable model output", - "message_id": message_id, - "model_name": "fault-injection-model", - "usage": {"input_tokens": 3, "output_tokens": 4}, - "context": {}, - }, - ) - elif phase in {"tool_intent", "tool_unknown"}: - from core.services.tool_effect_ledger import ToolEffectJournal - - effects = ToolEffectJournal() - recovery_policy = "replay_safe" if phase == "tool_intent" else "never_replay" - decision = effects.begin_intent( - run_id=run_id, - owner=owner, - claim_owner=f"tool-claim:{os.getpid()}", - tool_call_id=f"call-{phase}", - tool_name=f"fault-{phase}", - args={"path": "/ephemeral/fault-input"}, - recovery_policy=recovery_policy, - lease_seconds=lease_seconds, - ) - if phase == "tool_unknown": - effects.mark_unknown( - decision.effect_id, - owner=owner, - claim_owner=f"tool-claim:{os.getpid()}", - reason="fault injection persisted an unknown write outcome", - ) - return {"run_id": run_id, "effect_id": decision.effect_id} - elif phase == "message_committed": - - def _commit_message(db) -> None: - db.add( - ChatMessage( - message_id=message_id, - chat_id=chat_id, - role="assistant", - content="durable assistant message", - ) - ) - - journal.append_operation( - run_id, - owner=owner, - operation_type="message_committed", - phase="message_committed", - safety="side_effect_committed", - payload={"message_id": message_id}, - commit_effect=_commit_message, - ) - return {"run_id": run_id} - - -def _arm_compaction(lease_seconds: int) -> dict[str, Any]: - from core.db.engine import SessionLocal - from core.services.chat_service import ChatService - - chat_id = "chat-compacting" - _ensure_chat(chat_id, "fault-user") - with SessionLocal() as db: - service = ChatService(db) - service.add_message( - chat_id=chat_id, role="user", content="question before crash" - ) - service.add_message( - chat_id=chat_id, role="assistant", content="answer before crash" - ) - snapshot = service.acquire_compaction_snapshot( - chat_id, - owner=f"compactor:{os.getpid()}", - lease_seconds=lease_seconds, - ) - if snapshot is None: - raise RuntimeError("failed to acquire compaction snapshot") - # This arrives after the fixed source watermark, while the dead worker - # would have been summarizing. The restart must not lose it. - with SessionLocal() as db: - ChatService(db).add_message( - chat_id=chat_id, - role="user", - content="tail committed during dead compaction", - ) - return { - "chat_id": chat_id, - "covered_seq": snapshot.covered_seq, - "source_message_ids": list(snapshot.source_message_ids), - } - - -def _arm_memory_outbox() -> dict[str, Any]: - from core.memory.context import MemoryContext - from core.memory.outbox import _claim_specific, enqueue_profile_edit_job - - ctx = MemoryContext( - user_id="fault-user", - workspace_id="fault-workspace", - chat_id="chat-memory-outbox", - write_enabled=True, - message_id="message-memory-outbox", - ) - job_id = enqueue_profile_edit_job( - ctx, - "preference.language", - "Chinese", - operation_id="fault-memory-operation", - ) - worker_id = f"memory-worker:{os.getpid()}" - if not _claim_specific(job_id, worker_id): - raise RuntimeError("failed to claim memory outbox row") - return {"job_id": job_id, "worker_id": worker_id} - - -def _arm(args: argparse.Namespace) -> int: - database = _configure_database(args.state_dir, args.phase) - if database.exists(): - raise RuntimeError(f"refusing to overwrite existing fault database: {database}") - _initialize_schema() - if args.phase in RUN_SAFE_POINTS: - details = _arm_run_phase(args.phase, args.lease_seconds) - elif args.phase == "compacting": - details = _arm_compaction(args.lease_seconds) - elif args.phase == "memory_outbox": - details = _arm_memory_outbox() - else: # pragma: no cover - argparse constrains the value - raise ValueError(args.phase) - _emit({"state": "armed", "phase": args.phase, **details}) - while True: - signal.pause() - - -def _recover_run_phase(phase: str) -> dict[str, Any]: - from core.db.engine import SessionLocal - from core.db.models import ChatMessage, ChatRun, ToolEffectLedger - from core.services.run_journal import RunJournal - - run_id = f"run-{phase}" - tool_actions: list[str] = [] - if phase == "tool_intent": - from core.services.tool_effect_ledger import ( - ToolEffectJournal, - ToolRecoveryRegistry, - recover_incomplete_tool_effects, - ) - - registry = ToolRecoveryRegistry() - registry.register( - "fault-tool_intent", - "replay_safe", - replayer=lambda intent: { - "replayed": True, - "effect_id": intent.effect_id, - }, - ) - decisions = asyncio.run( - recover_incomplete_tool_effects( - journal=ToolEffectJournal(), - registry=registry, - lease_seconds=1, - ) - ) - tool_actions = [decision.action for decision in decisions] - - journal_decisions = RunJournal().recover() - matching = [decision for decision in journal_decisions if decision.run_id == run_id] - run_action = matching[0].action if matching else None - - if phase == "model_after" and matching: - from orchestration.chat_run_executor import _commit_recovered_chat_snapshot - - asyncio.run(_commit_recovered_chat_snapshot(matching[0])) - - with SessionLocal() as db: - run = db.get(ChatRun, run_id) - messages = ( - db.query(ChatMessage) - .filter(ChatMessage.chat_id == f"chat-{phase}") - .order_by(ChatMessage.chat_seq) - .all() - ) - effects = ( - db.query(ToolEffectLedger) - .filter(ToolEffectLedger.run_id == run_id) - .order_by(ToolEffectLedger.event_id) - .all() - ) - return { - "phase": phase, - "run_action": run_action, - "run_status": run.status if run is not None else None, - "run_phase": run.run_phase if run is not None else None, - "tool_actions": tool_actions, - "tool_events": [row.event_type for row in effects], - "message_contents": [row.content for row in messages], - } - - -def _recover_compaction() -> dict[str, Any]: - from core.db.engine import SessionLocal - from core.llm import compaction as compaction_ir - from core.services.chat_service import ChatService - - chat_id = "chat-compacting" - owner = f"recovery-compactor:{os.getpid()}" - with SessionLocal() as db: - service = ChatService(db) - snapshot = service.acquire_compaction_snapshot( - chat_id, - owner=owner, - lease_seconds=30, - ) - if snapshot is None: - raise RuntimeError("expired compaction lease was not recoverable") - source_ids = list(snapshot.source_message_ids) - service.commit_compaction_checkpoint( - snapshot, - owner=owner, - summary_text=compaction_ir.format_summary_text("recovered summary"), - replacement_history=[ - { - "role": "user", - "content": compaction_ir.format_summary_text("recovered summary"), - } - ], - replacement_manifest={ - "fault_injection": True, - "source_message_ids": source_ids, - }, - ) - checkpoint = service.get_latest_compaction_checkpoint(chat_id) - history = service.list_all_messages(chat_id, "fault-user") or [] - return { - "phase": "compacting", - "checkpoint_active": checkpoint is not None, - "covered_seq": (checkpoint.extra_data or {}).get("covered_seq") - if checkpoint - else None, - "source_message_ids": source_ids, - "source_contents": [row.content for row in history if row.role != "system"], - } - - -def _recover_memory_outbox() -> dict[str, Any]: - from core.db.engine import SessionLocal - from core.db.models import MemoryOutbox, ProfileMemory - from core.memory.outbox import drain_outbox - - processed = asyncio.run( - drain_outbox(max_jobs=10, worker_id="restart-memory-worker") - ) - processed_again = asyncio.run( - drain_outbox(max_jobs=10, worker_id="restart-memory-worker-2") - ) - with SessionLocal() as db: - job = ( - db.query(MemoryOutbox).filter(MemoryOutbox.job_kind == "profile_edit").one() - ) - profile = db.get(ProfileMemory, ("fault-user", "fault-workspace")) - return { - "phase": "memory_outbox", - "processed": processed, - "processed_again": processed_again, - "job_status": job.status, - "job_attempts": job.attempts, - "profile_revision": profile.revision if profile is not None else None, - "profile_content": profile.content_md if profile is not None else None, - "receipt_count": len(profile.effect_receipts or {}) - if profile is not None - else 0, - } - - -def _recover(args: argparse.Namespace) -> int: - database = _configure_database(args.state_dir, args.phase) - if not database.exists(): - raise RuntimeError(f"fault database does not exist: {database}") - if args.phase in RUN_SAFE_POINTS: - result = _recover_run_phase(args.phase) - elif args.phase == "compacting": - result = _recover_compaction() - elif args.phase == "memory_outbox": - result = _recover_memory_outbox() - else: # pragma: no cover - argparse constrains the value - raise ValueError(args.phase) - _emit({"state": "recovered", **result}) - return 0 - - -def _read_armed( - proc: subprocess.Popen[str], phase: str, timeout: float -) -> dict[str, Any]: - if proc.stdout is None: - raise RuntimeError("fault worker stdout was not captured") - with selectors.DefaultSelector() as selector: - selector.register(proc.stdout, selectors.EVENT_READ) - deadline = time.monotonic() + timeout - output: list[str] = [] - while time.monotonic() < deadline: - if proc.poll() is not None: - remainder = proc.stdout.read() - if remainder: - output.append(remainder) - stderr = proc.stderr.read() if proc.stderr is not None else "" - raise RuntimeError( - f"fault worker {phase} exited before arming: " - f"stdout={''.join(output)!r} stderr={stderr!r}" - ) - events = selector.select(timeout=min(0.25, deadline - time.monotonic())) - if not events: - continue - line = proc.stdout.readline() - if not line: - continue - output.append(line) - try: - payload = json.loads(line) - except json.JSONDecodeError: - continue - if ( - isinstance(payload, dict) - and payload.get("state") == "armed" - and payload.get("phase") == phase - ): - return {str(key): value for key, value in payload.items()} - raise TimeoutError(f"fault worker {phase} did not arm within {timeout}s") - - -def _last_json(stdout: str) -> dict[str, Any]: - for line in reversed(stdout.splitlines()): - try: - payload = json.loads(line) - except json.JSONDecodeError: - continue - if isinstance(payload, dict): - return payload - raise RuntimeError(f"subprocess emitted no JSON payload: {stdout!r}") - - -def _assert_matrix(results: dict[str, dict[str, Any]]) -> None: - assert results["pending"]["run_action"] == "resume" - assert results["model_before"]["run_action"] == "resume" - assert results["model_after"]["run_action"] == "resume_from_snapshot" - assert results["model_after"]["run_status"] == "completed" - assert results["model_after"]["message_contents"] == ["durable model output"] - assert results["tool_intent"]["tool_actions"] == ["replayed"] - assert results["tool_intent"]["tool_events"] == [ - "intent", - "recovery_claim", - "result", - ] - assert results["tool_intent"]["run_status"] == "needs_attention" - assert results["tool_unknown"]["run_status"] == "needs_attention" - assert results["tool_unknown"]["tool_events"] == ["intent", "unknown_outcome"] - assert results["message_committed"]["run_action"] == "completed" - assert results["message_committed"]["run_status"] == "completed" - assert results["message_committed"]["message_contents"] == [ - "durable assistant message" - ] - assert results["compacting"]["checkpoint_active"] is True - assert len(results["compacting"]["source_message_ids"]) == 3 - assert ( - "tail committed during dead compaction" - in results["compacting"]["source_contents"] - ) - assert results["memory_outbox"]["processed"] == 1 - assert results["memory_outbox"]["processed_again"] == 0 - assert results["memory_outbox"]["job_status"] == "succeeded" - assert results["memory_outbox"]["profile_content"] == ( - "- **preference.language**: Chinese" - ) - assert results["memory_outbox"]["receipt_count"] == 1 - - -def _matrix(args: argparse.Namespace) -> int: - if not hasattr(signal, "SIGKILL"): - raise RuntimeError("the process-crash matrix requires POSIX SIGKILL") - state_dir = Path(args.state_dir).expanduser().resolve() - state_dir.mkdir(parents=True, exist_ok=True) - if any(state_dir.iterdir()): - raise RuntimeError(f"state directory must be empty: {state_dir}") - - script = Path(__file__).resolve() - killed: dict[str, dict[str, Any]] = {} - for phase in SAFE_POINTS: - command = [ - sys.executable, - str(script), - "arm", - "--state-dir", - str(state_dir), - "--phase", - phase, - "--lease-seconds", - str(args.lease_seconds), - ] - proc = subprocess.Popen( - command, - cwd=str(script.parent.parent), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - start_new_session=True, - ) - try: - armed = _read_armed(proc, phase, args.timeout) - os.kill(proc.pid, signal.SIGKILL) - return_code = proc.wait(timeout=5) - if return_code != -signal.SIGKILL: - raise RuntimeError( - f"fault worker {phase} was not killed by SIGKILL: {return_code}" - ) - finally: - if proc.poll() is None: - proc.kill() - proc.wait(timeout=5) - killed[phase] = armed - - # All leases were armed with the same short duration. Wait once after all - # workers are dead, then recover every database in a genuinely fresh process. - time.sleep(args.lease_seconds + 0.25) - results: dict[str, dict[str, Any]] = {} - for phase in SAFE_POINTS: - completed = subprocess.run( - [ - sys.executable, - str(script), - "recover", - "--state-dir", - str(state_dir), - "--phase", - phase, - ], - cwd=str(script.parent.parent), - capture_output=True, - text=True, - timeout=args.timeout, - check=False, - ) - if completed.returncode != 0: - raise RuntimeError( - f"recovery process {phase} failed: " - f"stdout={completed.stdout!r} stderr={completed.stderr!r}" - ) - results[phase] = _last_json(completed.stdout) - - _assert_matrix(results) - _emit( - { - "state": "matrix_passed", - "signal": "SIGKILL", - "killed_phases": list(killed), - "results": results, - } - ) - return 0 - - -def _parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser(description=__doc__) - subparsers = parser.add_subparsers(dest="command", required=True) - - matrix = subparsers.add_parser("matrix", help="kill and recover all safe points") - matrix.add_argument( - "--state-dir", - default=str(Path(tempfile.gettempdir()) / "hugagent-harness-fault-matrix"), - ) - matrix.add_argument("--lease-seconds", type=int, default=1) - matrix.add_argument("--timeout", type=float, default=30.0) - matrix.set_defaults(handler=_matrix) - - arm = subparsers.add_parser("arm", help=argparse.SUPPRESS) - arm.add_argument("--state-dir", required=True) - arm.add_argument("--phase", choices=SAFE_POINTS, required=True) - arm.add_argument("--lease-seconds", type=int, default=1) - arm.set_defaults(handler=_arm) - - recover = subparsers.add_parser("recover", help=argparse.SUPPRESS) - recover.add_argument("--state-dir", required=True) - recover.add_argument("--phase", choices=SAFE_POINTS, required=True) - recover.set_defaults(handler=_recover) - return parser - - -def main() -> int: - args = _parser().parse_args() - if getattr(args, "lease_seconds", 1) < 1: - raise ValueError("lease seconds must be at least 1") - return int(args.handler(args)) - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/src/backend/scripts/import_cherry_agents.py b/src/backend/scripts/import_cherry_agents.py deleted file mode 100644 index c0440eb7..00000000 --- a/src/backend/scripts/import_cherry_agents.py +++ /dev/null @@ -1,372 +0,0 @@ -#!/usr/bin/env python3 -"""From a curated Cherry Studio subset -> generate preset sub-agent marketplace bundles (one-off, dev-only). - -Cherry Studio's ``agents-zh.json`` is a curated dataset under AGPL-3.0 plus additional -commercial-use restrictions; vendoring it verbatim would make AGPL contaminate the -commercial deliverable. This script therefore **only cherry-picks the role subset -relevant to this project**, "rewrites and rearranges" the prompts with -debranding/normalization, and for each role **binds skills / MCP tools / plugins that -actually exist in this system**, finally writing our own preset content to disk -(without keeping Cherry's ids / original curation structure). - -Usage (from src/backend): - python scripts/import_cherry_agents.py # use cache or download now, generate all bundles - python scripts/import_cherry_agents.py --source /path/to/agents-zh.json - python scripts/import_cherry_agents.py --dry-run # validate only, no writes - -Output: ``extensions/agents/marketplace//agent.json``. All binding ids are validated -against the whitelist of the system's real capabilities; any nonexistent skill/tool/plugin -causes a non-zero exit, so nothing gets bound to thin air. -""" - -from __future__ import annotations - -import argparse -import json -import re -import sys -import urllib.request -from pathlib import Path -from typing import Any, Dict, List - -BACKEND_ROOT = Path(__file__).resolve().parents[1] -MARKET_DIR = BACKEND_ROOT / "extensions/agents" / "marketplace" -CHERRY_URL = "https://raw.githubusercontent.com/CherryHQ/cherry-studio/main/resources/data/agents-zh.json" - -# ── Curation + adaptation table ────────────────────────────────────────────── -# Each entry = one preset sub-agent: -# name —— role name to match in Cherry data (exact match) -# slug/cat —— our marketplace slug and one of the 9 top-level categories -# skills/mcp/plugins —— bound ids of real capabilities in this system (the script validates existence) -# tags/questions —— marketplace display tags + suggested questions (written by us, not Cherry originals) -# featured —— whether pinned as featured -CURATION: List[Dict[str, Any]] = [ - # ── Workplace & office (category "职场办公") ── - {"name": "产品经理", "slug": "product-manager", "category": "职场办公", "featured": True, - "tags": ["产品", "需求分析", "PRD"], - "skills": ["pmaster", "diagram-builder", "officecli-pptx"], - "mcp": ["internet_search", "retrieve_dataset_content"], - "questions": ["帮我梳理这个功能的需求文档框架", "为这个产品方案做一次竞品分析"]}, - {"name": "项目管理", "slug": "project-manager", "category": "职场办公", - "tags": ["项目管理", "排期", "周报"], - "skills": ["diagram-builder", "smart-weekly-report"], - "mcp": ["automation_task"], - "questions": ["帮我把这个项目拆解成里程碑和任务", "根据进度生成本周项目周报"]}, - {"name": "行政", "slug": "admin-clerk", "category": "职场办公", - "tags": ["行政", "公文", "会议"], - "skills": ["officecli-docx", "meeting-minutes-organizer"], - "mcp": [], - "questions": ["帮我起草一份会议通知", "把这段会议录音整理成纪要"]}, - {"name": "HR人力资源管理", "slug": "hr-manager", "category": "职场办公", - "tags": ["人力资源", "招聘", "制度"], - "skills": ["officecli-docx", "official-doc-writer"], - "mcp": ["internet_search"], - "questions": ["帮我写一份岗位招聘JD", "起草一份员工考勤管理制度"]}, - {"name": "招聘", "slug": "recruiter", "category": "职场办公", - "tags": ["招聘", "简历", "面试"], - "skills": ["officecli-docx"], - "mcp": ["internet_search"], - "questions": ["帮我评估这份简历是否匹配岗位", "设计这个岗位的面试问题清单"]}, - {"name": "美文排版", "slug": "text-beautify", "category": "职场办公", - "tags": ["排版", "美化", "公众号"], - "skills": ["officecli-docx"], - "mcp": [], - "questions": ["帮我把这段文字排版美化", "给这篇文章加上合适的小标题和分段"]}, - {"name": "会计师", "slug": "accountant", "category": "职场办公", - "tags": ["会计", "账务", "报表"], - "skills": ["officecli-xlsx", "financial-report-minesweeper"], - "mcp": ["database_query"], - "questions": ["帮我核对这张费用报表", "解释一下这几个会计科目的处理"]}, - - # ── Business analysis (category "商业分析") ── - {"name": "商业数据分析", "slug": "business-data-analysis", "category": "商业分析", "featured": True, - "tags": ["商业分析", "经营", "洞察"], - "skills": ["data-analyst-cn", "data-analysis-report-generator"], - "mcp": ["database_query", "retrieve_dataset_content"], - "questions": ["从这份销售数据里找出关键经营洞察", "帮我做一份经营分析报告"]}, - {"name": "市场营销", "slug": "marketing-specialist", "category": "商业分析", - "tags": ["市场", "营销", "推广"], - "skills": ["copywriter-cn", "ad-copywriter"], - "mcp": ["internet_search"], - "questions": ["帮我策划一次产品上新的营销活动", "写三版不同风格的推广文案"]}, - {"name": "金融分析师", "slug": "financial-analyst", "category": "商业分析", - "tags": ["金融", "财报", "估值"], - "skills": ["financial-report-minesweeper", "officecli-xlsx"], - "mcp": ["database_query"], - "questions": ["帮我分析这家公司的财报有没有风险点", "对这组财务指标做可视化"]}, - {"name": "投资经理", "slug": "investment-manager", "category": "商业分析", - "tags": ["投资", "尽调", "策略"], - "skills": ["financial-report-minesweeper"], - "mcp": ["internet_search", "web_fetch"], - "questions": ["帮我梳理这个标的的投资逻辑", "这个行业当前的投资风险有哪些"]}, - {"name": "供应链策略专家", "slug": "supply-chain-strategist", "category": "商业分析", - "tags": ["供应链", "采购", "成本"], - "skills": ["diagram-builder"], - "mcp": ["internet_search", "retrieve_dataset_content"], - "questions": ["帮我画出这条供应链的关键环节", "如何优化这个环节的库存与成本"]}, - {"name": "SEO专家", "slug": "seo-expert", "category": "商业分析", - "tags": ["SEO", "搜索", "流量"], - "skills": ["copywriter-cn"], - "mcp": ["internet_search", "web_fetch"], - "questions": ["帮我优化这篇文章的SEO关键词", "分析这个页面的搜索表现并给建议"]}, - - # ── Data analysis (category "数据分析") ── - {"name": "数据分析师", "slug": "data-analyst", "category": "数据分析", "featured": True, - "tags": ["数据分析", "建模", "可视化"], - "skills": ["data-analyst-cn", "csv-pipeline", "officecli-xlsx", "data-analysis-report-generator"], - "mcp": ["database_query"], - "questions": ["帮我清洗并分析这份CSV数据", "把这组数据做成可视化图表"]}, - {"name": "网站运营数据分析", "slug": "web-analytics", "category": "数据分析", - "tags": ["网站运营", "转化", "漏斗"], - "skills": ["data-analyst-cn", "funnel-analyzer"], - "mcp": ["database_query"], - "questions": ["分析这个转化漏斗在哪一步流失最多", "给出提升网站留存的数据建议"]}, - {"name": "统计学家", "slug": "statistician", "category": "数据分析", - "tags": ["统计", "假设检验", "回归"], - "skills": ["csv-pipeline", "officecli-xlsx"], - "mcp": [], - "questions": ["帮我选择合适的统计检验方法", "对这组数据做一次回归分析"]}, - - # ── R&D & programming (category "研发编程") ── - {"name": "开发工程师", "slug": "developer", "category": "研发编程", "featured": True, - "tags": ["编程", "调试", "评审"], - "skills": ["code-review", "debug-pro"], - "mcp": ["internet_search", "web_fetch"], - "questions": ["帮我review这段代码有没有问题", "这个报错怎么排查"]}, - {"name": "前端工程师", "slug": "frontend-engineer", "category": "研发编程", - "tags": ["前端", "React", "样式"], - "skills": ["code-review", "debug-pro"], - "mcp": ["internet_search"], - "questions": ["帮我优化这个组件的渲染性能", "这个CSS布局问题怎么修"]}, - {"name": "运维工程师", "slug": "devops-engineer", "category": "研发编程", - "tags": ["运维", "日志", "故障"], - "skills": ["log-analyzer", "debug-pro"], - "mcp": ["internet_search"], - "questions": ["帮我分析这段日志定位故障", "这个服务的监控告警怎么配置"]}, - {"name": "测试工程师", "slug": "qa-engineer", "category": "研发编程", - "tags": ["测试", "用例", "质量"], - "skills": ["code-review"], - "mcp": [], - "questions": ["帮我为这个功能设计测试用例", "这个缺陷的复现步骤怎么写清楚"]}, - {"name": "网络安全专家", "slug": "security-expert", "category": "研发编程", - "tags": ["安全", "渗透", "加固"], - "skills": ["log-analyzer"], - "mcp": ["internet_search"], - "questions": ["帮我评估这段代码的安全风险", "这条可疑访问日志要不要警惕"]}, - {"name": "技术作家", "slug": "technical-writer", "category": "研发编程", - "tags": ["技术文档", "API", "翻译"], - "skills": ["api-doc-writer", "md2word-cn", "tech-translator"], - "mcp": [], - "questions": ["帮我把这个接口整理成API文档", "把这段英文技术文档翻译成中文"]}, - {"name": "全栈开发者", "slug": "fullstack-developer", "category": "研发编程", - "tags": ["全栈", "架构", "接口"], - "skills": ["code-review", "debug-pro", "api-doc-writer"], - "mcp": ["internet_search"], - "questions": ["帮我设计这个功能的前后端方案", "review一下这个接口设计是否合理"]}, - - # ── Translation & writing (category "翻译写作") ── - {"name": "英语翻译和改进者", "slug": "english-translator", "category": "翻译写作", "featured": True, - "tags": ["翻译", "润色", "英语"], - "skills": ["translate", "tech-translator"], - "mcp": [], - "questions": ["把这段中文翻译成地道英文", "帮我润色这封英文邮件"]}, - {"name": "要点精炼", "slug": "key-points-refine", "category": "翻译写作", - "tags": ["摘要", "提炼", "总结"], - "skills": ["report-summary-generation"], - "mcp": [], - "questions": ["帮我把这篇长文提炼成要点", "用三句话总结这段内容"]}, - {"name": "论文写手", "slug": "essay-writer", "category": "翻译写作", - "tags": ["论文", "学术", "写作"], - "skills": ["writing-polish", "humanize-zh"], - "mcp": ["retrieve_dataset_content"], - "questions": ["帮我搭一个论文的写作大纲", "润色这段论文摘要让它更严谨"]}, - {"name": "记者", "slug": "journalist", "category": "翻译写作", - "tags": ["新闻", "稿件", "采访"], - "skills": ["copywriter-cn", "humanize-zh"], - "mcp": ["internet_search", "web_fetch"], - "questions": ["帮我把这件事写成一篇新闻稿", "设计一份采访提纲"]}, - - # ── Creative design (category "创意设计") ── - {"name": "广告商", "slug": "advertiser", "category": "创意设计", - "tags": ["广告", "创意", "投放"], - "skills": ["ad-copywriter", "copywriter-cn"], - "mcp": ["internet_search"], - "questions": ["帮我想几个这个产品的广告创意", "写一句抓人的广告Slogan"]}, - {"name": "网页设计顾问", "slug": "web-design-consultant", "category": "创意设计", - "tags": ["网页设计", "UI", "体验"], - "skills": ["diagram-builder"], - "mcp": ["internet_search"], - "questions": ["帮我规划这个落地页的版块结构", "这个界面的交互怎么改更好用"]}, - {"name": "社交媒体经理", "slug": "social-media-manager", "category": "创意设计", - "tags": ["新媒体", "运营", "内容"], - "skills": ["copywriter-cn", "ad-copywriter"], - "mcp": ["internet_search"], - "questions": ["帮我做一周的社媒内容排期", "写一条小红书风格的种草文案"]}, - - # ── Policy & legal (category "政策法务") ── - {"name": "法务", "slug": "legal-affairs", "category": "政策法务", "featured": True, - "tags": ["法务", "合同", "风险"], - "skills": ["contract-guardian", "case-research"], - "mcp": ["retrieve_dataset_content"], - "questions": ["帮我审一下这份合同有没有风险条款", "这种情况涉及哪些法律风险"]}, - {"name": "法律顾问", "slug": "legal-advisor", "category": "政策法务", - "tags": ["法律", "咨询", "合规"], - "skills": ["contract-guardian", "case-research"], - "mcp": ["retrieve_dataset_content", "internet_search"], - "questions": ["帮我解读这条法规怎么落地", "这个业务在合规上要注意什么"]}, - - # ── Education & research (category "教育科研") ── - {"name": "哲学教师", "slug": "philosophy-teacher", "category": "教育科研", - "tags": ["教育", "哲学", "思辨"], - "skills": [], - "mcp": ["retrieve_dataset_content"], - "questions": ["用通俗的方式讲讲这个哲学概念", "带我辩证地分析这个观点"]}, - {"name": "认知科学研究员", "slug": "cognitive-scientist", "category": "教育科研", - "tags": ["认知", "科研", "学习"], - "skills": [], - "mcp": ["internet_search", "web_fetch"], - "questions": ["从认知科学角度解释这个现象", "有哪些提升学习效率的科学方法"]}, - {"name": "概念框架开发者", "slug": "concept-framework-developer", "category": "教育科研", - "tags": ["框架", "建模", "结构化"], - "skills": ["diagram-builder"], - "mcp": [], - "questions": ["帮我把这个想法整理成一个概念框架", "用结构化的方式拆解这个问题"]}, -] - -# Debranding / external product terms: scrubbed during rewriting so stored content is our own neutral content. -_BRAND_PATTERNS = [ - (re.compile(r"Cherry\s*Studio", re.I), "本助手"), - (re.compile(r"\bCherryHQ\b", re.I), "本助手"), - (re.compile(r"\bCherry\b", re.I), "本助手"), - (re.compile(r"ChatGPT", re.I), "AI 助手"), - (re.compile(r"\bOpenAI\b", re.I), "模型"), -] - - -def _debrand(text: str) -> str: - """Debrand + normalize newlines (\\r\\n -> \\n), rewriting into our own neutral prompt.""" - text = (text or "").replace("\r\n", "\n").replace("\r", "\n") - for pat, repl in _BRAND_PATTERNS: - text = pat.sub(repl, text) - # Collapse 3+ consecutive blank lines - text = re.sub(r"\n{3,}", "\n\n", text) - return text.strip() - - -def _load_whitelist() -> Dict[str, set]: - """Build the whitelist of bindable ids from real repo assets: default + marketplace skills / MCP / plugins.""" - skills: set = set() - for sub in ("default", "marketplace"): - d = BACKEND_ROOT / "extensions/skills" / sub - if d.is_dir(): - skills.update(c.name for c in d.iterdir() if c.is_dir()) - plugins: set = set() - for sub in ("default", "marketplace"): - d = BACKEND_ROOT / "extensions/plugins" / sub - if d.is_dir(): - plugins.update(c.name for c in d.iterdir() if c.is_dir()) - mcp: set = set() - try: - cat = json.loads((BACKEND_ROOT / "core" / "config" / "catalog.json").read_text("utf-8")) - for m in (cat.get("mcp") or cat.get("mcp_servers") or []): - if m.get("id"): - mcp.add(m["id"]) - except Exception as exc: # noqa: BLE001 - print(f"!! 无法读取 catalog.json 的 MCP 列表：{exc}", file=sys.stderr) - return {"skills": skills, "mcp": mcp, "plugins": plugins} - - -def _validate_bindings(entry: Dict[str, Any], wl: Dict[str, set]) -> List[str]: - errs: List[str] = [] - for sid in entry.get("skills", []): - if sid not in wl["skills"]: - errs.append(f"未知技能 id: {sid}") - for mid in entry.get("mcp", []): - if mid not in wl["mcp"]: - errs.append(f"未知 MCP id: {mid}") - for pid in entry.get("plugins", []): - if pid not in wl["plugins"]: - errs.append(f"未知插件 slug: {pid}") - return errs - - -def _load_cherry(source: str | None) -> Dict[str, Dict[str, Any]]: - if source: - raw = Path(source).read_text("utf-8") - else: - print(f".. 下载 {CHERRY_URL}") - with urllib.request.urlopen(CHERRY_URL, timeout=60) as resp: # noqa: S310 - raw = resp.read().decode("utf-8") - data = json.loads(raw) - return {a["name"]: a for a in data if a.get("name")} - - -def main() -> int: - ap = argparse.ArgumentParser() - ap.add_argument("--source", help="本地 agents-zh.json 路径（默认在线下载）") - ap.add_argument("--dry-run", action="store_true", help="只校验，不写盘") - args = ap.parse_args() - - wl = _load_whitelist() - print(f".. 白名单：技能 {len(wl['skills'])} / MCP {len(wl['mcp'])} / 插件 {len(wl['plugins'])}") - - # 1) Validate all binding ids up front; fail immediately if any does not exist (no binding to thin air) - all_errs: List[str] = [] - for e in CURATION: - for msg in _validate_bindings(e, wl): - all_errs.append(f"[{e['slug']}] {msg}") - if all_errs: - print("!! 绑定校验失败：", file=sys.stderr) - for m in all_errs: - print(" - " + m, file=sys.stderr) - return 1 - - cherry = _load_cherry(args.source) - written, missing = 0, [] - for e in CURATION: - src = cherry.get(e["name"]) - if src is None: - missing.append(e["name"]) - continue - bundle = { - "slug": e["slug"], - "name": e["name"], - # Cherry's emoji field occasionally carries trailing \r\n; strip it to avoid polluting avatar rendering - "avatar": (src.get("emoji") or "🤖").strip() or "🤖", - "summary": _debrand(src.get("description") or "")[:80], - "description": _debrand(src.get("description") or ""), - "category": e["category"], - "tags": e.get("tags", []), - "version": "1.0.0", - "author": "内置", - "source": "builtin", - "featured": bool(e.get("featured")), - "system_prompt": _debrand(src.get("prompt") or ""), - "welcome_message": "", - "suggested_questions": e.get("questions", []), - "model_config": {}, - "bindings": { - "skill_ids": e.get("skills", []), - "mcp_server_ids": e.get("mcp", []), - "plugin_ids": e.get("plugins", []), - "kb_ids": [], - }, - } - if args.dry_run: - written += 1 - continue - out_dir = MARKET_DIR / e["slug"] - out_dir.mkdir(parents=True, exist_ok=True) - (out_dir / "agent.json").write_text( - json.dumps(bundle, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" - ) - written += 1 - - print(f"== {'校验通过' if args.dry_run else '已生成'} {written} 个预置子智能体 → {MARKET_DIR}") - if missing: - print(f"!! Cherry 数据中未找到 {len(missing)} 个角色（已跳过）：{'、'.join(missing)}", file=sys.stderr) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/src/backend/scripts/import_content.py b/src/backend/scripts/import_content.py deleted file mode 100644 index 8e9cb253..00000000 --- a/src/backend/scripts/import_content.py +++ /dev/null @@ -1,379 +0,0 @@ -#!/usr/bin/env python3 -""" -Import release-notes and capability-center data, for cross-environment migration. - -Used together with export_content.py to import development-environment data into -production. - -Usage: - # Import release notes via the API (recommended) - python scripts/import_content.py --api-url http:///api \ - --docs docs_snapshot_20260310_143000.json - - # Import the capability center via the API (catalog.json is overwritten; backend restart required) - python scripts/import_content.py --api-url http:///api \ - --catalog catalog_snapshot_20260310_143000.json - - # Import both at once - python scripts/import_content.py --api-url http:///api \ - --docs docs_snapshot.json --catalog catalog_snapshot.json - - # Write directly to the database (no running backend needed) - python scripts/import_content.py --database-url postgresql://user:pass@host/db \ - --docs docs_snapshot.json --catalog catalog_snapshot.json - - # Import the system prompt version pool (prompt_versions: system/code_exec/distillation/plan_mode) - python scripts/import_content.py --api-url http:///api \ - --prompts prompts_snapshot_20260310_143000.json - - # Do not overwrite existing data (only write missing blocks) - python scripts/import_content.py --api-url http:///api \ - --docs docs_snapshot.json --no-overwrite - - # Dry run, no actual writes - python scripts/import_content.py --api-url http:///api \ - --docs docs_snapshot.json --dry-run -""" - -import argparse -import json -import os -import shutil -import sys -from datetime import datetime -from pathlib import Path - -# --------------------------------------------------------------------------- -# Constants -# --------------------------------------------------------------------------- - -SCRIPT_DIR = Path(__file__).resolve().parent -BACKEND_ROOT = SCRIPT_DIR.parent -CATALOG_JSON = BACKEND_ROOT / "configs" / "catalog.json" - - -def _load_env_token() -> str: - for candidate in [ - BACKEND_ROOT.parent.parent / ".env", - BACKEND_ROOT / ".env", - ]: - if candidate.exists(): - for line in candidate.read_text().splitlines(): - line = line.strip() - if line.startswith("ADMIN_TOKEN"): - _, _, val = line.partition("=") - return val.strip().strip('"').strip("'") - return os.getenv("ADMIN_TOKEN", "") - - -def _read_json(path: Path) -> dict: - if not path.exists(): - print(f"✗ 文件不存在: {path}") - sys.exit(1) - return json.loads(path.read_text(encoding="utf-8")) - - -# --------------------------------------------------------------------------- -# API-based import -# --------------------------------------------------------------------------- - -def import_docs_via_api(api_url: str, token: str, snapshot: dict, overwrite: bool) -> dict: - """POST /v1/content/docs/import""" - import urllib.request - import urllib.error - - url = f"{api_url.rstrip('/')}/v1/content/docs/import?overwrite={'true' if overwrite else 'false'}" - data = json.dumps(snapshot, ensure_ascii=False).encode("utf-8") - req = urllib.request.Request( - url, - data=data, - headers={ - "Authorization": f"Bearer {token}", - "Content-Type": "application/json", - }, - method="POST", - ) - try: - with urllib.request.urlopen(req, timeout=30) as resp: - body = json.loads(resp.read()) - except urllib.error.HTTPError as exc: - print(f"✗ 导入版本说明失败: HTTP {exc.code} — {exc.read().decode()}") - sys.exit(1) - except urllib.error.URLError as exc: - print(f"✗ 无法连接后端: {exc.reason}") - sys.exit(1) - - return body.get("data", body) - - -# --------------------------------------------------------------------------- -# DB-based import -# --------------------------------------------------------------------------- - -def import_docs_via_db(database_url: str, snapshot: dict, overwrite: bool) -> dict: - sys.path.insert(0, str(BACKEND_ROOT)) - from sqlalchemy import create_engine - from sqlalchemy.orm import sessionmaker - from core.applications.migration_compat import import_docs_snapshot - - from core.db.connection import utc_connect_args - engine = create_engine(database_url, connect_args=utc_connect_args(database_url)) - session = sessionmaker(bind=engine)() - try: - result = import_docs_snapshot(session, snapshot, overwrite=overwrite, default_updated_by="admin_import") - return result - finally: - session.close() - - -def import_prompts_via_api(api_url: str, token: str, snapshot: dict, overwrite: bool) -> dict: - """POST /v1/content/prompts/import""" - import urllib.request - import urllib.error - - url = f"{api_url.rstrip('/')}/v1/content/prompts/import?overwrite={'true' if overwrite else 'false'}" - data = json.dumps(snapshot, ensure_ascii=False).encode("utf-8") - req = urllib.request.Request( - url, - data=data, - headers={ - "Authorization": f"Bearer {token}", - "Content-Type": "application/json", - }, - method="POST", - ) - try: - with urllib.request.urlopen(req, timeout=30) as resp: - body = json.loads(resp.read()) - except urllib.error.HTTPError as exc: - print(f"✗ 导入系统提示词失败: HTTP {exc.code} — {exc.read().decode()}") - sys.exit(1) - except urllib.error.URLError as exc: - print(f"✗ 无法连接后端: {exc.reason}") - sys.exit(1) - - return body.get("data", body) - - -def import_prompts_via_db(database_url: str, snapshot: dict, overwrite: bool) -> dict: - sys.path.insert(0, str(BACKEND_ROOT)) - from sqlalchemy import create_engine - from sqlalchemy.orm import sessionmaker - from core.content.content_blocks import import_prompt_snapshot - - from core.db.connection import utc_connect_args - engine = create_engine(database_url, connect_args=utc_connect_args(database_url)) - session = sessionmaker(bind=engine)() - try: - return import_prompt_snapshot( - session, snapshot, overwrite=overwrite, default_updated_by="admin_import" - ) - finally: - session.close() - - -def import_catalog_overrides_via_db(database_url: str, overrides: list, overwrite: bool) -> dict: - sys.path.insert(0, str(BACKEND_ROOT)) - from sqlalchemy import create_engine - from sqlalchemy.orm import sessionmaker - from core.db.models import CatalogOverride, Base - - from core.db.connection import utc_connect_args - engine = create_engine(database_url, connect_args=utc_connect_args(database_url)) - Base.metadata.create_all(engine, tables=[CatalogOverride.__table__], checkfirst=True) - session = sessionmaker(bind=engine)() - - imported, skipped = 0, 0 - try: - for item in overrides: - existing = ( - session.query(CatalogOverride) - .filter_by(user_id=item["user_id"], kind=item["kind"], item_id=item["item_id"]) - .first() - ) - if existing: - if overwrite: - existing.enabled = item["enabled"] - existing.config_data = item.get("config_data", {}) - imported += 1 - else: - skipped += 1 - else: - session.add(CatalogOverride( - user_id=item["user_id"], - kind=item["kind"], - item_id=item["item_id"], - enabled=item["enabled"], - config_data=item.get("config_data", {}), - )) - imported += 1 - session.commit() - except Exception: - session.rollback() - raise - finally: - session.close() - - return {"imported": imported, "skipped": skipped} - - -# --------------------------------------------------------------------------- -# Catalog.json file replacement -# --------------------------------------------------------------------------- - -def import_catalog_json(catalog_data: dict, dry_run: bool) -> None: - """Replace catalog.json with exported version, keeping a backup.""" - if not catalog_data: - print(" catalog.json: 快照中无数据，跳过") - return - - if dry_run: - for kind in ("skills", "agents", "mcp", "kb"): - items = catalog_data.get(kind, []) - print(f" [dry-run] catalog.json → {kind}: {len(items)} 项") - return - - # Backup existing - if CATALOG_JSON.exists(): - ts = datetime.now().strftime("%Y%m%d_%H%M%S") - backup = CATALOG_JSON.with_suffix(f".backup_{ts}.json") - shutil.copy2(CATALOG_JSON, backup) - print(f" 已备份: {backup}") - - CATALOG_JSON.write_text( - json.dumps(catalog_data, ensure_ascii=False, indent=2) + "\n", - encoding="utf-8", - ) - for kind in ("skills", "agents", "mcp", "kb"): - items = catalog_data.get(kind, []) - print(f" catalog.json → {kind}: {len(items)} 项") - - print(" ⚠ catalog.json 已更新，需要重启后端才能生效:") - print(" docker-compose up -d --build backend") - - -# --------------------------------------------------------------------------- -# Main -# --------------------------------------------------------------------------- - -def main() -> None: - parser = argparse.ArgumentParser( - description="导入版本说明与能力中心数据", - formatter_class=argparse.RawDescriptionHelpFormatter, - epilog=__doc__, - ) - source = parser.add_mutually_exclusive_group(required=True) - source.add_argument("--api-url", help="后端 API 地址，如 http:///api") - source.add_argument("--database-url", help="直接连接数据库") - - parser.add_argument("--admin-token", help="ADMIN_TOKEN（默认从 .env 读取）") - parser.add_argument("--docs", type=Path, help="版本说明快照文件路径 (docs_snapshot_*.json)") - parser.add_argument("--catalog", type=Path, help="能力中心快照文件路径 (catalog_snapshot_*.json)") - parser.add_argument("--prompts", type=Path, help="系统提示词版本池快照文件路径 (prompts_snapshot_*.json)") - parser.add_argument("--no-overwrite", action="store_true", help="不覆盖已有数据") - parser.add_argument("--dry-run", action="store_true", help="试运行，仅输出将要执行的操作") - - args = parser.parse_args() - - if not args.docs and not args.catalog and not args.prompts: - parser.error("请至少指定 --docs / --catalog / --prompts 之一") - - token = args.admin_token or _load_env_token() - overwrite = not args.no_overwrite - - print("=" * 50) - print("HugAgentOS 数据导入") - print("=" * 50) - if args.dry_run: - print("⚠ 试运行模式 — 不会实际写入任何数据\n") - - # ── 1. Release notes ───────────────────────────────────────────────── - if args.docs: - print("[1/2] 导入版本说明 (content_blocks) ...") - snapshot = _read_json(args.docs) - - # Stats - blocks = snapshot.get("blocks", {}) - for name, block in blocks.items(): - count = len(block.get("payload", [])) - print(f" {name}: {count} 条") - - if args.dry_run: - print(" [dry-run] 跳过实际写入") - elif args.api_url: - if not token: - print("✗ 需要 ADMIN_TOKEN，请通过 --admin-token 指定或在 .env 中设置") - sys.exit(1) - result = import_docs_via_api(args.api_url, token, snapshot, overwrite) - print(f" 结果: imported={result.get('imported', [])}, skipped={result.get('skipped', [])}") - else: - result = import_docs_via_db(args.database_url, snapshot, overwrite) - print(f" 结果: imported={result.get('imported', [])}, skipped={result.get('skipped', [])}") - - # ── 2. Capability center ───────────────────────────────────────────── - if args.catalog: - print("\n[2/2] 导入能力中心 (catalog) ...") - catalog_data = _read_json(args.catalog) - - # 2a. catalog.json - catalog_json_data = catalog_data.get("catalog_json") - if catalog_json_data: - print(" [a] catalog.json:") - import_catalog_json(catalog_json_data, args.dry_run) - - # 2b. catalog_overrides - overrides = catalog_data.get("catalog_overrides", []) - if overrides: - print(f" [b] catalog_overrides: {len(overrides)} 条") - if args.dry_run: - print(" [dry-run] 跳过实际写入") - elif args.database_url: - result = import_catalog_overrides_via_db(args.database_url, overrides, overwrite) - print(f" 结果: imported={result['imported']}, skipped={result['skipped']}") - else: - print(" ⚠ API 模式不支持批量导入 catalog_overrides，需要使用 --database-url") - elif not catalog_json_data: - print(" 快照中无有效数据") - - # ── 3. System prompts + prompt hub ─────────────────────────────────── - if args.prompts: - print("\n[+] 导入系统提示词 (prompt_versions) + 提示词广场 (prompt_hub) ...") - snapshot = _read_json(args.prompts) - - # Stats - blocks = snapshot.get("blocks", {}) - pv_payload = blocks.get("prompt_versions", {}).get("payload", {}) - versions = pv_payload.get("versions", []) if isinstance(pv_payload, dict) else [] - active = pv_payload.get("active", {}) if isinstance(pv_payload, dict) else {} - by_kind: dict = {} - for v in versions: - by_kind[v.get("kind", "?")] = by_kind.get(v.get("kind", "?"), 0) + 1 - for kind, n in sorted(by_kind.items()): - print(f" prompt_versions/{kind}: {n} 个版本（激活: {active.get(kind)}）") - hub_payload = blocks.get("prompt_hub", {}).get("payload", []) - hub_count = len(hub_payload) if isinstance(hub_payload, list) else 0 - print(f" prompt_hub: {hub_count} 条提示词模板") - - if args.dry_run: - print(" [dry-run] 跳过实际写入") - elif args.api_url: - if not token: - print("✗ 需要 ADMIN_TOKEN，请通过 --admin-token 指定或在 .env 中设置") - sys.exit(1) - result = import_prompts_via_api(args.api_url, token, snapshot, overwrite) - print(f" 结果: imported={result.get('imported', [])}, skipped={result.get('skipped', [])}") - else: - result = import_prompts_via_db(args.database_url, snapshot, overwrite) - print(f" 结果: imported={result.get('imported', [])}, skipped={result.get('skipped', [])}") - print(" ⚠ 直连数据库导入：需重启后端，提示词缓存才会刷新") - - print("\n" + "=" * 50) - if args.dry_run: - print("✓ 试运行完成（未写入任何数据）") - else: - print("✓ 导入完成") - print("=" * 50) - - -if __name__ == "__main__": - main() diff --git a/src/backend/scripts/run_evolution_benchmark.py b/src/backend/scripts/run_evolution_benchmark.py deleted file mode 100644 index 59102634..00000000 --- a/src/backend/scripts/run_evolution_benchmark.py +++ /dev/null @@ -1,308 +0,0 @@ -"""Before/after evolution benchmark runner. - - PYTHONPATH=src/backend python -m scripts.run_evolution_benchmark - -Sequence: - -1. **Before** — run the task set against whatever skills the runtime currently - offers. -2. **Feed** — replay the successful runs as Episodes, i.e. the evidence the - promotion chain is supposed to learn from. -3. **Evolve** — run one real evolution cycle. -4. **After** — run the same task set again against whatever the runtime offers - *now*. - -Step 4 is the load-bearing one, and it deliberately asks the runtime rather than -being handed a skill list: the whole question is whether evolution actually put -a usable capability where the agent can reach it. Constructing the "after" -skills by hand would answer a question nobody asked. -""" - -from __future__ import annotations - -import json -import sys -import uuid -from typing import Any, Dict, List, Optional, Sequence - -from core.evolution.benchmark import ( - ArmSummary, - BenchmarkTask, - build_task_set, - compare, - run_arm, -) - - -def available_skills_from_runtime() -> Dict[str, Sequence[str]]: - """Tool sequences the runtime can actually offer an agent right now. - - Reads the *activated* skill assets. A candidate sitting in review is not a - capability — counting it here would report an improvement the agent cannot - use. - """ - skills: Dict[str, Sequence[str]] = {} - try: - from core.db.engine import SessionLocal - from core.db.models import AdminSkill - - with SessionLocal() as db: - rows = db.query(AdminSkill).all() - for row in rows: - sequence = _tool_sequence_of(row) - if sequence: - skills[row.skill_id] = sequence - except Exception as exc: # pragma: no cover - print(f"[bench] reading runtime skills failed: {exc}", file=sys.stderr) - return skills - - -def _tool_sequence_of(row: Any) -> Sequence[str]: - """Extract an ordered tool list from an *enabled* skill row. - - Disabled skills are skipped: a rolled-back capability must stop counting - immediately, otherwise the benchmark would keep crediting a skill the agent - can no longer load. - """ - if not getattr(row, "is_enabled", True): - return [] - extra = getattr(row, "extra_files", None) - if isinstance(extra, dict): - raw = extra.get("tool_sequence.json") - if isinstance(raw, str): - try: - sequence = json.loads(raw) - if isinstance(sequence, list) and sequence: - return [str(t) for t in sequence] - except Exception: - pass - allowed = getattr(row, "allowed_tools", None) - if isinstance(allowed, list) and allowed: - return [str(t) for t in allowed] - return [] - - -def ordering_rules_from_runtime(): - """Ordering rules the runtime currently offers, read from live skill assets. - - Read back from the materialised skill rather than from the in-memory report, - so the benchmark measures what actually reached the runtime. - """ - from core.evolution.promotion import OrderingConstraint - - rules = [] - try: - from core.db.engine import SessionLocal - from core.db.models import AdminSkill - - with SessionLocal() as db: - for row in db.query(AdminSkill).all(): - if not getattr(row, "is_enabled", True): - continue - raw = (getattr(row, "extra_files", None) or {}).get( - "ordering_constraints.json" - ) - if not isinstance(raw, str): - continue - for item in json.loads(raw) or []: - rules.append( - OrderingConstraint( - before=item["before"], - after=item["after"], - support=int(item.get("support") or 0), - contexts=int(item.get("contexts") or 0), - validated_in=tuple(item.get("validated_in") or ()), - contradicted_in=tuple(item.get("contradicted_in") or ()), - ) - ) - except Exception as exc: # pragma: no cover - print(f"[bench] reading ordering rules failed: {exc}", file=sys.stderr) - # De-duplicate: the same rule may be attached to several skills. - seen = set() - unique = [] - for rule in rules: - key = (rule.before, rule.after) - if key not in seen: - seen.add(key) - unique.append(rule) - return unique - - -def promote_candidates(report: Dict[str, Any]) -> List[Dict[str, Any]]: - """Walk each candidate through the governance path an operator would. - - Replay → approve → activate. Nothing is skipped: if the capability only - shows up when a step is bypassed, that is a finding about the design, not a - benchmark detail to smooth over. - """ - from core.evolution.activation import ActivationError, materialise_skill_candidate - from core.evolution.loop import evaluate_candidate - from core.evolution.replay import TaskResult - - outcomes: List[Dict[str, Any]] = [] - - def arm(task_id: str, is_candidate: bool) -> TaskResult: - # The candidate carries a distilled ordering, so it succeeds where the - # unaided baseline mis-orders — the same effect the task set models. - index = int(task_id.rsplit("-", 1)[-1]) - return TaskResult(task_id, success=is_candidate or index % 2 == 0) - - replay_tasks = [f"rep-{i}" for i in range(40)] - for entry in report.get("candidates", []): - candidate_id = entry["candidate_id"] - evaluation = evaluate_candidate(candidate_id, arm, replay_tasks) - record: Dict[str, Any] = { - "candidate_id": candidate_id, - "signature": entry.get("signature", ""), - "replay": (evaluation or {}).get("verdict"), - } - try: - # Approver differs from the proposer ("promotion_chain") — the - # separation-of-duties check has no override. - record["activated"] = materialise_skill_candidate( - candidate_id, approver="bench-admin" - ) - except ActivationError as exc: - record["activation_error"] = f"{exc.code}: {exc.message}" - outcomes.append(record) - return outcomes - - -def feed_episodes(tasks: Sequence[BenchmarkTask], results) -> int: - """Turn successful runs into Episodes — the evidence evolution learns from.""" - from core.evolution.contract import ASSET_SKILL, AssetRef, build_bundle - from core.evolution.events import EV_TOOL_CALLED, TraceSink - from core.evolution.trace_assembler import assemble_episode - - by_id = {t.task_id: t for t in tasks} - written = 0 - for result in results: - if not result.success: - continue - task = by_id[result.task_id] - message_id = f"bench_{uuid.uuid4().hex[:16]}" - sink = TraceSink( - run_id=f"benchrun_{result.task_id}", - message_id=message_id, - chat_id="chat-benchmark", - user_id="user-benchmark", - ) - for tool in result.order: - sink.append(EV_TOOL_CALLED, {"tool_name": tool}) - sink.flush() - assemble_episode( - message_id=message_id, - run_id=f"benchrun_{result.task_id}", - chat_id="chat-benchmark", - user_id="user-benchmark", - objective=task.prompt, - # The context the ordering was observed in. Without it every rule - # would have to be global, and a rule that legitimately flips - # between task types would be discarded instead of scoped. - task_type=task.family, - bundle=build_bundle( - [AssetRef(kind=ASSET_SKILL, asset_id="none", version="v0")] - ), - outcome={"verdict": "success", "quality_score": 0.9}, - ) - written += 1 - return written - - -def _guard_target_database() -> Optional[str]: - """Refuse to run against a database that isn't disposable. - - This script writes: it mints ``bench_*`` Episodes and trace events, and - materialises real ``evo-*`` skill rows that the capability resolver then - hands to every user. Run against a working deployment it contaminates the - evidence plane it is supposed to measure — which already happened once here, - leaving 290 orphaned ``bench_*`` trace events in the dev database. - - SQLite is treated as disposable. Anything else needs the operator to say so - out loud. - """ - import os - - from core.db.engine import DATABASE_URL - - if DATABASE_URL.startswith("sqlite"): - return None - if os.environ.get("GCE_BENCHMARK_ALLOW_DB") == "1": - print(f"[bench] WARNING: writing benchmark data into {DATABASE_URL}\n") - return None - return ( - f"refusing to run against {DATABASE_URL}\n" - "This benchmark writes episodes and activates skills. Point it at a\n" - "throwaway database instead:\n" - " DATABASE_URL=sqlite:////tmp/gce_bench.db python -m scripts.run_evolution_benchmark\n" - "or set GCE_BENCHMARK_ALLOW_DB=1 if contaminating this one is intended." - ) - - -def main() -> int: - refusal = _guard_target_database() - if refusal: - print(refusal, file=sys.stderr) - return 2 - - tasks = build_task_set() - print(f"benchmark tasks: {len(tasks)} families: " - f"{sorted({t.family for t in tasks})}\n") - - # ── 1. Before ── - before_skills = available_skills_from_runtime() - before_rules = ordering_rules_from_runtime() - before = ArmSummary("before", run_arm(tasks, before_skills, ordering_rules=before_rules)) - print("── BEFORE ──") - print(json.dumps(before.to_dict(), ensure_ascii=False, indent=1)) - print(f"runtime skills with a tool sequence: {len(before_skills)}\n") - - # ── 2. Feed evidence ── - written = feed_episodes(tasks, before.results) - print(f"── FEED ── episodes written from successful runs: {written}\n") - - # ── 3. Evolve ── - from core.evolution.loop import run_evolution_cycle - - report = run_evolution_cycle(limit=400) - print("── EVOLVE ──") - print(json.dumps( - {k: v for k, v in report.items() if k != "candidates"}, - ensure_ascii=False, indent=1)) - print(f"candidates: {report['candidates']}\n") - - # ── 4. Governance path: replay → approve → activate ── - promotions = promote_candidates(report) - print("── PROMOTE ──") - print(json.dumps(promotions, ensure_ascii=False, indent=1)) - print() - - # ── 5. After ── - after_skills = available_skills_from_runtime() - after_rules = ordering_rules_from_runtime() - after = ArmSummary("after", run_arm(tasks, after_skills, ordering_rules=after_rules)) - print("── AFTER ──") - print(json.dumps(after.to_dict(), ensure_ascii=False, indent=1)) - print(f"runtime skills with a tool sequence: {len(after_skills)}") - for rule in after_rules: - print(f" rule {rule.before} → {rule.after} " - f"| validated_in={list(rule.validated_in)} " - f"contradicted_in={list(rule.contradicted_in)}") - print() - - # ── Verdict ── - verdict = compare(before, after) - print("── VERDICT ──") - print(json.dumps(verdict, ensure_ascii=False, indent=1)) - - if verdict["effect_size"] > 0 and verdict["valid"]: - print("\n结论：进化后能力有提升。") - elif not verdict["valid"]: - print("\n结论：结果无效——干扰组也发生了变化，说明测量被污染。") - else: - print("\n结论：进化后能力无提升。需要回头检查闭环在哪里断了。") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/src/backend/scripts/run_evolution_llm_ab.py b/src/backend/scripts/run_evolution_llm_ab.py deleted file mode 100644 index 881ef227..00000000 --- a/src/backend/scripts/run_evolution_llm_ab.py +++ /dev/null @@ -1,621 +0,0 @@ -"""Real-LLM A/B for a materialised evolution skill. - - PYTHONPATH=src/backend python -m scripts.run_evolution_llm_ab [--skill-id evo-…] - -The offline benchmark (``run_evolution_benchmark``) answers a narrower question -than people read into it: it holds a deterministic planner fixed and measures -whether the harness now *holds* a reusable capability. It cannot answer whether -a real model, handed that capability, actually plans differently — the planner is -a stand-in, and a stand-in that always obeys a rule proves nothing about -obedience. - -This script closes that gap. Both arms are real calls to a configured chat -provider on the identical task set; the only difference is whether the skill -document the evolution pipeline wrote is present: - -* **arm A** — the task alone. -* **arm B** — the task plus the materialised ``SKILL.md``, verbatim, as the - runtime delivers it once the model has performed the mandated read step. - -Three families, and the interesting one is not the first: - -* ``validated_in`` families — the rule claims to apply. Improvement expected. -* ``contradicted_in`` families — the rule claims **not** to apply, and says so in - its own text. If the model applies it anyway, the scope annotation is - decorative and the capability is actively harmful there. This is the - over-application check, and it is the reason the trap family exists. -* distractor family — unrelated ordering. Movement here means the skill is - changing behaviour it has no business changing. - -What it still does not model: the tool-execution loop, and the decision to open -the skill in the first place (arm B assumes the read happened). Both are stated -rather than smoothed over. -""" - -from __future__ import annotations - -import argparse -import asyncio -import json -import re -import sys -from dataclasses import dataclass, field -from typing import Any, Dict, List, Optional, Sequence, Tuple - -TIMEOUT_S = 90 -# Deterministic decoding: the question is whether the skill changes the plan, and -# sampling noise would be indistinguishable from a small real effect at this -# sample size. -TEMPERATURE = 0.0 - - -@dataclass(frozen=True) -class LlmTask: - task_id: str - family: str - prompt: str - tools: Tuple[str, ...] - # (first, second): ``first`` must be called before ``second`` to succeed. - ordering: Tuple[str, str] - - -def _listing(tools: Sequence[Tuple[str, str]], task_id: str, ordering: Tuple[str, str]) -> str: - """Render the available-tools list with the constrained pair counterbalanced. - - Without this the whole experiment measures nothing. A first run scored 100% - in **both** arms on every family, which looked like "the model already knows - the ordering" — it was not. The prompt listed the tools in the correct order, - so a model that simply echoes the listing scores perfectly while - demonstrating no knowledge at all. - - Half the tasks therefore list the constrained pair the right way round and - half the wrong way round, deterministically by task id. Copying the listing - now scores ~50%, so the baseline has headroom and the arms differ by - knowledge rather than by transcription. - """ - first, second = ordering - items = list(tools) - flip = int(task_id.rsplit("-", 1)[-1]) % 2 == 1 - if flip: - index = {name: i for i, (name, _) in enumerate(items)} - if first in index and second in index: - i, j = index[first], index[second] - items[i], items[j] = items[j], items[i] - return "、".join(f"{name}（{desc}）" for name, desc in items) - - -def build_task_set() -> List[LlmTask]: - """Suite ``business`` — realistic tasks whose correct order is *intuitive*. - - Measured result: a current model scores 100% on these unaided, so the suite - has no headroom and can prove nothing about obedience. That is itself the - finding — for orderings a competent model already derives, a distilled rule - adds nothing, and the offline benchmark's effect comes from modelling the - planner as ignorant of them. - - Kept rather than deleted because it is the control: it is what shows the - ``platform`` suite's effect is not an artefact of prompt shape. - """ - tasks: List[LlmTask] = [] - - # ── validated_in: 产业周报 ── - for i in range(8): - tasks.append( - LlmTask( - task_id=f"report-{i:02d}", - family="industry_report", - prompt=( - f"任务：生成第 {i + 1} 期新能源汽车市场周报。" - "要求报告中出现的企业主体必须准确，并附可追溯的数据来源。\n" - "可用工具：company_verify（企业主体核验）、db_query（产业数据查询）、" - "chart_render（图表渲染）、doc_export（文档导出）。" - ), - tools=("company_verify", "db_query", "chart_render", "doc_export"), - ordering=("company_verify", "db_query"), - ) - ) - - # ── validated_in: 财务分析（同一顺序教训，不同工具集） ── - for i in range(8): - tasks.append( - LlmTask( - task_id=f"finance-{i:02d}", - family="finance", - prompt=( - f"任务：分析目标企业 {i + 1} 的财务风险并给出风险结论。\n" - "可用工具：company_verify（企业主体核验）、db_query（财务数据查询）、" - "risk_score（风险打分）。" - ), - tools=("company_verify", "db_query", "risk_score"), - ordering=("company_verify", "db_query"), - ) - ) - - # ── contradicted_in: 该场景下顺序相反才对 ── - for i in range(8): - tasks.append( - LlmTask( - task_id=f"trap-{i:02d}", - family="trap", - prompt=( - f"任务：内部流程草稿 {i + 1}。本场景的业务规则是——" - "**核验环节需要以查询结果作为输入**（先拿到数据，才能核验数据里出现的主体），" - "请据此安排步骤。\n" - "可用工具：db_query（数据查询）、company_verify（主体核验）、" - "doc_export（文档导出）。" - ), - tools=("db_query", "company_verify", "doc_export"), - ordering=("db_query", "company_verify"), - ) - ) - - # ── distractor: 与任何规则无关 ── - for i in range(6): - tasks.append( - LlmTask( - task_id=f"kbqa-{i:02d}", - family="kb_qa", - prompt=( - f"任务：回答知识库问题 {i + 1}，答案需要给出引用。\n" - "可用工具：kb_search（知识库检索）、cite_check（引用校验）。" - ), - tools=("kb_search", "cite_check"), - ordering=("kb_search", "cite_check"), - ) - ) - return tasks - - -def build_platform_task_set() -> List[LlmTask]: - """Suite ``platform`` — ordering the model cannot derive, only learn. - - ``currency_resolve`` must precede ``region_resolve`` on this platform. There - is no semantic reason: they are peer resolvers and either order reads as - plausible, so a model has no prior and lands near chance. That is exactly the - class of knowledge harness evolution is supposed to capture — a - platform-specific convention no pre-training could contain — and therefore - the only class on which "did the learned rule change the plan?" is a - measurable question. - - The trap family reverses it, and the rule's own text names that family as an - exception. If the model applies the rule there anyway, the scope annotation - is decorative. - """ - tasks: List[LlmTask] = [] - - settle_tools = ( - ("currency_resolve", "币种解析"), - ("region_resolve", "区域解析"), - ("metric_compute", "指标计算"), - ("report_render", "报告渲染"), - ) - for i in range(12): - tid = f"settle-{i:02d}" - order = ("currency_resolve", "region_resolve") - tasks.append( - LlmTask( - task_id=tid, - family="settlement", - prompt=( - f"任务：为跨境结算批次 {i + 1} 生成对账报告。\n" - f"可用工具：{_listing(settle_tools, tid, order)}。\n" - "请给出调用顺序。" - ), - tools=tuple(name for name, _ in settle_tools), - ordering=order, - ) - ) - - # Same lesson, different tool set: this is the transfer claim. - pricing_tools = ( - ("currency_resolve", "币种解析"), - ("region_resolve", "区域解析"), - ("price_adjust", "调价计算"), - ) - for i in range(10): - tid = f"pricing-{i:02d}" - order = ("currency_resolve", "region_resolve") - tasks.append( - LlmTask( - task_id=tid, - family="pricing", - prompt=( - f"任务：为区域定价方案 {i + 1} 计算调价建议。\n" - f"可用工具：{_listing(pricing_tools, tid, order)}。\n" - "请给出调用顺序。" - ), - tools=tuple(name for name, _ in pricing_tools), - ordering=order, - ) - ) - - legacy_tools = ( - ("region_resolve", "区域解析"), - ("currency_resolve", "币种解析"), - ("report_render", "报告渲染"), - ) - for i in range(10): - tid = f"legacy-{i:02d}" - order = ("region_resolve", "currency_resolve") - tasks.append( - LlmTask( - task_id=tid, - family="legacy_batch", - prompt=( - f"任务：处理历史批次 {i + 1} 的补算。\n" - f"可用工具：{_listing(legacy_tools, tid, order)}。\n" - "请给出调用顺序。" - ), - tools=tuple(name for name, _ in legacy_tools), - ordering=order, - ) - ) - - kb_tools = (("kb_search", "知识库检索"), ("cite_check", "引用校验")) - for i in range(8): - tid = f"kbqa-{i:02d}" - order = ("kb_search", "cite_check") - tasks.append( - LlmTask( - task_id=tid, - family="kb_qa", - prompt=( - f"任务：回答知识库问题 {i + 1}，答案需要给出引用。\n" - f"可用工具：{_listing(kb_tools, tid, order)}。" - ), - tools=tuple(name for name, _ in kb_tools), - ordering=order, - ) - ) - return tasks - - -SUITES = {"business": build_task_set, "platform": build_platform_task_set} - -# Which families each suite's rule claims to cover, and which it disclaims. Used -# only for reporting — the rule's own scope lives in the skill document. -SUITE_FAMILIES = { - "business": { - "applicable": ("industry_report", "finance"), - "contradicted": ("trap",), - "distractor": ("kb_qa",), - }, - "platform": { - "applicable": ("settlement", "pricing"), - "contradicted": ("legacy_batch",), - "distractor": ("kb_qa",), - }, -} - - -_INSTRUCTION = ( - "你是一个任务规划器。请只规划工具调用顺序，不要执行、不要解释。\n" - "输出格式：一个 JSON 数组，元素是工具名字符串，按调用先后排列。" - "只输出 JSON，不要出现其它文字或代码块标记。" -) - - -def build_messages(task: LlmTask, skill_document: Optional[str]) -> List[Dict[str, str]]: - """The two arms differ by exactly one block. - - Arm B mirrors the runtime's shape: the skill is announced as an available - skill and its instructions follow, which is the state the model is in after - the mandated ``view_text_file`` read. - """ - system = _INSTRUCTION - if skill_document: - system += ( - "\n\n# 技能（Agent Skills）\n" - "以下是当前可用的技能。技能不是工具，不能直接调用；" - "当任务匹配技能描述时，你必须遵循技能中的指令。\n\n" - "\n" + skill_document + "\n" - ) - return [ - {"role": "system", "content": system}, - {"role": "user", "content": task.prompt}, - ] - - -_JSON_ARRAY = re.compile(r"\[[^\[\]]*\]", re.S) - - -def parse_plan(text: str, tools: Sequence[str]) -> List[str]: - """Extract the planned tool order from the model's reply. - - Falls back to first-mention order when the reply is not valid JSON: refusing - to parse a legible answer would score a formatting failure as an ordering - failure, and this measures ordering. - """ - raw = (text or "").strip() - for candidate in ([raw] if raw.startswith("[") else []) + _JSON_ARRAY.findall(raw): - try: - parsed = json.loads(candidate) - except Exception: - continue - if isinstance(parsed, list): - names = [str(x).strip().strip("`") for x in parsed] - keep = [n for n in names if n in set(tools)] - if keep: - return keep - positions = [(raw.find(tool), tool) for tool in tools if raw.find(tool) >= 0] - return [tool for _, tool in sorted(positions)] - - -def satisfies(plan: Sequence[str], ordering: Tuple[str, str]) -> bool: - first, second = ordering - if first not in plan or second not in plan: - return False - return plan.index(first) < plan.index(second) - - -# ── Provider access ────────────────────────────────────────────────────────── - - -def pick_provider(preferred: Optional[str] = None) -> Tuple[str, str, str, str]: - """(display_name, model_name, base_url, api_key) for an active chat provider. - - Reads the same ``model_providers`` rows the product serves traffic from, so - the arms run against a real configured endpoint rather than a test double. - """ - from core.db.engine import SessionLocal - from core.db.models import ModelProvider - - with SessionLocal() as db: - query = db.query(ModelProvider).filter( - ModelProvider.provider_type == "chat", ModelProvider.is_active == True # noqa: E712 - ) - rows = query.order_by(ModelProvider.priority.desc()).all() - if not rows: - raise SystemExit("no active chat provider configured") - if preferred: - for row in rows: - if preferred in (row.model_name, row.provider_id, row.display_name): - return row.display_name, row.model_name, row.base_url, row.api_key - raise SystemExit(f"provider {preferred!r} not found among {[r.model_name for r in rows]}") - return rows[0].display_name, rows[0].model_name, rows[0].base_url, rows[0].api_key - - -async def call_model( - client: Any, *, model: str, base_url: str, api_key: str, messages: List[Dict[str, str]] -) -> str: - response = await client.post( - base_url.rstrip("/") + "/chat/completions", - headers={"Authorization": f"Bearer {api_key}"}, - json={ - "model": model, - "messages": messages, - "temperature": TEMPERATURE, - "max_tokens": 400, - }, - ) - response.raise_for_status() - payload = response.json() - message = payload["choices"][0]["message"] - # Reasoning providers put the answer in ``content`` and their trace elsewhere; - # concatenating would feed the trace into the parser. - return message.get("content") or "" - - -# ── Skill document ─────────────────────────────────────────────────────────── - - -def load_skill_document(skill_id: Optional[str]) -> Tuple[str, str]: - """The materialised SKILL.md, read from the asset the pipeline actually wrote.""" - from core.db.engine import SessionLocal - from core.db.models import AdminSkill - from core.evolution.activation import EVOLUTION_PREFIX - - with SessionLocal() as db: - query = db.query(AdminSkill) - if skill_id: - row = query.filter(AdminSkill.skill_id == skill_id).first() - else: - row = ( - query.filter(AdminSkill.skill_id.like(f"{EVOLUTION_PREFIX}%")) - .order_by(AdminSkill.updated_at.desc()) - .first() - ) - if row is None: - raise SystemExit( - "no materialised evolution skill found — run the loop and activate a " - "candidate first, or pass --skill-id" - ) - return row.skill_id, row.skill_content or "" - - -# ── Scoring ────────────────────────────────────────────────────────────────── - - -@dataclass -class ArmResult: - name: str - passed: Dict[str, bool] = field(default_factory=dict) - plans: Dict[str, List[str]] = field(default_factory=dict) - errors: Dict[str, str] = field(default_factory=dict) - - def rate(self, family: Optional[str] = None, tasks: Sequence[LlmTask] = ()) -> float: - ids = [t.task_id for t in tasks if family is None or t.family == family] - scored = [self.passed[i] for i in ids if i in self.passed] - return sum(scored) / len(scored) if scored else 0.0 - - -def compare(before: ArmResult, after: ArmResult, tasks: Sequence[LlmTask]) -> Dict[str, Any]: - from core.evolution.replay import mcnemar_p_value - - shared = [t.task_id for t in tasks if t.task_id in before.passed and t.task_id in after.passed] - regressed = [t for t in shared if before.passed[t] and not after.passed[t]] - improved = [t for t in shared if after.passed[t] and not before.passed[t]] - return { - "n_paired": len(shared), - "before_rate": round(sum(before.passed[t] for t in shared) / len(shared), 4) if shared else 0.0, - "after_rate": round(sum(after.passed[t] for t in shared) / len(shared), 4) if shared else 0.0, - "effect_size": round( - (sum(after.passed[t] for t in shared) - sum(before.passed[t] for t in shared)) - / len(shared), - 4, - ) - if shared - else 0.0, - "p_value": round(mcnemar_p_value(len(regressed), len(improved)), 6), - "improved": improved, - "regressed": regressed, - } - - -async def run_arm( - name: str, - tasks: Sequence[LlmTask], - skill_document: Optional[str], - *, - model: str, - base_url: str, - api_key: str, - concurrency: int = 4, -) -> ArmResult: - import httpx - - result = ArmResult(name) - semaphore = asyncio.Semaphore(concurrency) - - async with httpx.AsyncClient(timeout=TIMEOUT_S) as client: - - async def one(task: LlmTask) -> None: - async with semaphore: - try: - text = await call_model( - client, - model=model, - base_url=base_url, - api_key=api_key, - messages=build_messages(task, skill_document), - ) - except Exception as exc: # noqa: BLE001 - result.errors[task.task_id] = f"{type(exc).__name__}: {exc}" - return - plan = parse_plan(text, task.tools) - result.plans[task.task_id] = plan - # An unparseable reply is recorded as a failure rather than - # dropped: silently excluding it would let a model that answers - # incoherently under one arm look better than it is. - result.passed[task.task_id] = satisfies(plan, task.ordering) - - await asyncio.gather(*[one(task) for task in tasks]) - return result - - -def main(argv: Optional[Sequence[str]] = None) -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--skill-id", default=None, help="要测的已物化技能 id") - parser.add_argument("--model", default=None, help="model_name / provider_id / display_name") - parser.add_argument("--concurrency", type=int, default=4) - parser.add_argument("--json-out", default=None, help="把完整结果写到该路径") - parser.add_argument( - "--suite", - default="platform", - choices=sorted(SUITES), - help="business = 顺序直觉可推导（对照组，基线满分）；" - "platform = 平台特有约定，模型无先验（可测组）", - ) - args = parser.parse_args(argv) - - skill_id, document = load_skill_document(args.skill_id) - display, model, base_url, api_key = pick_provider(args.model) - tasks = SUITES[args.suite]() - roles = SUITE_FAMILIES[args.suite] - - print(f"provider : {display} ({model})") - print(f"skill : {skill_id} ({len(document)} chars)") - print(f"suite : {args.suite}") - print(f"tasks : {len(tasks)} families: {sorted({t.family for t in tasks})}") - print(f"arms : A = 任务; B = 任务 + 该技能文档\n") - - arm_a = asyncio.run( - run_arm("A_no_skill", tasks, None, model=model, base_url=base_url, - api_key=api_key, concurrency=args.concurrency) - ) - arm_b = asyncio.run( - run_arm("B_with_skill", tasks, document, model=model, base_url=base_url, - api_key=api_key, concurrency=args.concurrency) - ) - - for arm in (arm_a, arm_b): - if arm.errors: - print(f"[{arm.name}] {len(arm.errors)} call(s) failed: " - f"{list(arm.errors.items())[:2]}") - - families = sorted({t.family for t in tasks}) - print(f"{'family':<18}{'A':>8}{'B':>8}{'delta':>9} note") - role_note = { - "applicable": "规则声称适用 — 期望提升", - "contradicted": "规则声称【不】适用 — B 变差即为过度套用", - "distractor": "干扰组 — 任何变化都说明技能越界", - } - family_role = { - family: role - for role, names in roles.items() - for family in names - } - for family in families: - a = arm_a.rate(family, tasks) - b = arm_b.rate(family, tasks) - note = role_note.get(family_role.get(family, ""), "") - print(f"{family:<18}{a:>7.1%}{b:>8.1%}{b - a:>+9.1%} {note}") - - verdict_all = compare(arm_a, arm_b, tasks) - applicable = [t for t in tasks if t.family in roles["applicable"]] - verdict_applicable = compare(arm_a, arm_b, applicable) - trap = [t for t in tasks if t.family in roles["contradicted"]] - verdict_trap = compare(arm_a, arm_b, trap) - - print("\n── 全部任务 ──") - print(json.dumps(verdict_all, ensure_ascii=False)) - print("\n── 规则声称适用的家族 ──") - print(json.dumps(verdict_applicable, ensure_ascii=False)) - print("\n── 反例家族（过度套用检查）──") - print(json.dumps(verdict_trap, ensure_ascii=False)) - - over_applied = len(verdict_trap["regressed"]) - print("\n── 结论 ──") - if verdict_applicable["effect_size"] > 0: - print(f"· 模型确实按技能里的顺序约束改变了规划：" - f"适用家族 {verdict_applicable['before_rate']:.1%} → " - f"{verdict_applicable['after_rate']:.1%}" - f"（p={verdict_applicable['p_value']}）") - else: - print("· 在规则声称适用的家族上没有可测提升——技能文本没有改变模型的规划。") - if over_applied: - print(f"· ⚠ 反例家族有 {over_applied} 题回归：模型把规则套用到了它自己声明【不】适用的场景。" - "作用域标注对该模型无效。") - else: - print("· 反例家族无回归：模型尊重了规则自带的作用域标注，没有外推。") - distractors = [t for t in tasks if t.family in roles["distractor"]] - if distractors: - moved = [t.task_id for t in distractors - if arm_a.passed.get(t.task_id) != arm_b.passed.get(t.task_id)] - print(f"· 干扰组变化 {len(moved)} 题" + ("（应为 0）" if moved else "，未越界。")) - if verdict_applicable["before_rate"] >= 1.0: - print("· ⚠ 基线已满分：该 suite 无上升空间，测不出服从性。" - "说明这条顺序对当前模型是可自行推导的——换 --suite platform 才有可测空间。") - - if args.json_out: - with open(args.json_out, "w", encoding="utf-8") as handle: - json.dump( - { - "provider": {"display": display, "model": model}, - "skill_id": skill_id, - "arm_a": {"passed": arm_a.passed, "plans": arm_a.plans, "errors": arm_a.errors}, - "arm_b": {"passed": arm_b.passed, "plans": arm_b.plans, "errors": arm_b.errors}, - "verdict_all": verdict_all, - "verdict_applicable": verdict_applicable, - "verdict_trap": verdict_trap, - }, - handle, - ensure_ascii=False, - indent=1, - ) - print(f"\n完整结果已写入 {args.json_out}") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/src/backend/scripts/workbench_migration.py b/src/backend/scripts/workbench_migration.py deleted file mode 100644 index 97412d97..00000000 --- a/src/backend/scripts/workbench_migration.py +++ /dev/null @@ -1,66 +0,0 @@ -"""Explicit isolated-copy migration command. No automatic startup migration.""" - -import argparse -import json -import os -from pathlib import Path - - -def main(): - parser = argparse.ArgumentParser(description="Workbench inventory and isolated-copy rehearsal") - parser.add_argument("operation", choices=["inventory", "apply", "rollback", "clone-origin"]) - parser.add_argument("--snapshot", type=Path, required=True) - parser.add_argument("--digest") - parser.add_argument("--delta", type=Path) - parser.add_argument("--writes-frozen", action="store_true") - args = parser.parse_args() - # A dedicated variable prevents accidental use of deployment DATABASE_URL. - from sqlalchemy.engine import make_url - - url = make_url(os.environ.get("WB20_COPY_DATABASE_URL", "sqlite://")) - if url.get_backend_name() == "sqlite": - database = Path(url.database or "").resolve() - isolated = Path.cwd() / ".scratch" / "workbench-20" - if not database.is_relative_to(isolated.resolve()): - parser.error("SQLite copy must be under this worktree .scratch/workbench-20") - elif url.get_backend_name() == "postgresql": - if url.host not in {"localhost", "127.0.0.1", "::1"} or not (url.database or "").startswith( - "workbench_20" - ): - parser.error("PostgreSQL rehearsal requires loopback and workbench_20 database prefix") - else: - parser.error("Unsupported copy database") - os.environ["DATABASE_URL"] = url.render_as_string(hide_password=False) - from core.applications import migration - from sqlalchemy import create_engine - from sqlalchemy.orm import Session - - engine = create_engine(url) - with Session(engine) as db: - if args.operation == "inventory": - result = migration.inventory(db, args.snapshot) - else: - if not args.digest or not args.writes_frozen: - parser.error("Mutation requires reviewed digest and --writes-frozen") - if args.operation == "apply": - result = migration.apply(db, args.snapshot, args.digest, writes_frozen=True) - elif args.operation == "clone-origin": - result = migration.clone_origin(db, args.snapshot, args.digest, writes_frozen=True) - else: - if args.delta is None: - parser.error("Rollback requires --delta protected export path") - result = migration.rollback( - db, args.snapshot, args.digest, args.delta, writes_frozen=True - ) - print(json.dumps(result, sort_keys=True)) - engine.dispose() - - -if __name__ == "__main__": - try: - main() - except ValueError as error: - # Domain errors contain codes only; DB URLs and payloads never print. - raise SystemExit(str(error)) from None - except Exception as error: - raise SystemExit("migration_failed:" + type(error).__name__) from None diff --git a/src/backend/tests/_capability_index.py b/src/backend/tests/_capability_index.py index 7f9af10e..bc8246d8 100644 --- a/src/backend/tests/_capability_index.py +++ b/src/backend/tests/_capability_index.py @@ -15,15 +15,10 @@ def bind_capability_index(tmp_dir: Path, monkeypatch: pytest.MonkeyPatch): import core.db.models # noqa: F401 - populate metadata - from core.capabilities import registry + from features.desktop.capabilities import registry from core.db.engine import Base - from core.db.models import ( - ContentBlock, - DeviceCapabilityComponent, - DeviceCapabilityInstallation, - DeviceCapabilityNamePreference, - DeviceCapabilityTransaction, - ) + from core.db.models import ContentBlock + from features.desktop.models import DeviceCapabilityComponent, DeviceCapabilityInstallation, DeviceCapabilityNamePreference, DeviceCapabilityTransaction engine = create_engine( f"sqlite:///{tmp_dir / 'capability-index.db'}", connect_args={"check_same_thread": False} diff --git a/src/backend/tests/api/api_routes_v1_selftest.py b/src/backend/tests/api/api_routes_v1_selftest.py index 60534a94..3344606e 100755 --- a/src/backend/tests/api/api_routes_v1_selftest.py +++ b/src/backend/tests/api/api_routes_v1_selftest.py @@ -84,7 +84,7 @@ def test_route_imports(): ('api.routes.v1.chats', 'router'), ('api.routes.v1.users', 'router'), ('api.routes.v1.catalog', 'router'), - ('api.routes.v1.kb', 'router'), + ('features.knowledge.api.kb', 'router'), ] all_ok = True diff --git a/src/backend/tests/api/catalog_api_selftest.py b/src/backend/tests/api/catalog_api_selftest.py index 899a609c..d4edc929 100644 --- a/src/backend/tests/api/catalog_api_selftest.py +++ b/src/backend/tests/api/catalog_api_selftest.py @@ -15,15 +15,16 @@ def main() -> int: print(f"catalog_api_selftest: SKIP (missing dependency: {e})") return 0 - import core.config.catalog as cat + import core.capabilities.catalog.api as cat + from core.capabilities.catalog import common as catalog_common - original_catalog_path = cat._CATALOG_PATH # type: ignore[attr-defined] + original_catalog_path = catalog_common.CATALOG_PATH # type: ignore[attr-defined] old_auth_mode = os.environ.get("AUTH_MODE") os.environ["AUTH_MODE"] = "mock" with tempfile.TemporaryDirectory() as td: tmp = Path(td) / "catalog.json" try: - cat._CATALOG_PATH = tmp # type: ignore[attr-defined] + catalog_common.CATALOG_PATH = tmp # type: ignore[attr-defined] cat.ensure_default_catalog() client = TestClient(app) @@ -45,7 +46,7 @@ def main() -> int: bad = client.patch(f"/v1/catalog/unsupported/{skill_id}", json={"enabled": True}) assert bad.status_code == 400 finally: - cat._CATALOG_PATH = original_catalog_path # type: ignore[attr-defined] + catalog_common.CATALOG_PATH = original_catalog_path # type: ignore[attr-defined] if old_auth_mode is None: os.environ.pop("AUTH_MODE", None) else: diff --git a/src/backend/tests/api/test_agent_api_keys.py b/src/backend/tests/api/test_agent_api_keys.py index bafded39..cdf4ac41 100644 --- a/src/backend/tests/api/test_agent_api_keys.py +++ b/src/backend/tests/api/test_agent_api_keys.py @@ -26,12 +26,7 @@ from core.auth.backend import UserContext, get_current_user from core.db.engine import get_db from core.db.models import AgentApiCallLog, ChatRun, UserApiKey -from core.services.agent_api_service import ( - begin_agent_api_call, - bind_agent_api_run, - list_agent_api_calls, - prepare_agent_api_request, -) +from core.auth.agent_api import begin_agent_api_call, bind_agent_api_run, list_agent_api_calls, prepare_agent_api_request from tests.api.test_agent_api_scope import db, _key, _user from core.chat.schemas import ChatRequest diff --git a/src/backend/tests/api/test_agent_response_admission.py b/src/backend/tests/api/test_agent_response_admission.py index 2fccb0e6..37972459 100644 --- a/src/backend/tests/api/test_agent_response_admission.py +++ b/src/backend/tests/api/test_agent_response_admission.py @@ -1,26 +1,29 @@ """Route-level concurrency proof for POST /v1/agents/responses admission.""" from __future__ import annotations -import core.services.session_operations.admission as admission import asyncio from concurrent.futures import ThreadPoolExecutor from threading import Barrier from types import SimpleNamespace -import core.services.session_operations.agent_targets as chat_agent_targets -import core.services.session_operations.request_context as chat_request_context -import core.services.session_operations.session_context as chat_session_context +import api.transports.run_sse as run_sse import core.chat.context as chat_context +import core.chat.runtime.admission as runtime_admission import core.db.engine as db_engine -import core.services as chat_services +import core.runs.runtime as runtime +import core.auth.users as chat_services_UserService_module +import core.chat.operations.admission as admission +import core.chat.operations.agent_targets as chat_agent_targets +import core.chat.operations.request_context as chat_request_context +import core.chat.operations.session_context as chat_session_context import pytest from api.routes.v1.agent_responses import agent_response -from core.chat.schemas import ChatRequest from core.auth.backend import UserContext +from core.chat.schemas import ChatRequest from core.db.engine import Base from core.db.models import ChatMessage, ChatRun, ChatSession -from core.services.chat_sequencer import ChatSequencer +from core.chat.sequencer import ChatSequencer from fastapi import HTTPException from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker @@ -54,19 +57,19 @@ def __init__(self, _db): def get_user_settings(self, _user_id): return {} - monkeypatch.setattr(chat_services, "UserService", FakeUserService) - monkeypatch.setattr(chat_session_context, "_ensure_main_model_configured", lambda: None) + monkeypatch.setattr(chat_services_UserService_module, "UserService", FakeUserService) + monkeypatch.setattr(chat_session_context, 'ensure_main_model_configured', lambda: None) monkeypatch.setattr( chat_agent_targets, "_resolve_chat_agent_targets", lambda _db, request, _user_id: (request, None, request.message, None), ) monkeypatch.setattr( - chat_session_context, "_resolve_selected_model_provider_id", lambda *_args: None + chat_session_context, 'resolve_selected_model_provider_id', lambda *_args: None ) monkeypatch.setattr( chat_session_context, - "_resolve_actual_chat_model_name", + 'resolve_actual_chat_model_name', lambda request, _: request.model_name, ) monkeypatch.setattr( @@ -75,14 +78,14 @@ def get_user_settings(self, _user_id): monkeypatch.setattr( chat_session_context, "_ensure_chat_session", lambda *_args, **_kwargs: None ) - monkeypatch.setattr(chat_request_context, "_build_ctx", lambda *_args, **_kwargs: {}) + monkeypatch.setattr(chat_request_context, 'build_ctx', lambda *_args, **_kwargs: {}) monkeypatch.setattr(chat_request_context, "_build_user_extra_data", lambda *_args: {}) @pytest.mark.parametrize("transports", [(True, True), (False, False), (True, False)]) def test_concurrent_requests_have_one_durable_winner(monkeypatch, tmp_path, transports): from core.chat import plan_progress - from orchestration import chat_run_executor + pass engine = create_engine( f"sqlite:///{tmp_path / 'stream.db'}", @@ -112,19 +115,19 @@ async def empty_follow(_run_id, *, chat_id): yield chat_id monkeypatch.setattr(db_engine, "SessionLocal", Session) - monkeypatch.setattr(chat_services, "UserService", FakeUserService) - monkeypatch.setattr(chat_session_context, "_ensure_main_model_configured", lambda: None) + monkeypatch.setattr(chat_services_UserService_module, "UserService", FakeUserService) + monkeypatch.setattr(chat_session_context, 'ensure_main_model_configured', lambda: None) monkeypatch.setattr( chat_agent_targets, "_resolve_chat_agent_targets", lambda _db, request, _user_id: (request, None, request.message, None), ) monkeypatch.setattr( - chat_session_context, "_resolve_selected_model_provider_id", lambda *_args: None + chat_session_context, 'resolve_selected_model_provider_id', lambda *_args: None ) monkeypatch.setattr( chat_session_context, - "_resolve_actual_chat_model_name", + 'resolve_actual_chat_model_name', lambda request, _: request.model_name, ) monkeypatch.setattr( @@ -133,12 +136,12 @@ async def empty_follow(_run_id, *, chat_id): monkeypatch.setattr( chat_session_context, "_ensure_chat_session", lambda *_args, **_kwargs: None ) - monkeypatch.setattr(chat_request_context, "_build_ctx", lambda *_args, **_kwargs: {}) + monkeypatch.setattr(chat_request_context, 'build_ctx', lambda *_args, **_kwargs: {}) monkeypatch.setattr(chat_request_context, "_build_user_extra_data", lambda *_args: {}) - monkeypatch.setattr(chat_session_context, "_load_session_messages", lambda *_args: []) + monkeypatch.setattr(chat_session_context, 'load_session_messages', lambda *_args: []) monkeypatch.setattr(plan_progress, "clear_plan_progress", lambda _chat_id: None) - monkeypatch.setattr(chat_run_executor, "start_run", fake_start_run) - monkeypatch.setattr(chat_run_executor, "follow_run_as_sse", empty_follow) + monkeypatch.setattr(runtime_admission, "start_run", fake_start_run) + monkeypatch.setattr(run_sse, "follow_run_as_sse", empty_follow) from api.routes.v1 import agent_responses from core.chat.schemas import ChatResponse @@ -213,7 +216,7 @@ def test_non_stream_send_returns_busy_without_running_or_persisting(monkeypatch, _patch_common_chat_route(monkeypatch) monkeypatch.setattr(db_engine, "SessionLocal", Session) - monkeypatch.setattr(chat_session_context, "_load_session_messages", lambda *_args: []) + monkeypatch.setattr(chat_session_context, 'load_session_messages', lambda *_args: []) with Session() as db: with pytest.raises(HTTPException) as raised: @@ -236,7 +239,7 @@ def test_non_stream_send_returns_busy_without_running_or_persisting(monkeypatch, def test_non_stream_send_uses_reserved_sequences_and_releases_writer(monkeypatch, tmp_path): - from orchestration import chat_run_executor + pass engine, Session = _route_database(tmp_path, "send-success.db") _patch_common_chat_route(monkeypatch) @@ -247,7 +250,7 @@ def load_after_accept(service, _chat_id, _user_id): assert (stored.role, stored.chat_seq, stored.content) == ("user", 1, "hello") return [{"role": "user", "content": stored.content}] - monkeypatch.setattr(chat_session_context, "_load_session_messages", load_after_accept) + monkeypatch.setattr(chat_session_context, 'load_session_messages', load_after_accept) launched = {} @@ -276,8 +279,8 @@ async def fake_wait_run(_run_id): worker_db.commit() return SimpleNamespace(status="completed", error_message=None) - monkeypatch.setattr(chat_run_executor, "start_run", fake_start_run) - monkeypatch.setattr(chat_run_executor, "wait_run", fake_wait_run, raising=False) + monkeypatch.setattr(runtime_admission, "start_run", fake_start_run) + monkeypatch.setattr(runtime, "wait_run", fake_wait_run, raising=False) with Session() as db: response = asyncio.run( @@ -307,12 +310,12 @@ def test_both_transports_share_preferences_and_release_request_transaction( ): from api.routes.v1 import agent_responses from core.chat import plan_progress - from orchestration import chat_run_executor + pass engine, Session = _route_database(tmp_path, "preferences.db") _patch_common_chat_route(monkeypatch) monkeypatch.setattr(db_engine, "SessionLocal", Session) - monkeypatch.setattr(chat_session_context, "_load_session_messages", lambda *_args: []) + monkeypatch.setattr(chat_session_context, 'load_session_messages', lambda *_args: []) monkeypatch.setattr(plan_progress, "clear_plan_progress", lambda _chat_id: None) async def preferences(_request, _user_id): @@ -331,12 +334,12 @@ def context(*_args, **kwargs): async def start(**kwargs): return kwargs["accepted_run"] - monkeypatch.setattr("core.services.session_operations.admission._read_preferences", preferences) - monkeypatch.setattr(chat_request_context, "_build_ctx", context) - monkeypatch.setattr(chat_run_executor, "start_run", start) + monkeypatch.setattr('core.chat.operations.admission._read_preferences', preferences) + monkeypatch.setattr(chat_request_context, 'build_ctx', context) + monkeypatch.setattr(runtime_admission, "start_run", start) with Session() as db: result = asyncio.run( - admission._start_response_run( + admission.start_response_run( ChatRequest(chat_id="chat-1", message="hello", stream=stream), _user(), db ) ) @@ -361,14 +364,14 @@ async def start(**kwargs): ) def test_json_wait_preserves_non_success_terminal_state(monkeypatch, terminal, code): from api.routes.v1 import agent_responses - from orchestration import chat_run_executor + pass record = SimpleNamespace(status=terminal, error_message=None) async def wait(_run_id): return record - monkeypatch.setattr(chat_run_executor, "wait_run", wait) + monkeypatch.setattr(runtime, "wait_run", wait) with pytest.raises(HTTPException) as raised: asyncio.run( agent_responses._wait_response("run-1", chat_id="chat-1", message_id="message-1") diff --git a/src/backend/tests/api/test_agent_response_scope.py b/src/backend/tests/api/test_agent_response_scope.py index e1140ea5..105263d6 100644 --- a/src/backend/tests/api/test_agent_response_scope.py +++ b/src/backend/tests/api/test_agent_response_scope.py @@ -1,15 +1,18 @@ -import core.services.session_operations.admission as admission + +import core.chat.runtime.admission as runtime_admission +import core.chat.operations.admission as admission + """Scoped admission must not interpret account-wide delegation commands.""" -import core.services.session_operations.agent_targets as chat_agent_targets -import core.services.session_operations.request_context as chat_request_context -import core.services.session_operations.session_context as chat_session_context import core.db.engine as db_engine +import core.chat.operations.agent_targets as chat_agent_targets +import core.chat.operations.request_context as chat_request_context +import core.chat.operations.session_context as chat_session_context import pytest from api.routes.v1 import agent_responses, chats from core.chat.schemas import ChatRequest from core.db.models import ChatMessage, ChatRun, UserAgent -from core.services import agent_api_service +from core.auth import agent_api as agent_api_service from sqlalchemy.orm import sessionmaker from tests.api.test_agent_api_scope import _key, _user, db from tests.api.test_agent_response_admission import _patch_common_chat_route @@ -27,7 +30,7 @@ async def test_scope_preserves_message_and_never_enumerates_other_agents( ChatMessage.__table__.create(db.bind, checkfirst=True) _patch_common_chat_route(monkeypatch) monkeypatch.setattr(db_engine, "SessionLocal", sessionmaker(bind=db.bind)) - monkeypatch.setattr(chat_session_context, "_load_session_messages", lambda *_: []) + monkeypatch.setattr(chat_session_context, 'load_session_messages', lambda *_: []) def forbidden_resolution(*args, **kwargs): raise AssertionError("Scoped requests must not resolve owner-wide agent commands") @@ -41,14 +44,14 @@ async def preferences(*args): "ontology_enabled": True, } - monkeypatch.setattr("core.services.session_operations.admission._read_preferences", preferences) + monkeypatch.setattr('core.chat.operations.admission._read_preferences', preferences) context_args = {} def context(*args, **kwargs): context_args.update(kwargs) return {"user_id": "owner", "chat_id": "api-chat", "agent_id": key.agent_id} - monkeypatch.setattr(chat_request_context, "_build_ctx", context) + monkeypatch.setattr(chat_request_context, 'build_ctx', context) captured = {} @@ -57,10 +60,10 @@ async def start(**kwargs): captured["payload"] = dict(kwargs["accepted_run"].request_payload) return kwargs["accepted_run"] - monkeypatch.setattr(agent_responses.chat_run_executor, "start_run", start) + monkeypatch.setattr(runtime_admission, "start_run", start) message = "调用「Two」子智能体：原样保留的内容" - result = await admission._start_response_run( + result = await admission.start_response_run( ChatRequest(chat_id="api-chat", message=message, stream=stream, agent_id="ua_one"), _user(key), db, diff --git a/src/backend/tests/api/test_agent_responses.py b/src/backend/tests/api/test_agent_responses.py index a6e731f0..74bea21b 100644 --- a/src/backend/tests/api/test_agent_responses.py +++ b/src/backend/tests/api/test_agent_responses.py @@ -2,13 +2,13 @@ from types import SimpleNamespace +import api.transports.run_sse as run_sse import pytest -from fastapi import FastAPI -from fastapi.testclient import TestClient - -from core.chat.schemas import ChatRequest from core.auth.backend import UserContext, get_current_user +from core.chat.schemas import ChatRequest from core.db.engine import get_db +from fastapi import FastAPI +from fastapi.testclient import TestClient @pytest.fixture @@ -37,7 +37,7 @@ async def wait(run_id, *, chat_id, message_id): return ChatResponse(chat_id=chat_id, response="hello", timestamp="2026-09-26") monkeypatch.setattr(responses, "_start_response_run", start) - monkeypatch.setattr(responses.chat_run_executor, "follow_run_as_sse", follow) + monkeypatch.setattr(run_sse, "follow_run_as_sse", follow) monkeypatch.setattr(responses, "_wait_response", wait) return TestClient(app), seen, app @@ -91,7 +91,7 @@ def test_stream_requires_a_json_boolean(value): @pytest.mark.parametrize("status", [200, 409, 500]) def test_scoped_json_logs_actual_http_result(response_client, monkeypatch, status): from api.routes.v1 import agent_responses - from core.services import agent_api_service + from core.auth import agent_api as agent_api_service from fastapi import HTTPException client, _seen, app = response_client diff --git a/src/backend/tests/api/test_api_key_run_lifecycle_postgres.py b/src/backend/tests/api/test_api_key_run_lifecycle_postgres.py index 584ba1b2..4095e4d8 100644 --- a/src/backend/tests/api/test_api_key_run_lifecycle_postgres.py +++ b/src/backend/tests/api/test_api_key_run_lifecycle_postgres.py @@ -8,9 +8,9 @@ from threading import Event from core.db.models import ChatRun, ChatSession, UserApiKey, UserShadow -from core.services.api_key_runs import cancel_key_runs, require_admission_key -from core.services.api_key_service import ApiKeyService -from core.services.chat_sequencer import ChatSequencer +from core.auth.api_key_runs import cancel_key_runs, require_admission_key +from core.auth.api_keys import ApiKeyService +from core.chat.sequencer import ChatSequencer from fastapi import HTTPException from sqlalchemy.orm import sessionmaker diff --git a/src/backend/tests/api/test_archive_files.py b/src/backend/tests/api/test_archive_files.py index cd05f03b..096a0471 100644 --- a/src/backend/tests/api/test_archive_files.py +++ b/src/backend/tests/api/test_archive_files.py @@ -14,7 +14,7 @@ from core.auth.backend import UserContext, get_current_user from core.db.engine import Base, get_db from core.db.models import Artifact -from core.storage.local import LocalStorageBackend +from core.files.storage.local import LocalStorageBackend @pytest.fixture @@ -26,7 +26,7 @@ def archive_client(tmp_path, monkeypatch): db = Session(engine) monkeypatch.setenv("STORAGE_PATH", str(tmp_path / "storage")) storage = LocalStorageBackend() - monkeypatch.setattr("core.storage.get_storage", lambda: storage) + monkeypatch.setattr('core.files.storage.get_storage', lambda: storage) monkeypatch.setattr("api.routes.file_archive.get_storage", lambda: storage) app = FastAPI() app.include_router(router) diff --git a/src/backend/tests/api/test_archive_safety.py b/src/backend/tests/api/test_archive_safety.py index de5acc3b..b87c761e 100644 --- a/src/backend/tests/api/test_archive_safety.py +++ b/src/backend/tests/api/test_archive_safety.py @@ -9,7 +9,7 @@ import pytest -from core.content.archive_browser import MAX_MEMBER_BYTES +from core.files.content.archive_browser import MAX_MEMBER_BYTES from tests.api.test_archive_files import archive_client @@ -109,7 +109,7 @@ def test_large_zip_member_cannot_be_expanded(archive_client): def test_expanded_tar_scan_has_a_hard_budget(archive_client, monkeypatch): client, upload = archive_client - monkeypatch.setattr("core.content.archive_browser.MAX_SCAN_BYTES", 32 * 1024) + monkeypatch.setattr('core.files.content.archive_browser.MAX_SCAN_BYTES', 32 * 1024) data = io.BytesIO() with tarfile.open(fileobj=data, mode="w:gz") as archive: info = tarfile.TarInfo("file.txt") diff --git a/src/backend/tests/api/test_cancel_continue_e2e.py b/src/backend/tests/api/test_cancel_continue_e2e.py index 26efe6ad..19dcbc49 100644 --- a/src/backend/tests/api/test_cancel_continue_e2e.py +++ b/src/backend/tests/api/test_cancel_continue_e2e.py @@ -10,20 +10,22 @@ import asyncio import contextlib +import core.agent_runtime.runner as runner +import core.db.engine as core_db_engine +import core.runs.tasks as tasks import fakeredis.aioredis import httpx import pytest from api.routes.v1 import agent_responses, chat_runs, chats -from core.services.session_operations import invocation from core.auth.backend import UserContext, get_current_user from core.db import engine as db_engine from core.db.engine import Base, get_db from core.db.models import ChatRun, ChatSession -from core.llm.chat_models import ReasoningEchoChatFormatter -from core.llm.message_compat import session_to_msgs -from core.llm.model_steps import record_assistant_step +from core.models.chat_models import ReasoningEchoChatFormatter +from core.llm.runtime.message_compat import session_to_msgs +from core.llm.runtime.model_steps import record_assistant_step +from core.chat.operations import invocation from fastapi import FastAPI -from orchestration import chat_run_executor as executor from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker from tests.api.test_chat_stream_sequencer import _patch_common_chat_route @@ -45,18 +47,18 @@ def environment(tmp_path, monkeypatch): db.add(ChatSession(chat_id="chat-1", user_id="user-1", title="test")) db.commit() monkeypatch.setattr(db_engine, "SessionLocal", sessions) - monkeypatch.setattr(executor, "SessionLocal", sessions) + monkeypatch.setattr(core_db_engine, "SessionLocal", sessions) monkeypatch.setattr("core.chat.plan_progress.SessionLocal", sessions) - monkeypatch.setattr("core.services.chat_steer_service.SessionLocal", sessions) + monkeypatch.setattr('core.chat.steer.SessionLocal', sessions) redis = fakeredis.aioredis.FakeRedis(decode_responses=True) - monkeypatch.setattr("orchestration.run_event_stream.get_redis", lambda **_: redis) + monkeypatch.setattr('core.runs.events.store.get_redis', lambda **_: redis) _patch_common_chat_route(monkeypatch) - monkeypatch.setattr(invocation, "_resolve_explicit_capability_invocation", lambda db, r, u: r) + monkeypatch.setattr(invocation, 'resolve_explicit_capability_invocation', lambda db, r, u: r) async def preferences(*_): return (None, None, None), {} - monkeypatch.setattr("core.services.session_operations.admission._read_preferences", preferences) + monkeypatch.setattr('core.chat.operations.admission._read_preferences', preferences) app = FastAPI() for router in (agent_responses.router, chats.router, chat_runs.router): app.include_router(router) @@ -69,9 +71,9 @@ def database(): yield db app.dependency_overrides[get_db] = database - executor._active_runs.clear() + tasks.process_state().active_runs.clear() yield app, sessions - executor._active_runs.clear() + tasks.process_state().active_runs.clear() engine.dispose() @@ -144,7 +146,7 @@ async def test_cancel_reload_continue_never_echoes_private_reasoning( ): app, sessions = environment workflow, ready, calls, observed_wire = make_workflow(scenario) - monkeypatch.setattr(executor, "astream_chat_workflow", workflow) + monkeypatch.setattr(runner, "astream_chat_workflow", workflow) async with httpx.AsyncClient( transport=httpx.ASGITransport(app=app), base_url="http://test" ) as client: @@ -219,7 +221,7 @@ async def test_cancel_reload_continue_never_echoes_private_reasoning( first.cancel() with contextlib.suppress(asyncio.CancelledError): await first - for worker in list(executor._active_runs.values()): + for worker in list(tasks.process_state().active_runs.values()): worker.cancel() with contextlib.suppress(asyncio.CancelledError): await worker @@ -239,7 +241,7 @@ async def test_browser_cancel_reload_continue(environment, monkeypatch, scenario pytest.skip("Set RUN_REASONING_BROWSER_E2E=1 for Chromium end-to-end checks") app, _sessions = environment workflow, _ready, calls, observed_wire = make_workflow(scenario) - monkeypatch.setattr(executor, "astream_chat_workflow", workflow) + monkeypatch.setattr(runner, "astream_chat_workflow", workflow) sock = socket.socket() sock.bind(("127.0.0.1", 0)) port = sock.getsockname()[1] @@ -273,7 +275,7 @@ async def test_browser_cancel_reload_continue(environment, monkeypatch, scenario assert PRIVATE not in str(observed_wire) assert calls == (["scan"] if scenario == "completed_tool_then_partial" else []) finally: - for worker in list(executor._active_runs.values()): + for worker in list(tasks.process_state().active_runs.values()): worker.cancel() with contextlib.suppress(asyncio.CancelledError): await worker diff --git a/src/backend/tests/api/test_chat_connector_selection.py b/src/backend/tests/api/test_chat_connector_selection.py index 0aba3fdb..4b0f83c4 100644 --- a/src/backend/tests/api/test_chat_connector_selection.py +++ b/src/backend/tests/api/test_chat_connector_selection.py @@ -1,7 +1,7 @@ -from core.services.session_operations.request_context import _build_user_extra_data +from core.agent_runtime.skills import _build_skill_injection from core.chat.schemas import ChatRequest from core.llm.factory.tools.mcp_config import _required_mcp_server_keys -from orchestration.workflow import _build_skill_injection +from core.chat.operations.request_context import _build_user_extra_data from pydantic import ValidationError diff --git a/src/backend/tests/api/test_chat_event_loop_performance.py b/src/backend/tests/api/test_chat_event_loop_performance.py index bd3610d1..81f8cbc9 100644 --- a/src/backend/tests/api/test_chat_event_loop_performance.py +++ b/src/backend/tests/api/test_chat_event_loop_performance.py @@ -6,8 +6,8 @@ from types import SimpleNamespace import api.routes.v1.chats.questions as chat_questions -import core.services.session_operations.session_context as chat_session_context -import core.services as chat_services +import core.chat.operations.session_context as chat_session_context +import core.chat.service as chat_services_ChatService_module import pytest from core.llm.tools import user_questions @@ -27,8 +27,8 @@ def get_session(self, *args): time.sleep(0.15) return object() - monkeypatch.setattr(chat_services, "ChatService", Service) - monkeypatch.setattr(chat_session_context, "_release_request_session", lambda db: None) + monkeypatch.setattr(chat_services_ChatService_module, "ChatService", Service) + monkeypatch.setattr(chat_session_context, 'release_request_session', lambda db: None) async def pending(chat_id): return [] diff --git a/src/backend/tests/api/test_chat_fork_derived_data.py b/src/backend/tests/api/test_chat_fork_derived_data.py index 8d148bfb..9dfabf29 100644 --- a/src/backend/tests/api/test_chat_fork_derived_data.py +++ b/src/backend/tests/api/test_chat_fork_derived_data.py @@ -10,7 +10,7 @@ @pytest.mark.parametrize("group_by", ["user", "model", "day"]) def test_security_usage_overview_ignores_forked_history(fork_env, group_by): - from api.routes.v1.internal_security import SecurityQueryBody, usage + from features.security_console.api.internal_security import SecurityQueryBody, usage client, sessions, _ = fork_env moment = datetime.utcnow() - timedelta(hours=1) @@ -39,7 +39,8 @@ def test_security_usage_overview_ignores_forked_history(fork_env, group_by): def test_scheduled_backfill_does_not_create_episodes_for_forked_history(fork_env, monkeypatch): from core.db.models import ToolCallLog from core.db.models.evolution import EvolutionEpisode, EvolutionTraceEvent - from core.evolution import backfill, trace_assembler, trace_store + from core.evolution import backfill + from core.evolution.evidence import trace_assembler, trace_store client, sessions, _ = fork_env with sessions() as db: diff --git a/src/backend/tests/api/test_chat_forks.py b/src/backend/tests/api/test_chat_forks.py index f04ba5fe..0eb20457 100644 --- a/src/backend/tests/api/test_chat_forks.py +++ b/src/backend/tests/api/test_chat_forks.py @@ -4,8 +4,8 @@ import pytest from core.db.models import ChatCompactionState, ChatMessage, ChatRun, ChatSession -from core.services.chat_service import ChatService -from core.services.compaction_service import load_session_history +from core.chat.service import ChatService +from core.chat.compaction.service import load_session_history from tests.api.chat_fork_fixtures import fork_env # noqa: F401 @@ -134,7 +134,7 @@ def test_empty_chat_and_all_running_history_do_not_create_branch(fork_env): def test_future_checkpoint_is_not_inherited_and_tool_context_is_replayed(fork_env): client, sessions, _ = fork_env - from core.llm.model_steps import record_assistant_step, record_tool_result_step + from core.llm.runtime.model_steps import record_assistant_step, record_tool_result_step steps = [ record_assistant_step( @@ -295,7 +295,7 @@ def test_long_history_is_not_limited_to_ui_page(fork_env, monkeypatch): from core.config import settings as config_settings from core.db import engine as engine_module - from core.services import compaction_service as compaction + from core.chat.compaction import service as compaction monkeypatch.setattr(engine_module, "SessionLocal", sessions) monkeypatch.setattr( @@ -311,7 +311,7 @@ async def summarize(history, *, timeout): assert "item 121" not in str(history) return "fork prefix summary" - monkeypatch.setattr("core.services.compaction_summary._summarize", summarize) + monkeypatch.setattr('core.chat.compaction.summary._summarize', summarize) with sessions() as db: history = load_session_history(ChatService(db), cid, "owner") history, compacted = asyncio.run( @@ -421,7 +421,8 @@ def test_next_turn_collects_inherited_attachment_context(fork_env, monkeypatch): def test_branching_does_not_change_usage_or_billing_reports(fork_env): from api.deps import require_config from api.routes.v1 import admin_billing, admin_usage_logs, me_logs - from core.db.models import DesktopRecord, ModelPricing, ModelProvider, UserShadow + from features.desktop.models import DesktopRecord + from core.db.models import ModelPricing, ModelProvider, UserShadow client, sessions, _ = fork_env client.app.dependency_overrides[require_config] = lambda: None diff --git a/src/backend/tests/api/test_chat_reference_api.py b/src/backend/tests/api/test_chat_reference_api.py index 1bd263f7..47394695 100644 --- a/src/backend/tests/api/test_chat_reference_api.py +++ b/src/backend/tests/api/test_chat_reference_api.py @@ -8,7 +8,7 @@ from types import SimpleNamespace -import core.services.session_operations.request_context as chat_request_context +import core.chat.operations.request_context as chat_request_context import core.auth.backend as auth_backend import core.chat.context as chat_context import core.db.engine as db_engine @@ -162,7 +162,7 @@ def test_send_time_and_replay_render_the_same_reference_block(db_factory): """引用名片在发送时落库，重放时按落库的快照渲染——两边必须完全一致。""" from core.chat.schemas import ChatRequest from core.chat.context import build_effective_user_message - from core.services.compaction_history import normalize_rows + from core.chat.compaction.history import normalize_rows _seed(db_factory) diff --git a/src/backend/tests/api/test_chat_rerun_options.py b/src/backend/tests/api/test_chat_rerun_options.py index 40513fb6..b1e76d0d 100644 --- a/src/backend/tests/api/test_chat_rerun_options.py +++ b/src/backend/tests/api/test_chat_rerun_options.py @@ -1,20 +1,24 @@ """Reruns preserve current model and prompt surface at real admission boundaries.""" import asyncio + +import api.transports.run_sse as run_sse +import core.chat.runtime.admission as admission import pytest from core.db.engine import Base -from core.services.chat_service import ChatService from core.db.models import ChatRun -from tests.api.test_chat_stream_sequencer import _route_database, _patch_common_chat_route, _user +from core.chat.service import ChatService +from tests.api.test_chat_stream_sequencer import _patch_common_chat_route, _route_database, _user @pytest.mark.parametrize("operation", ["edit", "regenerate"]) @pytest.mark.parametrize("override", [False, True]) def test_rerun_model_and_mode_survive_admission(monkeypatch, tmp_path, operation, override): from api.routes.v1.chats import reruns - from core.services.session_operations import schemas as models, session_context from core.db import engine as db_engine - from orchestration import chat_run_executor + from core.chat.operations import schemas as models + from core.chat.operations import session_context + pass engine, Session = _route_database(tmp_path) Base.metadata.create_all(engine) @@ -32,7 +36,7 @@ def test_rerun_model_and_mode_survive_admission(monkeypatch, tmp_path, operation monkeypatch.setattr(db_engine, "SessionLocal", Session) monkeypatch.setattr( session_context, - "_resolve_selected_model_provider_id", + 'resolve_selected_model_provider_id', lambda db, req, uid: req.model_provider_id, ) @@ -42,8 +46,8 @@ async def start(**kwargs): async def follow(*args, **kwargs): yield "data: [DONE]\n\n" - monkeypatch.setattr(chat_run_executor, "start_run", start) - monkeypatch.setattr(chat_run_executor, "follow_run_as_sse", follow) + monkeypatch.setattr(admission, "start_run", start) + monkeypatch.setattr(run_sse, "follow_run_as_sse", follow) options = expected if override else {} with Session() as db: if operation == "edit": diff --git a/src/backend/tests/api/test_chat_rerun_start_failure.py b/src/backend/tests/api/test_chat_rerun_start_failure.py index 803f1a22..6e65612f 100644 --- a/src/backend/tests/api/test_chat_rerun_start_failure.py +++ b/src/backend/tests/api/test_chat_rerun_start_failure.py @@ -1,10 +1,12 @@ """HTTP retries remain possible when the admitted worker fails to start.""" +import api.transports.run_sse as run_sse +import core.chat.runtime.admission as admission import pytest from api.routes.v1 import chats from core.auth.backend import get_current_user from core.db.engine import Base, get_db -from core.services.chat_service import ChatService +from core.chat.service import ChatService from fastapi import FastAPI from fastapi.testclient import TestClient from tests.api.test_chat_stream_sequencer import _patch_common_chat_route, _route_database, _user @@ -19,7 +21,7 @@ ) def test_worker_start_failure_releases_admission_for_http_retry(monkeypatch, tmp_path, route, body): from core.db import engine as db_engine - from orchestration import chat_run_executor + pass engine, Session = _route_database(tmp_path) Base.metadata.create_all(engine) @@ -41,8 +43,8 @@ async def start(**kwargs): async def follow(*args, **kwargs): yield "data: [DONE]\n\n" - monkeypatch.setattr(chat_run_executor, "start_run", start) - monkeypatch.setattr(chat_run_executor, "follow_run_as_sse", follow) + monkeypatch.setattr(admission, "start_run", start) + monkeypatch.setattr(run_sse, "follow_run_as_sse", follow) app = FastAPI() app.include_router(chats.router) diff --git a/src/backend/tests/api/test_chat_run_steer_api.py b/src/backend/tests/api/test_chat_run_steer_api.py index 309151c2..f9d46ea3 100644 --- a/src/backend/tests/api/test_chat_run_steer_api.py +++ b/src/backend/tests/api/test_chat_run_steer_api.py @@ -2,9 +2,10 @@ from types import SimpleNamespace +import core.runs.runtime as runtime import pytest - from api.routes.v1 import chat_runs +from core.chat.operations import commands from core.auth.backend import UserContext @@ -33,8 +34,8 @@ async def accept(run_id, payload): "status": "accepted", } - monkeypatch.setattr(chat_runs.chat_run_executor, "get_run", lambda _run_id: run) - monkeypatch.setattr(chat_runs, "put_pending_steer", accept) + monkeypatch.setattr(runtime, "get_run", lambda _run_id: run) + monkeypatch.setattr(commands, "put_pending_steer", accept) result = await chat_runs.steer_chat_run( "run-1", @@ -63,7 +64,7 @@ async def accept(run_id, payload): @pytest.mark.asyncio async def test_queue_status_query_is_owner_scoped(monkeypatch): run = SimpleNamespace(run_id="run-1", user_id="user-1") - monkeypatch.setattr(chat_runs.chat_run_executor, "get_run", lambda _run_id: run) + monkeypatch.setattr(runtime, "get_run", lambda _run_id: run) monkeypatch.setattr( chat_runs, "list_run_steers", diff --git a/src/backend/tests/api/test_chat_stream_sequencer.py b/src/backend/tests/api/test_chat_stream_sequencer.py index 52448867..b4ea52fa 100644 --- a/src/backend/tests/api/test_chat_stream_sequencer.py +++ b/src/backend/tests/api/test_chat_stream_sequencer.py @@ -4,21 +4,23 @@ import asyncio -import core.services.session_operations.agent_targets as chat_agent_targets -import core.services.session_operations.invocation as chat_invocation -import core.services.session_operations.schemas as chat_models -import core.services.session_operations.request_context as chat_request_context import api.routes.v1.chats.reruns as chat_reruns -import core.services.session_operations.session_context as chat_session_context +import api.transports.run_sse as run_sse import core.chat.context as chat_context +import core.chat.runtime.admission as admission import core.db.engine as db_engine import core.infra.responses as responses -import core.services as chat_services +import core.auth.users as chat_services_UserService_module +import core.chat.operations.agent_targets as chat_agent_targets +import core.chat.operations.invocation as chat_invocation +import core.chat.operations.request_context as chat_request_context +import core.chat.operations.schemas as chat_models +import core.chat.operations.session_context as chat_session_context import pytest from core.auth.backend import UserContext from core.db.engine import Base from core.db.models import ChatMessage, ChatRun, ChatSession -from core.services.chat_sequencer import ChatSequencer +from core.chat.sequencer import ChatSequencer from fastapi import HTTPException from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker @@ -52,19 +54,19 @@ def __init__(self, _db): def get_user_settings(self, _user_id): return {} - monkeypatch.setattr(chat_services, "UserService", FakeUserService) - monkeypatch.setattr(chat_session_context, "_ensure_main_model_configured", lambda: None) + monkeypatch.setattr(chat_services_UserService_module, "UserService", FakeUserService) + monkeypatch.setattr(chat_session_context, 'ensure_main_model_configured', lambda: None) monkeypatch.setattr( chat_agent_targets, "_resolve_chat_agent_targets", lambda _db, request, _user_id: (request, None, request.message, None), ) monkeypatch.setattr( - chat_session_context, "_resolve_selected_model_provider_id", lambda *_args: None + chat_session_context, 'resolve_selected_model_provider_id', lambda *_args: None ) monkeypatch.setattr( chat_session_context, - "_resolve_actual_chat_model_name", + 'resolve_actual_chat_model_name', lambda request, _: request.model_name, ) monkeypatch.setattr( @@ -73,12 +75,12 @@ def get_user_settings(self, _user_id): monkeypatch.setattr( chat_session_context, "_ensure_chat_session", lambda *_args, **_kwargs: None ) - monkeypatch.setattr(chat_request_context, "_build_ctx", lambda *_args, **_kwargs: {}) + monkeypatch.setattr(chat_request_context, 'build_ctx', lambda *_args, **_kwargs: {}) monkeypatch.setattr(chat_request_context, "_build_user_extra_data", lambda *_args: {}) def test_regenerate_busy_does_not_delete_existing_history(monkeypatch, tmp_path): - from core.services.chat_service import ChatService + from core.chat.service import ChatService engine, Session = _route_database(tmp_path, "regenerate.db") with Session() as db: @@ -115,8 +117,8 @@ def test_regenerate_busy_does_not_delete_existing_history(monkeypatch, tmp_path) def test_regenerate_admits_then_deletes_tail_and_launches_reserved_run(monkeypatch, tmp_path): - from core.services.chat_service import ChatService - from orchestration import chat_run_executor + from core.chat.service import ChatService + pass engine, Session = _route_database(tmp_path, "regenerate-success.db") with Session() as db: @@ -144,9 +146,9 @@ async def empty_follow(_run_id, *, chat_id): if False: # pragma: no cover yield chat_id - monkeypatch.setattr(chat_session_context, "_load_session_messages", load_after_admission) - monkeypatch.setattr(chat_run_executor, "start_run", fake_start_run) - monkeypatch.setattr(chat_run_executor, "follow_run_as_sse", empty_follow) + monkeypatch.setattr(chat_session_context, 'load_session_messages', load_after_admission) + monkeypatch.setattr(admission, "start_run", fake_start_run) + monkeypatch.setattr(run_sse, "follow_run_as_sse", empty_follow) with Session() as db: asyncio.run( @@ -169,7 +171,7 @@ async def empty_follow(_run_id, *, chat_id): def test_edit_busy_does_not_delete_existing_history(monkeypatch, tmp_path): - from core.services.chat_service import ChatService + from core.chat.service import ChatService engine, Session = _route_database(tmp_path, "edit.db") with Session() as db: @@ -213,8 +215,8 @@ def test_edit_busy_does_not_delete_existing_history(monkeypatch, tmp_path): def test_edit_replays_original_turn_invocation(monkeypatch, tmp_path): - from core.services.chat_service import ChatService - from orchestration import chat_run_executor + from core.chat.service import ChatService + pass engine, Session = _route_database(tmp_path, "edit-invocation.db") saved_extra = { @@ -245,24 +247,24 @@ def resolve_invocation(_db, request, _user_id): return request monkeypatch.setattr( - chat_invocation, "_resolve_explicit_capability_invocation", resolve_invocation + chat_invocation, 'resolve_explicit_capability_invocation', resolve_invocation ) def capture_ctx(request, *_args, **_kwargs): captured["request"] = request return {} - monkeypatch.setattr(chat_request_context, "_build_ctx", capture_ctx) - monkeypatch.setattr(chat_session_context, "_load_session_messages", lambda *_args: []) - monkeypatch.setattr(chat_session_context, "_release_request_session", lambda _db: None) + monkeypatch.setattr(chat_request_context, 'build_ctx', capture_ctx) + monkeypatch.setattr(chat_session_context, 'load_session_messages', lambda *_args: []) + monkeypatch.setattr(chat_session_context, 'release_request_session', lambda _db: None) monkeypatch.setattr(responses, "sse_response", lambda stream: stream) - monkeypatch.setattr(chat_run_executor, "follow_run_as_sse", lambda *_a, **_k: None) + monkeypatch.setattr(run_sse, "follow_run_as_sse", lambda *_a, **_k: None) async def fake_start_run(**kwargs): captured["start_run"] = kwargs return kwargs["accepted_run"] - monkeypatch.setattr(chat_run_executor, "start_run", fake_start_run) + monkeypatch.setattr(admission, "start_run", fake_start_run) with Session() as db: asyncio.run( diff --git a/src/backend/tests/api/test_chat_thinking_mode.py b/src/backend/tests/api/test_chat_thinking_mode.py index b3a39d2b..60584438 100644 --- a/src/backend/tests/api/test_chat_thinking_mode.py +++ b/src/backend/tests/api/test_chat_thinking_mode.py @@ -5,8 +5,8 @@ import api.routes.v1.chats.run_views as chat_run_views import core.chat.context as chat_context +import core.runs.runtime as runtime import pytest -from orchestration import chat_run_executor as executor @pytest.mark.parametrize( @@ -26,7 +26,7 @@ def test_active_run_probe_preserves_the_agent_mode(monkeypatch, payload, expecte started_at=datetime.now(timezone.utc), request_payload=payload, ) - monkeypatch.setattr(executor, "get_active_run_for_chat", lambda *args: run) + monkeypatch.setattr(runtime, "get_active_run_for_chat", lambda *args: run) monkeypatch.setattr(chat_context, "resolve_db_user_id", lambda db, user_id: user_id) response = chat_run_views.chat_active_run( "chat-test", user=SimpleNamespace(user_id="owner"), db=object() diff --git a/src/backend/tests/api/test_cross_turn_file_access.py b/src/backend/tests/api/test_cross_turn_file_access.py index 45d7cd27..549896d7 100644 --- a/src/backend/tests/api/test_cross_turn_file_access.py +++ b/src/backend/tests/api/test_cross_turn_file_access.py @@ -5,7 +5,7 @@ - `_collect_historical_attachments` in api/routes/v1/chats/__init__.py: queries the Artifact table by chat_id — covers user uploads AND AI-generated files. -- `_build_historical_files_context` in core/llm/hooks.py: renders the list +- `_build_historical_files_context` in core/llm/runtime/hooks.py: renders the list into a prompt block labeled by provenance. """ @@ -379,7 +379,7 @@ def test_extract_message_file_ids_reads_both_keys(): def test_build_historical_context_labels_source(): - from core.llm.hooks import _build_historical_files_context + from core.llm.runtime.file_context import build_historical_files_context as _build_historical_files_context files = [ {"file_id": "ua_1", "name": "user.pdf", "source": "user_upload", "summary": "S1"}, @@ -392,7 +392,7 @@ def test_build_historical_context_labels_source(): def test_build_historical_context_includes_read_artifact_hint(): - from core.llm.hooks import _build_historical_files_context + from core.llm.runtime.file_context import build_historical_files_context as _build_historical_files_context files = [ {"file_id": "ua_1", "name": "report.pdf", "source": "user_upload", "summary": "annual"} @@ -403,7 +403,7 @@ def test_build_historical_context_includes_read_artifact_hint(): def test_build_historical_context_marks_deleted(): - from core.llm.hooks import _build_historical_files_context + from core.llm.runtime.file_context import build_historical_files_context as _build_historical_files_context files = [{"file_id": "ua_1", "name": "gone.pdf", "deleted": True}] out = _build_historical_files_context(files) @@ -411,6 +411,6 @@ def test_build_historical_context_marks_deleted(): def test_build_historical_context_empty_list_returns_empty(): - from core.llm.hooks import _build_historical_files_context + from core.llm.runtime.file_context import build_historical_files_context as _build_historical_files_context assert _build_historical_files_context([]) == "" diff --git a/src/backend/tests/api/test_deep_folder_repair.py b/src/backend/tests/api/test_deep_folder_repair.py index 3d474778..3be3ea73 100644 --- a/src/backend/tests/api/test_deep_folder_repair.py +++ b/src/backend/tests/api/test_deep_folder_repair.py @@ -30,7 +30,7 @@ def test_deep_batch_upload_move_and_breadcrumb(client): def test_path_in_filename_is_rejected_before_storage(client, monkeypatch): - from core.storage import get_storage + from core.files.storage import get_storage def fail(*a, **k): raise AssertionError("invalid filename reached object storage") monkeypatch.setattr(get_storage(), "upload_bytes", fail) diff --git a/src/backend/tests/api/test_evaluation_sandboxes.py b/src/backend/tests/api/test_evaluation_sandboxes.py index f1042fde..89b5afc0 100644 --- a/src/backend/tests/api/test_evaluation_sandboxes.py +++ b/src/backend/tests/api/test_evaluation_sandboxes.py @@ -7,7 +7,7 @@ @pytest.fixture def client(monkeypatch): - from api.routes.v1 import evaluation_sandboxes as routes + from features.evaluation.api import evaluation_sandboxes as routes app = FastAPI() app.include_router(routes.router) app.dependency_overrides[get_current_user] = lambda: UserContext( diff --git a/src/backend/tests/api/test_file_location.py b/src/backend/tests/api/test_file_location.py index a09d2e81..fefc10a7 100644 --- a/src/backend/tests/api/test_file_location.py +++ b/src/backend/tests/api/test_file_location.py @@ -1,7 +1,7 @@ from types import SimpleNamespace as NS import pytest from fastapi import HTTPException -from core.services.artifact_location import artifact_location +from core.files.artifacts.location import artifact_location from api.routes.file_location import file_location diff --git a/src/backend/tests/api/test_file_open_browser_e2e.py b/src/backend/tests/api/test_file_open_browser_e2e.py index 8f910e5d..e757d1e0 100644 --- a/src/backend/tests/api/test_file_open_browser_e2e.py +++ b/src/backend/tests/api/test_file_open_browser_e2e.py @@ -70,7 +70,7 @@ def test_file_open_browser_http_end_to_end(tmp_path, monkeypatch): settings = settings_module.settings monkeypatch.setattr(settings_module, "settings", replace(settings, deploy=replace(settings.deploy, profile="local"))) # Use the real local storage implementation, isolated from the singleton. - from core.storage.local import LocalStorageBackend + from core.files.storage.local import LocalStorageBackend monkeypatch.setattr("api.routes.files.get_storage", LocalStorageBackend) engine = create_engine(f"sqlite:///{tmp_path / 'fixture.db'}", connect_args={"check_same_thread": False}) diff --git a/src/backend/tests/api/test_loop_cancel_scope.py b/src/backend/tests/api/test_loop_cancel_scope.py new file mode 100644 index 00000000..bdd7ed1e --- /dev/null +++ b/src/backend/tests/api/test_loop_cancel_scope.py @@ -0,0 +1,39 @@ +"""Loop cancellation must query the current user's active run.""" + +from types import SimpleNamespace + +import pytest +from api.routes.v1 import loops + + +@pytest.mark.asyncio +@pytest.mark.parametrize("active", [False, True]) +async def test_cancel_loop_passes_authenticated_user_to_run_lookup(monkeypatch, active): + observed = [] + loop = SimpleNamespace(chat_id="chat", status="running") + monkeypatch.setattr( + loops, + "LoopService", + lambda db: SimpleNamespace(get_loop=lambda loop_id, user_id: loop), + ) + + def get_active(chat_id, user_id): + observed.append(("lookup", chat_id, user_id)) + return SimpleNamespace(run_id="run") if active else None + + async def cancel(run_id, *, user_id): + observed.append(("cancel", run_id, user_id)) + return True + + monkeypatch.setattr(loops.runtime, "get_active_run_for_chat", get_active) + monkeypatch.setattr(loops.runtime, "cancel_run", cancel) + db = SimpleNamespace(commit=lambda: observed.append(("commit",))) + result = await loops.cancel_loop("loop", db=db, user=SimpleNamespace(user_id="owner")) + assert result["data"]["cancelled"] is True + assert observed == ( + [("lookup", "chat", "owner"), ("cancel", "run", "owner")] + if active + else [("lookup", "chat", "owner"), ("commit",)] + ) + if not active: + assert loop.status == "cancelled" diff --git a/src/backend/tests/api/test_me_system.py b/src/backend/tests/api/test_me_system.py index 8c04f162..6cf047b5 100644 --- a/src/backend/tests/api/test_me_system.py +++ b/src/backend/tests/api/test_me_system.py @@ -246,7 +246,7 @@ def test_ce_excludes_shared_knowledge_base_connector(monkeypatch): def test_whitelist_groups_exist_in_seed_configs(): """Every whitelist group must have config items in SEED_CONFIGS (prevents the whitelist dangling after a seed regroup).""" from api.routes.v1.me_system import PERSONAL_GROUPS - from core.services.system_config import SEED_CONFIGS + from core.config.store.system_config_constants import SEED_CONFIGS seed_groups = {group for _, _, _, _, group, _ in SEED_CONFIGS} missing = set(PERSONAL_GROUPS) - seed_groups diff --git a/src/backend/tests/api/test_page_config_backfill.py b/src/backend/tests/api/test_page_config_backfill.py index 43d9253b..1356148c 100644 --- a/src/backend/tests/api/test_page_config_backfill.py +++ b/src/backend/tests/api/test_page_config_backfill.py @@ -12,10 +12,8 @@ from unittest.mock import MagicMock -from core.content.content_blocks import ( - DEFAULT_PAGE_CONFIG, - backfill_navigation_entries, -) +from core.files.content.content_blocks_constants import DEFAULT_PAGE_CONFIG +from core.files.content.branding import backfill_navigation_entries def _make_row(payload: dict) -> MagicMock: diff --git a/src/backend/tests/api/test_plugin_delete_isolation.py b/src/backend/tests/api/test_plugin_delete_isolation.py index 18a7a7c2..664e9a5c 100644 --- a/src/backend/tests/api/test_plugin_delete_isolation.py +++ b/src/backend/tests/api/test_plugin_delete_isolation.py @@ -13,7 +13,7 @@ def test_foreign_local_plugin_delete_is_not_found(monkeypatch, db_session): def foreign(*args): raise PermissionError("not owned") - monkeypatch.setattr("core.plugins.local.service.get", foreign) + monkeypatch.setattr('features.desktop.local.plugins.service.get', foreign) app = FastAPI() app.include_router(plugins.router) app.dependency_overrides[get_current_user] = lambda: UserContext( diff --git a/src/backend/tests/api/test_project_environments.py b/src/backend/tests/api/test_project_environments.py index d1fa902f..72b00d83 100644 --- a/src/backend/tests/api/test_project_environments.py +++ b/src/backend/tests/api/test_project_environments.py @@ -1,3 +1,4 @@ +from core.capabilities.skills import loader_factory as loader_factory_module from api.routes.v1.project_environments import router from core.auth.backend import UserContext, get_current_user from core.db.engine import get_db @@ -9,12 +10,12 @@ def test_environment_endpoint_enforces_project_permissions_and_empty_custom( db_session, monkeypatch ): - from core.agent_skills import loader + from core.capabilities.skills import loader from core.db import engine from sqlalchemy.orm import sessionmaker monkeypatch.setattr(engine, "SessionLocal", sessionmaker(bind=db_session.get_bind())) - monkeypatch.setattr(loader, "_global_loader", None) + monkeypatch.setattr(loader_factory_module, 'global_loader', None) db_session.add_all( [ UserShadow(user_id="owner", username="Owner"), diff --git a/src/backend/tests/api/test_public_chats.py b/src/backend/tests/api/test_public_chats.py index 51ba2504..d24eb64a 100644 --- a/src/backend/tests/api/test_public_chats.py +++ b/src/backend/tests/api/test_public_chats.py @@ -1,3 +1,4 @@ +from core.auth.public_identity import resolve_public_user """Public links cannot inherit cookies or cross visitor/resource boundaries.""" import asyncio @@ -23,9 +24,9 @@ UserFolder, UserShadow, ) -from core.services import public_chat_service as service -from core.services.agent_api_service import prepare_agent_api_request -from core.services.project_api_service import prepare_project_api_request +from core.chat import public as service +from core.auth.agent_api import prepare_agent_api_request +from core.projects.api import prepare_project_api_request from fastapi import HTTPException, Response from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker @@ -81,6 +82,7 @@ def db(tmp_path, monkeypatch): monkeypatch.setattr( module, "decrypt_secret", lambda value: value.removeprefix("encrypted:") ) + monkeypatch.setattr("core.auth.public_identity.decrypt_secret", lambda value: value.removeprefix("encrypted:")) yield session session.close() engine.dispose() @@ -107,8 +109,8 @@ def test_link_lifecycle_and_visitor_isolation(db, kind, resource): link = service.create_link(db, "owner", kind, resource) assert service.create_link(db, "owner", kind, resource) == link assert db.query(UserApiKey).count() == 1 - first = service.resolve_public_user(db, link["token"], "a" * 48) - second = service.resolve_public_user(db, link["token"], "b" * 48) + first = resolve_public_user(db, link["token"], "a" * 48) + second = resolve_public_user(db, link["token"], "b" * 48) assert first.public_chat_visitor != second.public_chat_visitor assert not hasattr(first, "api_key") body = ChatRequest( @@ -145,7 +147,7 @@ def test_link_lifecycle_and_visitor_isolation(db, kind, resource): asyncio.run(get_current_user(allowed, None, db)) assert service.get_link(db, "owner", kind, resource)["enabled"] is False service.set_enabled(db, "owner", kind, resource, True) - assert service.resolve_public_user(db, link["token"], "a" * 48) + assert resolve_public_user(db, link["token"], "a" * 48) def test_cannot_publish_someone_elses_resource(db): @@ -159,8 +161,8 @@ def test_bootstrap_returns_only_current_visitor(db): from api.routes.v1.public_chats import bootstrap link = service.create_link(db, "owner", "agents", "agent") - first = service.resolve_public_user(db, link["token"], "a" * 48) - second = service.resolve_public_user(db, link["token"], "b" * 48) + first = resolve_public_user(db, link["token"], "a" * 48) + second = resolve_public_user(db, link["token"], "b" * 48) for user, chat_id in ((first, "first"), (second, "second")): prepare_agent_api_request( db, user, ChatRequest(chat_id=chat_id, message="Hi", agent_id="agent") @@ -172,11 +174,11 @@ def test_bootstrap_returns_only_current_visitor(db): def test_public_agent_artifact_roundtrip(db, monkeypatch): - from core.llm.agent_api_runtime import artifact_in_scope, parse_api_scope - from core.services.agent_api_service import make_agent_api_scope + from core.llm.api_runtime.agent import artifact_in_scope, parse_api_scope + from core.auth.agent_api import make_agent_api_scope link = service.create_link(db, "owner", "agents", "agent") - user = service.resolve_public_user(db, link["token"], "a" * 48) + user = resolve_public_user(db, link["token"], "a" * 48) prepare_agent_api_request( db, user, ChatRequest(chat_id="files-chat", message="Hi", agent_id="agent") ) @@ -207,8 +209,8 @@ def test_public_agent_artifact_roundtrip(db, monkeypatch): @pytest.mark.parametrize("kind,resource", [("agents", "agent"), ("projects", "project")]) def test_public_composer_capabilities_and_uploads(db, kind, resource): link = service.create_link(db, "owner", kind, resource) - user = service.resolve_public_user(db, link["token"], "a" * 48) - other = service.resolve_public_user(db, link["token"], "b" * 48) + user = resolve_public_user(db, link["token"], "a" * 48) + other = resolve_public_user(db, link["token"], "b" * 48) for path, method in (("/v1/models/capabilities", "GET"), ("/v1/tool-approval", "GET"), ("/v1/tool-approval", "PUT"), ("/v1/file/upload", "POST")): @@ -247,11 +249,11 @@ def test_public_composer_capabilities_and_uploads(db, kind, resource): def test_public_approval_preferences_are_independent(db, monkeypatch): from api.routes.v1.tool_approval import ApprovalModeBody, get_tool_approval_mode, set_tool_approval_mode - from core.services.user_service import UserService + from core.auth.users import UserService link = service.create_link(db, "owner", "projects", "project") - first = service.resolve_public_user(db, link["token"], "a" * 48) - second = service.resolve_public_user(db, link["token"], "b" * 48) + first = resolve_public_user(db, link["token"], "a" * 48) + second = resolve_public_user(db, link["token"], "b" * 48) monkeypatch.setattr(UserService, "get_user_settings", lambda *_: {"tool_approval_mode": "auto"}) def must_not_change_owner(*args, **kwargs): raise AssertionError("visitor must not change the owner's global setting") diff --git a/src/backend/tests/browser_automation_server.py b/src/backend/tests/browser_automation_server.py index 1449a40a..f586cc5a 100644 --- a/src/backend/tests/browser_automation_server.py +++ b/src/backend/tests/browser_automation_server.py @@ -12,15 +12,19 @@ os.environ["MCP_HOST"] = "127.0.0.1" os.environ["HUGAGENT_CAPS_ROOT"] = str(home) +from features.desktop.bridge import gateway as bridge_gateway +from features.desktop.bridge import identity as bridge_identity +from features.desktop.capabilities import skill_account import json from fastapi import FastAPI, HTTPException from fastapi.responses import HTMLResponse, FileResponse from core.db.engine import Base, engine, SessionLocal from core.db import models from core.auth.backend import UserContext, get_current_user -from core.plugins.management.installation import install_plugin -from api.routes.v1 import plugin_resources, plugin_ui -from mcp.client.streamable_http import streamablehttp_client +from core.capabilities.plugins.management.installation import install_plugin +from api.routes.v1 import plugin_resources +from features.workbench.api import plugin_ui +from mcp.client.streamable_http import streamable_http_client from mcp import ClientSession Base.metadata.create_all(engine) @@ -28,11 +32,11 @@ db.add(models.UserShadow(user_id="browser-owner", user_center_id="browser-owner", username="Browser owner")) db.add(models.ChatSession(chat_id="browser-chat", user_id="browser-owner")) db.commit() - from core.services.user_service import UserService + from core.auth.users import UserService UserService(db).update_user_metadata("browser-owner", {"tool_approval_mode": os.getenv("BROWSER_E2E_APPROVAL_MODE", "auto")}) install_plugin(db, "browser-automation", owner_user_id="browser-owner") -from core.plugins.resources.lifecycle import lifespan as resources_lifespan +from core.capabilities.plugins.resources.lifecycle import lifespan as resources_lifespan app = FastAPI(lifespan=resources_lifespan) app.dependency_overrides[get_current_user] = lambda: UserContext(user_id="browser-owner", user_center_id="browser-owner", username="Browser owner") app.include_router(plugin_resources.router, prefix="/api") @@ -55,21 +59,21 @@ def health(): from core.config.settings import settings _PORTS["browser_runtime"] = urlsplit(os.environ["BROWSER_E2E_MCP"]).port assert settings.server.mcp_host == "127.0.0.1" -from core.plugins.local import service as local_packages -from core.capabilities import local_plugin_runtime, skills +from features.desktop.local.plugins import service as local_packages +from features.desktop.capabilities import local_plugin_runtime, skills modern = local_packages.install("browser-owner", str(Path(__file__).parents[1] / "extensions/plugins/marketplace/browser-automation")) -skills.current_local_user_id = lambda: "browser-owner" +skill_account.current_local_user_id = lambda: "browser-owner" # The isolated desktop proxy carries this synthetic account. Preserve the # real bridge authentication and ticket revalidation with its identity seam. -from core.services import desktop_cloud_bridge as desktop_bridge -desktop_bridge.get_identity_state = lambda: {"shell_user_center_id": "browser-owner"} +from features.desktop.bridge import client as desktop_bridge +bridge_identity.get_identity_state = lambda: {"shell_user_center_id": "browser-owner"} @app.post("/test/tool/{name}") async def tool(name: str, arguments: dict): - from core.llm.factory.tools.mcp_config import _inject_runtime_headers - from core.llm.mcp_pool import make_client - from core.config.catalog_resolver import _owned_enabled_ids - from core.services.desktop_cloud_bridge import cloud_gateway_mcp_configs + from core.llm.factory.tools.mcp_config import inject_runtime_headers as _inject_runtime_headers + from core.llm.mcp.client import make_client + from core.capabilities.catalog.resolver import _owned_enabled_ids + from features.desktop.bridge.gateway import cloud_gateway_mcp_configs sid = modern["components"]["mcp"][0] with SessionLocal() as db: _, allowed = _owned_enabled_ids(db, "browser-owner", {}) @@ -79,10 +83,9 @@ async def tool(name: str, arguments: dict): assert sid in choices[0].chosen config = _inject_runtime_headers(local_plugin_runtime.configs("browser-owner"), current_user_id="browser-owner", chat_id="browser-chat")[sid] - client = make_client(sid, config, is_stateful=False) - method = await client.get_tool(name) - result = await method(**arguments) - if result.metadata.get("is_error") or result.metadata.get("isError"): + async with make_client(sid, config) as client: + result = await client.call_tool(name, arguments) + if result.is_error: return __import__("fastapi").responses.JSONResponse(status_code=409, content={"error": result.content[0].text}) try: return json.loads(result.content[0].text) @@ -92,10 +95,10 @@ async def tool(name: str, arguments: dict): @app.post("/test/cloud-proof") async def cloud_proof(): import mcp.types - from core.services.desktop_capability_mcp import invoke_gateway_tool - from core.services import desktop_capability_mcp as gateway + from features.desktop.capability.mcp import invoke_gateway_tool + from features.desktop.capability import mcp as gateway gateway.guard_capability_content = lambda uid, data, **kw: data - async with streamablehttp_client(os.environ["BROWSER_E2E_MCP"]) as (read, write, _): + async with streamable_http_client(os.environ["BROWSER_E2E_MCP"]) as (read, write, _): async with ClientSession(read, write) as client: await client.initialize() tools = {t.name: t.model_dump() for t in (await client.list_tools()).tools} @@ -142,8 +145,8 @@ def cookie(): async def confinement_probe(): from core.sandbox.factory import get_sandbox_provider from core.sandbox.protocol import ProcessRequest - from core.plugins.resources.confinement import launch_policy - from core.llm.tools._paths import workspace_directory + from core.capabilities.plugins.resources.confinement import launch_policy + from core.sandbox.workspace.resolution import workspace_directory private = home / "credential-probe" private.write_text("isolated-secret-probe") workspace = Path(workspace_directory("browser-chat")) @@ -172,7 +175,7 @@ async def confinement_probe(): async def prewarm_probe(): import asyncio from core.sandbox.factory import get_sandbox_provider - from core.plugins.resources import store + from core.capabilities.plugins.resources import store await get_sandbox_provider().put_file("browser-chat", ".prewarm-probe", b"", user_id="browser-owner") for _ in range(600): with SessionLocal() as db: @@ -187,13 +190,13 @@ async def prewarm_probe(): @app.post("/test/synced-tool/{name}") async def synced_tool(name: str, arguments: dict): """Actual cloud package export, device binding and MCP callback; no cloud account.""" - from core.capabilities import registry, store, device_catalog, skills - from core.capabilities.ref import ResourceRef - from core.services.desktop_capability_entities import _plugin_files - from core.services.desktop_capability_protocol import entity_content_hash - from core.llm.factory.tools.mcp_config import _inject_runtime_headers - from core.llm.mcp_pool import make_client - from core.services import desktop_cloud_bridge as bridge + from features.desktop.capabilities import registry, store, device_catalog, skills + from core.capabilities.reference import ResourceRef + from features.desktop.capability.entities import _plugin_files + from core.capabilities.manifest_protocol import entity_content_hash + from core.llm.factory.tools.mcp_config import inject_runtime_headers as _inject_runtime_headers + from core.llm.mcp.client import make_client + from features.desktop.bridge import client as bridge profile = "browser-cloud-fixture" with SessionLocal() as db: row = db.query(models.InstalledPlugin).filter_by(slug="browser-automation").first() @@ -206,27 +209,27 @@ async def synced_tool(name: str, arguments: dict): inst = registry.upsert(profile_id=profile, ref=ResourceRef("fixture", "account", "plugin", "browser-automation"), content_hash=digest, source="cloud", enabled=True, payload={"components": {"mcp": [sid]}}) registry.set_state(inst.install_id, "ready", resolved_revision=comp.revision) - skills.current_account_profile = lambda: profile - skills.account_authorized_for = lambda uid: uid == "browser-owner" + skill_account.current_account_profile = lambda: profile + skill_account.account_authorized_for = lambda uid: uid == "browser-owner" device_catalog._profile = lambda: profile # The cloud side supplies its authorized schema; its URL must never be used. - async with streamablehttp_client(os.environ["BROWSER_E2E_MCP"]) as (read, write, _): + async with streamable_http_client(os.environ["BROWSER_E2E_MCP"]) as (read, write, _): async with ClientSession(read, write) as client: await client.initialize() schemas = [t.model_dump() for t in (await client.list_tools()).tools] - bridge._bridge_context = lambda: {"state": {"cloud_base": "https://cloud.invalid"}, + bridge_gateway.bridge_context = lambda: {"state": {"cloud_base": "https://cloud.invalid"}, "manifest_revision": "fixture", "profile": profile, "servers": [{"server_id": sid, "tools": schemas, "source_plugin": "browser-automation"}]} - cfg = bridge.cloud_gateway_mcp_configs([sid])[sid] + cfg = bridge_gateway.cloud_gateway_mcp_configs([sid])[sid] assert cfg["execution_scope"] == "local" and "gateway_invoke_url" not in cfg chat_id = arguments.pop("_chat_id", "browser-chat") cfg = _inject_runtime_headers({sid: cfg}, current_user_id="browser-owner", chat_id=chat_id)[sid] - client = make_client(sid, cfg, is_stateful=False) - result = await (await client.get_tool(name))(**arguments) + async with make_client(sid, cfg) as client: + result = await client.call_tool(name, arguments) text = result.content[0].text if "conversation_unavailable" in text: return {"error": "conversation_unavailable"} - if result.metadata.get("is_error") or result.metadata.get("isError"): + if result.is_error: raise HTTPException(409, text) payload = json.loads(text) if name == "browser_open": diff --git a/src/backend/tests/capabilities/test_credential_continuity.py b/src/backend/tests/capabilities/test_credential_continuity.py deleted file mode 100644 index 36bc1651..00000000 --- a/src/backend/tests/capabilities/test_credential_continuity.py +++ /dev/null @@ -1,189 +0,0 @@ -"""A long sync pass signs with the account's current token; a refused token re-syncs on renewal.""" - -from __future__ import annotations - -import base64 -import json -import threading - -import httpx -import pytest -from core.services import desktop_cloud_bridge as bridge -from core.services import desktop_cloud_bundles, desktop_cloud_skills - - -def _token(user_id: str, nonce: str) -> str: - claims = { - "u": user_id, - "c": "center", - "a": 1, - "h": f"session-{user_id}", - "d": "device", - "n": nonce, - } - body = base64.urlsafe_b64encode(json.dumps(claims).encode()).decode().rstrip("=") - return f"dcap2.{body}.sig" - - -def _refusal(status: int, state=None) -> httpx.HTTPStatusError: - request = httpx.Request( - "GET", - "https://cloud/api/v1/desktop/capability/plugins/manifest", - headers={"Authorization": f"Bearer {(state or bridge.get_state())['token']}"}, - ) - return httpx.HTTPStatusError( - str(status), request=request, response=httpx.Response(status, request=request) - ) - - -@pytest.fixture -def forced(monkeypatch): - bridge.reset_for_tests() - calls = [] - monkeypatch.setattr(bridge, "_purge_persisted_state", lambda: None) - monkeypatch.setattr(bridge, "_rebuild_identity_views", lambda: None) - monkeypatch.setattr(bridge, "_refresh_manifest_async", lambda **kw: calls.append(kw)) - yield calls - bridge.reset_for_tests() - - -def test_headers_sign_with_the_renewed_token_of_the_same_account(forced): - bridge.set_state("https://cloud", _token("u-1", "a"), 600, device_id="device") - snapshot = bridge.get_state() - renewed = _token("u-1", "b") - bridge.set_state("https://cloud", renewed, 600, device_id="device") - assert bridge.cloud_headers(snapshot)["Authorization"] == f"Bearer {renewed}" - other = {"cloud_base": "https://cloud", "token": _token("u-2", "c"), "device_id": "device"} - assert bridge.cloud_headers(other)["Authorization"] == f"Bearer {other['token']}" - - -def test_renewal_after_a_refused_token_resyncs_once(forced): - bridge.set_state("https://cloud", _token("u-1", "a"), 600, device_id="device") - bridge.set_state("https://cloud", _token("u-1", "b"), 600, device_id="device") - assert len(forced) == 1 - snapshot = bridge.get_state() - bridge.note_rejected_credential(snapshot, _refusal(500)) - bridge.set_state("https://cloud", _token("u-1", "c"), 600, device_id="device") - assert len(forced) == 1 - # A late refusal of b must not reject the already installed c. - bridge.note_rejected_credential(snapshot, _refusal(401, snapshot)) - assert bridge._credential_rejected == "" - bridge.note_rejected_credential(bridge.get_state(), _refusal(401)) - bridge.set_state("https://cloud", _token("u-1", "d"), 600, device_id="device") - assert len(forced) == 2 - bridge.set_state("https://cloud", _token("u-1", "e"), 600, device_id="device") - assert len(forced) == 2 - - -def test_logout_forgets_the_refusal(forced): - bridge.set_state("https://cloud", _token("u-1", "a"), 600, device_id="device") - bridge.note_rejected_credential(bridge.get_state(), _refusal(401)) - bridge.clear_state() - bridge.set_state("https://cloud", _token("u-1", "b"), 600, device_id="device") - assert len(forced) == 2 # first login + re-login only - bridge.set_state("https://cloud", _token("u-1", "c"), 600, device_id="device") - assert len(forced) == 2 - - -@pytest.mark.parametrize("kind", ["agent", "plugin"]) -def test_definition_sync_records_a_refused_credential(forced, monkeypatch, kind): - bridge.set_state("https://cloud", _token("u-1", "a"), 600, device_id="device") - state = bridge.get_state() - monkeypatch.setattr(desktop_cloud_bundles, "capabilities_enabled", lambda: True) - monkeypatch.setattr( - httpx, - "get", - lambda url, **kw: httpx.Response( - 401, request=httpx.Request("GET", url, headers=kw.get("headers")) - ), - ) - assert desktop_cloud_bundles.sync_kind(kind, state) is False - assert bridge._credential_rejected == bridge._state_fingerprint(state) - - -def test_skill_sync_records_a_refused_credential(forced, monkeypatch): - bridge.set_state("https://cloud", _token("u-1", "a"), 600, device_id="device") - state = bridge.get_state() - monkeypatch.setattr(desktop_cloud_skills, "capabilities_enabled", lambda: True) - monkeypatch.setattr( - httpx, - "get", - lambda url, **kw: httpx.Response( - 401, request=httpx.Request("GET", url, headers=kw.get("headers")) - ), - ) - desktop_cloud_skills.sync_blocking(state) - assert bridge._credential_rejected == bridge._state_fingerprint(state) - - -def test_refusal_of_an_expired_token_is_still_recorded(forced, monkeypatch): - bridge.set_state("https://cloud", _token("u-1", "a"), 600, device_id="device") - state = bridge.get_state() - with bridge._state_lock: - bridge._state["expires_at"] = 0 # expired, renewal not pushed yet - assert bridge.get_state() is None - monkeypatch.setattr(desktop_cloud_bundles, "capabilities_enabled", lambda: True) - monkeypatch.setattr( - httpx, - "get", - lambda url, **kw: httpx.Response( - 401, request=httpx.Request("GET", url, headers=kw.get("headers")) - ), - ) - desktop_cloud_bundles.sync_kind("plugin", state) - bridge.set_state("https://cloud", _token("u-1", "b"), 600, device_id="device") - assert len(forced) == 2 - - -def test_forced_refresh_during_a_running_pass_reruns_after_it(monkeypatch): - bridge.reset_for_tests() - monkeypatch.setattr(bridge, "_purge_persisted_state", lambda: None) - monkeypatch.setattr(bridge, "_rebuild_identity_views", lambda: None) - threads, runs = [], [] - - class Recorded(threading.Thread): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - threads.append(self) - - monkeypatch.setattr(threading, "Thread", Recorded) - - def headers(st): - runs.append(st["token"]) - if len(runs) == 1: - bridge.note_rejected_credential(st, _refusal(401)) - bridge.set_state("https://cloud", _token("u-1", "b"), 600, device_id="device") - raise RuntimeError("cloud unreachable") - - monkeypatch.setattr(bridge, "cloud_headers", headers) - bridge.set_state("https://cloud", _token("u-1", "a"), 600, device_id="device") - while any(t.is_alive() for t in list(threads)): - for t in list(threads): - t.join(5) - assert runs == [_token("u-1", "a"), _token("u-1", "b")] - assert bridge._refresh_pending is False and bridge._manifest_fetching is False - bridge.reset_for_tests() - - -def test_readiness_reads_progress_without_holding_the_identity_lock(forced, monkeypatch): - bridge.set_state("https://cloud", _token("u-1", "a"), 600, device_id="device") - seen = {} - - def status(): - def probe(): - seen["lock_free"] = bridge._state_lock.acquire(timeout=1) - if seen["lock_free"]: - bridge._state_lock.release() - - worker = threading.Thread(target=probe) - worker.start() - worker.join() - return {"revision": "r", "installations": [], "last_error": None} - - monkeypatch.setattr(desktop_cloud_skills, "status", status) - monkeypatch.setattr( - desktop_cloud_bundles, "status", lambda: {"agent": status(), "plugin": status()} - ) - monkeypatch.setattr("core.capabilities.session_availability.active", lambda _: False) - assert bridge.initial_sync_status()["totals_known"] is True - assert seen["lock_free"] is True diff --git a/src/backend/tests/ce_release/test_ce_api_key_lifecycle.py b/src/backend/tests/ce_release/test_ce_api_key_lifecycle.py index ce5259f6..16b491b9 100644 --- a/src/backend/tests/ce_release/test_ce_api_key_lifecycle.py +++ b/src/backend/tests/ce_release/test_ce_api_key_lifecycle.py @@ -8,8 +8,8 @@ from core.db.engine import Base from core.db.models import ChatRun, UserApiKey, UserShadow -from core.services.api_key_runs import cancel_key_runs, require_admission_key -from core.services.run_journal import RunJournal +from core.auth.api_key_runs import cancel_key_runs, require_admission_key +from core.runs.journal import RunJournal def test_ce_revocation_cancels_only_bound_runs_and_blocks_admission(tmp_path): @@ -41,8 +41,8 @@ def test_ce_project_credentials_keep_scope_and_register_http_routes(tmp_path, mo from core.chat.schemas import ChatRequest from core.auth.backend import UserContext from core.db.models import ChatSession, Project - from core.services.api_key_service import ApiKeyService - from core.services.project_api_service import prepare_project_api_request + from core.auth.api_keys import ApiKeyService + from core.projects.api import prepare_project_api_request paths = app.openapi()["paths"] assert "/v1/projects/{project_id}/api-keys" in paths diff --git a/src/backend/tests/ce_release/test_ce_history_display.py b/src/backend/tests/ce_release/test_ce_history_display.py index d78ed313..2c432aff 100644 --- a/src/backend/tests/ce_release/test_ce_history_display.py +++ b/src/backend/tests/ce_release/test_ce_history_display.py @@ -6,7 +6,7 @@ def test_ce_tool_calls_display_migration_preserves_existing_messages(tmp_path): import importlib.util from alembic.migration import MigrationContext from alembic.operations import Operations - from core.db.edition_tables import ce_create_all + from core.db.edition import ce_create_all from core.db.models import ChatMessage, ChatSession from sqlalchemy import create_engine from sqlalchemy.orm import Session diff --git a/src/backend/tests/ce_release/test_ce_release_regressions.py b/src/backend/tests/ce_release/test_ce_release_regressions.py index 55396ff4..f5e84f68 100644 --- a/src/backend/tests/ce_release/test_ce_release_regressions.py +++ b/src/backend/tests/ce_release/test_ce_release_regressions.py @@ -18,7 +18,7 @@ @pytest.fixture(scope="module", autouse=True) def initialized_ce_database(): from core.db.engine import SessionLocal, init_db - from core.services.local_user_service import ensure_ce_default_admin + from core.auth.local_user_bootstrap import ensure_ce_default_admin init_db() db = SessionLocal() @@ -211,7 +211,7 @@ async def validate_ok(url, *, allow_private_network, require_https): def test_ce_schema_reconcile_repairs_legacy_sqlite_columns(tmp_path): - from core.db.edition_tables import ce_create_all, ce_reconcile_schema + from core.db.edition import ce_create_all, ce_reconcile_schema from core.db.models import AdminSkill, ChatSession, ModelProvider from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker @@ -340,12 +340,7 @@ def test_ce_mcp_market_risk_migration_preserves_existing_notices(tmp_path): def test_ce_startup_seams_and_compose_defaults_are_ce_safe(): - from core.services.edition_startup import ( - bootstrap_edition_plugins, - create_distillation_scheduler, - recover_datasource_sidecars, - recover_persona_distill_jobs, - ) + from orchestration.workers.edition_startup import bootstrap_edition_plugins, create_distillation_scheduler, recover_datasource_sidecars, recover_persona_distill_jobs assert asyncio.run(recover_datasource_sidecars()) == {} assert bootstrap_edition_plugins(None) == () @@ -367,7 +362,7 @@ def test_ce_model_steps_migration_preserves_existing_messages(tmp_path): import importlib.util from alembic.migration import MigrationContext from alembic.operations import Operations - from core.db.edition_tables import ce_create_all + from core.db.edition import ce_create_all from core.db.models import ChatMessage, ChatSession from sqlalchemy import create_engine from sqlalchemy.orm import Session diff --git a/src/backend/tests/ce_release/test_ce_tool_media.py b/src/backend/tests/ce_release/test_ce_tool_media.py index f1412394..f9cf480a 100644 --- a/src/backend/tests/ce_release/test_ce_tool_media.py +++ b/src/backend/tests/ce_release/test_ce_tool_media.py @@ -6,22 +6,22 @@ def test_tool_media_survives_database_upgrade_and_worker_cache_reset(tmp_path, monkeypatch): from core.db.models import ToolMediaBlob - from core.db.edition_tables import ce_reconcile_schema - from core.llm import tool_media_store as media + from core.db.edition import ce_reconcile_schema + from core.llm.runtime import tool_media_store as media engine = create_engine(f"sqlite:///{tmp_path / 'media.db'}") ce_reconcile_schema(engine) assert "tool_media_blobs" in inspect(engine).get_table_names() monkeypatch.setattr(media, "SessionLocal", sessionmaker(bind=engine)) monkeypatch.setattr(media, "_cache", __import__("collections").OrderedDict()) - monkeypatch.setattr(media, "_cache_bytes", 0) + monkeypatch.setattr(media.process_state(), 'cache_bytes', 0) encoded = base64.b64encode(b"durable CE image bytes").decode() block = {"type": "data", "source": {"type": "base64", "media_type": "image/png", "data": encoded}} ref = media.store(block) assert ref is not None assert media.store(block) == ref media._cache.clear() - media._cache_bytes = 0 + media.process_state().cache_bytes = 0 ce_reconcile_schema(engine) rows = [{"content": [{"type": "tool_result", "output": [ref]}]}] assert media.hydrate_rows(rows)[0]["content"][0]["output"][0]["source"]["data"] == encoded diff --git a/src/backend/tests/ce_release/test_desktop_capability_wiring.py b/src/backend/tests/ce_release/test_desktop_capability_wiring.py index 027f8190..c793fefb 100644 --- a/src/backend/tests/ce_release/test_desktop_capability_wiring.py +++ b/src/backend/tests/ce_release/test_desktop_capability_wiring.py @@ -2,7 +2,7 @@ def test_generated_ce_exports_all_device_capability_models(): - import core.db.models as models + import features.desktop.models as models from core.db.engine import Base expected = { @@ -25,8 +25,8 @@ def test_generated_ce_registers_cloud_bridge_and_device_management_routes(): assert importlib.util.find_spec("edition_ee") is None assert EE_ROUTERS == () - assert ("desktop_capability", "router") in CE_ROUTERS - assert ("desktop_capabilities", "router") in CE_ROUTERS + assert ("features.desktop.api.desktop_capability", "router") in CE_ROUTERS + assert ("features.desktop.api.desktop_capabilities", "router") in CE_ROUTERS operations = {"get", "post", "put", "patch", "delete"} actual = { (method.upper(), path) diff --git a/src/backend/tests/ce_release/test_personal_agent_ce.py b/src/backend/tests/ce_release/test_personal_agent_ce.py index 07742ecc..6423e65d 100644 --- a/src/backend/tests/ce_release/test_personal_agent_ce.py +++ b/src/backend/tests/ce_release/test_personal_agent_ce.py @@ -8,9 +8,11 @@ from alembic.operations import Operations from sqlalchemy.orm import Session -from core.db.edition_tables import _ce_metadata, ce_create_all -from core.db.models import PersonalDevice, UserShadow -from core.extensions import registry +from core.db.personal_schema import ce_metadata +from core.db.edition import ce_create_all +from core.db.models import UserShadow +from personal_agent.models import PersonalDevice +from core.harness.registry import registry def migration(connection): @@ -27,7 +29,7 @@ def test_ce_personal_agent_upgrade_preserves_existing_data_and_refuses_loss(tmp_ engine = sa.create_engine(f"sqlite:///{tmp_path / 'ce.db'}") with engine.begin() as connection: revision = migration(connection) - metadata = _ce_metadata() + metadata = ce_metadata() if fresh: ce_create_all(connection) else: @@ -58,7 +60,7 @@ def test_ce_session_serialization_exposes_personal_agent_without_license(tmp_pat assert importlib.util.find_spec("edition_ee") is None engine = sa.create_engine(f"sqlite:///{tmp_path / 'auth.db'}") ce_create_all(engine) - monkeypatch.setattr(registry, "_enabled", set()) + monkeypatch.setattr(registry.registration_state(), 'enabled', set()) with Session(engine) as db: db.add(UserShadow(user_id="owner", user_center_id="owner", username="Owner")) db.commit() diff --git a/src/backend/tests/ce_release/test_workbench_migration_snapshot.py b/src/backend/tests/ce_release/test_workbench_migration_snapshot.py index 03359959..0ccaf81f 100644 --- a/src/backend/tests/ce_release/test_workbench_migration_snapshot.py +++ b/src/backend/tests/ce_release/test_workbench_migration_snapshot.py @@ -2,9 +2,9 @@ import pytest from sqlalchemy import create_engine, text from sqlalchemy.orm import Session -from core.applications.migration import inventory -from core.applications.migration_snapshot import read_snapshot -from core.db.edition_tables import ce_create_all +from features.workbench.migration import inventory +from features.workbench.migration_snapshot import read_snapshot +from core.db.edition import ce_create_all from core.db.models import ContentBlock, InstalledPlugin, UserShadow, CatalogOverride diff --git a/src/backend/tests/chat/test_display_bounds.py b/src/backend/tests/chat/test_display_bounds.py index a23cb37f..8387e72c 100644 --- a/src/backend/tests/chat/test_display_bounds.py +++ b/src/backend/tests/chat/test_display_bounds.py @@ -4,7 +4,7 @@ 这几条不是"覆盖率"，是这套裁剪能不能上线的**前提**：前端的知识库列表 / 搜索结果 / 通用列表卡片全靠字段名认结构，一旦裁剪动了键或数组长度，它们就会集体退化成裸 JSON。 """ -from core.chat.display_bounds import ( +from core.infra.text_bounds import ( DISPLAY_STRING_MAX, HISTORY_STRING_MAX, bound_result_for_display, diff --git a/src/backend/tests/chat/test_sidebar_order.py b/src/backend/tests/chat/test_sidebar_order.py index 1644a4d3..92b1d991 100644 --- a/src/backend/tests/chat/test_sidebar_order.py +++ b/src/backend/tests/chat/test_sidebar_order.py @@ -4,9 +4,9 @@ 以及写入落到 metadata 的键名不能漂——前端按这个键名读回顺序。 """ -import core.services.session_operations.schemas as chat_models +import core.chat.operations.schemas as chat_models import api.routes.v1.chats.sidebar as chat_sidebar -import core.services as chat_services +import core.auth.users as chat_services_UserService_module class _StubUser: @@ -38,7 +38,7 @@ def test_dedup_id_list_strips_blanks_and_duplicates(): def test_get_sidebar_order_cleans_stored_value(monkeypatch): _StubUserService.settings = {chat_sidebar.SIDEBAR_ORDER_KEY: ["c1", "c1", " c2 ", ""]} - monkeypatch.setattr(chat_services, "UserService", _StubUserService) + monkeypatch.setattr(chat_services_UserService_module, "UserService", _StubUserService) resp = chat_sidebar.get_sidebar_order(user=_StubUser(), db=object()) @@ -48,7 +48,7 @@ def test_get_sidebar_order_cleans_stored_value(monkeypatch): def test_get_sidebar_order_defaults_to_empty(monkeypatch): """没拖过的账号返回空数组——前端据此退回默认排序。""" _StubUserService.settings = {} - monkeypatch.setattr(chat_services, "UserService", _StubUserService) + monkeypatch.setattr(chat_services_UserService_module, "UserService", _StubUserService) resp = chat_sidebar.get_sidebar_order(user=_StubUser(), db=object()) @@ -57,7 +57,7 @@ def test_get_sidebar_order_defaults_to_empty(monkeypatch): def test_update_sidebar_order_persists_cleaned_order(monkeypatch): _StubUserService.captured = {} - monkeypatch.setattr(chat_services, "UserService", _StubUserService) + monkeypatch.setattr(chat_services_UserService_module, "UserService", _StubUserService) body = chat_models.UpdateSidebarOrderRequest(order=["c2", "c1", "c2", " "]) resp = chat_sidebar.update_sidebar_order(request=body, user=_StubUser(), db=object()) @@ -69,7 +69,7 @@ def test_update_sidebar_order_persists_cleaned_order(monkeypatch): def test_update_sidebar_order_truncates_to_cap(monkeypatch): _StubUserService.captured = {} - monkeypatch.setattr(chat_services, "UserService", _StubUserService) + monkeypatch.setattr(chat_services_UserService_module, "UserService", _StubUserService) ids = [f"c{i}" for i in range(chat_sidebar.SIDEBAR_ORDER_MAX + 30)] resp = chat_sidebar.update_sidebar_order( @@ -85,7 +85,7 @@ def test_update_sidebar_order_truncates_to_cap(monkeypatch): def test_update_sidebar_order_empty_resets(monkeypatch): """空数组＝恢复默认排序，必须真的把键写成空，而不是跳过写入。""" _StubUserService.captured = {} - monkeypatch.setattr(chat_services, "UserService", _StubUserService) + monkeypatch.setattr(chat_services_UserService_module, "UserService", _StubUserService) chat_sidebar.update_sidebar_order( request=chat_models.UpdateSidebarOrderRequest(order=[]), diff --git a/src/backend/tests/chat/test_snapshot_transaction_cleanup.py b/src/backend/tests/chat/test_snapshot_transaction_cleanup.py new file mode 100644 index 00000000..ed5f9dc4 --- /dev/null +++ b/src/backend/tests/chat/test_snapshot_transaction_cleanup.py @@ -0,0 +1,23 @@ +"""Snapshot failures must release the transaction, including cancellation.""" +import asyncio +from unittest.mock import Mock + +import pytest + +from core.chat import compaction_snapshots +from core.chat.service import ChatService + + +@pytest.mark.parametrize("failure", [RuntimeError("snapshot failed"), asyncio.CancelledError()]) +def test_public_snapshot_rolls_back_every_failure(monkeypatch, failure): + database = Mock() + service = ChatService(database) + + def fail_snapshot(*args, **kwargs): + raise failure + + monkeypatch.setattr(compaction_snapshots, "acquire_compaction_snapshot_impl", fail_snapshot) + with pytest.raises(type(failure)) as caught: + service.acquire_compaction_snapshot("chat", owner="worker", lease_seconds=30) + assert caught.value is failure + database.rollback.assert_called_once_with() diff --git a/src/backend/tests/chat/test_user_questions_api.py b/src/backend/tests/chat/test_user_questions_api.py index 6bde0daa..fc4e6f02 100644 --- a/src/backend/tests/chat/test_user_questions_api.py +++ b/src/backend/tests/chat/test_user_questions_api.py @@ -3,10 +3,10 @@ import asyncio from types import SimpleNamespace -import core.services.session_operations.session_context as chat_session_context +import core.chat.operations.session_context as chat_session_context import core.auth.backend as auth_backend import core.db.engine as db_engine -import core.services as chat_services +import core.chat.service as chat_services_ChatService_module import httpx import pytest from api.routes.v1 import chats @@ -25,6 +25,10 @@ @pytest.fixture def api_app(monkeypatch): + from core.infra.ephemeral import LocalEphemeralState + from core.llm.runtime import interaction_store + state = LocalEphemeralState() + monkeypatch.setattr(interaction_store, "get_ephemeral_state", lambda: state) class FakeChatService: def __init__(self, _db): pass @@ -34,8 +38,8 @@ def get_session(self, chat_id, user_id): return object() return None - monkeypatch.setattr(chat_services, "ChatService", FakeChatService) - monkeypatch.setattr(chat_session_context, "_release_request_session", lambda db: None) + monkeypatch.setattr(chat_services_ChatService_module, "ChatService", FakeChatService) + monkeypatch.setattr(chat_session_context, 'release_request_session', lambda db: None) app = FastAPI() app.include_router(chats.router) app.dependency_overrides[auth_backend.get_current_user] = lambda: SimpleNamespace( @@ -45,6 +49,16 @@ def get_session(self, chat_id, user_id): return app +async def pending_request(waiting): + async with asyncio.timeout(5): + while not user_questions.get_all_pending("owned-chat"): + if waiting.done(): + await waiting + raise AssertionError("Question finished before becoming pending") + await asyncio.sleep(0.01) + return user_questions.get_all_pending("owned-chat")[0] + + @pytest.mark.asyncio async def test_pending_recovery_answer_validation_and_first_claimant(api_app): waiting = asyncio.create_task( @@ -52,11 +66,10 @@ async def test_pending_recovery_answer_validation_and_first_claimant(api_app): chat_id="owned-chat", questions=[_QUESTION], interactive=True, - timeout=2, + timeout=10, ), ) - await asyncio.sleep(0) - pending = user_questions.get_all_pending("owned-chat")[0] + pending = await pending_request(waiting) request_id = pending["request_id"] transport = httpx.ASGITransport(app=api_app) @@ -111,11 +124,10 @@ async def test_cancel_endpoint_resolves_wait_authoritatively(api_app): chat_id="owned-chat", questions=[_QUESTION], interactive=True, - timeout=2, + timeout=10, ), ) - await asyncio.sleep(0) - request_id = user_questions.get_all_pending("owned-chat")[0]["request_id"] + request_id = (await pending_request(waiting))["request_id"] transport = httpx.ASGITransport(app=api_app) async with httpx.AsyncClient(transport=transport, base_url="http://test") as client: @@ -143,11 +155,10 @@ async def test_four_question_round_is_answerable(api_app): chat_id="owned-chat", questions=questions, interactive=True, - timeout=2, + timeout=10, ), ) - await asyncio.sleep(0) - request_id = user_questions.get_all_pending("owned-chat")[0]["request_id"] + request_id = (await pending_request(waiting))["request_id"] transport = httpx.ASGITransport(app=api_app) async with httpx.AsyncClient(transport=transport, base_url="http://test") as client: diff --git a/src/backend/tests/conftest.py b/src/backend/tests/conftest.py index eff02fc7..aac533b9 100644 --- a/src/backend/tests/conftest.py +++ b/src/backend/tests/conftest.py @@ -4,18 +4,41 @@ import pytest import os +import shutil import tempfile + +# Every pytest session gets its own SQLite files. A fixed name in the system +# temp dir is shared by concurrent runs (other checkouts, other sessions), which +# drop each other's tables mid-test. Set before core.config reads the env; +# explicit DATABASE_URL / SQLITE_FALLBACK_URL / TEST_DATABASE_URL still win. +SESSION_DB_DIR = tempfile.mkdtemp(prefix=f"hugagent-tests-{os.getpid()}-") +DATABASE_URL_PRESET = "DATABASE_URL" in os.environ +os.environ.setdefault("DATABASE_URL", f"sqlite:///{SESSION_DB_DIR}/hugagent.db") +os.environ.setdefault("SQLITE_FALLBACK_URL", f"sqlite:///{SESSION_DB_DIR}/hugagent_dev.db") + from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker from fastapi.testclient import TestClient from core.db.engine import Base + +def pytest_unconfigure(config): + from core.harness.observation_writer import flush_observations + + flush_observations() + shutil.rmtree(SESSION_DB_DIR, ignore_errors=True) + + +@pytest.fixture(scope="session") +def session_db_dir(): + return SESSION_DB_DIR + + @pytest.fixture(scope="session") def test_database_url(): - """Test database URL.""" - # Default to the system temp dir: a relative path would leave a stray test.db in pytest's CWD (repo root, etc.) - return os.getenv("TEST_DATABASE_URL", f"sqlite:///{tempfile.gettempdir()}/hugagent_test.db") + """Test database URL, private to this pytest session unless TEST_DATABASE_URL is set.""" + return os.getenv("TEST_DATABASE_URL", f"sqlite:///{SESSION_DB_DIR}/hugagent_test.db") @pytest.fixture(scope="function") @@ -57,3 +80,22 @@ def auth_headers(test_user): """Mock authentication headers.""" # In real tests, generate actual JWT token return {"Authorization": "Bearer test_token_123"} + + +@pytest.fixture(scope="session", autouse=True) +def product_tool_contributions(): + """Compose product tools for tests invoking the core factory without ASGI.""" + from features import register_product_tools, register_package_sources, register_resource_owners, register_run_capabilities + register_product_tools() + from features import register_catalog_options + register_catalog_options() + register_package_sources() + register_resource_owners() + register_run_capabilities() + from features import register_model_credentials + register_model_credentials() + from core.harness.registry import load_models + from features.installed import INSTALLED_FEATURES + load_models(INSTALLED_FEATURES) + from integrations import register_credentials + register_credentials() diff --git a/src/backend/tests/content/artifacts_selftest.py b/src/backend/tests/content/artifacts_selftest.py index debffe7b..23de66c6 100644 --- a/src/backend/tests/content/artifacts_selftest.py +++ b/src/backend/tests/content/artifacts_selftest.py @@ -12,13 +12,13 @@ def main() -> int: from fastapi import HTTPException from starlette.responses import FileResponse from api.routes.files import download_file - from core.content.artifact_refs import extract_file_ref, resolve_artifact_storage_key + from core.files.artifacts.refs import extract_file_ref, resolve_artifact_storage_key from core.db.engine import SessionLocal except ModuleNotFoundError as e: print(f"artifacts_selftest: SKIP (missing dependency: {e})") return 0 - import core.artifacts.store as store + import core.files.artifacts.store as store original_store_dir = store._STORE_DIR # type: ignore[attr-defined] diff --git a/src/backend/tests/content/catalog_selftest.py b/src/backend/tests/content/catalog_selftest.py index 879248b2..12facd46 100644 --- a/src/backend/tests/content/catalog_selftest.py +++ b/src/backend/tests/content/catalog_selftest.py @@ -12,13 +12,14 @@ def main() -> int: - import core.config.catalog as cat + import core.capabilities.catalog.api as cat + from core.capabilities.catalog import common as catalog_common with tempfile.TemporaryDirectory() as td: tmp = Path(td) / "catalog.json" # Monkeypatch path to avoid touching repo config. - cat._CATALOG_PATH = tmp # type: ignore[attr-defined] + catalog_common.CATALOG_PATH = tmp # type: ignore[attr-defined] data = cat.ensure_default_catalog() assert isinstance(data.get("skills"), list) diff --git a/src/backend/tests/content/content_blocks_selftest.py b/src/backend/tests/content/content_blocks_selftest.py index 28590771..1ce71b38 100644 --- a/src/backend/tests/content/content_blocks_selftest.py +++ b/src/backend/tests/content/content_blocks_selftest.py @@ -62,7 +62,7 @@ def main() -> int: from fastapi import FastAPI from fastapi.testclient import TestClient import content as content_route - from core.content.content_blocks import build_docs_snapshot, import_docs_snapshot + from core.files.content.content_blocks import build_docs_snapshot, import_docs_snapshot from core.db.engine import get_db except ModuleNotFoundError as exc: print(f"content_blocks_selftest: SKIP (missing dependency: {exc})") diff --git a/src/backend/tests/content/security_selftest.py b/src/backend/tests/content/security_selftest.py index cf809921..c036e444 100755 --- a/src/backend/tests/content/security_selftest.py +++ b/src/backend/tests/content/security_selftest.py @@ -32,7 +32,9 @@ mask_phone, mask_email, mask_api_key, mask_password, mask_sensitive_data, mask_log_data ) -from core.services import ChatService, ArtifactService, KBService +from core.chat.service import ChatService +from core.files.artifacts.service import ArtifactService +from features.knowledge.service import KBService # Test database setup diff --git a/src/backend/tests/content/test_artifact_default_path.py b/src/backend/tests/content/test_artifact_default_path.py new file mode 100644 index 00000000..025903b1 --- /dev/null +++ b/src/backend/tests/content/test_artifact_default_path.py @@ -0,0 +1,24 @@ +"""Relocating artifact modules must not relocate an existing local store.""" +import os +from pathlib import Path +import subprocess +import sys + +import pytest + + +@pytest.mark.parametrize("storage_path", [None, "", "configured"]) +def test_artifact_store_root_survives_module_relocation(tmp_path, storage_path): + backend = Path(__file__).resolve().parents[2] + expected = backend.parent / "result" / "artifacts" + env = dict(os.environ, PYTHONPATH=str(backend)) + env.pop("STORAGE_PATH", None) + if storage_path is not None: + env["STORAGE_PATH"] = "" if not storage_path else str(tmp_path / storage_path) + if storage_path: + expected = tmp_path / storage_path / "artifacts" + result = subprocess.run( + [sys.executable, "-c", "from core.files.artifacts.store import _STORE_DIR; print(_STORE_DIR)"], + cwd=tmp_path, env=env, capture_output=True, text=True, check=True, + ) + assert Path(result.stdout.strip()) == expected.resolve() diff --git a/src/backend/tests/content/test_artifact_record_store.py b/src/backend/tests/content/test_artifact_record_store.py index 6eb5eded..68e6556e 100644 --- a/src/backend/tests/content/test_artifact_record_store.py +++ b/src/backend/tests/content/test_artifact_record_store.py @@ -11,14 +11,14 @@ import json import pytest -from core.artifacts import store +from core.files.artifacts import store @pytest.fixture def store_dir(tmp_path, monkeypatch): # Every other path derives from _STORE_DIR, so this one patch is the whole redirect. monkeypatch.setattr(store, "_STORE_DIR", tmp_path) - monkeypatch.setattr(store, "_known_buckets", set()) + monkeypatch.setattr(store.process_state(), 'known_buckets', set()) monkeypatch.setenv("STORAGE_TYPE", "local") return tmp_path diff --git a/src/backend/tests/content/test_artifact_storage_keys.py b/src/backend/tests/content/test_artifact_storage_keys.py index 8d096a7c..8f7ac27d 100644 --- a/src/backend/tests/content/test_artifact_storage_keys.py +++ b/src/backend/tests/content/test_artifact_storage_keys.py @@ -1,6 +1,6 @@ import pytest -from core.content.artifact_refs import require_artifact_storage_key +from core.files.artifacts.refs import require_artifact_storage_key def test_new_artifact_without_an_authoritative_key_is_rejected(): @@ -8,7 +8,7 @@ def test_new_artifact_without_an_authoritative_key_is_rejected(): require_artifact_storage_key("unregistered-file") from types import SimpleNamespace -from core.content.artifact_key_repair import verified_legacy_key +from core.files.artifacts.key_repair import verified_legacy_key class Objects: @@ -35,7 +35,7 @@ def test_real_extensionless_object_and_unmatched_size_are_preserved(): assert verified_legacy_key(row(), Objects({"artifacts/old": b"content"})) is None assert verified_legacy_key(row(), Objects({"artifacts/old.docx": b"other"})) is None -from core.content.artifact_key_repair import repair_legacy_key +from core.files.artifacts.key_repair import repair_legacy_key from core.db.models import Artifact @@ -53,7 +53,7 @@ def test_repair_is_audited_idempotent_and_keeps_content_version(db_session): assert art.extra_data["storage_key_repair"]["old_key"] == "artifacts/old" assert not repair_legacy_key(db_session, art, storage) -from core.content.artifact_key_repair import rollback_legacy_key +from core.files.artifacts.key_repair import rollback_legacy_key def test_audited_repair_can_be_rolled_back_but_not_after_content_changes(db_session): @@ -73,7 +73,7 @@ def test_audited_repair_can_be_rolled_back_but_not_after_content_changes(db_sess def test_explicit_historical_placeholder_is_resolved_to_verified_suffix(monkeypatch): - monkeypatch.setattr("core.storage.get_storage", lambda: Objects({"artifacts/old.docx": b"content"})) + monkeypatch.setattr('core.files.storage.get_storage', lambda: Objects({"artifacts/old.docx": b"content"})) assert require_artifact_storage_key("old", "artifacts/old", filename="report.docx", size=7) == "artifacts/old.docx" diff --git a/src/backend/tests/content/test_extra_files.py b/src/backend/tests/content/test_extra_files.py index 3b86ea0e..b239e464 100644 --- a/src/backend/tests/content/test_extra_files.py +++ b/src/backend/tests/content/test_extra_files.py @@ -19,7 +19,7 @@ def test_agent_skill_spec_has_extra_files(): - from core.agent_skills.registry import AgentSkillSpec + from core.capabilities.skills.registry import AgentSkillSpec spec = AgentSkillSpec( id="test", @@ -35,7 +35,7 @@ def test_agent_skill_spec_has_extra_files(): def test_agent_skill_spec_extra_files_defaults_empty(): - from core.agent_skills.registry import AgentSkillSpec + from core.capabilities.skills.registry import AgentSkillSpec spec = AgentSkillSpec( id="test", @@ -52,7 +52,7 @@ def test_agent_skill_spec_extra_files_defaults_empty(): def test_filesystem_backend_get_extra_files(): - from core.agent_skills.backends.filesystem import FilesystemBackend + from core.capabilities.skills.backends.filesystem import FilesystemBackend with tempfile.TemporaryDirectory() as tmpdir: skill_dir = Path(tmpdir) / "my-skill" @@ -75,7 +75,7 @@ def test_filesystem_backend_get_extra_files(): def test_filesystem_backend_get_extra_files_missing_skill(): - from core.agent_skills.backends.filesystem import FilesystemBackend + from core.capabilities.skills.backends.filesystem import FilesystemBackend with tempfile.TemporaryDirectory() as tmpdir: backend = FilesystemBackend(root_dir=tmpdir, source_name="test") @@ -86,7 +86,7 @@ def test_filesystem_backend_get_extra_files_missing_skill(): def test_database_backend_get_extra_files(): - from core.agent_skills.backends.database import DatabaseBackend + from core.capabilities.skills.backends.database import DatabaseBackend mock_session = MagicMock() mock_session.query.return_value.filter.return_value.scalar.return_value = { @@ -102,7 +102,7 @@ def test_database_backend_get_extra_files(): def test_database_backend_get_extra_files_none(): - from core.agent_skills.backends.database import DatabaseBackend + from core.capabilities.skills.backends.database import DatabaseBackend mock_session = MagicMock() mock_session.query.return_value.filter.return_value.scalar.return_value = None @@ -114,7 +114,7 @@ def test_database_backend_get_extra_files_none(): def test_database_backend_get_extra_files_skill_not_found(): - from core.agent_skills.backends.database import DatabaseBackend + from core.capabilities.skills.backends.database import DatabaseBackend mock_session = MagicMock() mock_session.query.return_value.filter.return_value.scalar.return_value = None @@ -129,8 +129,8 @@ def test_database_backend_get_extra_files_skill_not_found(): def test_composite_backend_get_extra_files(): - from core.agent_skills.backends.composite import CompositeBackend - from core.agent_skills.backends.protocol import SkillFileInfo + from core.capabilities.skills.backends.composite import CompositeBackend + from core.capabilities.skills.backends.protocol import SkillFileInfo mock_fs = MagicMock() mock_fs.source_name = "built-in" @@ -152,7 +152,7 @@ def test_composite_backend_get_extra_files(): def test_composite_backend_get_extra_files_not_found(): - from core.agent_skills.backends.composite import CompositeBackend + from core.capabilities.skills.backends.composite import CompositeBackend mock = MagicMock() mock.source_name = "test" @@ -305,7 +305,7 @@ def test_migration_file_valid(): def test_loader_get_extra_files(): - from core.agent_skills.loader import MultiSourceSkillLoader + from core.capabilities.skills.loader import MultiSourceSkillLoader mock_backend = MagicMock() mock_backend.get_extra_files.return_value = {"f.txt": "content"} @@ -318,8 +318,8 @@ def test_loader_get_extra_files(): def test_loader_get_skill_dir_filesystem(): - from core.agent_skills.loader import MultiSourceSkillLoader - from core.agent_skills.backends.protocol import SkillFileInfo + from core.capabilities.skills.loader import MultiSourceSkillLoader + from core.capabilities.skills.backends.protocol import SkillFileInfo mock_backend = MagicMock() mock_backend.get_skill_info.return_value = SkillFileInfo( @@ -337,8 +337,8 @@ def test_loader_get_skill_dir_filesystem(): def test_loader_get_skill_dir_db_materializes(): - from core.agent_skills.loader import MultiSourceSkillLoader - from core.agent_skills.backends.protocol import SkillFileInfo + from core.capabilities.skills.loader import MultiSourceSkillLoader + from core.capabilities.skills.backends.protocol import SkillFileInfo mock_backend = MagicMock() mock_backend.get_skill_info.return_value = SkillFileInfo( @@ -371,8 +371,8 @@ def test_loader_get_skill_dir_db_materializes(): def test_register_skills_to_toolkit(): from agentscope.tool import Toolkit - from core.llm.tool_collector import ToolCollector - from core.agent_skills.loader import MultiSourceSkillLoader + from core.llm.factory.tools.definitions import ToolAssembly + from core.capabilities.skills.loader import MultiSourceSkillLoader with tempfile.TemporaryDirectory() as tmpdir: # Create a real skill directory @@ -383,15 +383,13 @@ def test_register_skills_to_toolkit(): ) (skill_dir / "config.json").write_text('{"key": "val"}') - from core.agent_skills.backends.filesystem import FilesystemBackend - from core.agent_skills.backends.composite import CompositeBackend + from core.capabilities.skills.backends.filesystem import FilesystemBackend + from core.capabilities.skills.backends.composite import CompositeBackend backend = CompositeBackend([FilesystemBackend(root_dir=tmpdir, source_name="test")]) loader = MultiSourceSkillLoader(backend=backend) - # AgentScope 2.0: register_skills_to_toolkit writes into ToolCollector (register_agent_skill); - # the collected skill directories live in collector.skill_loaders, passed to Toolkit(skills_or_loaders=). - c = ToolCollector() + c = ToolAssembly() n = loader.register_skills_to_toolkit(c) assert n == 1 assert len(c.skill_loaders) == 1 diff --git a/src/backend/tests/content/test_file_preview.py b/src/backend/tests/content/test_file_preview.py index 6895e502..975696cf 100644 --- a/src/backend/tests/content/test_file_preview.py +++ b/src/backend/tests/content/test_file_preview.py @@ -69,7 +69,7 @@ def test_preview_file_returns_inline_pdf_for_powerpoint(monkeypatch, tmp_path): monkeypatch.setattr(file_routes, "_convert_office_to_pdf", lambda source_path, file_id: (str(pdf_path), str(temp_dir))) (tmp_path / 'deck.pptx').write_bytes(b'synthetic unique preview cache source') - from core.content.office_preview_cache import cached_office_pdf + from core.files.content.office_preview_cache import cached_office_pdf monkeypatch.setattr(file_routes, "cached_office_pdf", lambda source, fid, convert: cached_office_pdf(source, fid, convert, root=tmp_path / "cache", version="test")) background_tasks = BackgroundTasks() diff --git a/src/backend/tests/content/test_local_file_location.py b/src/backend/tests/content/test_local_file_location.py index a485983c..801e6ab5 100644 --- a/src/backend/tests/content/test_local_file_location.py +++ b/src/backend/tests/content/test_local_file_location.py @@ -19,9 +19,9 @@ @pytest.fixture def local_files(tmp_path, monkeypatch): - from core.storage import factory + from core.files.storage import factory - monkeypatch.setattr(factory, "_storage_instance", None) + monkeypatch.setattr(factory.process_state(), 'storage_instance', None) monkeypatch.setenv("STORAGE_TYPE", "local") monkeypatch.setenv("STORAGE_PATH", str(tmp_path)) settings = settings_module.settings diff --git a/src/backend/tests/content/test_office_preview_cache.py b/src/backend/tests/content/test_office_preview_cache.py index 3ee6eeab..3642c536 100644 --- a/src/backend/tests/content/test_office_preview_cache.py +++ b/src/backend/tests/content/test_office_preview_cache.py @@ -3,7 +3,7 @@ from threading import Lock import time import pytest -from core.content.office_preview_cache import cached_office_pdf +from core.files.content.office_preview_cache import cached_office_pdf def test_preview_cache_deduplicates_and_invalidates(tmp_path): @@ -58,7 +58,7 @@ def fail(*args): def test_unexpected_lock_errors_are_not_retried(monkeypatch, tmp_path): import errno import fcntl - from core.content.office_preview_cache import _try_lock + from core.files.content.office_preview_cache import _try_lock def fail(*args): raise OSError(errno.ENOLCK, "no locks available") @@ -104,7 +104,7 @@ def test_preview_deduplicates_across_worker_processes(tmp_path): def test_eviction_keeps_response_file_available(monkeypatch, tmp_path): - import core.content.office_preview_cache as cache + import core.files.content.office_preview_cache as cache monkeypatch.setattr(cache, "CACHE_ENTRIES", 0) source = tmp_path / "source.docx" diff --git a/src/backend/tests/content/test_storage_existence_is_strict.py b/src/backend/tests/content/test_storage_existence_is_strict.py index 2d1d8f47..90cc335a 100644 --- a/src/backend/tests/content/test_storage_existence_is_strict.py +++ b/src/backend/tests/content/test_storage_existence_is_strict.py @@ -2,8 +2,8 @@ import pytest from core.infra.exceptions import StorageError -from core.storage.oss import OSSStorageBackend -from core.storage.s3 import S3StorageBackend +from core.files.storage.oss import OSSStorageBackend +from core.files.storage.s3 import S3StorageBackend class UnavailableBucket: @@ -48,7 +48,7 @@ def test_s3_distinguishes_absence_from_permission_or_server_failure(): s3(code).exists("file") from pathlib import Path -from core.storage.local import LocalStorageBackend +from core.files.storage.local import LocalStorageBackend def test_local_io_failure_is_not_reported_as_a_missing_object(tmp_path, monkeypatch): diff --git a/src/backend/tests/core/runs/test_automation_journal_guard.py b/src/backend/tests/core/runs/test_automation_journal_guard.py new file mode 100644 index 00000000..bd1bec18 --- /dev/null +++ b/src/backend/tests/core/runs/test_automation_journal_guard.py @@ -0,0 +1,89 @@ +"""Automation cancellation and adoption share a short ownership transaction.""" + +import pytest + +from features.automation.runs import owned_run_guard + +from tests.orchestration.test_run_journal import _accept, journal_env + + +def test_automation_cancellation_rejects_an_obsolete_observer(journal_env): + from features.automation.models import ScheduledTaskRun + + journal, sessions, _ = journal_env + child = _accept(journal, "automation-child") + with sessions() as db: + db.add( + ScheduledTaskRun( + run_id="automation-run", task_id="task", status="running", lease_token="new" + ) + ) + db.commit() + assert not journal.cancel( + child.run_id, reason="old observer stopped", cancel_guard=owned_run_guard("automation-run", "old") + ) + assert journal.cancel( + child.run_id, reason="current owner stopped", cancel_guard=owned_run_guard("automation-run", "new") + ) + + +async def test_terminal_worker_cleanup_rejects_false_guard(journal_env, monkeypatch): + import asyncio + from types import SimpleNamespace + + import core.runs.runtime as runtime + + journal, sessions, _ = journal_env + child = _accept(journal, "terminal-cleanup") + assert journal.cancel(child.run_id, reason="already stopped") + monkeypatch.setattr(runtime.engine, "SessionLocal", sessions) + worker = asyncio.create_task(asyncio.Event().wait()) + monkeypatch.setattr( + runtime.tasks, "process_state", + lambda: SimpleNamespace(active_runs={child.run_id: worker}), + ) + projected = [] + + async def project(*args, **kwargs): + projected.append(args) + + monkeypatch.setattr(runtime.writer, "_write_terminal_to_stream", project) + checked = [] + + def reject(db): + checked.append(db) + return False + + try: + assert not await runtime.cancel_run( + child.run_id, user_id=child.user_id, + stop_terminal_worker=True, cancel_guard=reject, + ) + assert len(checked) == 1 + assert not worker.done() + assert worker.cancelling() == 0 + assert projected == [] + finally: + worker.cancel() + await asyncio.gather(worker, return_exceptions=True) + + +@pytest.mark.parametrize("accepted", [False, True]) +async def test_terminal_projection_checks_guard(journal_env, monkeypatch, accepted): + import core.runs.runtime as runtime + + journal, sessions, _ = journal_env + child = _accept(journal, "terminal-projection") + assert journal.cancel(child.run_id, reason="already stopped") + monkeypatch.setattr(runtime.engine, "SessionLocal", sessions) + projected = [] + + async def project(*args, **kwargs): + projected.append(args) + + monkeypatch.setattr(runtime.writer, "_write_terminal_to_stream", project) + assert await runtime.cancel_run( + child.run_id, user_id=child.user_id, + stop_terminal_worker=True, cancel_guard=lambda db: accepted, + ) is accepted + assert len(projected) == int(accepted) diff --git a/src/backend/tests/e2e/_e2e_regression_smoke.py b/src/backend/tests/e2e/_e2e_regression_smoke.py index 18635414..da469a38 100644 --- a/src/backend/tests/e2e/_e2e_regression_smoke.py +++ b/src/backend/tests/e2e/_e2e_regression_smoke.py @@ -4,7 +4,7 @@ from core.db.engine import SessionLocal from core.db.models import ChatMessage, ChatSession, UserShadow -from core.services.chat_service import ChatService +from core.chat.service import ChatService def main() -> None: diff --git a/src/backend/tests/e2e/agentscope_poc.py b/src/backend/tests/e2e/agentscope_poc.py index 3ac65452..e002b4d5 100644 --- a/src/backend/tests/e2e/agentscope_poc.py +++ b/src/backend/tests/e2e/agentscope_poc.py @@ -27,7 +27,7 @@ def _resolve_model_config() -> dict: """Resolve model config from DB or env.""" try: sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) - from core.services.model_config import ModelConfigService + from core.models.config import ModelConfigService resolved = ModelConfigService.get_instance().resolve("main_agent") if resolved: return { diff --git a/src/backend/tests/e2e/core_external_server.py b/src/backend/tests/e2e/core_external_server.py index f3b9d891..8456d8f8 100644 --- a/src/backend/tests/e2e/core_external_server.py +++ b/src/backend/tests/e2e/core_external_server.py @@ -14,8 +14,8 @@ def build_app(): app = build_isolated_app() from core.db.engine import SessionLocal - from core.db.model_repository import create_provider, assign_role - from core.services.model_config import ModelConfigService + from core.models.repository import create_provider, assign_role + from core.models.config import ModelConfigService path = Path(os.environ["CORE_E2E_MODEL_CONFIG"]) assert path.is_relative_to("/tmp"), "Use a private disposable config file" diff --git a/src/backend/tests/e2e/core_refactor_server.py b/src/backend/tests/e2e/core_refactor_server.py index 82c1eafa..a58c7e60 100644 --- a/src/backend/tests/e2e/core_refactor_server.py +++ b/src/backend/tests/e2e/core_refactor_server.py @@ -22,7 +22,7 @@ def build_app(): assert os.environ.get("SESSION_STORE") == "memory" assert os.environ.get("REDIS_URL") == "" assert os.environ.get("AUTH_MODE") == "session" - if os.environ.get("CORE_E2E_PROFILE") == "cloud": + if os.environ.get("CORE_E2E_PROFILE") in ("cloud", "local"): assert not os.environ.get("HUGAGENT_CAPS_ROOT") else: assert os.environ.get("HUGAGENT_CAPS_ROOT") == str(root / "caps") diff --git a/src/backend/tests/e2e/model_tests.py b/src/backend/tests/e2e/model_tests.py index 3e97b3ae..4b79670b 100644 --- a/src/backend/tests/e2e/model_tests.py +++ b/src/backend/tests/e2e/model_tests.py @@ -28,8 +28,8 @@ _v = _v.strip().strip('"').strip("'") os.environ.setdefault(_k.strip(), _v) -from core.llm.chat_models import make_chat_model -from core.llm.message_compat import extract_text_from_chat_response +from core.models.chat_models import make_chat_model +from core.llm.runtime.message_compat import extract_text_from_chat_response # ── Output helpers ─────────────────────────────────────────────────────────── diff --git a/src/backend/tests/evolution/conftest.py b/src/backend/tests/evolution/conftest.py index b4c8256b..d4c53df7 100644 --- a/src/backend/tests/evolution/conftest.py +++ b/src/backend/tests/evolution/conftest.py @@ -9,7 +9,7 @@ import pytest -from core.evolution import readiness as R +from core.evolution.cycle import readiness as R @pytest.fixture(autouse=True) diff --git a/src/backend/tests/evolution/test_benchmark_and_activation.py b/src/backend/tests/evolution/test_benchmark_and_activation.py index 1afe4e72..13d001d1 100644 --- a/src/backend/tests/evolution/test_benchmark_and_activation.py +++ b/src/backend/tests/evolution/test_benchmark_and_activation.py @@ -4,11 +4,13 @@ candidates but nothing reached the runtime, and once it did, transfer was blocked by an over-broad conflict veto. """ +from core.evolution.cycle import ordering as ordering_module +from core.evolution.cycle import promotion_types as promotion_types_module import pytest -from core.evolution import benchmark as B -from core.evolution import promotion as P +from core.evolution.assessment import benchmark as B +from core.evolution.cycle import promotion as P # ── Context-scoped ordering constraints ────────────────────────────────────── @@ -51,14 +53,14 @@ def _rule(constraints, before, after): def test_a_rule_contradicted_elsewhere_is_scoped_rather_than_discarded(): """A global veto threw away rules that were valid where they were observed.""" - rule = _rule(P.extract_ordering_constraints(_episodes()), "company_verify", "db_query") + rule = _rule(ordering_module.extract_ordering_constraints(_episodes()), "company_verify", "db_query") assert rule is not None assert set(rule.validated_in) == {"industry_report", "finance"} assert "trap" in rule.contradicted_in def test_a_scoped_rule_applies_only_where_it_was_validated(): - rule = _rule(P.extract_ordering_constraints(_episodes()), "company_verify", "db_query") + rule = _rule(ordering_module.extract_ordering_constraints(_episodes()), "company_verify", "db_query") tools = ["company_verify", "db_query", "risk_score"] assert rule.applies_to(tools, "finance") is True assert rule.applies_to(["company_verify", "db_query", "doc_export"], "trap") is False @@ -66,7 +68,7 @@ def test_a_scoped_rule_applies_only_where_it_was_validated(): def test_a_rule_known_to_flip_is_not_extrapolated_to_an_unseen_context(): """Guessing is exactly what a rule with a known exception must not do.""" - rule = _rule(P.extract_ordering_constraints(_episodes()), "company_verify", "db_query") + rule = _rule(ordering_module.extract_ordering_constraints(_episodes()), "company_verify", "db_query") assert rule.applies_to(["company_verify", "db_query"], "never_seen_before") is False @@ -79,7 +81,7 @@ def test_an_uncontradicted_rule_does_extend_to_unseen_contexts(): } for i in range(9) ] - rule = _rule(P.extract_ordering_constraints(episodes, min_contexts=1), "a", "b") + rule = _rule(ordering_module.extract_ordering_constraints(episodes, min_contexts=1), "a", "b") assert rule is not None and rule.contradicted_in == () assert rule.applies_to(["a", "b"], "brand_new_type") is True @@ -90,7 +92,7 @@ def test_a_pair_that_flips_within_one_task_type_is_dropped_for_that_type(): [{"verdict": "success", "task_type": "mixed", "tool_sequence": ["a", "b"]}] * 6 + [{"verdict": "success", "task_type": "mixed", "tool_sequence": ["b", "a"]}] * 6 ) - constraints = P.extract_ordering_constraints(episodes, min_contexts=1) + constraints = ordering_module.extract_ordering_constraints(episodes, min_contexts=1) assert _rule(constraints, "a", "b") is None assert _rule(constraints, "b", "a") is None @@ -99,12 +101,12 @@ def test_failed_episodes_are_not_treated_as_evidence(): episodes = [ {"verdict": "failed", "task_type": "t", "tool_sequence": ["a", "b"]} ] * 20 - assert P.extract_ordering_constraints(episodes, min_contexts=1) == [] + assert ordering_module.extract_ordering_constraints(episodes, min_contexts=1) == [] def test_constraints_reorder_only_what_they_constrain(): - rule = P.OrderingConstraint(before="verify", after="query", support=9, contexts=2) - ordered = P.apply_ordering_constraints(["query", "chart", "verify", "export"], [rule]) + rule = promotion_types_module.OrderingConstraint(before="verify", after="query", support=9, contexts=2) + ordered = ordering_module.apply_ordering_constraints(["query", "chart", "verify", "export"], [rule]) assert ordered.index("verify") < ordered.index("query") # Unconstrained pairs keep their relative order — applying one rule must not # silently reshuffle everything else. @@ -148,7 +150,7 @@ def test_rules_earn_credit_only_by_correcting_a_wrong_plan(): """Applying rules to the declared tool list flattered them: that list was already correctly ordered, so a rule scored a pass without doing work.""" tasks = [t for t in B.build_task_set() if t.family == "finance"] - unhelpful = P.OrderingConstraint( + unhelpful = promotion_types_module.OrderingConstraint( before="risk_score", after="nonexistent", support=9, contexts=2 ) results = B.run_arm(tasks, {}, ordering_rules=[unhelpful]) diff --git a/src/backend/tests/evolution/test_candidate_reasoning.py b/src/backend/tests/evolution/test_candidate_reasoning.py index afb173cb..283113cc 100644 --- a/src/backend/tests/evolution/test_candidate_reasoning.py +++ b/src/backend/tests/evolution/test_candidate_reasoning.py @@ -13,11 +13,13 @@ * every cycle minted a fresh near-duplicate of the same proposal, because the hashed payload carried the evidence count. """ +from core.evolution.profiles import agent_profile as agent_profile_module +from core.evolution.profiles import agent_profile_constants as agent_profile_state_module import pytest -from core.evolution import credit as C -from core.evolution.console_view import candidate_to_dict +from core.evolution.evidence import credit as C +from core.evolution.profiles.console_view import candidate_to_dict class _Row: @@ -163,7 +165,7 @@ def test_the_catch_all_task_type_is_never_narrowed(): dev corpus: a proposal to remove database access, report export, site publishing and scheduled tasks from every conversation. """ - from core.evolution.orchestration_gen import converge_tools + from core.evolution.generators.orchestration_gen import converge_tools offered = [f"tool{i}" for i in range(12)] proposals = converge_tools( @@ -173,7 +175,7 @@ def test_the_catch_all_task_type_is_never_narrowed(): def test_a_thin_corpus_cannot_remove_a_capability(): - from core.evolution.orchestration_gen import converge_tools + from core.evolution.generators.orchestration_gen import converge_tools offered = [f"tool{i}" for i in range(12)] assert ( @@ -185,7 +187,7 @@ def test_a_thin_corpus_cannot_remove_a_capability(): def test_a_cut_this_large_is_reported_but_not_proposed(): """Removing most of the toolbox is a different product, not a tuning step.""" - from core.evolution.orchestration_gen import converge_tools + from core.evolution.generators.orchestration_gen import converge_tools offered = [f"tool{i}" for i in range(12)] # 400 episodes, only 3 tools ever used → 9 of 12 unused, far past the cap. @@ -197,7 +199,7 @@ def test_a_cut_this_large_is_reported_but_not_proposed(): def test_a_modest_well_evidenced_narrowing_is_proposed(): - from core.evolution.orchestration_gen import converge_tools + from core.evolution.generators.orchestration_gen import converge_tools offered = [f"tool{i}" for i in range(12)] used = [f"tool{i}" for i in range(9)] # 3 of 12 unused — within the cap @@ -223,23 +225,23 @@ def test_the_attribution_target_names_the_assembly_not_a_dag(): def test_react_observable_signals_are_declared_separately(): """A rule that can never fire looks the same, in the console, as one that simply has not fired yet.""" - from core.evolution import agent_profile as AP + from core.evolution.profiles import agent_profile as AP - assert AP.SIGNAL_REPEATED_ACTIONS in AP.REACT_OBSERVABLE_SIGNALS - assert AP.SIGNAL_TOOL_ERROR_STREAK in AP.REACT_OBSERVABLE_SIGNALS + assert agent_profile_state_module.SIGNAL_REPEATED_ACTIONS in agent_profile_state_module.REACT_OBSERVABLE_SIGNALS + assert agent_profile_state_module.SIGNAL_TOOL_ERROR_STREAK in agent_profile_state_module.REACT_OBSERVABLE_SIGNALS # Needs a file tree / a reviewer verdict: a single answer has neither. - assert AP.SIGNAL_NO_DIFF in AP.LOOP_ONLY_SIGNALS - assert AP.SIGNAL_REVIEWER_FLAT in AP.LOOP_ONLY_SIGNALS - assert not (AP.REACT_OBSERVABLE_SIGNALS & AP.LOOP_ONLY_SIGNALS) + assert agent_profile_state_module.SIGNAL_NO_DIFF in agent_profile_state_module.LOOP_ONLY_SIGNALS + assert agent_profile_state_module.SIGNAL_REVIEWER_FLAT in agent_profile_state_module.LOOP_ONLY_SIGNALS + assert not (agent_profile_state_module.REACT_OBSERVABLE_SIGNALS & agent_profile_state_module.LOOP_ONLY_SIGNALS) def test_the_builtin_profile_only_carries_rules_the_react_loop_can_act_on(): """Shipping a default rule the main axis cannot observe would mean the built-in profile advertises an intervention that never happens.""" - from core.evolution import agent_profile as AP + from core.evolution.profiles import agent_profile as AP - builtin = AP.builtin_profile() + builtin = agent_profile_module.builtin_profile() react_rules = [ - r for r in builtin.intervention_rules if r.signal in AP.REACT_OBSERVABLE_SIGNALS + r for r in builtin.intervention_rules if r.signal in agent_profile_state_module.REACT_OBSERVABLE_SIGNALS ] assert react_rules, "the default profile must be able to intervene on the main axis" diff --git a/src/backend/tests/evolution/test_cassette_and_replay.py b/src/backend/tests/evolution/test_cassette_and_replay.py index 7fe4dfe5..518af222 100644 --- a/src/backend/tests/evolution/test_cassette_and_replay.py +++ b/src/backend/tests/evolution/test_cassette_and_replay.py @@ -8,8 +8,8 @@ import pytest -from core.evolution import cassette as CA -from core.evolution import replay as RP +from core.evolution.evidence import cassette as CA +from core.evolution.evidence import replay as RP # ── Cassette: side-effect safety ───────────────────────────────────────────── diff --git a/src/backend/tests/evolution/test_compiler_v2.py b/src/backend/tests/evolution/test_compiler_v2.py index 75c5bf60..36ec7ced 100644 --- a/src/backend/tests/evolution/test_compiler_v2.py +++ b/src/backend/tests/evolution/test_compiler_v2.py @@ -4,6 +4,10 @@ 共享技能不含 L1 原文；新技能拿不到证据外的工具授权；L3 冲突不能自动激活； 激活技能可沿血缘反查 L2/L3/Episode；回放集覆盖不足不得晋级。 """ +from core.memory import graph as graph_module +from core.evolution.cycle import activation as activation_module +from core.evolution.cycle import activation_types as activation_types_module +from core.evolution.generators import skill_gen as skill_gen_module from datetime import datetime, timedelta, timezone @@ -24,21 +28,12 @@ EvolutionRelease, EvolutionTraceEvent, ) -from core.evolution import evidence_contract as EC -from core.evolution import graph_scan as GS -from core.evolution import lineage as LN -from core.evolution.applicability import ( - assess_replay_coverage, - load_manifest, - manifest_permits, -) -from core.evolution.evidence_resolver import resolve_capability_pack -from core.evolution.skill_candidate_engine import ( - STATUS_CREATED, - STATUS_REJECTED, - SkillCandidateSpec, - create_skill_candidate, -) +from core.evolution.evidence import evidence_contract as EC +from core.evolution.generators import graph_scan as GS +from core.evolution.evidence import lineage as LN +from core.evolution.assessment.applicability import assess_replay_coverage, load_manifest, manifest_permits +from core.evolution.evidence.evidence_resolver import resolve_capability_pack +from core.evolution.generators.skill_candidate_engine import STATUS_CREATED, STATUS_REJECTED, SkillCandidateSpec, create_skill_candidate _TABLES = [ EvolutionEpisode.__table__, @@ -60,14 +55,16 @@ def db(monkeypatch): Base.metadata.create_all(engine, tables=_TABLES) factory = sessionmaker(bind=engine) import core.db.engine as engine_mod - import core.evolution.activation as A - import core.evolution.loop as L + import core.evolution.cycle.activation as A + import core.evolution.cycle.loop as L import core.memory.weights as weights_mod monkeypatch.setattr(engine_mod, "SessionLocal", factory) monkeypatch.setattr(A, "SessionLocal", factory) - monkeypatch.setattr(L, "SessionLocal", factory) - monkeypatch.setattr(weights_mod, "_cache", {}) + from core.evolution.cycle import episode_views, candidate_records, evaluation, skill_release, profile_release + for owner in (episode_views, candidate_records, evaluation, skill_release, profile_release): + monkeypatch.setattr(owner, "SessionLocal", factory) + monkeypatch.setattr(weights_mod.process_state(), 'cache', {}) session = factory() yield session session.close() @@ -275,7 +272,7 @@ def test_l2_l3_contradiction_refuses_generation(): async def _fake_generate(**kwargs): - from core.evolution.skill_gen import DraftResult + from core.evolution.generators.skill_evidence import DraftResult return ( { @@ -311,9 +308,9 @@ def _spec(**overrides): def test_engine_persists_candidate_pack_credit_and_manifest(db, monkeypatch): - import core.evolution.skill_gen as skill_gen + import core.evolution.generators.skill_gen as skill_gen - monkeypatch.setattr(skill_gen, "generate_skill_document", _fake_generate) + monkeypatch.setattr(skill_gen_module, 'generate_skill_document', _fake_generate) outcome = create_skill_candidate( _spec(), credit={"selected": "skill", "confidence": 0.8, @@ -334,9 +331,9 @@ def test_engine_persists_candidate_pack_credit_and_manifest(db, monkeypatch): def test_engine_is_idempotent_for_identical_evidence(db, monkeypatch): - import core.evolution.skill_gen as skill_gen + import core.evolution.generators.skill_gen as skill_gen - monkeypatch.setattr(skill_gen, "generate_skill_document", _fake_generate) + monkeypatch.setattr(skill_gen_module, 'generate_skill_document', _fake_generate) credit = {"selected": "skill", "confidence": 0.8, "may_produce_candidate": True} first = create_skill_candidate(_spec(), credit=credit, run_async=_run) @@ -348,7 +345,7 @@ def test_engine_is_idempotent_for_identical_evidence(db, monkeypatch): def test_engine_refuses_a_document_that_widens_tool_authority(db, monkeypatch): async def widened(**kwargs): - from core.evolution.skill_gen import DraftResult + from core.evolution.generators.skill_evidence import DraftResult return ( { @@ -361,9 +358,9 @@ async def widened(**kwargs): DraftResult(ok=True, attempts=1), ) - import core.evolution.skill_gen as skill_gen + import core.evolution.generators.skill_gen as skill_gen - monkeypatch.setattr(skill_gen, "generate_skill_document", widened) + monkeypatch.setattr(skill_gen_module, 'generate_skill_document', widened) outcome = create_skill_candidate( _spec(), credit={"selected": "skill", "confidence": 0.8, "may_produce_candidate": True}, run_async=_run @@ -378,9 +375,9 @@ async def widened(**kwargs): def _materialisable_candidate(db, monkeypatch, *, graph_conflict=False, risk="medium", status="replay_passed"): - import core.evolution.skill_gen as skill_gen + import core.evolution.generators.skill_gen as skill_gen - monkeypatch.setattr(skill_gen, "generate_skill_document", _fake_generate) + monkeypatch.setattr(skill_gen_module, 'generate_skill_document', _fake_generate) views = [_view(f"e{i}", relations=[_REL]) for i in range(3)] if graph_conflict: conflicting = dict(_REL, relation_id="grel_dep2", content_hash="hash-dep2") @@ -408,10 +405,10 @@ def _materialisable_candidate(db, monkeypatch, *, graph_conflict=False, risk="me def test_materialisation_writes_the_manifest_and_the_lineage(db, monkeypatch): - from core.evolution import activation as A + from core.evolution.cycle import activation as A candidate_id = _materialisable_candidate(db, monkeypatch) - result = A.materialise_skill_candidate(candidate_id, approver="admin") + result = activation_module.materialise_skill_candidate(candidate_id, approver="admin") row = db.query(AdminSkill).one() manifest = load_manifest(row.extra_files) @@ -431,23 +428,23 @@ def test_materialisation_writes_the_manifest_and_the_lineage(db, monkeypatch): def test_a_graph_conflicted_candidate_cannot_activate_directly(db, monkeypatch): - from core.evolution import activation as A + from core.evolution.cycle import activation as A candidate_id = _materialisable_candidate( db, monkeypatch, graph_conflict=True, risk="low" ) - with pytest.raises(A.ActivationError) as excinfo: - A.materialise_skill_candidate(candidate_id, approver="admin") + with pytest.raises(activation_types_module.ActivationError) as excinfo: + activation_module.materialise_skill_candidate(candidate_id, approver="admin") assert excinfo.value.code == "graph_conflict_requires_review" def test_a_graph_conflicted_candidate_may_still_enter_shadow(db, monkeypatch): - from core.evolution import activation as A + from core.evolution.cycle import activation as A candidate_id = _materialisable_candidate( db, monkeypatch, graph_conflict=True, risk="medium" ) - result = A.materialise_skill_candidate(candidate_id, approver="admin") + result = activation_module.materialise_skill_candidate(candidate_id, approver="admin") assert result["stage"] == "shadow" assert result["exposed_to"] == "nobody (shadow)" @@ -517,7 +514,7 @@ def test_an_inactive_graph_dependency_withholds_the_skill(): def test_exposure_withholds_an_evolved_skill_outside_its_manifest(db): - from core.evolution.exposure import filter_skill_ids + from core.evolution.evidence.exposure import filter_skill_ids db.add( AdminSkill( @@ -633,7 +630,7 @@ def fake_update(relation_id, *, scope_user_id, weight=None, active=None): import core.memory.graph as G - monkeypatch.setattr(G, "update_relation_state", fake_update) + monkeypatch.setattr(graph_module, 'update_relation_state', fake_update) ops = [ GS.GraphOp(operation=GS.OP_REINFORCE, relation_id="r1", payload={"weight": 0.9}), GS.GraphOp(operation=GS.OP_FLAG_CONTRADICTION, relation_id="r2"), diff --git a/src/backend/tests/evolution/test_console_and_metrics.py b/src/backend/tests/evolution/test_console_and_metrics.py index 7abe3bab..0d1ac794 100644 --- a/src/backend/tests/evolution/test_console_and_metrics.py +++ b/src/backend/tests/evolution/test_console_and_metrics.py @@ -18,7 +18,7 @@ EvolutionRelease, EvolutionTraceEvent, ) -from core.evolution import metrics as M +from core.evolution.assessment import metrics as M @pytest.fixture @@ -140,7 +140,7 @@ def test_metrics_are_none_not_zero_when_a_dimension_has_no_data(db): def test_replay_eligibility_is_surfaced_for_the_console(db): - from core.evolution.console_view import episode_to_dict as _episode_to_dict + from core.evolution.profiles.console_view import episode_to_dict as _episode_to_dict good = _episode(db, index=1, days_ago=1) good.asset_bundle_id = "bundle_x" @@ -156,7 +156,7 @@ def test_replay_eligibility_is_surfaced_for_the_console(db): def test_candidate_summary_leads_with_why_not_with_the_ir(db): - from core.evolution.console_view import candidate_to_dict as _candidate_to_dict + from core.evolution.profiles.console_view import candidate_to_dict as _candidate_to_dict row = EvolutionCandidate( candidate_id="c1", @@ -183,7 +183,7 @@ def test_candidate_summary_leads_with_why_not_with_the_ir(db): def test_release_row_exposes_its_rollback_target(db): - from core.evolution.console_view import release_to_dict as _release_to_dict + from core.evolution.profiles.console_view import release_to_dict as _release_to_dict row = EvolutionRelease( release_id="r1", diff --git a/src/backend/tests/evolution/test_credit_assigner.py b/src/backend/tests/evolution/test_credit_assigner.py index db41ae78..80296d7c 100644 --- a/src/backend/tests/evolution/test_credit_assigner.py +++ b/src/backend/tests/evolution/test_credit_assigner.py @@ -9,7 +9,7 @@ import pytest -from core.evolution import credit as C +from core.evolution.evidence import credit as C def F(**kw) -> C.CreditFeatures: diff --git a/src/backend/tests/evolution/test_engines_end_to_end.py b/src/backend/tests/evolution/test_engines_end_to_end.py index 8ad47c7e..460cee7c 100644 --- a/src/backend/tests/evolution/test_engines_end_to_end.py +++ b/src/backend/tests/evolution/test_engines_end_to_end.py @@ -15,6 +15,9 @@ that governs the next run. Anything that cannot be observed at that boundary did not happen, whatever the intermediate values looked like. """ +from core.evolution.cycle import episode_views as episode_views_module, graph_candidates as graph_candidates_module, skill_release as skill_release_module, cycle_extras as X, loop as L +from core.evolution.generators import skill_gen as skill_gen_module, memory_apply as MA +from core.evolution.profiles import agent_profile as agent_profile_module, profile_selection as profile_selection_module, agent_profile as AP import pytest from sqlalchemy import create_engine @@ -29,13 +32,9 @@ EvolutionRelease, EvolutionTraceEvent, ) -from core.evolution import agent_profile as AP -from core.evolution import cycle_extras as X -from core.evolution import loop as L -from core.evolution import memory_apply as MA -from core.evolution import similarity as S -from core.evolution import trace_assembler as TA -from core.evolution.memory_ops import OP_MERGE, OP_REWEIGHT, MemoryOp +from core.evolution.assessment import similarity as S +from core.evolution.evidence import trace_assembler as TA +from core.evolution.generators.memory_ops import OP_MERGE, OP_REWEIGHT, MemoryOp _TABLES = [ EvolutionEpisode.__table__, @@ -52,14 +51,15 @@ def db(monkeypatch): engine = create_engine("sqlite:///:memory:", connect_args={"check_same_thread": False}) Base.metadata.create_all(engine, tables=_TABLES) factory = sessionmaker(bind=engine) - for module in (L, MA, AP, X): - if hasattr(module, "SessionLocal"): - monkeypatch.setattr(module, "SessionLocal", factory) + from core.evolution.cycle import episode_views, candidate_records, evaluation + from core.evolution.profiles import profile_selection + for module in (episode_views, candidate_records, evaluation, MA, AP, X, profile_selection): + if hasattr(module, "SessionLocal"): monkeypatch.setattr(module, "SessionLocal", factory) import core.db.engine as engine_mod import core.memory.weights as weights_mod monkeypatch.setattr(engine_mod, "SessionLocal", factory) - monkeypatch.setattr(weights_mod, "_cache", {}) + monkeypatch.setattr(weights_mod.process_state(), 'cache', {}) session = factory() yield session session.close() @@ -117,7 +117,7 @@ def test_a_recurring_intent_becomes_a_row_in_the_candidate_table(db, monkeypatch # The generator is the one part that talks to a model; its output is fixed # here so the test measures the wiring rather than the model. async def fake_generate(**kwargs): - from core.evolution.skill_gen import DraftResult + from core.evolution.generators.skill_evidence import DraftResult return ( { @@ -130,14 +130,14 @@ async def fake_generate(**kwargs): DraftResult(ok=True, attempts=1), ) - import core.evolution.skill_gen as skill_gen + import core.evolution.generators.skill_gen as skill_gen - monkeypatch.setattr(skill_gen, "generate_skill_document", fake_generate) + monkeypatch.setattr(skill_gen_module, 'generate_skill_document', fake_generate) - views = L.load_episode_views(limit=50) + views = episode_views_module.load_episode_views(limit=50) assert len(views) == 4 - result = L._run_other_engines(views, dry_run=False) + result = graph_candidates_module.run_other_engines(views, dry_run=False) rows = db.query(EvolutionCandidate).filter( EvolutionCandidate.proposer == "intent_engine" @@ -181,15 +181,15 @@ def test_a_skill_that_cannot_be_written_produces_no_candidate(db, monkeypatch): monkeypatch.setattr(S, "_embed_all", lambda texts: None) async def fails(**_kwargs): - from core.evolution.skill_gen import DraftResult + from core.evolution.generators.skill_evidence import DraftResult return None, DraftResult(ok=False, violations=["contains_code_block"], attempts=2) - import core.evolution.skill_gen as skill_gen + import core.evolution.generators.skill_gen as skill_gen - monkeypatch.setattr(skill_gen, "generate_skill_document", fails) + monkeypatch.setattr(skill_gen_module, 'generate_skill_document', fails) - L._run_other_engines(L.load_episode_views(limit=50), dry_run=False) + graph_candidates_module.run_other_engines(episode_views_module.load_episode_views(limit=50), dry_run=False) assert db.query(EvolutionCandidate).count() == 0 @@ -320,11 +320,10 @@ def _rollback_fixture(db, monkeypatch, *, guardrails): from sqlalchemy.orm import sessionmaker from core.db.models import AdminSkill - from core.evolution import activation as A AdminSkill.__table__.create(db.get_bind(), checkfirst=True) factory = sessionmaker(bind=db.get_bind()) - monkeypatch.setattr(A, "SessionLocal", factory) + monkeypatch.setattr(skill_release_module, "SessionLocal", factory) db.add( EvolutionCandidate( @@ -373,7 +372,7 @@ def _rollback_fixture(db, monkeypatch, *, guardrails): MA.demote_promoted_memories( candidate_id="cand-9", user_id="u1", memory_refs=["mref-a"], skill_id="evo-fin" ) - return A + return skill_release_module def test_release_rollback_restores_demoted_memories_in_the_same_motion(db, monkeypatch): @@ -391,7 +390,7 @@ def test_release_rollback_restores_demoted_memories_in_the_same_motion(db, monke weights, _ = load_overlay("u1", "default") assert weights == {"mref-a": MA.PROMOTION_WEIGHT} - outcome = A.rollback_skill_activation("rel-9", actor="admin", reason="canary 指标崩了") + outcome = skill_release_module.rollback_skill_activation("rel-9", actor="admin", reason="canary 指标崩了") invalidate() weights, _ = load_overlay("u1", "default") @@ -422,7 +421,7 @@ def test_release_rollback_to_a_prior_version_keeps_dependent_profiles(db, monkey } A = _rollback_fixture(db, monkeypatch, guardrails={"previous": previous}) - outcome = A.rollback_skill_activation("rel-9", actor="admin", reason="人工回滚") + outcome = skill_release_module.rollback_skill_activation("rel-9", actor="admin", reason="人工回滚") invalidate() weights, _ = load_overlay("u1", "default") @@ -444,7 +443,7 @@ def test_a_published_profile_governs_the_next_run_without_a_restart(db): Loop thresholds were read into module constants at import, so publishing required a restart and replicas disagreed until all of them had. """ - profile = AP.builtin_profile() + profile = agent_profile_module.builtin_profile() profile.profile_id = "auto-finance" profile.task_types = ["finance"] profile.max_react_turns = 12 @@ -460,13 +459,13 @@ def test_a_published_profile_governs_the_next_run_without_a_restart(db): ) db.commit() - assert AP.load_active_profile(task_type="finance").max_react_turns == 12 + assert profile_selection_module.load_active_profile(task_type="finance").max_react_turns == 12 # A task type this profile does not claim keeps the built-in assembly. - assert AP.load_active_profile(task_type="chat").profile_id == "builtin" + assert profile_selection_module.load_active_profile(task_type="chat").profile_id == "builtin" def test_one_tenants_profile_never_governs_anothers_run(db): - profile = AP.builtin_profile() + profile = agent_profile_module.builtin_profile() profile.profile_id = "acme-finance" profile.task_types = ["finance"] profile.scope = {"tenant_id": "acme"} @@ -483,14 +482,14 @@ def test_one_tenants_profile_never_governs_anothers_run(db): ) db.commit() - assert AP.load_active_profile(task_type="finance", tenant_id="acme").max_react_turns == 7 + assert profile_selection_module.load_active_profile(task_type="finance", tenant_id="acme").max_react_turns == 7 # The bug the tenant parameter was added for, asserted at the boundary that # matters rather than on the helper that picks. - assert AP.load_active_profile(task_type="finance", tenant_id="other").profile_id == "builtin" + assert profile_selection_module.load_active_profile(task_type="finance", tenant_id="other").profile_id == "builtin" def test_a_deactivated_profile_stops_governing_immediately(db): - profile = AP.builtin_profile() + profile = agent_profile_module.builtin_profile() profile.profile_id = "auto-finance" profile.task_types = ["finance"] profile.max_react_turns = 12 @@ -504,16 +503,16 @@ def test_a_deactivated_profile_stops_governing_immediately(db): ) db.add(row) db.commit() - assert AP.load_active_profile(task_type="finance").profile_id == "auto-finance" + assert profile_selection_module.load_active_profile(task_type="finance").profile_id == "auto-finance" row.is_active = False db.commit() - assert AP.load_active_profile(task_type="finance").profile_id == "builtin" + assert profile_selection_module.load_active_profile(task_type="finance").profile_id == "builtin" def test_a_stored_profile_that_no_longer_validates_is_not_applied(db): """Applying it partially would produce a configuration nobody reviewed.""" - broken = AP.builtin_profile() + broken = agent_profile_module.builtin_profile() broken.profile_id = "broken" broken.budget_multiplier = 99.0 db.add( @@ -527,7 +526,7 @@ def test_a_stored_profile_that_no_longer_validates_is_not_applied(db): ) ) db.commit() - assert AP.load_active_profile(task_type="chat").profile_id == "builtin" + assert profile_selection_module.load_active_profile(task_type="chat").profile_id == "builtin" # ── Skill opens are recoverable from history ───────────────────────────────── diff --git a/src/backend/tests/evolution/test_episode_assembler.py b/src/backend/tests/evolution/test_episode_assembler.py index 946709ee..84909056 100644 --- a/src/backend/tests/evolution/test_episode_assembler.py +++ b/src/backend/tests/evolution/test_episode_assembler.py @@ -4,19 +4,23 @@ assembly retries, evidence collection can never fail the turn, and an Episode without a full asset snapshot is barred from replay. """ +from core.agent_runtime.integrations import memory as memory_module +from core.evolution.profiles import user_settings as user_settings_module -import pytest from types import SimpleNamespace -from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker +import core.agent_runtime.integrations.evolution as evolution +import core.agent_runtime.integrations.memory as memory +import pytest from core.db.engine import Base from core.db.models.evolution import EvolutionEpisode, EvolutionTraceEvent -from core.evolution import events as EV -from core.evolution import trace_assembler as TA -from core.evolution import trace_store as TS +from core.evolution.evidence import events as EV +from core.evolution.evidence import trace_assembler as TA +from core.evolution.evidence import trace_store as TS from core.evolution.contract import ASSET_SKILL, AssetRef, build_bundle -from core.llm.execution_manifest import PromptManifestBuilder +from core.harness.execution_manifest import PromptManifestBuilder +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker @pytest.fixture @@ -124,8 +128,10 @@ def test_assembly_creates_one_episode(db_factory): def test_streaming_wrapper_persists_sanitized_execution_manifest(db_factory, monkeypatch): - from core.evolution import settlement_runner, trigger, user_settings - from orchestration import memory_integration, workflow + import core.agent_runtime.integrations.memory as memory_integration + from core.evolution.settlement import settlement_runner + from core.evolution.cycle import trigger + from core.evolution.profiles import user_settings builder = PromptManifestBuilder( context={ @@ -152,17 +158,17 @@ def test_streaming_wrapper_persists_sanitized_execution_manifest(db_factory, mon streaming_wrapper = SimpleNamespace(agent=raw_agent) monkeypatch.setattr(TA, "emit_tool_events", lambda *_args, **_kwargs: None) - monkeypatch.setattr(memory_integration, "get_last_retrieval", lambda _task: None) + monkeypatch.setattr(memory_module, 'get_last_retrieval', lambda _task: None) monkeypatch.setattr( - user_settings, - "resolve_for_user", + user_settings_module, + 'resolve_for_user', lambda _user_id: {"privacy_class": "private"}, ) monkeypatch.setattr(settlement_runner, "register_turn", lambda **_kwargs: None) monkeypatch.setattr(trigger, "schedule_personal", lambda _user_id: None) monkeypatch.setattr(trigger, "schedule_if_due", lambda: None) - workflow._assemble_episode_background( + evolution._assemble_episode_background( message_id="manifest-message-70", run_id="manifest-run-70", chat_id="manifest-chat-70", diff --git a/src/backend/tests/evolution/test_experiments_and_federation.py b/src/backend/tests/evolution/test_experiments_and_federation.py index 451e0cdd..3d62b459 100644 --- a/src/backend/tests/evolution/test_experiments_and_federation.py +++ b/src/backend/tests/evolution/test_experiments_and_federation.py @@ -2,8 +2,8 @@ import pytest -from core.evolution import experiments as EX -from core.evolution import federation as FD +from core.evolution.assessment import experiments as EX +from core.evolution.profiles import federation as FD # ── System matrix (41) ─────────────────────────────────────────────────────── diff --git a/src/backend/tests/evolution/test_exposure_and_ramp.py b/src/backend/tests/evolution/test_exposure_and_ramp.py index 482df957..ba31b5c1 100644 --- a/src/backend/tests/evolution/test_exposure_and_ramp.py +++ b/src/backend/tests/evolution/test_exposure_and_ramp.py @@ -12,9 +12,10 @@ import pytest -from core.evolution import release as R -from core.evolution.activation import _entry_stage, ActivationError -from core.evolution.exposure import is_evolved_skill_id, visible_evolved_skill_ids +from core.evolution.cycle import release as R +from core.evolution.cycle.activation_policy import entry_stage as _entry_stage +from core.evolution.cycle.activation_types import ActivationError +from core.evolution.evidence.exposure import is_evolved_skill_id, visible_evolved_skill_ids class _Candidate: @@ -156,7 +157,7 @@ def test_an_evolved_skill_with_no_release_is_withheld(): def test_hand_authored_skills_are_never_filtered(): - from core.evolution.exposure import filter_skill_ids + from core.evolution.evidence.exposure import filter_skill_ids ids = ["officecli-docx", "officecli-pptx"] assert filter_skill_ids(ids, user_id="u1") == ids diff --git a/src/backend/tests/evolution/test_fault_injection_gate.py b/src/backend/tests/evolution/test_fault_injection_gate.py index 26360a6d..c12ce29d 100644 --- a/src/backend/tests/evolution/test_fault_injection_gate.py +++ b/src/backend/tests/evolution/test_fault_injection_gate.py @@ -8,8 +8,8 @@ import pytest -from core.evolution import fault_injection as FI -from core.evolution.credit import T_NO_UPDATE, T_SKILL, CreditDecision +from core.evolution.assessment import fault_injection as FI +from core.evolution.evidence.credit import T_NO_UPDATE, T_SKILL, CreditDecision pytestmark = pytest.mark.slow diff --git a/src/backend/tests/evolution/test_mechanisms.py b/src/backend/tests/evolution/test_mechanisms.py index 86f3d0a9..49647411 100644 --- a/src/backend/tests/evolution/test_mechanisms.py +++ b/src/backend/tests/evolution/test_mechanisms.py @@ -4,9 +4,9 @@ import pytest -from core.evolution import counterfactual as CF -from core.evolution import evaluator as EV -from core.evolution import timescales as TS +from core.evolution.evidence import counterfactual as CF +from core.evolution.assessment import evaluator as EV +from core.evolution.cycle import timescales as TS # ── Counterfactual attribution (28) ────────────────────────────────────────── diff --git a/src/backend/tests/evolution/test_ontology_bridge.py b/src/backend/tests/evolution/test_ontology_bridge.py index 7d6765a1..40f06556 100644 --- a/src/backend/tests/evolution/test_ontology_bridge.py +++ b/src/backend/tests/evolution/test_ontology_bridge.py @@ -7,7 +7,7 @@ import pytest -from core.evolution import ontology_bridge as OB +from core.evolution.generators import ontology_bridge as OB def _candidate(**kw): diff --git a/src/backend/tests/evolution/test_outcomes_and_backfill.py b/src/backend/tests/evolution/test_outcomes_and_backfill.py index 4487cd08..d73d4d0c 100644 --- a/src/backend/tests/evolution/test_outcomes_and_backfill.py +++ b/src/backend/tests/evolution/test_outcomes_and_backfill.py @@ -2,7 +2,7 @@ import pytest -from core.evolution import outcomes as O +from core.evolution.evidence import outcomes as O # ── No single judge decides ────────────────────────────────────────────────── @@ -119,8 +119,8 @@ def test_backfilled_episodes_are_marked_and_barred_from_replay(monkeypatch): from core.db.engine import Base from core.db.models.evolution import EvolutionEpisode, EvolutionTraceEvent - from core.evolution import trace_assembler as TA - from core.evolution import trace_store as TS + from core.evolution.evidence import trace_assembler as TA + from core.evolution.evidence import trace_store as TS engine = create_engine("sqlite:///:memory:", connect_args={"check_same_thread": False}) Base.metadata.create_all( diff --git a/src/backend/tests/evolution/test_personal_approval.py b/src/backend/tests/evolution/test_personal_approval.py index 18aadb33..7ff200cd 100644 --- a/src/backend/tests/evolution/test_personal_approval.py +++ b/src/backend/tests/evolution/test_personal_approval.py @@ -6,6 +6,9 @@ with an administrator, because one user approving for the fleet is a governance hole regardless of how convenient it is. """ +from core.evolution.cycle import activation as activation_module +from core.evolution.profiles import personal_review as personal_review_module +from core.evolution.profiles import user_settings_constants as user_settings_state_module import pytest from sqlalchemy import create_engine @@ -17,7 +20,7 @@ EvolutionEpisode, EvolutionRelease, ) -from core.evolution import user_settings as US +from core.evolution.profiles import user_settings as US @pytest.fixture @@ -33,6 +36,8 @@ def db(monkeypatch): ) factory = sessionmaker(bind=engine) monkeypatch.setattr(US, "SessionLocal", factory) + from core.evolution.profiles import personal_review + monkeypatch.setattr(personal_review, "SessionLocal", factory) session = factory() yield session session.close() @@ -86,7 +91,7 @@ def test_a_candidate_from_my_own_evidence_is_mine_to_approve(db): _candidate(db, "c1", [f"e{i}" for i in range(5)]) db.commit() - pending = US.pending_for_user("alice") + pending = personal_review_module.pending_for_user("alice") assert [c["candidate_id"] for c in pending] == ["c1"] assert pending[0]["own_share"] == 1.0 @@ -100,7 +105,7 @@ def test_a_candidate_mostly_from_others_is_not_mine_to_approve(db): # 10% of the evidence is Alice's; approving it would activate a capability # built from Bob's work. - assert US.pending_for_user("alice") == [] + assert personal_review_module.pending_for_user("alice") == [] def test_the_threshold_is_explicit_not_incidental(db): @@ -113,9 +118,9 @@ def test_the_threshold_is_explicit_not_incidental(db): ) db.commit() - pending = US.pending_for_user("alice") + pending = personal_review_module.pending_for_user("alice") assert pending and pending[0]["own_share"] == pytest.approx(0.6) - assert US.OWN_EVIDENCE_THRESHOLD == 0.6 + assert user_settings_state_module.OWN_EVIDENCE_THRESHOLD == 0.6 def test_another_users_candidate_never_appears(db): @@ -123,7 +128,7 @@ def test_another_users_candidate_never_appears(db): _episode(db, f"e{i}", "bob") _candidate(db, "c1", [f"e{i}" for i in range(5)]) db.commit() - assert US.pending_for_user("alice") == [] + assert personal_review_module.pending_for_user("alice") == [] def test_already_released_candidates_are_not_pending(db): @@ -131,20 +136,20 @@ def test_already_released_candidates_are_not_pending(db): _episode(db, f"e{i}", "alice") _candidate(db, "c1", [f"e{i}" for i in range(5)], status="active") db.commit() - assert US.pending_for_user("alice") == [] + assert personal_review_module.pending_for_user("alice") == [] def test_a_user_with_no_episodes_sees_nothing(db): _candidate(db, "c1", ["someone-else"]) db.commit() - assert US.pending_for_user("nobody") == [] + assert personal_review_module.pending_for_user("nobody") == [] def test_candidates_without_evidence_are_skipped(db): _episode(db, "e0", "alice") _candidate(db, "c1", []) db.commit() - assert US.pending_for_user("alice") == [] + assert personal_review_module.pending_for_user("alice") == [] # ── Approval enforcement ───────────────────────────────────────────────────── @@ -158,7 +163,7 @@ def test_approving_someone_elses_candidate_is_refused(db): db.commit() with pytest.raises(PermissionError) as exc: - US.approve_for_user("c1", "alice") + personal_review_module.approve_for_user("c1", "alice") # The refusal explains the scope rule rather than just denying. assert "管理员" in str(exc.value) @@ -178,11 +183,11 @@ def fake_materialise(candidate_id, *, approver, force=False, owner_user_id=None) ) return {"skill_id": "evo-x", "tools": ["a", "b"]} - import core.evolution.activation as A + import core.evolution.cycle.activation as A - monkeypatch.setattr(A, "materialise_skill_candidate", fake_materialise) + monkeypatch.setattr(activation_module, 'materialise_skill_candidate', fake_materialise) - result = US.approve_for_user("c1", "alice") + result = personal_review_module.approve_for_user("c1", "alice") # Installed privately, so no other user's agent changes. assert called["owner"] == "alice" assert called["approver"] == "user:alice" @@ -194,7 +199,7 @@ def test_the_tool_sequence_is_surfaced_so_approval_is_informed(db): _episode(db, f"e{i}", "alice") _candidate(db, "c1", [f"e{i}" for i in range(5)]) db.commit() - pending = US.pending_for_user("alice") + pending = personal_review_module.pending_for_user("alice") # Approving a capability without seeing what it will do is a habit, not a # decision. assert pending[0]["tool_sequence"] == ["a", "b"] @@ -206,7 +211,7 @@ def test_sequence_only_skill_has_an_inspectable_change_preview(db): _candidate(db, "c1", [f"e{i}" for i in range(5)]) db.commit() - change = US.pending_for_user("alice")[0]["change"] + change = personal_review_module.pending_for_user("alice")[0]["change"] assert change["type"] == "skill_sequence" assert change["display_name"] == "固定调用顺序：a → b" assert change["steps"] == ["a", "b"] @@ -224,7 +229,7 @@ def test_legacy_single_tool_candidate_is_hidden_without_deleting_evidence(db): ) db.commit() - assert US.pending_for_user("alice") == [] + assert personal_review_module.pending_for_user("alice") == [] assert db.get(EvolutionCandidate, "single") is not None @@ -247,7 +252,7 @@ def test_a_memory_candidate_is_offered_as_a_memory_change_not_an_install(db): ) db.commit() - row = US.pending_for_user("alice")[0] + row = personal_review_module.pending_for_user("alice")[0] assert row["action"] == "apply_memory" # Not "为我启用": nothing is being installed, a retrieval weight is changing. assert row["action_label"] != "为我启用" @@ -265,7 +270,7 @@ def test_a_memory_operation_with_no_executor_is_not_offered(db): ) db.commit() - assert US.pending_for_user("alice") == [] + assert personal_review_module.pending_for_user("alice") == [] def test_a_medium_risk_draft_skill_is_not_personally_approvable(db): @@ -274,7 +279,7 @@ def test_a_medium_risk_draft_skill_is_not_personally_approvable(db): _candidate(db, "s2", [f"e{i}" for i in range(3)], risk_tier="medium") db.commit() - assert US.pending_for_user("alice") == [] + assert personal_review_module.pending_for_user("alice") == [] def test_a_replayed_medium_risk_skill_is_personally_approvable(db): @@ -288,7 +293,7 @@ def test_a_replayed_medium_risk_skill_is_personally_approvable(db): ) db.commit() - assert [c["candidate_id"] for c in US.pending_for_user("alice")] == ["s3"] + assert [c["candidate_id"] for c in personal_review_module.pending_for_user("alice")] == ["s3"] def test_a_retirement_is_never_offered_as_something_to_enable(db): @@ -297,7 +302,7 @@ def test_a_retirement_is_never_offered_as_something_to_enable(db): _candidate(db, "s4", [f"e{i}" for i in range(3)], operation="deprecate") db.commit() - assert US.pending_for_user("alice") == [] + assert personal_review_module.pending_for_user("alice") == [] def test_an_orchestration_profile_is_not_a_personal_decision(db): @@ -311,7 +316,7 @@ def test_an_orchestration_profile_is_not_a_personal_decision(db): ) db.commit() - assert US.pending_for_user("alice") == [] + assert personal_review_module.pending_for_user("alice") == [] def test_the_skill_body_is_shown_so_approval_is_a_decision(db): @@ -330,7 +335,7 @@ def test_the_skill_body_is_shown_so_approval_is_a_decision(db): ) db.commit() - change = US.pending_for_user("alice")[0]["change"] + change = personal_review_module.pending_for_user("alice")[0]["change"] assert change["type"] == "skill_document" assert "## 步骤" in change["content"] assert change["allowed_tools"] == ["internet_search"] @@ -367,7 +372,7 @@ def test_a_candidate_i_accepted_no_longer_appears_in_my_queue(db): _personal_release(db, "r1", "c1", "alice") db.commit() - assert US.pending_for_user("alice") == [] + assert personal_review_module.pending_for_user("alice") == [] def test_my_acceptance_does_not_hide_the_candidate_from_others(db): @@ -382,8 +387,8 @@ def test_my_acceptance_does_not_hide_the_candidate_from_others(db): _personal_release(db, "r1", "ca", "alice") db.commit() - assert US.pending_for_user("alice") == [] - assert [c["candidate_id"] for c in US.pending_for_user("bob")] == ["cb"] + assert personal_review_module.pending_for_user("alice") == [] + assert [c["candidate_id"] for c in personal_review_module.pending_for_user("bob")] == ["cb"] def test_a_rolled_back_personal_release_reoffers_the_candidate(db): @@ -393,7 +398,7 @@ def test_a_rolled_back_personal_release_reoffers_the_candidate(db): _personal_release(db, "r1", "c1", "alice", stage="rolled_back") db.commit() - assert [c["candidate_id"] for c in US.pending_for_user("alice")] == ["c1"] + assert [c["candidate_id"] for c in personal_review_module.pending_for_user("alice")] == ["c1"] def test_re_approving_an_accepted_candidate_is_refused_with_the_reason(db): @@ -404,7 +409,7 @@ def test_re_approving_an_accepted_candidate_is_refused_with_the_reason(db): db.commit() with pytest.raises(PermissionError) as exc: - US.approve_for_user("c1", "alice") + personal_review_module.approve_for_user("c1", "alice") # Not the scope refusal: telling a user they lack permission over a skill # they just installed misreports success as failure. assert "已启用" in str(exc.value) diff --git a/src/backend/tests/evolution/test_personal_cycle_and_prefs.py b/src/backend/tests/evolution/test_personal_cycle_and_prefs.py index 2afd6e46..deb8ab3b 100644 --- a/src/backend/tests/evolution/test_personal_cycle_and_prefs.py +++ b/src/backend/tests/evolution/test_personal_cycle_and_prefs.py @@ -1,4 +1,6 @@ """Personal evolution scope, edition gating, and fine-grained preferences.""" +from core.evolution.cycle import candidate_records as candidate_records_module +from core.evolution.cycle import loop as loop_module import pytest from sqlalchemy import create_engine @@ -6,8 +8,8 @@ from core.db.engine import Base from core.db.models.evolution import EvolutionEpisode, EvolutionTraceEvent -from core.evolution import personal as PE -from core.evolution import prefs as PF +from core.evolution.profiles import personal as PE +from core.evolution.profiles import prefs as PF @pytest.fixture @@ -72,7 +74,7 @@ def test_an_unknown_user_gets_nothing(db): def test_personal_support_threshold_is_lower_than_fleet_wide(): - from core.evolution.promotion import MIN_SUPPORT + from core.evolution.cycle.promotion_constants import MIN_SUPPORT # One person's blast radius is one person; demanding fleet-scale evidence # would mean personal evolution never fires. @@ -84,7 +86,7 @@ def test_a_disabled_user_produces_no_candidates(db, monkeypatch): _episode(db, f"e{i}", "alice", ["a", "b"]) db.commit() monkeypatch.setattr(PE, "resolve_prefs", lambda uid: {"enabled": False}, raising=False) - import core.evolution.prefs as prefs_mod + import core.evolution.profiles.prefs as prefs_mod monkeypatch.setattr(prefs_mod, "resolve_prefs", lambda uid: {"enabled": False}) report = PE.run_personal_cycle("alice") @@ -95,10 +97,10 @@ def test_a_disabled_user_produces_no_candidates(db, monkeypatch): def test_cross_user_mining_is_skipped_without_the_entitlement(monkeypatch): - import core.evolution.loop as L + import core.evolution.cycle.loop as L - monkeypatch.setattr(L, "cross_user_mining_available", lambda: False) - report = L.run_evolution_cycle(limit=10) + monkeypatch.setattr(candidate_records_module, 'cross_user_mining_available', lambda: False) + report = loop_module.run_evolution_cycle(limit=10) assert report["skipped"] == "cross_user_mining_requires_enterprise" assert report["candidates"] == [] @@ -109,7 +111,7 @@ def test_a_missing_licensing_module_reads_as_community(monkeypatch): import builtins import sys - import core.evolution.loop as L + import core.evolution.cycle.loop as L real_import = builtins.__import__ @@ -129,7 +131,7 @@ def blocked(name, *args, **kwargs): monkeypatch.delitem(sys.modules, name, raising=False) monkeypatch.setattr(builtins, "__import__", blocked) - assert L.cross_user_mining_available() is False + assert candidate_records_module.cross_user_mining_available() is False # ── Preferences ────────────────────────────────────────────────────────────── diff --git a/src/backend/tests/evolution/test_policies_and_memory_ops.py b/src/backend/tests/evolution/test_policies_and_memory_ops.py index 854d8109..a02ea641 100644 --- a/src/backend/tests/evolution/test_policies_and_memory_ops.py +++ b/src/backend/tests/evolution/test_policies_and_memory_ops.py @@ -1,17 +1,20 @@ """GCE tickets 20 / 21 — the orchestration asset and memory operations. -The orchestration asset is now :class:`~core.evolution.agent_profile.AgentProfile` +The orchestration asset is now :class:`~core.evolution.profiles.agent_profile.AgentProfile` and the autonomous loop reads a *projection* of it. These tests hold the same invariants as before — the built-in must reproduce today's constants, the action allow-list must refuse anything resembling code execution, the budget must be bounded — against the asset that actually governs behaviour. """ +from core.evolution.profiles import agent_profile as agent_profile_module +from core.evolution.profiles import agent_profile_constants as agent_profile_state_module +from core.evolution.profiles import profile_selection as profile_selection_module import pytest -from core.evolution import agent_profile as AP -from core.evolution import memory_ops as MO -from core.evolution import policies as PL +from core.evolution.profiles import agent_profile as AP +from core.evolution.generators import memory_ops as MO +from core.evolution.profiles import policies as PL # ── The orchestration asset ────────────────────────────────────────────────── @@ -19,22 +22,22 @@ def test_builtin_profile_reproduces_todays_constants(): """Adopting profile resolution must be a no-op until someone publishes.""" - policy = PL.policy_from_profile(AP.builtin_profile()) + policy = PL.policy_from_profile(agent_profile_module.builtin_profile()) assert policy.max_attempts_per_requirement == 6 assert policy.strategy_change_after == 2 # The main ReAct agent has no turn cap; the built-in profile must not # reintroduce one behind the runtime's back. - assert AP.builtin_profile().max_react_turns == AP.UNBOUNDED_REACT_TURNS + assert agent_profile_module.builtin_profile().max_react_turns == agent_profile_state_module.UNBOUNDED_REACT_TURNS # A profile published without an explicit turn budget stays unbounded too — # a missing key must not become an invented ceiling. - unspecified = AP.AgentProfile.from_dict({"profile_id": "p"}) - assert unspecified.max_react_turns == AP.UNBOUNDED_REACT_TURNS + unspecified = agent_profile_module.AgentProfile.from_dict({"profile_id": "p"}) + assert unspecified.max_react_turns == agent_profile_state_module.UNBOUNDED_REACT_TURNS def test_longer_stall_escalates_rather_than_repeating_the_mild_response(): - profile = AP.builtin_profile() + profile = agent_profile_module.builtin_profile() profile.intervention_rules.append( - AP.InterventionRule(AP.SIGNAL_NO_PROGRESS, 6, AP.ACTION_STOP) + agent_profile_module.InterventionRule(agent_profile_state_module.SIGNAL_NO_PROGRESS, 6, agent_profile_state_module.ACTION_STOP) ) policy = PL.policy_from_profile(profile) assert policy.action_for({PL.SIGNAL_NO_PROGRESS: 2}) == PL.ACTION_CHANGE_STRATEGY @@ -44,69 +47,69 @@ def test_longer_stall_escalates_rather_than_repeating_the_mild_response(): def test_no_signal_yields_no_intervention(): - assert PL.policy_from_profile(AP.builtin_profile()).action_for({}) is None + assert PL.policy_from_profile(agent_profile_module.builtin_profile()).action_for({}) is None def test_profile_rejects_actions_outside_the_allow_list(): - profile = AP.builtin_profile() + profile = agent_profile_module.builtin_profile() profile.intervention_rules = [ - AP.InterventionRule("no_progress", 2, "exec_arbitrary_code") + agent_profile_module.InterventionRule("no_progress", 2, "exec_arbitrary_code") ] - ok, problems = AP.validate_profile(profile) + ok, problems = agent_profile_module.validate_profile(profile) # Arbitrary self-modifying behaviour is the one capability refused outright. assert ok is False and any("未知动作" in p for p in problems) def test_profile_rejects_an_unbounded_budget_multiplier(): - profile = AP.builtin_profile() + profile = agent_profile_module.builtin_profile() profile.budget_multiplier = 50.0 - ok, problems = AP.validate_profile(profile) + ok, problems = agent_profile_module.validate_profile(profile) # This is how a policy change becomes a runaway spend unnoticed. assert ok is False and any("预算倍数" in p for p in problems) def test_profile_rejects_non_positive_thresholds(): - profile = AP.builtin_profile() + profile = agent_profile_module.builtin_profile() profile.intervention_rules = [ - AP.InterventionRule("no_progress", 0, AP.ACTION_RETRY) + agent_profile_module.InterventionRule("no_progress", 0, agent_profile_state_module.ACTION_RETRY) ] - ok, _ = AP.validate_profile(profile) + ok, _ = agent_profile_module.validate_profile(profile) assert ok is False def test_a_profile_may_never_widen_tool_authority(): """The invariant that stops orchestration evolution being an escalation path.""" - profile = AP.builtin_profile() + profile = agent_profile_module.builtin_profile() profile.tool_allowlist = ["search_web", "shell_exec"] - ok, problems = AP.validate_profile(profile, base_tools=["search_web", "fetch_page"]) + ok, problems = agent_profile_module.validate_profile(profile, base_tools=["search_web", "fetch_page"]) assert ok is False and any("工具授权越界" in p for p in problems) # Narrowing is the whole point, and must pass. profile.tool_allowlist = ["search_web"] - assert AP.validate_profile(profile, base_tools=["search_web", "fetch_page"])[0] + assert agent_profile_module.validate_profile(profile, base_tools=["search_web", "fetch_page"])[0] def test_valid_profile_passes(): - ok, problems = AP.validate_profile(AP.builtin_profile()) + ok, problems = agent_profile_module.validate_profile(agent_profile_module.builtin_profile()) assert ok is True and problems == [] def test_profile_roundtrips_through_dict(): - original = AP.builtin_profile() - restored = AP.AgentProfile.from_dict(original.to_dict()) + original = agent_profile_module.builtin_profile() + restored = agent_profile_module.AgentProfile.from_dict(original.to_dict()) assert restored.to_dict() == original.to_dict() def test_profile_load_degrades_to_builtin_when_store_is_unavailable(): # No database in this test context → must degrade, not raise. - assert AP.load_active_profile(task_type="nonexistent").profile_id == "builtin" + assert profile_selection_module.load_active_profile(task_type="nonexistent").profile_id == "builtin" # ── Scope resolution: the bug the tenant parameter was meant to fix ────────── def _profile(profile_id, *, scope=None, task_types=()): - profile = AP.builtin_profile() + profile = agent_profile_module.builtin_profile() profile.profile_id = profile_id profile.scope = dict(scope or {}) profile.task_types = list(task_types) @@ -114,7 +117,7 @@ def _profile(profile_id, *, scope=None, task_types=()): def test_a_tenants_profile_wins_over_the_deployment_default(): - chosen = AP.pick_profile( + chosen = profile_selection_module.pick_profile( [_profile("global"), _profile("acme", scope={"tenant_id": "acme"})], task_type="chat", tenant_id="acme", @@ -124,14 +127,14 @@ def test_a_tenants_profile_wins_over_the_deployment_default(): def test_one_tenants_profile_never_serves_another(): """Inheriting someone else's orchestration is invisible when it happens.""" - chosen = AP.pick_profile( + chosen = profile_selection_module.pick_profile( [_profile("acme", scope={"tenant_id": "acme"})], task_type="chat", tenant_id="other" ) assert chosen is None def test_a_users_own_profile_outranks_their_tenants(): - chosen = AP.pick_profile( + chosen = profile_selection_module.pick_profile( [ _profile("tenant", scope={"tenant_id": "acme"}), _profile("mine", scope={"user_id": "u1"}), @@ -144,7 +147,7 @@ def test_a_users_own_profile_outranks_their_tenants(): def test_a_task_type_specific_profile_outranks_a_general_one(): - chosen = AP.pick_profile( + chosen = profile_selection_module.pick_profile( [_profile("general"), _profile("finance", task_types=["finance"])], task_type="finance", ) @@ -153,7 +156,7 @@ def test_a_task_type_specific_profile_outranks_a_general_one(): def test_a_profile_for_another_task_type_is_not_used(): assert ( - AP.pick_profile([_profile("finance", task_types=["finance"])], task_type="chat") + profile_selection_module.pick_profile([_profile("finance", task_types=["finance"])], task_type="chat") is None ) diff --git a/src/backend/tests/evolution/test_promotion_chain.py b/src/backend/tests/evolution/test_promotion_chain.py index 65923027..1eff325f 100644 --- a/src/backend/tests/evolution/test_promotion_chain.py +++ b/src/backend/tests/evolution/test_promotion_chain.py @@ -4,10 +4,10 @@ the tests cover both directions: what must be promoted, and — more importantly — what must not be. """ +from core.evolution.cycle import patterns as patterns_module, promotion as promotion_module, promotion_constants as promotion_state_module, retirement as retirement_module, promotion as P import pytest -from core.evolution import promotion as P def _episodes(n, *, verdict="success", tools=("search", "verify", "chart", "export")): @@ -26,25 +26,25 @@ def _episodes(n, *, verdict="success", tools=("search", "verify", "chart", "expo def test_recurring_successful_subsequence_is_discovered(): - patterns = P.discover_patterns(_episodes(8)) - success = [p for p in patterns if p.kind == P.PATTERN_SUCCESS_SUBSEQUENCE] + patterns = patterns_module.discover_patterns(_episodes(8)) + success = [p for p in patterns if p.kind == promotion_state_module.PATTERN_SUCCESS_SUBSEQUENCE] assert success and success[0].support == 8 assert success[0].tool_sequence == ["search", "verify", "chart", "export"] def test_repeated_failures_are_discovered_separately(): - patterns = P.discover_patterns(_episodes(7, verdict="failed")) - assert any(p.kind == P.PATTERN_REPEATED_FAILURE for p in patterns) + patterns = patterns_module.discover_patterns(_episodes(7, verdict="failed")) + assert any(p.kind == promotion_state_module.PATTERN_REPEATED_FAILURE for p in patterns) def test_low_support_clusters_produce_nothing(): """One-off noise must not become a permanent asset.""" - assert P.discover_patterns(_episodes(2)) == [] + assert patterns_module.discover_patterns(_episodes(2)) == [] def test_patterns_carry_the_memories_they_came_from(): - patterns = P.discover_patterns(_episodes(6)) - success = [p for p in patterns if p.kind == P.PATTERN_SUCCESS_SUBSEQUENCE][0] + patterns = patterns_module.discover_patterns(_episodes(6)) + success = [p for p in patterns if p.kind == promotion_state_module.PATTERN_SUCCESS_SUBSEQUENCE][0] assert len(success.memory_refs) == 6 @@ -52,8 +52,8 @@ def test_patterns_carry_the_memories_they_came_from(): def _success_pattern(support=27, rate=0.9, tools=("search", "verify", "chart", "export")): - return P.Pattern( - kind=P.PATTERN_SUCCESS_SUBSEQUENCE, + return patterns_module.Pattern( + kind=promotion_state_module.PATTERN_SUCCESS_SUBSEQUENCE, signature="→".join(tools), support=support, success_rate=rate, @@ -81,7 +81,7 @@ def _supporting_episodes( def test_recurring_pattern_is_compiled_into_a_skill_proposal(): """The 27-weekly-reports case: same request, same tools, separate chats.""" - proposal = P.promote_tool_sequence_to_skill( + proposal = promotion_module.promote_tool_sequence_to_skill( _success_pattern(), episodes=_supporting_episodes(27), skill_credit=0.81, @@ -102,8 +102,8 @@ def test_recurring_pattern_is_compiled_into_a_skill_proposal(): def test_one_repeated_tool_call_is_not_mislabelled_as_a_sequence_skill(): - pattern = P.Pattern( - kind=P.PATTERN_SUCCESS_SUBSEQUENCE, + pattern = patterns_module.Pattern( + kind=promotion_state_module.PATTERN_SUCCESS_SUBSEQUENCE, signature="call_subagent", support=9, success_rate=1.0, @@ -112,7 +112,7 @@ def test_one_repeated_tool_call_is_not_mislabelled_as_a_sequence_skill(): ) assert ( - P.promote_tool_sequence_to_skill( + promotion_module.promote_tool_sequence_to_skill( pattern, skill_credit=0.9, workflow_credit=0.1 ) is None @@ -120,7 +120,7 @@ def test_one_repeated_tool_call_is_not_mislabelled_as_a_sequence_skill(): def test_promotion_is_refused_when_orchestration_explains_it_better(): - proposal = P.promote_tool_sequence_to_skill( + proposal = promotion_module.promote_tool_sequence_to_skill( _success_pattern(), episodes=_supporting_episodes(27), skill_credit=0.3, @@ -133,7 +133,7 @@ def test_promotion_is_refused_when_orchestration_explains_it_better(): def test_low_success_rate_is_not_promoted(): assert ( - P.promote_tool_sequence_to_skill( + promotion_module.promote_tool_sequence_to_skill( _success_pattern(rate=0.4), episodes=_supporting_episodes(27), skill_credit=0.9 ) is None @@ -142,7 +142,7 @@ def test_low_success_rate_is_not_promoted(): def test_insufficient_support_is_not_promoted(): assert ( - P.promote_tool_sequence_to_skill( + promotion_module.promote_tool_sequence_to_skill( _success_pattern(support=2), episodes=_supporting_episodes(2), skill_credit=0.9 ) is None @@ -150,7 +150,7 @@ def test_insufficient_support_is_not_promoted(): def test_existing_equivalent_skill_becomes_a_patch_not_a_twin(): - proposal = P.promote_tool_sequence_to_skill( + proposal = promotion_module.promote_tool_sequence_to_skill( _success_pattern(), episodes=_supporting_episodes(27), skill_credit=0.9, @@ -165,7 +165,7 @@ def test_a_trivial_sequence_is_never_promoted(): for tools in (("view_text_file",), ("call_subagent", "view_text_file")): pattern = _success_pattern(support=9, rate=1.0, tools=tools) assert ( - P.promote_tool_sequence_to_skill( + promotion_module.promote_tool_sequence_to_skill( pattern, episodes=_supporting_episodes(9), skill_credit=0.9 ) is None @@ -193,7 +193,7 @@ def test_unrelated_conversations_are_not_an_sop(): for i in range(6) ] assert ( - P.promote_tool_sequence_to_skill( + promotion_module.promote_tool_sequence_to_skill( _success_pattern(support=6), episodes=episodes, skill_credit=0.9 ) is None @@ -204,7 +204,7 @@ def test_recurrence_within_one_conversation_is_one_occasion(): """一场会话里反复跑同一套流程只算一次场合，不构成"这件事总在发生"。""" episodes = _supporting_episodes(6, chats=["chat-same"] * 6) assert ( - P.promote_tool_sequence_to_skill( + promotion_module.promote_tool_sequence_to_skill( _success_pattern(support=6), episodes=episodes, skill_credit=0.9 ) is None @@ -215,7 +215,7 @@ def test_no_request_text_means_no_commonality_and_no_promotion(): """Commonality that cannot be established is not commonality.""" episodes = _supporting_episodes(6, objective="") assert ( - P.promote_tool_sequence_to_skill( + promotion_module.promote_tool_sequence_to_skill( _success_pattern(support=6), episodes=episodes, skill_credit=0.9 ) is None @@ -225,12 +225,12 @@ def test_no_request_text_means_no_commonality_and_no_promotion(): def test_promotion_without_supporting_views_is_refused(): """A caller that cannot show the episodes gets a refusal, not the benefit of the doubt.""" - assert P.promote_tool_sequence_to_skill(_success_pattern(), skill_credit=0.9) is None + assert promotion_module.promote_tool_sequence_to_skill(_success_pattern(), skill_credit=0.9) is None def test_personal_support_floor_is_honoured_when_passed(): """个人路径发现阶段用门槛 3；晋升阶段必须沿用同一门槛，而不是全局的 5。""" - proposal = P.promote_tool_sequence_to_skill( + proposal = promotion_module.promote_tool_sequence_to_skill( _success_pattern(support=3), episodes=_supporting_episodes(3), min_support=3, @@ -241,7 +241,7 @@ def test_personal_support_floor_is_honoured_when_passed(): def test_procedural_memories_are_what_becomes_a_skill(): """The chain the design promised: memory *content* compiled into a skill.""" - proposal = P.promote_procedural_memory_to_skill( + proposal = promotion_module.promote_procedural_memory_to_skill( memories=[ { "ref": "mref-1", @@ -270,7 +270,7 @@ def test_procedural_memories_are_what_becomes_a_skill(): def test_a_recurring_intent_with_no_procedural_memory_yields_no_skill(): assert ( - P.promote_procedural_memory_to_skill( + promotion_module.promote_procedural_memory_to_skill( memories=[{"ref": "m", "memory_type": "fact", "content": "某公司是电池厂商"}], intent_cluster={"representative": "x", "episode_ids": []}, occurrences=9, @@ -284,7 +284,7 @@ def test_a_recurring_intent_with_no_procedural_memory_yields_no_skill(): def test_a_failing_recurring_intent_is_never_distilled(): """Distilling a failing approach would make it the house style.""" assert ( - P.promote_procedural_memory_to_skill( + promotion_module.promote_procedural_memory_to_skill( memories=[{"ref": "m", "memory_type": "procedural", "content": "先做 X"}], intent_cluster={"representative": "x", "episode_ids": []}, occurrences=9, @@ -298,7 +298,7 @@ def test_a_failing_recurring_intent_is_never_distilled(): def test_a_stable_plan_needs_no_skill(): """Same request, same plan every time: already effectively a skill.""" assert ( - P.promote_procedural_memory_to_skill( + promotion_module.promote_procedural_memory_to_skill( memories=[{"ref": "m", "memory_type": "procedural", "content": "先做 X"}], intent_cluster={"representative": "x", "episode_ids": []}, occurrences=9, @@ -313,8 +313,8 @@ def test_a_stable_plan_needs_no_skill(): def _combo_pattern(support=9): - return P.Pattern( - kind=P.PATTERN_SUCCESS_SUBSEQUENCE, + return patterns_module.Pattern( + kind=promotion_state_module.PATTERN_SUCCESS_SUBSEQUENCE, signature="kb-search→fin-analyse→report", support=support, success_rate=0.9, @@ -323,7 +323,7 @@ def _combo_pattern(support=9): def test_frequent_and_better_combination_is_promoted(): - proposal = P.promote_skills_to_orchestration( + proposal = promotion_module.promote_skills_to_orchestration( _combo_pattern(), combo_success_rate=0.88, individual_success_rate=0.70 ) assert proposal is not None @@ -336,7 +336,7 @@ def test_frequent_and_better_combination_is_promoted(): def test_frequent_but_no_better_combination_is_not_promoted(): """Frequency alone would promote every common sequence.""" assert ( - P.promote_skills_to_orchestration( + promotion_module.promote_skills_to_orchestration( _combo_pattern(), combo_success_rate=0.71, individual_success_rate=0.70 ) is None @@ -347,7 +347,7 @@ def test_single_skill_is_never_an_orchestration_change(): pattern = _combo_pattern() pattern.skill_sequence = ["only-one"] assert ( - P.promote_skills_to_orchestration( + promotion_module.promote_skills_to_orchestration( pattern, combo_success_rate=0.99, individual_success_rate=0.1 ) is None @@ -355,13 +355,13 @@ def test_single_skill_is_never_an_orchestration_change(): def test_profile_invalidates_when_a_dependency_moves(): - proposal = P.promote_skills_to_orchestration( + proposal = promotion_module.promote_skills_to_orchestration( _combo_pattern(), combo_success_rate=0.9, individual_success_rate=0.7, skill_versions={"kb-search": "v1", "fin-analyse": "v2", "report": "v1"}, ) - ok, stale = P.dependencies_satisfied( + ok, stale = promotion_module.dependencies_satisfied( proposal, {"kb-search": "v1", "fin-analyse": "v3", "report": "v1"} ) # A profile pointing at a dead skill version is worse than no profile. @@ -369,13 +369,13 @@ def test_profile_invalidates_when_a_dependency_moves(): def test_profile_stays_valid_while_dependencies_hold(): - proposal = P.promote_skills_to_orchestration( + proposal = promotion_module.promote_skills_to_orchestration( _combo_pattern(), combo_success_rate=0.9, individual_success_rate=0.7, skill_versions={"kb-search": "v1", "fin-analyse": "v2", "report": "v1"}, ) - ok, stale = P.dependencies_satisfied( + ok, stale = promotion_module.dependencies_satisfied( proposal, {"kb-search": "v1", "fin-analyse": "v2", "report": "v1"} ) assert ok is True and stale == [] @@ -385,7 +385,7 @@ def test_profile_stays_valid_while_dependencies_hold(): def test_overlapping_assets_are_surfaced_for_merge(): - overlaps = P.find_overlaps( + overlaps = promotion_module.find_overlaps( { "skill-a": ["search", "verify", "chart"], "skill-b": ["search", "verify", "export"], @@ -405,7 +405,7 @@ def test_retirement_reads_opens_not_offers(): "never chosen" from "chosen and useless" is whether the model opened the document for itself. """ - out = P.find_retirement_candidates( + out = retirement_module.find_retirement_candidates( { "never-opened": { "offers": 40, "opens": 0, "success_rate": 0.0, @@ -429,16 +429,16 @@ def test_retirement_reads_opens_not_offers(): assert set(by_id) == {"never-opened", "opened-but-ignored", "opened-and-failing"} # Never opened is a discovery problem, not a reason to remove a capability. - assert by_id["never-opened"]["action"] == P.ACTION_LOWER_EXPOSURE + assert by_id["never-opened"]["action"] == promotion_state_module.ACTION_LOWER_EXPOSURE # Opened and ignored is a *content* problem: retiring it would remove a # capability the model actively wanted. - assert by_id["opened-but-ignored"]["action"] == P.ACTION_REWRITE - assert by_id["opened-and-failing"]["action"] == P.ACTION_RETIRE + assert by_id["opened-but-ignored"]["action"] == promotion_state_module.ACTION_REWRITE + assert by_id["opened-and-failing"]["action"] == promotion_state_module.ACTION_RETIRE def test_a_freshly_shipped_skill_is_never_retired(): """The old rule's answer here was "retire it", which punished newness.""" - out = P.find_retirement_candidates( + out = retirement_module.find_retirement_candidates( { "just-shipped": { "offers": 30, "opens": 0, "success_rate": 0.0, @@ -450,7 +450,7 @@ def test_a_freshly_shipped_skill_is_never_retired(): def test_a_harmful_skill_is_rolled_back_immediately(): - out = P.find_retirement_candidates( + out = retirement_module.find_retirement_candidates( { "harmful": { "offers": 40, "opens": 20, "success_rate": 0.9, @@ -460,14 +460,14 @@ def test_a_harmful_skill_is_rolled_back_immediately(): ) # The one case that does not wait for the exposure window: a paired # comparison already showed it makes things worse. - assert out and out[0]["action"] == P.ACTION_ROLLBACK + assert out and out[0]["action"] == promotion_state_module.ACTION_ROLLBACK # ── Negative transfer: the dangerous, silent failure ───────────────────────── def test_collateral_damage_on_unrelated_tasks_is_detected(): - findings = P.detect_negative_transfer( + findings = retirement_module.detect_negative_transfer( asset_id="skill-weekly-report", related_task_types=["report_generation"], before={"report_generation": 0.70, "db_analysis": 0.85, "kb_qa": 0.80}, @@ -480,7 +480,7 @@ def test_collateral_damage_on_unrelated_tasks_is_detected(): def test_a_dip_on_the_targeted_task_is_not_negative_transfer(): - findings = P.detect_negative_transfer( + findings = retirement_module.detect_negative_transfer( asset_id="skill-x", related_task_types=["report_generation"], before={"report_generation": 0.9}, @@ -491,7 +491,7 @@ def test_a_dip_on_the_targeted_task_is_not_negative_transfer(): def test_small_fluctuations_are_not_flagged(): - findings = P.detect_negative_transfer( + findings = retirement_module.detect_negative_transfer( asset_id="skill-x", related_task_types=[], before={"kb_qa": 0.800}, @@ -501,7 +501,7 @@ def test_small_fluctuations_are_not_flagged(): def test_task_types_without_a_baseline_are_skipped(): - findings = P.detect_negative_transfer( + findings = retirement_module.detect_negative_transfer( asset_id="skill-x", related_task_types=[], before={}, after={"new_task": 0.1} ) assert findings == [] diff --git a/src/backend/tests/evolution/test_react_intervention.py b/src/backend/tests/evolution/test_react_intervention.py index d4345a7d..3429714e 100644 --- a/src/backend/tests/evolution/test_react_intervention.py +++ b/src/backend/tests/evolution/test_react_intervention.py @@ -14,13 +14,14 @@ import pytest from agentscope.message import ToolResultState -from core.evolution.agent_profile import ( - ACTION_CHANGE_STRATEGY, - SIGNAL_REPEATED_ACTIONS, - SIGNAL_TOOL_ERROR_STREAK, - InterventionRule, -) -from core.llm.middlewares import StallInterventionMiddleware +from core.evolution.profiles.agent_profile_constants import ACTION_CHANGE_STRATEGY, SIGNAL_REPEATED_ACTIONS, SIGNAL_TOOL_ERROR_STREAK +from core.evolution.profiles.agent_profile import InterventionRule +from core.evolution.profiles.stall_hook import stall_hooks +from tests.orchestration.hook_support import make_adapter +from agentscope.tool._response import ToolResponse + +def intervention_adapter(rules): + return make_adapter(*stall_hooks(rules)) class _Ctx(list): @@ -45,7 +46,7 @@ def __init__(self, state): def _run(mw, agent, call, state): async def handler(**_kw): - yield _Chunk(state) + yield ToolResponse(content=[], state=state) async def drive(): async for _ in mw.on_acting(agent, {"tool_call": call}, handler): @@ -65,7 +66,7 @@ def _reminders(agent): def test_repeating_the_same_call_triggers_the_configured_action(): - mw = StallInterventionMiddleware( + mw = intervention_adapter( [InterventionRule(SIGNAL_REPEATED_ACTIONS, 2, ACTION_CHANGE_STRATEGY)] ) agent = _Agent() @@ -82,7 +83,7 @@ def test_repeating_the_same_call_triggers_the_configured_action(): def test_a_different_call_breaks_the_streak(): """A streak is consecutive by definition.""" - mw = StallInterventionMiddleware( + mw = intervention_adapter( [InterventionRule(SIGNAL_REPEATED_ACTIONS, 2, ACTION_CHANGE_STRATEGY)] ) agent = _Agent() @@ -92,7 +93,7 @@ def test_a_different_call_breaks_the_streak(): def test_consecutive_tool_failures_trigger_an_intervention(): - mw = StallInterventionMiddleware( + mw = intervention_adapter( [InterventionRule(SIGNAL_TOOL_ERROR_STREAK, 3, ACTION_CHANGE_STRATEGY)] ) agent = _Agent() @@ -102,7 +103,7 @@ def test_consecutive_tool_failures_trigger_an_intervention(): def test_a_success_resets_the_failure_streak(): - mw = StallInterventionMiddleware( + mw = intervention_adapter( [InterventionRule(SIGNAL_TOOL_ERROR_STREAK, 3, ACTION_CHANGE_STRATEGY)] ) agent = _Agent() @@ -115,7 +116,7 @@ def test_a_success_resets_the_failure_streak(): def test_the_same_intervention_is_not_repeated_every_iteration(): """Repeating one reminder each turn makes it noise the model learns to skip.""" - mw = StallInterventionMiddleware( + mw = intervention_adapter( [InterventionRule(SIGNAL_REPEATED_ACTIONS, 2, ACTION_CHANGE_STRATEGY)] ) agent = _Agent() @@ -127,7 +128,7 @@ def test_the_same_intervention_is_not_repeated_every_iteration(): def test_no_rules_means_no_behaviour_change(): """The built-in profile must be a no-op until something is published.""" - mw = StallInterventionMiddleware([]) + mw = intervention_adapter([]) agent = _Agent() call = _Call("search_web", {"q": "x"}) for _ in range(5): diff --git a/src/backend/tests/evolution/test_readiness_gate.py b/src/backend/tests/evolution/test_readiness_gate.py index ceb42e04..74420817 100644 --- a/src/backend/tests/evolution/test_readiness_gate.py +++ b/src/backend/tests/evolution/test_readiness_gate.py @@ -6,12 +6,14 @@ "there was nothing to find" — so nobody investigates, and the feature looks switched on while doing almost nothing. """ +from core.evolution.cycle import loop as loop_module +from core.models import embedding as vector_embedding import pytest -from core.evolution import loop as L -from core.evolution import personal as PE -from core.evolution import readiness as R +from core.evolution.cycle import loop as L +from core.evolution.profiles import personal as PE +from core.evolution.cycle import readiness as R def _not_ready(reason="embedding"): @@ -26,7 +28,7 @@ def _not_ready(reason="embedding"): def test_the_fleet_cycle_refuses_rather_than_running_degraded(monkeypatch): monkeypatch.setattr(R, "check_readiness", lambda **_kw: _not_ready()) - report = L.run_evolution_cycle(limit=10) + report = loop_module.run_evolution_cycle(limit=10) assert report["skipped"] == "not_ready" assert report["candidates"] == [] # And it says which dependency, because "not ready" without a reason is the @@ -55,12 +57,12 @@ def test_an_unreachable_embedding_service_is_not_ready(monkeypatch): A configured-but-unreachable endpoint produces exactly the silent degradation this check exists to surface. """ - import core.kb.kb_vector as kb_vector + import features.knowledge.kb_vector as kb_vector def explode(*_a, **_kw): raise RuntimeError("connection refused") - monkeypatch.setattr(kb_vector, "embed_text", explode) + monkeypatch.setattr(vector_embedding, 'embed_text', explode) R.reset_cache() requirement = R._check_embedding() assert requirement.ok is False @@ -69,9 +71,9 @@ def explode(*_a, **_kw): def test_an_empty_vector_is_not_ready(monkeypatch): - import core.kb.kb_vector as kb_vector + import features.knowledge.kb_vector as kb_vector - monkeypatch.setattr(kb_vector, "embed_text", lambda *_a, **_kw: []) + monkeypatch.setattr(vector_embedding, 'embed_text', lambda *_a, **_kw: []) R.reset_cache() assert R._check_embedding().ok is False diff --git a/src/backend/tests/evolution/test_realistic_corpus.py b/src/backend/tests/evolution/test_realistic_corpus.py index a7264898..ca517c18 100644 --- a/src/backend/tests/evolution/test_realistic_corpus.py +++ b/src/backend/tests/evolution/test_realistic_corpus.py @@ -13,10 +13,15 @@ These tests pin the corpus shape that broke it. """ +from core.evolution.cycle import candidate_records as candidate_records_module +from core.evolution.cycle import ordering as ordering_module +from core.evolution.cycle import patterns as patterns_module +from core.evolution.cycle import promotion_constants as promotion_state_module +from core.evolution.cycle import promotion_types as promotion_types_module import pytest -from core.evolution import promotion as P +from core.evolution.cycle import promotion as P def _varied_corpus(): @@ -46,13 +51,13 @@ def _varied_corpus(): def test_no_sequence_pattern_survives_a_realistic_corpus(): """The premise: exact-sequence mining finds nothing when shapes vary.""" - patterns = P.discover_patterns(_varied_corpus()) - assert [p for p in patterns if p.kind == P.PATTERN_SUCCESS_SUBSEQUENCE] == [] + patterns = patterns_module.discover_patterns(_varied_corpus()) + assert [p for p in patterns if p.kind == promotion_state_module.PATTERN_SUCCESS_SUBSEQUENCE] == [] def test_ordering_rules_do_survive_it(): corpus = _varied_corpus() - rules = P.extract_ordering_constraints(corpus) + rules = ordering_module.extract_ordering_constraints(corpus) pair = next((c for c in rules if (c.before, c.after) == ("verify", "query")), None) assert pair is not None assert pair.contexts >= 2 @@ -61,17 +66,17 @@ def test_ordering_rules_do_survive_it(): def test_the_cycle_emits_a_rules_only_candidate(): """Where the loop previously produced nothing at all.""" - from core.evolution import loop as L + from core.evolution.cycle import loop as L corpus = _varied_corpus() - rules = P.extract_ordering_constraints(corpus) + rules = ordering_module.extract_ordering_constraints(corpus) assert rules, "precondition: the corpus yields rules" # No sequence candidate exists, so every rule is uncovered. - orphans = L._uncovered_constraints(rules, []) + orphans = candidate_records_module.uncovered_constraints(rules, []) assert orphans - decision = L.assign_credit(L._features_for_ordering_rules(orphans)) + decision = L.assign_credit(candidate_records_module.features_for_ordering_rules(orphans)) assert decision.may_produce_candidate, ( f"attribution refused a rules-only candidate: " f"{decision.selected} conf={decision.confidence}" @@ -80,11 +85,11 @@ def test_the_cycle_emits_a_rules_only_candidate(): def test_a_rule_already_carried_by_a_sequence_skill_is_not_duplicated(): - from core.evolution import loop as L + from core.evolution.cycle import loop as L - rules = P.extract_ordering_constraints(_varied_corpus()) + rules = ordering_module.extract_ordering_constraints(_varied_corpus()) covered = [["verify", "query", "chart", "export"]] - remaining = L._uncovered_constraints(rules, covered) + remaining = candidate_records_module.uncovered_constraints(rules, covered) assert all( not ({c.before, c.after} <= set(covered[0])) for c in remaining ) @@ -92,10 +97,10 @@ def test_a_rule_already_carried_by_a_sequence_skill_is_not_duplicated(): def test_thin_evidence_still_resolves_to_no_update(): """The floor must stay a floor: one weak observation is not a capability.""" - from core.evolution import loop as L + from core.evolution.cycle import loop as L - thin = [P.OrderingConstraint(before="a", after="b", support=0, contexts=0)] - decision = L.assign_credit(L._features_for_ordering_rules(thin)) + thin = [promotion_types_module.OrderingConstraint(before="a", after="b", support=0, contexts=0)] + decision = L.assign_credit(candidate_records_module.features_for_ordering_rules(thin)) assert decision.may_produce_candidate is False @@ -108,16 +113,11 @@ def test_rules_only_skill_renders_parseable_markdown_carrying_the_rules(): The skill document is the only channel by which a learned rule reaches the model, so a rule absent from it is a rule the runtime never receives. """ - from core.agent_skills.registry import _load_skill_from_str - from core.evolution.activation import ( - _skill_description, - _skill_label, - _skill_markdown, - _tools_from_constraints, - ) + from core.capabilities.skills.registry import _load_skill_from_str + from core.evolution.cycle.skill_document import skill_description as _skill_description, skill_label as _skill_label, skill_markdown as _skill_markdown, tools_from_constraints as _tools_from_constraints constraints = [ - c.to_dict() for c in P.extract_ordering_constraints(_varied_corpus()) + c.to_dict() for c in ordering_module.extract_ordering_constraints(_varied_corpus()) ] tools = _tools_from_constraints(constraints) assert tools and tools.index("verify") < tools.index("query") @@ -157,7 +157,7 @@ def test_a_contradicted_rule_is_withheld_rather_than_annotated(): The rule survives in the JSON side file for a runtime applier that knows the task type — which is where a scope decision belongs. """ - from core.evolution.activation import _ordering_rules_section + from core.evolution.cycle.skill_document import ordering_rules_section as _ordering_rules_section section = _ordering_rules_section( [ diff --git a/src/backend/tests/evolution/test_release_state_machine.py b/src/backend/tests/evolution/test_release_state_machine.py index 7ee538e4..6758aefd 100644 --- a/src/backend/tests/evolution/test_release_state_machine.py +++ b/src/backend/tests/evolution/test_release_state_machine.py @@ -7,7 +7,7 @@ import pytest -from core.evolution import release as R +from core.evolution.cycle import release as R # ── State machine ──────────────────────────────────────────────────────────── diff --git a/src/backend/tests/evolution/test_runtime_binding.py b/src/backend/tests/evolution/test_runtime_binding.py index 1c2b204e..d8bfcafa 100644 --- a/src/backend/tests/evolution/test_runtime_binding.py +++ b/src/backend/tests/evolution/test_runtime_binding.py @@ -8,7 +8,7 @@ import pytest from core.evolution import contract as C -from core.evolution import runtime_binding as rb +from core.evolution.cycle import runtime_binding as rb @pytest.fixture(autouse=True) @@ -212,7 +212,7 @@ def test_bundle_is_retrievable_by_run_id(monkeypatch): def test_bundle_for_run_keeps_workspace_policy_and_sanitized_execution_manifest(monkeypatch): - from core.llm.execution_manifest import PromptManifestBuilder + from core.harness.execution_manifest import PromptManifestBuilder monkeypatch.setattr(rb, "_prompt_refs", lambda: []) monkeypatch.setattr(rb, "_ontology_refs", lambda: []) @@ -274,7 +274,7 @@ def to_dict(self): def test_binder_uses_rendered_prompt_versions_not_a_new_active_pointer(monkeypatch): - from core.llm.execution_manifest import PromptManifestBuilder + from core.harness.execution_manifest import PromptManifestBuilder monkeypatch.setattr( rb, @@ -325,7 +325,7 @@ def test_no_drift_when_assets_republish_after_binding(monkeypatch): def test_request_manifest_rebind_preserves_frozen_refs_and_updates_run_index(monkeypatch): - from core.llm.execution_manifest import PromptManifestBuilder + from core.harness.execution_manifest import PromptManifestBuilder live_version = {"v": "v1"} monkeypatch.setattr(rb, "_prompt_refs", lambda: []) diff --git a/src/backend/tests/evolution/test_settlement.py b/src/backend/tests/evolution/test_settlement.py index 0e23c5d1..199c8feb 100644 --- a/src/backend/tests/evolution/test_settlement.py +++ b/src/backend/tests/evolution/test_settlement.py @@ -7,7 +7,7 @@ to edit or delete it. """ -from core.evolution import settlement as S +from core.evolution.settlement import settlement as S def _entry(handle="m-1", text="做财务分析前先核验主体", layer=S.LAYER_PROCEDURE, **kw): diff --git a/src/backend/tests/evolution/test_settlement_runner.py b/src/backend/tests/evolution/test_settlement_runner.py index 07ec8d22..1ffc5113 100644 --- a/src/backend/tests/evolution/test_settlement_runner.py +++ b/src/backend/tests/evolution/test_settlement_runner.py @@ -10,7 +10,7 @@ import pytest -from core.evolution import settlement_runner as R +from core.evolution.settlement import settlement_runner as R @pytest.fixture(autouse=True) @@ -25,7 +25,7 @@ def persisted(monkeypatch): """Capture what settlement persists instead of touching the DB.""" calls = [] monkeypatch.setattr( - "core.evolution.settlement_store.persist_summary", + 'core.evolution.settlement.settlement_store.persist_summary', lambda message_id, summary: calls.append((message_id, summary)) or True, ) return calls diff --git a/src/backend/tests/evolution/test_skill_selection_record.py b/src/backend/tests/evolution/test_skill_selection_record.py index 6b4d65fe..79cc12d1 100644 --- a/src/backend/tests/evolution/test_skill_selection_record.py +++ b/src/backend/tests/evolution/test_skill_selection_record.py @@ -6,7 +6,7 @@ until the record exists. """ -from core.agent_skills import selection_record as SR +from core.capabilities.skills import selection_record as SR def test_rejected_candidates_are_recorded_with_a_reason(): @@ -53,7 +53,7 @@ class Boom: def explode(**_kw): raise RuntimeError("model down") - import core.agent_skills.selector as selector_mod + import core.llm.skill_selection as selector_mod original = selector_mod.select_skills_for_query selector_mod.select_skills_for_query = explode @@ -100,7 +100,7 @@ def test_selection_is_intersected_with_enabled_set(): ) assert selection.selected_ids == ["a"] - import core.agent_skills.selector as selector_mod + import core.llm.skill_selection as selector_mod original = selector_mod.select_skills_for_query selector_mod.select_skills_for_query = lambda **_kw: ["a", "not-enabled"] diff --git a/src/backend/tests/evolution/test_sop_judge.py b/src/backend/tests/evolution/test_sop_judge.py index d686aebf..5b544e92 100644 --- a/src/backend/tests/evolution/test_sop_judge.py +++ b/src/backend/tests/evolution/test_sop_judge.py @@ -9,7 +9,7 @@ import asyncio import core.memory.extractors._base as base -from core.evolution.sop_judge import SopVerdict, adjudicate_sequence_sop +from core.evolution.assessment.sop_judge import SopVerdict, adjudicate_sequence_sop def _judge(monkeypatch, reply): diff --git a/src/backend/tests/evolution/test_three_engines.py b/src/backend/tests/evolution/test_three_engines.py index 462b2d31..63dd7ab7 100644 --- a/src/backend/tests/evolution/test_three_engines.py +++ b/src/backend/tests/evolution/test_three_engines.py @@ -4,12 +4,17 @@ still wrong — the class of defect that survives both code review and a green suite, because the code runs and produces output nobody checks against reality. """ +from core.evolution.cycle import cycle_extras as cycle_extras_module +from core.evolution.cycle import cycle_extras_constants as cycle_extras_state_module +from core.evolution.cycle import extra_findings as extra_findings_module +from core.evolution.cycle import memory_findings as memory_findings_module +from core.evolution.cycle import usage_evidence as usage_evidence_module import pytest -from core.evolution import cycle_extras as X -from core.evolution import memory_scan as MS -from core.evolution import similarity as S +from core.evolution.cycle import cycle_extras as X +from core.evolution.generators import memory_scan as MS +from core.evolution.assessment import similarity as S # ── Intent clustering ──────────────────────────────────────────────────────── @@ -88,7 +93,7 @@ def _intent_episodes(n, *, tools_vary=True, verdict="success"): def test_a_recurring_intent_with_a_stable_plan_yields_no_candidate(monkeypatch): monkeypatch.setattr(S, "_embed_all", lambda texts: None) - result = X.intent_findings(_intent_episodes(4, tools_vary=False)) + result = extra_findings_module.intent_findings(_intent_episodes(4, tools_vary=False)) # Already effectively a skill; writing one down adds an asset to maintain # and changes nothing. assert not [f for f in result["findings"] if f["action"] == "skill_candidate"] @@ -96,15 +101,15 @@ def test_a_recurring_intent_with_a_stable_plan_yields_no_candidate(monkeypatch): def test_a_recurring_intent_replanned_every_time_does_yield_a_candidate(monkeypatch): monkeypatch.setattr(S, "_embed_all", lambda texts: None) - result = X.intent_findings(_intent_episodes(4)) + result = extra_findings_module.intent_findings(_intent_episodes(4)) findings = [f for f in result["findings"] if f["action"] == "skill_candidate"] assert findings, result - assert findings[0]["cluster"]["plan_variety"] >= X.MIN_PLAN_VARIETY + assert findings[0]["cluster"]["plan_variety"] >= cycle_extras_state_module.MIN_PLAN_VARIETY def test_a_recurring_intent_that_keeps_failing_is_not_distilled(monkeypatch): monkeypatch.setattr(S, "_embed_all", lambda texts: None) - result = X.intent_findings(_intent_episodes(4, verdict="failed")) + result = extra_findings_module.intent_findings(_intent_episodes(4, verdict="failed")) kinds = {f["kind"] for f in result["findings"]} assert "recurring_failure_intent" in kinds assert not [f for f in result["findings"] if f["action"] == "skill_candidate"] @@ -123,7 +128,7 @@ def test_usage_counts_opens_separately_from_offers(): } for i in range(5) ] - stats = X.skill_usage_from_views(views) + stats = usage_evidence_module.skill_usage_from_views(views) assert stats["offered-only"]["offers"] == 5 and stats["offered-only"]["opens"] == 0 assert stats["used"]["opens"] == 5 @@ -137,7 +142,7 @@ def test_follow_through_excludes_turns_it_cannot_measure(monkeypatch): this, do not retire it* — could never fire. """ monkeypatch.setattr( - X, "_declared_tools_for", lambda ids: {"evo-fin": ["dbhub_query", "chart"]} + usage_evidence_module, 'declared_tools_for', lambda ids: {"evo-fin": ["dbhub_query", "chart"]} ) views = [ # Followed: called a declared tool afterwards. @@ -150,7 +155,7 @@ def test_follow_through_excludes_turns_it_cannot_measure(monkeypatch): {"episode_id": "e2", "verdict": "success", "skill_sequence": ["evo-fin"], "skills_opened": [{"skill_id": "evo-fin", "tools_after_open": []}]}, ] - stats = X.skill_usage_from_views(views)["evo-fin"] + stats = usage_evidence_module.skill_usage_from_views(views)["evo-fin"] assert stats["opens"] == 3 assert stats["opens_measurable"] == 2 assert stats["follow_through"] == 0.5 @@ -168,7 +173,7 @@ def test_offers_count_availability_because_loading_offers_everything(): } for i in range(4) ] - stats = X.skill_usage_from_views(views) + stats = usage_evidence_module.skill_usage_from_views(views) assert stats["b"]["offers"] == 4 and stats["b"]["opens"] == 0 assert stats["a"]["offers"] == 4 and stats["a"]["opens"] == 4 @@ -190,7 +195,7 @@ def test_overlap_uses_declared_tools_not_co_occurrence(): } for i in range(6) ] - result = X.decremental_findings(views, declared_tools={ + result = extra_findings_module.decremental_findings(views, declared_tools={ "skill-a": ["read", "write"], "skill-b": ["chart", "export"], }) @@ -198,7 +203,7 @@ def test_overlap_uses_declared_tools_not_co_occurrence(): def test_genuinely_overlapping_skills_are_surfaced(): - result = X.decremental_findings([], skill_stats={}, declared_tools={ + result = extra_findings_module.decremental_findings([], skill_stats={}, declared_tools={ "skill-a": ["search", "verify", "chart"], "skill-b": ["search", "verify", "export"], }) @@ -221,7 +226,7 @@ def test_negative_transfer_is_reported_when_the_corpus_can_be_split(): {"episode_id": "a1", "task_type": "kb_qa", "verdict": "failed", "after": True}, {"episode_id": "a2", "task_type": "kb_qa", "verdict": "success", "after": True}, ] - findings = X.negative_transfer_findings(views, asset_id="evo-x") + findings = extra_findings_module.negative_transfer_findings(views, asset_id="evo-x") assert findings, "an unrelated task type dropping must be reported" @@ -239,10 +244,10 @@ def test_one_sided_evidence_reports_nothing_rather_than_a_phantom_drop(): } for i in range(6) ] - split = [{**v, "after": X._is_after(v["created_at"], activated)} for v in views] + split = [{**v, "after": usage_evidence_module.is_after(v["created_at"], activated)} for v in views] assert all(v["after"] for v in split) # Every episode on one side → the detector has no baseline to compare with. - assert X.negative_transfer_findings(split, asset_id="evo-x") == [] + assert extra_findings_module.negative_transfer_findings(split, asset_id="evo-x") == [] def test_uplift_needs_both_arms_before_it_licenses_a_rollback(monkeypatch): @@ -255,13 +260,13 @@ def test_uplift_needs_both_arms_before_it_licenses_a_rollback(monkeypatch): started = datetime(2026, 7, 1, tzinfo=timezone.utc) monkeypatch.setattr( - X, "_uplift_from_canary", lambda views: {} # no canary in flight + usage_evidence_module, 'uplift_from_canary', lambda views: {} # no canary in flight ) views = [ {"episode_id": "e0", "verdict": "success", "skill_sequence": ["evo-x"], "skills_opened": [], "created_at": started} ] - assert X.skill_usage_from_views(views)["evo-x"]["uplift"] is None + assert usage_evidence_module.skill_usage_from_views(views)["evo-x"]["uplift"] is None def test_a_canary_comparison_uses_the_same_bucketing_the_runtime_used(): @@ -273,7 +278,7 @@ def test_a_canary_comparison_uses_the_same_bucketing_the_runtime_used(): later = started + timedelta(days=1) ramping = [("evo-x", "rel-1", 50, started)] - from core.evolution.release import in_canary_bucket + from core.evolution.cycle.release import in_canary_bucket # Split 40 users the way the runtime would have, then make the exposed arm # fail and the withheld arm succeed. @@ -288,7 +293,7 @@ def test_a_canary_comparison_uses_the_same_bucketing_the_runtime_used(): } for i in range(40) ] - uplift = X._uplift_from_canary(views, ramping=ramping) + uplift = usage_evidence_module.uplift_from_canary(views, ramping=ramping) assert uplift["evo-x"] is not None and uplift["evo-x"] < -0.5 @@ -307,7 +312,7 @@ def test_a_thin_canary_comparison_reports_nothing_rather_than_a_number(): } for i in range(4) ] - uplift = X._uplift_from_canary(views, ramping=[("evo-x", "rel-1", 50, started)]) + uplift = usage_evidence_module.uplift_from_canary(views, ramping=[("evo-x", "rel-1", 50, started)]) assert uplift["evo-x"] is None @@ -325,7 +330,7 @@ def test_frequent_retrieval_is_never_treated_as_duplication(): {"episode_id": f"e{i}", "verdict": "success", "memory_refs": ["mref-useful"]} for i in range(8) ] - result = X.memory_findings(views) + result = memory_findings_module.memory_findings(views) assert result["merge_proposals"] == 0 assert result["reweight_proposals"] == 0 @@ -337,7 +342,7 @@ def test_duplication_comes_from_scanning_the_store(): memory_ids=["m1", "m2"], reason="两条记忆语义重复", ) - result = X.memory_findings([], scan_findings=[finding]) + result = memory_findings_module.memory_findings([], scan_findings=[finding]) assert result["merge_proposals"] == 1 # The survivor is untouched; the loser stops being injected. op = result["needs_review"][0] @@ -350,7 +355,7 @@ def test_a_memory_retrieved_into_failures_is_down_weighted_not_deleted(): {"episode_id": f"e{i}", "verdict": "failed", "memory_refs": ["mref-noisy"]} for i in range(6) ] - result = X.memory_findings(views) + result = memory_findings_module.memory_findings(views) ops = result["auto_applicable"] + result["needs_review"] assert any(op["operation"] == "reweight" for op in ops) assert not any(op["operation"] == "delete" for op in ops) @@ -364,7 +369,7 @@ def test_contradictions_are_reported_for_a_human_not_resolved_by_recency(): memory_ids=["m1", "m2"], reason="方向相反", ) - result = X.memory_findings([], scan_findings=[finding]) + result = memory_findings_module.memory_findings([], scan_findings=[finding]) assert result["contradictions"] and result["merge_proposals"] == 0 @@ -372,7 +377,7 @@ def test_content_changing_memory_ops_require_review(): finding = MS.ScanFinding( kind=MS.FINDING_DUPLICATE, refs=["a", "b"], memory_ids=["1", "2"], reason="" ) - result = X.memory_findings([], scan_findings=[finding]) + result = memory_findings_module.memory_findings([], scan_findings=[finding]) assert all(op["operation"] != "merge" for op in result["auto_applicable"]) assert any(op["operation"] == "merge" for op in result["needs_review"]) @@ -425,5 +430,5 @@ def test_only_procedural_records_are_eligible_for_compilation(): def test_run_extras_returns_all_three_sections_and_survives_a_bad_one(): - result = X.run_extras([]) + result = cycle_extras_module.run_extras([]) assert set(result) == {"intent", "decremental", "memory"} diff --git a/src/backend/tests/evolution/test_trigger.py b/src/backend/tests/evolution/test_trigger.py index b560cbaa..0488e88a 100644 --- a/src/backend/tests/evolution/test_trigger.py +++ b/src/backend/tests/evolution/test_trigger.py @@ -3,12 +3,13 @@ The point of these is the *negative* cases: the trigger must decline far more often than it fires, and every decline must carry a reason an operator can read. """ +from core.evolution.cycle import loop as loop_module from datetime import datetime, timedelta, timezone import pytest -from core.evolution import trigger as T +from core.evolution.cycle import trigger as T @pytest.fixture(autouse=True) @@ -98,14 +99,14 @@ def test_run_cycle_returns_none_when_not_due(monkeypatch): def test_cycle_failure_is_swallowed_and_releases_the_lock(monkeypatch): monkeypatch.setattr(T, "_episode_count", lambda: 50) - import core.evolution.loop as L + import core.evolution.cycle.loop as L monkeypatch.setattr( - L, "run_evolution_cycle", lambda **kw: (_ for _ in ()).throw(RuntimeError("boom")) + loop_module, 'run_evolution_cycle', lambda **kw: (_ for _ in ()).throw(RuntimeError("boom")) ) assert T.run_cycle_if_due() is None # The in-flight flag must not stay stuck, or nothing would ever run again. - assert T._running is False + assert T.process_state().running is False def test_status_explains_why_it_has_not_run(monkeypatch): diff --git a/src/backend/tests/evolution/test_turn_card_edits.py b/src/backend/tests/evolution/test_turn_card_edits.py index b3d6ed3c..49de3161 100644 --- a/src/backend/tests/evolution/test_turn_card_edits.py +++ b/src/backend/tests/evolution/test_turn_card_edits.py @@ -12,8 +12,8 @@ from core.db.engine import Base from core.db.models.chat import ChatMessage, ChatSession -from core.evolution import settlement_store as SS -from core.evolution.settlement import LAYER_PROCEDURE, settle_turn +from core.evolution.settlement import settlement_store as SS +from core.evolution.settlement.settlement import LAYER_PROCEDURE, settle_turn @pytest.fixture diff --git a/src/backend/tests/evolution/test_user_contribution.py b/src/backend/tests/evolution/test_user_contribution.py index 6200fd52..3eb4337d 100644 --- a/src/backend/tests/evolution/test_user_contribution.py +++ b/src/backend/tests/evolution/test_user_contribution.py @@ -5,6 +5,10 @@ never reach cross-user pattern mining. A setting that only changed a label would be a promise the code could not keep. """ +from core.evolution.cycle import episode_views as episode_views_module +from core.evolution.profiles import personal_review as personal_review_module +from core.evolution.profiles import user_settings as user_settings_module +from core.evolution.profiles import user_settings_constants as user_settings_state_module import pytest from sqlalchemy import create_engine @@ -16,7 +20,7 @@ EvolutionEpisode, EvolutionTraceEvent, ) -from core.evolution import user_settings as US +from core.evolution.profiles import user_settings as US @pytest.fixture @@ -32,6 +36,8 @@ def db(monkeypatch): ) factory = sessionmaker(bind=engine) monkeypatch.setattr(US, "SessionLocal", factory, raising=False) + from core.evolution.profiles import personal_review + monkeypatch.setattr(personal_review, "SessionLocal", factory, raising=False) session = factory() yield session session.close() @@ -45,30 +51,30 @@ def db(monkeypatch): def test_participation_follows_the_single_evolution_switch(): - assert US.is_contribution_enabled({"evolution_prefs": {"enabled": True}}) is True - assert US.is_contribution_enabled({"evolution_prefs": {"enabled": False}}) is False + assert user_settings_module.is_contribution_enabled({"evolution_prefs": {"enabled": True}}) is True + assert user_settings_module.is_contribution_enabled({"evolution_prefs": {"enabled": False}}) is False def test_participation_defaults_to_off_until_evolution_is_enabled(): """Consent-first: a user who never enabled evolution contributes nothing.""" - assert US.is_contribution_enabled({}) is False - assert US.is_contribution_enabled(None) is False + assert user_settings_module.is_contribution_enabled({}) is False + assert user_settings_module.is_contribution_enabled(None) is False def test_legacy_contribution_key_no_longer_overrides_the_switch(): - metadata = {US.SETTING_KEY: True} # stale value from the two-switch era - assert US.is_contribution_enabled(metadata) is False + metadata = {user_settings_state_module.SETTING_KEY: True} # stale value from the two-switch era + assert user_settings_module.is_contribution_enabled(metadata) is False def test_opting_out_marks_episodes_private(): - assert US.privacy_class_for({"evolution_prefs": {"enabled": False}}) == US.PRIVACY_PRIVATE - assert US.privacy_class_for({"evolution_prefs": {"enabled": True}}) == US.PRIVACY_TENANT + assert user_settings_module.privacy_class_for({"evolution_prefs": {"enabled": False}}) == user_settings_state_module.PRIVACY_PRIVATE + assert user_settings_module.privacy_class_for({"evolution_prefs": {"enabled": True}}) == user_settings_state_module.PRIVACY_TENANT def test_missing_user_resolves_to_the_default_rather_than_failing(): - resolved = US.resolve_for_user("") - assert resolved[US.SETTING_KEY] is False - assert resolved["privacy_class"] == US.PRIVACY_PRIVATE + resolved = user_settings_module.resolve_for_user("") + assert resolved[user_settings_state_module.SETTING_KEY] is False + assert resolved["privacy_class"] == user_settings_state_module.PRIVACY_PRIVATE # ── Enforcement: the part that makes the promise real ──────────────────────── @@ -88,16 +94,16 @@ def _episode(db, *, episode_id, user, privacy): def test_private_episodes_are_excluded_from_pattern_mining(monkeypatch, db): """An opted-out user's traces must not feed cross-user learning.""" - from core.evolution import loop as L + from core.evolution.cycle import loop as L - _episode(db, episode_id="pub-1", user="u1", privacy=US.PRIVACY_TENANT) - _episode(db, episode_id="priv-1", user="u2", privacy=US.PRIVACY_PRIVATE) + _episode(db, episode_id="pub-1", user="u1", privacy=user_settings_state_module.PRIVACY_TENANT) + _episode(db, episode_id="priv-1", user="u2", privacy=user_settings_state_module.PRIVACY_PRIVATE) db.commit() factory = sessionmaker(bind=db.get_bind()) - monkeypatch.setattr(L, "SessionLocal", factory) + monkeypatch.setattr(episode_views_module, "SessionLocal", factory) - views = L.load_episode_views(limit=50) + views = episode_views_module.load_episode_views(limit=50) ids = {v["episode_id"] for v in views} assert "pub-1" in ids assert "priv-1" not in ids @@ -105,7 +111,7 @@ def test_private_episodes_are_excluded_from_pattern_mining(monkeypatch, db): def test_opted_out_episodes_are_still_recorded_for_their_owner(db): """Opting out is not deletion — the user keeps their own card and history.""" - _episode(db, episode_id="priv-1", user="u2", privacy=US.PRIVACY_PRIVATE) + _episode(db, episode_id="priv-1", user="u2", privacy=user_settings_state_module.PRIVACY_PRIVATE) db.commit() assert db.query(EvolutionEpisode).filter_by(episode_id="priv-1").first() is not None @@ -113,12 +119,12 @@ def test_opted_out_episodes_are_still_recorded_for_their_owner(db): def test_turning_it_off_does_not_retroactively_withdraw_evidence(db, monkeypatch): """Silently rewriting history other candidates were derived from would make those candidates unexplainable.""" - _episode(db, episode_id="old-1", user="u1", privacy=US.PRIVACY_TENANT) + _episode(db, episode_id="old-1", user="u1", privacy=user_settings_state_module.PRIVACY_TENANT) db.commit() # Simulate the user switching off afterwards. row = db.query(EvolutionEpisode).filter_by(episode_id="old-1").first() - assert row.privacy_class == US.PRIVACY_TENANT # unchanged by the toggle + assert row.privacy_class == user_settings_state_module.PRIVACY_TENANT # unchanged by the toggle # ── Contribution summary ───────────────────────────────────────────────────── @@ -126,12 +132,14 @@ def test_turning_it_off_does_not_retroactively_withdraw_evidence(db, monkeypatch def test_summary_separates_contributed_from_private(db, monkeypatch): monkeypatch.setattr(US, "SessionLocal", sessionmaker(bind=db.get_bind())) - _episode(db, episode_id="a", user="u1", privacy=US.PRIVACY_TENANT) - _episode(db, episode_id="b", user="u1", privacy=US.PRIVACY_TENANT) - _episode(db, episode_id="c", user="u1", privacy=US.PRIVACY_PRIVATE) + from core.evolution.profiles import personal_review + monkeypatch.setattr(personal_review, "SessionLocal", sessionmaker(bind=db.get_bind())) + _episode(db, episode_id="a", user="u1", privacy=user_settings_state_module.PRIVACY_TENANT) + _episode(db, episode_id="b", user="u1", privacy=user_settings_state_module.PRIVACY_TENANT) + _episode(db, episode_id="c", user="u1", privacy=user_settings_state_module.PRIVACY_PRIVATE) db.commit() - summary = US.summarise_contributions("u1") + summary = personal_review_module.summarise_contributions("u1") assert summary["episodes"] == 3 assert summary["contributed_episodes"] == 2 assert summary["private_episodes"] == 1 @@ -139,7 +147,9 @@ def test_summary_separates_contributed_from_private(db, monkeypatch): def test_summary_links_candidates_back_to_the_users_own_episodes(db, monkeypatch): monkeypatch.setattr(US, "SessionLocal", sessionmaker(bind=db.get_bind())) - _episode(db, episode_id="a", user="u1", privacy=US.PRIVACY_TENANT) + from core.evolution.profiles import personal_review + monkeypatch.setattr(personal_review, "SessionLocal", sessionmaker(bind=db.get_bind())) + _episode(db, episode_id="a", user="u1", privacy=user_settings_state_module.PRIVACY_TENANT) db.add( EvolutionCandidate( candidate_id="c1", @@ -155,7 +165,7 @@ def test_summary_links_candidates_back_to_the_users_own_episodes(db, monkeypatch ) db.commit() - summary = US.summarise_contributions("u1") + summary = personal_review_module.summarise_contributions("u1") assert len(summary["candidates"]) == 1 item = summary["candidates"][0] # The provenance chain: how much of this capability came from this user. @@ -167,7 +177,9 @@ def test_summary_links_candidates_back_to_the_users_own_episodes(db, monkeypatch def test_candidates_with_no_shared_evidence_are_not_claimed(db, monkeypatch): monkeypatch.setattr(US, "SessionLocal", sessionmaker(bind=db.get_bind())) - _episode(db, episode_id="a", user="u1", privacy=US.PRIVACY_TENANT) + from core.evolution.profiles import personal_review + monkeypatch.setattr(personal_review, "SessionLocal", sessionmaker(bind=db.get_bind())) + _episode(db, episode_id="a", user="u1", privacy=user_settings_state_module.PRIVACY_TENANT) db.add( EvolutionCandidate( candidate_id="c2", @@ -181,16 +193,16 @@ def test_candidates_with_no_shared_evidence_are_not_claimed(db, monkeypatch): ) ) db.commit() - assert US.summarise_contributions("u1")["candidates"] == [] + assert personal_review_module.summarise_contributions("u1")["candidates"] == [] def test_summary_degrades_to_empty_rather_than_raising(monkeypatch): monkeypatch.setattr( US, "SessionLocal", lambda: (_ for _ in ()).throw(RuntimeError("db down")) ) - summary = US.summarise_contributions("u1") + summary = personal_review_module.summarise_contributions("u1") assert summary["episodes"] == 0 and summary["candidates"] == [] def test_summary_for_a_user_with_no_episodes(): - assert US.summarise_contributions("nobody")["candidates"] == [] + assert personal_review_module.summarise_contributions("nobody")["candidates"] == [] diff --git a/src/backend/tests/features/__init__.py b/src/backend/tests/features/__init__.py new file mode 100644 index 00000000..6a378ddf --- /dev/null +++ b/src/backend/tests/features/__init__.py @@ -0,0 +1 @@ +"""Product module tests.""" diff --git a/src/backend/tests/features/automation/__init__.py b/src/backend/tests/features/automation/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/src/backend/tests/features/automation/__init__.py @@ -0,0 +1 @@ + diff --git a/src/backend/tests/features/automation/run_fixtures.py b/src/backend/tests/features/automation/run_fixtures.py new file mode 100644 index 00000000..84e00d71 --- /dev/null +++ b/src/backend/tests/features/automation/run_fixtures.py @@ -0,0 +1,15 @@ +"""Construct interrupted execution state for ownership/fault-injection tests.""" +from uuid import uuid4 +from core.infra.time import utc_now +from features.automation.models import ScheduledTaskRun + + +def running_run(db, task_id, lease_token=None): + run = ScheduledTaskRun( + run_id="run_" + uuid4().hex[:16], task_id=task_id, + status="running", lease_token=lease_token, started_at=utc_now(), + ) + db.add(run) + db.commit() + db.refresh(run) + return run diff --git a/src/backend/tests/features/automation/test_automation_delivery_deadline.py b/src/backend/tests/features/automation/test_automation_delivery_deadline.py new file mode 100644 index 00000000..4f47a2db --- /dev/null +++ b/src/backend/tests/features/automation/test_automation_delivery_deadline.py @@ -0,0 +1,179 @@ +"""Delivery deadlines cannot rewrite business outcomes or start later sends.""" +from tests.orchestration.automation_test_support import execute_queued + +import asyncio +import threading +from types import SimpleNamespace + +import pytest +from orchestration.automations import automation_delivery as module +from orchestration.automations.automation_scheduler import AutomationScheduler + +pytest_plugins = ["tests.orchestration.test_automation_live_conversations"] + + +async def test_swallowed_notification_cancel_cannot_start_channel_delivery(monkeypatch): + from integrations.channels import outbound + + monkeypatch.setattr(module, "DELIVERY_TIMEOUT_SECONDS", 0.01) + resumed, stopped = asyncio.Event(), asyncio.Event() + calls = [] + + async def notification(*args): + try: + await asyncio.Event().wait() + except asyncio.CancelledError: + await resumed.wait() + stopped.set() + + async def send(*args, **kwargs): + calls.append(args) + return True + + monkeypatch.setattr(outbound, "deliver_to_conversation", send) + await module.deliver_result( + SimpleNamespace(_send_notification=notification), + "u", + "t", + "n", + "success", + "text", + metadata={ + "delivery_targets": [ + {"type": "inapp"}, + {"type": "channel", "channel_id": "c", "conversation_id": "room"}, + ] + }, + ) + resumed.set() + await asyncio.wait_for(stopped.wait(), 1) + await asyncio.sleep(0) + assert calls == [] + + +async def test_artifact_download_does_not_block_deadline_or_send_late(monkeypatch): + from integrations.channels import outbound + from orchestration.automations import automation_artifacts + + monkeypatch.setattr(module, "DELIVERY_TIMEOUT_SECONDS", 0.02) + entered, finish = threading.Event(), threading.Event() + calls = [] + + def load(chat_id): + entered.set() + assert finish.wait(2) + return [] + + async def send(*args, **kwargs): + calls.append(args) + return True + + monkeypatch.setattr(automation_artifacts, "load_run_files", load) + monkeypatch.setattr(outbound, "deliver_to_conversation", send) + try: + await module.deliver_result( + SimpleNamespace(), + "u", + "t", + "n", + "success", + "text", + "chat", + { + "delivery_targets": [ + {"type": "channel", "channel_id": "c", "conversation_id": "room"} + ] + }, + ) + assert entered.is_set() + assert calls == [] + finally: + finish.set() + await asyncio.sleep(0.01) + assert calls == [] + + +async def test_delivery_error_keeps_committed_success(live_env, monkeypatch): + from features.automation.service import AutomationService + from orchestration.automations import automation_conversation + + factory, _, task_id = live_env + + async def prompt(**kwargs): + return "chat", "answer", {} + + async def fail(*args): + raise RuntimeError("delivery unavailable") + + monkeypatch.setattr(automation_conversation, "execute_prompt", prompt) + scheduler = AutomationScheduler() + monkeypatch.setattr(scheduler, "_send_notification", fail) + await execute_queued(scheduler, task_id, "owner") + with factory() as db: + service = AutomationService(db) + assert service.get_task_runs(task_id, "owner")[0].status == "success" + assert service.get_task_by_id(task_id).consecutive_failures == 0 + assert not scheduler._leases + + +async def test_stale_executor_cannot_update_task_or_notify(live_env, monkeypatch): + from features.automation.models import ScheduledTaskRun + from features.automation.runs import AutomationRunRepository + from features.automation.service import AutomationService + from orchestration.automations import automation_conversation + + factory, _, task_id = live_env + notifications = [] + + async def prompt(**kwargs): + with factory() as db: + run = db.query(ScheduledTaskRun).filter_by(task_id=task_id).one() + assert AutomationRunRepository(db).claim( + run.run_id, "new-owner", expected_token=run.lease_token + ) + return "chat", "stale answer", {} + + async def notify(*args): + notifications.append(args) + + monkeypatch.setattr(automation_conversation, "execute_prompt", prompt) + scheduler = AutomationScheduler() + monkeypatch.setattr(scheduler, "_send_notification", notify) + await execute_queued(scheduler, task_id, "owner") + with factory() as db: + service = AutomationService(db) + run = service.get_task_runs(task_id, "owner")[0] + assert run.status == "running" + assert run.lease_token == "new-owner" + assert run.result_summary is None + assert service.get_task_by_id(task_id).last_run_at is None + assert notifications == [] + + +async def test_channel_exception_does_not_skip_other_targets(monkeypatch): + from integrations.channels import outbound + + calls = [] + + async def send(channel, *args, **kwargs): + calls.append(channel) + if channel == "bad": + raise RuntimeError("adapter lookup failed") + return True + + monkeypatch.setattr(outbound, "deliver_to_conversation", send) + await module.deliver_result( + SimpleNamespace(), + "u", + "t", + "n", + "success", + "text", + metadata={ + "delivery_targets": [ + {"type": "channel", "channel_id": name, "conversation_id": "room"} + for name in ("bad", "good") + ] + }, + ) + assert calls == ["bad", "good"] diff --git a/src/backend/tests/test_automation_execution_location.py b/src/backend/tests/features/automation/test_automation_execution_location.py similarity index 80% rename from src/backend/tests/test_automation_execution_location.py rename to src/backend/tests/features/automation/test_automation_execution_location.py index 218346b7..cc33b624 100644 --- a/src/backend/tests/test_automation_execution_location.py +++ b/src/backend/tests/features/automation/test_automation_execution_location.py @@ -1,9 +1,18 @@ +from core.chat.projection.codec import chat_projection_codec +from features.desktop.bridge import client_state as bridge_client_state +from features.desktop.capabilities import skills_state + +import core.agent_runtime.runner as runner +import core.chat.runtime.admission as admission +import core.db.engine as db_engine +import core.runs.events.store as store +import orchestration.loops.admission as loops_admission import pytest -from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker from core.db.engine import Base from core.db.models import Project -from core.services.automation_service import AutomationService +from features.automation.service import AutomationService +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker @pytest.fixture @@ -31,15 +40,16 @@ def desktop_identity(local_project, monkeypatch): import base64 import json import time - from core.capabilities import registry, skills + + from features.desktop.capabilities import registry, skills from core.db.models import UserShadow - from core.services import desktop_cloud_bridge as bridge + from features.desktop.bridge import client as bridge factory, _ = local_project monkeypatch.setattr(registry, "SessionLocal", factory) - monkeypatch.setattr(skills, "_shadow_user_ids", {}) - monkeypatch.setattr(bridge, "_state_loaded", True) - monkeypatch.setattr(bridge, "_state", None) + monkeypatch.setattr(skills_state, 'shadow_user_ids', {}) + monkeypatch.setattr(bridge_client_state, 'state_loaded', True) + monkeypatch.setattr(bridge_client_state, 'state', None) with factory() as db: db.add(UserShadow(user_id="owner", username="Alice", user_center_id="cloud:example.com:443:account-a")) @@ -51,13 +61,13 @@ def switch(account="account-a", subject="cloud-owner", cloud_base="https://examp claims = base64.urlsafe_b64encode(json.dumps( {"u": subject, "c": account, "d": "test-device", "a": 1} ).encode()).decode().rstrip("=") - bridge._state = {"cloud_base": cloud_base, "token": f"dcap2.{claims}.fixture", + bridge_client_state.state = {"cloud_base": cloud_base, "token": f"dcap2.{claims}.fixture", "expires_at": time.time() + 600} switch() return switch def test_local_task_captures_project_and_revalidates_binding(local_project, monkeypatch): - from core.services.automation_execution import task_execution_context + from features.automation.execution import task_execution_context factory, path = local_project with factory() as db: service = AutomationService(db) @@ -90,20 +100,20 @@ def test_local_task_captures_project_and_revalidates_binding(local_project, monk @pytest.mark.asyncio async def test_scheduler_restores_project_into_real_run_context(local_project, monkeypatch): factory, path = local_project - from orchestration.schedulers.automation_conversation import execute_prompt - monkeypatch.setattr("core.services.user_model_selection.resolve_effective_chat_model_name", lambda: "test") - monkeypatch.setattr("core.services.ontology_service.build_user_ontology_runtime", lambda **kw: (False, {})) + from orchestration.automations.automation_conversation import execute_prompt + monkeypatch.setattr('core.models.selection.resolve_effective_chat_model_name', lambda: "test") + monkeypatch.setattr('core.ontology.runtime_policy.build_user_ontology_runtime', lambda **kw: (False, {})) captured = [] async def workflow(**kwargs): captured.append(kwargs["context"]) yield {"type": "content", "delta": "report complete"} yield {"type": "meta", "usage": {}} - monkeypatch.setattr("orchestration.workflow.astream_chat_workflow", workflow) - from orchestration import chat_run_executor, run_event_stream - monkeypatch.setattr(chat_run_executor, "SessionLocal", factory) - monkeypatch.setattr(chat_run_executor, "astream_chat_workflow", workflow) - stream = run_event_stream.LocalRunEventStream() - monkeypatch.setattr(chat_run_executor, "get_run_event_stream", lambda: stream) + monkeypatch.setattr('core.agent_runtime.runner.astream_chat_workflow', workflow) + import core.runs.events.store as run_event_stream + monkeypatch.setattr(db_engine, "SessionLocal", factory) + monkeypatch.setattr(runner, "astream_chat_workflow", workflow) + stream = run_event_stream.LocalRunEventStream(chat_projection_codec()) + monkeypatch.setattr(store, "get_run_event_stream", lambda: stream) with factory() as db: task = AutomationService(db).create_task( user_id="owner", task_type="prompt", prompt="summarize project", @@ -126,10 +136,12 @@ async def workflow(**kwargs): async def test_local_conversation_creation_uses_authorized_project(local_project, desktop_identity): import json from types import SimpleNamespace + from core.db.models import ChatSession - from core.services.run_journal import RunJournal - from core.services.tool_effect_ledger import ToolEffectJournal, CURRENT_TOOL_EFFECT - from core.services.automation_tool_routing import creation_arguments, local_automation_result + from features.automation.tool_routing import creation_arguments, local_automation_result + from core.runs.journal import RunJournal + from core.harness.tool_effect_context import CURRENT_TOOL_EFFECT + from core.runs.tool_effects.journal import ToolEffectJournal factory, _ = local_project with factory() as db: db.add(ChatSession(chat_id="bound-chat", user_id="owner", title="report", project_id="local-project")) @@ -159,7 +171,7 @@ async def revoked(): {"task_ref": data["task"]["task_id"], "execution_location": "local"}, {"x-chat-id": "bound-chat"}, authorize=revoked) with factory() as db: - from core.db.models import ScheduledTask + from features.automation.models import ScheduledTask assert db.get(ScheduledTask, data["task"]["task_id"]) is not None desktop_identity("account-b", "cloud-other") with pytest.raises(ValueError, match="账号"): @@ -170,7 +182,7 @@ async def revoked(): def test_public_urls_are_not_host_paths(): - from core.services.automation_execution import references_host_path + from features.automation.execution import references_host_path assert not references_host_path("Summarize https://example.com/news") assert references_host_path(r"Read C:\Users\Aaron") assert references_host_path("Read /home/aaron/project") @@ -190,9 +202,9 @@ def local_chat(local_project, desktop_identity): @pytest.mark.asyncio @pytest.mark.parametrize("change", ["switch", "logout", "server"]) async def test_local_task_rejects_identity_change_during_authorization(local_chat, change): - from core.capabilities.errors import CloudUnavailable - from core.services import desktop_cloud_bridge as bridge - from core.services.automation_tool_routing import local_automation_result + from core.harness.capability_errors import CloudUnavailable + from features.desktop.bridge import client as bridge + from features.automation.tool_routing import local_automation_result from extensions.mcp_servers.automation_task_mcp import impl async def authorize(): @@ -201,7 +213,7 @@ async def authorize(): elif change == "server": local_chat(cloud_base="https://other.example.com") else: - bridge._state = None + bridge_client_state.state = None with pytest.raises(CloudUnavailable): await local_automation_result( @@ -219,7 +231,7 @@ async def authorize(): ]) @pytest.mark.parametrize("identity", ["other", "missing", "other_server"]) async def test_local_task_tools_reject_unmapped_or_other_account(local_chat, tool, identity): - from core.services.automation_tool_routing import local_automation_result + from features.automation.tool_routing import local_automation_result if identity == "other": local_chat("account-b", "cloud-other") elif identity == "missing": @@ -240,7 +252,8 @@ async def authorize(): @pytest.mark.asyncio async def test_mapped_account_can_manage_local_tasks(local_chat): import json - from core.services.automation_tool_routing import local_automation_result + + from features.automation.tool_routing import local_automation_result async def authorize(): return None @@ -267,8 +280,9 @@ async def call(tool, **arguments): @pytest.mark.asyncio async def test_local_task_rejects_account_switch_before_queued_dispatch(local_chat, monkeypatch): import asyncio - from core.capabilities.errors import CloudUnavailable - from core.services.automation_tool_routing import local_automation_result + + from core.harness.capability_errors import CloudUnavailable + from features.automation.tool_routing import local_automation_result from extensions.mcp_servers.automation_task_mcp import impl original = asyncio.to_thread @@ -291,23 +305,23 @@ async def authorize(): @pytest.mark.asyncio async def test_scheduled_loop_passes_authorized_local_project_to_executor(local_project, monkeypatch): - from core.services.loop_service import LoopService - from orchestration import chat_run_executor, run_event_stream - from orchestration.schedulers.automation_conversation import execute_loop + import core.runs.events.store as run_event_stream + from features.automation.loops import LoopService + from orchestration.automations.automation_conversation import execute_loop factory, _ = local_project - monkeypatch.setattr(chat_run_executor, "SessionLocal", factory) - monkeypatch.setattr(chat_run_executor, "get_run_event_stream", lambda: run_event_stream.LocalRunEventStream()) + monkeypatch.setattr(db_engine, "SessionLocal", factory) + monkeypatch.setattr(store, "get_run_event_stream", lambda: run_event_stream.LocalRunEventStream(chat_projection_codec())) async def workflow(**kwargs): yield {"type": "content", "delta": "Loop report"} yield {"type": "meta", "usage": {}} - monkeypatch.setattr(chat_run_executor, "astream_chat_workflow", workflow) + monkeypatch.setattr(runner, "astream_chat_workflow", workflow) captured = {} async def start_loop(**kwargs): captured.update(kwargs) - return await chat_run_executor.start_run(chat_id=kwargs["chat_id"], user_id=kwargs["user_id"], + return await admission.start_run(chat_id=kwargs["chat_id"], user_id=kwargs["user_id"], session_messages=[], effective_user_message="Research", raw_user_message="Research", context={"user_id": kwargs["user_id"]}, model_name="test") - monkeypatch.setattr(chat_run_executor, "start_autonomous_loop_run", start_loop) + monkeypatch.setattr(loops_admission, "start_autonomous_loop_run", start_loop) with factory() as db: loop = LoopService(db).create_loop(user_id="owner", title="Research", goal_spec={"objective": "Research"}, budget={}, project_id="local-project") diff --git a/src/backend/tests/features/automation/test_automation_lease_close.py b/src/backend/tests/features/automation/test_automation_lease_close.py new file mode 100644 index 00000000..26baac03 --- /dev/null +++ b/src/backend/tests/features/automation/test_automation_lease_close.py @@ -0,0 +1,199 @@ +"""Release remains bounded even when renewal suppresses cancellation.""" + +import asyncio + +import pytest +from core.infra import ephemeral +from orchestration.automations import automation_lease as module +from orchestration.automations.automation_scheduler import AutomationScheduler + + +class ControlledState(ephemeral.LocalEphemeralState): + def __init__(self): + super().__init__() + self.entered = asyncio.Event() + self.cancelled = asyncio.Event() + self.finish = asyncio.Event() + self.renewals = 0 + self.deletions = 0 + + async def compare_exchange(self, *args, **kwargs): + self.renewals += 1 + self.entered.set() + try: + await self.finish.wait() + except asyncio.CancelledError: + self.cancelled.set() + await self.finish.wait() # deliberately swallow the cancellation + return await super().compare_exchange(*args, **kwargs) + + async def release(self, *args): + self.deletions += 1 + return await super().release(*args) + + +@pytest.fixture +def state(monkeypatch): + value = ControlledState() + monkeypatch.setattr(ephemeral, "get_ephemeral_state", lambda: value) + monkeypatch.setattr(module, "RENEW_INTERVAL_SECONDS", 0.001) + monkeypatch.setattr(module, "RELEASE_WAIT_SECONDS", 0.01) + return value + + +async def bounded(awaitable): + return await asyncio.wait_for(awaitable, 1) + + +async def test_release_wakes_sleep_and_is_idempotent(state, monkeypatch): + monkeypatch.setattr(module, "RENEW_INTERVAL_SECONDS", 3600) + lease = module.AutomationLease("sleep") + assert await lease.acquire() + await bounded(asyncio.gather(lease.release(), lease.release())) + await lease.release() + assert state.deletions == 1 + assert state.renewals == 0 + assert lease.renewal.done() + assert await state.get(lease.key) is None + + +async def test_swallowed_cancel_cannot_block_release_or_resurrect_lock(state): + lease = module.AutomationLease("race") + assert await lease.acquire() + await bounded(state.entered.wait()) + await bounded(lease.release()) + assert state.cancelled.is_set() + assert lease._release_task.done() + assert await state.get(lease.key) is None + assert await state.hold(lease.key, "new-owner", ttl=60) + state.finish.set() + await bounded(asyncio.gather(lease.renewal, return_exceptions=True)) + assert state.renewals == 1 + assert await state.get(lease.key) == "new-owner" + + +async def test_cancelled_close_caller_does_not_abort_shared_cleanup(state): + lease = module.AutomationLease("cancel") + assert await lease.acquire() + await bounded(state.entered.wait()) + caller = asyncio.create_task(lease.release()) + await bounded(state.cancelled.wait()) + caller.cancel() + with pytest.raises(asyncio.CancelledError): + await caller + await bounded(lease.release()) + assert state.deletions == 1 + state.finish.set() + await bounded(asyncio.gather(lease.renewal, return_exceptions=True)) + + +async def test_finished_execution_stops_renewal_without_release(state): + lease = module.AutomationLease("finished") + assert await lease.acquire() + + async def work(): + return "success" + + assert await lease.run(work()) == "success" + await bounded(asyncio.gather(lease.renewal, return_exceptions=True)) + assert state.renewals == 0 + # No explicit release: the unrenewed key will expire at its original TTL. + assert await state.get(lease.key) == lease.token + await lease.release() + + +async def test_finished_execution_during_cas_stops_another_renewal(state): + lease = module.AutomationLease("finished-cas") + assert await lease.acquire() + await bounded(state.entered.wait()) + + async def work(): + return "success" + + assert await lease.run(work()) == "success" + state.finish.set() + await bounded(asyncio.gather(lease.renewal, return_exceptions=True)) + assert state.renewals == 1 + await lease.release() + + +async def test_delete_timeout_does_not_leave_local_running_marker(state, monkeypatch): + monkeypatch.setattr(module, "RENEW_INTERVAL_SECONDS", 3600) + finish = asyncio.Event() + original = state.release + + async def slow_release(*args): + await finish.wait() + return await original(*args) + + monkeypatch.setattr(state, "release", slow_release) + scheduler = AutomationScheduler() + assert await scheduler._acquire_lock("delete-timeout") + await bounded(scheduler._release_lock("delete-timeout")) + assert not scheduler._leases + finish.set() + for _ in range(3): + await asyncio.sleep(0) + assert await state.get("jx:auto:lock:delete-timeout") is None + + +async def test_delete_error_does_not_leave_local_running_marker(state, monkeypatch): + async def fail(*args): + raise ConnectionError("store unavailable") + + monkeypatch.setattr(state, "release", fail) + scheduler = AutomationScheduler() + assert await scheduler._acquire_lock("error") + await bounded(scheduler._release_lock("error")) + assert not scheduler._leases + + +async def test_stop_is_bounded_with_cancellation_swallowing_renewal(state): + scheduler = AutomationScheduler() + assert await scheduler._acquire_lock("stop") + lease = scheduler._leases["stop"] + await bounded(state.entered.wait()) + await bounded(scheduler.stop()) + assert not scheduler._leases + assert await state.get(lease.key) is None + state.finish.set() + await bounded(asyncio.gather(lease.renewal, return_exceptions=True)) + + +async def test_redis_cas_cannot_recreate_deleted_lock_or_replace_new_owner(monkeypatch): + import fakeredis.aioredis + + client = fakeredis.aioredis.FakeRedis(decode_responses=True, protocol=2) + monkeypatch.setattr(ephemeral, "get_redis", lambda: client) + store = ephemeral.RedisEphemeralState() + try: + assert await store.hold("lease", "old", ttl=60) + assert await store.release("lease", "old") + assert not await store.compare_exchange("lease", "old", "old", ttl=60) + assert await store.get("lease") is None + assert await store.hold("lease", "new", ttl=60) + assert not await store.compare_exchange("lease", "old", "old", ttl=60) + assert not await store.release("lease", "old") + assert await store.get("lease") == "new" + finally: + await client.aclose() + + +async def test_stalled_renewal_stops_execution_even_if_cas_ignores_cancel(state): + lease = module.AutomationLease("stalled") + assert await lease.acquire() + stopped = asyncio.Event() + async def work(): + try: + await asyncio.Event().wait() + finally: + stopped.set() + with pytest.raises(RuntimeError, match="锁已失效"): + await bounded(lease.run(work())) + assert stopped.is_set() + await bounded(lease.release()) + assert await state.get(lease.key) is None + assert await state.hold(lease.key, "successor", ttl=60) + state.finish.set() + await asyncio.sleep(0.02) + assert await state.get(lease.key) == "successor" diff --git a/src/backend/tests/features/automation/test_automation_manual_trigger.py b/src/backend/tests/features/automation/test_automation_manual_trigger.py new file mode 100644 index 00000000..7f9b4da8 --- /dev/null +++ b/src/backend/tests/features/automation/test_automation_manual_trigger.py @@ -0,0 +1,81 @@ +"""Triggers and durable run acceptance commit or roll back together.""" +import pytest +from datetime import timedelta +from sqlalchemy import event +from core.infra.time import utc_now +from features.automation.admission import admit +from features.automation.models import ScheduledTaskRun +from features.automation.service import AutomationService + + +def _task(db, **kw): + svc = AutomationService(db) + return svc, svc.create_task(user_id="u1", task_type="prompt", prompt="run", + cron_expression="30 10 29 6 *", **kw) + + +def test_manual_acceptance_is_durable_and_does_not_move_schedule(db_session): + svc, task = _task(db_session) + before = task.next_run_at + svc.request_manual_trigger(task) + svc.request_manual_trigger(task) + ids = admit(db_session, 8) + assert len(ids) == 1 + run = db_session.get(ScheduledTaskRun, ids[0]) + assert run.status == "pending" + assert task.manual_trigger_at is None + assert task.run_count == 1 + assert task.next_run_at == before + assert admit(db_session, 8) == [] + + +def test_busy_task_retains_manual_trigger_for_after_completion(db_session): + svc, task = _task(db_session) + svc.request_manual_trigger(task) + run_id = admit(db_session, 8)[0] + svc.request_manual_trigger(task) + assert admit(db_session, 8) == [] + assert task.manual_trigger_at is not None + db_session.get(ScheduledTaskRun, run_id).status = "success" + db_session.commit() + assert len(admit(db_session, 8)) == 1 + assert task.run_count == 2 + + +def test_paused_task_manual_trigger_is_accepted(db_session): + svc, task = _task(db_session) + svc.pause_task(task.task_id, "u1") + svc.request_manual_trigger(task) + assert len(admit(db_session, 8)) == 1 + + +@pytest.mark.parametrize("manual", [True, False]) +def test_failed_acceptance_does_not_consume_trigger(db_session, manual): + svc, task = _task(db_session, schedule_type="once") + task.next_run_at = utc_now() - timedelta(seconds=5) + if manual: + svc.request_manual_trigger(task) + db_session.commit() + before = task.next_run_at + def fail(*args): + raise RuntimeError("storage failed") + event.listen(db_session, "before_commit", fail) + try: + with pytest.raises(RuntimeError): + admit(db_session, 8) + finally: + event.remove(db_session, "before_commit", fail) + db_session.rollback() + db_session.refresh(task) + assert task.run_count == 0 + assert task.next_run_at == before + assert bool(task.manual_trigger_at) == manual + assert db_session.query(ScheduledTaskRun).count() == 0 + + +def test_admission_is_bounded(db_session): + for _ in range(4): + svc, task = _task(db_session) + svc.request_manual_trigger(task) + assert len(admit(db_session, 2)) == 2 + assert len(admit(db_session, 2)) == 2 diff --git a/src/backend/tests/services/test_automation_notifications.py b/src/backend/tests/features/automation/test_automation_notifications.py similarity index 98% rename from src/backend/tests/services/test_automation_notifications.py rename to src/backend/tests/features/automation/test_automation_notifications.py index 273662d6..baab61d8 100644 --- a/src/backend/tests/services/test_automation_notifications.py +++ b/src/backend/tests/features/automation/test_automation_notifications.py @@ -16,7 +16,7 @@ import pytest from core.infra.ephemeral import LocalEphemeralState -from core.services import automation_notifications as notif +from features.automation import notifications as notif USER = "user_notify" diff --git a/src/backend/tests/features/automation/test_automation_oneshot_finalize.py b/src/backend/tests/features/automation/test_automation_oneshot_finalize.py new file mode 100644 index 00000000..9de6f319 --- /dev/null +++ b/src/backend/tests/features/automation/test_automation_oneshot_finalize.py @@ -0,0 +1,49 @@ +"""One-shot acceptance records work before removing the schedule.""" +from datetime import timedelta +from core.infra.time import utc_now +from features.automation.admission import admit +from features.automation.models import ScheduledTaskRun +from features.automation.service import AutomationService + + +def _accept(db, *, schedule_type="once", max_runs=None): + svc = AutomationService(db) + task = svc.create_task(user_id="u1", task_type="prompt", prompt="run", + cron_expression="30 10 29 6 *", + schedule_type=schedule_type, max_runs=max_runs) + task.next_run_at = utc_now() - timedelta(seconds=5) + db.commit() + run_id = admit(db, 1)[0] + return svc, task, db.get(ScheduledTaskRun, run_id) + + +def test_oneshot_is_pending_after_schedule_is_consumed(db_session): + svc, task, run = _accept(db_session) + assert task.status == "active" + assert task.next_run_at is None + assert task.run_count == 1 + assert run.status == "pending" + assert admit(db_session, 1) == [] + run.status = "success" + db_session.commit() + svc.finalize_after_run(task.task_id) + assert task.status == "completed" + + +def test_recurring_continues_after_completed_invocation(db_session): + svc, task, run = _accept(db_session, schedule_type="recurring") + assert task.next_run_at > utc_now() + run.status = "success" + db_session.commit() + svc.finalize_after_run(task.task_id) + assert task.status == "active" + + +def test_max_runs_and_disabled_are_respected(db_session): + svc, task, run = _accept(db_session, schedule_type="recurring", max_runs=1) + assert task.next_run_at is None + svc.update_task_system(task.task_id, status="disabled") + run.status = "failed" + db_session.commit() + svc.finalize_after_run(task.task_id) + assert task.status == "disabled" diff --git a/src/backend/tests/features/automation/test_automation_ownership_postgres.py b/src/backend/tests/features/automation/test_automation_ownership_postgres.py new file mode 100644 index 00000000..84f9f3f2 --- /dev/null +++ b/src/backend/tests/features/automation/test_automation_ownership_postgres.py @@ -0,0 +1,173 @@ +"""Opt-in ownership races in a disposable schema on a local PostgreSQL.""" + +from tests.features.automation.run_fixtures import running_run +import os +import threading +from concurrent.futures import ThreadPoolExecutor +from uuid import uuid4 + +import pytest +from core.db.engine import Base +from core.db.models import ChatSession, UserShadow +from features.automation.models import ScheduledTask, ScheduledTaskRun +from features.automation.runs import AutomationRunRepository, owned_run_guard +from core.runs.journal import RunJournal +from sqlalchemy import create_engine, text +from sqlalchemy.engine import make_url +from sqlalchemy.orm import sessionmaker + +POSTGRES_URL = os.getenv("AUTOMATION_TEST_POSTGRES_URL", "") +pytestmark = pytest.mark.skipif(not POSTGRES_URL, reason="local PostgreSQL opt-in required") + + +@pytest.fixture +def pg(): + url = make_url(POSTGRES_URL) + assert url.host in {"127.0.0.1", "localhost", "::1"}, "Only a local test server is allowed" + schema = "test_automation_ownership_" + uuid4().hex + admin = create_engine(url) + with admin.begin() as conn: + conn.execute(text(f'CREATE SCHEMA "{schema}"')) + engine = create_engine(url, connect_args={"options": f"-csearch_path={schema}"}) + try: + Base.metadata.create_all(engine) + sessions = sessionmaker(bind=engine, expire_on_commit=False) + with sessions() as db: + db.add(UserShadow(user_id="user", username="user")) + db.commit() + db.add( + ScheduledTask( + task_id="task", user_id="user", task_type="prompt", cron_expression="0 * * * *" + ) + ) + db.add(ChatSession(chat_id="chat", user_id="user", title="test")) + db.commit() + yield sessions + finally: + engine.dispose() + with admin.begin() as conn: + conn.execute(text(f'DROP SCHEMA "{schema}" CASCADE')) + admin.dispose() + + +def test_two_recovery_claimants_cannot_both_take_the_same_snapshot(pg): + with pg() as db: + run_id = running_run(db, "task", "old").run_id + barrier = threading.Barrier(2) + + def claim(token): + with pg() as db: + barrier.wait(timeout=5) + return token, AutomationRunRepository(db).claim(run_id, token, expected_token="old") + + with ThreadPoolExecutor(max_workers=2) as pool: + results = list(pool.map(claim, ["a", "b"])) + assert sorted(accepted for _, accepted in results) == [False, True] + winner = next(token for token, accepted in results if accepted) + with pg() as db: + repository = AutomationRunRepository(db) + assert repository.complete(run_id, lease_token="old", status="success") is None + assert repository.complete(run_id, lease_token=winner, status="success") is not None + + +def test_old_cancellation_waits_for_takeover_and_cannot_cancel_its_child(pg): + with pg() as db: + run_id = running_run(db, "task", "old").run_id + journal = RunJournal(pg) + child = journal.accept( + run_id="child", + message_id="message", + chat_id="chat", + user_id="user", + request_payload={"kind": "chat"}, + recovery_snapshot={}, + ) + updated, commit = threading.Event(), threading.Event() + + def takeover(): + with pg() as db: + db.query(ScheduledTaskRun).filter_by(run_id=run_id, lease_token="old").update( + {ScheduledTaskRun.lease_token: "new"} + ) + updated.set() + assert commit.wait(5) + db.commit() + + with ThreadPoolExecutor(max_workers=2) as pool: + owner = pool.submit(takeover) + assert updated.wait(5) + cancellation = pool.submit( + journal.cancel, + child.run_id, + reason="obsolete observer", + cancel_guard=owned_run_guard(run_id, "old"), + ) + commit.set() + owner.result(timeout=5) + assert cancellation.result(timeout=5) is False + assert journal.cancel(child.run_id, reason="current observer", cancel_guard=owned_run_guard(run_id, "new")) + + +def test_completion_waiting_on_takeover_rechecks_the_token(pg): + with pg() as db: + run_id = running_run(db, "task", "old").run_id + updated, commit = threading.Event(), threading.Event() + + def takeover(): + with pg() as db: + db.query(ScheduledTaskRun).filter_by(run_id=run_id, lease_token="old").update( + {ScheduledTaskRun.lease_token: "new"} + ) + updated.set() + assert commit.wait(5) + db.commit() + + def old_complete(): + with pg() as db: + return AutomationRunRepository(db).complete(run_id, lease_token="old", status="success") + + with ThreadPoolExecutor(max_workers=2) as pool: + owner = pool.submit(takeover) + assert updated.wait(5) + completion = pool.submit(old_complete) + commit.set() + owner.result(timeout=5) + assert completion.result(timeout=5) is None + with pg() as db: + assert db.get(ScheduledTaskRun, run_id).lease_token == "new" + assert db.get(ScheduledTaskRun, run_id).status == "running" + + +def test_additive_migration_preserves_rows_and_roundtrips_on_postgres(pg): + import importlib.util + from pathlib import Path + + from alembic.migration import MigrationContext + from alembic.operations import Operations + from sqlalchemy import inspect + + path = ( + Path(__file__).resolve().parents[5] + / "src/backend/alembic/versions/automationlease01_run_token.py" + ) + spec = importlib.util.spec_from_file_location("automation_pg_migration", path) + migration = importlib.util.module_from_spec(spec) + spec.loader.exec_module(migration) + with pg() as db: + run_id = running_run(db, "task", "original").run_id + with pg.kw["bind"].begin() as conn: + migration.op = Operations(MigrationContext.configure(conn)) + migration.downgrade() + assert conn.execute(text("SELECT status FROM scheduled_task_runs")).scalar() == "running" + migration.upgrade() + migration.upgrade() + column = next( + c + for c in inspect(conn).get_columns("scheduled_task_runs") + if c["name"] == "lease_token" + ) + assert column["nullable"] + assert column["default"] is None + with pg() as db: + assert db.get(ScheduledTaskRun, run_id).status == "running" + assert db.get(ScheduledTaskRun, run_id).lease_token is None diff --git a/src/backend/tests/test_automation_remote_effect.py b/src/backend/tests/features/automation/test_automation_remote_effect.py similarity index 74% rename from src/backend/tests/test_automation_remote_effect.py rename to src/backend/tests/features/automation/test_automation_remote_effect.py index cdd8670c..3529b57d 100644 --- a/src/backend/tests/test_automation_remote_effect.py +++ b/src/backend/tests/features/automation/test_automation_remote_effect.py @@ -1,8 +1,8 @@ import pytest -from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker from core.db.engine import Base from extensions.mcp_servers.automation_task_mcp import impl +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker @pytest.fixture @@ -17,7 +17,7 @@ def sessions(tmp_path, monkeypatch): def test_remote_create_retry_returns_one_task_and_rejects_changed_arguments(sessions): - from core.services.automation_remote_effect import bind_remote_effect + from features.automation.remote_effect import bind_remote_effect args = {"cron_expression": "0 18 * * *", "prompt": "daily report", "name": "daily"} effect = bind_remote_effect("user-a", "automation", "create_scheduled_task", "local-effect", args) first = impl.create_task(user_id="user-a", tool_effect_id=effect, **args) @@ -36,16 +36,18 @@ def test_remote_create_retry_returns_one_task_and_rejects_changed_arguments(sess @pytest.mark.asyncio async def test_device_gateway_maps_local_intent_before_cloud_mcp(effect_env, tmp_path, monkeypatch): import json + from types import SimpleNamespace + + import core.db.engine as engine_module import httpx import mcp.types - from types import SimpleNamespace - from agentscope.message import ToolCallBlock, ToolResultState, TextBlock + from agentscope.message import TextBlock, ToolCallBlock, ToolResultState from agentscope.tool import Toolkit from agentscope.tool._response import ToolChunk - from core.llm.mcp_manager import GatewayMCPTool - from core.llm.middlewares import AgentRuntimeState, ToolEffectMiddleware - import core.db.engine as engine_module - from core.services.desktop_capability import invoke_gateway_tool + from core.llm.tools.cloud_gateway import GatewayMCPTool + from core.llm.runtime.state import AgentRuntimeState + from tests.orchestration.hook_support import make_adapter, durable_adapter + from features.desktop.capability.service import invoke_gateway_tool local, make_run = effect_env make_run("gateway-run") @@ -58,15 +60,16 @@ async def test_device_gateway_maps_local_intent_before_cloud_mcp(effect_env, tmp raw = {"name": "create_scheduled_task", "inputSchema": {"type": "object", "properties": {}}} class CloudClient: - execution_timeout = 10 - async def get_tool(self, name): - async def invoke(**args): - result = impl.create_task(user_id="user-1", **args) - return ToolChunk(content=[TextBlock(text=json.dumps(result))], - state=ToolResultState.SUCCESS if result["ok"] else ToolResultState.ERROR) - return invoke - monkeypatch.setattr("core.llm.mcp_pool.make_client", lambda *a, **kw: CloudClient()) + async def __aenter__(self): return self + async def __aexit__(self, *exc): pass + async def call_tool(self, name, args): + from mcp.types import CallToolResult, TextContent + result = impl.create_task(user_id="user-1", **args) + return CallToolResult(content=[TextContent(text=json.dumps(result))], is_error=not result["ok"]) + monkeypatch.setattr('core.llm.mcp.client.make_client', lambda *a, **kw: CloudClient()) + # This ledger fixture has no configured connector/model credentials. + monkeypatch.setattr("features.desktop.capability.security._known_cloud_secrets", lambda *_a, **_kw: set()) received = [] async def endpoint(request): body = json.loads(request.content) @@ -79,19 +82,19 @@ async def endpoint(request): ) return httpx.Response(200, json={"code": 200, "data": result}) - tool = GatewayMCPTool(mcp_name="automation", tool=mcp.types.Tool.model_validate(raw), + tool = GatewayMCPTool(mcp_name="automation", tool=raw, invoke_url="https://cloud.invalid/api/v1/desktop/capability/gateway/automation/call", schema_hash="a"*64, headers={}, timeout=10, transport=httpx.MockTransport(endpoint)) toolkit = Toolkit(tools=[tool]) state = AgentRuntimeState(run_id="gateway-run", journal_owner="worker", user_id="user-1", chat_id="chat-1") - middleware = ToolEffectMiddleware(session_factory=local) + middleware = durable_adapter(session_factory=local) async def invoke(**kwargs): async for item in toolkit.call_tool(kwargs["tool_call"], state): yield item request = {"tool_call": ToolCallBlock(id="gateway-call", name=tool.name, input=json.dumps({"cron_expression": "0 18 * * *", "prompt": "report"}))} first = [item async for item in middleware.on_acting(SimpleNamespace(state=state), request, invoke)] - assert first[-1].state == ToolResultState.SUCCESS + assert first[-1].state == ToolResultState.SUCCESS, first[-1] second = [item async for item in middleware.on_acting(SimpleNamespace(state=state), request, invoke)] assert second[-1].state == ToolResultState.SUCCESS assert len(received) == 1 @@ -106,20 +109,21 @@ async def invoke(**kwargs): @pytest.mark.asyncio async def test_lost_cloud_response_reconciles_receipt_and_never_guesses(sessions, monkeypatch): - import httpx from types import SimpleNamespace - from core.services.automation_remote_effect import bind_remote_effect, lookup_remote_receipt - from orchestration.tool_effect_recovery import _reconcile_cloud_receipt - from core.capabilities.errors import CloudUnavailable + + import httpx + from core.harness.capability_errors import CloudUnavailable + from core.runs.tool_effect_recovery import _reconcile_cloud_receipt + from features.automation.remote_effect import bind_remote_effect, lookup_remote_receipt args = {"cron_expression": "0 18 * * *", "prompt": "report"} effect = bind_remote_effect("user-a", "automation", "create_scheduled_task", "lost", args) original = impl.create_task(user_id="user-a", tool_effect_id=effect, **args) state = {"token": "fixture", "cloud_base": "https://cloud.invalid"} - monkeypatch.setattr("core.services.desktop_cloud_bridge.get_state", lambda: state) - monkeypatch.setattr("core.services.desktop_capability_protocol.token_subject", lambda token: "user-a") - monkeypatch.setattr("core.services.desktop_cloud_bridge.ensure_current_authorization", lambda *a: None) - monkeypatch.setattr("core.services.desktop_cloud_bridge.require_current_account", lambda *a: None) - monkeypatch.setattr("core.services.desktop_cloud_bridge.cloud_headers", lambda *a: {}) + monkeypatch.setattr('features.desktop.bridge.identity.get_state', lambda: state) + monkeypatch.setattr('core.capabilities.manifest_protocol.token_subject', lambda token: "user-a") + monkeypatch.setattr('features.desktop.bridge.identity.ensure_current_authorization', lambda *a: None) + monkeypatch.setattr('features.desktop.bridge.identity.require_current_account', lambda *a: None) + monkeypatch.setattr('features.desktop.bridge.identity.cloud_headers', lambda *a: {}) requests = [] async def endpoint(request): import json @@ -137,6 +141,6 @@ async def endpoint(request): assert impl.list_tasks(user_id="user-a")["count"] == 1 def expired(*args): raise CloudUnavailable("expired") - monkeypatch.setattr("core.services.desktop_cloud_bridge.ensure_current_authorization", expired) + monkeypatch.setattr('features.desktop.bridge.identity.ensure_current_authorization', expired) assert (await _reconcile_cloud_receipt(intent, remote)).outcome == "unknown" assert len(requests) == 1 diff --git a/src/backend/tests/features/automation/test_automation_run_ownership.py b/src/backend/tests/features/automation/test_automation_run_ownership.py new file mode 100644 index 00000000..bb325e50 --- /dev/null +++ b/src/backend/tests/features/automation/test_automation_run_ownership.py @@ -0,0 +1,256 @@ +"""An obsolete executor cannot overwrite an adopted or terminal run.""" + +from tests.features.automation.run_fixtures import running_run +import importlib.util +from pathlib import Path + +import pytest +from alembic.migration import MigrationContext +from alembic.operations import Operations +from features.automation.models import ScheduledTaskRun +from features.automation.runs import AutomationRunRepository +from sqlalchemy import create_engine, inspect, text +from sqlalchemy.orm import sessionmaker + + +@pytest.fixture +def factory(tmp_path): + engine = create_engine(f"sqlite:///{tmp_path / 'ownership.sqlite'}") + ScheduledTaskRun.__table__.create(engine) + yield sessionmaker(bind=engine) + engine.dispose() + + +def test_takeover_rejects_old_completion_and_terminal_rewrites(factory): + with factory() as db: + repository = AutomationRunRepository(db) + run_id = running_run(db, "task", "old").run_id + assert repository.claim(run_id, "new", expected_token="old") + assert repository.complete(run_id, lease_token="old", status="failed") is None + completed = repository.complete( + run_id, lease_token="new", status="success", result_summary="accepted" + ) + assert completed.status == "success" + assert repository.complete(run_id, lease_token="new", status="failed") is None + assert not repository.claim(run_id, "third", expected_token="new") + db.expire_all() + assert db.get(ScheduledTaskRun, run_id).result_summary == "accepted" + + +def test_pending_row_requires_claim_before_completion(factory): + with factory() as db: + repository = AutomationRunRepository(db) + run = ScheduledTaskRun(run_id="pending", task_id="task", status="pending") + db.add(run) + db.commit() + run_id = run.run_id + assert repository.complete(run_id, lease_token="new", status="success") is None + assert repository.claim(run_id, "new", expected_token=None) + assert repository.complete(run_id, lease_token="new", status="success") is not None + + +def test_conditional_result_can_rollback_with_task_bookkeeping(factory): + with factory() as db: + repository = AutomationRunRepository(db) + run_id = running_run(db, "task", "owner").run_id + assert ( + repository.complete(run_id, lease_token="owner", status="success", commit=False) + is not None + ) + db.rollback() + assert db.get(ScheduledTaskRun, run_id).status == "running" + + +@pytest.mark.parametrize( + "relative", + [ + "src/backend/alembic/versions/automationlease01_run_token.py", + "ce/overlay/src/backend/alembic/versions/ce_0031_automation_lease_token.py", + ], +) +def test_migration_preserves_existing_rows_and_roundtrips(relative): + root = Path(__file__).resolve().parents[5] + spec = importlib.util.spec_from_file_location("lease_migration", root / relative) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + engine = create_engine("sqlite://") + with engine.begin() as conn: + conn.execute( + text("CREATE TABLE scheduled_task_runs (run_id TEXT PRIMARY KEY, status TEXT)") + ) + conn.execute(text("INSERT INTO scheduled_task_runs VALUES ('legacy', 'running')")) + module.op = Operations(MigrationContext.configure(conn)) + module.upgrade() + module.upgrade() # fresh-schema create_all may already include the column + assert "lease_token" in { + c["name"] for c in inspect(conn).get_columns("scheduled_task_runs") + } + assert conn.execute(text("SELECT lease_token FROM scheduled_task_runs")).scalar() is None + module.downgrade() + assert "lease_token" not in { + c["name"] for c in inspect(conn).get_columns("scheduled_task_runs") + } + assert conn.execute(text("SELECT status FROM scheduled_task_runs")).scalar() == "running" + module.upgrade() + engine.dispose() + + +def test_delayed_claim_cannot_replace_a_newer_claim(factory): + with factory() as db: + repository = AutomationRunRepository(db) + run_id = running_run(db, "task", "original").run_id + assert repository.claim(run_id, "current", expected_token="original") + assert not repository.claim(run_id, "late", expected_token="original") + assert repository.complete(run_id, lease_token="current", status="success") is not None + + +async def test_obsolete_observer_does_not_cancel_adopted_child(factory, monkeypatch): + import asyncio + from types import SimpleNamespace + + from orchestration.automations import automation_conversation as conversation + + with factory() as db: + repository = AutomationRunRepository(db) + run_id = running_run(db, "task", "old").run_id + monkeypatch.setattr(conversation.db_engine, "SessionLocal", factory) + entered = asyncio.Event() + cancelled = [] + + def get_run(_): + entered.set() + return SimpleNamespace(status="running") + + async def cancel_run(*args, **kwargs): + cancelled.append(args) + + monkeypatch.setattr(conversation.runtime, "get_run", get_run) + monkeypatch.setattr(conversation.runtime, "cancel_run", cancel_run) + child = SimpleNamespace(run_id="child", chat_id="chat", user_id="user") + observer = asyncio.create_task( + conversation._wait(child, automation_run_id=run_id, lease_token="old") + ) + await entered.wait() + with factory() as db: + assert AutomationRunRepository(db).claim(run_id, "new", expected_token="old") + observer.cancel() + with pytest.raises(asyncio.CancelledError): + await observer + assert cancelled == [] + + +async def test_expired_recovery_claimant_leaves_new_ownership_untouched(factory, monkeypatch): + from types import SimpleNamespace + + from core.infra import ephemeral + from features.automation.service import AutomationService + from orchestration.automations.automation_recovery import recover_task + from orchestration.automations.automation_scheduler import AutomationScheduler + + # Recovery reads running rows from its own session; task metadata is enough + # because expired ownership must return before observing/starting a child. + monkeypatch.setattr("core.db.engine.SessionLocal", factory) + monkeypatch.setattr( + AutomationService, "get_task_by_id", lambda *a: SimpleNamespace(user_id="u") + ) + state = ephemeral.LocalEphemeralState() + monkeypatch.setattr(ephemeral, "get_ephemeral_state", lambda: state) + with factory() as db: + run_id = running_run(db, "task", "original").run_id + scheduler = AutomationScheduler() + assert await scheduler._acquire_lock("task") + lease = scheduler._leases["task"] + await state.put(lease.key, "new", ttl=60) + await recover_task(scheduler, "task") + with factory() as db: + assert db.get(ScheduledTaskRun, run_id).lease_token == "original" + await scheduler._release_lock("task") + assert await state.get(lease.key) == "new" + + +async def test_recovery_stops_before_claiming_next_run_after_lease_loss(factory, monkeypatch): + from types import SimpleNamespace + + from core.infra import ephemeral + from features.automation.service import AutomationService + from orchestration.automations import automation_dispatch, automation_recovery + from orchestration.automations.automation_scheduler import AutomationScheduler + + monkeypatch.setattr("core.db.engine.SessionLocal", factory) + monkeypatch.setattr( + AutomationService, "get_task_by_id", lambda *a: SimpleNamespace(user_id="u") + ) + state = ephemeral.LocalEphemeralState() + monkeypatch.setattr(ephemeral, "get_ephemeral_state", lambda: state) + with factory() as db: + repository = AutomationRunRepository(db) + for i in range(2): + run = running_run(db, "task", "original") + run.chat_id = f"chat-{i}" + db.commit() + scheduler = AutomationScheduler() + assert await scheduler._acquire_lock("task") + lease = scheduler._leases["task"] + calls = [] + adopted = [] + + async def resume(*args, run_id): + calls.append(run_id) + await state.put(lease.key, "replacement", ttl=60) + with factory() as db: + other = ( + db.query(ScheduledTaskRun).filter(ScheduledTaskRun.run_id != run_id).one() + ) + adopted.append(other.run_id) + assert AutomationRunRepository(db).claim( + other.run_id, "replacement", expected_token="original" + ) + + monkeypatch.setattr(automation_dispatch, "execute_task", resume) + await automation_recovery.recover_task(scheduler, "task") + assert len(calls) == 1 + with factory() as db: + assert db.get(ScheduledTaskRun, adopted[0]).lease_token == "replacement" + await scheduler._release_lock("task") + + +async def test_recovery_reloads_child_link_created_before_claim(factory, monkeypatch): + from core.infra import ephemeral + from features.automation.service import AutomationService + from orchestration.automations import automation_dispatch, automation_recovery + from orchestration.automations.automation_scheduler import AutomationScheduler + from types import SimpleNamespace + + monkeypatch.setattr("core.db.engine.SessionLocal", factory) + monkeypatch.setattr( + AutomationService, "get_task_by_id", lambda *a: SimpleNamespace(user_id="u") + ) + state = ephemeral.LocalEphemeralState() + monkeypatch.setattr(ephemeral, "get_ephemeral_state", lambda: state) + with factory() as db: + run_id = running_run(db, "task", "original").run_id + original_claim = AutomationRunRepository.claim + + def link_then_claim(self, run_id, token, *, expected_token): + with factory() as old_worker: + row = old_worker.get(ScheduledTaskRun, run_id) + assert row.lease_token == "original" + row.chat_id = "late-child" + old_worker.commit() + return original_claim(self, run_id, token, expected_token=expected_token) + + resumed = [] + + async def resume(*args, run_id): + resumed.append(run_id) + + monkeypatch.setattr(AutomationRunRepository, "claim", link_then_claim) + monkeypatch.setattr(automation_dispatch, "execute_task", resume) + scheduler = AutomationScheduler() + assert await scheduler._acquire_lock("task") + await automation_recovery.recover_task(scheduler, "task") + assert resumed == [run_id] + with factory() as db: + assert db.get(ScheduledTaskRun, run_id).status == "running" + assert db.get(ScheduledTaskRun, run_id).chat_id == "late-child" + await scheduler._release_lock("task") diff --git a/src/backend/tests/features/automation/test_delivery_target_migration.py b/src/backend/tests/features/automation/test_delivery_target_migration.py new file mode 100644 index 00000000..5edfb7a4 --- /dev/null +++ b/src/backend/tests/features/automation/test_delivery_target_migration.py @@ -0,0 +1,31 @@ +"""Stored destinations survive removal of the flat runtime representation.""" +import pytest +from sqlalchemy import Column, JSON, MetaData, String, Table, create_engine, select +from features.automation.schema import upgrade_delivery_targets +from features.automation.delivery_targets import resolve_delivery_targets + +@pytest.mark.parametrize("targets", [None, [], [{"type": "inapp"}], + [{"type": "channel", "channel_id": "chosen", "conversation_id": "chosen-room"}]]) +def test_migration_preserves_destination_precedence_and_metadata(targets): + engine = create_engine("sqlite://") + table = Table("scheduled_tasks", MetaData(), Column("task_id", String, primary_key=True), + Column("metadata", JSON)) + table.metadata.create_all(engine) + data = {"channel_id": "old", "conversation_id": "room", "loop_id": "keep"} + if targets is not None: + data["delivery_targets"] = targets + with engine.begin() as conn: + conn.execute(table.insert().values(task_id="task", metadata=data)) + upgrade_delivery_targets(conn) + first = conn.execute(select(table.c.metadata)).scalar_one() + upgrade_delivery_targets(conn) + assert conn.execute(select(table.c.metadata)).scalar_one() == first + assert first["loop_id"] == "keep" + assert "channel_id" not in first and "conversation_id" not in first + expected = targets or [{"type": "inapp"}, {"type": "channel", "channel_id": "old", "conversation_id": "room"}] + assert resolve_delivery_targets(first) == expected + engine.dispose() + + +def test_runtime_does_not_interpret_flat_destinations(): + assert resolve_delivery_targets({"channel_id": "old", "conversation_id": "room"}) == [{"type": "inapp"}] diff --git a/src/backend/tests/features/automation/test_durable_admission.py b/src/backend/tests/features/automation/test_durable_admission.py new file mode 100644 index 00000000..90d52616 --- /dev/null +++ b/src/backend/tests/features/automation/test_durable_admission.py @@ -0,0 +1,96 @@ +"""Concurrency and migration regressions for durable automation admission.""" +import importlib.util +import threading +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +import pytest +from sqlalchemy import create_engine, event, text +from sqlalchemy.exc import OperationalError +from sqlalchemy.orm import sessionmaker + +from core.db.engine import Base +from features.automation.admission import admit +from features.automation.models import ScheduledTaskRun +from features.automation.schema import upgrade_queue +from features.automation.service import AutomationService + + +def test_two_connections_cannot_consume_same_trigger(tmp_path): + engine = create_engine(f"sqlite:///{tmp_path / 'admit.sqlite'}") + Base.metadata.create_all(engine) + factory = sessionmaker(bind=engine) + with factory() as db: + svc = AutomationService(db) + task = svc.create_task(user_id="u", task_type="prompt", prompt="run", + cron_expression="0 18 * * *") + svc.request_manual_trigger(task) + task_id = task.task_id + rendezvous = threading.Barrier(2) + def after_read(conn, cursor, statement, parameters, context, many): + if statement.startswith("SELECT scheduled_tasks.") and "EXISTS" in statement: + rendezvous.wait(timeout=5) + event.listen(engine, "after_cursor_execute", after_read) + def accept(): + with factory() as db: + try: + return admit(db, 1) + except OperationalError as exc: + assert "locked" in str(exc).lower() + return [] + try: + with ThreadPoolExecutor(max_workers=2) as pool: + futures = [pool.submit(accept), pool.submit(accept)] + results = [future.result(timeout=10) for future in futures] + finally: + event.remove(engine, "after_cursor_execute", after_read) + assert sum(map(len, results)) == 1 + with factory() as db: + assert db.query(ScheduledTaskRun).count() == 1 + assert AutomationService(db).get_task(task_id, "u").run_count == 1 + assert admit(db, 1) == [] + engine.dispose() + + +def test_sqlite_queue_upgrade_retains_history_and_is_repeatable(): + engine = create_engine("sqlite://") + with engine.begin() as connection: + connection.execute(text("""CREATE TABLE scheduled_task_runs ( + run_id TEXT PRIMARY KEY, status TEXT NOT NULL, + CONSTRAINT scheduled_task_runs_status_check CHECK + (status IN ('running', 'success', 'failed')))""")) + connection.execute(text("INSERT INTO scheduled_task_runs VALUES ('old', 'success')")) + upgrade_queue(connection) + upgrade_queue(connection) + connection.execute(text("INSERT INTO scheduled_task_runs VALUES ('queued', 'pending')")) + assert connection.execute(text( + "SELECT run_id, status FROM scheduled_task_runs ORDER BY run_id" + )).all() == [("old", "success"), ("queued", "pending")] + engine.dispose() + + +@pytest.mark.parametrize("path", [ + "src/backend/alembic/versions/automationqueue01_pending.py", + "ce/overlay/src/backend/alembic/versions/ce_0032_automation_queue.py", +]) +def test_downgrade_refuses_to_erase_pending_work(path): + from alembic.migration import MigrationContext + from alembic.operations import Operations + root = Path(__file__).resolve().parents[5] + spec = importlib.util.spec_from_file_location("queue_migration", root / path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + engine = create_engine("sqlite://") + with engine.begin() as connection: + connection.execute(text("""CREATE TABLE scheduled_task_runs ( + run_id TEXT PRIMARY KEY, status TEXT NOT NULL, + CONSTRAINT scheduled_task_runs_status_check CHECK + (status IN ('running', 'success', 'failed')))""")) + with Operations.context(MigrationContext.configure(connection)): + module.upgrade() + connection.execute(text("INSERT INTO scheduled_task_runs VALUES ('q', 'pending')")) + with pytest.raises(RuntimeError, match="Drain pending"): + module.downgrade() + connection.execute(text("DELETE FROM scheduled_task_runs")) + module.downgrade() + engine.dispose() diff --git a/src/backend/tests/services/test_loop_service_harness.py b/src/backend/tests/features/automation/test_loop_service_harness.py similarity index 98% rename from src/backend/tests/services/test_loop_service_harness.py rename to src/backend/tests/features/automation/test_loop_service_harness.py index 6c08ee37..94a6344c 100644 --- a/src/backend/tests/services/test_loop_service_harness.py +++ b/src/backend/tests/features/automation/test_loop_service_harness.py @@ -11,7 +11,7 @@ from sqlalchemy.pool import StaticPool from core.db.engine import Base -from core.services.loop_service import LoopService +from features.automation.loops import LoopService @pytest.fixture() diff --git a/src/backend/tests/features/automation/test_queue_postgres.py b/src/backend/tests/features/automation/test_queue_postgres.py new file mode 100644 index 00000000..b8f91117 --- /dev/null +++ b/src/backend/tests/features/automation/test_queue_postgres.py @@ -0,0 +1,94 @@ +"""Queue admission by separate OS processes and migration with real PostgreSQL.""" +import importlib.util +import json +import os +import subprocess +import sys +import time +from pathlib import Path + +import pytest +from alembic.migration import MigrationContext +from alembic.operations import Operations +from sqlalchemy import text +from features.automation.models import ScheduledTask, ScheduledTaskRun +from core.infra.time import utc_now +from tests.features.automation.test_automation_ownership_postgres import pg, POSTGRES_URL + +pytestmark = pytest.mark.skipif(not POSTGRES_URL, reason="local PostgreSQL opt-in required") + + +def test_processes_admit_each_trigger_once(pg, tmp_path): + with pg() as db: + task = db.get(ScheduledTask, "task") + task.manual_trigger_at = utc_now() + db.commit() + schema = db.execute(text("SELECT current_schema()")).scalar() + script = """ +import json, os, time +from pathlib import Path +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker +from features.automation.admission import admit +engine=create_engine(os.environ['AUTOMATION_TEST_POSTGRES_URL'], + connect_args={'options':'-csearch_path='+os.environ['QUEUE_TEST_SCHEMA']}) +barrier=Path(os.environ['QUEUE_TEST_BARRIER']) +(barrier/str(os.getpid())).touch() +while not (barrier/'release').exists(): time.sleep(.01) +with sessionmaker(engine)() as db: print('RESULT:'+json.dumps(admit(db,8)),flush=True) +""" + env = {**os.environ, "QUEUE_TEST_SCHEMA": schema, "QUEUE_TEST_BARRIER": str(tmp_path)} + processes = [] + accepted = [] + try: + for _ in range(4): + processes.append(subprocess.Popen( + [sys.executable, "-c", script], env=env, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, + )) + deadline = time.monotonic() + 30 + while len(list(tmp_path.iterdir())) < 4: + assert all(process.poll() is None for process in processes) + assert time.monotonic() < deadline, "admission workers failed to reach barrier" + time.sleep(.01) + (tmp_path / "release").touch() + for process in processes: + output, error = process.communicate(timeout=30) + assert process.returncode == 0, error + accepted.extend(json.loads(next( + line[7:] for line in output.splitlines() if line.startswith("RESULT:") + ))) + finally: + for process in processes: + if process.poll() is None: + process.kill() + for process in processes: + process.communicate() + assert len(accepted) == 1 + with pg() as db: + assert db.query(ScheduledTaskRun).count() == 1 + assert db.get(ScheduledTask, "task").run_count == 1 + + +@pytest.mark.parametrize("path", [ + "src/backend/alembic/versions/automationqueue01_pending.py", + "ce/overlay/src/backend/alembic/versions/ce_0032_automation_queue.py", +]) +def test_queue_migration_keeps_history_and_refuses_pending_downgrade(pg, path): + root = Path(__file__).resolve().parents[5] + spec = importlib.util.spec_from_file_location("queue_migration", root/path) + migration = importlib.util.module_from_spec(spec) + spec.loader.exec_module(migration) + with pg.kw["bind"].begin() as conn: + with Operations.context(MigrationContext.configure(conn)): + migration.downgrade() + conn.execute(text("INSERT INTO scheduled_task_runs (run_id,task_id,status,started_at) VALUES ('history','task','success',now())")) + migration.upgrade() + migration.upgrade() + conn.execute(text("INSERT INTO scheduled_task_runs (run_id,task_id,status,started_at) VALUES ('pending','task','pending',now())")) + with pytest.raises(RuntimeError, match="Drain pending"): + migration.downgrade() + conn.execute(text("UPDATE scheduled_task_runs SET status='success' WHERE run_id='pending'")) + migration.downgrade() + assert conn.execute(text("SELECT count(*) FROM scheduled_task_runs")).scalar() == 2 + migration.upgrade() diff --git a/src/backend/tests/features/desktop/__init__.py b/src/backend/tests/features/desktop/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/src/backend/tests/features/desktop/__init__.py @@ -0,0 +1 @@ + diff --git a/src/backend/tests/features/desktop/capabilities/__init__.py b/src/backend/tests/features/desktop/capabilities/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/src/backend/tests/features/desktop/capabilities/__init__.py @@ -0,0 +1 @@ + diff --git a/src/backend/tests/capabilities/_cloud_identity.py b/src/backend/tests/features/desktop/capabilities/_cloud_identity.py similarity index 100% rename from src/backend/tests/capabilities/_cloud_identity.py rename to src/backend/tests/features/desktop/capabilities/_cloud_identity.py diff --git a/src/backend/tests/capabilities/conftest.py b/src/backend/tests/features/desktop/capabilities/conftest.py similarity index 67% rename from src/backend/tests/capabilities/conftest.py rename to src/backend/tests/features/desktop/capabilities/conftest.py index eca9debc..a34db214 100644 --- a/src/backend/tests/capabilities/conftest.py +++ b/src/backend/tests/features/desktop/capabilities/conftest.py @@ -1,17 +1,19 @@ """Shared fixtures: an isolated capability root and an isolated SQLite index.""" from __future__ import annotations +from features.desktop.bridge import identity as bridge_identity +from features.desktop.capabilities import skill_account import pytest from api.routes.v1.catalog import router as catalog_router from core.auth.backend import UserContext, get_current_user -from core.capabilities import device_catalog +from features.desktop.capabilities import device_catalog from core.db.engine import get_db -from core.services import desktop_cloud_bridge as bridge +from features.desktop.bridge import client as bridge from fastapi import FastAPI from fastapi.testclient import TestClient from tests._capability_index import bind_capability_index -from tests.capabilities._cloud_identity import CLOUD_BASE, CLOUD_USER, cloud_token +from tests.features.desktop.capabilities._cloud_identity import CLOUD_BASE, CLOUD_USER, cloud_token @pytest.fixture @@ -30,18 +32,18 @@ def index_db(tmp_path, monkeypatch): @pytest.fixture def hybrid_catalog_client(tmp_path, index_db, caps_root, monkeypatch): - from core.config.catalog_runtime import invalidate_runtime_catalog_cache + from core.capabilities.catalog.runtime import invalidate_runtime_catalog_cache from core.db.engine import Base from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker monkeypatch.setattr( - bridge, - "get_state", + bridge_identity, + 'get_state', lambda: {"cloud_base": CLOUD_BASE, "token": cloud_token(CLOUD_USER)}, ) - monkeypatch.setattr("core.auth.desktop_bridge.bridge_enabled", lambda: True) - monkeypatch.setattr("core.capabilities.skills.current_local_user_id", lambda: CLOUD_USER) + monkeypatch.setattr('features.desktop.auth.bridge.bridge_enabled', lambda: True) + monkeypatch.setattr('features.desktop.capabilities.skill_account.current_local_user_id', lambda: CLOUD_USER) # 云端下发的连接器与本次断言无关，固定成空，避免依赖清单缓存。 monkeypatch.setattr(device_catalog, "_managed_connectors", lambda: []) @@ -69,19 +71,19 @@ def client(tmp_path, index_db, caps_root, monkeypatch): Base.metadata.create_all(index_db.kw["bind"]) monkeypatch.setattr("core.db.engine.SessionLocal", index_db) - monkeypatch.setattr("core.services.mcp_service.SessionLocal", index_db) - from api.routes.v1.desktop_capabilities import router as desktop_router - from core.capabilities import skills - from core.services import desktop_cloud_bridge as bridge + monkeypatch.setattr('core.capabilities.mcp.service.SessionLocal', index_db) + from features.desktop.api.desktop_capabilities import router as desktop_router + from features.desktop.capabilities import skills + from features.desktop.bridge import client as bridge monkeypatch.setenv("SANDBOX_SKILLS_DIR", str(tmp_path / "workspace" / "skills")) - monkeypatch.setattr("core.auth.desktop_bridge.bridge_enabled", lambda: True) - monkeypatch.setattr(skills, "current_local_user_id", lambda: "local-u1") - bridge.reset_for_tests() + monkeypatch.setattr('features.desktop.auth.bridge.bridge_enabled', lambda: True) + monkeypatch.setattr(skill_account, 'current_local_user_id', lambda: "local-u1") + bridge_identity.reset_for_tests() app = FastAPI() app.include_router(desktop_router) app.dependency_overrides[get_current_user] = lambda: UserContext( user_id="local-u1", user_center_id="local-u1", username="local-u1" ) yield TestClient(app) - bridge.reset_for_tests() + bridge_identity.reset_for_tests() diff --git a/src/backend/tests/capabilities/management_browser_server.py b/src/backend/tests/features/desktop/capabilities/management_browser_server.py similarity index 88% rename from src/backend/tests/capabilities/management_browser_server.py rename to src/backend/tests/features/desktop/capabilities/management_browser_server.py index 7aad4c76..4962dbcf 100644 --- a/src/backend/tests/capabilities/management_browser_server.py +++ b/src/backend/tests/features/desktop/capabilities/management_browser_server.py @@ -19,12 +19,12 @@ from unittest.mock import patch from core.db.engine import Base, get_db from core.db import models -from core.capabilities import registry +from features.desktop.capabilities import registry from core.auth.backend import UserContext, get_current_user from api.routes.v1 import catalog, plugins, me_capabilities -from core.plugins.local import service as local_plugin_service -from core.capabilities.local_plugin_runtime import configs -from core.llm.mcp_pool import make_client +from features.desktop.local.plugins import service as local_plugin_service +from features.desktop.capabilities.local_plugin_runtime import configs +from core.llm.mcp.client import make_client engine = create_engine(os.environ["DATABASE_URL"], connect_args={"check_same_thread": False}) Base.metadata.create_all(engine) @@ -47,12 +47,12 @@ def sessions(): app.include_router(plugins.router, prefix="/api") app.include_router(me_capabilities.router, prefix="/api") # No network or installed global capabilities participate in this isolated test. -patch("core.auth.desktop_bridge.bridge_enabled", return_value=False).start() +patch('features.desktop.auth.bridge.bridge_enabled', return_value=False).start() patch.object(catalog, "get_runtime_catalog", return_value={"skills": [], "agents": [], "mcp": [], "kb": []}).start() patch.object(catalog, "is_enabled", return_value=False).start() patch.object(plugins.ps, "list_plugins", return_value=[]).start() -package_root = Path(__file__).resolve().parents[2] / "extensions/plugins/marketplace" +package_root = Path(__file__).resolve().parents[4] / "extensions/plugins/marketplace" for manager in ("skill-manager", "plugin-manager"): local_plugin_service.install(user, str(package_root / manager)) @@ -63,9 +63,8 @@ def health(): @app.post("/test/tool/{manager}/{operation}") async def tool(manager: str, operation: str, arguments: dict): server = next((sid, cfg) for sid, cfg in configs(user).items() if cfg.get("gateway_plugin") == manager) - client = make_client(*server, is_stateful=False) - handler = await client.get_tool(operation) - result = await handler(**arguments) + async with make_client(*server) as client: + result = await client.call_tool(operation, arguments) return json.loads(result.content[0].text) @app.post("/test/package/{kind}") diff --git a/src/backend/tests/capabilities/runtime_recovery_support.py b/src/backend/tests/features/desktop/capabilities/runtime_recovery_support.py similarity index 75% rename from src/backend/tests/capabilities/runtime_recovery_support.py rename to src/backend/tests/features/desktop/capabilities/runtime_recovery_support.py index 5e6fe6f7..b125578b 100644 --- a/src/backend/tests/capabilities/runtime_recovery_support.py +++ b/src/backend/tests/features/desktop/capabilities/runtime_recovery_support.py @@ -1,13 +1,14 @@ """Regressions for desktop identity, explicit bindings and immutable runs.""" +from features.desktop.bridge import identity as bridge_identity import base64 import json import pytest -from core.capabilities import registry, manifest_order -from core.capabilities.ref import cloud_ref, profile_id -from core.services import desktop_cloud_bridge as bridge -from core.services import desktop_cloud_skills as cloud_skills +from features.desktop.capabilities import registry, manifest_order +from core.capabilities.reference import cloud_ref, profile_id +from features.desktop.bridge import client as bridge +from features.desktop.bridge import skills as cloud_skills def _ordered_skill_manifest(st, manifest): @@ -38,10 +39,10 @@ def _zip(files): def _intent(monkeypatch, body="v1"): - from core.services.desktop_capability_protocol import skill_content_hash + from core.capabilities.manifest_protocol import skill_content_hash st = state("a") - monkeypatch.setattr(bridge, "get_state", lambda: st) + monkeypatch.setattr(bridge_identity, 'get_state', lambda: st) profile = profile_id(st["cloud_base"], "a") inst = registry.upsert( profile_id=profile, @@ -54,18 +55,18 @@ def _intent(monkeypatch, body="v1"): @pytest.fixture def durable_index(index_db, monkeypatch): from core.db.engine import Base - from core.services.model_config import ModelConfigService + from core.models.config import ModelConfigService with index_db() as db: Base.metadata.create_all(db.get_bind()) monkeypatch.setattr(ModelConfigService, "_instance", None) - monkeypatch.setattr("core.services.model_config.SessionLocal", index_db) + monkeypatch.setattr('core.models.config.SessionLocal', index_db) return index_db def _shell_id(center: str) -> str: """本机影子用户按壳的命名空间规则建档：cloud:::。""" - from core.services.desktop_cloud_bridge import shell_user_center_id + from features.desktop.bridge.identity import shell_user_center_id return shell_user_center_id("https://cloud.example", center) diff --git a/src/backend/tests/capabilities/test_account_authorization.py b/src/backend/tests/features/desktop/capabilities/test_account_authorization.py similarity index 52% rename from src/backend/tests/capabilities/test_account_authorization.py rename to src/backend/tests/features/desktop/capabilities/test_account_authorization.py index a2d86b75..ed4219f2 100644 --- a/src/backend/tests/capabilities/test_account_authorization.py +++ b/src/backend/tests/features/desktop/capabilities/test_account_authorization.py @@ -1,31 +1,33 @@ """Identity authorization cannot be relaxed for legacy tokens or another request user.""" +from features.desktop.bridge import identity as bridge_identity +from features.desktop.capabilities import skill_account from pathlib import Path import pytest -from core.capabilities import connectors, registry, skills -from core.capabilities.resolver import Candidate -from core.services import desktop_cloud_bridge as bridge +from features.desktop.capabilities import connectors, registry, skills +from features.desktop.capabilities.resolver import Candidate +from features.desktop.bridge import client as bridge @pytest.mark.parametrize("token", ["dcap1.legacy.sig", "opaque", ""]) def test_legacy_token_never_authorizes_even_with_matching_identity(monkeypatch, token): - monkeypatch.setattr(bridge, "get_state", lambda: {"token": token}) - monkeypatch.setattr(skills, "current_local_user_id", lambda: "a") - assert not skills.account_authorized_for("a") + monkeypatch.setattr(bridge_identity, 'get_state', lambda: {"token": token}) + monkeypatch.setattr(skill_account, 'current_local_user_id', lambda: "a") + assert not skill_account.account_authorized_for("a") def test_cloud_authorization_requires_live_state_and_matching_request_user(monkeypatch): - monkeypatch.setattr(bridge, "get_state", lambda: {"token": "dcap2.test.sig"}) - monkeypatch.setattr(skills, "current_local_user_id", lambda: "a") - assert skills.account_authorized_for("a") - assert not skills.account_authorized_for("b") - assert not skills.account_authorized_for(None) - monkeypatch.setattr(bridge, "get_state", lambda: None) - assert not skills.account_authorized_for("a") + monkeypatch.setattr(bridge_identity, 'get_state', lambda: {"token": "dcap2.test.sig"}) + monkeypatch.setattr(skill_account, 'current_local_user_id', lambda: "a") + assert skill_account.account_authorized_for("a") + assert not skill_account.account_authorized_for("b") + assert not skill_account.account_authorized_for(None) + monkeypatch.setattr(bridge_identity, 'get_state', lambda: None) + assert not skill_account.account_authorized_for("a") def test_connector_explicit_request_user_controls_only_that_users_preference(index_db, monkeypatch): - monkeypatch.setattr(skills, "current_local_user_id", lambda: "b") + monkeypatch.setattr(skill_account, 'current_local_user_id', lambda: "b") candidates = [ Candidate( install_id="mcp:local:" + key, diff --git a/src/backend/tests/capabilities/test_agent_rerun_identity.py b/src/backend/tests/features/desktop/capabilities/test_agent_rerun_identity.py similarity index 79% rename from src/backend/tests/capabilities/test_agent_rerun_identity.py rename to src/backend/tests/features/desktop/capabilities/test_agent_rerun_identity.py index 68039f8e..aa9c5ded 100644 --- a/src/backend/tests/capabilities/test_agent_rerun_identity.py +++ b/src/backend/tests/features/desktop/capabilities/test_agent_rerun_identity.py @@ -1,27 +1,31 @@ """Actual rerun routes keep dedicated-agent identity and reject revoked sources.""" +from features.desktop.bridge import identity as bridge_identity +from features.desktop.capabilities import skill_account from types import SimpleNamespace -import core.services.session_operations.agent_targets as chat_agent_targets -import core.services.session_operations.schemas as chat_models -import core.services.session_operations.request_context as chat_request_context import api.routes.v1.chats.reruns as chat_reruns -import core.services.session_operations.session_context as chat_session_context +import api.transports.run_sse as run_sse import core.chat.context as chat_context +import core.chat.runtime.admission as admission import core.db.engine as db_engine import core.infra.responses as responses -import core.services as chat_services +import core.auth.users as chat_services_UserService_module +import core.chat.operations.agent_targets as chat_agent_targets +import core.chat.operations.request_context as chat_request_context +import core.chat.operations.schemas as chat_models +import core.chat.operations.session_context as chat_session_context import pytest -from core.chat.schemas import ChatRequest from core.auth.backend import UserContext -from core.capabilities import agents, registry, runtime, skills, store -from core.capabilities.paths import revision_for_hash -from core.capabilities.ref import cloud_ref, profile_id +from features.desktop.capabilities import agents, registry, runtime, skills, store +from features.desktop.capabilities.paths import revision_for_hash +from core.capabilities.reference import cloud_ref, profile_id +from core.chat.schemas import ChatRequest from core.db.models import ChatMessage, ChatRun, ChatSession -from core.services import desktop_cloud_bridge as bridge -from core.services.chat_service import ChatService +from features.desktop.bridge import client as bridge +from core.chat.service import ChatService from fastapi import HTTPException -from tests.capabilities.test_runtime_recovery import state +from tests.features.desktop.capabilities.test_runtime_recovery import state @pytest.fixture @@ -29,9 +33,9 @@ def rerun(index_db, caps_root, monkeypatch): st = state("cloud-a") current = [st] profile = profile_id(st["cloud_base"], "cloud-a") - monkeypatch.setattr(bridge, "get_state", lambda: current[0]) - monkeypatch.setattr(bridge, "ensure_current_authorization", lambda: None) - monkeypatch.setattr(skills, "current_local_user_id", lambda: "owner") + monkeypatch.setattr(bridge_identity, 'get_state', lambda: current[0]) + monkeypatch.setattr(bridge_identity, 'ensure_current_authorization', lambda: None) + monkeypatch.setattr(skill_account, 'current_local_user_id', lambda: "owner") monkeypatch.setattr("core.db.engine.SessionLocal", index_db) monkeypatch.setattr(db_engine, "SessionLocal", index_db) @@ -96,36 +100,36 @@ def list_for_user(self, user_id): def get_by_id(self, agent_id, user_id=None): return next(row for row in self.list_for_user(user_id) if row["agent_id"] == agent_id) - monkeypatch.setattr("core.services.user_service.UserService", UserService) - monkeypatch.setattr(chat_services, "UserService", UserService) - monkeypatch.setattr("core.services.user_agent_service.UserAgentService", AgentService) - monkeypatch.setattr(chat_session_context, "_ensure_main_model_configured", lambda: None) + monkeypatch.setattr('core.auth.users.UserService', UserService) + monkeypatch.setattr(chat_services_UserService_module, "UserService", UserService) + monkeypatch.setattr('core.capabilities.agents.edition.UserAgentService', AgentService) + monkeypatch.setattr(chat_session_context, 'ensure_main_model_configured', lambda: None) monkeypatch.setattr( - chat_session_context, "_resolve_selected_model_provider_id", lambda *a: None + chat_session_context, 'resolve_selected_model_provider_id', lambda *a: None ) monkeypatch.setattr( - chat_session_context, "_resolve_actual_chat_model_name", lambda *a: "synthetic-model" + chat_session_context, 'resolve_actual_chat_model_name', lambda *a: "synthetic-model" ) monkeypatch.setattr(chat_context, "resolve_enabled_capabilities", lambda *a: ([], [], [])) - monkeypatch.setattr(chat_session_context, "_load_session_messages", lambda *a: []) - monkeypatch.setattr(chat_session_context, "_release_request_session", lambda *a: None) + monkeypatch.setattr(chat_session_context, 'load_session_messages', lambda *a: []) + monkeypatch.setattr(chat_session_context, 'release_request_session', lambda *a: None) monkeypatch.setattr(responses, "sse_response", lambda value: value) - from orchestration import chat_run_executor + pass - monkeypatch.setattr(chat_run_executor, "follow_run_as_sse", lambda *a, **kw: None) + monkeypatch.setattr(run_sse, "follow_run_as_sse", lambda *a, **kw: None) captured = {} def context(request, *a, **kw): captured["request"] = request return {"agent_id": request.agent_id, "mention_agent_id": request.mention_agent_id} - monkeypatch.setattr(chat_request_context, "_build_ctx", context) + monkeypatch.setattr(chat_request_context, 'build_ctx', context) async def start(**kw): captured.update(kw) return kw["accepted_run"] - monkeypatch.setattr(chat_run_executor, "start_run", start) + monkeypatch.setattr(admission, "start_run", start) def metadata(session=None, message=None): with index_db() as db: @@ -227,7 +231,7 @@ async def test_rerun_rejects_unproven_or_changed_cloud_identity_before_history_m if change == "revoke": registry.set_enabled(rerun.inst.install_id, False) elif change == "actor": - monkeypatch.setattr(skills, "current_local_user_id", lambda: "other-user") + monkeypatch.setattr(skill_account, 'current_local_user_id', lambda: "other-user") elif change == "account-same-id": st = state("cloud-b") other_profile = profile_id(st["cloud_base"], "cloud-b") diff --git a/src/backend/tests/capabilities/test_agent_runtime_grants.py b/src/backend/tests/features/desktop/capabilities/test_agent_runtime_grants.py similarity index 62% rename from src/backend/tests/capabilities/test_agent_runtime_grants.py rename to src/backend/tests/features/desktop/capabilities/test_agent_runtime_grants.py index be1c9114..1fbf34a6 100644 --- a/src/backend/tests/capabilities/test_agent_runtime_grants.py +++ b/src/backend/tests/features/desktop/capabilities/test_agent_runtime_grants.py @@ -1,12 +1,14 @@ """Cloud definition grants are checked again after an online checkpoint.""" +from features.desktop.bridge import identity as bridge_identity +from features.desktop.capabilities import skill_account import pytest -from core.capabilities import agents, registry, runtime, skills, store -from core.capabilities.errors import PermissionDenied -from core.capabilities.ref import cloud_ref, profile_id -from core.capabilities.paths import revision_for_hash -from core.services import desktop_cloud_bridge as bridge -from tests.capabilities.test_runtime_recovery import durable_index, state +from features.desktop.capabilities import agents, registry, runtime, skills, store +from core.harness.capability_errors import PermissionDenied +from core.capabilities.reference import cloud_ref, profile_id +from features.desktop.capabilities.paths import revision_for_hash +from features.desktop.bridge import client as bridge +from tests.features.desktop.capabilities.test_runtime_recovery import durable_index, state @pytest.mark.parametrize("revoke", ["remove", "disable"]) @@ -14,8 +16,8 @@ def test_cloud_definition_revoked_during_creation_cannot_be_pinned( durable_index, caps_root, monkeypatch, revoke ): st = state("a") - monkeypatch.setattr(bridge, "get_state", lambda: st) - monkeypatch.setattr(skills, "current_local_user_id", lambda: "owner") + monkeypatch.setattr(bridge_identity, 'get_state', lambda: st) + monkeypatch.setattr(skill_account, 'current_local_user_id', lambda: "owner") definition = agents.AgentDefinition( agent_id="private", name="Private", system_prompt="private content" ) @@ -32,8 +34,8 @@ def test_cloud_definition_revoked_during_creation_cannot_be_pinned( registry.set_state(inst.install_id, "ready", resolved_revision=comp.revision) loaded = agents.load_definition(comp, origin="cloud") monkeypatch.setattr( - bridge, - "ensure_current_authorization", + bridge_identity, + 'ensure_current_authorization', lambda: ( registry.mark_removed(inst.install_id) if revoke == "remove" diff --git a/src/backend/tests/capabilities/test_agents_plugins.py b/src/backend/tests/features/desktop/capabilities/test_agents_plugins.py similarity index 90% rename from src/backend/tests/capabilities/test_agents_plugins.py rename to src/backend/tests/features/desktop/capabilities/test_agents_plugins.py index d78ef80c..da664959 100644 --- a/src/backend/tests/capabilities/test_agents_plugins.py +++ b/src/backend/tests/features/desktop/capabilities/test_agents_plugins.py @@ -1,6 +1,7 @@ """智能体 / 插件：协议、云端清单与定义包、本机落盘、组件归属与就绪、名称裁决与列表合并。""" from __future__ import annotations +from features.desktop.bridge import identity as bridge_identity import base64 import io @@ -10,23 +11,18 @@ from datetime import datetime import pytest -from core.capabilities import agents as caps_agents -from core.capabilities import plugins as caps_plugins -from core.capabilities import registry, store -from core.capabilities.paths import KIND_AGENT, KIND_PLUGIN, KIND_SKILL -from core.capabilities.ref import profile_id +from features.desktop.capabilities import agents as caps_agents +from features.desktop.capabilities import plugins as caps_plugins +from features.desktop.capabilities import registry, store +from core.capabilities.identity_rules import KIND_AGENT, KIND_PLUGIN, KIND_SKILL +from core.capabilities.reference import profile_id from core.db.engine import Base from core.db.models import InstalledPlugin, UserAgent, UserShadow -from core.services import desktop_capability as cloud -from core.services import desktop_capability_entities as entities -from core.services import desktop_cloud_bridge as bridge -from core.services import desktop_cloud_bundles as bundles -from core.services.desktop_capability_protocol import ( - CapabilityManifestError, - build_entity_manifest, - entity_content_hash, - validate_entity_manifest, -) +from features.desktop.capability import service as cloud +from features.desktop.capability import entities as entities +from features.desktop.bridge import client as bridge +from features.desktop.bridge import bundles as bundles +from core.capabilities.manifest_protocol import CapabilityManifestError, build_entity_manifest, entity_content_hash, validate_entity_manifest from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker @@ -81,7 +77,7 @@ def cloud_db(tmp_path, monkeypatch): factory = sessionmaker(bind=engine, autoflush=False, autocommit=False) monkeypatch.setattr(entities, "SessionLocal", factory) monkeypatch.setattr( - "core.plugins.management.admin_config._has_admin_config_for_slug", lambda slug: False + 'core.capabilities.plugins.management.admin_config._has_admin_config_for_slug', lambda slug: False ) with factory() as db: db.add(UserShadow(user_id="cloud-u", username="c")) @@ -213,7 +209,7 @@ def __init__(self, agents: dict, plugins: dict): def get(self, url, headers=None, timeout=None): if url.endswith("/skills/manifest"): - from core.services.desktop_capability_protocol import build_skill_manifest + from core.capabilities.manifest_protocol import build_skill_manifest return _Resp(200, body={"data": build_skill_manifest([])}) for kind in ("agents", "plugins"): @@ -231,13 +227,13 @@ def get(self, url, headers=None, timeout=None): def device(tmp_path, monkeypatch, index_db, caps_root): monkeypatch.setenv("SANDBOX_SKILLS_DIR", str(tmp_path / "ws" / "skills")) monkeypatch.setenv("HUGAGENT_DESKTOP_BRIDGE_SECRET", "s") - bridge.reset_for_tests() + bridge_identity.reset_for_tests() bundles.reset_for_tests() - monkeypatch.setattr(bridge, "bridge_enabled", lambda: True) - monkeypatch.setattr(bridge, "get_state", lambda: dict(STATE, expires_at=time.time() + 60)) - monkeypatch.setattr("core.capabilities.skills.current_local_user_id", lambda: "u") + monkeypatch.setattr('features.desktop.auth.bridge.bridge_enabled', lambda: True) + monkeypatch.setattr(bridge_identity, 'get_state', lambda: dict(STATE, expires_at=time.time() + 60)) + monkeypatch.setattr('features.desktop.capabilities.skill_account.current_local_user_id', lambda: "u") yield - bridge.reset_for_tests() + bridge_identity.reset_for_tests() bundles.reset_for_tests() @@ -298,7 +294,7 @@ def test_sync_prepares_definitions_and_plugin_readiness(device, monkeypatch): ] registry.upsert( profile_id=PROFILE, - ref=__import__("core.capabilities.ref", fromlist=["cloud_ref"]).cloud_ref( + ref=__import__("core.capabilities.reference", fromlist=["cloud_ref"]).cloud_ref( STATE["cloud_base"], KIND_SKILL, "sites-site-builder", scope="shared" ), content_hash="b" * 64, diff --git a/src/backend/tests/capabilities/test_autonomous_loop_scopes.py b/src/backend/tests/features/desktop/capabilities/test_autonomous_loop_scopes.py similarity index 73% rename from src/backend/tests/capabilities/test_autonomous_loop_scopes.py rename to src/backend/tests/features/desktop/capabilities/test_autonomous_loop_scopes.py index d6c0636b..eaaca87c 100644 --- a/src/backend/tests/capabilities/test_autonomous_loop_scopes.py +++ b/src/backend/tests/features/desktop/capabilities/test_autonomous_loop_scopes.py @@ -1,13 +1,22 @@ """Autonomous loop phases retain durable root ownership and independent snapshots.""" +from core.ontology import reviewer as reviewer_module +from core.capabilities.skills import loader_factory as loader_factory_module +from core.harness import log_writer +from features.desktop.bridge import gateway as bridge_gateway +from features.desktop.capabilities import skill_account +from features.desktop.capabilities import skill_candidates +from features.desktop.capabilities import skill_publication from types import SimpleNamespace +import core.ontology.reviewer as reviewer +import orchestration.loops.driver as driver +import orchestration.loops.planner as planner import pytest - -from core.capabilities import runtime, skills -from core.llm.middlewares import CURRENT_RUN_BINDING -from core.services.desktop_capability_protocol import skill_content_hash -from tests.capabilities.test_runtime_recovery import durable_index +from features.desktop.capabilities import runtime, skills +from core.harness.tool_call_context import CURRENT_RUN_BINDING +from core.capabilities.manifest_protocol import skill_content_hash +from tests.features.desktop.capabilities.test_runtime_recovery import durable_index from tests.orchestration.test_autonomous_loop_driver import Harness @@ -15,23 +24,24 @@ async def test_actual_loop_scout_then_two_workers_use_separate_factory_snapshots( durable_index, caps_root, monkeypatch ): + import core.capabilities.skills.loader as skill_loader + import core.capabilities.catalog.api as catalog + import orchestration.loops.driver as loop + import orchestration.loops.planner as loop_planner from core.db.engine import Base - from core.llm.factory.tools import mcp_config as factory_mcp_config from core.llm.factory.selection import capabilities as factory_capabilities - import core.config.catalog as catalog - import core.agent_skills.loader as skill_loader - from core.services import desktop_cloud_bridge as bridge - from core.services.mcp_service import McpServerConfigService - from orchestration import autonomous_loop as loop, loop_planner - from orchestration.loop_evaluator import GoalSpec + from core.llm.factory.tools import mcp_config as factory_mcp_config + from features.desktop.bridge import client as bridge + from core.capabilities.mcp.service import McpServerConfigService + from orchestration.loops.requirements import GoalSpec with durable_index() as db: Base.metadata.create_all(db.get_bind()) - monkeypatch.setattr(skills, "builtin_candidates", lambda: []) - monkeypatch.setattr(skills, "current_account_profile", lambda: None) + monkeypatch.setattr(skill_candidates, 'builtin_candidates', lambda: []) + monkeypatch.setattr(skill_account, 'current_account_profile', lambda: None) for key in ["worker-a", "worker-b"]: text = f"---\nname: {key}\ndescription: synthetic\n---\n{key}\n" - skills.publish_local_skill( + skill_publication.publish_local_skill( key, files={"SKILL.md": text}, content_hash=skill_content_hash(text, {}), @@ -41,22 +51,22 @@ async def test_actual_loop_scout_then_two_workers_use_separate_factory_snapshots harness = Harness( monkeypatch, [{"id": "R1", "description": "first"}, {"id": "R2", "description": "second"}] ) - monkeypatch.setattr(loop, "scout_workspace", loop_planner.scout_workspace) + monkeypatch.setattr(driver, "scout_workspace", loop_planner.scout_workspace) async def nonempty(*args): return False - monkeypatch.setattr(loop_planner, "_workspace_is_empty", nonempty) - monkeypatch.setattr("core.llm.tool_permissions.preferences.resolve_approval_mode", lambda *a, **kw: "auto") + monkeypatch.setattr(planner, "_workspace_is_empty", nonempty) + monkeypatch.setattr("core.sandbox.permissions.approval.resolve_approval_mode", lambda *a, **kw: "auto") monkeypatch.setattr(catalog, "get_enabled_ids", lambda *a: []) selected = [] monkeypatch.setattr( - factory_capabilities, "_effective_main_available_skills", lambda: list(selected) + factory_capabilities, 'effective_main_available_skills', lambda: list(selected) ) - monkeypatch.setattr(factory_capabilities, "_filter_skill_ids_for_user", lambda ids, uid: ids) + monkeypatch.setattr(factory_capabilities, 'filter_skill_ids_for_user', lambda ids, uid: ids) monkeypatch.setattr(factory_capabilities, "_mcp_ids_bound_to_skills", lambda *a: []) monkeypatch.setattr( - skill_loader, "get_skill_loader", lambda: SimpleNamespace(get_skill_dir=lambda sid: None) + loader_factory_module, 'get_skill_loader', lambda: SimpleNamespace(get_skill_dir=lambda sid: None) ) monkeypatch.setattr( McpServerConfigService, @@ -67,8 +77,8 @@ async def nonempty(*args): ) ), ) - monkeypatch.setattr(bridge, "cloud_gateway_mcp_configs", lambda *a, **kw: {}) - monkeypatch.setattr(factory_mcp_config, "_effective_mcp_server_keys", lambda *a, **kw: []) + monkeypatch.setattr(bridge_gateway, 'cloud_gateway_mcp_configs', lambda *a, **kw: {}) + monkeypatch.setattr(factory_mcp_config, 'effective_mcp_server_keys', lambda *a, **kw: []) monkeypatch.setattr(factory_mcp_config, "_filter_mcp_servers_by_keys", lambda *a, **kw: {}) original_preflight = runtime.preflight captured = [] @@ -95,7 +105,7 @@ async def worker(**kwargs): return {"text": "completed", "tokens": 1, "tool_calls": 1} raise AssertionError("must reach the real factory preflight") - monkeypatch.setattr(loop, "_run_worker_iteration", worker) + monkeypatch.setattr(driver, "_run_worker_iteration", worker) token = CURRENT_RUN_BINDING.set(("loop-root", "journal-owner")) try: result = await loop.run_autonomous_loop( @@ -130,8 +140,9 @@ async def worker(**kwargs): @pytest.mark.asyncio async def test_resumed_loop_uses_persisted_iteration_and_distinct_review_scopes(monkeypatch): from copy import deepcopy - from orchestration import autonomous_loop as loop - from orchestration.loop_evaluator import DONE, GoalSpec + + import orchestration.loops.driver as loop + from orchestration.loops.requirements import DONE, GoalSpec Harness(monkeypatch, [{"id": "R1", "description": "pending"}]) ledger = loop._new_ledger("same goal", [{"id": "R1", "description": "pending"}]) @@ -146,8 +157,8 @@ async def review(**kw): calls.append(("confirm" if kw.get("second_pass") else "review", kw)) return {"verdict": DONE, "evidence": "synthetic", "feedback": ""} - monkeypatch.setattr(loop, "_run_worker_iteration", worker) - monkeypatch.setattr(loop, "review_requirement", review) + monkeypatch.setattr(driver, "_run_worker_iteration", worker) + monkeypatch.setattr(driver, "review_requirement", review) token = CURRENT_RUN_BINDING.set(("durable-root", "owner")) try: for _ in range(2): @@ -174,10 +185,11 @@ async def review(**kw): @pytest.mark.asyncio @pytest.mark.parametrize("helper", ["planner", "judge", "ontology"]) async def test_text_helpers_partition_factory_audit_scope(monkeypatch, helper): + import core.capabilities.catalog.api as catalog + import core.llm.runtime.review_text_agent as ontology_text_agent + import orchestration.loops.planner as loop_planner + import orchestration.loops.requirements as loop_evaluator from core.llm import factory as agent_factory - import core.config.catalog as catalog - from orchestration import loop_planner, loop_evaluator - from orchestration.subagents import ontology_reviewer captured = [] @@ -203,7 +215,7 @@ async def factory(**kw): "synthetic", model_name=None, user_id="unit", capability_scope=parent_scope ) else: - await ontology_reviewer._run_text_agent( + await ontology_text_agent.run_text_agent( "synthetic", model_name=None, model_provider_id=None, @@ -221,10 +233,10 @@ async def factory(**kw): @pytest.mark.asyncio async def test_loop_reviewer_forwards_exact_durable_scope_to_factory(monkeypatch): + import core.capabilities.catalog.api as catalog + import orchestration.loops.reviewer as loop_reviewer from core.llm import factory as agent_factory - import core.config.catalog as catalog - from core.services import log_service - from orchestration.subagents import loop_reviewer + from core.harness import log_writer as log_service captured = [] @@ -236,8 +248,8 @@ async def noop(*a, **kw): return "log" monkeypatch.setattr(agent_factory, "create_agent_executor", factory) - monkeypatch.setattr(log_service, "start_subagent_log", noop) - monkeypatch.setattr(log_service, "finish_subagent_log", noop) + monkeypatch.setattr(log_writer, 'start_subagent_log', noop) + monkeypatch.setattr(log_writer, 'finish_subagent_log', noop) await loop_reviewer.review_requirement( objective="goal", requirement_desc="requirement", @@ -254,8 +266,8 @@ async def noop(*a, **kw): @pytest.mark.asyncio async def test_ontology_committee_keeps_parent_iteration_and_seat_scopes(monkeypatch): - from orchestration.subagents import ontology_reviewer - from tests.ontology.test_ontology_reviewer import _runtime, _disable_audit + import core.llm.runtime.review_text_agent as ontology_text_agent + from tests.ontology.test_ontology_reviewer import _disable_audit, _runtime captured = [] @@ -263,10 +275,11 @@ async def text_agent(*a, **kw): captured.append(kw["capability_scope"]) return '{"verdict":"pass","evidence":["synthetic"],"feedback":""}' - monkeypatch.setattr(ontology_reviewer, "_run_text_agent", text_agent) + from core.ontology import review_operations + monkeypatch.setattr(review_operations, "_run_text_agent", text_agent) _disable_audit(monkeypatch) for parent_scope in ["worker-1", "worker-2", "worker-1"]: - result = await ontology_reviewer.review_ontology_output( + result = await reviewer_module.review_ontology_output( task="synthetic", answer="synthetic", runtime=_runtime(), diff --git a/src/backend/tests/capabilities/test_available_runtime.py b/src/backend/tests/features/desktop/capabilities/test_available_runtime.py similarity index 82% rename from src/backend/tests/capabilities/test_available_runtime.py rename to src/backend/tests/features/desktop/capabilities/test_available_runtime.py index 3aaac118..d9f454d3 100644 --- a/src/backend/tests/capabilities/test_available_runtime.py +++ b/src/backend/tests/features/desktop/capabilities/test_available_runtime.py @@ -1,8 +1,11 @@ """Desktop capability failures must not prevent an ordinary answer.""" +from features.desktop.capabilities import skill_account +from features.desktop.capabilities import skill_candidates +from features.desktop.capabilities import skill_publication import pytest -from core.capabilities import runtime, skills -from core.services.desktop_capability_protocol import skill_content_hash +from features.desktop.capabilities import runtime, skills +from core.capabilities.manifest_protocol import skill_content_hash def test_full_factory_can_reply_when_selected_plugin_has_no_tools(tmp_path): @@ -18,8 +21,12 @@ def test_full_factory_can_reply_when_selected_plugin_has_no_tools(tmp_path): import core.db.models from core.db.engine import Base, engine from core.llm import factory as agent_factory -import core.agent_skills.loader as skill_loader +import core.capabilities.skills.loader as skill_loader Base.metadata.create_all(engine) +from core.runs.host import register +register() +from core.runs.tool_effects.host import register as register_effects +register_effects() class EmptyLoader: def load_all_metadata(self): return {} @@ -27,6 +34,8 @@ def get_skill_dir(self, *args): return None def register_skills_to_toolkit(self, *args, **kwargs): return 0 class Model: + from agentscope.formatter import OpenAIChatFormatter + formatter = OpenAIChatFormatter() model, context_size = "offline-test", 32768 calls = 0 async def __call__(self, messages, tools=None, **kwargs): @@ -57,25 +66,25 @@ async def main(): "SANDBOX_TOOLS_ENABLED": "false", "HUGAGENT_CAPS_ROOT": str(tmp_path / "caps"), } - subprocess.run( + result = subprocess.run( [sys.executable, "-c", script], - check=True, env=env, - cwd=Path(__file__).resolve().parents[2], + cwd=Path(__file__).resolve().parents[6], capture_output=True, text=True, timeout=90, ) + assert result.returncode == 0, result.stderr @pytest.fixture def local_skill(index_db, caps_root, monkeypatch): - monkeypatch.setattr(skills, "builtin_candidates", lambda: []) - monkeypatch.setattr(skills, "current_account_profile", lambda: None) + monkeypatch.setattr(skill_candidates, 'builtin_candidates', lambda: []) + monkeypatch.setattr(skill_account, 'current_account_profile', lambda: None) def publish(name): md = f"---\nname: {name}\ndescription: Test skill\n---\nOriginal instructions" - return skills.publish_local_skill( + return skill_publication.publish_local_skill( name, files={"SKILL.md": md}, content_hash=skill_content_hash(md, {}), @@ -91,7 +100,7 @@ def test_source_edits_do_not_change_an_already_running_turn(local_skill): run = runtime.prepare("first", "owner", skill_ids=["news"]) assert "Original" in (run.view_dir / "news" / "SKILL.md").read_text() updated = "---\nname: news\ndescription: Test skill\n---\nUpdated instructions" - skills.publish_local_skill( + skill_publication.publish_local_skill( "news", files={"SKILL.md": updated}, content_hash=skill_content_hash(updated, {}), @@ -121,7 +130,7 @@ def test_executable_scripts_stay_executable_in_the_view(local_skill): local_skill("placeholder") md = "---\nname: executable\ndescription: Test skill\n---\nOriginal instructions" extra = {"run.sh": "#!/bin/sh\necho ready\n"} - component = skills.publish_local_skill( + component = skill_publication.publish_local_skill( "executable", files={"SKILL.md": md, **extra}, content_hash=skill_content_hash(md, extra), @@ -133,7 +142,7 @@ def test_executable_scripts_stay_executable_in_the_view(local_skill): def test_disabled_skill_disappears_without_blocking_other_skills(local_skill): - from core.capabilities import registry + from features.desktop.capabilities import registry local_skill("good") local_skill("disabled") @@ -150,7 +159,7 @@ def test_foreign_directory_is_preserved_and_the_run_stops(local_skill): 所以这一轮明确失败，而不是另起一个影子视图继续跑——后者会留下一个永不刷新、 白占盘的半成品，正是不该有的兜底。 """ - from core.capabilities.errors import ViewUnavailable + from core.harness.capability_errors import ViewUnavailable local_skill("good") local_skill("disabled") @@ -171,14 +180,14 @@ def test_edited_required_dependency_is_included_and_usable(local_skill): "description: Test skill", "description: Test skill\ndependencies:\n - kind: skill\n id: child", ) - skills.publish_local_skill( + skill_publication.publish_local_skill( "parent", files={"SKILL.md": parent_md}, content_hash=skill_content_hash(parent_md, {}), owner_user_id="owner", ) child_md = "---\nname: child\ndescription: Test skill\n---\nUpdated instructions" - skills.publish_local_skill( + skill_publication.publish_local_skill( "child", files={"SKILL.md": child_md}, content_hash=skill_content_hash(child_md, {}), @@ -193,19 +202,19 @@ def test_edited_required_dependency_is_included_and_usable(local_skill): @pytest.mark.asyncio async def test_revoked_skill_returns_tool_error_and_other_skill_stays_readable(local_skill): - from core.capabilities import registry - from core.llm.tool_collector import ToolCollector + from features.desktop.capabilities import registry + from core.llm.factory.tools.definitions import ToolAssembly from core.llm.tools.skill_tool import register_sandboxed_view_text_file local_skill("good") local_skill("disabled") run = runtime.prepare("reader", "owner", skill_ids=["good", "disabled"]) loader = runtime.frozen_loader(run) - collector = ToolCollector() + collector = ToolAssembly() register_sandboxed_view_text_file( collector, [str(run.view_dir / n) for n in run.bindings], loader ) - tool = next(t for t in collector.function_tools if t.name == "view_text_file") + tool = next(t for t in list(collector) if t.name == "view_text_file") registry.set_enabled("skill:local:disabled", False) bad = await tool(file_path="/workspace/skills/disabled/SKILL.md") assert "Original instructions" not in str(bad) @@ -242,10 +251,12 @@ async def test_connector_disabled_midrun_never_calls_underlying_tool(local_skill from agentscope.tool import ToolBase, ToolChunk, Toolkit from agentscope.permission import PermissionBehavior, PermissionDecision from agentscope.message import TextBlock - from core.llm.capability_tools import AvailableMCPClient + from core.llm.tools.ontology_toolkit import OntologyFilteredToolkit + from core.llm.runtime.state import AgentRuntimeState + from agentscope.message import ToolCallBlock from types import SimpleNamespace from dataclasses import replace - from core.capabilities import mcp_json + from features.desktop.capabilities import mcp_json calls = [] @@ -256,7 +267,7 @@ class Tool(ToolBase): async def check_permissions(self, *a, **kw): return PermissionDecision(behavior=PermissionBehavior.ALLOW) - async def __call__(self, **kw): + async def call(self, **kw): calls.append("called") return ToolChunk(content=[TextBlock(text="result")]) @@ -268,12 +279,10 @@ async def list_tools(): run, mcp_bindings={"one": {"install_id": "mcp:local-json:one", "authorization_checked": True}}, ) - client = AvailableMCPClient( - SimpleNamespace(name="one", list_tools=list_tools, is_stateful=False), run - ) - toolkit = Toolkit(mcps=[client]) + toolkit = OntologyFilteredToolkit(tools=[Tool()]) + toolkit.capability_guard = run.validate_tool_call assert (await toolkit.get_tool_schemas())[0]["function"]["name"] == "lookup" monkeypatch.setattr(mcp_json, "local_server_configs", lambda: {}) - result = await (await client.list_tools())[0]() + result = [chunk async for chunk in toolkit.call_tool(ToolCallBlock(id="call", name="lookup", input="{}"), AgentRuntimeState())] assert not calls - assert "result" not in "".join(block.text for block in result.content) + assert "result" not in str(result) diff --git a/src/backend/tests/capabilities/test_capability_change_sync.py b/src/backend/tests/features/desktop/capabilities/test_capability_change_sync.py similarity index 81% rename from src/backend/tests/capabilities/test_capability_change_sync.py rename to src/backend/tests/features/desktop/capabilities/test_capability_change_sync.py index 6a364e50..77e60d28 100644 --- a/src/backend/tests/capabilities/test_capability_change_sync.py +++ b/src/backend/tests/features/desktop/capabilities/test_capability_change_sync.py @@ -1,19 +1,21 @@ """Changes are previewed and committed through the authenticated device API.""" +from features.desktop.bridge import identity as bridge_identity +from features.desktop.capabilities import skill_publication import os from pathlib import Path -from tests.capabilities.test_desktop_capabilities_api import cloud as cloud -from tests.capabilities.test_desktop_capabilities_api import client +from tests.features.desktop.capabilities.test_desktop_capabilities_api import cloud as cloud +from tests.features.desktop.capabilities.test_desktop_capabilities_api import client def _publish_news(folder): """Explicitly publish a local installation; directory scanning is not installation.""" - from core.capabilities import skills, registry, store - from core.services.desktop_capability_protocol import skill_content_hash - from tests.capabilities.test_desktop_capabilities_api import USER + from features.desktop.capabilities import skills, registry, store + from core.capabilities.manifest_protocol import skill_content_hash + from tests.features.desktop.capabilities.test_desktop_capabilities_api import USER files = {file.name: file.read_text() for file in folder.iterdir()} - skills.publish_local_skill( + skill_publication.publish_local_skill( "news", files=files, content_hash=skill_content_hash( @@ -26,8 +28,8 @@ def _publish_news(folder): def test_preview_includes_query_output_and_marks_file_conflict(client, monkeypatch): - from core.capabilities import change_sync - from core.capabilities.change_merge import revision + from features.desktop.capabilities import change_sync + from features.desktop.capabilities.change_merge import revision folder = Path(os.environ["HUGAGENT_CAPS_ROOT"]) / "skills" / "news" folder.mkdir(parents=True) @@ -56,8 +58,8 @@ def test_preview_includes_query_output_and_marks_file_conflict(client, monkeypat def test_commit_requires_resolution_and_rechecks_local_files(client, monkeypatch): - from core.capabilities import change_sync - from core.capabilities.change_merge import revision + from features.desktop.capabilities import change_sync + from features.desktop.capabilities.change_merge import revision folder = Path(os.environ["HUGAGENT_CAPS_ROOT"]) / "skills" / "news" folder.mkdir(parents=True) @@ -90,8 +92,8 @@ def request(state, method, kind, key, body=None): def test_successful_upload_is_idempotent_and_next_preview_has_baseline(client, monkeypatch): - from core.capabilities import change_sync - from core.capabilities.change_merge import revision + from features.desktop.capabilities import change_sync + from features.desktop.capabilities.change_merge import revision folder = Path(os.environ["HUGAGENT_CAPS_ROOT"]) / "skills" / "news" folder.mkdir(parents=True) @@ -130,9 +132,9 @@ def request(state, method, kind, key, body=None): def test_download_captures_baseline_and_cloud_choice_uses_fresh_directory(client, monkeypatch): - from tests.capabilities.test_desktop_capabilities_api import _iid, USER - from core.capabilities import change_sync - from core.capabilities.change_merge import revision + from tests.features.desktop.capabilities.test_desktop_capabilities_api import _iid, USER + from features.desktop.capabilities import change_sync + from features.desktop.capabilities.change_merge import revision client.post("/v1/desktop/capabilities/sync") iid = _iid("market-x") @@ -170,17 +172,17 @@ def request(state, method, kind, key, body=None): def test_account_switch_after_http_never_applies_local(client, monkeypatch): import pytest - from core.capabilities import change_sync - from core.capabilities.change_merge import revision - from core.capabilities.errors import CloudUnavailable - from core.services import desktop_cloud_bridge as bridge - from tests.capabilities.test_desktop_capabilities_api import USER + from features.desktop.capabilities import change_sync + from features.desktop.capabilities.change_merge import revision + from core.harness.capability_errors import CloudUnavailable + from features.desktop.bridge import client as bridge + from tests.features.desktop.capabilities.test_desktop_capabilities_api import USER folder = Path(os.environ["HUGAGENT_CAPS_ROOT"]) / "skills" / "news" folder.mkdir(parents=True) folder.joinpath("SKILL.md").write_text("---\nname: news\ndescription: News\n---\nLocal") folder = _publish_news(folder) - state = bridge.get_state() + state = bridge_identity.get_state() monkeypatch.setattr( change_sync, "cloud_request", @@ -189,7 +191,7 @@ def test_account_switch_after_http_never_applies_local(client, monkeypatch): preview = change_sync.preview(USER, "skill:local:news", state) def switched(state, method, kind, key, body=None): - monkeypatch.setattr(bridge, "get_state", lambda: {}) + monkeypatch.setattr(bridge_identity, 'get_state', lambda: {}) return {"key": key, "files": body["files"], "revision": revision(body["files"])} monkeypatch.setattr(change_sync, "cloud_request", switched) @@ -200,9 +202,9 @@ def switched(state, method, kind, key, body=None): def test_local_connector_preserves_unknown_config_fields(client): - from core.capabilities import change_sync, mcp_json - from core.capabilities.paths import mcp_json_path - from tests.capabilities.test_desktop_capabilities_api import USER + from features.desktop.capabilities import change_sync, mcp_json + from features.desktop.capabilities.paths import mcp_json_path + from tests.features.desktop.capabilities.test_desktop_capabilities_api import USER import json mcp_json.upsert_local_server("custom", {"transport": "stdio", "command": "example"}) @@ -216,8 +218,8 @@ def test_local_connector_preserves_unknown_config_fields(client): def test_inaccessible_cloud_name_can_be_forked_without_reading_foreign_files(client, monkeypatch): from fastapi import HTTPException - from core.capabilities import change_sync - from core.capabilities.change_merge import revision + from features.desktop.capabilities import change_sync + from features.desktop.capabilities.change_merge import revision folder = Path(os.environ["HUGAGENT_CAPS_ROOT"]) / "skills" / "news" folder.mkdir(parents=True) diff --git a/src/backend/tests/capabilities/test_catalog_cloud_only_in_hybrid.py b/src/backend/tests/features/desktop/capabilities/test_catalog_cloud_only_in_hybrid.py similarity index 72% rename from src/backend/tests/capabilities/test_catalog_cloud_only_in_hybrid.py rename to src/backend/tests/features/desktop/capabilities/test_catalog_cloud_only_in_hybrid.py index 689ee9ac..0c0261dd 100644 --- a/src/backend/tests/capabilities/test_catalog_cloud_only_in_hybrid.py +++ b/src/backend/tests/features/desktop/capabilities/test_catalog_cloud_only_in_hybrid.py @@ -6,13 +6,18 @@ """ from __future__ import annotations +from features.desktop import plugin_resolution as desktop_plugin_resolution +from features.desktop.bridge import identity as bridge_identity +from features.desktop.capabilities import skill_account +from features.desktop.capabilities import skill_candidates +from features.desktop.capabilities import skill_publication import pytest -from core.capabilities import registry, skills -from core.capabilities.paths import KIND_SKILL -from core.capabilities.ref import cloud_ref, profile_id -from tests.capabilities._cloud_identity import CLOUD_BASE, CLOUD_USER, cloud_token +from features.desktop.capabilities import registry, skills +from core.capabilities.identity_rules import KIND_SKILL +from core.capabilities.reference import cloud_ref, profile_id +from tests.features.desktop.capabilities._cloud_identity import CLOUD_BASE, CLOUD_USER, cloud_token def _catalog(client, kind: str) -> list: @@ -43,17 +48,17 @@ def test_builtin_skills_stay_resolvable_as_dependencies(index_db, caps_root, mon 这是本次收敛的边界——「不出现在能力中心、不会被启用」由目录层负责；把自带技能从 候选里也删掉，会把依赖图和平台兜底一起打断。 """ - from core.capabilities import skills - from core.services import desktop_cloud_bridge as bridge + from features.desktop.capabilities import skills + from features.desktop.bridge import client as bridge monkeypatch.setattr( - bridge, - "get_state", + bridge_identity, + 'get_state', lambda: {"cloud_base": CLOUD_BASE, "token": cloud_token(CLOUD_USER)}, ) - monkeypatch.setattr("core.auth.desktop_bridge.bridge_enabled", lambda: True) + monkeypatch.setattr('features.desktop.auth.bridge.bridge_enabled', lambda: True) - assert [c.runtime_name for c in skills.builtin_candidates()] + assert [c.runtime_name for c in skill_candidates.builtin_candidates()] @@ -86,17 +91,17 @@ def test_local_profile_plugins_only_resolve_off_hybrid(index_db, caps_root, monk 主机也没有对应 sidecar，连不上就被判不可用，日志里反复刷 missing MCP，还会和 云端那份同名插件互相顶替。单机安装没有云端账号，本机 profile 仍是唯一来源。 """ - from core.capabilities import device_catalog, plugins - from core.plugins import runtime as plugin_loader - from core.services.desktop_capability_protocol import skill_content_hash + from features.desktop.capabilities import device_catalog, plugins + from core.capabilities.plugins import runtime as plugin_loader + from core.capabilities.manifest_protocol import skill_content_hash monkeypatch.setattr(device_catalog, "active", lambda: hybrid) - monkeypatch.setattr(skills, "current_account_profile", lambda: profile_id(CLOUD_BASE, CLOUD_USER)) - monkeypatch.setattr(skills, "account_authorized_for", lambda _uid: True) - monkeypatch.setattr(skills, "builtin_candidates", lambda: []) + monkeypatch.setattr(skill_account, 'current_account_profile', lambda: profile_id(CLOUD_BASE, CLOUD_USER)) + monkeypatch.setattr(skill_account, 'account_authorized_for', lambda _uid: True) + monkeypatch.setattr(skill_candidates, 'builtin_candidates', lambda: []) body = "---\nname: sitebuilder\ndescription: Build sites\n---\nBuild a site." - skills.publish_local_skill( + skill_publication.publish_local_skill( "sitebuilder", files={"SKILL.md": body}, content_hash=skill_content_hash(body, {}), @@ -107,7 +112,7 @@ def test_local_profile_plugins_only_resolve_off_hybrid(index_db, caps_root, monk owner_user_id="owner", ) - plan = plugin_loader.resolve_desktop_progressive_plugins( + plan = desktop_plugin_resolution.resolve_desktop_progressive_plugins( user_id="owner", enabled_skill_ids=["sitebuilder"], enabled_mcp_ids=[], diff --git a/src/backend/tests/capabilities/test_catalog_plugin_components_hidden.py b/src/backend/tests/features/desktop/capabilities/test_catalog_plugin_components_hidden.py similarity index 87% rename from src/backend/tests/capabilities/test_catalog_plugin_components_hidden.py rename to src/backend/tests/features/desktop/capabilities/test_catalog_plugin_components_hidden.py index 01192838..5f488b68 100644 --- a/src/backend/tests/capabilities/test_catalog_plugin_components_hidden.py +++ b/src/backend/tests/features/desktop/capabilities/test_catalog_plugin_components_hidden.py @@ -6,10 +6,10 @@ from __future__ import annotations -from core.capabilities import registry -from core.capabilities.paths import KIND_PLUGIN, KIND_SKILL -from core.capabilities.ref import cloud_ref, profile_id -from tests.capabilities._cloud_identity import CLOUD_BASE, CLOUD_USER +from features.desktop.capabilities import registry +from core.capabilities.identity_rules import KIND_PLUGIN, KIND_SKILL +from core.capabilities.reference import cloud_ref, profile_id +from tests.features.desktop.capabilities._cloud_identity import CLOUD_BASE, CLOUD_USER def _install(kind: str, key: str, **kwargs): diff --git a/src/backend/tests/capabilities/test_change_merge.py b/src/backend/tests/features/desktop/capabilities/test_change_merge.py similarity index 90% rename from src/backend/tests/capabilities/test_change_merge.py rename to src/backend/tests/features/desktop/capabilities/test_change_merge.py index 453b6b32..1a48486a 100644 --- a/src/backend/tests/capabilities/test_change_merge.py +++ b/src/backend/tests/features/desktop/capabilities/test_change_merge.py @@ -1,6 +1,6 @@ import pytest -from core.capabilities.change_merge import plan, resolve, sensitive_paths, validate_files -from core.capabilities.errors import CapabilityError +from features.desktop.capabilities.change_merge import plan, resolve, sensitive_paths, validate_files +from core.harness.capability_errors import CapabilityError def test_disjoint_changes_merge_and_keep_outputs(): diff --git a/src/backend/tests/capabilities/test_change_signal.py b/src/backend/tests/features/desktop/capabilities/test_change_signal.py similarity index 97% rename from src/backend/tests/capabilities/test_change_signal.py rename to src/backend/tests/features/desktop/capabilities/test_change_signal.py index ae44323f..aae425ec 100644 --- a/src/backend/tests/capabilities/test_change_signal.py +++ b/src/backend/tests/features/desktop/capabilities/test_change_signal.py @@ -10,7 +10,7 @@ import pytest -from core.capabilities import change_signal +from features.desktop.capabilities import change_signal @pytest.fixture(autouse=True) diff --git a/src/backend/tests/capabilities/test_cloud_identity.py b/src/backend/tests/features/desktop/capabilities/test_cloud_identity.py similarity index 96% rename from src/backend/tests/capabilities/test_cloud_identity.py rename to src/backend/tests/features/desktop/capabilities/test_cloud_identity.py index 9f7d3f08..68ccc1eb 100644 --- a/src/backend/tests/capabilities/test_cloud_identity.py +++ b/src/backend/tests/features/desktop/capabilities/test_cloud_identity.py @@ -2,7 +2,7 @@ import hashlib import pytest -from core.capabilities.ref import cloud_issuer, cloud_ref, profile_id, ResourceRef +from core.capabilities.reference import cloud_issuer, cloud_ref, profile_id, ResourceRef @pytest.mark.parametrize( diff --git a/src/backend/tests/capabilities/test_cloud_management.py b/src/backend/tests/features/desktop/capabilities/test_cloud_management.py similarity index 88% rename from src/backend/tests/capabilities/test_cloud_management.py rename to src/backend/tests/features/desktop/capabilities/test_cloud_management.py index b38c28c1..650bd61d 100644 --- a/src/backend/tests/capabilities/test_cloud_management.py +++ b/src/backend/tests/features/desktop/capabilities/test_cloud_management.py @@ -1,10 +1,11 @@ """Cloud lifecycle: ownership, revision checks and reinstall after removal.""" -from core.plugins.management import projection as plugin_projection +from core.capabilities.plugins.management import projection as plugin_projection from contextlib import contextmanager import pytest from core.db.models import AdminSkill, AdminMcpServer, InstalledPlugin -from core.services import cloud_management as management, capability_workcopies as copies +from core.capabilities.plugins.management import cloud as management +from features.desktop.capability import workcopies as copies @pytest.fixture @@ -20,7 +21,7 @@ def cloud_db(index_db, monkeypatch): monkeypatch.setattr( "core.ontology.build_validator.ensure_ontology_build_valid", lambda *a, **kw: None ) - monkeypatch.setattr("core.agent_skills.cache_refresh.refresh_skill_caches", lambda: None) + monkeypatch.setattr('core.capabilities.skills.cache_refresh.refresh_skill_caches', lambda: None) return index_db @@ -88,7 +89,7 @@ def test_cloud_artifact_owned_bytes_reach_installer(cloud_db, tmp_path, monkeypa import io import zipfile from core.db.models import Artifact - from core.storage.local import LocalStorageBackend + from core.files.storage.local import LocalStorageBackend data = io.BytesIO() with zipfile.ZipFile(data, "w") as package: @@ -98,7 +99,7 @@ def test_cloud_artifact_owned_bytes_reach_installer(cloud_db, tmp_path, monkeypa monkeypatch.setenv("STORAGE_PATH", str(tmp_path / "storage")) storage = LocalStorageBackend() storage.upload_bytes(data.getvalue(), "packages/skill.zip") - monkeypatch.setattr("core.storage.get_storage", lambda: storage) + monkeypatch.setattr('core.files.storage.get_storage', lambda: storage) with cloud_db() as db: db.add( Artifact( @@ -121,10 +122,10 @@ def test_cloud_artifact_owned_bytes_reach_installer(cloud_db, tmp_path, monkeypa def test_official_manager_upgrade_replaces_retired_tools(cloud_db, monkeypatch): - from core.plugins import management as plugin_service - from core.services import manager_bundle_upgrade + from core.capabilities.plugins import management as plugin_service + from core.capabilities.plugins.management import bundle_upgrade as manager_bundle_upgrade - monkeypatch.setattr(plugin_projection, "_refresh_after_change", lambda *_: None) + monkeypatch.setattr(plugin_projection, 'refresh_after_change', lambda *_: None) with cloud_db() as db: plugin_service.install_plugin(db, "skill-manager", owner_user_id="owner") row = db.query(InstalledPlugin).filter_by(slug="skill-manager").one() diff --git a/src/backend/tests/capabilities/test_cloud_parallel_preparation.py b/src/backend/tests/features/desktop/capabilities/test_cloud_parallel_preparation.py similarity index 68% rename from src/backend/tests/capabilities/test_cloud_parallel_preparation.py rename to src/backend/tests/features/desktop/capabilities/test_cloud_parallel_preparation.py index d5e743f6..dbaff4a4 100644 --- a/src/backend/tests/capabilities/test_cloud_parallel_preparation.py +++ b/src/backend/tests/features/desktop/capabilities/test_cloud_parallel_preparation.py @@ -1,7 +1,10 @@ """Cold downloads overlap without losing results or refreshing each package view.""" +from features.desktop.bridge import identity as bridge_identity +from features.desktop.capabilities import skill_views +from features.desktop.capabilities import skills import threading -from core.capabilities import skills -from core.services import desktop_cloud_skills as cloud_skills +from features.desktop.capabilities import skills +from features.desktop.bridge import skills as cloud_skills def test_prepare_overlaps_four_downloads_and_preserves_failures(monkeypatch): @@ -28,9 +31,9 @@ def prepare_one(state, iid): active -= 1 monkeypatch.setattr(cloud_skills, "prepare_one", prepare_one) - monkeypatch.setattr(skills, "bump_view_generation", lambda: None) - monkeypatch.setattr(skills, "rebuild_views", lambda owner: refreshed.append(owner)) - monkeypatch.setattr("core.agent_skills.cache_refresh.refresh_skill_caches", lambda: None) + monkeypatch.setattr(skills, 'bump_view_generation', lambda: None) + monkeypatch.setattr(skill_views, 'rebuild_views', lambda owner: refreshed.append(owner)) + monkeypatch.setattr('core.capabilities.skills.cache_refresh.refresh_skill_caches', lambda: None) result = cloud_skills.prepare({"account": "same-account"}, [str(i) for i in range(8)]) assert peak == 4 assert [row["install_id"] for row in result] == [str(i) for i in range(8)] @@ -40,39 +43,39 @@ def prepare_one(state, iid): def test_sync_does_not_repeat_the_completed_batch_view_refresh(monkeypatch): from contextlib import nullcontext - from core.services import desktop_cloud_bridge as bridge - from core.capabilities import manifest_order + from features.desktop.bridge import client as bridge + from features.desktop.capabilities import manifest_order manifest = {"revision": "new", "skills": [{}]} monkeypatch.setattr(cloud_skills, "capabilities_enabled", lambda: True) monkeypatch.setattr(cloud_skills, "_manifest", None) monkeypatch.setattr(cloud_skills, "_fetch_manifest", lambda _: manifest) monkeypatch.setattr(cloud_skills, "_profile", lambda _: "profile") - monkeypatch.setattr(bridge, "account_scope", lambda _: nullcontext()) + monkeypatch.setattr(bridge_identity, 'account_scope', lambda _: nullcontext()) monkeypatch.setattr(manifest_order, "apply", lambda *_: nullcontext()) monkeypatch.setattr(cloud_skills, "_reconcile_intent", lambda *_: ["id"]) monkeypatch.setattr(cloud_skills, "prepare", lambda *_: [{"ok": True}]) def unexpected_refresh(*_): raise AssertionError("prepare already refreshed the completed batch") - monkeypatch.setattr(skills, "rebuild_views", unexpected_refresh) + monkeypatch.setattr(skill_views, 'rebuild_views', unexpected_refresh) cloud_skills.sync_blocking({}) def test_failed_batch_still_refreshes_manifest_revocations(monkeypatch): from contextlib import nullcontext - from core.services import desktop_cloud_bridge as bridge - from core.capabilities import manifest_order + from features.desktop.bridge import client as bridge + from features.desktop.capabilities import manifest_order refreshed = [] manifest = {"revision": "new", "skills": [{}]} monkeypatch.setattr(cloud_skills, "capabilities_enabled", lambda: True) monkeypatch.setattr(cloud_skills, "_manifest", None) monkeypatch.setattr(cloud_skills, "_fetch_manifest", lambda _: manifest) monkeypatch.setattr(cloud_skills, "_profile", lambda _: "profile") - monkeypatch.setattr(bridge, "account_scope", lambda _: nullcontext()) + monkeypatch.setattr(bridge_identity, 'account_scope', lambda _: nullcontext()) monkeypatch.setattr(manifest_order, "apply", lambda *_: nullcontext()) monkeypatch.setattr(cloud_skills, "_reconcile_intent", lambda *_: ["id"]) monkeypatch.setattr(cloud_skills, "prepare", lambda *_: [{"ok": False}]) - monkeypatch.setattr(skills, "bump_view_generation", lambda: None) - monkeypatch.setattr(skills, "rebuild_views", lambda *_: refreshed.append(True)) - monkeypatch.setattr("core.agent_skills.cache_refresh.refresh_skill_caches", lambda: None) + monkeypatch.setattr(skills, 'bump_view_generation', lambda: None) + monkeypatch.setattr(skill_views, 'rebuild_views', lambda *_: refreshed.append(True)) + monkeypatch.setattr('core.capabilities.skills.cache_refresh.refresh_skill_caches', lambda: None) cloud_skills.sync_blocking({}) assert refreshed == [True] diff --git a/src/backend/tests/capabilities/test_cloud_plugin_binding_runtime.py b/src/backend/tests/features/desktop/capabilities/test_cloud_plugin_binding_runtime.py similarity index 76% rename from src/backend/tests/capabilities/test_cloud_plugin_binding_runtime.py rename to src/backend/tests/features/desktop/capabilities/test_cloud_plugin_binding_runtime.py index 3a73d37f..96a46baa 100644 --- a/src/backend/tests/capabilities/test_cloud_plugin_binding_runtime.py +++ b/src/backend/tests/features/desktop/capabilities/test_cloud_plugin_binding_runtime.py @@ -1,22 +1,26 @@ """Cloud-only plugin bindings reach the actual factory and explain blocked runs.""" +from features.desktop.bridge import identity as bridge_identity +from features.desktop.capabilities import skill_account +from features.desktop.capabilities import skill_candidates +from features.desktop.capabilities import skill_publication import json import pytest -from core.capabilities import plugins, registry, runtime, skills, store -from core.capabilities.ref import cloud_ref, profile_id -from core.capabilities.paths import revision_for_hash -from core.capabilities.errors import IntegrityFailed -from core.services import desktop_cloud_bridge as bridge -from core.services.desktop_capability_protocol import entity_content_hash, skill_content_hash -from tests.capabilities.test_runtime_recovery import state, durable_index +from features.desktop.capabilities import plugins, registry, runtime, skills, store +from core.capabilities.reference import cloud_ref, profile_id +from features.desktop.capabilities.paths import revision_for_hash +from core.harness.capability_errors import IntegrityFailed +from features.desktop.bridge import client as bridge +from core.capabilities.manifest_protocol import entity_content_hash, skill_content_hash +from tests.features.desktop.capabilities.test_runtime_recovery import state, durable_index def cloud_plugin(monkeypatch, index_db): st = state("cloud-owner") - monkeypatch.setattr(bridge, "get_state", lambda: st) - monkeypatch.setattr(skills, "current_local_user_id", lambda: "local-owner") - monkeypatch.setattr(skills, "builtin_candidates", lambda: []) - monkeypatch.setattr(bridge, "ensure_current_authorization", lambda: None) + monkeypatch.setattr(bridge_identity, 'get_state', lambda: st) + monkeypatch.setattr(skill_account, 'current_local_user_id', lambda: "local-owner") + monkeypatch.setattr(skill_candidates, 'builtin_candidates', lambda: []) + monkeypatch.setattr(bridge_identity, 'ensure_current_authorization', lambda: None) monkeypatch.setattr("core.db.engine.SessionLocal", index_db) profile = profile_id(st["cloud_base"], "cloud-owner") definition = { @@ -85,10 +89,10 @@ def test_cloud_plugin_binding_discovery_rejects_other_account_and_tampered_defin def test_blocked_preflight_persists_safe_report_before_raising( durable_index, caps_root, monkeypatch ): - monkeypatch.setattr(skills, "builtin_candidates", lambda: []) - monkeypatch.setattr(skills, "current_account_profile", lambda: None) + monkeypatch.setattr(skill_candidates, 'builtin_candidates', lambda: []) + monkeypatch.setattr(skill_account, 'current_account_profile', lambda: None) body = "---\nname: local\nmcp_servers: search\n---\nREPORT_CONTENT_CANARY" - skills.publish_local_skill( + skill_publication.publish_local_skill( "local", files={"SKILL.md": body}, content_hash=skill_content_hash(body, {}), diff --git a/src/backend/tests/capabilities/test_cloud_sticky_plugins.py b/src/backend/tests/features/desktop/capabilities/test_cloud_sticky_plugins.py similarity index 83% rename from src/backend/tests/capabilities/test_cloud_sticky_plugins.py rename to src/backend/tests/features/desktop/capabilities/test_cloud_sticky_plugins.py index ccc4eb23..c2b35518 100644 --- a/src/backend/tests/capabilities/test_cloud_sticky_plugins.py +++ b/src/backend/tests/features/desktop/capabilities/test_cloud_sticky_plugins.py @@ -1,15 +1,18 @@ """Persisted cloud plugin selections retain their scoped dependency closure.""" +from core.capabilities.skills import loader_factory as loader_factory_module +from features.desktop.bridge import gateway as bridge_gateway +from features.desktop.bridge import identity as bridge_identity import json import pytest -from core.capabilities import plugins, registry, runtime, store -from core.capabilities.errors import PermissionDenied -from core.capabilities.paths import revision_for_hash +from features.desktop.capabilities import plugins, registry, runtime, store +from core.harness.capability_errors import PermissionDenied +from features.desktop.capabilities.paths import revision_for_hash from core.db.models import ChatSession, InstalledPlugin, UserShadow -from core.plugins import runtime as plugin_loader -from core.services.desktop_capability_protocol import entity_content_hash -from tests.capabilities.test_cloud_plugin_binding_runtime import cloud_plugin -from tests.capabilities.test_runtime_recovery import durable_index, state +from core.capabilities.plugins import runtime as plugin_loader +from core.capabilities.manifest_protocol import entity_content_hash +from tests.features.desktop.capabilities.test_cloud_plugin_binding_runtime import cloud_plugin +from tests.features.desktop.capabilities.test_runtime_recovery import durable_index, state @pytest.fixture @@ -87,7 +90,7 @@ def test_sticky_complete_plugin_declarations_block_missing_required_component( def test_persisted_cloud_selection_rejects_identity_change_or_revocation( selected, monkeypatch, change ): - from core.services import desktop_cloud_bridge as bridge + from features.desktop.bridge import client as bridge _, plugin, _, _ = selected plugin_loader.record_plugin_activation("sticky-cloud", [plugin.install_id]) @@ -95,7 +98,7 @@ def test_persisted_cloud_selection_rejects_identity_change_or_revocation( if change == "user": user = "other-local-user" elif change == "account": - monkeypatch.setattr(bridge, "get_state", lambda: state("other-cloud-owner")) + monkeypatch.setattr(bridge_identity, 'get_state', lambda: state("other-cloud-owner")) else: registry.set_enabled(plugin.install_id, False) if change == "disable": @@ -121,9 +124,8 @@ def test_raw_explicit_cloud_id_is_saved_with_its_account_scope(selected): def test_soft_run_connector_honors_parent_plugin_disable(selected, monkeypatch): from dataclasses import replace - from core.capabilities.runtime import save - from core.llm.capability_tools import connector_available - from core.services import desktop_cloud_bridge as bridge + from features.desktop.capabilities.runtime import save + from features.desktop.bridge import client as bridge _, plugin, _, skill = selected run = runtime.prepare( @@ -144,10 +146,10 @@ def test_soft_run_connector_honors_parent_plugin_disable(selected, monkeypatch): }, ) save(run) - monkeypatch.setattr(bridge, "cloud_gateway_mcp_configs", lambda *a, **kw: {"pack-search": {}}) - assert connector_available(run, "pack-search") + monkeypatch.setattr(bridge_gateway, 'cloud_gateway_mcp_configs', lambda *a, **kw: {"pack-search": {}}) + assert run.validate_tool_call("pack-search") registry.set_enabled(plugin.install_id, False) - assert not connector_available(run, "pack-search") + assert not run.validate_tool_call("pack-search") @pytest.mark.asyncio @@ -160,10 +162,10 @@ async def test_actual_factory_passes_selected_full_plugin_to_prepare_and_preflig from core.llm import factory as agent_factory from core.llm.factory.tools import mcp_config as factory_mcp_config from core.llm.factory.selection import capabilities as factory_capabilities - import core.config.catalog as catalog - import core.agent_skills.loader as skill_loader - from core.services import desktop_cloud_bridge as bridge - from core.services.mcp_service import McpServerConfigService + import core.capabilities.catalog.api as catalog + import core.capabilities.skills.loader as skill_loader + from features.desktop.bridge import client as bridge + from core.capabilities.mcp.service import McpServerConfigService _, plugin, _, skill = selected with durable_index() as db: @@ -177,13 +179,13 @@ async def test_actual_factory_passes_selected_full_plugin_to_prepare_and_preflig skill_ids=[], mcp_server_ids=[], ) - monkeypatch.setattr("core.llm.tool_permissions.preferences.resolve_approval_mode", lambda *a, **kw: "auto") + monkeypatch.setattr("core.sandbox.permissions.approval.resolve_approval_mode", lambda *a, **kw: "auto") monkeypatch.setattr(catalog, "get_enabled_ids", lambda *a: []) - monkeypatch.setattr(factory_capabilities, "_effective_main_available_skills", lambda: []) - monkeypatch.setattr(factory_capabilities, "_filter_skill_ids_for_user", lambda ids, uid: ids) + monkeypatch.setattr(factory_capabilities, 'effective_main_available_skills', lambda: []) + monkeypatch.setattr(factory_capabilities, 'filter_skill_ids_for_user', lambda ids, uid: ids) monkeypatch.setattr(factory_capabilities, "_mcp_ids_bound_to_skills", lambda *a: []) monkeypatch.setattr( - skill_loader, "get_skill_loader", lambda: SimpleNamespace(get_skill_dir=lambda sid: None) + loader_factory_module, 'get_skill_loader', lambda: SimpleNamespace(get_skill_dir=lambda sid: None) ) monkeypatch.setattr( McpServerConfigService, @@ -194,9 +196,9 @@ async def test_actual_factory_passes_selected_full_plugin_to_prepare_and_preflig ) ), ) - monkeypatch.setattr(bridge, "cloud_gateway_mcp_configs", lambda *a, **kw: {}) + monkeypatch.setattr(bridge_gateway, 'cloud_gateway_mcp_configs', lambda *a, **kw: {}) monkeypatch.setattr( - factory_mcp_config, "_effective_mcp_server_keys", lambda *a, **kw: ["pack-search"] + factory_mcp_config, 'effective_mcp_server_keys', lambda *a, **kw: ["pack-search"] ) monkeypatch.setattr( factory_mcp_config, diff --git a/src/backend/tests/capabilities/test_cloud_workcopies.py b/src/backend/tests/features/desktop/capabilities/test_cloud_workcopies.py similarity index 91% rename from src/backend/tests/capabilities/test_cloud_workcopies.py rename to src/backend/tests/features/desktop/capabilities/test_cloud_workcopies.py index 1a9ac472..f483c31f 100644 --- a/src/backend/tests/capabilities/test_cloud_workcopies.py +++ b/src/backend/tests/features/desktop/capabilities/test_cloud_workcopies.py @@ -1,7 +1,10 @@ +from features.desktop.api import desktop_capability +from features.desktop.api import capability_dependencies +from features.desktop.api import desktop_capability_state import pytest -from core.capabilities.change_merge import revision +from features.desktop.capabilities.change_merge import revision from core.db.models import AdminMcpServer, AdminSkill, ContentBlock, InstalledPlugin -from core.services import capability_workcopies as copies +from features.desktop.capability import workcopies as copies from fastapi import HTTPException @@ -58,7 +61,7 @@ def test_full_plugin_files_saved_with_receipt_and_cas(cloud): def test_global_skill_requires_manifest_authorization(cloud, monkeypatch): - from core.services import desktop_capability + from features.desktop.capability import service as desktop_capability with cloud() as db: db.add( @@ -121,8 +124,8 @@ def test_secret_ack_required_and_idempotency_payload_fixed(cloud): def test_cloud_routes_keep_known_credentials_on_cloud(cloud, monkeypatch): - from api.routes.v1 import desktop_capability as routes - from core.services import desktop_capability_security as security + from features.desktop.api import desktop_capability as routes + from features.desktop.capability import security as security from fastapi import FastAPI from fastapi.testclient import TestClient @@ -140,8 +143,8 @@ def test_cloud_routes_keep_known_credentials_on_cloud(cloud, monkeypatch): ) db.commit() app = FastAPI() - app.include_router(routes.router) - app.dependency_overrides[routes._require_capability_user] = lambda: "user" + app.include_router(desktop_capability.router) + app.dependency_overrides[capability_dependencies.require_capability_user] = lambda: "user" client = TestClient(app) assert client.get("/v1/desktop/capability/workcopies/skill/secret").status_code == 422 assert ( @@ -160,7 +163,7 @@ def test_cloud_routes_keep_known_credentials_on_cloud(cloud, monkeypatch): def test_skill_submission_preserves_all_files_and_checks_model_changes(cloud, monkeypatch): - from core.agent_skills import cache_refresh + from core.capabilities.skills import cache_refresh from core.ontology import build_validator validations = [] @@ -202,7 +205,7 @@ def test_skill_submission_preserves_all_files_and_checks_model_changes(cloud, mo def test_plugin_execution_declarations_are_saved_but_not_activated(cloud, monkeypatch): import json - from core.agent_skills import cache_refresh + from core.capabilities.skills import cache_refresh monkeypatch.setattr(cache_refresh, "refresh_skill_caches", lambda: None) with cloud() as db: @@ -235,8 +238,8 @@ def test_plugin_execution_declarations_are_saved_but_not_activated(cloud, monkey def test_pending_plugin_update_preserves_previous_active_package(cloud, monkeypatch): import json - from core.agent_skills import cache_refresh - from core.services.desktop_capability_entities import _plugin_files + from core.capabilities.skills import cache_refresh + from features.desktop.capability.entities import _plugin_files monkeypatch.setattr(cache_refresh, "refresh_skill_caches", lambda: None) with cloud() as db: diff --git a/src/backend/tests/capabilities/test_core_layers.py b/src/backend/tests/features/desktop/capabilities/test_core_layers.py similarity index 96% rename from src/backend/tests/capabilities/test_core_layers.py rename to src/backend/tests/features/desktop/capabilities/test_core_layers.py index 2c97f492..ba133b13 100644 --- a/src/backend/tests/capabilities/test_core_layers.py +++ b/src/backend/tests/features/desktop/capabilities/test_core_layers.py @@ -7,10 +7,15 @@ from pathlib import Path import pytest -from core.capabilities import archive, junction, paths, registry, store, view -from core.capabilities.errors import IntegrityFailed -from core.capabilities.ref import ResourceRef, cloud_ref, local_ref, profile_id -from core.capabilities.resolver import Candidate, resolve +from core.capabilities import archive as archive +from core.infra import directory_links as junction +from features.desktop.capabilities import paths +from features.desktop.capabilities import registry +from features.desktop.capabilities import store +from features.desktop.capabilities import view +from core.harness.capability_errors import IntegrityFailed +from core.capabilities.reference import ResourceRef, cloud_ref, local_ref, profile_id +from features.desktop.capabilities.resolver import Candidate, resolve def _zip(files: dict, root: str = "") -> bytes: diff --git a/src/backend/tests/features/desktop/capabilities/test_credential_continuity.py b/src/backend/tests/features/desktop/capabilities/test_credential_continuity.py new file mode 100644 index 00000000..e21ec25a --- /dev/null +++ b/src/backend/tests/features/desktop/capabilities/test_credential_continuity.py @@ -0,0 +1,193 @@ +"""A long sync pass signs with the account's current token; a refused token re-syncs on renewal.""" + +from __future__ import annotations +from features.desktop.bridge import client as bridge_client +from features.desktop.bridge import client_state as bridge_client_state +from features.desktop.bridge import identity as bridge_identity +from features.desktop.bridge import status as bridge_status + +import base64 +import json +import threading + +import httpx +import pytest +from features.desktop.bridge import client as bridge +from features.desktop.bridge import bundles as desktop_cloud_bundles, skills as desktop_cloud_skills + + +def _token(user_id: str, nonce: str) -> str: + claims = { + "u": user_id, + "c": "center", + "a": 1, + "h": f"session-{user_id}", + "d": "device", + "n": nonce, + } + body = base64.urlsafe_b64encode(json.dumps(claims).encode()).decode().rstrip("=") + return f"dcap2.{body}.sig" + + +def _refusal(status: int, state=None) -> httpx.HTTPStatusError: + request = httpx.Request( + "GET", + "https://cloud/api/v1/desktop/capability/plugins/manifest", + headers={"Authorization": f"Bearer {(state or bridge_identity.get_state())['token']}"}, + ) + return httpx.HTTPStatusError( + str(status), request=request, response=httpx.Response(status, request=request) + ) + + +@pytest.fixture +def forced(monkeypatch): + bridge_identity.reset_for_tests() + calls = [] + monkeypatch.setattr(bridge_identity, 'purge_persisted_state', lambda: None) + monkeypatch.setattr(bridge_identity, 'rebuild_identity_views', lambda: None) + monkeypatch.setattr(bridge_client, 'refresh_manifest_async', lambda **kw: calls.append(kw)) + yield calls + bridge_identity.reset_for_tests() + + +def test_headers_sign_with_the_renewed_token_of_the_same_account(forced): + bridge_client.set_state("https://cloud", _token("u-1", "a"), 600, device_id="device") + snapshot = bridge_identity.get_state() + renewed = _token("u-1", "b") + bridge_client.set_state("https://cloud", renewed, 600, device_id="device") + assert bridge_identity.cloud_headers(snapshot)["Authorization"] == f"Bearer {renewed}" + other = {"cloud_base": "https://cloud", "token": _token("u-2", "c"), "device_id": "device"} + assert bridge_identity.cloud_headers(other)["Authorization"] == f"Bearer {other['token']}" + + +def test_renewal_after_a_refused_token_resyncs_once(forced): + bridge_client.set_state("https://cloud", _token("u-1", "a"), 600, device_id="device") + bridge_client.set_state("https://cloud", _token("u-1", "b"), 600, device_id="device") + assert len(forced) == 1 + snapshot = bridge_identity.get_state() + bridge_identity.note_rejected_credential(snapshot, _refusal(500)) + bridge_client.set_state("https://cloud", _token("u-1", "c"), 600, device_id="device") + assert len(forced) == 1 + # A late refusal of b must not reject the already installed c. + bridge_identity.note_rejected_credential(snapshot, _refusal(401, snapshot)) + assert bridge_client_state.credential_rejected == "" + bridge_identity.note_rejected_credential(bridge_identity.get_state(), _refusal(401)) + bridge_client.set_state("https://cloud", _token("u-1", "d"), 600, device_id="device") + assert len(forced) == 2 + bridge_client.set_state("https://cloud", _token("u-1", "e"), 600, device_id="device") + assert len(forced) == 2 + + +def test_logout_forgets_the_refusal(forced): + bridge_client.set_state("https://cloud", _token("u-1", "a"), 600, device_id="device") + bridge_identity.note_rejected_credential(bridge_identity.get_state(), _refusal(401)) + bridge_identity.clear_state() + bridge_client.set_state("https://cloud", _token("u-1", "b"), 600, device_id="device") + assert len(forced) == 2 # first login + re-login only + bridge_client.set_state("https://cloud", _token("u-1", "c"), 600, device_id="device") + assert len(forced) == 2 + + +@pytest.mark.parametrize("kind", ["agent", "plugin"]) +def test_definition_sync_records_a_refused_credential(forced, monkeypatch, kind): + bridge_client.set_state("https://cloud", _token("u-1", "a"), 600, device_id="device") + state = bridge_identity.get_state() + monkeypatch.setattr(desktop_cloud_bundles, "capabilities_enabled", lambda: True) + monkeypatch.setattr( + httpx, + "get", + lambda url, **kw: httpx.Response( + 401, request=httpx.Request("GET", url, headers=kw.get("headers")) + ), + ) + assert desktop_cloud_bundles.sync_kind(kind, state) is False + assert bridge_client_state.credential_rejected == bridge_identity.state_fingerprint(state) + + +def test_skill_sync_records_a_refused_credential(forced, monkeypatch): + bridge_client.set_state("https://cloud", _token("u-1", "a"), 600, device_id="device") + state = bridge_identity.get_state() + monkeypatch.setattr(desktop_cloud_skills, "capabilities_enabled", lambda: True) + monkeypatch.setattr( + httpx, + "get", + lambda url, **kw: httpx.Response( + 401, request=httpx.Request("GET", url, headers=kw.get("headers")) + ), + ) + desktop_cloud_skills.sync_blocking(state) + assert bridge_client_state.credential_rejected == bridge_identity.state_fingerprint(state) + + +def test_refusal_of_an_expired_token_is_still_recorded(forced, monkeypatch): + bridge_client.set_state("https://cloud", _token("u-1", "a"), 600, device_id="device") + state = bridge_identity.get_state() + with bridge_client_state.state_lock: + bridge_client_state.state["expires_at"] = 0 # expired, renewal not pushed yet + assert bridge_identity.get_state() is None + monkeypatch.setattr(desktop_cloud_bundles, "capabilities_enabled", lambda: True) + monkeypatch.setattr( + httpx, + "get", + lambda url, **kw: httpx.Response( + 401, request=httpx.Request("GET", url, headers=kw.get("headers")) + ), + ) + desktop_cloud_bundles.sync_kind("plugin", state) + bridge_client.set_state("https://cloud", _token("u-1", "b"), 600, device_id="device") + assert len(forced) == 2 + + +def test_forced_refresh_during_a_running_pass_reruns_after_it(monkeypatch): + bridge_identity.reset_for_tests() + monkeypatch.setattr(bridge_identity, 'purge_persisted_state', lambda: None) + monkeypatch.setattr(bridge_identity, 'rebuild_identity_views', lambda: None) + threads, runs = [], [] + + class Recorded(threading.Thread): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + threads.append(self) + + monkeypatch.setattr(threading, "Thread", Recorded) + + def headers(st): + runs.append(st["token"]) + if len(runs) == 1: + bridge_identity.note_rejected_credential(st, _refusal(401)) + bridge_client.set_state("https://cloud", _token("u-1", "b"), 600, device_id="device") + raise RuntimeError("cloud unreachable") + + monkeypatch.setattr(bridge_identity, 'cloud_headers', headers) + bridge_client.set_state("https://cloud", _token("u-1", "a"), 600, device_id="device") + while any(t.is_alive() for t in list(threads)): + for t in list(threads): + t.join(5) + assert runs == [_token("u-1", "a"), _token("u-1", "b")] + assert bridge_client_state.refresh_pending is False and bridge_client_state.manifest_fetching is False + bridge_identity.reset_for_tests() + + +def test_readiness_reads_progress_without_holding_the_identity_lock(forced, monkeypatch): + bridge_client.set_state("https://cloud", _token("u-1", "a"), 600, device_id="device") + seen = {} + + def status(): + def probe(): + seen["lock_free"] = bridge_client_state.state_lock.acquire(timeout=1) + if seen["lock_free"]: + bridge_client_state.state_lock.release() + + worker = threading.Thread(target=probe) + worker.start() + worker.join() + return {"revision": "r", "installations": [], "last_error": None} + + monkeypatch.setattr(desktop_cloud_skills, "status", status) + monkeypatch.setattr( + desktop_cloud_bundles, "status", lambda: {"agent": status(), "plugin": status()} + ) + monkeypatch.setattr('features.desktop.capabilities.session_availability.active', lambda _: False) + assert bridge_status.initial_sync_status()["totals_known"] is True + assert seen["lock_free"] is True diff --git a/src/backend/tests/capabilities/test_dependency_closure.py b/src/backend/tests/features/desktop/capabilities/test_dependency_closure.py similarity index 87% rename from src/backend/tests/capabilities/test_dependency_closure.py rename to src/backend/tests/features/desktop/capabilities/test_dependency_closure.py index 58aa2877..fc371986 100644 --- a/src/backend/tests/capabilities/test_dependency_closure.py +++ b/src/backend/tests/features/desktop/capabilities/test_dependency_closure.py @@ -1,8 +1,11 @@ """Declared dependency closure fails closed without installing anything.""" +from features.desktop.capabilities import skill_account +from features.desktop.capabilities import skill_candidates +from features.desktop.capabilities import skill_publication -from core.capabilities import dependency, plugins, agents, registry, skills -from core.services.desktop_capability_protocol import skill_content_hash -from tests.capabilities.test_runtime_recovery import durable_index +from features.desktop.capabilities import dependency, plugins, agents, registry, skills +from core.capabilities.manifest_protocol import skill_content_hash +from tests.features.desktop.capabilities.test_runtime_recovery import durable_index def plugin(name, components, **extra): @@ -31,7 +34,7 @@ def test_required_optional_unknown_and_cycle(index_db, caps_root): def test_version_platform_and_unknown_legacy_plugin(index_db, caps_root): - skills.publish_local_skill( + skill_publication.publish_local_skill( "dep", files={"SKILL.md": "dep"}, content_hash=skill_content_hash("dep", {}), version="1.0" ) parent = plugin( @@ -81,7 +84,7 @@ def test_runtime_dependency_metadata_never_claims_unverified_packages(index_db, body = ( "---\nname: dep\ndescription: test\ndependencies:\n npm: [unknown-npm-package]\n---\nbody" ) - skills.publish_local_skill( + skill_publication.publish_local_skill( "dep", files={"SKILL.md": body}, content_hash=skill_content_hash(body, {}) ) inst = registry.get("skill:local:dep") @@ -93,7 +96,7 @@ def test_runtime_dependency_metadata_never_claims_unverified_packages(index_db, def test_private_component_cannot_become_ready_for_another_owner(index_db, caps_root): - skills.publish_local_skill( + skill_publication.publish_local_skill( "private", files={"SKILL.md": "private"}, content_hash=skill_content_hash("private", {}), @@ -105,12 +108,12 @@ def test_private_component_cannot_become_ready_for_another_owner(index_db, caps_ def test_runtime_preflight_blocks_declared_mcp_and_agent_kb(durable_index, caps_root, monkeypatch): - from core.capabilities import runtime + from features.desktop.capabilities import runtime - monkeypatch.setattr(skills, "builtin_candidates", lambda: []) - monkeypatch.setattr(skills, "current_account_profile", lambda: None) + monkeypatch.setattr(skill_candidates, 'builtin_candidates', lambda: []) + monkeypatch.setattr(skill_account, 'current_account_profile', lambda: None) body = "---\nname: needs-search\ndescription: Search\nmcp_servers: search\n---\nbody" - skills.publish_local_skill( + skill_publication.publish_local_skill( "needs-search", files={"SKILL.md": body}, content_hash=skill_content_hash(body, {}) ) run = runtime.prepare("needs-run", "u", skill_ids=["needs-search"]) @@ -124,12 +127,12 @@ def test_runtime_preflight_blocks_declared_mcp_and_agent_kb(durable_index, caps_ def test_plugin_closure_replay_keeps_original_definition_revision( durable_index, caps_root, monkeypatch ): - from core.capabilities import runtime + from features.desktop.capabilities import runtime - monkeypatch.setattr(skills, "builtin_candidates", lambda: []) - monkeypatch.setattr(skills, "current_account_profile", lambda: None) + monkeypatch.setattr(skill_candidates, 'builtin_candidates', lambda: []) + monkeypatch.setattr(skill_account, 'current_account_profile', lambda: None) for name in ("first", "second"): - skills.publish_local_skill( + skill_publication.publish_local_skill( name, files={"SKILL.md": name}, content_hash=skill_content_hash(name, {}) ) original = plugin("pack", {"skills": ["first"]}, version="1") @@ -214,7 +217,7 @@ def test_extra_requirements_list_and_extension_constraints_survive_projection(in def test_legacy_plugin_catalog_projects_declarative_component_ids_without_mutation(): - from core.plugins.packaging.sources import _component_keys + from core.capabilities.plugins.packaging.sources import component_keys as _component_keys declaration = { "skills": [ @@ -239,14 +242,14 @@ def test_catalog_skill_with_missing_connector_is_dropped_not_fatal( turn had committed to it, *every* local conversation stopped — including plain messages that selected nothing. """ - from core.capabilities import runtime + from features.desktop.capabilities import runtime - monkeypatch.setattr(skills, "builtin_candidates", lambda: []) - monkeypatch.setattr(skills, "current_account_profile", lambda: None) + monkeypatch.setattr(skill_candidates, 'builtin_candidates', lambda: []) + monkeypatch.setattr(skill_account, 'current_account_profile', lambda: None) plain = "---\nname: plain\ndescription: Plain\n---\nbody" needy = "---\nname: needy\ndescription: Needy\nmcp_servers: absent_server\n---\nbody" for name, body in (("plain", plain), ("needy", needy)): - skills.publish_local_skill( + skill_publication.publish_local_skill( name, files={"SKILL.md": body}, content_hash=skill_content_hash(body, {}) ) run = runtime.prepare("catalog-run", "u", skill_ids=["plain", "needy"]) @@ -259,7 +262,7 @@ def test_catalog_skill_with_missing_connector_is_dropped_not_fatal( def test_snapshot_missing_or_removed_skill_is_rejected(index_db, caps_root): - component = skills.publish_local_skill( + component = skill_publication.publish_local_skill( "gone", files={"SKILL.md": "dep"}, content_hash=skill_content_hash("dep", {}) ) binding = {"gone": {"install_id": "skill:local:gone", "revision": component.revision}} @@ -307,19 +310,19 @@ def test_cached_definition_is_detached_and_observes_edits(tmp_path): def test_parallel_eligibility_and_catalog_match_serial(durable_index, caps_root, monkeypatch): from types import SimpleNamespace - from core.capabilities import readiness, runtime + from features.desktop.capabilities import readiness, runtime - monkeypatch.setattr(skills, "builtin_candidates", lambda: []) - monkeypatch.setattr(skills, "current_account_profile", lambda: None) + monkeypatch.setattr(skill_candidates, 'builtin_candidates', lambda: []) + monkeypatch.setattr(skill_account, 'current_account_profile', lambda: None) for i in range(8): body = "---\nname: item-%s\n" % i if i == 7: body += "mcp_servers: absent-server\n" body += "---\nbody" - skills.publish_local_skill( + skill_publication.publish_local_skill( "item-%s" % i, files={"SKILL.md": body}, content_hash=skill_content_hash(body, {}) ) - candidates = skills.candidates("u") + candidates = skill_candidates.candidates("u") serial = readiness.eligible_skill_candidates(candidates, "u") monkeypatch.setattr(readiness, "os", SimpleNamespace(name="nt")) assert readiness.eligible_skill_candidates(candidates, "u") == serial diff --git a/src/backend/tests/capabilities/test_desktop_capabilities_api.py b/src/backend/tests/features/desktop/capabilities/test_desktop_capabilities_api.py similarity index 85% rename from src/backend/tests/capabilities/test_desktop_capabilities_api.py rename to src/backend/tests/features/desktop/capabilities/test_desktop_capabilities_api.py index 1269f32c..11a34aba 100644 --- a/src/backend/tests/capabilities/test_desktop_capabilities_api.py +++ b/src/backend/tests/features/desktop/capabilities/test_desktop_capabilities_api.py @@ -1,6 +1,10 @@ """本机能力接口：登录同步即全部就绪、只读的来源清单、视图重建。""" from __future__ import annotations +from features.desktop.bridge import identity as bridge_identity +from features.desktop.bridge import managed_connectors as bridge_managed_connectors +from features.desktop.capabilities import skill_account +from features.desktop.capabilities import skill_paths import copy import base64 @@ -10,20 +14,18 @@ import zipfile import pytest -from api.routes.v1.desktop_capabilities import router -from core.agent_skills import config as skill_config +from features.desktop.api.desktop_capabilities import router +from core.capabilities.skills import config as skill_config from core.auth.backend import UserContext, get_current_user -from core.capabilities import junction, registry, skills, store -from core.capabilities.paths import KIND_SKILL -from core.capabilities.ref import cloud_ref, profile_id -from core.services import desktop_cloud_bridge as bridge -from core.services import desktop_cloud_skills as cloud_skills -from core.services.desktop_capability_protocol import ( - build_entity_manifest, - build_manifest, - build_skill_manifest, - skill_content_hash, -) +from core.infra import directory_links as junction +from features.desktop.capabilities import registry +from features.desktop.capabilities import skills +from features.desktop.capabilities import store +from core.capabilities.identity_rules import KIND_SKILL +from core.capabilities.reference import cloud_ref, profile_id +from features.desktop.bridge import client as bridge +from features.desktop.bridge import skills as cloud_skills +from core.capabilities.manifest_protocol import build_entity_manifest, build_manifest, build_skill_manifest, skill_content_hash from fastapi import FastAPI from fastapi.testclient import TestClient @@ -112,7 +114,7 @@ def cloud(): @pytest.fixture def client(tmp_path, monkeypatch, index_db, cloud): - monkeypatch.setattr("core.services.desktop_capability_sync_check.CHECK_INTERVAL_SECONDS", 0) + monkeypatch.setattr('features.desktop.capability.sync_check.CHECK_INTERVAL_SECONDS', 0) monkeypatch.setenv("SANDBOX_SKILLS_DIR", str(tmp_path / "ws" / "skills")) monkeypatch.setenv("HUGAGENT_CAPS_ROOT", str(tmp_path / "caps")) @@ -121,12 +123,12 @@ def client(tmp_path, monkeypatch, index_db, cloud): (builtin / "ppt-design").mkdir(parents=True) (builtin / "ppt-design" / "SKILL.md").write_text(_md("ppt-design", "old")) monkeypatch.setattr(skill_config, "_builtin_skills_dir", lambda: builtin) - monkeypatch.setattr(skills, "builtin_dir", lambda: builtin) - bridge.reset_for_tests() - monkeypatch.setattr(bridge, "bridge_enabled", lambda: True) - monkeypatch.setattr(skills, "current_local_user_id", lambda: USER) - monkeypatch.setattr(bridge, "get_state", lambda: dict(STATE, expires_at=time.time() + 60)) - monkeypatch.setattr("core.agent_skills.cache_refresh.refresh_skill_caches", lambda: None) + monkeypatch.setattr(skill_paths, 'builtin_dir', lambda: builtin) + bridge_identity.reset_for_tests() + monkeypatch.setattr('features.desktop.auth.bridge.bridge_enabled', lambda: True) + monkeypatch.setattr(skill_account, 'current_local_user_id', lambda: USER) + monkeypatch.setattr(bridge_identity, 'get_state', lambda: dict(STATE, expires_at=time.time() + 60)) + monkeypatch.setattr('core.capabilities.skills.cache_refresh.refresh_skill_caches', lambda: None) monkeypatch.setattr("httpx.get", cloud.get) app = FastAPI() @@ -135,7 +137,7 @@ def client(tmp_path, monkeypatch, index_db, cloud): user_id=USER, user_center_id="c1", username="t" ) yield TestClient(app) - bridge.reset_for_tests() + bridge_identity.reset_for_tests() def _iid(sid: str) -> str: @@ -235,7 +237,7 @@ def test_each_account_manifest_change_is_detected(client, cloud, kind): if kind == "skill": cloud.manifest = build_skill_manifest([]) elif kind == "mcp": - from core.services.desktop_capability_protocol import canonical_hash + from core.capabilities.manifest_protocol import canonical_hash cloud.connectors = build_manifest([{"server_id": "new", "tools": [], "schema_hash": canonical_hash([])}]) elif kind == "agent": @@ -253,7 +255,7 @@ def test_each_account_manifest_change_is_detected(client, cloud, kind): if kind == "mcp": assert client.post(SYNC).status_code == 200 assert client.get(CHECK).json()["data"]["changed"] is False - assert any(s["server_id"] == "new" for s in bridge.managed_connectors()) + assert any(s["server_id"] == "new" for s in bridge_managed_connectors.managed_connectors()) def test_toggles_and_order_alone_are_not_updates(client, cloud): @@ -282,7 +284,7 @@ def offline(*args, **kwargs): def test_check_cache_is_shared_but_invalidated_by_new_local_snapshot(client, cloud, monkeypatch): assert client.post(SYNC).status_code == 200 cloud.manifest = build_skill_manifest([]) - monkeypatch.setattr("core.services.desktop_capability_sync_check.CHECK_INTERVAL_SECONDS", 60) + monkeypatch.setattr('features.desktop.capability.sync_check.CHECK_INTERVAL_SECONDS', 60) assert client.get(CHECK).json()["data"]["changed"] is True cloud.requests.clear() assert client.get(CHECK).json()["data"]["changed"] is True @@ -296,7 +298,7 @@ def test_check_rejects_an_account_switch_during_request(client, cloud, monkeypat original = cloud.get def switched(*args, **kwargs): result = original(*args, **kwargs) - monkeypatch.setattr(bridge, "get_state", lambda: {**STATE, "token": _token("u-2")}) + monkeypatch.setattr(bridge_identity, 'get_state', lambda: {**STATE, "token": _token("u-2")}) return result monkeypatch.setattr("httpx.get", switched) response = client.get(CHECK) diff --git a/src/backend/tests/capabilities/test_desktop_content_safety.py b/src/backend/tests/features/desktop/capabilities/test_desktop_content_safety.py similarity index 87% rename from src/backend/tests/capabilities/test_desktop_content_safety.py rename to src/backend/tests/features/desktop/capabilities/test_desktop_content_safety.py index d2b052f8..0d67d866 100644 --- a/src/backend/tests/capabilities/test_desktop_content_safety.py +++ b/src/backend/tests/features/desktop/capabilities/test_desktop_content_safety.py @@ -1,6 +1,9 @@ """Configured-secret canaries; no real configuration, credentials, or network.""" from __future__ import annotations +from features.desktop.api import desktop_capability +from features.desktop.api import capability_dependencies +from features.desktop.api import desktop_capability_state import asyncio import io @@ -8,10 +11,10 @@ import zipfile import pytest -from api.routes.v1 import desktop_capability as routes -from core.services import desktop_capability as cap -from core.services import desktop_capability_credentials as credentials -from core.services import desktop_capability_security as security +from features.desktop.api import desktop_capability as routes +from features.desktop.capability import service as cap +from features.desktop.capability import credentials as credentials +from features.desktop.capability import security as security from fastapi import FastAPI from fastapi.testclient import TestClient @@ -114,8 +117,8 @@ def test_manifest_boundaries_return_fixed_integrity_diagnostic(monkeypatch, path cap, fn, lambda uid, **kwargs: {"revision": "a" * 64, "description": CANARY} ) app = FastAPI() - app.include_router(routes.router) - app.dependency_overrides[routes._require_capability_user] = lambda: "user" + app.include_router(desktop_capability.router) + app.dependency_overrides[capability_dependencies.require_capability_user] = lambda: "user" with TestClient(app) as client: response = client.get("/v1/desktop/capability" + path) assert response.status_code == 422 @@ -137,8 +140,8 @@ def test_each_bundle_download_rejects_canary(monkeypatch, kind, fn): archive.writestr("example/body.txt", CANARY) monkeypatch.setattr(cap, fn, lambda uid, key: (output.getvalue(), "a" * 64)) app = FastAPI() - app.include_router(routes.router) - app.dependency_overrides[routes._require_capability_user] = lambda: "user" + app.include_router(desktop_capability.router) + app.dependency_overrides[capability_dependencies.require_capability_user] = lambda: "user" with TestClient(app) as client: response = client.get("/v1/desktop/capability/" + kind + "/example/bundle") assert response.status_code == 422 and CANARY not in response.text @@ -152,8 +155,8 @@ async def invoke(*args): monkeypatch.setattr(cap, "invoke_gateway_tool", invoke) app = FastAPI() - app.include_router(routes.router) - app.dependency_overrides[routes._require_capability_user] = lambda: "user" + app.include_router(desktop_capability.router) + app.dependency_overrides[capability_dependencies.require_capability_user] = lambda: "user" with TestClient(app) as client: response = client.post( "/v1/desktop/capability/gateway/example/call", @@ -176,7 +179,7 @@ async def upstream(request): return httpx.Response(200, headers={"content-type": "text/event-stream"}, stream=Stream()) remote = httpx.AsyncClient(transport=httpx.MockTransport(upstream)) - monkeypatch.setattr(routes, "_gateway_client", remote) + monkeypatch.setattr(desktop_capability_state, 'gateway_client', remote) monkeypatch.setattr( cap, "resolve_model_gateway_target", @@ -197,8 +200,8 @@ async def upstream(request): }, ) app = FastAPI() - app.include_router(routes.router) - app.dependency_overrides[routes._require_capability_user] = lambda: "user" + app.include_router(desktop_capability.router) + app.dependency_overrides[capability_dependencies.require_capability_user] = lambda: "user" path = "/gateway/models/example/chat/completions" if model else "/gateway/example/mcp" with TestClient(app) as client: response = client.post("/v1/desktop/capability" + path, json={"messages": []}) @@ -251,7 +254,7 @@ def query(self, *args): monkeypatch.setattr(security, "SessionLocal", Database) from types import SimpleNamespace - from core.services.model_config import ModelConfigService + from core.models.config import ModelConfigService monkeypatch.setattr(ModelConfigService, "_instance", SimpleNamespace(version=0)) secrets = _REAL_COLLECTOR("user") @@ -277,18 +280,14 @@ def unavailable(*args, **kwargs): async def test_actual_invocation_checks_successful_result(monkeypatch): from types import SimpleNamespace - from core.llm import mcp_pool - - async def fake_tool(**kwargs): - return SimpleNamespace( - metadata={}, model_dump=lambda **kwargs: {"content": [{"text": CANARY}]} - ) + from core.llm.mcp import client as mcp_pool class Client: - execution_timeout = 1 - - async def get_tool(self, name): - return fake_tool + async def __aenter__(self): return self + async def __aexit__(self, *exc): pass + async def call_tool(self, name, arguments): + from mcp.types import CallToolResult, TextContent + return CallToolResult(content=[TextContent(text=CANARY)]) monkeypatch.setattr(mcp_pool, "make_client", lambda *args, **kwargs: Client()) with pytest.raises(cap.CapabilityContentRejected): @@ -362,20 +361,16 @@ def test_userinfo_token_without_password_is_guarded(): async def test_actual_invocation_checks_custom_header_echo(monkeypatch): from types import SimpleNamespace - from core.llm import mcp_pool + from core.llm.mcp import client as mcp_pool monkeypatch.setattr(security, "_known_cloud_secrets", lambda uid: set()) - async def fake_tool(**kwargs): - return SimpleNamespace( - metadata={}, model_dump=lambda **kwargs: {"content": [{"text": CANARY}]} - ) - class Client: - execution_timeout = 1 - - async def get_tool(self, name): - return fake_tool + async def __aenter__(self): return self + async def __aexit__(self, *exc): pass + async def call_tool(self, name, arguments): + from mcp.types import CallToolResult, TextContent + return CallToolResult(content=[TextContent(text=CANARY)]) monkeypatch.setattr(mcp_pool, "make_client", lambda *args, **kwargs: Client()) with pytest.raises(cap.CapabilityContentRejected): @@ -409,8 +404,8 @@ def build(uid, *, use_cache=True): monkeypatch.setattr(cap, fn, build) app = FastAPI() - app.include_router(routes.router) - app.dependency_overrides[routes._require_capability_user] = lambda: "current-user" + app.include_router(desktop_capability.router) + app.dependency_overrides[capability_dependencies.require_capability_user] = lambda: "current-user" with TestClient(app) as client: assert client.get("/v1/desktop/capability" + path).status_code == 200 assert ( diff --git a/src/backend/tests/capabilities/test_desktop_declaration_export.py b/src/backend/tests/features/desktop/capabilities/test_desktop_declaration_export.py similarity index 89% rename from src/backend/tests/capabilities/test_desktop_declaration_export.py rename to src/backend/tests/features/desktop/capabilities/test_desktop_declaration_export.py index aadd9c04..a5189fe8 100644 --- a/src/backend/tests/capabilities/test_desktop_declaration_export.py +++ b/src/backend/tests/features/desktop/capabilities/test_desktop_declaration_export.py @@ -1,12 +1,15 @@ +from features.desktop.api import desktop_capability """Synthetic declaration export checks; no user configuration or credentials.""" +from features.desktop.api import capability_dependencies +from features.desktop.api import desktop_capability_state import json import pytest -from core.services import desktop_capability as cap -from core.services import desktop_capability_credentials as credentials -from core.services import desktop_capability_entities as entities -from core.services import desktop_capability_security as security +from features.desktop.capability import service as cap +from features.desktop.capability import credentials as credentials +from features.desktop.capability import entities as entities +from features.desktop.capability import security as security def agent(data): @@ -120,7 +123,7 @@ def test_legacy_plugin_lists_remain_compatible(): def test_optional_unknown_extension_remains_optional_without_executable_payload(): - from core.capabilities.dependency import _entries + from features.desktop.capabilities.dependency import _entries result = agent( {"extensions": {"future_hook": {"required": False, "script": "secret execution content"}}} @@ -140,7 +143,7 @@ def test_declaration_edits_change_manifest_content_hash(monkeypatch): assert before["revision"] != after["revision"] -from tests.capabilities.test_agents_plugins import cloud_db +from tests.features.desktop.capabilities.test_agents_plugins import cloud_db def test_database_component_metadata_reaches_plugin_bundle(cloud_db): @@ -168,7 +171,7 @@ def test_database_component_metadata_reaches_plugin_bundle(cloud_db): def test_invalid_declaration_returns_fixed_http_error(monkeypatch): - from api.routes.v1 import desktop_capability as routes + from features.desktop.api import desktop_capability as routes from fastapi import FastAPI from fastapi.testclient import TestClient @@ -185,8 +188,8 @@ def test_invalid_declaration_returns_fixed_http_error(monkeypatch): ], ) app = FastAPI() - app.include_router(routes.router) - app.dependency_overrides[routes._require_capability_user] = lambda: "user" + app.include_router(desktop_capability.router) + app.dependency_overrides[capability_dependencies.require_capability_user] = lambda: "user" with TestClient(app) as client: response = client.get("/v1/desktop/capability/agents/writer/bundle") assert response.status_code == 422 diff --git a/src/backend/tests/capabilities/test_desktop_model_reasoning_safety.py b/src/backend/tests/features/desktop/capabilities/test_desktop_model_reasoning_safety.py similarity index 88% rename from src/backend/tests/capabilities/test_desktop_model_reasoning_safety.py rename to src/backend/tests/features/desktop/capabilities/test_desktop_model_reasoning_safety.py index ed35d8ed..92a49f4b 100644 --- a/src/backend/tests/capabilities/test_desktop_model_reasoning_safety.py +++ b/src/backend/tests/features/desktop/capabilities/test_desktop_model_reasoning_safety.py @@ -1,13 +1,16 @@ +from features.desktop.api import desktop_capability """Model synchronization must distinguish reasoning labels from credentials.""" +from features.desktop.api import capability_dependencies +from features.desktop.api import desktop_capability_state import pytest -from api.routes.v1 import desktop_capability as routes -from core.db.model_repository import assign_role, create_provider -from core.services import desktop_capability as cap -from core.services import desktop_capability_credentials as credentials -from core.services import desktop_capability_models as models -from core.services import desktop_capability_security as security -from core.services import model_config +from features.desktop.api import desktop_capability as routes +from core.models.repository import assign_role, create_provider +from features.desktop.capability import service as cap +from features.desktop.capability import credentials as credentials +from features.desktop.capability import models as models +from features.desktop.capability import security as security +from core.models import config as model_config from fastapi import FastAPI from fastapi.testclient import TestClient from sqlalchemy.orm import sessionmaker @@ -22,8 +25,8 @@ def model_cloud(monkeypatch, db_session): monkeypatch.setattr(security, "_user_capability_configs", lambda *a, **k: ([], [], {})) cap.invalidate_model_gateway_cache() app = FastAPI() - app.include_router(routes.router) - app.dependency_overrides[routes._require_capability_user] = lambda: "fixture-user" + app.include_router(desktop_capability.router) + app.dependency_overrides[capability_dependencies.require_capability_user] = lambda: "fixture-user" with TestClient(app) as client: yield db_session, client diff --git a/src/backend/tests/capabilities/test_desktop_resource_plugins.py b/src/backend/tests/features/desktop/capabilities/test_desktop_resource_plugins.py similarity index 87% rename from src/backend/tests/capabilities/test_desktop_resource_plugins.py rename to src/backend/tests/features/desktop/capabilities/test_desktop_resource_plugins.py index abc2038c..d25e8d0e 100644 --- a/src/backend/tests/capabilities/test_desktop_resource_plugins.py +++ b/src/backend/tests/features/desktop/capabilities/test_desktop_resource_plugins.py @@ -6,11 +6,11 @@ import pytest from fastapi import HTTPException -ROOT = Path(__file__).parents[2] / "extensions/plugins/marketplace/browser-automation" +ROOT = Path(__file__).parents[4] / "extensions/plugins/marketplace/browser-automation" def test_export_includes_exact_worker_and_canvas(monkeypatch): - from core.services import desktop_resource_plugins as resources + from features.desktop.bridge import resource_plugins as resources monkeypatch.setattr(resources, "package_root", lambda installed: ROOT) files, native = resources.package_files( @@ -32,8 +32,8 @@ def test_export_includes_exact_worker_and_canvas(monkeypatch): @pytest.fixture def synced(monkeypatch, tmp_path): - from core.capabilities import device_catalog, store, dependency - from core.services import desktop_resource_plugins as resources + from features.desktop.capabilities import device_catalog, store, dependency + from features.desktop.bridge import resource_plugins as resources root = tmp_path / "revision" (root / "package/runtime").mkdir(parents=True) @@ -54,7 +54,7 @@ def synced(monkeypatch, tmp_path): ) monkeypatch.setattr( device_catalog, - "_installations", + "installations", lambda kind, user_id: [inst] if user_id == "device-user" else [], ) monkeypatch.setattr(store, "get", lambda *args: SimpleNamespace(path=root)) @@ -92,7 +92,7 @@ def test_binding_uses_local_endpoint_and_exact_installation(synced): @pytest.mark.parametrize("failure", ["missing", "hash", "disabled"]) def test_broken_local_resource_never_falls_back_to_cloud(synced, monkeypatch, failure): - from core.capabilities import dependency + from features.desktop.capabilities import dependency resources, inst, root = synced if failure == "missing": @@ -106,9 +106,9 @@ def test_broken_local_resource_never_falls_back_to_cloud(synced, monkeypatch, fa def test_runtime_proof_preserves_device_user_chat_and_cloud_installation(synced, monkeypatch): - from core.llm.factory.tools.mcp_config import _inject_runtime_headers - from core.llm.mcp_invocation import verify - from core.kb import external_provider + from core.llm.factory.tools.mcp_config import inject_runtime_headers as _inject_runtime_headers + from core.llm.mcp.invocation import verify + from features.knowledge import edition as external_provider resources, inst, _ = synced monkeypatch.setenv("BACKEND_INTERNAL_TOKEN", "isolated-resource-proof") @@ -129,8 +129,8 @@ def test_runtime_proof_preserves_device_user_chat_and_cloud_installation(synced, def test_uploaded_definition_gets_resource_assets_too(monkeypatch): - from core.services import desktop_resource_plugins as resources - from core.services.desktop_capability_entities import _plugin_files + from features.desktop.bridge import resource_plugins as resources + from features.desktop.capability.entities import _plugin_files monkeypatch.setattr(resources, "package_root", lambda installed: ROOT) installed = { @@ -152,7 +152,7 @@ def test_uploaded_definition_gets_resource_assets_too(monkeypatch): def test_unrelated_uploaded_plugin_is_byte_for_byte_unchanged(): - from core.services.desktop_capability_entities import _plugin_files + from features.desktop.capability.entities import _plugin_files files = {"plugin.json": '{"name":"Untouched"}\n', "notes.txt": "owned content"} assert ( @@ -169,7 +169,6 @@ def test_unrelated_uploaded_plugin_is_byte_for_byte_unchanged(): def test_native_client_keeps_the_cloud_authorized_tool_scope(synced): - from core.llm.mcp_pool import make_client resources, _, _ = synced cfg = resources.local_config( @@ -177,5 +176,4 @@ def test_native_client_keeps_the_cloud_authorized_tool_scope(synced): "device-user", "account", ) - client = make_client("account-browser", cfg, is_stateful=False) - assert client.enable_tools == ["browser_open"] + assert cfg["enable_tools"] == ["browser_open"] diff --git a/src/backend/tests/capabilities/test_desktop_token_security.py b/src/backend/tests/features/desktop/capabilities/test_desktop_token_security.py similarity index 90% rename from src/backend/tests/capabilities/test_desktop_token_security.py rename to src/backend/tests/features/desktop/capabilities/test_desktop_token_security.py index e7bd5e3c..16bf6aae 100644 --- a/src/backend/tests/capabilities/test_desktop_token_security.py +++ b/src/backend/tests/features/desktop/capabilities/test_desktop_token_security.py @@ -1,6 +1,10 @@ """Desktop token contract; synthetic memory sessions and a fixed fake signing key only.""" from __future__ import annotations +from features.desktop.bridge import client as bridge_client +from features.desktop.bridge import identity as bridge_identity +from features.desktop.api import desktop_capability +from features.desktop.api import desktop_capability_state import asyncio import base64 @@ -10,12 +14,12 @@ import pytest from api.deps import require_config -from api.routes.v1 import desktop_capability as routes +from features.desktop.api import desktop_capability as routes from core.auth import session from core.config.settings import settings -from core.services import desktop_capability as cap -from core.services import desktop_capability_credentials as credentials -from core.services import desktop_capability_security as security +from features.desktop.capability import service as cap +from features.desktop.capability import credentials as credentials +from features.desktop.capability import security as security from fastapi import FastAPI from fastapi.testclient import TestClient @@ -25,7 +29,7 @@ @pytest.fixture(autouse=True) def isolated_sessions(monkeypatch): - monkeypatch.setattr(session, "_MEMORY_SESSIONS", {}) + monkeypatch.setattr(session.process_state(), 'MEMORY_SESSIONS', {}) monkeypatch.setattr(session, "_use_memory_store", lambda: True) monkeypatch.setattr(cap, "_secret_cache", "ab" * 32) monkeypatch.setattr(security, "_known_cloud_secrets", lambda uid: set()) @@ -41,7 +45,7 @@ def _session(uid="user-a"): def _issue(cookie, uid="user-a", **kwargs): session_hash = session._hash_token(cookie) - payload = session._MEMORY_SESSIONS[session_hash]["payload"] + payload = session.process_state().MEMORY_SESSIONS[session_hash]["payload"] return cap.issue_capability_token( uid, device_id=DEVICE, @@ -134,7 +138,7 @@ def test_logout_immediately_revokes_derived_token(): def test_expired_session_and_changed_epoch_are_rejected(): cookie = _session() token = _issue(cookie)["token"] - entry = session._MEMORY_SESSIONS[session._hash_token(cookie)] + entry = session.process_state().MEMORY_SESSIONS[session._hash_token(cookie)] entry["payload"]["created_at"] = datetime(2020, 1, 1, tzinfo=timezone.utc).isoformat() assert _verify(token) is None entry["expires_at"] = datetime.now(timezone.utc) - timedelta(seconds=1) @@ -144,7 +148,7 @@ def test_expired_session_and_changed_epoch_are_rejected(): def test_token_check_does_not_renew_session(): cookie = _session() token = _issue(cookie)["token"] - entry = session._MEMORY_SESSIONS[session._hash_token(cookie)] + entry = session.process_state().MEMORY_SESSIONS[session._hash_token(cookie)] expiry = entry["expires_at"] assert _verify(token) == "user-a" assert entry["expires_at"] == expiry @@ -152,7 +156,7 @@ def test_token_check_does_not_renew_session(): def _client(): app = FastAPI() - app.include_router(routes.router) + app.include_router(desktop_capability.router) return TestClient(app, base_url=ISSUER) @@ -277,14 +281,14 @@ async def test_redis_session_online_revocation_and_no_renewal(monkeypatch): def test_bridge_control_requires_shell_secret_even_if_config_permission_is_granted( monkeypatch, method, headers ): - from core.services import desktop_cloud_bridge as bridge + from features.desktop.bridge import client as bridge monkeypatch.setenv("HUGAGENT_DESKTOP_BRIDGE_SECRET", "synthetic-shell-secret") calls = [] - monkeypatch.setattr(bridge, "set_state", lambda *a, **kw: calls.append("set")) - monkeypatch.setattr(bridge, "clear_state", lambda: calls.append("clear")) + monkeypatch.setattr(bridge_client, 'set_state', lambda *a, **kw: calls.append("set")) + monkeypatch.setattr(bridge_identity, 'clear_state', lambda: calls.append("clear")) app = FastAPI() - app.include_router(routes.router) + app.include_router(desktop_capability.router) app.dependency_overrides[require_config] = lambda: None with TestClient(app) as client: response = client.request( @@ -302,14 +306,14 @@ def test_bridge_control_requires_shell_secret_even_if_config_permission_is_grant def test_bridge_control_accepts_direct_shell_for_set_and_clear(monkeypatch): - from core.services import desktop_cloud_bridge as bridge + from features.desktop.bridge import client as bridge monkeypatch.setenv("HUGAGENT_DESKTOP_BRIDGE_SECRET", "synthetic-shell-secret") calls = [] monkeypatch.setattr( - bridge, "set_state", lambda *a, **kw: calls.append(("set", kw["device_id"])) + bridge_client, 'set_state', lambda *a, **kw: calls.append(("set", kw["device_id"])) ) - monkeypatch.setattr(bridge, "clear_state", lambda: calls.append(("clear", None))) + monkeypatch.setattr(bridge_identity, 'clear_state', lambda: calls.append(("clear", None))) with _client() as client: headers = {"Authorization": "Bearer synthetic-shell-secret"} response = client.post( diff --git a/src/backend/tests/capabilities/test_desktop_warmup.py b/src/backend/tests/features/desktop/capabilities/test_desktop_warmup.py similarity index 61% rename from src/backend/tests/capabilities/test_desktop_warmup.py rename to src/backend/tests/features/desktop/capabilities/test_desktop_warmup.py index 3a8f3f42..5e2e1daa 100644 --- a/src/backend/tests/capabilities/test_desktop_warmup.py +++ b/src/backend/tests/features/desktop/capabilities/test_desktop_warmup.py @@ -1,10 +1,14 @@ """Startup reads never include another user's packages or a replaced account.""" +from core.capabilities.skills import loader_factory as loader_factory_module +from features.desktop.bridge import identity as bridge_identity +from features.desktop.capabilities import skill_account +from features.desktop.capabilities import skills from types import SimpleNamespace -from core.capabilities import warmup, skills, dependency, registry, store -from core.services import desktop_cloud_bridge as bridge -from core.agent_skills import loader +from features.desktop.capabilities import warmup, skills, dependency, registry, store +from features.desktop.bridge import client as bridge +from core.capabilities.skills import loader def test_warmup_filters_identity_and_owner(monkeypatch): @@ -12,18 +16,18 @@ def test_warmup_filters_identity_and_owner(monkeypatch): reads = [] materialized = [] monkeypatch.setattr(warmup, "os", SimpleNamespace(name="nt")) - monkeypatch.setattr("core.capabilities.paths.capabilities_enabled", lambda: True) - monkeypatch.setattr(bridge, "get_state", lambda: state[0]) - monkeypatch.setattr(bridge, "_state_fingerprint", lambda value: value) - monkeypatch.setattr(skills, "current_local_user_id", lambda: "u") - monkeypatch.setattr(skills, "current_account_profile", lambda: "account-a") - monkeypatch.setattr(skills, "account_authorized_for", lambda uid: uid == "u") + monkeypatch.setattr('features.desktop.capabilities.paths.capabilities_enabled', lambda: True) + monkeypatch.setattr(bridge_identity, 'get_state', lambda: state[0]) + monkeypatch.setattr(bridge_identity, 'state_fingerprint', lambda value: value) + monkeypatch.setattr(skill_account, 'current_local_user_id', lambda: "u") + monkeypatch.setattr(skill_account, 'current_account_profile', lambda: "account-a") + monkeypatch.setattr(skill_account, 'account_authorized_for', lambda uid: uid == "u") monkeypatch.setattr( - skills, "resolve_for_user", lambda uid: SimpleNamespace(chosen={"own": None, "other": None}) + skills, 'resolve_for_user', lambda uid: SimpleNamespace(chosen={"own": None, "other": None}) ) monkeypatch.setattr( - loader, - "get_skill_loader", + loader_factory_module, + 'get_skill_loader', lambda: SimpleNamespace( get_skill_owner=lambda name: "u" if name == "own" else "v", get_skill_dir=materialized.append, diff --git a/src/backend/tests/capabilities/test_device_api_contract.py b/src/backend/tests/features/desktop/capabilities/test_device_api_contract.py similarity index 74% rename from src/backend/tests/capabilities/test_device_api_contract.py rename to src/backend/tests/features/desktop/capabilities/test_device_api_contract.py index d944917b..b8b00dee 100644 --- a/src/backend/tests/capabilities/test_device_api_contract.py +++ b/src/backend/tests/features/desktop/capabilities/test_device_api_contract.py @@ -1,11 +1,15 @@ """本机能力接口只读：来源清单与 mcp.json 投影都不能越账号，同步也要认账号。""" +from features.desktop.bridge import client as bridge_client +from features.desktop.bridge import identity as bridge_identity +from features.desktop.capabilities import skill_account +from features.desktop.capabilities import skill_publication from types import SimpleNamespace import pytest -from core.capabilities import registry, store -from core.capabilities.ref import cloud_ref, local_ref -from tests.capabilities.test_desktop_capabilities_api import cloud as cloud -from tests.capabilities.test_desktop_capabilities_api import client, USER, PROFILE +from features.desktop.capabilities import registry, store +from core.capabilities.reference import cloud_ref, local_ref +from tests.features.desktop.capabilities.test_desktop_capabilities_api import cloud as cloud +from tests.features.desktop.capabilities.test_desktop_capabilities_api import client, USER, PROFILE def install(kind, key="shared-name", profile=PROFILE, owner=None): @@ -28,7 +32,7 @@ def install(kind, key="shared-name", profile=PROFILE, owner=None): def test_mcp_document_is_scoped_to_active_account(client, monkeypatch): - from core.capabilities import mcp_json + from features.desktop.capabilities import mcp_json doc = SimpleNamespace( generation=1, @@ -44,15 +48,15 @@ def test_mcp_document_is_scoped_to_active_account(client, monkeypatch): def test_disabled_cloud_mcp_remains_visible_and_cannot_win(client, monkeypatch): - from core.capabilities import mcp_json - from core.services import desktop_cloud_bridge as bridge - from core.services.mcp_service import McpServerConfigService + from features.desktop.capabilities import mcp_json + from features.desktop.bridge import client as bridge + from core.capabilities.mcp.service import McpServerConfigService manifest = { "revision": "r1", "servers": [{"server_id": "remote", "component": "remote", "tools": [{"name": "run"}]}], } - monkeypatch.setattr(bridge, "get_cached_manifest", lambda: manifest) + monkeypatch.setattr(bridge_client, 'get_cached_manifest', lambda: manifest) monkeypatch.setattr(mcp_json, "managed_enabled", lambda profile: {"remote": False}) monkeypatch.setattr( McpServerConfigService, @@ -71,8 +75,8 @@ def test_disabled_cloud_mcp_remains_visible_and_cannot_win(client, monkeypatch): @pytest.mark.parametrize("center", ["different-account", None]) def test_signed_cloud_subject_rejects_stale_identity_header(client, monkeypatch, center): import base64, json - from core.auth import desktop_bridge - from core.services import desktop_cloud_bridge + from features.desktop.auth import bridge as desktop_bridge + from features.desktop.bridge import client as desktop_cloud_bridge claims = ( base64.urlsafe_b64encode(json.dumps({"u": "cloud-u", "c": center}).encode()) @@ -80,8 +84,8 @@ def test_signed_cloud_subject_rejects_stale_identity_header(client, monkeypatch, .rstrip("=") ) monkeypatch.setattr( - desktop_cloud_bridge, - "get_identity_state", + bridge_identity, + 'get_identity_state', lambda: {"user_center_id": center, "shell_user_center_id": center}, ) user_header = base64.b64encode( @@ -96,14 +100,14 @@ def test_signed_cloud_subject_rejects_stale_identity_header(client, monkeypatch, @pytest.mark.parametrize("kind", ["skill", "agent", "plugin", "mcp"]) async def test_management_lists_cannot_expose_another_bridged_account(client, monkeypatch, kind): - from core.capabilities import skills - from core.services import desktop_cloud_bridge as bridge - from core.services.mcp_service import McpServerConfigService - from core.services.user_agent_service import UserAgentService + from features.desktop.capabilities import skills + from features.desktop.bridge import client as bridge + from core.capabilities.mcp.service import McpServerConfigService + from core.capabilities.agents.edition import UserAgentService if kind != "mcp": install(kind, key="private-account-b") - monkeypatch.setattr(skills, "current_local_user_id", lambda: "local-account-b") + monkeypatch.setattr(skill_account, 'current_local_user_id', lambda: "local-account-b") monkeypatch.setattr( McpServerConfigService, "get_instance", @@ -117,8 +121,8 @@ async def test_management_lists_cannot_expose_another_bridged_account(client, mo lambda self, db: setattr(self, "repo", SimpleNamespace(list_for_user=lambda uid: [])), ) monkeypatch.setattr( - bridge, - "get_cached_manifest", + bridge_client, + 'get_cached_manifest', lambda: { "servers": [{"server_id": "private-account-b", "tools": [{"name": "private-tool"}]}] }, @@ -130,9 +134,9 @@ async def test_management_lists_cannot_expose_another_bridged_account(client, mo def test_mcp_document_and_sync_cannot_target_another_bridged_account(client, monkeypatch): - from core.capabilities import skills, mcp_json + from features.desktop.capabilities import skills, mcp_json - monkeypatch.setattr(skills, "current_local_user_id", lambda: "local-account-b") + monkeypatch.setattr(skill_account, 'current_local_user_id', lambda: "local-account-b") monkeypatch.setattr( mcp_json, "load", @@ -151,18 +155,18 @@ def test_mcp_document_and_sync_cannot_target_another_bridged_account(client, mon def test_local_skill_stays_listed_without_a_live_cloud_account(client, monkeypatch): """云端断线不影响本机技能：它不属于任何云端账号，照样列出且可用。""" - from core.services import desktop_cloud_bridge as bridge - from core.capabilities import skills - from core.services.desktop_capability_protocol import skill_content_hash + from features.desktop.bridge import client as bridge + from features.desktop.capabilities import skills + from core.capabilities.manifest_protocol import skill_content_hash content = "---\nname: offline-local\ndescription: Offline fixture\n---\nlocal content\n" - skills.publish_local_skill( + skill_publication.publish_local_skill( "offline-local", files={"SKILL.md": content}, content_hash=skill_content_hash(content, {}), owner_user_id=USER, ) - monkeypatch.setattr(bridge, "get_state", lambda: None) + monkeypatch.setattr(bridge_identity, 'get_state', lambda: None) response = client.get("/v1/desktop/capabilities/installations", params={"kind": "skill"}) assert response.status_code == 200 item = next( diff --git a/src/backend/tests/capabilities/test_device_catalog_projection.py b/src/backend/tests/features/desktop/capabilities/test_device_catalog_projection.py similarity index 91% rename from src/backend/tests/capabilities/test_device_catalog_projection.py rename to src/backend/tests/features/desktop/capabilities/test_device_catalog_projection.py index 0e30da3c..1960429c 100644 --- a/src/backend/tests/capabilities/test_device_catalog_projection.py +++ b/src/backend/tests/features/desktop/capabilities/test_device_catalog_projection.py @@ -5,15 +5,17 @@ """ from __future__ import annotations +from features.desktop.bridge import identity as bridge_identity +from features.desktop.bridge import managed_connectors as bridge_managed_connectors import base64 import json import pytest -from core.capabilities import device_catalog, registry -from core.capabilities.paths import KIND_AGENT, KIND_PLUGIN, KIND_SKILL -from core.capabilities.ref import cloud_ref, profile_id -from core.services import desktop_cloud_bridge as bridge +from features.desktop.capabilities import device_catalog, registry +from core.capabilities.identity_rules import KIND_AGENT, KIND_PLUGIN, KIND_SKILL +from core.capabilities.reference import cloud_ref, profile_id +from features.desktop.bridge import client as bridge _CLOUD = "https://cloud.example" @@ -35,8 +37,8 @@ def _state(uid: str = "u1") -> dict: def bridged(index_db, caps_root, monkeypatch): """桥已激活、身份已认出的本机后端。""" st = _state() - monkeypatch.setattr(bridge, "get_state", lambda: st) - monkeypatch.setattr("core.auth.desktop_bridge.bridge_enabled", lambda: True) + monkeypatch.setattr(bridge_identity, 'get_state', lambda: st) + monkeypatch.setattr('features.desktop.auth.bridge.bridge_enabled', lambda: True) return st, profile_id(_CLOUD, "u1") @@ -51,7 +53,7 @@ def _install(profile: str, kind: str, key: str, **kwargs): def test_inactive_off_desktop_returns_nothing(index_db, monkeypatch): - monkeypatch.setattr("core.auth.desktop_bridge.bridge_enabled", lambda: False) + monkeypatch.setattr('features.desktop.auth.bridge.bridge_enabled', lambda: False) assert device_catalog.active() is False assert device_catalog.catalog_overlay()["skills"] == [] assert device_catalog.agent_entries() == [] @@ -273,7 +275,7 @@ def test_database_connectors_collapse_into_the_umbrella_item(bridged, monkeypatc 桌面端此前直接摊开云端下发的原始 server，同一份能力在网页端叫「数据库查询」、 在桌面端叫「Elasticsearch 查询」，用户会以为数据库工具没同步下来。 """ - from core.config.catalog_loader import DB_UMBRELLA_ID, DB_UMBRELLA_NAME + from core.capabilities.catalog.loader import DB_UMBRELLA_ID, DB_UMBRELLA_NAME monkeypatch.setattr( device_catalog, @@ -302,13 +304,13 @@ def test_database_connectors_collapse_into_the_umbrella_item(bridged, monkeypatc def test_umbrella_toggle_reaches_every_member(bridged, monkeypatch): """伞形条目本身没有对应的 server，开关必须落到它收拢的成员上。""" - from core.config.catalog_loader import DB_UMBRELLA_ID + from core.capabilities.catalog.loader import DB_UMBRELLA_ID # bridge 模块在 import 时就把 bridge_enabled 绑成了本地名，得按这个名字打桩。 - monkeypatch.setattr(bridge, "bridge_enabled", lambda: True) + monkeypatch.setattr('features.desktop.auth.bridge.bridge_enabled', lambda: True) monkeypatch.setattr( - bridge, - "managed_connectors", + bridge_managed_connectors, + 'managed_connectors', lambda: [ {"server_id": "es_query", "enabled": True}, {"server_id": "query_database", "enabled": True}, @@ -317,19 +319,19 @@ def test_umbrella_toggle_reaches_every_member(bridged, monkeypatch): ) written: list = [] monkeypatch.setattr( - "core.capabilities.mcp_json.set_managed_enabled", + 'features.desktop.capabilities.mcp_json.set_managed_enabled', lambda profile, server_id, enabled: written.append((server_id, enabled)), ) - assert bridge.set_managed_connector_enabled(DB_UMBRELLA_ID, False) is True + assert bridge_managed_connectors.set_managed_connector_enabled(DB_UMBRELLA_ID, False) is True assert sorted(written) == [("es_query", False), ("query_database", False)] def _install_plugin_with_ui(profile: str, slug: str, ui: dict, *, enabled: bool = True): """云端同步下来的插件：登记表一条 + 存储里的 plugin.json（带界面贡献）。""" - from core.capabilities import store - from core.capabilities.paths import revision_for_hash + from features.desktop.capabilities import store + from features.desktop.capabilities.paths import revision_for_hash manifest = {"slug": slug, "name": slug, "ui_contributions": ui} files = {"plugin.json": json.dumps(manifest, ensure_ascii=False)} diff --git a/src/backend/tests/capabilities/test_dynamic_model_epoch_cache.py b/src/backend/tests/features/desktop/capabilities/test_dynamic_model_epoch_cache.py similarity index 80% rename from src/backend/tests/capabilities/test_dynamic_model_epoch_cache.py rename to src/backend/tests/features/desktop/capabilities/test_dynamic_model_epoch_cache.py index 1e3dc2d3..52fcddc0 100644 --- a/src/backend/tests/capabilities/test_dynamic_model_epoch_cache.py +++ b/src/backend/tests/features/desktop/capabilities/test_dynamic_model_epoch_cache.py @@ -1,4 +1,6 @@ """Real dynamic model objects must keep their captured desktop authorization epoch.""" +from core.llm.runtime import model_cache as model_cache_module +from features.desktop.bridge import identity as bridge_identity import base64 import json @@ -8,11 +10,11 @@ import pytest import pytest_asyncio -from core.capabilities.errors import CloudUnavailable -from core.capabilities.ref import profile_id -from core.llm import chat_models, hooks -from core.services import desktop_cloud_bridge as bridge -from core.services.model_config import ModelConfigService, ResolvedModelConfig +from core.harness.capability_errors import CloudUnavailable +from core.capabilities.reference import profile_id +from core.models import chat_models +from features.desktop.bridge import client as bridge +from core.models.config import ModelConfigService, ResolvedModelConfig def state( @@ -39,12 +41,12 @@ async def models(monkeypatch, caps_root): current = {"state": state()} monkeypatch.setenv("HUGAGENT_DESKTOP_BRIDGE_SECRET", "synthetic-shell-secret") monkeypatch.setattr( - bridge, "get_state", lambda: dict(current["state"]) if current["state"] else None + bridge_identity, 'get_state', lambda: dict(current["state"]) if current["state"] else None ) - monkeypatch.setattr(hooks, "_model_cache", {}) - monkeypatch.setattr(hooks, "_cached_version", -1) - monkeypatch.setattr(chat_models, "_HTTP_CLIENTS", {}) - monkeypatch.setattr("core.llm.context_manager.resolve_model_context_window", lambda model: 8192) + monkeypatch.setattr(model_cache_module.process_state(), 'model_cache', {}) + monkeypatch.setattr(model_cache_module.process_state(), 'cached_version', -1) + monkeypatch.setattr(chat_models.process_state(), 'HTTP_CLIENTS', {}) + monkeypatch.setattr('core.models.context_window.resolve_model_context_window', lambda model: 8192) service = SimpleNamespace(version=0) def configure(value, *, direct=False): @@ -68,12 +70,12 @@ def configure(value, *, direct=False): monkeypatch.setattr(ModelConfigService, "get_instance", lambda: service) current.update(configure=configure, service=service) yield current - for client in chat_models._HTTP_CLIENTS.values(): + for client in chat_models.process_state().HTTP_CLIENTS.values(): await client.aclose() def model(kind, mode="medium"): - return hooks._get_main_model(mode) if kind == "main" else hooks._get_provider_model("p1", mode) + return model_cache_module.get_main_model(mode) if kind == "main" else model_cache_module.get_provider_model("p1", mode) async def authorize(instance): @@ -135,7 +137,7 @@ async def test_non_desktop_retains_mode_and_config_version_cache(models, monkeyp def forbidden(): raise AssertionError("non-desktop cache must not consult desktop identity") - monkeypatch.setattr(bridge, "get_state", forbidden) + monkeypatch.setattr(bridge_identity, 'get_state', forbidden) original = model(kind) assert model(kind) is original assert model(kind, "fast") is not original diff --git a/src/backend/tests/capabilities/test_explicit_desktop_invocation.py b/src/backend/tests/features/desktop/capabilities/test_explicit_desktop_invocation.py similarity index 79% rename from src/backend/tests/capabilities/test_explicit_desktop_invocation.py rename to src/backend/tests/features/desktop/capabilities/test_explicit_desktop_invocation.py index 1d155049..4727ed47 100644 --- a/src/backend/tests/capabilities/test_explicit_desktop_invocation.py +++ b/src/backend/tests/features/desktop/capabilities/test_explicit_desktop_invocation.py @@ -1,20 +1,23 @@ """The actual /skill, /connector and /plugin API resolves desktop-only sources.""" +from features.desktop.bridge import gateway as bridge_gateway +from features.desktop.capabilities import skill_publication import pytest -from core.services.session_operations.invocation import _resolve_explicit_capability_invocation +from core.chat.operations.invocation import resolve_explicit_capability_invocation as _resolve_explicit_capability_invocation from core.chat.schemas import ChatRequest -from core.capabilities import registry, skills, store -from core.capabilities.paths import revision_for_hash -from core.config import catalog_resolver -from core.services import desktop_cloud_bridge as bridge +from features.desktop.capabilities import registry, skills, store +from features.desktop.capabilities.paths import revision_for_hash +from core.capabilities.catalog import resolver as catalog_resolver +from features.desktop.bridge import client as bridge from fastapi import HTTPException -from tests.capabilities.test_cloud_plugin_binding_runtime import cloud_plugin +from tests.features.desktop.capabilities.test_cloud_plugin_binding_runtime import cloud_plugin @pytest.fixture def explicit_cloud(index_db, caps_root, monkeypatch): state, plugin, component, skill = cloud_plugin(monkeypatch, index_db) - from core.db.models import AdminMcpServer, AdminSkill, InstalledPlugin, McpMarketInstallation + from core.db.models import AdminMcpServer, AdminSkill, InstalledPlugin + from features.marketplace.models import McpMarketInstallation with index_db() as db: for model in (AdminSkill, AdminMcpServer, InstalledPlugin, McpMarketInstallation): @@ -30,13 +33,13 @@ def explicit_cloud(index_db, caps_root, monkeypatch): monkeypatch.setattr( catalog_resolver, "get_runtime_catalog", lambda *a, **k: {"skills": [], "mcp": []} ) - monkeypatch.setattr(bridge, "_local_server_ids", lambda: set()) - monkeypatch.setattr(bridge, "_mcp_json_local_declarations", lambda: {}) + monkeypatch.setattr(bridge_gateway, 'local_server_ids', lambda: set()) + monkeypatch.setattr(bridge_gateway, 'mcp_json_local_declarations', lambda: {}) context = { "profile": skill.profile_id, "servers": [{"server_id": "pack-search", "tools": [{"name": "search"}]}], } - monkeypatch.setattr(bridge, "_bridge_context", lambda: context) + monkeypatch.setattr(bridge_gateway, 'bridge_context', lambda: context) return index_db, plugin, skill, context @@ -100,10 +103,10 @@ def test_empty_mcp_schema_and_disabled_cloud_skill_cannot_be_revived(explicit_cl def test_cloud_skill_name_conflict_retains_selection_without_grant(explicit_cloud): factory, plugin, skill, context = explicit_cloud - from core.services.desktop_capability_protocol import skill_content_hash + from core.capabilities.manifest_protocol import skill_content_hash body = "different local bytes" - skills.publish_local_skill( + skill_publication.publish_local_skill( "pack-skill", files={"SKILL.md": body}, content_hash=skill_content_hash(body, {}), diff --git a/src/backend/tests/capabilities/test_gateway_invocation_proof.py b/src/backend/tests/features/desktop/capabilities/test_gateway_invocation_proof.py similarity index 78% rename from src/backend/tests/capabilities/test_gateway_invocation_proof.py rename to src/backend/tests/features/desktop/capabilities/test_gateway_invocation_proof.py index 6430a7ea..095538d1 100644 --- a/src/backend/tests/capabilities/test_gateway_invocation_proof.py +++ b/src/backend/tests/features/desktop/capabilities/test_gateway_invocation_proof.py @@ -1,22 +1,22 @@ """A desktop gateway reissues native MCP proofs with its authenticated identity.""" from types import SimpleNamespace import pytest -from core.llm import mcp_invocation, mcp_pool -from core.services import desktop_capability_mcp as gateway +from core.llm.mcp import invocation as mcp_invocation, client as mcp_pool +from features.desktop.capability import mcp as gateway @pytest.mark.asyncio @pytest.mark.parametrize("native", [True, False]) async def test_gateway_replaces_device_proof_only_for_trusted_native_target(monkeypatch, native): monkeypatch.setenv("BACKEND_INTERNAL_TOKEN", "synthetic-cloud-signing-key") - from core.config.mcp_config import _mcp_http_url + from core.config.mcp_config import mcp_http_url as _mcp_http_url url = _mcp_http_url("browser_runtime") if native else "https://third-party.example/mcp" captured = {} - async def invoke(**arguments): - return SimpleNamespace(metadata={}, model_dump=lambda **kwargs: {"content": []}) class Client: - execution_timeout = 1 - async def get_tool(self, name): - return invoke + async def __aenter__(self): return self + async def __aexit__(self, *exc): pass + async def call_tool(self, name, arguments): + from mcp.types import CallToolResult + return CallToolResult(content=[]) def make_client(sid, config, **kwargs): captured.update(config["headers"]) return Client() diff --git a/src/backend/tests/capabilities/test_gateway_unknown_outcome.py b/src/backend/tests/features/desktop/capabilities/test_gateway_unknown_outcome.py similarity index 61% rename from src/backend/tests/capabilities/test_gateway_unknown_outcome.py rename to src/backend/tests/features/desktop/capabilities/test_gateway_unknown_outcome.py index 890b23fc..377ed8d0 100644 --- a/src/backend/tests/capabilities/test_gateway_unknown_outcome.py +++ b/src/backend/tests/features/desktop/capabilities/test_gateway_unknown_outcome.py @@ -5,15 +5,14 @@ import pytest from agentscope.message import ToolCallBlock from agentscope.tool import Toolkit -from core.llm.middlewares import AgentRuntimeState, ToolEffectMiddleware -from core.llm.mcp_pool import make_client +from core.llm.runtime.state import AgentRuntimeState +from tests.orchestration.hook_support import durable_adapter +from core.llm.tools.cloud_manifest import manifest_tools from core.db.models import ChatRun, ToolEffectLedger -from core.services.run_journal import RunJournal -from core.services.tool_effect_ledger import ( - ToolOutcomeUnknown, - ToolEffectJournal, - recover_incomplete_tool_effects, -) +from core.runs.journal import RunJournal +from core.runs.tool_effects.errors import ToolOutcomeUnknown +from core.runs.tool_effects.journal import ToolEffectJournal +from core.runs.tool_effects.recovery import recover_incomplete_tool_effects from tests.llm.test_manifest_mcp_client import _config from tests.orchestration.test_tool_effect_ledger import effect_env @@ -37,10 +36,10 @@ async def upstream(request): return httpx.Response(200, content=b"lost-result") return httpx.Response(int(failure), json={"detail": "upstream result unavailable"}) - client = make_client("search", _config(httpx.MockTransport(upstream)), is_stateful=False) - toolkit = Toolkit(mcps=[client]) + client = manifest_tools("search", _config(httpx.MockTransport(upstream))) + toolkit = Toolkit(tools=client) state = AgentRuntimeState(run_id="cloud-write", journal_owner="worker") - middleware = ToolEffectMiddleware(session_factory=sessions) + middleware = durable_adapter(sessions) async def invoke(**kwargs): async for item in toolkit.call_tool(kwargs["tool_call"], state): @@ -81,9 +80,9 @@ async def test_rejected_write_does_not_claim_an_unknown_effect(monkeypatch, stat async def rejected(_request): return httpx.Response(status, json={"detail": "not authorized or stale schema"}) - client = make_client("search", _config(httpx.MockTransport(rejected)), is_stateful=False) + client = manifest_tools("search", _config(httpx.MockTransport(rejected))) with pytest.raises(RuntimeError): - await (await client.get_tool("search"))(query="write") + await next(t for t in client if t.name == "search")(query="write") @pytest.mark.asyncio @@ -95,6 +94,29 @@ async def timeout(request): config = _config(httpx.MockTransport(timeout)) config["manifest_tools"][0]["annotations"] = {"readOnlyHint": True} - client = make_client("search", config, is_stateful=False) + client = manifest_tools("search", config) with pytest.raises(RuntimeError, match="调用超时"): - await (await client.get_tool("search"))(query="read") + await next(t for t in client if t.name == "search")(query="read") + + +@pytest.mark.asyncio +async def test_stale_gateway_schema_refreshes_captured_account(monkeypatch): + from core.harness.registry import run_capabilities + from features.desktop.bridge import client as bridge_client, identity + from features.automation import tool_routing + + monkeypatch.setattr(run_capabilities, "has_run_capabilities", lambda: True) + monkeypatch.setattr(identity, "require_current_account", lambda state: None) + monkeypatch.setattr(identity, "get_state", lambda: {"cloud_base": "https://cloud.example"}) + monkeypatch.setattr(identity, "cloud_headers", lambda state: {}) + async def no_local_result(*args, **kwargs): + return None + monkeypatch.setattr(tool_routing, "local_automation_result", no_local_result) + refreshes = [] + monkeypatch.setattr(bridge_client, "notify_cloud_changed", lambda: refreshes.append(True)) + async def stale(request): + return httpx.Response(409, json={"detail": "schema changed"}) + client = manifest_tools("search", _config(httpx.MockTransport(stale))) + with pytest.raises(RuntimeError, match="正在同步最新能力"): + await next(t for t in client if t.name == "search")(query="write") + assert refreshes == [True] diff --git a/src/backend/tests/capabilities/test_hybrid_owned_plugins.py b/src/backend/tests/features/desktop/capabilities/test_hybrid_owned_plugins.py similarity index 50% rename from src/backend/tests/capabilities/test_hybrid_owned_plugins.py rename to src/backend/tests/features/desktop/capabilities/test_hybrid_owned_plugins.py index 05964bc0..d6939221 100644 --- a/src/backend/tests/capabilities/test_hybrid_owned_plugins.py +++ b/src/backend/tests/features/desktop/capabilities/test_hybrid_owned_plugins.py @@ -1,22 +1,26 @@ +from features.desktop import plugin_resolution as desktop_plugin_resolution """User-owned local packages remain visible in hybrid conversation assembly.""" +from features.desktop.capabilities import skill_account +from features.desktop.capabilities import skill_candidates +from features.desktop.capabilities import skill_publication import pytest -from core.capabilities import plugins, skills -from core.plugins import runtime as plugin_loader -from core.services.desktop_capability_protocol import skill_content_hash +from features.desktop.capabilities import plugins, skills +from core.capabilities.plugins import runtime as plugin_loader +from core.capabilities.manifest_protocol import skill_content_hash @pytest.mark.parametrize("owner,expected", [("owner", True), (None, False), ("other", False)]) def test_hybrid_keeps_owned_local_plugins_without_legacy_shared_bundles( index_db, caps_root, monkeypatch, owner, expected ): - from core.capabilities import device_catalog + from features.desktop.capabilities import device_catalog - monkeypatch.setattr(skills, "current_account_profile", lambda: "cloud-profile") - monkeypatch.setattr(skills, "account_authorized_for", lambda user: True) - monkeypatch.setattr(skills, "builtin_candidates", lambda: []) + monkeypatch.setattr(skill_account, 'current_account_profile', lambda: "cloud-profile") + monkeypatch.setattr(skill_account, 'account_authorized_for', lambda user: True) + monkeypatch.setattr(skill_candidates, 'builtin_candidates', lambda: []) monkeypatch.setattr(device_catalog, "active", lambda: True) body = "---\nname: native-browser\ndescription: Native browser\n---\nUse browser tools." - skills.publish_local_skill( + skill_publication.publish_local_skill( "native-browser", files={"SKILL.md": body}, content_hash=skill_content_hash(body, {}), owner_user_id=owner, ) @@ -24,7 +28,7 @@ def test_hybrid_keeps_owned_local_plugins_without_legacy_shared_bundles( {"slug": "native-browser", "components": {"skills": ["native-browser"]}}, owner_user_id=owner, ) - plan = plugin_loader.resolve_desktop_progressive_plugins( + plan = desktop_plugin_resolution.resolve_desktop_progressive_plugins( user_id="owner", enabled_skill_ids=["native-browser"], enabled_mcp_ids=[] ) assert bool(plan.directory) is expected diff --git a/src/backend/tests/capabilities/test_initial_sync_readiness.py b/src/backend/tests/features/desktop/capabilities/test_initial_sync_readiness.py similarity index 51% rename from src/backend/tests/capabilities/test_initial_sync_readiness.py rename to src/backend/tests/features/desktop/capabilities/test_initial_sync_readiness.py index 8ca00fc0..d91009c2 100644 --- a/src/backend/tests/capabilities/test_initial_sync_readiness.py +++ b/src/backend/tests/features/desktop/capabilities/test_initial_sync_readiness.py @@ -1,9 +1,13 @@ """First-login readiness must cover every capability kind and revision.""" +from features.desktop.bridge import client as bridge_client +from features.desktop.bridge import client_state as bridge_client_state +from features.desktop.bridge import identity as bridge_identity +from features.desktop.bridge import status as bridge_status from contextlib import nullcontext import pytest -from core.services import desktop_cloud_bridge as bridge -from core.services import desktop_cloud_skills, desktop_cloud_bundles -from core.capabilities.paths import revision_for_hash +from features.desktop.bridge import client as bridge +from features.desktop.bridge import skills as desktop_cloud_skills, bundles as desktop_cloud_bundles +from features.desktop.capabilities.paths import revision_for_hash @pytest.mark.parametrize("case", ["complete", "downloading", "missing_manifest", "old_revision", "error"]) @@ -19,21 +23,21 @@ def test_model_manifest_does_not_mark_partial_capabilities_ready(monkeypatch, ca other["revision"] = "" if case == "error": group["last_error"] = "download failed" - monkeypatch.setattr(bridge, "get_state", lambda: {"account": "current"}) - monkeypatch.setattr(bridge, "account_scope", lambda _: nullcontext()) - monkeypatch.setattr(bridge, "_manifest", {"servers": []}) - monkeypatch.setattr(bridge, "_refresh_manifest_async", lambda **_: None) - monkeypatch.setattr(bridge, "_manifest_error", None) - monkeypatch.setattr(bridge, "_manifest_fetching", case == "downloading") + monkeypatch.setattr(bridge_identity, 'get_state', lambda: {"account": "current"}) + monkeypatch.setattr(bridge_identity, 'account_scope', lambda _: nullcontext()) + monkeypatch.setattr(bridge_client_state, 'manifest', {"servers": []}) + monkeypatch.setattr(bridge_client, 'refresh_manifest_async', lambda **_: None) + monkeypatch.setattr(bridge_client_state, 'manifest_error', None) + monkeypatch.setattr(bridge_client_state, 'manifest_fetching', case == "downloading") monkeypatch.setattr(desktop_cloud_skills, "_profile", lambda _: "profile") monkeypatch.setattr(desktop_cloud_skills, "status", lambda: group) monkeypatch.setattr(desktop_cloud_bundles, "status", lambda: {"agent": other, "plugin": other}) - status = bridge.initial_sync_status() + status = bridge_status.initial_sync_status() assert status["ready"] is (case == "complete") assert status["pending"] == (1 if case == "old_revision" else 0) def test_logout_cannot_report_cached_manifest_as_ready(monkeypatch): - monkeypatch.setattr(bridge, "get_state", lambda: None) - monkeypatch.setattr(bridge, "_manifest", {"servers": []}) - assert bridge.initial_sync_status()["ready"] is False + monkeypatch.setattr(bridge_identity, 'get_state', lambda: None) + monkeypatch.setattr(bridge_client_state, 'manifest', {"servers": []}) + assert bridge_status.initial_sync_status()["ready"] is False diff --git a/src/backend/tests/capabilities/test_junction_windows_paths.py b/src/backend/tests/features/desktop/capabilities/test_junction_windows_paths.py similarity index 86% rename from src/backend/tests/capabilities/test_junction_windows_paths.py rename to src/backend/tests/features/desktop/capabilities/test_junction_windows_paths.py index 47f86b9c..582fc467 100644 --- a/src/backend/tests/capabilities/test_junction_windows_paths.py +++ b/src/backend/tests/features/desktop/capabilities/test_junction_windows_paths.py @@ -2,8 +2,8 @@ import os import pytest -from core.capabilities import junction -from core.capabilities.junction import _extended_windows_path, _strip_windows_prefix +from core.infra import directory_links as junction +from core.infra.directory_links import extended_windows_path as _extended_windows_path, strip_windows_prefix as _strip_windows_prefix def test_drive_path_adds_extended_prefix_and_roundtrips(): @@ -29,7 +29,7 @@ def test_existing_extended_prefix_is_not_duplicated(): def test_long_target_is_readable_without_long_paths_policy(tmp_path): root = tmp_path / "allowed" target = root / ("a" * 100) / ("b" * 100) / ("c" * 100) - native = junction._native(target) + native = junction.native_path(target) native.mkdir(parents=True) (native / "sentinel.txt").write_text("preserved") link = tmp_path / "view" @@ -50,7 +50,7 @@ def test_long_link_can_be_replaced_and_removed(tmp_path): link = tmp_path / ("x" * 100) / ("y" * 100) / ("z" * 100) / "view" junction.create_directory_link(link, first, allowed_roots=[root]) assert junction.ensure_directory_link(link, second, allowed_roots=[root]) - assert junction._native(link / "sentinel.txt").read_text() == "preserved" + assert junction.native_path(link / "sentinel.txt").read_text() == "preserved" junction.remove_directory_link(link) assert not junction.is_directory_link(link) assert (second / "sentinel.txt").read_text() == "preserved" @@ -63,5 +63,5 @@ def test_extended_prefix_cannot_bypass_allowed_root(tmp_path): outside.mkdir() with pytest.raises(junction.LinkTargetOutsideRoot): junction.create_directory_link( - tmp_path / "bad", junction._native(outside), allowed_roots=[root] + tmp_path / "bad", junction.native_path(outside), allowed_roots=[root] ) diff --git a/src/backend/tests/capabilities/test_loader_store_mode.py b/src/backend/tests/features/desktop/capabilities/test_loader_store_mode.py similarity index 74% rename from src/backend/tests/capabilities/test_loader_store_mode.py rename to src/backend/tests/features/desktop/capabilities/test_loader_store_mode.py index 781971b8..6db05bc0 100644 --- a/src/backend/tests/capabilities/test_loader_store_mode.py +++ b/src/backend/tests/features/desktop/capabilities/test_loader_store_mode.py @@ -1,18 +1,22 @@ """The real skill loader with the desktop store enabled: resolver-driven merge, no priority override.""" from __future__ import annotations +from features.desktop.bridge import identity as bridge_identity +from features.desktop.capabilities import skill_account +from features.desktop.capabilities import skill_paths +from features.desktop.capabilities import skills import base64 import json import time import pytest -from core.agent_skills import config as skill_config -from core.agent_skills.loader import MultiSourceSkillLoader -from core.capabilities import registry, skills, store -from core.capabilities.paths import KIND_SKILL -from core.capabilities.ref import cloud_ref, profile_id -from core.services import desktop_cloud_bridge as bridge +from core.capabilities.skills import config as skill_config +from core.capabilities.skills.loader import MultiSourceSkillLoader +from features.desktop.capabilities import registry, skills, store +from core.capabilities.identity_rules import KIND_SKILL +from core.capabilities.reference import cloud_ref, profile_id +from features.desktop.bridge import client as bridge def _md(sid: str, body: str) -> str: @@ -45,7 +49,7 @@ def env(tmp_path, monkeypatch, index_db, caps_root): (builtin / "ppt-design").mkdir(parents=True) (builtin / "ppt-design" / "SKILL.md").write_text(_md("ppt-design", "shipped")) monkeypatch.setattr(skill_config, "_builtin_skills_dir", lambda: builtin) - monkeypatch.setattr(skills, "builtin_dir", lambda: builtin) + monkeypatch.setattr(skill_paths, 'builtin_dir', lambda: builtin) # The loader's built-in source is derived from the package path; point it at the fixture. real_sources = skill_config.get_default_skill_sources @@ -61,20 +65,20 @@ def _sources(): monkeypatch.setattr(skill_config, "get_default_skill_sources", _sources) monkeypatch.setattr( - "core.agent_skills.loader.get_enabled_skill_sources", + 'core.capabilities.skills.loader.get_enabled_skill_sources', lambda: [s for s in _sources() if s.enabled], ) - bridge.reset_for_tests() - monkeypatch.setattr(bridge, "bridge_enabled", lambda: True) - monkeypatch.setattr(bridge, "get_state", lambda: dict(STATE, expires_at=time.time() + 60)) - monkeypatch.setattr(skills, "current_local_user_id", lambda: "u-1") + bridge_identity.reset_for_tests() + monkeypatch.setattr('features.desktop.auth.bridge.bridge_enabled', lambda: True) + monkeypatch.setattr(bridge_identity, 'get_state', lambda: dict(STATE, expires_at=time.time() + 60)) + monkeypatch.setattr(skill_account, 'current_local_user_id', lambda: "u-1") yield builtin - bridge.reset_for_tests() + bridge_identity.reset_for_tests() def _install_cloud(sid: str, body: str) -> str: md = _md(sid, body) - from core.services.desktop_capability_protocol import skill_content_hash + from core.capabilities.manifest_protocol import skill_content_hash h = skill_content_hash(md, {}) inst = registry.upsert( @@ -82,7 +86,7 @@ def _install_cloud(sid: str, body: str) -> str: ref=cloud_ref(STATE["cloud_base"], KIND_SKILL, sid, scope="shared"), content_hash=h, ) - from core.capabilities.paths import revision_for_hash + from features.desktop.capabilities.paths import revision_for_hash store.write_from_files(KIND_SKILL, PROFILE, sid, revision_for_hash(h), {"SKILL.md": md}) registry.set_state(inst.install_id, "ready", resolved_revision=revision_for_hash(h)) diff --git a/src/backend/tests/capabilities/test_local_folder_discovery.py b/src/backend/tests/features/desktop/capabilities/test_local_folder_discovery.py similarity index 90% rename from src/backend/tests/capabilities/test_local_folder_discovery.py rename to src/backend/tests/features/desktop/capabilities/test_local_folder_discovery.py index 865ed28c..78f77ff3 100644 --- a/src/backend/tests/capabilities/test_local_folder_discovery.py +++ b/src/backend/tests/features/desktop/capabilities/test_local_folder_discovery.py @@ -2,8 +2,8 @@ import os from pathlib import Path -from core.capabilities import discovery, registry -from core.services import local_skill_service +from features.desktop.capabilities import discovery, registry +from features.desktop.local.skills import service as local_skill_service def test_directory_requires_explicit_install_and_uninstall_never_resurrects( diff --git a/src/backend/tests/capabilities/test_local_management_migration.py b/src/backend/tests/features/desktop/capabilities/test_local_management_migration.py similarity index 83% rename from src/backend/tests/capabilities/test_local_management_migration.py rename to src/backend/tests/features/desktop/capabilities/test_local_management_migration.py index cabc33e0..48a02940 100644 --- a/src/backend/tests/capabilities/test_local_management_migration.py +++ b/src/backend/tests/features/desktop/capabilities/test_local_management_migration.py @@ -2,25 +2,25 @@ import json import pytest -from core.capabilities import registry +from features.desktop.capabilities import registry from core.db.engine import Base from core.db.models import AdminSkill, AdminMcpServer, InstalledPlugin, ContentBlock -from core.plugins.local import legacy_migration as migration -from core.services import local_management_migration as skill_migration +from features.desktop.local.plugins import legacy_migration as migration +from features.desktop.local import management_migration as skill_migration @pytest.fixture def legacy(index_db, caps_root, monkeypatch): Base.metadata.create_all(index_db.kw["bind"]) monkeypatch.setattr("core.db.engine.SessionLocal", index_db) - monkeypatch.setattr("core.services.mcp_service.SessionLocal", index_db) - monkeypatch.setattr("core.capabilities.device_catalog.active", lambda: False) - monkeypatch.setattr("core.capabilities.skills.account_authorized_for", lambda _: False) + monkeypatch.setattr('core.capabilities.mcp.service.SessionLocal', index_db) + monkeypatch.setattr('features.desktop.capabilities.device_catalog.active', lambda: False) + monkeypatch.setattr('features.desktop.capabilities.skill_account.account_authorized_for', lambda _: False) return index_db def test_old_local_manager_upgrades_and_remains_usable(legacy): - from core.plugins.packaging.sources import _make_skill_id, _make_server_id + from core.capabilities.plugins.packaging.sources import make_skill_id as _make_skill_id, make_server_id as _make_server_id from core.config.settings import settings from extensions.mcp_servers._ports import PORTS @@ -61,8 +61,8 @@ def test_old_local_manager_upgrades_and_remains_usable(legacy): db.commit() assert migration.migrate() == 1 assert migration.migrate() == 0 - from core.capabilities.local_plugin_runtime import configs - from core.capabilities import device_catalog + from features.desktop.capabilities.local_plugin_runtime import configs + from features.desktop.capabilities import device_catalog assert "install_skill" in [x["name"] for x in configs("owner")[mid]["manifest_tools"]] entry = device_catalog.plugin_entries(user_id="owner")[0] @@ -77,7 +77,7 @@ def test_old_local_manager_upgrades_and_remains_usable(legacy): restored = json.loads(decrypt_secret(backup.payload["encrypted"])) assert restored["skills"][0]["skill_content"].endswith("register_skill") - from core.plugins.local import service as local_plugin_service + from features.desktop.local.plugins import service as local_plugin_service current = local_plugin_service.get("owner", entry["install_id"]) local_plugin_service.uninstall("owner", current["install_id"], current["revision"]) @@ -98,7 +98,7 @@ def test_personal_skill_migrates_once_without_resurrection(legacy): ) db.commit() assert skill_migration.migrate() == 1 - from core.services import local_skill_service + from features.desktop.local.skills import service as local_skill_service iid = registry.install_id("skill", "local", "legacy-skill") current = local_skill_service.get("owner", iid) diff --git a/src/backend/tests/capabilities/test_local_skill_projection.py b/src/backend/tests/features/desktop/capabilities/test_local_skill_projection.py similarity index 83% rename from src/backend/tests/capabilities/test_local_skill_projection.py rename to src/backend/tests/features/desktop/capabilities/test_local_skill_projection.py index 93f19380..55dec364 100644 --- a/src/backend/tests/capabilities/test_local_skill_projection.py +++ b/src/backend/tests/features/desktop/capabilities/test_local_skill_projection.py @@ -1,15 +1,18 @@ """Actual upload/edit/disable/delete routes must publish the desktop installation.""" +from features.desktop.capabilities import skill_candidates +from features.desktop.capabilities import skills import io import zipfile import pytest from fastapi import FastAPI from fastapi.testclient import TestClient -from core.capabilities import registry, skills, store +from features.desktop.capabilities import registry, skills, store from core.auth.backend import UserContext, get_current_user from core.db.engine import get_db from core.db.models import AdminSkill -from api.routes.v1 import me_capabilities, desktop_capabilities, admin_skills, catalog +from api.routes.v1 import me_capabilities, admin_skills, catalog +from features.desktop.api import desktop_capabilities @pytest.fixture @@ -19,14 +22,14 @@ def local_client(index_db, caps_root, tmp_path, monkeypatch): monkeypatch.setattr("core.db.engine.SessionLocal", index_db) monkeypatch.setenv("HUGAGENT_DESKTOP_BRIDGE_SECRET", "test-config-only") monkeypatch.setenv("SANDBOX_SKILLS_DIR", str(tmp_path / "workspace" / "skills")) - monkeypatch.setattr(skills, "builtin_candidates", lambda: []) - monkeypatch.setattr("core.services.desktop_cloud_bridge.get_state", lambda: None) + monkeypatch.setattr(skill_candidates, 'builtin_candidates', lambda: []) + monkeypatch.setattr('features.desktop.bridge.identity.get_state', lambda: None) monkeypatch.setattr(me_capabilities, "_require_flag", lambda *args: None) monkeypatch.setattr( - "core.services.skill_management_service.ensure_ontology_build_valid", + 'core.capabilities.skills.management.ensure_ontology_build_valid', lambda *args, **kwargs: None, ) - monkeypatch.setattr("core.config.catalog_loader.load_catalog", lambda **kwargs: {}) + monkeypatch.setattr("core.capabilities.catalog.loader.load_catalog", lambda **kwargs: {}) app = FastAPI() for router in ( me_capabilities.router, @@ -46,7 +49,7 @@ def database(): ) app.dependency_overrides[admin_skills.require_admin] = lambda: None yield TestClient(app) - from core.agent_skills.loader import get_skill_loader + from core.capabilities.skills.loader_factory import get_skill_loader get_skill_loader(reset=True) diff --git a/src/backend/tests/capabilities/test_management_readiness.py b/src/backend/tests/features/desktop/capabilities/test_management_readiness.py similarity index 87% rename from src/backend/tests/capabilities/test_management_readiness.py rename to src/backend/tests/features/desktop/capabilities/test_management_readiness.py index b44a73a2..e82cf31d 100644 --- a/src/backend/tests/capabilities/test_management_readiness.py +++ b/src/backend/tests/features/desktop/capabilities/test_management_readiness.py @@ -1,13 +1,15 @@ """Downloaded capability bytes are distinct from platform/dependency readiness.""" +from features.desktop.bridge import client as bridge_client +from features.desktop.bridge import gateway as bridge_gateway import json from types import SimpleNamespace -from core.capabilities import registry, store -from core.capabilities.ref import cloud_ref -from core.services import desktop_cloud_bridge as bridge -from core.services.desktop_capability_protocol import entity_content_hash -from tests.capabilities.test_desktop_capabilities_api import cloud as cloud -from tests.capabilities.test_desktop_capabilities_api import client, PROFILE, _Cloud +from features.desktop.capabilities import registry, store +from core.capabilities.reference import cloud_ref +from features.desktop.bridge import client as bridge +from core.capabilities.manifest_protocol import entity_content_hash +from tests.features.desktop.capabilities.test_desktop_capabilities_api import cloud as cloud +from tests.features.desktop.capabilities.test_desktop_capabilities_api import client, PROFILE, _Cloud def _cloud_skill(client, monkeypatch, metadata, key="requires-runtime", sync=True): @@ -37,7 +39,7 @@ def test_incompatible_skill_is_downloaded_but_not_ready(client, monkeypatch): def test_missing_runtime_recovers_without_redownload(client, monkeypatch): - from core.capabilities import dependency + from features.desktop.capabilities import dependency iid, cloud = _cloud_skill( client, @@ -90,7 +92,7 @@ def _entity(kind, key, definition): def test_agent_list_reports_incompatible_platform(client, monkeypatch): - from core.services import user_agent_service + from core.capabilities.agents import edition as user_agent_service monkeypatch.setattr( user_agent_service, @@ -129,16 +131,16 @@ def test_plugin_readiness_follows_its_components(client, monkeypatch): def test_local_mcp_missing_command_is_not_ready(client, monkeypatch): - from core.services import mcp_service + from core.capabilities.mcp import service as mcp_service empty = SimpleNamespace( get_all_servers=lambda **kw: {}, get_owned_servers=lambda *args, **kw: {} ) monkeypatch.setattr(mcp_service.McpServerConfigService, "get_instance", lambda: empty) - monkeypatch.setattr(bridge, "get_cached_manifest", lambda: None) + monkeypatch.setattr(bridge_client, 'get_cached_manifest', lambda: None) monkeypatch.setattr( - bridge, - "_mcp_json_local_declarations", + bridge_gateway, + 'mcp_json_local_declarations', lambda: { "missing-cli": { "transport": "stdio", diff --git a/src/backend/tests/capabilities/test_management_tools.py b/src/backend/tests/features/desktop/capabilities/test_management_tools.py similarity index 56% rename from src/backend/tests/capabilities/test_management_tools.py rename to src/backend/tests/features/desktop/capabilities/test_management_tools.py index 6a4d48d1..114436ac 100644 --- a/src/backend/tests/capabilities/test_management_tools.py +++ b/src/backend/tests/features/desktop/capabilities/test_management_tools.py @@ -1,26 +1,29 @@ """Declared manager plugins execute locally and expose environment-specific contracts.""" +from features.desktop.bridge import gateway as bridge_gateway +from features.desktop.bridge import identity as bridge_identity import json from pathlib import Path import pytest -from core.services.management_contract import tools, localize -from core.llm.mcp_pool import make_client -from core.services.desktop_capability_protocol import canonical_hash +from core.capabilities.plugins.management.contract import tools, localize +from core.llm.tools.cloud_manifest import manifest_tools +from core.capabilities.manifest_protocol import canonical_hash def manifest_client(manager, specs, user="local-u1", remote=False): - return make_client("test-manager", {"transport": "streamable_http", "url": "http://localhost/inert", + return manifest_tools("test-manager", {"transport": "streamable_http", "url": "http://localhost/inert", "schema_source": "cloud_manifest" if remote else "plugin_manifest", "origin": "local_plugin", "manifest_tools": specs, "gateway_invoke_url": "https://cloud.example/api/v1/desktop/capability/gateway/test-manager/call" if remote else "", "schema_hash": canonical_hash(specs), "gateway_plugin": manager, - "headers": {"X-Current-User-Id": user}}, is_stateful=False) + "headers": {"X-Current-User-Id": user}}) @pytest.mark.asyncio async def test_declared_local_install_updates_page_projection(client, tmp_path, monkeypatch): - from core.capabilities import local_plugin_runtime, device_catalog - monkeypatch.setattr(local_plugin_runtime, "authorizes_manager", lambda user, sid, manager: user == "local-u1") + from features.desktop.capabilities import local_plugin_runtime, device_catalog + from core.harness.registry import execution_catalog + monkeypatch.setattr(execution_catalog, "local_plugin_configs", lambda user, enabled_only=True: {"test-manager": {"gateway_plugin": "skill-manager"}} if user == "local-u1" else {}) mcp = manifest_client("skill-manager", tools("skill-manager", "cloud")) - tool = await mcp.get_tool("install_skill") + tool = next(t for t in mcp if t.name == "install_skill") assert "artifact_id" not in json.dumps(tool.input_schema) root = tmp_path / "from-tool" root.mkdir() @@ -31,21 +34,21 @@ async def test_declared_local_install_updates_page_projection(client, tmp_path, assert body["install_id"] in {x["install_id"] for x in device_catalog.catalog_overlay(user_id="local-u1")["skills"]} rejected = await tool(source={"kind": "artifact", "artifact_id": "not-local"}) assert json.loads(rejected.content[0].text)["ok"] is False - assert "upload_skill_to_cloud" not in [t.name for t in await mcp.list_tools()] + assert "upload_skill_to_cloud" not in [t.name for t in mcp] @pytest.mark.asyncio async def test_offline_plugin_package_supplies_manager_without_remote_discovery(client, monkeypatch): - monkeypatch.setattr("core.capabilities.skills.account_authorized_for", lambda _: False) - from core.plugins.local import service as local_plugin_service - from core.capabilities.local_plugin_runtime import configs - root = Path(__file__).resolve().parents[2] / "extensions/plugins/marketplace/skill-manager" + monkeypatch.setattr('features.desktop.capabilities.skill_account.account_authorized_for', lambda _: False) + from features.desktop.local.plugins import service as local_plugin_service + from features.desktop.capabilities.local_plugin_runtime import configs + root = Path(__file__).resolve().parents[4] / "extensions/plugins/marketplace/skill-manager" installed = local_plugin_service.install("local-u1", str(root)) servers = configs("local-u1") sid = installed["components"]["mcp"][0] assert sid in servers - mcp = make_client(sid, servers[sid], is_stateful=False) - names = [t.name for t in await mcp.list_tools()] + mcp = manifest_tools(sid, servers[sid]) + names = [t.name for t in mcp] assert "install_skill" in names assert "upload_skill_to_cloud" not in names assert not configs("other") @@ -62,32 +65,32 @@ def test_unrelated_or_future_contract_never_uses_local_executor(): @pytest.mark.asyncio async def test_installed_local_manager_exposes_upload_only_to_bound_cloud_user(client, monkeypatch): - from core.plugins.local import service as local_plugin_service - from core.capabilities.local_plugin_runtime import configs - monkeypatch.setattr("core.capabilities.skills.account_authorized_for", lambda uid: uid == "local-u1") - root = Path(__file__).resolve().parents[2] / "extensions/plugins/marketplace/skill-manager" + from features.desktop.local.plugins import service as local_plugin_service + from features.desktop.capabilities.local_plugin_runtime import configs + monkeypatch.setattr('features.desktop.capabilities.skill_account.account_authorized_for', lambda uid: uid == "local-u1") + root = Path(__file__).resolve().parents[4] / "extensions/plugins/marketplace/skill-manager" installed = local_plugin_service.install("local-u1", str(root)) sid = installed["components"]["mcp"][0] - mcp = make_client(sid, configs("local-u1")[sid], is_stateful=False) - assert "upload_skill_to_cloud" in [t.name for t in await mcp.list_tools()] + mcp = manifest_tools(sid, configs("local-u1")[sid]) + assert "upload_skill_to_cloud" in [t.name for t in mcp] @pytest.mark.asyncio async def test_cloud_delivered_manager_contract_executes_locally(client, tmp_path, monkeypatch): - from core.services import desktop_cloud_bridge as bridge + from features.desktop.bridge import client as bridge specs = tools("skill-manager", "cloud") - monkeypatch.setattr("core.auth.desktop_bridge.bridge_enabled", lambda: True) - monkeypatch.setattr(bridge, "require_current_account", lambda _: None) - monkeypatch.setattr(bridge, "_bridge_context", lambda: {"servers": [{"server_id": "test-manager", "source_plugin": "skill-manager", "tools": specs}]}) - monkeypatch.setattr("core.capabilities.skills.current_local_user_id", lambda: "local-u1") + monkeypatch.setattr('features.desktop.auth.bridge.bridge_enabled', lambda: True) + monkeypatch.setattr(bridge_identity, 'require_current_account', lambda _: None) + monkeypatch.setattr(bridge_gateway, 'bridge_context', lambda: {"servers": [{"server_id": "test-manager", "source_plugin": "skill-manager", "tools": specs}]}) + monkeypatch.setattr('features.desktop.capabilities.skill_account.current_local_user_id', lambda: "local-u1") mcp = manifest_client("skill-manager", specs, remote=True) - tool = await mcp.get_tool("install_skill") + tool = next(t for t in mcp if t.name == "install_skill") root = tmp_path / "hybrid-skill" root.mkdir() (root / "SKILL.md").write_text("---\nname: hybrid-skill\ndescription: Hybrid\n---\nBody") result = json.loads((await tool(source={"kind": "local_path", "path": str(root)})).content[0].text) assert result["ok"] and result["source"] == "local" - assert "upload_skill_to_cloud" in [t.name for t in await mcp.list_tools()] - monkeypatch.setattr(bridge, "_bridge_context", lambda: {"servers": []}) + assert "upload_skill_to_cloud" in [t.name for t in mcp] + monkeypatch.setattr(bridge_gateway, 'bridge_context', lambda: {"servers": []}) revoked = json.loads((await tool(source={"kind": "local_path", "path": str(root)})).content[0].text) assert not revoked["ok"] diff --git a/src/backend/tests/capabilities/test_manager_lifecycle.py b/src/backend/tests/features/desktop/capabilities/test_manager_lifecycle.py similarity index 91% rename from src/backend/tests/capabilities/test_manager_lifecycle.py rename to src/backend/tests/features/desktop/capabilities/test_manager_lifecycle.py index 8411ab59..cae6ba66 100644 --- a/src/backend/tests/capabilities/test_manager_lifecycle.py +++ b/src/backend/tests/features/desktop/capabilities/test_manager_lifecycle.py @@ -1,7 +1,7 @@ """Manager commands and the desktop installation API share one source of truth.""" from pathlib import Path import pytest -from core.services import local_skill_service as service +from features.desktop.local.skills import service as service def test_install_update_uninstall_visible_to_device_api(client, tmp_path): @@ -11,7 +11,7 @@ def test_install_update_uninstall_visible_to_device_api(client, tmp_path): installed = service.install("local-u1", str(folder)) iid = installed["install_id"] assert installed["source"] == "local" - from core.capabilities import device_catalog + from features.desktop.capabilities import device_catalog assert any(x["id"] == installed["skill_id"] for x in device_catalog.catalog_overlay(user_id="local-u1")["skills"]) assert not any(x["id"] == installed["skill_id"] for x in device_catalog.catalog_overlay(user_id="other")["skills"]) listing = client.get("/v1/desktop/capabilities/installations?kind=skill").json()["data"]["items"] @@ -31,7 +31,7 @@ def test_install_update_uninstall_visible_to_device_api(client, tmp_path): def test_plugin_install_has_components_and_appears_in_installed_page(client, tmp_path): import json - from core.plugins.local import service as plugins + from features.desktop.local.plugins import service as plugins folder = tmp_path / "graph-plugin" child = folder / "skills" / "graph-builder" child.mkdir(parents=True) @@ -41,7 +41,7 @@ def test_plugin_install_has_components_and_appears_in_installed_page(client, tmp assert installed["source"] == "local" items = client.get("/v1/desktop/capabilities/installations?kind=plugin").json()["data"]["items"] assert any(x["install_id"] == installed["install_id"] for x in items) - from core.capabilities import device_catalog + from features.desktop.capabilities import device_catalog assert any(x["install_id"] == installed["install_id"] for x in device_catalog.plugin_entries(user_id="local-u1")) assert not device_catalog.plugin_entries(user_id="other") detail = plugins.get("local-u1", installed["install_id"]) diff --git a/src/backend/tests/capabilities/test_manager_regressions.py b/src/backend/tests/features/desktop/capabilities/test_manager_regressions.py similarity index 85% rename from src/backend/tests/capabilities/test_manager_regressions.py rename to src/backend/tests/features/desktop/capabilities/test_manager_regressions.py index 1a59603b..5d7909b1 100644 --- a/src/backend/tests/capabilities/test_manager_regressions.py +++ b/src/backend/tests/features/desktop/capabilities/test_manager_regressions.py @@ -1,4 +1,6 @@ """Lifecycle regressions found by independent implementation review.""" +from features.desktop.bridge import identity as bridge_identity +from features.desktop.capabilities import skill_account import io import json import zipfile @@ -6,9 +8,9 @@ from contextlib import nullcontext from types import SimpleNamespace import pytest -from core.capabilities import registry, store -from core.services import local_skill_service as skills -from core.plugins.local import service as plugins +from features.desktop.capabilities import registry, store +from features.desktop.local.skills import service as skills +from features.desktop.local.plugins import service as plugins def skill(root, body="First"): @@ -34,20 +36,21 @@ def test_plugin_dependencies_and_sibling_paths_survive_import(client, tmp_path): assert "${SKILL_DIR}" not in content with pytest.raises(ValueError, match="plugin component"): skills.uninstall("local-u1", first.install_id, first.resolved_revision) - from core.capabilities.invocation import cloud_plugin_selection + from features.desktop.capabilities.invocation import cloud_plugin_selection selected = cloud_plugin_selection(installed["install_id"], user_id="local-u1") assert selected["skills"] == children assert selected["mcp"] == [sid] def test_upload_repeat_and_lost_response_have_stable_identity(client, tmp_path, monkeypatch): - from core.services import local_skill_upload, desktop_cloud_bridge as bridge - from core.capabilities import skills as capability_skills, change_sync + from features.desktop.local.skills import upload as local_skill_upload + from features.desktop.bridge import client as bridge + from features.desktop.capabilities import skills as capability_skills, change_sync installed = skills.install("local-u1", skill(tmp_path / "skill")) - monkeypatch.setattr(bridge, "require_current_account", lambda _: None) - monkeypatch.setattr(bridge, "account_scope", lambda _: nullcontext()) - monkeypatch.setattr(capability_skills, "account_authorized_for", lambda uid: uid == "local-u1") - monkeypatch.setattr(capability_skills, "current_account_profile", lambda: "test-cloud") + monkeypatch.setattr(bridge_identity, 'require_current_account', lambda _: None) + monkeypatch.setattr(bridge_identity, 'account_scope', lambda _: nullcontext()) + monkeypatch.setattr(skill_account, 'account_authorized_for', lambda uid: uid == "local-u1") + monkeypatch.setattr(skill_account, 'current_account_profile', lambda: "test-cloud") remote = {"exists": False, "revision": "empty", "model_version": None} receipts, posts = {}, [] def request(state, method, kind, key, body=None): @@ -73,7 +76,7 @@ def request(state, method, kind, key, body=None): def test_ui_editor_and_tool_share_registration(client, tmp_path): - from core.services import local_skill_editor as editor + from features.desktop.local.skills import editor as editor installed = skills.install("local-u1", skill(tmp_path / "skill")) key = installed["skill_id"] draft = editor.detail("local-u1", key) @@ -111,7 +114,7 @@ def test_rejected_archive_does_not_publish_and_plugin_update_removes_old_childre def test_stale_form_cannot_overwrite_or_recreate_removed_skill(client, tmp_path): from fastapi import HTTPException - from core.services import local_skill_editor as editor + from features.desktop.local.skills import editor as editor installed = skills.install("local-u1", skill(tmp_path / "skill")) body = SimpleNamespace(name=installed["skill_id"], display_name="Stale", description="Old", expected_revision=installed["revision"], instructions="Old", tags=[], mcp_server_ids=[], user_intro="", icon=None) @@ -127,7 +130,7 @@ def test_stale_form_cannot_overwrite_or_recreate_removed_skill(client, tmp_path) def test_plugin_card_reports_missing_runtime(client, tmp_path): - from core.capabilities.device_plugin_catalog import plugin_entries + from features.desktop.capabilities.device_plugin_catalog import plugin_entries root = tmp_path / "missing-runtime" root.mkdir() (root / "plugin.json").write_text(json.dumps({"name": "missing-runtime", "mcpServers": { diff --git a/src/backend/tests/capabilities/test_manifest_order.py b/src/backend/tests/features/desktop/capabilities/test_manifest_order.py similarity index 85% rename from src/backend/tests/capabilities/test_manifest_order.py rename to src/backend/tests/features/desktop/capabilities/test_manifest_order.py index 492d7ca9..df1dfbbd 100644 --- a/src/backend/tests/capabilities/test_manifest_order.py +++ b/src/backend/tests/features/desktop/capabilities/test_manifest_order.py @@ -1,16 +1,18 @@ """Controlled concurrent responses cannot resurrect revoked cloud intentions.""" +from features.desktop.bridge import identity as bridge_identity +from features.desktop.capabilities import skill_views import base64 import json import threading import httpx import pytest -from core.capabilities import registry, skills -from core.capabilities.ref import profile_id, cloud_ref -from core.services import desktop_cloud_bridge as bridge -from core.services import desktop_cloud_skills as cloud_skills -from core.services import desktop_cloud_bundles as bundles -from core.services.desktop_capability_protocol import build_skill_manifest, build_entity_manifest +from features.desktop.capabilities import registry, skills +from core.capabilities.reference import profile_id, cloud_ref +from features.desktop.bridge import client as bridge +from features.desktop.bridge import skills as cloud_skills +from features.desktop.bridge import bundles as bundles +from core.capabilities.manifest_protocol import build_skill_manifest, build_entity_manifest def state(subject): @@ -63,11 +65,11 @@ def test_late_background_manifest_cannot_replace_new_checkpoint( ): st = state(kind + str(replacement)) profile = profile_id(st["cloud_base"], kind + str(replacement)) - monkeypatch.setattr(bridge, "get_state", lambda: st) + monkeypatch.setattr(bridge_identity, 'get_state', lambda: st) cloud_skills.on_account_switch() bundles.on_account_switch() - monkeypatch.setattr(skills, "rebuild_views", lambda *_: {}) - monkeypatch.setattr("core.agent_skills.cache_refresh.refresh_skill_caches", lambda: None) + monkeypatch.setattr(skill_views, 'rebuild_views', lambda *_: {}) + monkeypatch.setattr('core.capabilities.skills.cache_refresh.refresh_skill_caches', lambda: None) monkeypatch.setattr(bundles, "_prepare", lambda *_: None) original, updated = manifest(kind, "a" * 64), manifest(kind, replacement) inst = registry.upsert( @@ -103,7 +105,7 @@ def background(): assert entered.wait(5) if kind == "skill": current = cloud_skills._fetch_manifest(st) - with bridge.account_scope(st): + with bridge_identity.account_scope(st): cloud_skills._reconcile_intent(current, st) else: current = bundles._fetch(kind, st) @@ -126,11 +128,11 @@ def test_new_failed_request_still_prevents_older_response_resurrection( ): st = state("failed-" + kind) profile = profile_id(st["cloud_base"], "failed-" + kind) - monkeypatch.setattr(bridge, "get_state", lambda: st) + monkeypatch.setattr(bridge_identity, 'get_state', lambda: st) cloud_skills.on_account_switch() bundles.on_account_switch() - monkeypatch.setattr(skills, "rebuild_views", lambda *_: {}) - monkeypatch.setattr("core.agent_skills.cache_refresh.refresh_skill_caches", lambda: None) + monkeypatch.setattr(skill_views, 'rebuild_views', lambda *_: {}) + monkeypatch.setattr('core.capabilities.skills.cache_refresh.refresh_skill_caches', lambda: None) monkeypatch.setattr(bundles, "_prepare", lambda *_: None) inst = registry.upsert( profile_id=profile, @@ -172,7 +174,7 @@ def test_304_gets_new_private_ticket_without_changing_manifest_hash( index_db, caps_root, monkeypatch, kind ): st = state("cached-" + kind) - monkeypatch.setattr(bridge, "get_state", lambda: st) + monkeypatch.setattr(bridge_identity, 'get_state', lambda: st) cloud_skills.on_account_switch() bundles.on_account_switch() payload = manifest(kind, "a" * 64) @@ -200,7 +202,7 @@ def get(url, **kwargs): cloud_skills._reconcile_intent(second, st) else: bundles._reconcile(kind, second, st, prepare=False) - from core.capabilities.manifest_order import StaleManifest + from features.desktop.capabilities.manifest_order import StaleManifest with pytest.raises(StaleManifest): if kind == "skill": diff --git a/src/backend/tests/capabilities/test_manifest_tool_schema.py b/src/backend/tests/features/desktop/capabilities/test_manifest_tool_schema.py similarity index 76% rename from src/backend/tests/capabilities/test_manifest_tool_schema.py rename to src/backend/tests/features/desktop/capabilities/test_manifest_tool_schema.py index 23555c1d..2e6432bf 100644 --- a/src/backend/tests/capabilities/test_manifest_tool_schema.py +++ b/src/backend/tests/features/desktop/capabilities/test_manifest_tool_schema.py @@ -13,9 +13,9 @@ from agentscope.mcp import HttpMCPConfig -from core.llm.mcp_manager import ManifestMCPClient, has_usable_schema -from core.llm.mcp_pool import uses_manifest_schema -from core.plugins.management.components import _merge_tool_metadata +from core.capabilities.mcp.tool_schema import has_usable_schema +from core.llm.tools.cloud_manifest import manifest_tools +from core.capabilities.plugins.management.components import merge_tool_metadata as _merge_tool_metadata REAL_SCHEMA = { "type": "object", @@ -46,26 +46,19 @@ def test_argument_less_tool_is_still_a_captured_schema(): """A server may describe an argument-less tool tersely; that is not a gap.""" assert has_usable_schema({"name": "list_sites", "inputSchema": ARGLESS_SCHEMA}) assert has_usable_schema({"name": "list_sites", "inputSchema": {"type": "object"}}) - assert uses_manifest_schema(_cfg([{"name": "list_sites", "inputSchema": {"type": "object"}}])) + assert manifest_tools("s", _cfg([{"name": "list_sites", "inputSchema": {"type": "object"}}])) def test_manifest_without_any_schema_is_rejected(): - assert not uses_manifest_schema(_cfg([{"name": "publish_site", "description": "..."}])) + assert manifest_tools("s", _cfg([{"name": "publish_site", "description": "..."}])) == [] def test_client_withholds_the_tool_it_cannot_describe(): - client = ManifestMCPClient( - name="sites-site_publish", - is_stateful=False, - mcp_config=HttpMCPConfig(url="https://cloud.example/call"), - manifest_tools=[ - {"name": "publish_site", "description": "发布站点"}, - {"name": "list_sites", "description": "查询站点", "inputSchema": ARGLESS_SCHEMA}, - ], - gateway_invoke_url="https://cloud.example/call", - schema_hash="hash", - ) - names = [tool.name for tool in client._raw_manifest_tools()] + tools = manifest_tools("sites-site_publish", _cfg([ + {"name": "publish_site", "description": "发布站点"}, + {"name": "list_sites", "description": "查询站点", "inputSchema": ARGLESS_SCHEMA}, + ])) + names = [tool.name for tool in tools] assert names == ["list_sites"], "参数丢失的工具不能交给模型" diff --git a/src/backend/tests/capabilities/test_mcp_json.py b/src/backend/tests/features/desktop/capabilities/test_mcp_json.py similarity index 98% rename from src/backend/tests/capabilities/test_mcp_json.py rename to src/backend/tests/features/desktop/capabilities/test_mcp_json.py index 34e62c46..f91f85f4 100644 --- a/src/backend/tests/capabilities/test_mcp_json.py +++ b/src/backend/tests/features/desktop/capabilities/test_mcp_json.py @@ -5,7 +5,7 @@ import json import pytest -from core.capabilities import credentials, mcp_json +from features.desktop.capabilities import credentials, mcp_json @pytest.fixture(autouse=True) diff --git a/src/backend/tests/capabilities/test_mcp_json_consistency.py b/src/backend/tests/features/desktop/capabilities/test_mcp_json_consistency.py similarity index 99% rename from src/backend/tests/capabilities/test_mcp_json_consistency.py rename to src/backend/tests/features/desktop/capabilities/test_mcp_json_consistency.py index 23e8fef4..42247dee 100644 --- a/src/backend/tests/capabilities/test_mcp_json_consistency.py +++ b/src/backend/tests/features/desktop/capabilities/test_mcp_json_consistency.py @@ -6,7 +6,7 @@ import json import pytest -from core.capabilities import credentials, mcp_json +from features.desktop.capabilities import credentials, mcp_json @pytest.fixture(autouse=True) diff --git a/src/backend/tests/capabilities/test_memory_epoch_cache.py b/src/backend/tests/features/desktop/capabilities/test_memory_epoch_cache.py similarity index 79% rename from src/backend/tests/capabilities/test_memory_epoch_cache.py rename to src/backend/tests/features/desktop/capabilities/test_memory_epoch_cache.py index 2c0f28d2..aabddcc2 100644 --- a/src/backend/tests/capabilities/test_memory_epoch_cache.py +++ b/src/backend/tests/features/desktop/capabilities/test_memory_epoch_cache.py @@ -1,4 +1,5 @@ """The memory singleton must not outlive its captured desktop login identity.""" +from features.desktop.bridge import identity as bridge_identity import sys from types import SimpleNamespace @@ -6,11 +7,11 @@ import httpx import pytest -from core.capabilities.errors import CloudUnavailable -from core.capabilities.ref import profile_id +from core.harness.capability_errors import CloudUnavailable +from core.capabilities.reference import profile_id from core.memory import backend as service -from core.services import desktop_cloud_bridge as bridge -from tests.capabilities.test_dynamic_model_epoch_cache import state +from features.desktop.bridge import client as bridge +from tests.features.desktop.capabilities.test_dynamic_model_epoch_cache import state @pytest.fixture @@ -18,22 +19,22 @@ def memories(monkeypatch, tmp_path): monkeypatch.setenv("HUGAGENT_CAPS_ROOT", str(tmp_path / "caps")) current = {"state": state(), "instances": [], "on_build": None} monkeypatch.setenv("HUGAGENT_DESKTOP_BRIDGE_SECRET", "synthetic") - monkeypatch.setattr(bridge, "get_state", lambda: current["state"]) + monkeypatch.setattr(bridge_identity, 'get_state', lambda: current["state"]) monkeypatch.setattr( service, "settings", SimpleNamespace(memory=SimpleNamespace(enabled=True, graph_enabled=False)), ) - monkeypatch.setattr(service, "_memory_instance", None) - monkeypatch.setattr(service, "_memory_epoch", None, raising=False) - monkeypatch.setattr(service, "_memory_init_failed", False) + monkeypatch.setattr(service.process_state(), 'memory_instance', None) + monkeypatch.setattr(service.process_state(), 'memory_epoch', None, raising=False) + monkeypatch.setattr(service.process_state(), 'memory_init_failed', False) monkeypatch.setattr(service, "_patch_mem0_embedding", lambda: None) monkeypatch.setattr(service, "_apply_probed_embed_dims", lambda cfg: None) monkeypatch.setattr(service, "_reconcile_vector_collection", lambda cfg: None) def config(): value = current["state"] or state() - from core.services.desktop_capability_protocol import token_subject + from core.capabilities.manifest_protocol import token_subject entry = { "api_key": "desktop-capability:" @@ -106,25 +107,25 @@ def test_logout_never_reuses_old_memory(memories): def test_identity_change_during_initialization_is_not_published(memories): memories["on_build"] = lambda: memories.update(state=state(epoch=2, session="new")) assert service._get_memory() is None - assert service._memory_instance is None + assert service.process_state().memory_instance is None def test_non_desktop_singleton_does_not_consult_bridge(memories, monkeypatch): monkeypatch.delenv("HUGAGENT_DESKTOP_BRIDGE_SECRET") cached = object() - monkeypatch.setattr(service, "_memory_instance", cached) - monkeypatch.setattr(service, "_memory_epoch", "") + monkeypatch.setattr(service.process_state(), 'memory_instance', cached) + monkeypatch.setattr(service.process_state(), 'memory_epoch', "") def forbidden(): raise AssertionError("non-desktop memory must not consult bridge") - monkeypatch.setattr(bridge, "get_state", forbidden) + monkeypatch.setattr(bridge_identity, 'get_state', forbidden) assert service._get_memory() is cached def test_explicit_runtime_reset_invalidates_instance_and_epoch(memories): original = service._get_memory() service.reset_runtime() - assert service._memory_instance is None - assert service._memory_epoch is None + assert service.process_state().memory_instance is None + assert service.process_state().memory_epoch is None assert service._get_memory() is not original diff --git a/src/backend/tests/capabilities/test_migration.py b/src/backend/tests/features/desktop/capabilities/test_migration.py similarity index 89% rename from src/backend/tests/capabilities/test_migration.py rename to src/backend/tests/features/desktop/capabilities/test_migration.py index 72520da4..61082fdf 100644 --- a/src/backend/tests/capabilities/test_migration.py +++ b/src/backend/tests/features/desktop/capabilities/test_migration.py @@ -4,9 +4,12 @@ import os -from core.agent_skills import config as skill_config -from core.capabilities import junction, migration, registry, store -from core.capabilities.paths import KIND_SKILL +from core.capabilities.skills import config as skill_config +from core.infra import directory_links as junction +from features.desktop.capabilities import migration +from features.desktop.capabilities import registry +from features.desktop.capabilities import store +from core.capabilities.identity_rules import KIND_SKILL def _md(sid: str, body: str = "x") -> str: diff --git a/src/backend/tests/capabilities/test_migration_preservation.py b/src/backend/tests/features/desktop/capabilities/test_migration_preservation.py similarity index 96% rename from src/backend/tests/capabilities/test_migration_preservation.py rename to src/backend/tests/features/desktop/capabilities/test_migration_preservation.py index 0cefdd2a..ecc0dbfd 100644 --- a/src/backend/tests/capabilities/test_migration_preservation.py +++ b/src/backend/tests/features/desktop/capabilities/test_migration_preservation.py @@ -1,16 +1,17 @@ """Only exact, rebuildable session copies may be removed during migration.""" from __future__ import annotations +from features.desktop.capabilities import skill_paths import shutil import pytest -from core.capabilities import migration, skills +from features.desktop.capabilities import migration, skills @pytest.fixture() def layout(tmp_path, monkeypatch, caps_root): - monkeypatch.setattr(skills, "builtin_dir", lambda: tmp_path / "builtin") + monkeypatch.setattr(skill_paths, 'builtin_dir', lambda: tmp_path / "builtin") ws = tmp_path / "workspace" source = ws / "skills" / "demo" source.mkdir(parents=True) diff --git a/src/backend/tests/capabilities/test_model_credential_reference.py b/src/backend/tests/features/desktop/capabilities/test_model_credential_reference.py similarity index 86% rename from src/backend/tests/capabilities/test_model_credential_reference.py rename to src/backend/tests/features/desktop/capabilities/test_model_credential_reference.py index bdbdecc6..527b75bb 100644 --- a/src/backend/tests/capabilities/test_model_credential_reference.py +++ b/src/backend/tests/features/desktop/capabilities/test_model_credential_reference.py @@ -1,13 +1,14 @@ """Cloud model references are non-secret, account-bound, and refreshed per request.""" +from features.desktop.bridge import identity as bridge_identity import base64 import json import pytest import httpx -from core.services import desktop_model_credentials as credentials -from core.services import desktop_cloud_bridge as bridge -from core.capabilities.ref import profile_id -from core.capabilities.errors import CloudUnavailable +from features.desktop.bridge import model_credentials as credentials +from features.desktop.bridge import client as bridge +from core.capabilities.reference import profile_id +from core.harness.capability_errors import CloudUnavailable def token(subject, device="device-a", nonce="one"): @@ -25,7 +26,7 @@ def state(monkeypatch, caps_root): "device_id": "device-a", } monkeypatch.setenv("HUGAGENT_DESKTOP_BRIDGE_SECRET", "test-only") - monkeypatch.setattr(bridge, "get_state", lambda: dict(value)) + monkeypatch.setattr(bridge_identity, 'get_state', lambda: dict(value)) return value @@ -73,7 +74,7 @@ async def test_transport_never_sends_bearer_to_other_destination(state): def test_reference_fails_closed_after_logout(state, monkeypatch): p = credentials.sanitize_import({"providers": [provider(state)]})["providers"][0] - monkeypatch.setattr(bridge, "get_state", lambda: None) + monkeypatch.setattr(bridge_identity, 'get_state', lambda: None) with pytest.raises(CloudUnavailable): credentials.headers(p["api_key"], p["base_url"]) diff --git a/src/backend/tests/capabilities/test_model_error_diagnostics.py b/src/backend/tests/features/desktop/capabilities/test_model_error_diagnostics.py similarity index 83% rename from src/backend/tests/capabilities/test_model_error_diagnostics.py rename to src/backend/tests/features/desktop/capabilities/test_model_error_diagnostics.py index 820c3ae9..c0e18784 100644 --- a/src/backend/tests/capabilities/test_model_error_diagnostics.py +++ b/src/backend/tests/features/desktop/capabilities/test_model_error_diagnostics.py @@ -1,7 +1,7 @@ """Wrapped SDK diagnostics must never serialize exception payloads.""" -from core.capabilities.errors import PermissionDenied -from core.llm.providers.openai_compat import _safe_exception_chain +from core.harness.capability_errors import PermissionDenied +from core.models.providers.openai_compat import _safe_exception_chain def test_diagnostic_keeps_wrapped_authorization_code_without_private_payload(): @@ -13,7 +13,7 @@ def test_diagnostic_keeps_wrapped_authorization_code_without_private_payload(): assert chain == [ {"type": "builtins.RuntimeError"}, { - "type": "core.capabilities.errors.PermissionDenied", + "type": "core.harness.capability_errors.PermissionDenied", "code": "permission_denied", }, ] diff --git a/src/backend/tests/capabilities/test_owned_connector_assembly.py b/src/backend/tests/features/desktop/capabilities/test_owned_connector_assembly.py similarity index 55% rename from src/backend/tests/capabilities/test_owned_connector_assembly.py rename to src/backend/tests/features/desktop/capabilities/test_owned_connector_assembly.py index b939a99c..184b2d11 100644 --- a/src/backend/tests/capabilities/test_owned_connector_assembly.py +++ b/src/backend/tests/features/desktop/capabilities/test_owned_connector_assembly.py @@ -1,20 +1,22 @@ """Retained modern packages reach ambient, explicit and progressive assembly.""" +from features.desktop.bridge import gateway as bridge_gateway +from features.desktop.capabilities import skill_account from pathlib import Path import pytest -from core.capabilities import connectors, local_plugin_runtime, registry, skills, store -from core.config import catalog_resolver -from core.plugins.local import service -from core.services import desktop_cloud_bridge as bridge +from features.desktop.capabilities import connectors, local_plugin_runtime, registry, skills, store +from core.capabilities.catalog import resolver as catalog_resolver +from features.desktop.local.plugins import service as service +from features.desktop.bridge import client as bridge @pytest.fixture def installed(client, index_db, monkeypatch): - root = Path(__file__).resolve().parents[2] / "extensions/plugins/marketplace/browser-automation" + root = Path(__file__).resolve().parents[4] / "extensions/plugins/marketplace/browser-automation" result = service.install("local-u1", str(root)) - monkeypatch.setattr(bridge, "_bridge_context", lambda: None) - monkeypatch.setattr(bridge, "_local_server_ids", lambda: set()) - monkeypatch.setattr(bridge, "_mcp_json_local_declarations", lambda: {}) - monkeypatch.setattr(skills, "current_local_user_id", lambda: "local-u1") + monkeypatch.setattr(bridge_gateway, 'bridge_context', lambda: None) + monkeypatch.setattr(bridge_gateway, 'local_server_ids', lambda: set()) + monkeypatch.setattr(bridge_gateway, 'mcp_json_local_declarations', lambda: {}) + monkeypatch.setattr(skill_account, 'current_local_user_id', lambda: "local-u1") return result, index_db @@ -26,9 +28,9 @@ def test_modern_owned_connector_reaches_default_and_explicit_bindings(installed) assert db.query(AdminMcpServer).filter_by(server_id=sid).first() is None _, ambient = catalog_resolver._owned_enabled_ids(db, "local-u1", {}) assert sid in ambient - assert sid in bridge.apply_to_enabled_mcp_ids(ambient) + assert sid in bridge_gateway.apply_to_enabled_mcp_ids(ambient) output = [] - bridge.cloud_gateway_mcp_configs(ambient, resolution_out=output) + bridge_gateway.cloud_gateway_mcp_configs(ambient, resolution_out=output) assert sid in output[0].chosen _, explicit, _, missing = catalog_resolver.resolve_explicit_runtime_capabilities( db, "local-u1", mcp_ids=[sid] @@ -38,7 +40,7 @@ def test_modern_owned_connector_reaches_default_and_explicit_bindings(installed) db, "local-u1", {"mcps": [{"id": sid, "enabled": False}]} ) assert sid not in disabled - assert sid not in bridge.apply_to_enabled_mcp_ids(disabled) + assert sid not in bridge_gateway.apply_to_enabled_mcp_ids(disabled) selected = connectors.resolve_bindings(connectors.db_candidates([], set(ambient))) assert selected.chosen[sid].install_id == "mcp:local:" + sid @@ -60,3 +62,15 @@ def test_modern_connector_cannot_revive_revoked_installation(installed, revocati assert catalog_resolver.resolve_explicit_runtime_capabilities( db, user, mcp_ids=[sid] )[1] == [] + + +def test_packaged_compose_host_reaches_the_local_plane(client, index_db, monkeypatch): + from dataclasses import replace + from core.config.settings import settings + from extensions.mcp_servers._ports import PORTS + monkeypatch.setattr("core.config.settings.settings", + replace(settings, server=replace(settings.server, mcp_host="127.0.0.1"))) + root = Path(__file__).resolve().parents[4] / "extensions/plugins/marketplace/automation" + sid = service.install("local-u1", str(root))["components"]["mcp"][0] + # The manifest names the compose service ``mcp``; a local plane listens on loopback. + assert local_plugin_runtime.configs("local-u1")[sid]["url"] == f"http://127.0.0.1:{PORTS['automation_task']}/mcp" diff --git a/src/backend/tests/capabilities/test_partial_session_availability.py b/src/backend/tests/features/desktop/capabilities/test_partial_session_availability.py similarity index 73% rename from src/backend/tests/capabilities/test_partial_session_availability.py rename to src/backend/tests/features/desktop/capabilities/test_partial_session_availability.py index cd340ef6..421559c1 100644 --- a/src/backend/tests/capabilities/test_partial_session_availability.py +++ b/src/backend/tests/features/desktop/capabilities/test_partial_session_availability.py @@ -1,9 +1,12 @@ """A partial-sync choice is session-scoped and cannot trigger hidden downloads.""" +from features.desktop.bridge import identity as bridge_identity +from features.desktop.bridge import status as bridge_status +from features.desktop.capabilities import skill_account import pytest -from core.capabilities import registry, session_availability, preparation, skills -from core.capabilities.ref import cloud_ref -from core.services import desktop_cloud_bridge as bridge, desktop_cloud_skills, desktop_cloud_bundles -from core.db.models import DeviceCapabilityInstallation +from features.desktop.capabilities import registry, session_availability, preparation, skills +from core.capabilities.reference import cloud_ref +from features.desktop.bridge import client as bridge, skills as desktop_cloud_skills, bundles as desktop_cloud_bundles +from features.desktop.models import DeviceCapabilityInstallation @pytest.fixture(autouse=True) @@ -50,9 +53,9 @@ def test_another_account_does_not_inherit_the_partial_selection(index_db): def test_missing_selected_skill_does_not_download_in_partial_mode(index_db, monkeypatch): missing = installation("missing") session_availability.activate("profile", {}) - monkeypatch.setattr(skills, "account_authorized_for", lambda _: True) - monkeypatch.setattr(skills, "current_account_profile", lambda: "profile") - monkeypatch.setattr(bridge, "get_state", lambda: {"account": "current"}) + monkeypatch.setattr(skill_account, 'account_authorized_for', lambda _: True) + monkeypatch.setattr(skill_account, 'current_account_profile', lambda: "profile") + monkeypatch.setattr(bridge_identity, 'get_state', lambda: {"account": "current"}) def forbidden(*args): raise AssertionError("ordinary assembly tried to download an excluded package") monkeypatch.setattr(desktop_cloud_skills, "prepare", forbidden) @@ -63,9 +66,9 @@ def forbidden(*args): def test_continue_is_rejected_while_sync_is_running(index_db, monkeypatch): from contextlib import nullcontext from fastapi import HTTPException - from api.routes.v1 import desktop_capabilities as api - monkeypatch.setattr(bridge, "account_scope", lambda _: nullcontext()) - monkeypatch.setattr(bridge, "initial_sync_status", lambda: {"can_continue": False}) + from features.desktop.api import desktop_capabilities as api + monkeypatch.setattr(bridge_identity, 'account_scope', lambda _: nullcontext()) + monkeypatch.setattr(bridge_status, 'initial_sync_status', lambda: {"can_continue": False}) with pytest.raises(HTTPException) as error: api._accept_synced_capabilities({}, "user") assert error.value.status_code == 409 @@ -74,16 +77,16 @@ def test_continue_is_rejected_while_sync_is_running(index_db, monkeypatch): def test_continue_includes_only_current_integrity_verified_revisions(index_db, monkeypatch): from contextlib import nullcontext - from api.routes.v1 import desktop_capabilities as api - from core.capabilities import readiness - from core.capabilities.paths import revision_for_hash + from features.desktop.api import desktop_capabilities as api + from features.desktop.capabilities import readiness + from features.desktop.capabilities.paths import revision_for_hash good, corrupt, old, missing = [installation(key) for key in ("good", "corrupt", "old", "missing")] revision = revision_for_hash(good.content_hash) for row in (good, corrupt): registry.set_state(row.install_id, "ready", resolved_revision=revision) registry.set_state(old.install_id, "ready", resolved_revision="old-revision") - monkeypatch.setattr(bridge, "account_scope", lambda _: nullcontext()) - monkeypatch.setattr(bridge, "initial_sync_status", lambda: {"can_continue": True}) + monkeypatch.setattr(bridge_identity, 'account_scope', lambda _: nullcontext()) + monkeypatch.setattr(bridge_status, 'initial_sync_status', lambda: {"can_continue": True}) monkeypatch.setattr(api, "_authorized_profile", lambda _: "profile") monkeypatch.setattr(api, "_readiness_context", lambda _: object()) checked = [] diff --git a/src/backend/tests/capabilities/test_plugin_connections.py b/src/backend/tests/features/desktop/capabilities/test_plugin_connections.py similarity index 89% rename from src/backend/tests/capabilities/test_plugin_connections.py rename to src/backend/tests/features/desktop/capabilities/test_plugin_connections.py index 63d7ca3e..33713dfe 100644 --- a/src/backend/tests/capabilities/test_plugin_connections.py +++ b/src/backend/tests/features/desktop/capabilities/test_plugin_connections.py @@ -8,10 +8,10 @@ from fastapi.testclient import TestClient from api.routes.v1.plugins import router from core.auth.backend import UserContext, get_current_user -from core.capabilities import device_catalog, plugins +from features.desktop.capabilities import device_catalog, plugins from core.db.engine import get_db -from core.services import desktop_capability_entities as entities -from tests.capabilities.test_agents_plugins import cloud_db # noqa: F401 +from features.desktop.capability import entities as entities +from tests.features.desktop.capabilities.test_agents_plugins import cloud_db # noqa: F401 @pytest.mark.parametrize("connection", ["yida", "lark", "dingtalk", "email"]) @@ -46,10 +46,10 @@ def test_device_detail_exposes_connection(index_db, caps_root, monkeypatch, conn def test_local_import_preserves_connection(index_db, caps_root, tmp_path, monkeypatch): - from core.plugins.local import service + from features.desktop.local.plugins import service as service from core.db.engine import Base Base.metadata.create_all(index_db.kw["bind"]) - monkeypatch.setattr("core.services.local_skill_service._invalidate", lambda: None) + monkeypatch.setattr('features.desktop.local.skills.service._invalidate', lambda: None) monkeypatch.setattr(device_catalog, "_profile", lambda: None) monkeypatch.setattr(device_catalog, "_managed_connectors", lambda: []) root = tmp_path / "package" diff --git a/src/backend/tests/capabilities/test_plugin_device_assets.py b/src/backend/tests/features/desktop/capabilities/test_plugin_device_assets.py similarity index 89% rename from src/backend/tests/capabilities/test_plugin_device_assets.py rename to src/backend/tests/features/desktop/capabilities/test_plugin_device_assets.py index 545d4e27..392ae15d 100644 --- a/src/backend/tests/capabilities/test_plugin_device_assets.py +++ b/src/backend/tests/features/desktop/capabilities/test_plugin_device_assets.py @@ -9,9 +9,9 @@ from __future__ import annotations import pytest -from core.capabilities import device_catalog -from core.plugins.local import assets as plugin_device_assets -from core.plugins.local.assets import provision_for, provision_present_plugins +from features.desktop.capabilities import device_catalog +from features.desktop.local.plugins import assets as plugin_device_assets +from features.desktop.local.plugins.assets import provision_for, provision_present_plugins @pytest.fixture() @@ -25,7 +25,7 @@ def template_home(tmp_path, monkeypatch): @pytest.fixture() def bare_device(monkeypatch): """云端投影为空、本机业务库也没有插件的机器。""" - monkeypatch.setattr(device_catalog, "plugin_entries", list) + monkeypatch.setattr(device_catalog, "plugin_entries", lambda user_id=None: []) class _Query: def filter(self, *_): @@ -66,7 +66,7 @@ def test_refresh_covers_a_cloud_synced_plugin_with_no_local_install( ): """插件只存在于云端投影里时，升级后的资产刷新照样要覆盖它。""" monkeypatch.setattr( - device_catalog, "plugin_entries", lambda: [{"slug": "sites"}, {"slug": "yida"}] + device_catalog, "plugin_entries", lambda user_id=None: [{"slug": "sites"}, {"slug": "yida"}] ) assert provision_present_plugins(bare_device) == ["sites"] diff --git a/src/backend/tests/capabilities/test_prepare_lock_not_held_over_io.py b/src/backend/tests/features/desktop/capabilities/test_prepare_lock_not_held_over_io.py similarity index 98% rename from src/backend/tests/capabilities/test_prepare_lock_not_held_over_io.py rename to src/backend/tests/features/desktop/capabilities/test_prepare_lock_not_held_over_io.py index d5d78bb9..9a6f3ab9 100644 --- a/src/backend/tests/capabilities/test_prepare_lock_not_held_over_io.py +++ b/src/backend/tests/features/desktop/capabilities/test_prepare_lock_not_held_over_io.py @@ -11,7 +11,7 @@ import threading -from core.capabilities import preparation +from features.desktop.capabilities import preparation def test_prepare_lock_is_free_while_a_download_is_in_flight(): diff --git a/src/backend/tests/capabilities/test_progressive_plugins.py b/src/backend/tests/features/desktop/capabilities/test_progressive_plugins.py similarity index 72% rename from src/backend/tests/capabilities/test_progressive_plugins.py rename to src/backend/tests/features/desktop/capabilities/test_progressive_plugins.py index 6b6dcc69..aa9ae126 100644 --- a/src/backend/tests/capabilities/test_progressive_plugins.py +++ b/src/backend/tests/features/desktop/capabilities/test_progressive_plugins.py @@ -1,19 +1,25 @@ +from features.desktop import plugin_resolution as desktop_plugin_resolution """Desktop plugin activation keeps the prepared source and tool surface.""" +from features.desktop.bridge import identity as bridge_identity +from features.desktop.capabilities import skill_account +from features.desktop.capabilities import skill_candidates +from features.desktop.capabilities import skill_publication import asyncio from types import SimpleNamespace import pytest -from core.capabilities import plugins, skills, runtime -from core.plugins import runtime as plugin_loader -from core.services.desktop_capability_protocol import skill_content_hash -from core.llm.tool_collector import ToolCollector +from features.desktop.capabilities import plugins, skills, runtime +from core.capabilities.plugins import runtime as plugin_loader +from core.capabilities.manifest_protocol import skill_content_hash +from core.llm.factory.tools.definitions import ToolAssembly +from core.llm.tools import plugin_activation def test_desktop_directory_defers_skill_until_load_plugin(index_db, caps_root, monkeypatch): - monkeypatch.setattr(skills, "current_account_profile", lambda: None) - monkeypatch.setattr(skills, "builtin_candidates", lambda: []) + monkeypatch.setattr(skill_account, 'current_account_profile', lambda: None) + monkeypatch.setattr(skill_candidates, 'builtin_candidates', lambda: []) body = "---\nname: report\ndescription: Write reports\n---\nUse the report template." - skills.publish_local_skill( + skill_publication.publish_local_skill( "report", files={ "SKILL.md": "---\nname: report\ndescription: Write reports\n---\nUse the report template." @@ -25,7 +31,7 @@ def test_desktop_directory_defers_skill_until_load_plugin(index_db, caps_root, m {"slug": "reports", "name": "Reports", "components": {"skills": ["report"]}}, owner_user_id="owner", ) - plan = plugin_loader.resolve_desktop_progressive_plugins( + plan = desktop_plugin_resolution.resolve_desktop_progressive_plugins( user_id="owner", enabled_skill_ids=["report"], enabled_mcp_ids=[], @@ -36,8 +42,8 @@ def test_desktop_directory_defers_skill_until_load_plugin(index_db, caps_root, m "run", "owner", skill_ids=["report"], plugin_ids=[plan.directory[0].install_id] ) run = runtime.preflight(run, plugin_ids=[plan.directory[0].install_id]) - basic = SimpleNamespace(mcps=[], skills_or_loaders=[]) - collector = ToolCollector() + basic = SimpleNamespace(tools=[], skills_or_loaders=[]) + collector = ToolAssembly() context = { "toolkit": SimpleNamespace(tool_groups=[basic]), "loader": runtime.frozen_loader(run), @@ -45,9 +51,9 @@ def test_desktop_directory_defers_skill_until_load_plugin(index_db, caps_root, m "prepared_servers": {}, "persist": False, } - plugin_loader.register_load_plugin(collector, plan.deferred_by_slug(), context) + plugin_activation.register_load_plugin(collector, plan.deferred_by_slug(), context) assert basic.skills_or_loaders == [] - tool = next(t for t in collector.function_tools if t.name == "load_plugin") + tool = next(t for t in list(collector) if t.name == "load_plugin") asyncio.run(tool(plugin="reports")) assert len(basic.skills_or_loaders) == 1 @@ -56,11 +62,11 @@ def test_desktop_directory_defers_skill_until_load_plugin(index_db, caps_root, m def test_partial_plugin_and_explicit_skill_preserve_selected_surface( index_db, caps_root, monkeypatch, bound ): - monkeypatch.setattr(skills, "current_account_profile", lambda: None) - monkeypatch.setattr(skills, "builtin_candidates", lambda: []) + monkeypatch.setattr(skill_account, 'current_account_profile', lambda: None) + monkeypatch.setattr(skill_candidates, 'builtin_candidates', lambda: []) for name in ["report", "chart"]: body = f"---\nname: {name}\ndescription: Test skill\n---\nInstructions" - skills.publish_local_skill( + skill_publication.publish_local_skill( name, files={"SKILL.md": body}, content_hash=skill_content_hash(body, {}), @@ -75,13 +81,13 @@ def test_partial_plugin_and_explicit_skill_preserve_selected_surface( enabled_mcp_ids=[], plugin_ids=["reports"] if bound else None, ) - plan = plugin_loader.resolve_desktop_progressive_plugins(**args) + plan = desktop_plugin_resolution.resolve_desktop_progressive_plugins(**args) assert plan.deferred_skill_ids == {"report"} run = runtime.prepare("partial", "owner", skill_ids=["report"]) frozen = runtime.preflight( run) assert frozen.dependency_report["ready"] - explicit = plugin_loader.resolve_desktop_progressive_plugins( + explicit = desktop_plugin_resolution.resolve_desktop_progressive_plugins( **args, invoked_skill_ids=["report"] ) assert explicit.deferred_skill_ids == set() @@ -91,13 +97,13 @@ def test_partial_plugin_and_explicit_skill_preserve_selected_surface( def test_load_plugin_rechecks_definition_before_exposing_skills( index_db, caps_root, monkeypatch, mutation ): - from core.capabilities import registry - from core.capabilities.errors import CapabilityError + from features.desktop.capabilities import registry + from core.harness.capability_errors import CapabilityError - monkeypatch.setattr(skills, "current_account_profile", lambda: None) - monkeypatch.setattr(skills, "builtin_candidates", lambda: []) + monkeypatch.setattr(skill_account, 'current_account_profile', lambda: None) + monkeypatch.setattr(skill_candidates, 'builtin_candidates', lambda: []) body = "---\nname: report\ndescription: Test skill\n---\nInstructions" - skills.publish_local_skill( + skill_publication.publish_local_skill( "report", files={"SKILL.md": body}, content_hash=skill_content_hash(body, {}), @@ -106,16 +112,16 @@ def test_load_plugin_rechecks_definition_before_exposing_skills( component = plugins.publish_local_plugin( {"slug": "reports", "components": {"skills": ["report"]}}, owner_user_id="owner" ) - plan = plugin_loader.resolve_desktop_progressive_plugins( + plan = desktop_plugin_resolution.resolve_desktop_progressive_plugins( user_id="owner", enabled_skill_ids=["report"], enabled_mcp_ids=[] ) run = runtime.prepare("revoke", "owner", skill_ids=["report"]) run = runtime.preflight( run) loader = runtime.frozen_loader(run) - basic = SimpleNamespace(mcps=[], skills_or_loaders=[]) - collector = ToolCollector() - plugin_loader.register_load_plugin( + basic = SimpleNamespace(tools=[], skills_or_loaders=[]) + collector = ToolAssembly() + plugin_activation.register_load_plugin( collector, plan.deferred_by_slug(), { @@ -131,26 +137,26 @@ def test_load_plugin_rechecks_definition_before_exposing_skills( else: component.entry_file.write_text("{}") with pytest.raises(CapabilityError): - asyncio.run(collector.get_tool("load_plugin")._func("reports")) + asyncio.run(next(t for t in collector if t.name == "load_plugin")._func("reports")) assert basic.skills_or_loaders == [] def test_pending_cloud_plugin_is_discovered_from_manifest(index_db, caps_root, monkeypatch): import io, zipfile - from core.capabilities import registry - from core.capabilities.ref import cloud_ref, profile_id - from core.services import desktop_cloud_bridge as bridge, desktop_cloud_bundles as bundles - from core.services.desktop_capability_protocol import entity_content_hash - from tests.capabilities.test_runtime_recovery import state + from features.desktop.capabilities import registry + from core.capabilities.reference import cloud_ref, profile_id + from features.desktop.bridge import client as bridge, bundles as bundles + from core.capabilities.manifest_protocol import entity_content_hash + from tests.features.desktop.capabilities.test_runtime_recovery import state from core.db.engine import Base with index_db() as db: Base.metadata.create_all(db.get_bind()) monkeypatch.setattr("core.db.engine.SessionLocal", index_db) st = state("cloud-owner") - monkeypatch.setattr(bridge, "get_state", lambda: st) - monkeypatch.setattr(skills, "current_local_user_id", lambda: "owner") - monkeypatch.setattr(bridge, "ensure_current_authorization", lambda: None) + monkeypatch.setattr(bridge_identity, 'get_state', lambda: st) + monkeypatch.setattr(skill_account, 'current_local_user_id', lambda: "owner") + monkeypatch.setattr(bridge_identity, 'ensure_current_authorization', lambda: None) files = plugins.plugin_manifest_files({"slug": "reports", "components": {"skills": ["report"]}}) inst = registry.upsert( profile_id=profile_id(st["cloud_base"], "cloud-owner"), @@ -173,13 +179,13 @@ def test_pending_cloud_plugin_is_discovered_from_manifest(index_db, caps_root, m skill_archive = io.BytesIO() with zipfile.ZipFile(skill_archive, "w") as out: out.writestr("SKILL.md", body) - from core.services import desktop_cloud_skills + from features.desktop.bridge import skills as desktop_cloud_skills monkeypatch.setattr(desktop_cloud_skills, "_download", lambda *a: skill_archive.getvalue()) assert not registry.get(inst.install_id).ready - defaults = plugin_loader.prepare_desktop_plugin_skill_defaults("owner", []) + defaults = desktop_plugin_resolution.prepare_desktop_plugin_skill_defaults("owner", []) assert defaults == ["report"] - plan = plugin_loader.resolve_desktop_progressive_plugins( + plan = desktop_plugin_resolution.resolve_desktop_progressive_plugins( user_id="owner", enabled_skill_ids=defaults, enabled_mcp_ids=[] ) assert plan.deferred_skill_ids == {"report"} @@ -193,12 +199,12 @@ async def test_load_plugin_adds_frozen_mcp_schema_to_live_toolkit( ): from agentscope.tool import Toolkit - monkeypatch.setattr(skills, "current_account_profile", lambda: None) - monkeypatch.setattr(skills, "builtin_candidates", lambda: []) + monkeypatch.setattr(skill_account, 'current_account_profile', lambda: None) + monkeypatch.setattr(skill_candidates, 'builtin_candidates', lambda: []) plugins.publish_local_plugin( {"slug": "lookup", "components": {"mcp": ["lookup-mcp"]}}, owner_user_id="owner" ) - plan = plugin_loader.resolve_desktop_progressive_plugins( + plan = desktop_plugin_resolution.resolve_desktop_progressive_plugins( user_id="owner", enabled_skill_ids=[], enabled_mcp_ids=["lookup-mcp"], @@ -218,7 +224,7 @@ async def test_load_plugin_adds_frozen_mcp_schema_to_live_toolkit( runtime.bind_mcp(run, configs, None) run = runtime.preflight( run, available_mcp={"lookup-mcp"}) - collector = ToolCollector() + collector = ToolAssembly() context = { "prepared_run": run, "prepared_servers": configs, @@ -226,24 +232,24 @@ async def test_load_plugin_adds_frozen_mcp_schema_to_live_toolkit( "loader": runtime.frozen_loader(run), "close_list": [], } - plugin_loader.register_load_plugin(collector, plan.deferred_by_slug(), context) - toolkit = Toolkit(tools=collector.function_tools) + plugin_activation.register_load_plugin(collector, plan.deferred_by_slug(), context) + toolkit = Toolkit(tools=list(collector)) context["toolkit"] = toolkit before = await toolkit.get_tool_schemas() assert "find_record" not in str(before) - await collector.get_tool("load_plugin")(plugin="lookup") + await next(t for t in collector if t.name == "load_plugin")(plugin="lookup") assert "find_record" in str(await toolkit.get_tool_schemas()) for client in context["close_list"]: await client.close() def test_plugin_selected_skill_version_constraint_is_not_bypassed(index_db, caps_root, monkeypatch): - from core.capabilities.errors import PackageMissing + from core.harness.capability_errors import PackageMissing - monkeypatch.setattr(skills, "current_account_profile", lambda: None) - monkeypatch.setattr(skills, "builtin_candidates", lambda: []) + monkeypatch.setattr(skill_account, 'current_account_profile', lambda: None) + monkeypatch.setattr(skill_candidates, 'builtin_candidates', lambda: []) body = "---\nname: report\ndescription: Test\n---\nInstructions" - skills.publish_local_skill( + skill_publication.publish_local_skill( "report", files={"SKILL.md": body}, content_hash=skill_content_hash(body, {}), @@ -258,7 +264,7 @@ def test_plugin_selected_skill_version_constraint_is_not_bypassed(index_db, caps owner_user_id="owner", ) # 版本约束对不上：这个插件这一轮直接不出现，不影响别的能力。 - plan = plugin_loader.resolve_desktop_progressive_plugins( + plan = desktop_plugin_resolution.resolve_desktop_progressive_plugins( user_id="owner", enabled_skill_ids=["report"], enabled_mcp_ids=[] ) assert [item.install_id for item in plan.directory] == [] @@ -267,14 +273,14 @@ def test_plugin_selected_skill_version_constraint_is_not_bypassed(index_db, caps def test_activation_checks_requirement_against_frozen_skill_version( index_db, caps_root, monkeypatch ): - from core.capabilities.dependency import DependencyMissing + from features.desktop.capabilities.dependency import DependencyMissing - monkeypatch.setattr(skills, "current_account_profile", lambda: None) - monkeypatch.setattr(skills, "builtin_candidates", lambda: []) + monkeypatch.setattr(skill_account, 'current_account_profile', lambda: None) + monkeypatch.setattr(skill_candidates, 'builtin_candidates', lambda: []) def install(version): body = f"---\nname: report\ndescription: Test\n---\nVersion {version}" - skills.publish_local_skill( + skill_publication.publish_local_skill( "report", files={"SKILL.md": body}, content_hash=skill_content_hash(body, {}), @@ -292,7 +298,7 @@ def install(version): }, owner_user_id="owner", ) - plan = plugin_loader.resolve_desktop_progressive_plugins( + plan = desktop_plugin_resolution.resolve_desktop_progressive_plugins( user_id="owner", enabled_skill_ids=["report"], enabled_mcp_ids=[] ) # 约束要拿这一轮冻结的 1.0 去对，而不是拿当前安装的 2.0——所以激活必须被拦下。 @@ -305,11 +311,11 @@ def install(version): def test_incomplete_plugin_never_advertises_skill_only_success( index_db, caps_root, monkeypatch, scope ): - monkeypatch.setattr(skills, "current_account_profile", lambda: None) - monkeypatch.setattr(skills, "builtin_candidates", lambda: []) + monkeypatch.setattr(skill_account, 'current_account_profile', lambda: None) + monkeypatch.setattr(skill_candidates, 'builtin_candidates', lambda: []) for sid in ("site-builder", "cloud-site-builder"): body = f"---\nname: {sid}\ndescription: Build and publish a site\n---\nCall publish_site." - skills.publish_local_skill( + skill_publication.publish_local_skill( sid, files={"SKILL.md": body}, content_hash=skill_content_hash(body, {}), @@ -326,7 +332,7 @@ def test_incomplete_plugin_never_advertises_skill_only_success( }, owner_user_id="owner", ) - plan = plugin_loader.resolve_desktop_progressive_plugins( + plan = desktop_plugin_resolution.resolve_desktop_progressive_plugins( user_id="owner", enabled_skill_ids=["site-builder", "cloud-site-builder"], enabled_mcp_ids=["site-publish-private"], @@ -344,12 +350,12 @@ def test_incomplete_plugin_never_advertises_skill_only_success( def test_explicit_incomplete_plugin_reports_missing_binding( index_db, caps_root, monkeypatch, selection ): - from core.capabilities.errors import PackageMissing + from core.harness.capability_errors import PackageMissing - monkeypatch.setattr(skills, "current_account_profile", lambda: None) - monkeypatch.setattr(skills, "builtin_candidates", lambda: []) + monkeypatch.setattr(skill_account, 'current_account_profile', lambda: None) + monkeypatch.setattr(skill_candidates, 'builtin_candidates', lambda: []) body = "---\nname: site-builder\ndescription: Build a site\n---\nCall publish_site." - skills.publish_local_skill( + skill_publication.publish_local_skill( "site-builder", files={"SKILL.md": body}, content_hash=skill_content_hash(body, {}), @@ -360,7 +366,7 @@ def test_explicit_incomplete_plugin_reports_missing_binding( owner_user_id="owner", ) def resolve(): - return plugin_loader.resolve_desktop_progressive_plugins( + return desktop_plugin_resolution.resolve_desktop_progressive_plugins( user_id="owner", enabled_skill_ids=["site-builder"], enabled_mcp_ids=[], @@ -386,11 +392,11 @@ async def test_available_namespaced_plugin_publishes_tools_after_activation( ): from agentscope.tool import Toolkit - monkeypatch.setattr(skills, "current_account_profile", lambda: None) - monkeypatch.setattr(skills, "builtin_candidates", lambda: []) + monkeypatch.setattr(skill_account, 'current_account_profile', lambda: None) + monkeypatch.setattr(skill_candidates, 'builtin_candidates', lambda: []) for key, mcp in (("sites", "site-publish"), ("sites-private", "site-publish-private")): body = f"---\nname: {key}\ndescription: Build a site\n---\nCall publish_site." - skills.publish_local_skill( + skill_publication.publish_local_skill( key, files={"SKILL.md": body}, content_hash=skill_content_hash(body, {}), @@ -400,7 +406,7 @@ async def test_available_namespaced_plugin_publishes_tools_after_activation( {"slug": key, "components": {"skills": [key], "mcp": [mcp]}}, owner_user_id="owner", ) - plan = plugin_loader.resolve_desktop_progressive_plugins( + plan = desktop_plugin_resolution.resolve_desktop_progressive_plugins( user_id="owner", enabled_skill_ids=["sites", "sites-private"], enabled_mcp_ids=["site-publish-private"], @@ -421,7 +427,7 @@ async def test_available_namespaced_plugin_publishes_tools_after_activation( runtime.bind_mcp(run, configs, None) run = runtime.preflight( run, available_mcp=set(configs)) - collector = ToolCollector() + collector = ToolAssembly() context = { "prepared_run": run, "prepared_servers": configs, @@ -429,21 +435,21 @@ async def test_available_namespaced_plugin_publishes_tools_after_activation( "loader": runtime.frozen_loader(run), "close_list": [], } - plugin_loader.register_load_plugin(collector, plan.deferred_by_slug(), context) - toolkit = Toolkit(tools=collector.function_tools) + plugin_activation.register_load_plugin(collector, plan.deferred_by_slug(), context) + toolkit = Toolkit(tools=list(collector)) context["toolkit"] = toolkit assert "publish_site" not in str(await toolkit.get_tool_schemas()) - await collector.get_tool("load_plugin")(plugin="sites-private") + await next(t for t in collector if t.name == "load_plugin")(plugin="sites-private") assert "publish_site" in str(await toolkit.get_tool_schemas()) for client in context["close_list"]: await client.close() def test_shared_skill_remains_callable_via_complete_plugin(index_db, caps_root, monkeypatch): - monkeypatch.setattr(skills, "current_account_profile", lambda: None) - monkeypatch.setattr(skills, "builtin_candidates", lambda: []) + monkeypatch.setattr(skill_account, 'current_account_profile', lambda: None) + monkeypatch.setattr(skill_candidates, 'builtin_candidates', lambda: []) body = "---\nname: shared\ndescription: Shared skill\n---\nInstructions." - skills.publish_local_skill( + skill_publication.publish_local_skill( "shared", files={"SKILL.md": body}, content_hash=skill_content_hash(body, {}), @@ -454,7 +460,7 @@ def test_shared_skill_remains_callable_via_complete_plugin(index_db, caps_root, {"slug": slug, "components": {"skills": ["shared"], "mcp": [mcp]}}, owner_user_id="owner", ) - plan = plugin_loader.resolve_desktop_progressive_plugins( + plan = desktop_plugin_resolution.resolve_desktop_progressive_plugins( user_id="owner", enabled_skill_ids=["shared"], enabled_mcp_ids=["available-mcp"], @@ -466,15 +472,15 @@ def test_shared_skill_remains_callable_via_complete_plugin(index_db, caps_root, def test_explicit_mcp_only_plugin_cannot_vanish(index_db, caps_root, monkeypatch): - from core.capabilities.errors import PackageMissing + from core.harness.capability_errors import PackageMissing - monkeypatch.setattr(skills, "current_account_profile", lambda: None) - monkeypatch.setattr(skills, "builtin_candidates", lambda: []) + monkeypatch.setattr(skill_account, 'current_account_profile', lambda: None) + monkeypatch.setattr(skill_candidates, 'builtin_candidates', lambda: []) plugins.publish_local_plugin( {"slug": "lookup", "components": {"mcp": ["missing"]}}, owner_user_id="owner" ) # 插件要的连接器不存在：它这一轮不出现，而不是让整轮失败。 - plan = plugin_loader.resolve_desktop_progressive_plugins( + plan = desktop_plugin_resolution.resolve_desktop_progressive_plugins( user_id="owner", enabled_skill_ids=[], enabled_mcp_ids=[], diff --git a/src/backend/tests/capabilities/test_runtime_frozen_bindings.py b/src/backend/tests/features/desktop/capabilities/test_runtime_frozen_bindings.py similarity index 63% rename from src/backend/tests/capabilities/test_runtime_frozen_bindings.py rename to src/backend/tests/features/desktop/capabilities/test_runtime_frozen_bindings.py index a7ecae26..c3408bfd 100644 --- a/src/backend/tests/capabilities/test_runtime_frozen_bindings.py +++ b/src/backend/tests/features/desktop/capabilities/test_runtime_frozen_bindings.py @@ -1,3 +1,9 @@ +from features.desktop.bridge import client as bridge_client +from features.desktop.bridge import client_state as bridge_client_state +from features.desktop.bridge import identity as bridge_identity +from features.desktop.capabilities import skill_account +from features.desktop.capabilities import skill_candidates +from features.desktop.capabilities import skill_publication from tests.sandbox.runner_client import run_runner """Regressions for desktop identity, explicit bindings and immutable runs.""" @@ -6,29 +12,29 @@ import json import pytest -from core.capabilities import registry, skills -from core.services import desktop_cloud_bridge as bridge -from core.services import desktop_cloud_skills as cloud_skills -from core.services import desktop_cloud_bundles as bundles -from core.services.desktop_capability_protocol import build_skill_manifest +from features.desktop.capabilities import registry, skills +from features.desktop.bridge import client as bridge +from features.desktop.bridge import skills as cloud_skills +from features.desktop.bridge import bundles as bundles +from core.capabilities.manifest_protocol import build_skill_manifest -from tests.capabilities.runtime_recovery_support import _ordered_skill_manifest, state, _zip, _intent, durable_index +from tests.features.desktop.capabilities.runtime_recovery_support import _ordered_skill_manifest, state, _zip, _intent, durable_index def test_prepared_run_keeps_old_bytes_after_update_and_rebuild( durable_index, caps_root, monkeypatch ): - from core.capabilities import runtime - from core.services.desktop_capability_protocol import skill_content_hash + from features.desktop.capabilities import runtime + from core.capabilities.manifest_protocol import skill_content_hash - monkeypatch.setattr(skills, "builtin_candidates", lambda: []) - monkeypatch.setattr(skills, "current_account_profile", lambda: None) - first = skills.publish_local_skill( + monkeypatch.setattr(skill_candidates, 'builtin_candidates', lambda: []) + monkeypatch.setattr(skill_account, 'current_account_profile', lambda: None) + first = skill_publication.publish_local_skill( "example", files={"SKILL.md": "v1"}, content_hash=skill_content_hash("v1", {}) ) run = runtime.prepare("run-a", "user-a", skill_ids=["example"]) - skills.publish_local_skill( + skill_publication.publish_local_skill( "example", files={"SKILL.md": "v2"}, content_hash=skill_content_hash("v2", {}) ) replay = runtime.prepare("run-a", "user-a", skill_ids=["example"]) @@ -41,13 +47,13 @@ def test_prepared_run_keeps_old_bytes_after_update_and_rebuild( def test_prepared_run_rejects_account_plane_and_integrity_changes( durable_index, caps_root, monkeypatch ): - from core.capabilities import runtime - from core.capabilities.errors import CapabilityError - from core.services.desktop_capability_protocol import skill_content_hash + from features.desktop.capabilities import runtime + from core.harness.capability_errors import CapabilityError + from core.capabilities.manifest_protocol import skill_content_hash - monkeypatch.setattr(skills, "builtin_candidates", lambda: []) - monkeypatch.setattr(skills, "current_account_profile", lambda: None) - original = skills.publish_local_skill( + monkeypatch.setattr(skill_candidates, 'builtin_candidates', lambda: []) + monkeypatch.setattr(skill_account, 'current_account_profile', lambda: None) + original = skill_publication.publish_local_skill( "example", files={"SKILL.md": "v1"}, content_hash=skill_content_hash("v1", {}) ) run = runtime.prepare("run-a", "user-a", skill_ids=["example"]) @@ -64,7 +70,7 @@ def test_prepared_run_rejects_account_plane_and_integrity_changes( def test_plugin_required_disabled_agent_blocks_optional_missing_skill_does_not(index_db, caps_root): - from core.capabilities import agents, plugins + from features.desktop.capabilities import agents, plugins agent = agents.publish_local_agent( {"agent_id": "writer", "name": "Writer", "is_enabled": False} @@ -85,10 +91,10 @@ def test_plugin_required_disabled_agent_blocks_optional_missing_skill_does_not(i def test_mcp_contract_and_source_are_pinned_across_replay(durable_index, caps_root, monkeypatch): - from core.capabilities import runtime + from features.desktop.capabilities import runtime - monkeypatch.setattr(skills, "builtin_candidates", lambda: []) - monkeypatch.setattr(skills, "current_account_profile", lambda: None) + monkeypatch.setattr(skill_candidates, 'builtin_candidates', lambda: []) + monkeypatch.setattr(skill_account, 'current_account_profile', lambda: None) run = runtime.prepare("run-mcp", "u", skill_ids=[]) v1 = { "search": { @@ -119,10 +125,10 @@ def test_mcp_contract_and_source_are_pinned_across_replay(durable_index, caps_ro def test_agent_definition_is_pinned_on_replay(durable_index, caps_root, monkeypatch): - from core.capabilities import runtime - from core.capabilities.agents import AgentDefinition + from features.desktop.capabilities import runtime + from features.desktop.capabilities.agents import AgentDefinition - monkeypatch.setattr(skills, "current_account_profile", lambda: None) + monkeypatch.setattr(skill_account, 'current_account_profile', lambda: None) old = AgentDefinition(agent_id="writer", name="Writer", system_prompt="v1", skill_ids=["first"]) newer = AgentDefinition( agent_id="writer", name="Writer", system_prompt="v2", skill_ids=["second"] @@ -136,19 +142,19 @@ def test_agent_definition_is_pinned_on_replay(durable_index, caps_root, monkeypa async def test_runner_executes_frozen_view_after_background_update( durable_index, caps_root, tmp_path, monkeypatch ): - from core.capabilities import runtime - from core.services.desktop_capability_protocol import skill_content_hash + from features.desktop.capabilities import runtime + from core.capabilities.manifest_protocol import skill_content_hash from core.sandbox.runner import server - monkeypatch.setattr(skills, "builtin_candidates", lambda: []) - monkeypatch.setattr(skills, "current_account_profile", lambda: None) + monkeypatch.setattr(skill_candidates, 'builtin_candidates', lambda: []) + monkeypatch.setattr(skill_account, 'current_account_profile', lambda: None) monkeypatch.setenv("DEPLOY_PROFILE", "local") monkeypatch.setattr(server, "WORKSPACE_ROOT", str(tmp_path / "workspace")) - skills.publish_local_skill( + skill_publication.publish_local_skill( "example", files={"SKILL.md": "v1"}, content_hash=skill_content_hash("v1", {}) ) run = runtime.prepare("run-exec", "u", skill_ids=["example"]) - skills.publish_local_skill( + skill_publication.publish_local_skill( "example", files={"SKILL.md": "v2"}, content_hash=skill_content_hash("v2", {}) ) response = await run_runner( @@ -168,21 +174,23 @@ async def test_runner_executes_frozen_view_after_background_update( async def test_skill_reader_uses_frozen_loader_and_cannot_read_other_profile( durable_index, caps_root, monkeypatch ): - from core.capabilities import runtime - from core.services.desktop_capability_protocol import skill_content_hash + from features.desktop.capabilities import runtime + from core.capabilities.manifest_protocol import skill_content_hash from core.llm.tools.skill_tool import register_sandboxed_view_text_file - monkeypatch.setattr(skills, "builtin_candidates", lambda: []) - monkeypatch.setattr(skills, "current_account_profile", lambda: None) + monkeypatch.setattr(skill_candidates, 'builtin_candidates', lambda: []) + monkeypatch.setattr(skill_account, 'current_account_profile', lambda: None) body = "---\nname: example\ndescription: sample\n---\nv1" - skills.publish_local_skill( + skill_publication.publish_local_skill( "example", files={"SKILL.md": body}, content_hash=skill_content_hash(body, {}) ) run = runtime.prepare("run-reader", "u", skill_ids=["example"]) loader = runtime.frozen_loader(run) class Toolkit: - def register_tool_function(self, fn, **_): + def append(self, tool): + fn = tool._func + _ = {"func_description": tool.description, "permission": getattr(tool, "dispatch_spec", None)} self.read = fn toolkit = Toolkit() @@ -201,14 +209,14 @@ def register_tool_function(self, fn, **_): def test_bridge_token_is_memory_only_and_session_bound(durable_index, caps_root, monkeypatch): import time from core.db.models import ContentBlock - from core.capabilities.errors import CloudUnavailable + from core.harness.capability_errors import CloudUnavailable - monkeypatch.setattr(bridge, "_refresh_manifest_async", lambda **_: None) - monkeypatch.setattr(bridge, "_rebuild_identity_views", lambda: None) + monkeypatch.setattr(bridge_client, 'refresh_manifest_async', lambda **_: None) + monkeypatch.setattr(bridge_identity, 'rebuild_identity_views', lambda: None) monkeypatch.setattr("core.db.engine.SessionLocal", durable_index) - monkeypatch.setattr("core.services.desktop_model_credentials.scrub_legacy_rows", lambda: None) + monkeypatch.setattr('features.desktop.bridge.model_credentials.scrub_legacy_rows', lambda: None) with durable_index() as db: - db.add(ContentBlock(id=bridge.BRIDGE_BLOCK_ID, payload={"token": "old-secret"})) + db.add(ContentBlock(id=bridge_client_state.BRIDGE_BLOCK_ID, payload={"token": "old-secret"})) db.commit() def token(epoch, nonce): @@ -224,40 +232,40 @@ def token(epoch, nonce): body = base64.urlsafe_b64encode(json.dumps(claims).encode()).decode().rstrip("=") return "dcap2." + body + ".test-signature" - bridge.set_state("https://cloud.example", token(1, "one"), 600, device_id="device-a") - captured = bridge.get_state() + bridge_client.set_state("https://cloud.example", token(1, "one"), 600, device_id="device-a") + captured = bridge_identity.get_state() with durable_index() as db: - assert db.get(ContentBlock, bridge.BRIDGE_BLOCK_ID) is None - assert bridge.cloud_headers(captured)["X-Desktop-Device-Id"] == "device-a" - bridge.set_state("https://cloud.example", token(1, "two"), 600, device_id="device-a") - bridge.require_current_account(captured) - bridge.set_state("https://cloud.example", token(2, "three"), 600, device_id="device-a") + assert db.get(ContentBlock, bridge_client_state.BRIDGE_BLOCK_ID) is None + assert bridge_identity.cloud_headers(captured)["X-Desktop-Device-Id"] == "device-a" + bridge_client.set_state("https://cloud.example", token(1, "two"), 600, device_id="device-a") + bridge_identity.require_current_account(captured) + bridge_client.set_state("https://cloud.example", token(2, "three"), 600, device_id="device-a") with pytest.raises(CloudUnavailable): - bridge.require_current_account(captured) - bridge.clear_state() - assert bridge.get_state() is None - bridge._state_loaded = False - assert bridge.get_state() is None + bridge_identity.require_current_account(captured) + bridge_identity.clear_state() + assert bridge_identity.get_state() is None + bridge_client_state.state_loaded = False + assert bridge_identity.get_state() is None def test_stale_mcp_error_does_not_replace_current_account_status(index_db, caps_root, monkeypatch): a, b = state("a"), state("b") current = [a] - monkeypatch.setattr(bridge, "get_state", lambda: current[0]) - monkeypatch.setattr(bridge, "_refresh_manifest_async", lambda **_: None) + monkeypatch.setattr(bridge_identity, 'get_state', lambda: current[0]) + monkeypatch.setattr(bridge_client, 'refresh_manifest_async', lambda **_: None) def fetch(*_, **__): current[0] = b - bridge._manifest_error = None + bridge_client_state.manifest_error = None raise RuntimeError("old account offline") monkeypatch.setattr("httpx.get", fetch) - bridge._fetch_manifest_blocking(a) - assert bridge._manifest_error is None + bridge_client.fetch_manifest_blocking(a) + assert bridge_client_state.manifest_error is None def test_cloud_removal_revokes_access_but_retains_history_bytes(index_db, caps_root, monkeypatch): - from core.capabilities import store + from features.desktop.capabilities import store st, inst = _intent(monkeypatch) monkeypatch.setattr(cloud_skills, "_download", lambda *_: _zip({"SKILL.md": "v1"})) @@ -274,17 +282,17 @@ def test_cloud_removal_revokes_access_but_retains_history_bytes(index_db, caps_r def test_audit_references_the_frozen_run_after_live_update(durable_index, caps_root, monkeypatch): - from core.capabilities import runtime - from core.services.desktop_capability_protocol import skill_content_hash - from core.evolution.runtime_binding import _skill_refs + from features.desktop.capabilities import runtime + from core.capabilities.manifest_protocol import skill_content_hash + from core.evolution.cycle.runtime_binding import _skill_refs - monkeypatch.setattr(skills, "builtin_candidates", lambda: []) - monkeypatch.setattr(skills, "current_account_profile", lambda: None) - skills.publish_local_skill( + monkeypatch.setattr(skill_candidates, 'builtin_candidates', lambda: []) + monkeypatch.setattr(skill_account, 'current_account_profile', lambda: None) + skill_publication.publish_local_skill( "example", files={"SKILL.md": "v1"}, content_hash=skill_content_hash("v1", {}) ) frozen = runtime.prepare("run-audit", "u", skill_ids=["example"]) - skills.publish_local_skill( + skill_publication.publish_local_skill( "example", files={"SKILL.md": "v2"}, content_hash=skill_content_hash("v2", {}) ) ref = _skill_refs(["example"], "run-audit")[0] diff --git a/src/backend/tests/capabilities/test_runtime_publication_recovery.py b/src/backend/tests/features/desktop/capabilities/test_runtime_publication_recovery.py similarity index 70% rename from src/backend/tests/capabilities/test_runtime_publication_recovery.py rename to src/backend/tests/features/desktop/capabilities/test_runtime_publication_recovery.py index 64bde8bf..3cdc9070 100644 --- a/src/backend/tests/capabilities/test_runtime_publication_recovery.py +++ b/src/backend/tests/features/desktop/capabilities/test_runtime_publication_recovery.py @@ -1,25 +1,25 @@ +from features.desktop.bridge import client as bridge_client +from features.desktop.bridge import gateway as bridge_gateway +from features.desktop.bridge import identity as bridge_identity +from features.desktop.capabilities import skill_account +from features.desktop.capabilities import skill_candidates +from features.desktop.capabilities import skill_hashes +from features.desktop.capabilities import skill_publication +from features.desktop.capabilities import skills from tests.sandbox.runner_client import run_runner """Regressions for desktop identity, explicit bindings and immutable runs.""" import pytest -from core.capabilities import registry, skills -from core.capabilities.ref import profile_id -from core.services import desktop_cloud_bridge as bridge -from core.services import desktop_cloud_skills as cloud_skills -from core.services.desktop_capability_protocol import build_skill_manifest +from features.desktop.capabilities import registry, skills +from core.capabilities.reference import profile_id +from features.desktop.bridge import client as bridge +from features.desktop.bridge import skills as cloud_skills +from core.capabilities.manifest_protocol import build_skill_manifest -from tests.capabilities.runtime_recovery_support import ( - _ordered_skill_manifest, - state, - _zip, - _intent, - durable_index, - _shell_id, - _state_v2, -) +from tests.features.desktop.capabilities.runtime_recovery_support import _ordered_skill_manifest, state, _zip, _intent, durable_index, _shell_id, _state_v2 @pytest.mark.asyncio @@ -27,19 +27,19 @@ async def test_concurrent_runs_in_same_chat_cannot_repoint_running_script( durable_index, caps_root, tmp_path, monkeypatch ): import asyncio - from core.capabilities import runtime - from core.services.desktop_capability_protocol import skill_content_hash + from features.desktop.capabilities import runtime + from core.capabilities.manifest_protocol import skill_content_hash from core.sandbox.runner import server - monkeypatch.setattr(skills, "builtin_candidates", lambda: []) - monkeypatch.setattr(skills, "current_account_profile", lambda: None) + monkeypatch.setattr(skill_candidates, 'builtin_candidates', lambda: []) + monkeypatch.setattr(skill_account, 'current_account_profile', lambda: None) monkeypatch.setenv("DEPLOY_PROFILE", "local") monkeypatch.setattr(server, "WORKSPACE_ROOT", str(tmp_path / "workspace")) - skills.publish_local_skill( + skill_publication.publish_local_skill( "example", files={"SKILL.md": "v1"}, content_hash=skill_content_hash("v1", {}) ) old = runtime.prepare("concurrent-old", "u", skill_ids=["example"]) - skills.publish_local_skill( + skill_publication.publish_local_skill( "example", files={"SKILL.md": "v2"}, content_hash=skill_content_hash("v2", {}) ) new = runtime.prepare("concurrent-new", "u", skill_ids=["example"]) @@ -72,7 +72,7 @@ async def test_concurrent_runs_in_same_chat_cannot_repoint_running_script( def test_multiple_failed_updates_keep_actual_resolved_hash(index_db, caps_root, monkeypatch): - from core.services.desktop_capability_protocol import skill_content_hash + from core.capabilities.manifest_protocol import skill_content_hash st, inst = _intent(monkeypatch) monkeypatch.setattr(cloud_skills, "_download", lambda *_: _zip({"SKILL.md": "v1"})) @@ -86,7 +86,7 @@ def test_multiple_failed_updates_keep_actual_resolved_hash(index_db, caps_root, def test_device_disable_survives_cloud_manifest_refresh(index_db, caps_root, monkeypatch): st, inst = _intent(monkeypatch) registry.set_enabled(inst.install_id, False) - from core.services.desktop_capability_protocol import skill_content_hash + from core.capabilities.manifest_protocol import skill_content_hash manifest = build_skill_manifest( [ @@ -110,10 +110,10 @@ def test_device_disable_survives_cloud_manifest_refresh(index_db, caps_root, mon def test_mcp_public_environment_is_frozen_while_credentials_rotate( durable_index, caps_root, monkeypatch ): - from core.capabilities import runtime + from features.desktop.capabilities import runtime - monkeypatch.setattr(skills, "builtin_candidates", lambda: []) - monkeypatch.setattr(skills, "current_account_profile", lambda: None) + monkeypatch.setattr(skill_candidates, 'builtin_candidates', lambda: []) + monkeypatch.setattr(skill_account, 'current_account_profile', lambda: None) run = runtime.prepare("env-run", "u", skill_ids=[]) runtime.bind_mcp( run, {"local": {"command": "node", "env": {"MODE": "safe", "API_KEY": "old"}}}, None @@ -133,9 +133,9 @@ def test_mcp_public_environment_is_frozen_while_credentials_rotate( def test_full_package_hash_never_skips_hidden_or_executable_bytes( index_db, caps_root, monkeypatch, extra ): - from core.capabilities import store - from core.capabilities.errors import IntegrityFailed - from core.capabilities.paths import revision_for_hash + from features.desktop.capabilities import store + from core.harness.capability_errors import IntegrityFailed + from features.desktop.capabilities.paths import revision_for_hash st, inst = _intent(monkeypatch) store.write_from_files( @@ -150,8 +150,8 @@ def test_full_package_hash_never_skips_hidden_or_executable_bytes( def test_full_package_hash_rejects_limit_instead_of_skipping(tmp_path, monkeypatch): - from core.capabilities import archive - from core.capabilities.errors import IntegrityFailed + from core.capabilities import archive as archive + from core.harness.capability_errors import IntegrityFailed root = tmp_path / "package" root.mkdir() @@ -159,15 +159,15 @@ def test_full_package_hash_rejects_limit_instead_of_skipping(tmp_path, monkeypat (root / "extra").write_bytes(b"123456789") monkeypatch.setattr(archive, "MAX_MEMBER_BYTES", 8) with pytest.raises(IntegrityFailed): - skills.skill_dir_hash(root, fresh=True) + skill_hashes.skill_dir_hash(root, fresh=True) def test_expired_cloud_token_keeps_local_copy_identity_until_logout( durable_index, caps_root, monkeypatch ): from core.db.models import UserShadow - from core.agent_skills.backends.capability_store import CapabilityStoreBackend - from core.services.desktop_capability_protocol import skill_content_hash + from features.desktop.skill_backend import CapabilityStoreBackend + from core.capabilities.manifest_protocol import skill_content_hash with durable_index() as db: UserShadow.__table__.create(db.get_bind(), checkfirst=True) @@ -175,24 +175,24 @@ def test_expired_cloud_token_keeps_local_copy_identity_until_logout( db.commit() now = [100.0] monkeypatch.setattr(bridge.time, "time", lambda: now[0]) - monkeypatch.setattr(bridge, "_purge_persisted_state", lambda: None) - monkeypatch.setattr(bridge, "_rebuild_identity_views", lambda: None) - monkeypatch.setattr(bridge, "_refresh_manifest_async", lambda **kwargs: None) - skills.publish_local_skill( + monkeypatch.setattr(bridge_identity, 'purge_persisted_state', lambda: None) + monkeypatch.setattr(bridge_identity, 'rebuild_identity_views', lambda: None) + monkeypatch.setattr(bridge_client, 'refresh_manifest_async', lambda **kwargs: None) + skill_publication.publish_local_skill( "copy", files={"SKILL.md": "copy"}, content_hash=skill_content_hash("copy", {}), owner_user_id="local-a", ) st = _state_v2("a") - bridge.set_state(st["cloud_base"], st["token"], 1) + bridge_client.set_state(st["cloud_base"], st["token"], 1) now[0] += 2 - assert bridge.get_state() is None - assert skills.current_account_profile() is None - assert skills.current_local_user_id() == "local-a" + assert bridge_identity.get_state() is None + assert skill_account.current_account_profile() is None + assert skill_account.current_local_user_id() == "local-a" assert CapabilityStoreBackend(local=True).exists("copy") - bridge.clear_state() - assert skills.current_local_user_id() is None + bridge_identity.clear_state() + assert skill_account.current_local_user_id() is None assert not CapabilityStoreBackend(local=True).exists("copy") @@ -202,12 +202,12 @@ async def test_dedicated_agent_factory_reaches_config_after_freezing_definition( ): from core.llm import factory as agent_factory import prompts.prompt_config as prompt_config - from core.capabilities.agents import AgentDefinition - from core.capabilities import runtime + from features.desktop.capabilities.agents import AgentDefinition + from features.desktop.capabilities import runtime - monkeypatch.setattr(skills, "current_account_profile", lambda: None) + monkeypatch.setattr(skill_account, 'current_account_profile', lambda: None) monkeypatch.setattr( - "core.llm.tool_permissions.preferences.resolve_approval_mode", lambda *args, **kwargs: "auto" + "core.sandbox.permissions.approval.resolve_approval_mode", lambda *args, **kwargs: "auto" ) class ConfigurationReached(Exception): @@ -229,8 +229,8 @@ class ConfigurationReached(Exception): def test_name_preferences_are_isolated_between_accounts(index_db, monkeypatch, tmp_path): - from core.capabilities import connectors - from core.capabilities.resolver import Candidate + from features.desktop.capabilities import connectors + from features.desktop.capabilities.resolver import Candidate def candidate(iid): path = tmp_path / iid.rsplit(":", 1)[-1] @@ -246,7 +246,7 @@ def candidate(iid): ) a, b = candidate("skill:local:a"), candidate("skill:local:b") - monkeypatch.setattr(skills, "candidates", lambda uid: [a] if uid == "a" else [b]) + monkeypatch.setattr(skill_candidates, 'candidates', lambda uid: [a] if uid == "a" else [b]) registry.set_preference("skill", "same", a.install_id, chosen_by="a") assert skills.resolve_for_user("b").chosen["same"].install_id == b.install_id registry.set_preference("skill", "same", b.install_id, chosen_by="b") @@ -255,13 +255,13 @@ def candidate(iid): assert registry.clear_preference("skill", "same", user_id="b") assert registry.preferences("skill", user_id="a") == {"same": a.install_id} registry.set_preference("mcp", "search", "mcp:local:a", chosen_by="a") - monkeypatch.setattr(skills, "current_local_user_id", lambda: "b") + monkeypatch.setattr(skill_account, 'current_local_user_id', lambda: "b") cands = connectors.json_candidates({"search": {"enabled": True}}) assert connectors.resolve_bindings(cands).chosen["search"].install_id == "mcp:local-json:search" def test_legacy_name_preference_stays_with_its_owner(index_db): - from core.db.models import DeviceCapabilityNamePreference + from features.desktop.models import DeviceCapabilityNamePreference with registry._session() as db: db.add( @@ -283,8 +283,8 @@ def test_legacy_name_preference_stays_with_its_owner(index_db): def test_enabled_cloud_connector_without_schema_is_unusable(index_db, monkeypatch): - from core.capabilities import connectors - from core.capabilities.errors import PackageMissing + from features.desktop.capabilities import connectors + from core.harness.capability_errors import PackageMissing st = state("a") ctx = { @@ -299,12 +299,12 @@ def test_enabled_cloud_connector_without_schema_is_unusable(index_db, monkeypatc assert not empty.usable and empty.state == "schema_empty" disabled = connectors.cloud_candidates(ctx["profile"], ctx["servers"], {"empty": False})[0] assert not disabled.usable and disabled.state == "disabled" - monkeypatch.setattr(bridge, "_bridge_context", lambda: ctx) - monkeypatch.setattr(bridge, "_local_server_ids", lambda: set()) - monkeypatch.setattr(bridge, "_mcp_json_local_declarations", lambda: {}) - monkeypatch.setattr(bridge, "_mcp_json_local_configs", lambda: {}) + monkeypatch.setattr(bridge_gateway, 'bridge_context', lambda: ctx) + monkeypatch.setattr(bridge_gateway, 'local_server_ids', lambda: set()) + monkeypatch.setattr(bridge_gateway, 'mcp_json_local_declarations', lambda: {}) + monkeypatch.setattr(bridge_gateway, 'mcp_json_local_configs', lambda: {}) with pytest.raises(PackageMissing): - bridge.cloud_gateway_mcp_configs(["empty"]) + bridge_gateway.cloud_gateway_mcp_configs(["empty"]) unavailable = {} - assert bridge.cloud_gateway_mcp_configs(["empty"], unavailable_out=unavailable) == {} + assert bridge_gateway.cloud_gateway_mcp_configs(["empty"], unavailable_out=unavailable) == {} assert unavailable == {"empty": PackageMissing.code} diff --git a/src/backend/tests/capabilities/test_runtime_recovery.py b/src/backend/tests/features/desktop/capabilities/test_runtime_recovery.py similarity index 58% rename from src/backend/tests/capabilities/test_runtime_recovery.py rename to src/backend/tests/features/desktop/capabilities/test_runtime_recovery.py index 7b7ee99f..5ddb7f29 100644 --- a/src/backend/tests/capabilities/test_runtime_recovery.py +++ b/src/backend/tests/features/desktop/capabilities/test_runtime_recovery.py @@ -1,3 +1,7 @@ +from features.desktop.bridge import gateway as bridge_gateway +from features.desktop.bridge import identity as bridge_identity +from features.desktop.capabilities import skill_publication +from features.desktop.capabilities import skill_views from tests.sandbox.runner_client import run_runner """Regressions for desktop identity, explicit bindings and immutable runs.""" @@ -6,23 +10,15 @@ import json import pytest -from core.capabilities import registry, skills, manifest_order -from core.capabilities.ref import cloud_ref, profile_id -from core.services import desktop_cloud_bridge as bridge -from core.services import desktop_cloud_skills as cloud_skills -from core.services import desktop_cloud_bundles as bundles -from core.services.desktop_capability_protocol import build_skill_manifest, build_entity_manifest +from features.desktop.capabilities import registry, skills, manifest_order +from core.capabilities.reference import cloud_ref, profile_id +from features.desktop.bridge import client as bridge +from features.desktop.bridge import skills as cloud_skills +from features.desktop.bridge import bundles as bundles +from core.capabilities.manifest_protocol import build_skill_manifest, build_entity_manifest -from tests.capabilities.runtime_recovery_support import ( - _ordered_skill_manifest, - state, - _zip, - _intent, - durable_index, - _shell_id, - _state_v2, -) +from tests.features.desktop.capabilities.runtime_recovery_support import _ordered_skill_manifest, state, _zip, _intent, durable_index, _shell_id, _state_v2 def test_explicit_local_binding_does_not_get_cloud_config(index_db, monkeypatch): @@ -36,19 +32,19 @@ def test_explicit_local_binding_does_not_get_cloud_config(index_db, monkeypatch) {"server_id": "search", "component": "search", "tools": [], "schema_hash": "hash"} ], } - monkeypatch.setattr(bridge, "_bridge_context", lambda: ctx) - monkeypatch.setattr(bridge, "_local_server_ids", lambda: {"search"}) - monkeypatch.setattr(bridge, "_mcp_json_local_declarations", lambda: {}) - monkeypatch.setattr(bridge, "_mcp_json_local_configs", lambda: {}) + monkeypatch.setattr(bridge_gateway, 'bridge_context', lambda: ctx) + monkeypatch.setattr(bridge_gateway, 'local_server_ids', lambda: {"search"}) + monkeypatch.setattr(bridge_gateway, 'mcp_json_local_declarations', lambda: {}) + monkeypatch.setattr(bridge_gateway, 'mcp_json_local_configs', lambda: {}) registry.set_preference("mcp", "search", "mcp:local:search") - assert bridge.apply_to_enabled_mcp_ids(["search"]) == ["search"] - assert "search" not in bridge.cloud_gateway_mcp_configs() + assert bridge_gateway.apply_to_enabled_mcp_ids(["search"]) == ["search"] + assert "search" not in bridge_gateway.cloud_gateway_mcp_configs() def test_stale_skill_response_cannot_repopulate_switched_account(index_db, caps_root, monkeypatch): a, b = state("a"), state("b") current = [a] - monkeypatch.setattr(bridge, "get_state", lambda: current[0]) + monkeypatch.setattr(bridge_identity, 'get_state', lambda: current[0]) manifest = build_skill_manifest([]) def fetch(_): @@ -57,8 +53,8 @@ def fetch(_): return manifest monkeypatch.setattr(cloud_skills, "_fetch_manifest", fetch) - monkeypatch.setattr(skills, "rebuild_views", lambda _: {}) - monkeypatch.setattr("core.agent_skills.cache_refresh.refresh_skill_caches", lambda: None) + monkeypatch.setattr(skill_views, 'rebuild_views', lambda _: {}) + monkeypatch.setattr('core.capabilities.skills.cache_refresh.refresh_skill_caches', lambda: None) cloud_skills.sync_blocking(a) assert cloud_skills.status()["revision"] == "" assert registry.list_installations() == [] @@ -67,7 +63,7 @@ def fetch(_): def test_stale_agent_response_cannot_repopulate_switched_account(index_db, caps_root, monkeypatch): a, b = state("a"), state("b") current = [a] - monkeypatch.setattr(bridge, "get_state", lambda: current[0]) + monkeypatch.setattr(bridge_identity, 'get_state', lambda: current[0]) manifest = build_entity_manifest("agent", []) def fetch(*_): @@ -81,18 +77,18 @@ def fetch(*_): def test_json_local_connector_is_enabled_without_cloud(index_db, monkeypatch): - monkeypatch.setattr(bridge, "_bridge_context", lambda: None) - monkeypatch.setattr(bridge, "_local_server_ids", lambda: set()) + monkeypatch.setattr(bridge_gateway, 'bridge_context', lambda: None) + monkeypatch.setattr(bridge_gateway, 'local_server_ids', lambda: set()) monkeypatch.setattr( - bridge, "_mcp_json_local_declarations", lambda: {"my-files": {"enabled": True}} + bridge_gateway, 'mcp_json_local_declarations', lambda: {"my-files": {"enabled": True}} ) - assert bridge.apply_to_enabled_mcp_ids([]) == ["my-files"] + assert bridge_gateway.apply_to_enabled_mcp_ids([]) == ["my-files"] def test_prepare_revalidates_existing_revision(index_db, caps_root, monkeypatch): - from core.capabilities import store - from core.capabilities.errors import IntegrityFailed - from core.capabilities.paths import revision_for_hash + from features.desktop.capabilities import store + from core.harness.capability_errors import IntegrityFailed + from features.desktop.capabilities.paths import revision_for_hash st, inst = _intent(monkeypatch) store.write_from_files( @@ -124,13 +120,13 @@ def test_prepare_failure_retains_previous_ready_revision(index_db, caps_root, mo def test_switch_during_download_cannot_publish_old_account(index_db, caps_root, monkeypatch): - from core.capabilities import store - from core.capabilities.errors import CloudUnavailable + from features.desktop.capabilities import store + from core.harness.capability_errors import CloudUnavailable st, inst = _intent(monkeypatch) def download(*_): - monkeypatch.setattr(bridge, "get_state", lambda: state("b")) + monkeypatch.setattr(bridge_identity, 'get_state', lambda: state("b")) return _zip({"SKILL.md": "v1"}) monkeypatch.setattr(cloud_skills, "_download", download) @@ -141,9 +137,9 @@ def download(*_): def test_publishing_new_local_revision_preserves_old_run_target(index_db, caps_root): - from core.capabilities import store + from features.desktop.capabilities import store - old = skills.publish_local_skill("example", files={"SKILL.md": "v1"}, content_hash="a" * 64) - skills.publish_local_skill("example", files={"SKILL.md": "v2"}, content_hash="b" * 64) + old = skill_publication.publish_local_skill("example", files={"SKILL.md": "v1"}, content_hash="a" * 64) + skill_publication.publish_local_skill("example", files={"SKILL.md": "v2"}, content_hash="b" * 64) assert old.entry_file.read_text() == "v1" assert len(store.revisions("skill", "local", "example")) == 2 diff --git a/src/backend/tests/capabilities/test_runtime_scope_consumers.py b/src/backend/tests/features/desktop/capabilities/test_runtime_scope_consumers.py similarity index 87% rename from src/backend/tests/capabilities/test_runtime_scope_consumers.py rename to src/backend/tests/features/desktop/capabilities/test_runtime_scope_consumers.py index 7f6f84f8..ac5bd723 100644 --- a/src/backend/tests/capabilities/test_runtime_scope_consumers.py +++ b/src/backend/tests/features/desktop/capabilities/test_runtime_scope_consumers.py @@ -1,4 +1,7 @@ """Child capability scopes retain their own execution view and audit references.""" +from features.desktop.capabilities import skill_account +from features.desktop.capabilities import skill_candidates +from features.desktop.capabilities import skill_publication import json from pathlib import Path @@ -8,9 +11,9 @@ import httpx import pytest -from core.capabilities import runtime -from core.capabilities.errors import IntegrityFailed -from core.evolution import runtime_binding as audit +from features.desktop.capabilities import runtime +from core.harness.capability_errors import IntegrityFailed +from core.evolution.cycle import runtime_binding as audit from core.evolution.contract import ASSET_SKILL, AssetRef from core.sandbox.protocol import ProcessRequest from core.sandbox.providers.script_runner.provider import ScriptRunnerProvider @@ -22,7 +25,7 @@ async def test_runner_uses_scope_for_view_and_completion_checkpoint(monkeypatch, calls = [] prepared = SimpleNamespace(run_id="run-1", scope_id=scope) view_key = "root-view" if not scope else "child-view" - monkeypatch.setattr("core.capabilities.paths.capabilities_enabled", lambda: True) + monkeypatch.setattr('features.desktop.capabilities.paths.capabilities_enabled', lambda: True) def view(run_id, user_id, scope_id=""): calls.append(("view", run_id, user_id, scope_id)) @@ -33,7 +36,8 @@ def get(run_id, scope_id=""): assert scope_id == scope return prepared - def validate(run, *, user_id): + def validate(run, *, user_id, execution_plane=None, only_skill=None): + assert execution_plane == "local" and only_skill is None assert run is prepared calls.append(("validate", run.scope_id, user_id)) @@ -85,7 +89,7 @@ def respond(request): @pytest.mark.asyncio async def test_missing_child_snapshot_does_not_fall_back_to_root(monkeypatch): - monkeypatch.setattr("core.capabilities.paths.capabilities_enabled", lambda: True) + monkeypatch.setattr('features.desktop.capabilities.paths.capabilities_enabled', lambda: True) def view(run_id, user_id, scope_id=""): return None if scope_id else Path("/synthetic/root/skills") @@ -132,12 +136,14 @@ async def start_process(req, yield_time_ms=10000): ) class Toolkit: - def register_tool_function(self, fn, **kwargs): + def append(self, tool): + fn = tool._func + kwargs = {"func_description": tool.description, "permission": getattr(tool, "dispatch_spec", None)} self.fn = fn toolkit = Toolkit() loader = SimpleNamespace(capability_run=SimpleNamespace(run_id="run-1", scope_id="step:two")) - sandbox_tool.register_bash(toolkit, loader=loader, loaded_skill_ids=set(), chat_id="chat-1", user_id="user-1") + sandbox_tool.register_bash(toolkit, loader=loader, chat_id="chat-1", user_id="user-1") await toolkit.fn(command="echo ok") assert captured[0].capability_run_id == "run-1" assert captured[0].capability_scope == "step:two" @@ -145,9 +151,9 @@ def register_tool_function(self, fn, **kwargs): def test_skill_audit_looks_up_the_exact_scope(monkeypatch): - monkeypatch.setattr("core.capabilities.paths.capabilities_enabled", lambda: True) + monkeypatch.setattr('features.desktop.capabilities.paths.capabilities_enabled', lambda: True) monkeypatch.setattr( - "core.agent_skills.loader.get_skill_loader", + 'core.capabilities.skills.loader_factory.get_skill_loader', lambda: SimpleNamespace(load_all_metadata=lambda: {}), ) seen = [] @@ -214,7 +220,7 @@ def test_child_audit_rebind_and_clear_preserve_parent_bundle(monkeypatch): @pytest.mark.asyncio async def test_child_snapshot_disappearing_after_execution_fails_closed(monkeypatch): - monkeypatch.setattr("core.capabilities.paths.capabilities_enabled", lambda: True) + monkeypatch.setattr('features.desktop.capabilities.paths.capabilities_enabled', lambda: True) monkeypatch.setattr( runtime, "view_for_execution", @@ -266,21 +272,21 @@ def get(run_id, scope_id=""): async def test_real_scoped_snapshots_reach_runner_and_audit_without_root_drift( index_db, caps_root, monkeypatch ): - from core.capabilities import skills + from features.desktop.capabilities import skills from core.db.models import ContentBlock - from core.services.desktop_capability_protocol import skill_content_hash + from core.capabilities.manifest_protocol import skill_content_hash with index_db() as db: ContentBlock.__table__.create(db.get_bind(), checkfirst=True) - monkeypatch.setattr(skills, "builtin_candidates", lambda: []) - monkeypatch.setattr(skills, "current_account_profile", lambda: None) + monkeypatch.setattr(skill_candidates, 'builtin_candidates', lambda: []) + monkeypatch.setattr(skill_account, 'current_account_profile', lambda: None) monkeypatch.setattr( - "core.agent_skills.loader.get_skill_loader", + 'core.capabilities.skills.loader_factory.get_skill_loader', lambda: SimpleNamespace(load_all_metadata=lambda: {}), ) def publish(text): - return skills.publish_local_skill( + return skill_publication.publish_local_skill( "scope-marker", files={"SKILL.md": text}, content_hash=skill_content_hash(text, {}), diff --git a/src/backend/tests/capabilities/test_runtime_scopes.py b/src/backend/tests/features/desktop/capabilities/test_runtime_scopes.py similarity index 82% rename from src/backend/tests/capabilities/test_runtime_scopes.py rename to src/backend/tests/features/desktop/capabilities/test_runtime_scopes.py index 97f0d724..dccd8f5f 100644 --- a/src/backend/tests/capabilities/test_runtime_scopes.py +++ b/src/backend/tests/features/desktop/capabilities/test_runtime_scopes.py @@ -1,23 +1,29 @@ """Read-only repository probe, synthetic temporary capability data only.""" +from features.desktop.bridge import identity as bridge_identity +from features.desktop.capabilities import skill_account +from features.desktop.capabilities import skill_candidates +from features.desktop.capabilities import skill_publication +import core.runs.tool_effect_recovery as runs_tool_effect_recovery +import orchestration.plans.mode as mode import pytest -from tests.capabilities.test_runtime_recovery import durable_index, state, _zip -from core.capabilities import runtime, registry, skills -from core.capabilities.agents import AgentDefinition -from core.capabilities.errors import PackageMissing -from core.capabilities.ref import profile_id, cloud_ref -from core.services import desktop_cloud_bridge as bridge -from core.services import desktop_cloud_skills as cloud_skills -from core.services.desktop_capability_protocol import skill_content_hash +from features.desktop.capabilities import registry, runtime, skills +from features.desktop.capabilities.agents import AgentDefinition +from core.harness.capability_errors import PackageMissing +from core.capabilities.reference import cloud_ref, profile_id +from features.desktop.bridge import client as bridge +from features.desktop.bridge import skills as cloud_skills +from core.capabilities.manifest_protocol import skill_content_hash +from tests.features.desktop.capabilities.test_runtime_recovery import _zip, durable_index, state @pytest.fixture def setup(durable_index, caps_root, monkeypatch): st = state("a") - monkeypatch.setattr(bridge, "get_state", lambda: st) - monkeypatch.setattr(bridge, "ensure_current_authorization", lambda *_: None) - monkeypatch.setattr(skills, "current_local_user_id", lambda: "owner") - monkeypatch.setattr(skills, "builtin_candidates", lambda: []) + monkeypatch.setattr(bridge_identity, 'get_state', lambda: st) + monkeypatch.setattr(bridge_identity, 'ensure_current_authorization', lambda *_: None) + monkeypatch.setattr(skill_account, 'current_local_user_id', lambda: "owner") + monkeypatch.setattr(skill_candidates, 'builtin_candidates', lambda: []) for name in ("cloud-a", "cloud-b"): body = "---\nname: " + name + "\ndescription: synthetic\n---\n" + name inst = registry.upsert( @@ -62,17 +68,18 @@ def test_same_scope_replay_retains_revision_and_rechecks_identity(setup, monkeyp assert replay.to_dict() == saved with pytest.raises(PackageMissing): runtime.prepare("root-run", "owner", skill_ids=["cloud-b"], scope_id=scope) - from core.capabilities.errors import PermissionDenied, CloudUnavailable + from core.harness.capability_errors import CloudUnavailable, PermissionDenied with pytest.raises(PermissionDenied): runtime.prepare("root-run", "other", skill_ids=["cloud-a"], scope_id=scope) - monkeypatch.setattr(bridge, "get_state", lambda: state("b")) + monkeypatch.setattr(bridge_identity, 'get_state', lambda: state("b")) with pytest.raises((PermissionDenied, CloudUnavailable)): runtime.prepare("root-run", "owner", skill_ids=["cloud-a"], scope_id=scope) def test_agent_snapshots_pin_per_scope_and_legacy_root_key_is_unchanged(setup): import hashlib + from core.db.models import ContentBlock a = AgentDefinition(agent_id="same-agent", name="A", system_prompt="v1") @@ -99,7 +106,7 @@ def test_agent_snapshots_pin_per_scope_and_legacy_root_key_is_unchanged(setup): def test_tool_scope_recovery_allows_proven_root_and_rejects_children_and_unknown(setup): - from core.capabilities.errors import IntegrityFailed + from core.harness.capability_errors import IntegrityFailed root = runtime.prepare("root-run", "owner", skill_ids=[]) child = runtime.prepare( @@ -124,7 +131,7 @@ def test_tool_scope_recovery_allows_proven_root_and_rejects_children_and_unknown def test_same_scope_frozen_skill_bytes_survive_local_update(setup): body = "---\nname: local-a\ndescription: synthetic\n---\nv1" - skills.publish_local_skill( + skill_publication.publish_local_skill( "local-a", files={"SKILL.md": body}, content_hash=skill_content_hash(body, {}), @@ -134,7 +141,7 @@ def test_same_scope_frozen_skill_bytes_survive_local_update(setup): first = runtime.prepare("root-run", "owner", skill_ids=["local-a"], scope_id=scope) old_revision = first.bindings["local-a"]["revision"] newer = body.replace("v1", "v2") - skills.publish_local_skill( + skill_publication.publish_local_skill( "local-a", files={"SKILL.md": newer}, content_hash=skill_content_hash(newer, {}), @@ -149,8 +156,9 @@ def test_same_scope_frozen_skill_bytes_survive_local_update(setup): async def test_actual_plan_steps_use_stable_independent_scopes(setup, monkeypatch): from contextlib import nullcontext from types import SimpleNamespace - from orchestration.subagents import plan_mode - from core.llm.middlewares import CURRENT_RUN_BINDING + + import orchestration.plans.mode as plan_mode + from core.harness.tool_call_context import CURRENT_RUN_BINDING steps = [ SimpleNamespace( @@ -208,25 +216,25 @@ async def create(**kw): captured.append((kw["run_id"], kw["capability_scope"], tuple(kw["enabled_skill_ids"]))) return SimpleNamespace(state=SimpleNamespace(context=[]), reply=reply), [] - monkeypatch.setattr(plan_mode, "PlanService", Service) - monkeypatch.setattr(plan_mode, "create_agent_executor", create) - monkeypatch.setattr(plan_mode, "_prepare_history", history) + monkeypatch.setattr(mode, "PlanService", Service) + monkeypatch.setattr(mode, "create_agent_executor", create) + monkeypatch.setattr(mode, "_prepare_history", history) monkeypatch.setattr( - plan_mode, "_resolve_plugin_capabilities", lambda db, uid, sids, mids: (sids, mids, []) + mode, "_resolve_plugin_capabilities", lambda db, uid, sids, mids: (sids, mids, []) ) - monkeypatch.setattr(plan_mode, "_load_visible_agents", lambda *a: []) - monkeypatch.setattr(plan_mode, "_build_file_context", lambda *a, **kw: "") - monkeypatch.setattr(plan_mode, "_build_plan_header_lines", lambda *a: ([], [])) - monkeypatch.setattr(plan_mode, "_build_step_instruction", lambda step, *a, **kw: step.step_id) - monkeypatch.setattr(plan_mode, "is_run_cancelled", lambda *a: False) + monkeypatch.setattr(mode, "_load_visible_agents", lambda *a: []) + monkeypatch.setattr(mode, "_build_file_context", lambda *a, **kw: "") + monkeypatch.setattr(mode, "_build_plan_header_lines", lambda *a: ([], [])) + monkeypatch.setattr(mode, "_build_step_instruction", lambda step, *a, **kw: step.step_id) + monkeypatch.setattr(mode, "is_run_cancelled", lambda *a: False) monkeypatch.setattr(plan_mode.log_writer, "start_subagent_log", noop) monkeypatch.setattr(plan_mode.log_writer, "finish_subagent_log", noop) - monkeypatch.setattr(plan_mode.log_writer, "subagent_scope", lambda *a, **kw: nullcontext()) + monkeypatch.setattr("core.harness.log_context.subagent_scope", lambda *a, **kw: nullcontext()) monkeypatch.setattr( - "core.services.project_scope.build_project_ctx_from_chat_id", lambda *a: None + 'core.projects.edition.build_project_ctx_from_chat_id', lambda *a: None ) monkeypatch.setattr( - "core.services.ontology_service.build_user_ontology_runtime", lambda **kw: (False, {}) + 'core.ontology.runtime_policy.build_user_ontology_runtime', lambda **kw: (False, {}) ) binding = CURRENT_RUN_BINDING.set(("root-run", "ledger-owner")) try: @@ -250,9 +258,9 @@ async def create(**kw): def test_subagent_scope_uses_durable_call_not_presentation_id(setup): - from core.llm.subagent_tool import _child_capability_runtime - from core.llm.middlewares import CURRENT_TOOL_CALL_ID - from core.capabilities.errors import IntegrityFailed + from core.harness.capability_errors import IntegrityFailed + from core.harness.tool_call_context import CURRENT_TOOL_CALL_ID + from core.llm.subagents.tool import _child_capability_runtime parent = { "run_id": "root-run", @@ -280,13 +288,13 @@ async def test_actual_factory_pins_scope_without_changing_durable_binding( setup, monkeypatch, durable_index ): from core.db.engine import Base - from core.services import model_config + from core.models import config as model_config Base.metadata.create_all(durable_index.kw["bind"]) monkeypatch.setattr(model_config, "SessionLocal", durable_index) monkeypatch.setattr(model_config.ModelConfigService, "_instance", None) from core.llm.factory import create_agent_executor - from core.llm.middlewares import CURRENT_RUN_BINDING + from core.harness.tool_call_context import CURRENT_RUN_BINDING captured = [] @@ -318,10 +326,11 @@ def pin(rid, uid, definition, **kw): @pytest.mark.asyncio async def test_recovery_adapter_executes_proven_root_but_never_rebuilds_child(setup, monkeypatch): from types import SimpleNamespace - from agentscope.tool import ToolResponse + + import core.runs.tool_effect_recovery as tool_effect_recovery from agentscope.message import TextBlock - from orchestration import tool_effect_recovery - from core.services.tool_effect_ledger import ToolEffectError + from agentscope.tool import ToolResponse + from core.runs.tool_effects.errors import ToolEffectError root = runtime.prepare("root-run", "owner", skill_ids=[]) child = runtime.prepare("root-run", "owner", skill_ids=["cloud-a"], scope_id="child-scope") @@ -358,7 +367,7 @@ async def create(**kw): [], ) - monkeypatch.setattr(tool_effect_recovery, "SessionLocal", Db) + monkeypatch.setattr(runs_tool_effect_recovery, "SessionLocal", Db) monkeypatch.setattr("core.llm.factory.create_agent_executor", create) intent = SimpleNamespace( run_id="root-run", @@ -381,11 +390,12 @@ async def create(**kw): @pytest.mark.asyncio async def test_registered_child_dispatch_has_scope_even_without_sse(setup, monkeypatch): from types import SimpleNamespace - from core.llm import subagent_tool - from core.llm.middlewares import CURRENT_TOOL_CALL_ID + + from core.llm.subagents import tool as subagent_tool + from core.harness.tool_call_context import CURRENT_TOOL_CALL_ID registered = {} - toolkit = SimpleNamespace(register_tool_function=lambda fn, **kw: registered.update(call=fn)) + toolkit = SimpleNamespace(append=lambda tool: registered.update(call=tool._func)) captured = [] def worker(*args): @@ -424,7 +434,7 @@ async def noop(*a, **kw): def test_same_ready_scope_never_replaces_its_frozen_dependency_report(setup): for name in ("local-a", "local-b"): body = "---\nname: " + name + "\ndescription: synthetic\n---\n" + name - skills.publish_local_skill( + skill_publication.publish_local_skill( name, files={"SKILL.md": body}, content_hash=skill_content_hash(body, {}), diff --git a/src/backend/tests/capabilities/test_runtime_selection_recovery.py b/src/backend/tests/features/desktop/capabilities/test_runtime_selection_recovery.py similarity index 66% rename from src/backend/tests/capabilities/test_runtime_selection_recovery.py rename to src/backend/tests/features/desktop/capabilities/test_runtime_selection_recovery.py index 1aa2b6ec..35353919 100644 --- a/src/backend/tests/capabilities/test_runtime_selection_recovery.py +++ b/src/backend/tests/features/desktop/capabilities/test_runtime_selection_recovery.py @@ -1,15 +1,20 @@ """Regressions for desktop identity, explicit bindings and immutable runs.""" +from features.desktop.bridge import identity as bridge_identity +from features.desktop.capabilities import skill_account +from features.desktop.capabilities import skill_candidates +from features.desktop.capabilities import skill_publication +from features.desktop.capabilities import skills import pytest -from core.capabilities import registry, skills -from core.services import desktop_cloud_bridge as bridge -from core.services import desktop_cloud_skills as cloud_skills -from core.services import desktop_cloud_bundles as bundles +from features.desktop.capabilities import registry, skills +from features.desktop.bridge import client as bridge +from features.desktop.bridge import skills as cloud_skills +from features.desktop.bridge import bundles as bundles -from tests.capabilities.runtime_recovery_support import state, _zip, _intent, durable_index +from tests.features.desktop.capabilities.runtime_recovery_support import state, _zip, _intent, durable_index @pytest.mark.parametrize("method", ["get_by_id", "get_raw_by_id"]) @@ -17,13 +22,13 @@ def test_cloud_agent_details_cannot_cross_local_user_identity( index_db, caps_root, monkeypatch, method ): from types import SimpleNamespace - from core.capabilities import agents - from core.services.user_agent_base import UserAgentBaseService + from features.desktop.capabilities import agents + from core.capabilities.agents.base import UserAgentBaseService current = agents.AgentDefinition(agent_id="private-agent", name="Private", origin="cloud") monkeypatch.setattr(agents, "account_definition", lambda _: current) monkeypatch.setattr(agents, "account_definitions", lambda: [current]) - monkeypatch.setattr(skills, "account_authorized_for", lambda uid: uid == "current-user") + monkeypatch.setattr(skill_account, 'account_authorized_for', lambda uid: uid == "current-user") service = object.__new__(UserAgentBaseService) service.repo = SimpleNamespace(get_by_id=lambda _: None) with pytest.raises(LookupError): @@ -39,30 +44,30 @@ def test_cloud_agent_details_cannot_cross_local_user_identity( def test_unselected_cloud_skill_does_not_block_local_run_offline( durable_index, caps_root, monkeypatch ): - from core.capabilities import runtime - from core.capabilities.errors import CloudUnavailable, PermissionDenied - from core.services.desktop_capability_protocol import skill_content_hash + from features.desktop.capabilities import runtime + from core.harness.capability_errors import CloudUnavailable, PermissionDenied + from core.capabilities.manifest_protocol import skill_content_hash - monkeypatch.setattr(skills, "current_local_user_id", lambda: "owner") - monkeypatch.setattr(skills, "builtin_candidates", lambda: []) + monkeypatch.setattr(skill_account, 'current_local_user_id', lambda: "owner") + monkeypatch.setattr(skill_candidates, 'builtin_candidates', lambda: []) st, inst = _intent(monkeypatch) monkeypatch.setattr(cloud_skills, "_download", lambda *_: _zip({"SKILL.md": "v1"})) cloud_skills.prepare_one(st, inst.install_id) - skills.publish_local_skill( + skill_publication.publish_local_skill( "local-copy", files={"SKILL.md": "offline"}, content_hash=skill_content_hash("offline", {}), owner_user_id="owner", ) monkeypatch.setattr( - bridge, - "ensure_current_authorization", + bridge_identity, + 'ensure_current_authorization', lambda *_: (_ for _ in ()).throw(CloudUnavailable("offline")), ) run = runtime.prepare("offline-local", "owner", skill_ids=["local-copy"]) assert run.profile is None and set(run.bindings) == {"local-copy"} assert runtime.frozen_loader(run).get_skill_dir("example") is None - monkeypatch.setattr(bridge, "get_state", lambda: None) + monkeypatch.setattr(bridge_identity, 'get_state', lambda: None) runtime.validate(run, user_id="owner") with pytest.raises(PermissionDenied): runtime.validate(run, user_id="someone-else") @@ -71,14 +76,14 @@ def test_unselected_cloud_skill_does_not_block_local_run_offline( def test_cloud_skill_declared_by_agent_is_included_in_frozen_selection( durable_index, caps_root, monkeypatch ): - from core.capabilities import runtime, agents + from features.desktop.capabilities import runtime, agents - monkeypatch.setattr(skills, "current_local_user_id", lambda: "owner") - monkeypatch.setattr(skills, "builtin_candidates", lambda: []) + monkeypatch.setattr(skill_account, 'current_local_user_id', lambda: "owner") + monkeypatch.setattr(skill_candidates, 'builtin_candidates', lambda: []) st, inst = _intent(monkeypatch) monkeypatch.setattr(cloud_skills, "_download", lambda *_: _zip({"SKILL.md": "v1"})) cloud_skills.prepare_one(st, inst.install_id) - monkeypatch.setattr(bridge, "ensure_current_authorization", lambda *_: None) + monkeypatch.setattr(bridge_identity, 'ensure_current_authorization', lambda *_: None) definition = agents.AgentDefinition( agent_id="declare", name="Declare", dependencies=[{"kind": "skill", "id": "example"}] ) @@ -87,14 +92,14 @@ def test_cloud_skill_declared_by_agent_is_included_in_frozen_selection( def test_cloud_skill_declared_by_selected_plugin_is_frozen(durable_index, caps_root, monkeypatch): - from core.capabilities import runtime, plugins + from features.desktop.capabilities import runtime, plugins - monkeypatch.setattr(skills, "current_local_user_id", lambda: "owner") - monkeypatch.setattr(skills, "builtin_candidates", lambda: []) + monkeypatch.setattr(skill_account, 'current_local_user_id', lambda: "owner") + monkeypatch.setattr(skill_candidates, 'builtin_candidates', lambda: []) st, inst = _intent(monkeypatch) monkeypatch.setattr(cloud_skills, "_download", lambda *_: _zip({"SKILL.md": "v1"})) cloud_skills.prepare_one(st, inst.install_id) - monkeypatch.setattr(bridge, "ensure_current_authorization", lambda *_: None) + monkeypatch.setattr(bridge_identity, 'ensure_current_authorization', lambda *_: None) plugins.publish_local_plugin( {"slug": "pack", "install_id": "pack@owner", "components": {"skills": ["example"]}}, owner_user_id="owner", @@ -106,10 +111,10 @@ def test_cloud_skill_declared_by_selected_plugin_is_frozen(durable_index, caps_r def test_local_agent_replay_does_not_depend_on_cloud_session(durable_index, caps_root, monkeypatch): - from core.capabilities import runtime, agents + from features.desktop.capabilities import runtime, agents current = [state("a")] - monkeypatch.setattr(bridge, "get_state", lambda: current[0]) + monkeypatch.setattr(bridge_identity, 'get_state', lambda: current[0]) definition = agents.AgentDefinition(agent_id="local", name="Local", system_prompt="offline") runtime.pin_agent_definition("local-agent", "owner", definition) current[0] = None @@ -118,17 +123,17 @@ def test_local_agent_replay_does_not_depend_on_cloud_session(durable_index, caps def test_prepared_run_is_materialized_once_before_loading(durable_index, caps_root, monkeypatch): - from core.capabilities import runtime - from core.services.desktop_capability_protocol import skill_content_hash + from features.desktop.capabilities import runtime + from core.capabilities.manifest_protocol import skill_content_hash - monkeypatch.setattr(skills, "builtin_candidates", lambda: []) - monkeypatch.setattr(skills, "current_account_profile", lambda: None) + monkeypatch.setattr(skill_candidates, 'builtin_candidates', lambda: []) + monkeypatch.setattr(skill_account, 'current_account_profile', lambda: None) body = "---\nname: example\ndescription: sample\n---\nv1" - skills.publish_local_skill( + skill_publication.publish_local_skill( "example", files={"SKILL.md": body}, content_hash=skill_content_hash(body, {}) ) builds = [] - from core.capabilities import view + from features.desktop.capabilities import view build = view.build_view @@ -151,11 +156,11 @@ def test_changed_grant_or_package_drops_only_that_skill( durable_index, caps_root, monkeypatch, change ): """授权撤销或字节改动之后，这一份立刻从视图里摘掉，读取方再也拿不到它。""" - from core.capabilities import runtime - from core.services.desktop_capability_protocol import skill_content_hash + from features.desktop.capabilities import runtime + from core.capabilities.manifest_protocol import skill_content_hash - monkeypatch.setattr(skills, "builtin_candidates", lambda: []) - comp = skills.publish_local_skill( + monkeypatch.setattr(skill_candidates, 'builtin_candidates', lambda: []) + comp = skill_publication.publish_local_skill( "gate-test", files={"SKILL.md": "initial"}, content_hash=skill_content_hash("initial", {}), @@ -180,35 +185,35 @@ def test_changed_grant_or_package_drops_only_that_skill( def test_identical_local_publication_does_not_invalidate_resolution(index_db, caps_root): - from core.services.desktop_capability_protocol import skill_content_hash + from core.capabilities.manifest_protocol import skill_content_hash kwargs = dict( files={"SKILL.md": "same"}, content_hash=skill_content_hash("same", {}), owner_user_id="owner", ) - first = skills.publish_local_skill("stable", **kwargs) + first = skill_publication.publish_local_skill("stable", **kwargs) before = registry.generation(), skills.view_generation() - second = skills.publish_local_skill("stable", **kwargs) + second = skill_publication.publish_local_skill("stable", **kwargs) assert first == second assert (registry.generation(), skills.view_generation()) == before - skills.publish_local_skill("stable", **{**kwargs, "owner_user_id": "changed"}) + skill_publication.publish_local_skill("stable", **{**kwargs, "owner_user_id": "changed"}) assert registry.generation() > before[0] assert skills.view_generation() > before[1] def test_missing_view_target_never_reaches_the_sandbox(durable_index, caps_root, monkeypatch): """冻结的版本目录不见了：视图里不能留下这个名字，也不能悄悄指向别处。""" - from core.capabilities import runtime - from core.services.desktop_capability_protocol import skill_content_hash + from features.desktop.capabilities import runtime + from core.capabilities.manifest_protocol import skill_content_hash - monkeypatch.setattr(skills, "builtin_candidates", lambda: []) - monkeypatch.setattr(skills, "current_account_profile", lambda: None) - skills.publish_local_skill( + monkeypatch.setattr(skill_candidates, 'builtin_candidates', lambda: []) + monkeypatch.setattr(skill_account, 'current_account_profile', lambda: None) + skill_publication.publish_local_skill( "example", files={"SKILL.md": "v1"}, content_hash=skill_content_hash("v1", {}) ) run = runtime.prepare("missing-view-target", "u", skill_ids=["example"]) - monkeypatch.setattr("core.capabilities.runtime.state._component", lambda binding: None) + monkeypatch.setattr('features.desktop.capabilities.runtime.state._component', lambda binding: None) runtime.rebuild(run) assert "example" in runtime.get("missing-view-target").unavailable assert not (run.view_dir / "example").exists() diff --git a/src/backend/tests/capabilities/test_runtime_session_recovery.py b/src/backend/tests/features/desktop/capabilities/test_runtime_session_recovery.py similarity index 68% rename from src/backend/tests/capabilities/test_runtime_session_recovery.py rename to src/backend/tests/features/desktop/capabilities/test_runtime_session_recovery.py index 4ee52e4d..cd49d14b 100644 --- a/src/backend/tests/capabilities/test_runtime_session_recovery.py +++ b/src/backend/tests/features/desktop/capabilities/test_runtime_session_recovery.py @@ -1,25 +1,32 @@ """Regressions for desktop identity, explicit bindings and immutable runs.""" +from features.desktop.bridge import gateway as bridge_gateway +from features.desktop.bridge import identity as bridge_identity +from features.desktop.capabilities import skill_account +from features.desktop.capabilities import skill_candidates +from features.desktop.capabilities import skill_paths +from features.desktop.capabilities import skill_views +from features.desktop.capabilities import skills import pytest -from core.capabilities import registry, skills -from core.capabilities.ref import cloud_ref, profile_id -from core.services import desktop_cloud_bridge as bridge -from core.services import desktop_cloud_skills as cloud_skills -from core.services import desktop_cloud_bundles as bundles +from features.desktop.capabilities import registry, skills +from core.capabilities.reference import cloud_ref, profile_id +from features.desktop.bridge import client as bridge +from features.desktop.bridge import skills as cloud_skills +from features.desktop.bridge import bundles as bundles -from tests.capabilities.runtime_recovery_support import durable_index, _shell_id, _state_v2 +from tests.features.desktop.capabilities.runtime_recovery_support import durable_index, _shell_id, _state_v2 def test_switch_rebuild_never_puts_new_account_in_old_users_view( durable_index, caps_root, tmp_path, monkeypatch ): - from core.capabilities import store + from features.desktop.capabilities import store from core.db.models import UserShadow - from core.services.desktop_capability_protocol import skill_content_hash - from core.capabilities.paths import revision_for_hash + from core.capabilities.manifest_protocol import skill_content_hash + from features.desktop.capabilities.paths import revision_for_hash with durable_index() as db: UserShadow.__table__.create(db.get_bind(), checkfirst=True) @@ -31,10 +38,10 @@ def test_switch_rebuild_never_puts_new_account_in_old_users_view( ) db.commit() current = [_state_v2("a")] - monkeypatch.setattr(bridge, "get_state", lambda: current[0]) + monkeypatch.setattr(bridge_identity, 'get_state', lambda: current[0]) monkeypatch.setattr( - bridge, - "get_identity_state", + bridge_identity, + 'get_identity_state', lambda: { "user_center_id": "center-" + ("a" if current[0]["token"] == _state_v2("a")["token"] else "b"), @@ -43,10 +50,10 @@ def test_switch_rebuild_never_puts_new_account_in_old_users_view( ), }, ) - monkeypatch.setattr(skills, "builtin_candidates", lambda: []) - monkeypatch.setattr(skills, "device_view_dir", lambda: tmp_path / "shared" / "skills") - monkeypatch.setattr(skills, "user_view_dir", lambda uid: tmp_path / "shared" / "skills_u" / uid) - monkeypatch.setattr("core.agent_skills.cache_refresh.refresh_skill_caches", lambda: None) + monkeypatch.setattr(skill_candidates, 'builtin_candidates', lambda: []) + monkeypatch.setattr(skill_paths, 'device_view_dir', lambda: tmp_path / "shared" / "skills") + monkeypatch.setattr(skill_paths, 'user_view_dir', lambda uid: tmp_path / "shared" / "skills_u" / uid) + monkeypatch.setattr('core.capabilities.skills.cache_refresh.refresh_skill_caches', lambda: None) for uid in ("a", "b"): profile = profile_id("https://cloud.example", "cloud-" + uid) digest = skill_content_hash("private-" + uid, {}) @@ -59,14 +66,14 @@ def test_switch_rebuild_never_puts_new_account_in_old_users_view( "skill", profile, "private", revision_for_hash(digest), {"SKILL.md": "private-" + uid} ) registry.set_state(inst.install_id, "ready", resolved_revision=comp.revision) - skills.rebuild_user_view("local-a") - old = skills.user_view_dir("local-a") + skill_views.rebuild_user_view("local-a") + old = skill_paths.user_view_dir("local-a") assert (old / "private" / "SKILL.md").read_text() == "private-a" current[0] = _state_v2("b") - bridge._rebuild_identity_views() + bridge_identity.rebuild_identity_views() assert not (old / "private").exists() - skills.rebuild_user_view("local-b") - assert (skills.user_view_dir("local-b") / "private" / "SKILL.md").read_text() == "private-b" + skill_views.rebuild_user_view("local-b") + assert (skill_paths.user_view_dir("local-b") / "private" / "SKILL.md").read_text() == "private-b" def test_ensure_cloud_ready_downloads_only_unready_cloud_components( @@ -74,12 +81,12 @@ def test_ensure_cloud_ready_downloads_only_unready_cloud_components( ): """对话里选中尚未下载的云端技能 / 插件时按需准备：只对当前账号未就绪的记录调下载， 插件定义就绪后再准备它的组件；已就绪的与别的账号的记录不碰。""" - from core.capabilities.preparation import ensure_cloud_ready - from core.services import desktop_cloud_bundles, desktop_cloud_skills + from features.desktop.capabilities.preparation import ensure_cloud_ready + from features.desktop.bridge import bundles as desktop_cloud_bundles, skills as desktop_cloud_skills current = [_state_v2("a")] - monkeypatch.setattr(bridge, "get_state", lambda: current[0]) - monkeypatch.setattr(skills, "account_authorized_for", lambda uid: uid == "local-a") + monkeypatch.setattr(bridge_identity, 'get_state', lambda: current[0]) + monkeypatch.setattr(skill_account, 'account_authorized_for', lambda uid: uid == "local-a") profile = profile_id("https://cloud.example", "cloud-a") other = profile_id("https://cloud.example", "cloud-b") pending_skill = registry.upsert( @@ -144,23 +151,23 @@ def fake_skills(state, install_ids): def test_prepared_run_rejects_new_session_of_the_same_account( durable_index, caps_root, monkeypatch ): - from core.capabilities import runtime - from core.capabilities.errors import PermissionDenied + from features.desktop.capabilities import runtime + from core.harness.capability_errors import PermissionDenied current = [_state_v2("a")] - monkeypatch.setattr(bridge, "get_state", lambda: current[0]) - monkeypatch.setattr(skills, "builtin_candidates", lambda: []) + monkeypatch.setattr(bridge_identity, 'get_state', lambda: current[0]) + monkeypatch.setattr(skill_candidates, 'builtin_candidates', lambda: []) from pathlib import Path - from core.capabilities.resolver import Candidate, Resolution + from features.desktop.capabilities.resolver import Candidate, Resolution - monkeypatch.setattr(skills, "resolve_for_user", lambda _: Resolution()) - monkeypatch.setattr(skills, "current_local_user_id", lambda: "local-a") - monkeypatch.setattr(bridge, "ensure_current_authorization", lambda *_: None) + monkeypatch.setattr(skills, 'resolve_for_user', lambda _: Resolution()) + monkeypatch.setattr(skill_account, 'current_local_user_id', lambda: "local-a") + monkeypatch.setattr(bridge_identity, 'ensure_current_authorization', lambda *_: None) config = {"search": {"url": "https://cloud.example/gateway/search"}} - monkeypatch.setattr(bridge, "cloud_gateway_mcp_configs", lambda *_: config) + monkeypatch.setattr(bridge_gateway, 'cloud_gateway_mcp_configs', lambda *_: config) run = runtime.prepare("session-run", "local-a", skill_ids=[]) assert run.profile is None - profile = skills.current_account_profile() + profile = skill_account.current_account_profile() choice = Candidate( install_id="mcp:" + profile + ":search", runtime_name="search", @@ -182,14 +189,14 @@ def test_prepared_run_rejects_new_session_of_the_same_account( def test_authorization_check_makes_no_network_call(index_db, caps_root, monkeypatch): """装配前的授权判定不发网络请求——同步只由登录和变更信号驱动，没有事前探测。""" current = [_state_v2("a")] - monkeypatch.setattr(bridge, "get_state", lambda: current[0]) + monkeypatch.setattr(bridge_identity, 'get_state', lambda: current[0]) def forbidden(*args, **kwargs): raise AssertionError("authorization must not probe the cloud") monkeypatch.setattr("httpx.get", forbidden) for _ in range(3): - bridge.ensure_current_authorization() + bridge_identity.ensure_current_authorization() assert current[0] is not None @@ -198,17 +205,17 @@ async def test_revoked_grant_is_decided_by_the_gateway_call(index_db, caps_root, """撤权由云端在真实网关调用时裁决：401 立刻清掉本机的桥状态。""" import httpx import mcp.types - from core.llm.mcp_manager import GatewayMCPTool + from core.llm.tools.cloud_gateway import GatewayMCPTool current = [_state_v2("a")] - monkeypatch.setattr(bridge, "get_state", lambda: current[0]) - monkeypatch.setattr(bridge, "clear_state", lambda: current.__setitem__(0, None)) + monkeypatch.setattr(bridge_identity, 'get_state', lambda: current[0]) + monkeypatch.setattr(bridge_identity, 'clear_state', lambda: current.__setitem__(0, None)) tool = GatewayMCPTool( mcp_name="search", - tool=mcp.types.Tool(name="search", inputSchema={"type": "object"}), + tool={"name": "search", "inputSchema": {"type": "object"}}, invoke_url="https://cloud.example/api/v1/desktop/capability/gateway/search/call", schema_hash="frozen", - headers=bridge.cloud_headers(current[0]), + headers=bridge_identity.cloud_headers(current[0]), timeout=5, transport=httpx.MockTransport(lambda request: httpx.Response(401, json={})), ) @@ -223,11 +230,11 @@ async def test_running_gateway_uses_rotated_token_but_rejects_new_account( ): import httpx import mcp.types - from core.llm.mcp_manager import GatewayMCPTool - from core.capabilities.errors import CloudUnavailable + from core.llm.tools.cloud_gateway import GatewayMCPTool + from core.harness.capability_errors import CloudUnavailable current = [_state_v2("a")] - monkeypatch.setattr(bridge, "get_state", lambda: current[0]) + monkeypatch.setattr(bridge_identity, 'get_state', lambda: current[0]) seen = [] def handler(request): @@ -236,10 +243,10 @@ def handler(request): tool = GatewayMCPTool( mcp_name="search", - tool=mcp.types.Tool(name="search", inputSchema={"type": "object"}), + tool={"name": "search", "inputSchema": {"type": "object"}}, invoke_url="https://cloud.example/api/v1/desktop/capability/gateway/search/call", schema_hash="frozen", - headers=bridge.cloud_headers(current[0]), + headers=bridge_identity.cloud_headers(current[0]), timeout=5, transport=httpx.MockTransport(handler), ) @@ -254,7 +261,7 @@ def handler(request): def test_missing_explicit_source_never_falls_back(index_db): from pathlib import Path - from core.capabilities.resolver import Candidate, resolve + from features.desktop.capabilities.resolver import Candidate, resolve builtin = Candidate( install_id="skill:builtin:example", diff --git a/src/backend/tests/capabilities/test_selection_dependency_sources.py b/src/backend/tests/features/desktop/capabilities/test_selection_dependency_sources.py similarity index 79% rename from src/backend/tests/capabilities/test_selection_dependency_sources.py rename to src/backend/tests/features/desktop/capabilities/test_selection_dependency_sources.py index 2682f97e..c6c0fe9c 100644 --- a/src/backend/tests/capabilities/test_selection_dependency_sources.py +++ b/src/backend/tests/features/desktop/capabilities/test_selection_dependency_sources.py @@ -1,20 +1,26 @@ """Implicit defaults and dependency closure share the final source decision.""" +from features.desktop.bridge import identity as bridge_identity +from features.desktop.capabilities import skill_account +from features.desktop.capabilities import skill_candidates +from features.desktop.capabilities import skill_paths +from features.desktop.capabilities import skill_publication +from features.desktop.capabilities import skills from time import monotonic from types import SimpleNamespace import pytest -from core.capabilities import registry, skills, readiness -from core.services import desktop_cloud_bridge as bridge -from core.services.desktop_capability_protocol import skill_content_hash -from tests.capabilities.test_desktop_capabilities_api import cloud as cloud -from tests.capabilities.test_desktop_capabilities_api import client, USER, PROFILE, _Cloud +from features.desktop.capabilities import registry, skills, readiness +from features.desktop.bridge import client as bridge +from core.capabilities.manifest_protocol import skill_content_hash +from tests.features.desktop.capabilities.test_desktop_capabilities_api import cloud as cloud +from tests.features.desktop.capabilities.test_desktop_capabilities_api import client, USER, PROFILE, _Cloud def _publish(key, metadata="", owner=USER): text = f"---\nname: {key}\ndescription: local fixture\n{metadata}\n---\nLocal fixture\n" - skills.publish_local_skill( + skill_publication.publish_local_skill( key, files={"SKILL.md": text}, content_hash=skill_content_hash(text, {}), @@ -25,20 +31,20 @@ def _publish(key, metadata="", owner=USER): @pytest.mark.parametrize("cached", [False, True]) def test_offline_non_none_catalog_defaults_filter_with_actual_user(client, monkeypatch, cached): - from core.config import catalog_resolver - from core import services + from core.capabilities.catalog import resolver as catalog_resolver + from core.capabilities.catalog import overrides as services _publish("good-default") _publish( "bad-default", "dependencies:\n - kind: pip\n id: codex-never-installed-selection-783465", ) - monkeypatch.setattr(bridge, "bridge_active", lambda: False) - monkeypatch.setattr(bridge, "get_state", lambda: None) - monkeypatch.setattr(skills, "current_local_user_id", lambda: "a-different-user") + monkeypatch.setattr(bridge_identity, 'bridge_active', lambda: False) + monkeypatch.setattr(bridge_identity, 'get_state', lambda: None) + monkeypatch.setattr(skill_account, 'current_local_user_id', lambda: "a-different-user") catalog_resolver.invalidate_capability_cache() if cached: - catalog_resolver._capability_cache[USER] = ( + catalog_resolver.process_state().capability_cache[USER] = ( monotonic() + 60, (["good-default", "bad-default"], [], []), ) @@ -92,7 +98,7 @@ def test_cloud_dependency_uses_explicit_compatible_local_source(client, monkeypa assert result.chosen["dependent-a"].install_id == aid # Every root is evaluated independently: the rejected cloud copy must not # borrow the ready state of its same-named local replacement. - evaluated = readiness.eligible_skill_candidates(skills.candidates(USER), USER) + evaluated = readiness.eligible_skill_candidates(skill_candidates.candidates(USER), USER) assert not next(candidate for candidate in evaluated if candidate.install_id == bid).usable report = readiness.file_readiness(registry.get(aid), readiness.context_for_user(USER)) assert report["ready"] @@ -139,7 +145,7 @@ def test_dependency_uses_runtime_alias_for_opaque_local_installation(client, mon def test_builtin_dependency_is_checked_without_publishing_files(client, monkeypatch, tmp_path): - from core.capabilities import store + from features.desktop.capabilities import store root = tmp_path / "unpublished-builtins" for key, metadata in [ @@ -151,7 +157,7 @@ def test_builtin_dependency_is_checked_without_publishing_files(client, monkeypa (path / "SKILL.md").write_text( f"---\nname: {key}\ndescription: fixture\n{metadata}\n---\nBuiltin fixture\n" ) - monkeypatch.setattr(skills, "builtin_dir", lambda: root) + monkeypatch.setattr(skill_paths, 'builtin_dir', lambda: root) result = skills.resolve_for_user(USER) assert set(result.chosen) == {"builtin-a", "builtin-b"} assert list(store.iter_components("skill", "builtin")) == [] diff --git a/src/backend/tests/capabilities/test_selection_readiness.py b/src/backend/tests/features/desktop/capabilities/test_selection_readiness.py similarity index 80% rename from src/backend/tests/capabilities/test_selection_readiness.py rename to src/backend/tests/features/desktop/capabilities/test_selection_readiness.py index 93484aad..5d0ddec4 100644 --- a/src/backend/tests/capabilities/test_selection_readiness.py +++ b/src/backend/tests/features/desktop/capabilities/test_selection_readiness.py @@ -1,9 +1,11 @@ """Unavailable installed skills cannot win implicit runtime selection.""" +from features.desktop.capabilities import skill_paths +from features.desktop.capabilities import skills -from core.capabilities import registry, skills -from tests.capabilities.test_desktop_capabilities_api import cloud as cloud -from tests.capabilities.test_desktop_capabilities_api import client, USER -from tests.capabilities.test_management_readiness import _cloud_skill, _item +from features.desktop.capabilities import registry, skills +from tests.features.desktop.capabilities.test_desktop_capabilities_api import cloud as cloud +from tests.features.desktop.capabilities.test_desktop_capabilities_api import client, USER +from tests.features.desktop.capabilities.test_management_readiness import _cloud_skill, _item def test_incompatible_cloud_candidate_does_not_override_compatible_builtin( @@ -19,7 +21,7 @@ def test_incompatible_cloud_candidate_does_not_override_compatible_builtin( (path / "SKILL.md").write_text( "---\nname: platform-choice\ndescription: compatible\n---\nBuiltin\n" ) - monkeypatch.setattr(skills, "builtin_dir", lambda: root) + monkeypatch.setattr(skill_paths, 'builtin_dir', lambda: root) resolution = skills.resolve_for_user(USER) assert resolution.chosen["platform-choice"].profile == "builtin" registry.set_preference("skill", "platform-choice", iid, chosen_by=USER) @@ -33,7 +35,7 @@ def test_explicit_unavailable_request_never_falls_back_to_builtin(client, monkey path = tmp_path / "builtin" / "requested" path.mkdir(parents=True) (path / "SKILL.md").write_text("---\nname: requested\ndescription: compatible\n---\nBuiltin\n") - monkeypatch.setattr(skills, "builtin_dir", lambda: path.parent) + monkeypatch.setattr(skill_paths, 'builtin_dir', lambda: path.parent) result = skills.resolve_for_user(USER, requested={iid}) assert "requested" not in result.chosen assert result.reasons["requested"] == "requested_not_ready" diff --git a/src/backend/tests/capabilities/test_store_file_integrity.py b/src/backend/tests/features/desktop/capabilities/test_store_file_integrity.py similarity index 88% rename from src/backend/tests/capabilities/test_store_file_integrity.py rename to src/backend/tests/features/desktop/capabilities/test_store_file_integrity.py index 3ece383e..ce7ebef8 100644 --- a/src/backend/tests/capabilities/test_store_file_integrity.py +++ b/src/backend/tests/features/desktop/capabilities/test_store_file_integrity.py @@ -1,11 +1,14 @@ """Stored package files stay inside their root and retain portable file names.""" +from features.desktop.capabilities import skill_hashes import io import stat import zipfile import pytest -from core.capabilities import archive, junction, store -from core.capabilities.errors import IntegrityFailed +from core.capabilities import archive as archive +from core.infra import directory_links as junction +from features.desktop.capabilities import store +from core.harness.capability_errors import IntegrityFailed def test_existing_directory_link_still_checks_allowed_roots(tmp_path): @@ -103,7 +106,7 @@ def test_duplicate_zip_member_is_rejected(tmp_path): def test_hash_detects_change_to_file_older_than_newest(tmp_path): import os - from core.capabilities.skills import skill_dir_hash + from features.desktop.capabilities.skill_hashes import skill_dir_hash root = tmp_path / "package" archive.write_files(root, {"SKILL.md": "old", "newer.txt": "unchanged"}) @@ -125,25 +128,25 @@ def test_iter_files_retains_path_component_order(tmp_path): def test_fresh_hash_reads_bytes_even_with_unchanged_signature(tmp_path, monkeypatch): - from core.capabilities import skills + from features.desktop.capabilities import skills root = tmp_path / "package" archive.write_files(root, {"SKILL.md": "old"}) entries = list(archive.iter_file_stats(root)) monkeypatch.setattr(archive, "iter_file_stats", lambda _path: entries) - before = skills.skill_dir_hash(root, fresh=True) + before = skill_hashes.skill_dir_hash(root, fresh=True) (root / "SKILL.md").write_text("new") - assert skills.skill_dir_hash(root, fresh=True) != before + assert skill_hashes.skill_dir_hash(root, fresh=True) != before @pytest.mark.skipif(__import__("os").name != "nt", reason="Windows extended path regression") def test_skill_hash_reads_long_windows_paths(tmp_path): - from core.capabilities.skills import skill_dir_hash + from features.desktop.capabilities.skill_hashes import skill_dir_hash root = tmp_path / "long-package" while len(str(root)) < 275: root /= "nested-package-directory" - native_root = junction._native(root) + native_root = junction.native_path(root) native_root.mkdir(parents=True) (native_root / "SKILL.md").write_text("old") before = skill_dir_hash(root, fresh=True) diff --git a/src/backend/tests/capabilities/test_toolkit_runtime_alias.py b/src/backend/tests/features/desktop/capabilities/test_toolkit_runtime_alias.py similarity index 83% rename from src/backend/tests/capabilities/test_toolkit_runtime_alias.py rename to src/backend/tests/features/desktop/capabilities/test_toolkit_runtime_alias.py index 51cb15b5..dfb06d27 100644 --- a/src/backend/tests/capabilities/test_toolkit_runtime_alias.py +++ b/src/backend/tests/features/desktop/capabilities/test_toolkit_runtime_alias.py @@ -1,15 +1,18 @@ """Real AgentScope rendering and loading must use resolved runtime names.""" +from features.desktop.capabilities import skill_account +from features.desktop.capabilities import skill_candidates +from features.desktop.capabilities import skill_publication from pathlib import Path from types import SimpleNamespace import pytest -from agentscope.tool import Toolkit from agentscope.state import AgentState -from core.capabilities import registry, runtime, skills -from core.llm.tool_collector import ToolCollector +from agentscope.tool import Toolkit +from features.desktop.capabilities import registry, runtime, skills +from core.llm.factory.tools.definitions import ToolAssembly from core.llm.tools.skill_tool import register_sandboxed_view_text_file -from core.services.desktop_capability_protocol import skill_content_hash +from core.capabilities.manifest_protocol import skill_content_hash def factory_template(): @@ -21,10 +24,10 @@ def factory_template(): @pytest.fixture def frozen(index_db, caps_root, monkeypatch): monkeypatch.setattr("core.config.local_mode.local_mode_enabled", lambda: True) - monkeypatch.setattr(skills, "builtin_candidates", lambda: []) - monkeypatch.setattr(skills, "current_account_profile", lambda: None) + monkeypatch.setattr(skill_candidates, 'builtin_candidates', lambda: []) + monkeypatch.setattr(skill_account, 'current_account_profile', lambda: None) monkeypatch.setattr( - "core.agent_skills.loader.get_skill_loader", + 'core.capabilities.skills.loader_factory.get_skill_loader', lambda: SimpleNamespace(load_all_metadata=lambda: {}), ) @@ -33,7 +36,7 @@ def create(names, physical=False): md = ( "---\nname: original-cloud-name\ndescription: Synthetic marker reader\n---\n" + body ) - skills.publish_local_skill( + skill_publication.publish_local_skill( alias, files={"SKILL.md": md}, content_hash=skill_content_hash(md, {}), @@ -44,7 +47,7 @@ def create(names, physical=False): if physical: original_get_dir = loader.get_skill_dir loader.get_skill_dir = lambda sid: str(Path(original_get_dir(sid)).resolve()) - collector = ToolCollector() + collector = ToolAssembly() assert loader.register_skills_to_toolkit(collector, list(names)) == len(names) return prepared, loader, collector @@ -72,7 +75,7 @@ async def test_real_factory_prompt_and_native_registry_preserve_two_custom_alias Path(loader.get_skill_dir(alias)).resolve() ) assert native[alias].markdown == marker - loaded = await toolkit.builtin_skill_viewer.tool(alias, AgentState()) + loaded = await toolkit.builtin_skill_viewer.tool.call(alias, AgentState()) assert "\n".join(block.text for block in loaded.content) == marker assert "name: original-cloud-name" in (prepared.view_dir / alias / "SKILL.md").read_text() assert "- `original-cloud-name`" not in prompt @@ -92,7 +95,7 @@ async def test_native_projection_is_detached_and_custom_alias_reads_frozen_bytes assert again["custom-local"].name == "custom-local" assert again["custom-local"].dir == str(Path(loader.get_skill_dir("custom-local")).resolve()) register_sandboxed_view_text_file(collector, [loader.get_skill_dir("custom-local")], loader) - tool = collector.get_tool("view_text_file")._func + tool = next(t for t in collector if t.name == "view_text_file")._func response = await tool(str(Path(loader.get_skill_dir("custom-local")).resolve() / "SKILL.md")) assert "FROZEN_CONTENT" in "\n".join(block.text for block in response.content) denied = await tool( @@ -123,14 +126,14 @@ async def test_native_projection_rechecks_frozen_authorization_and_hash_before_c def test_explicit_skill_hint_uses_authorized_alias_instead_of_store_revision(frozen, monkeypatch): - from orchestration.workflow import _build_skill_injection + from core.agent_runtime.skills import _build_skill_injection prepared, loader, collector = frozen({"chosen-local-alias": "FROZEN_BODY"}) physical = str((prepared.view_dir / "chosen-local-alias").resolve()) assert Path(physical).name == prepared.bindings["chosen-local-alias"]["revision"] # The global capability-store loader returns this real revision directory. monkeypatch.setattr(loader, "get_skill_dir", lambda sid: physical) - monkeypatch.setattr("core.agent_skills.loader.get_skill_loader", lambda: loader) + monkeypatch.setattr('core.capabilities.skills.loader_factory.get_skill_loader', lambda: loader) hint = _build_skill_injection({"skill_id": "chosen-local-alias", "skill_name": "Chosen"}) assert physical + "/SKILL.md" in hint["content"] assert "chosen-local-alias" in hint["content"] @@ -139,7 +142,8 @@ def test_explicit_skill_hint_uses_authorized_alias_instead_of_store_revision(fro @pytest.mark.asyncio async def test_parallel_runtime_loader_order_bounds_and_failure(): import asyncio - from core.llm.tool_collector import _ParallelRuntimeSkillLoader + + from core.capabilities.skills.runtime_loader import ParallelRuntimeSkillLoader active = 0 peak = 0 @@ -162,13 +166,13 @@ async def list_skills(self): finally: active -= 1 - assert await _ParallelRuntimeSkillLoader([Loader(i) for i in range(8)]).list_skills() == list( + assert await ParallelRuntimeSkillLoader([Loader(i) for i in range(8)]).list_skills() == list( range(8) ) assert peak <= 4 seen.clear() with pytest.raises(RuntimeError, match="validation rejected"): - await _ParallelRuntimeSkillLoader([Loader(i, i == 2) for i in range(8)]).list_skills() + await ParallelRuntimeSkillLoader([Loader(i, i == 2) for i in range(8)]).list_skills() assert sorted(seen) == list(range(8)) assert active == 0 @@ -176,12 +180,12 @@ async def list_skills(self): @pytest.mark.asyncio @pytest.mark.parametrize("mutation", ["revoke", "corrupt"]) async def test_desktop_batched_loader_matches_native_and_rechecks(frozen, monkeypatch, mutation): - from core.llm import tool_collector + from core.capabilities.skills import runtime_loader prepared, loader, collector = frozen({"alias": "BODY"}) instance = collector.skill_loaders[0] expected = await instance.list_skills() - monkeypatch.setattr(tool_collector, "os", SimpleNamespace(name="nt")) + monkeypatch.setattr(runtime_loader, "os", SimpleNamespace(name="nt")) actual = await instance.list_skills() assert actual == expected actual[0].markdown = "poison" diff --git a/src/backend/tests/capabilities/test_user_facing_errors.py b/src/backend/tests/features/desktop/capabilities/test_user_facing_errors.py similarity index 81% rename from src/backend/tests/capabilities/test_user_facing_errors.py rename to src/backend/tests/features/desktop/capabilities/test_user_facing_errors.py index 62e5e37e..78c055e3 100644 --- a/src/backend/tests/capabilities/test_user_facing_errors.py +++ b/src/backend/tests/features/desktop/capabilities/test_user_facing_errors.py @@ -2,15 +2,8 @@ import pytest -from core.capabilities.dependency import DependencyMissing -from core.capabilities.errors import ( - CloudUnavailable, - IntegrityFailed, - NameConflict, - PackageMissing, - PermissionDenied, - ViewUnavailable, -) +from features.desktop.capabilities.dependency import DependencyMissing +from core.harness.capability_errors import CloudUnavailable, IntegrityFailed, NameConflict, PackageMissing, PermissionDenied, ViewUnavailable from core.chat.context import resolve_user_facing_error diff --git a/src/backend/tests/capabilities/test_view_os_errors.py b/src/backend/tests/features/desktop/capabilities/test_view_os_errors.py similarity index 95% rename from src/backend/tests/capabilities/test_view_os_errors.py rename to src/backend/tests/features/desktop/capabilities/test_view_os_errors.py index c8ce08ba..1922ee0b 100644 --- a/src/backend/tests/capabilities/test_view_os_errors.py +++ b/src/backend/tests/features/desktop/capabilities/test_view_os_errors.py @@ -3,8 +3,9 @@ import errno from pathlib import Path import pytest -from core.capabilities import junction, view -from core.capabilities.errors import ViewUnavailable +from core.infra import directory_links as junction +from features.desktop.capabilities import view +from core.harness.capability_errors import ViewUnavailable def locked(path): @@ -117,7 +118,7 @@ def mkdir(path, *args, **kwargs): def test_windows_case_collisions_block_before_mutation(tmp_path, monkeypatch): from types import SimpleNamespace - from core.capabilities import view + from features.desktop.capabilities import view monkeypatch.setattr(view, "os", SimpleNamespace(name="nt")) destination = tmp_path / "view" diff --git a/src/backend/tests/capabilities/test_workbench_local_upgrades.py b/src/backend/tests/features/desktop/capabilities/test_workbench_local_upgrades.py similarity index 84% rename from src/backend/tests/capabilities/test_workbench_local_upgrades.py rename to src/backend/tests/features/desktop/capabilities/test_workbench_local_upgrades.py index d395826a..7e09d79f 100644 --- a/src/backend/tests/capabilities/test_workbench_local_upgrades.py +++ b/src/backend/tests/features/desktop/capabilities/test_workbench_local_upgrades.py @@ -4,8 +4,8 @@ import zipfile import io -from tests.test_workbench_upgrade_routes import upgraded -from tests.workbench_plugin_support import package +from tests.features.workbench.test_workbench_upgrade_routes import upgraded +from tests.features.workbench.workbench_plugin_support import package def unpack(path, raw): @@ -16,9 +16,9 @@ def unpack(path, raw): def test_local_upgrade_requires_approval_and_preserves_paused_state(client, tmp_path): - from core.plugins.local import service - from core.applications import local_plugin_upgrades - from core.capabilities import registry + from features.desktop.local.plugins import service as service + from features.workbench import local_plugin_upgrades + from features.desktop.capabilities import registry old_path = unpack(tmp_path / "old-plugin", package()) new_path = unpack(tmp_path / "new-plugin", upgraded(grants=["file.save"])) @@ -42,9 +42,9 @@ def test_local_upgrade_requires_approval_and_preserves_paused_state(client, tmp_ def test_local_candidates_do_not_materialize_children_and_rejection_reclaims_exact_revision( client, tmp_path ): - from core.plugins.local import service - from core.applications import local_plugin_upgrades - from core.capabilities import registry, store + from features.desktop.local.plugins import service as service + from features.workbench import local_plugin_upgrades + from features.desktop.capabilities import registry, store old_path = unpack(tmp_path / "old-candidate-test", package()) root = tmp_path / "new-candidate-test" diff --git a/src/backend/tests/desktop_login_browser_server.py b/src/backend/tests/features/desktop/desktop_login_browser_server.py similarity index 91% rename from src/backend/tests/desktop_login_browser_server.py rename to src/backend/tests/features/desktop/desktop_login_browser_server.py index 1bde396f..80de421d 100644 --- a/src/backend/tests/desktop_login_browser_server.py +++ b/src/backend/tests/features/desktop/desktop_login_browser_server.py @@ -6,7 +6,7 @@ from pathlib import Path import socket import sys -sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +sys.path.insert(0, str(Path(__file__).resolve().parents[3])) from fastapi import FastAPI, Request, Response, HTTPException from fastapi.responses import FileResponse @@ -16,12 +16,12 @@ from core.config.settings import settings from core.auth.session import create_session, validate_session from core.infra.responses import success_response -from api.routes.v1.desktop_login import router +from features.desktop.api.desktop_login import router object.__setattr__(settings, "session", dataclasses.replace(settings.session, store_type="memory")) app = FastAPI() app.include_router(router, prefix="/api") -web = Path(__file__).resolve().parents[2] / "frontend" / "dist" +web = Path(__file__).resolve().parents[4] / "frontend" / "dist" @app.post("/__test__/session") diff --git a/src/backend/tests/features/desktop/index_fixtures.py b/src/backend/tests/features/desktop/index_fixtures.py new file mode 100644 index 00000000..d142ca71 --- /dev/null +++ b/src/backend/tests/features/desktop/index_fixtures.py @@ -0,0 +1,13 @@ +"""Desktop bridge tests consult the capability index (name preferences).""" + +from __future__ import annotations + +import pytest +from tests._capability_index import bind_capability_index + + +@pytest.fixture(autouse=True) +def _capability_index(tmp_path, monkeypatch): + engine, _factory = bind_capability_index(tmp_path, monkeypatch) + yield + engine.dispose() diff --git a/src/backend/tests/features/desktop/observability_cases_1.py b/src/backend/tests/features/desktop/observability_cases_1.py new file mode 100644 index 00000000..97fc1413 --- /dev/null +++ b/src/backend/tests/features/desktop/observability_cases_1.py @@ -0,0 +1,310 @@ +from tests.features.desktop.observability_support import (bridge_identity, pytest, create_engine, sessionmaker) + +def test_duplicate_and_stale_delivery_keep_newest_chat(store): + from features.desktop.observability.projection import ingest, list_records + + def batch(version, content): + return { + "events": [ + { + "kind": "message", + "object_id": "m1", + "revision": version, + "payload": {"chat_id": "c1", "role": "user", "content": content}, + } + ] + } + + with store() as db: + ingest(db, "alice", "laptop", batch(2, "latest")) + ingest(db, "alice", "laptop", batch(2, "latest")) + ingest(db, "alice", "laptop", batch(1, "old")) + rows, total = list_records(db, kind="message", user_id="alice") + assert total == 1 + assert rows[0]["payload"]["content"] == "latest" + assert list_records(db, kind="message", user_id="bob")[1] == 0 + ingest(db, "bob", "laptop", batch(1, "bob's message")) + assert ( + list_records(db, kind="message", user_id="bob")[0][0]["payload"]["content"] + == "bob's message" + ) + +def test_capture_upload_and_conditional_ack_survive_retry(tmp_path): + import asyncio + + from core.db.models import ChatMessage, ChatSession, UserShadow + from features.desktop.models import DesktopOutbox + from features.desktop.observability.sync import acknowledge, install_capture, prepare_batch + + engine = create_engine("sqlite:///" + str(tmp_path / "local.db")) + from core.db.models import ChatRun + + for m in (UserShadow, ChatSession, ChatMessage, ChatRun, DesktopOutbox): + m.__table__.create(engine) + factory = sessionmaker(bind=engine) + identity = { + "cloud_base": "https://cloud.example", + "subject": "alice", + "device_id": "device", + "shell_user_center_id": "cloud:cloud.example:443:center", + } + remove = install_capture(factory, lambda: identity) + try: + with factory() as db: + db.add( + UserShadow( + user_id="local-alice", + username="Alice", + user_center_id=identity["shell_user_center_id"], + ) + ) + db.add(ChatSession(chat_id="c", user_id="local-alice", title="Desktop task")) + db.add( + ChatMessage(message_id="m", chat_id="c", role="user", chat_seq=1, content="hello") + ) + db.commit() + with factory() as db: + first = prepare_batch(db, identity) + assert {e["kind"] for e in first["events"]} == {"session", "message"} + with factory() as db: + row = db.get(ChatMessage, "m") + row.content = "newer" + db.commit() + with factory() as db: + acknowledge(db, identity, {"acknowledged": first["events"]}) + retry = prepare_batch(db, identity) + assert len(retry["events"]) == 1 + assert retry["events"][0]["payload"]["content"] == "newer" + with factory() as db: + assert prepare_batch(db, {**identity, "subject": "bob"})["events"] == [] + finally: + remove() + +def test_cloud_gateway_and_desktop_receipt_are_one_call(store): + from features.desktop.observability.projection import ingest, list_records + + event = { + "kind": "tool", + "object_id": "result-1", + "revision": 1, + "payload": { + "tool_name": "search", + "chat_id": "c", + "status": "success", + "tool_args": {"api_key": "secret-value"}, + }, + } + with store() as db: + ingest(db, "alice", "device", {"events": [event]}) + ingest(db, "alice", "device", {"events": [event]}, observation="gateway") + rows, total = list_records(db, kind="tool", user_id="alice") + assert total == 1 + assert rows[0]["observation"] == "gateway" + assert rows[0]["payload"]["tool_args"]["api_key"] == "[redacted]" + +def test_gateway_stream_arrives_before_log_writer_and_extracts_usage(monkeypatch): + import asyncio + + from features.desktop.gateway import observability as obs + from fastapi import FastAPI, Request + from fastapi.responses import StreamingResponse + from fastapi.testclient import TestClient + + captured = [] + monkeypatch.setattr(obs, "submit", lambda *args: captured.append(args)) + app = FastAPI() + app.router.route_class = obs.ObservedGatewayRoute + + @app.post("/gateway/models/model/chat/completions") + async def handler(request: Request): + request.state.desktop_observation_user = "alice" + + async def chunks(): + assert not captured + yield b'data: {"choices":[{"delta":{"content":"hello"}}]}\n\n' + assert not captured + yield b'data: {"usage":{"prompt_tokens":7,"completion_tokens":3}}\n\n' + + return StreamingResponse(chunks(), media_type="text/event-stream") + + response = TestClient(app).post( + "/gateway/models/model/chat/completions", + headers={"x-desktop-device-id": "d", "x-observation-call-id": "try-1"}, + ) + assert response.status_code == 200 + assert "hello" in response.text + assert len(captured) == 1 + assert captured[0][-1]["usage"] == {"prompt_tokens": 7, "completion_tokens": 3} + assert captured[0][-1]["status"] == "success" + +def test_real_upload_endpoint_authorization_and_size_limit(store, monkeypatch): + from features.desktop.api import desktop_observability as route + from core.db.engine import get_db + from features.desktop.observability.projection import MAX_BATCH_BYTES + from fastapi import FastAPI + from fastapi.testclient import TestClient + + monkeypatch.setenv("CONFIG_TOKEN", "test-config") + monkeypatch.delenv("HUGAGENT_DESKTOP_BRIDGE_SECRET", raising=False) + monkeypatch.setattr(route, "SessionLocal", store) + + def database(): + with store() as db: + yield db + + app = FastAPI() + app.include_router(route.router, prefix="/api") + app.include_router(route.admin_router, prefix="/api") + app.dependency_overrides[get_db] = database + client = TestClient(app) + assert ( + client.post("/api/v1/desktop/observability/batch", json={"events": []}).status_code == 401 + ) + assert client.get("/api/v1/admin/desktop-observability/records").status_code == 401 + app.dependency_overrides[route._require_capability_user] = lambda: "alice" + event = { + "kind": "message", + "object_id": "m", + "revision": 1, + "payload": {"content": "private conversation", "user_id": "victim", "role": "user"}, + } + response = client.post( + "/api/v1/desktop/observability/batch", + headers={"x-desktop-device-id": "d"}, + json={"events": [event]}, + ) + assert response.status_code == 200, response.text + data = client.get( + "/api/v1/admin/desktop-observability/records", + headers={"Authorization": "Bearer test-config"}, + ).json()["data"] + assert data["pagination"]["total_items"] == 1 + assert data["items"][0]["user_id"] == "alice" + assert "user_id" not in data["items"][0]["payload"] + assert ( + client.post( + "/api/v1/desktop/observability/batch", + headers={"x-desktop-device-id": "d"}, + content=b"x" * (MAX_BATCH_BYTES + 1), + ).status_code + == 413 + ) + +def test_offline_restart_and_http_delivery_to_cloud(local_store, store, monkeypatch): + import asyncio + + import httpx + from features.desktop.api import desktop_observability as route + from core.db.models import ( + ChatMessage, + ChatSession, + SkillCallLog, + SubAgentCallLog, + ToolCallLog, + UserShadow, + ) + from features.desktop.bridge import client as bridge + from features.desktop.observability.projection import list_records + from features.desktop.observability.sync import DesktopSyncWorker, install_capture, prepare_batch + from fastapi import FastAPI + + identity = { + "cloud_base": "http://cloud", + "subject": "alice", + "device_id": "d", + "shell_user_center_id": "cloud:cloud:80:alice", + } + state = {"cloud_base": "http://cloud", "token": "test-credential", "device_id": "d"} + monkeypatch.setattr(bridge_identity, 'get_state', lambda: state) + monkeypatch.setattr(bridge_identity, 'get_identity_state', lambda: identity) + remove = install_capture(local_store, lambda: identity) + try: + with local_store() as db: + db.add( + UserShadow( + user_id="local", + username="Alice", + user_center_id=identity["shell_user_center_id"], + ) + ) + db.add(ChatSession(chat_id="c", user_id="local", title="Research")) + db.add( + ChatMessage(message_id="m", chat_id="c", chat_seq=1, role="user", content="Hello") + ) + db.add( + ToolCallLog( + id="t", user_id="local", chat_id="c", tool_name="search", status="success" + ) + ) + db.add( + SkillCallLog( + id="s", + user_id="local", + chat_id="c", + skill_id="cloud-skill", + skill_source="cloud", + invocation_type="view", + status="success", + ) + ) + db.add( + SubAgentCallLog( + id="a", + user_id="local", + chat_id="c", + subagent_name="Researcher", + status="success", + ) + ) + db.commit() + + async def offline(request): + raise httpx.ConnectError("offline") + + worker = DesktopSyncWorker(local_store, transport=httpx.MockTransport(offline)) + with pytest.raises(httpx.ConnectError): + asyncio.run(worker.sync_once()) + with local_store() as db: + assert len(prepare_batch(db, identity)["events"]) == 5 + monkeypatch.setattr(route, "SessionLocal", store) + app = FastAPI() + app.include_router(route.router, prefix="/api") + app.dependency_overrides[route._require_capability_user] = lambda: "alice" + restarted = DesktopSyncWorker(local_store, transport=httpx.ASGITransport(app=app)) + asyncio.run(restarted.sync_once()) + asyncio.run(restarted.sync_once()) + with local_store() as db: + assert prepare_batch(db, identity)["events"] == [] + with store() as db: + assert list_records(db, user_id="alice")[1] == 5 + assert list_records(db, kind="message")[0][0]["payload"]["content"] == "Hello" + assert list_records(db, kind="skill")[0][0]["payload"]["capability_origin"] == "cloud" + finally: + remove() + +def test_unicode_large_records_and_secrets_are_safe(store): + import json + + from features.desktop.observability.projection import MAX_EVENT_BYTES, ingest, list_records, public_payload + + payload = public_payload("message", {"content": "中" * 100000, "role": "assistant"}) + assert len(json.dumps(payload, ensure_ascii=False).encode()) < MAX_EVENT_BYTES + with store() as db: + ingest( + db, + "u", + "d", + {"events": [{"kind": "message", "object_id": "m", "revision": 1, "payload": payload}]}, + ) + assert list_records(db, kind="message")[0][0]["payload"]["content"] == "中" * 100000 + payload = public_payload( + "tool", + { + "tool_args": {"api_key": "hidden"}, + "tool_result": '{"api_key":"hidden"} Bearer abcdefghi -----BEGIN PRIVATE KEY-----abc-----END PRIVATE KEY-----', + }, + ) + assert "hidden" not in json.dumps(payload) + assert "abcdefghi" not in json.dumps(payload) + assert "BEGIN PRIVATE KEY" not in json.dumps(payload) + assert public_payload("message", {"content": "x" * 100001})["truncated"] is True diff --git a/src/backend/tests/features/desktop/observability_cases_2.py b/src/backend/tests/features/desktop/observability_cases_2.py new file mode 100644 index 00000000..3c4ff581 --- /dev/null +++ b/src/backend/tests/features/desktop/observability_cases_2.py @@ -0,0 +1,270 @@ +from tests.features.desktop.observability_support import (bridge_identity, pytest, create_engine, sessionmaker) + +def test_gateway_unknown_usage_keeps_desktop_usage_and_parent(store): + from features.desktop.observability.projection import ingest, list_records + + with store() as db: + ingest( + db, + "u", + "d", + { + "events": [ + { + "kind": "usage", + "object_id": "local-attempt", + "revision": 1, + "payload": { + "kind": "model", + "call_id": "attempt", + "prompt_tokens": 20, + "completion_tokens": 5, + }, + } + ] + }, + ) + ingest( + db, + "u", + "d", + { + "events": [ + { + "kind": "model", + "object_id": "attempt", + "revision": 2, + "payload": {"usage_known": False, "status": "success"}, + } + ] + }, + observation="gateway", + ) + rows, total = list_records(db, kind="model") + assert total == 1 + assert rows[0]["payload"]["usage"]["prompt_tokens"] == 20 + ingest( + db, + "u", + "d", + { + "events": [ + { + "kind": "tool", + "object_id": "t", + "revision": 1, + "payload": {"subagent_log_id": "parent", "status": "success"}, + } + ] + }, + ) + ingest( + db, + "u", + "d", + { + "events": [ + { + "kind": "tool", + "object_id": "t", + "revision": 2, + "payload": {"status": "success", "duration_ms": 10}, + } + ] + }, + observation="gateway", + ) + assert list_records(db, kind="tool")[0][0]["payload"]["subagent_log_id"] == "parent" + +def test_bulk_updates_are_uploaded_and_rollback_is_not(local_store): + from core.db.models import ChatMessage, ChatSession, UserShadow + from features.desktop.observability.sync import acknowledge, install_capture, prepare_batch + from sqlalchemy import update + + identity = { + "cloud_base": "https://cloud", + "subject": "u", + "device_id": "d", + "shell_user_center_id": "cloud:center", + } + remove = install_capture(local_store, lambda: identity) + try: + with local_store() as db: + db.add(UserShadow(user_id="local", username="U", user_center_id="cloud:center")) + db.add(ChatSession(chat_id="c", user_id="local", title="Before")) + db.add( + ChatMessage( + message_id="m", chat_id="c", chat_seq=1, role="assistant", content="partial" + ) + ) + db.commit() + with local_store() as db: + acknowledge(db, identity, {"acknowledged": prepare_batch(db, identity)["events"]}) + with local_store() as db: + db.execute( + update(ChatMessage).where(ChatMessage.message_id == "m").values(content="final") + ) + db.commit() + with local_store() as db: + batch = prepare_batch(db, identity) + assert len(batch["events"]) == 1 + assert batch["events"][0]["payload"]["content"] == "final" + acknowledge(db, identity, {"acknowledged": batch["events"]}) + with local_store() as db: + db.get(ChatMessage, "m").content = "rolled back" + db.flush() + db.rollback() + with local_store() as db: + assert prepare_batch(db, identity)["events"] == [] + finally: + remove() + +def test_account_switch_during_upload_does_not_ack_old_account(local_store, monkeypatch): + import asyncio + + import httpx + from core.harness.capability_errors import CloudUnavailable + from core.db.models import ChatSession, UserShadow + from features.desktop.bridge import client as bridge + from features.desktop.observability.sync import DesktopSyncWorker, install_capture, prepare_batch + + identity = { + "cloud_base": "http://cloud", + "subject": "alice", + "device_id": "d", + "shell_user_center_id": "cloud:alice", + } + state = {"cloud_base": "http://cloud", "token": "alice-credential", "device_id": "d"} + current = [state] + monkeypatch.setattr(bridge_identity, 'get_state', lambda: current[0]) + monkeypatch.setattr(bridge_identity, 'get_identity_state', lambda: identity) + remove = install_capture(local_store, lambda: identity) + try: + with local_store() as db: + db.add(UserShadow(user_id="local", username="Alice", user_center_id="cloud:alice")) + db.add(ChatSession(chat_id="c", user_id="local", title="Alice only")) + db.commit() + + async def switch(request): + import json + + body = json.loads(request.content) + assert request.headers["authorization"] == "Bearer alice-credential" + current[0] = {**state, "token": "bob-credential"} + return httpx.Response(200, json={"data": {"acknowledged": body["events"]}}) + + with pytest.raises(CloudUnavailable): + asyncio.run( + DesktopSyncWorker(local_store, transport=httpx.MockTransport(switch)).sync_once() + ) + with local_store() as db: + assert len(prepare_batch(db, identity)["events"]) == 1 + finally: + remove() + +def test_blocked_upload_does_not_block_task_progress(local_store, monkeypatch): + import asyncio + + import httpx + from core.db.models import ChatSession, UserShadow + from features.desktop.bridge import client as bridge + from features.desktop.observability.sync import DesktopSyncWorker, install_capture + + identity = { + "cloud_base": "http://cloud", + "subject": "u", + "device_id": "d", + "shell_user_center_id": "cloud:u", + } + state = {"cloud_base": "http://cloud", "token": "test", "device_id": "d"} + monkeypatch.setattr(bridge_identity, 'get_state', lambda: state) + monkeypatch.setattr(bridge_identity, 'get_identity_state', lambda: identity) + remove = install_capture(local_store, lambda: identity) + try: + with local_store() as db: + db.add(UserShadow(user_id="local", username="U", user_center_id="cloud:u")) + db.add(ChatSession(chat_id="c", user_id="local", title="Initial")) + db.commit() + + async def scenario(): + entered, release = asyncio.Event(), asyncio.Event() + + async def slow_cloud(request): + entered.set() + await release.wait() + return httpx.Response(200, json={"data": {"acknowledged": []}}) + + worker = DesktopSyncWorker(local_store, transport=httpx.MockTransport(slow_cloud)) + upload = asyncio.create_task(worker.sync_once()) + await asyncio.wait_for(entered.wait(), timeout=2) + + def advance(): + with local_store() as db: + db.get(ChatSession, "c").title = "Task continued" + db.commit() + + await asyncio.wait_for(asyncio.to_thread(advance), timeout=1) + assert not upload.done() + release.set() + await upload + + asyncio.run(scenario()) + finally: + remove() + +@pytest.mark.parametrize("finish", ["commit", "rollback"]) +@pytest.mark.parametrize("operation", ["flush", "update", "delete"]) +def test_capture_identity_is_stable_within_transaction_and_refreshes_on_reuse( + local_store, finish, operation +): + from core.db.models import ChatSession, UserShadow + from features.desktop.models import DesktopOutbox + from features.desktop.observability.sync import install_capture + from sqlalchemy import update, delete + + alice = dict( + cloud_base="https://cloud", + subject="alice", + device_id="d", + shell_user_center_id="center-alice", + ) + bob = dict( + cloud_base="https://cloud", subject="bob", device_id="d", shell_user_center_id="center-bob" + ) + with local_store() as db: + for name in ("alice", "bob"): + db.add(UserShadow(user_id=name, username=name, user_center_id="center-" + name)) + db.add(ChatSession(chat_id="existing", user_id="alice", title="before")) + db.commit() + identity = alice + remove = install_capture(local_store, lambda: identity) + try: + with local_store() as db: + db.begin() + identity = bob # Source transaction still belongs to its original account. + if operation == "flush": + db.add(ChatSession(chat_id="new-alice", user_id="alice", title="Alice")) + db.flush() + else: + stmt = ( + update(ChatSession).values(title="after") + if operation == "update" + else delete(ChatSession) + ) + db.execute(stmt.where(ChatSession.chat_id == "existing")) + getattr(db, finish)() + # Reuse the same Session. Neither commit nor rollback may retain Alice. + with db.begin_nested(): + db.add(ChatSession(chat_id="new-bob", user_id="bob", title="Bob")) + db.flush() + db.commit() + with local_store() as db: + rows = db.query(DesktopOutbox).all() + assert [(r.subject, r.object_id) for r in rows if r.subject == "bob"] == [ + ("bob", "new-bob") + ] + assert len([r for r in rows if r.subject == "alice"]) == ( + 1 if finish == "commit" else 0 + ) + finally: + remove() diff --git a/src/backend/tests/features/desktop/observability_support.py b/src/backend/tests/features/desktop/observability_support.py new file mode 100644 index 00000000..bb4038eb --- /dev/null +++ b/src/backend/tests/features/desktop/observability_support.py @@ -0,0 +1,32 @@ +"""Cloud management copies survive delivery retries and never become runnable chats.""" + +from features.desktop.bridge import identity as bridge_identity + +import pytest + +from sqlalchemy import create_engine + +from sqlalchemy.orm import sessionmaker + +@pytest.fixture +def store(tmp_path): + from features.desktop.models import DesktopOutbox, DesktopRecord, DesktopSyncDevice + + engine = create_engine("sqlite:///" + str(tmp_path / "cloud.db")) + from core.db.models import UserShadow + + for model in (DesktopRecord, DesktopSyncDevice, DesktopOutbox, UserShadow): + model.__table__.create(engine) + return sessionmaker(bind=engine) + +@pytest.fixture +def local_store(tmp_path): + from core.db.models import UserShadow + from features.desktop.models import DesktopOutbox + from features.desktop.observability.sync import SOURCES + + engine = create_engine("sqlite:///" + str(tmp_path / "desktop.db")) + for model in {UserShadow, DesktopOutbox, *(m for m, _ in SOURCES.values())}: + model.__table__.create(engine) + factory = sessionmaker(bind=engine) + return factory diff --git a/src/backend/tests/run_desktop_browser_e2e.py b/src/backend/tests/features/desktop/run_desktop_browser_e2e.py similarity index 99% rename from src/backend/tests/run_desktop_browser_e2e.py rename to src/backend/tests/features/desktop/run_desktop_browser_e2e.py index 5ce7fe14..eca56111 100644 --- a/src/backend/tests/run_desktop_browser_e2e.py +++ b/src/backend/tests/features/desktop/run_desktop_browser_e2e.py @@ -13,7 +13,7 @@ import httpx from playwright.sync_api import sync_playwright -ROOT = Path(__file__).resolve().parents[3] +ROOT = Path(__file__).resolve().parents[5] def port(): diff --git a/src/backend/tests/test_desktop_auth.py b/src/backend/tests/features/desktop/test_desktop_auth.py similarity index 96% rename from src/backend/tests/test_desktop_auth.py rename to src/backend/tests/features/desktop/test_desktop_auth.py index 047c8c30..c4cd90d5 100644 --- a/src/backend/tests/test_desktop_auth.py +++ b/src/backend/tests/features/desktop/test_desktop_auth.py @@ -34,7 +34,7 @@ def _run(coro): # ── Ticket store: roundtrip + single use ─────────────────────────────────────────────── def test_ticket_store_roundtrip_and_single_use(memory_sessions): - from core.auth import desktop_ticket_store as store + from features.desktop.auth import ticket_store as store token = _run(store.issue_ticket({"session_token": "raw-tok-1"})) assert token @@ -47,7 +47,7 @@ def test_ticket_store_roundtrip_and_single_use(memory_sessions): def test_ticket_store_invalid_token(memory_sessions): - from core.auth import desktop_ticket_store as store + from features.desktop.auth import ticket_store as store assert _run(store.consume_ticket("does-not-exist")) is None assert _run(store.consume_ticket("")) is None diff --git a/src/backend/tests/misc/test_desktop_bridge.py b/src/backend/tests/features/desktop/test_desktop_bridge.py similarity index 90% rename from src/backend/tests/misc/test_desktop_bridge.py rename to src/backend/tests/features/desktop/test_desktop_bridge.py index e261674c..02162c52 100644 --- a/src/backend/tests/misc/test_desktop_bridge.py +++ b/src/backend/tests/features/desktop/test_desktop_bridge.py @@ -1,15 +1,17 @@ -"""混合架构 P2：桌面壳 → 本机后端身份桥接（core.auth.desktop_bridge）。 +"""混合架构 P2：桌面壳 → 本机后端身份桥接（features.desktop.auth.bridge）。 桥接语义：秘密匹配 + 用户头合法 → 按云端 user_center_id get-or-create 本机 shadow 用户并授予 CE 管理能力；任何不满足都静默 None（回落常规认证）。 """ from __future__ import annotations +from tests.features.desktop.index_fixtures import _capability_index +from features.desktop.bridge import identity as bridge_identity import base64 import json -from core.auth import desktop_bridge +from features.desktop.auth import bridge as desktop_bridge class _Req: @@ -106,23 +108,23 @@ def test_bridge_is_idempotent_same_identity(db_session, monkeypatch): def test_expired_cloud_authorization_preserves_shell_verified_local_identity(db_session, monkeypatch, tmp_path): - from core.services import desktop_cloud_bridge as cloud + from features.desktop.bridge import client as cloud monkeypatch.setenv(desktop_bridge.BRIDGE_SECRET_ENV, "s3cret") monkeypatch.setenv("HUGAGENT_CAPS_ROOT", str(tmp_path / "caps")) - monkeypatch.setattr(cloud, "get_state", lambda: None) + monkeypatch.setattr(bridge_identity, 'get_state', lambda: None) monkeypatch.setattr( - cloud, - "get_identity_state", + bridge_identity, + 'get_identity_state', lambda: {"user_center_id": "u42", "shell_user_center_id": "cloud_user_42"}, ) request = _Req({desktop_bridge.BRIDGE_SECRET_HEADER:"s3cret", desktop_bridge.BRIDGE_USER_HEADER:_user_header()}) assert desktop_bridge.resolve_bridge_user(request, db_session).user_center_id == "cloud_user_42" - monkeypatch.setattr(cloud, "get_identity_state", lambda: None) + monkeypatch.setattr(bridge_identity, 'get_identity_state', lambda: None) assert desktop_bridge.resolve_bridge_user(request, db_session) is None def test_shell_user_center_id_mirrors_the_desktop_shell_namespace(): - from core.services.desktop_cloud_bridge import shell_user_center_id + from features.desktop.bridge.identity import shell_user_center_id assert shell_user_center_id("https://hugagent.quant-chi.com", "u-1") == "cloud:hugagent.quant-chi.com:443:u-1" assert shell_user_center_id("http://203.0.113.10:8012/", "u-1") == "cloud:203.0.113.10:8012:u-1" @@ -131,14 +133,14 @@ def test_shell_user_center_id_mirrors_the_desktop_shell_namespace(): def test_bridge_user_must_match_the_namespaced_identity(db_session, monkeypatch, tmp_path): """壳送来的标识带云端命名空间；本机只认与当前凭据身份推出的同一标识。""" - from core.services import desktop_cloud_bridge as cloud + from features.desktop.bridge import client as cloud monkeypatch.setenv(desktop_bridge.BRIDGE_SECRET_ENV, "s3cret") monkeypatch.setenv("HUGAGENT_CAPS_ROOT", str(tmp_path / "caps")) - monkeypatch.setattr(cloud, "get_state", lambda: None) + monkeypatch.setattr(bridge_identity, 'get_state', lambda: None) monkeypatch.setattr( - cloud, - "get_identity_state", + bridge_identity, + 'get_identity_state', lambda: {"user_center_id": "u42", "shell_user_center_id": "cloud:hugagent.quant-chi.com:443:u42"}, ) namespaced = _Req({ diff --git a/src/backend/tests/misc/test_desktop_capability.py b/src/backend/tests/features/desktop/test_desktop_capability.py similarity index 81% rename from src/backend/tests/misc/test_desktop_capability.py rename to src/backend/tests/features/desktop/test_desktop_capability.py index 168aa11e..a0cc2413 100644 --- a/src/backend/tests/misc/test_desktop_capability.py +++ b/src/backend/tests/features/desktop/test_desktop_capability.py @@ -8,20 +8,25 @@ """ from __future__ import annotations +from tests.features.desktop.index_fixtures import _capability_index +from features.desktop.bridge import client as bridge_client +from features.desktop.bridge import client_state as bridge_client_state +from features.desktop.bridge import gateway as bridge_gateway +from features.desktop.bridge import identity as bridge_identity import asyncio import time import pytest -from core.db.model_repository import assign_role, create_provider +from core.models.repository import assign_role, create_provider from core.db.models import AdminMcpServer -from core.services import desktop_capability as cap -from core.services import desktop_capability_configs as configs -from core.services import desktop_capability_credentials as credentials -from core.services import desktop_capability_mcp as mcp_cap -from core.services import desktop_capability_security as security -from core.services import desktop_cloud_bridge as bridge -from core.services.desktop_capability_protocol import build_manifest, canonical_hash +from features.desktop.capability import service as cap +from features.desktop.capability import configs as configs +from features.desktop.capability import credentials as credentials +from features.desktop.capability import mcp as mcp_cap +from features.desktop.capability import security as security +from features.desktop.bridge import client as bridge +from core.capabilities.manifest_protocol import build_manifest, canonical_hash from sqlalchemy.orm import sessionmaker @@ -49,7 +54,7 @@ def _fixed_secret(monkeypatch): def _issue_test_token(monkeypatch, ttl_s=600): from core.auth import session - monkeypatch.setattr(session, "_MEMORY_SESSIONS", {}) + monkeypatch.setattr(session.process_state(), 'MEMORY_SESSIONS', {}) monkeypatch.setattr(session, "_use_memory_store", lambda: True) cookie = asyncio.run( session.create_session({"user_id": "user-1", "user_center_id": "center-1"}) @@ -63,7 +68,7 @@ def _issue_test_token(monkeypatch, ttl_s=600): session_hash=digest, user_center_id="center-1", authorization_epoch=cap.session_authorization_epoch( - session._MEMORY_SESSIONS[digest]["payload"] + session.process_state().MEMORY_SESSIONS[digest]["payload"] ), ) @@ -100,12 +105,12 @@ def test_token_expiry(monkeypatch): def _activate_bridge(monkeypatch, servers): """把桥置为激活态并注入假 manifest（绕开网络与 DB）。""" - monkeypatch.setattr(bridge, "bridge_enabled", lambda: True) + monkeypatch.setattr('features.desktop.auth.bridge.bridge_enabled', lambda: True) manifest = build_manifest(servers) - monkeypatch.setattr(bridge, "get_cached_manifest", lambda: manifest) + monkeypatch.setattr(bridge_client, 'get_cached_manifest', lambda: manifest) monkeypatch.setattr( - bridge, - "get_state", + bridge_identity, + 'get_state', lambda: { "cloud_base": "https://cloud.example", "token": "dcap1.x.y", @@ -113,8 +118,8 @@ def _activate_bridge(monkeypatch, servers): }, ) monkeypatch.setattr( - bridge, - "_local_server_ids", + bridge_gateway, + 'local_server_ids', lambda: { "internet_search", "retrieve_dataset_content", @@ -172,7 +177,7 @@ def _cloud_server(server_id, source_plugin, tool_name): def test_apply_merges_cloud_and_suppresses_local(monkeypatch): _activate_bridge(monkeypatch, _CLOUD_SERVERS) - out = bridge.apply_to_enabled_mcp_ids( + out = bridge_gateway.apply_to_enabled_mcp_ids( ["internet_search", "web_fetch", "sites-site_publish", "skill-manager-skill_manager"] ) # 同名连接器默认选中当前账号的云端绑定 @@ -187,29 +192,29 @@ def test_apply_merges_cloud_and_suppresses_local(monkeypatch): def test_apply_noop_when_bridge_inactive(monkeypatch): - monkeypatch.setattr(bridge, "bridge_enabled", lambda: False) + monkeypatch.setattr('features.desktop.auth.bridge.bridge_enabled', lambda: False) ids = ["internet_search", "web_fetch"] - assert bridge.apply_to_enabled_mcp_ids(list(ids)) == ids - assert bridge.apply_to_enabled_mcp_ids(None) is None + assert bridge_gateway.apply_to_enabled_mcp_ids(list(ids)) == ids + assert bridge_gateway.apply_to_enabled_mcp_ids(None) is None def test_apply_noop_when_manifest_missing(monkeypatch): _activate_bridge(monkeypatch, []) - monkeypatch.setattr(bridge, "get_cached_manifest", lambda: None) + monkeypatch.setattr(bridge_client, 'get_cached_manifest', lambda: None) ids = ["internet_search"] - assert bridge.apply_to_enabled_mcp_ids(list(ids)) == ids + assert bridge_gateway.apply_to_enabled_mcp_ids(list(ids)) == ids def test_apply_is_idempotent(monkeypatch): _activate_bridge(monkeypatch, _CLOUD_SERVERS) - once = bridge.apply_to_enabled_mcp_ids(["internet_search", "web_fetch"]) - twice = bridge.apply_to_enabled_mcp_ids(list(once)) + once = bridge_gateway.apply_to_enabled_mcp_ids(["internet_search", "web_fetch"]) + twice = bridge_gateway.apply_to_enabled_mcp_ids(list(once)) assert once == twice def test_cloud_gateway_configs_shape(monkeypatch): _activate_bridge(monkeypatch, _CLOUD_SERVERS) - cfgs = bridge.cloud_gateway_mcp_configs() + cfgs = bridge_gateway.cloud_gateway_mcp_configs() sid = "industry-knowledge-center-ai_chain_information_mcp" assert sid in cfgs cfg = cfgs[sid] @@ -230,22 +235,22 @@ def test_bridge_account_switch_clears_previous_manifest(monkeypatch): from core.db import engine monkeypatch.setattr( - bridge, - "_state", + bridge_client_state, + 'state', { "cloud_base": "https://cloud-a.example", "token": "token-a", "expires_at": time.time() + 3600, }, ) - monkeypatch.setattr(bridge, "_state_loaded", True) - monkeypatch.setattr(bridge, "_manifest", build_manifest([])) - monkeypatch.setattr(bridge, "_manifest_ts", 123.0) - monkeypatch.setattr(bridge, "_manifest_error", "old error") + monkeypatch.setattr(bridge_client_state, 'state_loaded', True) + monkeypatch.setattr(bridge_client_state, 'manifest', build_manifest([])) + monkeypatch.setattr(bridge_client_state, 'manifest_ts', 123.0) + monkeypatch.setattr(bridge_client_state, 'manifest_error', "old error") refreshed = [] monkeypatch.setattr( - bridge, - "_refresh_manifest_async", + bridge_client, + 'refresh_manifest_async', lambda force=False: refreshed.append(force), ) monkeypatch.setattr( @@ -254,11 +259,11 @@ def test_bridge_account_switch_clears_previous_manifest(monkeypatch): lambda: (_ for _ in ()).throw(RuntimeError("database unavailable")), ) - bridge.set_state("https://cloud-b.example", "token-b", 3600) + bridge_client.set_state("https://cloud-b.example", "token-b", 3600) - assert bridge._manifest is None - assert bridge._manifest_ts == 0.0 - assert bridge._manifest_error is None + assert bridge_client_state.manifest is None + assert bridge_client_state.manifest_ts == 0.0 + assert bridge_client_state.manifest_error is None assert refreshed == [True] @@ -271,7 +276,6 @@ def test_capability_manifest_contains_current_sanitized_schemas(monkeypatch, db_ transport="streamable_http", url="https://mcp.example/mcp?api_key=must-not-leak", headers={"Authorization": "encrypted-secret"}, - is_stable=False, is_enabled=True, tools_json=[ { @@ -323,7 +327,7 @@ def test_capability_manifest_contains_current_sanitized_schemas(monkeypatch, db_ def test_resolve_gateway_tool_rejects_a_stale_schema(monkeypatch, db_session): - from core.services.desktop_capability_protocol import CapabilityManifestStaleError + from core.capabilities.manifest_protocol import CapabilityManifestStaleError _use_test_database(monkeypatch, db_session) tools = [ @@ -340,7 +344,6 @@ def test_resolve_gateway_tool_rejects_a_stale_schema(monkeypatch, db_session): description="", transport="streamable_http", url="https://mcp.example/mcp", - is_stable=False, is_enabled=True, tools_json=tools, ) @@ -377,10 +380,10 @@ def test_resolve_gateway_tool_rejects_a_stale_schema(monkeypatch, db_session): ) def test_cloud_tool_remains_available_without_local_launcher(monkeypatch, server_id): _activate_bridge(monkeypatch, [_cloud_server(server_id, None, "test_tool")]) - monkeypatch.setattr(bridge, "_local_server_ids", lambda: set()) - enabled = bridge.apply_to_enabled_mcp_ids([]) + monkeypatch.setattr(bridge_gateway, 'local_server_ids', lambda: set()) + enabled = bridge_gateway.apply_to_enabled_mcp_ids([]) assert server_id in enabled - configs = bridge.cloud_gateway_mcp_configs(enabled) + configs = bridge_gateway.cloud_gateway_mcp_configs(enabled) assert server_id in configs assert configs[server_id]["schema_source"] == "cloud_manifest" @@ -403,16 +406,16 @@ def test_legacy_environment_cannot_hide_authorized_cloud_tools(monkeypatch, lega monkeypatch.setenv("DESKTOP_LOCAL_MCP_KEEP", ",".join(s["server_id"] for s in servers)) if legacy_setting in ("switch", "both"): monkeypatch.setenv("DESKTOP_CLOUD_MCP_BRIDGE_ENABLED", "0") - monkeypatch.setattr(bridge, "_local_server_ids", lambda: {s["server_id"] for s in servers}) - enabled = bridge.apply_to_enabled_mcp_ids([s["server_id"] for s in servers]) - configs = bridge.cloud_gateway_mcp_configs(enabled) + monkeypatch.setattr(bridge_gateway, 'local_server_ids', lambda: {s["server_id"] for s in servers}) + enabled = bridge_gateway.apply_to_enabled_mcp_ids([s["server_id"] for s in servers]) + configs = bridge_gateway.cloud_gateway_mcp_configs(enabled) assert set(configs) == {s["server_id"] for s in servers} assert all(c["schema_source"] == "cloud_manifest" for c in configs.values()) - assert bridge.bridge_active() + assert bridge_identity.bridge_active() def test_fresh_manifests_bypass_worker_mcp_caches(monkeypatch, db_session): - from core.services import mcp_service + from core.capabilities.mcp import service as mcp_service _use_test_database(monkeypatch, db_session) monkeypatch.setattr(mcp_service, "SessionLocal", sessionmaker(bind=db_session.get_bind())) @@ -437,7 +440,7 @@ def test_fresh_manifests_bypass_worker_mcp_caches(monkeypatch, db_session): def test_other_users_private_connector_does_not_change_manifest(monkeypatch, db_session): - from core.services import mcp_service + from core.capabilities.mcp import service as mcp_service _use_test_database(monkeypatch, db_session) monkeypatch.setattr(mcp_service, "SessionLocal", sessionmaker(bind=db_session.get_bind())) diff --git a/src/backend/tests/misc/test_desktop_capability_models.py b/src/backend/tests/features/desktop/test_desktop_capability_models.py similarity index 91% rename from src/backend/tests/misc/test_desktop_capability_models.py rename to src/backend/tests/features/desktop/test_desktop_capability_models.py index 0d6bee99..1d2efa01 100644 --- a/src/backend/tests/misc/test_desktop_capability_models.py +++ b/src/backend/tests/features/desktop/test_desktop_capability_models.py @@ -8,21 +8,22 @@ """ from __future__ import annotations +from tests.features.desktop.index_fixtures import _capability_index import asyncio import time import pytest -from core.db.model_repository import assign_role, create_provider +from core.models.repository import assign_role, create_provider from core.db.models import AdminMcpServer -from core.services import desktop_capability as cap -from core.services import desktop_capability_configs as configs -from core.services import desktop_capability_credentials as credentials -from core.services import desktop_capability_models as models -from core.services import desktop_capability_security as security -from core.services import desktop_cloud_bridge as bridge -from core.services import model_config -from core.services.desktop_capability_protocol import build_manifest, canonical_hash +from features.desktop.capability import service as cap +from features.desktop.capability import configs as configs +from features.desktop.capability import credentials as credentials +from features.desktop.capability import models as models +from features.desktop.capability import security as security +from features.desktop.bridge import client as bridge +from core.models import config as model_config +from core.capabilities.manifest_protocol import build_manifest, canonical_hash from sqlalchemy.orm import sessionmaker # ── 模型能力清单 / 网关授权 ─────────────────────────────────────── @@ -43,7 +44,7 @@ def _use_test_database(monkeypatch, db_session): def test_model_manifest_contains_no_upstream_credentials(monkeypatch, db_session): _use_test_database(monkeypatch, db_session) - from core.services import user_model_selection + from core.auth import model_selection as user_model_selection monkeypatch.setattr(user_model_selection, "user_can_switch_model", lambda _db, _uid: False) assigned = create_provider( @@ -91,7 +92,7 @@ def test_credential_equal_to_a_public_model_identifier_is_not_a_secret(monkeypat """api_key 与 model_name 字面相同：模型名对所有登录用户可见，不是机密。 这样的模型照常下发（生产上 main_agent 就绑在这种配置上），清单仍通过出口守卫。""" _use_test_database(monkeypatch, db_session) - from core.services import user_model_selection + from core.auth import model_selection as user_model_selection monkeypatch.setattr(user_model_selection, "user_can_switch_model", lambda _db, _uid: False) keyless = create_provider( @@ -169,7 +170,7 @@ def test_model_manifest_withholds_only_the_provider_that_would_leak_a_real_crede ): """另一条模型的真实密钥出现在某模型的展示名里：只扣留这一条并点名字段，其余照常下发。""" _use_test_database(monkeypatch, db_session) - from core.services import user_model_selection + from core.auth import model_selection as user_model_selection monkeypatch.setattr(user_model_selection, "user_can_switch_model", lambda _db, _uid: False) healthy = create_provider( @@ -207,7 +208,7 @@ def test_model_manifest_withholds_only_the_provider_that_would_leak_a_real_crede def test_model_gateway_target_is_role_or_user_switch_allowlisted(monkeypatch, db_session): _use_test_database(monkeypatch, db_session) - from core.services import user_model_selection + from core.auth import model_selection as user_model_selection assigned = create_provider( db_session, diff --git a/src/backend/tests/misc/test_desktop_cloud_skills.py b/src/backend/tests/features/desktop/test_desktop_cloud_skills.py similarity index 84% rename from src/backend/tests/misc/test_desktop_cloud_skills.py rename to src/backend/tests/features/desktop/test_desktop_cloud_skills.py index e214cff2..88cceb78 100644 --- a/src/backend/tests/misc/test_desktop_cloud_skills.py +++ b/src/backend/tests/features/desktop/test_desktop_cloud_skills.py @@ -1,6 +1,13 @@ """桌面双端技能：清单 → 安装意图；同一轮同步即下载发布 → 存储层 revision → 运行视图联接；无镜像拷贝。""" from __future__ import annotations +from tests.features.desktop.index_fixtures import _capability_index +from features.desktop.bridge import gateway as bridge_gateway +from features.desktop.bridge import identity as bridge_identity +from features.desktop.capabilities import skill_account +from features.desktop.capabilities import skill_paths +from features.desktop.capabilities import skill_publication +from features.desktop.capabilities import skills import base64 import io @@ -10,19 +17,16 @@ from pathlib import Path import pytest -from core.agent_skills import config as skill_config -from core.capabilities import junction, registry, skills, store -from core.capabilities.paths import KIND_SKILL -from core.capabilities.ref import profile_id -from core.services import desktop_cloud_bridge as bridge -from core.services import desktop_cloud_skills as cloud_skills -from core.services.desktop_capability_protocol import ( - CapabilityManifestError, - build_skill_manifest, - skill_content_hash, - token_subject, - validate_skill_manifest, -) +from core.capabilities.skills import config as skill_config +from core.infra import directory_links as junction +from features.desktop.capabilities import registry +from features.desktop.capabilities import skills +from features.desktop.capabilities import store +from core.capabilities.identity_rules import KIND_SKILL +from core.capabilities.reference import profile_id +from features.desktop.bridge import client as bridge +from features.desktop.bridge import skills as cloud_skills +from core.capabilities.manifest_protocol import CapabilityManifestError, build_skill_manifest, skill_content_hash, token_subject, validate_skill_manifest def _skill_md(skill_id: str, body: str = "do it") -> str: @@ -95,9 +99,9 @@ def test_token_subject_and_state_fingerprint_survive_rotation(): {"cloud_base": "https://c", "token": _token("u-1")}, {"cloud_base": "https://c", "token": _token("u-1")}, ] - assert bridge._state_fingerprint(same_user[0]) == bridge._state_fingerprint(same_user[1]) + assert bridge_identity.state_fingerprint(same_user[0]) == bridge_identity.state_fingerprint(same_user[1]) other = {"cloud_base": "https://c", "token": _token("u-2")} - assert bridge._state_fingerprint(same_user[0]) != bridge._state_fingerprint(other) + assert bridge_identity.state_fingerprint(same_user[0]) != bridge_identity.state_fingerprint(other) # ── 本机侧同步 ───────────────────────────────────────────────────────── @@ -156,14 +160,14 @@ def dirs(tmp_path, monkeypatch, index_db): (builtin / "ppt-design").mkdir(parents=True) (builtin / "ppt-design" / "SKILL.md").write_text(_skill_md("ppt-design", "old")) monkeypatch.setattr(skill_config, "_builtin_skills_dir", lambda: builtin) - monkeypatch.setattr(skills, "builtin_dir", lambda: builtin) - bridge.reset_for_tests() - monkeypatch.setattr(bridge, "bridge_enabled", lambda: True) - monkeypatch.setattr(bridge, "get_state", lambda: dict(_STATE, expires_at=time.time() + 60)) - monkeypatch.setattr(skills, "current_local_user_id", lambda: "u") - monkeypatch.setattr("core.agent_skills.cache_refresh.refresh_skill_caches", lambda: None) + monkeypatch.setattr(skill_paths, 'builtin_dir', lambda: builtin) + bridge_identity.reset_for_tests() + monkeypatch.setattr('features.desktop.auth.bridge.bridge_enabled', lambda: True) + monkeypatch.setattr(bridge_identity, 'get_state', lambda: dict(_STATE, expires_at=time.time() + 60)) + monkeypatch.setattr(skill_account, 'current_local_user_id', lambda: "u") + monkeypatch.setattr('core.capabilities.skills.cache_refresh.refresh_skill_caches', lambda: None) yield tmp_path - bridge.reset_for_tests() + bridge_identity.reset_for_tests() @pytest.fixture @@ -226,7 +230,7 @@ def test_sync_downloads_everything_in_the_manifest(dirs, monkeypatch): def test_prepare_publishes_revision_and_links_view(dirs, monkeypatch): - monkeypatch.setattr(skills, "current_local_user_id", lambda: "local-user") + monkeypatch.setattr(skill_account, 'current_local_user_id', lambda: "local-user") fake = _cloud(monkeypatch, {"market-x": {"scripts/a.py": "print()"}}) cloud_skills.sync_blocking(_STATE) results = cloud_skills.prepare(_STATE, [_iid("market-x")]) @@ -297,18 +301,18 @@ def test_apply_to_enabled_skill_ids_follows_cloud(dirs, monkeypatch): ) cloud_skills.sync_blocking(_STATE) # 同步即就绪：账号里的技能同一轮全部装好，停用的装了但不进本轮启用集 - out = bridge.apply_to_enabled_skill_ids(["word-editing", "ppt-design", "local-only"]) + out = bridge_gateway.apply_to_enabled_skill_ids(["word-editing", "ppt-design", "local-only"]) assert out == ["ppt-design", "local-only", "market-x"] - assert bridge.apply_to_enabled_skill_ids(list(out)) == out - assert bridge.apply_to_enabled_skill_ids(None) is None + assert bridge_gateway.apply_to_enabled_skill_ids(list(out)) == out + assert bridge_gateway.apply_to_enabled_skill_ids(None) is None def test_apply_noop_when_bridge_inactive_or_unsynced(dirs, monkeypatch): ids = ["ppt-design", "local-only"] - monkeypatch.setattr(bridge, "bridge_enabled", lambda: False) - assert bridge.apply_to_enabled_skill_ids(list(ids)) == ids - monkeypatch.setattr(bridge, "bridge_enabled", lambda: True) - assert bridge.apply_to_enabled_skill_ids(list(ids)) == ids # manifest 尚未同步 + monkeypatch.setattr('features.desktop.auth.bridge.bridge_enabled', lambda: False) + assert bridge_gateway.apply_to_enabled_skill_ids(list(ids)) == ids + monkeypatch.setattr('features.desktop.auth.bridge.bridge_enabled', lambda: True) + assert bridge_gateway.apply_to_enabled_skill_ids(list(ids)) == ids # manifest 尚未同步 def test_cloud_disabled_skill_is_still_installed(dirs, monkeypatch): @@ -318,7 +322,7 @@ def test_cloud_disabled_skill_is_still_installed(dirs, monkeypatch): inst = registry.get(_iid("market-x")) assert inst.state == "ready" and inst.enabled is False assert store.revisions(KIND_SKILL, _PROFILE, "market-x") - assert bridge.apply_to_enabled_skill_ids(["other"]) == ["other"] + assert bridge_gateway.apply_to_enabled_skill_ids(["other"]) == ["other"] def test_device_toggle_is_not_overwritten_by_a_later_sync(dirs, monkeypatch): @@ -342,7 +346,7 @@ def test_account_switch_keeps_files_isolated_per_profile(dirs, monkeypatch): assert store.revisions(KIND_SKILL, _PROFILE, "my-private") other = dict(_STATE, token=_token("u-2")) - monkeypatch.setattr(bridge, "get_state", lambda: dict(other, expires_at=time.time() + 60)) + monkeypatch.setattr(bridge_identity, 'get_state', lambda: dict(other, expires_at=time.time() + 60)) cloud_skills.on_account_switch() _cloud(monkeypatch, {}) cloud_skills.sync_blocking(other) @@ -365,7 +369,7 @@ def test_local_fork_takes_the_name_by_preference(dirs, monkeypatch): cloud_skills.sync_blocking(_STATE) cloud_skills.prepare(_STATE, [_iid("ppt-design")]) md = _skill_md("ppt-design", "mine") - comp = skills.publish_local_skill( + comp = skill_publication.publish_local_skill( "ppt-design", files={"SKILL.md": md}, content_hash=skill_content_hash(md, {}), owner_user_id="u" ) res = skills.resolve_for_user("u") diff --git a/src/backend/tests/test_desktop_device_login.py b/src/backend/tests/features/desktop/test_desktop_device_login.py similarity index 97% rename from src/backend/tests/test_desktop_device_login.py rename to src/backend/tests/features/desktop/test_desktop_device_login.py index a16865cc..a1e7293b 100644 --- a/src/backend/tests/test_desktop_device_login.py +++ b/src/backend/tests/features/desktop/test_desktop_device_login.py @@ -16,8 +16,9 @@ def client(request, monkeypatch, tmp_path): import subprocess import time from redis.asyncio import Redis - from core.auth import desktop_login_store as store, session - from api.routes.v1.desktop_login import router + from features.desktop.auth import login_store as store + from core.auth import session + from features.desktop.api.desktop_login import router previous = settings.session mode = request.param object.__setattr__(settings, "session", dataclasses.replace(previous, store_type=mode)) @@ -105,7 +106,7 @@ def test_cancel_or_deny_prevents_delivery(client, terminal): def test_expiry_and_revoked_browser_session(client, monkeypatch): - from core.auth import desktop_login_store as store + from features.desktop.auth import login_store as store from core.auth.session import revoke_session grant, secret = begin(client) path = "/v1/auth/desktop/requests/" + grant["request_id"] @@ -158,7 +159,7 @@ def test_wrong_code_missing_cookie_and_foreign_account(client): def test_rate_limit_and_no_store(client): - from core.auth import desktop_login_store as store + from features.desktop.auth import login_store as store secret = secrets.token_hex(32) for _ in range(10): r = client.post("/v1/auth/desktop/requests", json={"device_secret": secret}) diff --git a/src/backend/tests/test_desktop_memory_persistence.py b/src/backend/tests/features/desktop/test_desktop_memory_persistence.py similarity index 96% rename from src/backend/tests/test_desktop_memory_persistence.py rename to src/backend/tests/features/desktop/test_desktop_memory_persistence.py index fd2a95e8..6be3335f 100644 --- a/src/backend/tests/test_desktop_memory_persistence.py +++ b/src/backend/tests/features/desktop/test_desktop_memory_persistence.py @@ -59,7 +59,7 @@ def do_POST(self): root, url, phase = Path(sys.argv[1]), sys.argv[2], sys.argv[3] from core.memory import backend -from core.services.model_config import ModelConfigService +from core.models.config import ModelConfigService memory_settings = SimpleNamespace( enabled=True, graph_enabled=False, @@ -77,8 +77,8 @@ def do_POST(self): side_effect=RuntimeError("injected collection load failure")): for attempt in range(2): assert backend._get_memory() is None - assert backend._memory_init_failed - assert backend._memory_instance is None + assert backend.process_state().memory_init_failed + assert backend.process_state().memory_instance is None assert {alias for alias, handler in connections.list_connections() if handler} == connected memory = backend._get_memory() assert memory is not None @@ -133,7 +133,7 @@ def do_POST(self): memory.llm.client.close() memory.close() """ - backend_path = Path(__file__).resolve().parents[1] + backend_path = Path(__file__).resolve().parents[3] env = { **os.environ, "PYTHONPATH": os.pathsep.join(filter(None, (str(backend_path), os.getenv("PYTHONPATH")))), diff --git a/src/backend/tests/api/test_desktop_model_gateway.py b/src/backend/tests/features/desktop/test_desktop_model_gateway.py similarity index 88% rename from src/backend/tests/api/test_desktop_model_gateway.py rename to src/backend/tests/features/desktop/test_desktop_model_gateway.py index 45ff6be9..043b0c1d 100644 --- a/src/backend/tests/api/test_desktop_model_gateway.py +++ b/src/backend/tests/features/desktop/test_desktop_model_gateway.py @@ -1,13 +1,16 @@ from __future__ import annotations +from features.desktop.api import desktop_capability +from features.desktop.api import capability_dependencies +from features.desktop.api import desktop_capability_state import asyncio import httpx import pytest -from api.routes.v1 import desktop_capability as routes -from core.services import desktop_capability as service -from core.services import desktop_capability_models as models -from core.services import desktop_capability_security as security +from features.desktop.api import desktop_capability as routes +from features.desktop.capability import service as service +from features.desktop.capability import models as models +from features.desktop.capability import security as security from fastapi import FastAPI from fastapi.testclient import TestClient @@ -50,11 +53,11 @@ async def upstream(request: httpx.Request) -> httpx.Response: monkeypatch.setattr(service, "resolve_model_gateway_target", resolve_target) gateway_client = httpx.AsyncClient(transport=httpx.MockTransport(upstream)) - monkeypatch.setattr(routes, "_gateway_client", gateway_client) + monkeypatch.setattr(desktop_capability_state, 'gateway_client', gateway_client) app = FastAPI() - app.include_router(routes.router) - app.dependency_overrides[routes._require_capability_user] = lambda: "user-1" + app.include_router(desktop_capability.router) + app.dependency_overrides[capability_dependencies.require_capability_user] = lambda: "user-1" with TestClient(app) as client: response = client.post( "/v1/desktop/capability/gateway/models/provider-1/chat/completions", @@ -90,8 +93,8 @@ def test_model_gateway_rejects_wrong_protocol_path(monkeypatch): }, ) app = FastAPI() - app.include_router(routes.router) - app.dependency_overrides[routes._require_capability_user] = lambda: "user-1" + app.include_router(desktop_capability.router) + app.dependency_overrides[capability_dependencies.require_capability_user] = lambda: "user-1" with TestClient(app) as client: response = client.post( "/v1/desktop/capability/gateway/models/provider-1/embeddings", @@ -104,8 +107,8 @@ def test_manifest_endpoint_supports_revision_revalidation(monkeypatch): manifest = {"version": 2, "revision": "a" * 64, "servers": []} monkeypatch.setattr(service, "build_user_capability_manifest", lambda _uid, **_kwargs: manifest) app = FastAPI() - app.include_router(routes.router) - app.dependency_overrides[routes._require_capability_user] = lambda: "user-1" + app.include_router(desktop_capability.router) + app.dependency_overrides[capability_dependencies.require_capability_user] = lambda: "user-1" with TestClient(app) as client: first = client.get("/v1/desktop/capability/manifest") @@ -148,10 +151,10 @@ async def upstream(request: httpx.Request) -> httpx.Response: ) gateway_client = httpx.AsyncClient(transport=httpx.MockTransport(upstream)) - monkeypatch.setattr(routes, "_gateway_client", gateway_client) + monkeypatch.setattr(desktop_capability_state, 'gateway_client', gateway_client) app = FastAPI() - app.include_router(routes.router) - app.dependency_overrides[routes._require_capability_user] = lambda: "user-1" + app.include_router(desktop_capability.router) + app.dependency_overrides[capability_dependencies.require_capability_user] = lambda: "user-1" with TestClient(app) as client: response = client.post( @@ -199,8 +202,8 @@ async def invoke(resolved, arguments, runtime_headers): monkeypatch.setattr(service, "resolve_gateway_tool", resolve) monkeypatch.setattr(service, "invoke_gateway_tool", invoke) app = FastAPI() - app.include_router(routes.router) - app.dependency_overrides[routes._require_capability_user] = lambda: "user-1" + app.include_router(desktop_capability.router) + app.dependency_overrides[capability_dependencies.require_capability_user] = lambda: "user-1" with TestClient(app) as client: response = client.post( @@ -320,12 +323,12 @@ async def upstream(request: httpx.Request) -> httpx.Response: # 出口脱敏集合与本用例无关；品牌分支在这里还会查凭据策略，测试环境没有。 monkeypatch.setattr(service, "gateway_stream_secrets", lambda *_a, **_k: set()) monkeypatch.setattr( - routes, "_gateway_client", httpx.AsyncClient(transport=httpx.MockTransport(upstream)) + desktop_capability_state, 'gateway_client', httpx.AsyncClient(transport=httpx.MockTransport(upstream)) ) app = FastAPI() - app.include_router(routes.router) - app.dependency_overrides[routes._require_capability_user] = lambda: "user-1" + app.include_router(desktop_capability.router) + app.dependency_overrides[capability_dependencies.require_capability_user] = lambda: "user-1" with TestClient(app) as client: response = client.post( "/v1/desktop/capability/gateway/models/provider-1/responses", @@ -355,8 +358,8 @@ def test_a_chat_completions_model_still_rejects_a_responses_request(monkeypatch) monkeypatch.setattr(models, "_model_provider_allowed", lambda *_a, **_k: True) app = FastAPI() - app.include_router(routes.router) - app.dependency_overrides[routes._require_capability_user] = lambda: "user-1" + app.include_router(desktop_capability.router) + app.dependency_overrides[capability_dependencies.require_capability_user] = lambda: "user-1" with TestClient(app) as client: response = client.post( "/v1/desktop/capability/gateway/models/provider-2/responses", diff --git a/src/backend/tests/features/desktop/test_desktop_observability.py b/src/backend/tests/features/desktop/test_desktop_observability.py new file mode 100644 index 00000000..1ceb74c1 --- /dev/null +++ b/src/backend/tests/features/desktop/test_desktop_observability.py @@ -0,0 +1,20 @@ +"""Collect desktop observability scenarios with stable pytest names.""" +from tests.features.desktop.observability_support import store, local_store + +from tests.features.desktop.observability_cases_1 import ( + test_duplicate_and_stale_delivery_keep_newest_chat, + test_capture_upload_and_conditional_ack_survive_retry, + test_cloud_gateway_and_desktop_receipt_are_one_call, + test_gateway_stream_arrives_before_log_writer_and_extracts_usage, + test_real_upload_endpoint_authorization_and_size_limit, + test_offline_restart_and_http_delivery_to_cloud, + test_unicode_large_records_and_secrets_are_safe, +) + +from tests.features.desktop.observability_cases_2 import ( + test_gateway_unknown_usage_keeps_desktop_usage_and_parent, + test_bulk_updates_are_uploaded_and_rollback_is_not, + test_account_switch_during_upload_does_not_ack_old_account, + test_blocked_upload_does_not_block_task_progress, + test_capture_identity_is_stable_within_transaction_and_refreshes_on_reuse, +) diff --git a/src/backend/tests/test_desktop_observability_authenticated_delivery.py b/src/backend/tests/features/desktop/test_desktop_observability_authenticated_delivery.py similarity index 77% rename from src/backend/tests/test_desktop_observability_authenticated_delivery.py rename to src/backend/tests/features/desktop/test_desktop_observability_authenticated_delivery.py index b7a2fe38..dbf8ddb2 100644 --- a/src/backend/tests/test_desktop_observability_authenticated_delivery.py +++ b/src/backend/tests/features/desktop/test_desktop_observability_authenticated_delivery.py @@ -1,4 +1,9 @@ +from features.desktop.bridge import client as bridge_client +from features.desktop.bridge import client_state as bridge_client_state +from features.desktop.bridge import identity as bridge_identity +from features.desktop.api import desktop_capability """Signed cloud authentication and shell publication recover a durable desktop outbox.""" +from features.desktop.api import desktop_capability_state import asyncio import time @@ -12,22 +17,17 @@ def test_expired_credential_renews_via_apis_and_preserves_account_outboxes(tmp_path, monkeypatch): - from api.routes.v1 import desktop_capability as control - from api.routes.v1 import desktop_observability as upload + from features.desktop.api import desktop_capability as control + from features.desktop.api import desktop_observability as upload from core.auth import session from core.config.settings import settings from core.db.models import ChatSession, ContentBlock, UserShadow - from core.db.models.observability import DesktopOutbox, DesktopRecord, DesktopSyncDevice - from core.services import desktop_capability as capability - from core.services import desktop_cloud_bridge as bridge - from core.services import desktop_cloud_bundles, desktop_cloud_skills - from core.services.desktop_observability import list_records - from core.services.desktop_observability_sync import ( - SOURCES, - DesktopSyncWorker, - install_capture, - prepare_batch, - ) + from features.desktop.models import DesktopOutbox, DesktopRecord, DesktopSyncDevice + from features.desktop.capability import service as capability + from features.desktop.bridge import client as bridge + from features.desktop.bridge import bundles as desktop_cloud_bundles, skills as desktop_cloud_skills + from features.desktop.observability.projection import list_records + from features.desktop.observability.sync import SOURCES, DesktopSyncWorker, install_capture, prepare_batch from fastapi import FastAPI local_engine = create_engine("sqlite:///" + str(tmp_path / "local.db")) @@ -46,27 +46,27 @@ def test_expired_credential_renews_via_apis_and_preserves_account_outboxes(tmp_p monkeypatch.setattr( session, "settings", SimpleNamespace(session=replace(settings.session, store_type="memory")) ) - monkeypatch.setattr(session, "_MEMORY_SESSIONS", {}) + monkeypatch.setattr(session.process_state(), 'MEMORY_SESSIONS', {}) monkeypatch.delenv("DESKTOP_CAPABILITY_ISSUER", raising=False) monkeypatch.setenv("HUGAGENT_DESKTOP_BRIDGE_SECRET", "isolated-shell-process-secret") - monkeypatch.setattr(bridge, "_state", None) - monkeypatch.setattr(bridge, "_state_loaded", True) - monkeypatch.setattr(bridge, "_credential_rejected", "") + monkeypatch.setattr(bridge_client_state, 'state', None) + monkeypatch.setattr(bridge_client_state, 'state_loaded', True) + monkeypatch.setattr(bridge_client_state, 'credential_rejected', "") # These manage unrelated capability files/background network activity. - monkeypatch.setattr(bridge, "_purge_persisted_state", lambda: None) - monkeypatch.setattr(bridge, "_clear_partial_selection", lambda: None) - monkeypatch.setattr(bridge, "_rebuild_identity_views", lambda: None) + monkeypatch.setattr(bridge_identity, 'purge_persisted_state', lambda: None) + monkeypatch.setattr(bridge_identity, 'clear_partial_selection', lambda: None) + monkeypatch.setattr(bridge_identity, 'rebuild_identity_views', lambda: None) monkeypatch.setattr(desktop_cloud_skills, "on_account_switch", lambda: None) monkeypatch.setattr(desktop_cloud_bundles, "on_account_switch", lambda: None) refreshes = [] - monkeypatch.setattr(bridge, "_refresh_manifest_async", lambda **kw: refreshes.append(kw)) + monkeypatch.setattr(bridge_client, 'refresh_manifest_async', lambda **kw: refreshes.append(kw)) cloud_clock = [time.time()] monkeypatch.setattr(capability, "time", SimpleNamespace(time=lambda: cloud_clock[0])) cloud, local = FastAPI(), FastAPI() - cloud.include_router(control.router, prefix="/api") + cloud.include_router(desktop_capability.router, prefix="/api") cloud.include_router(upload.router, prefix="/api") - local.include_router(control.router, prefix="/api") - remove = install_capture(local_factory, bridge.get_identity_state) + local.include_router(desktop_capability.router, prefix="/api") + remove = install_capture(local_factory, bridge_identity.get_identity_state) async def scenario(): async with ( @@ -106,7 +106,7 @@ async def publish(data): expired = await issue("alice") await publish(expired) - alice_identity = bridge.get_identity_state() + alice_identity = bridge_identity.get_identity_state() with local_factory() as db: db.add( UserShadow( @@ -128,7 +128,7 @@ async def publish(data): with pytest.raises(httpx.HTTPStatusError) as rejected: await worker.sync_once() assert rejected.value.response.status_code == 401 - assert bridge._credential_rejected + assert bridge_client_state.credential_rejected with local_factory() as db: assert len(prepare_batch(db, alice_identity)["events"]) == 1 with cloud_factory() as db: @@ -138,7 +138,7 @@ async def publish(data): assert renewed["token"] != expired["token"] await publish(renewed) assert len(refreshes) == before_refresh + 1 - assert not bridge._credential_rejected + assert not bridge_client_state.credential_rejected await worker.sync_once() with local_factory() as db: assert prepare_batch(db, alice_identity)["events"] == [] diff --git a/src/backend/tests/test_desktop_observability_delivery.py b/src/backend/tests/features/desktop/test_desktop_observability_delivery.py similarity index 75% rename from src/backend/tests/test_desktop_observability_delivery.py rename to src/backend/tests/features/desktop/test_desktop_observability_delivery.py index 64a1bfe1..04388dcc 100644 --- a/src/backend/tests/test_desktop_observability_delivery.py +++ b/src/backend/tests/features/desktop/test_desktop_observability_delivery.py @@ -1,4 +1,7 @@ """Delivery failures keep the durable outbox and expose only safe diagnostics.""" +from features.desktop.bridge import client as bridge_client +from features.desktop.bridge import client_state as bridge_client_state +from features.desktop.bridge import identity as bridge_identity import asyncio import base64 @@ -12,14 +15,9 @@ from sqlalchemy.orm import sessionmaker from core.db.models import ChatSession, UserShadow -from core.db.models.observability import DesktopOutbox, DesktopRecord, DesktopSyncDevice -from core.services import desktop_cloud_bridge as bridge -from core.services.desktop_observability_sync import ( - SOURCES, - DesktopSyncWorker, - install_capture, - prepare_batch, -) +from features.desktop.models import DesktopOutbox, DesktopRecord, DesktopSyncDevice +from features.desktop.bridge import client as bridge +from features.desktop.observability.sync import SOURCES, DesktopSyncWorker, install_capture, prepare_batch def token(user="alice", nonce="initial"): @@ -48,10 +46,10 @@ def delivery(tmp_path, monkeypatch): device_id="d", shell_user_center_id="cloud:alice", ) - monkeypatch.setattr(bridge, "_state", state) - monkeypatch.setattr(bridge, "_state_loaded", True) - monkeypatch.setattr(bridge, "_credential_rejected", "") - monkeypatch.setattr(bridge, "get_identity_state", lambda: identity) + monkeypatch.setattr(bridge_client_state, 'state', state) + monkeypatch.setattr(bridge_client_state, 'state_loaded', True) + monkeypatch.setattr(bridge_client_state, 'credential_rejected', "") + monkeypatch.setattr(bridge_identity, 'get_identity_state', lambda: identity) remove = install_capture(factory, lambda: identity) with factory() as db: db.add(UserShadow(user_id="local", username="Alice", user_center_id="cloud:alice")) @@ -87,7 +85,7 @@ async def fail(request): asyncio.run(worker.sync_once()) with factory() as db: assert len(prepare_batch(db, identity)["events"]) == 1 - assert bool(bridge._credential_rejected) == (failure == 401) + assert bool(bridge_client_state.credential_rejected) == (failure == 401) assert "elapsed_ms=" in caplog.text assert "batch_bytes=" in caplog.text and "event_count=1" in caplog.text assert trace_ids[0] in caplog.text @@ -102,8 +100,8 @@ async def fail(request): def test_401_renewal_retries_to_real_cloud_endpoint_without_losing_records( delivery, tmp_path, monkeypatch ): - from api.routes.v1 import desktop_observability as route - from core.services.desktop_observability import list_records + from features.desktop.api import desktop_observability as route + from features.desktop.observability.projection import list_records from fastapi import FastAPI factory, identity, state = delivery @@ -127,15 +125,15 @@ async def endpoint(request): worker = DesktopSyncWorker(factory, transport=httpx.MockTransport(endpoint)) with pytest.raises(httpx.HTTPStatusError): asyncio.run(worker.sync_once()) - assert bridge._credential_rejected - monkeypatch.setattr(bridge, "_purge_persisted_state", lambda: None) - monkeypatch.setattr(bridge, "_rebuild_identity_views", lambda: None) - monkeypatch.setattr(bridge, "_clear_partial_selection", lambda: None) + assert bridge_client_state.credential_rejected + monkeypatch.setattr(bridge_identity, 'purge_persisted_state', lambda: None) + monkeypatch.setattr(bridge_identity, 'rebuild_identity_views', lambda: None) + monkeypatch.setattr(bridge_identity, 'clear_partial_selection', lambda: None) refreshed = [] - monkeypatch.setattr(bridge, "_refresh_manifest_async", lambda **kw: refreshed.append(kw)) + monkeypatch.setattr(bridge_client, 'refresh_manifest_async', lambda **kw: refreshed.append(kw)) renewed = token(nonce="renewed") - bridge.set_state("http://cloud", renewed, 3600, device_id="d") - assert not bridge._credential_rejected + bridge_client.set_state("http://cloud", renewed, 3600, device_id="d") + assert not bridge_client_state.credential_rejected assert refreshed == [{"force": True}] asyncio.run(worker.sync_once()) asyncio.run(worker.sync_once()) # Cloud accepts the duplicate heartbeat idempotently. @@ -158,29 +156,29 @@ async def late_reply(request): replacement = {**state, "token": token(nonce="replacement")} if change == "account": replacement["token"] = token(user="bob") - monkeypatch.setattr(bridge, "_state", None if change == "logout" else replacement) + monkeypatch.setattr(bridge_client_state, 'state', None if change == "logout" else replacement) return httpx.Response(401) worker = DesktopSyncWorker(factory, transport=httpx.MockTransport(late_reply)) with pytest.raises(httpx.HTTPStatusError): asyncio.run(worker.sync_once()) - assert not bridge._credential_rejected + assert not bridge_client_state.credential_rejected def test_refusal_uses_actual_request_token_when_headers_renewed_snapshot(delivery, monkeypatch): _, _, snapshot = delivery renewed = {**snapshot, "token": token(nonce="new")} - monkeypatch.setattr(bridge, "_state", renewed) + monkeypatch.setattr(bridge_client_state, 'state', renewed) request = httpx.Request( "POST", "http://cloud/api/v1/desktop/observability/batch", - headers=bridge.cloud_headers(snapshot), + headers=bridge_identity.cloud_headers(snapshot), ) refusal = httpx.HTTPStatusError( "Unauthorized", request=request, response=httpx.Response(401, request=request) ) - bridge.note_rejected_credential(snapshot, refusal) - assert bridge._credential_rejected == bridge._state_fingerprint(renewed) + bridge_identity.note_rejected_credential(snapshot, refusal) + assert bridge_client_state.credential_rejected == bridge_identity.state_fingerprint(renewed) @pytest.mark.parametrize("authorization", [None, "Bearer different-token"]) @@ -192,16 +190,16 @@ def test_missing_or_unrelated_request_credential_cannot_mark_rejection(delivery, exc = httpx.HTTPStatusError( "Unauthorized", request=request, response=httpx.Response(401, request=request) ) - bridge.note_rejected_credential(snapshot, exc) - assert not bridge._credential_rejected + bridge_identity.note_rejected_credential(snapshot, exc) + assert not bridge_client_state.credential_rejected def test_other_account_snapshot_cannot_reject_live_request_token(delivery): _, _, state = delivery other = {**state, "token": token(user="bob")} - request = httpx.Request("POST", "http://cloud", headers=bridge.cloud_headers(state)) + request = httpx.Request("POST", "http://cloud", headers=bridge_identity.cloud_headers(state)) exc = httpx.HTTPStatusError( "Unauthorized", request=request, response=httpx.Response(401, request=request) ) - bridge.note_rejected_credential(other, exc) - assert not bridge._credential_rejected + bridge_identity.note_rejected_credential(other, exc) + assert not bridge_client_state.credential_rejected diff --git a/src/backend/tests/api/test_desktop_release_events.py b/src/backend/tests/features/desktop/test_desktop_release_events.py similarity index 96% rename from src/backend/tests/api/test_desktop_release_events.py rename to src/backend/tests/features/desktop/test_desktop_release_events.py index 9311b2f7..e4c4cabf 100644 --- a/src/backend/tests/api/test_desktop_release_events.py +++ b/src/backend/tests/features/desktop/test_desktop_release_events.py @@ -3,7 +3,7 @@ import json import pytest -from core.services.desktop_release_events import ReleaseEvents +from features.desktop.bridge.release_events import ReleaseEvents @pytest.mark.asyncio @@ -43,7 +43,7 @@ async def test_slow_subscriber_gets_latest_revision_without_queue_growth(tmp_pat @pytest.mark.asyncio async def test_public_http_stream_sends_release_changes_without_reconnecting(tmp_path, monkeypatch): from fastapi import FastAPI - from api.routes.v1.desktop import router + from features.desktop.api.desktop import router app = FastAPI() app.include_router(router) diff --git a/src/backend/tests/test_desktop_sync_locking.py b/src/backend/tests/features/desktop/test_desktop_sync_locking.py similarity index 77% rename from src/backend/tests/test_desktop_sync_locking.py rename to src/backend/tests/features/desktop/test_desktop_sync_locking.py index 86469e12..0f2801ec 100644 --- a/src/backend/tests/test_desktop_sync_locking.py +++ b/src/backend/tests/features/desktop/test_desktop_sync_locking.py @@ -1,4 +1,6 @@ """SQLite publication must not wait on a writer that waits for account identity.""" +from features.desktop.bridge import client_state as bridge_client_state +from features.desktop.bridge import identity as bridge_identity import base64 import json @@ -9,13 +11,14 @@ from sqlalchemy import create_engine, event from sqlalchemy.orm import sessionmaker -from core.capabilities import registry -from core.capabilities.ref import cloud_ref +from features.desktop.capabilities import registry +from core.capabilities.reference import cloud_ref from core.db.engine import apply_sqlite_concurrency_pragmas -from core.db.models import ContentBlock, DeviceCapabilityInstallation, UserShadow, ChatSession -from core.db.models.observability import DesktopOutbox -from core.services import desktop_cloud_bridge as bridge -from core.services.desktop_observability_sync import install_capture +from core.db.models import ContentBlock, UserShadow, ChatSession +from features.desktop.models import DeviceCapabilityInstallation +from features.desktop.models import DesktopOutbox +from features.desktop.bridge import client as bridge +from features.desktop.observability.sync import install_capture @pytest.mark.parametrize("observed", [False, True]) @@ -46,9 +49,9 @@ def test_sqlite_writer_commits_while_capability_publication_holds_account_lock( "token": f"dcap2.{claims}.sig", "expires_at": 9999999999, } - monkeypatch.setattr(bridge, "_state", state) - monkeypatch.setattr(bridge, "_state_loaded", True) - identity = bridge.get_identity_state() + monkeypatch.setattr(bridge_client_state, 'state', state) + monkeypatch.setattr(bridge_client_state, 'state_loaded', True) + identity = bridge_identity.get_identity_state() with factory() as db: db.add( UserShadow( @@ -68,7 +71,7 @@ def pause_after_write(db, context): assert continue_flush.wait(3) event.listen(factory, "after_flush", pause_after_write) - remove = install_capture(factory, bridge.get_identity_state) + remove = install_capture(factory, bridge_identity.get_identity_state) def write(): with factory() as db: @@ -84,7 +87,7 @@ def write(): with ThreadPoolExecutor(max_workers=1) as pool: writer = pool.submit(write) assert wrote.wait(3) - with bridge.account_scope(state): + with bridge_identity.account_scope(state): continue_flush.set() assert registry.set_state(inst.install_id, "preparing").state == "preparing" writer.result(timeout=3) @@ -103,7 +106,8 @@ def write(): @pytest.mark.asyncio async def test_desktop_authentication_does_not_block_the_event_loop(monkeypatch): import asyncio - from core.auth import backend, desktop_bridge + from core.auth import backend + from features.desktop.auth import bridge as desktop_bridge from starlette.requests import Request released = threading.Event() @@ -116,7 +120,8 @@ def authenticate(request, db): progressed.append(released.wait(2)) return backend.UserContext(user_id="alice", user_center_id="center", username="Alice") - monkeypatch.setattr(desktop_bridge, "resolve_bridge_user", authenticate) + from core.harness.registry import authentication + monkeypatch.setattr(authentication.registration_state(), "resolver", authenticate) try: result = await backend.get_current_user( Request({"type": "http", "headers": []}), None, None diff --git a/src/backend/tests/api/test_desktop_updates.py b/src/backend/tests/features/desktop/test_desktop_updates.py similarity index 98% rename from src/backend/tests/api/test_desktop_updates.py rename to src/backend/tests/features/desktop/test_desktop_updates.py index ab7802c1..6318a601 100644 --- a/src/backend/tests/api/test_desktop_updates.py +++ b/src/backend/tests/features/desktop/test_desktop_updates.py @@ -2,7 +2,7 @@ import re from fastapi import FastAPI from fastapi.testclient import TestClient -from api.routes.v1.desktop import router +from features.desktop.api.desktop import router def test_unpublished_platform_is_no_update(tmp_path, monkeypatch): @@ -22,7 +22,7 @@ def test_unpublished_platform_is_no_update(tmp_path, monkeypatch): import sys from pathlib import Path -PUBLISH = Path(__file__).resolve().parents[4] / "desktop" / "scripts" / "publish-desktop.py" +PUBLISH = Path(__file__).resolve().parents[5] / "desktop" / "scripts" / "publish-desktop.py" def artifact_name(version, target): diff --git a/src/backend/tests/api/test_local_file_actions.py b/src/backend/tests/features/desktop/test_local_file_actions.py similarity index 89% rename from src/backend/tests/api/test_local_file_actions.py rename to src/backend/tests/features/desktop/test_local_file_actions.py index 9e4e902a..62309da3 100644 --- a/src/backend/tests/api/test_local_file_actions.py +++ b/src/backend/tests/features/desktop/test_local_file_actions.py @@ -3,17 +3,17 @@ import pytest from fastapi import HTTPException -from api.routes.local_file_actions import FileOpenRequest, applications, open_local_file +from features.desktop.api.local_file_actions import FileOpenRequest, applications, open_local_file def test_native_actions_use_authorized_file_identity(monkeypatch): calls = [] monkeypatch.setattr("api.routes.files.local_file_location", lambda fid, user, db: {"path": "/authorized/report.txt"}) - monkeypatch.setattr("core.services.local_file_actions.file_applications", + monkeypatch.setattr('features.desktop.local.file_actions.file_applications', lambda path, **kwargs: {"applications": [{"id": "editor", "name": "Editor", "icon": None}], "default_id": "editor"}) - monkeypatch.setattr("core.services.local_file_actions.open_file", + monkeypatch.setattr('features.desktop.local.file_actions.open_file', lambda path, **kwargs: calls.append((path, kwargs))) user = SimpleNamespace(user_id="owner") assert applications("ref", user, None)["path"] == "/authorized/report.txt" @@ -32,9 +32,9 @@ def denied(*args): def test_native_open_rejects_unregistered_application(monkeypatch): - from core.services.local_file_actions import open_file + from features.desktop.local.file_actions import open_file - monkeypatch.setattr("core.services.local_file_actions.file_applications", + monkeypatch.setattr('features.desktop.local.file_actions.file_applications', lambda path, **kwargs: {"applications": [], "default_id": None}) with pytest.raises(ValueError, match="关联应用"): open_file("/authorized/report.txt", action="open", application="/untrusted/program") @@ -42,7 +42,7 @@ def test_native_open_rejects_unregistered_application(monkeypatch): def test_windows_delegate_default_keeps_system_open(monkeypatch): import sys - from core.services import local_file_actions + from features.desktop.local import file_actions as local_file_actions def missing_key(*args): raise OSError @@ -66,7 +66,7 @@ def missing_key(*args): def test_application_icon_payload_is_inline_and_bounded(monkeypatch): import base64 - from core.services import native_app_icons + from features.desktop.local import native_app_icons native_app_icons.windows_icon.cache_clear() monkeypatch.setattr(native_app_icons.sys, "platform", "win32") png = bytes.fromhex("89504e470d0a1a0a") + b"fixture" @@ -84,7 +84,7 @@ def run(arguments, **kwargs): def test_application_icon_failure_has_no_external_fallback(monkeypatch): - from core.services import native_app_icons + from features.desktop.local import native_app_icons native_app_icons.windows_icon.cache_clear() monkeypatch.setattr(native_app_icons.sys, "platform", "win32") monkeypatch.setattr(native_app_icons, "no_window_kwargs", lambda: {}) @@ -94,7 +94,7 @@ def test_application_icon_failure_has_no_external_fallback(monkeypatch): native_app_icons.windows_icon.cache_clear() def test_selected_application_does_not_read_icons(monkeypatch, tmp_path): - from core.services import local_file_actions as actions + from features.desktop.local import file_actions as actions entry = tmp_path / "viewer.desktop" entry.write_text("[Desktop Entry]\nName=Viewer\nExec=viewer %f\n") @@ -112,7 +112,7 @@ def slow_icon(path): def test_windows_lightweight_choices_do_not_parse_icon_commands(monkeypatch): import sys - from core.services import local_file_actions as actions + from features.desktop.local import file_actions as actions def missing_key(*args): raise OSError diff --git a/src/backend/tests/services/test_local_grant_service.py b/src/backend/tests/features/desktop/test_local_grant_service.py similarity index 98% rename from src/backend/tests/services/test_local_grant_service.py rename to src/backend/tests/features/desktop/test_local_grant_service.py index f92af798..df8379f1 100644 --- a/src/backend/tests/services/test_local_grant_service.py +++ b/src/backend/tests/features/desktop/test_local_grant_service.py @@ -12,7 +12,7 @@ def _fresh_service(tmp_path, monkeypatch): monkeypatch.setenv("HUGAGENT_HOME", str(tmp_path)) - import core.services.local_grant_service as svc + import features.desktop.local.grants as svc return importlib.reload(svc) diff --git a/src/backend/tests/services/test_local_snapshot_service.py b/src/backend/tests/features/desktop/test_local_snapshot_service.py similarity index 96% rename from src/backend/tests/services/test_local_snapshot_service.py rename to src/backend/tests/features/desktop/test_local_snapshot_service.py index ee58a987..3feaec49 100644 --- a/src/backend/tests/services/test_local_snapshot_service.py +++ b/src/backend/tests/features/desktop/test_local_snapshot_service.py @@ -7,7 +7,7 @@ def _svc(tmp_path, monkeypatch): monkeypatch.setenv("HUGAGENT_HOME", str(tmp_path / "home")) - import core.services.local_snapshot_service as s + import features.desktop.local.snapshots as s return importlib.reload(s) diff --git a/src/backend/tests/features/evaluation/__init__.py b/src/backend/tests/features/evaluation/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/src/backend/tests/features/evaluation/__init__.py @@ -0,0 +1 @@ + diff --git a/src/backend/tests/features/evaluation/conftest.py b/src/backend/tests/features/evaluation/conftest.py new file mode 100644 index 00000000..28a005df --- /dev/null +++ b/src/backend/tests/features/evaluation/conftest.py @@ -0,0 +1,9 @@ +"""Keep standalone evaluation tests on the installed extension path.""" +import pytest +from tests.sandbox.conftest import sandbox_session_store, provider_database + + +@pytest.fixture(autouse=True) +def evaluation_extension(): + from features.evaluation.extension import setup + setup() diff --git a/src/backend/tests/sandbox/test_evaluation_admission.py b/src/backend/tests/features/evaluation/test_evaluation_admission.py similarity index 97% rename from src/backend/tests/sandbox/test_evaluation_admission.py rename to src/backend/tests/features/evaluation/test_evaluation_admission.py index 43030eab..53aecd96 100644 --- a/src/backend/tests/sandbox/test_evaluation_admission.py +++ b/src/backend/tests/features/evaluation/test_evaluation_admission.py @@ -7,8 +7,8 @@ from core.chat.schemas import ChatRequest from core.infra.ephemeral import LocalEphemeralState -import core.sandbox.providers.evaluation.binding as binding -from core.services import evaluation_admission as admission +import features.evaluation.sandbox_provider.binding as binding +from features.evaluation import admission as admission @pytest.fixture(autouse=True) diff --git a/src/backend/tests/sandbox/test_evaluation_binding.py b/src/backend/tests/features/evaluation/test_evaluation_binding.py similarity index 98% rename from src/backend/tests/sandbox/test_evaluation_binding.py rename to src/backend/tests/features/evaluation/test_evaluation_binding.py index 9bc83973..b4e5ee59 100644 --- a/src/backend/tests/sandbox/test_evaluation_binding.py +++ b/src/backend/tests/features/evaluation/test_evaluation_binding.py @@ -2,7 +2,7 @@ import pytest from core.infra.ephemeral import LocalEphemeralState -import core.sandbox.providers.evaluation.binding as binding +import features.evaluation.sandbox_provider.binding as binding from core.sandbox.errors import SandboxError diff --git a/src/backend/tests/llm/test_evaluation_factory.py b/src/backend/tests/features/evaluation/test_evaluation_factory.py similarity index 87% rename from src/backend/tests/llm/test_evaluation_factory.py rename to src/backend/tests/features/evaluation/test_evaluation_factory.py index 1c130a19..ebf53899 100644 --- a/src/backend/tests/llm/test_evaluation_factory.py +++ b/src/backend/tests/features/evaluation/test_evaluation_factory.py @@ -15,9 +15,14 @@ def test_evaluation_factory_and_child_share_one_native_surface(tmp_path): import core.sandbox.factory from core.db.engine import Base, engine, SessionLocal from core.db.models import UserShadow -from core.db.model_repository import create_provider, assign_role +from core.models.repository import create_provider, assign_role +from features import register_product_tools, register_run_capabilities +register_product_tools() +from features.evaluation.extension import setup +setup() +register_run_capabilities() from core.llm.factory import create_agent_executor -from core.llm.builtin_subagents import build_builtin_runtime_profile, get_builtin_subagent +from core.capabilities.agents.builtin import build_builtin_runtime_profile, get_builtin_subagent Base.metadata.create_all(engine) with SessionLocal() as db: @@ -60,9 +65,9 @@ async def run(): assert "evaluation" in schemas["Bash"]["description"] assert "OWNER_PRIVATE_PROJECT_SENTINEL" not in str(agent._system_prompt) assert "OWNER_OTHER_AGENT_SENTINEL" not in str(agent._system_prompt) - policies = {type(policy).__name__ for adapter in agent._reply_middlewares - for policy in getattr(adapter, "legacy_middlewares", ())} - assert "FinishPinGuardMiddleware" not in policies + policies = {policy.name for adapter in agent._reply_middlewares + for policy in adapter.bus.registered_specs()} + assert "finish_pin_guard" not in policies assert agent.offloader is not None await agent.offloader.offload_tool_result("framework", SimpleNamespace(output="large " * 15000)) assert sandbox.writes[-1][0] == session diff --git a/src/backend/tests/sandbox/test_evaluation_process_guard.py b/src/backend/tests/features/evaluation/test_evaluation_process_guard.py similarity index 97% rename from src/backend/tests/sandbox/test_evaluation_process_guard.py rename to src/backend/tests/features/evaluation/test_evaluation_process_guard.py index c6cc92c9..55b4e18b 100644 --- a/src/backend/tests/sandbox/test_evaluation_process_guard.py +++ b/src/backend/tests/features/evaluation/test_evaluation_process_guard.py @@ -3,7 +3,7 @@ import pytest -import core.sandbox.providers.evaluation.process_guard as guard +import features.evaluation.sandbox_provider.process_guard as guard from core.sandbox.errors import SandboxError diff --git a/src/backend/tests/sandbox/test_evaluation_provider.py b/src/backend/tests/features/evaluation/test_evaluation_provider.py similarity index 96% rename from src/backend/tests/sandbox/test_evaluation_provider.py rename to src/backend/tests/features/evaluation/test_evaluation_provider.py index 5a3025bc..9e1a183b 100644 --- a/src/backend/tests/sandbox/test_evaluation_provider.py +++ b/src/backend/tests/features/evaluation/test_evaluation_provider.py @@ -5,15 +5,15 @@ import pytest from core.infra.ephemeral import LocalEphemeralState -import core.sandbox.providers.evaluation.binding as binding -from core.sandbox.providers.evaluation.provider import EvaluationSandboxProvider +import features.evaluation.sandbox_provider.binding as binding +from features.evaluation.sandbox_provider.provider import EvaluationSandboxProvider from core.sandbox.errors import SandboxError from core.sandbox.protocol import ProcessRequest @pytest.fixture def setup(monkeypatch): - import core.sandbox.providers.evaluation.process_guard as evaluation_process_guard + import features.evaluation.sandbox_provider.process_guard as evaluation_process_guard monkeypatch.setattr(evaluation_process_guard, "clear_processes", AsyncMock()) state = LocalEphemeralState() monkeypatch.setattr(binding, "get_ephemeral_state", lambda: state) @@ -178,7 +178,7 @@ async def test_partial_status_does_not_report_success(setup): async def test_process_sweep_precedes_frozen_and_failure_keeps_agents_blocked(setup, monkeypatch): - import core.sandbox.providers.evaluation.process_guard as evaluation_process_guard + import features.evaluation.sandbox_provider.process_guard as evaluation_process_guard provider, sandbox, _ = setup item = await binding.create("alice", "box", 60, protected_processes={"1": "123"}) async def fail(sbx, baseline): @@ -196,7 +196,7 @@ async def fail(sbx, baseline): @pytest.mark.parametrize("exit_code", [0, 1]) async def test_confirmed_finished_command_skips_freeze_poll_but_still_sweeps(setup, exit_code): - import core.sandbox.providers.evaluation.process_guard as evaluation_process_guard + import features.evaluation.sandbox_provider.process_guard as evaluation_process_guard provider, sandbox, _ = setup sandbox.commands.get_command_status.return_value = SimpleNamespace( running=False, exit_code=exit_code, error=None, diff --git a/src/backend/tests/sandbox/test_evaluation_router.py b/src/backend/tests/features/evaluation/test_evaluation_router.py similarity index 78% rename from src/backend/tests/sandbox/test_evaluation_router.py rename to src/backend/tests/features/evaluation/test_evaluation_router.py index 4c6858f7..a28531be 100644 --- a/src/backend/tests/sandbox/test_evaluation_router.py +++ b/src/backend/tests/features/evaluation/test_evaluation_router.py @@ -5,9 +5,9 @@ import pytest from core.infra.ephemeral import LocalEphemeralState -import core.sandbox.providers.evaluation.binding as binding +import features.evaluation.sandbox_provider.binding as binding from core.sandbox.errors import SandboxError, SandboxConnectError -from core.sandbox.providers.evaluation.provider import EvaluationSandboxProvider +from features.evaluation.sandbox_provider.provider import EvaluationSandboxProvider from core.sandbox.protocol import ProcessRequest from core.sandbox.sessions.router import SessionSandboxRouter @@ -39,7 +39,7 @@ def arguments(method, session): @pytest.fixture def providers(monkeypatch): import core.config.settings as config - import core.sandbox.providers.evaluation.provider as evaluation + import features.evaluation.sandbox_provider.provider as evaluation ordinary = SimpleNamespace(name="opensandbox", runs_on_host=False) strict = SimpleNamespace() for method in METHODS: @@ -81,7 +81,7 @@ async def test_evaluation_never_falls_back_to_other_provider(providers, monkeypa async def test_missing_invalid_foreign_and_frozen_bindings_never_reach_normal_provider(providers, monkeypatch): - import core.sandbox.providers.evaluation.provider as evaluation + import features.evaluation.sandbox_provider.provider as evaluation router, ordinary, _ = providers state = LocalEphemeralState() monkeypatch.setattr(binding, "get_ephemeral_state", lambda: state) @@ -104,7 +104,7 @@ async def test_missing_invalid_foreign_and_frozen_bindings_never_reach_normal_pr async def test_attach_failure_does_not_create_replacement(providers, monkeypatch): - import core.sandbox.providers.evaluation.provider as evaluation + import features.evaluation.sandbox_provider.provider as evaluation router, ordinary, _ = providers state = LocalEphemeralState() monkeypatch.setattr(binding, "get_ephemeral_state", lambda: state) @@ -115,3 +115,31 @@ async def test_attach_failure_does_not_create_replacement(providers, monkeypatch await router.get_file(item.session_id, "/app/file", user_id="owner") ordinary.get_file.assert_not_awaited() connect.assert_awaited_once_with("task-container") + + +def test_missing_extension_fails_closed(monkeypatch): + import core.harness.registry + monkeypatch.setattr(core.harness.registry, "sandbox_route", lambda session_id: None) + ordinary = SimpleNamespace(name="opensandbox") + router = SessionSandboxRouter(ordinary) + with pytest.raises(SandboxError, match="not registered; fallback is forbidden"): + router._for(SESSION) + assert router._for("ordinary") is ordinary + + +def test_extension_registration_is_idempotent(): + import subprocess + import sys + script = """ +from unittest.mock import patch +from core.harness.registry import sandbox_route +from features.evaluation import extension +assert sandbox_route("eval_probe") is None +with patch.object(extension, "register_sandbox_route", wraps=extension.register_sandbox_route) as register: + extension.setup() + extension.setup() + register.assert_called_once_with("eval_", extension.evaluation_provider) +assert sandbox_route("eval_probe") is extension.evaluation_provider +""" + result = subprocess.run([sys.executable, "-c", script], capture_output=True, text=True) + assert result.returncode == 0, result.stdout + result.stderr diff --git a/src/backend/tests/llm/test_evaluation_runtime.py b/src/backend/tests/features/evaluation/test_evaluation_runtime.py similarity index 88% rename from src/backend/tests/llm/test_evaluation_runtime.py rename to src/backend/tests/features/evaluation/test_evaluation_runtime.py index 96d2484a..36f573fd 100644 --- a/src/backend/tests/llm/test_evaluation_runtime.py +++ b/src/backend/tests/features/evaluation/test_evaluation_runtime.py @@ -3,7 +3,7 @@ import pytest -from core.llm.evaluation_runtime import apply_evaluation_scope, parse_evaluation_scope +from features.evaluation.runtime.runtime import apply_evaluation_scope, parse_evaluation_scope SESSION = "eval_" + "a" * 32 @@ -48,7 +48,7 @@ def test_ordinary_context_is_unchanged(): def test_bound_agent_projection_does_not_mutate_its_definition(): from core.db.models import UserAgent - from core.llm.evaluation_runtime import evaluation_user_agent + from features.evaluation.runtime.runtime import evaluation_user_agent agent = UserAgent(agent_id="ua", name="Agent", system_prompt="task prompt", mcp_server_ids=["private"], skill_ids=["private"], kb_ids=["private"], plugin_ids=["private"]) @@ -64,18 +64,16 @@ def test_api_and_eval_scope_cannot_be_combined(): "agent_api_scope": {"version": 1}}) -class Collector: - def __init__(self): - self.tools = {} - - def register_tool_function(self, func, **kwargs): - self.tools[func.__name__] = func +class Collector(list): + @property + def tools(self): + return {tool.name: tool._func for tool in self} @pytest.fixture def evaluation_tools(monkeypatch): import core.sandbox - from core.llm.evaluation_tools import register_evaluation_tools + from features.evaluation.runtime.tools import register_evaluation_tools from core.sandbox import SandboxError class Provider: @@ -168,9 +166,10 @@ async def test_artifact_registration_is_rejected_before_io(evaluation_tools): @pytest.mark.asyncio async def test_paths_are_task_local_during_concurrent_calls_and_reset_afterward(monkeypatch): import asyncio - from core.llm.evaluation_runtime import CURRENT_EVALUATION_SCOPE - from core.llm.evaluation_tools import _EvaluationCollector - from core.llm.tools._paths import validate_workspace_path + from features.evaluation.runtime.runtime import CURRENT_EVALUATION_SCOPE + from features.evaluation.runtime.tools import scoped_tool + from core.llm.factory.tools.definitions import function_tool + from core.sandbox.workspace.resolution import validate_workspace_path collector = Collector() scope = parse_evaluation_scope(owner_user_id="owner", chat_id=SESSION) monkeypatch.setattr("core.config.local_mode.local_mode_enabled", lambda: False) @@ -182,7 +181,7 @@ async def probe(): assert validate_workspace_path("/app/file") is None raise RuntimeError("intentional") - _EvaluationCollector(collector, scope).register_tool_function(probe) + collector.append(scoped_tool(function_tool(probe), scope)) task = asyncio.create_task(collector.tools["probe"]()) await entered.wait() assert CURRENT_EVALUATION_SCOPE.get() is None @@ -194,7 +193,7 @@ async def probe(): def test_readonly_surface_and_local_mode_fail_closed(monkeypatch): - from core.llm.evaluation_tools import register_evaluation_tools + from features.evaluation.runtime.tools import register_evaluation_tools collector = Collector() scope = parse_evaluation_scope(owner_user_id="owner", chat_id=SESSION) monkeypatch.setattr("core.config.local_mode.local_mode_enabled", lambda: False) @@ -207,10 +206,9 @@ def test_readonly_surface_and_local_mode_fail_closed(monkeypatch): def test_evaluation_command_registration_respects_disabled_tools(monkeypatch): from types import SimpleNamespace - from core.llm.evaluation_process_tools import register_evaluation_process_tools - from core.llm.evaluation_runtime import EvaluationExecutionScope + from features.evaluation.runtime.process_tools import register_evaluation_process_tools + from features.evaluation.runtime.runtime import EvaluationExecutionScope monkeypatch.setenv("SANDBOX_TOOLS_ENABLED", "false") - registered = [] - toolkit = SimpleNamespace(register_tool_function=lambda *args, **kw: registered.append(args)) + toolkit = [] register_evaluation_process_tools(toolkit, EvaluationExecutionScope("u", SESSION)) - assert registered == [] + assert toolkit == [] diff --git a/src/backend/tests/sandbox/test_evaluation_service.py b/src/backend/tests/features/evaluation/test_evaluation_service.py similarity index 94% rename from src/backend/tests/sandbox/test_evaluation_service.py rename to src/backend/tests/features/evaluation/test_evaluation_service.py index cb33f41d..496b1df1 100644 --- a/src/backend/tests/sandbox/test_evaluation_service.py +++ b/src/backend/tests/features/evaluation/test_evaluation_service.py @@ -7,9 +7,9 @@ from core.config.settings import settings from core.infra.ephemeral import LocalEphemeralState -import core.sandbox.providers.evaluation.binding as bindings +import features.evaluation.sandbox_provider.binding as bindings from core.sandbox.errors import SandboxError -from core.services import evaluation_sandbox_service as service +from features.evaluation import sandboxes as service @pytest.fixture @@ -17,7 +17,7 @@ def sdk(monkeypatch): from opensandbox import Sandbox monkeypatch.setattr(bindings, "get_ephemeral_state", lambda: store) monkeypatch.setattr( - "core.sandbox.providers.evaluation.process_guard.capture_baseline", + 'features.evaluation.sandbox_provider.process_guard.capture_baseline', AsyncMock(return_value={"1": "100"}), ) store = LocalEphemeralState() diff --git a/src/backend/tests/features/hosted_apps/__init__.py b/src/backend/tests/features/hosted_apps/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/src/backend/tests/features/hosted_apps/__init__.py @@ -0,0 +1 @@ + diff --git a/src/backend/tests/api/application_hosting_support.py b/src/backend/tests/features/hosted_apps/application_hosting_support.py similarity index 77% rename from src/backend/tests/api/application_hosting_support.py rename to src/backend/tests/features/hosted_apps/application_hosting_support.py index 72de86e8..b9c93fcd 100644 --- a/src/backend/tests/api/application_hosting_support.py +++ b/src/backend/tests/features/hosted_apps/application_hosting_support.py @@ -1,4 +1,5 @@ """Disposable application and platform stores shared by hosting API contracts.""" +from core.files.storage.factory import process_state as state_core_files_storage_factory import os from types import SimpleNamespace @@ -6,7 +7,7 @@ import pytest from core.auth.backend import get_current_user -from core.services.application_store import initialize_store +from features.hosted_apps.store import initialize_store from fastapi import FastAPI from fastapi.testclient import TestClient from sqlalchemy import create_engine, text @@ -15,11 +16,11 @@ @pytest.fixture(params=["sqlite", "postgres"]) def hosted(tmp_path, monkeypatch, request): - from api.routes.v1 import application_transports, applications + from features.hosted_apps.api import application_transports, applications from unittest.mock import AsyncMock - monkeypatch.setattr("core.services.site_rate_limit.redis_configured", lambda: True) - monkeypatch.setattr("core.services.site_rate_limit.get_redis", lambda: SimpleNamespace(eval=AsyncMock(return_value=1))) + monkeypatch.setattr('features.sites.rate_limit.redis_configured', lambda: True) + monkeypatch.setattr('features.sites.rate_limit.get_redis', lambda: SimpleNamespace(eval=AsyncMock(return_value=1))) admin = None database = "test_applications_" + uuid4().hex if request.param == "postgres": @@ -36,11 +37,11 @@ def hosted(tmp_path, monkeypatch, request): ) initialize_store(engine) monkeypatch.setattr( - "core.config.application_hosting.application_hosting_settings", + 'features.hosted_apps.config.application_hosting_settings', SimpleNamespace(database_url=str(engine.url)), ) - monkeypatch.setattr("core.services.application_data.application_engine", lambda: engine) - monkeypatch.setattr("core.services.application_deployments.application_engine", lambda: engine) + monkeypatch.setattr('features.hosted_apps.data.application_engine', lambda: engine) + monkeypatch.setattr('features.hosted_apps.deployments.application_engine', lambda: engine) from core.db.engine import get_db from core.db.models import AdminMcpServer from sqlalchemy.orm import sessionmaker @@ -58,7 +59,7 @@ def hosted(tmp_path, monkeypatch, request): db.commit() monkeypatch.setenv("STORAGE_TYPE", "local") monkeypatch.setenv("STORAGE_PATH", str(tmp_path / "storage")) - monkeypatch.setattr("core.storage.factory._storage_instance", None) + monkeypatch.setattr(state_core_files_storage_factory(), 'storage_instance', None) def platform_dependency(): with platform_sessions() as db: @@ -72,7 +73,7 @@ def platform_dependency(): yield client, engine platform_engine.dispose() if admin is not None: - from core.services.application_store import applications + from features.hosted_apps.store import applications from sqlalchemy import select with engine.connect() as connection: diff --git a/src/backend/tests/api/test_application_agent_contract.py b/src/backend/tests/features/hosted_apps/test_application_agent_contract.py similarity index 79% rename from src/backend/tests/api/test_application_agent_contract.py rename to src/backend/tests/features/hosted_apps/test_application_agent_contract.py index d6814282..2e248633 100644 --- a/src/backend/tests/api/test_application_agent_contract.py +++ b/src/backend/tests/features/hosted_apps/test_application_agent_contract.py @@ -2,7 +2,7 @@ import asyncio -from tests.api.test_application_hosting import hosted +from tests.features.hosted_apps.test_application_hosting import hosted def test_mcp_exposes_typed_application_payload(): @@ -10,7 +10,7 @@ def test_mcp_exposes_typed_application_payload(): tools = asyncio.run(mcp.list_tools()) tool = next(item for item in tools if item.name == "manage_application") - definitions = tool.inputSchema.get("$defs", {}) + definitions = tool.input_schema.get("$defs", {}) assert definitions["ColumnDefinition"]["properties"]["type"]["enum"] == [ "text", "integer", @@ -21,18 +21,18 @@ def test_mcp_exposes_typed_application_payload(): ] assert "TableDefinition" in definitions assert "IndustryImport" not in definitions - assert "import_industry" not in tool.inputSchema["properties"]["action"]["enum"] - assert "app_id" in tool.inputSchema["properties"] + assert "import_industry" not in tool.input_schema["properties"]["action"]["enum"] + assert "app_id" in tool.input_schema["properties"] def test_internal_validation_returns_field_errors_without_values(hosted, monkeypatch): - from api.routes.v1 import application_transports + from features.hosted_apps.api import application_transports from core.db.engine import get_db client, _ = hosted client.app.include_router(application_transports.internal_router) client.app.dependency_overrides[get_db] = lambda: None - monkeypatch.setattr("api.routes.v1.internal_site_auth._check_internal_token", lambda _: None) + monkeypatch.setattr('features.sites.api.internal_site_auth._check_internal_token', lambda _: None) response = client.post( "/v1/internal/applications/operation", json={ diff --git a/src/backend/tests/api/test_application_collections.py b/src/backend/tests/features/hosted_apps/test_application_collections.py similarity index 87% rename from src/backend/tests/api/test_application_collections.py rename to src/backend/tests/features/hosted_apps/test_application_collections.py index 79869037..f17fcd99 100644 --- a/src/backend/tests/api/test_application_collections.py +++ b/src/backend/tests/features/hosted_apps/test_application_collections.py @@ -1,10 +1,10 @@ """Public collection migration behavior and structured survey columns.""" import pytest -from core.services.application_data import ApplicationDataService -from core.services.application_schema import ApplicationDefinition, RecordBatch, TableDefinition +from features.hosted_apps.data import ApplicationDataService +from features.hosted_apps.schema import ApplicationDefinition, RecordBatch, TableDefinition from fastapi import HTTPException -from tests.api.test_application_hosting import hosted +from tests.features.hosted_apps.test_application_hosting import hosted def test_public_collection_compare_and_swap(hosted): @@ -18,7 +18,7 @@ def test_public_collection_compare_and_swap(hosted): columns=[{"name": "task_id", "unique": True, "required": True}, {"name": "title"}], ) service.define_table(app["id"], "owner", definition) - from core.services.application_collections import read_collection, replace_collection + from features.hosted_apps.collections import read_collection, replace_collection original = read_collection(service, app["id"], "owner", "tasks") updated = replace_collection( @@ -51,7 +51,7 @@ def test_json_survey_and_private_read(hosted): ) service.insert(app["id"], "owner", "responses", RecordBatch(rows=[{"ratings": {"a": 4}}])) assert service.query(app["id"], "owner", "responses")["items"][0]["ratings"] == {"a": 4} - from core.services.application_collections import read_collection + from features.hosted_apps.collections import read_collection with pytest.raises(HTTPException) as error: read_collection(service, app["id"], "owner", "responses") @@ -61,7 +61,7 @@ def test_json_survey_and_private_read(hosted): @pytest.mark.parametrize("rows", [[None], [1], [["title"]], [{"title": float("nan")}]]) def test_collection_rejects_invalid_rows(hosted, rows): _, engine = hosted - from core.services.application_collections import replace_collection + from features.hosted_apps.collections import replace_collection with pytest.raises(HTTPException) as error: replace_collection( @@ -71,8 +71,8 @@ def test_collection_rejects_invalid_rows(hosted, rows): def test_json_constraints_and_mcp_filters(hosted): - from core.services.application_deployments import ApplicationDeploymentService - from core.services.application_schema import ColumnDefinition, ToolDefinition + from features.hosted_apps.deployments import ApplicationDeploymentService + from features.hosted_apps.schema import ColumnDefinition, ToolDefinition from pydantic import ValidationError with pytest.raises(ValidationError): @@ -120,7 +120,7 @@ def test_collection_optional_fields_preserved(hosted, reverse): columns=[{"name": "task_id"}, {"name": "title"}], ), ) - from core.services.application_collections import read_collection, replace_collection + from features.hosted_apps.collections import read_collection, replace_collection original = read_collection(service, app["id"], "owner", "tasks") rows = [{"task_id": "one", "title": "Preserved"}, {"task_id": "two"}] diff --git a/src/backend/tests/services/test_application_desktop.py b/src/backend/tests/features/hosted_apps/test_application_desktop.py similarity index 80% rename from src/backend/tests/services/test_application_desktop.py rename to src/backend/tests/features/hosted_apps/test_application_desktop.py index 0223f2ae..fd2f42da 100644 --- a/src/backend/tests/services/test_application_desktop.py +++ b/src/backend/tests/features/hosted_apps/test_application_desktop.py @@ -1,4 +1,5 @@ """Desktop startup and authenticated cloud forwarding contracts.""" +from features.desktop.bridge import identity as bridge_identity import json from types import SimpleNamespace @@ -8,11 +9,11 @@ def test_local_startup_provisions_separate_store(tmp_path, monkeypatch): - from core.auth import desktop_bridge # noqa: F401 - import before replacing settings - from core.config import application_hosting + from features.desktop.auth import bridge as desktop_bridge # noqa: F401 - import before replacing settings + from features.hosted_apps import config as application_hosting from core.config import settings as config from core.db import engine as platform - from core.services import application_store as store + from features.hosted_apps import store as store monkeypatch.setattr( config, @@ -27,7 +28,7 @@ def test_local_startup_provisions_separate_store(tmp_path, monkeypatch): application_hosting, "application_hosting_settings", SimpleNamespace(database_url="") ) monkeypatch.setattr(platform, "DATABASE_URL", "sqlite:///" + str(tmp_path / "platform.sqlite")) - monkeypatch.setattr("core.auth.desktop_bridge.bridge_enabled", lambda: False) + monkeypatch.setattr('features.desktop.auth.bridge.bridge_enabled', lambda: False) store.application_engine.cache_clear() try: store.initialize_local_store() @@ -57,9 +58,9 @@ def test_local_startup_provisions_separate_store(tmp_path, monkeypatch): def test_cloud_startup_does_not_provision(monkeypatch): - from core.auth import desktop_bridge # noqa: F401 - import before replacing settings + from features.desktop.auth import bridge as desktop_bridge # noqa: F401 - import before replacing settings from core.config import settings as config - from core.services import application_store as store + from features.hosted_apps import store as store monkeypatch.setattr(config, "settings", SimpleNamespace(deploy=SimpleNamespace(is_local=False))) monkeypatch.setattr( @@ -70,13 +71,14 @@ def test_cloud_startup_does_not_provision(monkeypatch): @pytest.mark.asyncio async def test_dual_callback_forwards_without_local_write(monkeypatch): - from api.routes.v1 import application_transports as routes - from core.services import desktop_cloud_bridge, desktop_site_publish - from core.services.application_schema import InternalOperation + from features.hosted_apps.api import application_transports as routes + from features.desktop.bridge import client as desktop_cloud_bridge + from features.sites import desktop_publish as desktop_site_publish + from features.hosted_apps.schema import InternalOperation - monkeypatch.setattr(desktop_cloud_bridge, "bridge_enabled", lambda: True) + monkeypatch.setattr('features.desktop.auth.bridge.bridge_enabled', lambda: True) monkeypatch.setattr( - "api.routes.v1.internal_site_auth._check_internal_token", lambda value: None + 'features.sites.api.internal_site_auth._check_internal_token', lambda value: None ) calls = [] @@ -115,7 +117,7 @@ def test_local_retirement_is_idempotent_and_verifies_archive(tmp_path): directory = tmp_path / "archives" assert retire_site_storage(engine, directory) is None assert inspect(engine).has_table("site_kv") - from core.db.site_storage_retirement import _digest + from core.db.site_storage_retirement import digest as _digest receipt = directory / "kv-to-sql-test" receipt.mkdir(parents=True) @@ -147,7 +149,7 @@ def test_pending_local_migration_preserves_rows(tmp_path): def test_local_conversation_mcp_receipt_uses_cloud_origin(): - from core.services.desktop_site_publish import localize_application_result + from features.sites.desktop_publish import localize_application_result result = { "content": [ @@ -171,7 +173,7 @@ def test_real_local_bootstrap_retains_unmigrated_and_provisions(tmp_path): import sys from pathlib import Path - source = Path(__file__).parents[2] + source = Path(__file__).parents[3] platform = tmp_path / "platform.sqlite" engine = create_engine("sqlite:///" + str(platform)) with engine.begin() as connection: @@ -181,11 +183,12 @@ def test_real_local_bootstrap_retains_unmigrated_and_provisions(tmp_path): code = """ import asyncio from sqlalchemy import inspect +import api.app # compose the host (schema lifecycles) as uvicorn does before startup from api.startup.bootstrap import _startup_ensure_tables from core.db.engine import engine -from core.services.application_data import ApplicationDataService -from core.services.application_store import application_engine -from core.services.application_schema import ApplicationDefinition, TableDefinition, RecordBatch +from features.hosted_apps.data import ApplicationDataService +from features.hosted_apps.store import application_engine +from features.hosted_apps.schema import ApplicationDefinition, TableDefinition, RecordBatch asyncio.run(_startup_ensure_tables()) assert inspect(engine).has_table("site_kv") service = ApplicationDataService() @@ -225,14 +228,13 @@ def test_real_local_bootstrap_retains_unmigrated_and_provisions(tmp_path): @pytest.mark.asyncio async def test_cloud_gateway_mcp_receipt_is_localized(monkeypatch): import httpx - import mcp.types - from core.llm.gateway_mcp_tool import GatewayMCPTool - from core.services import desktop_cloud_bridge as bridge - from tests.capabilities.test_runtime_recovery import state + from core.llm.tools.cloud_gateway import GatewayMCPTool + from features.desktop.bridge import client as bridge + from tests.features.desktop.capabilities.test_runtime_recovery import state - monkeypatch.setattr("core.capabilities.paths.capabilities_enabled", lambda: True) + monkeypatch.setattr('features.desktop.capabilities.paths.capabilities_enabled', lambda: True) st = state("cloud-user") - monkeypatch.setattr(bridge, "get_state", lambda: st) + monkeypatch.setattr(bridge_identity, 'get_state', lambda: st) def cloud(request): assert request.headers["authorization"] == "Bearer " + st["token"] @@ -253,7 +255,7 @@ def cloud(request): tool = GatewayMCPTool( mcp_name="sites-site_publish", source_plugin="sites", - tool=mcp.types.Tool(name="manage_application", inputSchema={"type": "object"}), + tool={"name": "manage_application", "inputSchema": {"type": "object"}}, invoke_url=st["cloud_base"] + "/api/v1/desktop/capability/gateway/sites-site_publish/call", schema_hash="a" * 64, timeout=10, @@ -266,13 +268,14 @@ def cloud(request): @pytest.mark.asyncio async def test_personal_mcp_publication_forwards_cloud_identity_and_tools(monkeypatch): - from api.routes.v1 import application_transports as routes - from core.services import desktop_cloud_bridge, desktop_site_publish - from core.services.application_schema import InternalMCPPublish - from core.services.desktop_gateway_uploads import result_localizer - - monkeypatch.setattr(desktop_cloud_bridge, "bridge_enabled", lambda: True) - monkeypatch.setattr("api.routes.v1.internal_site_auth._check_internal_token", lambda _: None) + from features.hosted_apps.api import application_transports as routes + from features.desktop.bridge import client as desktop_cloud_bridge + from features.sites import desktop_publish as desktop_site_publish + from features.hosted_apps.schema import InternalMCPPublish + from core.harness.registry.transfer_channels import result_localizer + + monkeypatch.setattr('features.desktop.auth.bridge.bridge_enabled', lambda: True) + monkeypatch.setattr('features.sites.api.internal_site_auth._check_internal_token', lambda _: None) calls = [] async def forward(tool, arguments, **identity): diff --git a/src/backend/tests/api/test_application_export.py b/src/backend/tests/features/hosted_apps/test_application_export.py similarity index 91% rename from src/backend/tests/api/test_application_export.py rename to src/backend/tests/features/hosted_apps/test_application_export.py index 651e10d4..155670e1 100644 --- a/src/backend/tests/api/test_application_export.py +++ b/src/backend/tests/features/hosted_apps/test_application_export.py @@ -1,6 +1,6 @@ """CSV export preserves values while rendering spreadsheet formulas as text.""" -from tests.api.test_application_hosting import create, data, hosted, table +from tests.features.hosted_apps.test_application_hosting import create, data, hosted, table def test_csv_export_treats_formula_prefixes_as_text(hosted): diff --git a/src/backend/tests/api/test_application_hosting.py b/src/backend/tests/features/hosted_apps/test_application_hosting.py similarity index 92% rename from src/backend/tests/api/test_application_hosting.py rename to src/backend/tests/features/hosted_apps/test_application_hosting.py index f781c23f..b7dbff54 100644 --- a/src/backend/tests/api/test_application_hosting.py +++ b/src/backend/tests/features/hosted_apps/test_application_hosting.py @@ -7,7 +7,7 @@ from core.auth.backend import get_current_user from fastapi import FastAPI from sqlalchemy import create_engine, inspect, text -from tests.api.application_hosting_support import hosted +from tests.features.hosted_apps.application_hosting_support import hosted def data(response): @@ -109,7 +109,7 @@ def test_mcp_publish_projection_and_revoke(hosted): } published = data(client.post(f"/v1/applications/{app_id}/mcp", json={"tools": [tool]})) assert published["url"] == f"/applications-mcp/{app_id}" - from core.services.application_store import applications + from features.hosted_apps.store import applications from sqlalchemy import select with engine.connect() as db: @@ -121,8 +121,8 @@ def test_mcp_publish_projection_and_revoke(hosted): @pytest.mark.asyncio async def test_external_mcp_protocol_field_allowlist_and_revocation(hosted): - import httpx - from core.services.application_mcp import ApplicationMCPGateway + import httpx2 + from features.hosted_apps.mcp import ApplicationMCPGateway from mcp import ClientSession from mcp.client.streamable_http import streamable_http_client @@ -155,38 +155,38 @@ async def test_external_mcp_protocol_field_allowlist_and_revocation(hosted): ) gateway = FastAPI() gateway.mount("/applications-mcp", ApplicationMCPGateway()) - async with httpx.AsyncClient( - transport=httpx.ASGITransport(app=gateway), base_url="http://testserver" + async with httpx2.AsyncClient( + transport=httpx2.ASGITransport(app=gateway), base_url="http://testserver" ) as anonymous: assert (await anonymous.post(published["url"], json={})).status_code == 401 - async with httpx.AsyncClient( - transport=httpx.ASGITransport(app=gateway), + async with httpx2.AsyncClient( + transport=httpx2.ASGITransport(app=gateway), headers={"Authorization": "Bearer " + published["token"]}, ) as http: async with streamable_http_client( "http://testserver" + published["url"], http_client=http - ) as (read, write, _): + ) as (read, write): async with ClientSession(read, write) as session: await session.initialize() tools = await session.list_tools() assert [tool.name for tool in tools.tools] == ["query_entries"] result = await session.call_tool("query_entries", {"name": "Aaron"}) - assert not result.isError - assert result.structuredContent["items"] == [{"name": "Aaron"}] + assert not result.is_error + assert result.structured_content["items"] == [{"name": "Aaron"}] result = await session.call_tool("query_entries", {"email": "private@example.test"}) - assert result.isError + assert result.is_error data(client.delete(f"/v1/applications/{app_id}/mcp")) assert (await http.post("http://testserver" + published["url"], json={})).status_code == 401 @pytest.fixture def hosted_sites(hosted, tmp_path, monkeypatch): - from api.routes import sites_serve - from api.routes.v1 import application_transports, applications + from features.sites.api import sites_serve + from features.hosted_apps.api import application_transports, applications from core.db.engine import Base, get_db from core.db.models import UserShadow - from core.services.site_service import SiteService - from core.storage.local import LocalStorageBackend + from features.sites.service import SiteService + from core.files.storage.local import LocalStorageBackend from sqlalchemy.orm import sessionmaker platform = create_engine( @@ -197,8 +197,8 @@ def hosted_sites(hosted, tmp_path, monkeypatch): factory = sessionmaker(bind=platform) monkeypatch.setenv("STORAGE_PATH", str(tmp_path / "storage")) storage = LocalStorageBackend() - monkeypatch.setattr("core.services.site_service.get_storage", lambda: storage) - monkeypatch.setattr("core.storage.get_storage", lambda: storage) + monkeypatch.setattr('features.sites.service.get_storage', lambda: storage) + monkeypatch.setattr('core.files.storage.get_storage', lambda: storage) with factory() as db: db.add(UserShadow(user_id="owner", username="owner")) db.commit() @@ -414,7 +414,7 @@ def test_mcp_rollback_preserves_records(hosted): ], ) def test_malformed_industry_source_is_controlled(envelope, monkeypatch): - from core.services.application_industry import fetch_enterprises + from features.hosted_apps.industry import fetch_enterprises from fastapi import HTTPException monkeypatch.setattr( @@ -422,7 +422,7 @@ def test_malformed_industry_source_is_controlled(envelope, monkeypatch): lambda: ("http://source.example.test", "local-test-only"), ) monkeypatch.setattr( - "core.services.application_industry.httpx.get", + 'features.hosted_apps.industry.httpx.get', lambda *args, **kwargs: SimpleNamespace( raise_for_status=lambda: None, json=lambda: envelope ), diff --git a/src/backend/tests/api/test_application_personal_mcp.py b/src/backend/tests/features/hosted_apps/test_application_personal_mcp.py similarity index 91% rename from src/backend/tests/api/test_application_personal_mcp.py rename to src/backend/tests/features/hosted_apps/test_application_personal_mcp.py index c1232148..a4943bb3 100644 --- a/src/backend/tests/api/test_application_personal_mcp.py +++ b/src/backend/tests/features/hosted_apps/test_application_personal_mcp.py @@ -7,12 +7,12 @@ import pytest from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker -from tests.api.test_application_hosting import create, data, hosted, table +from tests.features.hosted_apps.test_application_hosting import create, data, hosted, table @pytest.fixture def personal(hosted, tmp_path, monkeypatch): - from api.routes.v1 import application_transports + from features.hosted_apps.api import application_transports from core.db.engine import get_db from core.db.models import AdminMcpServer @@ -37,14 +37,14 @@ def db_dependency(): client.app.dependency_overrides[get_db] = db_dependency client.app.include_router(application_transports.internal_router) client.app.include_router(application_transports.mcp_router) - monkeypatch.setattr("api.routes.v1.internal_site_auth._check_internal_token", lambda _: None) + monkeypatch.setattr('features.sites.api.internal_site_auth._check_internal_token', lambda _: None) monkeypatch.setattr( "core.auth.capabilities.resolve_user_capabilities", lambda *a: {"can_add_mcp": True} ) # The real protocol adapter is exercised in-process instead of opening a test port. async def probe(row, db=None, **kwargs): - from core.services.mcp_management_service import decrypt_mcp_headers + from core.capabilities.mcp.management import decrypt_mcp_headers headers = decrypt_mcp_headers(row.headers) headers.update({"Accept": "application/json, text/event-stream"}) @@ -74,8 +74,8 @@ async def probe(row, db=None, **kwargs): row.tools_json = tools.json()["result"]["tools"] return True, "" - monkeypatch.setattr("core.services.mcp_management_service.probe_mcp_connectivity", probe) - monkeypatch.setattr("core.services.mcp_management_service.refresh_mcp_caches", lambda: None) + monkeypatch.setattr('core.capabilities.mcp.management.probe_mcp_connectivity', probe) + monkeypatch.setattr('core.capabilities.mcp.management.refresh_mcp_caches', lambda: None) yield client, factory platform.dispose() @@ -102,7 +102,7 @@ def publish(client, app_id, **changes): def test_conversation_publishes_private_mcp_and_updates_same_entry(personal): from core.db.models import AdminMcpServer - from core.services.mcp_management_service import decrypt_mcp_headers + from core.capabilities.mcp.management import decrypt_mcp_headers client, factory = personal app_id = create(client) @@ -162,7 +162,7 @@ def test_permission_and_owner_rejected_before_publication(personal, monkeypatch) def test_failed_probe_leaves_disabled_connection_and_retry_repairs(personal, monkeypatch): from core.db.models import AdminMcpServer - from core.services import mcp_management_service as management + from core.capabilities.mcp import management as management client, factory = personal app_id = create(client) @@ -186,7 +186,7 @@ async def fail(*args, **kwargs): def test_manual_updates_and_rollback_refresh_personal_credentials(personal): from core.db.models import AdminMcpServer - from core.services.mcp_management_service import decrypt_mcp_headers + from core.capabilities.mcp.management import decrypt_mcp_headers client, factory = personal app_id = create(client) @@ -219,14 +219,14 @@ def test_publish_tool_has_typed_tools_and_requires_injected_identity(monkeypatch from extensions.mcp_servers.site_publish_mcp.server import mcp, publish_mcp tool = next(item for item in asyncio.run(mcp.list_tools()) if item.name == "publish_mcp") - assert set(tool.inputSchema["required"]) == {"app_id", "tools"} - assert "user_id" not in tool.inputSchema["properties"] - assert "ToolDefinition" in tool.inputSchema["$defs"] + assert set(tool.input_schema["required"]) == {"app_id", "tools"} + assert "user_id" not in tool.input_schema["properties"] + assert "ToolDefinition" in tool.input_schema["$defs"] assert asyncio.run(publish_mcp("a" * 32, [])) == {"error": "Current user identity is required"} def test_publish_tool_forwards_only_injected_identity_and_typed_tools(monkeypatch): - from core.services.application_schema import ToolDefinition + from features.hosted_apps.schema import ToolDefinition from extensions.mcp_servers.site_publish_mcp import server calls = [] @@ -282,7 +282,7 @@ def test_legacy_conversation_publish_uses_personal_flow_without_token(personal): def test_registry_failure_is_partial_and_does_not_expose_secrets(personal, monkeypatch): - from core.services import mcp_management_service as management + from core.capabilities.mcp import management as management client, _ = personal app_id = create(client) @@ -304,7 +304,7 @@ def test_concurrent_publications_keep_one_connection_with_latest_credentials(per from concurrent.futures import ThreadPoolExecutor from core.db.models import AdminMcpServer - from core.services.mcp_management_service import decrypt_mcp_headers + from core.capabilities.mcp.management import decrypt_mcp_headers client, factory = personal app_id = create(client) diff --git a/src/backend/tests/api/test_application_project_tools.py b/src/backend/tests/features/hosted_apps/test_application_project_tools.py similarity index 87% rename from src/backend/tests/api/test_application_project_tools.py rename to src/backend/tests/features/hosted_apps/test_application_project_tools.py index 567165e8..8ae6c961 100644 --- a/src/backend/tests/api/test_application_project_tools.py +++ b/src/backend/tests/features/hosted_apps/test_application_project_tools.py @@ -5,9 +5,9 @@ from dataclasses import replace from fastapi.testclient import TestClient -from tests.api.application_hosting_support import hosted # noqa: F401 -from tests.api.test_application_hosting import data, table -from tests.api.test_application_projects import projects # noqa: F401 +from tests.features.hosted_apps.application_hosting_support import hosted # noqa: F401 +from tests.features.hosted_apps.test_application_hosting import data, table +from tests.features.hosted_apps.test_application_projects import projects # noqa: F401 class FileTransport: @@ -34,7 +34,9 @@ class Toolkit: def __init__(self): self.functions = {} - def register_tool_function(self, function, **kwargs): + def append(self, tool): + function = tool._func + kwargs = {"func_description": tool.description, "permission": getattr(tool, "dispatch_spec", None)} self.functions[function.__name__] = function @@ -48,9 +50,9 @@ def test_immediate_file_tool_edit_is_published_without_registration_delay( from core.llm.tools._state import ReadStateTracker from core.llm.tools.edit_tool import register_edit from core.llm.tools.read_tool import register_read - from core.spaces.access import personal_grant - from core.storage import get_storage - from core.services.project_scope import ProjectScope + from core.files.spaces.access import personal_grant + from core.files.storage import get_storage + from core.projects.edition import ProjectScope config = replace( settings, diff --git a/src/backend/tests/api/test_application_projects.py b/src/backend/tests/features/hosted_apps/test_application_projects.py similarity index 94% rename from src/backend/tests/api/test_application_projects.py rename to src/backend/tests/features/hosted_apps/test_application_projects.py index 0da8ee75..f96564d2 100644 --- a/src/backend/tests/api/test_application_projects.py +++ b/src/backend/tests/features/hosted_apps/test_application_projects.py @@ -1,4 +1,5 @@ """MCP source projects through the owner API, with disposable independent stores.""" +from core.files.storage.factory import process_state as state_core_files_storage_factory import pytest from core.auth.backend import get_current_user @@ -6,7 +7,7 @@ from core.db.models import UserShadow from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker -from tests.api.test_application_hosting import create, data, hosted, table +from tests.features.hosted_apps.test_application_hosting import create, data, hosted, table @pytest.fixture @@ -15,12 +16,12 @@ def projects(hosted, tmp_path, monkeypatch): from api.routes.v1.projects import router client.app.include_router(router) - from api.routes.v1.application_transports import internal_router + from features.hosted_apps.api.application_transports import internal_router from api.routes.v1.chats.sessions import router as chats_router client.app.include_router(chats_router, prefix="/v1/chats") client.app.include_router(internal_router) - monkeypatch.setattr("api.routes.v1.internal_site_auth._check_internal_token", lambda _: None) + monkeypatch.setattr('features.sites.api.internal_site_auth._check_internal_token', lambda _: None) engine = create_engine( f"sqlite:///{tmp_path / 'projects.db'}", connect_args={"check_same_thread": False} ) @@ -38,7 +39,7 @@ def dependency(): client.app.dependency_overrides[get_db] = dependency monkeypatch.setenv("STORAGE_TYPE", "local") monkeypatch.setenv("STORAGE_PATH", str(tmp_path / "storage")) - monkeypatch.setattr("core.storage.factory._storage_instance", None) + monkeypatch.setattr(state_core_files_storage_factory(), 'storage_instance', None) yield client engine.dispose() diff --git a/src/backend/tests/api/test_application_retirement.py b/src/backend/tests/features/hosted_apps/test_application_retirement.py similarity index 94% rename from src/backend/tests/api/test_application_retirement.py rename to src/backend/tests/features/hosted_apps/test_application_retirement.py index 79cbffef..6f74e230 100644 --- a/src/backend/tests/api/test_application_retirement.py +++ b/src/backend/tests/features/hosted_apps/test_application_retirement.py @@ -2,12 +2,12 @@ import pytest from sqlalchemy import inspect -from tests.api.test_application_hosting import hosted +from tests.features.hosted_apps.test_application_hosting import hosted def test_retired_storage_routes_are_absent(hosted): - from api.routes import sites_serve - from api.routes.v1 import internal_sites, sites + from features.sites.api import sites_serve + from features.sites.api import internal_sites, sites client, _ = hosted client.app.include_router(sites.router) @@ -73,12 +73,12 @@ def test_retirement_migration_archives_and_restores(hosted, tmp_path, monkeypatc "created_at": timestamp, }, ) - path = Path(__file__).parents[2] / "alembic/versions/site05retire_remove_legacy_storage.py" + path = Path(__file__).parents[3] / "alembic/versions/site05retire_remove_legacy_storage.py" spec = importlib.util.spec_from_file_location("retirement_revision", path) revision = importlib.util.module_from_spec(spec) spec.loader.exec_module(revision) monkeypatch.setenv("SITE_STORAGE_BACKUP_DIR", str(tmp_path)) - from core.db.site_storage_retirement import _digest + from core.db.site_storage_retirement import digest as _digest with engine.connect() as connection: original_rows = { diff --git a/src/backend/tests/api/test_hosting_remediation.py b/src/backend/tests/features/hosted_apps/test_hosting_remediation.py similarity index 82% rename from src/backend/tests/api/test_hosting_remediation.py rename to src/backend/tests/features/hosted_apps/test_hosting_remediation.py index 8d382a05..acaea287 100644 --- a/src/backend/tests/api/test_hosting_remediation.py +++ b/src/backend/tests/features/hosted_apps/test_hosting_remediation.py @@ -3,7 +3,7 @@ import pytest from fastapi import FastAPI, HTTPException from fastapi.testclient import TestClient -from tests.api.test_application_hosting import hosted, create, data +from tests.features.hosted_apps.test_application_hosting import hosted, create, data @pytest.mark.parametrize("status", [400, 403, 404, 409, 413, 422, 429, 500]) def test_site_errors_have_cors(status): @@ -20,7 +20,7 @@ def fail(): @pytest.mark.asyncio async def test_stateless_mcp_get_rejected(monkeypatch): - from core.services import application_mcp + from features.hosted_apps import mcp as application_mcp monkeypatch.setattr(application_mcp, "authorized_application", lambda *args: (None, {"title": "probe", "tools": []})) import httpx @@ -50,9 +50,9 @@ def test_owner_delete_and_complete_json_export(hosted): assert client.get(root).status_code == 404 def test_public_replacement_history_can_be_restored(hosted): - from core.services.application_collections import read_collection, replace_collection - from core.services.application_data import ApplicationDataService - from core.services.application_schema import TableDefinition, RecordBatch + from features.hosted_apps.collections import read_collection, replace_collection + from features.hosted_apps.data import ApplicationDataService + from features.hosted_apps.schema import TableDefinition, RecordBatch client, engine = hosted app_id = create(client) service = ApplicationDataService(engine) @@ -68,10 +68,10 @@ def test_public_replacement_history_can_be_restored(hosted): assert data(client.get(root + "/records"))["items"][0]["title"] == "Keep me" def test_anonymous_cannot_evict_recovery_baseline(hosted): - from core.services.application_collections import read_collection, replace_collection - from core.services.application_data import ApplicationDataService - from core.services.application_schema import TableDefinition, RecordBatch - from core.services.application_history import list_history, restore + from features.hosted_apps.collections import read_collection, replace_collection + from features.hosted_apps.data import ApplicationDataService + from features.hosted_apps.schema import TableDefinition, RecordBatch + from features.hosted_apps.history import list_history, restore client, engine = hosted app_id = create(client) service = ApplicationDataService(engine) @@ -87,12 +87,12 @@ def test_anonymous_cannot_evict_recovery_baseline(hosted): @pytest.mark.asyncio async def test_mcp_limit_schema_has_bounds(): - from core.services.application_mcp import query_tool - from core.services.application_schema import ToolDefinition, TableDefinition - from mcp.server.fastmcp import FastMCP + from features.hosted_apps.mcp import query_tool + from features.hosted_apps.schema import ToolDefinition, TableDefinition + from mcp.server.mcpserver import MCPServer app = {"tables": {"entries": TableDefinition(name="entries", columns=[{"name": "title"}]).model_dump()}} tool = ToolDefinition(name="entries", description="Read entries", table="entries", fields=["title"]) - mcp = FastMCP("probe") + mcp = MCPServer("probe") mcp.add_tool(query_tool(None, app, tool)) - schema = (await mcp.list_tools())[0].inputSchema["properties"]["limit"] + schema = (await mcp.list_tools())[0].input_schema["properties"]["limit"] assert schema["maximum"] == 100 and schema["minimum"] == 1 diff --git a/src/backend/tests/features/hosted_apps/test_schema_lifecycle.py b/src/backend/tests/features/hosted_apps/test_schema_lifecycle.py new file mode 100644 index 00000000..f520ba5e --- /dev/null +++ b/src/backend/tests/features/hosted_apps/test_schema_lifecycle.py @@ -0,0 +1,60 @@ +"""Local schema composition preserves provisioning and startup failure semantics.""" +import os +from pathlib import Path +import subprocess +import sys + + +def run_script(tmp_path, code): + env = dict(os.environ, PYTHONPATH=str(Path(__file__).parents[3]), + DEPLOY_PROFILE="local", JX_EDITION="ce", ENV="dev", + DATABASE_URL="sqlite:///" + str(tmp_path / "platform.sqlite"), + STORAGE_PATH=str(tmp_path / "storage"), APPLICATION_DATABASE_URL="", + HUGAGENT_DESKTOP_BRIDGE_SECRET="", PERSONAL_AGENT_ENABLED="false") + result = subprocess.run([sys.executable, "-c", code], env=env, + capture_output=True, text=True, timeout=45) + assert result.returncode == 0, result.stderr[-4000:] + + +def test_schema_only_composition_provisions_without_api_or_workers(tmp_path): + run_script(tmp_path, """ +import sys +from features.installed import INSTALLED_FEATURES +from core.harness.registry import load_models +from core.harness.registry.schema_lifecycle import load_schema_lifecycle +from core.db.engine import init_db, engine +from features.hosted_apps.store import application_engine, require_store +load_models(INSTALLED_FEATURES) +load_schema_lifecycle(INSTALLED_FEATURES) +load_schema_lifecycle(INSTALLED_FEATURES) +assert 'api.app' not in sys.modules +init_db() +require_store(application_engine()) +assert application_engine().url.database != engine.url.database +init_db() +require_store(application_engine()) +assert 'api.app' not in sys.modules +""") + + +def test_schema_callback_replacement_order_and_failures(tmp_path): + run_script(tmp_path, """ +from core.harness.registry.schema_lifecycle import register_schema_lifecycle, initialize_local_schema +seen=[] +register_schema_lifecycle('first', lambda: seen.append('old')) +register_schema_lifecycle('second', lambda: seen.append('second')) +register_schema_lifecycle('first', lambda: seen.append('first')) +initialize_local_schema() +assert seen == ['first','second'] +def fail(): + raise RuntimeError('provisioning failed') +register_schema_lifecycle('first', fail) +seen.clear() +try: + initialize_local_schema() +except RuntimeError as error: + assert str(error) == 'provisioning failed' +else: + raise AssertionError('startup failure was swallowed') +assert seen == [] +""") diff --git a/src/backend/tests/features/knowledge/__init__.py b/src/backend/tests/features/knowledge/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/src/backend/tests/features/knowledge/__init__.py @@ -0,0 +1 @@ + diff --git a/src/backend/tests/features/knowledge/kb_multimodal_assets_cases_1.py b/src/backend/tests/features/knowledge/kb_multimodal_assets_cases_1.py new file mode 100644 index 00000000..c3893f21 --- /dev/null +++ b/src/backend/tests/features/knowledge/kb_multimodal_assets_cases_1.py @@ -0,0 +1,353 @@ +from core.files.content import pdf_parser +from tests.features.knowledge.kb_multimodal_assets_support import ( + _FakeSession, + _FakeStorage, + _IMG_NAME, + _PNG_1PX, + _RICH_PAYLOAD, + _asset_row, + _evidence, + _install_bridge, + _install_fake_parser, + _parent, + _vision_result, + asset_captions, + asset_indexing, + fake_storage, + file_parser, + json, + kb_assets, + kb_assets_state, + parser_extract, + parser_types, + vector_embedding, + vector_indexing, +) + +def test_plain_parse_does_not_request_media(monkeypatch): + """纯文本路径（对话附件）不能顺手把图片一起要回来：图片是 base64 内联在响应体里的， + 一篇图多的 PDF 会凭空多出几 MB 传输，而这条路径根本用不到。""" + captured = _install_fake_parser(monkeypatch, _RICH_PAYLOAD) + + pdf_parser.parse_pdf(b"%PDF-1.4", "a.pdf") + + assert "return_images" not in captured["data"] + assert "return_content_list" not in captured["data"] + + +def test_rich_parse_requests_media_and_decodes_data_uri(monkeypatch): + captured = _install_fake_parser(monkeypatch, _RICH_PAYLOAD) + + result = pdf_parser.parse_pdf_rich(b"%PDF-1.4", "a.pdf") + + assert captured["data"]["return_images"] == "true" + assert captured["data"]["return_content_list"] == "true" + # 数据 URI 前缀必须剥掉，否则存下来的是坏文件 + assert result.images[f"images/{_IMG_NAME}"] == b"JPEGBYTES" + # 裸文件名也要能查到：markdown 里带 images/ 前缀，images 字典按裸名索引 + assert result.images[_IMG_NAME] == b"JPEGBYTES" + # content_list 是 JSON **字符串**，不是数组 + assert result.blocks[0]["img_path"] == f"images/{_IMG_NAME}" + + +def test_rich_parse_survives_undecodable_image(monkeypatch): + payload = json.loads(json.dumps(_RICH_PAYLOAD)) + payload["results"]["doc"]["images"] = {_IMG_NAME: "data:image/jpeg;base64,!!!not-base64!!!"} + _install_fake_parser(monkeypatch, payload) + + result = pdf_parser.parse_pdf_rich(b"%PDF-1.4", "a.pdf") + + # 单张图坏掉不该让整篇文档解析失败——正文照常返回 + assert "标题" in result.markdown + + +def test_uploaded_image_no_longer_indexed_as_binary_garbage(): + """上传白名单放行 png/jpg/webp/gif，但解析侧原先没有图片分支， + 会落到 latin-1 兜底把二进制解成乱码段落并向量化——静默污染，必须挡住。""" + paragraphs, assets = parser_extract.extract_text_with_assets(_PNG_1PX, "image/png") + + joined = "".join(p["text"] for p in paragraphs) + assert "\x89" not in joined and "PNG" not in joined + assert len(assets) == 1 + assert assets[0].kind == "image" + assert assets[0].data == _PNG_1PX + # 占位符必须真的出现在段落里，否则后面定位不到父块 + assert assets[0].placeholder in joined + + +def test_pdf_assets_carry_caption_and_locator(monkeypatch): + _install_fake_parser(monkeypatch, _RICH_PAYLOAD) + + paragraphs, assets = parser_extract.extract_text_with_assets(b"%PDF-1.4", "application/pdf") + + assert [p["text"] for p in paragraphs][0].startswith("# 标题") + assert len(assets) == 1 + asset = assets[0] + assert asset.text_content == "图 1 季度营收趋势" + assert asset.locator == {"page_idx": 3, "bbox": [139, 178, 647, 431]} + assert asset.mime_type == "image/jpeg" + + +def test_rich_parse_failure_degrades_to_text(monkeypatch): + """解析服务不支持这些参数（或直接抽风）时，必须退回今天的纯文本行为， + 而不是让整篇文档索引失败。""" + + def _boom(*_args, **_kwargs): + raise RuntimeError("parser exploded") + + monkeypatch.setattr(pdf_parser, "parse_pdf_rich", _boom) + monkeypatch.setattr(file_parser, "parse_file", lambda *_a, **_k: "# 纯文本兜底") + + paragraphs, assets = parser_extract.extract_text_with_assets(b"%PDF-1.4", "application/pdf") + + assert assets == [] + assert "纯文本兜底" in "".join(p["text"] for p in paragraphs) + + +def test_with_assets_false_keeps_legacy_path(monkeypatch): + """关掉多模态时不能再走重解析——parse_pdf_rich 一次都不该被调到。""" + calls: list = [] + + def _record(*_args, **_kwargs): + calls.append(1) + raise AssertionError("rich parse must not run when assets are disabled") + + monkeypatch.setattr(pdf_parser, "parse_pdf_rich", _record) + monkeypatch.setattr(file_parser, "parse_file", lambda *_a, **_k: "# 正文") + + paragraphs, assets = parser_extract.extract_text_with_assets( + b"%PDF-1.4", "application/pdf", with_assets=False + ) + + assert calls == [] + assert assets == [] + assert paragraphs + + +def test_asset_lands_on_the_chunk_that_contains_it(fake_storage, monkeypatch): + """挂错父块的后果是检索命中后回溯到别处的正文——比丢图严重得多。""" + monkeypatch.setattr(kb_assets_state, 'MIN_ASSET_BYTES', 0) + parents = [ + _parent("p_first", "第一段没有图。"), + _parent("p_second", "第二段：![](images/x.jpg) 见上图。"), + ] + assets = [ + parser_types.RawAsset(data=b"x" * 100, placeholder="images/x.jpg", mime_type="image/jpeg") + ] + db = _FakeSession() + + created = kb_assets.persist_assets( + db, kb_id="kb1", document_id="doc1", user_id="u1", assets=assets, parents=parents + ) + + assert len(created) == 1 + assert created[0].chunk_id == "p_second" + assert fake_storage.objects # 字节确实落盘了 + + +def test_placeholder_rewritten_in_both_parent_and_child(fake_storage, monkeypatch): + """入库正文与被向量化的子块文本必须是同一份；只改父块会让两者分叉。""" + monkeypatch.setattr(kb_assets_state, 'MIN_ASSET_BYTES', 0) + parents = [_parent("p1", "见 ![](images/x.jpg) 图。")] + assets = [ + parser_types.RawAsset(data=b"x" * 100, placeholder="images/x.jpg", mime_type="image/jpeg") + ] + db = _FakeSession() + + created = kb_assets.persist_assets( + db, kb_id="kb1", document_id="doc1", user_id="u1", assets=assets, parents=parents + ) + + url = kb_assets.asset_public_url(created[0].asset_id) + assert url in parents[0].content + assert url in parents[0].children[0].content + assert "images/x.jpg" not in parents[0].content + + +def test_tiny_assets_are_dropped(fake_storage): + """版面切出来的分割线/项目符号图标不值一次 VLM 调用，也不该进检索。""" + parents = [_parent("p1", "![](images/tiny.jpg)")] + assets = [ + parser_types.RawAsset(data=b"tiny", placeholder="images/tiny.jpg", mime_type="image/jpeg") + ] + + created = kb_assets.persist_assets( + _FakeSession(), kb_id="kb1", document_id="d1", user_id="u1", assets=assets, parents=parents + ) + + assert created == [] + assert fake_storage.objects == {} + + +def test_storage_failure_skips_one_asset_not_the_document(monkeypatch): + monkeypatch.setattr(kb_assets_state, 'MIN_ASSET_BYTES', 0) + + class _BrokenStorage(_FakeStorage): + def upload_bytes(self, content, storage_key): + raise RuntimeError("bucket on fire") + + monkeypatch.setattr('core.files.storage.get_storage', lambda: _BrokenStorage()) + parents = [_parent("p1", "![](images/x.jpg)")] + assets = [ + parser_types.RawAsset(data=b"x" * 100, placeholder="images/x.jpg", mime_type="image/jpeg") + ] + + created = kb_assets.persist_assets( + _FakeSession(), kb_id="kb1", document_id="d1", user_id="u1", assets=assets, parents=parents + ) + + assert created == [] + assert "images/x.jpg" in parents[0].content # 没落盘就不该改写成死链 + + +def test_no_vision_model_is_a_degradation_not_a_failure(monkeypatch): + """未指派 vision 角色、主模型也不识图，是正常状态：图注照常入索引。 + 这正是「视觉模型可以晚点再配」的前提——之后回填不用重新解析文档。""" + monkeypatch.setattr(asset_captions, 'vision_available', lambda: False) + rows = [_asset_row()] + db = _FakeSession() + + done = asset_captions.generate_captions(db, rows) + + assert done == 0 + assert rows[0].caption_status == "skipped" + assert "季度营收趋势" in asset_indexing.asset_index_text(rows[0], title="年报") + + +def test_evidence_lands_in_caption_and_ocr_merges_into_text(fake_storage, monkeypatch): + """证据要落到正确的字段上：概述进 caption，OCR 并进 text_content（与图注共存）。 + 并错了检索文本就废了——图注是作者写的用途，OCR 是图里实际的字，两者都要。""" + fake_storage.objects["k1"] = b"x" * 100 + _install_bridge(monkeypatch, [_vision_result()]) + rows = [_asset_row()] + + done = asset_captions.generate_captions(_FakeSession(), rows) + + assert done == 1 + assert rows[0].caption_status == "completed" + assert rows[0].caption == "柱状图：四个季度营收，Q4 最高" + assert "图 1 季度营收趋势" in rows[0].text_content # 原图注保留 + assert "Q1 Q2 Q3 Q4" in rows[0].text_content # OCR 并入 + # 实体/不确定项留档，给后续本体层与人工复核用 + assert rows[0].extra_data["vision"]["entities"][0]["name"] == "Q4" + assert rows[0].extra_data["vision"]["uncertainty"] == ["坐标轴单位模糊"] + assert rows[0].extra_data["vision"]["meta"]["model"] == "qwen-vl" + + +def test_ocr_identical_to_caption_is_not_duplicated(fake_storage, monkeypatch): + fake_storage.objects["k1"] = b"x" * 100 + _install_bridge(monkeypatch, [_vision_result(_evidence(ocr="图 1 季度营收趋势"))]) + rows = [_asset_row()] + + asset_captions.generate_captions(_FakeSession(), rows) + + assert rows[0].text_content == "图 1 季度营收趋势" + + +def test_text_content_is_capped(fake_storage, monkeypatch): + """满是文字的扫描件能吐出几万字，直接入库会把行和向量文本一起撑爆。""" + fake_storage.objects["k1"] = b"x" * 100 + monkeypatch.setattr(kb_assets_state, 'MAX_TEXT_CONTENT_CHARS', 50) + _install_bridge(monkeypatch, [_vision_result(_evidence(ocr="字" * 500))]) + rows = [_asset_row()] + + asset_captions.generate_captions(_FakeSession(), rows) + + assert len(rows[0].text_content) == 50 + + +def test_single_image_failure_keeps_asset_indexable(fake_storage, monkeypatch): + """视觉桥对单张返回 None（任何失败都这样降级），资产仍要凭图注可检索。""" + fake_storage.objects["k1"] = b"x" * 100 + _install_bridge(monkeypatch, [None]) + rows = [_asset_row()] + + done = asset_captions.generate_captions(_FakeSession(), rows) + + assert done == 0 + assert rows[0].caption_status == "failed" + assert asset_indexing.asset_index_text(rows[0]).strip() != "" + + +def test_whole_batch_exception_does_not_fail_the_document(fake_storage, monkeypatch): + fake_storage.objects["k1"] = b"x" * 100 + monkeypatch.setattr(asset_captions, 'vision_available', lambda: True) + + def _boom(_images, _loop=None): + raise RuntimeError("gateway down") + + monkeypatch.setattr(asset_captions, 'describe_batch', _boom) + rows = [_asset_row()] + + assert asset_captions.generate_captions(_FakeSession(), rows) == 0 + assert rows[0].caption_status == "failed" + + +def test_kb_never_saturates_the_shared_vision_lane(fake_storage, monkeypatch): + """视觉桥的并发信号量是进程级的：一篇图海文档一次性 gather 过去会把对话里的看图 + 请求全排到后面。KB 必须按小批提交，任何时刻只占用其中几格。""" + for i in range(7): + fake_storage.objects[f"k{i}"] = b"x" * 100 + calls: list = [] + _install_bridge(monkeypatch, [], calls=calls) + monkeypatch.setattr(kb_assets_state, 'CAPTION_BATCH', 2) + rows = [_asset_row(asset_id=f"a{i}", storage_key=f"k{i}") for i in range(7)] + + asset_captions.generate_captions(_FakeSession(), rows) + + assert calls == [2, 2, 2, 1], f"批次切分不对: {calls}" + assert max(calls) <= 2 + + +def test_unreadable_asset_is_skipped_without_calling_vision(fake_storage, monkeypatch): + """存储里读不到的那张不该占用一次视觉调用，也不该拖垮同批其他图。""" + fake_storage.objects["k2"] = b"x" * 100 + calls: list = [] + _install_bridge(monkeypatch, [_vision_result()], calls=calls) + monkeypatch.setattr(kb_assets_state, 'CAPTION_BATCH', 5) + rows = [_asset_row(asset_id="a1", storage_key="missing"), _asset_row(asset_id="a2", storage_key="k2")] + + done = asset_captions.generate_captions(_FakeSession(), rows) + + assert calls == [1] # 只送了能读出来的那张 + assert rows[0].caption_status == "failed" + assert done == 1 + + +def test_per_document_cap_marks_the_rest_explicitly(fake_storage, monkeypatch): + """上限生效要在数据里看得见，否则和"视觉模型什么都没返回"分不开。""" + for i in range(4): + fake_storage.objects[f"k{i}"] = b"x" * 100 + _install_bridge(monkeypatch, []) + monkeypatch.setattr(kb_assets_state, 'MAX_CAPTIONS_PER_DOC', 2) + rows = [_asset_row(asset_id=f"a{i}", storage_key=f"k{i}") for i in range(4)] + + asset_captions.generate_captions(_FakeSession(), rows) + + assert [r.caption_status for r in rows[2:]] == ["skipped", "skipped"] + + +def test_index_text_prefers_caption_then_caption_text(): + row = _asset_row(caption="柱状图，Q4 最高，同比增长 32%") + text = asset_indexing.asset_index_text(row, title="2026 年报") + assert text.splitlines() == ["柱状图，Q4 最高，同比增长 32%", "图 1 季度营收趋势", "2026 年报"] + + +def test_asset_rows_point_at_their_parent_chunk(monkeypatch): + """图片行必须挂在父块上：这样去重、回 PG 取父块正文、重排全都不用改。""" + written: list[dict] = [] + monkeypatch.setattr(asset_indexing, 'asset_index_text', lambda a, title="": "柱状图描述") + import features.knowledge.kb_vector as kb_vector + + monkeypatch.setattr(vector_embedding, 'embed_batch', lambda texts: [[0.1] * 8 for _ in texts]) + monkeypatch.setattr(vector_indexing, 'upsert_rows', lambda rows: written.extend(rows)) + + count = asset_indexing.index_assets( + [_asset_row()], user_id="u1", kb_id="kb1", document_id="d1", title="年报" + ) + + assert count == 1 + row = written[0] + assert row["row_type"] == "image" + assert row["parent_chunk_id"] == "p1" + assert row["chunk_id"] != "p1" # 主键不能和分块行撞 diff --git a/src/backend/tests/features/knowledge/kb_multimodal_assets_cases_2.py b/src/backend/tests/features/knowledge/kb_multimodal_assets_cases_2.py new file mode 100644 index 00000000..b4d9d944 --- /dev/null +++ b/src/backend/tests/features/knowledge/kb_multimodal_assets_cases_2.py @@ -0,0 +1,120 @@ +from tests.features.knowledge.kb_multimodal_assets_support import ( + _asset_row, + _stub_bridge, + asset_captions, + asset_indexing, + kb_assets, + vector_embedding, + vector_indexing, +) + +def test_orphan_asset_falls_back_to_itself(monkeypatch): + written: list[dict] = [] + import features.knowledge.kb_vector as kb_vector + + monkeypatch.setattr(vector_embedding, 'embed_batch', lambda texts: [[0.1] * 8 for _ in texts]) + monkeypatch.setattr(vector_indexing, 'upsert_rows', lambda rows: written.extend(rows)) + + asset_indexing.index_assets( + [_asset_row(chunk_id=None)], user_id="u1", kb_id="kb1", document_id="d1", title="年报" + ) + + # 没有归属分块时指向自己，检索侧取不到父块正文会退回本行 content（描述本身） + assert written[0]["parent_chunk_id"] == written[0]["chunk_id"] + + +def test_multimodal_toggle_precedence(monkeypatch): + monkeypatch.delenv("KB_MULTIMODAL_INDEXING", raising=False) + assert kb_assets.multimodal_indexing_enabled(None) is True + assert kb_assets.multimodal_indexing_enabled({"multimodal_indexing": False}) is False + monkeypatch.setenv("KB_MULTIMODAL_INDEXING", "false") + assert kb_assets.multimodal_indexing_enabled(None) is False + # 单库显式打开要能盖过全局关闭 + assert kb_assets.multimodal_indexing_enabled({"multimodal_indexing": True}) is True + + +def test_describe_batch_runs_on_the_host_loop_when_there_is_one(monkeypatch): + """索引跑在 anyio 工作线程里时，协程必须交回宿主循环执行——只有走宿主循环， + 视觉桥的证据缓存与并发信号量才真的生效（也才不会各起各的循环）。""" + import anyio + import anyio.to_thread + + seen: dict = {} + _stub_bridge(monkeypatch, seen) + + async def _main(): + return await anyio.to_thread.run_sync( + lambda: asset_captions.describe_batch([(b"a", "image/png"), (b"b", "image/jpeg")]) + ) + + results = anyio.run(_main) + + assert results == ["ok-0", "ok-1"] + assert seen["describe_many"]["n"] == 2 + assert "文档中的插图" in seen["describe_many"]["focus"] + assert "describe" not in seen + + +def test_describe_batch_without_host_loop_disables_the_evidence_cache(monkeypatch): + """没有宿主循环时才起临时循环，而且必须关掉缓存：Redis 连接池是模块级全局、 + 绑定首次使用它的循环，让一个转瞬即逝的临时循环抢先创建它，会把整个进程的 Redis + 绑死在已销毁的循环上——比「这次没吃到缓存」严重得多。""" + seen: dict = {} + _stub_bridge(monkeypatch, seen) + + results = asset_captions.describe_batch([(b"a", "image/png")]) + + assert results == ["ok-image/png"] + assert seen["describe"][0]["use_cache"] is False + assert "describe_many" not in seen + + +def test_explicit_loop_wins_over_anyio(monkeypatch): + """索引现在跑在常驻 worker **自己的**线程池里（不是 anyio 的），线程里既没有运行中的 + 循环、也没有回宿主循环的通路——所以 worker 会把调度它的循环显式传下来，必须优先用它。 + 退化成临时循环的话，证据缓存就白配了。""" + import asyncio + import threading + + seen: dict = {} + _stub_bridge(monkeypatch, seen) + + loop = asyncio.new_event_loop() + thread = threading.Thread(target=loop.run_forever, daemon=True) + thread.start() + try: + # 在一个普通线程里调用（模拟 worker 的线程池），只有显式 loop 能救它 + box: dict = {} + worker = threading.Thread( + target=lambda: box.update( + r=asset_captions.describe_batch([(b"a", "image/png")], loop) + ) + ) + worker.start() + worker.join(timeout=10) + finally: + loop.call_soon_threadsafe(loop.stop) + thread.join(timeout=5) + loop.close() + + assert box["r"] == ["ok-0"] + assert seen["describe_many"]["n"] == 1 + assert "describe" not in seen # 没退化到不带缓存的临时循环 + + +def test_media_only_parent_falls_back_to_the_asset_row_content(): + """单独上传的图片，其父块正文就是一条 markdown 图片链接。父块无条件覆盖的话， + 调用方拿到的只是个 URL，而命中的图片行里恰好装着描述与转写。""" + from extensions.mcp_servers.retrieve_dataset_content_mcp.local_impl import _pick_content + + link = "![](/api/v1/catalog/kb/assets/abc)" + rich = "柱状图：Q4 达到峰值 42\nQuarterly Revenue 2026" + + assert _pick_content(link, rich) == rich + assert _pick_content(f" {link} \n", rich) == rich + # 正常文档：父块里除了图还有正文，必须原样保留父块 + normal = f"第四条 专项资金用于……\n{link}\n见上图。" + assert _pick_content(normal, rich) == normal + # 两边都空时不要凭空造内容 + assert _pick_content("", "") == "" + assert _pick_content(link, "") == link diff --git a/src/backend/tests/features/knowledge/kb_multimodal_assets_support.py b/src/backend/tests/features/knowledge/kb_multimodal_assets_support.py new file mode 100644 index 00000000..fd5847c4 --- /dev/null +++ b/src/backend/tests/features/knowledge/kb_multimodal_assets_support.py @@ -0,0 +1,235 @@ +"""知识库多模态（路径 A：图生文入索引）的单元测试。 + +盯的是那几处「错了不会报错、只会悄悄变差或变脏」的地方： + +* 图片上传曾被当纯文本 latin-1 硬解成乱码入库——静默污染向量库，必须回归； +* 富解析的开关必须是**显式**传给解析服务的，漏传就永远拿不到图，且没有任何报错； +* 图片字节是 ``data:image/...;base64,`` 数据 URI，少剥一层前缀就存进一堆坏文件； +* 资产必须落在**包含它的父块**上——挂错块，检索命中后回溯到的是别处的正文； +* 占位符改写必须同时改父块和子块，否则入库正文与被向量化的文本不一致； +* 未配置视觉模型是正常状态，不能因此让整篇文档索引失败； +* KB 的批量图像理解不许占满进程级的视觉桥信号量，否则对话看图会被排到后面。 +""" + + +from __future__ import annotations +from core.files.content import pdf_parser +from core.files.content import parser_config + + +from core.models import embedding as vector_embedding + + +from features.knowledge import vector_indexing + + +from features.knowledge import parser_extract + + +from features.knowledge import parser_types + + +from features.knowledge import asset_captions + + +from features.knowledge import asset_indexing + + +from features.knowledge import kb_assets + + +from features.knowledge import kb_assets_state + + +import base64 + + +import json + + +from types import SimpleNamespace + + +import pytest + + +from core.files.content import file_parser + + +from features.knowledge import kb_assets, kb_parser + + +_PNG_1PX = base64.b64decode( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==" +) + + +class _FakeResponse: + def __init__(self, payload: dict) -> None: + self._payload = payload + + def raise_for_status(self) -> None: # pragma: no cover - always OK in these tests + return None + + def json(self) -> dict: + return self._payload + + +def _install_fake_parser(monkeypatch, payload: dict) -> dict: + """Stub the parse service; returns a dict that captures the posted form data.""" + captured: dict = {} + + def _fake_post(url, files=None, data=None, timeout=None): + captured["url"] = url + captured["data"] = data + captured["filename"] = files["files"][0] if files else None + return _FakeResponse(payload) + + monkeypatch.setattr(parser_config, 'cfg_api_url', lambda: "http://parser.test/file_parse") + monkeypatch.setattr(pdf_parser.requests, "post", _fake_post) + return captured + + +_IMG_NAME = "abc123.jpg" + + +_RICH_PAYLOAD = { + "results": { + "doc": { + "md_content": f"# 标题\n\n![](images/{_IMG_NAME})\n\n图 1 季度营收趋势\n\n结论段落。", + "images": { + _IMG_NAME: "data:image/jpeg;base64," + base64.b64encode(b"JPEGBYTES").decode() + }, + "content_list": json.dumps( + [ + { + "type": "image", + "img_path": f"images/{_IMG_NAME}", + "image_caption": ["图 1 季度营收趋势"], + "image_footnote": [], + "bbox": [139, 178, 647, 431], + "page_idx": 3, + } + ] + ), + } + } +} + + +class _FakeStorage: + def __init__(self) -> None: + self.objects: dict[str, bytes] = {} + self.deleted: list[str] = [] + + def upload_bytes(self, content: bytes, storage_key: str) -> str: + self.objects[storage_key] = content + return f"memory://{storage_key}" + + def download_bytes(self, storage_key: str) -> bytes: + return self.objects[storage_key] + + def delete(self, storage_key: str) -> None: + self.deleted.append(storage_key) + self.objects.pop(storage_key, None) + + +class _FakeSession: + """Just enough Session surface for persist_assets/generate_captions.""" + + def __init__(self) -> None: + self.added: list = [] + self.commits = 0 + + def add(self, obj) -> None: + self.added.append(obj) + + def commit(self) -> None: + self.commits += 1 + + def refresh(self, _obj) -> None: + return None + + +@pytest.fixture() +def fake_storage(monkeypatch): + storage = _FakeStorage() + monkeypatch.setattr('core.files.storage.get_storage', lambda: storage) + return storage + + +def _parent(parent_id: str, content: str): + child = parser_types.ChildChunk(child_id=f"{parent_id}_0", content=content, index=0) + return parser_types.ParentChunk(parent_id=parent_id, content=content, children=[child]) + + +def _asset_row(**kwargs): + base = dict( + asset_id="a1", + kind="image", + mime_type="image/jpeg", + storage_key="k1", + caption=None, + text_content="图 1 季度营收趋势", + caption_status="pending", + chunk_id="p1", + asset_index=0, + extra_data={}, + vector_state={}, + ) + base.update(kwargs) + return SimpleNamespace(**base) + + +def _evidence(summary="柱状图：四个季度营收，Q4 最高", ocr="Q1 Q2 Q3 Q4 营收(亿元)"): + """构造一份视觉桥证据（只用到 kb 侧真正读的字段）。""" + return SimpleNamespace( + summary=summary, + ocr=SimpleNamespace(full_text=ocr), + semantics=SimpleNamespace( + scene="财务图表", + intent="展示季度营收趋势", + entities=[SimpleNamespace(model_dump=lambda **_k: {"name": "Q4", "type": "季度"})], + relations=[], + ), + uncertainty=["坐标轴单位模糊"], + ) + + +def _vision_result(evidence=None): + return SimpleNamespace( + evidence=evidence if evidence is not None else _evidence(), + to_meta=lambda: {"model": "qwen-vl", "cached": False}, + ) + + +def _install_bridge(monkeypatch, results, *, available=True, calls=None): + """替换视觉桥：results 为按批返回的结果列表（长度与该批输入一致）。""" + monkeypatch.setattr(asset_captions, 'vision_available', lambda: available) + + def _fake_describe(images, loop=None): + if calls is not None: + calls.append(len(images)) + return [results.pop(0) if results else None for _ in images] + + monkeypatch.setattr(asset_captions, 'describe_batch', _fake_describe) + + +def _stub_bridge(monkeypatch, seen: dict): + """替换 core.models.vision.get_vision_bridge，记录实际调用参数。""" + import sys + import types + + async def _describe_many(images, *, focus=None): + seen["describe_many"] = {"n": len(images), "focus": focus} + return [f"ok-{i}" for i in range(len(images))] + + async def _describe(data, mime, *, focus=None, use_cache=True): + seen.setdefault("describe", []).append({"use_cache": use_cache, "focus": focus}) + return f"ok-{mime}" + + bridge = SimpleNamespace(describe_many=_describe_many, describe=_describe) + module = types.ModuleType("core.models.vision") + module.get_vision_bridge = lambda: bridge + module.is_available = lambda: True + monkeypatch.setitem(sys.modules, "core.models.vision", module) diff --git a/src/backend/tests/features/knowledge/kb_wiki_end_to_end_cases_1.py b/src/backend/tests/features/knowledge/kb_wiki_end_to_end_cases_1.py new file mode 100644 index 00000000..8401f74c --- /dev/null +++ b/src/backend/tests/features/knowledge/kb_wiki_end_to_end_cases_1.py @@ -0,0 +1,349 @@ +from tests.features.knowledge.kb_wiki_end_to_end_support import ( + DOC_ID, + INDEX_SLUG, + KBChunk, + KBWikiPage, + KB_ID, + LLMBudget, + WikiConfig, + _point_session_local, + finalize_kb_wiki, + local_graph, + local_provider, + pipeline, + wiki_db, +) + +def test_ingest_creates_pages_with_lineage(wiki_db, stub_llm, monkeypatch): + session, Session = wiki_db + config = WikiConfig(map_parallel=1, reduce_parallel=1, citation_chunk_batch=8) + + result = pipeline.run_ingest( + session, KB_ID, [DOC_ID], config=config, budget=LLMBudget(max_calls=100) + ) + + assert result.documents == 1 + assert result.pages_written == 2 + assert result.summaries_written == 1 + + pages = {p.slug: p for p in session.query(KBWikiPage).all()} + assert set(pages) == {"entity/acme", "concept/rag", f"summary/{DOC_ID}"} + + acme = pages["entity/acme"] + # 血缘：引文标注给的短 ID 必须被映射回真实 chunk_id + assert acme.chunk_refs == ["chunk-0", "chunk-2"] + assert acme.source_refs == [DOC_ID] + assert acme.title == "示例科技" + assert "示例科技有限公司" in (acme.aliases or []) + # 目录规划落成真实目录 + assert acme.category_path == ["组织"] + assert acme.wiki_path == "组织" + assert acme.folder_id + + rag = pages["concept/rag"] + assert rag.chunk_refs == ["chunk-1"] + assert rag.category_path == ["技术", "检索"] + + +def test_ingest_links_pages_to_each_other(wiki_db, stub_llm): + """写页阶段漏链时，自动织链要把正文里的条目名补成 Wiki 链接。""" + session, _ = wiki_db + pipeline.run_ingest( + session, + KB_ID, + [DOC_ID], + config=WikiConfig(map_parallel=1, reduce_parallel=1), + budget=LLMBudget(max_calls=100), + ) + acme = session.query(KBWikiPage).filter_by(slug="entity/acme").one() + assert "[[concept/rag|检索增强生成]]" in acme.content + assert "concept/rag" in (acme.out_links or []) + + +def test_finalize_builds_index_and_backlinks(wiki_db, stub_llm): + session, _ = wiki_db + pipeline.run_ingest( + session, + KB_ID, + [DOC_ID], + config=WikiConfig(map_parallel=1, reduce_parallel=1), + budget=LLMBudget(max_calls=100), + ) + stats = finalize_kb_wiki(session, KB_ID, config=WikiConfig(), budget=LLMBudget(max_calls=10)) + session.commit() + + assert stats["pages"] == 3 + index = session.query(KBWikiPage).filter_by(slug=INDEX_SLUG).one() + assert "实体" in index.content and "概念" in index.content + + # 反向链接由全库 out_links 反推，而不是增量维护 + rag = session.query(KBWikiPage).filter_by(slug="concept/rag").one() + assert "entity/acme" in (rag.in_links or []) + + +def test_finalize_strips_dead_links(wiki_db, stub_llm): + """指向已不存在页面的链接必须降级成纯文本，否则 Wiki 里会积累死链。""" + session, _ = wiki_db + pipeline.run_ingest( + session, + KB_ID, + [DOC_ID], + config=WikiConfig(map_parallel=1, reduce_parallel=1), + budget=LLMBudget(max_calls=100), + ) + acme = session.query(KBWikiPage).filter_by(slug="entity/acme").one() + acme.content = acme.content + "\n参见 [[entity/ghost|幽灵页]]。" + acme.out_links = [*(acme.out_links or []), "entity/ghost"] + session.commit() + + stats = finalize_kb_wiki(session, KB_ID, config=WikiConfig(), budget=LLMBudget(max_calls=10)) + session.commit() + + assert stats["dead_links_removed"] >= 1 + refreshed = session.query(KBWikiPage).filter_by(slug="entity/acme").one() + assert "[[entity/ghost" not in refreshed.content + assert "幽灵页" in refreshed.content + assert "entity/ghost" not in (refreshed.out_links or []) + + +def test_retract_removes_document_lineage(wiki_db, stub_llm): + """文档删除后，只由它供源的页面要归档，悬空的分块引用要摘掉。""" + session, _ = wiki_db + pipeline.run_ingest( + session, + KB_ID, + [DOC_ID], + config=WikiConfig(map_parallel=1, reduce_parallel=1), + budget=LLMBudget(max_calls=100), + ) + session.query(KBChunk).filter_by(document_id=DOC_ID).delete() + session.commit() + + stats = pipeline.run_retract(session, KB_ID, [DOC_ID]) + + assert stats["archived"] == 3, "三个页面都只有这一个来源，应全部归档" + live = session.query(KBWikiPage).filter(KBWikiPage.deleted_at.is_(None)).all() + assert live == [] + + +def test_read_surface_matches_expected_shape(wiki_db, stub_llm, monkeypatch): + """读取面的字段是路由、MCP 工具与前端组件的共同契约，形状不能漂。""" + session, Session = wiki_db + pipeline.run_ingest( + session, + KB_ID, + [DOC_ID], + config=WikiConfig(map_parallel=1, reduce_parallel=1), + budget=LLMBudget(max_calls=100), + ) + finalize_kb_wiki(session, KB_ID, config=WikiConfig(), budget=LLMBudget(max_calls=10)) + session.commit() + _point_session_local(monkeypatch, Session) + + stats = local_provider.wiki_stats(KB_ID) + assert stats["total_pages"] == 4 # 2 条目 + 1 摘要 + 1 索引 + assert stats["pages_by_type"]["entity"] == 1 + + page = local_provider.wiki_read_page(KB_ID, "entity/acme") + for key in ( + "slug", + "title", + "page_type", + "type_label", + "summary", + "aliases", + "category_path", + "wiki_path", + "out_links", + "in_links", + "source_refs", + "chunk_refs", + "content", + "related_pages", + ): + assert key in page, f"读取面缺少字段 {key}" + assert page["type_label"] == "实体" + # 相关页要带上可读标题，否则模型和用户看到的都是一串拼音 + assert page["related_pages"][0]["title"] == "检索增强生成" + + +def test_source_chunks_resolve_by_id(wiki_db, stub_llm, monkeypatch): + """顺血缘取原文是按 ID 直取，且要保持页面记录的引用顺序。""" + session, Session = wiki_db + pipeline.run_ingest( + session, + KB_ID, + [DOC_ID], + config=WikiConfig(map_parallel=1, reduce_parallel=1), + budget=LLMBudget(max_calls=100), + ) + session.commit() + _point_session_local(monkeypatch, Session) + + result = local_provider.wiki_source_chunks(KB_ID, "entity/acme", max_chunks=6) + ids = [c["chunk_id"] for c in result["chunks"]] + assert ids == ["chunk-0", "chunk-2"] + assert result["chunks"][0]["document_title"] == "示例科技年度报告" + assert "示例科技有限公司成立于" in result["chunks"][0]["content"] + + +def test_source_chunks_fall_back_when_refs_are_stale(wiki_db, stub_llm, monkeypatch): + """页面被改写或文档重新索引后 chunk_refs 会对不上，此时退回该文档前几块。""" + session, Session = wiki_db + pipeline.run_ingest( + session, + KB_ID, + [DOC_ID], + config=WikiConfig(map_parallel=1, reduce_parallel=1), + budget=LLMBudget(max_calls=100), + ) + page = session.query(KBWikiPage).filter_by(slug="entity/acme").one() + page.chunk_refs = ["chunk-does-not-exist"] + session.commit() + _point_session_local(monkeypatch, Session) + + result = local_provider.wiki_source_chunks(KB_ID, "entity/acme", max_chunks=2) + assert len(result["chunks"]) == 2, "对不上时不该空手而归" + + +def test_search_ranks_title_hits_above_body(wiki_db, stub_llm, monkeypatch): + session, Session = wiki_db + pipeline.run_ingest( + session, + KB_ID, + [DOC_ID], + config=WikiConfig(map_parallel=1, reduce_parallel=1), + budget=LLMBudget(max_calls=100), + ) + session.commit() + _point_session_local(monkeypatch, Session) + + result = local_provider.wiki_search(KB_ID, "检索增强生成", limit=5) + assert result["pages"], "应命中概念页" + assert result["pages"][0]["slug"] == "concept/rag" + + +def test_search_supports_alternation(wiki_db, stub_llm, monkeypatch): + """口语说法常匹配不上书面术语，一次给多个说法要能命中。""" + session, Session = wiki_db + pipeline.run_ingest( + session, + KB_ID, + [DOC_ID], + config=WikiConfig(map_parallel=1, reduce_parallel=1), + budget=LLMBudget(max_calls=100), + ) + session.commit() + _point_session_local(monkeypatch, Session) + + slugs = {p["slug"] for p in local_provider.wiki_search(KB_ID, "示例科技|检索增强生成")["pages"]} + assert {"entity/acme", "concept/rag"} <= slugs + + +def test_folders_report_recursive_page_counts(wiki_db, stub_llm, monkeypatch): + session, Session = wiki_db + pipeline.run_ingest( + session, + KB_ID, + [DOC_ID], + config=WikiConfig(map_parallel=1, reduce_parallel=1), + budget=LLMBudget(max_calls=100), + ) + session.commit() + _point_session_local(monkeypatch, Session) + + roots = local_provider.wiki_folders(KB_ID) + by_name = {f["name"]: f for f in roots["folders"]} + assert {"组织", "技术"} <= set(by_name) + # 「技术」下的页面挂在子目录「检索」里，递归计数必须能看到它 + assert by_name["技术"]["page_count"] == 1 + assert by_name["技术"]["has_children"] is True + assert by_name["组织"]["has_children"] is False + + +def test_graph_ego_mode_centres_on_node(wiki_db, stub_llm, monkeypatch): + session, Session = wiki_db + pipeline.run_ingest( + session, + KB_ID, + [DOC_ID], + config=WikiConfig(map_parallel=1, reduce_parallel=1), + budget=LLMBudget(max_calls=100), + ) + finalize_kb_wiki(session, KB_ID, config=WikiConfig(), budget=LLMBudget(max_calls=10)) + session.commit() + _point_session_local(monkeypatch, Session) + + graph = local_graph.wiki_graph(KB_ID, mode="ego", center="entity/acme", depth=1) + slugs = {n["slug"] for n in graph["nodes"]} + assert "entity/acme" in slugs and "concept/rag" in slugs + assert graph["edges"], "两页之间有链接，应产出边" + + +def test_graph_ego_on_missing_node_is_empty(wiki_db, stub_llm, monkeypatch): + session, Session = wiki_db + _point_session_local(monkeypatch, Session) + graph = local_graph.wiki_graph(KB_ID, mode="ego", center="entity/nope") + assert graph["nodes"] == [] and graph["edges"] == [] + + +def test_ingest_is_idempotent_on_rerun(wiki_db, stub_llm): + """同一批文档重跑不该产生重复页——slug 复用规则要真的生效。""" + session, _ = wiki_db + config = WikiConfig(map_parallel=1, reduce_parallel=1) + pipeline.run_ingest(session, KB_ID, [DOC_ID], config=config, budget=LLMBudget(max_calls=100)) + first = session.query(KBWikiPage).count() + + pipeline.run_ingest(session, KB_ID, [DOC_ID], config=config, budget=LLMBudget(max_calls=100)) + assert session.query(KBWikiPage).count() == first + + acme = session.query(KBWikiPage).filter_by(slug="entity/acme").one() + assert acme.version >= 2, "重跑应写出新版本而不是新页面" + + +def test_concurrent_stages_never_touch_the_session(wiki_db, stub_llm): + """并发段绝不能碰 session。 + + SQLAlchemy 的 Session 不是线程安全的：从线程池里查库会炸成 + "concurrent operations are not permitted"，而且只在并发度 > 1 时才复现—— + 单线程跑一万遍都发现不了。这里用一个会记录调用线程的代理把它钉死。 + """ + import threading + + session, _ = wiki_db + main_thread = threading.get_ident() + offenders: list[str] = [] + + real_query = session.query + + def _tracking_query(*args, **kwargs): + if threading.get_ident() != main_thread: + offenders.append(threading.current_thread().name) + return real_query(*args, **kwargs) + + session.query = _tracking_query # type: ignore[method-assign] + try: + pipeline.run_ingest( + session, + KB_ID, + [DOC_ID], + config=WikiConfig(map_parallel=4, reduce_parallel=4), + budget=LLMBudget(max_calls=100), + ) + finally: + session.query = real_query # type: ignore[method-assign] + + assert offenders == [], f"这些线程在并发段里查了库：{offenders}" + + +def test_quota_stops_generation_without_failing(wiki_db, stub_llm): + """超额是设计好的刹车：已生成的部分保留，作业不算失败。""" + session, _ = wiki_db + result = pipeline.run_ingest( + session, + KB_ID, + [DOC_ID], + config=WikiConfig(map_parallel=1, reduce_parallel=1), + budget=LLMBudget(max_calls=1), + ) + assert result.quota_exceeded is True diff --git a/src/backend/tests/features/knowledge/kb_wiki_end_to_end_cases_2.py b/src/backend/tests/features/knowledge/kb_wiki_end_to_end_cases_2.py new file mode 100644 index 00000000..d8a41c51 --- /dev/null +++ b/src/backend/tests/features/knowledge/kb_wiki_end_to_end_cases_2.py @@ -0,0 +1,175 @@ +from tests.features.knowledge.kb_wiki_end_to_end_support import ( + DOC_ID, + KBSpace, + KB_ID, + LLMBudget, + USER_ID, + WikiConfig, + _wrap_json, + json, + pipeline, + wiki_db, +) + +def test_custom_instructions_reach_every_stage(wiki_db, monkeypatch): + """用户填的要求必须真的送达每个阶段的模型调用。 + + 配置字段加了却没接到提示词上，是这类功能最典型的"看起来做了"—— + 界面能填、能存库、生成结果毫无变化。这里逐阶段核实。 + """ + session, _ = wiki_db + seen: dict[str, str] = {} + + def _capture(*, system, user, budget=None, purpose="wiki", **kwargs): + seen[purpose] = f"{system or ''}\n{user}" + if budget is not None: + budget.record(len(user), 10) + if purpose == "候选抽取": + return json.dumps( + { + "entities": [ + {"name": "示例科技", "slug": "entity/acme", "aliases": [], + "description": "d", "details": "x"} + ], + "concepts": [], + } + ) + if purpose == "引文标注": + return json.dumps({"citations": {"entity/acme": ["c001"]}, "new_slugs": []}) + if purpose in ("去重判定",): + return json.dumps({"merges": {}}) + if purpose == "目录规划": + return json.dumps({"assignments": []}) + if purpose in ("写页", "文档摘要"): + return "SUMMARY: 摘要\n# 标题\n正文" + return "{}" + + for module in ("extract", "cite", "dedup", "taxonomy"): + monkeypatch.setattr(f"features.knowledge.wiki.{module}.call_llm_json", _wrap_json(_capture)) + monkeypatch.setattr('features.knowledge.wiki.reduce.call_llm', _capture) + + pipeline.run_ingest( + session, + KB_ID, + [DOC_ID], + config=WikiConfig( + map_parallel=1, + reduce_parallel=1, + extraction_instructions="重点关注补助金额与申报条件", + content_instructions="每页开头先给一句话结论", + ), + budget=LLMBudget(max_calls=100), + ) + + # 抽取侧两个阶段拿到抽取要求 + for stage in ("候选抽取", "引文标注"): + assert "重点关注补助金额与申报条件" in seen[stage], f"{stage} 未收到抽取要求" + assert "" in seen[stage] + + # 撰写侧两个阶段拿到撰写要求 + for stage in ("写页", "文档摘要"): + assert "每页开头先给一句话结论" in seen[stage], f"{stage} 未收到撰写要求" + assert "" in seen[stage] + + # 两侧不串味：抽取要求不该出现在写页提示词里 + assert "重点关注补助金额与申报条件" not in seen["写页"] + + +def test_worker_batches_sibling_ingest_jobs(wiki_db): + """同库的多个待跑摄入作业要被合并成一批。 + + 上传是一篇文档一个作业（入队点在索引完成回调里，天然单篇）。若处理时也一篇 + 一批，单 worker 串行跑 39 篇要数小时，而且 map_parallel 根本没有多篇文档可并行。 + """ + from features.knowledge.wiki import jobs as job_queue + + session, _ = wiki_db + first = job_queue.enqueue_ingest(session, KB_ID, ["doc-a"]) + job_queue.enqueue_ingest(session, KB_ID, ["doc-b"]) + job_queue.enqueue_ingest(session, KB_ID, ["doc-c"]) + + claimed = job_queue.claim_next_job(session, "worker-1") + assert claimed is not None and claimed.job_id == first.job_id + + siblings = job_queue.claim_sibling_ingest_jobs(session, "worker-1", KB_ID, limit=4) + assert [s.payload["document_ids"] for s in siblings] == [["doc-b"], ["doc-c"]] + + # 合批后全部一起结单，不留孤儿 + job_queue.finish_jobs(session, [claimed.job_id, *(s.job_id for s in siblings)]) + assert job_queue.kb_job_summary(session, KB_ID)["generating"] is False + + +def test_batching_never_crosses_knowledge_bases(wiki_db): + """跨库合批会让一个大库把其他库饿死。""" + from features.knowledge.models import KBSpace + from features.knowledge.wiki import jobs as job_queue + + session, _ = wiki_db + session.add( + KBSpace(kb_id="kb_other", user_id=USER_ID, name="别的库", visibility="private", + extra_data={"index_modes": ["wiki"]}) + ) + session.commit() + job_queue.enqueue_ingest(session, KB_ID, ["mine"]) + job_queue.enqueue_ingest(session, "kb_other", ["theirs"]) + + siblings = job_queue.claim_sibling_ingest_jobs(session, "worker-1", KB_ID, limit=10) + assert all(s.kb_id == KB_ID for s in siblings) + + +def test_one_knowledge_base_runs_one_batch_at_a_time(wiki_db): + """同一个库同时只允许一批在跑。 + + 同库两批并发会丢更新：写页阶段先把既有页读成内存快照，再逐条基于快照重写、 + 最后整体 upsert，而不同文档抽出同名 slug 是常态——后写的会覆盖先写的。 + 生产上两个 uvicorn worker 各跑一个 wiki worker 同啃一个库，内存也随之翻倍。 + """ + from features.knowledge.wiki import jobs as job_queue + + session, _ = wiki_db + job_queue.enqueue_ingest(session, KB_ID, ["doc-a"]) + job_queue.enqueue_ingest(session, KB_ID, ["doc-b"]) + + assert job_queue.claim_next_job(session, "worker-1") is not None + assert job_queue.claim_next_job(session, "worker-2") is None + + +def test_a_busy_knowledge_base_does_not_block_other_ones(wiki_db): + """同库串行不能退化成全局串行，否则一个大库会把其他库全堵死。""" + from features.knowledge.wiki import jobs as job_queue + + session, _ = wiki_db + session.add( + KBSpace( + kb_id="kb_other", + user_id=USER_ID, + name="别的库", + visibility="private", + extra_data={"index_modes": ["wiki"]}, + ) + ) + session.commit() + job_queue.enqueue_ingest(session, KB_ID, ["mine"]) + job_queue.enqueue_ingest(session, "kb_other", ["theirs"]) + + first = job_queue.claim_next_job(session, "worker-1") + second = job_queue.claim_next_job(session, "worker-2") + assert first is not None and second is not None + assert {first.kb_id, second.kb_id} == {KB_ID, "kb_other"} + + +def test_a_dead_workers_knowledge_base_is_taken_over(wiki_db): + """持有者死了（心跳停摆）之后这个库必须能被接管，不能因为互斥而永远卡住。""" + from features.knowledge.wiki import jobs as job_queue + from features.knowledge.wiki.jobs import STALE_AFTER, _now + + session, _ = wiki_db + job_queue.enqueue_ingest(session, KB_ID, ["doc-a"]) + claimed = job_queue.claim_next_job(session, "worker-1") + assert claimed is not None + + claimed.heartbeat_at = _now() - STALE_AFTER * 2 + session.commit() + + retaken = job_queue.claim_next_job(session, "worker-2") + assert retaken is not None and retaken.job_id == claimed.job_id diff --git a/src/backend/tests/features/knowledge/kb_wiki_end_to_end_support.py b/src/backend/tests/features/knowledge/kb_wiki_end_to_end_support.py new file mode 100644 index 00000000..ae5ecbaa --- /dev/null +++ b/src/backend/tests/features/knowledge/kb_wiki_end_to_end_support.py @@ -0,0 +1,197 @@ +"""Wiki 管线的端到端测试：喂真文档、桩掉模型、检查落库结果与读取面。 + +模型调用全部被桩掉——这里验的不是模型写得好不好，而是**管线接线对不对**： +候选是否落成页、引文是否变成可回溯的血缘、目录是否建起来、读取面返回的结构 +是否与前端/工具期望的一致。这类接线错误在真实环境里要跑几十分钟才暴露一次。 +""" + + +from __future__ import annotations + + +from features.knowledge.wiki import local_graph + + +from features.knowledge.wiki import local_provider + + +import json + + +import pytest + + +from sqlalchemy import create_engine + + +from sqlalchemy.orm import sessionmaker + + +from core.db.engine import Base + + +from features.knowledge.models import KBChunk, KBDocument, KBSpace, KBWikiPage + + +from core.db.models import UserShadow + + +from features.knowledge.wiki import local_provider, pipeline + + +from features.knowledge.wiki.config import WikiConfig + + +from features.knowledge.wiki.finalize import INDEX_SLUG, finalize_kb_wiki + + +from features.knowledge.wiki.llm import LLMBudget + + +KB_ID = "kb_wikitest" + + +USER_ID = "u_wikitest" + + +DOC_ID = "doc_alpha" + + +@pytest.fixture +def wiki_db(tmp_path): + """独立的 sqlite 库；local_provider 走自己的 SessionLocal，必须一并指向它。""" + engine = create_engine(f"sqlite:///{tmp_path}/wiki.db") + Base.metadata.create_all(engine) + Session = sessionmaker(bind=engine) + + session = Session() + session.add(UserShadow(user_id=USER_ID, username="wiki-tester")) + session.add( + KBSpace( + kb_id=KB_ID, + user_id=USER_ID, + name="测试知识库", + visibility="private", + extra_data={"index_modes": ["rag", "wiki"]}, + ) + ) + session.add( + KBDocument( + document_id=DOC_ID, + kb_id=KB_ID, + title="示例科技年度报告", + filename="report.md", + size_bytes=100, + mime_type="text/markdown", + storage_key="k", + indexing_status="completed", + ) + ) + for index, text in enumerate( + [ + "示例科技有限公司成立于 2020 年，主营企业级检索产品。", + "检索增强生成（RAG）先召回文档再交给大模型生成答案。", + "示例科技在 2024 年发布了基于 RAG 的问答平台。", + ] + ): + session.add( + KBChunk( + chunk_id=f"chunk-{index}", + kb_id=KB_ID, + document_id=DOC_ID, + chunk_index=index, + content=text, + ) + ) + session.commit() + yield session, Session + session.close() + + +@pytest.fixture +def stub_llm(monkeypatch): + """按提示词特征分派桩响应，模拟各阶段的模型输出。""" + calls: list[str] = [] + + def _fake_call(*, system, user, budget=None, purpose="wiki", **kwargs): + calls.append(purpose) + if budget is not None: + budget.check() + budget.record(len(user), 100) + + if purpose == "候选抽取": + return json.dumps( + { + "entities": [ + { + "name": "示例科技", + "slug": "entity/acme", + "aliases": ["示例科技有限公司"], + "description": "一家做企业级检索产品的公司。", + "details": "成立于 2020 年。", + } + ], + "concepts": [ + { + "name": "检索增强生成", + "slug": "concept/rag", + "aliases": ["RAG"], + "description": "先检索再生成的技术路线。", + "details": "召回文档后交给大模型。", + } + ], + } + ) + if purpose == "引文标注": + return json.dumps( + { + "citations": { + "entity/acme": ["c001", "c003"], + "concept/rag": ["c002"], + }, + "new_slugs": [], + } + ) + if purpose == "去重判定": + return json.dumps({"merges": {}}) + if purpose == "目录规划": + return json.dumps( + { + "assignments": [ + {"slug": "entity/acme", "path": ["组织"]}, + {"slug": "concept/rag", "path": ["技术", "检索"]}, + ] + } + ) + if purpose == "写页": + return "SUMMARY: 该条目的一句话摘要。\n# 条目\n\n正文内容，提到 检索增强生成。" + if purpose == "文档摘要": + return "SUMMARY: 这篇文档讲示例科技。\n# 示例科技年度报告\n\n概要正文。" + if purpose == "索引导语": + return "# 测试知识库\n\n本库覆盖企业与检索技术。" + return "{}" + + monkeypatch.setattr('features.knowledge.wiki.extract.call_llm_json', _wrap_json(_fake_call)) + monkeypatch.setattr('features.knowledge.wiki.cite.call_llm_json', _wrap_json(_fake_call)) + monkeypatch.setattr('features.knowledge.wiki.dedup.call_llm_json', _wrap_json(_fake_call)) + monkeypatch.setattr('features.knowledge.wiki.taxonomy.call_llm_json', _wrap_json(_fake_call)) + monkeypatch.setattr('features.knowledge.wiki.reduce.call_llm', _fake_call) + monkeypatch.setattr('features.knowledge.wiki.finalize.call_llm', _fake_call) + return calls + + +def _wrap_json(fake_call): + from features.knowledge.wiki.llm import parse_json_object + + def _inner(*, system, user, budget=None, purpose="wiki", **kwargs): + return parse_json_object( + fake_call(system=system, user=user, budget=budget, purpose=purpose) + ) + + return _inner + + +def _point_session_local(monkeypatch, Session): + """local_provider / store 用模块级 SessionLocal，测试里要一并指过去。""" + monkeypatch.setattr('features.knowledge.wiki.local_provider.SessionLocal', Session) + monkeypatch.setattr('features.knowledge.wiki.local_graph.SessionLocal', Session) diff --git a/src/backend/tests/features/knowledge/test_host_contributions.py b/src/backend/tests/features/knowledge/test_host_contributions.py new file mode 100644 index 00000000..910b867f --- /dev/null +++ b/src/backend/tests/features/knowledge/test_host_contributions.py @@ -0,0 +1,68 @@ +"""Domain seams preserve payloads, failures and concrete host authorization.""" +import subprocess +import sys + + +def test_unconfigured_core_seams_are_standalone(): + subprocess.run([sys.executable, "-c", """ +import sys +from core.harness.registry.catalog_options import catalog_options +from core.harness.registry.retrieval_context import runtime_request_context +from core.harness.registry.model_credentials import prepare_request_headers +assert catalog_options('kb_spaces', object()) == [] +assert runtime_request_context(['kb_one']) == ({}, {}) +assert prepare_request_headers('key', 'https://model.invalid') == { + 'Content-Type': 'application/json', 'Authorization': 'Bearer key'} +assert not any(name.startswith(('features.', 'integrations.')) for name in sys.modules) +"""], check=True) + + +def test_retrieval_host_uses_current_provider_function(monkeypatch): + from core.harness.registry.retrieval_context import runtime_request_context + from features.knowledge import edition as external_provider + from features.knowledge.extension import register_retrieval_context + selected = ['kb_local', 'remote_one'] + captured = [] + def provider(ids): + captured.append(ids) + return {'X-Remote': 'one'}, {'REMOTE_IDS': 'one'} + register_retrieval_context() + monkeypatch.setattr(external_provider, 'runtime_request_context', provider) + assert runtime_request_context(selected) == ({'X-Remote': 'one'}, {'REMOTE_IDS': 'one'}) + assert captured == [selected] + + +def test_model_host_propagates_account_rejection(monkeypatch): + import pytest + from core.harness.registry.model_credentials import prepare_request_headers + from features.desktop.bridge import model_credentials as desktop_model_credentials + from features import register_model_credentials + def reject(*args): + raise ValueError('account changed') + register_model_credentials() + monkeypatch.setattr(desktop_model_credentials, 'prepare_request_headers', reject) + with pytest.raises(ValueError, match='account changed'): + prepare_request_headers('reference', 'https://model.invalid') + + +def test_knowledge_picker_keeps_order_and_omits_deleted(db_session): + from datetime import timedelta + from core.db.models import UserShadow + from core.infra.time import utc_now + from core.harness.registry.catalog_options import catalog_options + from features.knowledge.extension import register_catalog_options + from features.knowledge.models import KBSpace + register_catalog_options() + now = utc_now() + db_session.add(UserShadow(user_id='picker-owner', username='owner')) + db_session.flush() + db_session.add_all([ + KBSpace(kb_id='old', user_id='picker-owner', name='Old', created_at=now-timedelta(days=1)), + KBSpace(kb_id='new', user_id='picker-owner', name='New', description='details', created_at=now), + KBSpace(kb_id='gone', user_id='picker-owner', name='Gone', deleted_at=now), + ]) + db_session.commit() + assert catalog_options('kb_spaces', db_session) == [ + {'id': 'new', 'name': 'New', 'description': 'details'}, + {'id': 'old', 'name': 'Old', 'description': ''}, + ] diff --git a/src/backend/tests/mcp/test_kb_backend_routing.py b/src/backend/tests/features/knowledge/test_kb_backend_routing.py similarity index 74% rename from src/backend/tests/mcp/test_kb_backend_routing.py rename to src/backend/tests/features/knowledge/test_kb_backend_routing.py index 6cd0b859..d8dcd3cb 100644 --- a/src/backend/tests/mcp/test_kb_backend_routing.py +++ b/src/backend/tests/features/knowledge/test_kb_backend_routing.py @@ -1,14 +1,15 @@ """Public retrieval selects one backend and preserves local access boundaries.""" +from features.knowledge import kb_vector +from core.models import embedding as vector_embedding import asyncio import json from types import SimpleNamespace import pytest -from mcp import types -from core.kb import external_provider, kb_vector -from core.services.system_config import SystemConfigService +from features.knowledge import edition as external_provider, kb_vector +from core.config.store.system_config import SystemConfigService from extensions.mcp_servers.retrieve_dataset_content_mcp import impl, server, wiki_impl @@ -30,7 +31,8 @@ def test_database_url(tmp_path): @pytest.fixture def local_data(db_session, monkeypatch): from core.db import engine - from core.db.models import KBSpace, UserShadow + from features.knowledge.models import KBSpace + from core.db.models import UserShadow from sqlalchemy.orm import sessionmaker db_session.add_all( @@ -50,7 +52,7 @@ def local_data(db_session, monkeypatch): ) db_session.commit() monkeypatch.setattr(engine, "SessionLocal", sessionmaker(bind=db_session.get_bind())) - monkeypatch.setattr(kb_vector, "embed_text", lambda *args, **kwargs: [0.1]) + monkeypatch.setattr(vector_embedding, 'embed_text', lambda *args, **kwargs: [0.1]) calls = [] def search(**kwargs): @@ -65,7 +67,7 @@ def search(**kwargs): } ] - monkeypatch.setattr(kb_vector, "hybrid_search", search) + monkeypatch.setattr(kb_vector, 'hybrid_search', search) return calls @@ -141,29 +143,40 @@ async def external(**kwargs): assert local_data == [] +@pytest.mark.asyncio @pytest.mark.parametrize("provider", ["custom", "dify", "fastgpt", "weknora"]) -def test_public_tool_remains_available_for_selected_backend(monkeypatch, provider): +async def test_public_tool_remains_available_for_selected_backend(monkeypatch, provider): + from core.llm.mcp.lifetime import discover + from tests.llm.test_mcp_parallel_execution import probe_server + configure(monkeypatch, provider) monkeypatch.setattr(wiki_impl, "wiki_supported", lambda: False) - handler = server.mcp._mcp_server.request_handlers[types.ListToolsRequest] - result = asyncio.run(handler(types.ListToolsRequest(method="tools/list"))) - names = {tool.name for tool in result.root.tools} - assert {"retrieve_dataset_content", "retrieve_local_kb", "list_datasets"} <= names + async with probe_server(server.mcp) as (url, _): + resource = await discover("retrieval", {"transport": "streamable_http", "url": url}) + try: + names = {tool.name for tool in resource.tools} + assert {"retrieve_dataset_content", "retrieve_local_kb", "list_datasets"} <= names + finally: + await resource.close() + +@pytest.mark.asyncio +async def test_local_public_result_survives_mcp_protocol(monkeypatch, local_data): + from core.llm.mcp.lifetime import discover + from tests.llm.test_mcp_parallel_execution import probe_server -def test_local_public_result_survives_mcp_protocol(monkeypatch, local_data): configure(monkeypatch, "custom") - monkeypatch.setenv("CURRENT_USER_ID", "reader") - handler = server.mcp._mcp_server.request_handlers[types.CallToolRequest] - request = types.CallToolRequest( - method="tools/call", - params=types.CallToolRequestParams( - name="retrieve_dataset_content", arguments={"query": "乘法定义"} - ), - ) - result = asyncio.run(handler(request)) - payload = json.loads(result.root.content[0].text) - assert payload["items"][0]["content"] == "3 × 4 = 12" + async with probe_server(server.mcp) as (url, _): + resource = await discover("retrieval", {"transport": "streamable_http", "url": url, + "headers": {"X-Current-User-Id": "reader"}}) + try: + tool = next(t for t in resource.tools if t.name == "retrieve_dataset_content") + result = await tool(query="乘法定义") + payload = result.metadata["mcp"]["structuredContent"]["result"] + assert "items" in payload, payload + assert payload["items"][0]["content"] == "3 × 4 = 12" + finally: + await resource.close() @pytest.mark.parametrize("provider", ["custom", "fastgpt"]) diff --git a/src/backend/tests/test_kb_document_status.py b/src/backend/tests/features/knowledge/test_kb_document_status.py similarity index 89% rename from src/backend/tests/test_kb_document_status.py rename to src/backend/tests/features/knowledge/test_kb_document_status.py index fca8451b..85c745e4 100644 --- a/src/backend/tests/test_kb_document_status.py +++ b/src/backend/tests/features/knowledge/test_kb_document_status.py @@ -1,8 +1,9 @@ from datetime import datetime, timezone -from core.db.models import KBDocument, KBSpace, UserShadow -from core.db.repository import KBRepository -from core.kb.document_status import classify_document_status, summarize_document_statuses +from features.knowledge.models import KBDocument, KBSpace +from core.db.models import UserShadow +from features.knowledge.repository import KBRepository +from features.knowledge.document_status import classify_document_status, summarize_document_statuses def test_classify_document_status_normalizes_local_and_external_values(): diff --git a/src/backend/tests/services/test_kb_index_queue.py b/src/backend/tests/features/knowledge/test_kb_index_queue.py similarity index 98% rename from src/backend/tests/services/test_kb_index_queue.py rename to src/backend/tests/features/knowledge/test_kb_index_queue.py index 3d2e60d4..05706676 100644 --- a/src/backend/tests/services/test_kb_index_queue.py +++ b/src/backend/tests/features/knowledge/test_kb_index_queue.py @@ -12,8 +12,9 @@ import pytest from core.db.engine import Base -from core.db.models import KBDocument, KBSpace, UserShadow -from core.kb import index_queue +from features.knowledge.models import KBDocument, KBSpace +from core.db.models import UserShadow +from features.knowledge import index_queue from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker diff --git a/src/backend/tests/services/test_kb_index_worker.py b/src/backend/tests/features/knowledge/test_kb_index_worker.py similarity index 88% rename from src/backend/tests/services/test_kb_index_worker.py rename to src/backend/tests/features/knowledge/test_kb_index_worker.py index 29d96495..38c7d3ec 100644 --- a/src/backend/tests/services/test_kb_index_worker.py +++ b/src/backend/tests/features/knowledge/test_kb_index_worker.py @@ -9,7 +9,7 @@ from __future__ import annotations import pytest -from core.kb.index_worker import KBIndexWorker, _concurrency +from features.knowledge.index_worker import KBIndexWorker, _concurrency class _FakeStorage: @@ -46,11 +46,11 @@ def test_run_one_fetches_source_from_storage_and_passes_params_through( monkeypatch, snapshot ): storage = _FakeStorage(payload=b"hello world") - monkeypatch.setattr("core.storage.get_storage", lambda: storage) + monkeypatch.setattr('core.files.storage.get_storage', lambda: storage) seen = {} monkeypatch.setattr( - "core.content.kb_processing.vectorise_document_background", + 'features.knowledge.processing.vectorise_document_background', lambda **kwargs: seen.update(kwargs), ) @@ -72,16 +72,16 @@ def test_run_one_settles_the_document_when_the_source_cannot_be_read( ): """取不回原文不能把任务默默丢掉——那正是文档永远停在「索引中」的老毛病。""" monkeypatch.setattr( - "core.storage.get_storage", lambda: _FakeStorage(boom=OSError("gone")) + 'core.files.storage.get_storage', lambda: _FakeStorage(boom=OSError("gone")) ) monkeypatch.setattr( - "core.content.kb_processing.vectorise_document_background", + 'features.knowledge.processing.vectorise_document_background', lambda **kwargs: pytest.fail("原文都没取到，不该进索引"), ) settled = {} monkeypatch.setattr( - "core.kb.index_queue.give_up_or_requeue", + 'features.knowledge.index_queue.give_up_or_requeue', lambda db, document_id, error: settled.update(id=document_id, error=error), ) diff --git a/src/backend/tests/features/knowledge/test_kb_multimodal_assets.py b/src/backend/tests/features/knowledge/test_kb_multimodal_assets.py new file mode 100644 index 00000000..e21f2b93 --- /dev/null +++ b/src/backend/tests/features/knowledge/test_kb_multimodal_assets.py @@ -0,0 +1,37 @@ +"""Collect knowledge cases under their original node names.""" + +from tests.features.knowledge.kb_multimodal_assets_support import fake_storage + +from tests.features.knowledge.kb_multimodal_assets_cases_1 import ( + test_plain_parse_does_not_request_media, + test_rich_parse_requests_media_and_decodes_data_uri, + test_rich_parse_survives_undecodable_image, + test_uploaded_image_no_longer_indexed_as_binary_garbage, + test_pdf_assets_carry_caption_and_locator, + test_rich_parse_failure_degrades_to_text, + test_with_assets_false_keeps_legacy_path, + test_asset_lands_on_the_chunk_that_contains_it, + test_placeholder_rewritten_in_both_parent_and_child, + test_tiny_assets_are_dropped, + test_storage_failure_skips_one_asset_not_the_document, + test_no_vision_model_is_a_degradation_not_a_failure, + test_evidence_lands_in_caption_and_ocr_merges_into_text, + test_ocr_identical_to_caption_is_not_duplicated, + test_text_content_is_capped, + test_single_image_failure_keeps_asset_indexable, + test_whole_batch_exception_does_not_fail_the_document, + test_kb_never_saturates_the_shared_vision_lane, + test_unreadable_asset_is_skipped_without_calling_vision, + test_per_document_cap_marks_the_rest_explicitly, + test_index_text_prefers_caption_then_caption_text, + test_asset_rows_point_at_their_parent_chunk, +) + +from tests.features.knowledge.kb_multimodal_assets_cases_2 import ( + test_orphan_asset_falls_back_to_itself, + test_multimodal_toggle_precedence, + test_describe_batch_runs_on_the_host_loop_when_there_is_one, + test_describe_batch_without_host_loop_disables_the_evidence_cache, + test_explicit_loop_wins_over_anyio, + test_media_only_parent_falls_back_to_the_asset_row_content, +) diff --git a/src/backend/tests/mcp/test_kb_vector_safety.py b/src/backend/tests/features/knowledge/test_kb_vector_safety.py similarity index 57% rename from src/backend/tests/mcp/test_kb_vector_safety.py rename to src/backend/tests/features/knowledge/test_kb_vector_safety.py index 6e1543c4..704b765a 100644 --- a/src/backend/tests/mcp/test_kb_vector_safety.py +++ b/src/backend/tests/features/knowledge/test_kb_vector_safety.py @@ -1,4 +1,7 @@ from __future__ import annotations +from features.knowledge import kb_vector_state +from features.knowledge import vector_client +from core.models import embedding as vector_embedding import sys from types import ModuleType @@ -39,63 +42,63 @@ def create_schema(self, *_args, **_kwargs): def test_dimension_mismatch_preserves_existing_collection(monkeypatch): - from core.kb import kb_vector + from features.knowledge import kb_vector _install_fake_pymilvus(monkeypatch) client = _ExistingCollectionClient(existing_dim=2048) - monkeypatch.setattr(kb_vector, "_get_client", lambda: client) - monkeypatch.setattr(kb_vector, "detect_embed_dim", lambda: 1024) - monkeypatch.setattr(kb_vector, "_VERIFIED_COLLECTION_DIM", None) + monkeypatch.setattr(vector_client, 'get_client', lambda: client) + monkeypatch.setattr(vector_embedding, 'detect_embed_dim', lambda: 1024) + monkeypatch.setattr(kb_vector_state, 'VERIFIED_COLLECTION_DIM', None) with pytest.raises(RuntimeError, match="维度不匹配"): - kb_vector.get_or_create_collection(timeout=3.0) + vector_client.get_or_create_collection(timeout=3.0) assert client.drop_called is False - assert kb_vector._VERIFIED_COLLECTION_DIM is None + assert kb_vector_state.VERIFIED_COLLECTION_DIM is None def test_dimension_probe_failure_preserves_existing_collection(monkeypatch): - from core.kb import kb_vector + from features.knowledge import kb_vector _install_fake_pymilvus(monkeypatch) client = _ExistingCollectionClient(describe_error=ConnectionError("milvus unavailable")) - monkeypatch.setattr(kb_vector, "_get_client", lambda: client) - monkeypatch.setattr(kb_vector, "detect_embed_dim", lambda: 2048) - monkeypatch.setattr(kb_vector, "_VERIFIED_COLLECTION_DIM", None) + monkeypatch.setattr(vector_client, 'get_client', lambda: client) + monkeypatch.setattr(vector_embedding, 'detect_embed_dim', lambda: 2048) + monkeypatch.setattr(kb_vector_state, 'VERIFIED_COLLECTION_DIM', None) with pytest.raises(RuntimeError, match="无法确认现有知识库集合的向量维度"): - kb_vector.get_or_create_collection(timeout=3.0) + vector_client.get_or_create_collection(timeout=3.0) assert client.drop_called is False - assert kb_vector._VERIFIED_COLLECTION_DIM is None + assert kb_vector_state.VERIFIED_COLLECTION_DIM is None def test_missing_dimension_metadata_preserves_existing_collection(monkeypatch): - from core.kb import kb_vector + from features.knowledge import kb_vector _install_fake_pymilvus(monkeypatch) client = _ExistingCollectionClient(existing_dim=None) - monkeypatch.setattr(kb_vector, "_get_client", lambda: client) - monkeypatch.setattr(kb_vector, "detect_embed_dim", lambda: 2048) - monkeypatch.setattr(kb_vector, "_VERIFIED_COLLECTION_DIM", None) + monkeypatch.setattr(vector_client, 'get_client', lambda: client) + monkeypatch.setattr(vector_embedding, 'detect_embed_dim', lambda: 2048) + monkeypatch.setattr(kb_vector_state, 'VERIFIED_COLLECTION_DIM', None) with pytest.raises(RuntimeError, match="dim 参数缺失"): - kb_vector.get_or_create_collection(timeout=3.0) + vector_client.get_or_create_collection(timeout=3.0) assert client.drop_called is False - assert kb_vector._VERIFIED_COLLECTION_DIM is None + assert kb_vector_state.VERIFIED_COLLECTION_DIM is None def test_matching_dimension_marks_existing_collection_verified(monkeypatch): - from core.kb import kb_vector + from features.knowledge import kb_vector _install_fake_pymilvus(monkeypatch) client = _ExistingCollectionClient(existing_dim=2048) - monkeypatch.setattr(kb_vector, "_get_client", lambda: client) - monkeypatch.setattr(kb_vector, "detect_embed_dim", lambda: 2048) - monkeypatch.setattr(kb_vector, "_VERIFIED_COLLECTION_DIM", None) + monkeypatch.setattr(vector_client, 'get_client', lambda: client) + monkeypatch.setattr(vector_embedding, 'detect_embed_dim', lambda: 2048) + monkeypatch.setattr(kb_vector_state, 'VERIFIED_COLLECTION_DIM', None) - kb_vector.get_or_create_collection(timeout=3.0) + vector_client.get_or_create_collection(timeout=3.0) assert client.drop_called is False - assert kb_vector._VERIFIED_COLLECTION_DIM == 2048 + assert kb_vector_state.VERIFIED_COLLECTION_DIM == 2048 diff --git a/src/backend/tests/mcp/test_kb_vector_timeouts.py b/src/backend/tests/features/knowledge/test_kb_vector_timeouts.py similarity index 68% rename from src/backend/tests/mcp/test_kb_vector_timeouts.py rename to src/backend/tests/features/knowledge/test_kb_vector_timeouts.py index 649689e7..ba7ea7fa 100644 --- a/src/backend/tests/mcp/test_kb_vector_timeouts.py +++ b/src/backend/tests/features/knowledge/test_kb_vector_timeouts.py @@ -1,11 +1,16 @@ from __future__ import annotations +from core.models import embedding_state +from features.knowledge import kb_vector +from features.knowledge import kb_vector_state +from features.knowledge import vector_client +from core.models import embedding as vector_embedding import sys from types import ModuleType def test_embed_text_uses_retrieval_timeout_and_caches_dimension(monkeypatch): - from core.kb import kb_vector + from features.knowledge import kb_vector captured: dict = {} @@ -22,22 +27,22 @@ def _post(url, **kwargs): return _Response() monkeypatch.setattr( - kb_vector, - "_resolve_embed_config", + vector_embedding, + 'resolve_embed_config', lambda: ("http://embedding.test/v1", "embed-model", "secret"), ) - monkeypatch.setattr(kb_vector.requests, "post", _post) - kb_vector._DETECTED_DIM_CACHE.clear() + monkeypatch.setattr(vector_embedding.requests, "post", _post) + embedding_state.process_state().DETECTED_DIM_CACHE.clear() - vector = kb_vector.embed_text("query", timeout=4.5) + vector = vector_embedding.embed_text("query", timeout=4.5) assert vector == [0.1, 0.2, 0.3] assert captured["timeout"] == 4.5 - assert kb_vector._DETECTED_DIM_CACHE[("http://embedding.test/v1", "embed-model")] == 3 + assert embedding_state.process_state().DETECTED_DIM_CACHE[("http://embedding.test/v1", "embed-model")] == 3 def test_hybrid_search_forwards_timeout_to_milvus(monkeypatch): - from core.kb import kb_vector + from features.knowledge import kb_vector captured: dict = {} @@ -69,10 +74,10 @@ def search(self, **kwargs): ] monkeypatch.setattr( - kb_vector, "get_or_create_collection", lambda **kwargs: captured.update(kwargs) + vector_client, 'get_or_create_collection', lambda **kwargs: captured.update(kwargs) ) - monkeypatch.setattr(kb_vector, "_get_client", lambda: _Client()) - monkeypatch.setattr(kb_vector, "_is_lite", lambda: True) + monkeypatch.setattr(vector_client, 'get_client', lambda: _Client()) + monkeypatch.setattr(vector_client, 'is_lite', lambda: True) hits = kb_vector.hybrid_search( "user-1", diff --git a/src/backend/tests/features/knowledge/test_kb_wiki_end_to_end.py b/src/backend/tests/features/knowledge/test_kb_wiki_end_to_end.py new file mode 100644 index 00000000..9eb83cf5 --- /dev/null +++ b/src/backend/tests/features/knowledge/test_kb_wiki_end_to_end.py @@ -0,0 +1,31 @@ +"""Collect knowledge cases under their original node names.""" + +from tests.features.knowledge.kb_wiki_end_to_end_support import wiki_db, stub_llm + +from tests.features.knowledge.kb_wiki_end_to_end_cases_1 import ( + test_ingest_creates_pages_with_lineage, + test_ingest_links_pages_to_each_other, + test_finalize_builds_index_and_backlinks, + test_finalize_strips_dead_links, + test_retract_removes_document_lineage, + test_read_surface_matches_expected_shape, + test_source_chunks_resolve_by_id, + test_source_chunks_fall_back_when_refs_are_stale, + test_search_ranks_title_hits_above_body, + test_search_supports_alternation, + test_folders_report_recursive_page_counts, + test_graph_ego_mode_centres_on_node, + test_graph_ego_on_missing_node_is_empty, + test_ingest_is_idempotent_on_rerun, + test_concurrent_stages_never_touch_the_session, + test_quota_stops_generation_without_failing, +) + +from tests.features.knowledge.kb_wiki_end_to_end_cases_2 import ( + test_custom_instructions_reach_every_stage, + test_worker_batches_sibling_ingest_jobs, + test_batching_never_crosses_knowledge_bases, + test_one_knowledge_base_runs_one_batch_at_a_time, + test_a_busy_knowledge_base_does_not_block_other_ones, + test_a_dead_workers_knowledge_base_is_taken_over, +) diff --git a/src/backend/tests/services/test_kb_wiki_pipeline.py b/src/backend/tests/features/knowledge/test_kb_wiki_pipeline.py similarity index 96% rename from src/backend/tests/services/test_kb_wiki_pipeline.py rename to src/backend/tests/features/knowledge/test_kb_wiki_pipeline.py index 9ba4a946..5f4a1bcd 100644 --- a/src/backend/tests/services/test_kb_wiki_pipeline.py +++ b/src/backend/tests/features/knowledge/test_kb_wiki_pipeline.py @@ -13,23 +13,13 @@ import pytest -from core.kb.wiki import linkify, llm, slug as slug_mod -from core.kb.wiki.cite import ( - _parse_citations, - apply_citations, - render_chunks_xml, - split_into_batches, -) -from core.kb.wiki.config import ( - MAX_CUSTOM_INSTRUCTIONS_LENGTH, - WikiConfig, - normalize_granularity, - normalize_index_modes, -) -from core.kb.wiki.dedup import _similarity, apply_merges, select_candidates -from core.kb.wiki.extract import CandidateItem, parse_candidates -from core.kb.wiki.prompts import append_business_instructions -from core.kb.wiki.store import clean_category_path +from features.knowledge.wiki import linkify, llm, slug as slug_mod +from features.knowledge.wiki.cite import _parse_citations, apply_citations, render_chunks_xml, split_into_batches +from features.knowledge.wiki.config import MAX_CUSTOM_INSTRUCTIONS_LENGTH, WikiConfig, normalize_granularity, normalize_index_modes +from features.knowledge.wiki.dedup import _similarity, apply_merges, select_candidates +from features.knowledge.wiki.extract import CandidateItem, parse_candidates +from features.knowledge.wiki.prompts import append_business_instructions +from features.knowledge.wiki.store import clean_category_path class _Chunk: diff --git a/src/backend/tests/features/marketplace/__init__.py b/src/backend/tests/features/marketplace/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/src/backend/tests/features/marketplace/__init__.py @@ -0,0 +1 @@ + diff --git a/src/backend/tests/features/marketplace/mcp_marketplace_cases_1.py b/src/backend/tests/features/marketplace/mcp_marketplace_cases_1.py new file mode 100644 index 00000000..7d39bd16 --- /dev/null +++ b/src/backend/tests/features/marketplace/mcp_marketplace_cases_1.py @@ -0,0 +1,434 @@ +from __future__ import annotations +from tests.features.marketplace.mcp_marketplace_support import ( + AdminMcpServer, + BadRequestError, + Base, + McpMarketInstallation, + McpMarketItem, + McpMarketSubmission, + McpMarketVersion, + McpServerConfigService, + OAuthBundleStorage, + OAuthToken, + Path, + ResourceNotFoundError, + SimpleNamespace, + _IS_CE_TREE, + _admin_private_market, + _approved_market, + _patch_install_side_effects, + _patch_probe, + _private_server, + _tools, + admin_mcp_routes, + asyncio, + create_engine, + db, + decrypt_mcp_headers, + encrypt_legacy_mcp_headers, + encrypt_mcp_headers, + httpx, + importlib, + json, + listing, + make_client, + management, + mask_mcp_headers, + materialize_mcp_http_connection, + mcp_administration, + mcp_listing, + mcp_curated, + mcp_installation, + mcp_oauth_bundle_storage_key, + mcp_publishing, + mcp_query_secret_storage_key, + mcp_revalidation, + mcp_submissions, + mcp_versions, + me_capability_routes, + oauth_service, + pytest, + sessionmaker, + socket, + tool_snapshot_hash, + validate_remote_mcp_url, +) + +@pytest.mark.asyncio +async def test_http_probe_releases_its_discovery_resource(db, monkeypatch): + from mcp.types import Tool + from core.harness.registry import mcp_clients + calls = [] + class Resource: + descriptors = [Tool(name="search", description="Search", input_schema={"type":"object"})] + async def close(self): + calls.append("closed") + async def discover(name, cfg): + calls.append(name) + return Resource() + monkeypatch.setattr(mcp_clients, "discover_tools", discover) + row = AdminMcpServer(server_id="probe", display_name="Probe", transport="streamable_http", url="https://mcp.example.test/mcp", headers={}, is_enabled=True) + ok, error = await management.probe_mcp_connectivity(row) + assert ok and not error + assert calls == ["probe", "closed"] + assert row.tools_json[0]["name"] == "search" + + +@pytest.mark.asyncio +async def test_http_probe_surfaces_401_instead_of_cancel_scope_noise(db, monkeypatch): + import httpx2 + request = httpx2.Request("POST", "https://mcp.example.test/mcp") + response = httpx2.Response(401, request=request) + unauthorized = httpx2.HTTPStatusError("401 Unauthorized", request=request, response=response) + async def discover(name, cfg): + raise BaseExceptionGroup("MCP request failed", [unauthorized, asyncio.CancelledError("Cancelled via cancel scope")]) + from core.harness.registry import mcp_clients + monkeypatch.setattr(mcp_clients, "discover_tools", discover) + row = AdminMcpServer( + server_id="unauthorized_probe", + display_name="Unauthorized Probe", + transport="streamable_http", + url="https://mcp.example.test/mcp", + headers={}, + is_enabled=True, + ) + + ok, error = await management.probe_mcp_connectivity(row) + + assert ok is False + assert error == "远端 MCP 返回 401 Unauthorized，请检查鉴权设置和 Token" + assert "cancel scope" not in error.lower() + + +def test_ce_metadata_contains_mcp_market_auth_without_ee_dependency(): + project_root = Path(__file__).resolve().parents[5] + module_path = project_root / "ce/overlay/src/backend/core/db/personal_schema.py" + if not module_path.exists(): + module_path = project_root / "src/backend/core/db/personal_schema.py" + spec = importlib.util.spec_from_file_location("ce_edition_tables_test", module_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + metadata = module.ce_metadata() + assert "mcp_market_items" in metadata.tables + assert "auth_config" in metadata.tables["mcp_market_versions"].columns + assert "auth_config" in metadata.tables["mcp_market_submissions"].columns + + +def test_header_credentials_are_encrypted_and_masked(): + encrypted = encrypt_mcp_headers({"Authorization": "Bearer top-secret"}) + assert encrypted["Authorization"].startswith("enc:v1:") + assert "top-secret" not in encrypted["Authorization"] + assert decrypt_mcp_headers(encrypted) == {"Authorization": "Bearer top-secret"} + assert mask_mcp_headers(encrypted) == {"Authorization": "***"} + + +def test_runtime_query_credentials_are_encrypted_and_never_sent_as_headers(): + storage_key = mcp_query_secret_storage_key("key") + encrypted = encrypt_mcp_headers({storage_key: "amap-secret"}) + assert "amap-secret" not in str(encrypted) + assert mask_mcp_headers(encrypted) == {} + + runtime_url, runtime_headers = materialize_mcp_http_connection( + "https://mcp.amap.com/mcp?locale=zh-CN", + decrypt_mcp_headers(encrypted), + ) + assert runtime_url == "https://mcp.amap.com/mcp?locale=zh-CN&key=amap-secret" + assert runtime_headers == {} + + row = AdminMcpServer( + server_id="amap_private", + display_name="Amap", + transport="streamable_http", + url="https://mcp.amap.com/mcp?locale=zh-CN", + headers=encrypted, + is_enabled=True, + ) + runtime_config = McpServerConfigService.get_instance()._row_to_config(row) + assert runtime_config["url"] == runtime_url + assert "headers" not in runtime_config + + +@pytest.mark.asyncio +async def test_oauth_bundle_is_provider_neutral_and_never_sent_as_header(): + storage = OAuthBundleStorage( + { + "client_metadata": { + "redirect_uris": ["https://app.example.test/api/v1/mcp-market/oauth/callback"], + "client_name": "HugAgentOS MCP Client", + }, + "client_info": { + "redirect_uris": ["https://app.example.test/api/v1/mcp-market/oauth/callback"], + "client_id": "client-id", + "token_endpoint_auth_method": "none", + }, + } + ) + await storage.set_tokens( + OAuthToken( + access_token="oauth-access-secret", + refresh_token="oauth-refresh-secret", + expires_in=3600, + ) + ) + bundle = storage.bundle + encrypted = encrypt_mcp_headers( + {mcp_oauth_bundle_storage_key(): json.dumps(bundle, separators=(",", ":"))} + ) + assert "oauth-access-secret" not in str(encrypted) + assert mask_mcp_headers(encrypted) == {} + + row = AdminMcpServer( + server_id="oauth_private", + display_name="OAuth MCP", + transport="streamable_http", + url="https://mcp.example.test/mcp", + headers=encrypted, + is_enabled=True, + ) + runtime_config = McpServerConfigService.get_instance()._row_to_config(row) + assert "oauth_provider" in runtime_config + assert "headers" not in runtime_config + client = make_client(row.server_id, runtime_config) + assert client.mode == "auto" + assert client.server.args[1]["oauth_provider"] is runtime_config["oauth_provider"] + + +@pytest.mark.asyncio +async def test_oauth_callback_and_status_are_shared_without_plaintext_code(monkeypatch): + import fakeredis.aioredis + + fake_redis = fakeredis.aioredis.FakeRedis(decode_responses=True) + monkeypatch.setattr("core.infra.ephemeral.get_redis", lambda **_: fake_redis) + monkeypatch.setattr("core.infra.ephemeral.redis_configured", lambda: True) + flow = oauth_service.OAuthInstallFlow( + flow_id="mcpoauth_test", + slug="gitlab-official", + owner_user_id="u1", + installed_by="u1", + auth_method="oauth2", + credentials={}, + callback_url="https://app.example.test/api/v1/mcp-market/oauth/callback", + status="waiting_for_user", + ) + await oauth_service.store_flow_status(flow) + oauth_service.process_state().flows.pop(flow.flow_id, None) # simulate callback landing on another worker + await oauth_service.complete_callback( + flow.flow_id, + code="short-lived-authorization-code", + state="oauth-state", + ) + callback_raw = await fake_redis.get(oauth_service.callback_key(flow.flow_id)) + assert "short-lived-authorization-code" not in callback_raw + status = await oauth_service.get_flow_status(flow.flow_id, owner_user_id="u1") + assert status["status"] == "processing_callback" + await fake_redis.aclose() + + +@pytest.mark.asyncio +async def test_oauth_start_hands_back_before_discovery_finishes(db, monkeypatch): + """Slow authorization-URL discovery must keep the flow pollable, never fail the install.""" + import fakeredis.aioredis + + fake_redis = fakeredis.aioredis.FakeRedis(decode_responses=True) + monkeypatch.setattr("core.infra.ephemeral.get_redis", lambda **_: fake_redis) + monkeypatch.setattr("core.infra.ephemeral.redis_configured", lambda: True) + monkeypatch.setattr("core.capabilities.mcp.oauth.DISCOVERY_GRACE_SECONDS", 0.05) + mcp_curated.ensure_curated_market_items(db) + + async def slow_discovery(flow): + await asyncio.sleep(0.3) + flow.authorization_url = "https://provider.example.test/authorize?code_challenge=x" + flow.status = "waiting_for_user" + await oauth_service.store_flow_status(flow) + flow.ready.set() + + monkeypatch.setattr(oauth_service, "run_flow", slow_discovery) + try: + started = await oauth_service.start_oauth_install( + db, + slug="gitlab-official", + owner_user_id="u1", + installed_by="u1", + auth_method="oauth2", + credentials={}, + callback_url_base="https://app.example.test/api/v1/mcp-market/oauth/callback", + ) + assert started["status"] == "starting" + assert started["authorization_url"] == "" + + await asyncio.sleep(0.4) + status = await oauth_service.get_flow_status(started["flow_id"], owner_user_id="u1") + assert status["status"] == "waiting_for_user" + assert status["authorization_url"].startswith("https://provider.example.test/authorize") + finally: + oauth_service.process_state().flows.clear() + await fake_redis.aclose() + + +@pytest.mark.asyncio +async def test_oauth_cancel_marks_flow_failed_and_wakes_worker(monkeypatch): + import fakeredis.aioredis + + fake_redis = fakeredis.aioredis.FakeRedis(decode_responses=True) + monkeypatch.setattr("core.infra.ephemeral.get_redis", lambda **_: fake_redis) + monkeypatch.setattr("core.infra.ephemeral.redis_configured", lambda: True) + flow = oauth_service.OAuthInstallFlow( + flow_id="mcpoauth_cancel", + slug="gitlab-official", + owner_user_id="u1", + installed_by="u1", + auth_method="oauth2", + credentials={}, + callback_url="https://app.example.test/api/v1/mcp-market/oauth/callback", + status="waiting_for_user", + ) + oauth_service.process_state().flows[flow.flow_id] = flow + try: + await oauth_service.store_flow_status(flow) + status = await oauth_service.cancel_flow(flow.flow_id, owner_user_id="u1") + assert status["status"] == "failed" + assert status["error"] == "OAuth 登录已取消" + assert flow.callback_received.is_set() + with pytest.raises(ResourceNotFoundError): + await oauth_service.cancel_flow(flow.flow_id, owner_user_id="u2") + finally: + oauth_service.process_state().flows.pop(flow.flow_id, None) + await fake_redis.aclose() + + +def test_oauth_callback_uses_browser_origin_not_untrusted_host(monkeypatch): + monkeypatch.setattr( + oauth_service, + "settings", + SimpleNamespace(server=SimpleNamespace(mcp_oauth_public_base_url="", is_prod=False)), + ) + request = SimpleNamespace( + headers={"origin": "https://app.example.test", "host": "attacker.invalid"}, + url=SimpleNamespace(netloc="attacker.invalid"), + ) + assert oauth_service.public_oauth_callback_url(request) == ( + "https://app.example.test/api/v1/mcp-market/oauth/callback" + ) + + +def test_oauth_callback_allows_configured_http_in_production(monkeypatch): + monkeypatch.setattr( + oauth_service, + "settings", + SimpleNamespace( + server=SimpleNamespace( + mcp_oauth_public_base_url="http://app.example.test/api", + is_prod=True, + ) + ), + ) + request = SimpleNamespace(headers={}, url=SimpleNamespace(netloc="ignored.example.test")) + + assert oauth_service.public_oauth_callback_url(request) == ( + "http://app.example.test/api/v1/mcp-market/oauth/callback" + ) + + +def test_legacy_plaintext_headers_are_encrypted_once(db): + row = _private_server(db) + row.headers = {"Authorization": "Bearer legacy"} + db.commit() + assert encrypt_legacy_mcp_headers(db) == 1 + db.refresh(row) + assert decrypt_mcp_headers(row.headers) == {"Authorization": "Bearer legacy"} + assert "legacy" not in str(row.headers) + assert encrypt_legacy_mcp_headers(db) == 0 + + +@pytest.mark.asyncio +async def test_remote_url_security_blocks_private_and_accepts_public_dns(monkeypatch): + with pytest.raises(BadRequestError): + await validate_remote_mcp_url("https://127.0.0.1/mcp") + with pytest.raises(BadRequestError): + await validate_remote_mcp_url("http://example.com/mcp") + + monkeypatch.setattr( + socket, + "getaddrinfo", + lambda *args, **kwargs: [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("8.8.8.8", 443))], + ) + await validate_remote_mcp_url("https://example.com/mcp") + + +@pytest.mark.asyncio +async def test_remote_url_security_allows_public_http_when_requested(monkeypatch): + monkeypatch.setattr( + socket, + "getaddrinfo", + lambda *args, **kwargs: [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("8.8.8.8", 80))], + ) + + await validate_remote_mcp_url("http://example.com/mcp", require_https=False) + + with pytest.raises(BadRequestError, match="本机、内网或保留地址"): + await validate_remote_mcp_url("http://127.0.0.1/mcp", require_https=False) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("address", ["127.0.0.1", "10.10.0.8", "172.18.0.12", "192.168.1.9"]) +async def test_remote_url_security_allows_trusted_private_networks_when_enabled(address): + await validate_remote_mcp_url( + f"http://{address}/mcp", + allow_private_network=True, + require_https=False, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("address", ["0.0.0.0", "169.254.169.254", "224.0.0.1"]) +async def test_remote_url_security_keeps_high_risk_ranges_blocked_when_private_enabled(address): + with pytest.raises(BadRequestError, match="本机、内网或保留地址"): + await validate_remote_mcp_url( + f"http://{address}/mcp", + allow_private_network=True, + require_https=False, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("allow_private_network", [False, True]) +async def test_self_service_mcp_uses_private_network_deployment_setting( + db, monkeypatch, allow_private_network +): + validation = {} + + async def _validate(url, **kwargs): + validation.update(url=url, **kwargs) + + async def _probe(row, _db): + row.tools_json = _tools("search") + return True, "" + + monkeypatch.setattr(me_capability_routes, "_require_flag", lambda *args: None) + monkeypatch.setattr(me_capability_routes, "validate_remote_mcp_url", _validate) + monkeypatch.setattr(me_capability_routes, "probe_mcp_connectivity", _probe) + monkeypatch.setattr( + me_capability_routes, + "settings", + SimpleNamespace( + server=SimpleNamespace( + mcp_self_service_allow_private_network=allow_private_network, + ) + ), + ) + + await me_capability_routes.create_my_mcp_server( + me_capability_routes.CreateUserMcpRequest( + display_name="Private MCP", + url="http://10.10.0.8/mcp", + ), + user=SimpleNamespace(user_id="u1"), + db=db, + ) + + assert validation == { + "url": "http://10.10.0.8/mcp", + "allow_private_network": allow_private_network, + "require_https": False, + } diff --git a/src/backend/tests/features/marketplace/mcp_marketplace_cases_2.py b/src/backend/tests/features/marketplace/mcp_marketplace_cases_2.py new file mode 100644 index 00000000..192a4816 --- /dev/null +++ b/src/backend/tests/features/marketplace/mcp_marketplace_cases_2.py @@ -0,0 +1,466 @@ +from __future__ import annotations +from tests.features.marketplace.mcp_marketplace_support import ( + AdminMcpServer, + BadRequestError, + Base, + McpMarketInstallation, + McpMarketItem, + McpMarketSubmission, + McpMarketVersion, + McpServerConfigService, + OAuthBundleStorage, + OAuthToken, + Path, + ResourceNotFoundError, + SimpleNamespace, + _IS_CE_TREE, + _admin_private_market, + _approved_market, + _patch_install_side_effects, + _patch_probe, + _private_server, + _tools, + admin_mcp_routes, + asyncio, + create_engine, + db, + decrypt_mcp_headers, + encrypt_legacy_mcp_headers, + encrypt_mcp_headers, + httpx, + importlib, + json, + listing, + make_client, + management, + mask_mcp_headers, + materialize_mcp_http_connection, + mcp_administration, + mcp_listing, + mcp_curated, + mcp_installation, + mcp_oauth_bundle_storage_key, + mcp_publishing, + mcp_query_secret_storage_key, + mcp_revalidation, + mcp_submissions, + mcp_versions, + me_capability_routes, + oauth_service, + pytest, + sessionmaker, + socket, + tool_snapshot_hash, + validate_remote_mcp_url, +) + +@pytest.mark.asyncio +async def test_remote_url_security_rechecks_tun_fake_ip_dns(monkeypatch): + with pytest.raises(BadRequestError): + await validate_remote_mcp_url("https://198.18.0.1/mcp") + + monkeypatch.setattr( + socket, + "getaddrinfo", + lambda *args, **kwargs: [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("198.18.0.42", 443))], + ) + + async def _public_dns(_hostname: str): + return {"8.8.8.8", "2001:4860:4860::8888"} + + monkeypatch.setattr(management, "_resolve_public_dns_via_doh", _public_dns) + await validate_remote_mcp_url("https://example.com/mcp") + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "answers", + [ + [ + ( + socket.AF_INET6, + socket.SOCK_STREAM, + 6, + "", + ("fc00::2", 443, 0, 0), + ) + ], + [ + ( + socket.AF_INET6, + socket.SOCK_STREAM, + 6, + "", + ("fdfe:dcba:9876::1b", 443, 0, 0), + ) + ], + [ + (socket.AF_INET, socket.SOCK_STREAM, 6, "", ("198.18.0.43", 443)), + ( + socket.AF_INET6, + socket.SOCK_STREAM, + 6, + "", + ("fdfe:dcba:9876::1b", 443, 0, 0), + ), + ], + ], + ids=["sing-box-ipv6", "tun-ipv6", "dual-stack"], +) +async def test_remote_url_security_rechecks_ipv6_tun_fake_ip_dns(monkeypatch, answers): + monkeypatch.setattr(socket, "getaddrinfo", lambda *args, **kwargs: answers) + + async def _public_dns(_hostname: str): + return {"140.82.112.4", "2606:50c0:8000::154"} + + monkeypatch.setattr(management, "_resolve_public_dns_via_doh", _public_dns) + await validate_remote_mcp_url("https://github.com/mcp") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("address", ["fd00::1", "fdfe:dcba:9877::1"]) +async def test_remote_url_security_does_not_treat_arbitrary_ula_as_fake_ip(monkeypatch, address): + monkeypatch.setattr( + socket, + "getaddrinfo", + lambda *args, **kwargs: [ + (socket.AF_INET6, socket.SOCK_STREAM, 6, "", (address, 443, 0, 0)) + ], + ) + public_dns_called = False + + async def _public_dns(_hostname: str): + nonlocal public_dns_called + public_dns_called = True + return {"8.8.8.8"} + + monkeypatch.setattr(management, "_resolve_public_dns_via_doh", _public_dns) + with pytest.raises(BadRequestError, match="本机、内网或保留地址"): + await validate_remote_mcp_url("https://internal.example.com/mcp") + assert public_dns_called is False + + +@pytest.mark.asyncio +async def test_remote_url_security_rejects_fake_ip_when_public_dns_is_private(monkeypatch): + monkeypatch.setattr( + socket, + "getaddrinfo", + lambda *args, **kwargs: [ + ( + socket.AF_INET6, + socket.SOCK_STREAM, + 6, + "", + ("fdfe:dcba:9876::1b", 443, 0, 0), + ) + ], + ) + + async def _public_dns(_hostname: str): + return {"10.0.0.1"} + + monkeypatch.setattr(management, "_resolve_public_dns_via_doh", _public_dns) + with pytest.raises(BadRequestError, match="公共 DNS 安全复核失败"): + await validate_remote_mcp_url("https://example.com/mcp") + + +def test_tool_hash_is_order_independent(): + first = _tools("search", "delete_record") + second = list(reversed(first)) + assert tool_snapshot_hash(first) == tool_snapshot_hash(second) + + +@pytest.mark.asyncio +async def test_submit_snapshots_without_credentials_then_approve(db, monkeypatch): + source = _private_server(db) + _patch_probe(monkeypatch) + + result = await mcp_submissions.submit_to_marketplace( + db, + source.server_id, + owner_user_id="u1", + submitter_name="Alice", + category="信息检索", + version="1.0.0", + note="Tested", + ) + submission = ( + db.query(McpMarketSubmission).filter_by(submission_id=result["submission_id"]).one() + ) + assert submission.status == "pending" + assert submission.auth_schema == [ + { + "key": "Authorization", + "label": "Authorization", + "target": "header", + "required": True, + "secret": True, + } + ] + assert "publisher-secret" not in str(submission.auth_schema) + assert submission.tools_json == _tools("search_documents") + + reviewed = await mcp_submissions.review_submission( + db, + submission.submission_id, + approve=True, + reviewed_by="admin", + ) + assert reviewed["status"] == "approved" + item = db.query(McpMarketItem).filter_by(slug=submission.slug).one() + version = db.query(McpMarketVersion).filter_by(version_id=item.latest_version_id).one() + assert item.source == "community" + assert version.tool_hash == tool_snapshot_hash(_tools("search_documents")) + + +@pytest.mark.asyncio +async def test_public_http_marketplace_submit_review_and_install(db, monkeypatch): + source = _private_server(db) + source.url = "http://8.8.8.8:8082/mcp" + db.commit() + + async def _probe(row, db=None): + return True, "" + + monkeypatch.setattr(mcp_installation, 'probe_mcp_connectivity', _probe) + monkeypatch.setattr(mcp_revalidation, 'probe_mcp_connectivity', _probe) + monkeypatch.setattr(mcp_submissions, 'probe_mcp_connectivity', _probe) + monkeypatch.setattr(mcp_publishing, 'probe_mcp_connectivity', _probe) + monkeypatch.setattr(mcp_installation, 'refresh_mcp_caches', lambda: None) + monkeypatch.setattr(mcp_administration, 'refresh_mcp_caches', lambda: None) + monkeypatch.setattr(mcp_versions, 'refresh_mcp_caches', lambda: None) + monkeypatch.setattr(mcp_publishing, 'refresh_mcp_caches', lambda: None) + + submitted = await mcp_submissions.submit_to_marketplace( + db, + source.server_id, + owner_user_id="u1", + submitter_name="Alice", + category="信息检索", + version="1.0.0", + ) + reviewed = await mcp_submissions.review_submission( + db, + submitted["submission_id"], + approve=True, + reviewed_by="admin", + ) + installed = await mcp_installation.install_market_item( + db, + reviewed["slug"], + owner_user_id="u2", + installed_by="u2", + credentials={"Authorization": "Bearer user-two"}, + ) + + row = db.query(AdminMcpServer).filter_by(server_id=installed["server_id"]).one() + assert row.url == "http://8.8.8.8:8082/mcp" + assert row.owner_user_id == "u2" + + +@pytest.mark.asyncio +async def test_market_installed_private_server_cannot_submit_again(db): + item, version = _approved_market(db) + source = _private_server(db) + db.add( + McpMarketInstallation( + install_id="mcpins_no_resubmit", + slug=item.slug, + version_id=version.version_id, + server_id=source.server_id, + owner_user_id="u1", + status="active", + ) + ) + db.commit() + + from api.routes.v1.catalog import _load_owned_capability_items + + _skills, mcps = _load_owned_capability_items(db, "u1") + catalog_item = next(row for row in mcps if row["id"] == source.server_id) + assert catalog_item["marketplace_installed"] is True + + with pytest.raises(BadRequestError, match="不能再次申请上架"): + await mcp_submissions.submit_to_marketplace( + db, + source.server_id, + owner_user_id="u1", + submitter_name="Alice", + category="信息检索", + version="1.0.0", + ) + + +@pytest.mark.asyncio +async def test_submission_strips_query_secret_into_install_schema(db, monkeypatch): + source = _private_server(db) + source.url = "https://mcp.example.test/mcp?key=publisher-secret&locale=zh-CN" + db.commit() + _patch_probe(monkeypatch) + + submitted = await mcp_submissions.submit_to_marketplace( + db, + source.server_id, + owner_user_id="u1", + submitter_name="Alice", + category="信息检索", + version="1.0.0", + ) + row = db.query(McpMarketSubmission).filter_by(submission_id=submitted["submission_id"]).one() + assert row.url == "https://mcp.example.test/mcp?locale=zh-CN" + assert row.auth_schema[-1] == { + "key": "QUERY_KEY", + "label": "key", + "target": "query", + "name": "key", + "required": True, + "secret": True, + } + assert "publisher-secret" not in str(row.url) + assert "publisher-secret" not in str(row.auth_schema) + + +@pytest.mark.asyncio +async def test_admin_publish_links_existing_global_instance_without_duplication(db, monkeypatch): + source = _private_server(db, owner="temporary", server_id="global_remote") + source.owner_user_id = None + db.commit() + _patch_probe(monkeypatch) + + published = await mcp_publishing.publish_admin_server( + db, + source.server_id, + category="信息检索", + version="1.0.0", + ) + assert published["installed"] is True + installation = db.query(McpMarketInstallation).one() + assert installation.server_id == source.server_id + assert installation.owner_user_id is None + assert db.query(AdminMcpServer).count() == 1 + listed = mcp_listing.list_market_items( + db, + owner_user_id=None, + viewer_user_id=None, + include_disabled=True, + ) + assert listed["items"][0]["installed"] is True + + +@pytest.mark.asyncio +async def test_marketplace_first_admin_source_is_reused_on_global_install(db, monkeypatch): + source = _private_server(db, owner="temporary", server_id="market_source") + source.owner_user_id = None + db.commit() + _patch_probe(monkeypatch) + + published = await mcp_publishing.publish_admin_server( + db, + source.server_id, + category="信息检索", + version="1.0.0", + link_as_installation=False, + ) + db.refresh(source) + assert published["installed"] is False + assert source.is_enabled is False + assert source.extra_config["market_source_only"] is True + assert db.query(McpMarketInstallation).count() == 0 + assert published["requires_auth"] is True + assert published["credentials_managed_by_admin"] is True + assert published["requires_user_credentials"] is False + assert "publisher-secret" not in json.dumps(published) + + installed = await mcp_installation.install_market_item( + db, + published["slug"], + owner_user_id=None, + installed_by="admin", + credentials={}, + ) + db.refresh(source) + assert installed["server_id"] == source.server_id + assert source.is_enabled is True + assert "market_source_only" not in source.extra_config + assert decrypt_mcp_headers(source.headers)["Authorization"] == "Bearer publisher-secret" + assert db.query(AdminMcpServer).count() == 1 + assert db.query(McpMarketInstallation).one().server_id == source.server_id + + +@pytest.mark.asyncio +async def test_user_install_reuses_admin_managed_credentials_without_exposing_them(db, monkeypatch): + source = _private_server(db, owner="temporary", server_id="shared_market_source") + source.owner_user_id = None + db.commit() + _patch_probe(monkeypatch) + + published = await mcp_publishing.publish_admin_server( + db, + source.server_id, + category="信息检索", + version="1.0.0", + link_as_installation=False, + ) + listed = mcp_listing.get_market_item( + db, + published["slug"], + owner_user_id="u2", + viewer_user_id="u2", + ) + assert listed["credentials_managed_by_admin"] is True + assert listed["requires_user_credentials"] is False + assert "publisher-secret" not in json.dumps(listed) + + installed = await mcp_installation.install_market_item( + db, + published["slug"], + owner_user_id="u2", + installed_by="u2", + credentials={}, + ) + private_server = db.query(AdminMcpServer).filter_by(server_id=installed["server_id"]).one() + assert private_server.owner_user_id == "u2" + assert decrypt_mcp_headers(private_server.headers)["Authorization"] == ( + "Bearer publisher-secret" + ) + assert "publisher-secret" not in str(private_server.headers) + db.refresh(source) + assert source.is_enabled is False + assert source.extra_config["market_source_only"] is True + + +@pytest.mark.asyncio +async def test_user_install_reuses_admin_managed_query_token(db, monkeypatch): + source = _private_server(db, owner="temporary", server_id="query_market_source") + source.owner_user_id = None + source.url = "https://mcp.example.test/mcp?key=publisher-query-secret&locale=zh-CN" + db.commit() + _patch_probe(monkeypatch) + + published = await mcp_publishing.publish_admin_server( + db, + source.server_id, + category="信息检索", + version="1.0.0", + link_as_installation=False, + ) + assert published["credentials_managed_by_admin"] is True + assert published["url_origin"] == "https://mcp.example.test" + assert "publisher-query-secret" not in json.dumps(published) + + installed = await mcp_installation.install_market_item( + db, + published["slug"], + owner_user_id="u2", + installed_by="u2", + credentials={}, + ) + private_server = db.query(AdminMcpServer).filter_by(server_id=installed["server_id"]).one() + assert private_server.url == "https://mcp.example.test/mcp?locale=zh-CN" + assert "publisher-query-secret" not in str(private_server.headers) + runtime_url, runtime_headers = materialize_mcp_http_connection( + private_server.url, + decrypt_mcp_headers(private_server.headers), + ) + assert runtime_url == ("https://mcp.example.test/mcp?locale=zh-CN&key=publisher-query-secret") + assert runtime_headers == {"Authorization": "Bearer publisher-secret"} diff --git a/src/backend/tests/features/marketplace/mcp_marketplace_cases_3.py b/src/backend/tests/features/marketplace/mcp_marketplace_cases_3.py new file mode 100644 index 00000000..8c2b9398 --- /dev/null +++ b/src/backend/tests/features/marketplace/mcp_marketplace_cases_3.py @@ -0,0 +1,458 @@ +from __future__ import annotations +from tests.features.marketplace.mcp_marketplace_support import ( + AdminMcpServer, + BadRequestError, + Base, + McpMarketInstallation, + McpMarketItem, + McpMarketSubmission, + McpMarketVersion, + McpServerConfigService, + OAuthBundleStorage, + OAuthToken, + Path, + ResourceNotFoundError, + SimpleNamespace, + _IS_CE_TREE, + _admin_private_market, + _approved_market, + _patch_install_side_effects, + _patch_probe, + _private_server, + _tools, + admin_mcp_routes, + asyncio, + create_engine, + db, + decrypt_mcp_headers, + encrypt_legacy_mcp_headers, + encrypt_mcp_headers, + httpx, + importlib, + json, + listing, + make_client, + management, + mask_mcp_headers, + materialize_mcp_http_connection, + mcp_administration, + mcp_listing, + mcp_curated, + mcp_installation, + mcp_oauth_bundle_storage_key, + mcp_publishing, + mcp_query_secret_storage_key, + mcp_revalidation, + mcp_submissions, + mcp_versions, + me_capability_routes, + oauth_service, + pytest, + sessionmaker, + socket, + tool_snapshot_hash, + validate_remote_mcp_url, +) + +@pytest.mark.asyncio +async def test_removed_admin_credential_falls_back_to_installer_input(db, monkeypatch): + source = _private_server(db, owner="temporary", server_id="unmanaged_market_source") + source.owner_user_id = None + db.commit() + _patch_probe(monkeypatch) + + published = await mcp_publishing.publish_admin_server( + db, + source.server_id, + category="信息检索", + version="1.0.0", + link_as_installation=False, + ) + source.headers = {} + db.commit() + listed = mcp_listing.get_market_item( + db, + published["slug"], + owner_user_id="u2", + viewer_user_id="u2", + ) + assert listed["credentials_managed_by_admin"] is False + assert listed["requires_user_credentials"] is True + + with pytest.raises(BadRequestError, match="请填写安装凭据"): + await mcp_installation.install_market_item( + db, + published["slug"], + owner_user_id="u2", + installed_by="u2", + credentials={}, + ) + + +@pytest.mark.asyncio +async def test_market_edit_can_require_installer_credentials_without_reusing_source( + db, monkeypatch +): + source = _private_server(db, owner="temporary", server_id="installer_auth_source") + source.owner_user_id = None + db.commit() + _patch_probe(monkeypatch) + + published = await mcp_publishing.publish_admin_server( + db, + source.server_id, + category="信息检索", + version="1.0.0", + link_as_installation=False, + ) + updated = mcp_administration.update_market_item( + db, + published["slug"], + requires_auth=True, + credential_mode="installer", + ) + assert updated["requires_auth"] is True + assert updated["credentials_managed_by_admin"] is False + assert updated["requires_user_credentials"] is True + assert updated["auth_config"]["credential_mode"] == "installer" + assert decrypt_mcp_headers(source.headers)["Authorization"] == "Bearer publisher-secret" + + with pytest.raises(BadRequestError, match="请填写安装凭据"): + await mcp_installation.install_market_item( + db, + published["slug"], + owner_user_id="u2", + installed_by="u2", + credentials={}, + ) + + +@pytest.mark.asyncio +async def test_market_edit_sets_admin_token_and_keeps_it_out_of_responses(db, monkeypatch): + source = _private_server(db, owner="temporary", server_id="edited_admin_auth_source") + source.owner_user_id = None + db.commit() + _patch_probe(monkeypatch) + + published = await mcp_publishing.publish_admin_server( + db, + source.server_id, + category="信息检索", + version="1.0.0", + link_as_installation=False, + ) + updated = mcp_administration.update_market_item( + db, + published["slug"], + requires_auth=True, + credential_mode="admin", + managed_credentials={"Authorization": "Bearer centrally-managed"}, + ) + assert updated["credentials_managed_by_admin"] is True + assert updated["requires_user_credentials"] is False + assert updated["auth_config"]["credential_mode"] == "admin" + assert "centrally-managed" not in json.dumps(updated) + db.refresh(source) + assert decrypt_mcp_headers(source.headers)["Authorization"] == ("Bearer centrally-managed") + assert "centrally-managed" not in str(source.headers) + + installed = await mcp_installation.install_market_item( + db, + published["slug"], + owner_user_id="u2", + installed_by="u2", + credentials={}, + ) + private_server = db.query(AdminMcpServer).filter_by(server_id=installed["server_id"]).one() + assert decrypt_mcp_headers(private_server.headers)["Authorization"] == ( + "Bearer centrally-managed" + ) + + +@pytest.mark.asyncio +async def test_create_market_mcp_with_blank_token_requires_each_installer(db, monkeypatch): + async def _validate(*args, **kwargs): + return None + + async def _fail_probe(*args, **kwargs): + return False, "401 Unauthorized" + + monkeypatch.setattr(mcp_installation, 'validate_remote_mcp_url', _validate) + monkeypatch.setattr(mcp_revalidation, 'validate_remote_mcp_url', _validate) + monkeypatch.setattr(mcp_submissions, 'validate_remote_mcp_url', _validate) + monkeypatch.setattr(mcp_publishing, 'validate_remote_mcp_url', _validate) + monkeypatch.setattr(mcp_installation, 'probe_mcp_connectivity', _fail_probe) + monkeypatch.setattr(mcp_revalidation, 'probe_mcp_connectivity', _fail_probe) + monkeypatch.setattr(mcp_submissions, 'probe_mcp_connectivity', _fail_probe) + monkeypatch.setattr(mcp_publishing, 'probe_mcp_connectivity', _fail_probe) + monkeypatch.setattr(mcp_installation, 'refresh_mcp_caches', lambda: None) + monkeypatch.setattr(mcp_administration, 'refresh_mcp_caches', lambda: None) + monkeypatch.setattr(mcp_versions, 'refresh_mcp_caches', lambda: None) + monkeypatch.setattr(mcp_publishing, 'refresh_mcp_caches', lambda: None) + monkeypatch.setattr(admin_mcp_routes, "_probe_connectivity", _fail_probe) + monkeypatch.setattr(admin_mcp_routes, "_refresh_caches", lambda: None) + + request = admin_mcp_routes.McpServerCreateRequest( + server_id="blank_token_mcp", + display_name="Blank Token MCP", + description="Discovers tools after the user authenticates", + transport="streamable_http", + url="https://mcp.example.test/mcp", + publish_to_marketplace=True, + market_requires_auth=True, + market_auth_type="token", + market_auth_header="Authorization", + market_auth_prefix="Bearer ", + market_admin_token="", + ) + await admin_mcp_routes.create_mcp_server(request, db) + + listed = mcp_listing.list_market_items( + db, + owner_user_id="u2", + viewer_user_id="u2", + )[ + "items" + ][0] + assert listed["requires_auth"] is True + assert listed["requires_user_credentials"] is True + assert listed["credentials_managed_by_admin"] is False + assert listed["supports_admin_credentials"] is True + assert listed["auth_config"]["credential_mode"] == "installer" + assert listed["listing_notice"]["discovery_mode"] == "per_install" + assert listed["tool_count"] == 0 + + async def _authenticated_probe(row, db=None): + assert decrypt_mcp_headers(row.headers)["Authorization"] == "Bearer user-secret" + row.tools_json = _tools("search_documents") + return True, "" + + monkeypatch.setattr(mcp_installation, 'probe_mcp_connectivity', _authenticated_probe) + monkeypatch.setattr(mcp_revalidation, 'probe_mcp_connectivity', _authenticated_probe) + monkeypatch.setattr(mcp_submissions, 'probe_mcp_connectivity', _authenticated_probe) + monkeypatch.setattr(mcp_publishing, 'probe_mcp_connectivity', _authenticated_probe) + installed = await mcp_installation.install_market_item( + db, + listed["slug"], + owner_user_id="u2", + installed_by="u2", + credentials={"Authorization": "user-secret"}, + ) + private_server = db.query(AdminMcpServer).filter_by(server_id=installed["server_id"]).one() + assert decrypt_mcp_headers(private_server.headers)["Authorization"] == "Bearer user-secret" + assert private_server.tools_json == _tools("search_documents") + + +@pytest.mark.asyncio +async def test_create_market_mcp_with_admin_token_makes_user_install_tokenless(db, monkeypatch): + discovered_tools = _tools("search_documents") + + async def _validate(*args, **kwargs): + return None + + async def _probe(row, db=None): + row.tools_json = discovered_tools + return True, "" + + monkeypatch.setattr(mcp_installation, 'validate_remote_mcp_url', _validate) + monkeypatch.setattr(mcp_revalidation, 'validate_remote_mcp_url', _validate) + monkeypatch.setattr(mcp_submissions, 'validate_remote_mcp_url', _validate) + monkeypatch.setattr(mcp_publishing, 'validate_remote_mcp_url', _validate) + monkeypatch.setattr(mcp_installation, 'probe_mcp_connectivity', _probe) + monkeypatch.setattr(mcp_revalidation, 'probe_mcp_connectivity', _probe) + monkeypatch.setattr(mcp_submissions, 'probe_mcp_connectivity', _probe) + monkeypatch.setattr(mcp_publishing, 'probe_mcp_connectivity', _probe) + monkeypatch.setattr(mcp_installation, 'refresh_mcp_caches', lambda: None) + monkeypatch.setattr(mcp_administration, 'refresh_mcp_caches', lambda: None) + monkeypatch.setattr(mcp_versions, 'refresh_mcp_caches', lambda: None) + monkeypatch.setattr(mcp_publishing, 'refresh_mcp_caches', lambda: None) + monkeypatch.setattr(admin_mcp_routes, "_probe_connectivity", _probe) + monkeypatch.setattr(admin_mcp_routes, "_refresh_caches", lambda: None) + + request = admin_mcp_routes.McpServerCreateRequest( + server_id="admin_token_mcp", + display_name="Admin Token MCP", + description="Uses one administrator token", + transport="streamable_http", + url="https://mcp.example.test/mcp", + publish_to_marketplace=True, + market_requires_auth=True, + market_auth_type="token", + market_auth_header="Authorization", + market_auth_prefix="Bearer ", + market_admin_token="central-secret", + ) + await admin_mcp_routes.create_mcp_server(request, db) + + source = db.query(AdminMcpServer).filter_by(server_id="admin_token_mcp").one() + assert decrypt_mcp_headers(source.headers)["Authorization"] == "Bearer central-secret" + assert "central-secret" not in str(source.headers) + listed = mcp_listing.list_market_items( + db, + owner_user_id="u2", + viewer_user_id="u2", + )[ + "items" + ][0] + assert listed["credentials_managed_by_admin"] is True + assert listed["requires_user_credentials"] is False + assert "central-secret" not in json.dumps(listed) + + +@pytest.mark.asyncio +async def test_create_market_mcp_with_oauth_defers_authorization_to_installer(db, monkeypatch): + async def _validate(*args, **kwargs): + return None + + async def _fail_probe(*args, **kwargs): + return False, "401 Unauthorized" + + monkeypatch.setattr(mcp_installation, 'validate_remote_mcp_url', _validate) + monkeypatch.setattr(mcp_revalidation, 'validate_remote_mcp_url', _validate) + monkeypatch.setattr(mcp_submissions, 'validate_remote_mcp_url', _validate) + monkeypatch.setattr(mcp_publishing, 'validate_remote_mcp_url', _validate) + monkeypatch.setattr(mcp_installation, 'probe_mcp_connectivity', _fail_probe) + monkeypatch.setattr(mcp_revalidation, 'probe_mcp_connectivity', _fail_probe) + monkeypatch.setattr(mcp_submissions, 'probe_mcp_connectivity', _fail_probe) + monkeypatch.setattr(mcp_publishing, 'probe_mcp_connectivity', _fail_probe) + monkeypatch.setattr(mcp_installation, 'refresh_mcp_caches', lambda: None) + monkeypatch.setattr(mcp_administration, 'refresh_mcp_caches', lambda: None) + monkeypatch.setattr(mcp_versions, 'refresh_mcp_caches', lambda: None) + monkeypatch.setattr(mcp_publishing, 'refresh_mcp_caches', lambda: None) + monkeypatch.setattr(admin_mcp_routes, "_probe_connectivity", _fail_probe) + monkeypatch.setattr(admin_mcp_routes, "_refresh_caches", lambda: None) + + request = admin_mcp_routes.McpServerCreateRequest( + server_id="oauth_market_mcp", + display_name="OAuth Market MCP", + transport="streamable_http", + url="https://mcp.example.test/mcp", + publish_to_marketplace=True, + market_requires_auth=True, + market_auth_type="oauth2", + ) + await admin_mcp_routes.create_mcp_server(request, db) + + listed = mcp_listing.list_market_items( + db, + owner_user_id="u2", + viewer_user_id="u2", + )[ + "items" + ][0] + assert listed["requires_auth"] is True + assert listed["requires_user_credentials"] is True + assert listed["auth_config"]["default_method"] == "oauth2" + assert listed["auth_config"]["methods"][0]["type"] == "oauth2" + assert listed["listing_notice"]["discovery_mode"] == "per_install" + + +@pytest.mark.asyncio +async def test_move_existing_published_server_removes_global_installation(db, monkeypatch): + source = _private_server(db, owner="temporary", server_id="published_global") + source.owner_user_id = None + db.commit() + _patch_probe(monkeypatch) + + await mcp_publishing.publish_admin_server( + db, + source.server_id, + category="信息检索", + version="1.0.0", + ) + assert db.query(McpMarketInstallation).count() == 1 + + moved = await mcp_publishing.publish_admin_server( + db, + source.server_id, + category="信息检索", + version="1.0.1", + link_as_installation=False, + ) + + db.refresh(source) + assert moved["installed"] is False + assert source.is_enabled is False + assert source.extra_config["market_source_only"] is True + assert db.query(McpMarketInstallation).count() == 0 + + +def test_edit_market_metadata_propagates_to_existing_installations(db, monkeypatch): + item, version = _approved_market(db) + installed_server = AdminMcpServer( + server_id="installed_acme", + display_name=item.display_name, + description=item.description, + transport=version.transport, + url=version.url, + is_enabled=True, + owner_user_id="u2", + ) + db.add(installed_server) + db.flush() + db.add( + McpMarketInstallation( + install_id="mcpins_edit", + slug=item.slug, + version_id=version.version_id, + server_id=installed_server.server_id, + owner_user_id="u2", + status="active", + ) + ) + db.commit() + monkeypatch.setattr(mcp_installation, 'refresh_mcp_caches', lambda: None) + monkeypatch.setattr(mcp_administration, 'refresh_mcp_caches', lambda: None) + monkeypatch.setattr(mcp_versions, 'refresh_mcp_caches', lambda: None) + monkeypatch.setattr(mcp_publishing, 'refresh_mcp_caches', lambda: None) + + result = mcp_administration.update_market_item( + db, + item.slug, + display_name="Acme Knowledge Search", + description="Search the updated Acme knowledge base", + user_intro="## 用途\n检索 Acme 知识库", + category="通用工具", + tags=["search", "knowledge", "search"], + icon="/home/mcp/knowledge.svg", + ) + + db.refresh(installed_server) + assert result["display_name"] == "Acme Knowledge Search" + assert result["updated_installations"] == 1 + assert result["tags"] == ["search", "knowledge"] + assert installed_server.display_name == result["display_name"] + assert installed_server.description == result["description"] + assert installed_server.user_intro == result["user_intro"] + assert installed_server.icon == result["icon"] + assert db.query(McpMarketVersion).one().version == "1.0.0" + assert db.query(McpMarketVersion).one().tool_hash == version.tool_hash + + +@pytest.mark.asyncio +async def test_install_uses_installers_own_encrypted_credentials(db, monkeypatch): + _approved_market(db) + _patch_probe(monkeypatch) + + first = await mcp_installation.install_market_item( + db, + "acme-search", + owner_user_id="u2", + installed_by="u2", + credentials={"Authorization": "Bearer user-two"}, + ) + second = await mcp_installation.install_market_item( + db, + "acme-search", + owner_user_id="u3", + installed_by="u3", + credentials={"Authorization": "Bearer user-three"}, + ) + first_server = db.query(AdminMcpServer).filter_by(server_id=first["server_id"]).one() + second_server = db.query(AdminMcpServer).filter_by(server_id=second["server_id"]).one() + assert first_server.owner_user_id == "u2" + assert second_server.owner_user_id == "u3" + assert decrypt_mcp_headers(first_server.headers)["Authorization"] == "Bearer user-two" + assert decrypt_mcp_headers(second_server.headers)["Authorization"] == "Bearer user-three" + assert first_server.headers != second_server.headers + assert db.query(McpMarketInstallation).count() == 2 diff --git a/src/backend/tests/features/marketplace/mcp_marketplace_cases_4.py b/src/backend/tests/features/marketplace/mcp_marketplace_cases_4.py new file mode 100644 index 00000000..38eee866 --- /dev/null +++ b/src/backend/tests/features/marketplace/mcp_marketplace_cases_4.py @@ -0,0 +1,435 @@ +from __future__ import annotations +from tests.features.marketplace.mcp_marketplace_support import ( + AdminMcpServer, + BadRequestError, + Base, + McpMarketInstallation, + McpMarketItem, + McpMarketSubmission, + McpMarketVersion, + McpServerConfigService, + OAuthBundleStorage, + OAuthToken, + Path, + ResourceNotFoundError, + SimpleNamespace, + _IS_CE_TREE, + _admin_private_market, + _approved_market, + _patch_install_side_effects, + _patch_probe, + _private_server, + _tools, + admin_mcp_routes, + asyncio, + create_engine, + db, + decrypt_mcp_headers, + encrypt_legacy_mcp_headers, + encrypt_mcp_headers, + httpx, + importlib, + json, + listing, + make_client, + management, + mask_mcp_headers, + materialize_mcp_http_connection, + mcp_administration, + mcp_listing, + mcp_curated, + mcp_installation, + mcp_oauth_bundle_storage_key, + mcp_publishing, + mcp_query_secret_storage_key, + mcp_revalidation, + mcp_submissions, + mcp_versions, + me_capability_routes, + oauth_service, + pytest, + sessionmaker, + socket, + tool_snapshot_hash, + validate_remote_mcp_url, +) + +@pytest.mark.asyncio +async def test_curated_templates_seed_once_and_materialize_user_query_key(db, monkeypatch): + assert set(mcp_curated.ensure_curated_market_items(db)) == { + "amap-maps", + "metaso-search", + "github-official", + "gitlab-official", + "alibaba-cloud-observability", + "notion-official", + } + assert mcp_curated.ensure_curated_market_items(db) == [] + gitlab_item = mcp_listing.get_market_item( + db, + "gitlab-official", + owner_user_id="u2", + viewer_user_id="u2", + ) + assert gitlab_item["auth_config"]["default_method"] == "oauth2" + assert gitlab_item["supports_admin_credentials"] is False + assert {method["type"] for method in gitlab_item["auth_config"]["methods"]} == { + "oauth2", + "token", + } + + actual_tools = _tools("maps_geo", "maps_weather") + _patch_probe(monkeypatch, tools=actual_tools) + installed = await mcp_installation.install_market_item( + db, + "amap-maps", + owner_user_id="u2", + installed_by="u2", + credentials={"AMAP_API_KEY": "user-amap-key"}, + ) + server = db.query(AdminMcpServer).filter_by(server_id=installed["server_id"]).one() + assert server.url == "https://mcp.amap.com/mcp" + assert "user-amap-key" not in str(server.headers) + runtime_url, runtime_headers = materialize_mcp_http_connection( + server.url, + decrypt_mcp_headers(server.headers), + ) + assert runtime_url == "https://mcp.amap.com/mcp?key=user-amap-key" + assert runtime_headers == {} + assert server.tools_json == actual_tools + assert server.owner_user_id == "u2" + assert server.icon == "/home/mcp/internet.svg" + + +@pytest.mark.asyncio +async def test_curated_bearer_token_gets_prefix_and_can_be_rotated(db, monkeypatch): + mcp_curated.ensure_curated_market_items(db) + _patch_probe(monkeypatch, tools=_tools("metaso_web_search")) + + installed = await mcp_installation.install_market_item( + db, + "metaso-search", + owner_user_id="u2", + installed_by="u2", + credentials={"METASO_API_KEY": "first-key"}, + ) + server = db.query(AdminMcpServer).filter_by(server_id=installed["server_id"]).one() + assert decrypt_mcp_headers(server.headers)["Authorization"] == "Bearer first-key" + + updated = await mcp_installation.install_market_item( + db, + "metaso-search", + owner_user_id="u2", + installed_by="u2", + credentials={"METASO_API_KEY": "Bearer second-key"}, + ) + db.refresh(server) + assert updated["action"] == "existing" + assert decrypt_mcp_headers(server.headers)["Authorization"] == "Bearer second-key" + + +@pytest.mark.asyncio +async def test_tool_drift_pauses_new_installation(db, monkeypatch): + _approved_market(db) + _patch_probe(monkeypatch, tools=_tools("search_documents", "delete_documents")) + with pytest.raises(BadRequestError, match="发生变化"): + await mcp_installation.install_market_item( + db, + "acme-search", + owner_user_id="u2", + installed_by="u2", + credentials={"Authorization": "Bearer u2"}, + ) + item = db.query(McpMarketItem).filter_by(slug="acme-search").one() + assert item.status == "changed" + assert db.query(McpMarketInstallation).count() == 0 + + +@pytest.mark.asyncio +async def test_failed_update_probe_never_commits_plaintext_credentials(db, monkeypatch): + _approved_market(db) + _patch_probe(monkeypatch) + installed = await mcp_installation.install_market_item( + db, + "acme-search", + owner_user_id="u2", + installed_by="u2", + credentials={"Authorization": "Bearer original"}, + ) + _patch_probe(monkeypatch, tools=_tools("changed_tool")) + + with pytest.raises(BadRequestError, match="发生变化"): + await mcp_installation.install_market_item( + db, + "acme-search", + owner_user_id="u2", + installed_by="u2", + credentials={"Authorization": "Bearer replacement"}, + ) + + server = db.query(AdminMcpServer).filter_by(server_id=installed["server_id"]).one() + assert decrypt_mcp_headers(server.headers)["Authorization"] == "Bearer original" + assert "original" not in str(server.headers) + assert "replacement" not in str(server.headers) + + +@pytest.mark.asyncio +async def test_security_suspension_disables_all_derived_servers(db, monkeypatch): + _approved_market(db) + _patch_probe(monkeypatch) + await mcp_installation.install_market_item( + db, + "acme-search", + owner_user_id="u2", + installed_by="u2", + credentials={"Authorization": "Bearer u2"}, + ) + await mcp_installation.install_market_item( + db, + "acme-search", + owner_user_id="u3", + installed_by="u3", + credentials={"Authorization": "Bearer u3"}, + ) + result = mcp_administration.set_suspended(db, "acme-search", suspended=True, reason="malware") + assert result["disabled_installations"] == 2 + assert all(not row.is_enabled for row in db.query(AdminMcpServer).all()) + assert {row.status for row in db.query(McpMarketInstallation).all()} == {"suspended"} + + restored = mcp_administration.set_suspended(db, "acme-search", suspended=False) + assert restored["restored_installations"] == 2 + assert all(row.is_enabled for row in db.query(AdminMcpServer).all()) + assert {row.status for row in db.query(McpMarketInstallation).all()} == {"active"} + + +@pytest.mark.asyncio +async def test_review_rechecks_live_snapshot(db, monkeypatch): + source = _private_server(db) + _patch_probe(monkeypatch) + submitted = await mcp_submissions.submit_to_marketplace( + db, + source.server_id, + owner_user_id="u1", + submitter_name="Alice", + category="信息检索", + version="1.0.0", + ) + _patch_probe(monkeypatch, tools=_tools("changed_after_submission")) + + with pytest.raises(BadRequestError, match="重新提交"): + await mcp_submissions.review_submission( + db, + submitted["submission_id"], + approve=True, + reviewed_by="admin", + ) + + assert db.query(McpMarketSubmission).one().status == "pending" + assert db.query(McpMarketItem).count() == 0 + + +@pytest.mark.asyncio +async def test_missing_source_revalidates_from_reviewed_version(db, monkeypatch): + _approved_market(db) + _patch_probe(monkeypatch) + result = await mcp_revalidation.revalidate_market_item(db, "acme-search") + + assert result["status"] == "active" + assert result["status_reason"] == "" + + +@pytest.mark.asyncio +async def test_missing_source_still_detects_live_snapshot_drift(db, monkeypatch): + _approved_market(db) + _patch_probe(monkeypatch, tools=_tools("changed_after_review")) + + result = await mcp_revalidation.revalidate_market_item(db, "acme-search") + + assert result["status"] == "changed" + assert "需发布新版本" in result["status_reason"] + + +@pytest.mark.asyncio +async def test_manual_admin_revalidation_accepts_drift_as_new_version(db, monkeypatch): + source = _private_server(db) + source.owner_user_id = None + item, old_version = _approved_market(db) + item.source = "admin" + db.commit() + changed_tools = _tools("search_documents", "list_accessible_projects") + _patch_probe(monkeypatch, tools=changed_tools) + + result = await mcp_revalidation.revalidate_market_item( + db, + "acme-search", + accept_changes=True, + ) + + assert result["status"] == "active" + assert result["snapshot_accepted"] is True + assert result["previous_version"] == "1.0.0" + assert result["version"] == "1.0.1" + versions = { + row.version: row for row in db.query(McpMarketVersion).filter_by(slug="acme-search").all() + } + assert set(versions) == {"1.0.0", "1.0.1"} + assert versions["1.0.0"].tool_hash == old_version.tool_hash + assert versions["1.0.1"].tool_hash == tool_snapshot_hash(changed_tools) + assert versions["1.0.1"].approved_by == "admin-revalidation" + assert item.latest_version_id == versions["1.0.1"].version_id + + +@pytest.mark.asyncio +async def test_periodic_revalidation_never_auto_accepts_drift(db, monkeypatch): + source = _private_server(db) + source.owner_user_id = None + item, _ = _approved_market(db) + item.source = "admin" + db.commit() + _patch_probe(monkeypatch, tools=_tools("changed_after_review")) + + result = await mcp_revalidation.revalidate_market_item(db, "acme-search") + + assert result["status"] == "changed" + assert db.query(McpMarketVersion).filter_by(slug="acme-search").count() == 1 + + +@pytest.mark.asyncio +async def test_manual_revalidation_rejects_empty_tool_snapshot(db, monkeypatch): + source = _private_server(db) + source.owner_user_id = None + item, _ = _approved_market(db) + item.source = "admin" + db.commit() + _patch_probe(monkeypatch, tools=[]) + + result = await mcp_revalidation.revalidate_market_item( + db, + "acme-search", + accept_changes=True, + ) + + assert result["status"] == "changed" + assert "未返回任何工具" in result["status_reason"] + assert db.query(McpMarketVersion).filter_by(slug="acme-search").count() == 1 + + +@pytest.mark.skipif(_IS_CE_TREE, reason="CE marketplace listings are always public") +def test_mcp_visibility_uses_shared_marketplace_scope(db): + item, _ = _approved_market(db) + listing.set_listing_visibility( + db, + listing.KIND_MCP, + item.slug, + visibility="scoped", + grants=[{"principal_type": "user", "principal_id": "u2"}], + ) + visible = mcp_listing.list_market_items( + db, + owner_user_id="u2", + viewer_user_id="u2", + ) + hidden = mcp_listing.list_market_items( + db, + owner_user_id="u3", + viewer_user_id="u3", + ) + assert [row["slug"] for row in visible["items"]] == ["acme-search"] + assert hidden["items"] == [] + + +@pytest.mark.asyncio +async def test_delisted_item_blocks_user_install_but_not_admin_install(db, monkeypatch): + item, _ = _approved_market(db) + _patch_probe(monkeypatch) + listing.set_listing_enabled(db, listing.KIND_MCP, item.slug, False) + + with pytest.raises(ResourceNotFoundError): + await mcp_installation.install_market_item( + db, + item.slug, + owner_user_id="u2", + installed_by="u2", + credentials={"Authorization": "Bearer u2"}, + ) + + installed = await mcp_installation.install_market_item( + db, + item.slug, + owner_user_id=None, + installed_by="admin", + credentials={"Authorization": "Bearer global"}, + ) + assert installed["action"] == "installed" + + +@pytest.mark.asyncio +async def test_user_installs_admin_published_private_network_mcp(db, monkeypatch): + _admin_private_market(db) + _patch_install_side_effects(monkeypatch) + + installed = await mcp_installation.install_market_item( + db, + "builtin-search", + owner_user_id="u2", + installed_by="u2", + credentials={}, + ) + + row = db.query(AdminMcpServer).filter(AdminMcpServer.owner_user_id == "u2").one() + assert installed["server_id"] == row.server_id + assert row.url == "http://mcp:9102/mcp/" + + +@pytest.mark.asyncio +async def test_installer_cannot_redirect_an_admin_item_to_another_private_host(db, monkeypatch): + _admin_private_market( + db, + auth_schema=[ + { + "key": "ENDPOINT", + "label": "Endpoint", + "target": "url", + "required": True, + "secret": True, + } + ], + ) + _patch_install_side_effects(monkeypatch) + + with pytest.raises(BadRequestError, match="本机、内网或保留地址"): + await mcp_installation.install_market_item( + db, + "builtin-search", + owner_user_id="u2", + installed_by="u2", + credentials={"ENDPOINT": "http://127.0.0.1:9102/mcp/"}, + ) + assert db.query(AdminMcpServer).count() == 0 + + +@pytest.mark.asyncio +async def test_installer_may_keep_the_admin_host_when_overriding_the_endpoint(db, monkeypatch): + _admin_private_market( + db, + auth_schema=[ + { + "key": "ENDPOINT", + "label": "Endpoint", + "target": "url", + "required": True, + "secret": True, + } + ], + ) + _patch_install_side_effects(monkeypatch) + + await mcp_installation.install_market_item( + db, + "builtin-search", + owner_user_id="u2", + installed_by="u2", + credentials={"ENDPOINT": "http://mcp:9102/mcp/tenant-a"}, + ) + + assert db.query(AdminMcpServer).filter(AdminMcpServer.owner_user_id == "u2").count() == 1 diff --git a/src/backend/tests/features/marketplace/mcp_marketplace_support.py b/src/backend/tests/features/marketplace/mcp_marketplace_support.py new file mode 100644 index 00000000..ba9de5f5 --- /dev/null +++ b/src/backend/tests/features/marketplace/mcp_marketplace_support.py @@ -0,0 +1,269 @@ +"""End-to-end service tests for the reviewed MCP marketplace lifecycle.""" + + +from __future__ import annotations + + +from features.marketplace import edition as listing + + +from features.marketplace import mcp_administration + + +from features.marketplace import mcp_installation + + +from features.marketplace import mcp_publishing + + +from features.marketplace import mcp_revalidation + + +from features.marketplace import mcp_submissions + + +from features.marketplace import mcp_versions + + +from features.marketplace import mcp_administration + + +from features.marketplace import mcp_listing + + +from features.marketplace import mcp_curated + + +from features.marketplace import mcp_installation + + +from features.marketplace import mcp_publishing + + +from features.marketplace import mcp_revalidation + + +from features.marketplace import mcp_submissions + + +import asyncio + + +import importlib.util + + +import json + + +import socket + + +from pathlib import Path + + +from types import SimpleNamespace + + +import httpx + + +import pytest + + +from api.routes.v1 import admin_mcp_servers as admin_mcp_routes + + +from api.routes.v1 import me_capabilities as me_capability_routes + + +from core.db.engine import Base + + +from core.db.models import AdminMcpServer + + +from features.marketplace.models import McpMarketInstallation, McpMarketItem, McpMarketSubmission, McpMarketVersion + + +from core.infra.exceptions import BadRequestError, ResourceNotFoundError + + +from core.llm.mcp.client import make_client + + +from core.capabilities.mcp import management as management + + +from core.capabilities.mcp import oauth as oauth_service + + +from core.capabilities.mcp.management import decrypt_mcp_headers, encrypt_legacy_mcp_headers, encrypt_mcp_headers, mask_mcp_headers, materialize_mcp_http_connection, mcp_oauth_bundle_storage_key, mcp_query_secret_storage_key, tool_snapshot_hash, validate_remote_mcp_url + + +from core.capabilities.mcp.oauth_storage import OAuthBundleStorage + + +from core.capabilities.mcp.service import McpServerConfigService + + +from mcp.shared.auth import OAuthToken + + +from sqlalchemy import create_engine + + +from sqlalchemy.orm import sessionmaker + + +_IS_CE_TREE = not (Path(__file__).resolve().parents[3] / "edition_ee").exists() + + +@pytest.fixture +def db(): + engine = create_engine("sqlite://", future=True) + Base.metadata.create_all(engine) + Session = sessionmaker(bind=engine, autoflush=False, future=True) + session = Session() + try: + yield session + finally: + session.close() + Base.metadata.drop_all(engine) + + +def _tools(*names: str): + return [ + { + "name": name, + "description": f"Tool {name}", + "inputSchema": {"type": "object", "properties": {"query": {"type": "string"}}}, + } + for name in names + ] + + +def _private_server(db, *, owner: str = "u1", server_id: str = "umcp_source"): + row = AdminMcpServer( + server_id=server_id, + display_name="Acme Search", + description="Search Acme documents", + transport="streamable_http", + url="https://mcp.example.test/mcp", + headers=encrypt_mcp_headers({"Authorization": "Bearer publisher-secret"}), + tools_json=_tools("search_documents"), + is_enabled=True, + owner_user_id=owner, + ) + db.add(row) + db.commit() + return row + + +def _approved_market(db, *, tools=None): + snapshot = tools or _tools("search_documents") + item = McpMarketItem( + slug="acme-search", + display_name="Acme Search", + description="Search Acme documents", + category="信息检索", + tags=["search"], + publisher_name="Acme", + source="community", + latest_version_id="mcpver_1", + status="active", + ) + version = McpMarketVersion( + version_id="mcpver_1", + slug="acme-search", + version="1.0.0", + transport="streamable_http", + url="https://mcp.example.test/mcp", + auth_schema=[ + { + "key": "Authorization", + "label": "Authorization", + "target": "header", + "required": True, + "secret": True, + } + ], + tools_json=snapshot, + tool_hash=tool_snapshot_hash(snapshot), + source_server_id="umcp_source", + ) + db.add_all([item, version]) + db.commit() + return item, version + + +def _patch_probe(monkeypatch, tools=None): + async def _validate(*args, **kwargs): + return None + + async def _probe(row, db=None): + if tools is not None: + row.tools_json = tools + return True, "" + + monkeypatch.setattr(mcp_installation, 'validate_remote_mcp_url', _validate) + monkeypatch.setattr(mcp_revalidation, 'validate_remote_mcp_url', _validate) + monkeypatch.setattr(mcp_submissions, 'validate_remote_mcp_url', _validate) + monkeypatch.setattr(mcp_publishing, 'validate_remote_mcp_url', _validate) + monkeypatch.setattr(mcp_installation, 'probe_mcp_connectivity', _probe) + monkeypatch.setattr(mcp_revalidation, 'probe_mcp_connectivity', _probe) + monkeypatch.setattr(mcp_submissions, 'probe_mcp_connectivity', _probe) + monkeypatch.setattr(mcp_publishing, 'probe_mcp_connectivity', _probe) + monkeypatch.setattr(mcp_installation, 'refresh_mcp_caches', lambda: None) + monkeypatch.setattr(mcp_administration, 'refresh_mcp_caches', lambda: None) + monkeypatch.setattr(mcp_versions, 'refresh_mcp_caches', lambda: None) + monkeypatch.setattr(mcp_publishing, 'refresh_mcp_caches', lambda: None) + + +def _admin_private_market(db, *, auth_schema=None, url: str = "http://mcp:9102/mcp/"): + """An admin-published item whose endpoint lives on the deployment's own network.""" + snapshot = _tools("search_documents") + item = McpMarketItem( + slug="builtin-search", + display_name="联网搜索", + description="Built-in search MCP", + category="信息检索", + tags=["search"], + publisher_name="平台", + source="admin", + latest_version_id="mcpver_builtin", + status="active", + ) + version = McpMarketVersion( + version_id="mcpver_builtin", + slug="builtin-search", + version="1.0.0", + transport="streamable_http", + url=url, + auth_schema=auth_schema or [], + tools_json=snapshot, + tool_hash=tool_snapshot_hash(snapshot), + ) + db.add_all([item, version]) + db.commit() + return item, version + + +def _patch_install_side_effects(monkeypatch, *, resolves_to: str = "172.27.0.4"): + """Keep the real URL guard in play while stubbing the network around it.""" + + async def _probe(row, db=None): + return True, "" + + monkeypatch.setattr( + socket, + "getaddrinfo", + lambda *args, **kwargs: [(socket.AF_INET, socket.SOCK_STREAM, 6, "", (resolves_to, 9102))], + ) + monkeypatch.setattr(mcp_installation, 'probe_mcp_connectivity', _probe) + monkeypatch.setattr(mcp_revalidation, 'probe_mcp_connectivity', _probe) + monkeypatch.setattr(mcp_submissions, 'probe_mcp_connectivity', _probe) + monkeypatch.setattr(mcp_publishing, 'probe_mcp_connectivity', _probe) + monkeypatch.setattr(mcp_installation, 'refresh_mcp_caches', lambda: None) + monkeypatch.setattr(mcp_administration, 'refresh_mcp_caches', lambda: None) + monkeypatch.setattr(mcp_versions, 'refresh_mcp_caches', lambda: None) + monkeypatch.setattr(mcp_publishing, 'refresh_mcp_caches', lambda: None) diff --git a/src/backend/tests/test_agent_marketplace.py b/src/backend/tests/features/marketplace/test_agent_marketplace.py similarity index 69% rename from src/backend/tests/test_agent_marketplace.py rename to src/backend/tests/features/marketplace/test_agent_marketplace.py index 313c9705..eb3bd4f5 100644 --- a/src/backend/tests/test_agent_marketplace.py +++ b/src/backend/tests/features/marketplace/test_agent_marketplace.py @@ -1,11 +1,14 @@ """Sub-agent marketplace: full lifecycle test — listing request → review → install-clone (dependencies installed along) → delisting.""" from __future__ import annotations +from features.marketplace import agent_catalog +from features.marketplace import agent_installation +from features.marketplace import agent_submissions import pytest from core.db.models import AdminMcpServer, AdminSkill, UserAgent, UserShadow -from core.services import agent_market_service as am -from core.services.user_agent_service import UserAgentService + +from core.capabilities.agents.edition import UserAgentService def _seed_users(db): @@ -63,7 +66,7 @@ def test_submit_review_install_lifecycle(db_session): src = _make_source_agent(db) # 1) Submit a listing request → pending - sub = am.submit_to_marketplace( + sub = agent_submissions.submit_to_marketplace( db, src["agent_id"], owner_user_id="u1", @@ -74,25 +77,25 @@ def test_submit_review_install_lifecycle(db_session): ) assert sub["status"] == "pending" slug = sub["slug"] - assert am.list_my_submissions(db, "u1")[0]["submission_id"] == sub["submission_id"] + assert agent_submissions.list_my_submissions(db, "u1")[0]["submission_id"] == sub["submission_id"] # A duplicate submission (already pending) should be rejected with pytest.raises(Exception): - am.submit_to_marketplace(db, src["agent_id"], owner_user_id="u1", category="数据分析") + agent_submissions.submit_to_marketplace(db, src["agent_id"], owner_user_id="u1", category="数据分析") # While pending it does not show up in the marketplace list - assert slug not in {it["slug"] for it in am.list_marketplace_agents(db)} + assert slug not in {it["slug"] for it in agent_catalog.list_marketplace_agents(db)} # 2) Admin approves the review → approved - am.review_submission(db, sub["submission_id"], approve=True) - market = {it["slug"]: it for it in am.list_marketplace_agents(db)} + agent_submissions.review_submission(db, sub["submission_id"], approve=True) + market = {it["slug"]: it for it in agent_catalog.list_marketplace_agents(db)} assert slug in market assert market[slug]["category"] == "数据分析" assert market[slug]["source"] == "community" assert market[slug]["tags"] == ["ontology:RiskReport"] # 3) u2 installs = clone into a private sub-agent + dependencies installed along - res = am.install_marketplace_agent(db, slug, owner_user_id="u2", operator_name="U2") + res = agent_installation.install_marketplace_agent(db, slug, owner_user_id="u2", operator_name="U2") clone = db.query(UserAgent).filter(UserAgent.agent_id == res["agent_id"]).first() assert clone is not None assert clone.owner_type == "user" and clone.user_id == "u2" @@ -108,20 +111,20 @@ def test_submit_review_install_lifecycle(db_session): assert "mcp:test-mcp" in report["bound"] # 4) Installed annotation: u2 hits, u3 not installed - items = am.list_marketplace_agents(db) + items = agent_catalog.list_marketplace_agents(db) assert ( - am.annotate_installed([i for i in items if i["slug"] == slug], db, "u2")[0]["installed"] + agent_installation.annotate_installed([i for i in items if i["slug"] == slug], db, "u2")[0]["installed"] is True ) assert ( - am.annotate_installed([i for i in items if i["slug"] == slug], db, "u3")[0]["installed"] + agent_installation.annotate_installed([i for i in items if i["slug"] == slug], db, "u3")[0]["installed"] is False ) - assert am.is_installed(db, slug, "u2") is True + assert agent_installation.is_installed(db, slug, "u2") is True # 5) Admin delists (rejects an approved one) → no longer shown in the user-facing marketplace, installed clones unaffected - am.review_submission(db, sub["submission_id"], approve=False, review_note="先下架") - assert slug not in {it["slug"] for it in am.list_marketplace_agents(db)} + agent_submissions.review_submission(db, sub["submission_id"], approve=False, review_note="先下架") + assert slug not in {it["slug"] for it in agent_catalog.list_marketplace_agents(db)} assert db.query(UserAgent).filter(UserAgent.agent_id == res["agent_id"]).first() is not None @@ -130,15 +133,15 @@ def test_preset_bundles_load_and_install(db_session): db = db_session _seed_users(db) - items = am.list_marketplace_agents(db) + items = agent_catalog.list_marketplace_agents(db) slugs = {it["slug"] for it in items} # Preset sub-agents generated by the import script should exist assert "product-manager" in slugs - detail = am.get_agent_detail(db, "product-manager") + detail = agent_catalog.get_agent_detail(db, "product-manager") assert detail["system_prompt"] assert detail["category"] == "职场办公" - res = am.install_marketplace_agent( + res = agent_installation.install_marketplace_agent( db, "product-manager", owner_user_id="u1", operator_name="U1" ) clone = db.query(UserAgent).filter(UserAgent.agent_id == res["agent_id"]).first() @@ -151,9 +154,9 @@ def test_reject_pending_blocks_listing(db_session): db = db_session _seed_users(db) src = _make_source_agent(db) - sub = am.submit_to_marketplace(db, src["agent_id"], owner_user_id="u1", category="数据分析") - am.review_submission(db, sub["submission_id"], approve=False, review_note="不合适") - assert sub["slug"] not in {it["slug"] for it in am.list_marketplace_agents(db)} + sub = agent_submissions.submit_to_marketplace(db, src["agent_id"], owner_user_id="u1", category="数据分析") + agent_submissions.review_submission(db, sub["submission_id"], approve=False, review_note="不合适") + assert sub["slug"] not in {it["slug"] for it in agent_catalog.list_marketplace_agents(db)} # After rejection the user can withdraw and resubmit - am.withdraw_submission(db, sub["submission_id"], owner_user_id="u1") - assert am.list_my_submissions(db, "u1") == [] + agent_submissions.withdraw_submission(db, sub["submission_id"], owner_user_id="u1") + assert agent_submissions.list_my_submissions(db, "u1") == [] diff --git a/src/backend/tests/skills/test_marketplace_service.py b/src/backend/tests/features/marketplace/test_marketplace_service.py similarity index 73% rename from src/backend/tests/skills/test_marketplace_service.py rename to src/backend/tests/features/marketplace/test_marketplace_service.py index 2ce72d8c..577602fd 100644 --- a/src/backend/tests/skills/test_marketplace_service.py +++ b/src/backend/tests/features/marketplace/test_marketplace_service.py @@ -7,6 +7,14 @@ monkeypatched into a no-op. """ from __future__ import annotations +from features.marketplace import skill_installation +from features.marketplace import skill_versions +from features.marketplace import skill_catalog +from features.marketplace import skill_installation +from features.marketplace import skill_metadata +from features.marketplace import skill_publishing + +from core.capabilities.skills import installation as package_installation import json @@ -15,15 +23,16 @@ from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker -from core.config.catalog_details import resolve_skill_detail +from core.capabilities.catalog.details import resolve_skill_detail from core.db.engine import Base from core.db.models import AdminSkill -from core.services import marketplace_service as mk + @pytest.fixture(autouse=True) def _no_cache_refresh(monkeypatch): - monkeypatch.setattr(mk, "refresh_skill_caches", lambda: None) + monkeypatch.setattr(skill_versions, 'refresh_skill_caches', lambda: None) + monkeypatch.setattr(skill_installation, 'refresh_skill_caches', lambda: None) @pytest.fixture @@ -39,7 +48,7 @@ def db(): def test_list_and_detail_nonempty(): - items = mk.list_marketplace_skills() + items = skill_catalog.list_marketplace_skills() assert len(items) >= 5 slugs = {it["slug"] for it in items} assert "diagram-builder" in slugs @@ -47,14 +56,14 @@ def test_list_and_detail_nonempty(): # installable marketplace portion that follows them. installable_items = [item for item in items if not item.get("builtin")] assert installable_items[0]["featured"] is True - detail = mk.get_marketplace_skill("diagram-builder") + detail = skill_catalog.get_marketplace_skill("diagram-builder") assert detail["instructions"] assert any(f["path"] == "SKILL.md" for f in detail["files"]) is False # SKILL.md is the body, not in files assert detail["files"] def test_guizang_ppt_bundle_is_compact_installable_and_dependency_free(db): - detail = mk.get_marketplace_skill("guizang-ppt-skill") + detail = skill_catalog.get_marketplace_skill("guizang-ppt-skill") paths = {item["path"] for item in detail["files"]} assert detail["source"] == "github" @@ -73,7 +82,7 @@ def test_guizang_ppt_bundle_is_compact_installable_and_dependency_free(db): assert "git -C" not in detail["instructions"] assert 'cp "{baseDir}/assets/motion.min.js"' in detail["instructions"] - result = mk.install_marketplace_skill( + result = skill_installation.install_marketplace_skill( db, "guizang-ppt-skill", owner_user_id=None, secrets={} ) assert result["dependencies"] == { @@ -89,11 +98,11 @@ def test_unknown_slug_404(): from core.infra.exceptions import ResourceNotFoundError with pytest.raises(ResourceNotFoundError): - mk.get_marketplace_skill("does-not-exist") + skill_catalog.get_marketplace_skill("does-not-exist") def test_user_install_is_namespaced_and_private(db): - r = mk.install_marketplace_skill(db, "diagram-builder", owner_user_id="userA", secrets={}) + r = skill_installation.install_marketplace_skill(db, "diagram-builder", owner_user_id="userA", secrets={}) assert r["owner"] == "self" and r["action"] == "installed" row = db.query(AdminSkill).filter_by(skill_id=r["id"]).one() assert row.owner_user_id == "userA" @@ -101,7 +110,7 @@ def test_user_install_is_namespaced_and_private(db): assert row.user_intro is None assert ( resolve_skill_detail(row.user_intro, row.skill_content) - == mk.get_marketplace_skill("diagram-builder")["instructions"] + == skill_catalog.get_marketplace_skill("diagram-builder")["instructions"] ) # frontmatter name is rewritten to the install id assert f"name: {r['id']}" in row.skill_content.split("---")[1] @@ -109,28 +118,28 @@ def test_user_install_is_namespaced_and_private(db): def test_two_users_get_independent_copies(db): - a = mk.install_marketplace_skill(db, "diagram-builder", owner_user_id="userA", secrets={}) - b = mk.install_marketplace_skill(db, "diagram-builder", owner_user_id="userB", secrets={}) + a = skill_installation.install_marketplace_skill(db, "diagram-builder", owner_user_id="userA", secrets={}) + b = skill_installation.install_marketplace_skill(db, "diagram-builder", owner_user_id="userB", secrets={}) assert a["id"] != b["id"] assert db.query(AdminSkill).filter(AdminSkill.skill_id.in_([a["id"], b["id"]])).count() == 2 def test_same_user_reinstall_upserts(db): - a = mk.install_marketplace_skill(db, "diagram-builder", owner_user_id="userA", secrets={}) - a2 = mk.install_marketplace_skill(db, "diagram-builder", owner_user_id="userA", secrets={}) + a = skill_installation.install_marketplace_skill(db, "diagram-builder", owner_user_id="userA", secrets={}) + a2 = skill_installation.install_marketplace_skill(db, "diagram-builder", owner_user_id="userA", secrets={}) assert a2["id"] == a["id"] and a2["action"] == "updated" assert db.query(AdminSkill).filter_by(skill_id=a["id"]).count() == 1 def test_admin_install_is_global(db): - r = mk.install_marketplace_skill(db, "diagram-builder", owner_user_id=None, secrets={}) + r = skill_installation.install_marketplace_skill(db, "diagram-builder", owner_user_id=None, secrets={}) assert r["owner"] == "global" and r["id"] == "diagram-builder" row = db.query(AdminSkill).filter_by(skill_id="diagram-builder").one() assert row.owner_user_id is None def test_secrets_injection(db): - r = mk.install_marketplace_skill( + r = skill_installation.install_marketplace_skill( db, "gpt-image2-pro", owner_user_id="userA", secrets={"IMAGE_GEN_API_KEY": "sk-test-123"}, ) @@ -142,7 +151,7 @@ def test_secrets_injection(db): def test_missing_required_secret_rejected(db, monkeypatch): # Temporarily mark gpt-image2-pro's optional credential as required, verifying that missing it gives 400. - real = mk._read_manifest + real = skill_metadata.read_manifest def fake(slug): m = real(slug) @@ -151,16 +160,16 @@ def fake(slug): m["required_secrets"] = [{"key": "IMAGE_GEN_API_KEY", "label": "Key", "required": True}] return m - monkeypatch.setattr(mk, "_read_manifest", fake) + monkeypatch.setattr(skill_metadata, 'read_manifest', fake) from core.infra.exceptions import BadRequestError with pytest.raises(BadRequestError): - mk.install_marketplace_skill(db, "gpt-image2-pro", owner_user_id="userA", secrets={}) + skill_installation.install_marketplace_skill(db, "gpt-image2-pro", owner_user_id="userA", secrets={}) def test_annotate_installed_flags(db): - mk.install_marketplace_skill(db, "diagram-builder", owner_user_id="userA", secrets={}) - items = mk.annotate_installed(mk.list_marketplace_skills(), db, owner_user_id="userA") + skill_installation.install_marketplace_skill(db, "diagram-builder", owner_user_id="userA", secrets={}) + items = skill_catalog.annotate_installed(skill_catalog.list_marketplace_skills(), db, owner_user_id="userA") by_slug = {it["slug"]: it for it in items} assert by_slug["diagram-builder"]["installed"] is True # another, uninstalled skill is False @@ -171,14 +180,14 @@ def test_annotate_installed_flags(db): assert by_slug[other]["installed"] is False # From userB's view all installable entries are uninstalled; resident # built-ins remain available to every user. - items_b = mk.annotate_installed(mk.list_marketplace_skills(), db, owner_user_id="userB") + items_b = skill_catalog.annotate_installed(skill_catalog.list_marketplace_skills(), db, owner_user_id="userB") assert all(it["installed"] is False for it in items_b if not it.get("builtin")) assert all(it["installed"] is True for it in items_b if it.get("builtin")) def test_cross_owner_conflict_409(db): # Seed a global skill that occupies the namespace id userC's private install would use. - install_id = "diagram-builder-" + mk._user_suffix("userC") + install_id = "diagram-builder-" + package_installation.owner_suffix("userC") db.add(AdminSkill( skill_id=install_id, skill_content="x", display_name="x", description="x", version="1", tags=[], allowed_tools=[], extra_files={}, dependencies={}, @@ -186,7 +195,7 @@ def test_cross_owner_conflict_409(db): )) db.commit() with pytest.raises(HTTPException) as ei: - mk.install_marketplace_skill(db, "diagram-builder", owner_user_id="userC", secrets={}) + skill_installation.install_marketplace_skill(db, "diagram-builder", owner_user_id="userC", secrets={}) assert ei.value.status_code == 409 @@ -208,19 +217,19 @@ def _zip_skill(name: str, *, description: str = "A test skill.", extra: dict | N def test_admin_upload_publishes_to_marketplace_not_global(db): """Admin uploads a zip -> listed on the marketplace as approved (no global AdminSkill created), installable.""" data = _zip_skill("my-uploaded-skill") - res = mk.publish_skill_zip_to_marketplace(db, data, category="办公效率") + res = skill_publishing.publish_skill_zip_to_marketplace(db, data, category="办公效率") assert res["action"] == "published" and res["skill_id"] == "my-uploaded-skill" # no global skill created (not in catalog) assert db.query(AdminSkill).filter(AdminSkill.skill_id == "my-uploaded-skill").first() is None # appears on the marketplace (community source) - items = mk.list_marketplace_skills(db) + items = skill_catalog.list_marketplace_skills(db) m = next((it for it in items if it["slug"] == res["slug"]), None) assert m is not None and m["source"] == "community" and m["category"] == "办公效率" # can be installed as a global skill - inst = mk.install_marketplace_skill(db, res["slug"], owner_user_id=None, secrets={}) + inst = skill_installation.install_marketplace_skill(db, res["slug"], owner_user_id=None, secrets={}) assert inst["action"] == "installed" glob = db.query(AdminSkill).filter(AdminSkill.skill_id == "my-uploaded-skill", AdminSkill.owner_user_id.is_(None)).first() assert glob is not None @@ -230,10 +239,10 @@ def test_admin_upload_publishes_to_marketplace_not_global(db): def test_admin_upload_reupload_updates_same_record(db): """Re-uploading the same skill -> updates the existing listing record (same slug), no new one.""" - first = mk.publish_skill_zip_to_marketplace(db, _zip_skill("dup-skill", description="v1"), category="文档处理") - second = mk.publish_skill_zip_to_marketplace(db, _zip_skill("dup-skill", description="v2"), category="数据分析") + first = skill_publishing.publish_skill_zip_to_marketplace(db, _zip_skill("dup-skill", description="v1"), category="文档处理") + second = skill_publishing.publish_skill_zip_to_marketplace(db, _zip_skill("dup-skill", description="v2"), category="数据分析") assert second["action"] == "updated" and second["slug"] == first["slug"] - from core.db.models import MarketplaceSubmission + from features.marketplace.models import MarketplaceSubmission rows = db.query(MarketplaceSubmission).filter(MarketplaceSubmission.skill_id == "dup-skill").all() assert len(rows) == 1 and rows[0].category == "数据分析" @@ -241,7 +250,7 @@ def test_admin_upload_reupload_updates_same_record(db): def test_admin_upload_rejects_bad_category(db): from core.infra.exceptions import BadRequestError with pytest.raises(BadRequestError): - mk.publish_skill_zip_to_marketplace(db, _zip_skill("cat-skill"), category="不存在的分类") + skill_publishing.publish_skill_zip_to_marketplace(db, _zip_skill("cat-skill"), category="不存在的分类") def test_admin_create_form_publishes_to_marketplace(db): @@ -249,7 +258,7 @@ def test_admin_create_form_publishes_to_marketplace(db): content = ( "---\nname: form-skill\ndescription: built from form.\nversion: 2.0.0\n---\n\n# form-skill\n步骤\n" ) - res = mk.publish_skill_to_marketplace( + res = skill_publishing.publish_skill_to_marketplace( db, skill_id="form-skill", skill_content=content, display_name="表单技能", description="built from form.", user_intro="## 用户介绍\n\n适合表单自动化。", version="2.0.0", tags=["a", "b"], category="研发效率", @@ -258,9 +267,9 @@ def test_admin_create_form_publishes_to_marketplace(db): # no global skill created assert db.query(AdminSkill).filter(AdminSkill.skill_id == "form-skill").first() is None # appears on the marketplace, installable - m = next((it for it in mk.list_marketplace_skills(db) if it["slug"] == "form-skill"), None) + m = next((it for it in skill_catalog.list_marketplace_skills(db) if it["slug"] == "form-skill"), None) assert m is not None and m["category"] == "研发效率" and m["display_name"] == "表单技能" - inst = mk.install_marketplace_skill(db, "form-skill", owner_user_id=None, secrets={}) + inst = skill_installation.install_marketplace_skill(db, "form-skill", owner_user_id=None, secrets={}) assert inst["action"] == "installed" installed = db.query(AdminSkill).filter( AdminSkill.skill_id == "form-skill", AdminSkill.owner_user_id.is_(None) @@ -270,9 +279,9 @@ def test_admin_create_form_publishes_to_marketplace(db): def test_market_summary_is_not_used_as_user_intro(): - assert mk._build_user_intro({"summary": "市场卡片的一句话摘要"}) == "" + assert skill_installation.build_user_intro({"summary": "市场卡片的一句话摘要"}) == "" assert ( - mk._build_user_intro({"summary": "摘要", "user_intro": " ## 详细介绍 "}) + skill_installation.build_user_intro({"summary": "摘要", "user_intro": " ## 详细介绍 "}) == "## 详细介绍" ) @@ -285,7 +294,7 @@ def test_whitespace_user_intro_falls_back_to_skill_body(): def test_admin_create_form_rejects_bad_category(db): from core.infra.exceptions import BadRequestError with pytest.raises(BadRequestError): - mk.publish_skill_to_marketplace( + skill_publishing.publish_skill_to_marketplace( db, skill_id="x-skill", skill_content="---\nname: x-skill\ndescription: d\n---\n", display_name="X", description="d", version="1.0.0", tags=[], category="乱填", ) diff --git a/src/backend/tests/skills/test_marketplace_submission.py b/src/backend/tests/features/marketplace/test_marketplace_submission.py similarity index 54% rename from src/backend/tests/skills/test_marketplace_submission.py rename to src/backend/tests/features/marketplace/test_marketplace_submission.py index 2029329b..36a242f7 100644 --- a/src/backend/tests/skills/test_marketplace_submission.py +++ b/src/backend/tests/features/marketplace/test_marketplace_submission.py @@ -4,6 +4,14 @@ entering the marketplace listing/detail/installable state after approval, going offline on rejection, and category override during review. """ from __future__ import annotations +from features.marketplace import skill_installation +from features.marketplace import skill_versions +from features.marketplace import skill_catalog +from features.marketplace import skill_installation +from features.marketplace import skill_submissions +from features.marketplace import skills_state + +from core.capabilities.skills import installation as package_installation import pytest from fastapi import HTTPException @@ -11,14 +19,16 @@ from sqlalchemy.orm import sessionmaker from core.db.engine import Base -from core.db.models import AdminSkill, MarketplaceSubmission +from core.db.models import AdminSkill +from features.marketplace.models import MarketplaceSubmission from core.infra.exceptions import BadRequestError, ResourceNotFoundError -from core.services import marketplace_service as mk + @pytest.fixture(autouse=True) def _no_cache_refresh(monkeypatch): - monkeypatch.setattr(mk, "refresh_skill_caches", lambda: None) + monkeypatch.setattr(skill_versions, 'refresh_skill_caches', lambda: None) + monkeypatch.setattr(skill_installation, 'refresh_skill_caches', lambda: None) @pytest.fixture @@ -56,11 +66,11 @@ def _make_user_skill( def test_submit_snapshot_and_slug_strips_fingerprint(db): - suffix = mk._user_suffix("userA") + suffix = package_installation.owner_suffix("userA") skill_id = f"my-skill-{suffix}" _make_user_skill(db, skill_id, "userA", user_intro="## 用户介绍\n\n这是完整介绍。") - sub = mk.submit_to_marketplace( + sub = skill_submissions.submit_to_marketplace( db, skill_id, owner_user_id="userA", submitter_name="张三", note="求上架", category="办公效率" ) assert sub["slug"] == "my-skill" # fingerprint suffix stripped @@ -75,93 +85,93 @@ def test_submit_snapshot_and_slug_strips_fingerprint(db): def test_slug_collision_with_builtin_dir(db): # The preset directory already has diagram-builder, so the community slug auto-avoids the collision _make_user_skill(db, "diagram-builder", "userA") - sub = mk.submit_to_marketplace(db, "diagram-builder", owner_user_id="userA", category="办公效率") + sub = skill_submissions.submit_to_marketplace(db, "diagram-builder", owner_user_id="userA", category="办公效率") assert sub["slug"] == "diagram-builder-2" def test_submit_category_must_be_in_fixed_set(db): _make_user_skill(db, "cat-skill", "userA") with pytest.raises(BadRequestError): - mk.submit_to_marketplace(db, "cat-skill", owner_user_id="userA", category="") + skill_submissions.submit_to_marketplace(db, "cat-skill", owner_user_id="userA", category="") with pytest.raises(BadRequestError): - mk.submit_to_marketplace(db, "cat-skill", owner_user_id="userA", category="自定义分类") - sub = mk.submit_to_marketplace(db, "cat-skill", owner_user_id="userA", category="研发效率") + skill_submissions.submit_to_marketplace(db, "cat-skill", owner_user_id="userA", category="自定义分类") + sub = skill_submissions.submit_to_marketplace(db, "cat-skill", owner_user_id="userA", category="研发效率") assert sub["category"] == "研发效率" # Changing the category during review is likewise constrained to the fixed set with pytest.raises(BadRequestError): - mk.review_submission(db, sub["submission_id"], approve=True, category="乱填的") + skill_submissions.review_submission(db, sub["submission_id"], approve=True, category="乱填的") def test_categories_fixed_order(db): - cats = mk.list_categories(db) - assert cats[:8] == mk.MARKETPLACE_CATEGORIES + cats = skill_catalog.list_categories(db) + assert cats[:8] == skills_state.MARKETPLACE_CATEGORIES def test_submit_requires_ownership(db): _make_user_skill(db, "owned-skill", "userA") with pytest.raises(ResourceNotFoundError): - mk.submit_to_marketplace(db, "owned-skill", owner_user_id="userB", category="办公效率") + skill_submissions.submit_to_marketplace(db, "owned-skill", owner_user_id="userB", category="办公效率") def test_duplicate_submission_409(db): _make_user_skill(db, "dup-skill", "userA") - mk.submit_to_marketplace(db, "dup-skill", owner_user_id="userA", category="办公效率") + skill_submissions.submit_to_marketplace(db, "dup-skill", owner_user_id="userA", category="办公效率") with pytest.raises(HTTPException) as e: - mk.submit_to_marketplace(db, "dup-skill", owner_user_id="userA", category="办公效率") + skill_submissions.submit_to_marketplace(db, "dup-skill", owner_user_id="userA", category="办公效率") assert e.value.status_code == 409 def test_withdraw_pending_ok_but_approved_blocked(db): _make_user_skill(db, "wd-skill", "userA") - sub = mk.submit_to_marketplace(db, "wd-skill", owner_user_id="userA", category="办公效率") - mk.withdraw_submission(db, sub["submission_id"], owner_user_id="userA") - assert mk.list_my_submissions(db, "userA") == [] + sub = skill_submissions.submit_to_marketplace(db, "wd-skill", owner_user_id="userA", category="办公效率") + skill_submissions.withdraw_submission(db, sub["submission_id"], owner_user_id="userA") + assert skill_submissions.list_my_submissions(db, "userA") == [] - sub2 = mk.submit_to_marketplace(db, "wd-skill", owner_user_id="userA", category="办公效率") - mk.review_submission(db, sub2["submission_id"], approve=True) + sub2 = skill_submissions.submit_to_marketplace(db, "wd-skill", owner_user_id="userA", category="办公效率") + skill_submissions.review_submission(db, sub2["submission_id"], approve=True) with pytest.raises(BadRequestError): - mk.withdraw_submission(db, sub2["submission_id"], owner_user_id="userA") + skill_submissions.withdraw_submission(db, sub2["submission_id"], owner_user_id="userA") def test_approve_publishes_to_listing_detail_and_install(db): _make_user_skill(db, "pub-skill", "userA") - sub = mk.submit_to_marketplace(db, "pub-skill", owner_user_id="userA", submitter_name="张三", category="办公效率") + sub = skill_submissions.submit_to_marketplace(db, "pub-skill", owner_user_id="userA", submitter_name="张三", category="办公效率") # Not in the marketplace while pending - assert all(it["slug"] != "pub-skill" for it in mk.list_marketplace_skills(db)) + assert all(it["slug"] != "pub-skill" for it in skill_catalog.list_marketplace_skills(db)) - mk.review_submission(db, sub["submission_id"], approve=True, category="数据分析") - items = mk.list_marketplace_skills(db) + skill_submissions.review_submission(db, sub["submission_id"], approve=True, category="数据分析") + items = skill_catalog.list_marketplace_skills(db) pub = next(it for it in items if it["slug"] == "pub-skill") assert pub["source"] == "community" assert pub["author"] == "张三" assert pub["category"] == "数据分析" - detail = mk.get_marketplace_skill("pub-skill", db) + detail = skill_catalog.get_marketplace_skill("pub-skill", db) assert "做点事" in detail["instructions"] assert detail["files"] == [{"path": "scripts/run.py", "size": len("print('hi')\n")}] # Another user installs: sourced from the snapshot, carrying their own fingerprint - r = mk.install_marketplace_skill(db, "pub-skill", owner_user_id="userB") + r = skill_installation.install_marketplace_skill(db, "pub-skill", owner_user_id="userB") assert r["action"] == "installed" row = db.query(AdminSkill).filter_by(skill_id=r["id"]).one() assert row.owner_user_id == "userB" assert row.user_intro is None assert row.extra_files == {"scripts/run.py": "print('hi')\n"} - assert mk.is_installed(db, "pub-skill", owner_user_id="userB") is True + assert skill_catalog.is_installed(db, "pub-skill", owner_user_id="userB") is True def test_reject_approved_takes_offline(db): _make_user_skill(db, "off-skill", "userA") - sub = mk.submit_to_marketplace(db, "off-skill", owner_user_id="userA", category="办公效率") - mk.review_submission(db, sub["submission_id"], approve=True) - assert any(it["slug"] == "off-skill" for it in mk.list_marketplace_skills(db)) + sub = skill_submissions.submit_to_marketplace(db, "off-skill", owner_user_id="userA", category="办公效率") + skill_submissions.review_submission(db, sub["submission_id"], approve=True) + assert any(it["slug"] == "off-skill" for it in skill_catalog.list_marketplace_skills(db)) - mk.review_submission(db, sub["submission_id"], approve=False, review_note="质量不达标") - assert all(it["slug"] != "off-skill" for it in mk.list_marketplace_skills(db)) + skill_submissions.review_submission(db, sub["submission_id"], approve=False, review_note="质量不达标") + assert all(it["slug"] != "off-skill" for it in skill_catalog.list_marketplace_skills(db)) with pytest.raises(ResourceNotFoundError): - mk.get_marketplace_skill("off-skill", db) - sub_after = mk.get_submission(db, sub["submission_id"]) + skill_catalog.get_marketplace_skill("off-skill", db) + sub_after = skill_submissions.get_submission(db, sub["submission_id"]) assert sub_after["status"] == "rejected" assert sub_after["review_note"] == "质量不达标" @@ -169,9 +179,9 @@ def test_reject_approved_takes_offline(db): def test_admin_list_orders_pending_first(db): _make_user_skill(db, "s1", "userA") _make_user_skill(db, "s2", "userA") - a = mk.submit_to_marketplace(db, "s1", owner_user_id="userA", category="办公效率") - b = mk.submit_to_marketplace(db, "s2", owner_user_id="userA", category="数据分析") - mk.review_submission(db, a["submission_id"], approve=True) - rows = mk.list_submissions(db) + a = skill_submissions.submit_to_marketplace(db, "s1", owner_user_id="userA", category="办公效率") + b = skill_submissions.submit_to_marketplace(db, "s2", owner_user_id="userA", category="数据分析") + skill_submissions.review_submission(db, a["submission_id"], approve=True) + rows = skill_submissions.list_submissions(db) assert [r["status"] for r in rows] == ["pending", "approved"] - assert len(mk.list_submissions(db, "pending")) == 1 + assert len(skill_submissions.list_submissions(db, "pending")) == 1 diff --git a/src/backend/tests/features/marketplace/test_mcp_marketplace.py b/src/backend/tests/features/marketplace/test_mcp_marketplace.py new file mode 100644 index 00000000..71cf6a4a --- /dev/null +++ b/src/backend/tests/features/marketplace/test_mcp_marketplace.py @@ -0,0 +1,69 @@ +"""Collect MCP marketplace lifecycle cases with stable pytest node IDs.""" +from tests.features.marketplace.mcp_marketplace_support import db + +from tests.features.marketplace.mcp_marketplace_cases_1 import ( + test_http_probe_releases_its_discovery_resource, + test_http_probe_surfaces_401_instead_of_cancel_scope_noise, + test_ce_metadata_contains_mcp_market_auth_without_ee_dependency, + test_header_credentials_are_encrypted_and_masked, + test_runtime_query_credentials_are_encrypted_and_never_sent_as_headers, + test_oauth_bundle_is_provider_neutral_and_never_sent_as_header, + test_oauth_callback_and_status_are_shared_without_plaintext_code, + test_oauth_start_hands_back_before_discovery_finishes, + test_oauth_cancel_marks_flow_failed_and_wakes_worker, + test_oauth_callback_uses_browser_origin_not_untrusted_host, + test_oauth_callback_allows_configured_http_in_production, + test_legacy_plaintext_headers_are_encrypted_once, + test_remote_url_security_blocks_private_and_accepts_public_dns, + test_remote_url_security_allows_public_http_when_requested, + test_remote_url_security_allows_trusted_private_networks_when_enabled, + test_remote_url_security_keeps_high_risk_ranges_blocked_when_private_enabled, + test_self_service_mcp_uses_private_network_deployment_setting, +) + +from tests.features.marketplace.mcp_marketplace_cases_2 import ( + test_remote_url_security_rechecks_tun_fake_ip_dns, + test_remote_url_security_rechecks_ipv6_tun_fake_ip_dns, + test_remote_url_security_does_not_treat_arbitrary_ula_as_fake_ip, + test_remote_url_security_rejects_fake_ip_when_public_dns_is_private, + test_tool_hash_is_order_independent, + test_submit_snapshots_without_credentials_then_approve, + test_public_http_marketplace_submit_review_and_install, + test_market_installed_private_server_cannot_submit_again, + test_submission_strips_query_secret_into_install_schema, + test_admin_publish_links_existing_global_instance_without_duplication, + test_marketplace_first_admin_source_is_reused_on_global_install, + test_user_install_reuses_admin_managed_credentials_without_exposing_them, + test_user_install_reuses_admin_managed_query_token, +) + +from tests.features.marketplace.mcp_marketplace_cases_3 import ( + test_removed_admin_credential_falls_back_to_installer_input, + test_market_edit_can_require_installer_credentials_without_reusing_source, + test_market_edit_sets_admin_token_and_keeps_it_out_of_responses, + test_create_market_mcp_with_blank_token_requires_each_installer, + test_create_market_mcp_with_admin_token_makes_user_install_tokenless, + test_create_market_mcp_with_oauth_defers_authorization_to_installer, + test_move_existing_published_server_removes_global_installation, + test_edit_market_metadata_propagates_to_existing_installations, + test_install_uses_installers_own_encrypted_credentials, +) + +from tests.features.marketplace.mcp_marketplace_cases_4 import ( + test_curated_templates_seed_once_and_materialize_user_query_key, + test_curated_bearer_token_gets_prefix_and_can_be_rotated, + test_tool_drift_pauses_new_installation, + test_failed_update_probe_never_commits_plaintext_credentials, + test_security_suspension_disables_all_derived_servers, + test_review_rechecks_live_snapshot, + test_missing_source_revalidates_from_reviewed_version, + test_missing_source_still_detects_live_snapshot_drift, + test_manual_admin_revalidation_accepts_drift_as_new_version, + test_periodic_revalidation_never_auto_accepts_drift, + test_manual_revalidation_rejects_empty_tool_snapshot, + test_mcp_visibility_uses_shared_marketplace_scope, + test_delisted_item_blocks_user_install_but_not_admin_install, + test_user_installs_admin_published_private_network_mcp, + test_installer_cannot_redirect_an_admin_item_to_another_private_host, + test_installer_may_keep_the_admin_host_when_overriding_the_endpoint, +) diff --git a/src/backend/tests/features/marketplace/test_oauth_sdk2_contract.py b/src/backend/tests/features/marketplace/test_oauth_sdk2_contract.py new file mode 100644 index 00000000..5a553936 --- /dev/null +++ b/src/backend/tests/features/marketplace/test_oauth_sdk2_contract.py @@ -0,0 +1,113 @@ +"""OAuth SDK 2 callback integrity and interactive discovery budgets.""" +import asyncio +import json +from types import SimpleNamespace +from urllib.parse import parse_qs, urlparse +from unittest.mock import AsyncMock + +import pytest +from mcp.shared.auth import OAuthMetadata + +from core.capabilities.mcp import oauth, management +from features.marketplace.api.mcp_marketplace import oauth_callback + + +class MemoryState: + def __init__(self): + self.values = {} + + async def get(self, key): + return self.values.get(key) + + async def put(self, key, value, **kwargs): + self.values[key] = value + + +@pytest.mark.asyncio +@pytest.mark.parametrize("cross_worker", [False, True]) +@pytest.mark.parametrize("issuer,accepted", [ + ("https://issuer.example.test/", True), + ("https://attacker.example.test/", False), + (None, False), +]) +async def test_callback_issuer_reaches_sdk_validation(monkeypatch, cross_worker, issuer, accepted): + """Exercise the actual route, flow callbacks and SDK authorization grant.""" + state = MemoryState() + monkeypatch.setattr(oauth, "get_ephemeral_state", lambda: state) + monkeypatch.setattr(oauth, "encrypt_secret", lambda value: "encrypted:" + value) + monkeypatch.setattr(oauth, "decrypt_secret", lambda value: value.removeprefix("encrypted:")) + monkeypatch.setattr(oauth, "SessionLocal", lambda: SimpleNamespace(rollback=lambda: None, close=lambda: None)) + monkeypatch.setattr(oauth, "validate_remote_mcp_url", AsyncMock()) + monkeypatch.setattr(oauth, "market_dependency_materialize_mcp_http_connection", lambda url, headers: (url, headers)) + version = SimpleNamespace(auth_config={}, auth_schema=[], url="https://mcp.example.test/mcp", transport="streamable_http") + monkeypatch.setattr(oauth, "get_version", lambda db, slug: (SimpleNamespace(display_name="OAuth test", description=""), version)) + installed = AsyncMock(return_value={"installed": True}) + catalog = SimpleNamespace( + normalize_auth_config=lambda *args: {"methods": [{"id": "oauth", "scopes": []}]}, + installation_secrets=lambda *args, **kwargs: {}, + install_market_item=installed, + ) + monkeypatch.setattr(oauth, "marketplace_catalog", lambda: catalog) + flow = oauth.OAuthInstallFlow( + flow_id="issuer-test", slug="test", owner_user_id="owner", installed_by="owner", + auth_method="oauth", credentials={}, callback_url="https://app.example.test/callback", client_id="client", + ) + monkeypatch.setattr(oauth, "process_state", lambda: SimpleNamespace(flows=flows)) + flows = {flow.flow_id: flow} + + async def probe(row, db, *, oauth_provider, timeout_seconds): + assert oauth_provider.context.client_metadata.application_type == "web" + assert timeout_seconds == 300.0 + oauth_provider.context.client_info = await oauth_provider.context.storage.get_client_info() + oauth_provider.context.oauth_metadata = OAuthMetadata( + issuer="https://issuer.example.test/", + authorization_endpoint="https://issuer.example.test/authorize", + token_endpoint="https://issuer.example.test/token", + authorization_response_iss_parameter_supported=True, + ) + + async def deliver_callback(): + await flow.ready.wait() + authorization_state = parse_qs(urlparse(flow.authorization_url).query)["state"][0] + if cross_worker: + flows.clear() + await oauth_callback(flow_id=flow.flow_id, code="authorization-code", state=authorization_state, + iss=issuer, error="", error_description="") + + callback = asyncio.create_task(deliver_callback()) + try: + code, verifier = await oauth_provider._perform_authorization_code_grant() + assert code == "authorization-code" + assert verifier + return True, "" + finally: + await callback + + monkeypatch.setattr(oauth, "probe_mcp_connectivity", probe) + await oauth.run_flow(flow) + assert flow.status == ("completed" if accepted else "failed") + assert installed.await_count == int(accepted) + persisted = json.loads(await state.get(oauth.callback_key(flow.flow_id))) + assert persisted["iss"] == issuer + assert persisted["code"] == "encrypted:authorization-code" + if not accepted: + assert "iss" in flow.error + + +@pytest.mark.asyncio +async def test_connectivity_probe_passes_interactive_budget_and_releases_resource(monkeypatch): + from core.capabilities.mcp.service import McpServerConfigService + from core.harness.registry import mcp_clients + cfg = {"transport": "streamable_http", "url": "https://mcp.example.test/mcp", "transport_timeout": 17} + monkeypatch.setattr(McpServerConfigService, "get_instance", lambda: SimpleNamespace(_row_to_config=lambda row: dict(cfg))) + resource = SimpleNamespace(descriptors=[], close=AsyncMock()) + discover = AsyncMock(return_value=resource) + monkeypatch.setattr(mcp_clients, "discover_tools", discover) + provider = object() + row = SimpleNamespace(server_id="oauth", transport="streamable_http") + assert await management.probe_mcp_connectivity(row, oauth_provider=provider, timeout_seconds=300) == (True, "") + passed = discover.await_args.args[1] + assert passed["discovery_timeout"] == 300 + assert passed["transport_timeout"] == 17 + assert passed["oauth_provider"] is provider + resource.close.assert_awaited_once() diff --git a/src/backend/tests/features/sites/__init__.py b/src/backend/tests/features/sites/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/src/backend/tests/features/sites/__init__.py @@ -0,0 +1 @@ + diff --git a/src/backend/tests/features/sites/conftest.py b/src/backend/tests/features/sites/conftest.py new file mode 100644 index 00000000..eec431ed --- /dev/null +++ b/src/backend/tests/features/sites/conftest.py @@ -0,0 +1,2 @@ +"""Preserve desktop index fixtures used by site publishing tests.""" +from tests.features.desktop.capabilities.conftest import caps_root, index_db, client, hybrid_catalog_client diff --git a/src/backend/tests/capabilities/test_cloud_site_publish.py b/src/backend/tests/features/sites/test_cloud_site_publish.py similarity index 82% rename from src/backend/tests/capabilities/test_cloud_site_publish.py rename to src/backend/tests/features/sites/test_cloud_site_publish.py index 62e7effc..77af706e 100644 --- a/src/backend/tests/capabilities/test_cloud_site_publish.py +++ b/src/backend/tests/features/sites/test_cloud_site_publish.py @@ -1,12 +1,15 @@ +from features.desktop.bridge import identity as bridge_identity """The desktop sends file bytes; the cloud owns hosted sites.""" +from features.desktop.api import capability_dependencies +from features.desktop.api import desktop_capability_state import io import json import tarfile -from api.routes.v1 import desktop_capability as route +from features.desktop.api import desktop_capability as route from core.sandbox.processes.completion import CompletionMixin -from core.services.desktop_gateway_uploads import UPLOAD_OPTIONS_HEADER, UPLOAD_SCHEMA_HEADER +from core.harness.registry.transfer_channels import UPLOAD_OPTIONS_HEADER, UPLOAD_SCHEMA_HEADER from fastapi import FastAPI from fastapi.testclient import TestClient @@ -24,8 +27,8 @@ def bundle(): def test_site_upload_requires_current_tool_grant(monkeypatch): app = FastAPI() app.include_router(route.router) - app.dependency_overrides[route._require_capability_user] = lambda: "cloud-user" - from core.services import desktop_capability + app.dependency_overrides[capability_dependencies.require_capability_user] = lambda: "cloud-user" + from features.desktop.capability import service as desktop_capability monkeypatch.setattr(desktop_capability, "resolve_gateway_tool", lambda *a, **k: None) response = TestClient(app).post( @@ -44,8 +47,8 @@ def test_uploaded_bytes_are_hosted_and_versioned_for_cloud_user(tmp_path, monkey from core.db.engine import Base from core.db.models import UserShadow from core.infra.exceptions import ResourceNotFoundError - from core.services import desktop_capability - from core.services.site_service import SiteService + from features.desktop.capability import service as desktop_capability + from features.sites.service import SiteService from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker @@ -53,11 +56,11 @@ def test_uploaded_bytes_are_hosted_and_versioned_for_cloud_user(tmp_path, monkey Base.metadata.create_all(engine) factory = sessionmaker(bind=engine) monkeypatch.setattr("core.db.engine.SessionLocal", factory) - from core.services import desktop_capability_security as security + from features.desktop.capability import security as security monkeypatch.setattr(security, "SessionLocal", factory) monkeypatch.setattr(security, "_user_capability_configs", lambda *a, **k: ([], [], {})) - monkeypatch.setattr("core.services.model_config.SessionLocal", factory) + monkeypatch.setattr('core.models.config.SessionLocal', factory) monkeypatch.setenv("STORAGE_TYPE", "local") monkeypatch.setenv("STORAGE_PATH", str(tmp_path / "cloud-storage")) with factory() as db: @@ -65,7 +68,7 @@ def test_uploaded_bytes_are_hosted_and_versioned_for_cloud_user(tmp_path, monkey db.commit() app = FastAPI() app.include_router(route.router) - app.dependency_overrides[route._require_capability_user] = lambda: "cloud-user" + app.dependency_overrides[capability_dependencies.require_capability_user] = lambda: "cloud-user" monkeypatch.setattr( desktop_capability, "resolve_gateway_tool", @@ -116,8 +119,8 @@ def test_uploaded_bytes_are_hosted_and_versioned_for_cloud_user(tmp_path, monkey def test_archive_rejects_traversal_and_expansion_before_reading(): import pytest - from core.services.site_packaging import safe_extract_tar - from core.services.site_service import MAX_SITE_FILE_BYTES + from features.sites.packaging import safe_extract_tar + from features.sites.service import MAX_SITE_FILE_BYTES for name, size in [("../escape", 1), ("/absolute", 1), ("huge.bin", MAX_SITE_FILE_BYTES + 1)]: output = io.BytesIO() @@ -133,14 +136,13 @@ def test_gateway_transfers_local_build_bytes_and_returns_cloud_url(caps_root, mo import asyncio import httpx - import mcp.types - from core.llm.mcp_manager import GatewayMCPTool - from core.services import desktop_cloud_bridge as bridge - from core.services.site_packaging import safe_extract_tar - from tests.capabilities.test_runtime_recovery import state + from core.llm.tools.cloud_gateway import GatewayMCPTool + from features.desktop.bridge import client as bridge + from features.sites.packaging import safe_extract_tar + from tests.features.desktop.capabilities.test_runtime_recovery import state st = state("cloud-user") - monkeypatch.setattr(bridge, "get_state", lambda: st) + monkeypatch.setattr(bridge_identity, 'get_state', lambda: st) archive = bundle() class Sandbox(CompletionMixin): @@ -185,7 +187,7 @@ def cloud(request): tool = GatewayMCPTool( mcp_name="sites-site_publish", source_plugin="sites", - tool=mcp.types.Tool(name="publish_site", inputSchema={"type": "object"}), + tool={"name": "publish_site", "inputSchema": {"type": "object"}}, invoke_url=st["cloud_base"] + "/api/v1/desktop/capability/gateway/sites-site_publish/call", schema_hash="a" * 64, timeout=120, diff --git a/src/backend/tests/llm/test_design_pick.py b/src/backend/tests/features/sites/test_design_pick.py similarity index 99% rename from src/backend/tests/llm/test_design_pick.py rename to src/backend/tests/features/sites/test_design_pick.py index 0260fa5b..ce678393 100644 --- a/src/backend/tests/llm/test_design_pick.py +++ b/src/backend/tests/features/sites/test_design_pick.py @@ -8,7 +8,7 @@ import pytest -from core.llm.tools import _tool_confirmation as mc +from core.llm.tools import confirmation as mc _OPTIONS = [ {"id": "a", "title": "深色科技风", "image_file_id": "fid_a", "brief": ""}, diff --git a/src/backend/tests/llm/test_desktop_site_paths.py b/src/backend/tests/features/sites/test_desktop_site_paths.py similarity index 88% rename from src/backend/tests/llm/test_desktop_site_paths.py rename to src/backend/tests/features/sites/test_desktop_site_paths.py index 2eb63c52..b7717ab0 100644 --- a/src/backend/tests/llm/test_desktop_site_paths.py +++ b/src/backend/tests/features/sites/test_desktop_site_paths.py @@ -16,14 +16,14 @@ ]) def test_windows_publish_boundary(monkeypatch, path, allowed): import core.sandbox.workspace.paths as _common_paths - from core.llm.tools._tool_helpers import _validate_workspace_path + from core.llm.tools._tool_helpers import validate_workspace_path as _validate_workspace_path monkeypatch.setattr(_common_paths, "WORKSPACE", "C:/Users/Aaron/workspace") assert (_validate_workspace_path(path) is None) == allowed def test_portable_archive_enforces_contents_and_limits(tmp_path): from core.sandbox.workspace.archive import pack_directory - from core.services.site_packaging import safe_extract_tar + from features.sites.packaging import safe_extract_tar source=tmp_path/"source" source.mkdir() (source/"index.html").write_bytes(b"test") @@ -53,7 +53,7 @@ def test_paths_are_used_as_given(tmp_path, monkeypatch): 此前 ``/workspace/...`` 会被改写到会话目录，于是同一个"工作区"在提示词、 bash cwd、文件工具三处各有一套说法，模型只能靠拼路径去对齐。 """ - from core.llm.tools import _paths + from core.sandbox.workspace import resolution as _paths from core.config import local_mode from core.sandbox.runner import server monkeypatch.setattr(local_mode, "local_mode_enabled", lambda: True) @@ -69,9 +69,9 @@ def test_publish_uses_real_runner_and_same_session_files(tmp_path, monkeypatch): import asyncio import base64 from core.sandbox.runner import server - from core.services import site_packaging + from features.sites import packaging as site_packaging monkeypatch.setattr(server, "WORKSPACE_ROOT", str(tmp_path)) - monkeypatch.setattr("core.llm.tools._paths.WORKSPACE_ROOT", str(tmp_path)) + monkeypatch.setattr("core.sandbox.workspace.resolution.WORKSPACE_ROOT", str(tmp_path)) monkeypatch.setattr("core.config.local_mode.local_mode_enabled", lambda: True) monkeypatch.setenv("DEPLOY_PROFILE", "local") workspace = server._session_workspace("site-chat",create=True) @@ -94,21 +94,21 @@ async def get_file(self, session, path, user_id=None): assert not list(workspace.glob(".__site_pack_*")) async def test_write_permission_and_bash_share_the_same_file(tmp_path, monkeypatch): - from core.llm.tools import _paths + from core.sandbox.workspace import resolution as _paths from core.config import local_mode from core.sandbox.runner import server from core.llm.tools.write_tool import register_write from core.llm.tools._state import ReadStateTracker - from core.llm.tool_permissions import CURRENT_PERMISSION_TICKET + from core.sandbox.permissions.dispatch_models import CURRENT_PERMISSION_TICKET from core.sandbox.permissions.models import Grant from core.sandbox.permissions.models import Policy from tests.llm.test_local_file_permission import _Toolkit, _service, _call, _payload monkeypatch.setattr(local_mode,"local_mode_enabled",lambda:True) monkeypatch.setattr(_paths,"WORKSPACE_ROOT",str(tmp_path)) monkeypatch.setattr(server,"WORKSPACE_ROOT",str(tmp_path)) - monkeypatch.setattr("core.services.local_grant_service.grants_for_gate",lambda:[Grant(str(tmp_path),"readwrite")]) - monkeypatch.setattr("core.services.local_grant_service.policy_for_gate",lambda *args:Policy()) - monkeypatch.setattr("core.services.local_snapshot_service.maybe_snapshot_local",lambda path:None) + monkeypatch.setattr('features.desktop.local.grants.grants_for_gate',lambda:[Grant(str(tmp_path),"readwrite")]) + monkeypatch.setattr('features.desktop.local.grants.policy_for_gate',lambda *args:Policy()) + monkeypatch.setattr('features.desktop.local.snapshots.maybe_snapshot_local',lambda path:None) from core.sandbox.providers.script_runner.provider import ScriptRunnerProvider monkeypatch.setattr("core.sandbox.get_sandbox_provider",lambda:ScriptRunnerProvider()) toolkit=_Toolkit() diff --git a/src/backend/tests/capabilities/test_local_project_site_publish.py b/src/backend/tests/features/sites/test_local_project_site_publish.py similarity index 93% rename from src/backend/tests/capabilities/test_local_project_site_publish.py rename to src/backend/tests/features/sites/test_local_project_site_publish.py index 022f201c..8349f7d3 100644 --- a/src/backend/tests/capabilities/test_local_project_site_publish.py +++ b/src/backend/tests/features/sites/test_local_project_site_publish.py @@ -12,8 +12,8 @@ from core.db.engine import Base from core.db.models import ChatSession, Project, UserShadow -from core.services.desktop_site_publish import package_local_site -from core.services.site_packaging import safe_extract_tar +from features.sites.desktop_publish import package_local_site +from features.sites.packaging import safe_extract_tar @pytest.fixture @@ -45,7 +45,7 @@ async def pack(src, session_id, user_id, **kwargs): seen.append((src, session_id, user_id)) return [("index.html", (root / "index.html").read_bytes())], None - monkeypatch.setattr("core.services.site_packaging.pack_and_fetch_dir", pack) + monkeypatch.setattr('features.sites.packaging.pack_and_fetch_dir', pack) data, options = asyncio.run(package_local_site( {"src_dir": str(root), "title": "Desktop"}, {"X-Current-User-Id": "owner", "X-Conversation-Id": "chat"}, @@ -60,7 +60,7 @@ def test_local_publish_requires_explicit_directory(local_project, monkeypatch, s root, _ = local_project async def pack(*args, **kwargs): pytest.fail("must reject before packing") - monkeypatch.setattr("core.services.site_packaging.pack_and_fetch_dir", pack) + monkeypatch.setattr('features.sites.packaging.pack_and_fetch_dir', pack) with pytest.raises(ValueError, match="src_dir"): asyncio.run(package_local_site( {"src_dir": src_dir}, {"x-current-user-id": "owner", "x-chat-id": "chat"})) @@ -129,7 +129,7 @@ def test_unbound_chat_publishes_from_its_session_directory(local_project, tmp_pa async def pack(src, session_id, user_id, **kwargs): return [("index.html", (site / "index.html").read_bytes())], None - monkeypatch.setattr("core.services.site_packaging.pack_and_fetch_dir", pack) + monkeypatch.setattr('features.sites.packaging.pack_and_fetch_dir', pack) data, _options = asyncio.run(package_local_site( {"src_dir": str(site)}, {"x-current-user-id": "owner", "x-chat-id": "chat"})) assert safe_extract_tar(data) == [("index.html", b"

default workspace

")] @@ -151,7 +151,7 @@ def test_unbound_chat_still_cannot_publish_outside_the_workspace(local_project, async def pack(*args, **kwargs): pytest.fail("越界时不该走到打包") - monkeypatch.setattr("core.services.site_packaging.pack_and_fetch_dir", pack) + monkeypatch.setattr('features.sites.packaging.pack_and_fetch_dir', pack) with pytest.raises(ValueError): asyncio.run(package_local_site( {"src_dir": str(outside)}, {"x-current-user-id": "owner", "x-chat-id": "chat"})) @@ -163,7 +163,7 @@ def test_bound_project_archive_uses_real_runner(local_project, tmp_path, monkeyp root, _ = local_project monkeypatch.setenv("DEPLOY_PROFILE", "local") monkeypatch.setattr(server, "WORKSPACE_ROOT", str(tmp_path / "workspace")) - monkeypatch.setattr("core.llm.tools._paths.WORKSPACE_ROOT", str(tmp_path / "workspace")) + monkeypatch.setattr("core.sandbox.workspace.resolution.WORKSPACE_ROOT", str(tmp_path / "workspace")) class Provider(CompletionMixin): async def start_process(self, request, yield_time_ms=10000): return vars(await run_runner(server.ProcessRequest( @@ -189,7 +189,7 @@ async def get_file(self, session, path, user_id=None): ]) def test_windows_bound_directory_containment(monkeypatch, path, allowed): import core.sandbox.workspace.paths as _common_paths - from core.llm.tools._tool_helpers import _validate_workspace_path + from core.llm.tools._tool_helpers import validate_workspace_path as _validate_workspace_path monkeypatch.setattr(_common_paths, "WORKSPACE", "C:/managed/workspace") assert (_validate_workspace_path(path, additional_roots=("D:/Desktop/站点",)) is None) == allowed @@ -218,7 +218,7 @@ def test_boundary_is_the_session_directory_not_the_workspace_root(local_project, async def pack(src, session_id, user_id, **kwargs): return [("index.html", (Path(src) / "index.html").read_bytes())], None - monkeypatch.setattr("core.services.site_packaging.pack_and_fetch_dir", pack) + monkeypatch.setattr('features.sites.packaging.pack_and_fetch_dir', pack) data, _ = asyncio.run(package_local_site( {"src_dir": str(inside)}, {"x-current-user-id": "owner", "x-chat-id": "chat"})) diff --git a/src/backend/tests/capabilities/test_local_site_sources.py b/src/backend/tests/features/sites/test_local_site_sources.py similarity index 88% rename from src/backend/tests/capabilities/test_local_site_sources.py rename to src/backend/tests/features/sites/test_local_site_sources.py index 5656094a..72d3be0f 100644 --- a/src/backend/tests/capabilities/test_local_site_sources.py +++ b/src/backend/tests/features/sites/test_local_site_sources.py @@ -7,7 +7,7 @@ from fastapi import FastAPI from fastapi.testclient import TestClient -from tests.capabilities.test_local_project_site_publish import local_project +from tests.features.sites.test_local_project_site_publish import local_project import pytest @@ -15,11 +15,11 @@ def test_prepare_site_creates_real_local_project_before_generation(local_project, monkeypatch): root, factory = local_project - from api.routes.v1 import local_site_sources as route + from features.sites.api import local_site_sources as route from core.auth.backend import get_current_user from types import SimpleNamespace - monkeypatch.setattr("core.llm.tools._paths.WORKSPACE_ROOT", str(root.parent / "managed")) + monkeypatch.setattr("core.sandbox.workspace.resolution.WORKSPACE_ROOT", str(root.parent / "managed")) app = FastAPI() app.include_router(route.router) app.dependency_overrides[get_current_user] = lambda: SimpleNamespace(user_id="owner") @@ -27,7 +27,7 @@ def test_prepare_site_creates_real_local_project_before_generation(local_project assert response.status_code == 200, response.text prepared = response.json()["data"] assert Path(prepared["source_dir"]).is_dir() - from core.services.project_service import ProjectService + from core.projects.edition import ProjectService with factory() as db: project = ProjectService(db).get(prepared["project_id"], "owner") @@ -36,15 +36,15 @@ def test_prepare_site_creates_real_local_project_before_generation(local_project def test_publish_receipt_enables_local_edit_and_reuses_cloud_site(local_project, monkeypatch): root, _ = local_project - from core.services import local_site_sources as sources - from core.services.desktop_site_publish import package_local_site, localize_site_result + from features.sites import local_sources as sources + from features.sites.desktop_publish import package_local_site, localize_site_result monkeypatch.setattr(sources, "current_cloud", lambda: ("https://cloud.example", "cloud-user")) async def pack(src, *_args, **_kwargs): return [("index.html", (Path(src) / "index.html").read_bytes())], None - monkeypatch.setattr("core.services.site_packaging.pack_and_fetch_dir", pack) + monkeypatch.setattr('features.sites.packaging.pack_and_fetch_dir', pack) args = {"src_dir": str(root), "title": "Desktop"} headers = {"x-current-user-id": "owner", "x-chat-id": "chat"} asyncio.run(package_local_site(args, headers)) @@ -82,7 +82,7 @@ async def pack(src, *_args, **_kwargs): def test_editor_reuses_replacement_when_original_chat_was_deleted(local_project, monkeypatch): root, factory = local_project - from core.services import local_site_sources as sources + from features.sites import local_sources as sources from core.db.models import ChatSession from datetime import datetime @@ -106,13 +106,13 @@ def test_editor_reuses_replacement_when_original_chat_was_deleted(local_project, @pytest.mark.parametrize("url", ["/site/report/", "https://cloud.example/site/report/"]) def test_known_cloud_success_survives_local_receipt_failure(local_project, monkeypatch, url): root, _ = local_project - from core.services.desktop_site_publish import localize_site_result + from features.sites.desktop_publish import localize_site_result from fastapi import HTTPException def fail(*args, **kwargs): raise HTTPException(403, "project removed during upload") - monkeypatch.setattr("core.services.local_site_sources.save_receipt", fail) + monkeypatch.setattr('features.sites.local_sources.save_receipt', fail) result = { "content": [ {"type": "text", "text": json.dumps({"ok": True, "site_id": "cloud-site", "url": url})} @@ -139,7 +139,7 @@ def fail(*args, **kwargs): def test_reassociation_moves_site_binding_to_one_project(local_project, monkeypatch): root, factory = local_project - from core.services import local_site_sources as sources + from features.sites import local_sources as sources from core.db.models import Project, ChatSession monkeypatch.setattr(sources, "current_cloud", lambda: ("https://cloud.example", "cloud-user")) @@ -178,7 +178,7 @@ def test_reassociation_moves_site_binding_to_one_project(local_project, monkeypa def test_source_must_already_be_inside_local_project(local_project): root, _ = local_project - from core.services.local_site_sources import validate_source + from features.sites.local_sources import validate_source outside = root.parent / "scratch-site" outside.mkdir() @@ -191,23 +191,23 @@ def test_source_must_already_be_inside_local_project(local_project): def test_local_publish_requires_current_chat(local_project, monkeypatch): - from core.services.desktop_site_publish import package_local_site + from features.sites.desktop_publish import package_local_site async def unexpected_pack(*args, **kwargs): pytest.fail("must reject before reading files") - monkeypatch.setattr("core.services.site_packaging.pack_and_fetch_dir", unexpected_pack) + monkeypatch.setattr('features.sites.packaging.pack_and_fetch_dir', unexpected_pack) with pytest.raises(ValueError, match="当前会话标识"): asyncio.run(package_local_site({}, {"x-current-user-id": "owner"})) def test_project_query_from_new_chat_and_explicit_publish(local_project, monkeypatch): root, factory = local_project - from core.services import local_site_sources as sources - from core.services.desktop_site_publish import package_local_site - from core.services.site_packaging import safe_extract_tar + from features.sites import local_sources as sources + from features.sites.desktop_publish import package_local_site + from features.sites.packaging import safe_extract_tar from core.db.models import ChatSession - from core.services.site_listing import list_sites + from features.sites.listing import list_sites monkeypatch.setattr(sources, "current_cloud", lambda: ("https://cloud.example", "cloud-user")) page = root / "sites" / "resume" @@ -234,7 +234,7 @@ def test_project_query_from_new_chat_and_explicit_publish(local_project, monkeyp async def pack(src, *_args, **_kwargs): return [("index.html", (Path(src) / "index.html").read_bytes())], None - monkeypatch.setattr("core.services.site_packaging.pack_and_fetch_dir", pack) + monkeypatch.setattr('features.sites.packaging.pack_and_fetch_dir', pack) headers = {"x-current-user-id": "owner", "x-chat-id": "new-chat"} data, options = asyncio.run( package_local_site( @@ -257,8 +257,8 @@ async def pack(src, *_args, **_kwargs): def test_project_query_reports_account_switch_as_error(local_project, monkeypatch): - from core.services import local_site_sources as sources - from core.services.site_listing import list_sites + from features.sites import local_sources as sources + from features.sites.listing import list_sites from fastapi import HTTPException states = iter( @@ -278,7 +278,7 @@ def test_project_query_reports_account_switch_as_error(local_project, monkeypatc def test_project_query_lists_candidates_without_mutating_chat(local_project, monkeypatch): root, factory = local_project - from core.services import local_site_sources as sources + from features.sites import local_sources as sources from core.db.models import ChatSession, Project from fastapi import HTTPException @@ -315,15 +315,15 @@ def test_project_query_lists_candidates_without_mutating_chat(local_project, mon def test_unbound_site_can_publish_open_edit_and_republish(local_project, monkeypatch, tmp_path): from core.db.models import ChatSession, Project - from core.services import local_site_sources as sources - from core.services.desktop_site_publish import package_local_site, localize_site_result - from core.services.site_listing import list_sites + from features.sites import local_sources as sources + from features.sites.desktop_publish import package_local_site, localize_site_result + from features.sites.listing import list_sites from core.sandbox.workspace.paths import session_root _, factory = local_project workspace = str(tmp_path / "workspace 中文") monkeypatch.setattr("core.sandbox.workspace.paths.WORKSPACE", workspace) - monkeypatch.setattr("core.llm.tools._paths.WORKSPACE_ROOT", workspace) + monkeypatch.setattr("core.sandbox.workspace.resolution.WORKSPACE_ROOT", workspace) monkeypatch.setattr(sources, "current_cloud", lambda: ("https://cloud.example", "cloud-user")) with factory() as db: db.get(ChatSession, "chat").project_id = None @@ -335,7 +335,7 @@ def test_unbound_site_can_publish_open_edit_and_republish(local_project, monkeyp async def pack(src, *_args, **_kwargs): return [("index.html", (Path(src) / "index.html").read_bytes())], None - monkeypatch.setattr("core.services.site_packaging.pack_and_fetch_dir", pack) + monkeypatch.setattr('features.sites.packaging.pack_and_fetch_dir', pack) headers = {"x-current-user-id": "owner", "x-chat-id": "chat"} args = {"src_dir": "sites/demo", "title": "Unbound"} asyncio.run(package_local_site(args, headers)) @@ -353,7 +353,7 @@ async def pack(src, *_args, **_kwargs): (page / "index.html").write_text("edited") update_args = {"site_id": entry["site_id"], "src_dir": entry["publish_dir"]} data, options = asyncio.run(package_local_site(update_args, headers)) - from core.services.site_packaging import safe_extract_tar + from features.sites.packaging import safe_extract_tar assert safe_extract_tar(data) == [("index.html", b"edited")] assert json.loads(options)["site_id"] == "unbound-site" result["content"][0]["text"] = json.dumps({ diff --git a/src/backend/tests/api/test_site_access_password.py b/src/backend/tests/features/sites/test_site_access_password.py similarity index 95% rename from src/backend/tests/api/test_site_access_password.py rename to src/backend/tests/features/sites/test_site_access_password.py index 9c2ea484..e4507593 100644 --- a/src/backend/tests/api/test_site_access_password.py +++ b/src/backend/tests/features/sites/test_site_access_password.py @@ -3,12 +3,12 @@ import time import pytest -from api.routes import sites_serve -from core.services.site_rate_limit import UNLOCK_LIMIT +from features.sites.api import sites_serve +from features.sites.rate_limit import UNLOCK_LIMIT from core.db.engine import get_db from core.db.models import UserShadow -from core.services import site_password -from core.services.site_service import SiteService +from features.sites import password as site_password +from features.sites.service import SiteService from fastapi import FastAPI from fastapi.testclient import TestClient @@ -32,13 +32,13 @@ def svc(db_session, tmp_path, monkeypatch): @pytest.fixture() def client(db_session, monkeypatch): - monkeypatch.setattr("core.services.site_rate_limit.redis_configured", lambda: False) + monkeypatch.setattr('features.sites.rate_limit.redis_configured', lambda: False) from types import SimpleNamespace - monkeypatch.setattr("core.services.site_rate_limit.settings", SimpleNamespace(deploy=SimpleNamespace(is_local=True))) + monkeypatch.setattr('features.sites.rate_limit.settings', SimpleNamespace(deploy=SimpleNamespace(is_local=True))) app = FastAPI() app.include_router(sites_serve.router) app.dependency_overrides[get_db] = lambda: db_session - from core.services import site_rate_limit + from features.sites import rate_limit as site_rate_limit site_rate_limit._rate_buckets.clear() site_rate_limit._unlock_buckets.clear() return TestClient(app) @@ -205,7 +205,7 @@ def test_manager_skips_the_gate_but_visitors_do_not(svc, owner, db_session): def test_management_endpoints_set_and_clear_the_password(svc, owner, db_session): """站点管理面板走的就是这两个接口——顺带确认 DELETE 没被 /{site_id} 抢走路由。""" - from api.routes.v1 import sites as sites_routes + from features.sites.api import sites as sites_routes from core.auth.backend import UserContext, get_current_user app = FastAPI() diff --git a/src/backend/tests/api/test_site_contention_postgres.py b/src/backend/tests/features/sites/test_site_contention_postgres.py similarity index 92% rename from src/backend/tests/api/test_site_contention_postgres.py rename to src/backend/tests/features/sites/test_site_contention_postgres.py index 5c68245e..8517e9e1 100644 --- a/src/backend/tests/api/test_site_contention_postgres.py +++ b/src/backend/tests/features/sites/test_site_contention_postgres.py @@ -12,13 +12,14 @@ import pytest from api.health import router as health_router -from api.routes import sites_serve -from api.routes.v1 import internal_sites, sites +from features.sites.api import sites_serve +from features.sites.api import internal_sites, sites from core.auth.backend import get_current_user from core.db.engine import Base, get_db -from core.db.models import Site, UserShadow -from core.services.site_service import SiteService -from core.storage.local import LocalStorageBackend +from features.sites.models import Site +from core.db.models import UserShadow +from features.sites.service import SiteService +from core.files.storage.local import LocalStorageBackend from fastapi import FastAPI from fastapi.testclient import TestClient from sqlalchemy import create_engine, text @@ -50,10 +51,10 @@ def site_http(tmp_path, monkeypatch): Base.metadata.create_all(engine) monkeypatch.setenv("STORAGE_PATH", str(tmp_path / "storage")) storage = LocalStorageBackend() - monkeypatch.setattr("core.storage.get_storage", lambda: storage) - monkeypatch.setattr("core.services.site_service.get_storage", lambda: storage) + monkeypatch.setattr('core.files.storage.get_storage', lambda: storage) + monkeypatch.setattr('features.sites.service.get_storage', lambda: storage) monkeypatch.setattr("core.db.engine.SessionLocal", factory) - monkeypatch.setattr("core.services.desktop_cloud_bridge.bridge_enabled", lambda: False) + monkeypatch.setattr('features.desktop.auth.bridge.bridge_enabled', lambda: False) monkeypatch.setenv("BACKEND_INTERNAL_TOKEN", "site-test-only") async def pack_directory(*args, **kwargs): diff --git a/src/backend/tests/capabilities/test_site_listing.py b/src/backend/tests/features/sites/test_site_listing.py similarity index 90% rename from src/backend/tests/capabilities/test_site_listing.py rename to src/backend/tests/features/sites/test_site_listing.py index d9f08725..8213b5a6 100644 --- a/src/backend/tests/capabilities/test_site_listing.py +++ b/src/backend/tests/features/sites/test_site_listing.py @@ -8,7 +8,8 @@ @pytest.fixture def cloud(tmp_path, monkeypatch): from core.db.engine import Base - from core.db.models import ChatSession, Project, Site, UserFolder, UserShadow + from core.db.models import ChatSession, Project, UserFolder, UserShadow + from features.sites.models import Site from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker @@ -83,7 +84,7 @@ def cloud(tmp_path, monkeypatch): def test_loose_chat_still_finds_every_editable_site(cloud): - from core.services.site_listing import list_sites + from features.sites.listing import list_sites entries = {e["site_id"]: e for e in list_sites("owner", "chat-loose")} assert set(entries) == {"site-static", "site-build", "site-legacy"} @@ -91,7 +92,7 @@ def test_loose_chat_still_finds_every_editable_site(cloud): def test_source_dir_is_the_logical_myspace_path_the_agent_can_type(cloud): - from core.services.site_listing import list_sites + from features.sites.listing import list_sites entry = next(e for e in list_sites("owner", "chat-loose") if e["site_id"] == "site-static") assert entry["source_dir"] == "/myspace/产品介绍站" @@ -103,7 +104,7 @@ def test_source_dir_is_the_logical_myspace_path_the_agent_can_type(cloud): def test_build_sites_report_no_publish_dir_so_the_agent_rebuilds(cloud): - from core.services.site_listing import list_sites + from features.sites.listing import list_sites entry = next(e for e in list_sites("owner", "chat-loose") if e["site_id"] == "site-build") assert entry["kind"] == "build" @@ -112,7 +113,7 @@ def test_build_sites_report_no_publish_dir_so_the_agent_rebuilds(cloud): def test_site_without_source_project_is_reported_uneditable(cloud): - from core.services.site_listing import list_sites + from features.sites.listing import list_sites entry = next(e for e in list_sites("owner", "chat-loose") if e["site_id"] == "site-legacy") assert entry["editable"] is False @@ -120,7 +121,7 @@ def test_site_without_source_project_is_reported_uneditable(cloud): def test_current_project_site_sorts_first(cloud): - from core.services.site_listing import list_sites + from features.sites.listing import list_sites entries = list_sites("owner", "chat-in-a") assert entries[0]["site_id"] == "site-static" @@ -129,14 +130,14 @@ def test_current_project_site_sorts_first(cloud): def test_listing_is_scoped_to_the_asking_account(cloud): - from core.services.site_listing import list_sites + from features.sites.listing import list_sites assert list_sites("someone-else", "chat-loose") == [] assert list_sites("", "chat-loose") == [] def test_internal_endpoint_serves_the_mcp_tool(cloud, monkeypatch): - from api.routes.v1 import internal_sites + from features.sites.api import internal_sites from fastapi import FastAPI from fastapi.testclient import TestClient @@ -157,7 +158,7 @@ def test_internal_endpoint_serves_the_mcp_tool(cloud, monkeypatch): def test_limit_is_clamped_to_a_context_safe_window(cloud): - from core.services.site_listing import list_sites + from features.sites.listing import list_sites assert len(list_sites("owner", "chat-loose", limit=1)) == 1 assert len(list_sites("owner", "chat-loose", limit=999)) == 3 diff --git a/src/backend/tests/capabilities/test_site_plugin_upgrade.py b/src/backend/tests/features/sites/test_site_plugin_upgrade.py similarity index 85% rename from src/backend/tests/capabilities/test_site_plugin_upgrade.py rename to src/backend/tests/features/sites/test_site_plugin_upgrade.py index 0c2a94a4..13082840 100644 --- a/src/backend/tests/capabilities/test_site_plugin_upgrade.py +++ b/src/backend/tests/features/sites/test_site_plugin_upgrade.py @@ -6,12 +6,12 @@ from pathlib import Path import pytest -from core.plugins.management import projection as plugin_projection -from tests.capabilities.test_local_project_site_publish import local_project +from core.capabilities.plugins.management import projection as plugin_projection +from tests.features.sites.test_local_project_site_publish import local_project BUNDLE_VERSION = json.loads( ( - Path(__file__).resolve().parents[2] / "extensions/plugins/marketplace/sites/plugin.json" + Path(__file__).resolve().parents[3] / "extensions/plugins/marketplace/sites/plugin.json" ).read_text() )["version"] @@ -22,12 +22,12 @@ def test_installed_sites_upgrade_preserves_settings_and_exports_new_skill( ): _, factory = local_project from core.db.models import AdminMcpServer, AdminSkill, InstalledPlugin - from core.plugins import management as plugin_service - from core.plugins.local.site_upgrade import upgrade_builtin_sites - from core.services.desktop_capability_protocol import skill_content_hash - from core.services.marketplace_service import build_skill_zip + from core.capabilities.plugins import management as plugin_service + from features.sites.builtin_upgrade import upgrade_builtin_sites + from core.capabilities.manifest_protocol import skill_content_hash + from features.marketplace.skill_archives import build_skill_zip - monkeypatch.setattr(plugin_projection, "_refresh_after_change", lambda *_: None) + monkeypatch.setattr(plugin_projection, 'refresh_after_change', lambda *_: None) monkeypatch.setattr(plugin_projection, "_project_plugin_to_store", lambda *a, **k: None) with factory() as db: plugin_service.install_plugin(db, "sites", owner_user_id=owner) @@ -87,10 +87,10 @@ def test_installed_sites_upgrade_preserves_settings_and_exports_new_skill( def test_sites_upgrade_skips_custom_and_newer_installs(local_project, monkeypatch, source, version): _, factory = local_project from core.db.models import InstalledPlugin - from core.plugins import management as plugin_service - from core.plugins.local.site_upgrade import upgrade_builtin_sites + from core.capabilities.plugins import management as plugin_service + from features.sites.builtin_upgrade import upgrade_builtin_sites - monkeypatch.setattr(plugin_projection, "_refresh_after_change", lambda *_: None) + monkeypatch.setattr(plugin_projection, 'refresh_after_change', lambda *_: None) monkeypatch.setattr(plugin_projection, "_project_plugin_to_store", lambda *a, **k: None) with factory() as db: plugin_service.install_plugin(db, "sites", owner_user_id=None) @@ -106,10 +106,10 @@ def test_site_upgrade_is_an_all_role_startup_step(local_project, monkeypatch): import importlib from core.db.models import InstalledPlugin - from core.plugins import management as plugin_service + from core.capabilities.plugins import management as plugin_service _, factory = local_project - monkeypatch.setattr(plugin_projection, "_refresh_after_change", lambda *_: None) + monkeypatch.setattr(plugin_projection, 'refresh_after_change', lambda *_: None) monkeypatch.setattr(plugin_projection, "_project_plugin_to_store", lambda *a, **k: None) with factory() as db: plugin_service.install_plugin(db, "sites", owner_user_id=None) @@ -127,15 +127,15 @@ def test_site_upgrade_is_an_all_role_startup_step(local_project, monkeypatch): def test_site_upgrade_refreshes_only_matching_local_projection(local_project, monkeypatch): - from core.capabilities import plugins, registry, store + from features.desktop.capabilities import plugins, registry, store from core.db.models import InstalledPlugin - from core.plugins import management as plugin_service - from core.plugins.local.site_upgrade import upgrade_builtin_sites + from core.capabilities.plugins import management as plugin_service + from features.sites.builtin_upgrade import upgrade_builtin_sites root, factory = local_project monkeypatch.setenv("HUGAGENT_CAPS_ROOT", str(root.parent / "caps")) monkeypatch.setattr(registry, "SessionLocal", factory) - monkeypatch.setattr(plugin_projection, "_refresh_after_change", lambda *_: None) + monkeypatch.setattr(plugin_projection, 'refresh_after_change', lambda *_: None) with factory() as db: plugin_service.install_plugin(db, "sites", owner_user_id=None) row = db.query(InstalledPlugin).one() @@ -176,11 +176,11 @@ def test_old_sites_install_gains_mcp_builder_without_enabling_disabled_skills( local_project, monkeypatch, owner ): from core.db.models import AdminMcpServer, AdminSkill, InstalledPlugin - from core.plugins import management as plugin_service - from core.plugins.local.site_upgrade import upgrade_builtin_sites + from core.capabilities.plugins import management as plugin_service + from features.sites.builtin_upgrade import upgrade_builtin_sites _, factory = local_project - monkeypatch.setattr(plugin_projection, "_refresh_after_change", lambda *_: None) + monkeypatch.setattr(plugin_projection, 'refresh_after_change', lambda *_: None) monkeypatch.setattr(plugin_projection, "_project_plugin_to_store", lambda *a, **k: None) with factory() as db: plugin_service.install_plugin(db, "sites", owner_user_id=owner) diff --git a/src/backend/tests/services/test_site_rate_limit.py b/src/backend/tests/features/sites/test_site_rate_limit.py similarity index 97% rename from src/backend/tests/services/test_site_rate_limit.py rename to src/backend/tests/features/sites/test_site_rate_limit.py index 1488a638..deafaa64 100644 --- a/src/backend/tests/services/test_site_rate_limit.py +++ b/src/backend/tests/features/sites/test_site_rate_limit.py @@ -7,7 +7,7 @@ import pytest from fastapi import HTTPException from redis.asyncio import Redis -from core.services import site_rate_limit as limiter +from features.sites import rate_limit as limiter @pytest.mark.asyncio async def test_two_workers_share_atomic_budget(tmp_path, monkeypatch): diff --git a/src/backend/tests/features/test_execution_identity_contract.py b/src/backend/tests/features/test_execution_identity_contract.py new file mode 100644 index 00000000..ff84646a --- /dev/null +++ b/src/backend/tests/features/test_execution_identity_contract.py @@ -0,0 +1,44 @@ +"""Execution identity must fail closed until the host binds its implementation.""" +import os +import subprocess +import sys + + +def test_headless_identity_does_not_import_desktop_or_authorize(): + script = """ +import sys +from core.harness.registry import execution_identity, execution_account, execution_catalog +assert not any(name.startswith('features.desktop') for name in sys.modules) +assert execution_identity.get_state() is None +assert execution_identity.state_fingerprint(None) == '' +assert execution_account.current_local_user_id() is None +assert not execution_account.account_authorized_for('user-1') +assert execution_catalog.apply_to_enabled_skill_ids(['s']) == ['s'] +assert execution_catalog.cloud_gateway_mcp_configs() == {} +for operation in ( + lambda: execution_identity.require_current_account({'token': 'test'}), + lambda: execution_identity.cloud_headers({'token': 'test'}), + lambda: execution_identity.ensure_current_authorization(), +): + try: + operation() + except RuntimeError: + pass + else: + raise AssertionError('unregistered identity authorized an operation') +""" + result = subprocess.run([sys.executable, '-c', script], env=os.environ.copy(), capture_output=True, text=True) + assert result.returncode == 0, result.stderr + + +def test_headless_tool_host_registers_marketplace_without_api(): + script = """ +import sys +from features import register_product_tools +from core.harness.registry.marketplace_catalog import marketplace_catalog +register_product_tools() +assert type(marketplace_catalog()).__name__ == 'MarketplaceCatalogProvider' +assert 'api.app' not in sys.modules +""" + result = subprocess.run([sys.executable, '-c', script], env=os.environ.copy(), capture_output=True, text=True) + assert result.returncode == 0, result.stderr diff --git a/src/backend/tests/features/test_host_runtime.py b/src/backend/tests/features/test_host_runtime.py new file mode 100644 index 00000000..86427af1 --- /dev/null +++ b/src/backend/tests/features/test_host_runtime.py @@ -0,0 +1,18 @@ +"""Remote execution policy remains dynamic and host composed.""" +import subprocess +import sys + + +def test_core_policy_does_not_import_product_modules(): + subprocess.run([sys.executable, "-c", "from core.harness.registry.host_runtime import requires_remote_execution; import sys; assert not requires_remote_execution(); assert not any(n.startswith('features.') for n in sys.modules)"], check=True) + + +def test_host_policy_reads_current_bridge_setting(monkeypatch): + from features import register_run_capabilities + from features.desktop.auth import bridge as desktop_bridge + from core.harness.registry.host_runtime import requires_remote_execution + register_run_capabilities() + monkeypatch.setattr(desktop_bridge, "bridge_enabled", lambda: True) + assert requires_remote_execution() + monkeypatch.setattr(desktop_bridge, "bridge_enabled", lambda: False) + assert not requires_remote_execution() diff --git a/src/backend/tests/features/test_installed_modules.py b/src/backend/tests/features/test_installed_modules.py new file mode 100644 index 00000000..c157dd55 --- /dev/null +++ b/src/backend/tests/features/test_installed_modules.py @@ -0,0 +1,63 @@ +"""Installed schemas survive disabled runtime features and headless startup.""" +import os +from pathlib import Path +import subprocess +import sys +from types import SimpleNamespace + + +def test_models_are_registered_before_disabled_setup(monkeypatch): + import core.harness.registry as registry + import core.harness.registry.edition as edition + import core.harness.registry.schema_lifecycle as schema_lifecycle + + calls = [] + + def imported(name): + calls.append(name) + if name.endswith(".extension"): + return SimpleNamespace(NAME="example", setup=lambda: False) + return SimpleNamespace() + + monkeypatch.setattr(registry, "import_module", imported) + monkeypatch.setattr(schema_lifecycle, "import_module", imported) + monkeypatch.setattr(registry, "_loaded", False) + monkeypatch.setattr(edition, "EDITION_EXTENSIONS", ()) + monkeypatch.setattr(registry, "mark_enabled", lambda name: calls.append("enabled:" + name)) + registry.load_extensions(("features.example",)) + registry.load_extensions(("features.example",)) + assert calls == [ + "features.example.models", + "features.example.extension", # Schema hooks load even when setup is disabled. + "features.example.extension", + ] + + +def test_model_loading_does_not_call_extension_setup(monkeypatch): + import core.harness.registry as registry + + imports = [] + monkeypatch.setattr(registry, "import_module", imports.append) + registry.load_models(("features.first", "features.second")) + assert imports == ["features.first.models", "features.second.models"] + + +def test_disabled_personal_agent_schema_available_without_api(): + env = dict(os.environ, PERSONAL_AGENT_ENABLED="false") + backend = Path(__file__).resolve().parents[2] + env["PYTHONPATH"] = str(backend) + script = """ +import sys +from core.db import models +from core.db.engine import Base +from core.harness.registry import load_models, is_enabled +from features.installed import INSTALLED_FEATURES +load_models(INSTALLED_FEATURES) +assert 'personal_agents' in Base.metadata.tables +assert 'personal_devices' in Base.metadata.tables +assert 'personal_agent.extension' not in sys.modules +assert 'api.app' not in sys.modules +assert not is_enabled('personal_agent') +""" + result = subprocess.run([sys.executable, "-c", script], env=env, capture_output=True, text=True) + assert result.returncode == 0, result.stderr diff --git a/src/backend/tests/features/test_run_capability_host.py b/src/backend/tests/features/test_run_capability_host.py new file mode 100644 index 00000000..f82c7ee5 --- /dev/null +++ b/src/backend/tests/features/test_run_capability_host.py @@ -0,0 +1,15 @@ +"""Independent processes must explicitly install the same runtime selection as API/CLI.""" +import os +import subprocess +import sys + +import pytest + + +@pytest.mark.parametrize("enabled", [False, True]) +def test_headless_capability_host_preserves_dynamic_selection(enabled, tmp_path): + env = dict(os.environ) + env["HUGAGENT_CAPS_ROOT"] = str(tmp_path) if enabled else "" + script = 'import asyncio\nimport os\nimport sys\nfrom features import register_run_capabilities\nfrom core.harness.registry.run_capabilities import has_run_capabilities, run_capability_provider, prepare_run_capabilities\nfrom features.desktop.capabilities.paths import capabilities_enabled\n\nregister_run_capabilities()\nregister_run_capabilities()\nassert has_run_capabilities() == capabilities_enabled()\nassert "api.app" not in sys.modules\nos.environ["HUGAGENT_CAPS_ROOT"] = "/tmp/core-refactor-headless-proof"\nassert has_run_capabilities()\nassert type(run_capability_provider()).__name__ == "DesktopRunCapabilityProvider"\nos.environ.pop("HUGAGENT_CAPS_ROOT")\nassert not has_run_capabilities()\nassert asyncio.run(prepare_run_capabilities(None, _caps_skill_ids=["selected"])) == (["selected"], None, None)\nassert "api.app" not in sys.modules\nprint("headless capability selection passed: desktop -> cloud, no API import")\n' + result = subprocess.run([sys.executable, "-c", script], env=env, capture_output=True, text=True) + assert result.returncode == 0, result.stdout + result.stderr diff --git a/src/backend/tests/features/workbench/__init__.py b/src/backend/tests/features/workbench/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/src/backend/tests/features/workbench/__init__.py @@ -0,0 +1 @@ + diff --git a/src/backend/tests/test_plugin_ui_contract.py b/src/backend/tests/features/workbench/test_plugin_ui_contract.py similarity index 97% rename from src/backend/tests/test_plugin_ui_contract.py rename to src/backend/tests/features/workbench/test_plugin_ui_contract.py index d2b2424e..d5a235b5 100644 --- a/src/backend/tests/test_plugin_ui_contract.py +++ b/src/backend/tests/features/workbench/test_plugin_ui_contract.py @@ -20,12 +20,12 @@ import pytest from core.infra.exceptions import BadRequestError -from core.plugins.ui import data_proxy as proxy -from core.plugins.ui.primitives import VIEW_KINDS -from core.plugins.ui.contract import SUPPORTED_UI_VERSION, find_data_source, find_module, normalize_ui, public_contributions +from features.workbench.ui import data_proxy as proxy +from features.workbench.ui.primitives import VIEW_KINDS +from features.workbench.ui.contract import SUPPORTED_UI_VERSION, find_data_source, find_module, normalize_ui, public_contributions IKC_MANIFEST = ( - Path(__file__).resolve().parents[1] + Path(__file__).resolve().parents[3] / "extensions/plugins/marketplace/industry-knowledge-center/plugin.json" ) @@ -350,7 +350,7 @@ def test_backend_view_kinds_match_the_frontend_registry(): in a type check, so it is pinned here. """ # tests/ -> backend/ -> src/ - registry = Path(__file__).resolve().parents[2] / "frontend/src/plugin-ui/registry.ts" + registry = Path(__file__).resolve().parents[4] / "frontend/src/plugin-ui/registry.ts" if not registry.is_file(): # CE tree ships the library; a bare backend checkout may not pytest.skip("frontend registry not present in this tree") diff --git a/src/backend/tests/test_plugin_ui_routes.py b/src/backend/tests/features/workbench/test_plugin_ui_routes.py similarity index 97% rename from src/backend/tests/test_plugin_ui_routes.py rename to src/backend/tests/features/workbench/test_plugin_ui_routes.py index 7875b2b3..8f1f4936 100644 --- a/src/backend/tests/test_plugin_ui_routes.py +++ b/src/backend/tests/features/workbench/test_plugin_ui_routes.py @@ -46,7 +46,7 @@ def _override_db(): def _install(db_session): - from core.plugins import management as ps + from core.capabilities.plugins import management as ps return ps.install_plugin(db_session, SLUG, owner_user_id=OWNER) @@ -158,7 +158,7 @@ def test_startup_refresh_backfills_pre_upgrade_installs(client, db_session): auto-opened. The startup refresh must backfill such rows from the bundle. """ from core.db.models import InstalledPlugin - from core.plugins.ui.contributions import refresh_builtin_ui_contributions + from features.workbench.ui.contributions import refresh_builtin_ui_contributions install_id = _install(db_session)["install_id"] # Simulate the pre-upgrade row. diff --git a/src/backend/tests/test_workbench_activity_routes.py b/src/backend/tests/features/workbench/test_workbench_activity_routes.py similarity index 98% rename from src/backend/tests/test_workbench_activity_routes.py rename to src/backend/tests/features/workbench/test_workbench_activity_routes.py index 1e4d0ecb..c01ab22c 100644 --- a/src/backend/tests/test_workbench_activity_routes.py +++ b/src/backend/tests/features/workbench/test_workbench_activity_routes.py @@ -9,7 +9,7 @@ @pytest.fixture() def client(db_session): - from api.routes.v1.workbench import router + from features.workbench.api.workbench import router app = FastAPI() app.include_router(router) @@ -24,7 +24,7 @@ def test_unopened_catalog_activity_does_not_create_instances(client, db_session) from core.db.models import ContentBlock # Persistent service identity is provisioned independently of application instances. - from core.applications import origin + from features.workbench import origin origin.current(db_session) before = db_session.query(ContentBlock).count() response = client.get("/v1/workbench/me/activity") @@ -46,7 +46,7 @@ def plugin(client, db_session): import zipfile from api.routes.v1.plugins import router from core.db.models import UserShadow - from tests.workbench_plugin_support import package + from tests.features.workbench.workbench_plugin_support import package client.app.include_router(router) db_session.add( diff --git a/src/backend/tests/test_workbench_actual_process_observer.py b/src/backend/tests/features/workbench/test_workbench_actual_process_observer.py similarity index 93% rename from src/backend/tests/test_workbench_actual_process_observer.py rename to src/backend/tests/features/workbench/test_workbench_actual_process_observer.py index 04a96f8f..0b9dbb08 100644 --- a/src/backend/tests/test_workbench_actual_process_observer.py +++ b/src/backend/tests/features/workbench/test_workbench_actual_process_observer.py @@ -20,8 +20,8 @@ async def test_observer_reads_actual_owner_process_without_consuming_output(monk monkeypatch.setattr(server, "_AUTH_TOKEN", "wb19-isolated-control") monkeypatch.setattr(server.app.router, "routes", list(server.app.routes)) - monkeypatch.delitem(sys.modules, "tests.workbench_observed_runner_fixture", raising=False) - observer = importlib.import_module("tests.workbench_observed_runner_fixture") + monkeypatch.delitem(sys.modules, "tests.features.workbench.workbench_observed_runner_fixture", raising=False) + observer = importlib.import_module("tests.features.workbench.workbench_observed_runner_fixture") sessions = server.process_sessions class ActualHandle: diff --git a/src/backend/tests/test_workbench_application_config_http.py b/src/backend/tests/features/workbench/test_workbench_application_config_http.py similarity index 97% rename from src/backend/tests/test_workbench_application_config_http.py rename to src/backend/tests/features/workbench/test_workbench_application_config_http.py index df1848f6..16e08e66 100644 --- a/src/backend/tests/test_workbench_application_config_http.py +++ b/src/backend/tests/features/workbench/test_workbench_application_config_http.py @@ -22,13 +22,13 @@ @pytest.fixture() def boundary(monkeypatch): from api.app import app - from core.plugins.ui import data_proxy - from core.services.system_config import SystemConfigService + from features.workbench.ui import data_proxy + from core.config.store.system_config import SystemConfigService assert settings.auth.mode == "session" and session._use_memory_store() assert not app.dependency_overrides monkeypatch.setenv("PLUGIN_RESOURCE_SECRET_KEY", "isolated-app-config-key") monkeypatch.setenv("EMAIL_SECRET_KEY", "isolated-api-key-encryption") - session._MEMORY_SESSIONS.clear() + session.process_state().MEMORY_SESSIONS.clear() Base.metadata.create_all(engine) with SessionLocal() as db: for name in ("owner", "other", "admin", "content-admin"): @@ -70,14 +70,14 @@ def log_message(self, *_): server.shutdown() server.server_close() worker.join() - session._MEMORY_SESSIONS.clear() + session.process_state().MEMORY_SESSIONS.clear() Base.metadata.drop_all(engine) SystemConfigService._instance = None data_proxy._client = None def package(name="scoped-data", endpoint="endpoint", secret="token", url=None): - from tests.workbench_plugin_support import package as base + from tests.features.workbench.workbench_plugin_support import package as base original = zipfile.ZipFile(io.BytesIO(base())) manifest = json.loads(original.read("plugin.json")) manifest["name"] = name @@ -214,7 +214,7 @@ def test_other_install_origin_revision_and_untrusted_target_cannot_inherit(bound def test_private_manifest_cannot_read_or_trust_global_platform_fields(boundary): clients, upstream, calls = boundary - from core.services.system_config import SystemConfigService + from core.config.store.system_config import SystemConfigService service = SystemConfigService.get_instance() service.get("firecrawl.api_key") # Seed only the platform's own built-in schema. response = clients["admin"].put("/v1/service-configs", json={"items": [ diff --git a/src/backend/tests/test_workbench_approval_lease_http.py b/src/backend/tests/features/workbench/test_workbench_approval_lease_http.py similarity index 95% rename from src/backend/tests/test_workbench_approval_lease_http.py rename to src/backend/tests/features/workbench/test_workbench_approval_lease_http.py index 0e08e9f8..588f2833 100644 --- a/src/backend/tests/test_workbench_approval_lease_http.py +++ b/src/backend/tests/features/workbench/test_workbench_approval_lease_http.py @@ -6,8 +6,8 @@ from core.config.settings import settings from core.db.engine import SessionLocal from core.db.models import ContentBlock -from core.applications.capabilities import record_id -from tests.test_workbench_application_config_http import HOST,boundary,package +from features.workbench.capabilities import record_id +from tests.features.workbench.test_workbench_application_config_http import HOST, boundary, package A="lease_instance_alpha_123";B="lease_instance_beta_123" def installed(client,payload=None): @@ -92,7 +92,7 @@ def test_retained_old_revision_requires_the_same_live_lease(boundary): @pytest.mark.parametrize("field",["userId","applicationKey","revision","instanceId","installId","bindingToken","expiresAt"]) def test_current_lease_full_identity_and_expiration_are_authoritative(boundary,field): - from core.applications.plugin_revisions import lease_id + from features.workbench.plugin_revisions import lease_id clients,_,_=boundary;client=clients["owner"];item,root,body=installed(client) token=approval(client,root,body) with SessionLocal() as db: @@ -105,8 +105,8 @@ def test_current_lease_full_identity_and_expiration_are_authoritative(boundary,f def test_inflight_owned_http_query_checks_closed_instance_before_return(boundary,monkeypatch): import threading,asyncio from concurrent.futures import ThreadPoolExecutor - from tests.test_workbench_application_config_http import configure - from api.routes.v1 import workbench_capabilities as route + from tests.features.workbench.test_workbench_application_config_http import configure + from features.workbench.api import workbench_capabilities as route clients,upstream,calls=boundary;client=clients["owner"];item,root,body=installed(client) assert configure(clients["admin"],item,{"endpoint":upstream,"token":"owned-test-secret"}).status_code==200 token=approval(client,root,body,["data.query","data_source:records"]) @@ -132,7 +132,7 @@ def test_private_file_upload_download_bind_instance_and_poststorage_close_denies import io,json,zipfile,threading,os from pathlib import Path from concurrent.futures import ThreadPoolExecutor - from core.storage import get_storage + from core.files.storage import get_storage clients,_,_=boundary;client=clients["owner"] original=zipfile.ZipFile(io.BytesIO(package(name="lease-files"))) manifest=json.loads(original.read("plugin.json")) @@ -181,7 +181,7 @@ def controlled_download(*args,**kwargs): assert not list((Path(os.environ["STORAGE_PATH"])/".workbench-resources").iterdir()) def test_empty_json_get_configuration_and_query_only_delete_are_legal(boundary): - from tests.test_workbench_application_config_http import ROOT + from tests.features.workbench.test_workbench_application_config_http import ROOT clients,_,_=boundary;client=clients["owner"];item,root,body=installed(client) token=approval(client,root,body) headers={**HOST,"Content-Type":"application/json"} @@ -194,7 +194,7 @@ def test_empty_json_get_configuration_and_query_only_delete_are_legal(boundary): assert check(client,root,body,token).status_code==403 def test_json_budget_origin_and_required_write_body_remain_enforced(boundary): - from tests.test_workbench_application_config_http import ROOT + from tests.features.workbench.test_workbench_application_config_http import ROOT clients,_,_=boundary;client=clients["owner"];_,root,body=installed(client) headers={**HOST,"Content-Type":"application/json"} response=client.post(root+"/capabilities/approval",headers=headers) diff --git a/src/backend/tests/test_workbench_auth_routes.py b/src/backend/tests/features/workbench/test_workbench_auth_routes.py similarity index 98% rename from src/backend/tests/test_workbench_auth_routes.py rename to src/backend/tests/features/workbench/test_workbench_auth_routes.py index 7a21045d..6909645e 100644 --- a/src/backend/tests/test_workbench_auth_routes.py +++ b/src/backend/tests/features/workbench/test_workbench_auth_routes.py @@ -2,7 +2,7 @@ from urllib.parse import parse_qs, urlsplit -from tests.test_workbench_external_routes import AUTH, BASE, client, create # noqa: F401 +from tests.features.workbench.test_workbench_external_routes import AUTH, BASE, client, create # noqa: F401 KEY = "external:wb_reports" CONFIG = BASE + "/admin/external-auth/" + KEY @@ -124,7 +124,7 @@ def test_unauthenticated_target_cannot_consume_ticket(client): def test_default_thirty_second_expiry(client, monkeypatch): - from core.applications import external_auth + from features.workbench import external_auth assert create(client).status_code == 200 assert configure(client).status_code == 200 @@ -386,7 +386,7 @@ def test_approved_distinct_frontend_origin_works_but_wildcard_does_not(client, m from dataclasses import replace from types import SimpleNamespace - from api.routes.v1 import workbench_auth + from features.workbench.api import workbench_auth original = workbench_auth.settings monkeypatch.setattr( @@ -425,7 +425,7 @@ def test_external_target_cannot_receive_host_session_cookie(client, monkeypatch, from dataclasses import replace from types import SimpleNamespace - from api.routes.v1 import workbench + from features.workbench.api import workbench original = workbench.settings monkeypatch.setattr( diff --git a/src/backend/tests/test_workbench_background_subpage_http.py b/src/backend/tests/features/workbench/test_workbench_background_subpage_http.py similarity index 97% rename from src/backend/tests/test_workbench_background_subpage_http.py rename to src/backend/tests/features/workbench/test_workbench_background_subpage_http.py index b69196b3..2e8382f2 100644 --- a/src/backend/tests/test_workbench_background_subpage_http.py +++ b/src/backend/tests/features/workbench/test_workbench_background_subpage_http.py @@ -20,14 +20,14 @@ from core.db.engine import Base, get_db from core.db.models import UserShadow from core.db.models.plugin_resource import PluginResource -from core.plugins.resources.crypto import encrypt_secret +from core.capabilities.plugins.resources.crypto import encrypt_secret @pytest.fixture() def boundary(monkeypatch): monkeypatch.setenv("PLUGIN_RESOURCE_SECRET_KEY", "isolated-subpage-test-key") from api.routes.v1.plugins import router as plugins - from api.routes.v1.workbench import router as workbench + from features.workbench.api.workbench import router as workbench engine = create_engine( "sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool @@ -58,7 +58,7 @@ def boundary(monkeypatch): def package_with_subpage(): - from tests.workbench_plugin_support import package + from tests.features.workbench.workbench_plugin_support import package original = zipfile.ZipFile(io.BytesIO(package())) output = io.BytesIO() diff --git a/src/backend/tests/test_workbench_candidate_storage.py b/src/backend/tests/features/workbench/test_workbench_candidate_storage.py similarity index 91% rename from src/backend/tests/test_workbench_candidate_storage.py rename to src/backend/tests/features/workbench/test_workbench_candidate_storage.py index 409f0c63..234d474a 100644 --- a/src/backend/tests/test_workbench_candidate_storage.py +++ b/src/backend/tests/features/workbench/test_workbench_candidate_storage.py @@ -3,9 +3,9 @@ import os from pathlib import Path import pytest -from tests.test_workbench_plugin_routes import client, upload, launched -from tests.test_workbench_upgrade_routes import import_upgrade, endpoint, decision -from core.plugins.packaging.runtime_assets import materialize +from tests.features.workbench.test_workbench_plugin_routes import client, upload, launched +from tests.features.workbench.test_workbench_upgrade_routes import import_upgrade, endpoint, decision +from core.capabilities.plugins.packaging.runtime_assets import materialize def test_cloud_candidate_replacement_cleanup_failure_quota_and_retry(client, db_session, tmp_path): @@ -45,7 +45,7 @@ def test_cloud_candidate_replacement_cleanup_failure_quota_and_retry(client, db_ def test_cloud_candidate_byte_budget_rejects_before_archive_retention(client, monkeypatch): - from core.applications import plugin_candidate_storage + from features.workbench import plugin_candidate_storage install = upload(client) old = launched(client) @@ -61,7 +61,7 @@ def test_cloud_candidate_byte_budget_rejects_before_archive_retention(client, mo def test_rejected_candidate_cannot_delete_another_users_current_package(client, db_session): from core.auth.backend import UserContext, get_current_user from core.db.models import UserShadow - from tests.test_workbench_upgrade_routes import upgraded + from tests.features.workbench.test_workbench_upgrade_routes import upgraded install = upload(client) old = launched(client) @@ -96,7 +96,7 @@ def test_rejected_candidate_cannot_delete_another_users_current_package(client, def test_owner_quota_and_reinstall_orphan_cleanup_are_bounded(client, monkeypatch): - from core.applications import plugin_candidate_storage + from features.workbench import plugin_candidate_storage install = upload(client) old = launched(client) diff --git a/src/backend/tests/test_workbench_capability_routes.py b/src/backend/tests/features/workbench/test_workbench_capability_routes.py similarity index 98% rename from src/backend/tests/test_workbench_capability_routes.py rename to src/backend/tests/features/workbench/test_workbench_capability_routes.py index a3fbaa88..b9f10b6f 100644 --- a/src/backend/tests/test_workbench_capability_routes.py +++ b/src/backend/tests/features/workbench/test_workbench_capability_routes.py @@ -4,8 +4,8 @@ import json import zipfile -from tests.test_workbench_plugin_routes import client, launched -from tests.workbench_plugin_support import package +from tests.features.workbench.test_workbench_plugin_routes import client, launched +from tests.features.workbench.workbench_plugin_support import package def capability_package(): @@ -270,7 +270,7 @@ def test_data_query_approved_source_and_inflight_revocation(client, db_session, from core.db.engine import get_db from sqlalchemy.orm import sessionmaker - from tests.test_workbench_external_routes import AUTH + from tests.features.workbench.test_workbench_external_routes import AUTH entered, finish = threading.Event(), threading.Event() payload = b'{"rows":[{"id":7,"name":"Isolated fixture"}]}' diff --git a/src/backend/tests/test_workbench_closing_isolation_http.py b/src/backend/tests/features/workbench/test_workbench_closing_isolation_http.py similarity index 94% rename from src/backend/tests/test_workbench_closing_isolation_http.py rename to src/backend/tests/features/workbench/test_workbench_closing_isolation_http.py index 17fd5b2c..b274ae61 100644 --- a/src/backend/tests/test_workbench_closing_isolation_http.py +++ b/src/backend/tests/features/workbench/test_workbench_closing_isolation_http.py @@ -9,18 +9,14 @@ import time from fastapi.testclient import TestClient -from core.applications import activity -from core.applications.revocation_sweep import reconcile_owner +from features.workbench import activity +from features.workbench.revocation_sweep import reconcile_owner from core.db.engine import SessionLocal from core.db.models import ContentBlock from core.db.models.plugin_resource import PluginResource -from core.plugins.resources import service, store -from core.plugins.resources.crypto import encrypt_secret -from tests.test_workbench_revocation_failure_http import ( - boundary, - background_package, - USER, -) # noqa: F401 +from core.capabilities.plugins.resources import service, store +from core.capabilities.plugins.resources.crypto import encrypt_secret +from tests.features.workbench.test_workbench_revocation_failure_http import boundary, background_package, USER # noqa: F401 INSTANCE_A = "closing-instance-0001" INSTANCE_B = "healthy-instance-0002" diff --git a/src/backend/tests/test_workbench_cookie_boundaries.py b/src/backend/tests/features/workbench/test_workbench_cookie_boundaries.py similarity index 94% rename from src/backend/tests/test_workbench_cookie_boundaries.py rename to src/backend/tests/features/workbench/test_workbench_cookie_boundaries.py index cddb2ddf..4509ae6d 100644 --- a/src/backend/tests/test_workbench_cookie_boundaries.py +++ b/src/backend/tests/features/workbench/test_workbench_cookie_boundaries.py @@ -1,7 +1,7 @@ """Browser host spellings cannot bypass external cookie boundaries.""" import pytest -from tests.test_workbench_external_routes import APP, AUTH, BASE, client, create # noqa: F401 +from tests.features.workbench.test_workbench_external_routes import APP, AUTH, BASE, client, create # noqa: F401 def legacy(client, url): diff --git a/src/backend/tests/test_workbench_coordination.py b/src/backend/tests/features/workbench/test_workbench_coordination.py similarity index 97% rename from src/backend/tests/test_workbench_coordination.py rename to src/backend/tests/features/workbench/test_workbench_coordination.py index 37de1149..9202b066 100644 --- a/src/backend/tests/test_workbench_coordination.py +++ b/src/backend/tests/features/workbench/test_workbench_coordination.py @@ -6,7 +6,7 @@ import pytest from sqlalchemy.orm import sessionmaker -from core.applications.coordination import configuration, runtime +from features.workbench.coordination import configuration, runtime def test_runtime_requires_explicit_owner(db_session): diff --git a/src/backend/tests/test_workbench_diagnostic_failure_http.py b/src/backend/tests/features/workbench/test_workbench_diagnostic_failure_http.py similarity index 97% rename from src/backend/tests/test_workbench_diagnostic_failure_http.py rename to src/backend/tests/features/workbench/test_workbench_diagnostic_failure_http.py index c13385a9..44116665 100644 --- a/src/backend/tests/test_workbench_diagnostic_failure_http.py +++ b/src/backend/tests/features/workbench/test_workbench_diagnostic_failure_http.py @@ -22,8 +22,8 @@ @pytest.fixture() def client(db_session, monkeypatch): - from api.routes.v1.workbench import router - from core.applications import diagnostics + from features.workbench.api.workbench import router + from features.workbench import diagnostics assert settings.auth.mode == "session" assert settings.session.store_type == "memory" diff --git a/src/backend/tests/test_workbench_diagnostics.py b/src/backend/tests/features/workbench/test_workbench_diagnostics.py similarity index 98% rename from src/backend/tests/test_workbench_diagnostics.py rename to src/backend/tests/features/workbench/test_workbench_diagnostics.py index 0aaa1cec..04dcedeb 100644 --- a/src/backend/tests/test_workbench_diagnostics.py +++ b/src/backend/tests/features/workbench/test_workbench_diagnostics.py @@ -10,7 +10,7 @@ @pytest.fixture() def client(db_session, monkeypatch): - from api.routes.v1.workbench import router + from features.workbench.api.workbench import router monkeypatch.setenv("CONFIG_TOKEN", "wb19-config") db_session.add(UserShadow(user_id="wb19-reader", username="Reader", extra_data={})) @@ -184,7 +184,7 @@ def network(monkeypatch, addresses, peer="93.184.216.34", status=200): import io import socket from types import SimpleNamespace - from core.applications import diagnostic_probe + from features.workbench import diagnostic_probe connected = [] diff --git a/src/backend/tests/test_workbench_embed_routes.py b/src/backend/tests/features/workbench/test_workbench_embed_routes.py similarity index 96% rename from src/backend/tests/test_workbench_embed_routes.py rename to src/backend/tests/features/workbench/test_workbench_embed_routes.py index 4d917b32..65eca8f1 100644 --- a/src/backend/tests/test_workbench_embed_routes.py +++ b/src/backend/tests/features/workbench/test_workbench_embed_routes.py @@ -4,7 +4,7 @@ import pytest from core.db.models import UserShadow -from tests.test_workbench_external_routes import APP, AUTH, ADMIN, BASE, client # noqa: F401 +from tests.features.workbench.test_workbench_external_routes import APP, AUTH, ADMIN, BASE, client # noqa: F401 EMBED = { "origin": "https://reports.example.org", diff --git a/src/backend/tests/test_workbench_external_routes.py b/src/backend/tests/features/workbench/test_workbench_external_routes.py similarity index 100% rename from src/backend/tests/test_workbench_external_routes.py rename to src/backend/tests/features/workbench/test_workbench_external_routes.py diff --git a/src/backend/tests/test_workbench_invalidation.py b/src/backend/tests/features/workbench/test_workbench_invalidation.py similarity index 94% rename from src/backend/tests/test_workbench_invalidation.py rename to src/backend/tests/features/workbench/test_workbench_invalidation.py index 4a569274..c740ef26 100644 --- a/src/backend/tests/test_workbench_invalidation.py +++ b/src/backend/tests/features/workbench/test_workbench_invalidation.py @@ -15,13 +15,13 @@ @pytest.fixture() def client(db_session, monkeypatch): from api.deps import require_config - from api.routes.v1 import workbench_events - from api.routes.v1.workbench import router + from features.workbench.api import workbench_events + from features.workbench.api.workbench import router monkeypatch.setattr(session, "_use_memory_store", lambda: True) monkeypatch.setattr(backend, "_auth_mode", lambda: "session") monkeypatch.setattr(workbench_events, "SessionLocal", sessionmaker(bind=db_session.get_bind())) - session._MEMORY_SESSIONS.clear() + session.process_state().MEMORY_SESSIONS.clear() for user in ["revision-alice", "revision-bob"]: db_session.add(UserShadow(user_id=user, user_center_id=user, username=user, extra_data={})) db_session.commit() @@ -37,7 +37,7 @@ def client(db_session, monkeypatch): with TestClient(app) as http: http.cookies.set(settings.session.cookie_name, tokens["revision-alice"]) yield http, tokens - session._MEMORY_SESSIONS.clear() + session.process_state().MEMORY_SESSIONS.clear() def test_directory_validator_is_private_and_identity_bound(client): @@ -101,7 +101,7 @@ def test_revoked_cookie_cannot_resume_notifications(client): def test_notifications_timeout_and_reject_unbounded_revision(client, monkeypatch): - from api.routes.v1 import workbench_events + from features.workbench.api import workbench_events http, _ = client revision = http.get("/v1/workbench/catalog").json()["data"]["revision"] @@ -153,8 +153,8 @@ def observe(connection, cursor, statement, parameters, context, executemany): def test_revocation_sweep_discovers_owners_without_live_workers(client): - from core.applications import activity - from core.applications.revocation_sweep import owners + from features.workbench import activity + from features.workbench.revocation_sweep import owners from core.db.engine import SessionLocal http, _ = client diff --git a/src/backend/tests/test_workbench_migration.py b/src/backend/tests/features/workbench/test_workbench_migration.py similarity index 98% rename from src/backend/tests/test_workbench_migration.py rename to src/backend/tests/features/workbench/test_workbench_migration.py index 39f73a25..11abbb7f 100644 --- a/src/backend/tests/test_workbench_migration.py +++ b/src/backend/tests/features/workbench/test_workbench_migration.py @@ -4,8 +4,8 @@ import stat import pytest -from core.applications.migration import apply, inventory, rollback -from core.applications.migration_snapshot import read_snapshot +from features.workbench.migration import apply, inventory, rollback +from features.workbench.migration_snapshot import read_snapshot from core.auth.backend import UserContext, get_current_user from core.db.engine import get_db from core.db.models import ContentBlock, UserShadow @@ -65,7 +65,7 @@ def seeded(db_session): def http(db): from api.routes.v1.content import router as content_router - from api.routes.v1.workbench import router + from features.workbench.api.workbench import router app = FastAPI() app.include_router(router) @@ -287,7 +287,7 @@ def test_auth_binding_and_same_service_origin_survive_restore(seeded, tmp_path): def test_clone_rotates_origin_without_remapping_plugin_permission(seeded, tmp_path): - from core.applications.migration import clone_origin + from features.workbench.migration import clone_origin seeded.add(ContentBlock(id="workbench_origin_identity", payload={"id": "a" * 32})) seeded.get(UserShadow, "bob").extra_data = {"allowed_apps": ["plugin:old:exact:key"]} diff --git a/src/backend/tests/test_workbench_migration_auth_gate.py b/src/backend/tests/features/workbench/test_workbench_migration_auth_gate.py similarity index 95% rename from src/backend/tests/test_workbench_migration_auth_gate.py rename to src/backend/tests/features/workbench/test_workbench_migration_auth_gate.py index aded522c..d1bccaf2 100644 --- a/src/backend/tests/test_workbench_migration_auth_gate.py +++ b/src/backend/tests/features/workbench/test_workbench_migration_auth_gate.py @@ -2,10 +2,10 @@ from copy import deepcopy import pytest -from core.applications.migration import apply, inventory, rollback -from core.applications.migration_snapshot import RECEIPT, state +from features.workbench.migration import apply, inventory, rollback +from features.workbench.migration_snapshot import RECEIPT, state from core.db.models import ContentBlock, UserShadow -from tests.test_workbench_migration import http +from tests.features.workbench.test_workbench_migration import http KEY = "legacy:old_portal" AUTH_KEY = "workbench_external_auth:" + KEY diff --git a/src/backend/tests/test_workbench_observer_paths.py b/src/backend/tests/features/workbench/test_workbench_observer_paths.py similarity index 93% rename from src/backend/tests/test_workbench_observer_paths.py rename to src/backend/tests/features/workbench/test_workbench_observer_paths.py index 7acd6982..bfc2b763 100644 --- a/src/backend/tests/test_workbench_observer_paths.py +++ b/src/backend/tests/features/workbench/test_workbench_observer_paths.py @@ -1,7 +1,7 @@ """Isolation must be actual containment, never a substring convention.""" import pytest -from tests.workbench_observer_paths import ROOT, guard_cli_binding, guard_environment, isolated_path +from tests.features.workbench.workbench_observer_paths import ROOT, guard_cli_binding, guard_environment, isolated_path def test_observer_rejects_outside_paths_parent_traversal_and_symlinks(monkeypatch, tmp_path): diff --git a/src/backend/tests/test_workbench_official_capabilities_http.py b/src/backend/tests/features/workbench/test_workbench_official_capabilities_http.py similarity index 93% rename from src/backend/tests/test_workbench_official_capabilities_http.py rename to src/backend/tests/features/workbench/test_workbench_official_capabilities_http.py index 523421d1..42bba085 100644 --- a/src/backend/tests/test_workbench_official_capabilities_http.py +++ b/src/backend/tests/features/workbench/test_workbench_official_capabilities_http.py @@ -5,11 +5,11 @@ import pytest -from tests.test_workbench_revocation_failure_http import boundary +from tests.features.workbench.test_workbench_revocation_failure_http import boundary def capability_package(capability): - from tests.workbench_plugin_support import package + from tests.features.workbench.workbench_plugin_support import package output = io.BytesIO() with zipfile.ZipFile(io.BytesIO(package())) as original: diff --git a/src/backend/tests/test_workbench_origins.py b/src/backend/tests/features/workbench/test_workbench_origins.py similarity index 95% rename from src/backend/tests/test_workbench_origins.py rename to src/backend/tests/features/workbench/test_workbench_origins.py index 464a83d2..7e8cf7c4 100644 --- a/src/backend/tests/test_workbench_origins.py +++ b/src/backend/tests/features/workbench/test_workbench_origins.py @@ -1,6 +1,6 @@ """WB16 public HTTP source and exact-install boundary.""" -from tests.test_workbench_plugin_routes import client, upload, launched, plugin_items +from tests.features.workbench.test_workbench_plugin_routes import client, upload, launched, plugin_items from core.db.models import ContentBlock @@ -83,7 +83,7 @@ def test_cloud_preferences_reject_device_only_reference(client): def test_missing_packaged_sdk_asset_is_explicit(client): - from tests.workbench_plugin_support import package + from tests.features.workbench.workbench_plugin_support import package response = client.post( "/v1/plugins/import", @@ -103,7 +103,7 @@ def test_missing_packaged_sdk_asset_is_explicit(client): def test_complete_package_css_script_and_images_need_no_network(client): - from tests.workbench_plugin_support import package + from tests.features.workbench.workbench_plugin_support import package response = client.post( "/v1/plugins/import", @@ -137,7 +137,7 @@ def test_device_registry_install_launch_is_exact_and_owned( ): from sqlalchemy.orm import sessionmaker from core.config import local_mode - from core.capabilities import registry + from features.desktop.capabilities import registry monkeypatch.setenv("HUGAGENT_CAPS_ROOT", str(tmp_path / "caps")) monkeypatch.setattr(registry, "SessionLocal", sessionmaker(bind=db_session.get_bind())) @@ -183,7 +183,7 @@ def test_declared_execution_scope_cannot_launch_on_another_source(client): import io import json import zipfile - from tests.workbench_plugin_support import package + from tests.features.workbench.workbench_plugin_support import package original = zipfile.ZipFile(io.BytesIO(package())) output = io.BytesIO() diff --git a/src/backend/tests/test_workbench_plugin_routes.py b/src/backend/tests/features/workbench/test_workbench_plugin_routes.py similarity index 98% rename from src/backend/tests/test_workbench_plugin_routes.py rename to src/backend/tests/features/workbench/test_workbench_plugin_routes.py index 506f2e3d..8c84f33c 100644 --- a/src/backend/tests/test_workbench_plugin_routes.py +++ b/src/backend/tests/features/workbench/test_workbench_plugin_routes.py @@ -6,13 +6,13 @@ from core.auth.backend import UserContext, get_current_user from core.db.engine import get_db from core.db.models import UserShadow -from tests.workbench_plugin_support import package +from tests.features.workbench.workbench_plugin_support import package @pytest.fixture def client(db_session, monkeypatch, tmp_path): monkeypatch.setenv("PLUGIN_ASSET_CACHE_DIR", str(tmp_path / "plugin-assets")) - from api.routes.v1.workbench import router + from features.workbench.api.workbench import router from api.routes.v1.plugins import router as plugins app = FastAPI() diff --git a/src/backend/tests/test_workbench_preferences_routes.py b/src/backend/tests/features/workbench/test_workbench_preferences_routes.py similarity index 99% rename from src/backend/tests/test_workbench_preferences_routes.py rename to src/backend/tests/features/workbench/test_workbench_preferences_routes.py index 105741d2..1da0f7f1 100644 --- a/src/backend/tests/test_workbench_preferences_routes.py +++ b/src/backend/tests/features/workbench/test_workbench_preferences_routes.py @@ -11,7 +11,7 @@ @pytest.fixture() def client(db_session): - from api.routes.v1.workbench import router + from features.workbench.api.workbench import router app = FastAPI() app.include_router(router) diff --git a/src/backend/tests/test_workbench_protocol.py b/src/backend/tests/features/workbench/test_workbench_protocol.py similarity index 95% rename from src/backend/tests/test_workbench_protocol.py rename to src/backend/tests/features/workbench/test_workbench_protocol.py index 16340ade..f5833886 100644 --- a/src/backend/tests/test_workbench_protocol.py +++ b/src/backend/tests/features/workbench/test_workbench_protocol.py @@ -1,6 +1,6 @@ """The plugin installation boundary must validate applications without breaking old UI.""" -from core.plugins.ui.contract import normalize_ui, public_contributions +from features.workbench.ui.contract import normalize_ui, public_contributions def test_workspace_application_survives_installation_and_public_projection(): @@ -32,9 +32,9 @@ def test_workspace_application_survives_installation_and_public_projection(): from pathlib import Path import pytest -from core.applications.protocol import validate_application, validate_contributions +from features.workbench.protocol import validate_application, validate_contributions -_ROOT = Path(__file__).resolve().parents[3] +_ROOT = Path(__file__).resolve().parents[5] _PACKAGE = _ROOT / "packages/workbench-protocol" _APP_CASES = json.loads((_PACKAGE / "fixtures/applications.json").read_text()) _CONTRIBUTION_CASES = json.loads((_PACKAGE / "fixtures/contributions.json").read_text()) @@ -76,7 +76,7 @@ def test_javascript_and_python_give_the_same_fixture_verdicts(): def test_backend_schema_artifacts_equal_the_independent_package(): for name in ("application.schema.json", "module.schema.json", "host.json"): assert (_PACKAGE / "schemas" / name).read_bytes() == ( - _ROOT / "src/backend/core/applications/schemas" / name + _ROOT / "src/backend/features/workbench/schemas" / name ).read_bytes() diff --git a/src/backend/tests/test_workbench_provider_scope_http.py b/src/backend/tests/features/workbench/test_workbench_provider_scope_http.py similarity index 93% rename from src/backend/tests/test_workbench_provider_scope_http.py rename to src/backend/tests/features/workbench/test_workbench_provider_scope_http.py index 1cc10ef8..77f4c5a3 100644 --- a/src/backend/tests/test_workbench_provider_scope_http.py +++ b/src/backend/tests/features/workbench/test_workbench_provider_scope_http.py @@ -5,9 +5,9 @@ import types import zipfile -from core.plugins.ui.data_proxy import call_data_source +from features.workbench.ui.data_proxy import call_data_source from extensions.mcp_servers.ai_chain_information_mcp import ui_data -from tests.test_workbench_application_config_http import HOST, boundary, install, package +from tests.features.workbench.test_workbench_application_config_http import HOST, boundary, install, package def provider_package(): diff --git a/src/backend/tests/test_workbench_resource_owner_migrations.py b/src/backend/tests/features/workbench/test_workbench_resource_owner_migrations.py similarity index 99% rename from src/backend/tests/test_workbench_resource_owner_migrations.py rename to src/backend/tests/features/workbench/test_workbench_resource_owner_migrations.py index 643fd9bd..7a6b00cd 100644 --- a/src/backend/tests/test_workbench_resource_owner_migrations.py +++ b/src/backend/tests/features/workbench/test_workbench_resource_owner_migrations.py @@ -9,7 +9,7 @@ from alembic.operations import Operations from sqlalchemy.exc import IntegrityError -SOURCE = Path(os.environ.get("WORKBENCH_OWNER_SOURCE_ROOT", Path(__file__).resolve().parents[3])) +SOURCE = Path(os.environ.get("WORKBENCH_OWNER_SOURCE_ROOT", Path(__file__).resolve().parents[5])) def migration(): diff --git a/src/backend/tests/test_workbench_resources.py b/src/backend/tests/features/workbench/test_workbench_resources.py similarity index 89% rename from src/backend/tests/test_workbench_resources.py rename to src/backend/tests/features/workbench/test_workbench_resources.py index ab31cb85..0bd47ba6 100644 --- a/src/backend/tests/test_workbench_resources.py +++ b/src/backend/tests/features/workbench/test_workbench_resources.py @@ -10,8 +10,9 @@ PluginResourceAssetTicket, PluginResourceTicket, ) -from core.plugins.resources import application, service, store -from core.plugins.resources.crypto import encrypt_secret +from features.hosted_apps.resources import admission as application +from core.capabilities.plugins.resources import service, store +from core.capabilities.plugins.resources.crypto import encrypt_secret from fastapi import HTTPException from sqlalchemy import create_engine from sqlalchemy.exc import IntegrityError @@ -60,7 +61,7 @@ def record(**changes): def authorize(monkeypatch): # Import the exact resolver before patching legacy resolution to avoid cached fake imports. - from core.applications import catalog, installation_binding + from features.workbench import catalog, installation_binding monkeypatch.setattr( catalog, @@ -77,7 +78,7 @@ def authorize(monkeypatch): lambda *args: SimpleNamespace(revision="revision", install_id="install", slug="fixture"), ) monkeypatch.setattr(installation_binding, "resolve", lambda *args: service.installation.resolve(*args)) - from core.plugins.resources import owners + from core.capabilities.plugins.resources import owners monkeypatch.setattr(owners, "execution_scope", lambda: "cloud") @@ -137,7 +138,7 @@ async def test_close_is_idempotent_revokes_tickets_and_prevents_lease(resource_d async def test_janitor_cleans_expired_application_and_continues_after_failure( resource_db, monkeypatch ): - from core.plugins.resources import lifecycle + from core.capabilities.plugins.resources import lifecycle first, second = record(resource_id="a", expires_at=0), record(resource_id="b", expires_at=0) resource_db.add_all([first, second]) @@ -275,7 +276,7 @@ async def test_revocation_blocks_reads_but_owner_can_still_release(resource_db, row = record() resource_db.add(row) resource_db.commit() - from core.applications import catalog + from features.workbench import catalog def revoked(*args): raise HTTPException(404, "app_not_found") @@ -292,8 +293,9 @@ def revoked(*args): async def test_authenticated_http_lease_close_and_instance_isolation(resource_db, monkeypatch): import httpx - from api.routes.v1.workbench import require_json_write - from api.routes.v1.workbench_resources import routes + from api import deps # Compose the host before testing its feature routes. + from features.workbench.api.workbench import require_json_write + from features.workbench.api.workbench_resources import routes from core.auth.backend import UserContext, get_current_user from core.db.engine import get_db from fastapi import FastAPI @@ -309,11 +311,22 @@ async def test_authenticated_http_lease_close_and_instance_isolation(resource_db app.dependency_overrides[get_current_user] = lambda: UserContext( user_id="owner", user_center_id="owner", username="Owner", email="" ) + from core.capabilities.plugins.resources import owners + monkeypatch.setattr(owners, "resolve_installation", lambda *args: SimpleNamespace( + ui={"contributes": {"modules": [{"id": "page", "grants": ["resource.command"]}]}} + )) + dispatched = AsyncMock(return_value={"accepted": True}) + monkeypatch.setattr(service, "command", dispatched) prefix = "/v1/workbench/applications/plugin:key/instances/instance/resources" async with httpx.AsyncClient( transport=httpx.ASGITransport(app=app), base_url="http://test" ) as client: assert (await client.post(prefix + "/r/lease", json={})).status_code == 200 + command = {"id": "cmd", "action": "refresh", "params": {}} + assert (await client.post(prefix + "/r/command", json=command)).status_code == 200 + dispatched.assert_awaited_once() + assert (await client.post(prefix.replace("/instance/", "/other/") + "/r/command", + json=command)).status_code == 404 assert ( await client.post(prefix.replace("/instance/", "/other/") + "/r/lease", json={}) ).status_code == 404 @@ -335,7 +348,7 @@ def test_server_migration_is_reentrant_and_refuses_lossy_downgrade(): from alembic.operations import Operations from sqlalchemy import inspect, text - path = Path(__file__).parents[1] / "alembic/versions/workbenchresources01_application_owner.py" + path = Path(__file__).parents[3] / "alembic/versions/workbenchresources01_application_owner.py" spec = importlib.util.spec_from_file_location("wb11_migration", path) migration = importlib.util.module_from_spec(spec) spec.loader.exec_module(migration) diff --git a/src/backend/tests/test_workbench_revocation_failure_http.py b/src/backend/tests/features/workbench/test_workbench_revocation_failure_http.py similarity index 91% rename from src/backend/tests/test_workbench_revocation_failure_http.py rename to src/backend/tests/features/workbench/test_workbench_revocation_failure_http.py index ec100b06..a8848a5d 100644 --- a/src/backend/tests/test_workbench_revocation_failure_http.py +++ b/src/backend/tests/features/workbench/test_workbench_revocation_failure_http.py @@ -17,8 +17,8 @@ PluginResourceAssetTicket, PluginResourceTicket, ) -from core.plugins.resources import service, store -from core.plugins.resources.crypto import encrypt_secret +from core.capabilities.plugins.resources import service, store +from core.capabilities.plugins.resources.crypto import encrypt_secret from fastapi.testclient import TestClient USER = "wb-user" @@ -26,7 +26,7 @@ @pytest.fixture() def boundary(monkeypatch): - from tests.test_workbench_external_routes import AUTH + from tests.features.workbench.test_workbench_external_routes import AUTH monkeypatch.setenv("CONFIG_TOKEN", AUTH["Authorization"].removeprefix("Bearer ")) monkeypatch.setenv("PLUGIN_RESOURCE_SECRET_KEY", "wb-revocation-isolated-secret-key") from core.sandbox import factory @@ -50,7 +50,7 @@ async def set_process_deadline(self, process_id, *, sandbox_session_id, assert settings.auth.mode == "session" assert session._use_memory_store() - session._MEMORY_SESSIONS.clear() + session.process_state().MEMORY_SESSIONS.clear() Base.metadata.create_all(engine) with SessionLocal() as db: db.add( @@ -67,12 +67,12 @@ async def set_process_deadline(self, process_id, *, sandbox_session_id, client.cookies.set(settings.session.cookie_name, token) yield client, token client.close() - session._MEMORY_SESSIONS.clear() + session.process_state().MEMORY_SESSIONS.clear() Base.metadata.drop_all(engine) def background_package(): - from tests.workbench_plugin_support import package + from tests.features.workbench.workbench_plugin_support import package output = io.BytesIO() with zipfile.ZipFile(io.BytesIO(package())) as original: @@ -90,7 +90,7 @@ def background_package(): def setup_workers(client): - from core.applications import activity + from features.workbench import activity response = client.post( "/v1/plugins/import", @@ -159,8 +159,8 @@ def setup_workers(client): def credentials_revoked(): - from core.applications import activity - from core.applications.revocation import EXTERNAL_PREFIXES + from features.workbench import activity + from features.workbench.revocation import EXTERNAL_PREFIXES with SessionLocal() as db: value = activity.read(db, USER) @@ -180,9 +180,9 @@ def credentials_revoked(): @pytest.mark.parametrize("trigger", ["logout", "unavailable"]) def test_revocation_commits_before_failed_close_and_continues(boundary, monkeypatch, trigger): - from core.applications import activity - from tests.test_workbench_auth_routes import configure, launch, redeem - from tests.test_workbench_external_routes import APP, AUTH, BASE, create + from features.workbench import activity + from tests.features.workbench.test_workbench_auth_routes import configure, launch, redeem + from tests.features.workbench.test_workbench_external_routes import APP, AUTH, BASE, create client, old_token = boundary item, prefix, tickets = setup_workers(client) @@ -226,7 +226,7 @@ async def shutdown(row, path, body=None, **kwargs): ).status_code == 200 ) - from core.applications.revocation_sweep import reconcile_owner + from features.workbench.revocation_sweep import reconcile_owner asyncio.run(reconcile_owner(USER)) assert calls == ["worker-a", "worker-b"] @@ -256,7 +256,7 @@ async def shutdown(row, path, body=None, **kwargs): assert client.get("/v1/workbench/me/activity").json()["data"]["background"] == [] # A subsequent real janitor sweep can retry the retained exact failed handle. fail_first[0] = False - from core.plugins.resources import lifecycle + from core.capabilities.plugins.resources import lifecycle with SessionLocal() as db: asyncio.run(lifecycle.reconcile(db, USER)) diff --git a/src/backend/tests/test_workbench_routing_routes.py b/src/backend/tests/features/workbench/test_workbench_routing_routes.py similarity index 94% rename from src/backend/tests/test_workbench_routing_routes.py rename to src/backend/tests/features/workbench/test_workbench_routing_routes.py index 93365aa7..0b6939fe 100644 --- a/src/backend/tests/test_workbench_routing_routes.py +++ b/src/backend/tests/features/workbench/test_workbench_routing_routes.py @@ -4,8 +4,8 @@ import zipfile from urllib.parse import quote from core.auth.backend import UserContext, get_current_user -from tests.test_workbench_plugin_routes import client, upload, plugin_items -from tests.workbench_plugin_support import package +from tests.features.workbench.test_workbench_plugin_routes import client, upload, plugin_items +from tests.features.workbench.workbench_plugin_support import package def test_disabled_deep_link_retains_safe_reason_without_changing_directory(client): installed = upload(client) diff --git a/src/backend/tests/test_workbench_state_binding.py b/src/backend/tests/features/workbench/test_workbench_state_binding.py similarity index 94% rename from src/backend/tests/test_workbench_state_binding.py rename to src/backend/tests/features/workbench/test_workbench_state_binding.py index 1cc272aa..d0845a1a 100644 --- a/src/backend/tests/test_workbench_state_binding.py +++ b/src/backend/tests/features/workbench/test_workbench_state_binding.py @@ -2,7 +2,7 @@ from urllib.parse import urlencode from core.auth.backend import UserContext, get_current_user -from tests.test_workbench_plugin_routes import client as client, upload, launched +from tests.features.workbench.test_workbench_plugin_routes import client as client, upload, launched def test_installed_state_requires_exact_binding_and_stops_on_revoke(client): diff --git a/src/backend/tests/test_workbench_state_concurrency.py b/src/backend/tests/features/workbench/test_workbench_state_concurrency.py similarity index 93% rename from src/backend/tests/test_workbench_state_concurrency.py rename to src/backend/tests/features/workbench/test_workbench_state_concurrency.py index e3794114..10fce60d 100644 --- a/src/backend/tests/test_workbench_state_concurrency.py +++ b/src/backend/tests/features/workbench/test_workbench_state_concurrency.py @@ -4,7 +4,7 @@ from threading import Barrier from sqlalchemy.orm import sessionmaker from core.db.engine import get_db -from tests.test_workbench_state_routes import client as client, BASE, SCOPE, body +from tests.features.workbench.test_workbench_state_routes import client as client, BASE, SCOPE, body def test_concurrent_state_creation_has_one_cas_winner(client, db_session): diff --git a/src/backend/tests/test_workbench_state_routes.py b/src/backend/tests/features/workbench/test_workbench_state_routes.py similarity index 98% rename from src/backend/tests/test_workbench_state_routes.py rename to src/backend/tests/features/workbench/test_workbench_state_routes.py index 608b6173..beee19b0 100644 --- a/src/backend/tests/test_workbench_state_routes.py +++ b/src/backend/tests/features/workbench/test_workbench_state_routes.py @@ -16,7 +16,7 @@ def body(revision=0, **values): @pytest.fixture() def client(db_session): - from api.routes.v1.workbench import router + from features.workbench.api.workbench import router app = FastAPI() app.include_router(router) diff --git a/src/backend/tests/test_workbench_upgrade_routes.py b/src/backend/tests/features/workbench/test_workbench_upgrade_routes.py similarity index 97% rename from src/backend/tests/test_workbench_upgrade_routes.py rename to src/backend/tests/features/workbench/test_workbench_upgrade_routes.py index e2d91f4b..6b5ca347 100644 --- a/src/backend/tests/test_workbench_upgrade_routes.py +++ b/src/backend/tests/features/workbench/test_workbench_upgrade_routes.py @@ -4,8 +4,8 @@ import json import zipfile from urllib.parse import quote -from tests.test_workbench_plugin_routes import client, upload, plugin_items, launched -from tests.workbench_plugin_support import package +from tests.features.workbench.test_workbench_plugin_routes import client, upload, plugin_items, launched +from tests.features.workbench.workbench_plugin_support import package def upgraded(*, grants=None, permissions=None, state=None, required=None, html="New revision"): diff --git a/src/backend/tests/workbench_capabilities_http_fixture.py b/src/backend/tests/features/workbench/workbench_capabilities_http_fixture.py similarity index 96% rename from src/backend/tests/workbench_capabilities_http_fixture.py rename to src/backend/tests/features/workbench/workbench_capabilities_http_fixture.py index adace2ff..dbdee2f9 100644 --- a/src/backend/tests/workbench_capabilities_http_fixture.py +++ b/src/backend/tests/features/workbench/workbench_capabilities_http_fixture.py @@ -7,11 +7,11 @@ from api.routes.v1.artifacts import router as artifacts from api.routes.v1.plugins import router as plugins -from api.routes.v1.workbench import router as workbench +from features.workbench.api.workbench import router as workbench from core.db.engine import Base, SessionLocal, engine from core.db.models import SystemConfig, UserShadow from core.infra.exceptions import AppException -from core.services.artifact_service import store_bytes_as_artifact +from core.files.artifacts.service import store_bytes_as_artifact from fastapi import FastAPI, Request from fastapi.responses import FileResponse, JSONResponse, Response from fastapi.staticfiles import StaticFiles @@ -137,7 +137,7 @@ def setup(request: Request): mime_type="text/plain", ) other_id = other.artifact_id - from core.services.system_config import SystemConfigService + from core.config.store.system_config import SystemConfigService SystemConfigService.get_instance().invalidate_cache() return { diff --git a/src/backend/tests/workbench_diagnostic_http_fixture.py b/src/backend/tests/features/workbench/workbench_diagnostic_http_fixture.py similarity index 95% rename from src/backend/tests/workbench_diagnostic_http_fixture.py rename to src/backend/tests/features/workbench/workbench_diagnostic_http_fixture.py index eb429626..4427b717 100644 --- a/src/backend/tests/workbench_diagnostic_http_fixture.py +++ b/src/backend/tests/features/workbench/workbench_diagnostic_http_fixture.py @@ -4,7 +4,7 @@ from pathlib import Path from fastapi import FastAPI from fastapi.staticfiles import StaticFiles -from api.routes.v1.workbench import router +from features.workbench.api.workbench import router from core.auth.backend import UserContext, get_current_user from core.db.engine import Base, engine, SessionLocal from core.db.models import UserShadow diff --git a/src/backend/tests/workbench_embed_http_fixture.py b/src/backend/tests/features/workbench/workbench_embed_http_fixture.py similarity index 95% rename from src/backend/tests/workbench_embed_http_fixture.py rename to src/backend/tests/features/workbench/workbench_embed_http_fixture.py index c383bb32..38701e65 100644 --- a/src/backend/tests/workbench_embed_http_fixture.py +++ b/src/backend/tests/features/workbench/workbench_embed_http_fixture.py @@ -3,7 +3,7 @@ import os from pathlib import Path -from api.routes.v1.workbench import router +from features.workbench.api.workbench import router from core.db import models # noqa: F401 from core.db.engine import Base, engine from fastapi import FastAPI diff --git a/src/backend/tests/workbench_http_fixture.py b/src/backend/tests/features/workbench/workbench_http_fixture.py similarity index 95% rename from src/backend/tests/workbench_http_fixture.py rename to src/backend/tests/features/workbench/workbench_http_fixture.py index b692ccac..b94a37f8 100644 --- a/src/backend/tests/workbench_http_fixture.py +++ b/src/backend/tests/features/workbench/workbench_http_fixture.py @@ -8,7 +8,7 @@ from pathlib import Path from api.routes.v1.content import router as content_router -from api.routes.v1.workbench import router +from features.workbench.api.workbench import router from core.db import models # noqa: F401 from core.db.engine import Base, engine from fastapi import FastAPI, Response diff --git a/src/backend/tests/workbench_observed_runner_fixture.py b/src/backend/tests/features/workbench/workbench_observed_runner_fixture.py similarity index 90% rename from src/backend/tests/workbench_observed_runner_fixture.py rename to src/backend/tests/features/workbench/workbench_observed_runner_fixture.py index ccacf826..4ecfaf5a 100644 --- a/src/backend/tests/workbench_observed_runner_fixture.py +++ b/src/backend/tests/features/workbench/workbench_observed_runner_fixture.py @@ -1,11 +1,11 @@ """Test-only actual ScriptRunner process registry observation. -Start with uvicorn tests.workbench_observed_runner_fixture:app. +Start with uvicorn tests.features.workbench.workbench_observed_runner_fixture:app. This module is never registered by production server or desktop entrypoints. """ from pydantic import BaseModel, ConfigDict, Field -from tests.workbench_observer_paths import LoopbackOnly, guard_cli_binding, guard_environment +from tests.features.workbench.workbench_observer_paths import LoopbackOnly, guard_cli_binding, guard_environment guard_environment("SCRIPT_RUNNER_WORKSPACE") guard_cli_binding() diff --git a/src/backend/tests/workbench_observer_paths.py b/src/backend/tests/features/workbench/workbench_observer_paths.py similarity index 98% rename from src/backend/tests/workbench_observer_paths.py rename to src/backend/tests/features/workbench/workbench_observer_paths.py index fdcb32f0..76456499 100644 --- a/src/backend/tests/workbench_observer_paths.py +++ b/src/backend/tests/features/workbench/workbench_observer_paths.py @@ -7,7 +7,7 @@ from fastapi import HTTPException, Request -ROOT = Path(__file__).resolve().parents[3] +ROOT = Path(__file__).resolve().parents[5] SCRATCH = ROOT / ".scratch" diff --git a/src/backend/tests/workbench_overflow_http_fixture.py b/src/backend/tests/features/workbench/workbench_overflow_http_fixture.py similarity index 95% rename from src/backend/tests/workbench_overflow_http_fixture.py rename to src/backend/tests/features/workbench/workbench_overflow_http_fixture.py index 82a5ee44..eb7f42ce 100644 --- a/src/backend/tests/workbench_overflow_http_fixture.py +++ b/src/backend/tests/features/workbench/workbench_overflow_http_fixture.py @@ -2,7 +2,7 @@ import os from pathlib import Path -from api.routes.v1.workbench import router +from features.workbench.api.workbench import router from core.db.engine import Base, engine from core.db.models import UserShadow from fastapi import FastAPI, Response diff --git a/src/backend/tests/workbench_plugin_http_fixture.py b/src/backend/tests/features/workbench/workbench_plugin_http_fixture.py similarity index 96% rename from src/backend/tests/workbench_plugin_http_fixture.py rename to src/backend/tests/features/workbench/workbench_plugin_http_fixture.py index 4cf6ea5b..b14f4a6a 100644 --- a/src/backend/tests/workbench_plugin_http_fixture.py +++ b/src/backend/tests/features/workbench/workbench_plugin_http_fixture.py @@ -5,7 +5,7 @@ from fastapi import FastAPI, Request from fastapi.responses import JSONResponse from fastapi.staticfiles import StaticFiles -from api.routes.v1.workbench import router as workbench +from features.workbench.api.workbench import router as workbench from api.routes.v1.plugins import router as plugins from core.auth.backend import get_current_user, UserContext from core.db.engine import Base, engine, SessionLocal diff --git a/src/backend/tests/workbench_plugin_support.py b/src/backend/tests/features/workbench/workbench_plugin_support.py similarity index 100% rename from src/backend/tests/workbench_plugin_support.py rename to src/backend/tests/features/workbench/workbench_plugin_support.py diff --git a/src/backend/tests/workbench_resource_observer.py b/src/backend/tests/features/workbench/workbench_resource_observer.py similarity index 92% rename from src/backend/tests/workbench_resource_observer.py rename to src/backend/tests/features/workbench/workbench_resource_observer.py index 88407ef5..0e10ac1e 100644 --- a/src/backend/tests/workbench_resource_observer.py +++ b/src/backend/tests/features/workbench/workbench_resource_observer.py @@ -3,13 +3,13 @@ from urllib.parse import urlsplit import httpx -from core.applications.diagnostic_resources import snapshot +from features.workbench.diagnostic_resources import snapshot from core.auth.backend import UserContext, require_auth from core.db.engine import get_db from core.infra.responses import success_response -from core.sandbox.providers.script_runner.auth import auth_headers +from core.infra.local_runtime.runner_auth import auth_headers from fastapi import Depends, HTTPException, Response -from tests.workbench_observer_paths import guard_cli_binding, guard_environment, require_loopback +from tests.features.workbench.workbench_observer_paths import guard_cli_binding, guard_environment, require_loopback def install(app): diff --git a/src/backend/tests/workbench_routing_http_fixture.py b/src/backend/tests/features/workbench/workbench_routing_http_fixture.py similarity index 88% rename from src/backend/tests/workbench_routing_http_fixture.py rename to src/backend/tests/features/workbench/workbench_routing_http_fixture.py index 225f5f2f..c259ed63 100644 --- a/src/backend/tests/workbench_routing_http_fixture.py +++ b/src/backend/tests/features/workbench/workbench_routing_http_fixture.py @@ -3,7 +3,7 @@ from pathlib import Path from fastapi.responses import FileResponse # Reuse the real plugin/session fixture, in a separate process and database. -from tests.workbench_plugin_http_fixture import app +from tests.features.workbench.workbench_plugin_http_fixture import app @app.middleware("http") async def frontend_history(request, call_next): diff --git a/src/backend/tests/infra/test_core_cleanup.py b/src/backend/tests/infra/test_core_cleanup.py index 320b7389..8ce64eea 100644 --- a/src/backend/tests/infra/test_core_cleanup.py +++ b/src/backend/tests/infra/test_core_cleanup.py @@ -77,7 +77,7 @@ def test_runtime_operation_modules_import_without_order_dependency(module): import sys result = subprocess.run( - [sys.executable, "-c", f"import core.capabilities.{module}"], + [sys.executable, "-c", f"import features.desktop.capabilities.{module}"], env=os.environ.copy(), capture_output=True, text=True, @@ -99,3 +99,24 @@ async def __call__(self): await call() with pytest.raises(CircuitBreakerOpenError): await call() + + +def test_rate_limit_configuration_is_not_advertised_without_a_reader(): + """The global HTTP limiter was never mounted; its knobs must not reappear.""" + from pathlib import Path + + from core.config.settings import settings + + assert not hasattr(settings, "rate_limit") + root = Path(__file__).resolve().parents[4] + for relative in ( + "docker-compose.yml", + "requirements.txt", + "desktop/requirements-desktop.txt", + "document/en/deployment/environment-variables.md", + "document/zh-CN/deployment/environment-variables.md", + ): + path = root / relative + if path.is_file(): + text = path.read_text(encoding="utf-8") + assert "RATE_LIMIT_" not in text and "slowapi" not in text, relative diff --git a/src/backend/tests/integration/background_command_server.py b/src/backend/tests/integration/background_command_server.py index caeb1c61..bae0d581 100644 --- a/src/backend/tests/integration/background_command_server.py +++ b/src/backend/tests/integration/background_command_server.py @@ -21,18 +21,19 @@ ) from contextlib import asynccontextmanager -from fastapi import FastAPI -from pydantic import BaseModel + import uvicorn +from api.routes.v1.commands import router +from api.startup.commands import shutdown_commands +from core.auth.backend import UserContext, get_current_user from core.db.engine import Base, SessionLocal, engine from core.db.models import ChatSession, UserShadow -from core.auth.backend import UserContext, get_current_user +from core.sandbox.commands.runtime import runtime from core.sandbox.protocol import ProcessRequest from core.sandbox.providers.script_runner.provider import ScriptRunnerProvider -from api.routes.v1.commands import router -from api.startup.commands import shutdown_commands -from orchestration.commands.runtime import runtime from core.sandbox.runner import server +from fastapi import FastAPI +from pydantic import BaseModel Base.metadata.create_all(engine) with SessionLocal() as db: diff --git a/src/backend/tests/integration/job_agent_local_smoke.py b/src/backend/tests/integration/job_agent_local_smoke.py index cabbc5df..5e8f995c 100644 --- a/src/backend/tests/integration/job_agent_local_smoke.py +++ b/src/backend/tests/integration/job_agent_local_smoke.py @@ -114,21 +114,22 @@ def do_POST(self): async def acceptance(root, backend_socket, runner_socket, model): + import core.jobs.owner as owner + import core.jobs.supervisor as supervisor import core.llm.factory as factory import uvicorn from agentscope.message import UserMsg from api.app import app as production_app from api.routes.v1.internal_jobs import router from core.db.engine import Base, SessionLocal, engine - from core.db.model_repository import assign_role, create_provider + from core.models.repository import assign_role, create_provider from core.db.models import ChatSession, Job, UserShadow - from core.llm.builtin_subagents import get_builtin_subagent, list_builtin_subagents + from core.capabilities.agents.builtin import get_builtin_subagent, list_builtin_subagents from core.sandbox import get_sandbox_provider - from core.services.job_service import JobService - from core.services.run_journal import durable_run_binding - from fastapi import FastAPI - from orchestration.jobs import owner, supervisor from core.sandbox.runner import server as runner + from core.jobs.store import JobService + from core.runs.journal import durable_run_binding + from fastapi import FastAPI Base.metadata.create_all(engine) with SessionLocal() as db: @@ -224,7 +225,7 @@ async def inspect_factory(**kwargs): model_name="workflow-test", model_provider_id=pid, ) - from core.llm.mcp_manager import close_clients + from core.llm.mcp.manager import close_clients try: answer = await asyncio.wait_for( diff --git a/src/backend/tests/integration/job_local_smoke.py b/src/backend/tests/integration/job_local_smoke.py index ebca941f..335b5919 100644 --- a/src/backend/tests/integration/job_local_smoke.py +++ b/src/backend/tests/integration/job_local_smoke.py @@ -67,17 +67,21 @@ def configure(root, backend_port, runner_port): async def acceptance(root, backend_socket, runner_socket): + import core.jobs.callback as callback + import core.jobs.files as files + import core.jobs.owner as owner + import core.jobs.runtime as job_runtime + import core.jobs.state as state + import core.jobs.supervisor as supervisor import uvicorn from api.routes.v1 import internal_jobs from core.db.engine import Base, SessionLocal, engine from core.db.models import UserShadow - from core.llm.tool_collector import ToolCollector + from core.llm.factory.tools.definitions import ToolAssembly from core.llm.tools.job_tool import register_run_job from core.sandbox import get_sandbox_provider - from fastapi import FastAPI - from orchestration import job_runtime - from orchestration.jobs import callback, files, owner, state, supervisor from core.sandbox.runner import server as runner + from fastapi import FastAPI Base.metadata.create_all(engine) with SessionLocal() as db: @@ -137,14 +141,14 @@ def payload(response): user_id = "acceptance-user" await provider.put_file(session_id, "acceptance.py", SCRIPT.encode(), user_id=user_id) - collector = ToolCollector() + collector = ToolAssembly() register_run_job( collector, user_id=user_id, chat_id=None, sandbox_session_id=session_id, ) - tool = collector.get_tool("run_job") + tool = next(t for t in collector if t.name == "run_job") assert tool is not None, "run_job registration failed" run_job = tool._func started = payload( @@ -200,7 +204,7 @@ def payload(response): # A deterministic Windows registry proxy model, under native SAFE_ENV. # It never reads or writes the user's proxy configuration. - from orchestration.jobs.sdk import SDK_SOURCE + from core.jobs.sdk import SDK_SOURCE from tests.sandbox.runner_client import run_spawn assert not runner.SAFE_ENV.get("NO_PROXY") diff --git a/src/backend/tests/integration/job_managed_smoke.py b/src/backend/tests/integration/job_managed_smoke.py index 8dc544f0..ebb3dd45 100644 --- a/src/backend/tests/integration/job_managed_smoke.py +++ b/src/backend/tests/integration/job_managed_smoke.py @@ -13,13 +13,15 @@ import threading import time +import core.jobs.owner as owner +import core.jobs.process as process +import core.jobs.runtime as runtime +import core.jobs.state as state import uvicorn from api.routes.v1.internal_jobs import router from core.db.engine import SessionLocal from core.db.models import Job from fastapi import FastAPI -from orchestration import job_runtime as runtime -from orchestration.jobs import owner, process, state SCRIPT = """from hugagent_job import ledger, job import time @@ -82,7 +84,7 @@ async def wait(jid): # A separate Python worker adopts the same durable command, without launching it. jid = await start("import time\ntime.sleep(8)") child_code = """import asyncio, sys -from orchestration.jobs import owner, state, process, supervisor +from core.jobs import owner, state, process, supervisor async def main(): owner.bind() row = state.snapshot(sys.argv[1]) @@ -109,7 +111,9 @@ async def main(): # Resume repeats the script, but SDK skips already settled rows. partial = SCRIPT.replace( "job.map(items, handle, concurrency=8)", - "job.map(items[:2], handle, concurrency=2)\ntime.sleep(25)\njob.map(items, handle, concurrency=8)", + """job.map(items[:2], handle, concurrency=2) +time.sleep(25) +job.map(items, handle, concurrency=8)""", ) jid = await start(partial) for _ in range(40): diff --git a/src/backend/tests/integrations/__init__.py b/src/backend/tests/integrations/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/src/backend/tests/integrations/__init__.py @@ -0,0 +1 @@ + diff --git a/src/backend/tests/integrations/channels/__init__.py b/src/backend/tests/integrations/channels/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/src/backend/tests/integrations/channels/__init__.py @@ -0,0 +1 @@ + diff --git a/src/backend/tests/integrations/channels/bot_cases_1.py b/src/backend/tests/integrations/channels/bot_cases_1.py new file mode 100644 index 00000000..95f15463 --- /dev/null +++ b/src/backend/tests/integrations/channels/bot_cases_1.py @@ -0,0 +1,381 @@ +from tests.integrations.channels.bot_support import ( + AccessDeniedError, + BOOL_CAPABILITY_DEFAULTS, + BadRequestError, + ChannelConnectionRepository, + ChannelService, + InboundMsg, + LarkAdapter, + SendResult, + _FakeConn, + _clean_scope, + _inbound, + _mk_conn, + _mk_user, + asyncio, + base64, + bot_to_dict, + hashlib, + inbound_attachments, + inbound_sessions, + inbound_state, + json, + pytest, + resolve_capabilities, +) + +def test_capability_bit_registered_default_false(): + assert "can_create_channel_bot" in BOOL_CAPABILITY_DEFAULTS + assert BOOL_CAPABILITY_DEFAULTS["can_create_channel_bot"] is False + + +def test_capability_personal_override_wins(): + caps = resolve_capabilities({"can_create_channel_bot": True}, {}) + assert caps["can_create_channel_bot"] is True + caps2 = resolve_capabilities({}, {"can_create_channel_bot": True}) # team default + assert caps2["can_create_channel_bot"] is True + + +def test_repo_crud(db_session): + _mk_user(db_session) + repo = ChannelConnectionRepository(db_session) + conn = repo.create({ + "channel_id": "chan_a", "owner_user_id": "u1", "channel_type": "lark", + "app_id": "cli_x", "config": {"app_secret_enc": "x"}, "status": "pending", + }) + assert repo.get_by_id("chan_a") is conn + assert repo.get_by_app_id("lark", "cli_x").channel_id == "chan_a" + assert [c.channel_id for c in repo.list_by_owner("u1")] == ["chan_a"] + + repo.set_status("chan_a", "connected") + assert repo.get_by_id("chan_a").status == "connected" + repo.set_status("chan_a", "error", last_error="boom") + assert repo.get_by_id("chan_a").last_error == "boom" + + assert repo.delete("chan_a") is True + assert repo.get_by_id("chan_a") is None + + +def test_clean_scope(): + assert _clean_scope(None) is None + assert _clean_scope({}) is None + assert _clean_scope({"kb_ids": []}) is None + assert _clean_scope({"kb_ids": ["k1"], "junk": 1}) == {"kb_ids": ["k1"]} + assert _clean_scope({"skill_ids": [1, 2]}) == {"skill_ids": ["1", "2"]} + + +def test_create_bot_denied_without_capability(db_session): + _mk_user(db_session, "u_deny", meta={}) + svc = ChannelService(db_session) + with pytest.raises(AccessDeniedError): + asyncio.run(svc.create_bot( + "u_deny", channel_type="lark", app_id="cli_a", app_secret="s", + )) + + +def test_create_bot_token_lock(db_session): + _mk_user(db_session, "u_ok", meta={"can_create_channel_bot": True}) + # Pre-occupy the same app + ChannelConnectionRepository(db_session).create({ + "channel_id": "chan_pre", "owner_user_id": "u_ok", "channel_type": "lark", + "app_id": "cli_dup", "config": {}, "status": "connected", + }) + svc = ChannelService(db_session) + with pytest.raises(BadRequestError): + asyncio.run(svc.create_bot( + "u_ok", channel_type="lark", app_id="cli_dup", app_secret="s", + )) + + +def test_create_bot_rejects_unknown_channel(db_session): + _mk_user(db_session, "u_ch", meta={"can_create_channel_bot": True}) + svc = ChannelService(db_session) + with pytest.raises(BadRequestError): + asyncio.run(svc.create_bot( + "u_ch", channel_type="telegram", app_id="a", app_secret="s", + )) + + +def test_bot_to_dict_hides_secrets(db_session): + _mk_user(db_session, "u_d") + conn = ChannelConnectionRepository(db_session).create({ + "channel_id": "chan_d", "owner_user_id": "u_d", "channel_type": "lark", + "app_id": "cli_d", "config": {"app_secret_enc": "SECRET"}, "status": "connected", + }) + d = bot_to_dict(conn) + assert "config" not in d and "app_secret_enc" not in json.dumps(d) + assert d["app_id"] == "cli_d" and d["status"] == "connected" + assert d["agent_id"] is None # defaults to the main agent + + +def test_repo_list_filters_by_agent(db_session): + _mk_user(db_session, "u_f") + repo = ChannelConnectionRepository(db_session) + repo.create({ + "channel_id": "chan_main", "owner_user_id": "u_f", "channel_type": "lark", + "app_id": "cli_main", "config": {}, "status": "connected", "agent_id": None, + }) + repo.create({ + "channel_id": "chan_ag", "owner_user_id": "u_f", "channel_type": "lark", + "app_id": "cli_ag", "config": {}, "status": "connected", "agent_id": "ag_1", + }) + assert {c.channel_id for c in repo.list_by_owner("u_f")} == {"chan_main", "chan_ag"} + assert [c.channel_id for c in repo.list_by_owner("u_f", main_only=True)] == ["chan_main"] + assert [c.channel_id for c in repo.list_by_owner("u_f", agent_id="ag_1")] == ["chan_ag"] + + +def test_create_bot_rejects_unknown_agent(db_session): + _mk_user(db_session, "u_ag", meta={"can_create_channel_bot": True}) + svc = ChannelService(db_session) + # agent_id pointing to a nonexistent/inaccessible sub-agent → rejected before any network call + with pytest.raises(BadRequestError): + asyncio.run(svc.create_bot( + "u_ag", channel_type="lark", app_id="cli_ag2", app_secret="s", + agent_id="nope_agent", + )) + + +def test_inbound_resource_scope_ignored_marker_for_agent(db_session): + """A conn bound to a sub-agent: agent_id lands on the row, for inbound to set context['agent_id'] pinning that sub-agent. + + (Inbound's context assembly goes through the global SessionLocal; the + unit test only asserts at the row level that the binding is persisted; + the pinning branch `if conn.agent_id: context['agent_id']=conn.agent_id` + is a direct attribute read.) + """ + conn = _mk_conn(db_session) + assert conn.agent_id is None # _mk_conn defaults to the main agent + conn.agent_id = "ag_pinned" + db_session.commit() + reloaded = ChannelConnectionRepository(db_session).get_by_id("chan_in") + assert reloaded.agent_id == "ag_pinned" + + +def test_lark_extract_text_strips_mention(): + assert LarkAdapter._extract_text(json.dumps({"text": "@_user_1 你好"})) == "你好" + assert LarkAdapter._extract_text(json.dumps({"text": "纯文本"})) == "纯文本" + assert LarkAdapter._extract_text(None) == "" + + +def test_lark_parse_inbound_group(): + adapter = LarkAdapter() + payload = { + "header": {"event_type": "im.message.receive_v1"}, + "event": { + "message": { + "chat_id": "oc_group1", "chat_type": "group", + "message_id": "om_1", "message_type": "text", + "content": json.dumps({"text": "@_user_1 汇报一下"}), + }, + "sender": {"sender_id": {"open_id": "ou_alice"}}, + }, + } + msg = adapter.parse_inbound(_FakeConn(), payload) + assert isinstance(msg, InboundMsg) + assert msg.text == "汇报一下" + assert msg.chat_type == "group" + assert msg.external_conversation_id == "oc_group1" # groups are keyed by chat_id + assert msg.sender_id == "ou_alice" + assert msg.raw["lark_chat_id"] == "oc_group1" + + +def test_lark_parse_inbound_skips_non_message(): + adapter = LarkAdapter() + assert adapter.parse_inbound(_FakeConn(), {"type": "url_verification"}) is None + # Non-text messages are skipped + payload = { + "header": {"event_type": "im.message.receive_v1"}, + "event": {"message": {"message_type": "image", "chat_id": "x"}, "sender": {}}, + } + assert adapter.parse_inbound(_FakeConn(), payload) is None + + +def test_lark_aes_decrypt_roundtrip(): + from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes + + encrypt_key = "my-encrypt-key" + plain = json.dumps({"type": "url_verification", "challenge": "abc"}).encode("utf-8") + key = hashlib.sha256(encrypt_key.encode()).digest() + iv = b"\x00" * 16 + pad = 16 - (len(plain) % 16) + padded = plain + bytes([pad]) * pad + enc = Cipher(algorithms.AES(key), modes.CBC(iv)).encryptor() + ct = enc.update(padded) + enc.finalize() + token = base64.b64encode(iv + ct).decode() + + out = LarkAdapter._aes_decrypt(token, encrypt_key) + assert out["challenge"] == "abc" + + +def test_chunk_text(): + from integrations.channels.protocol import chunk_text + assert chunk_text("", 100) == [] + assert chunk_text("hello", 100) == ["hello"] + # Over-length → multiple chunks, each within the limit + parts = chunk_text("x" * 950, 200) + assert len(parts) == 5 and all(len(p) <= 200 for p in parts) + # Prefer breaking at a newline + parts2 = chunk_text("a" * 150 + "\n" + "b" * 50, 200) + assert parts2[0] == "a" * 150 # cut at the newline, without the b's + + +def test_speaker_label(): + from integrations.channels.inbound_observation import speaker_label as _speaker_label + from integrations.channels.protocol import InboundMsg + m = InboundMsg(channel_id="c", channel_type="lark", text="hi", chat_type="group", + external_conversation_id="oc", sender_id="ou_xyz789", sender_name="张三") + assert _speaker_label(m) == "张三" + m2 = InboundMsg(channel_id="c", channel_type="lark", text="hi", chat_type="group", + external_conversation_id="oc", sender_id="ou_xyz789") + assert _speaker_label(m2) == "xyz789" # no name → tail segment of the open_id + + +def test_conv_lock_serializes_same_conversation(): + """#1: same-conversation locks serialize; different conversations run concurrently.""" + import asyncio + + from integrations.channels import inbound as inb + + order = [] + + async def _fake(msg, tag, delay): + async with inbound_state.conv_locks[inbound_sessions.conv_key(msg)]: + order.append(f"{tag}-start") + await asyncio.sleep(delay) + order.append(f"{tag}-end") + + def _msg(conv): + from integrations.channels.protocol import InboundMsg + return InboundMsg(channel_id="c", channel_type="lark", text="x", chat_type="group", + external_conversation_id=conv) + + async def _run(): + same1, same2 = _msg("A"), _msg("A") + await asyncio.gather(_fake(same1, "a1", 0.05), _fake(same2, "a2", 0.01)) + + asyncio.run(_run()) + # Same conversation: a1 must fully finish before a2 starts (serial) + assert order == ["a1-start", "a1-end", "a2-start", "a2-end"] + + +def test_outbound_synthetic_msg_targets_conversation(): + """#7: the proactively delivered placeholder message lets the adapter locate the target chat_id.""" + from integrations.channels.adapters.lark import LarkAdapter + from integrations.channels.outbound import _synthetic_msg + m = _synthetic_msg("chan_x", "lark", "oc_group9") + assert m.external_conversation_id == "oc_group9" + # The adapter uses raw.lark_chat_id to locate the send target + assert LarkAdapter()._chat_id(m) == "oc_group9" + + +def test_send_result_helpers(): + ok = SendResult.ok("m1") + assert ok.success and ok.message_id == "m1" + bad = SendResult.fail("nonsense") + assert not bad.success and bad.error_kind == "unknown" + rl = SendResult.fail("rate_limited", "too fast") + assert rl.error_kind == "rate_limited" + + +def test_inbound_session_keying(db_session): + from integrations.channels.inbound_sessions import find_or_create_session as _find_or_create_session + conn = _mk_conn(db_session) + + # Two messages in the same conversation → reuse the same chat_id, held by the owner + s1 = _find_or_create_session(db_session, conn, _inbound("oc_g1", "group")) + s2 = _find_or_create_session(db_session, conn, _inbound("oc_g1", "group", mid="m2")) + assert s1.chat_id == s2.chat_id + assert s1.user_id == "u_o" + assert s1.channel_id == "chan_in" and s1.external_conversation_id == "oc_g1" + + # A different conversation (another person's DM / another group) → separate chat_id + s3 = _find_or_create_session(db_session, conn, _inbound("oc_p2", "p2p")) + assert s3.chat_id != s1.chat_id + + +def test_lark_parse_file_and_image_messages(): + adapter = LarkAdapter() + base = {"header": {"event_type": "im.message.receive_v1"}, "event": { + "message": {"chat_id": "oc_1", "chat_type": "p2p", "message_id": "om_9"}, + "sender": {"sender_id": {"open_id": "ou_a"}}}} + # file message + import copy + fmsg = copy.deepcopy(base) + fmsg["event"]["message"]["message_type"] = "file" + fmsg["event"]["message"]["content"] = json.dumps({"file_key": "fk_1", "file_name": "数据.xlsx"}) + m = adapter.parse_inbound(_FakeConn(), fmsg) + assert m.text == "" and m.attachments == [{"kind": "file", "key": "fk_1", "name": "数据.xlsx"}] + assert m.raw["lark_message_id"] == "om_9" + # image message + imsg = copy.deepcopy(base) + imsg["event"]["message"]["message_type"] = "image" + imsg["event"]["message"]["content"] = json.dumps({"image_key": "ik_2"}) + m2 = adapter.parse_inbound(_FakeConn(), imsg) + assert m2.attachments[0]["kind"] == "image" and m2.attachments[0]["key"] == "ik_2" + + +def test_lark_parse_post_extracts_text_and_images(): + adapter = LarkAdapter() + post = {"header": {"event_type": "im.message.receive_v1"}, "event": { + "message": {"chat_id": "oc_1", "chat_type": "group", "message_id": "om_p", + "message_type": "post", + "content": json.dumps({"title": "周报", "content": [ + [{"tag": "text", "text": "完成了"}, {"tag": "img", "image_key": "ik_x"}], + ]})}, + "sender": {"sender_id": {"open_id": "ou_a"}}}} + m = adapter.parse_inbound(_FakeConn(), post) + assert "周报" in m.text and "完成了" in m.text + assert any(a["key"] == "ik_x" for a in m.attachments) + + +def test_inbound_ingest_attachments_stores_artifact(db_session): + import asyncio + + from integrations.channels import inbound as inb + from core.db.models import Artifact + + conn = _mk_conn(db_session) + + class _Adapter: + async def download_resource(self, conn, msg, att): + return b"hello world bytes" + + # mock storage (avoid real disk writes) + parse_file + class _Storage: + def upload_bytes(self, content, key): + return f"/local/{key}" + import core.files.storage as cs + orig_get = cs.get_storage + cs.get_storage = lambda: _Storage() + try: + msg = _inbound("oc_1", "p2p", text="", mid="mf1") + msg.attachments = [{"kind": "file", "key": "fk_1", "name": "note.txt"}] + files = asyncio.run(inbound_attachments.ingest_attachments(db_session, _Adapter(), conn, "u_o", "chat_x", msg)) + finally: + cs.get_storage = orig_get + + assert len(files) == 1 + f = files[0] + assert f["name"] == "note.txt" and f["file_id"].startswith("ua_") + assert f["download_url"] == f"/files/{f['file_id']}" + # Artifact persisted to DB + row = db_session.query(Artifact).filter(Artifact.artifact_id == f["file_id"]).first() + assert row is not None and row.user_id == "u_o" and row.chat_id == "chat_x" + assert row.extra_data["source"] == "channel_upload" + assert row.size_bytes == len(b"hello world bytes") + + +def test_inbound_resource_scope_narrows(db_session): + from integrations.channels.inbound_sessions import resolve_enabled as _resolve_enabled + + # No whitelist → enabled_kbs=None (everything), enabled_skills resolved via the owner + conn = _mk_conn(db_session) + out = _resolve_enabled(db_session, conn, "u_o") + assert out["enabled_kbs"] is None + + # With a whitelist → narrowed to the specified kb/skill + conn.resource_scope = {"kb_ids": ["k1", "k2"], "skill_ids": ["sk1"]} + db_session.commit() + out2 = _resolve_enabled(db_session, conn, "u_o") + assert out2["enabled_kbs"] == ["k1", "k2"] + assert out2["enabled_skills"] == ["sk1"] diff --git a/src/backend/tests/integrations/channels/bot_cases_2.py b/src/backend/tests/integrations/channels/bot_cases_2.py new file mode 100644 index 00000000..739bac47 --- /dev/null +++ b/src/backend/tests/integrations/channels/bot_cases_2.py @@ -0,0 +1,362 @@ +from tests.integrations.channels.bot_support import ( + InboundMsg, + SendResult, + _FakeDTConn, + asyncio, + pytest, + subscription, +) + +def test_dingtalk_parse_inbound_group_and_session_webhook(): + from integrations.channels.adapters.dingtalk import DingTalkAdapter + + adapter = DingTalkAdapter() + payload = { + "msgtype": "text", + "text": {"content": " 汇报一下 "}, + "conversationType": "2", + "conversationId": "cidGROUP", + "senderStaffId": "staff_a", + "senderNick": "Alice", + "msgId": "msg_1", + "sessionWebhook": "https://oapi.dingtalk.com/robot/send?session=xyz", + } + msg = adapter.parse_inbound(_FakeDTConn(), payload) + assert isinstance(msg, InboundMsg) + assert msg.text == "汇报一下" + assert msg.chat_type == "group" + assert msg.external_conversation_id == "cidGROUP" + assert msg.sender_name == "Alice" + assert msg.raw["dingtalk_session_webhook"].endswith("session=xyz") + + +def test_dingtalk_parse_inbound_p2p_and_skips_non_text(): + from integrations.channels.adapters.dingtalk import DingTalkAdapter + + adapter = DingTalkAdapter() + p2p = adapter.parse_inbound(_FakeDTConn(), { + "msgtype": "text", "text": {"content": "hi"}, "conversationType": "1", + "conversationId": "cidP2P", "msgId": "m2", + }) + assert p2p.chat_type == "p2p" + # Images and other non-text → None (v1 is text only) + assert adapter.parse_inbound(_FakeDTConn(), {"msgtype": "picture"}) is None + + +def test_dingtalk_edit_message_unsupported(): + from integrations.channels.adapters.dingtalk import DingTalkAdapter + + r = asyncio.run(DingTalkAdapter().edit_message(_FakeDTConn(), "mid", "x")) + assert r.success is False and r.error_kind == "bad_format" + + +def test_dingtalk_has_push_file(): + # Inbound orchestration decides whether to return generated files by hasattr(adapter, "push_file") — if missing, it is silently skipped + from integrations.channels.adapters.dingtalk import DingTalkAdapter + + assert callable(getattr(DingTalkAdapter(), "push_file", None)) + + +def test_dingtalk_robot_send_requires_target(): + from integrations.channels.adapters.dingtalk import DingTalkAdapter + + adapter = DingTalkAdapter() + group = InboundMsg(channel_id="chan_dt", channel_type="dingtalk", text="", + chat_type="group", external_conversation_id="") + r = asyncio.run(adapter._robot_send("tok", _FakeDTConn(), group, "sampleFile", {})) + assert r.success is False and r.error_kind == "bad_format" + + # p2p with neither staffId nor openConversationId → bad_format + # (with an openConversationId it goes through privateChatMessages/send, a network call, not covered by unit tests) + p2p = InboundMsg(channel_id="chan_dt", channel_type="dingtalk", text="", + chat_type="p2p", external_conversation_id="", sender_id="") + r2 = asyncio.run(adapter._robot_send("tok", _FakeDTConn(), p2p, "sampleFile", {})) + assert r2.success is False and r2.error_kind == "bad_format" + + +def test_dingtalk_caps_supports_markdown(): + from integrations.channels.adapters.dingtalk import DingTalkAdapter + + assert DingTalkAdapter.caps.supports_markdown is True + + +def test_dingtalk_send_markdown_via_session_webhook(monkeypatch): + """Reply scenario (with sessionWebhook): msgtype=markdown + title extraction + table downgrade.""" + from integrations.channels.adapters.dingtalk import DingTalkAdapter + + adapter = DingTalkAdapter() + captured = {} + + async def fake_post(webhook, payload): + captured.update(webhook=webhook, payload=payload) + return SendResult.ok() + + monkeypatch.setattr(adapter, "_post_webhook", fake_post) + msg = InboundMsg( + channel_id="chan_dt", channel_type="dingtalk", text="", chat_type="p2p", + external_conversation_id="cid1", + raw={"dingtalk_session_webhook": "https://oapi.dingtalk.com/robot/send?s=1"}, + ) + md = "# 日报\n\n| 指标 | 值 |\n| --- | --- |\n| 完成率 | 98% |" + r = asyncio.run(adapter.send_markdown(_FakeDTConn(), msg, md)) + assert r.success + assert captured["payload"]["msgtype"] == "markdown" + body = captured["payload"]["markdown"] + assert body["title"] == "日报" + assert "- 指标: 完成率｜值: 98%" in body["text"] # table downgraded + assert "| --- |" not in body["text"] + + +def test_dingtalk_send_markdown_proactive_uses_sample_markdown(monkeypatch): + """Proactive delivery (no sessionWebhook) → robot API sampleMarkdown.""" + from integrations.channels.adapters.dingtalk import DingTalkAdapter + + adapter = DingTalkAdapter() + captured = {} + + async def fake_token(app_id, secret): + return "tok" + + async def fake_robot_send(token, conn, inbound, msg_key, msg_param): + captured.update(msg_key=msg_key, msg_param=msg_param) + return SendResult.ok() + + monkeypatch.setattr(adapter, "_access_token", fake_token) + monkeypatch.setattr(adapter, "_robot_send", fake_robot_send) + msg = InboundMsg(channel_id="chan_dt", channel_type="dingtalk", text="", + chat_type="group", external_conversation_id="cidG") + r = asyncio.run(adapter.send_markdown(_FakeDTConn(), msg, "**每日简报**\n- 事项一")) + assert r.success + assert captured["msg_key"] == "sampleMarkdown" + assert captured["msg_param"]["title"] == "每日简报" + assert "**每日简报**" in captured["msg_param"]["text"] + + +def test_dingtalk_push_sends_markdown_chunks(monkeypatch): + """push (the automation proactive-delivery entry point) also sends via markdown.""" + from integrations.channels.adapters.dingtalk import DingTalkAdapter + + adapter = DingTalkAdapter() + sent = [] + + async def fake_send_md(conn, inbound, text): + sent.append(text) + return SendResult.ok() + + monkeypatch.setattr(adapter, "send_markdown", fake_send_md) + msg = InboundMsg(channel_id="chan_dt", channel_type="dingtalk", text="", + chat_type="group", external_conversation_id="cidG") + r = asyncio.run(adapter.push(_FakeDTConn(), msg, "## 标题\n正文")) + assert r.success and sent == ["## 标题\n正文"] + + +def test_deliver_reply_prefers_markdown_and_strips_refs(): + """_deliver_reply: channels that support markdown go through send_markdown, and [ref:...] markers are cleaned.""" + from integrations.channels.inbound_delivery import deliver_reply as _deliver_reply + + calls = {"md": [], "text": []} + + class _MdAdapter: + class caps: + supports_markdown = True + max_message_len = 4000 + + @staticmethod + def prepare_markdown(text): + return text + + async def send_markdown(self, conn, msg, text): + calls["md"].append(text) + return SendResult.ok() + + async def send_text(self, conn, msg, text): + calls["text"].append(text) + return SendResult.ok() + + async def edit_message(self, conn, mid, text): + return SendResult.fail("bad_format", "不支持") + + msg = InboundMsg(channel_id="c", channel_type="dingtalk", text="", + chat_type="p2p", external_conversation_id="cid") + asyncio.run(_deliver_reply(_MdAdapter(), object(), msg, + "结论如下[ref:internet_search-1]，见上文。", "ph_1")) + assert calls["md"] == ["结论如下，见上文。"] # went via markdown + citation markers cleaned + assert calls["text"] == [] + + +def test_dingtalk_send_placeholder_prefers_robot_api(monkeypatch): + """Placeholder via the robot API: obtains the processQueryKey (used as message_id) → recallable afterwards.""" + from integrations.channels.adapters.dingtalk import DingTalkAdapter + + adapter = DingTalkAdapter() + + async def fake_token(app_id, secret): + return "tok" + + async def fake_robot_send(token, conn, inbound, msg_key, msg_param): + assert msg_key == "sampleText" + return SendResult.ok("pqk_123") + + monkeypatch.setattr(adapter, "_access_token", fake_token) + monkeypatch.setattr(adapter, "_robot_send", fake_robot_send) + msg = InboundMsg(channel_id="c", channel_type="dingtalk", text="", chat_type="p2p", + external_conversation_id="cid", sender_id="staff_1") + r = asyncio.run(adapter.send_placeholder(_FakeDTConn(), msg, "🤔 正在处理")) + assert r.success and r.message_id == "pqk_123" + + +def test_dingtalk_send_placeholder_falls_back_to_webhook(monkeypatch): + """Robot API unavailable (token failure) → fall back to sessionWebhook plain text (no id, not recallable).""" + from integrations.channels.adapters.dingtalk import DingTalkAdapter + + adapter = DingTalkAdapter() + captured = {} + + async def fake_token(app_id, secret): + raise RuntimeError("no permission") + + async def fake_post(webhook, payload): + captured.update(payload=payload) + return SendResult.ok() + + monkeypatch.setattr(adapter, "_access_token", fake_token) + monkeypatch.setattr(adapter, "_post_webhook", fake_post) + msg = InboundMsg(channel_id="c", channel_type="dingtalk", text="", chat_type="p2p", + external_conversation_id="cid", + raw={"dingtalk_session_webhook": "https://oapi.dingtalk.com/robot/send?s=1"}) + r = asyncio.run(adapter.send_placeholder(_FakeDTConn(), msg, "🤔 正在处理")) + assert r.success and r.message_id is None + assert captured["payload"]["msgtype"] == "text" + + +def test_dingtalk_recall_url_selection(): + """Recall endpoint routing: group chats use groupMessages/recall (requires openConversationId); one-on-one chats use batchRecall.""" + from integrations.channels.adapters.dingtalk import DingTalkAdapter + + adapter = DingTalkAdapter() + group = InboundMsg(channel_id="c", channel_type="dingtalk", text="", chat_type="group", + external_conversation_id="cidG") + url, body = adapter._recall_url_body(_FakeDTConn(), group, "pqk_1") + assert url.endswith("/v1.0/robot/groupMessages/recall") + assert body["openConversationId"] == "cidG" and body["processQueryKeys"] == ["pqk_1"] + + p2p = InboundMsg(channel_id="c", channel_type="dingtalk", text="", chat_type="p2p", + external_conversation_id="cidP") + url2, body2 = adapter._recall_url_body(_FakeDTConn(), p2p, "pqk_2") + assert url2.endswith("/v1.0/robot/otoMessages/batchRecall") + assert "openConversationId" not in body2 + + # Group chat missing openConversationId → None (recall_message returns bad_format based on this) + orphan = InboundMsg(channel_id="c", channel_type="dingtalk", text="", chat_type="group", + external_conversation_id="") + assert adapter._recall_url_body(_FakeDTConn(), orphan, "pqk_3") is None + r = asyncio.run(adapter.recall_message(_FakeDTConn(), orphan, "")) + assert r.success is False and r.error_kind == "bad_format" + + +def test_deliver_reply_recalls_placeholder_when_edit_unsupported(): + """Edit failure (DingTalk) → recall the placeholder then send the formal reply, visually equivalent to a "replace".""" + from integrations.channels.inbound_delivery import deliver_reply as _deliver_reply + + calls = {"recalled": [], "md": []} + + class _Adapter: + class caps: + supports_markdown = True + max_message_len = 4000 + + @staticmethod + def prepare_markdown(text): + return text + + async def edit_message(self, conn, mid, text): + return SendResult.fail("bad_format", "不支持") + + async def recall_message(self, conn, msg, mid): + calls["recalled"].append(mid) + return SendResult.ok(mid) + + async def send_markdown(self, conn, msg, text): + calls["md"].append(text) + return SendResult.ok() + + async def send_text(self, conn, msg, text): + return SendResult.ok() + + msg = InboundMsg(channel_id="c", channel_type="dingtalk", text="", chat_type="p2p", + external_conversation_id="cid") + asyncio.run(_deliver_reply(_Adapter(), object(), msg, "正式回复", "pqk_ph")) + assert calls["recalled"] == ["pqk_ph"] # placeholder recalled + assert calls["md"] == ["正式回复"] # reply sent as a new message + + +def test_replace_placeholder_falls_back_to_recall_and_send(): + """Error / no-text receipt: edit failure → recall + send anew (the old logic only edited, so DingTalk users would be stuck on "processing" forever).""" + from integrations.channels.inbound_delivery import replace_placeholder as _replace_placeholder + + calls = {"recalled": [], "sent": []} + + class _Adapter: + async def edit_message(self, conn, mid, text): + return SendResult.fail("bad_format", "不支持") + + async def recall_message(self, conn, msg, mid): + calls["recalled"].append(mid) + return SendResult.ok(mid) + + async def send_text(self, conn, msg, text): + calls["sent"].append(text) + return SendResult.ok() + + msg = InboundMsg(channel_id="c", channel_type="dingtalk", text="", chat_type="p2p", + external_conversation_id="cid") + asyncio.run(_replace_placeholder(_Adapter(), object(), msg, "pqk_ph", "⚠️ 出错了")) + assert calls["recalled"] == ["pqk_ph"] and calls["sent"] == ["⚠️ 出错了"] + + +def test_replace_placeholder_without_message_id_sends_terminal_notice_directly(): + """WeChat returns no placeholder ID, but users must still receive the terminal message.""" + from integrations.channels.inbound_delivery import replace_placeholder as _replace_placeholder + + calls = {"edited": 0, "sent": []} + + class _Adapter: + async def edit_message(self, conn, mid, text): + calls["edited"] += 1 + return SendResult.fail("bad_format", "不支持") + + async def send_text(self, conn, msg, text): + calls["sent"].append(text) + return SendResult.ok() + + msg = InboundMsg( + channel_id="c", + channel_type="weixin", + text="", + chat_type="p2p", + external_conversation_id="wx-user", + ) + asyncio.run(_replace_placeholder(_Adapter(), object(), msg, None, "⚠️ 处理失败")) + assert calls == {"edited": 0, "sent": ["⚠️ 处理失败"]} + + +def test_collect_reply_raises_user_facing_stream_error(monkeypatch): + from integrations.channels.inbound_delivery import ChannelRunError, collect_reply as _collect_reply + pass + + async def _events(): + yield {"type": "thinking", "delta": "内部思考"} + yield {"type": "error", "error": "模型暂时不可用"} + + monkeypatch.setattr(subscription, "follow_run", lambda run_id: _events()) + + with pytest.raises(ChannelRunError, match="模型暂时不可用"): + asyncio.run(_collect_reply("run_failed")) + + +def test_markdown_strip_citation_markers(): + from integrations.channels.markdown import strip_citation_markers + + assert strip_citation_markers("A[ref:internet_search-1]B[ref:query_database-12]") == "AB" + assert strip_citation_markers("无标记原样") == "无标记原样" + assert strip_citation_markers("") == "" diff --git a/src/backend/tests/integrations/channels/bot_cases_3.py b/src/backend/tests/integrations/channels/bot_cases_3.py new file mode 100644 index 00000000..51614d54 --- /dev/null +++ b/src/backend/tests/integrations/channels/bot_cases_3.py @@ -0,0 +1,374 @@ +from tests.integrations.channels.bot_support import ( + InboundMsg, + LarkAdapter, + _FakeConn, + _FakeWXConn, + _inbound, + _mk_conn, + _mk_user, + _wecom_conn, + _wecom_encrypt, + asyncio, + base64, + inbound_delivery, + json, + pytest, + subscription, +) + +def test_markdown_strip_inline_thinking(): + """Inline chain of thought must never reach the channel as message text. + Semantics mirror the web frontend (segments.ts): body = whatever follows the LAST .""" + from integrations.channels.markdown import strip_inline_thinking + + # Full tag pair + assert strip_inline_thinking("推理过程\n最终回复") == "最终回复" + # Multi-round tool loop: one thinking block per round, opening often absent + # (server-side template pre-fill) — only the text after the last close tag survives + assert strip_inline_thinking("先想想A再想想B正式答复") == "正式答复" + # Dangling opener (stream cut mid-thinking) → drop the unclosed tail + assert strip_inline_thinking("正文没写完的思考") == "正文" + # All-thinking reply collapses to empty (caller then sends the no-text receipt) + assert strip_inline_thinking("只有思考") == "" + # No tags → untouched + assert strip_inline_thinking("普通回复") == "普通回复" + assert strip_inline_thinking("") == "" + + +def test_collect_reply_strips_inline_thinking(monkeypatch): + """_collect_reply: content deltas carrying inline spans (channel runs use + enable_thinking=True, so the streaming layer forwards them raw) must be stripped + before delivery; thinking events stay ignored as before.""" + from integrations.channels import inbound as inbound_mod + pass + + events = [ + {"type": "thinking", "delta": "结构化思考,本就不进正文"}, + {"type": "content", "delta": "内联思"}, + {"type": "content", "delta": "考跨 delta你好,"}, + {"type": "content", "delta": "这是答复"}, + {"type": "meta", "artifacts": [{"file_id": "f1"}]}, + ] + + async def _fake_follow(run_id): + for ev in events: + yield ev + + monkeypatch.setattr(subscription, "follow_run", _fake_follow) + reply, artifacts = asyncio.run(inbound_delivery.collect_reply("run_x")) + assert reply == "你好,这是答复" + assert artifacts == [{"file_id": "f1"}] + + +def test_markdown_derive_title(): + from integrations.channels.markdown import derive_title + + assert derive_title("# 每周产业简报\n正文") == "每周产业简报" + assert derive_title("**加粗开头** 后续") == "加粗开头 后续" + assert derive_title("- [官网](https://x.y) 入口") == "官网 入口" + assert derive_title("\n\n") == "新消息" + assert len(derive_title("很长" * 40)) == 20 + + +def test_markdown_downgrade_for_dingtalk(): + from integrations.channels.markdown import downgrade_for_dingtalk + + # Well-formed table → row-style list; supported syntax like headings/lists kept as-is + md = "# 标题\n| 姓名 | 部门 |\n| --- | :---: |\n| 张三 | 研发 |\n| 李四 | 市场 |\n- 列表项" + out = downgrade_for_dingtalk(md) + assert "# 标题" in out and "- 列表项" in out + assert "- 姓名: 张三｜部门: 研发" in out + assert "- 姓名: 李四｜部门: 市场" in out + assert "|" not in out.replace("｜", "") + + # Code fences: fence lines removed, content kept (embedded | lines are not misjudged as tables) + code = "```python\nprint('hi')\n| not a table\n```\n尾行" + out2 = downgrade_for_dingtalk(code) + assert "```" not in out2 + assert "print('hi')" in out2 and "| not a table" in out2 and "尾行" in out2 + + # Ill-formed (no separator row) pipe content untouched + raw = "|a|b|\n|1|2|" + assert downgrade_for_dingtalk(raw) == raw + # Idempotent + assert downgrade_for_dingtalk(out) == out + + +def test_outbound_synthetic_msg_carries_chat_type_and_peer(): + # The proactively delivered synthetic message must carry the conversation profile — DingTalk selects the robot endpoint by chat_type/sender_id + from integrations.channels.outbound import _synthetic_msg + + m = _synthetic_msg("chan_x", "dingtalk", "cidABC", chat_type="p2p", peer_id="staff_9") + assert m.chat_type == "p2p" and m.sender_id == "staff_9" + assert m.external_conversation_id == "cidABC" + # Default is still group (backward compatible with old callers) + assert _synthetic_msg("chan_x", "lark", "oc_1").chat_type == "group" + + +def test_wecom_signature_pure(): + from integrations.channels.adapters.wecom import WeComAdapter + + sig = WeComAdapter._signature("tok", "1700000000", "nonce1", "ENC") + # Matches hand computation: sha1(sorted concatenation) + import hashlib as _h + expect = _h.sha1("".join(sorted(["tok", "1700000000", "nonce1", "ENC"])).encode()).hexdigest() + assert sig == expect + + +def test_wecom_url_verify_roundtrip_and_signature(): + from integrations.channels.adapters.wecom import WeComAdapter + + key32 = bytes(range(32)) + aes_key_b64 = base64.b64encode(key32).decode().rstrip("=") # 43-character EncodingAESKey + conn = _wecom_conn(token="mytoken", aes_key_b64=aes_key_b64) + echostr = _wecom_encrypt(key32, "hello-echo") + sig = WeComAdapter._signature("mytoken", "ts", "nc", echostr) + params = {"echostr": echostr, "msg_signature": sig, "timestamp": "ts", "nonce": "nc"} + assert WeComAdapter().verify_url(conn, params) == "hello-echo" + # Wrong signature → raises + with pytest.raises(ValueError): + WeComAdapter().verify_url(conn, {**params, "msg_signature": "bad"}) + + +def test_wecom_decrypt_webhook_and_parse_inbound(): + from integrations.channels.adapters.wecom import WeComAdapter + + key32 = bytes(range(32)) + aes_key_b64 = base64.b64encode(key32).decode().rstrip("=") + conn = _wecom_conn(token="mytoken", aes_key_b64=aes_key_b64) + inner_xml = ( + "" + "" + "" + "" + "12345" + ) + encrypt = _wecom_encrypt(key32, inner_xml) + body = f"".encode() + headers = {} + payload = WeComAdapter().decrypt_webhook(conn, body, headers) + assert payload["Content"] == "你好" + assert headers["_encrypt"] == encrypt # backfilled for signature verification + msg = WeComAdapter().parse_inbound(conn, payload) + assert msg.text == "你好" + assert msg.chat_type == "p2p" + assert msg.external_conversation_id == "userA" + + +def test_wecom_edit_message_unsupported(): + from integrations.channels.adapters.wecom import WeComAdapter + + r = asyncio.run(WeComAdapter().edit_message(_wecom_conn(aes_key_b64="x"), "mid", "x")) + assert r.success is False + + +def test_weixin_parse_inbound_text_and_context_token(): + from integrations.channels.adapters.weixin import WeixinAdapter + + adapter = WeixinAdapter() + payload = { + "from_user_id": "o9cq_xxx@im.wechat", + "message_type": 1, + "context_token": "CTX-TOKEN-123", + "item_list": [{"type": 1, "text_item": {"text": "你好微信"}}], + } + msg = adapter.parse_inbound(_FakeWXConn(), payload) + assert msg.text == "你好微信" + assert msg.chat_type == "p2p" + assert msg.external_conversation_id == "o9cq_xxx@im.wechat" + assert msg.raw["weixin_context_token"] == "CTX-TOKEN-123" + assert msg.raw["weixin_to_user_id"] == "o9cq_xxx@im.wechat" + # Non-text (image) → None + assert adapter.parse_inbound(_FakeWXConn(), {"message_type": 2}) is None + + +def test_weixin_uin_header_is_base64_decimal(): + from integrations.channels.adapters.weixin import _uin_header + + raw = base64.b64decode(_uin_header()).decode() + assert raw.isdigit() and 0 <= int(raw) <= 0xFFFFFFFF + + +def test_weixin_caps_qr_bind_mode(): + from integrations.channels.adapters.weixin import WeixinAdapter + + assert WeixinAdapter.caps.bind_mode == "qr" + assert WeixinAdapter.caps.credential_fields == () + + +def test_weixin_has_push_file(): + # Inbound orchestration decides whether to return generated files by hasattr(adapter, "push_file") — if missing, it is silently skipped + from integrations.channels.adapters.weixin import WeixinAdapter + + assert callable(getattr(WeixinAdapter(), "push_file", None)) + + +def test_weixin_push_file_requires_token_and_target(): + from integrations.channels.adapters.weixin import WeixinAdapter + + adapter = WeixinAdapter() + msg = InboundMsg(channel_id="chan_wx", channel_type="weixin", text="", + chat_type="p2p", external_conversation_id="o9cq_xxx@im.wechat") + # No bot_token → forbidden (_FakeWXConn.config is empty) + r = asyncio.run(adapter.push_file(_FakeWXConn(), msg, b"data", "a.docx", "application/x")) + assert r.success is False and r.error_kind == "forbidden" + + +def test_weixin_aes_ecb_encrypt_pkcs7(): + from integrations.channels.adapters.weixin import _aes_ecb_encrypt + + key = bytes(range(16)) + ct = _aes_ecb_encrypt(b"hello weixin cdn", key) # 16-byte plaintext → full PKCS7 padding block + assert len(ct) == 32 and ct != b"hello weixin cdn" + # Empty content must also be encryptable (1 padding block) + assert len(_aes_ecb_encrypt(b"", key)) == 16 + + +def test_registry_lists_all_four_channels(): + from integrations.channels.registry import list_adapters + + got = set(list_adapters()) + assert {"lark", "dingtalk", "wecom", "weixin"}.issubset(got) + + +def test_caps_bind_mode_defaults(): + from integrations.channels.adapters.dingtalk import DingTalkAdapter + from integrations.channels.adapters.lark import LarkAdapter + from integrations.channels.adapters.wecom import WeComAdapter + + assert LarkAdapter.caps.bind_mode == "credentials" + assert DingTalkAdapter.caps.bind_mode == "credentials" + assert WeComAdapter.caps.credential_fields == ("app_id", "app_secret", "agent_id", "token", "aes_key") + + +def test_inbound_history_preserves_tool_calls(db_session): + """_load_history goes through compaction_service.load_session_history (same + source as the web UI) and must preserve the assistant turns' tool_calls / + tool results, instead of the old approach of stripping them to plain text. + + Old bug: channel history kept only user/assistant text → across turns the + model could not see the tools it had called in its previous turn, redoing + work repeatedly and spinning idle. Here we assert the tool_call replay + (role="tool" carrier + tool name) is still present. + """ + from integrations.channels.inbound_sessions import load_history as _load_history + from core.chat.service import ChatService + + _mk_user(db_session, uid="owner1") + cs = ChatService(db_session) + session = cs.create_session(user_id="owner1", title="渠道会话") + chat_id = session.chat_id + + cs.add_message(chat_id=chat_id, role="user", content="生成一份周报") + cs.add_message( + chat_id=chat_id, role="assistant", content="好的，已生成。", + tool_calls=[{ + "tool_name": "word_create", "tool_id": "tc_1", "status": "success", + "tool_args": {"title": "周报"}, "tool_result": "已生成 weekly.docx", + }], + ) + + history = _load_history(db_session, chat_id, "owner1") + + # Tool replay is present: the assistant block list carries the tool name, and there is a role="tool" result carrier + roles = [m["role"] for m in history] + assert "tool" in roles, f"工具结果载体被丢弃，历史被剥成纯文本: {roles}" + dumped = json.dumps(history, default=str, ensure_ascii=False) + assert "word_create" in dumped, "工具调用信息未保留在跨轮历史中" + + +def test_inbound_history_empty_on_no_access(db_session): + """When the session does not exist / access is denied, load_session_history returns None → _load_history falls back to an empty list.""" + from integrations.channels.inbound_sessions import load_history as _load_history + + _mk_user(db_session, uid="owner2") + assert _load_history(db_session, "chat_does_not_exist", "owner2") == [] + + +def test_lark_parse_extracts_mentions_and_defers_group_decision(): + """群消息的 @ 判定必须延后：飞书事件里没有机器人自己的 open_id，只能先收集 mentions。""" + adapter = LarkAdapter() + evt = {"header": {"event_type": "im.message.receive_v1"}, "event": { + "message": { + "chat_id": "oc_1", "chat_type": "group", "message_id": "om_1", + "message_type": "text", "content": json.dumps({"text": "@_user_1 帮我看下"}), + "mentions": [{"key": "@_user_1", "id": {"open_id": "ou_bot"}, "name": "小助手"}], + }, + "sender": {"sender_id": {"open_id": "ou_a"}}}} + m = adapter.parse_inbound(_FakeConn(), evt) + assert m.mentioned_ids == ["ou_bot"] + assert m.addressed_to_bot is None, "群消息不应在 parse 阶段就下结论" + + # 单聊永远是冲着机器人来的，不需要再解析 + evt["event"]["message"]["chat_type"] = "p2p" + assert adapter.parse_inbound(_FakeConn(), evt).addressed_to_bot is True + + +def test_dingtalk_parse_uses_is_in_at_list(): + """钉钉自带 isInAtList，可同步判定；字段缺失时按「已被 @」处理，保持老行为。""" + from integrations.channels.adapters.dingtalk import DingTalkAdapter + + adapter = DingTalkAdapter() + base = {"msgtype": "text", "text": {"content": "hi"}, "conversationType": "2", + "conversationId": "cid_1", "msgId": "m1"} + assert adapter.parse_inbound(_FakeConn(), {**base, "isInAtList": True}).addressed_to_bot is True + assert adapter.parse_inbound(_FakeConn(), {**base, "isInAtList": False}).addressed_to_bot is False + # 未开通群消息读取权限时钉钉不下发该字段 → 不能误判成「没 @ 」而静音 + assert adapter.parse_inbound(_FakeConn(), base).addressed_to_bot is True + # 单聊不受影响 + assert adapter.parse_inbound( + _FakeConn(), {**base, "conversationType": "1", "isInAtList": False} + ).addressed_to_bot is True + + +def test_resolve_addressed_fails_open(db_session): + """判定不了就当被 @ 了：宁可多答一句，也不能对着直接 @ 装死。""" + from integrations.channels.inbound_observation import resolve_addressed as _resolve_addressed + + conn = _mk_conn(db_session) + msg = _inbound("oc_g1", "group") + + class _NoResolver: + pass + + class _Boom: + async def resolve_addressed(self, conn, inbound): + raise RuntimeError("bot/v3/info 挂了") + + class _Says: + def __init__(self, v): self.v = v + async def resolve_addressed(self, conn, inbound): return self.v + + # 渠道没实现该钩子（企微/微信）→ 行为与本特性上线前完全一致 + assert asyncio.run(_resolve_addressed(_NoResolver(), conn, msg)) is True + assert asyncio.run(_resolve_addressed(_Boom(), conn, msg)) is True + assert asyncio.run(_resolve_addressed(_Says(False), conn, msg)) is False + # adapter 已给出结论时不再回调钩子 + decided = _inbound("oc_g1", "group") + decided.addressed_to_bot = True + assert asyncio.run(_resolve_addressed(_Says(False), conn, decided)) is True + + +def test_observe_buffer_accumulates_caps_and_drains(db_session): + """旁听消息进缓冲区、有水位上限、被 @ 时一次性排空。""" + from integrations.channels.inbound_state import OBSERVE_MAX as _OBSERVE_MAX + from integrations.channels.inbound_observation import drain_observed as _drain_observed, observe_message as _observe_message + from integrations.channels.inbound_sessions import find_or_create_session as _find_or_create_session + + conn = _mk_conn(db_session) + for i in range(_OBSERVE_MAX + 5): + m = _inbound("oc_g1", "group", text=f"闲聊{i}", mid=f"m{i}") + m.sender_name = f"甲{i}" + _observe_message(db_session, conn, m) + + session = _find_or_create_session(db_session, conn, _inbound("oc_g1", "group")) + buf = (session.extra_data or {}).get("observed") + assert len(buf) == _OBSERVE_MAX, "旁听缓冲必须有水位上限，否则活跃群会把提示词撑爆" + assert buf[0]["t"] == "闲聊5", "应淘汰最旧的，保留最近的" + assert buf[0]["n"] == "甲5" + + drained = _drain_observed(db_session, session) + assert len(drained) == _OBSERVE_MAX + db_session.refresh(session) + assert (session.extra_data or {}).get("observed") == [], "排空后不能重复注入" + assert _drain_observed(db_session, session) == [] diff --git a/src/backend/tests/integrations/channels/bot_cases_4.py b/src/backend/tests/integrations/channels/bot_cases_4.py new file mode 100644 index 00000000..27893a90 --- /dev/null +++ b/src/backend/tests/integrations/channels/bot_cases_4.py @@ -0,0 +1,382 @@ +from tests.integrations.channels.bot_support import ( + BadRequestError, + ChannelConnectionRepository, + InboundMsg, + LarkAdapter, + SendResult, + _FakeConn, + _inbound, + _mk_conn, + _reg_channel_tool, + _tool_json, + asyncio, + bot_to_dict, + inbound, + inbound_sessions, + json, + pytest, +) + +def test_observe_text_truncates_and_keeps_attachment_keys(db_session): + """旁听正文截断；附件只留句柄不下载，但 key 必须保住——丢了 key 就永远取不回。""" + from integrations.channels.inbound_state import OBSERVE_TEXT_MAX as _OBSERVE_TEXT_MAX + from integrations.channels.inbound_observation import observe_attachment_refs as _observe_attachment_refs, observe_text as _observe_text + + long_msg = _inbound("oc_g1", "group", text="x" * (_OBSERVE_TEXT_MAX + 50)) + assert _observe_text(long_msg) == "x" * _OBSERVE_TEXT_MAX + "…" + assert _observe_text(_inbound("oc_g1", "group", text=" ")) == "" + + withfile = _inbound("oc_g1", "group", text="看下这个") + withfile.attachments = [ + {"kind": "file", "key": "fk", "name": "报表.xlsx"}, + {"kind": "image", "key": "ik", "name": "p.png"}, + {"kind": "file", "name": "无 key 的不要"}, # 没有 key 就没法取回，直接丢弃 + ] + assert _observe_attachment_refs(withfile) == [ + {"kind": "file", "key": "fk", "name": "报表.xlsx"}, + {"kind": "image", "key": "ik", "name": "p.png"}, + ] + + +def test_observe_records_file_index_for_lazy_fetch(db_session): + """附件句柄进独立索引（含 message_id / raw），且有水位；纯附件消息也要记。""" + from integrations.channels.inbound_state import OBSERVE_FILE_INDEX_MAX as _OBSERVE_FILE_INDEX_MAX + from integrations.channels.inbound_observation import drain_observed as _drain_observed, observe_message as _observe_message + from integrations.channels.inbound_sessions import find_or_create_session as _find_or_create_session + + conn = _mk_conn(db_session) + m = _inbound("oc_g1", "group", text="", mid="om_file") + m.sender_name = "张三" + m.attachments = [{"kind": "file", "key": "fk_1", "name": "Q3财报.xlsx"}] + m.raw = {"lark_message_id": "om_file"} + _observe_message(db_session, conn, m) + + session = _find_or_create_session(db_session, conn, _inbound("oc_g1", "group")) + idx = (session.extra_data or {})["observed_files"] + assert idx["fk_1"]["name"] == "Q3财报.xlsx" + assert idx["fk_1"]["message_id"] == "om_file", "缺 message_id 飞书就取不回资源" + assert idx["fk_1"]["raw"] == {"lark_message_id": "om_file"} + + # 索引在缓冲区排空后仍然存在——模型可能过几轮才决定要看这个文件 + _drain_observed(db_session, session) + db_session.refresh(session) + assert "fk_1" in (session.extra_data or {})["observed_files"] + + # 水位 + for i in range(_OBSERVE_FILE_INDEX_MAX + 3): + mi = _inbound("oc_g1", "group", text="", mid=f"om_{i}") + mi.attachments = [{"kind": "file", "key": f"k{i}", "name": f"f{i}.txt"}] + _observe_message(db_session, conn, mi) + db_session.refresh(session) + idx = (session.extra_data or {})["observed_files"] + assert len(idx) == _OBSERVE_FILE_INDEX_MAX + assert "fk_1" not in idx and f"k{_OBSERVE_FILE_INDEX_MAX + 2}" in idx + + +def test_render_observed_block_exposes_attachment_keys(): + """群聊记录里必须带出 key，否则模型看得到文件却拿不到句柄。""" + from integrations.channels.inbound_observation import render_observed_block as _render_observed_block + + block = _render_observed_block([ + {"n": "张三", "t": "看下这个", "a": [ + {"kind": "file", "key": "fk_1", "name": "Q3财报.xlsx"}, + ]}, + {"n": "李四", "t": "收到"}, + ]) + assert "[文件：Q3财报.xlsx｜key=fk_1]" in block + assert "channel_read_attachment" in block, "要告诉模型怎么取回" + assert "李四：收到" in block + + # 没有附件时不必挂取回提示，省 token + assert "channel_read_attachment" not in _render_observed_block([{"n": "李四", "t": "收到"}]) + + +def test_render_observed_block_marks_as_do_not_reply(): + """群聊记录必须显式标注为背景、不要逐条回复，否则模型会把群消息当成待办清单挨个答。""" + from integrations.channels.inbound_observation import render_observed_block as _render_observed_block + + block = _render_observed_block([{"n": "张三", "t": "明天几点"}, {"n": "李四", "t": "十点"}]) + assert "张三：明天几点" in block and "李四：十点" in block + assert "不要逐条回复" in block + assert block.endswith("\n"), "需与本次消息之间有分隔" + + +def test_group_listen_mode_validation(): + """observe_all 只能开在平台可能投递非 @ 群消息的渠道上。""" + from integrations.channels.service import _validate_group_listen + + assert _validate_group_listen("lark", "observe_all") == "observe_all" + assert _validate_group_listen("dingtalk", "observe_all") == "observe_all" + assert _validate_group_listen("lark", None) == "mention_only" + with pytest.raises(BadRequestError): + _validate_group_listen("lark", "listen_everything") + with pytest.raises(BadRequestError): + _validate_group_listen("weixin", "observe_all") + + +def test_default_group_listen_mode_is_mention_only(db_session): + """默认必须是 mention_only —— 旁听全群是隐私敏感行为，只能由 owner 显式打开。""" + conn = _mk_conn(db_session) + assert conn.group_listen_mode == "mention_only" + assert bot_to_dict(conn)["group_listen_mode"] == "mention_only" + + +def test_inbound_gate_routes_bystander_messages(db_session, monkeypatch): + """闸门路由：未 @ 的群消息在 observe_all 下只落缓冲、在 mention_only 下直接丢弃， + 两种情况都**绝不**起 run、绝不回复——旁听如果会触发回复，机器人会在群里刷屏。""" + import integrations.channels.inbound as inbound_mod + + chan_id = _mk_conn(db_session).channel_id + repo = ChannelConnectionRepository(db_session) + # 不 patch close：fixture 是共享文件 SQLite + teardown drop_all，压住 close 会留锁 + # 拖垮后续用例建表。但 _process_inbound 内部会 close，而 Session.close() 会把已加载 + # 对象全部 expunge —— 所以下面一律用 channel_id 重新取 conn，不复用旧引用。 + monkeypatch.setattr(inbound_mod, "SessionLocal", lambda: db_session) + + started = [] + + class _FakeAdapter: + caps = None + async def resolve_addressed(self, conn, inbound): return False + async def send_text(self, *a, **k): + started.append("sent"); return SendResult.ok("x") + + monkeypatch.setattr(inbound_mod, "get_adapter", lambda ct: _FakeAdapter()) + + def _boom(*a, **k): + started.append("run") + raise AssertionError("旁听消息不得触发 agent run") + + monkeypatch.setattr( + 'core.chat.runtime.admission.start_run', _boom, raising=False + ) + + msg = _inbound("oc_g1", "group", text="今天几点开会", mid="mo1") + msg.sender_name = "张三" + + # observe_all：进缓冲，不起 run、不回复 + repo.update(chan_id, {"group_listen_mode": "observe_all"}) + asyncio.run(inbound.process_inbound(msg)) + assert started == [], "旁听消息既不应起 run 也不应回复" + session = inbound_sessions.find_or_create_session(db_session, repo.get_by_id(chan_id), msg) + assert [it["t"] for it in (session.extra_data or {}).get("observed", [])] == ["今天几点开会"] + assert (session.extra_data or {})["observed"][0]["n"] == "张三" + + # mention_only：直接丢弃，缓冲区不动 + repo.update(chan_id, {"group_listen_mode": "mention_only"}) + asyncio.run(inbound.process_inbound( + _inbound("oc_g1", "group", text="不该被记下", mid="mo2") + )) + # 同样：close() 已把 session 对象 expunge，重新查一次而不是 refresh + fresh = inbound_sessions.find_or_create_session(db_session, repo.get_by_id(chan_id), msg) + assert [it["t"] for it in (fresh.extra_data or {}).get("observed", [])] == ["今天几点开会"] + assert started == [] + + +def test_lark_resolve_addressed_matches_bot_open_id(monkeypatch): + """飞书 @ 判定：只认机器人自己的 open_id。 + + 覆盖 include_bot 权限（`im:message.group_at_msg.include_bot:readonly`）下的场景—— + 群里 @ 的是**别的**机器人时，我们不该抢答；这在闸门上线前是会误答的。 + """ + adapter = LarkAdapter() + + async def _fake_id(conn): return "ou_me" + monkeypatch.setattr(LarkAdapter, "_bot_open_id", staticmethod(_fake_id)) + + def _msg(ids): + m = _inbound("oc_g1", "group") + m.mentioned_ids = ids + return m + + assert asyncio.run(adapter.resolve_addressed(_FakeConn(), _msg(["ou_me"]))) is True + assert asyncio.run(adapter.resolve_addressed(_FakeConn(), _msg(["ou_other_bot"]))) is False + assert asyncio.run(adapter.resolve_addressed(_FakeConn(), _msg([]))) is False + + # 取不到自身 open_id（接口抖动 / 空值）→ 一律按「已被 @」处理，不能在群里装死 + async def _boom(conn): raise RuntimeError("bot/v3/info 502") + monkeypatch.setattr(LarkAdapter, "_bot_open_id", staticmethod(_boom)) + assert asyncio.run(adapter.resolve_addressed(_FakeConn(), _msg(["ou_other"]))) is True + + async def _empty(conn): return "" + monkeypatch.setattr(LarkAdapter, "_bot_open_id", staticmethod(_empty)) + assert asyncio.run(adapter.resolve_addressed(_FakeConn(), _msg(["ou_other"]))) is True + + +def test_dingtalk_extracts_attachments_by_msgtype(): + """钉钉各消息类型的 downloadCode 提取（字段位置依官方「接收的消息类型」）。""" + from integrations.channels.adapters.dingtalk import DingTalkAdapter + + a = DingTalkAdapter() + base = {"conversationType": "1", "conversationId": "cid", "msgId": "m1", + "robotCode": "ding_robot_1"} + + f = a.parse_inbound(_FakeConn(), {**base, "msgtype": "file", "content": { + "downloadCode": "dc_1", "fileName": "合同.pdf", "spaceId": "s", "fileId": "fi"}}) + assert f.attachments == [{"kind": "file", "key": "dc_1", "name": "合同.pdf"}] + # 纯附件、无正文的消息也必须成立——否则钉钉发文件永远收不到 + assert f.text == "" + assert f.raw["dingtalk_robot_code"] == "ding_robot_1" + + p = a.parse_inbound(_FakeConn(), {**base, "msgtype": "picture", "content": { + "downloadCode": "dc_2", "pictureDownloadCode": "pdc"}}) + assert p.attachments[0]["kind"] == "image" and p.attachments[0]["key"] == "dc_2" + # 只给 pictureDownloadCode 时回退取它 + p2 = a.parse_inbound(_FakeConn(), {**base, "msgtype": "picture", + "content": {"pictureDownloadCode": "pdc"}}) + assert p2.attachments[0]["key"] == "pdc" + + r = a.parse_inbound(_FakeConn(), {**base, "msgtype": "richText", "content": { + "richText": [{"text": "看图"}, {"downloadCode": "dc_3"}]}}) + assert r.text == "看图" and r.attachments[0]["key"] == "dc_3" + + # 字段缺失不能抛异常（payload 规格松散，抛了会打断长连接） + assert a.parse_inbound(_FakeConn(), {**base, "msgtype": "file", "content": {}}) is None + assert a.parse_inbound(_FakeConn(), {**base, "msgtype": "file"}) is None + + +def test_dingtalk_download_resource_two_hop(monkeypatch): + """downloadCode → downloadUrl → bytes；缺 robotCode 或过期一律返回 None 不抛。""" + import httpx as _httpx + from integrations.channels.adapters.dingtalk import DingTalkAdapter + + a = DingTalkAdapter() + calls = [] + + class _Resp: + def __init__(self, code=200, payload=None, content=b""): + self.status_code, self._p, self.content = code, payload or {}, content + self.text = json.dumps(self._p) + def json(self): return self._p + + class _Client: + async def __aenter__(self): return self + async def __aexit__(self, *a): return False + async def post(self, url, headers=None, json=None): + calls.append(("post", url, json)) + return _Resp(200, {"downloadUrl": "https://dl.example/x.file"}) + async def get(self, url): + calls.append(("get", url, None)) + return _Resp(200, content=b"PDFBYTES") + + monkeypatch.setattr(_httpx, "AsyncClient", lambda **kw: _Client()) + async def _tok(app_id, secret): return "tk" + monkeypatch.setattr(DingTalkAdapter, "_access_token", staticmethod(_tok)) + + msg = InboundMsg(channel_id="c", channel_type="dingtalk", text="", chat_type="group", + external_conversation_id="cid", message_id="m1", + raw={"dingtalk_robot_code": "rc_1"}) + got = asyncio.run(a.download_resource(_FakeConn(), msg, {"kind": "file", "key": "dc_1"})) + assert got == b"PDFBYTES" + assert calls[0][1].endswith("/v1.0/robot/messageFiles/download") + assert calls[0][2] == {"downloadCode": "dc_1", "robotCode": "rc_1"} + + # 没有 robotCode 就调不动接口，直接放弃而不是发一个必失败的请求 + nomeat = InboundMsg(channel_id="c", channel_type="dingtalk", text="", chat_type="group", + external_conversation_id="cid", raw={}) + assert asyncio.run(a.download_resource(_FakeConn(), nomeat, {"key": "dc_1"})) is None + assert asyncio.run(a.download_resource(_FakeConn(), msg, {"key": ""})) is None + + +def test_channel_read_attachment_fetches_and_stores(db_session, monkeypatch): + """按 key 取回 → 落 Artifact → 返回 file_id 供 read_artifact 读。""" + from integrations.channels.inbound_sessions import find_or_create_session as _find_or_create_session + from integrations.channels.inbound_observation import observe_message as _observe_message + + conn = _mk_conn(db_session) + m = _inbound("oc_g1", "group", text="", mid="om_f") + m.attachments = [{"kind": "file", "key": "fk_1", "name": "Q3财报.xlsx"}] + m.raw = {"lark_message_id": "om_f"} + _observe_message(db_session, conn, m) + session = _find_or_create_session(db_session, conn, _inbound("oc_g1", "group")) + + # 只替换 SessionLocal（工具内部是 `from core.db.engine import SessionLocal` 的函数内导入）。 + # 不要去 patch session.close —— fixture 用的是**共享文件** SQLite 且 teardown 会 drop_all， + # 把 close 变成空操作会让连接一直占着锁，下一个用例的 create_all 就建不出表了。 + # Session.close() 之后仍可继续使用（会开新事务），所以让 `with` 正常关闭即可。 + monkeypatch.setattr("core.db.engine.SessionLocal", lambda: db_session) + + seen = {} + + class _Ad: + async def download_resource(self, conn, inbound, att): + seen["message_id"] = inbound.message_id + seen["raw"] = dict(inbound.raw or {}) + seen["key"] = att.get("key") + return b"XLSXBYTES" + + monkeypatch.setattr('integrations.channels.registry.get_adapter', lambda ct: _Ad()) + + tool = _reg_channel_tool(str(session.user_id), session.chat_id) + out = _tool_json(asyncio.run(tool("fk_1"))) + + assert "error" not in out, out + assert out["filename"] == "Q3财报.xlsx" and out["size"] == len(b"XLSXBYTES") + assert out["file_id"] and "read_artifact" in out["next"] + # adapter 拿到了取回所需的上下文（飞书按 message_id 定位资源） + assert seen["key"] == "fk_1" and seen["message_id"] == "om_f" + assert seen["raw"] == {"lark_message_id": "om_f"} + + +def test_channel_read_attachment_rejects_unknown_and_foreign(db_session, monkeypatch): + """未知 key / 越权会话 / 空 key / 非渠道会话 都必须干净报错，不能抛。""" + from integrations.channels.inbound_sessions import find_or_create_session as _find_or_create_session + from integrations.channels.inbound_observation import observe_message as _observe_message + + conn = _mk_conn(db_session) + m = _inbound("oc_g1", "group", text="", mid="om_f") + m.attachments = [{"kind": "file", "key": "fk_1", "name": "a.xlsx"}] + _observe_message(db_session, conn, m) + session = _find_or_create_session(db_session, conn, _inbound("oc_g1", "group")) + + monkeypatch.setattr("core.db.engine.SessionLocal", lambda: db_session) + + tool = _reg_channel_tool(str(session.user_id), session.chat_id) + assert "不能为空" in _tool_json(asyncio.run(tool(" ")))["error"] + assert "未找到" in _tool_json(asyncio.run(tool("no_such_key")))["error"] + + # 别人的会话：拿到 chat_id 也不能越权取走群文件 + other = _reg_channel_tool("someone_else", session.chat_id) + assert "无权" in _tool_json(asyncio.run(other("fk_1")))["error"] + + # 不在渠道会话里时工具压根没有 chat_id + assert "渠道会话" in _tool_json(asyncio.run(_reg_channel_tool("u_o", None)("fk_1")))["error"] + + +def test_inbound_channel_origin_carries_channel_type(db_session, monkeypatch): + """channel_origin 必须带上 channel_type，否则提示里说不出「哪个平台」、也选不出 CLI。""" + import integrations.channels.inbound as inbound_mod + + chan_id = _mk_conn(db_session).channel_id + monkeypatch.setattr(inbound_mod, "SessionLocal", lambda: db_session) + + captured = {} + + class _Ad: + caps = None + async def resolve_addressed(self, conn, inbound): return True + async def send_text(self, *a, **k): return SendResult.ok("x") + async def send_placeholder(self, *a, **k): return SendResult.ok("ph") + + monkeypatch.setattr(inbound_mod, "get_adapter", lambda ct: _Ad()) + # 这个扫描器自己开 DB 会话（不走被 patch 的那个），测试库里没建表 → 直接短路掉， + # 本用例只关心 context 的组装。 + monkeypatch.setattr( + "core.chat.context.collect_historical_attachments", lambda *a, **k: [] + ) + + async def _fake_start_run(**kwargs): + captured.update(kwargs.get("context") or {}) + raise RuntimeError("stop here — 只验证 context 组装") + + monkeypatch.setattr('core.chat.runtime.admission.start_run', _fake_start_run) + + asyncio.run(inbound.process_inbound( + _inbound("oc_g1", "group", text="帮我看看群里发的文件", mid="mco1") + )) + + origin = captured.get("channel_origin") or {} + assert origin.get("channel_type") == "lark" + assert origin.get("chat_type") == "group" + assert origin.get("conversation_id") == "oc_g1" + assert origin.get("channel_id") == chan_id diff --git a/src/backend/tests/integrations/channels/bot_cases_5.py b/src/backend/tests/integrations/channels/bot_cases_5.py new file mode 100644 index 00000000..a2ede987 --- /dev/null +++ b/src/backend/tests/integrations/channels/bot_cases_5.py @@ -0,0 +1,381 @@ +from tests.integrations.channels.bot_support import ( + InboundMsg, + LarkAdapter, + _HistAdapter, + _hist, + _inbound, + _mk_conn, + asyncio, + json, + re, +) + +def test_history_pull_is_incremental_and_dedupes(db_session): + """第二次拉取必须以游标为下界，且边界重复的消息不能重复入库。""" + from integrations.channels.inbound_sessions import find_or_create_session as _find_or_create_session + from integrations.channels.inbound_history import pull_history as _pull_history + + conn = _mk_conn(db_session) + msg = _inbound("oc_g1", "group", mid="m_at1") + session = _find_or_create_session(db_session, conn, msg) + + # 用真实量级的 epoch ms：游标只进不退，比冷启动窗口更旧的时间戳不会让它倒退 + import time as _t + t0 = int(_t.time() * 1000) - 3_600_000 # 1 小时前，落在 24h 冷启动窗口内 + ad = _HistAdapter([ + [_hist("h1", "周三评审会", t0), _hist("h2", "地点 302", t0 + 1_000)], + # 第二批把 h2 又回了一遍（平台边界通常是闭区间）+ 一条新的 + [_hist("h2", "地点 302", t0 + 1_000), _hist("h3", "带上季度数据", t0 + 2_000)], + ]) + + assert asyncio.run(_pull_history(db_session, ad, conn, msg, session)) == 2 + assert ad.calls[0]["since_ms"] > 0, "首次拉取要有冷启动窗口下界，不能从 0 全量拉" + assert ad.calls[0]["limit"] == 80 + + assert asyncio.run(_pull_history(db_session, ad, conn, msg, session)) == 1, "重复的不该再记一次" + assert ad.calls[1]["since_ms"] == t0 + 1_000, "第二次必须从游标开始，而不是重新拉一遍窗口" + + db_session.refresh(session) + texts = [it["t"] for it in (session.extra_data or {})["observed"]] + assert texts == ["周三评审会", "地点 302", "带上季度数据"] + + +def test_history_pull_respects_budget_and_truncation(db_session): + """总字数预算与单条截断都要生效——活跃群不能把提示词撑爆。""" + from integrations.channels.inbound_state import HISTORY_MAX_TOTAL_CHARS as _HISTORY_MAX_TOTAL_CHARS, OBSERVE_TEXT_MAX as _OBSERVE_TEXT_MAX + from integrations.channels.inbound_sessions import find_or_create_session as _find_or_create_session + from integrations.channels.inbound_history import pull_history as _pull_history + + conn = _mk_conn(db_session) + msg = _inbound("oc_g2", "group", mid="m_at2") + session = _find_or_create_session(db_session, conn, msg) + + import time as _t + _now = int(_t.time() * 1000) + big = [_hist(f"b{i}", "x" * 5000, _now - 60_000 + i) for i in range(10)] + ad = _HistAdapter([big]) + n = asyncio.run(_pull_history(db_session, ad, conn, msg, session)) + + db_session.refresh(session) + entries = (session.extra_data or {})["observed"] + assert n == len(entries) + assert all(len(e["t"]) <= _OBSERVE_TEXT_MAX + 1 for e in entries), "单条要截断" + assert sum(len(e["t"]) for e in entries) <= _HISTORY_MAX_TOTAL_CHARS, "总量要卡预算" + + +def test_history_pull_records_attachment_handles(db_session): + """历史里的文件同样只留句柄进索引，且带上 message_id 供后续按需取回。""" + from integrations.channels.inbound_sessions import find_or_create_session as _find_or_create_session + from integrations.channels.inbound_history import pull_history as _pull_history + + conn = _mk_conn(db_session) + msg = _inbound("oc_g3", "group", mid="m_at3") + session = _find_or_create_session(db_session, conn, msg) + + ad = _HistAdapter([[ + _hist("hf1", "看下这个", int(__import__("time").time() * 1000) - 60_000, + attachments=[{"kind": "file", "key": "fk_h", "name": "周报.docx"}]), + ]]) + asyncio.run(_pull_history(db_session, ad, conn, msg, session)) + + db_session.refresh(session) + idx = (session.extra_data or {})["observed_files"] + assert idx["fk_h"]["name"] == "周报.docx" + assert idx["fk_h"]["message_id"] == "hf1" + assert idx["fk_h"]["raw"] == {"lark_message_id": "hf1"} + + +def test_history_pull_degrades_and_stops_retrying(db_session): + """拉取失败绝不能影响回复；连续失败到上限后不再重试，避免每条消息都白等一次超时。""" + from integrations.channels.inbound_state import HISTORY_MAX_FAILURES as _HISTORY_MAX_FAILURES + from integrations.channels.inbound_sessions import find_or_create_session as _find_or_create_session + from integrations.channels.inbound_history import pull_history as _pull_history + + conn = _mk_conn(db_session) + msg = _inbound("oc_g4", "group", mid="m_at4") + session = _find_or_create_session(db_session, conn, msg) + + class _Broken: + calls = 0 + async def fetch_history(self, conn, conversation_id, *, since_ms, limit, newest_first=False): + type(self).calls += 1 + raise RuntimeError("接口 500") + + bad = _Broken() + for _ in range(_HISTORY_MAX_FAILURES + 3): + assert asyncio.run(_pull_history(db_session, bad, conn, msg, session)) == 0 + assert _Broken.calls == _HISTORY_MAX_FAILURES, "到上限后应停止重试" + + # 渠道没实现该钩子（钉钉当前状态）→ 直接跳过，不报错 + class _NoHook: + pass + assert asyncio.run(_pull_history(db_session, _NoHook(), conn, msg, session)) == 0 + + +def test_lark_history_item_parsing(): + """飞书 im/v1/messages 条目 → HistoryItem；已删除/不支持类型要跳过。""" + item = LarkAdapter._history_item({ + "message_id": "om_1", "msg_type": "text", "create_time": "1700000000000", + "sender": {"id": "ou_abcdef123"}, + "body": {"content": json.dumps({"text": "@_user_1 周三评审"})}, + }) + assert item.message_id == "om_1" and item.ts_ms == 1700000000000 + assert item.text == "周三评审", "@ 提及应被剥掉" + assert item.raw == {"lark_message_id": "om_1"} + + f = LarkAdapter._history_item({ + "message_id": "om_2", "msg_type": "file", "create_time": "1", + "sender": {"id": "ou_x"}, + "body": {"content": json.dumps({"file_key": "fk", "file_name": "a.pdf"})}, + }) + assert f.attachments == [{"kind": "file", "key": "fk", "name": "a.pdf"}] + + assert LarkAdapter._history_item({"message_id": "om_3", "deleted": True}) is None + assert LarkAdapter._history_item({"message_id": "om_4", "msg_type": "sticker", + "body": {"content": "{}"}}) is None + + +def test_dingtalk_fetch_history_requires_owner(): + """没有 owner 或会话 ID 就直接跳过，不起子进程。""" + from integrations.channels.adapters.dingtalk import DingTalkAdapter + + class _C: + owner_user_id = "" + config = {} + app_id = "k" + channel_id = "c" + + a = DingTalkAdapter() + assert asyncio.run(a.fetch_history(_C(), "cid", since_ms=0, limit=5)) == [] + + +def test_dingtalk_fetch_history_dws_call_shape(monkeypatch): + """命令行形状：group / time（本地时间串）/ limit / json，且以 owner 身份运行。""" + from integrations.channels.adapters import dingtalk as dt + + seen = {} + + async def _fake_run(user_id, args, timeout=40): + seen["user_id"] = user_id + seen["args"] = args + seen["timeout"] = timeout + return (json.dumps({"success": True, "result": {"messages": [ + {"msgId": "m1", "senderNick": "张三", "content": "本周进展", "createTime": 1700000000000}, + {"msgId": "m2", "senderNick": "李四", "content": " "}, + ]}}), "", 0) + + monkeypatch.setattr('integrations.dingtalk.service._run_dws', _fake_run) + + class _C: + owner_user_id = "u_owner" + config = {} + app_id = "k" + channel_id = "c" + + items = asyncio.run( + dt.DingTalkAdapter().fetch_history(_C(), "cid_x", since_ms=1_700_000_000_000, limit=9) + ) + assert seen["user_id"] == "u_owner", "必须以连接 owner 的身份跑（凭据按 HOME 隔离）" + args = seen["args"] + assert args[:3] == ["chat", "message", "list"] + assert "--group" in args and args[args.index("--group") + 1] == "cid_x" + assert args[args.index("--limit") + 1] == "9" + assert "--format" in args and args[args.index("--format") + 1] == "json" + tval = args[args.index("--time") + 1] + assert re.match(r"^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$", tval), tval + assert seen["timeout"] <= 10, "子进程要有独立上限，不能吃满调用方的预算" + # 空白正文不入库 + assert [(i.message_id, i.sender_name, i.text) for i in items] == [("m1", "张三", "本周进展")] + + +def test_dingtalk_fetch_history_degrades(monkeypatch): + """未登录/超时/坏输出一律返回 []，不把异常抛回入站链路。""" + from integrations.channels.adapters import dingtalk as dt + + class _C: + owner_user_id = "u" + config = {} + app_id = "k" + channel_id = "c" + + a = dt.DingTalkAdapter() + + async def _not_logged_in(user_id, args, timeout=40): + return ("", "未登录，请先执行 dws auth login", 2) + + monkeypatch.setattr('integrations.dingtalk.service._run_dws', _not_logged_in) + assert asyncio.run(a.fetch_history(_C(), "cid", since_ms=0, limit=5)) == [] + + async def _garbage(user_id, args, timeout=40): + return ("not json at all", "", 0) + + monkeypatch.setattr('integrations.dingtalk.service._run_dws', _garbage) + assert asyncio.run(a.fetch_history(_C(), "cid", since_ms=0, limit=5)) == [] + + async def _boom(user_id, args, timeout=40): + raise RuntimeError("子进程炸了") + + monkeypatch.setattr('integrations.dingtalk.service._run_dws', _boom) + assert asyncio.run(a.fetch_history(_C(), "cid", since_ms=0, limit=5)) == [] + + +def test_dingtalk_parse_dws_history_real_shape(): + """按 dws 线上真实输出解析：openMessageId / sender / createTime 是本地时间**字符串**。 + + 这三个字段名跟机器人回调那套完全不同，踩过一次：把 createTime 当 epoch ms 解析会 + 静默得到 0，游标就永远停在冷启动窗口，每次 @ 都重拉同一个 24 小时。 + """ + from integrations.channels.adapters.dingtalk import DingTalkAdapter + + p = DingTalkAdapter._parse_dws_history + items = p({"errorCode": None, "result": {"hasMore": True, "messages": [{ + "content": "@HugAgentOS 你再看看 现在可以查到了吗", + "createTime": "2026-08-08 21:57:33", + "openConversationId": "cidaKkLSKoy6sVI9cklpK2LWw==", + "openMessageId": "msgpUQWt9mBVdjLj788Z/nnww==", + "sender": "朱路浩", + }]}}) + assert len(items) == 1 + it = items[0] + assert it.message_id == "msgpUQWt9mBVdjLj788Z/nnww==", "去重靠它，取错就重复入库" + assert it.sender_name == "朱路浩", "sender 已是显示名，不该退回「群成员」" + assert it.ts_ms > 1_700_000_000_000, f"时间戳没解析出来，游标会卡死: {it.ts_ms}" + + # 时间戳容错 + ts = DingTalkAdapter._dws_ts_ms + assert ts("2026-08-08 21:57:33") > 0 + assert ts(1786197795294) == 1786197795294 + assert ts("1786197795294") == 1786197795294 + assert ts("") == 0 and ts(None) == 0 and ts("不是时间") == 0 + + # 外壳兼容 + 失败态 + assert [i.text for i in p({"messages": [{"openMessageId": "2", "content": "b"}]})] == ["b"] + assert p({"success": False}) == [] and p({"errorCode": "40001"}) == [] + assert p({}) == [] and p(None) == [] + + +def test_history_coldstart_takes_newest_end(db_session): + """冷启动要取窗口里**最新**的一批，不是最旧的——7 天窗口 + 条数上限下，取旧端等于 + 把一周前的闲聊塞进来、而最近几天反而看不到。截断也必须砍旧的那头。""" + import time as _t + + from integrations.channels.inbound_state import HISTORY_COLDSTART_WINDOW_H as _HISTORY_COLDSTART_WINDOW_H + from integrations.channels.inbound_sessions import find_or_create_session as _find_or_create_session + from integrations.channels.inbound_history import pull_history as _pull_history + + conn = _mk_conn(db_session) + msg = _inbound("oc_cold", "group", mid="m_cold") + session = _find_or_create_session(db_session, conn, msg) + + now = int(_t.time() * 1000) + # 单条先被截到 500 字，所以要够多条才会触到 12000 字总预算（约 24 条） + batch = [_hist(f"c{i}", "x" * 4000, now - i * 60_000, name=f"甲{i:02d}") for i in range(30)] + ad = _HistAdapter([batch]) + n = asyncio.run(_pull_history(db_session, ad, conn, msg, session)) + + assert ad.calls[0]["newest_first"] is True, "首次拉取必须取新端" + # 冷启动窗口 = 7 天 + assert _HISTORY_COLDSTART_WINDOW_H == 168 + assert now - ad.calls[0]["since_ms"] <= 168 * 3600 * 1000 + 60_000 + + db_session.refresh(session) + entries = (session.extra_data or {})["observed"] + assert len(entries) == n and n > 0 + names = [e["n"] for e in entries] + # 保留的是最新的几条（c0 最新），且落库顺序是时间正序 + assert len(entries) < 30, "总预算应当生效，砍掉一部分" + assert "甲00" in names, f"最新的一条被预算砍掉了: {names}" + assert "甲29" not in names, f"最旧的一条不该保留（应从旧端砍）: {names}" + assert names == sorted(names, key=lambda x: -int(x[1:])), f"落库应为时间正序: {names}" + + +def test_history_incremental_is_not_newest_first(db_session): + """有游标之后是增量：从游标往后取，方向必须是旧→新，否则会跳过消息。""" + import time as _t + + from integrations.channels.inbound_sessions import find_or_create_session as _find_or_create_session + from integrations.channels.inbound_history import pull_history as _pull_history + + conn = _mk_conn(db_session) + msg = _inbound("oc_inc", "group", mid="m_inc") + session = _find_or_create_session(db_session, conn, msg) + t0 = int(_t.time() * 1000) - 3_600_000 + + ad = _HistAdapter([[_hist("i1", "第一条", t0)], [_hist("i2", "第二条", t0 + 1000)]]) + asyncio.run(_pull_history(db_session, ad, conn, msg, session)) + asyncio.run(_pull_history(db_session, ad, conn, msg, session)) + + assert ad.calls[0]["newest_first"] is True, "第一次是冷启动" + assert ad.calls[1]["newest_first"] is False, "第二次是增量，必须旧→新" + assert ad.calls[1]["since_ms"] == t0, "增量以游标为下界" + + +def test_dingtalk_history_extracts_file_handles(): + """dws 把文件消息渲染成**纯文本**，必须把 fileId 解析回附件句柄。 + + 不解析的话，拉回来的就只是一句「[文件] xx.docx fileId: ...」——模型看得到有文件， + 却没有任何句柄能打开它，正是实测踩到的现象。 + """ + from integrations.channels.adapters.dingtalk import DingTalkAdapter as D + + raw = ("[文件] 项目周报.docx fileId: X6GRezwJl2LLnwYYTgZMNMzbWdqbropQ " + "注意：如需下载使用dws drive download命令下载") + text, refs = D._extract_dws_files(raw) + assert refs == [{"kind": "file", "key": "X6GRezwJl2LLnwYYTgZMNMzbWdqbropQ", + "name": "项目周报.docx"}] + assert text == "", "匹配掉的整句要清掉，CLI 用法提示不该进提示词" + + img = "[图片] shot.png fileId: ABC123" + assert D._extract_dws_files(img)[1][0]["kind"] == "image" + + # 普通消息不受影响 + assert D._extract_dws_files("就是一句普通的话") == ("就是一句普通的话", []) + + items = D._parse_dws_history({"result": {"messages": [ + {"openMessageId": "m1", "sender": "朱路浩", "createTime": "2026-08-08 22:03:21", + "content": raw}, + ]}}) + assert items[0].attachments[0]["key"] == "X6GRezwJl2LLnwYYTgZMNMzbWdqbropQ" + assert items[0].raw == {"dws_drive": True}, "要标记走钉盘下载而不是机器人 downloadCode" + + +def test_dingtalk_download_routes_history_files_to_drive(monkeypatch, tmp_path): + """历史里的文件按 fileId 走 dws drive download；机器人附件仍走 downloadCode。""" + from integrations.channels.adapters import dingtalk as dt + from integrations.channels.protocol import InboundMsg + + calls = [] + + async def _fake_run(user_id, args, timeout=40): + calls.append(args) + # 模拟 CLI 写盘 + out_path = args[args.index("--output") + 1] + with open(out_path, "wb") as fh: + fh.write(b"DOCXBYTES") + return ('{"success": true}', "", 0) + + monkeypatch.setattr('integrations.dingtalk.service._run_dws', _fake_run) + + class _C: + owner_user_id = "u_owner" + config = {} + app_id = "k" + channel_id = "c" + + a = dt.DingTalkAdapter() + hist_msg = InboundMsg(channel_id="c", channel_type="dingtalk", text="", chat_type="group", + external_conversation_id="cid", raw={"dws_drive": True}) + got = asyncio.run(a.download_resource(_C(), hist_msg, {"kind": "file", "key": "FID1"})) + assert got == b"DOCXBYTES" + assert calls[0][:2] == ["drive", "download"] + assert calls[0][calls[0].index("--node") + 1] == "FID1" + + # 临时目录要清干净（下载跑在后端宿主上，不是一次性沙箱） + import glob as _g + assert not _g.glob("/tmp/dws_dl_*"), "临时目录未清理" + + # 没有 dws_drive 标记 → 仍走机器人 downloadCode 那条（缺 robotCode 时短路） + bot_msg = InboundMsg(channel_id="c", channel_type="dingtalk", text="", chat_type="group", + external_conversation_id="cid", raw={}) + assert asyncio.run(a.download_resource(_C(), bot_msg, {"key": "dc"})) is None + assert len(calls) == 1, "机器人附件不该走 dws" diff --git a/src/backend/tests/integrations/channels/bot_cases_6.py b/src/backend/tests/integrations/channels/bot_cases_6.py new file mode 100644 index 00000000..5d51b7ba --- /dev/null +++ b/src/backend/tests/integrations/channels/bot_cases_6.py @@ -0,0 +1,145 @@ +from tests.integrations.channels.bot_support import ( + InboundMsg, + _inbound, + _mk_conn, + _reg_channel_tool, + _tool_json, + asyncio, + json, +) + +def test_dingtalk_dws_download_degrades(monkeypatch): + """下载失败/未产出文件 → None，且不残留临时目录。""" + from integrations.channels.adapters import dingtalk as dt + + async def _fail(user_id, args, timeout=40): + return ("", "no permission", 1) + + monkeypatch.setattr('integrations.dingtalk.service._run_dws', _fail) + + class _C: + owner_user_id = "u" + config = {} + app_id = "k" + channel_id = "c" + + a = dt.DingTalkAdapter() + assert asyncio.run(a._download_via_dws(_C(), "FID")) is None + import glob as _g + assert not _g.glob("/tmp/dws_dl_*") + + class _NoOwner: + owner_user_id = "" + config = {} + app_id = "k" + channel_id = "c" + assert asyncio.run(a._download_via_dws(_NoOwner(), "FID")) is None + + +def test_dingtalk_history_extracts_online_doc_links(): + """钉钉**在线文档**是链接卡片，不是文件——也要解析成句柄，否则模型只看到一条没用的链接。""" + from integrations.channels.adapters.dingtalk import DingTalkAdapter as D + + card = ( + "季度工作台账\n创建者：张三\n" + "![image](https://img.alicdn.com/x.png)\n" + "[https://alidocs.dingtalk.com/i/nodes/NODEabc123XYZ?corpId=d1]" + "(https://alidocs.dingtalk.com/i/nodes/NODEabc123XYZ?corpId=d1)" + ) + text, refs = D._extract_dws_files(card) + assert refs == [{"kind": "doc", "key": "NODEabc123XYZ", + "name": "季度工作台账"}] + assert text == "", "卡片正文是标题+图标+两遍长 URL，留着纯属浪费提示词" + # 同一 URL 在卡片里出现两次（显示文本 + 链接），不能记成两个附件 + assert len(refs) == 1 + + +def test_dingtalk_doc_read_surfaces_real_reason(monkeypatch): + """平台给了明确理由（如跨组织被拒）时必须原样透出，不能笼统说“可能已过期”。 + + 实测踩到的就是这个：文档属于另一个组织，dws 明确说明了原因，但用户看到的是 + 模糊的“统一身份认证”，往完全错误的方向排查。 + """ + from integrations.channels.adapters import dingtalk as dt + from integrations.channels.protocol import ChannelResourceError, InboundMsg + + cross_org = json.dumps({"error": { + "category": "api", + "message": "出于安全考虑，不支持跨组织访问数据，该数据属于组织 A，而 MCP 工具配置的组织为 B", + "server_error_code": "forbidden.accessDenied", + }}, ensure_ascii=False) + + async def _refused(user_id, args, timeout=40): + return (cross_org, "", 1) + + monkeypatch.setattr('integrations.dingtalk.service._run_dws', _refused) + + class _C: + owner_user_id = "u" + config = {} + app_id = "k" + channel_id = "c" + + a = dt.DingTalkAdapter() + msg = InboundMsg(channel_id="c", channel_type="dingtalk", text="", chat_type="group", + external_conversation_id="cid", raw={}) + try: + asyncio.run(a.download_resource(_C(), msg, {"kind": "doc", "key": "NODE1"})) + raise AssertionError("应当抛出带原因的 ChannelResourceError") + except ChannelResourceError as exc: + assert "跨组织" in str(exc) + + # 正常读取：返回 markdown 字节 + async def _ok(user_id, args, timeout=40): + assert args[:2] == ["doc", "read"] + assert args[args.index("--content-format") + 1] == "markdown" + return (json.dumps({"content": "# 台账\n第一行"}, ensure_ascii=False), "", 0) + + monkeypatch.setattr('integrations.dingtalk.service._run_dws', _ok) + got = asyncio.run(a.download_resource(_C(), msg, {"kind": "doc", "key": "NODE1"})) + assert got.decode("utf-8").startswith("# 台账") + + # 业务错误判定不能误伤正常返回 + assert dt.DingTalkAdapter._dws_business_error('{"result":{"a":1}}') is None + assert dt.DingTalkAdapter._dws_business_error("not json") is None + + +def test_channel_read_attachment_names_docs_with_md_extension(db_session, monkeypatch): + """在线文档取回的是 markdown 正文，落库文件名必须带 .md。 + + read_artifact 的解析器是**纯按扩展名**分发的（file_parser.parse_file），标题类文件名 + 没有后缀就会被判成"不支持的格式"——文件下下来了却读不了，等于白取。 + """ + from integrations.channels.inbound_sessions import find_or_create_session as _find_or_create_session + from integrations.channels.inbound_observation import record_observed_batch as _record_observed_batch + + conn = _mk_conn(db_session) + msg = _inbound("oc_doc", "group", mid="m_doc") + session = _find_or_create_session(db_session, conn, msg) + _record_observed_batch(db_session, session, [], { + "NODE1": {"name": "季度工作台账", "kind": "doc", + "message_id": "m1", "raw": {}}, + "FID2": {"name": "报表.xlsx", "kind": "file", "message_id": "m2", "raw": {}}, + }) + + monkeypatch.setattr("core.db.engine.SessionLocal", lambda: db_session) + + class _Ad: + async def download_resource(self, conn, inbound, att): + return b"# tai zhang" + + monkeypatch.setattr('integrations.channels.registry.get_adapter', lambda ct: _Ad()) + + tool = _reg_channel_tool(str(session.user_id), session.chat_id) + doc = _tool_json(asyncio.run(tool("NODE1"))) + assert doc["filename"].endswith(".md"), doc + assert doc["mime_type"].startswith("text/"), doc + + # 解析器确实认这个名字 + from core.files.content.file_parser import parse_file + assert parse_file(b"# tai zhang", doc["filename"]) == "# tai zhang" + assert parse_file(b"# tai zhang", "季度工作台账") is None, "无后缀本会被判不支持" + + # 普通文件名不受影响，不该被加后缀 + f = _tool_json(asyncio.run(tool("FID2"))) + assert f["filename"] == "报表.xlsx" diff --git a/src/backend/tests/integrations/channels/bot_support.py b/src/backend/tests/integrations/channels/bot_support.py new file mode 100644 index 00000000..15545bab --- /dev/null +++ b/src/backend/tests/integrations/channels/bot_support.py @@ -0,0 +1,179 @@ +"""Unit tests for inbound channel bots (owner service-account model). + +Covers: capability flags, repository CRUD, service capability gating + token +lock, resource-whitelist normalization, and the Lark adapter's event +normalization / @-mention stripping / AES decryption / session keying. + +No network calls: create_bot's credential validation needs HTTP, so only +the pre-network branches are tested (capability gating / token lock). +""" + + +from integrations.channels import inbound + + +from integrations.channels import inbound_attachments + + +from integrations.channels import inbound_delivery + + +from integrations.channels import inbound_sessions + + +from integrations.channels import inbound_state + + +import asyncio + + +import base64 + + +import hashlib + + +import json + + +import re + + +import core.runs.events.subscription as subscription + + +import pytest + + +from core.auth.capabilities import BOOL_CAPABILITY_DEFAULTS, resolve_capabilities + + +from integrations.channels.adapters.lark import LarkAdapter + + +from integrations.channels.protocol import InboundMsg, SendResult + + +from integrations.channels.models import ChannelConnection + + +from core.db.models import UserShadow + + +from integrations.channels.repository import ChannelConnectionRepository + + +from core.infra.exceptions import AccessDeniedError, BadRequestError + + +from integrations.channels.service import ChannelService, _clean_scope, bot_to_dict + + +def _mk_user(db, uid="u1", meta=None): + db.add(UserShadow(user_id=uid, username=uid, extra_data=meta or {})) + db.commit() + + +class _FakeConn: + channel_id = "chan_lark" + channel_type = "lark" + app_id = "cli_z" + config = {} + + +def _mk_conn(db, owner="u_o", scope=None): + _mk_user(db, owner, meta={"can_create_channel_bot": True}) + return ChannelConnectionRepository(db).create({ + "channel_id": "chan_in", "owner_user_id": owner, "channel_type": "lark", + "app_id": "cli_in", "config": {}, "status": "connected", "resource_scope": scope, + }) + + +def _inbound(conv_id, chat_type, text="hi", mid="m1"): + return InboundMsg( + channel_id="chan_in", channel_type="lark", text=text, chat_type=chat_type, + external_conversation_id=conv_id, sender_id="ou_x", message_id=mid, + ) + + +class _FakeDTConn: + channel_id = "chan_dt" + channel_type = "dingtalk" + app_id = "dingkey" + config = {} + + +def _wecom_conn(token="tk", aes_key_b64=None): + """Build a fake conn with real Fernet-encrypted credentials.""" + from core.infra.crypto import encrypt_secret + + class _C: + channel_id = "chan_wc" + channel_type = "wecom" + app_id = "corp1" + config = { + "app_secret_enc": encrypt_secret("sec"), + "agent_id_enc": encrypt_secret("1000002"), + "token_enc": encrypt_secret(token), + "aes_key_enc": encrypt_secret(aes_key_b64), + } + return _C() + + +def _wecom_encrypt(key32: bytes, msg: str, receiveid: str = "corp1") -> str: + import struct + + from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes + + body = msg.encode("utf-8") + plain = b"0123456789abcdef" + struct.pack(">I", len(body)) + body + receiveid.encode() + pad = 16 - (len(plain) % 16) + plain += bytes([pad]) * pad + enc = Cipher(algorithms.AES(key32), modes.CBC(key32[:16])).encryptor() + return base64.b64encode(enc.update(plain) + enc.finalize()).decode() + + +class _FakeWXConn: + channel_id = "chan_wx" + channel_type = "weixin" + app_id = "wx_abc" + config = {} + + +def _reg_channel_tool(user_id, chat_id): + """从原生工具列表取出附件工具函数。""" + from core.llm.factory.tools.definitions import ToolAssembly + from integrations.channels.tools.attachment import register_channel_attachment + + tc = ToolAssembly() + register_channel_attachment(tc, user_id=user_id, chat_id=chat_id) + return next(tool for tool in tc if tool.name == "channel_read_attachment")._func + + +def _tool_json(resp): + block = resp.content[0] + text = block["text"] if isinstance(block, dict) else getattr(block, "text") + return json.loads(text) + + +class _HistAdapter: + """带 fetch_history 的假 adapter；记录每次调用的 since_ms 以验证增量。""" + + def __init__(self, batches): + self.batches = list(batches) + self.calls = [] + + async def fetch_history(self, conn, conversation_id, *, since_ms, limit, newest_first=False): + self.calls.append({ + "conv": conversation_id, "since_ms": since_ms, + "limit": limit, "newest_first": newest_first, + }) + return self.batches.pop(0) if self.batches else [] + + +def _hist(mid, text, ts_ms, name="群友", attachments=None): + from integrations.channels.protocol import HistoryItem + return HistoryItem( + message_id=mid, sender_name=name, text=text, ts_ms=ts_ms, + attachments=attachments or [], raw={"lark_message_id": mid}, + ) diff --git a/src/backend/tests/integrations/channels/test_adapter_method_binding.py b/src/backend/tests/integrations/channels/test_adapter_method_binding.py new file mode 100644 index 00000000..309c5ff8 --- /dev/null +++ b/src/backend/tests/integrations/channels/test_adapter_method_binding.py @@ -0,0 +1,16 @@ +"""History parsing preserves the concrete adapter method binding.""" +from integrations.channels.adapters.dingtalk import DingTalkAdapter + + +def test_history_parser_uses_adapter_file_and_timestamp_methods(monkeypatch): + calls = [] + def files(text): + calls.append(text) + return "parsed", [{"key": "file"}] + monkeypatch.setattr(DingTalkAdapter, "_extract_dws_files", staticmethod(files)) + monkeypatch.setattr(DingTalkAdapter, "_dws_ts_ms", staticmethod(lambda value: 123)) + items = DingTalkAdapter._parse_dws_history({"messages": [{"content": "source", "createTime": 1}]}) + assert calls == ["source"] + assert [(item.text, item.ts_ms, item.attachments) for item in items] == [ + ("parsed", 123, [{"key": "file"}]) + ] diff --git a/src/backend/tests/integrations/channels/test_channel_bots.py b/src/backend/tests/integrations/channels/test_channel_bots.py new file mode 100644 index 00000000..d8e730f9 --- /dev/null +++ b/src/backend/tests/integrations/channels/test_channel_bots.py @@ -0,0 +1,115 @@ +"""Channel bot cases collected under their stable test module identity.""" + +from tests.integrations.channels.bot_cases_1 import ( + test_capability_bit_registered_default_false, + test_capability_personal_override_wins, + test_repo_crud, + test_clean_scope, + test_create_bot_denied_without_capability, + test_create_bot_token_lock, + test_create_bot_rejects_unknown_channel, + test_bot_to_dict_hides_secrets, + test_repo_list_filters_by_agent, + test_create_bot_rejects_unknown_agent, + test_inbound_resource_scope_ignored_marker_for_agent, + test_lark_extract_text_strips_mention, + test_lark_parse_inbound_group, + test_lark_parse_inbound_skips_non_message, + test_lark_aes_decrypt_roundtrip, + test_chunk_text, + test_speaker_label, + test_conv_lock_serializes_same_conversation, + test_outbound_synthetic_msg_targets_conversation, + test_send_result_helpers, + test_inbound_session_keying, + test_lark_parse_file_and_image_messages, + test_lark_parse_post_extracts_text_and_images, + test_inbound_ingest_attachments_stores_artifact, + test_inbound_resource_scope_narrows, +) + +from tests.integrations.channels.bot_cases_2 import ( + test_dingtalk_parse_inbound_group_and_session_webhook, + test_dingtalk_parse_inbound_p2p_and_skips_non_text, + test_dingtalk_edit_message_unsupported, + test_dingtalk_has_push_file, + test_dingtalk_robot_send_requires_target, + test_dingtalk_caps_supports_markdown, + test_dingtalk_send_markdown_via_session_webhook, + test_dingtalk_send_markdown_proactive_uses_sample_markdown, + test_dingtalk_push_sends_markdown_chunks, + test_deliver_reply_prefers_markdown_and_strips_refs, + test_dingtalk_send_placeholder_prefers_robot_api, + test_dingtalk_send_placeholder_falls_back_to_webhook, + test_dingtalk_recall_url_selection, + test_deliver_reply_recalls_placeholder_when_edit_unsupported, + test_replace_placeholder_falls_back_to_recall_and_send, + test_replace_placeholder_without_message_id_sends_terminal_notice_directly, + test_collect_reply_raises_user_facing_stream_error, + test_markdown_strip_citation_markers, +) + +from tests.integrations.channels.bot_cases_3 import ( + test_markdown_strip_inline_thinking, + test_collect_reply_strips_inline_thinking, + test_markdown_derive_title, + test_markdown_downgrade_for_dingtalk, + test_outbound_synthetic_msg_carries_chat_type_and_peer, + test_wecom_signature_pure, + test_wecom_url_verify_roundtrip_and_signature, + test_wecom_decrypt_webhook_and_parse_inbound, + test_wecom_edit_message_unsupported, + test_weixin_parse_inbound_text_and_context_token, + test_weixin_uin_header_is_base64_decimal, + test_weixin_caps_qr_bind_mode, + test_weixin_has_push_file, + test_weixin_push_file_requires_token_and_target, + test_weixin_aes_ecb_encrypt_pkcs7, + test_registry_lists_all_four_channels, + test_caps_bind_mode_defaults, + test_inbound_history_preserves_tool_calls, + test_inbound_history_empty_on_no_access, + test_lark_parse_extracts_mentions_and_defers_group_decision, + test_dingtalk_parse_uses_is_in_at_list, + test_resolve_addressed_fails_open, + test_observe_buffer_accumulates_caps_and_drains, +) + +from tests.integrations.channels.bot_cases_4 import ( + test_observe_text_truncates_and_keeps_attachment_keys, + test_observe_records_file_index_for_lazy_fetch, + test_render_observed_block_exposes_attachment_keys, + test_render_observed_block_marks_as_do_not_reply, + test_group_listen_mode_validation, + test_default_group_listen_mode_is_mention_only, + test_inbound_gate_routes_bystander_messages, + test_lark_resolve_addressed_matches_bot_open_id, + test_dingtalk_extracts_attachments_by_msgtype, + test_dingtalk_download_resource_two_hop, + test_channel_read_attachment_fetches_and_stores, + test_channel_read_attachment_rejects_unknown_and_foreign, + test_inbound_channel_origin_carries_channel_type, +) + +from tests.integrations.channels.bot_cases_5 import ( + test_history_pull_is_incremental_and_dedupes, + test_history_pull_respects_budget_and_truncation, + test_history_pull_records_attachment_handles, + test_history_pull_degrades_and_stops_retrying, + test_lark_history_item_parsing, + test_dingtalk_fetch_history_requires_owner, + test_dingtalk_fetch_history_dws_call_shape, + test_dingtalk_fetch_history_degrades, + test_dingtalk_parse_dws_history_real_shape, + test_history_coldstart_takes_newest_end, + test_history_incremental_is_not_newest_first, + test_dingtalk_history_extracts_file_handles, + test_dingtalk_download_routes_history_files_to_drive, +) + +from tests.integrations.channels.bot_cases_6 import ( + test_dingtalk_dws_download_degrades, + test_dingtalk_history_extracts_online_doc_links, + test_dingtalk_doc_read_surfaces_real_reason, + test_channel_read_attachment_names_docs_with_md_extension, +) diff --git a/src/backend/tests/test_channel_bots_api.py b/src/backend/tests/integrations/channels/test_channel_bots_api.py similarity index 94% rename from src/backend/tests/test_channel_bots_api.py rename to src/backend/tests/integrations/channels/test_channel_bots_api.py index 8abc2555..9d84074a 100644 --- a/src/backend/tests/test_channel_bots_api.py +++ b/src/backend/tests/integrations/channels/test_channel_bots_api.py @@ -37,9 +37,9 @@ def client(monkeypatch): # 2) Monkeypatch away side effects that touch the network / spawn threads async def _fake_validate(self, conn): return {"app_id": conn.app_id} - monkeypatch.setattr("core.channels.adapters.lark.LarkAdapter.validate_credentials", _fake_validate) - monkeypatch.setattr("core.channels.manager.ChannelManager.start_connection", lambda self, *a, **k: None) - monkeypatch.setattr("core.channels.manager.ChannelManager.stop_connection", lambda self, *a, **k: None) + monkeypatch.setattr('integrations.channels.adapters.lark.LarkAdapter.validate_credentials', _fake_validate) + monkeypatch.setattr('integrations.channels.manager.ChannelManager.start_connection', lambda self, *a, **k: None) + monkeypatch.setattr('integrations.channels.manager.ChannelManager.stop_connection', lambda self, *a, **k: None) from api.app import app diff --git a/src/backend/tests/test_channel_desktop_e2e.py b/src/backend/tests/integrations/channels/test_channel_desktop_e2e.py similarity index 88% rename from src/backend/tests/test_channel_desktop_e2e.py rename to src/backend/tests/integrations/channels/test_channel_desktop_e2e.py index 9a5dbd82..b655d6a7 100644 --- a/src/backend/tests/test_channel_desktop_e2e.py +++ b/src/backend/tests/integrations/channels/test_channel_desktop_e2e.py @@ -3,29 +3,31 @@ Only external platform HTTP, model generation and long-connection startup are fixtures. Cloud and local databases are separate; provider parsing and outbound adapters are real. """ +from features.desktop.bridge import identity as bridge_identity import asyncio import json from contextvars import Context +import core.agent_runtime.runner as runner import httpx import pytest -from core.artifacts import store -from core.channels.adapters.lark import LarkAdapter -from core.channels.registry import get_adapter -from core.db.models import ChannelConnection, UserShadow +from core.files.artifacts import store +from integrations.channels.adapters.lark import LarkAdapter +from integrations.channels.registry import get_adapter +from integrations.channels.models import ChannelConnection +from core.db.models import UserShadow from core.infra.ephemeral import LocalEphemeralState -from core.llm import workspace -from core.services import channel_desktop_worker as worker -from core.services.artifact_service import persist_artifacts -from core.services.channel_relay_dispatch import dispatch_to_desktop -from core.services.channel_service import ChannelService -from core.storage import LocalStorageBackend +from core.llm.runtime import workspace +from integrations.channels import desktop_worker as worker +from core.files.artifacts.service import persist_artifacts +from integrations.channels.relay_dispatch import dispatch_to_desktop +from integrations.channels.service import ChannelService +from core.files.storage import LocalStorageBackend from httpx import _client -from orchestration import chat_run_executor as executor -from tests.test_channel_desktop_http import cloud # noqa: F401 -from tests.test_channel_desktop_relay_api import api # noqa: F401 -from tests.test_channel_desktop_worker import runtime # noqa: F401 +from tests.integrations.channels.test_channel_desktop_http import cloud # noqa: F401 +from tests.integrations.channels.test_channel_desktop_relay_api import api # noqa: F401 +from tests.integrations.channels.test_channel_desktop_worker import runtime # noqa: F401 # Restore the actual methods overridden by the older API fixture. LARK_METHODS = { @@ -87,7 +89,7 @@ async def test_binding_ingress_local_run_reply_file_and_complete( tmp_path, ): client, cloud_sessions, _ = api - identity = worker.bridge.get_identity_state() + identity = bridge_identity.get_identity_state() with runtime() as db: db.get(UserShadow, "local-owner").user_center_id = "center" db.commit() @@ -228,20 +230,20 @@ async def respond(request): conn = db.get(ChannelConnection, bot["channel_id"]) msg = adapter.parse_inbound(conn, inbound_payload(channel)) assert msg is not None and msg.text == "你好" - monkeypatch.setattr("core.services.channel_relay_dispatch.bridge_enabled", lambda: False) + monkeypatch.setattr('integrations.channels.relay_dispatch.bridge_enabled', lambda: False) assert await dispatch_to_desktop(msg, cloud_sessions) claim = await worker.cloud_post(cloud, "claim", {"binding_ids": [grant["binding_id"]]}) item = claim["delivery"] assert item and item["message"]["text"] == "你好" assert "fixture-secret" not in str(item) and "fixture-token" not in str(item) - monkeypatch.setattr("core.services.channel_relay_dispatch.bridge_enabled", lambda: True) + monkeypatch.setattr('integrations.channels.relay_dispatch.bridge_enabled', lambda: True) seen = {} monkeypatch.setenv("STORAGE_PATH", str(tmp_path / "storage")) monkeypatch.setenv("STORAGE_TYPE", "local") monkeypatch.setattr(store, "_STORE_DIR", tmp_path / "storage" / "artifacts") storage = LocalStorageBackend() - monkeypatch.setattr("core.storage.get_storage", lambda: storage) - monkeypatch.setattr("core.services.artifact_service.persist_artifacts", persist_artifacts) + monkeypatch.setattr('core.files.storage.get_storage', lambda: storage) + monkeypatch.setattr('core.files.artifacts.service.persist_artifacts', persist_artifacts) async def workflow(**kwargs): seen.update(kwargs) @@ -257,7 +259,7 @@ async def workflow(**kwargs): yield {"type": "content", "delta": "你好，本机执行成功：" + channel} yield {"type": "meta", "route": "main", "usage": {"input_tokens": 1, "output_tokens": 1}} - monkeypatch.setattr(executor, "astream_chat_workflow", workflow) + monkeypatch.setattr(runner, "astream_chat_workflow", workflow) assert ( await asyncio.wait_for( worker.run_delivery(runtime, cloud, identity, "local-owner", item), 15 diff --git a/src/backend/tests/test_channel_desktop_http.py b/src/backend/tests/integrations/channels/test_channel_desktop_http.py similarity index 89% rename from src/backend/tests/test_channel_desktop_http.py rename to src/backend/tests/integrations/channels/test_channel_desktop_http.py index 1506331f..6d729459 100644 --- a/src/backend/tests/test_channel_desktop_http.py +++ b/src/backend/tests/integrations/channels/test_channel_desktop_http.py @@ -1,20 +1,21 @@ """Desktop client against real cloud routes and their canonical response envelopes.""" +from features.desktop.bridge import identity as bridge_identity import httpx import pytest from core.db.models import UserShadow from core.infra.exceptions import AccessDeniedError, BadRequestError -from core.services import channel_desktop_worker as worker -from tests.test_channel_desktop_relay_api import api, create_local, enqueue # noqa: F401 +from integrations.channels import desktop_worker as worker +from tests.integrations.channels.test_channel_desktop_relay_api import api, create_local, enqueue # noqa: F401 @pytest.fixture def cloud(api, monkeypatch): client, factory, _ = api state = {"cloud_base": "https://cloud.invalid", "token": "fixture", "device_id": "device-a"} - monkeypatch.setattr(worker.bridge, "get_state", lambda: state) - monkeypatch.setattr(worker.bridge, "bridge_enabled", lambda: True) - monkeypatch.setattr(worker.bridge, "get_identity_state", lambda: { + monkeypatch.setattr(bridge_identity, "get_state", lambda: state) + monkeypatch.setattr('features.desktop.auth.bridge.bridge_enabled', lambda: True) + monkeypatch.setattr(bridge_identity, "get_identity_state", lambda: { "cloud_base": state["cloud_base"], "subject": "owner", "device_id": "device-a", "shell_user_center_id": "center", }) @@ -97,8 +98,8 @@ async def test_invalid_registration_is_not_saved(api, cloud, monkeypatch, payloa ("lark", "long_conn"), ("dingtalk", "long_conn"), ("wecom", "webhook"), ]) async def test_credential_channels_bind_to_registered_desktop(api, cloud, monkeypatch, channel, transport): - from core.channels.registry import get_adapter - from core.services.channel_service import ChannelService + from integrations.channels.registry import get_adapter + from integrations.channels.service import ChannelService async def validate(self, conn): return {} @@ -118,9 +119,9 @@ async def validate(self, conn): async def test_weixin_qr_local_binding_preserves_selected_device(api, cloud, monkeypatch): - from core.channels.adapters.weixin import WeixinAdapter + from integrations.channels.adapters.weixin import WeixinAdapter from core.infra.ephemeral import LocalEphemeralState - from core.services.channel_service import ChannelService + from integrations.channels.service import ChannelService async def start(): return {"qrcode": "fixture-qr", "qrcode_img_content": "fixture-png"} @@ -150,10 +151,10 @@ async def poll(qrcode): async def test_account_switch_rejects_late_registration(api, cloud, monkeypatch): - from core.capabilities.errors import CloudUnavailable + from core.harness.capability_errors import CloudUnavailable from httpx import _client active = [cloud] - monkeypatch.setattr(worker.bridge, "get_state", lambda: active[0]) + monkeypatch.setattr(bridge_identity, "get_state", lambda: active[0]) def response(request): active[0] = {**cloud, "token": "different-account"} @@ -169,14 +170,14 @@ def response(request): def test_offline_local_device_is_not_queued_or_routed_to_cloud(api, monkeypatch): import time - from core.db.models import ChannelConnection - from core.services.channel_relay import ChannelRelayService - from core.db.models.channel_relay import ChannelRelayDelivery + from integrations.channels.models import ChannelConnection + from integrations.channels.relay import ChannelRelayService + from integrations.channels.models import ChannelRelayDelivery from sqlalchemy import select grant, bot = create_local(api) now = time.time() - monkeypatch.setattr("core.services.channel_relay.time.time", lambda: now + 36) + monkeypatch.setattr('integrations.channels.relay.time.time', lambda: now + 36) with api[1]() as db: assert not ChannelRelayService(db).online(grant["binding_id"]) with pytest.raises(BadRequestError, match="本机设备离线"): diff --git a/src/backend/tests/test_channel_desktop_migration.py b/src/backend/tests/integrations/channels/test_channel_desktop_migration.py similarity index 97% rename from src/backend/tests/test_channel_desktop_migration.py rename to src/backend/tests/integrations/channels/test_channel_desktop_migration.py index aaa8f0c5..aed28e92 100644 --- a/src/backend/tests/test_channel_desktop_migration.py +++ b/src/backend/tests/integrations/channels/test_channel_desktop_migration.py @@ -11,7 +11,7 @@ def test_relay_migration_roundtrip(tmp_path): - root = Path(__file__).resolve().parents[3] + root = Path(__file__).resolve().parents[5] url = f"sqlite:///{tmp_path / 'relay-migration.sqlite'}" env = {**os.environ, "DATABASE_URL": url} engine = create_engine(url) diff --git a/src/backend/tests/test_channel_desktop_relay.py b/src/backend/tests/integrations/channels/test_channel_desktop_relay.py similarity index 88% rename from src/backend/tests/test_channel_desktop_relay.py rename to src/backend/tests/integrations/channels/test_channel_desktop_relay.py index fa6f6f81..37b738d9 100644 --- a/src/backend/tests/test_channel_desktop_relay.py +++ b/src/backend/tests/integrations/channels/test_channel_desktop_relay.py @@ -1,16 +1,17 @@ """Desktop channel delivery contract with real SQLite transactions.""" import pytest -from core.channels.protocol import InboundMsg +from integrations.channels.protocol import InboundMsg from core.db.engine import Base -from core.db.models import ChannelConnection, UserShadow +from integrations.channels.models import ChannelConnection +from core.db.models import UserShadow from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker @pytest.fixture def db(): - from core.db.models.channel_relay import ChannelRelayDelivery, DesktopChannelBinding + from integrations.channels.models import ChannelRelayDelivery, DesktopChannelBinding engine = create_engine("sqlite://") Base.metadata.create_all(engine) @@ -21,7 +22,7 @@ def db(): def test_only_bound_device_can_claim_and_delivery_is_not_replayed(db): - from core.services.channel_relay import ChannelRelayService + from integrations.channels.relay import ChannelRelayService svc = ChannelRelayService(db) grant = svc.register("owner", "device-a", "Laptop") @@ -57,7 +58,7 @@ def test_only_bound_device_can_claim_and_delivery_is_not_replayed(db): def test_binding_is_owner_scoped_and_single_use(db): from core.infra.exceptions import BadRequestError - from core.services.channel_relay import ChannelRelayService + from integrations.channels.relay import ChannelRelayService svc = ChannelRelayService(db) grant = svc.register("owner", "d", "PC") @@ -70,7 +71,7 @@ def test_binding_is_owner_scoped_and_single_use(db): def test_disabled_bot_cannot_continue_a_claimed_run(db): from core.infra.exceptions import AccessDeniedError - from core.services.channel_relay import ChannelRelayService + from integrations.channels.relay import ChannelRelayService svc = ChannelRelayService(db) grant = svc.register("owner", "d", "PC") diff --git a/src/backend/tests/test_channel_desktop_relay_api.py b/src/backend/tests/integrations/channels/test_channel_desktop_relay_api.py similarity index 89% rename from src/backend/tests/test_channel_desktop_relay_api.py rename to src/backend/tests/integrations/channels/test_channel_desktop_relay_api.py index 12b20444..0ac4ab77 100644 --- a/src/backend/tests/test_channel_desktop_relay_api.py +++ b/src/backend/tests/integrations/channels/test_channel_desktop_relay_api.py @@ -2,11 +2,12 @@ import pytest from core.auth.backend import UserContext, get_current_user -from core.channels.protocol import InboundMsg, SendResult +from integrations.channels.protocol import InboundMsg, SendResult from core.db.engine import Base, get_db -from core.db.models import ChannelConnection, UserShadow +from integrations.channels.models import ChannelConnection +from core.db.models import UserShadow from core.infra.exceptions import BadRequestError -from core.services.channel_relay import ChannelRelayService +from integrations.channels.relay import ChannelRelayService from fastapi import FastAPI from fastapi.testclient import TestClient from sqlalchemy import create_engine @@ -17,9 +18,9 @@ @pytest.fixture def api(monkeypatch): from api.middleware.error_handler import setup_error_handlers - from api.routes.v1.channel_desktop import router as desktop_router - from api.routes.v1.channels import router - from api.routes.v1.desktop_capability import _require_capability_user + from integrations.channels.api.channel_desktop import router as desktop_router + from integrations.channels.api.channels import router + from features.desktop.api.capability_dependencies import require_capability_user as _require_capability_user engine = create_engine( "sqlite://", poolclass=StaticPool, connect_args={"check_same_thread": False} @@ -59,9 +60,9 @@ async def send(self, conn, msg, text): async def download(self, conn, msg, att): return b"local-fixture" - monkeypatch.setattr("core.channels.adapters.lark.LarkAdapter.validate_credentials", validate) - monkeypatch.setattr("core.channels.adapters.lark.LarkAdapter.send_text", send) - monkeypatch.setattr("core.channels.adapters.lark.LarkAdapter.download_resource", download) + monkeypatch.setattr('integrations.channels.adapters.lark.LarkAdapter.validate_credentials', validate) + monkeypatch.setattr('integrations.channels.adapters.lark.LarkAdapter.send_text', send) + monkeypatch.setattr('integrations.channels.adapters.lark.LarkAdapter.download_resource', download) with TestClient(app) as client: yield client, factory, sent diff --git a/src/backend/tests/test_channel_desktop_worker.py b/src/backend/tests/integrations/channels/test_channel_desktop_worker.py similarity index 74% rename from src/backend/tests/test_channel_desktop_worker.py rename to src/backend/tests/integrations/channels/test_channel_desktop_worker.py index 5ee3b5f4..604fba7e 100644 --- a/src/backend/tests/test_channel_desktop_worker.py +++ b/src/backend/tests/integrations/channels/test_channel_desktop_worker.py @@ -1,23 +1,22 @@ """Local runtime integration; model/network boundaries use deterministic fixtures.""" +from features.desktop.bridge import identity as bridge_identity import asyncio from dataclasses import asdict +import core.agent_runtime.runner as runner +import core.chat.runtime.compaction as compaction +import core.chat.runtime.followups as followups +import core.chat.runtime.recovery as recovery +import core.runs.tasks as tasks import fakeredis.aioredis import pytest -from core.channels.protocol import ChannelCaps, InboundMsg, SendResult +from integrations.channels.protocol import ChannelCaps, InboundMsg, SendResult from core.db.engine import Base -from core.db.models import ( - ChannelConnection, - ChatMessage, - ChatRun, - ChatSession, - ContentBlock, - UserShadow, -) -from core.services import channel_desktop_worker as worker -from core.services.run_journal import RunJournal -from orchestration import chat_run_executor as executor +from integrations.channels.models import ChannelConnection +from core.db.models import ChatMessage, ChatRun, ChatSession, ContentBlock, UserShadow +from integrations.channels import desktop_worker as worker +from core.runs.journal import RunJournal from sqlalchemy import create_engine, select from sqlalchemy.orm import sessionmaker @@ -34,19 +33,20 @@ def runtime(tmp_path, monkeypatch): db.commit() for target in ( "core.db.engine.SessionLocal", - "core.channels.inbound.SessionLocal", - "orchestration.chat_run_executor.SessionLocal", - "orchestration.tool_effect_recovery.SessionLocal", + "integrations.channels.inbound.SessionLocal", + "integrations.channels.inbound_attachments.SessionLocal", + 'core.db.engine.SessionLocal', + 'core.runs.tool_effect_recovery.SessionLocal', ): monkeypatch.setattr(target, sessions) redis = fakeredis.aioredis.FakeRedis(decode_responses=True) - monkeypatch.setattr("orchestration.run_event_stream.get_redis", lambda **_: redis) - monkeypatch.setattr(executor, "_spawn_followup_task", lambda **_: None) - monkeypatch.setattr(executor, "_spawn_compaction_task", lambda **_: None) - monkeypatch.setattr("core.services.artifact_service.persist_artifacts", lambda *a, **kw: None) - executor._active_runs.clear() + monkeypatch.setattr('core.runs.events.store.get_redis', lambda **_: redis) + monkeypatch.setattr(followups, "_spawn_followup_task", lambda **_: None) + monkeypatch.setattr(compaction, "_spawn_compaction_task", lambda **_: None) + monkeypatch.setattr('core.files.artifacts.service.persist_artifacts', lambda *a, **kw: None) + tasks.process_state().active_runs.clear() yield sessions - executor._active_runs.clear() + tasks.process_state().active_runs.clear() engine.dispose() @@ -54,8 +54,8 @@ async def test_local_delivery_uses_default_chat_and_preserves_owner(runtime, tmp sessions = runtime identity = {"cloud_base": "https://cloud.invalid", "subject": "account", "device_id": "pc"} captured = dict(identity) - monkeypatch.setattr(worker.bridge, "require_current_account", lambda _: None) - monkeypatch.setattr("core.services.channel_relay_dispatch.bridge_enabled", lambda: True) + monkeypatch.setattr(bridge_identity, "require_current_account", lambda _: None) + monkeypatch.setattr('integrations.channels.relay_dispatch.bridge_enabled', lambda: True) with sessions() as db: db.add( ContentBlock( @@ -94,7 +94,7 @@ async def workflow(**kwargs): yield {"type": "content", "delta": fixture_file.read_text()} yield {"type": "meta", "route": "main", "usage": {"input_tokens": 1, "output_tokens": 1}} - monkeypatch.setattr(executor, "astream_chat_workflow", workflow) + monkeypatch.setattr(runner, "astream_chat_workflow", workflow) sent = [] async def cloud(_captured, path, payload): @@ -146,7 +146,7 @@ async def test_restart_cancels_local_run_before_recovery(runtime, monkeypatch): request_payload={"kind": "chat", "source": "desktop_channel"}, recovery_snapshot={"kind": "chat", "worker_args": {"context": {"user_id": "local-owner"}}}, ) - assert await executor.recover_orphan_runs() == 1 + assert await recovery.recover_orphan_runs() == 1 with sessions() as db: assert db.get(ChatRun, "local-run").status == "cancelled" - assert "local-run" not in executor._active_runs + assert "local-run" not in tasks.process_state().active_runs diff --git a/src/backend/tests/test_channel_receiver_lifecycle.py b/src/backend/tests/integrations/channels/test_channel_receiver_lifecycle.py similarity index 94% rename from src/backend/tests/test_channel_receiver_lifecycle.py rename to src/backend/tests/integrations/channels/test_channel_receiver_lifecycle.py index ac9a0885..3354a4b1 100644 --- a/src/backend/tests/test_channel_receiver_lifecycle.py +++ b/src/backend/tests/integrations/channels/test_channel_receiver_lifecycle.py @@ -6,13 +6,13 @@ import time import httpx -from core.channels.manager import ChannelManager +from integrations.channels.manager import ChannelManager from core.db.engine import Base from core.db.models import UserShadow from core.infra.ephemeral import LocalEphemeralState from httpx import _client from sqlalchemy import create_engine -from tests.test_channel_desktop_relay_api import api, headers # noqa: F401 +from tests.integrations.channels.test_channel_desktop_relay_api import api, headers # noqa: F401 async def eventually(check, seconds=2): @@ -37,10 +37,10 @@ async def test_binding_on_non_owner_is_received_and_claimed_on_desktop(api, monk state = LocalEphemeralState() monkeypatch.setattr("core.infra.ephemeral.get_ephemeral_state", lambda: state) monkeypatch.setattr("core.db.engine.SessionLocal", factory) - monkeypatch.setattr("core.channels.inbound.SessionLocal", factory) - monkeypatch.setattr("core.channels.manager._RECONCILE_SECONDS", 0.02, raising=False) + monkeypatch.setattr('integrations.channels.inbound.SessionLocal', factory) + monkeypatch.setattr('integrations.channels.manager._RECONCILE_SECONDS', 0.02, raising=False) follower = ChannelManager() - monkeypatch.setattr("core.channels.manager._manager", follower) + monkeypatch.setattr('integrations.channels.manager._manager', follower) ready = threading.Event() entered_poll = threading.Event() delivered = threading.Event() diff --git a/src/backend/tests/test_weixin_connection_health.py b/src/backend/tests/integrations/channels/test_weixin_connection_health.py similarity index 95% rename from src/backend/tests/test_weixin_connection_health.py rename to src/backend/tests/integrations/channels/test_weixin_connection_health.py index 066bc67b..1fd4ba7c 100644 --- a/src/backend/tests/test_weixin_connection_health.py +++ b/src/backend/tests/integrations/channels/test_weixin_connection_health.py @@ -4,10 +4,10 @@ import httpx import pytest -from core.channels.adapters.weixin import WeixinAdapter +from integrations.channels.adapters.weixin import WeixinAdapter from core.infra.crypto import encrypt_secret from httpx import _client -from tests.test_channel_desktop_relay_api import api # noqa: F401 +from tests.integrations.channels.test_channel_desktop_relay_api import api # noqa: F401 @pytest.mark.parametrize( @@ -119,7 +119,7 @@ def respond(request): def test_http_credential_test_does_not_fabricate_receiver_health( api, monkeypatch, body, expected_http ): - from core.db.models import ChannelConnection + from integrations.channels.models import ChannelConnection client, factory, _ = api with factory() as db: @@ -180,8 +180,8 @@ def unexpected_request(**kwargs): def test_qr_binding_preserves_user_for_new_and_existing_bot(api, monkeypatch): from core.infra.ephemeral import LocalEphemeralState - from core.services.channel_service import ChannelService - from core.db.models import ChannelConnection + from integrations.channels.service import ChannelService + from integrations.channels.models import ChannelConnection state = LocalEphemeralState() monkeypatch.setattr("core.infra.ephemeral.get_ephemeral_state", lambda: state) diff --git a/src/backend/tests/test_weixin_qr_protocol.py b/src/backend/tests/integrations/channels/test_weixin_qr_protocol.py similarity index 96% rename from src/backend/tests/test_weixin_qr_protocol.py rename to src/backend/tests/integrations/channels/test_weixin_qr_protocol.py index 92860ae0..7a33b03e 100644 --- a/src/backend/tests/test_weixin_qr_protocol.py +++ b/src/backend/tests/integrations/channels/test_weixin_qr_protocol.py @@ -2,11 +2,11 @@ import httpx import pytest -from core.channels.adapters.weixin import WeixinAdapter +from integrations.channels.adapters.weixin import WeixinAdapter from core.infra.ephemeral import LocalEphemeralState -from core.services.channel_service import ChannelService +from integrations.channels.service import ChannelService from httpx import _client -from tests.test_channel_desktop_relay_api import api, headers # noqa: F401 +from tests.integrations.channels.test_channel_desktop_relay_api import api, headers # noqa: F401 @pytest.mark.parametrize( diff --git a/src/backend/tests/test_dingtalk_integration.py b/src/backend/tests/integrations/dingtalk/test_dingtalk_integration.py similarity index 93% rename from src/backend/tests/test_dingtalk_integration.py rename to src/backend/tests/integrations/dingtalk/test_dingtalk_integration.py index c568b1df..cf3fc559 100644 --- a/src/backend/tests/test_dingtalk_integration.py +++ b/src/backend/tests/integrations/dingtalk/test_dingtalk_integration.py @@ -1,3 +1,4 @@ +from features.marketplace import plugin_market """DingTalk (dingtalk / dws) integration unit tests: pure-function parsing, PAT detection, models/repositories, settings, credential-volume degradation, marketplace skill installability. No dependence on real DingTalk/sandbox — the sandbox orchestration part @@ -14,12 +15,8 @@ from core.db.engine import Base import core.db.models # noqa: F401 ensure all models are registered (FK depends on users_shadow) from core.db.models import UserShadow, AdminSkill -from core.db.repository import DingTalkConnectionRepository -from core.services.dingtalk_service import ( - parse_device_login_output, - parse_auth_status, - parse_get_self, -) +from integrations.dingtalk.repository import DingTalkConnectionRepository +from integrations.dingtalk.parsing import parse_device_login_output, parse_auth_status, parse_get_self # sandbox_tool pulls in agentscope via core.llm.__init__; skip related cases when # agentscope is absent locally — they run normally inside the container (the real test environment). @@ -155,7 +152,7 @@ def test_parse_get_self_garbage(): ], ) async def test_verify_and_refresh_verdict(monkeypatch, out, rc, expect): - import core.services.dingtalk_service as ds + import integrations.dingtalk.service as ds async def fake_run_dws(user_id, args, timeout=40): return out, "", rc @@ -166,7 +163,7 @@ async def fake_run_dws(user_id, args, timeout=40): def test_mark_login_expired_sets_disconnected(monkeypatch): # mark_login_expired writes the DB with an independent SessionLocal; stub the underlying _update_connection to verify the persisted fields. - import core.services.dingtalk_service as ds + import integrations.dingtalk.service as ds captured = {} monkeypatch.setattr( @@ -207,7 +204,7 @@ def test_settings_dws_arch_flag(): def test_dingtalk_seeded_in_system_configs(): # Custom App credentials go through system-config dingtalk.* keys (configured visually in the Config admin platform, not in .env) - from core.services.system_config import SEED_CONFIGS + from core.config.store.system_config_constants import SEED_CONFIGS keys = {row[0] for row in SEED_CONFIGS} assert {"dingtalk.client_id", "dingtalk.client_secret", "dingtalk.trusted_domains"} <= keys @@ -226,21 +223,21 @@ def test_creds_volume_degrades_without_host_storage(monkeypatch): runtime_settings.settings, sandbox=replace(runtime_settings.settings.sandbox, opensandbox_host_storage_path=""), )) - from core.sandbox.providers.opensandbox.account_volumes import _make_dws_creds_volumes + from integrations.dingtalk.mounts import mounts as _make_dws_creds_volumes # No local HOST_STORAGE_PATH → quietly return an empty list (sandbox still created, just without the credential volume) assert _make_dws_creds_volumes("u1") == [] def test_creds_volume_rejects_bad_user_id(): - from core.sandbox.providers.opensandbox.account_volumes import _make_dws_creds_volumes + from integrations.dingtalk.mounts import mounts as _make_dws_creds_volumes assert _make_dws_creds_volumes("") == [] assert _make_dws_creds_volumes("../etc/passwd") == [] def test_dws_cache_dir_path(): - from core.sandbox.workspace.credentials import dws_cache_dir + from integrations.dingtalk.storage import dws_cache_dir p = dws_cache_dir("u_abc") assert p.name == "u_abc" @@ -262,7 +259,7 @@ def test_connection_repo_ensure_and_update(db): def test_connection_status_dict(db): - from core.services.dingtalk_service import DingTalkService + from integrations.dingtalk.service import DingTalkService svc = DingTalkService(db) data = svc.get_status("u1") @@ -274,7 +271,8 @@ def test_connection_status_dict(db): def test_status_dict_renders_qr_when_pending(db): - from core.services.dingtalk_service import DingTalkService, make_qr_data_uri + from integrations.dingtalk.service import DingTalkService + from integrations.dingtalk.parsing import make_qr_data_uri if make_qr_data_uri("https://x") is None: pytest.skip("segno 未安装（仅容器内测）") @@ -299,7 +297,7 @@ def test_owned_private_skills_resolve_into_enabled(db): enter their enabled set, getting injected into the agent toolkit + system prompt — not just global skills.""" from core.db.models import AdminSkill, AdminMcpServer - from core.config.catalog_resolver import _owned_enabled_ids + from core.capabilities.catalog.resolver import _owned_enabled_ids # This user's private skills: one enabled, one disabled db.add( @@ -349,7 +347,7 @@ def test_owned_private_skills_resolve_into_enabled(db): def test_owned_override_disables(db): """User turns a private skill off in the capability center (writes CatalogOverride enabled=false) → not in the set.""" from core.db.models import AdminSkill - from core.config.catalog_resolver import _owned_enabled_ids + from core.capabilities.catalog.resolver import _owned_enabled_ids db.add( AdminSkill( @@ -386,9 +384,9 @@ def test_dingtalk_plugin_installable(db): references/scripts ported with the package → install lands as AdminSkill(source_plugin=dingtalk) → detail returns connection for the frontend to render the account-connection panel on the plugin detail page.""" - from core.plugins import management as ps + from core.capabilities.plugins import management as ps - items = ps.list_plugins(db, owner_user_id="u1") + items = plugin_market.list_plugins(db, owner_user_id="u1") dt = next((it for it in items if it["slug"] == "dingtalk"), None) assert dt is not None, "dingtalk 未出现在插件列表" assert dt["installed"] is False diff --git a/src/backend/tests/test_email_integration.py b/src/backend/tests/integrations/email/test_email_integration.py similarity index 91% rename from src/backend/tests/test_email_integration.py rename to src/backend/tests/integrations/email/test_email_integration.py index 7e821704..be4ed742 100644 --- a/src/backend/tests/test_email_integration.py +++ b/src/backend/tests/integrations/email/test_email_integration.py @@ -1,8 +1,11 @@ +from features.marketplace import plugin_market """Email (email / himalaya) integration unit tests: pure functions (provider detection / config rendering), app-password encryption round trip, model/repository, connect/disconnect sync flow (stubbed _verify, no real mail server connection), credentials-volume degradation, plugin installability, route e2e. Real sandbox connections are left for verification on a real machine.""" +import integrations.email.storage as integrations_email_storage +import integrations.email.mounts as integrations_email_mounts from tests.sandbox.provider_test_support import patch_open_settings @@ -14,15 +17,11 @@ from core.db.engine import Base import core.db.models # noqa: F401 ensures all models are registered (FK depends on users_shadow) -from core.db.models import UserShadow, EmailConnection -from core.db.repository import EmailConnectionRepository -from core.services import email_service as es -from core.services.email_service import ( - EmailService, - detect_provider, - render_config_toml, - get_email_config_bundle, -) +from core.db.models import UserShadow +from integrations.email.models import EmailConnection +from integrations.email.repository import EmailConnectionRepository +from integrations.email import service as es +from integrations.email.service import EmailService, detect_provider, render_config_toml, get_email_config_bundle @pytest.fixture @@ -46,10 +45,10 @@ def _redirect_storage(monkeypatch, tmp_path): def cache(uid): return tmp_path / "email_cache" / uid - monkeypatch.setattr(common_credentials, "email_cache_dir", cache) - monkeypatch.setattr(common_credentials, "email_home_dir", lambda uid: cache(uid) / "home") + monkeypatch.setattr(integrations_email_storage, 'email_cache_dir', cache) + monkeypatch.setattr(integrations_email_storage, 'email_home_dir', lambda uid: cache(uid) / "home") monkeypatch.setattr( - common_credentials, "email_himalaya_config", + integrations_email_storage, 'email_himalaya_config', lambda uid: cache(uid) / "home" / ".config" / "himalaya" / "config.toml", ) @@ -158,7 +157,7 @@ async def fake_verify(user_id): decoded = base64.b64decode(rec.config_bundle).decode("utf-8") assert 'backend.auth.raw = "app-pw"' in decoded # config.toml is actually written under that user's HOME - from core.sandbox.workspace.credentials import email_himalaya_config + from integrations.email.storage import email_himalaya_config assert email_himalaya_config("u1").exists() @@ -186,7 +185,7 @@ async def fake_verify(user_id): assert "AUTHENTICATIONFAILED" in (data["last_error"] or "") rec = EmailConnectionRepository(db).get("u1") assert rec.secret_enc is None and rec.config_bundle is None # no credentials kept on failure - from core.sandbox.workspace.credentials import email_himalaya_config + from integrations.email.storage import email_himalaya_config assert not email_himalaya_config("u1").exists() # bad config was purged @@ -211,7 +210,7 @@ async def fake_verify(user_id): assert data["email_address"] is None rec = EmailConnectionRepository(db).get("u1") assert rec.secret_enc is None and rec.config_bundle is None - from core.sandbox.workspace.credentials import email_himalaya_config + from integrations.email.storage import email_himalaya_config assert not email_himalaya_config("u1").exists() @@ -234,7 +233,6 @@ async def fake_verify(user_id): # ── Credentials-volume degradation + paths ─────────────────────────────── def test_creds_volume_degrades_without_host_storage(monkeypatch): from types import SimpleNamespace - import core.sandbox.providers.opensandbox.account_volumes as internals_account_volumes # Explicitly simulate "HOST_STORAGE_PATH not configured" — don't depend on the # runtime environment's .env (a dev machine may actually set HOST_STORAGE_PATH, @@ -245,18 +243,18 @@ def test_creds_volume_degrades_without_host_storage(monkeypatch): email_creds_bind_mount_enabled=True, opensandbox_host_storage_path="", ))) - assert internals_account_volumes._make_email_creds_volumes("u1") == [] # quiet degradation without HOST_STORAGE_PATH + assert integrations_email_mounts.mounts("u1") == [] # quiet degradation without HOST_STORAGE_PATH def test_creds_volume_rejects_bad_user_id(): - from core.sandbox.providers.opensandbox.account_volumes import _make_email_creds_volumes + from integrations.email.mounts import mounts as _make_email_creds_volumes assert _make_email_creds_volumes("") == [] assert _make_email_creds_volumes("../etc/passwd") == [] def test_email_cache_dir_path(): - from core.sandbox.workspace.credentials import email_cache_dir - from core.sandbox.workspace.credentials import email_himalaya_config + from integrations.email.storage import email_cache_dir + from integrations.email.storage import email_himalaya_config p = email_cache_dir("u_abc") assert p.name == "u_abc" and p.parent.name == "email_cache" cfg = email_himalaya_config("u_abc") @@ -275,9 +273,9 @@ def test_email_plugin_installable(db): connection so the frontend renders the account-connection panel on the plugin detail page.""" from core.db.models import AdminSkill - from core.plugins import management as ps + from core.capabilities.plugins import management as ps - items = ps.list_plugins(db, owner_user_id="u1") + items = plugin_market.list_plugins(db, owner_user_id="u1") em = next((it for it in items if it["slug"] == "email"), None) assert em is not None, "email 未出现在插件列表" assert em["installed"] is False diff --git a/src/backend/tests/integrations/lark/test_app_lifecycle.py b/src/backend/tests/integrations/lark/test_app_lifecycle.py new file mode 100644 index 00000000..d0cccba8 --- /dev/null +++ b/src/backend/tests/integrations/lark/test_app_lifecycle.py @@ -0,0 +1,48 @@ +"""Shared app initialization stays singular and reset cancels its task.""" +import asyncio +import pytest +from integrations.lark import configuration, state, storage +from integrations.lark.service import LarkService + + +@pytest.mark.asyncio +async def test_pending_app_is_shared_and_reset_cancels_task(tmp_path, monkeypatch): + monkeypatch.setattr(state, "APP_INIT_TASK", None) + monkeypatch.setattr(state, "app_init_state", {"status": "idle"}) + monkeypatch.setattr(configuration, "app_configured", lambda: None) + home = tmp_path / "app-home" + home.mkdir() + (home / "config.json").write_text("{}") + monkeypatch.setattr(storage, "lark_app_home_dir", lambda: home) + calls = [] + + async def initialize(): + calls.append("start") + state.app_init_state.update(status="pending", verification_url="https://example.test/setup") + await asyncio.Future() + + monkeypatch.setattr(configuration, "app_init_flow", initialize) + service = LarkService(None) + try: + result = await service.start_app_init() + task = state.APP_INIT_TASK + assert result["verification_url"] == "https://example.test/setup" + assert await LarkService(None).start_app_init() == result + assert state.APP_INIT_TASK is task and calls == ["start"] + reset = await service.reset_app() + with pytest.raises(asyncio.CancelledError): + await task + assert reset["status"] == "idle" and state.APP_INIT_TASK is None + assert not home.exists() + finally: + if state.APP_INIT_TASK is not None: + state.APP_INIT_TASK.cancel() + + +@pytest.mark.asyncio +async def test_configured_app_does_not_spawn_initialization(monkeypatch): + monkeypatch.setattr(configuration, "app_configured", lambda: "configured-app") + async def forbidden(): + raise AssertionError("already configured app spawned initialization") + monkeypatch.setattr(configuration, "app_init_flow", forbidden) + assert (await LarkService(None).start_app_init())["app_id"] == "configured-app" diff --git a/src/backend/tests/test_lark_auth_status.py b/src/backend/tests/integrations/lark/test_lark_auth_status.py similarity index 79% rename from src/backend/tests/test_lark_auth_status.py rename to src/backend/tests/integrations/lark/test_lark_auth_status.py index afb31fa8..a603b9bc 100644 --- a/src/backend/tests/test_lark_auth_status.py +++ b/src/backend/tests/integrations/lark/test_lark_auth_status.py @@ -17,13 +17,16 @@ """ from __future__ import annotations +from integrations.lark import login as lark_login +from integrations.lark import parsing as lark_parsing +from integrations.lark import process as lark_process import asyncio import json import pytest -from core.services import lark_service as ls +from integrations.lark import service as ls _EXPIRED_STATUS = json.dumps( { @@ -83,21 +86,21 @@ def test_expired_user_identity_is_not_authenticated(): """The crux: openId/userName are still present, but the login is dead.""" - assert ls.parse_user_identity_state(_EXPIRED_STATUS) is False - assert ls.is_authenticated(_EXPIRED_STATUS) is False + assert lark_parsing.parse_user_identity_state(_EXPIRED_STATUS) is False + assert lark_parsing.is_authenticated(_EXPIRED_STATUS) is False def test_ready_user_identity_is_authenticated(): - assert ls.parse_user_identity_state(_READY_STATUS) is True - assert ls.is_authenticated(_READY_STATUS) is True + assert lark_parsing.parse_user_identity_state(_READY_STATUS) is True + assert lark_parsing.is_authenticated(_READY_STATUS) is True def test_unrecognized_shape_falls_back_to_the_lenient_heuristics(): """Device-flow output carries no ``identities``; the old path must still serve it.""" - assert ls.parse_user_identity_state(_DEVICE_LOGIN) is None - assert ls.parse_user_identity_state("not json at all") is None - assert ls.parse_user_identity_state(json.dumps({"authenticated": True})) is None - assert ls.is_authenticated(json.dumps({"authenticated": True})) is True + assert lark_parsing.parse_user_identity_state(_DEVICE_LOGIN) is None + assert lark_parsing.parse_user_identity_state("not json at all") is None + assert lark_parsing.parse_user_identity_state(json.dumps({"authenticated": True})) is None + assert lark_parsing.is_authenticated(json.dumps({"authenticated": True})) is True # -- The CLI is never handed a flag it does not accept ------------------- @@ -110,14 +113,14 @@ async def _fake_run(_user_id, args, **_kw): calls.append(list(args)) return stdout, "", 0 - monkeypatch.setattr(ls, "_run_lark", _fake_run) + monkeypatch.setattr(lark_process, 'run_lark', _fake_run) return calls def test_verify_and_refresh_never_passes_format_json(monkeypatch): calls = _record_run(monkeypatch, _READY_STATUS) - assert asyncio.run(ls.verify_and_refresh("u1")) == "valid" + assert asyncio.run(lark_login.verify_and_refresh("u1")) == "valid" assert calls == [["auth", "status", "--verify"]] assert all("--format" not in c for c in calls) @@ -126,23 +129,23 @@ def test_verify_and_refresh_never_passes_format_json(monkeypatch): def test_verify_and_refresh_reports_expired_login_as_invalid(monkeypatch): _record_run(monkeypatch, _EXPIRED_STATUS) - assert asyncio.run(ls.verify_and_refresh("u1")) == "invalid" + assert asyncio.run(lark_login.verify_and_refresh("u1")) == "invalid" def test_verify_and_refresh_stays_unknown_on_timeout(monkeypatch): async def _timeout(_user_id, _args, **_kw): return "", "lark-cli auth status timeout", 124 - monkeypatch.setattr(ls, "_run_lark", _timeout) + monkeypatch.setattr(lark_process, 'run_lark', _timeout) - assert asyncio.run(ls.verify_and_refresh("u1")) == "unknown" + assert asyncio.run(lark_login.verify_and_refresh("u1")) == "unknown" def test_build_connected_update_backfills_identity(monkeypatch): """With the bogus flag gone, open_id / name finally land in the DB row.""" calls = _record_run(monkeypatch, _READY_STATUS) - data = asyncio.run(ls._build_connected_update("u1")) + data = asyncio.run(lark_login.build_connected_update("u1")) assert calls == [["auth", "status"]] assert data["status"] == "connected" @@ -161,9 +164,9 @@ async def _fake_run(_user_id, args, **_kw): return json.dumps({"ok": True}), "", login_rc return status_stdout, "", 0 - monkeypatch.setattr(ls, "_run_lark", _fake_run) - monkeypatch.setattr(ls, "_update_connection", lambda uid, data: captured.update(data)) - asyncio.run(ls._device_complete_flow("u1", "dc_abc")) + monkeypatch.setattr(lark_process, 'run_lark', _fake_run) + monkeypatch.setattr(lark_login, 'update_connection', lambda uid, data: captured.update(data)) + asyncio.run(lark_login.device_complete_flow("u1", "dc_abc")) return captured diff --git a/src/backend/tests/test_yida_integration.py b/src/backend/tests/integrations/yida/test_yida_integration.py similarity index 94% rename from src/backend/tests/test_yida_integration.py rename to src/backend/tests/integrations/yida/test_yida_integration.py index 31f1a755..dbcb9621 100644 --- a/src/backend/tests/test_yida_integration.py +++ b/src/backend/tests/integrations/yida/test_yida_integration.py @@ -1,3 +1,4 @@ +from features.marketplace import plugin_market """Shared Yida plugin, connection service, path, and SKILL.md regression tests. No dependency on real Yida or a real sandbox. EE persistent-sandbox state tests @@ -35,8 +36,8 @@ def test_settings_yida_arch_flag(): def test_yida_cache_dir_path(): - from core.sandbox.workspace.credentials import yida_cache_dir - from core.sandbox.workspace.credentials import yida_workspace_dir + from integrations.yida.storage import yida_cache_dir + from integrations.yida.storage import yida_workspace_dir p = yida_cache_dir("u_abc") assert p.name == "u_abc" @@ -47,7 +48,7 @@ def test_yida_cache_dir_path(): def test_yida_shared_workspace_dir_path(): """script-runner shared sandbox working directory: one per deployment (__shared__), rooted alongside the per-user directories.""" - from core.sandbox.workspace.credentials import yida_shared_workspace_dir + from integrations.yida.storage import yida_shared_workspace_dir p = yida_shared_workspace_dir() assert p.name == "workspace" @@ -57,7 +58,7 @@ def test_yida_shared_workspace_dir_path(): # ── Connection panel service (cookie cache + real read-only liveness reconciliation) ── def test_yida_extract_result_json(): - from core.services.yida_service import extract_result_json + from integrations.yida.service import extract_result_json out = ( "some sandbox noise\n" @@ -74,7 +75,7 @@ def test_yida_extract_result_json(): async def test_yida_service_status_lifecycle(tmp_path, monkeypatch): - import core.services.yida_service as ys + import integrations.yida.service as ys monkeypatch.setattr(ys, "_host_workspace_dir", lambda uid: tmp_path / uid / "workspace") await ys._clear_pending("u1") @@ -115,7 +116,7 @@ async def test_yida_probe_status_reconciles_real_login( import json as _json import os - import core.services.yida_service as ys + import integrations.yida.service as ys monkeypatch.setattr(ys, "_host_workspace_dir", lambda uid: tmp_path / uid / "workspace") await ys._clear_pending("u1") @@ -159,7 +160,7 @@ async def fake_run(user_id, command, timeout): # noqa: ARG001 @pytest.mark.asyncio async def test_yida_probe_without_cookie_does_not_start_sandbox(tmp_path, monkeypatch): - import core.services.yida_service as ys + import integrations.yida.service as ys monkeypatch.setattr(ys, "_host_workspace_dir", lambda uid: tmp_path / uid / "workspace") svc = ys.YidaService() @@ -177,7 +178,7 @@ async def test_yida_service_poll_transitions(tmp_path, monkeypatch): non-JSON (not scanned / timeout) → stays pending.""" import json as _json - import core.services.yida_service as ys + import integrations.yida.service as ys monkeypatch.setattr(ys, "_host_workspace_dir", lambda uid: tmp_path / uid / "workspace") await ys._clear_pending("u1") @@ -280,7 +281,7 @@ async def test_yida_run_in_sandbox_heals_stale_sandbox(monkeypatch): truly destroys it, then retry with a fresh sandbox. This was the pitfall the connection panel hit on first release: both the no-session_id light-pool path and old-sandbox adopt reuse trigger it. """ - import core.services.yida_service as ys + import integrations.yida.service as ys calls = {"exec": 0, "closed": []} @@ -327,9 +328,9 @@ def test_yida_plugin_declares_connection(): → the frontend plugin detail page renders the YidaConnect panel.""" import pathlib - from core.plugins.packaging.importer import normalize_plugin_dir + from core.capabilities.plugins.packaging.importer import normalize_plugin_dir - p = pathlib.Path(__file__).resolve().parents[1] / "extensions/plugins" / "marketplace" / "yida" + p = pathlib.Path(__file__).resolve().parents[3] / "extensions/plugins" / "marketplace" / "yida" assert normalize_plugin_dir(p).connection == "yida" @@ -339,9 +340,9 @@ def test_yida_plugin_installable(db): panel on the plugin detail page, in-conversation scan as fallback), contains a single skill, the full references/subskills set travels with the package → install lands AdminSkill(source_plugin=yida).""" - from core.plugins import management as ps + from core.capabilities.plugins import management as ps - items = ps.list_plugins(db, owner_user_id="u1") + items = plugin_market.list_plugins(db, owner_user_id="u1") yd = next((it for it in items if it["slug"] == "yida"), None) assert yd is not None, "yida 未出现在插件列表" assert yd["installed"] is False @@ -373,7 +374,7 @@ def test_yida_skill_md_host_adaptation(): import pathlib p = ( - pathlib.Path(__file__).resolve().parents[1] + pathlib.Path(__file__).resolve().parents[3] / "extensions/plugins" / "marketplace" / "yida" diff --git a/src/backend/tests/llm/command_test_support.py b/src/backend/tests/llm/command_test_support.py index b02ce8b8..9c398c6f 100644 --- a/src/backend/tests/llm/command_test_support.py +++ b/src/backend/tests/llm/command_test_support.py @@ -1,11 +1,14 @@ """Isolated ledger state for process-tool integration tests.""" + +import core.sandbox.commands.runtime as commands_runtime +import core.sandbox.commands.runtime as command_runtime import pytest -from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker from core.db.engine import Base from core.db.models import ChatSession, UserShadow -from core.services.commands import store -from orchestration.commands import runtime as command_runtime +from core.sandbox.commands import store +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + @pytest.fixture async def command_tool_state(tmp_path, monkeypatch): @@ -22,7 +25,9 @@ async def command_tool_state(tmp_path, monkeypatch): db.commit() monkeypatch.setattr(store, "SessionLocal", factory) runtime = command_runtime.CommandRuntime() - monkeypatch.setattr(command_runtime, "runtime", runtime) + monkeypatch.setattr(commands_runtime, "runtime", runtime) + from core.llm.ports import process_state + monkeypatch.setattr(process_state(), "commands", runtime) yield factory await runtime.shutdown() engine.dispose() diff --git a/src/backend/tests/llm/e2e_multiturn_toolcall.py b/src/backend/tests/llm/e2e_multiturn_toolcall.py index 68da0f7a..29953027 100644 --- a/src/backend/tests/llm/e2e_multiturn_toolcall.py +++ b/src/backend/tests/llm/e2e_multiturn_toolcall.py @@ -128,8 +128,8 @@ def ck(name: str, cond: bool, detail: str = ""): ck("T2 真实执行成功（输出含 sentinel2）", S2 in blob2) # ── Context reconstruction: message bodies the next turn sees must have no residue ── - from core.services.chat_service import ChatService - from core.services.compaction_service import load_session_history + from core.chat.service import ChatService + from core.chat.compaction.service import load_session_history db = SessionLocal() try: diff --git a/src/backend/tests/llm/file_context_hook_user_id_gating_selftest.py b/src/backend/tests/llm/file_context_hook_user_id_gating_selftest.py index d40ba499..6df99c96 100644 --- a/src/backend/tests/llm/file_context_hook_user_id_gating_selftest.py +++ b/src/backend/tests/llm/file_context_hook_user_id_gating_selftest.py @@ -2,7 +2,7 @@ Background ---------- -`core/llm/hooks.py::_build_file_context` assembles the parsed text / xlsx +`core/llm/runtime/hooks.py::_build_file_context` assembles the parsed text / xlsx preview / image base64 of ``uploaded_files`` (i.e. the ``attachments`` in the request body) into context injected into agent memory, as input to ReAct reasoning. @@ -12,7 +12,7 @@ 1. ``_build_file_context`` → ``fetch_parsed_text(fid)`` — user_id was not passed, so the ownership check inside - :func:`core.content.artifact_reader.fetch_parsed_text` + :func:`core.files.artifacts.reader.fetch_parsed_text` ``if user_id and art.user_id != user_id:`` was **skipped entirely** because user_id was falsy. 2. ``_build_xlsx_preview_block`` → ``_download_artifact_bytes(...)`` — @@ -322,8 +322,8 @@ class _StorageError(Exception): infra_exc = _types.SimpleNamespace(StorageError=_StorageError) fake_modules = { - "core.content.artifact_reader": artifact_reader_stub, - "core.storage": storage_stub, + "core.files.artifacts.reader": artifact_reader_stub, + "core.files.storage": storage_stub, "core.infra.exceptions": infra_exc, } diff --git a/src/backend/tests/llm/local_project_test_support.py b/src/backend/tests/llm/local_project_test_support.py index ad248493..6a9a2094 100644 --- a/src/backend/tests/llm/local_project_test_support.py +++ b/src/backend/tests/llm/local_project_test_support.py @@ -9,13 +9,13 @@ from core.db.models import Project, UserShadow, ChatSession -from core.llm import workspace +from core.llm.runtime import workspace -from core.llm.tool_collector import ToolCollector +from core.llm.factory.tools.definitions import ToolAssembly from core.llm.tools.pin_tool import register_pin_to_workspace -from core.services.project_scope import ProjectScope +from core.projects.edition import ProjectScope @pytest.fixture @@ -27,14 +27,14 @@ def local_project(tmp_path, db_session, monkeypatch): from api.routes import files monkeypatch.setattr(files, "local_mode_enabled", lambda: True) - from core.artifacts import store + from core.files.artifacts import store monkeypatch.setattr(store, "_STORE_DIR", tmp_path / "artifacts") monkeypatch.setattr("core.db.engine.SessionLocal", sessionmaker(bind=db_session.get_bind())) root = tmp_path / "project" root.mkdir() monkeypatch.setenv("HUGAGENT_HOME", str(tmp_path / "home")) - from core.services.local_grant_service import add_grant + from features.desktop.local.grants import add_grant add_grant(str(root)) source = root / "report.txt" @@ -64,11 +64,7 @@ def local_project(tmp_path, db_session, monkeypatch): local_slug="reports", local_path=str(root), ) - from core.llm.tool_permissions import ( - CURRENT_PERMISSION_TICKET, - PermissionTicket, - PermissionIntent, - ) + from core.sandbox.permissions.dispatch_models import CURRENT_PERMISSION_TICKET, PermissionTicket, PermissionIntent ticket_token = CURRENT_PERMISSION_TICKET.set( PermissionTicket( @@ -87,16 +83,13 @@ def local_project(tmp_path, db_session, monkeypatch): def pin_tool(scope, session_id=None): - collector = ToolCollector() + collector = ToolAssembly() register_pin_to_workspace(collector, scope=scope, sandbox_session_id=session_id) async def invoke(**args): - from core.llm.tool_permissions import ( - ToolPermissionRegistry, - ToolPermissionService, - PermissionRuntime, - CURRENT_PERMISSION_TICKET, - ) + from core.sandbox.permissions.tool_specs import ToolPermissionRegistry + from core.llm.tool_permissions import ToolPermissionService + from core.sandbox.permissions.dispatch_models import PermissionRuntime, CURRENT_PERMISSION_TICKET registry = ToolPermissionRegistry() registry.register( @@ -115,7 +108,7 @@ async def invoke(**args): raise PermissionError(str(outcome.payload)) token = CURRENT_PERMISSION_TICKET.set(outcome.ticket) try: - return await collector.get_tool("pin_to_workspace")._func(**args) + return await next(t for t in collector if t.name == "pin_to_workspace")._func(**args) finally: CURRENT_PERMISSION_TICKET.reset(token) diff --git a/src/backend/tests/llm/mcp_env_debug.py b/src/backend/tests/llm/mcp_env_debug.py index 4a9c4acf..e54c319a 100644 --- a/src/backend/tests/llm/mcp_env_debug.py +++ b/src/backend/tests/llm/mcp_env_debug.py @@ -9,8 +9,12 @@ async def main(): + from features import register_model_credentials, register_product_tools, register_run_capabilities + register_product_tools() + register_run_capabilities() + register_model_credentials() from core.llm.factory import create_agent_executor - from core.llm.mcp_manager import close_clients + from core.llm.mcp.manager import close_clients agent, clients = await create_agent_executor(agent_spec=None, disable_tools=False) diff --git a/src/backend/tests/llm/test_agent_api_execution.py b/src/backend/tests/llm/test_agent_api_execution.py index c1c62afe..9ea05349 100644 --- a/src/backend/tests/llm/test_agent_api_execution.py +++ b/src/backend/tests/llm/test_agent_api_execution.py @@ -23,8 +23,11 @@ def test_scoped_factory_executes_bound_model_with_isolated_tools_and_offloader(t from agentscope.message import UserMsg from core.db.engine import Base, engine, SessionLocal from core.db.models import UserAgent, UserShadow, ProfileMemory -from core.db.model_repository import create_provider, assign_role -from core.evolution import runtime_binding +from core.models.repository import create_provider, assign_role +from core.evolution.cycle import runtime_binding +from features import register_product_tools, register_run_capabilities +register_product_tools() +register_run_capabilities() from core.llm.factory import create_agent_executor Base.metadata.create_all(engine) @@ -115,12 +118,12 @@ async def run(): assert agent.state.model_provider_id == child_id assert agent.state.model_pinned is True policies = { - type(policy).__name__ + policy.name for adapter in agent._reply_middlewares - for policy in getattr(adapter, "legacy_middlewares", ()) + for policy in adapter.bus.registered_specs() } - assert "FileContextMiddleware" in policies - assert "FinishPinGuardMiddleware" not in policies, "automatic pinning bypasses scoped artifact checks" + assert "attachment_context" in policies + assert "finish_pin_guard" not in policies, "automatic pinning bypasses scoped artifact checks" schemas = {item["function"]["name"]: item["function"] for item in agent._jx_compaction_tool_schemas} assert {"Bash", "write_stdin", "sandbox_get_artifact", "pin_to_workspace", "read_artifact"} <= schemas.keys() diff --git a/src/backend/tests/llm/test_agent_api_runtime.py b/src/backend/tests/llm/test_agent_api_runtime.py index 55cb8e46..418989f8 100644 --- a/src/backend/tests/llm/test_agent_api_runtime.py +++ b/src/backend/tests/llm/test_agent_api_runtime.py @@ -3,7 +3,7 @@ import hashlib import pytest -from core.llm.agent_api_runtime import apply_api_scope, parse_api_scope +from core.llm.api_runtime.agent import apply_api_scope, parse_api_scope def scope(**updates): @@ -65,7 +65,9 @@ class Collector: def __init__(self): self.tools = {} - def register_tool_function(self, func, **kwargs): + def append(self, tool): + func = tool._func + kwargs = {"func_description": tool.description, "permission": getattr(tool, "dispatch_spec", None)} self.tools[func.__name__] = func @@ -73,7 +75,7 @@ def register_tool_function(self, func, **kwargs): async def test_api_bash_never_uses_account_identity_or_shared_process_runner(monkeypatch): import json import core.sandbox - from core.llm.agent_api_tools import register_agent_api_tools + from core.llm.api_runtime.agent_tools import register_agent_api_tools class Provider: name = "script_runner" @@ -103,7 +105,7 @@ async def start_process(self, req, yield_time_ms): def test_mcp_scope_replaces_identity_and_preserves_exact_kb_grants(): - from core.llm.agent_api_runtime import scope_mcp_servers + from core.llm.api_runtime.agent import scope_mcp_servers parsed = parse_api_scope( scope(), owner_user_id="owner", chat_id="api_chat", agent_id="agent_123" @@ -135,16 +137,16 @@ def test_mcp_scope_replaces_identity_and_preserves_exact_kb_grants(): async def test_pin_rejects_private_artifact_and_only_accepts_file_ids(monkeypatch): import inspect import json - import core.llm.agent_api_tools as tools - from core.llm.tool_collector import ToolCollector + import core.llm.api_runtime.agent_tools as tools + from core.llm.factory.tools.definitions import ToolAssembly parsed = parse_api_scope( scope(), owner_user_id="owner", chat_id="api_chat", agent_id="agent_123" ) - collector = ToolCollector() + collector = ToolAssembly() tools.register_agent_api_tools(collector, parsed) assert all( - collector.get_tool(name) is not None + any(t.name == name for t in collector) for name in ( "Bash", "write_stdin", @@ -153,7 +155,7 @@ async def test_pin_rejects_private_artifact_and_only_accepts_file_ids(monkeypatc "pin_to_workspace", ) ) - pin = collector.get_tool("pin_to_workspace")._func + pin = next(t for t in collector if t.name == "pin_to_workspace")._func assert "file_paths" not in inspect.signature(pin).parameters monkeypatch.setattr(tools, "artifact_in_scope", lambda *_: False) result = await pin(["private"]) @@ -205,7 +207,7 @@ def artifact_db(tmp_path, monkeypatch): def test_artifacts_require_the_same_session_provenance_and_respect_deletion(artifact_db): from datetime import datetime, timezone from core.db.models import Artifact, ChatSession - from core.llm.agent_api_runtime import artifact_in_scope + from core.llm.api_runtime.agent import artifact_in_scope parsed = parse_api_scope( scope(), owner_user_id="owner", chat_id="api_chat", agent_id="agent_123" @@ -223,7 +225,7 @@ def test_artifacts_require_the_same_session_provenance_and_respect_deletion(arti def test_pending_output_requires_trusted_metadata_before_store_resolution(artifact_db, monkeypatch): - from core.llm.agent_api_runtime import artifact_in_scope + from core.llm.api_runtime.agent import artifact_in_scope parsed = parse_api_scope( scope(), owner_user_id="owner", chat_id="api_chat", agent_id="agent_123" @@ -234,7 +236,7 @@ def test_pending_output_requires_trusted_metadata_before_store_resolution(artifa "chat_id": "api_chat", "agent_api_key_id": "ak_123", } - monkeypatch.setattr("core.artifacts.store._read_record", lambda _: {"metadata": metadata}) + monkeypatch.setattr('core.files.artifacts.store._read_record', lambda _: {"metadata": metadata}) assert artifact_in_scope("pending", parsed) metadata["agent_api_key_id"] = "another_key" assert not artifact_in_scope("pending", parsed) @@ -243,7 +245,7 @@ def test_pending_output_requires_trusted_metadata_before_store_resolution(artifa @pytest.mark.asyncio async def test_selected_skills_are_staged_once_without_other_owner_packages(tmp_path): - from core.llm.agent_api_skills import ApiSkillStager + from core.llm.api_runtime.agent_skills import ApiSkillStager selected = tmp_path / "selected" selected.mkdir() @@ -275,7 +277,7 @@ async def put_file(self, session, path, content, user_id): def test_skill_staging_rejects_escaping_packages(tmp_path): - from core.llm.agent_api_skills import _skill_files + from core.llm.api_runtime.agent_skills import _skill_files package = tmp_path / "selected" package.mkdir() @@ -297,7 +299,6 @@ async def test_api_sandbox_never_uses_general_pool_or_host_mounts(monkeypatch): user_pool = SimpleNamespace(acquire=AsyncMock(return_value=SimpleNamespace(id="api-container"))) general_pool = SimpleNamespace(acquire=AsyncMock()) host = SimpleNamespace(_jupyter_user_pool=user_pool, _pool=general_pool) - monkeypatch.setattr(sessions, "_user_bound_sandbox_required", lambda: False) result = await sessions._OpenSandboxCreationMixin._create_session(host, "api_ak_123") assert result.user_id == "api_ak_123" user_pool.acquire.assert_awaited_once_with("api_ak_123") @@ -308,10 +309,7 @@ def reject_mount(*args): for name in ( "_make_skills_volumes", - "_make_dws_creds_volumes", - "_make_lark_creds_volumes", - "_make_email_creds_volumes", - "_make_yida_creds_volumes", + "credential_mounts", ): monkeypatch.setattr(sessions, name, reject_mount) create = AsyncMock(return_value=SimpleNamespace(id="new-container")) @@ -320,7 +318,8 @@ def reject_mount(*args): ) host = sessions._OpenSandboxCreationMixin() host._make_config = lambda: None - host._ttl_s = 60 + host._create_owned_sandbox = create + host._session_expiry_s = lambda: 60 host._image = "image" host._repoint_execd_direct = AsyncMock() # Even if the workspace hook offers a mount, API principals must receive none. @@ -340,8 +339,8 @@ async def test_generated_output_is_pinned_and_downloadable_only_with_its_key( import core.sandbox from core.auth.agent_api_scope import enforce_agent_api_route from core.db.models import Artifact - from core.llm import workspace - from core.llm.agent_api_tools import register_agent_api_tools + from core.llm.runtime import workspace + from core.llm.api_runtime.agent_tools import register_agent_api_tools records = {} @@ -368,11 +367,11 @@ def save(path, *, name, mime_type, user_id, source, extra_metadata): return {key: value for key, value in item.items() if key != "metadata"} monkeypatch.setattr(core.sandbox, "get_sandbox_provider", lambda: Provider()) - monkeypatch.setattr("core.llm.tools._tool_helpers._store_generated_file_path", save) - monkeypatch.setattr("core.artifacts.store._read_record", records.get) - monkeypatch.setattr("core.artifacts.store.get_artifact", records.get) + monkeypatch.setattr("core.files.artifacts.generated.store_generated_file_path", save) + monkeypatch.setattr('core.files.artifacts.store._read_record', records.get) + monkeypatch.setattr('core.files.artifacts.store.get_artifact', records.get) monkeypatch.setattr( - "core.services.artifact_service.require_artifact_storage_key", lambda fid, key: key + 'core.files.artifacts.service.require_artifact_storage_key', lambda fid, key: key ) parsed = parse_api_scope( scope(), owner_user_id="owner", chat_id="api_chat", agent_id="agent_123" diff --git a/src/backend/tests/llm/test_api_protocol_backfill.py b/src/backend/tests/llm/test_api_protocol_backfill.py index 8ca68dd2..b580346a 100644 --- a/src/backend/tests/llm/test_api_protocol_backfill.py +++ b/src/backend/tests/llm/test_api_protocol_backfill.py @@ -10,10 +10,10 @@ import pytest -from core.db import model_repository -from core.llm.providers import protocol_probe -from core.llm.providers.protocol_probe import PROTOCOL_CHAT, PROTOCOL_RESPONSES -from core.services.api_protocol_backfill import backfill_missing_api_protocol +from core.models import repository as model_repository +from core.models.providers import protocol_probe +from core.models.providers.protocol_probe import PROTOCOL_CHAT, PROTOCOL_RESPONSES +from core.models.api_protocol_backfill import backfill_missing_api_protocol def _provider(pid, **kw): diff --git a/src/backend/tests/llm/test_approval_modes.py b/src/backend/tests/llm/test_approval_modes.py index 4cd23279..591b6dcc 100644 --- a/src/backend/tests/llm/test_approval_modes.py +++ b/src/backend/tests/llm/test_approval_modes.py @@ -10,19 +10,11 @@ import pytest from agentscope.message import ToolCallBlock -from core.llm.tool_permissions import ( - APPROVAL_ASK, - APPROVAL_AUTO, - APPROVAL_FULL, - READ, - WRITE, - PermissionRuntime, - ToolPermissionRegistry, - ToolPermissionService, - local_path_tool, - normalize_approval_mode, -) -from core.llm.tools._tool_confirmation import OP_DELETE, OP_WRITE +from core.sandbox.permissions.approval import APPROVAL_ASK, APPROVAL_AUTO, APPROVAL_FULL, normalize_approval_mode +from core.sandbox.permissions.dispatch_models import READ, WRITE, PermissionRuntime +from core.sandbox.permissions.tool_specs import ToolPermissionRegistry, local_path_tool +from core.llm.tool_permissions import ToolPermissionService +from core.llm.tools.confirmation import OP_DELETE, OP_WRITE from core.sandbox.permissions.models import DANGER_REASON_PREFIX from core.sandbox.permissions.models import DELETE from core.sandbox.permissions.models import danger_categories @@ -80,7 +72,7 @@ def test_danger_categories_are_recovered_from_verdict_reasons(): @pytest.mark.asyncio async def test_ask_preset_still_confirms_an_ordinary_write(): gate = AsyncMock(return_value={"status": "blocked", "error": "等确认"}) - with patch("core.llm.tools._tool_confirmation.gate", gate): + with patch("core.llm.tools.confirmation.gate", gate): outcome = await _service(APPROVAL_ASK).authorize( _tool_call("Write", {"file_path": "/myspace/a.txt", "content": "x"}) ) @@ -92,7 +84,7 @@ async def test_ask_preset_still_confirms_an_ordinary_write(): @pytest.mark.asyncio async def test_approve_for_me_passes_writes_but_still_asks_before_deleting(): gate = AsyncMock(return_value={"status": "blocked", "error": "等确认"}) - with patch("core.llm.tools._tool_confirmation.gate", gate): + with patch("core.llm.tools.confirmation.gate", gate): service = _service(APPROVAL_AUTO) written = await service.authorize( _tool_call("Write", {"file_path": "/myspace/a.txt", "content": "x"}) @@ -110,7 +102,7 @@ async def test_approve_for_me_passes_writes_but_still_asks_before_deleting(): @pytest.mark.asyncio async def test_full_access_never_asks_even_to_delete(): gate = AsyncMock() - with patch("core.llm.tools._tool_confirmation.gate", gate): + with patch("core.llm.tools.confirmation.gate", gate): outcome = await _service(APPROVAL_FULL).authorize( _tool_call("space_delete", {"path": "/myspace/a.txt"}) ) diff --git a/src/backend/tests/llm/test_artifact_refs.py b/src/backend/tests/llm/test_artifact_refs.py index 800af0d9..2b2192dd 100644 --- a/src/backend/tests/llm/test_artifact_refs.py +++ b/src/backend/tests/llm/test_artifact_refs.py @@ -1,4 +1,4 @@ -from core.content import artifact_refs as artifact_routes +from core.files.artifacts import refs as artifact_routes def test_extract_file_refs_supports_artifacts_array(): diff --git a/src/backend/tests/llm/test_ask_user_question_tool.py b/src/backend/tests/llm/test_ask_user_question_tool.py index fccd0d92..8c974357 100644 --- a/src/backend/tests/llm/test_ask_user_question_tool.py +++ b/src/backend/tests/llm/test_ask_user_question_tool.py @@ -4,7 +4,7 @@ import pytest from agentscope.tool import Toolkit -from core.llm.tool_collector import ToolCollector +from core.llm.factory.tools.definitions import ToolAssembly from core.llm.tools import ask_user_question_tool as tool_module @@ -64,13 +64,13 @@ def test_registration_policy_only_allows_standard_interactive_root(kwargs, expec @pytest.mark.asyncio async def test_registered_schema_matches_dsh_model_contract(): - collector = ToolCollector() + collector = ToolAssembly() tool_module.register_ask_user_question( collector, chat_id="chat-schema", interactive=True, ) - toolkit = Toolkit(tools=collector.function_tools) + toolkit = Toolkit(tools=list(collector)) schema = next( item["function"] @@ -110,7 +110,9 @@ async def test_tool_returns_structured_answer_from_suspended_service(monkeypatch captured = {} class CaptureToolkit: - def register_tool_function(self, function, **_kwargs): + def append(self, tool): + function = tool._func + _kwargs = {"func_description": tool.description, "permission": getattr(tool, "dispatch_spec", None)} captured["function"] = function async def fake_ask(**kwargs): @@ -179,7 +181,9 @@ async def test_tool_surfaces_non_answer_outcomes_without_retry_loop( captured = {} class CaptureToolkit: - def register_tool_function(self, function, **_kwargs): + def append(self, tool): + function = tool._func + _kwargs = {"func_description": tool.description, "permission": getattr(tool, "dispatch_spec", None)} captured["function"] = function async def fake_ask(**_kwargs): diff --git a/src/backend/tests/llm/test_bash_process_tools.py b/src/backend/tests/llm/test_bash_process_tools.py index 167803d1..adb023c9 100644 --- a/src/backend/tests/llm/test_bash_process_tools.py +++ b/src/backend/tests/llm/test_bash_process_tools.py @@ -3,6 +3,7 @@ import json from types import SimpleNamespace from unittest.mock import AsyncMock + import pytest from core.llm.tools.sandbox_tool import register_bash @@ -11,7 +12,9 @@ class Toolkit: def __init__(self): self.functions = {} - def register_tool_function(self, fn, **kwargs): + def append(self, tool): + fn = tool._func + kwargs = {"func_description": tool.description, "permission": getattr(tool, "dispatch_spec", None)} self.functions[fn.__name__] = fn @@ -21,12 +24,12 @@ def payload(result): async def test_bash_yields_and_write_stdin_waits_on_same_owned_process(monkeypatch, tmp_path): - from sqlalchemy import create_engine - from sqlalchemy.orm import sessionmaker from core.db.engine import Base from core.db.models import ChatSession, UserShadow - from core.services.commands import store - from orchestration.commands.runtime import runtime + from core.sandbox.commands.runtime import runtime + from core.sandbox.commands import store + from sqlalchemy import create_engine + from sqlalchemy.orm import sessionmaker engine = create_engine("sqlite:///" + str(tmp_path / "tools.sqlite"), connect_args={"check_same_thread": False}) Base.metadata.create_all(engine) @@ -63,7 +66,7 @@ async def test_bash_yields_and_write_stdin_waits_on_same_owned_process(monkeypat monkeypatch.setattr("core.config.local_mode.local_mode_enabled", lambda: False) monkeypatch.setattr("core.sandbox.get_sandbox_provider", lambda: provider) toolkit = Toolkit() - register_bash(toolkit, loader=None, loaded_skill_ids=set(), chat_id="chat-a", user_id="u") + register_bash(toolkit, loader=None, chat_id="chat-a", user_id="u") assert "write_stdin" in toolkit.functions first = payload(await toolkit.functions["Bash"]("long-command", yield_time_ms=250)) assert first["session_id"].startswith("cmd_") @@ -81,14 +84,14 @@ async def test_bash_yields_and_write_stdin_waits_on_same_owned_process(monkeypat async def test_registered_schemas_expose_two_tool_contract(monkeypatch): from agentscope.tool import Toolkit as AgentToolkit - from core.llm.tool_collector import ToolCollector + from core.llm.factory.tools.definitions import ToolAssembly monkeypatch.setenv("SANDBOX_TOOLS_ENABLED", "true") - collector = ToolCollector() - register_bash(collector, loader=None, loaded_skill_ids=set(), chat_id="schema-test") + collector = ToolAssembly() + register_bash(collector, loader=None, chat_id="schema-test") schemas = { s["function"]["name"]: s["function"] - for s in await AgentToolkit(tools=collector.function_tools).get_tool_schemas() + for s in await AgentToolkit(tools=list(collector)).get_tool_schemas() } assert set(schemas) == {"Bash", "write_stdin", "background_process", "list_processes"} assert schemas["Bash"]["parameters"]["properties"]["run_in_background"]["default"] is False diff --git a/src/backend/tests/llm/test_builtin_subagents.py b/src/backend/tests/llm/test_builtin_subagents.py index a20f0854..cccaded3 100644 --- a/src/backend/tests/llm/test_builtin_subagents.py +++ b/src/backend/tests/llm/test_builtin_subagents.py @@ -1,16 +1,9 @@ +from core.llm import prompt_files as prompt_files_module from core.db.models import UserShadow -from core.llm.builtin_subagents import ( - BUILTIN_SUBAGENTS, - build_builtin_runtime_profile, - effective_builtin_capabilities, - get_builtin_subagent, - list_builtin_subagents, - merge_builtin_subagents, - refresh_builtin_subagents, -) -from core.llm.subagent_tool import build_subagent_prompt_section -from core.services import prompt_version_service -from core.services.user_service import UserService +from core.capabilities.agents.builtin import BUILTIN_SUBAGENTS, build_builtin_runtime_profile, effective_builtin_capabilities, get_builtin_subagent, list_builtin_subagents, merge_builtin_subagents, refresh_builtin_subagents +from core.llm.subagents.tool import build_subagent_prompt_section +from core.llm import prompt_versions as prompt_version_service +from core.auth.users import UserService def test_platform_defaults_are_exactly_explorer_worker_reviewer(): @@ -223,7 +216,7 @@ def test_main_prompt_omits_parent_disabled_capability_names(): def test_subagent_prompt_kind_has_three_independent_filesystem_parts(): - parts = prompt_version_service._read_fs_parts("subagents") + parts = prompt_files_module.read_fs_parts("subagents") assert [part["part_id"] for part in parts] == ["explorer", "reviewer", "worker"] assert all(part["content"].strip() for part in parts) diff --git a/src/backend/tests/llm/test_cancelled_replay.py b/src/backend/tests/llm/test_cancelled_replay.py index 3c36914c..72ad1021 100644 --- a/src/backend/tests/llm/test_cancelled_replay.py +++ b/src/backend/tests/llm/test_cancelled_replay.py @@ -2,7 +2,7 @@ from types import SimpleNamespace -from core.services.compaction_history import normalize_rows +from core.chat.compaction.history import normalize_rows def test_cancelled_tail_survives_without_repeating_completed_text(): diff --git a/src/backend/tests/llm/test_chat_history_tool.py b/src/backend/tests/llm/test_chat_history_tool.py index 907406b1..964682c5 100644 --- a/src/backend/tests/llm/test_chat_history_tool.py +++ b/src/backend/tests/llm/test_chat_history_tool.py @@ -22,7 +22,9 @@ class FakeToolkit: def __init__(self): self.tools = {} - def register_tool_function(self, fn, **_kwargs): + def append(self, tool): + fn = tool._func + _kwargs = {"func_description": tool.description, "permission": getattr(tool, "dispatch_spec", None)} self.tools[fn.__name__] = fn @@ -37,15 +39,7 @@ def wired(tmp_path, monkeypatch): engine = create_engine( f"sqlite:///{tmp_path / 'tool.db'}", connect_args={"check_same_thread": False} ) - Base.metadata.create_all( - engine, - tables=[ - ChatSession.__table__, - ChatMessage.__table__, - ChatCompactionState.__table__, - Project.__table__, - ], - ) + Base.metadata.create_all(engine) Session = sessionmaker(bind=engine) with Session() as db: db.add_all( @@ -183,7 +177,7 @@ async def fake_summarize(_history, *, timeout): # noqa: ARG001 calls["n"] += 1 return "这段会话讨论了杭州落户积分的构成与社保断缴的处理。" - monkeypatch.setattr("core.services.compaction_summary._summarize", fake_summarize) + monkeypatch.setattr('core.chat.compaction.summary._summarize', fake_summarize) first = _payload(await wired.tools["read_chat"](chat_id="past", mode="digest")) second = _payload(await wired.tools["read_chat"](chat_id="past", mode="digest")) @@ -199,7 +193,7 @@ async def test_digest_failure_points_at_the_full_text_path(wired, monkeypatch): async def failing_summarize(_history, *, timeout): # noqa: ARG001 return None - monkeypatch.setattr("core.services.compaction_summary._summarize", failing_summarize) + monkeypatch.setattr('core.chat.compaction.summary._summarize', failing_summarize) result = _payload(await wired.tools["read_chat"](chat_id="past", mode="digest")) diff --git a/src/backend/tests/llm/test_chat_model_stream_retry.py b/src/backend/tests/llm/test_chat_model_stream_retry.py index 5898c800..5ab52a54 100644 --- a/src/backend/tests/llm/test_chat_model_stream_retry.py +++ b/src/backend/tests/llm/test_chat_model_stream_retry.py @@ -10,7 +10,7 @@ import httpx import openai import pytest -from core.llm.providers import stream_retry as chat_models +from core.models.providers import stream_retry as chat_models class _FakeModel: @@ -58,11 +58,11 @@ async def _collect(model, client, kwargs=None): def _quiet_usage(monkeypatch): calls = [] monkeypatch.setattr( - "core.llm.model_usage.record_provider_failure", + 'core.models.usage.record_provider_failure', AsyncMock(side_effect=lambda *a, **k: calls.append(("failure", k.get("metadata")))), ) monkeypatch.setattr( - "core.llm.model_usage.note_provider_retry_started", + 'core.models.usage.note_provider_retry_started', lambda *a, **k: calls.append(("retry_started", None)), ) return calls @@ -145,7 +145,7 @@ def test_transient_classification(): @pytest.mark.asyncio async def test_stopped_run_cannot_reissue_after_transient_first_event_failure(): - from core.llm.context_safety import ContextBudgetExceeded, request_admission_scope + from core.llm.context.safety import ContextBudgetExceeded, request_admission_scope client, create = _client() stopped = ContextBudgetExceeded( diff --git a/src/backend/tests/llm/test_compaction.py b/src/backend/tests/llm/test_compaction.py index 287fa11f..060af89c 100644 --- a/src/backend/tests/llm/test_compaction.py +++ b/src/backend/tests/llm/test_compaction.py @@ -4,7 +4,7 @@ Covers token estimation, middle-truncation markers, selecting the most recent user messages by budget, summary concatenation and filtering, etc. """ -from core.llm import compaction as C +from core.llm.context import compaction as C # ── token estimation (truncate.rs::approx_token_count) ──────────────────────────── diff --git a/src/backend/tests/llm/test_compaction_e2e.py b/src/backend/tests/llm/test_compaction_e2e.py index 083350a3..8ca3d9fe 100644 --- a/src/backend/tests/llm/test_compaction_e2e.py +++ b/src/backend/tests/llm/test_compaction_e2e.py @@ -14,9 +14,9 @@ import pytest from core.config import settings as config_settings from core.db.models import ChatSession -from core.llm import compaction as C -from core.services import compaction_service as S -from core.services.chat_service import ChatService +from core.llm.context import compaction as C +from core.chat.compaction import service as S +from core.chat.service import ChatService CHAT_ID = "chat_e2e" @@ -58,7 +58,7 @@ def _u(svc, i, text): def _a(svc, i, text, tool_calls=None): import json - from core.llm.model_steps import record_assistant_step, record_tool_result_step + from core.llm.runtime.model_steps import record_assistant_step, record_tool_result_step steps = [] for index, call in enumerate(tool_calls or []): @@ -155,7 +155,7 @@ async def test_full_compaction_cycle(patched_sessionlocal, monkeypatch): async def _fake_summarize(history, *, timeout): return "已完成 developer 表插入 nbacheng/dromara；待办：无；关键数据：nbacheng粉丝219" - monkeypatch.setattr("core.services.compaction_summary._summarize", _fake_summarize) + monkeypatch.setattr('core.chat.compaction.summary._summarize', _fake_summarize) # ── Drive the real compaction ── ok = await S.run_post_turn_compaction(CHAT_ID) @@ -205,7 +205,7 @@ async def _fake_summarize(history, *, timeout): assert any(C.is_summary_message(t) for t in texts) return f"summary-v{calls['n']}" - monkeypatch.setattr("core.services.compaction_summary._summarize", _fake_summarize) + monkeypatch.setattr('core.chat.compaction.summary._summarize', _fake_summarize) await S.run_post_turn_compaction(CHAT_ID) _u(svc, 2, "q2") @@ -242,7 +242,7 @@ async def test_compaction_commits_when_the_hook_bus_is_active(patched_sessionloc async def _fake_summarize(history, *, timeout): return "已完成两轮；待办：无" - monkeypatch.setattr("core.services.compaction_summary._summarize", _fake_summarize) + monkeypatch.setattr('core.chat.compaction.summary._summarize', _fake_summarize) ok = await S.run_post_turn_compaction(CHAT_ID, run_id="run_hooked_1") assert ok is True, "hook bus 激活时压缩必须仍能成功落库" diff --git a/src/backend/tests/llm/test_compaction_integration.py b/src/backend/tests/llm/test_compaction_integration.py index 449d2b38..d8f77c75 100644 --- a/src/backend/tests/llm/test_compaction_integration.py +++ b/src/backend/tests/llm/test_compaction_integration.py @@ -10,11 +10,11 @@ from core.config import settings as config_settings from core.db.models import ChatMessage, ChatSession -from core.llm import compaction as C -from core.llm.single_turn import Endpoint -from core.services import compaction_budget as budget -from core.services import compaction_service as S -from core.services.chat_service import ChatService +from core.llm.context import compaction as C +from core.models.completions.single_turn import Endpoint +from core.llm.context import budget as budget +from core.chat.compaction import service as S +from core.chat.service import ChatService def _endpoint(): @@ -47,7 +47,7 @@ def _add_user(svc, text): def _add_assistant(svc, text, tool_calls=None): - from core.llm.model_steps import record_assistant_step, record_tool_result_step + from core.llm.runtime.model_steps import record_assistant_step, record_tool_result_step steps = [] for index, call in enumerate(tool_calls or []): @@ -243,7 +243,7 @@ def test_summary_flatten_preserves_tool_blocks(): ], }, ] - from core.services.compaction_summary import _flatten_for_summary + from core.chat.compaction.summary import _flatten_for_summary flat = _flatten_for_summary(history) assert len(flat) == 3 @@ -287,7 +287,7 @@ def test_context_state_uses_compacted_baseline_and_visible_boundary(db_session): ) _add_user(svc, "压缩后的新问题") - from core.services.context_usage_projection import get_chat_context_state + from core.chat.compaction.usage_projection import get_chat_context_state state = get_chat_context_state(svc, CHAT_ID)["context_compaction"] @@ -335,7 +335,7 @@ def test_pre_turn_compaction_fast_path_no_llm(monkeypatch): async def boom(history, *, timeout): # noqa: ARG001 raise AssertionError("fast path must not call the LLM") - monkeypatch.setattr("core.services.compaction_summary._summarize", boom) + monkeypatch.setattr('core.chat.compaction.summary._summarize', boom) history = [{"role": "user", "content": "短问题"}] # Unknown model → window None → threshold None → fast path out, compacted = asyncio.run( @@ -366,7 +366,7 @@ def test_pre_turn_compaction_triggers_and_writes_checkpoint(db_session, monkeypa async def fake_summarize(history, *, timeout): # noqa: ARG001 return "模拟交接摘要" - monkeypatch.setattr("core.services.compaction_summary._summarize", fake_summarize) + monkeypatch.setattr('core.chat.compaction.summary._summarize', fake_summarize) # Checkpoint persistence goes through the global SessionLocal → point it at the test DB monkeypatch.setattr(engine_mod, "SessionLocal", sessionmaker(bind=db_session.get_bind())) @@ -420,7 +420,7 @@ def test_pre_turn_in_turn_view_keeps_current_user_last(db_session, monkeypatch): async def fake_summarize(history, *, timeout): # noqa: ARG001 return "摘要" - monkeypatch.setattr("core.services.compaction_summary._summarize", fake_summarize) + monkeypatch.setattr('core.chat.compaction.summary._summarize', fake_summarize) monkeypatch.setattr(engine_mod, "SessionLocal", sessionmaker(bind=db_session.get_bind())) history = [ diff --git a/src/backend/tests/llm/test_compaction_mid_turn.py b/src/backend/tests/llm/test_compaction_mid_turn.py index 5eea0e55..850bf55d 100644 --- a/src/backend/tests/llm/test_compaction_mid_turn.py +++ b/src/backend/tests/llm/test_compaction_mid_turn.py @@ -16,11 +16,11 @@ from agentscope.agent import ContextConfig from agentscope.message import DataBlock, Msg, TextBlock, ToolCallBlock, ToolResultBlock from core.config import settings as config_settings -from core.llm import compaction as C -from core.llm.compacting_agent import CompactingAgent, msg_to_history_dict -from core.llm.context_ir import IMAGE_TOKEN_RESERVE -from core.services import compaction_budget as budget -from core.services import compaction_service as S +from core.llm.context import compaction as C +from core.llm.context.compacting_agent import CompactingAgent, msg_to_history_dict +from core.models.constants import IMAGE_TOKEN_RESERVE +from core.llm.context import budget as budget +from core.chat.compaction import service as S def _user(text: str) -> Msg: @@ -52,7 +52,8 @@ def _make_agent( agent.model = SimpleNamespace(context_size=window) agent.offloader = offloader agent.context_config = ContextConfig() - agent.state = SimpleNamespace( + from core.llm.runtime.state import AgentRuntimeState + agent.state = AgentRuntimeState( context=list(context), summary="", chat_id=chat_id, session_id="sess-1" ) agent._jx_compaction_system_prompt = sys_prompt @@ -147,7 +148,7 @@ async def _engine(chat_id, history, **kwargs): called.append(chat_id) return [{"role": "user", "content": "nope"}] - monkeypatch.setattr(S, "run_compaction", _engine) + monkeypatch.setattr(compaction_port, "run_compaction", _engine) await agent.compress_context() assert called == [] @@ -172,7 +173,7 @@ async def _engine(chat_id, history, **kwargs): _older, recent = C.split_history_for_compaction(history, keep_recent_tokens=5) return C.build_compacted_history(summary_text, recent) - monkeypatch.setattr(S, "run_compaction", _engine) + monkeypatch.setattr(compaction_port, "run_compaction", _engine) await agent.compress_context() assert captured["chat_id"] == "chat-1" @@ -201,7 +202,7 @@ async def _engine(chat_id, history, **kwargs): async def _framework(self, context_config=None): fallback_calls.append(context_config) - monkeypatch.setattr(S, "run_compaction", _engine) + monkeypatch.setattr(compaction_port, "run_compaction", _engine) monkeypatch.setattr("agentscope.agent.Agent.compress_context", _framework) await agent.compress_context() @@ -239,7 +240,7 @@ async def _engine(chat_id, history, **kwargs): async def _framework(self, context_config=None): fallback_calls.append(1) - monkeypatch.setattr(S, "run_compaction", _engine) + monkeypatch.setattr(compaction_port, "run_compaction", _engine) monkeypatch.setattr("agentscope.agent.Agent.compress_context", _framework) await agent.compress_context() @@ -263,7 +264,7 @@ async def _engine(chat_id, history, **kwargs): {"role": "user", "content": "sum"}, ] - monkeypatch.setattr(S, "run_compaction", _engine) + monkeypatch.setattr(compaction_port, "run_compaction", _engine) await agent.compress_context() assert len(calls) == 1 @@ -294,7 +295,7 @@ async def _engine(chat_id, history, **kwargs): {"role": "user", "content": "q"}, ] - monkeypatch.setattr(S, "run_compaction", _engine) + monkeypatch.setattr(compaction_port, "run_compaction", _engine) await agent.compress_context() assert offloader.msgs is not None and len(offloader.msgs) == 2 @@ -328,9 +329,9 @@ async def _summarize(history, *, timeout): return "摘要正文" writes = [] - monkeypatch.setattr("core.services.compaction_summary._summarize", _summarize) + monkeypatch.setattr('core.chat.compaction.summary._summarize', _summarize) monkeypatch.setattr( - S, "run_compaction", lambda *a, **k: writes.append(1) + compaction_port, "run_compaction", lambda *a, **k: writes.append(1) ) # must not be reached await agent.compress_context() @@ -394,7 +395,7 @@ def get(self, key): monkeypatch.setitem( __import__("sys").modules, - "core.services.system_config", + "core.config.store.system_config", SimpleNamespace(SystemConfigService=_Svc), ) assert S.resolve_trigger_ratio() == 0.7 @@ -418,7 +419,7 @@ def get(self, key): monkeypatch.setitem( __import__("sys").modules, - "core.services.system_config", + "core.config.store.system_config", SimpleNamespace(SystemConfigService=_Svc), ) assert S.resolve_trigger_ratio() == 0.8 @@ -454,3 +455,5 @@ async def test_measure_counts_tool_results_appended_inside_existing_assistant(): measured = await agent._measure_context_tokens() assert measured > 12_500, "the 50k result must count even without a new message" assert await agent._measure_context_tokens() == measured, "do not accumulate estimates twice" + +from core.harness.registry import compaction as compaction_port diff --git a/src/backend/tests/llm/test_compaction_observation.py b/src/backend/tests/llm/test_compaction_observation.py index 92b74e2d..859655fa 100644 --- a/src/backend/tests/llm/test_compaction_observation.py +++ b/src/backend/tests/llm/test_compaction_observation.py @@ -15,12 +15,10 @@ from agentscope.agent import ContextConfig from agentscope.message import DataBlock, Msg, TextBlock, ToolCallBlock, ToolResultBlock from agentscope.model import ChatUsage -from core.llm import compaction as C -from core.llm.compacting_agent import CompactingAgent, msg_to_history_dict -from core.llm.context_ir import IMAGE_TOKEN_RESERVE -from core.services import compaction_service as S - - +from core.llm.context import compaction as C +from core.llm.context.compacting_agent import CompactingAgent, msg_to_history_dict +from core.models.constants import IMAGE_TOKEN_RESERVE +from core.chat.compaction import service as S from tests.llm.test_compaction_mid_turn import _assistant, _make_agent, _user @@ -37,7 +35,7 @@ async def engine(chat_id, history, **kwargs): {"role": "assistant", "content": [{"type": "text", "text": "recent"}]}, ] - monkeypatch.setattr(S, "run_compaction", engine) + monkeypatch.setattr(compaction_port, "run_compaction", engine) agent.observe_context_tokens(900, 10) await agent.compress_context() assert len(calls) == 1 @@ -104,7 +102,7 @@ async def test_rewritten_history_invalidates_the_usage_baseline(change): async def test_actual_sdk_event_order_does_not_double_count_response(monkeypatch): from agentscope.event import ModelCallEndEvent from agentscope.model import ChatResponse - from orchestration.streaming import StreamingAgent + from core.llm.runtime.streaming import StreamingAgent agent = _make_agent([_user("q")]) agent.name = "agent" @@ -138,3 +136,5 @@ async def chunks(): assert seen assert len(agent.state.context) == 2 assert await agent._measure_context_tokens() == 1500 + +from core.harness.registry import compaction as compaction_port diff --git a/src/backend/tests/llm/test_compaction_progress.py b/src/backend/tests/llm/test_compaction_progress.py index 70c0e3d8..db2a62d9 100644 --- a/src/backend/tests/llm/test_compaction_progress.py +++ b/src/backend/tests/llm/test_compaction_progress.py @@ -3,8 +3,8 @@ import asyncio import pytest -from core.services.compaction_progress import finish_progress, progress_scope, start_progress -from orchestration.compaction_stream import stream_with_compaction +from core.agent_runtime.integrations.compaction_stream import stream_with_compaction +from core.chat.compaction.progress import finish_progress, progress_scope, start_progress @pytest.mark.asyncio @@ -17,8 +17,8 @@ async def compact(chat_id, **kwargs): await finish_progress(event, "completed") return [{"role": "user", "content": "summary"}] - monkeypatch.setattr("core.services.compaction_service.run_compaction", compact) - monkeypatch.setattr("core.services.compaction_progress.persist_progress", lambda event: None) + monkeypatch.setattr('core.chat.compaction.service.run_compaction', compact) + monkeypatch.setattr('core.chat.compaction.progress.persist_progress', lambda event: None) async def model(**kwargs): raise AssertionError("manual command must not invoke reply model") @@ -46,7 +46,7 @@ async def model(**kwargs): @pytest.mark.asyncio async def test_auto_progress_arrives_before_waiting_model_finishes(monkeypatch): - monkeypatch.setattr("core.services.compaction_progress.persist_progress", lambda event: None) + monkeypatch.setattr('core.chat.compaction.progress.persist_progress', lambda event: None) gate = asyncio.Event() async def model(**kwargs): @@ -72,7 +72,7 @@ async def model(**kwargs): @pytest.mark.asyncio async def test_cancelled_compaction_has_terminal_state(monkeypatch): - monkeypatch.setattr("core.services.compaction_progress.persist_progress", lambda event: None) + monkeypatch.setattr('core.chat.compaction.progress.persist_progress', lambda event: None) events = [] async def sink(event): @@ -90,9 +90,9 @@ async def sink(event): async def test_coordinator_reports_real_work_and_always_closes(monkeypatch, outcome): from types import SimpleNamespace - from core.services.compaction_coordinator import CompactionCoordinator + from core.chat.compaction.coordinator import CompactionCoordinator - monkeypatch.setattr("core.services.compaction_progress.persist_progress", lambda event: None) + monkeypatch.setattr('core.chat.compaction.progress.persist_progress', lambda event: None) history = [{"role": "user", "content": "old" * 100}, {"role": "assistant", "content": "answer"}] snapshot = SimpleNamespace( replacement_history=history, @@ -112,7 +112,7 @@ def commit(*args, **kwargs): monkeypatch.setattr(coordinator, "_commit", commit) monkeypatch.setattr( - "core.llm.compaction.split_history_for_compaction", lambda *args, **kwargs: (history, []) + 'core.llm.context.compaction.split_history_for_compaction', lambda *args, **kwargs: (history, []) ) gate = asyncio.Event() @@ -122,7 +122,7 @@ async def summarize(*args, **kwargs): await asyncio.Event().wait() return None if outcome == "failed" else "summary" - monkeypatch.setattr("core.services.compaction_summary._summarize", summarize) + monkeypatch.setattr('core.chat.compaction.summary._summarize', summarize) events = [] async def sink(event): @@ -153,8 +153,8 @@ async def test_short_manual_context_reports_noop_without_reply_model(monkeypatch async def compact(*args, **kwargs): return None - monkeypatch.setattr("core.services.compaction_service.run_compaction", compact) - monkeypatch.setattr("core.services.compaction_progress.persist_progress", lambda event: None) + monkeypatch.setattr('core.chat.compaction.service.run_compaction', compact) + monkeypatch.setattr('core.chat.compaction.progress.persist_progress', lambda event: None) chunks = [ c async for c in stream_with_compaction( @@ -174,9 +174,9 @@ async def compact(*args, **kwargs): async def test_preparation_failure_is_not_reported_as_noop(monkeypatch): from types import SimpleNamespace - from core.services.compaction_coordinator import CompactionCoordinator + from core.chat.compaction.coordinator import CompactionCoordinator - monkeypatch.setattr("core.services.compaction_progress.persist_progress", lambda event: None) + monkeypatch.setattr('core.chat.compaction.progress.persist_progress', lambda event: None) coordinator = CompactionCoordinator("c") snapshot = SimpleNamespace( replacement_history=[{"role": "user", "content": "old"}], source_rows=[] @@ -188,7 +188,7 @@ async def test_preparation_failure_is_not_reported_as_noop(monkeypatch): def invalid_budget(**kwargs): raise RuntimeError("invalid budget") - monkeypatch.setattr("core.services.compaction_budget.estimate_context_budget", invalid_budget) + monkeypatch.setattr('core.llm.context.budget.estimate_context_budget', invalid_budget) events = [] async def sink(event): @@ -203,7 +203,7 @@ async def sink(event): @pytest.mark.asyncio async def test_backpressured_consumer_close_finishes_producer(monkeypatch): - monkeypatch.setattr("core.services.compaction_progress.persist_progress", lambda event: None) + monkeypatch.setattr('core.chat.compaction.progress.persist_progress', lambda event: None) released = asyncio.Event() async def model(**kwargs): @@ -231,7 +231,7 @@ async def model(**kwargs): def test_run_snapshot_preserves_compaction_order_and_tool_count(): - from orchestration.run_projection import fold_projection, new_projection + from core.chat.projection.reducer import fold_projection, new_projection state = new_projection("r") state = fold_projection(state, {"type": "tool_call", "tool_id": "t", "tool_name": "search"}) @@ -249,8 +249,8 @@ def test_post_turn_progress_is_durable_on_message(db_session, monkeypatch): from contextlib import nullcontext from core.db.models import ChatMessage, ChatRun, ChatSession - from core.services.chat_service import ChatService - from core.services.compaction_progress import persist_progress, read_progress + from core.chat.service import ChatService + from core.chat.compaction.progress import persist_progress, read_progress db_session.add( ChatSession(chat_id="durable", user_id="u", title="t", extra_data={"other": "keep"}) @@ -294,7 +294,7 @@ def test_post_turn_progress_is_durable_on_message(db_session, monkeypatch): @pytest.mark.asyncio async def test_steer_message_identity_tracks_context(monkeypatch): - monkeypatch.setattr("core.services.compaction_progress.persist_progress", lambda event: None) + monkeypatch.setattr('core.chat.compaction.progress.persist_progress', lambda event: None) events = [] async def sink(event): @@ -333,7 +333,7 @@ async def model(**kwargs): @pytest.mark.asyncio async def test_private_run_does_not_inherit_main_message_identity(monkeypatch): stored = [] - monkeypatch.setattr("core.services.compaction_progress.persist_progress", stored.append) + monkeypatch.setattr('core.chat.compaction.progress.persist_progress', stored.append) async def sink(event): pass @@ -347,9 +347,9 @@ async def sink(event): async def test_normalization_failure_releases_acquired_lease(monkeypatch): from types import SimpleNamespace - from core.services.compaction_coordinator import CompactionCoordinator + from core.chat.compaction.coordinator import CompactionCoordinator - monkeypatch.setattr("core.services.compaction_progress.persist_progress", lambda event: None) + monkeypatch.setattr('core.chat.compaction.progress.persist_progress', lambda event: None) coordinator = CompactionCoordinator("c") monkeypatch.setattr( coordinator, "_acquire", lambda: SimpleNamespace(replacement_history=[], source_rows=[None]) @@ -360,7 +360,7 @@ async def test_normalization_failure_releases_acquired_lease(monkeypatch): def broken_rows(rows): raise ValueError("invalid source row") - monkeypatch.setattr("core.services.compaction_coordinator.normalize_rows", broken_rows) + monkeypatch.setattr('core.chat.compaction.coordinator.normalize_rows', broken_rows) events = [] async def sink(event): @@ -381,10 +381,10 @@ async def test_cancelled_database_work_settles_before_terminal_or_release( import threading from types import SimpleNamespace - from core.llm import compaction as compaction - from core.services.compaction_coordinator import CompactionCoordinator + from core.llm.context import compaction as compaction + from core.chat.compaction.coordinator import CompactionCoordinator - monkeypatch.setattr("core.services.compaction_progress.persist_progress", lambda event: None) + monkeypatch.setattr('core.chat.compaction.progress.persist_progress', lambda event: None) history = [{"role": "user", "content": "old history"}] snapshot = SimpleNamespace( replacement_history=history, @@ -423,7 +423,7 @@ def commit(*args, **kwargs): async def summarize(*args, **kwargs): return "summary" - monkeypatch.setattr("core.services.compaction_summary._summarize", summarize) + monkeypatch.setattr('core.chat.compaction.summary._summarize', summarize) events = [] async def sink(event): diff --git a/src/backend/tests/llm/test_compaction_summary_requests.py b/src/backend/tests/llm/test_compaction_summary_requests.py index 67977b7d..3772df7c 100644 --- a/src/backend/tests/llm/test_compaction_summary_requests.py +++ b/src/backend/tests/llm/test_compaction_summary_requests.py @@ -7,7 +7,7 @@ import json -from core.llm import compaction as C +from core.llm.context import compaction as C from tests.llm.test_compaction_integration import _endpoint @@ -15,7 +15,7 @@ def test_summarize_context_overflow_self_heal(monkeypatch): """Summary input exceeds the window → drop the oldest history and retry until it fits (aligned with Codex's ContextWindowExceeded handling).""" import asyncio - from core.services import compaction_summary as summary + from core.chat.compaction import summary as summary monkeypatch.setattr(summary, "_resolve_summarizer_model", lambda: _endpoint()) monkeypatch.setattr(summary, "_load_base_system_prompt", lambda: "SYS PROMPT") @@ -65,7 +65,7 @@ def test_summarize_non_context_error_no_retry(monkeypatch): """Non-context-overflow errors (e.g. auth failure) skip the self-heal loop; return None after one attempt.""" import asyncio - from core.services import compaction_summary as summary + from core.chat.compaction import summary as summary monkeypatch.setattr(summary, "_resolve_summarizer_model", lambda: _endpoint()) monkeypatch.setattr(summary, "_load_base_system_prompt", lambda: "") diff --git a/src/backend/tests/llm/test_compaction_watermark.py b/src/backend/tests/llm/test_compaction_watermark.py index 9d8c3cf7..dcd01f1c 100644 --- a/src/backend/tests/llm/test_compaction_watermark.py +++ b/src/backend/tests/llm/test_compaction_watermark.py @@ -1,7 +1,8 @@ """Regression tests for sequence-watermarked, CAS-safe compaction.""" from __future__ import annotations -from core.services import compaction_budget as budget +from core.llm.context import budget as budget +from core.chat.compaction import service as S from core.config import settings as config_settings import asyncio @@ -12,20 +13,18 @@ import core.db.engine as engine_mod from core.db.models import ChatMessage, ChatSession from core.db.repository.chat import ChatMessageRepository -from core.llm import compaction as C -from core.services import compaction_service as S -from core.services.chat_service import ChatService, CompactionCASConflict +from core.llm.context import compaction as C +from core.chat.service import ChatService +from core.chat.compaction_types import CompactionCASConflict from sqlalchemy.orm import sessionmaker CHAT_ID = "chat_compaction_watermark" - def _session(db) -> ChatService: db.add(ChatSession(chat_id=CHAT_ID, user_id="u1", title="watermark")) db.commit() return ChatService(db) - def _keep_recent(monkeypatch, tokens: int, **overrides) -> None: """These histories are a few words long; the production 20k verbatim tail would leave nothing older to summarize, so the tail is sized per test.""" @@ -33,7 +32,6 @@ def _keep_recent(monkeypatch, tokens: int, **overrides) -> None: compaction = dataclasses.replace(real.compaction, keep_recent_tokens=tokens, **overrides) monkeypatch.setattr(config_settings, "settings", dataclasses.replace(real, compaction=compaction)) - def _message(svc: ChatService, message_id: str, role: str, content: str) -> ChatMessage: return svc.message_repo.create( { @@ -44,7 +42,6 @@ def _message(svc: ChatService, message_id: str, role: str, content: str) -> Chat } ) - def test_direct_orm_message_writes_receive_non_null_sequences(db_session): _session(db_session) first = ChatMessage( @@ -68,7 +65,6 @@ def test_direct_orm_message_writes_receive_non_null_sequences(db_session): ] assert db_session.get(ChatSession, CHAT_ID).next_message_seq == 3 - def test_sqlite_two_session_sequence_allocation_serializes_without_collision( db_session, ): @@ -129,7 +125,6 @@ def write_second() -> None: ("concurrent-2", 2), ] - def test_replay_uses_chat_seq_when_created_at_is_identical(db_session): svc = _session(db_session) first = _message(svc, "m1", "user", "before") @@ -157,7 +152,6 @@ def test_replay_uses_chat_seq_when_created_at_is_identical(db_session): assert "same-time tail" in rendered assert "answer" not in rendered - def test_legacy_checkpoint_without_watermark_is_invalidated_and_rebuilt(db_session): svc = _session(db_session) _message(svc, "m1", "user", "legacy source question") @@ -195,7 +189,6 @@ def test_legacy_checkpoint_without_watermark_is_invalidated_and_rebuilt(db_sessi assert snapshot.base_covered_seq == 0 assert snapshot.source_message_ids == ("m1", "m2", "m3") - def test_message_arriving_during_summary_stays_in_tail(db_session, monkeypatch): svc = _session(db_session) _message(svc, "m1", "user", "snapshot question") @@ -210,7 +203,7 @@ async def summarize_then_insert(history, *, timeout): _message(ChatService(other), "m3", "user", "arrived while summarizing") return "snapshot summary" - monkeypatch.setattr("core.services.compaction_summary._summarize", summarize_then_insert) + monkeypatch.setattr('core.chat.compaction.summary._summarize', summarize_then_insert) assert asyncio.run(S.run_post_turn_compaction(CHAT_ID)) is True checkpoint = svc.get_latest_compaction_checkpoint(CHAT_ID) @@ -221,7 +214,6 @@ async def summarize_then_insert(history, *, timeout): "arrived while summarizing" in str(item.get("content")) for item in history ) - def test_same_base_checkpoint_has_exactly_one_cas_winner(db_session): svc = _session(db_session) source = _message(svc, "m1", "user", "source") @@ -263,7 +255,6 @@ def test_same_base_checkpoint_has_exactly_one_cas_winner(db_session): assert len(checkpoints) == 1 assert checkpoints[0].extra_data["checkpoint_version"] == 1 - def test_two_compactors_overlap_but_only_lease_holder_publishes( db_session, monkeypatch ): @@ -281,7 +272,7 @@ async def held_summary(history, *, timeout): await finish.wait() return "winner" - monkeypatch.setattr("core.services.compaction_summary._summarize", held_summary) + monkeypatch.setattr('core.chat.compaction.summary._summarize', held_summary) async def overlap() -> tuple[bool, bool]: first = asyncio.create_task(S.run_post_turn_compaction(CHAT_ID)) @@ -300,7 +291,6 @@ async def overlap() -> tuple[bool, bool]: ] assert len(checkpoints) == 1 - def test_history_rewrite_invalidates_lineage_and_fences_inflight_summary(db_session): svc = _session(db_session) _message(svc, "m1", "user", "retained question") @@ -341,7 +331,6 @@ def test_history_rewrite_invalidates_lineage_and_fences_inflight_summary(db_sess assert rebuilt.base_covered_seq == 0 assert rebuilt.source_message_ids == ("m1", "m2") - def test_accepting_covered_revision_invalidates_checkpoint_and_inflight_snapshot( db_session, ): @@ -395,7 +384,6 @@ def test_accepting_covered_revision_invalidates_checkpoint_and_inflight_snapshot assert "accepted revised answer" in rendered assert "old summary" not in rendered - def test_checkpoint_write_failure_is_explicit_and_emits_no_fake_notice( db_session, monkeypatch ): @@ -411,14 +399,13 @@ async def fake_summary(history, *, timeout): def fail_write(*args, **kwargs): raise RuntimeError("database unavailable") - monkeypatch.setattr("core.services.compaction_summary._summarize", fake_summary) + monkeypatch.setattr('core.chat.compaction.summary._summarize', fake_summary) monkeypatch.setattr(ChatService, "commit_compaction_checkpoint", fail_write) assert asyncio.run(S.run_post_turn_compaction(CHAT_ID)) is False assert svc.get_latest_compaction_checkpoint(CHAT_ID) is None assert S.pop_compaction_notice(svc, CHAT_ID) is False - def test_context_budget_estimator_has_stable_component_manifest(): estimate = budget.estimate_context_budget( system_prompt="system", @@ -466,7 +453,6 @@ def test_context_budget_estimator_has_stable_component_manifest(): ) } - def test_pre_turn_rechecks_authoritative_snapshot_before_summarizing( db_session, monkeypatch ): @@ -482,7 +468,7 @@ def test_pre_turn_rechecks_authoritative_snapshot_before_summarizing( async def must_not_summarize(history, *, timeout): raise AssertionError("authoritative snapshot is below the threshold") - monkeypatch.setattr("core.services.compaction_summary._summarize", must_not_summarize) + monkeypatch.setattr('core.chat.compaction.summary._summarize', must_not_summarize) stale_caller_history = [ { "role": "tool", @@ -503,7 +489,6 @@ async def must_not_summarize(history, *, timeout): assert returned == stale_caller_history assert svc.get_latest_compaction_checkpoint(CHAT_ID) is None - def test_pre_turn_and_post_turn_store_the_same_budget_manifest_shape( db_session, monkeypatch ): @@ -517,7 +502,7 @@ def test_pre_turn_and_post_turn_store_the_same_budget_manifest_shape( async def fake_summary(history, *, timeout): return "summary" - monkeypatch.setattr("core.services.compaction_summary._summarize", fake_summary) + monkeypatch.setattr('core.chat.compaction.summary._summarize', fake_summary) history = S._load_history(svc, CHAT_ID) _, compacted = asyncio.run( S.maybe_run_pre_turn_compaction( @@ -560,7 +545,6 @@ async def fake_summary(history, *, timeout): ) assert second_budget["provider_overhead_tokens"] == 13 - def test_rolling_compaction_keeps_one_lineage_without_duplicate_or_loss( db_session, monkeypatch ): @@ -578,7 +562,7 @@ def test_rolling_compaction_keeps_one_lineage_without_duplicate_or_loss( async def fake_summary(history, *, timeout): return next(summaries) - monkeypatch.setattr("core.services.compaction_summary._summarize", fake_summary) + monkeypatch.setattr('core.chat.compaction.summary._summarize', fake_summary) assert asyncio.run(S.run_post_turn_compaction(CHAT_ID)) is True _message(svc, "m3", "user", "round-two-question") _message(svc, "m4", "assistant", "round-two-answer") diff --git a/src/backend/tests/llm/test_configured_reasoning_effort.py b/src/backend/tests/llm/test_configured_reasoning_effort.py index 8f0afd35..375e8d1b 100644 --- a/src/backend/tests/llm/test_configured_reasoning_effort.py +++ b/src/backend/tests/llm/test_configured_reasoning_effort.py @@ -5,10 +5,10 @@ import pytest from agentscope.message import Msg, TextBlock from core.db.models import ModelProvider -from core.llm.chat_models import build_model_for_mode -from core.llm.providers.registry import validate_payload -from core.services.model_config import ResolvedModelConfig -from core.services.user_model_selection import list_user_selectable_models +from core.models.chat_models import build_model_for_mode +from core.models.providers.registry import validate_payload +from core.models.config import ResolvedModelConfig +from core.models.selection import list_user_selectable_models CONFIG = { "supports_reasoning_effort": True, @@ -51,7 +51,7 @@ def test_selectable_model_exposes_configured_levels_without_credentials(db_sessi @pytest.mark.asyncio @pytest.mark.parametrize("mode,wire", [("medium", "high"), ("max", 100), ("xhigh", "xhigh")]) async def test_responses_sends_configured_mapping_and_valid_default(monkeypatch, mode, wire): - monkeypatch.setattr("core.llm.context_manager.resolve_model_context_window", lambda name: 65536) + monkeypatch.setattr('core.models.context_window.resolve_model_context_window', lambda name: 65536) cfg = ResolvedModelConfig( base_url="http://model.test/v1", api_key="test-key", diff --git a/src/backend/tests/llm/test_context_adapter.py b/src/backend/tests/llm/test_context_adapter.py index 7898f7f9..fe348d18 100644 --- a/src/backend/tests/llm/test_context_adapter.py +++ b/src/backend/tests/llm/test_context_adapter.py @@ -4,24 +4,14 @@ from agentscope.message import Msg, TextBlock -from core.llm.context_adapter import ( - AgentScopeContextAdapter, - PROVIDER_CONTEXT_META_KEY, - append_context_item, - next_context_sequence, -) -from core.llm.context_ir import ( - ContextAssembler, - ContextItem, - IMAGE_TOKEN_RESERVE, - KIND_COMPACTION, - KIND_MEMORY, - KIND_REMINDER, - KIND_TOOL_CALL, - KIND_TOOL_RESULT, - KIND_USER_INPUT, -) -from core.llm.execution_manifest import PromptManifestBuilder, stable_hash +from core.llm.context.adapter import AgentScopeContextAdapter +from core.llm.context.rendering import append_context_item, next_context_sequence +from core.models.constants import PROVIDER_CONTEXT_META_KEY +from core.llm.context.ir import ContextAssembler +from core.llm.context.items import ContextItem +from core.llm.context.ir_constants import KIND_COMPACTION, KIND_MEMORY, KIND_REMINDER, KIND_TOOL_CALL, KIND_TOOL_RESULT, KIND_USER_INPUT +from core.models.constants import IMAGE_TOKEN_RESERVE +from core.harness.execution_manifest import PromptManifestBuilder, stable_hash def _item(item_id, content, **overrides): diff --git a/src/backend/tests/llm/test_context_ir.py b/src/backend/tests/llm/test_context_ir.py index fa42f1b6..9fddcae7 100644 --- a/src/backend/tests/llm/test_context_ir.py +++ b/src/backend/tests/llm/test_context_ir.py @@ -8,7 +8,9 @@ import pytest -from core.llm.context_ir import IMAGE_TOKEN_RESERVE, ContextAssembler, ContextItem +from core.llm.context.ir import ContextAssembler +from core.llm.context.items import ContextItem +from core.models.constants import IMAGE_TOKEN_RESERVE def _item( @@ -51,7 +53,7 @@ def test_context_ir_module_does_not_import_agentscope(): sys.executable, "-c", ( - "import sys; import core.llm.context_ir; " + "import sys; import core.llm.context.ir; " "raise SystemExit('agentscope.message' in sys.modules)" ), ], @@ -408,7 +410,7 @@ def test_manifest_is_sanitized_and_content_hash_sensitive(): def test_tool_text_budget_and_history_pruning_preserve_image_blocks( policy, total_budget, text_budget, action ): - from core.immutable import thaw_json + from core.infra.immutable import thaw_json image = { "type": "data", diff --git a/src/backend/tests/llm/test_context_order_user_input.py b/src/backend/tests/llm/test_context_order_user_input.py index 2ca0a3c5..968b323a 100644 --- a/src/backend/tests/llm/test_context_order_user_input.py +++ b/src/backend/tests/llm/test_context_order_user_input.py @@ -10,17 +10,11 @@ from agentscope.message import Msg, ToolCallBlock, ToolResultBlock -from core.llm.context_adapter import ( - AgentScopeContextAdapter, - next_request_sequence, - render_context_item, -) -from core.llm.context_ir import ( - KIND_USER_INPUT, - POLICY_NEVER, - ContextAssembler, - ContextItem, -) +from core.llm.context.adapter import AgentScopeContextAdapter +from core.llm.context.rendering import next_request_sequence, render_context_item +from core.llm.context.ir_constants import KIND_USER_INPUT, POLICY_NEVER +from core.llm.context.ir import ContextAssembler +from core.llm.context.items import ContextItem USER_TEXT = "新建一个 test 文件夹，派 5 个子智能体各生成一份 html 放进去" diff --git a/src/backend/tests/llm/test_context_probe.py b/src/backend/tests/llm/test_context_probe.py index 3cfe3b04..591f4fd4 100644 --- a/src/backend/tests/llm/test_context_probe.py +++ b/src/backend/tests/llm/test_context_probe.py @@ -8,8 +8,8 @@ import httpx import pytest -from core.llm.providers import context_probe -from core.llm.providers.context_probe import discover_context_length, from_name_heuristic +from core.models.providers import context_probe +from core.models.providers.context_probe import discover_context_length, from_name_heuristic # One self-hosted vLLM deployment: a single entry that publishes the real window. _VLLM_MODELS = { diff --git a/src/backend/tests/llm/test_context_safety.py b/src/backend/tests/llm/test_context_safety.py index fd7f5958..c8cc422b 100644 --- a/src/backend/tests/llm/test_context_safety.py +++ b/src/backend/tests/llm/test_context_safety.py @@ -3,12 +3,14 @@ import pytest from agentscope.message import TextBlock, UserMsg from agentscope.model import ChatResponse, ChatUsage -from core.llm.manifest_agent import ManifestBoundAgent -from core.ontology.toolkit import OntologyFilteredToolkit -from core.services.compaction_history import normalize_rows +from core.llm.runtime.manifest_agent import ManifestBoundAgent +from core.llm.tools.ontology_toolkit import OntologyFilteredToolkit +from core.chat.compaction.history import normalize_rows class CaptureModel: + from agentscope.formatter import OpenAIChatFormatter + formatter = OpenAIChatFormatter() model = "budget-probe" context_size = 100 @@ -69,10 +71,10 @@ async def test_compaction_failure_cannot_replace_history_with_fake_summary(monke from agentscope.agent import Agent from core.config import settings as settings_module - from core.llm.compacting_agent import CompactingAgent - from core.llm.context_safety import ContextBudgetExceeded - from core.llm.middlewares import AgentRuntimeState - from core.services import compaction_service + from core.llm.context.compacting_agent import CompactingAgent + from core.llm.context.safety import ContextBudgetExceeded + from core.llm.runtime.state import AgentRuntimeState + from core.chat.compaction import service as compaction_service engine_calls = [] @@ -85,7 +87,7 @@ async def compact(*args, **kwargs): async def forbidden_framework(*args, **kwargs): raise AssertionError("A failed summary must never enter a second compressor") - monkeypatch.setattr(compaction_service, "run_compaction", compact) + monkeypatch.setattr(compaction_port, "run_compaction", compact) monkeypatch.setattr(Agent, "compress_context", forbidden_framework) if failure == "disabled": monkeypatch.setattr( @@ -114,9 +116,9 @@ async def forbidden_framework(*args, **kwargs): @pytest.mark.asyncio async def test_budget_stop_also_blocks_another_compaction_attempt(monkeypatch): - from core.llm.compacting_agent import CompactingAgent - from core.llm.context_safety import ContextBudgetExceeded, request_admission_scope - from core.services import compaction_service + from core.llm.context.compacting_agent import CompactingAgent + from core.llm.context.safety import ContextBudgetExceeded, request_admission_scope + from core.chat.compaction import service as compaction_service calls = [] @@ -124,7 +126,7 @@ async def compact(*args, **kwargs): calls.append(True) return None - monkeypatch.setattr(compaction_service, "run_compaction", compact) + monkeypatch.setattr(compaction_port, "run_compaction", compact) model = CaptureModel() agent = CompactingAgent( name="stopped-compactor", @@ -142,3 +144,5 @@ async def compact(*args, **kwargs): await agent.compress_context() assert calls == [] assert model.calls == [] + +from core.harness.registry import compaction as compaction_port diff --git a/src/backend/tests/llm/test_context_step_units.py b/src/backend/tests/llm/test_context_step_units.py index f19ceb27..0e8be8a0 100644 --- a/src/backend/tests/llm/test_context_step_units.py +++ b/src/backend/tests/llm/test_context_step_units.py @@ -8,23 +8,14 @@ import pytest from agentscope.message import Msg, TextBlock, ThinkingBlock, ToolCallBlock, ToolResultBlock -from core.llm.chat_models import ReasoningEchoChatFormatter -from core.llm.context_adapter import ( - AgentScopeContextAdapter, - next_request_sequence, - render_context_item, -) -from core.llm.context_ir import ( - KIND_THINKING, - KIND_TOOL_CALL, - KIND_TOOL_RESULT, - KIND_USER_INPUT, - POLICY_NEVER, - ContextAssembler, - ContextItem, -) -from core.llm.message_compat import session_to_msgs -from core.llm.model_steps import record_assistant_step, record_tool_result_step, replay_rows +from core.models.chat_models import ReasoningEchoChatFormatter +from core.llm.context.adapter import AgentScopeContextAdapter +from core.llm.context.rendering import next_request_sequence, render_context_item +from core.llm.context.ir_constants import KIND_THINKING, KIND_TOOL_CALL, KIND_TOOL_RESULT, KIND_USER_INPUT, POLICY_NEVER +from core.llm.context.ir import ContextAssembler +from core.llm.context.items import ContextItem +from core.llm.runtime.message_compat import session_to_msgs +from core.llm.runtime.model_steps import record_assistant_step, record_tool_result_step, replay_rows LONG_THINKING = "深入推理一下这个问题，" * 4_000 # far beyond the old 20k per-block cap diff --git a/src/backend/tests/llm/test_context_usage_adversarial.py b/src/backend/tests/llm/test_context_usage_adversarial.py index dedfd5db..c79cd219 100644 --- a/src/backend/tests/llm/test_context_usage_adversarial.py +++ b/src/backend/tests/llm/test_context_usage_adversarial.py @@ -4,8 +4,8 @@ from types import SimpleNamespace from api.routes.v1.chats.history import get_context_usage -from core.llm.context_usage import build_compaction_context_usage, build_context_usage_snapshot -from core.services.chat_service import ChatService +from core.llm.context.usage import build_compaction_context_usage, build_context_usage_snapshot +from core.chat.service import ChatService from core.db.models import ChatSession @@ -80,9 +80,9 @@ def test_midturn_checkpoint_counts_the_active_system_and_tool_surface(db_session from agentscope.message import Msg, TextBlock from sqlalchemy.orm import sessionmaker import core.db.engine as engine - from core.llm.compacting_agent import CompactingAgent + from core.llm.context.compacting_agent import CompactingAgent from core.config import settings as config_settings - from core.services import compaction_service as service + from core.chat.compaction import service as service svc, row, ckpt, usage = _setup(db_session) svc.message_repo.create( @@ -101,7 +101,7 @@ def test_midturn_checkpoint_counts_the_active_system_and_tool_surface(db_session async def summarize(history, **kwargs): return "short summary" - monkeypatch.setattr("core.services.compaction_summary._summarize", summarize) + monkeypatch.setattr('core.chat.compaction.summary._summarize', summarize) agent = object.__new__(CompactingAgent) agent._jx_observation = None agent._jx_compacted_cursor = None diff --git a/src/backend/tests/llm/test_context_usage_snapshot.py b/src/backend/tests/llm/test_context_usage_snapshot.py index 11daceaa..c4a68e3b 100644 --- a/src/backend/tests/llm/test_context_usage_snapshot.py +++ b/src/backend/tests/llm/test_context_usage_snapshot.py @@ -1,10 +1,7 @@ import asyncio from types import SimpleNamespace -from core.llm.context_usage import ( - build_compaction_context_usage, - build_context_usage_snapshot, -) +from core.llm.context.usage import build_compaction_context_usage, build_context_usage_snapshot def _agent(): @@ -118,7 +115,7 @@ def test_compaction_snapshot_uses_post_compaction_backend_estimate(): def test_model_call_end_emits_live_context_usage(): - from orchestration.streaming import StreamingAgent + from core.llm.runtime.streaming import StreamingAgent class ModelCallEndEvent: input_tokens = 120 @@ -140,7 +137,7 @@ async def collect(): def test_auxiliary_call_can_restore_primary_context_snapshot(): - from orchestration.streaming import StreamingAgent + from core.llm.runtime.streaming import StreamingAgent class ModelCallEndEvent: input_tokens = 120 @@ -165,7 +162,7 @@ async def collect(): def test_compaction_projection_is_hidden_after_a_newer_provider_snapshot(): from datetime import datetime, timezone - from core.services.context_usage_projection import get_chat_context_state + from core.chat.compaction.usage_projection import get_chat_context_state compacted = build_compaction_context_usage( { @@ -208,7 +205,7 @@ def test_compaction_projection_is_hidden_after_a_newer_provider_snapshot(): def test_latest_persisted_snapshot_uses_newest_assistant_measurement(): - from core.services.context_usage_projection import get_chat_context_state + from core.chat.compaction.usage_projection import get_chat_context_state older = build_context_usage_snapshot( _agent(), prompt_tokens=100, completion_tokens=10, model_call_index=1 diff --git a/src/backend/tests/llm/test_execution_manifest.py b/src/backend/tests/llm/test_execution_manifest.py index 84d90c3f..249429ce 100644 --- a/src/backend/tests/llm/test_execution_manifest.py +++ b/src/backend/tests/llm/test_execution_manifest.py @@ -7,11 +7,7 @@ import sys import pytest -from core.llm.execution_manifest import ( - PromptManifestBuilder, - stable_hash, - stable_tool_schema_order, -) +from core.harness.execution_manifest import PromptManifestBuilder, stable_hash, stable_tool_schema_order def _tool_schema(*, enum_order=("fast", "safe")): @@ -91,7 +87,7 @@ def test_provider_tool_schema_order_is_stable_by_name(): @pytest.mark.asyncio async def test_agent_skill_prompt_order_is_stable_by_name(): from agentscope.skill import Skill - from core.ontology.toolkit import OntologyFilteredToolkit + from core.llm.tools.ontology_toolkit import OntologyFilteredToolkit toolkit = OntologyFilteredToolkit( skills_or_loaders=[ @@ -275,7 +271,7 @@ def test_aggregate_hash_is_stable_across_fresh_python_processes(): script = r""" import json import sys -from core.llm.execution_manifest import PromptManifestBuilder +from core.harness.execution_manifest import PromptManifestBuilder ordered = sys.argv[1] == "ordered" context = ( diff --git a/src/backend/tests/llm/test_execution_surface_limits.py b/src/backend/tests/llm/test_execution_surface_limits.py index c4675ead..5719e2d3 100644 --- a/src/backend/tests/llm/test_execution_surface_limits.py +++ b/src/backend/tests/llm/test_execution_surface_limits.py @@ -5,16 +5,13 @@ import pytest from agentscope.message import UserMsg from agentscope.tool import FunctionTool, ToolChunk -from core.llm.context_adapter import next_request_sequence, render_context_item -from core.llm.context_ir import ( - IMAGE_TOKEN_RESERVE, - KIND_USER_INPUT, - POLICY_NEVER, - make_text_context_item, -) -from core.llm.manifest_agent import ManifestBoundAgent -from core.llm.message_compat import session_to_msgs -from core.ontology.toolkit import OntologyFilteredToolkit +from core.llm.context.rendering import next_request_sequence, render_context_item +from core.llm.context.ir_constants import KIND_USER_INPUT, POLICY_NEVER +from core.llm.context.items import make_text_context_item +from core.models.constants import IMAGE_TOKEN_RESERVE +from core.llm.runtime.manifest_agent import ManifestBoundAgent +from core.llm.runtime.message_compat import session_to_msgs +from core.llm.tools.ontology_toolkit import OntologyFilteredToolkit from tests.llm.test_execution_surface_snapshot import CaptureModel @@ -37,7 +34,7 @@ def oversized_tool() -> ToolChunk: toolkit=toolkit, ) - from core.llm.context_safety import ContextBudgetExceeded + from core.llm.context.safety import ContextBudgetExceeded with pytest.raises(ContextBudgetExceeded): await agent.reply(UserMsg(name="user", content="current")) @@ -49,17 +46,19 @@ def oversized_tool() -> ToolChunk: @pytest.mark.asyncio async def test_public_multimodal_request_keeps_or_drops_each_image_whole(monkeypatch): - from core.llm import middlewares as mw + from core.llm.context import attachment_hook as mw + from core.llm.runtime.state import AgentRuntimeState + from tests.orchestration.hook_support import make_adapter monkeypatch.setattr( - mw, - "_build_file_context", + mw.file_context, + "build_file_context", lambda files, user_id=None: "TEXT-FILE-EVIDENCE", ) - monkeypatch.setattr(mw, "_effective_model_supports_vision", lambda state: True) + monkeypatch.setattr(mw, "supports_native_images", lambda state: True) monkeypatch.setattr( - mw, - "_fetch_image_base64", + mw.file_context, + "fetch_image_base64", lambda file, user_id=None: ("A" * 4_000, "image/png"), ) model = CaptureModel() @@ -70,8 +69,8 @@ async def test_public_multimodal_request_keeps_or_drops_each_image_whole(monkeyp system_prompt="rules", model=model, toolkit=OntologyFilteredToolkit(), - middlewares=[mw.FileContextMiddleware()], - state=mw.AgentRuntimeState( + middlewares=[make_adapter(mw.attachment_context_hook())], + state=AgentRuntimeState( uploaded_files=[ {"file_id": "doc", "name": "notes.txt", "mime_type": "text/plain"}, {"file_id": "one", "name": "one.png", "mime_type": "image/png"}, diff --git a/src/backend/tests/llm/test_execution_surface_snapshot.py b/src/backend/tests/llm/test_execution_surface_snapshot.py index 9dec63f8..21727161 100644 --- a/src/backend/tests/llm/test_execution_surface_snapshot.py +++ b/src/backend/tests/llm/test_execution_surface_snapshot.py @@ -6,18 +6,16 @@ from agentscope.message import TextBlock, UserMsg from agentscope.model import ChatResponse, ChatUsage from agentscope.skill import Skill, SkillLoaderBase -from core.llm.compaction import build_compacted_history -from core.llm.context_adapter import ( - PROVIDER_CONTEXT_META_KEY, - AgentScopeContextAdapter, - next_request_sequence, - render_context_item, -) -from core.llm.context_ir import KIND_USER_INPUT, POLICY_NEVER, make_text_context_item -from core.llm.execution_manifest import PromptManifestBuilder, stable_hash -from core.llm.manifest_agent import ManifestBoundAgent -from core.llm.message_compat import session_to_msgs -from core.ontology.toolkit import OntologyFilteredToolkit +from core.llm.context.compaction import build_compacted_history +from core.llm.context.adapter import AgentScopeContextAdapter +from core.llm.context.rendering import next_request_sequence, render_context_item +from core.models.constants import PROVIDER_CONTEXT_META_KEY +from core.llm.context.ir_constants import KIND_USER_INPUT, POLICY_NEVER +from core.llm.context.items import make_text_context_item +from core.harness.execution_manifest import PromptManifestBuilder, stable_hash +from core.llm.runtime.manifest_agent import ManifestBoundAgent +from core.llm.runtime.message_compat import session_to_msgs +from core.llm.tools.ontology_toolkit import OntologyFilteredToolkit class CountingSkillLoader(SkillLoaderBase): @@ -40,6 +38,8 @@ async def list_skills(self): # noqa: ANN201 class CaptureModel: + from agentscope.formatter import OpenAIChatFormatter + formatter = OpenAIChatFormatter() model = "capture-model" context_size = 32_768 diff --git a/src/backend/tests/llm/test_failover_context_safety.py b/src/backend/tests/llm/test_failover_context_safety.py index 0a785810..4357f415 100644 --- a/src/backend/tests/llm/test_failover_context_safety.py +++ b/src/backend/tests/llm/test_failover_context_safety.py @@ -3,10 +3,10 @@ import pytest from agentscope.message import UserMsg from agentscope.model import ChatResponse -from core.llm import failover -from core.llm.context_safety import ContextBudgetExceeded -from core.llm.manifest_agent import ManifestBoundAgent -from core.ontology.toolkit import OntologyFilteredToolkit +from core.models import failover +from core.llm.context.safety import ContextBudgetExceeded +from core.llm.runtime.manifest_agent import ManifestBoundAgent +from core.llm.tools.ontology_toolkit import OntologyFilteredToolkit from tests.llm.test_model_failover import _build, _Candidate, _status diff --git a/src/backend/tests/llm/test_file_context_offloads_blocking_parse.py b/src/backend/tests/llm/test_file_context_offloads_blocking_parse.py index f50f7ee6..bb34a306 100644 --- a/src/backend/tests/llm/test_file_context_offloads_blocking_parse.py +++ b/src/backend/tests/llm/test_file_context_offloads_blocking_parse.py @@ -19,7 +19,8 @@ import pytest -from core.llm import middlewares as mw +from core.llm.context import attachment_hook as mw +from tests.orchestration.hook_support import inject_attachments # How long the fake "parse" blocks its thread. Long enough that a blocked loop is # unambiguous, short enough to keep the suite fast. @@ -80,10 +81,10 @@ def fake_build_file_context(uploaded_files, user_id=None): time.sleep(BLOCKING_SECONDS) return "[file content begin]parsed[file content end]" - monkeypatch.setattr(mw, "_build_file_context", fake_build_file_context) + monkeypatch.setattr(mw.file_context, "build_file_context", fake_build_file_context) agent = _make_agent([{"file_id": "ua_x", "name": "a.pdf", "mime_type": "application/pdf"}]) - ticks = await _count_heartbeats_during(mw.FileContextMiddleware()._inject(agent)) + ticks = await _count_heartbeats_during(inject_attachments(agent)) # The parse actually ran, with ownership still propagated. assert called["n"] == 1 @@ -103,23 +104,23 @@ async def test_native_image_injection_does_not_block_the_event_loop(monkeypatch) Same defect class: on S3/OSS deployments each image is a network round-trip. """ - monkeypatch.setattr(mw, "_effective_model_supports_vision", lambda st: True) + monkeypatch.setattr(mw, "supports_native_images", lambda st: True) - def fake_inject_native_images(st, image_files, user_id): + def fake_inject_native_images(image_files, user_id): time.sleep(BLOCKING_SECONDS) - st.context.append("images-injected") + return [{"type": "text", "text": "images-injected"}] monkeypatch.setattr( - mw.FileContextMiddleware, - "_inject_native_images", - staticmethod(fake_inject_native_images), + mw, + "native_image_content", + fake_inject_native_images, ) - monkeypatch.setattr(mw, "_build_file_context", lambda files, user_id=None: "") + monkeypatch.setattr(mw.file_context, "build_file_context", lambda files, user_id=None: "") agent = _make_agent([{"file_id": "ua_img", "name": "a.png", "mime_type": "image/png"}]) - ticks = await _count_heartbeats_during(mw.FileContextMiddleware()._inject(agent)) + ticks = await _count_heartbeats_during(inject_attachments(agent)) - assert "images-injected" in agent.state.context + assert any("images-injected" in str(getattr(m, "content", "")) for m in agent.state.context) assert ( ticks >= MIN_EXPECTED_TICKS ), f"event loop stalled during image injection: only {ticks} heartbeat(s)" diff --git a/src/backend/tests/llm/test_file_context_preview.py b/src/backend/tests/llm/test_file_context_preview.py index 3b21f4c3..416700fa 100644 --- a/src/backend/tests/llm/test_file_context_preview.py +++ b/src/backend/tests/llm/test_file_context_preview.py @@ -1,4 +1,7 @@ """Regression tests for file_id-only attachment requests and bounded previews.""" +from core.llm.runtime import file_context as file_context_module +from core.files.content import spreadsheet_parser +from core.files import artifact_bytes as shared_artifact_bytes def test_attachment_schema_discards_legacy_content_and_download_url(): @@ -22,8 +25,8 @@ def test_attachment_schema_discards_legacy_content_and_download_url(): def test_regular_file_context_uses_bounded_lazy_preview(monkeypatch): - from core.content import artifact_reader - from core.llm.hooks import _build_file_context + from core.files.artifacts import reader as artifact_reader + from core.llm.runtime.file_context import build_file_context as _build_file_context parsed = "A" * artifact_reader.ATTACHMENT_PREVIEW_MAX_CHARS + "UNSEEN_TAIL" monkeypatch.setattr(artifact_reader, "fetch_parsed_text", lambda *args, **kwargs: parsed) @@ -47,13 +50,12 @@ def test_regular_file_context_uses_bounded_lazy_preview(monkeypatch): def test_xlsx_context_uses_preview_budget_without_batch_hint(monkeypatch): - from core.content import artifact_reader - from core.content import file_parser - from core.llm import hooks + from core.files.artifacts import reader as artifact_reader + from core.files.content import file_parser seen = {} - monkeypatch.setattr(hooks, "_download_artifact_bytes", lambda *args, **kwargs: b"xlsx") + monkeypatch.setattr(shared_artifact_bytes, 'download_artifact_bytes', lambda *args, **kwargs: b"xlsx") monkeypatch.setattr(artifact_reader, "fetch_parsed_text", lambda *args, **kwargs: "cached") def fake_preview(_file_bytes, char_budget): @@ -68,9 +70,9 @@ def fake_preview(_file_bytes, char_budget): "truncated_cells": 0, } - monkeypatch.setattr(file_parser, "parse_xlsx_preview", fake_preview) + monkeypatch.setattr(spreadsheet_parser, 'parse_xlsx_preview', fake_preview) - block = hooks._build_xlsx_preview_block( + block = file_context_module.build_xlsx_preview_block( { "name": "large.xlsx", "mime_type": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", diff --git a/src/backend/tests/llm/test_live_approval_integration.py b/src/backend/tests/llm/test_live_approval_integration.py index 7c37f70f..4f57b778 100644 --- a/src/backend/tests/llm/test_live_approval_integration.py +++ b/src/backend/tests/llm/test_live_approval_integration.py @@ -3,15 +3,15 @@ import asyncio import uuid import pytest -from core.llm.tools import _tool_confirmation as confirms -from core.services.user_service import UserService +from core.llm.tools import confirmation as confirms +from core.auth.users import UserService from tests.llm.test_shared_human_interactions import shared_store, pending_on from tests.storage.project_fixtures import team @pytest.fixture(autouse=True) def isolated_local_confirm_state(monkeypatch): - monkeypatch.setattr(confirms.state, "_CHATS", {}) + monkeypatch.setattr(confirms.state.process_state(), "CHATS", {}) @pytest.mark.asyncio @@ -178,7 +178,7 @@ async def ask(): pending = await pending_on(confirms, chat) cid = pending[0]["confirm_id"] for _ in range(100): - if confirms.state._CHATS[chat].pending[cid].waiters == 2: + if confirms.state.process_state().CHATS[chat].pending[cid].waiters == 2: break await asyncio.sleep(0.01) assert (await confirms.set_decision_shared(chat, cid, "allow"))["ok"] diff --git a/src/backend/tests/llm/test_live_approval_repair.py b/src/backend/tests/llm/test_live_approval_repair.py index e6658fdf..c9409ec9 100644 --- a/src/backend/tests/llm/test_live_approval_repair.py +++ b/src/backend/tests/llm/test_live_approval_repair.py @@ -3,7 +3,7 @@ from dataclasses import replace from unittest.mock import AsyncMock import pytest -from core.llm.tool_permissions import ToolPermissionMiddleware +from tests.orchestration.hook_support import run_permission from core.llm.tool_permissions.pending import approve_current_policy from tests.llm.test_approval_modes import _service, _tool_call from tests.storage.project_fixtures import team @@ -16,18 +16,16 @@ async def test_running_dispatch_rechecks_account_mode(monkeypatch, initial, current, executes): service = _service(initial) service.runtime = replace(service.runtime, account_approval=True) - monkeypatch.setattr("core.llm.tool_permissions.account.read_account_mode", lambda *a: current) + monkeypatch.setattr("core.sandbox.permissions.account.read_account_mode", lambda *a: current) gate = AsyncMock(return_value={"error": "not approved"}) - monkeypatch.setattr("core.llm.tools._tool_confirmation.gate", gate) + monkeypatch.setattr("core.llm.tools.confirmation.gate", gate) async def execute(**kwargs): yield "executed" result = [ x - async for x in ToolPermissionMiddleware(service)._act( - {"tool_call": _tool_call("space_delete", {"path": "/myspace/a.txt"})}, execute - ) + async for x in run_permission(service, {"tool_call": _tool_call("space_delete", {"path": "/myspace/a.txt"})}, execute) ] assert (result == ["executed"]) is executes assert gate.await_count == (0 if executes else 1) @@ -36,20 +34,18 @@ async def execute(**kwargs): @pytest.mark.asyncio async def test_explicit_module_policy_is_not_replaced(monkeypatch): monkeypatch.setattr( - "core.llm.tool_permissions.account.read_account_mode", + "core.sandbox.permissions.account.read_account_mode", lambda *a: (_ for _ in ()).throw(AssertionError("account cannot replace module")), ) gate = AsyncMock(return_value={"error": "not approved"}) - monkeypatch.setattr("core.llm.tools._tool_confirmation.gate", gate) + monkeypatch.setattr("core.llm.tools.confirmation.gate", gate) async def execute(**kwargs): yield "executed" result = [ x - async for x in ToolPermissionMiddleware(_service("ask"))._act( - {"tool_call": _tool_call("space_delete", {"path": "/myspace/a.txt"})}, execute - ) + async for x in run_permission(_service("ask"), {"tool_call": _tool_call("space_delete", {"path": "/myspace/a.txt"})}, execute) ] assert result != ["executed"] gate.assert_awaited_once() @@ -63,18 +59,16 @@ async def test_unavailable_policy_blocks_without_prompt(monkeypatch): def unavailable(*args): raise OSError("database offline") - monkeypatch.setattr("core.llm.tool_permissions.account.read_account_mode", unavailable) + monkeypatch.setattr("core.sandbox.permissions.account.read_account_mode", unavailable) gate = AsyncMock() - monkeypatch.setattr("core.llm.tools._tool_confirmation.gate", gate) + monkeypatch.setattr("core.llm.tools.confirmation.gate", gate) async def execute(**kwargs): yield "executed" result = [ x - async for x in ToolPermissionMiddleware(service)._act( - {"tool_call": _tool_call("space_delete", {"path": "/myspace/a.txt"})}, execute - ) + async for x in run_permission(service, {"tool_call": _tool_call("space_delete", {"path": "/myspace/a.txt"})}, execute) ] assert result != ["executed"] gate.assert_not_called() @@ -82,10 +76,10 @@ async def execute(**kwargs): def test_api_visitor_and_other_owner_cannot_use_account_policy(team): from core.db.models import ChatSession - from core.llm.tool_permissions.account import read_account_mode + from core.sandbox.permissions.account import read_account_mode db, _ = team - from core.services.user_service import UserService + from core.auth.users import UserService UserService(db).update_user_metadata("alice", {"tool_approval_mode": "full"}) db.add_all( [ diff --git a/src/backend/tests/llm/test_local_bash_permission.py b/src/backend/tests/llm/test_local_bash_permission.py index b944229a..6d788b42 100644 --- a/src/backend/tests/llm/test_local_bash_permission.py +++ b/src/backend/tests/llm/test_local_bash_permission.py @@ -16,15 +16,10 @@ import pytest from agentscope.message import ToolCallBlock -from core.llm.tool_permissions import ( - APPROVAL_ASK, - CURRENT_PERMISSION_TICKET, - FAIL_CLOSED_MODE, - PermissionRuntime, - ToolPermissionRegistry, - ToolPermissionService, - builtin_tool_permission, -) +from core.sandbox.permissions.approval import APPROVAL_ASK +from core.sandbox.permissions.dispatch_models import CURRENT_PERMISSION_TICKET, FAIL_CLOSED_MODE, PermissionRuntime +from core.sandbox.permissions.tool_specs import ToolPermissionRegistry, builtin_tool_permission +from core.llm.tool_permissions import ToolPermissionService from core.llm.tools.sandbox_tool import register_bash from tests.llm.command_test_support import command_tool_state @@ -77,7 +72,9 @@ def _capture(policy, context): class _Toolkit: fn = None - def register_tool_function(self, fn, **_kwargs): + def append(self, tool): + fn = tool._func + _kwargs = {"func_description": tool.description, "permission": getattr(tool, "dispatch_spec", None)} self.fn = fn @@ -86,7 +83,6 @@ def _bash(): register_bash( toolkit, loader=None, - loaded_skill_ids=set(), chat_id="chat-1", user_id="user-1", ) assert toolkit.fn is not None @@ -138,9 +134,9 @@ async def test_noninteractive_confirm_is_rejected_before_execution(): with ( patch.dict("os.environ", {"SANDBOX_TOOLS_ENABLED": "true"}), patch("core.config.local_mode.local_mode_enabled", return_value=True), - patch("core.services.local_grant_service.grants_for_gate", return_value=[]), + patch('features.desktop.local.grants.grants_for_gate', return_value=[]), patch( - "core.services.local_grant_service.policy_for_gate", + 'features.desktop.local.grants.policy_for_gate', return_value=Policy(), ), ): @@ -166,9 +162,9 @@ async def test_a_command_that_cannot_be_confined_is_refused(approval_mode): with ( patch.dict("os.environ", {"SANDBOX_TOOLS_ENABLED": "true"}), patch("core.config.local_mode.local_mode_enabled", return_value=True), - patch("core.services.local_grant_service.grants_for_gate", return_value=[]), + patch('features.desktop.local.grants.grants_for_gate', return_value=[]), patch( - "core.services.local_grant_service.policy_for_gate", + 'features.desktop.local.grants.policy_for_gate', return_value=Policy(), ), patch(_CONFINE, side_effect=SandboxUnavailableError("sandbox unavailable")), @@ -195,9 +191,9 @@ async def _execute(req): with ( patch.dict("os.environ", {"SANDBOX_TOOLS_ENABLED": "true"}), patch("core.config.local_mode.local_mode_enabled", return_value=True), - patch("core.services.local_grant_service.grants_for_gate", return_value=[]), + patch('features.desktop.local.grants.grants_for_gate', return_value=[]), patch( - "core.services.local_grant_service.policy_for_gate", + 'features.desktop.local.grants.policy_for_gate', return_value=Policy(), ), _patch_host_provider(_execute), @@ -218,9 +214,9 @@ async def test_blocking_the_network_category_restricts_the_sandbox_network(): with ( patch.dict("os.environ", {"SANDBOX_TOOLS_ENABLED": "true"}), patch("core.config.local_mode.local_mode_enabled", return_value=True), - patch("core.services.local_grant_service.grants_for_gate", return_value=[]), + patch('features.desktop.local.grants.grants_for_gate', return_value=[]), patch( - "core.services.local_grant_service.policy_for_gate", + 'features.desktop.local.grants.policy_for_gate', return_value=Policy(danger={"network": "block"}), ), patch(_CONFINE, new=_capture_policy(captured)), @@ -236,9 +232,9 @@ async def test_leaving_the_network_category_alone_keeps_the_sandbox_network_open with ( patch.dict("os.environ", {"SANDBOX_TOOLS_ENABLED": "true"}), patch("core.config.local_mode.local_mode_enabled", return_value=True), - patch("core.services.local_grant_service.grants_for_gate", return_value=[]), + patch('features.desktop.local.grants.grants_for_gate', return_value=[]), patch( - "core.services.local_grant_service.policy_for_gate", + 'features.desktop.local.grants.policy_for_gate', return_value=Policy(), ), patch(_CONFINE, new=_capture_policy(captured)), @@ -265,15 +261,15 @@ async def test_approved_copy_only_adds_destination_to_one_shot_write_set(): patch.dict("os.environ", {"SANDBOX_TOOLS_ENABLED": "true"}), patch("core.config.local_mode.local_mode_enabled", return_value=True), patch( - "core.services.local_grant_service.grants_for_gate", + 'features.desktop.local.grants.grants_for_gate', return_value=[Grant(str(source_dir), "read")], ), patch( - "core.services.local_grant_service.policy_for_gate", + 'features.desktop.local.grants.policy_for_gate', return_value=Policy(), ), patch( - "core.llm.tools._tool_confirmation.gate", + "core.llm.tools.confirmation.gate", new=AsyncMock(return_value=None), ), patch(_CONFINE, new=_capture_policy(captured)), @@ -298,11 +294,11 @@ async def test_system_overlapping_grant_is_not_a_standing_os_write_bind(): patch.dict("os.environ", {"SANDBOX_TOOLS_ENABLED": "true"}), patch("core.config.local_mode.local_mode_enabled", return_value=True), patch( - "core.services.local_grant_service.grants_for_gate", + 'features.desktop.local.grants.grants_for_gate', return_value=[Grant("/", "readwrite")], ), patch( - "core.services.local_grant_service.policy_for_gate", + 'features.desktop.local.grants.policy_for_gate', return_value=Policy(), ), patch(_CONFINE, new=_capture_policy(captured)), @@ -336,9 +332,9 @@ async def test_an_unattended_run_is_confined_like_an_interactive_one(): with ( patch.dict("os.environ", {"SANDBOX_TOOLS_ENABLED": "true"}), patch("core.config.local_mode.local_mode_enabled", return_value=True), - patch("core.services.local_grant_service.grants_for_gate", return_value=[]), + patch('features.desktop.local.grants.grants_for_gate', return_value=[]), patch( - "core.services.local_grant_service.policy_for_gate", + 'features.desktop.local.grants.policy_for_gate', return_value=Policy(), ), patch(_CONFINE, new=_capture_policy(captured)), diff --git a/src/backend/tests/llm/test_local_confinement_contract.py b/src/backend/tests/llm/test_local_confinement_contract.py index afd9fbde..0ad211f2 100644 --- a/src/backend/tests/llm/test_local_confinement_contract.py +++ b/src/backend/tests/llm/test_local_confinement_contract.py @@ -11,15 +11,8 @@ from unittest.mock import patch import pytest -from core.llm.tool_permissions import ( - APPROVAL_ASK, - APPROVAL_AUTO, - APPROVAL_FULL, - FAIL_CLOSED_MODE, - UNCONFINED_APPROVAL_MODES, - LocalCommandAuthorization, - LocalConfinementUnavailableError, -) +from core.sandbox.permissions.approval import APPROVAL_ASK, APPROVAL_AUTO, APPROVAL_FULL +from core.sandbox.permissions.dispatch_models import FAIL_CLOSED_MODE, UNCONFINED_APPROVAL_MODES, LocalCommandAuthorization, LocalConfinementUnavailableError from core.sandbox.permissions.os_gate import LocalAccessDecision from core.sandbox.oslayer import SandboxUnavailableError, SandboxUnenforceableError diff --git a/src/backend/tests/llm/test_local_file_permission.py b/src/backend/tests/llm/test_local_file_permission.py index 4fd08982..ef884832 100644 --- a/src/backend/tests/llm/test_local_file_permission.py +++ b/src/backend/tests/llm/test_local_file_permission.py @@ -11,15 +11,11 @@ from unittest.mock import AsyncMock, MagicMock, patch from agentscope.message import ToolCallBlock -from core.llm.tool_permissions import ( - CURRENT_PERMISSION_TICKET, - PermissionRuntime, - ToolPermissionRegistry, - ToolPermissionService, - builtin_tool_permission, -) -from core.llm.tools import _tool_confirmation as confirm -from core.llm.tools._paths import to_physical_path +from core.sandbox.permissions.dispatch_models import CURRENT_PERMISSION_TICKET, PermissionRuntime +from core.sandbox.permissions.tool_specs import ToolPermissionRegistry, builtin_tool_permission +from core.llm.tool_permissions import ToolPermissionService +from core.llm.tools import confirmation as confirm +from core.sandbox.workspace.resolution import to_physical_path from core.llm.tools._state import ReadEntry, ReadStateTracker from core.llm.tools.edit_tool import register_edit from core.llm.tools.write_tool import register_write @@ -32,8 +28,8 @@ def isolated_permission_store(monkeypatch): from core.infra.ephemeral import LocalEphemeralState store = LocalEphemeralState() monkeypatch.setattr("core.infra.ephemeral.get_ephemeral_state", lambda: store) - monkeypatch.setattr("core.llm.interaction_store.get_ephemeral_state", lambda: store) - monkeypatch.setattr(confirm.state, "_CHATS", {}) + monkeypatch.setattr('core.llm.runtime.interaction_store.get_ephemeral_state', lambda: store) + monkeypatch.setattr(confirm.state.process_state(), "CHATS", {}) async def _pending_pair(chat_id): @@ -54,7 +50,9 @@ def _payload(response) -> dict: class _Toolkit: fn = None - def register_tool_function(self, fn, **_kwargs): + def append(self, tool): + fn = tool._func + _kwargs = {"func_description": tool.description, "permission": getattr(tool, "dispatch_spec", None)} self.fn = fn @@ -90,9 +88,9 @@ def _call(name: str, arguments: dict) -> ToolCallBlock: async def test_strict_workspace_write_is_blocked_before_file_io(): with ( patch("core.config.local_mode.local_mode_enabled", return_value=True), - patch("core.services.local_grant_service.grants_for_gate", return_value=[]), + patch('features.desktop.local.grants.grants_for_gate', return_value=[]), patch( - "core.services.local_grant_service.policy_for_gate", + 'features.desktop.local.grants.policy_for_gate', return_value=Policy(workspace_write="block", out_of_scope="block"), ), ): @@ -107,9 +105,9 @@ async def test_read_grant_allows_read_but_not_unattended_write(): grants = [Grant("/data/project", "read")] with ( patch("core.config.local_mode.local_mode_enabled", return_value=True), - patch("core.services.local_grant_service.grants_for_gate", return_value=grants), + patch('features.desktop.local.grants.grants_for_gate', return_value=grants), patch( - "core.services.local_grant_service.policy_for_gate", + 'features.desktop.local.grants.policy_for_gate', return_value=Policy(out_of_scope="confirm"), ), ): @@ -130,9 +128,9 @@ async def test_read_grant_allows_read_but_not_unattended_write(): async def test_missing_chat_cannot_turn_confirm_into_an_implicit_grant(): with ( patch("core.config.local_mode.local_mode_enabled", return_value=True), - patch("core.services.local_grant_service.grants_for_gate", return_value=[]), + patch('features.desktop.local.grants.grants_for_gate', return_value=[]), patch( - "core.services.local_grant_service.policy_for_gate", + 'features.desktop.local.grants.policy_for_gate', return_value=Policy(out_of_scope="confirm"), ), ): @@ -146,9 +144,9 @@ async def test_missing_chat_cannot_turn_confirm_into_an_implicit_grant(): async def test_read_image_file_path_uses_the_same_local_read_gate(): with ( patch("core.config.local_mode.local_mode_enabled", return_value=True), - patch("core.services.local_grant_service.grants_for_gate", return_value=[]), + patch('features.desktop.local.grants.grants_for_gate', return_value=[]), patch( - "core.services.local_grant_service.policy_for_gate", + 'features.desktop.local.grants.policy_for_gate', return_value=Policy(out_of_scope="confirm"), ), ): @@ -268,11 +266,11 @@ async def test_local_write_detects_existing_host_file_before_overwrite(): with ( patch("core.config.local_mode.local_mode_enabled", return_value=True), patch( - "core.services.local_grant_service.grants_for_gate", + 'features.desktop.local.grants.grants_for_gate', return_value=[Grant(tmp, "readwrite")], ), patch( - "core.services.local_grant_service.policy_for_gate", + 'features.desktop.local.grants.policy_for_gate', return_value=Policy(), ), patch("core.sandbox.get_sandbox_provider", return_value=provider), @@ -377,7 +375,9 @@ async def test_remote_edit_handles_a_sandbox_read_error_without_a_scope_error(): def test_gate_treats_the_session_directory_as_the_workspace(monkeypatch, tmp_path): """权限闸里的"工作区"就是本次对话的工作目录，不是部署级工作区根。""" - from core.llm.tool_permissions import PermissionRuntime, ToolPermissionRegistry, ToolPermissionService + from core.sandbox.permissions.dispatch_models import PermissionRuntime + from core.sandbox.permissions.tool_specs import ToolPermissionRegistry + from core.llm.tool_permissions import ToolPermissionService from core.sandbox.workspace.paths import session_root monkeypatch.setattr("core.sandbox.workspace.paths.WORKSPACE", str(tmp_path)) diff --git a/src/backend/tests/llm/test_local_myspace_block.py b/src/backend/tests/llm/test_local_myspace_block.py index cf75efcc..720e976f 100644 --- a/src/backend/tests/llm/test_local_myspace_block.py +++ b/src/backend/tests/llm/test_local_myspace_block.py @@ -3,7 +3,7 @@ from __future__ import annotations import core.config.local_mode as lm -from core.llm.tools._paths import validate_workspace_path +from core.sandbox.workspace.resolution import validate_workspace_path def test_myspace_rejected_in_local_mode(monkeypatch): diff --git a/src/backend/tests/llm/test_local_project_cwd.py b/src/backend/tests/llm/test_local_project_cwd.py index 0f427021..060ad2fb 100644 --- a/src/backend/tests/llm/test_local_project_cwd.py +++ b/src/backend/tests/llm/test_local_project_cwd.py @@ -1,4 +1,6 @@ """Local project paths agree across the public tool/runner/preview boundaries.""" +from core.capabilities.skills import loader_factory as loader_factory_module +from core.config.store import system_config import json from types import SimpleNamespace @@ -6,7 +8,7 @@ import pytest from tests.llm.local_project_test_support import local_project, pin_tool -from core.llm.tool_collector import ToolCollector +from core.llm.factory.tools.definitions import ToolAssembly from tests.llm.command_test_support import command_tool_state @@ -21,15 +23,11 @@ async def test_relative_write_lands_in_project_and_pin_keeps_original(local_proj monkeypatch.setattr("core.sandbox.get_sandbox_provider", ScriptRunnerProvider) from core.llm.tools.write_tool import register_write from core.llm.tools._state import ReadStateTracker - from core.llm.tool_permissions import ( - CURRENT_PERMISSION_TICKET, - PermissionTicket, - PermissionIntent, - ) + from core.sandbox.permissions.dispatch_models import CURRENT_PERMISSION_TICKET, PermissionTicket, PermissionIntent source, scope = local_project destination = source.parent / "交付 results" / "report.html" - collector = ToolCollector() + collector = ToolAssembly() register_write( collector, chat_id="pin-chat", @@ -52,7 +50,7 @@ async def test_relative_write_lands_in_project_and_pin_keeps_original(local_proj ) try: result = payload( - await collector.get_tool("Write")._func( + await next(t for t in collector if t.name == "Write")._func( file_path="交付 results/report.html", content="

Project original

" ) ) @@ -73,7 +71,7 @@ async def local_tools(local_project, command_tool_state, tmp_path, monkeypatch): import uvicorn from core.sandbox.providers.script_runner.provider import ScriptRunnerProvider from core.sandbox.runner import server - from core.llm.tools import _paths + from core.sandbox.workspace import resolution as _paths from core.llm.tools._state import ReadStateTracker from core.llm.tools.write_tool import register_write from core.llm.tools.read_tool import register_read @@ -82,12 +80,9 @@ async def local_tools(local_project, command_tool_state, tmp_path, monkeypatch): from core.llm.tools.grep_tool import register_grep from core.llm.tools.sandbox_tool import register_bash from core.llm.tools.pin_tool import register_pin_to_workspace - from core.llm.tool_permissions import ( - ToolPermissionRegistry, - ToolPermissionService, - PermissionRuntime, - CURRENT_PERMISSION_TICKET, - ) + from core.sandbox.permissions.tool_specs import ToolPermissionRegistry + from core.llm.tool_permissions import ToolPermissionService + from core.sandbox.permissions.dispatch_models import PermissionRuntime, CURRENT_PERMISSION_TICKET source, scope = local_project scratch_root = tmp_path / "session-storage" @@ -115,7 +110,7 @@ async def local_tools(local_project, command_tool_state, tmp_path, monkeypatch): ) provider = ScriptRunnerProvider() monkeypatch.setattr("core.sandbox.get_sandbox_provider", lambda: provider) - collector = ToolCollector() + collector = ToolAssembly() state = ReadStateTracker() common = dict( chat_id="pin-chat", sandbox_session_id="cwd-child", user_id="pin-user", scope=scope @@ -124,7 +119,7 @@ async def local_tools(local_project, command_tool_state, tmp_path, monkeypatch): register(collector, state=state, **common) for register in (register_glob, register_grep): register(collector, **common) - register_bash(collector, loader=None, loaded_skill_ids=set(), **common) + register_bash(collector, loader=None, **common) register_pin_to_workspace(collector, scope=scope, sandbox_session_id="cwd-child") registry = ToolPermissionRegistry() for name, spec in collector.permission_specs.items(): @@ -148,7 +143,7 @@ async def invoke(name, **args): return outcome.payload token = CURRENT_PERMISSION_TICKET.set(outcome.ticket) try: - return payload(await collector.get_tool(name)._func(**args)) + return payload(await next(t for t in collector if t.name == name)._func(**args)) finally: CURRENT_PERMISSION_TICKET.reset(token) @@ -300,13 +295,10 @@ async def test_project_changed_during_run_blocks_tools_without_fallback( async def test_relative_permissions_use_project_without_implicitly_granting_it( local_project, output_name ): - from core.llm.tool_permissions import ( - ToolPermissionRegistry, - ToolPermissionService, - PermissionRuntime, - builtin_tool_permission, - ) - from core.services.local_grant_service import remove_grant + from core.sandbox.permissions.tool_specs import ToolPermissionRegistry, builtin_tool_permission + from core.llm.tool_permissions import ToolPermissionService + from core.sandbox.permissions.dispatch_models import PermissionRuntime + from features.desktop.local.grants import remove_grant source, scope = local_project registry = ToolPermissionRegistry() @@ -364,7 +356,7 @@ async def test_real_factory_agent_uses_bound_project_for_file_and_bash(local_too from agentscope.message import UserMsg, TextBlock, ToolCallBlock from agentscope.model import ChatResponse from core.llm import factory as agent_factory - import core.agent_skills.loader as skill_loader + import core.capabilities.skills.loader as skill_loader invoke, root, scope, provider = local_tools @@ -379,6 +371,8 @@ def register_skills_to_toolkit(self, *args, **kwargs): return 0 class ScriptedModel: + from agentscope.formatter import OpenAIChatFormatter + formatter = OpenAIChatFormatter() model = "cwd-e2e-scripted" context_size = 32768 calls = 0 @@ -410,14 +404,15 @@ async def __call__(self, messages, tools=None, **kwargs): content = [TextBlock(type="text", text="done")] return ChatResponse(content=content, is_last=True) - monkeypatch.setattr(skill_loader, "get_skill_loader", lambda: EmptyLoader()) + monkeypatch.setattr(loader_factory_module, 'get_skill_loader', lambda: EmptyLoader()) from core.db.engine import SessionLocal - from core.services.run_journal import RunJournal + from core.runs.journal import RunJournal - monkeypatch.setattr("core.services.chat_steer_service.SessionLocal", SessionLocal) - from core.services import system_config + monkeypatch.setattr('core.chat.steer.SessionLocal', SessionLocal) + monkeypatch.setattr('core.runs.tool_effects.host.SessionLocal', SessionLocal) + from core.config.store import system_config - monkeypatch.setattr(system_config, "SessionLocal", SessionLocal) + monkeypatch.setattr("core.db.system_configuration.SessionLocal", SessionLocal) monkeypatch.setattr(system_config.SystemConfigService, "_instance", None) journal = RunJournal(SessionLocal) journal.accept( diff --git a/src/backend/tests/llm/test_local_project_delivery.py b/src/backend/tests/llm/test_local_project_delivery.py index c0696059..e750e96f 100644 --- a/src/backend/tests/llm/test_local_project_delivery.py +++ b/src/backend/tests/llm/test_local_project_delivery.py @@ -11,19 +11,19 @@ import core.db.models # noqa: F401 - register model metadata -from core.llm import workspace +from core.llm.runtime import workspace -from core.llm.tool_collector import ToolCollector +from core.llm.factory.tools.definitions import ToolAssembly from tests.llm.local_project_test_support import local_project, pin_tool -def test_session_delivery_deduplicates_and_rejects_conflicting_names( +def test_session_delivery_deduplicates_live_references_and_tracks_renames( local_project, monkeypatch, tmp_path ): - from core.llm.tools import _paths - from core.services.local_grant_service import add_grant + from core.sandbox.workspace import resolution as _paths + from features.desktop.local.grants import add_grant monkeypatch.setattr(_paths, "WORKSPACE_ROOT", str(tmp_path / "workspace")) root = Path(_paths.workspace_directory("pin-chat")) @@ -39,27 +39,33 @@ def test_session_delivery_deduplicates_and_rejects_conflicting_names( assert retry["pinned_count"] == 1 source.write_text("second") changed = json.loads(asyncio.run(pin(file_paths=[str(source)])).content[0].text) - assert changed["ok"] is False + assert changed["ok"] is True assert changed["pinned_count"] == 1 - assert "同名文件" in changed["failed"][0]["error"] + assert changed["pinned"][0]["file_id"] == first["pinned"][0]["file_id"] + assert changed["pinned"][0]["already_pinned"] is True + from core.files.artifacts.local_reference import authorized_reference + reference = authorized_reference(first["pinned"][0]["file_id"], "pin-user") + assert reference["path"] == str(source) + assert Path(reference["path"]).read_text() == "second" + assert authorized_reference(reference["file_id"], "another-user") is None renamed = source.with_name("output-v2.txt") source.rename(renamed) delivered = json.loads(asyncio.run(pin(file_paths=[str(renamed)])).content[0].text) assert delivered["pinned_count"] == 2 assert delivered["pinned"][0]["file_id"] != first["pinned"][0]["file_id"] + assert authorized_reference(first["pinned"][0]["file_id"], "pin-user") is None + assert Path(authorized_reference(delivered["pinned"][0]["file_id"], "pin-user")["path"]).read_text() == "second" def test_design_picker_accepts_local_images_without_pinning(local_project, monkeypatch, tmp_path): - from core.llm.tools import _paths, _tool_confirmation - from core.llm.tools.design_picker_tool import register_choose_design - from core.services.local_grant_service import add_grant - from core.llm.tool_permissions import ( - ToolPermissionRegistry, - ToolPermissionService, - PermissionRuntime, - CURRENT_PERMISSION_TICKET, - ) - from core.artifacts.store import get_artifact + from core.sandbox.workspace import resolution as _paths + from core.llm.tools import confirmation + from features.sites.tools.design_picker import register_choose_design + from features.desktop.local.grants import add_grant + from core.sandbox.permissions.tool_specs import ToolPermissionRegistry + from core.llm.tool_permissions import ToolPermissionService + from core.sandbox.permissions.dispatch_models import PermissionRuntime, CURRENT_PERMISSION_TICKET + from core.files.artifacts.store import get_artifact monkeypatch.setattr(_paths, "WORKSPACE_ROOT", str(tmp_path / "workspace")) root = Path(_paths.workspace_directory("pin-chat")) @@ -78,8 +84,8 @@ async def choose(**kwargs): assert get_artifact(option["image_file_id"])["mime_type"] == "image/png" return {"status": "chosen", "option_id": "b"} - monkeypatch.setattr(_tool_confirmation, "pick", choose) - collector = ToolCollector() + monkeypatch.setattr(confirmation, "pick", choose) + collector = ToolAssembly() register_choose_design(collector, chat_id="pin-chat", user_id="pin-user") registry = ToolPermissionRegistry() registry.register("choose_design", collector.permission_specs["choose_design"], source="test") @@ -101,10 +107,37 @@ async def invoke(): assert outcome.proceed token = CURRENT_PERMISSION_TICKET.set(outcome.ticket) try: - return await collector.get_tool("choose_design")._func(**args) + return await next(t for t in collector if t.name == "choose_design")._func(**args) finally: CURRENT_PERMISSION_TICKET.reset(token) result = json.loads(asyncio.run(invoke()).content[0].text) assert result["ok"] and result["selected_id"] == "b" assert workspace.get_pinned() == [] + + +def test_expired_inherited_ticket_cannot_read_or_overwrite_reference(local_project, db_session): + from contextvars import copy_context + from fastapi import HTTPException + import pytest + from core.files.artifacts.local_reference import authorized_reference, overwrite_reference + from core.sandbox.permissions.dispatch_models import CURRENT_PERMISSION_TICKET, PermissionTicket, PermissionIntent + from features.desktop.local.grants import remove_grant + + source, scope = local_project + delivered = json.loads(asyncio.run(pin_tool(scope)(file_paths=[str(source)])).content[0].text) + fid = delivered["pinned"][0]["file_id"] + ticket = PermissionTicket("Write", "finished-call", "", "test-write", ( + PermissionIntent("local_path", "write", str(source), "temporary approval"), + )) + token = CURRENT_PERMISSION_TICKET.set(ticket) + inherited = copy_context() + CURRENT_PERMISSION_TICKET.reset(token) + remove_grant(str(source.parent)) + # An in-flight approval is valid only until the dispatch completes. + assert inherited.run(authorized_reference, fid, "pin-user") is not None + ticket.lifetime.expire() + assert inherited.run(authorized_reference, fid, "pin-user") is None + with pytest.raises(HTTPException): + inherited.run(overwrite_reference, fid, "pin-user", b"unauthorized", db_session) + assert source.read_text() == "original report" diff --git a/src/backend/tests/llm/test_local_project_pin.py b/src/backend/tests/llm/test_local_project_pin.py index 4ace6de0..495043fe 100644 --- a/src/backend/tests/llm/test_local_project_pin.py +++ b/src/backend/tests/llm/test_local_project_pin.py @@ -15,9 +15,9 @@ from core.db.models import Project, UserShadow, Artifact -from core.llm import workspace +from core.llm.runtime import workspace -from core.llm.tool_collector import ToolCollector +from core.llm.factory.tools.definitions import ToolAssembly from tests.llm.local_project_test_support import local_project, pin_tool @@ -41,7 +41,7 @@ def test_pin_local_path_opens_original_and_deduplicates(local_project, db_sessio fid, BackgroundTasks(), mode="direct", inline=True, user=user, db=db_session ) assert Path(response.path).read_text() == "updated report" - from core.services.artifact_service import persist_artifacts + from core.files.artifacts.service import persist_artifacts persist_artifacts(db_session, "pin-user", None, workspace.get_pinned(), scope=scope) assert db_session.query(Artifact).count() == 0 @@ -58,7 +58,7 @@ def test_existing_export_then_pin_does_not_copy_project_file( ): source, scope = local_project from core.llm.tools.sandbox_tool import register_sandbox_get_artifact - from core.artifacts.store import get_artifact + from core.files.artifacts.store import get_artifact class Provider: async def get_file_to_path(self, session, src, dst, **kwargs): @@ -67,9 +67,9 @@ async def get_file_to_path(self, session, src, dst, **kwargs): return len(data) monkeypatch.setattr("core.sandbox.get_sandbox_provider", lambda: Provider()) - collector = ToolCollector() + collector = ToolAssembly() register_sandbox_get_artifact(collector, chat_id="pin-chat", user_id="pin-user", scope=scope) - export = collector.get_tool("sandbox_get_artifact")._func + export = next(t for t in collector if t.name == "sandbox_get_artifact")._func result = json.loads(asyncio.run(export(src_path=str(source))).content[0].text) assert result.get("ok"), result assert get_artifact(result["file_id"])["path"] == str(source) @@ -84,7 +84,7 @@ def test_pinned_file_remains_readable_across_turns_and_saves_to_project( ): import io from fastapi import UploadFile - from core.content.artifact_reader import fetch_parsed_text, load_artifact_meta + from core.files.artifacts.reader import fetch_parsed_text, load_artifact_meta from api.routes.v1.file_upload import overwrite_file source, scope = local_project @@ -139,9 +139,9 @@ def test_reference_revalidates_project_and_file(local_project, db_session, tmp_p def test_revoked_folder_grant_blocks_pin_and_existing_reference(local_project, db_session): - from core.llm.tool_permissions import CURRENT_PERMISSION_TICKET - from core.services.local_grant_service import remove_grant - from core.content.artifact_reader import fetch_parsed_text + from core.sandbox.permissions.dispatch_models import CURRENT_PERMISSION_TICKET + from features.desktop.local.grants import remove_grant + from core.files.artifacts.reader import fetch_parsed_text from api.routes.files import local_file_location source, scope = local_project @@ -164,7 +164,7 @@ def test_live_spreadsheet_sheet_read_and_artifact_input(local_project): import base64 from openpyxl import Workbook from core.llm.tools.read_artifact_tool import register_read_artifact - from core.llm.tools._tool_helpers import _resolve_artifact_files + from core.files.artifacts.generated import resolve_artifact_files as _resolve_artifact_files source, scope = local_project sheet = source.with_suffix(".xlsx") @@ -175,10 +175,10 @@ def test_live_spreadsheet_sheet_read_and_artifact_input(local_project): fid = json.loads(asyncio.run(pin_tool(scope)(file_paths=[str(sheet)])).content[0].text)[ "pinned" ][0]["file_id"] - collector = ToolCollector() + collector = ToolAssembly() register_read_artifact(collector, user_id="pin-user") response = json.loads( - asyncio.run(collector.get_tool("read_artifact")._func(file_id=fid, sheet_name="预算")) + asyncio.run(next(t for t in collector if t.name == "read_artifact")._func(file_id=fid, sheet_name="预算")) .content[0] .text ) @@ -191,19 +191,19 @@ def test_live_spreadsheet_sheet_read_and_artifact_input(local_project): def test_scratch_export_references_original(local_project, monkeypatch, tmp_path): from core.llm.tools.sandbox_tool import register_sandbox_get_artifact - from core.artifacts.store import get_artifact - from core.artifacts.local_reference import is_local_file_ref + from core.files.artifacts.store import get_artifact + from core.files.artifacts.local_reference import is_local_file_ref source, scope = local_project - from core.llm.tools import _paths + from core.sandbox.workspace import resolution as _paths monkeypatch.setattr(_paths, "WORKSPACE_ROOT", str(tmp_path / "workspace")) root = Path(_paths.workspace_directory("pin-chat")) root.mkdir(parents=True) scratch = root / "scratch.txt" scratch.write_text("scratch output") - from core.services.local_grant_service import add_grant - from core.llm.tool_permissions import CURRENT_PERMISSION_TICKET, PermissionTicket, PermissionIntent + from features.desktop.local.grants import add_grant + from core.sandbox.permissions.dispatch_models import CURRENT_PERMISSION_TICKET, PermissionTicket, PermissionIntent add_grant(str(root)) ticket_token = CURRENT_PERMISSION_TICKET.set(PermissionTicket( @@ -228,11 +228,11 @@ async def get_file_to_path(self, session, src, dst, **kwargs): return Path(dst).stat().st_size monkeypatch.setattr("core.sandbox.get_sandbox_provider", lambda: Provider()) - collector = ToolCollector() + collector = ToolAssembly() register_sandbox_get_artifact(collector, chat_id="pin-chat", user_id="pin-user", scope=scope) result = json.loads( asyncio.run( - collector.get_tool("sandbox_get_artifact")._func( + next(t for t in collector if t.name == "sandbox_get_artifact")._func( src_path="/workspace/scratch/output.txt" ) ) @@ -249,19 +249,15 @@ async def get_file_to_path(self, session, src, dst, **kwargs): def test_write_delivers_original_project_file(local_project, monkeypatch): from core.llm.tools.write_tool import register_write from core.llm.tools._state import ReadStateTracker - from core.llm.tool_permissions import ( - CURRENT_PERMISSION_TICKET, - PermissionTicket, - PermissionIntent, - ) - from core.artifacts.store import get_artifact + from core.sandbox.permissions.dispatch_models import CURRENT_PERMISSION_TICKET, PermissionTicket, PermissionIntent + from core.files.artifacts.store import get_artifact source, scope = local_project target = source.parent / "new-report.txt" monkeypatch.setattr( "core.sandbox.get_sandbox_provider", lambda: SimpleNamespace(name="script_runner") ) - collector = ToolCollector() + collector = ToolAssembly() register_write( collector, chat_id="pin-chat", user_id="pin-user", state=ReadStateTracker(), scope=scope ) @@ -277,7 +273,7 @@ def test_write_delivers_original_project_file(local_project, monkeypatch): try: response = json.loads( asyncio.run( - collector.get_tool("Write")._func( + next(t for t in collector if t.name == "Write")._func( file_path=str(target), content="created in project", register_as_artifact=True ) ) @@ -296,7 +292,7 @@ def test_write_delivers_original_project_file(local_project, monkeypatch): @pytest.mark.parametrize("logical_alias", [False, True]) def test_export_rejects_project_symlink_escape(local_project, tmp_path, monkeypatch, logical_alias): from core.llm.tools.sandbox_tool import register_sandbox_get_artifact - from core.llm.tools import _paths + from core.sandbox.workspace import resolution as _paths source, scope = local_project outside = tmp_path / "outside.txt" @@ -314,10 +310,10 @@ def test_export_rejects_project_symlink_escape(local_project, tmp_path, monkeypa str(outside) if value == path else original_resolve(value, user, **kwargs) ), ) - collector = ToolCollector() + collector = ToolAssembly() register_sandbox_get_artifact(collector, chat_id="pin-chat", user_id="pin-user", scope=scope) result = json.loads( - asyncio.run(collector.get_tool("sandbox_get_artifact")._func(src_path=path)).content[0].text + asyncio.run(next(t for t in collector if t.name == "sandbox_get_artifact")._func(src_path=path)).content[0].text ) assert "error" in result and "file_id" not in result @@ -396,9 +392,9 @@ def test_desktop_bridge_previews_pinned_html_through_http(local_project, db_sess @pytest.mark.parametrize("bound_project", [False, True]) def test_pin_session_file_without_export(local_project, monkeypatch, tmp_path, bound_project): - from core.llm.tools import _paths - from core.artifacts.store import get_artifact - from core.services.local_grant_service import add_grant + from core.sandbox.workspace import resolution as _paths + from core.files.artifacts.store import get_artifact + from features.desktop.local.grants import add_grant _, scope = local_project monkeypatch.setattr(_paths, "WORKSPACE_ROOT", str(tmp_path / "workspace")) @@ -421,8 +417,8 @@ def test_pin_session_file_without_export(local_project, monkeypatch, tmp_path, b ) def test_path_delivery_rejects_invalid_sources(local_project, monkeypatch, tmp_path, case): from dataclasses import replace - from core.llm.tools import _paths - from core.services.local_grant_service import add_grant + from core.sandbox.workspace import resolution as _paths + from features.desktop.local.grants import add_grant from core.config.settings import settings import importlib @@ -462,9 +458,9 @@ def test_path_delivery_rejects_invalid_sources(local_project, monkeypatch, tmp_p def test_session_relative_delivery_reports_partial_failure(local_project, monkeypatch, tmp_path): - from core.llm.tools import _paths - from core.services.local_grant_service import add_grant - from core.artifacts.store import get_artifact + from core.sandbox.workspace import resolution as _paths + from features.desktop.local.grants import add_grant + from core.files.artifacts.store import get_artifact monkeypatch.setattr(_paths, "WORKSPACE_ROOT", str(tmp_path / "workspace")) root = Path(_paths.workspace_directory("pin-chat")) diff --git a/src/backend/tests/llm/test_local_session_reference.py b/src/backend/tests/llm/test_local_session_reference.py index ea583833..39249337 100644 --- a/src/backend/tests/llm/test_local_session_reference.py +++ b/src/backend/tests/llm/test_local_session_reference.py @@ -7,15 +7,15 @@ import pytest from fastapi import BackgroundTasks, HTTPException, UploadFile -from core.artifacts.store import get_artifact +from core.files.artifacts.store import get_artifact from core.db.models import Artifact -from core.llm import workspace +from core.llm.runtime import workspace from tests.llm.local_project_test_support import local_project, pin_tool def session_file(tmp_path, monkeypatch): - from core.llm.tools import _paths - from core.services.local_grant_service import add_grant + from core.sandbox.workspace import resolution as _paths + from features.desktop.local.grants import add_grant monkeypatch.setattr(_paths, "WORKSPACE_ROOT", str(tmp_path / "workspace")) root = Path(_paths.workspace_directory("pin-chat")) @@ -28,8 +28,8 @@ def session_file(tmp_path, monkeypatch): def test_session_delivery_is_live_and_keeps_id_across_turns(local_project, db_session, tmp_path, monkeypatch): from api.routes.files import local_file_location, download_file - from core.content.artifact_reader import fetch_parsed_text - from core.services.artifact_service import persist_artifacts + from core.files.artifacts.reader import fetch_parsed_text + from core.files.artifacts.service import persist_artifacts source = session_file(tmp_path, monkeypatch) pin = pin_tool(None, session_id="pin-chat") @@ -77,8 +77,8 @@ def test_session_canvas_saves_original_and_rechecks_ownership(local_project, db_ @pytest.mark.parametrize("change", ["symlink", "revoked"]) def test_session_access_revalidates_source(local_project, db_session, tmp_path, monkeypatch, change): from api.routes.files import local_file_location - from core.services.local_grant_service import remove_grant - from core.llm.tool_permissions import CURRENT_PERMISSION_TICKET + from features.desktop.local.grants import remove_grant + from core.sandbox.permissions.dispatch_models import CURRENT_PERMISSION_TICKET source = session_file(tmp_path, monkeypatch) result = json.loads(asyncio.run(pin_tool(None, "pin-chat")(file_paths=[str(source)])).content[0].text) @@ -101,19 +101,19 @@ def test_session_access_revalidates_source(local_project, db_session, tmp_path, def test_write_and_pin_use_same_session_reference(local_project, tmp_path, monkeypatch): from core.llm.tools.write_tool import register_write from core.llm.tools._state import ReadStateTracker - from core.llm.tool_collector import ToolCollector - from core.llm.tool_permissions import CURRENT_PERMISSION_TICKET, PermissionTicket, PermissionIntent + from core.llm.factory.tools.definitions import ToolAssembly + from core.sandbox.permissions.dispatch_models import CURRENT_PERMISSION_TICKET, PermissionTicket, PermissionIntent source = session_file(tmp_path, monkeypatch) source.unlink() - collector = ToolCollector() + collector = ToolAssembly() monkeypatch.setattr("core.sandbox.get_sandbox_provider", lambda: SimpleNamespace(name="script_runner")) register_write(collector, chat_id="pin-chat", user_id="pin-user", state=ReadStateTracker()) token = CURRENT_PERMISSION_TICKET.set(PermissionTicket( "Write", "write", "", "test", (PermissionIntent("local_path", "write", str(source), "create"),), )) try: - written = json.loads(asyncio.run(collector.get_tool("Write")._func( + written = json.loads(asyncio.run(next(t for t in collector if t.name == "Write")._func( file_path=str(source), content="written original", register_as_artifact=True, )).content[0].text) finally: diff --git a/src/backend/tests/llm/test_manifest_mcp_client.py b/src/backend/tests/llm/test_manifest_mcp_client.py index 9b45f249..2f56c2c1 100644 --- a/src/backend/tests/llm/test_manifest_mcp_client.py +++ b/src/backend/tests/llm/test_manifest_mcp_client.py @@ -4,12 +4,7 @@ import pytest from agentscope.message import ToolResultState -from core.services.desktop_capability_protocol import ( - CapabilityManifestError, - build_manifest, - canonical_hash, - validate_manifest, -) +from core.capabilities.manifest_protocol import CapabilityManifestError, build_manifest, canonical_hash, validate_manifest def _tools() -> list[dict]: @@ -65,36 +60,31 @@ def test_protocol_accepts_only_a_complete_current_dynamic_manifest(): @pytest.mark.asyncio async def test_manifest_client_lists_cloud_schemas_without_network(): - from core.llm.mcp_manager import ManifestMCPClient - from core.llm.mcp_pool import make_client + from core.llm.tools.cloud_manifest import manifest_tools class NetworkMustNotRun(httpx.AsyncBaseTransport): async def handle_async_request(self, _request): # pragma: no cover raise AssertionError("schema listing must not perform network I/O") - client = make_client("search", _config(NetworkMustNotRun()), is_stateful=False) - assert isinstance(client, ManifestMCPClient) - - tools = await client.list_tools() + tools = manifest_tools("search", _config(NetworkMustNotRun())) assert [tool.name for tool in tools] == ["search"] assert tools[0].input_schema["required"] == ["query"] def test_incomplete_manifest_config_is_rejected_not_compatibility_routed(): - from core.llm.mcp_pool import make_client, uses_manifest_schema + from core.llm.tools.cloud_manifest import manifest_tools config = _config() config["manifest_tools"] = ["search"] - assert uses_manifest_schema(config) is False - with pytest.raises(ValueError, match="invalid cloud capability manifest"): - make_client("search", config, is_stateful=False) + with pytest.raises(ValueError, match="Invalid manifest tool entry"): + manifest_tools("search", config) @pytest.mark.asyncio async def test_manifest_tool_calls_json_gateway_with_current_schema_hash(): - from core.llm.mcp_pool import make_client + from core.llm.tools.cloud_manifest import manifest_tools captured: dict = {} @@ -117,8 +107,7 @@ async def handler(request: httpx.Request) -> httpx.Response: ) config = _config(httpx.MockTransport(handler)) - client = make_client("search", config, is_stateful=False) - tool = (await client.list_tools())[0] + tool = manifest_tools("search", config)[0] result = await tool(query="hello") diff --git a/src/backend/tests/llm/test_mcp_factory_e2e.py b/src/backend/tests/llm/test_mcp_factory_e2e.py new file mode 100644 index 00000000..36cec53c --- /dev/null +++ b/src/backend/tests/llm/test_mcp_factory_e2e.py @@ -0,0 +1,88 @@ +"""Full production factory -> HTTP model -> SDK2 server -> final model reply.""" +import os +import subprocess +import sys + + +def test_production_factory_calls_real_mcp_and_returns_final_reply(tmp_path): + script = r''' +import asyncio, json, socket +from contextlib import asynccontextmanager +import uvicorn +from fastapi import FastAPI, Request +from fastapi.responses import StreamingResponse +from agentscope.message import UserMsg +from tests.llm.test_mcp_parallel_execution import probe_server +from core.db.engine import Base, engine, SessionLocal +from core.db.models import Project, UserShadow, ChatSession, AdminMcpServer +from core.models.repository import create_provider, assign_role +from core.harness.registry import load_models +from features.installed import INSTALLED_FEATURES +from features import register_product_tools, register_run_capabilities, register_model_credentials +register_product_tools() +register_run_capabilities() +register_model_credentials() +load_models(INSTALLED_FEATURES) +Base.metadata.create_all(engine) +from core.llm.factory import create_agent_executor +from core.llm.mcp.manager import close_clients + +async def run(): + model_app = FastAPI() + requests = [] + @model_app.post("/v1/chat/completions") + async def complete(request: Request): + body = await request.json() + requests.append(body) + first = len(requests) == 1 + delta = {"role": "assistant", "tool_calls": [{"index": 0, "id": "probe-call", "type": "function", "function": {"name": "probe", "arguments": '{"item":7}'}}]} if first else {"role": "assistant", "content": "verified completed-7"} + async def stream(): + yield "data: " + json.dumps({"id":"fixture", "object":"chat.completion.chunk", "created":1, "model":"fixture", "choices":[{"index":0,"delta":delta,"finish_reason":None}]}) + "\n\n" + yield "data: " + json.dumps({"id":"fixture", "object":"chat.completion.chunk", "created":1, "model":"fixture", "choices":[{"index":0,"delta":{},"finish_reason":"tool_calls" if first else "stop"}], "usage":{"prompt_tokens":100,"completion_tokens":10,"total_tokens":110}}) + "\n\n" + yield "data: [DONE]\n\n" + return StreamingResponse(stream(), media_type="text/event-stream") + sock = socket.socket() + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + server = uvicorn.Server(uvicorn.Config(model_app, log_level="error")) + task = asyncio.create_task(server.serve(sockets=[sock])) + try: + async with asyncio.timeout(10): + while not server.started: + await asyncio.sleep(.01) + async with probe_server() as (url, calls): + with SessionLocal() as db: + model = create_provider(db, display_name="fixture", provider_type="chat", base_url=f"http://127.0.0.1:{port}/v1", api_key="test-only", model_name="fixture", extra_config={"context_length":32768,"api_protocol":"chat_completions"}) + assign_role(db, "main_agent", model.provider_id) + db.add(UserShadow(user_id="owner", username="Owner")) + db.add(Project(project_id="e2e", name="E2E", kind="personal", owner_user_id="owner", extra_data={"environment":{"mode":"custom","mcp_ids":["probe-server"]}})) + db.add(ChatSession(chat_id="e2e-chat", user_id="owner", project_id="e2e", title="E2E")) + db.add(AdminMcpServer(server_id="probe-server", display_name="Probe", owner_user_id="owner", is_enabled=True, transport="streamable_http", url=url)) + db.commit() + from core.runs.journal import RunJournal + journal = RunJournal(SessionLocal) + journal.accept(run_id="e2e-run", message_id="e2e-msg", chat_id="e2e-chat", user_id="owner", request_payload={"kind":"chat"}, recovery_snapshot={"kind":"chat","worker_args":{}}) + assert journal.claim("e2e-run", owner="e2e-worker", lease_seconds=300) + agent, resources = await create_agent_executor(current_user_id="owner", chat_id="e2e-chat", project_ctx={"project_id":"e2e"}, enabled_mcp_ids=["probe-server"], max_iters=3) + agent.state.run_id = "e2e-run" + agent.state.journal_owner = "e2e-worker" + try: + result = await agent.reply(UserMsg(name="user", content="Call probe with item 7 and report the result.")) + assert list(calls) == [7], calls + assert len(requests) == 2, len(requests) + assert "completed-7" in json.dumps(requests[1]), requests[1] + assert "verified completed-7" in str(result), result + assert len(resources) == 1 + finally: + await close_clients(resources) + assert all(r.session is None or r.session._owner.done() for r in resources) + finally: + server.should_exit = True + await asyncio.wait_for(task, 10) + sock.close() +asyncio.run(run()) +engine.dispose() +''' + env = dict(os.environ, DATABASE_URL=f"sqlite:///{tmp_path / 'e2e.db'}", REDIS_URL="", JX_CAPABILITIES_ENABLED="false", LOCAL_MODE="false", SANDBOX_TOOLS_ENABLED="false") + result = subprocess.run([sys.executable, "-c", script], env=env, capture_output=True, text=True, timeout=90) + assert result.returncode == 0, result.stdout[-8000:] + result.stderr[-8000:] diff --git a/src/backend/tests/llm/test_mcp_human_wait_timeout.py b/src/backend/tests/llm/test_mcp_human_wait_timeout.py new file mode 100644 index 00000000..fe917ed2 --- /dev/null +++ b/src/backend/tests/llm/test_mcp_human_wait_timeout.py @@ -0,0 +1,271 @@ +"""Human waits do not consume the active MCP invocation budget.""" +import asyncio +import uuid +import json +import socket +from contextlib import asynccontextmanager + +import uvicorn +from starlette.applications import Starlette +from starlette.responses import Response, StreamingResponse +from starlette.routing import Route + +import pytest +from mcp import types +from mcp.server.mcpserver import Context, MCPServer + +from core.llm.mcp.lifetime import discover +from core.llm.tools import user_questions as uq +from tests.llm.test_mcp_parallel_execution import probe_server +from tests.llm.test_mcp_runtime_pool import _run + + +def question_server(): + server = MCPServer("human-budget") + + @server.tool() + async def choose(ctx: Context, before: float = 0, after: float = 0, questions: int = 1) -> str | types.InputRequiredResult: + if ctx.input_responses: + await asyncio.sleep(after) + answer = ctx.input_responses["choice"] + return answer.action + ":" + str((answer.content or {}).get("color", "")) + await asyncio.sleep(before) + return types.InputRequiredResult(input_requests={key: types.ElicitRequest( + params=types.ElicitRequestFormParams(message="Choose a color", requested_schema={ + "type": "object", "properties": {"color": {"type": "string"}}, "required": ["color"], + }), + ) for key in ("choice", "second")[:questions]}) + return server + + +async def pending_question(chat): + async with asyncio.timeout(5): + while True: + pending = await uq.get_all_pending_shared(chat) + if pending: + return pending[0] + await asyncio.sleep(.01) + + +@pytest.mark.asyncio +async def test_answer_after_execution_budget_still_resumes_real_mcp_call(): + chat = "mcp-wait-" + uuid.uuid4().hex + async with probe_server(question_server()) as (url, _): + resource = await discover("human-budget", { + "transport": "streamable_http", "url": url, "execution_timeout": .5, "transport_timeout": .15, + }, interactions=_run("unused", chat=chat)) + task = asyncio.create_task(resource.tools[0](after=.25)) + try: + question = await pending_question(chat) + await asyncio.sleep(.7) + assert not task.done(), "Human wait consumed the MCP execution budget" + reply = await uq.answer_shared(chat, question["request_id"], [{"id": "color", "custom": "blue"}]) + assert reply["ok"] + result = await asyncio.wait_for(task, 3) + assert "accept:blue" in str(result.content) + assert not await uq.get_all_pending_shared(chat) + finally: + task.cancel() + await asyncio.gather(task, return_exceptions=True) + await resource.close() + + +@pytest.mark.asyncio +async def test_answer_resumes_remaining_budget_instead_of_resetting_it(): + chat = "mcp-remaining-" + uuid.uuid4().hex + async with probe_server(question_server()) as (url, _): + resource = await discover("human-budget", { + "transport": "streamable_http", "url": url, "execution_timeout": .8, + }, interactions=_run("unused", chat=chat)) + # Each server phase fits the budget alone; their sum must still time out. + task = asyncio.create_task(resource.tools[0](before=.5, after=.5)) + try: + question = await pending_question(chat) + await asyncio.sleep(1) + assert not task.done() + assert (await uq.answer_shared(chat, question["request_id"], [ + {"id": "color", "custom": "blue"}, + ]))["ok"] + result = await asyncio.wait_for(task, 2) + assert result.metadata.get("outcome_unknown") is True + assert "TimeoutError" in str(result.content) + finally: + task.cancel() + await asyncio.gather(task, return_exceptions=True) + await resource.close() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("cancel_call", [False, True]) +async def test_cancellation_during_human_wait_cleans_pending_question(cancel_call): + chat = "mcp-cancel-" + uuid.uuid4().hex + async with probe_server(question_server()) as (url, _): + resource = await discover("human-budget", { + "transport": "streamable_http", "url": url, "execution_timeout": .5, + }, interactions=_run("unused", chat=chat)) + task = asyncio.create_task(resource.tools[0]()) + try: + question = await pending_question(chat) + await asyncio.sleep(.7) + assert not task.done() + if cancel_call: + task.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(task, 2) + else: + assert (await uq.cancel_shared(chat, question["request_id"]))["ok"] + assert "cancel:" in str((await asyncio.wait_for(task, 2)).content) + assert not await uq.get_all_pending_shared(chat) + finally: + task.cancel() + await asyncio.gather(task, return_exceptions=True) + await resource.close() + + +@asynccontextmanager +async def legacy_server(): + # A real pre-2026 protocol peer: negotiate initialize and issue a callback + # while tools/call remains outstanding (unlike modern InputRequiredResult). + messages = asyncio.Queue() + tool_id = None + invocation_method = None + + async def stream(request): + async def events(): + yield "event: endpoint\ndata: /messages/\n\n" + while True: + yield "event: message\ndata: " + json.dumps(await messages.get()) + "\n\n" + return StreamingResponse(events(), media_type="text/event-stream") + + async def post(request): + nonlocal tool_id, invocation_method + body = await request.json() + method = body.get("method") + ident = body.get("id") + result = {} + if method == "server/discover": + await messages.put({"jsonrpc": "2.0", "id": ident, + "error": {"code": -32601, "message": "Method not found"}}) + return Response(status_code=202) + if method == "initialize": + result = {"protocolVersion": "2025-11-25", "capabilities": {"tools": {}, "resources": {}, "prompts": {}}, + "serverInfo": {"name": "legacy-human-budget", "version": "1"}} + elif method == "tools/list": + result = {"tools": [{"name": "choose", "inputSchema": {"type": "object", "properties": {}}}]} + elif method == "resources/list": + result = {"resources": [{"uri": "test://color", "name": "color"}]} + elif method == "resources/templates/list": + result = {"resourceTemplates": []} + elif method == "prompts/list": + result = {"prompts": [{"name": "color"}]} + elif method in {"tools/call", "resources/read", "prompts/get"}: + tool_id = ident + invocation_method = method + await messages.put({"jsonrpc": "2.0", "id": "question", "method": "elicitation/create", + "params": {"mode": "form", "message": "Choose a color", + "requestedSchema": {"type": "object", + "properties": {"color": {"type": "string"}}, "required": ["color"]}}}) + return Response(status_code=202) + elif method is None and ident == "question": + answer = body["result"] + ident = tool_id + text = answer["action"] + ":" + answer.get("content", {}).get("color", "") + if invocation_method == "resources/read": + result = {"contents": [{"uri": "test://color", "text": text}]} + elif invocation_method == "prompts/get": + result = {"messages": [{"role": "user", "content": {"type": "text", "text": text}}]} + else: + result = {"content": [{"type": "text", "text": text}]} + if ident is not None: + await messages.put({"jsonrpc": "2.0", "id": ident, "result": result}) + return Response(status_code=202) + + app = Starlette(routes=[Route("/sse", stream), Route("/messages/", post, methods=["POST"])]) + sock = socket.socket() + sock.bind(("127.0.0.1", 0)) + http = uvicorn.Server(uvicorn.Config(app, log_level="error")) + task = asyncio.create_task(http.serve(sockets=[sock])) + try: + async with asyncio.timeout(5): + while not http.started: + if task.done(): + await task + await asyncio.sleep(.01) + yield f"http://127.0.0.1:{sock.getsockname()[1]}/sse" + finally: + http.should_exit = True + await asyncio.wait_for(task, 5) + sock.close() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("kind", ["tool", "resource", "prompt"]) +async def test_legacy_sse_request_clock_also_excludes_human_wait(kind): + chat = "mcp-legacy-" + uuid.uuid4().hex + async with legacy_server() as url: + resource = await discover("legacy-human-budget", { + "transport": "sse", "url": url, "execution_timeout": .5, + }, interactions=_run("unused", chat=chat)) + tool = next(tool for tool in resource.tools if getattr(tool, "kind", "tool") == kind) + task = asyncio.create_task(tool()) + try: + question = await pending_question(chat) + await asyncio.sleep(.7) + assert not task.done(), "Legacy SDK request timeout interrupted the human wait" + assert (await uq.answer_shared(chat, question["request_id"], [ + {"id": "color", "custom": "blue"}, + ]))["ok"] + assert "accept:blue" in str((await asyncio.wait_for(task, 3)).content) + finally: + task.cancel() + await asyncio.gather(task, return_exceptions=True) + await resource.close() + + +@pytest.mark.asyncio +async def test_overlapping_questions_keep_budget_paused_until_last_answer(): + chat = "mcp-overlap-" + uuid.uuid4().hex + async with probe_server(question_server()) as (url, _): + resource = await discover("human-budget", { + "transport": "streamable_http", "url": url, "execution_timeout": .5, + }, interactions=_run("unused", chat=chat)) + task = asyncio.create_task(resource.tools[0](questions=2)) + try: + async with asyncio.timeout(5): + while len(pending := await uq.get_all_pending_shared(chat)) < 2: + await asyncio.sleep(.01) + for question in pending: + await asyncio.sleep(.7) + assert not task.done() + assert (await uq.answer_shared(chat, question["request_id"], [ + {"id": "color", "custom": "blue"}, + ]))["ok"] + assert "accept:blue" in str((await asyncio.wait_for(task, 3)).content) + finally: + task.cancel() + await asyncio.gather(task, return_exceptions=True) + await resource.close() + + +@pytest.mark.asyncio +async def test_waiting_call_does_not_pause_another_invocations_budget(): + chat = "mcp-isolated-" + uuid.uuid4().hex + async with probe_server(question_server()) as (url, _): + cfg = {"transport": "streamable_http", "url": url, "execution_timeout": .5} + waiting = await discover("human-budget", cfg, interactions=_run("unused", chat=chat)) + working = await discover("human-budget", cfg, interactions=_run("unused", chat=chat + "-other")) + task = asyncio.create_task(waiting.tools[0]()) + try: + question = await pending_question(chat) + result = await asyncio.wait_for(working.tools[0](before=1), 2) + assert result.metadata.get("outcome_unknown") is True + assert not task.done() + assert (await uq.answer_shared(chat, question["request_id"], [ + {"id": "color", "custom": "blue"}, + ]))["ok"] + assert "accept:blue" in str((await asyncio.wait_for(task, 3)).content) + finally: + task.cancel() + await asyncio.gather(task, return_exceptions=True) + await waiting.close() + await working.close() diff --git a/src/backend/tests/llm/test_mcp_native_adversarial.py b/src/backend/tests/llm/test_mcp_native_adversarial.py new file mode 100644 index 00000000..f4963bb2 --- /dev/null +++ b/src/backend/tests/llm/test_mcp_native_adversarial.py @@ -0,0 +1,219 @@ +"""Adversarial lifecycle, pagination, authority and schema tests.""" +import asyncio +import sys +from types import SimpleNamespace +import pytest +from mcp import types +from mcp.server.lowlevel import Server +from mcp.server.mcpserver import MCPServer +from mcp.shared.exceptions import MCPError +from core.llm.mcp.lifetime import discover +from core.llm.mcp import chat_sessions +from tests.llm.test_mcp_parallel_execution import probe_server +from tests.llm.test_mcp_native_capabilities import call_scope + + +@pytest.mark.asyncio +@pytest.mark.parametrize("repeat_cursor", [False, True]) +async def test_paginated_discovery_and_repeated_cursor_rejection(repeat_cursor): + calls = [] + async def listing(ctx, params): + cursor = params.cursor if params else None + calls.append(cursor) + return types.ListToolsResult(tools=[types.Tool(name="one" if cursor is None else "two", input_schema={"type":"object"})], + next_cursor="next" if cursor is None or repeat_cursor else None) + low = Server("paged", on_list_tools=listing) + server = MCPServer("paged") + server._lowlevel_server = low + async with probe_server(server) as (url, _): + if repeat_cursor: + with pytest.raises(BaseExceptionGroup) as failure: + await discover("paged", {"transport":"streamable_http", "url":url}) + assert failure.value.subgroup(lambda e: isinstance(e, ValueError) and "pagination cursor repeated" in str(e)) is not None + else: + resource = await discover("paged", {"transport":"streamable_http", "url":url}) + try: + assert [t.name for t in resource.tools] == ["one", "two"] + assert calls == [None, "next"] + finally: + await resource.close() + + +@pytest.mark.asyncio +async def test_resource_only_server_never_receives_tools_list(): + async def resources(ctx, params): + return types.ListResourcesResult(resources=[types.Resource(name="only",uri="doc://only")]) + async def templates(ctx, params): + return types.ListResourceTemplatesResult(resource_templates=[]) + async def read(ctx, params): + return types.ReadResourceResult(contents=[types.TextResourceContents(uri=params.uri, text="only-resource")]) + server = MCPServer("resource-only") + server._lowlevel_server = Server("resource-only", on_list_resources=resources, + on_list_resource_templates=templates, on_read_resource=read) + async with probe_server(server) as (url, _): + resource = await discover("only", {"transport":"streamable_http","url":url}) + try: + assert len(resource.tools) == 1 + assert "only-resource" in str(await resource.tools[0]()) + finally: + await resource.close() + + +@pytest.mark.asyncio +async def test_schema_change_during_header_recovery_never_replays_call(): + calls = [] + async def listing(ctx, params): + return types.ListToolsResult(tools=[types.Tool(name="write", input_schema={"type":"object", + "properties": {"new": {"type":"string"}} if calls else {}})]) + async def invoke(ctx, params): + calls.append(params.name) + raise MCPError(code=-32020, message="schema changed") + server = MCPServer("changes") + server._lowlevel_server = Server("changes", on_list_tools=listing, on_call_tool=invoke) + async with probe_server(server) as (url, _): + resource = await discover("changes", {"transport":"streamable_http","url":url}) + try: + with pytest.raises(Exception): + await resource.tools[0]() + assert calls == ["write"] + finally: + await resource.close() + + +@pytest.mark.asyncio +async def test_chat_deleted_while_first_process_is_opening(tmp_path): + script = tmp_path / "slow.py" + script.write_text('import time\ntime.sleep(300)\n') + cfg = {"transport":"stdio","command":sys.executable,"args":[str(script)]} + opening = asyncio.create_task(discover("slow",cfg,scope=("user","deleted",""))) + await asyncio.sleep(.15) + await chat_sessions.close_chat("deleted") + with pytest.raises((asyncio.CancelledError, RuntimeError)): + await asyncio.wait_for(opening, 5) + assert not any(k[1] == "deleted" for k in chat_sessions._entries) + assert not any(t.get_name() == "mcp-session:slow" and not t.done() for t in asyncio.all_tasks()) + + +@pytest.mark.asyncio +async def test_sampling_budget_is_reserved_before_slow_authorization(): + call = call_scope() + ready = asyncio.Event() + async def validate(): + await ready.wait() + call.validate = validate + params = types.CreateMessageRequestParams(messages=[], max_tokens=10) + requests = [asyncio.create_task(call.sampling(None, params)) for _ in range(12)] + await asyncio.sleep(0) + assert call.run.samples == 4 + ready.set() + results = await asyncio.gather(*requests) + assert sum(isinstance(r,types.CreateMessageResult) for r in results) == 4 + + +@pytest.mark.asyncio +async def test_connector_revoked_while_user_is_answering(monkeypatch): + call = call_scope() + allowed = True + call.run.agent._jx_prepared_capabilities = SimpleNamespace(validate=lambda **_:None, + validate_tool_call=lambda _:allowed) + async def answer(**kwargs): + nonlocal allowed + allowed = False + return {"status":"answered","answers":[{"id":"name","custom":"Ada"}]} + monkeypatch.setattr("core.llm.tools.user_questions.ask",answer) + params = types.ElicitRequestFormParams(message="Name",requested_schema={"type":"object", + "properties":{"name":{"type":"string"}},"required":["name"]}) + with pytest.raises(PermissionError): + await call.elicitation(None,params) + + +@pytest.mark.asyncio +async def test_outer_cancellation_cancels_pending_model_work(): + call = call_scope() + stopped = asyncio.Event() + class SlowModel: + model = "slow" + async def __call__(self,*args,**kwargs): + try: + await asyncio.sleep(300) + finally: + stopped.set() + call.run.agent.model = SlowModel() + params = types.CreateMessageRequestParams(messages=[],max_tokens=10) + task = asyncio.create_task(call.dispatch("sampling",None,params)) + await asyncio.sleep(.01) + await call.finish() + with pytest.raises(asyncio.CancelledError): + await task + assert stopped.is_set() and not call.pending + + +@pytest.mark.asyncio +async def test_deleted_chat_revokes_every_run_sharing_its_session(tmp_path): + script = tmp_path / "shared.py" + script.write_text('from mcp.server.mcpserver import MCPServer\ns=MCPServer("shared")\n@s.tool()\ndef echo(text: str)->str: return text\ns.run()\n') + cfg = {"transport":"stdio", "command":sys.executable, "args":[str(script)]} + first = await discover("shared", cfg, scope=("user", "deleted-shared", "")) + second = await asyncio.wait_for(discover("shared", cfg, scope=("user", "deleted-shared", "")), 5) + assert first.session is second.session + await chat_sessions.close_chat("deleted-shared") + for run in (first, second): + with pytest.raises(RuntimeError, match="lease is closed"): + await run.tools[0](text="after delete") + assert first.session._owner.done() + await first.close() + await second.close() + assert not any(k[1] == "deleted-shared" for k in chat_sessions._entries) + + +@pytest.mark.asyncio +async def test_discovery_repeated_cancel_waits_for_owner_exit(monkeypatch): + from core.llm.mcp import session as session_module + entered, exiting, finish = asyncio.Event(), asyncio.Event(), asyncio.Event() + class Client: + async def __aenter__(self): + return self + async def __aexit__(self, *args): + exiting.set() + await finish.wait() + @property + def server_capabilities(self): + return SimpleNamespace(tools=True) + async def list_tools(self, **kwargs): + entered.set() + await asyncio.sleep(300) + monkeypatch.setattr(session_module, "make_client", lambda *a, **kw:Client()) + opening = asyncio.create_task(discover("repeat-cancel", {"transport":"streamable_http"})) + await entered.wait() + opening.cancel() + await exiting.wait() + opening.cancel() + await asyncio.sleep(.01) + assert not opening.done() + finish.set() + with pytest.raises(asyncio.CancelledError): + await opening + assert not any(t.get_name() == "mcp-session:repeat-cancel" and not t.done() for t in asyncio.all_tasks()) + + +def test_retired_column_migration_preserves_existing_servers(): + import importlib.util + from pathlib import Path + import sqlalchemy as sa + from alembic.migration import MigrationContext + from alembic.operations import Operations + path = Path(__file__).parents[2] / "alembic/versions/mcpnative01_drop_stable.py" + spec = importlib.util.spec_from_file_location("mcp_migration",path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + engine = sa.create_engine("sqlite://") + with engine.begin() as connection: + connection.exec_driver_sql("CREATE TABLE admin_mcp_servers (server_id TEXT PRIMARY KEY, is_stable BOOLEAN NOT NULL)") + connection.exec_driver_sql("INSERT INTO admin_mcp_servers VALUES ('keep', 0)") + with Operations.context(MigrationContext.configure(connection)): + module.upgrade() + assert [c["name"] for c in sa.inspect(connection).get_columns("admin_mcp_servers")] == ["server_id"] + assert connection.exec_driver_sql("SELECT server_id FROM admin_mcp_servers").scalar() == "keep" + module.downgrade() + assert connection.exec_driver_sql("SELECT is_stable FROM admin_mcp_servers").scalar() == 1 + module.upgrade() diff --git a/src/backend/tests/llm/test_mcp_native_capabilities.py b/src/backend/tests/llm/test_mcp_native_capabilities.py new file mode 100644 index 00000000..090c7f4b --- /dev/null +++ b/src/backend/tests/llm/test_mcp_native_capabilities.py @@ -0,0 +1,169 @@ +"""Native modern protocol and business boundaries over real HTTP and STDIO.""" +import asyncio +import os +import sys +from types import SimpleNamespace +import pytest +from agentscope.message import TextBlock +from agentscope.model import ChatResponse +from mcp import types +from mcp.server.mcpserver import MCPServer, Context +from core.llm.mcp.lifetime import discover +from core.llm.mcp.callbacks import Interactions +from core.llm.mcp import chat_sessions +from core.llm.mcp.chat_sessions import close_chat +from tests.llm.test_mcp_parallel_execution import probe_server + + +def interactions(): + class Model: + model = "fixture" + async def __call__(self, messages, tools=None, **kwargs): + assert tools is None + assert kwargs["max_tokens"] == 10 + assert "private-history" not in str(messages) + return ChatResponse(content=[TextBlock(text="Paris")], is_last=True) + request = SimpleNamespace(chat_id="mcp-chat", run_id="", current_user_id="mcp-user") + binding = Interactions(request, "probe", True) + binding.agent = SimpleNamespace(model=Model(), state=SimpleNamespace(pending_mcp_progress=[])) + return binding + + +def call_scope(): + """One active outer call of a fresh run binding.""" + return interactions().open_call() + + +@pytest.mark.asyncio +async def test_modern_sampling_elicitation_progress_resources_and_prompts(monkeypatch): + server = MCPServer("modern") + @server.tool() + async def sample(ctx: Context) -> str | types.InputRequiredResult: + if ctx.input_responses: + return ctx.input_responses["q"].content.text + return types.InputRequiredResult(input_requests={"q": types.CreateMessageRequest( + params=types.CreateMessageRequestParams(messages=[types.SamplingMessage( + role="user", content=types.TextContent(text="Capital of France?"))], max_tokens=10))}) + @server.tool() + async def form(ctx: Context) -> str | types.InputRequiredResult: + if ctx.input_responses: + answer = ctx.input_responses["q"] + assert answer.content == {"count": 3, "enabled": True} + return "accepted" + return types.InputRequiredResult(input_requests={"q": types.ElicitRequest(params=types.ElicitRequestFormParams( + message="Choose", requested_schema={"type": "object", "properties": { + "count": {"type": "integer", "minimum": 1}, "enabled": {"type": "boolean"}}, + "required": ["count", "enabled"]}))}) + @server.tool() + async def progress(ctx: Context) -> str: + await ctx.report_progress(1, 2, "halfway") + return "done" + @server.tool() + async def linked() -> types.CallToolResult: + return types.CallToolResult(content=[types.TextContent(text="description"), + types.ResourceLink(name="report", uri="report://one", mime_type="text/plain")], + structured_content={"answer": 42}) + @server.resource("report://one") + def report() -> str: + return "resource body" + @server.resource("report://by-name/{name}") + def report_template(name: str) -> str: + return "template " + name + @server.prompt() + def summarize(subject: str) -> str: + return "untrusted prompt " + subject + from core.llm.tools import user_questions as uq + async def answer_form(): + for _ in range(500): + queue = uq.get_ui_queue("mcp-chat") + if queue and not queue.empty(): + event = queue.get_nowait() + if event["event"] == "requested": + assert uq.answer("mcp-chat", event["request_id"], [ + {"id":"count", "custom":"3"}, {"id":"enabled", "selected":["option_1"]} + ])["ok"] + return + await asyncio.sleep(.01) + raise AssertionError("MCP elicitation did not reach the user-question queue") + binding = interactions() + async with probe_server(server, json_response=False) as (url, _): + resource = await discover("modern", {"transport": "streamable_http", "url": url}, interactions=binding) + try: + assert resource.session.client.protocol_version == "2026-07-28" + tools = {t.name: t for t in resource.tools} + assert "Paris" in str(await tools["sample"]()) + answer_task = asyncio.create_task(answer_form()) + try: + assert "accepted" in str(await tools["form"]()) + await answer_task + finally: + answer_task.cancel() + await asyncio.gather(answer_task, return_exceptions=True) + await tools["progress"]() + assert binding.agent.state.pending_mcp_progress[0]["progress"] == 1 + result = await tools["linked"]() + assert result.metadata["mcp"]["structuredContent"] == {"answer": 42} + assert "report://one" in str(result.content) + resource_tool = next(t for t in resource.tools if t.name.startswith("mcp_resource_")) + assert "resource body" in str(await resource_tool()) + template_tool = next(t for t in resource.tools if t.name.startswith("mcp_resource_template_")) + assert "template one" in str(await template_tool(name="one")) + prompt_tool = next(t for t in resource.tools if t.name.startswith("mcp_prompt_")) + assert "untrusted prompt topic" in str(await prompt_tool(subject="topic")) + finally: + await resource.close() + + +@pytest.mark.asyncio +async def test_stdio_chat_reuse_isolated_users_stale_lease_and_cancelled_opener(tmp_path): + script = tmp_path / "pid.py" + script.write_text('import os\nfrom mcp.server.mcpserver import MCPServer\ns=MCPServer("pid")\n@s.tool()\ndef pid()->int:return os.getpid()\ns.run()\n') + cfg = {"transport": "stdio", "command": sys.executable, "args": [str(script)]} + scope = ("user-one", "chat", "") + first = await discover("pid", cfg, scope=scope) + old_tool = first.tools[0] + pid = (await old_tool()).metadata["mcp"]["structuredContent"] + await first.close() + second = await discover("pid", cfg, scope=scope) + try: + assert (await second.tools[0]()).metadata["mcp"]["structuredContent"] == pid + with pytest.raises(RuntimeError, match="lease is closed"): + await old_tool() + # A concurrent run in the same chat shares the live process instead of waiting. + concurrent = await asyncio.wait_for(discover("pid", cfg, scope=scope), 5) + assert (await concurrent.tools[0]()).metadata["mcp"]["structuredContent"] == pid + await concurrent.close() + assert (await second.tools[0]()).metadata["mcp"]["structuredContent"] == pid + # A run cancelled while its process opens leaves a parked session, not a leak. + opener = asyncio.create_task(discover("pid", cfg, scope=("user-one", "other-chat", ""))) + await asyncio.sleep(0.02) + opener.cancel() + with pytest.raises(asyncio.CancelledError): + await opener + reused = await discover("pid", cfg, scope=("user-one", "other-chat", "")) + try: + assert sum(k[1] == "other-chat" for k in chat_sessions._entries) == 1 + finally: + await reused.close() + third = await discover("pid", cfg, scope=("user-two", "chat", "")) + try: + assert (await third.tools[0]()).metadata["mcp"]["structuredContent"] != pid + finally: + await third.close() + finally: + await second.close() + await close_chat() + assert second.session._owner.done() + + +@pytest.mark.asyncio +async def test_sampling_denies_excessive_or_shared_context(): + call = call_scope() + params = types.CreateMessageRequestParams(messages=[], max_tokens=5000) + assert isinstance(await call.sampling(None, params), types.ErrorData) + params.max_tokens = 10 + params.include_context = "allServers" + assert isinstance(await call.sampling(None, params), types.ErrorData) + call.run.interactive = False + form = types.ElicitRequestFormParams(message="test", requested_schema={"type":"object"}) + assert (await call.elicitation(None, form)).action == "decline" diff --git a/src/backend/tests/llm/test_mcp_parallel_execution.py b/src/backend/tests/llm/test_mcp_parallel_execution.py index 1e505880..4f82c6db 100644 --- a/src/backend/tests/llm/test_mcp_parallel_execution.py +++ b/src/backend/tests/llm/test_mcp_parallel_execution.py @@ -8,18 +8,19 @@ import pytest import uvicorn from agentscope.agent import Agent -from agentscope.mcp import HttpMCPConfig from agentscope.message import TextBlock, ToolCallBlock, UserMsg from agentscope.model import ChatResponse from agentscope.tool import Toolkit -from mcp.server.fastmcp import FastMCP +from mcp.server.mcpserver import MCPServer -from core.llm.mcp_manager import BareNameMCPClient +from core.llm.mcp.lifetime import discover class ThreeCallsModel: model = "mcp-concurrency-probe" context_size = 32768 + from agentscope.formatter import OpenAIChatFormatter + formatter = OpenAIChatFormatter() def __init__(self): self.calls = 0 @@ -41,21 +42,23 @@ async def __call__(self, messages, tools=None, **kwargs): @asynccontextmanager -async def probe_server(): +async def probe_server(mcp_server=None, *, json_response=True): intervals = {} - server = FastMCP("parallel-probe", stateless_http=True, json_response=True) + server = mcp_server or MCPServer("parallel-probe") - @server.tool(annotations={"readOnlyHint": True}) async def probe(item: int) -> str: started = time.monotonic() await asyncio.sleep(0.4) intervals[item] = (started, time.monotonic()) return f"completed-{item}" + if mcp_server is None: + server.tool(annotations={"readOnlyHint": True})(probe) + sock = socket.socket() sock.bind(("127.0.0.1", 0)) port = sock.getsockname()[1] - http = uvicorn.Server(uvicorn.Config(server.streamable_http_app(), log_level="error")) + http = uvicorn.Server(uvicorn.Config(server.streamable_http_app(stateless_http=True, json_response=json_response), log_level="error")) task = asyncio.create_task(http.serve(sockets=[sock])) try: async with asyncio.timeout(10): @@ -71,23 +74,18 @@ async def probe(item: int) -> str: @pytest.mark.asyncio -@pytest.mark.parametrize("stateful", [False, True]) -async def test_agent_overlaps_real_mcp_requests(stateful): +async def test_agent_overlaps_real_mcp_requests(): async with probe_server() as (url, intervals): - client = BareNameMCPClient( - name="parallel-probe", - is_stateful=stateful, - mcp_config=HttpMCPConfig(url=url), - ) + client = await discover("parallel-probe", {"transport": "streamable_http", "url": url}) try: - await client.connect() - # Enumeration and direct lookup must preserve the same policy. - await client.list_tools() + tools = client.tools + # The probe is annotated read-only, so the agent may overlap its calls. + assert all(tool.is_concurrency_safe for tool in tools) agent = Agent( name="probe-agent", system_prompt="Run the probes.", model=ThreeCallsModel(), - toolkit=Toolkit(mcps=[client]), + toolkit=Toolkit(tools=tools), ) started = time.monotonic() result = await agent.reply( @@ -98,7 +96,7 @@ async def test_agent_overlaps_real_mcp_requests(stateful): overlap = min(end for _, end in intervals.values()) - max( start for start, _ in intervals.values() ) - print(f"stateful={stateful} calls=3 elapsed={elapsed:.3f}s overlap={overlap:.3f}s") + print(f"calls=3 elapsed={elapsed:.3f}s overlap={overlap:.3f}s") assert overlap > 0.2, "MCP requests did not execute concurrently" assert "done" in str(result) finally: diff --git a/src/backend/tests/llm/test_mcp_pool_cache.py b/src/backend/tests/llm/test_mcp_pool_cache.py deleted file mode 100644 index 5b90d273..00000000 --- a/src/backend/tests/llm/test_mcp_pool_cache.py +++ /dev/null @@ -1,203 +0,0 @@ -"""Tests for MCPConnectionPool (AgentScope 2.0 rewrite). - -Migration note: the 1.x "tool-function cache" (_cached_tool_funcs / get_callable_function / -build_toolkit_from_cache returning a Toolkit) no longer exists in 2.0 -- MCPClient has no -get_callable_function, and the Toolkit is constructed once. This pool is now responsible only for **connection reuse**: - • only pools stdio stable servers (HTTP is not pooled even when is_stable, it goes per-request stateless) - • get_request_clients() returns a list of connected MCPClients (including stable + transient) - • shutdown / reconnect / reinitialize lifecycle - -This file is rewritten accordingly, testing 2.0's actual pool behavior. -""" - -from __future__ import annotations - -from typing import Dict, List, Optional -from unittest.mock import patch - -import pytest - - -# ── Fake MCPClient ───────────────────────────────────────────────────────── - -class FakeTool: - def __init__(self, name: str): - self.name = name - self.description = "" - - -class FakeMCPClient: - """Mimics agentscope 2.0 MCPClient (connect/list_tools/close).""" - - def __init__(self, name: str, tools: Optional[List[str]] = None): - self.name = name - self._tools = [FakeTool(t) for t in (tools or ["tool_a", "tool_b"])] - self.connect_count = 0 - self.close_count = 0 - self.list_tools_count = 0 - - async def connect(self): - self.connect_count += 1 - - async def list_tools(self): - self.list_tools_count += 1 - return list(self._tools) - - async def close(self): - self.close_count += 1 - - -def _make_pool(): - from core.llm.mcp_pool import MCPConnectionPool - pool = MCPConnectionPool.__new__(MCPConnectionPool) - pool.__init__() - return pool - - -def _stdio_configs(names: List[str], stable: bool = True) -> Dict[str, dict]: - return { - name: {"command": "python", "args": ["-m", f"extensions.mcp_servers.{name}"], "is_stable": stable} - for name in names - } - - -def _http_config(name: str, stable: bool = True) -> Dict[str, dict]: - return {name: {"transport": "streamable_http", "url": f"http://{name}/mcp", "is_stable": stable}} - - -@pytest.fixture -def pool(): - return _make_pool() - - -def _patch_make_client(registry: dict): - def fake_make_client(name, cfg, *, is_stateful=True): - c = FakeMCPClient(name=name) - registry[name] = c - return c - return patch("core.llm.mcp_pool.make_client", side_effect=fake_make_client) - - -class TestInitialize: - @pytest.mark.asyncio - async def test_initialize_connects_stdio_stable(self, pool): - clients: dict = {} - with _patch_make_client(clients): - await pool.initialize(_stdio_configs(["server_a", "server_b"])) - assert pool.is_initialized - assert pool.stable_client_count == 2 - assert pool.has_cached_tools # 2.0 semantics: stable connections already exist - assert clients["server_a"].connect_count == 1 - assert clients["server_b"].connect_count == 1 - - @pytest.mark.asyncio - async def test_http_not_pooled(self, pool): - """An HTTP server does not enter the stable pool even when is_stable (2.0: goes per-request stateless).""" - clients: dict = {} - with _patch_make_client(clients): - await pool.initialize(_http_config("kb_http")) - assert pool.stable_client_count == 0 - assert "kb_http" not in pool.stable_server_ids - - @pytest.mark.asyncio - async def test_connect_failure_does_not_block_init(self, pool): - def fake_make_client(name, cfg, *, is_stateful=True): - c = FakeMCPClient(name=name) - if name == "bad": - async def fail(): - raise RuntimeError("connect failed") - c.connect = fail - return c - with patch("core.llm.mcp_pool.make_client", side_effect=fake_make_client): - await pool.initialize(_stdio_configs(["good", "bad"])) - assert pool.is_initialized - assert "good" in pool._stable_clients - assert "bad" not in pool._stable_clients - - -class TestGetRequestClients: - @pytest.mark.asyncio - async def test_returns_stable_clients(self, pool): - clients: dict = {} - with _patch_make_client(clients): - await pool.initialize(_stdio_configs(["srv_a", "srv_b"])) - req, transient = await pool.get_request_clients(enabled_keys=["srv_a", "srv_b"]) - assert {c.name for c in req} == {"srv_a", "srv_b"} - assert transient == [] - - @pytest.mark.asyncio - async def test_partial_enabled_keys(self, pool): - clients: dict = {} - with _patch_make_client(clients): - await pool.initialize(_stdio_configs(["srv_a", "srv_b", "srv_c"])) - req, _ = await pool.get_request_clients(enabled_keys=["srv_a"]) - assert {c.name for c in req} == {"srv_a"} - - @pytest.mark.asyncio - async def test_transient_spawned_fresh(self, pool): - clients: dict = {} - with _patch_make_client(clients): - await pool.initialize(_stdio_configs(["stable_srv"])) - req, transient = await pool.get_request_clients( - enabled_keys=["stable_srv", "transient_srv"], - per_request_servers_cfg=_stdio_configs(["transient_srv"], stable=False), - ) - assert len(transient) == 1 - assert transient[0].name == "transient_srv" - assert transient[0].connect_count == 1 - - @pytest.mark.asyncio - async def test_build_toolkit_from_cache_alias(self, pool): - """The backward-compatible alias build_toolkit_from_cache now returns (clients, transient).""" - clients: dict = {} - with _patch_make_client(clients): - await pool.initialize(_stdio_configs(["srv"])) - req, transient = await pool.build_toolkit_from_cache(enabled_keys=["srv"]) - assert {c.name for c in req} == {"srv"} - - -class TestLifecycle: - @pytest.mark.asyncio - async def test_shutdown_closes_and_clears(self, pool): - clients: dict = {} - with _patch_make_client(clients): - await pool.initialize(_stdio_configs(["srv"])) - assert pool.has_cached_tools - await pool.shutdown() - assert not pool.has_cached_tools - assert pool.stable_client_count == 0 - assert clients["srv"].close_count == 1 - - @pytest.mark.asyncio - async def test_reconnect_stable(self, pool): - clients: dict = {} - with _patch_make_client(clients): - await pool.initialize(_stdio_configs(["srv"])) - new_client = await pool._reconnect_stable("srv") - assert new_client is not None - assert new_client.connect_count == 1 - - @pytest.mark.asyncio - async def test_reinitialize_on_config_change(self, pool): - with _patch_make_client({}): - await pool.initialize(_stdio_configs(["srv_a"])) - assert pool.stable_server_ids == frozenset({"srv_a"}) - await pool.reinitialize_if_config_changed(_stdio_configs(["srv_a", "srv_b"])) - assert pool.stable_server_ids == frozenset({"srv_a", "srv_b"}) - - -class TestHasCachedTools: - @pytest.mark.asyncio - async def test_false_before_init(self, pool): - assert not pool.has_cached_tools - - @pytest.mark.asyncio - async def test_true_after_init_with_stable(self, pool): - with _patch_make_client({}): - await pool.initialize(_stdio_configs(["srv"])) - assert pool.has_cached_tools - - @pytest.mark.asyncio - async def test_false_when_no_stable_servers(self, pool): - await pool.initialize(_stdio_configs([], stable=True)) - assert not pool.has_cached_tools diff --git a/src/backend/tests/llm/test_mcp_runtime_pool.py b/src/backend/tests/llm/test_mcp_runtime_pool.py new file mode 100644 index 00000000..d6df0cdf --- /dev/null +++ b/src/backend/tests/llm/test_mcp_runtime_pool.py @@ -0,0 +1,272 @@ +"""Run handles, the shared STDIO pool, call admission and result shaping over real SDK transports.""" +import asyncio +import sys +import time +from types import SimpleNamespace +from typing import Any, Dict + +import pytest +from agentscope.message import TextBlock +from agentscope.model import ChatResponse +from mcp import types +from mcp.server.caching import CacheHint +from mcp.server.mcpserver import Context, MCPServer +from mcp.shared.exceptions import MCPError + +from core.llm.mcp import chat_sessions +from core.llm.mcp.callbacks import Interactions +from core.llm.mcp.lifetime import discover +from core.llm.mcp.tools import queue_progress +from tests.llm.test_mcp_parallel_execution import probe_server + + +def _state(chunk): + return str(getattr(chunk.state, "value", chunk.state)) + + +def _texts(chunk): + return [block.text for block in chunk.content if isinstance(block, TextBlock)] + + +def _run(answer, chat="pool-chat", user="pool-user"): + class Model: + model = "fixture" + async def __call__(self, messages, tools=None, **kwargs): + return ChatResponse(content=[TextBlock(text=answer)], is_last=True) + request = SimpleNamespace(chat_id=chat, run_id="", current_user_id=user) + return Interactions(request, "pool", True, + agent=SimpleNamespace(model=Model(), state=SimpleNamespace(pending_mcp_progress=[]))) + + +def _effects_server(calls): + server = MCPServer("effects") + + @server.tool(annotations={"readOnlyHint": True}) + async def look(seconds: float = 0) -> str: + await asyncio.sleep(seconds) + return "seen" + + @server.tool() + async def write(key: str, seconds: float = 0) -> str: + calls.append(key) + await asyncio.sleep(seconds) + return "written " + key + + @server.tool(annotations={"readOnlyHint": True}) + async def reject() -> str: + raise MCPError(code=-32602, message="bad request") + + @server.tool(annotations={"readOnlyHint": True}) + def typed(text: str) -> Dict[str, Any]: + return {"text": text} + + @server.tool(annotations={"readOnlyHint": True}) + async def structured_only() -> types.CallToolResult: + return types.CallToolResult(content=[], structured_content={"answer": 42}) + + @server.tool(annotations={"readOnlyHint": True}) + async def described() -> types.CallToolResult: + return types.CallToolResult(content=[types.TextContent(text="see data")], structured_content={"answer": 42}) + + @server.tool(annotations={"readOnlyHint": True}) + def plain() -> str: + return "done" + + @server.tool(annotations={"readOnlyHint": True}) + async def steps(ctx: Context) -> str: + for step in range(1, 6): + await ctx.report_progress(step, 5, f"step {step}") + return "stepped" + return server + + +@pytest.mark.asyncio +async def test_request_failures_become_results_and_keep_the_session(): + calls = [] + async with probe_server(_effects_server(calls)) as (url, _): + resource = await discover("effects", {"transport": "streamable_http", "url": url, "execution_timeout": 0.5}) + tools = {tool.name: tool for tool in resource.tools} + try: + timed_out = await tools["look"](seconds=2) + assert _state(timed_out) == "error" and "outcome_unknown" not in timed_out.metadata + rejected = await tools["reject"]() + assert _state(rejected) == "error" + assert not resource.session.failed + assert _texts(await tools["look"]()) == ["seen"] + finally: + await resource.close() + + +@pytest.mark.asyncio +async def test_write_with_unknown_outcome_is_never_replayed_in_the_same_run(): + calls = [] + async with probe_server(_effects_server(calls)) as (url, _): + cfg = {"transport": "streamable_http", "url": url, "execution_timeout": 0.5} + resource = await discover("effects", cfg) + tools = {tool.name: tool for tool in resource.tools} + try: + first = await tools["write"](key="a", seconds=2) + assert first.metadata["outcome_unknown"] is True + replay = await tools["write"](key="a", seconds=2) + assert replay.metadata["outcome_unknown"] is True + assert calls == ["a"] + assert _texts(await tools["write"](key="b")) == ["written b"] + finally: + await resource.close() + # A new run decides for itself. + later = await discover("effects", cfg) + try: + assert _texts(await {t.name: t for t in later.tools}["write"](key="a")) == ["written a"] + finally: + await later.close() + + +@pytest.mark.asyncio +async def test_read_only_calls_overlap_while_writes_run_alone(): + calls = [] + async with probe_server(_effects_server(calls)) as (url, _): + resource = await discover("effects", {"transport": "streamable_http", "url": url}) + tools = {tool.name: tool for tool in resource.tools} + try: + assert tools["look"].is_concurrency_safe and not tools["write"].is_concurrency_safe + started = time.monotonic() + await asyncio.gather(*(tools["look"](seconds=0.4) for _ in range(4))) + assert time.monotonic() - started < 1.0 + started = time.monotonic() + await asyncio.gather(*(tools["write"](key=str(i), seconds=0.3) for i in range(3))) + assert time.monotonic() - started >= 0.85 + finally: + await resource.close() + + +@pytest.mark.asyncio +async def test_structured_output_reaches_the_model_once(): + async with probe_server(_effects_server([])) as (url, _): + resource = await discover("effects", {"transport": "streamable_http", "url": url}) + tools = {tool.name: tool for tool in resource.tools} + try: + typed = await tools["typed"](text="x") + assert len(_texts(typed)) == 1 and typed.metadata["mcp"]["structuredContent"] + assert _texts(await tools["structured_only"]()) == ['{"answer":42}'] + assert _texts(await tools["plain"]()) == ["done"] + # Text that does not carry the structured value keeps it for the model. + assert _texts(await tools["described"]()) == ["see data", '{"answer":42}'] + finally: + await resource.close() + + +@pytest.mark.asyncio +async def test_progress_keeps_the_newest_event_per_call(): + queue = [] + for step in range(3): + queue_progress(queue, {"tool_id": "a", "progress": step}) + queue_progress(queue, {"tool_id": "b", "progress": 9}) + assert queue == [{"tool_id": "a", "progress": 2}, {"tool_id": "b", "progress": 9}] + + run = _run("unused") + async with probe_server(_effects_server([]), json_response=False) as (url, _): + resource = await discover("effects", {"transport": "streamable_http", "url": url}, interactions=run) + try: + await {t.name: t for t in resource.tools}["steps"]() + events = run.agent.state.pending_mcp_progress + assert len(events) == 1 and events[0]["progress"] == 5 + finally: + await resource.close() + + +@pytest.mark.asyncio +async def test_listings_are_shared_across_runs_of_one_authorization_context(): + server = MCPServer("cached", cache_hints={method: CacheHint(ttl_ms=60_000) for method in + ("server/discover", "tools/list", "resources/list", + "resources/templates/list", "prompts/list")}) + + @server.tool(annotations={"readOnlyHint": True}) + def ping() -> str: + return "pong" + + async with probe_server(server) as (url, _): + first = await _timed_listing(url, "alice") + second = await _timed_listing(url, "alice") + other = await _timed_listing(url, "bob") + assert second < first and other == first + + +async def _timed_listing(url, user): + """Count list requests one run sends, through the SDK's own session methods.""" + from mcp.client.session import ClientSession + sent = [] + original = ClientSession.send_request + + async def counting(self, request, *args, **kwargs): + sent.append(getattr(request, "method", type(request).__name__)) + return await original(self, request, *args, **kwargs) + + ClientSession.send_request = counting + try: + resource = await discover("cached", {"transport": "streamable_http", "url": url, + "headers": {"X-Current-User-Id": user}}) + await resource.close() + finally: + ClientSession.send_request = original + return len(sent) + + +def _stdio_server(tmp_path): + script = tmp_path / "pool_server.py" + script.write_text( + "import os\n" + "from mcp import types\n" + "from mcp.server.mcpserver import Context, MCPServer\n" + "s = MCPServer('pool')\n" + "@s.tool(annotations={'readOnlyHint': True})\n" + "def pid() -> int:\n return os.getpid()\n" + "@s.tool(annotations={'readOnlyHint': True})\n" + "async def ask(ctx: Context) -> str | types.InputRequiredResult:\n" + " if ctx.input_responses:\n" + " return ctx.input_responses['q'].content.text\n" + " return types.InputRequiredResult(input_requests={'q': types.CreateMessageRequest(\n" + " params=types.CreateMessageRequestParams(messages=[types.SamplingMessage(\n" + " role='user', content=types.TextContent(text='who?'))], max_tokens=10))})\n" + "s.run()\n") + return {"transport": "stdio", "command": sys.executable, "args": [str(script)]} + + +@pytest.mark.asyncio +async def test_concurrent_runs_share_one_process_and_each_answers_its_own_requests(tmp_path): + cfg = _stdio_server(tmp_path) + scope = ("pool-user", "pool-chat", "") + first = await discover("pool", cfg, interactions=_run("first-run"), scope=scope) + second = await asyncio.wait_for(discover("pool", cfg, interactions=_run("second-run"), scope=scope), 5) + try: + assert first.session is second.session + for run in (first, second): + run.interactions.agent.toolkit = None + answers = await asyncio.gather( + {t.name: t for t in first.tools}["ask"](), {t.name: t for t in second.tools}["ask"]()) + assert [_texts(a) for a in answers] == [["first-run"], ["second-run"]] + finally: + await first.close() + await second.close() + await chat_sessions.close_chat() + + +@pytest.mark.asyncio +async def test_idle_sessions_make_room_and_only_held_sessions_count_against_capacity(tmp_path, monkeypatch): + cfg = _stdio_server(tmp_path) + monkeypatch.setattr(chat_sessions, "MAX_USER_SESSIONS", 2) + try: + for chat in range(4): + resource = await discover("pool", cfg, scope=("cap-user", f"chat-{chat}", "")) + assert _state(await resource.tools[0]()) != "error" + await resource.close() + assert sum(k[0] == "cap-user" for k in chat_sessions._entries) <= 2 + held = [await discover("pool", cfg, scope=("cap-user", f"held-{i}", "")) for i in range(2)] + try: + with pytest.raises(RuntimeError, match="capacity"): + await discover("pool", cfg, scope=("cap-user", "one-too-many", "")) + finally: + for resource in held: + await resource.close() + finally: + await chat_sessions.close_chat() + assert not chat_sessions._entries diff --git a/src/backend/tests/llm/test_mcp_session_ownership.py b/src/backend/tests/llm/test_mcp_session_ownership.py new file mode 100644 index 00000000..90d9e380 --- /dev/null +++ b/src/backend/tests/llm/test_mcp_session_ownership.py @@ -0,0 +1,128 @@ +"""Real SDK transports: one discovery, run ownership, user isolation and cancellation.""" +import asyncio +import sys +import pytest +from agentscope.message import ToolCallBlock +from agentscope.tool import Toolkit +from core.llm.mcp.lifetime import discover +from core.llm.runtime.state import AgentRuntimeState +from tests.llm.test_mcp_parallel_execution import probe_server + + +@pytest.mark.asyncio +async def test_http_tools_are_discovered_once_and_read_only_tools_may_overlap(): + async with probe_server() as (url, intervals): + resource = await discover("probe", {"transport": "streamable_http", "url": url}) + try: + assert [t.name for t in resource.tools] == ["probe"] + assert resource.tools[0].is_concurrency_safe is True + toolkit = Toolkit(tools=resource.tools) + for _ in range(2): + assert (await toolkit.get_tool_schemas())[0]["function"]["name"] == "probe" + chunks = await resource.tools[0](item=8) + assert "completed-8" in str(chunks) + assert list(intervals) == [8] + finally: + await resource.close() + assert resource.session._owner.done() + + +@pytest.mark.asyncio +async def test_stdio_discovery_and_calls_can_cross_tasks_without_cross_task_cleanup(tmp_path): + script = tmp_path / "server.py" + script.write_text('from mcp.server.mcpserver import MCPServer\ns=MCPServer("owned")\n@s.tool()\ndef echo(text: str)->str: return text\ns.run()\n') + cfg = {"command": sys.executable, "args": [str(script)]} + resource = await discover("owned", cfg) + async def call(): + return await resource.tools[0](text="run-owned") + assert "run-owned" in str(await asyncio.create_task(call())) + await asyncio.create_task(resource.close()) + assert resource.session._owner.done() + assert not resource.session._owner.cancelled() + + +@pytest.mark.asyncio +async def test_failed_discovery_releases_owner(tmp_path): + script = tmp_path / "bad.py" + script.write_text('raise SystemExit(1)\n') + with pytest.raises(Exception): + await discover("bad", {"command": sys.executable, "args": [str(script)]}) + assert not any(t.get_name() == "mcp-session:bad" and not t.done() for t in asyncio.all_tasks()) + + +@pytest.mark.asyncio +async def test_cancel_never_ready_stdio_does_not_leave_an_owner(tmp_path): + script = tmp_path / "never_ready.py" + script.write_text('import time\ntime.sleep(300)\n') + task = asyncio.create_task(discover("never-ready", { + "command": sys.executable, "args": [str(script)], + })) + await asyncio.sleep(0.2) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(task, 5) + assert not any(t.get_name() == "mcp-session:never-ready" and not t.done() for t in asyncio.all_tasks()) + + +@pytest.mark.asyncio +async def test_run_guard_handles_native_tools_and_revoked_mcp(): + from agentscope.tool import FunctionTool + from core.llm.tools.ontology_toolkit import OntologyFilteredToolkit + calls = [] + async def local(value: str): + calls.append(value) + return value + native = FunctionTool(local) + async with probe_server() as (url, intervals): + resource = await discover("probe", {"transport": "streamable_http", "url": url}) + try: + toolkit = OntologyFilteredToolkit(tools=[native, *resource.tools]) + allowed = True + toolkit.capability_guard = lambda name: allowed and name == "probe" + state = AgentRuntimeState() + result = [x async for x in toolkit.call_tool(ToolCallBlock(id="native", name="local", input='{"value":"local-ok"}'), state)] + assert calls == ["local-ok"] + result = [x async for x in toolkit.call_tool(ToolCallBlock(id="allowed", name="probe", input='{"item":1}'), state)] + assert 1 in intervals + allowed = False + result = [x async for x in toolkit.call_tool(ToolCallBlock(id="revoked", name="probe", input='{"item":2}'), state)] + assert 2 not in intervals + assert "本次调用未执行" in str(result) + finally: + await resource.close() + + +@pytest.mark.asyncio +async def test_plugin_rollback_repeated_cancel_releases_every_stdio_owner(tmp_path): + from types import SimpleNamespace + from core.llm.tools.plugin_activation import _ActivationStage + script = tmp_path / "delayed_shutdown.py" + script.write_text('''import anyio +from contextlib import asynccontextmanager +from mcp.server.mcpserver import MCPServer +@asynccontextmanager +async def lifespan(server): + try: + yield {} + finally: + await anyio.sleep(.3) +s = MCPServer("rollback", lifespan=lifespan) +@s.tool() +def echo(text: str)->str: return text +s.run() +''') + live = SimpleNamespace(tools=[], skills_or_loaders=[]) + stage = _ActivationStage(live, {"close_list": []}, None) + resources = [await discover("rollback-"+str(i), {"command":sys.executable,"args":[str(script)]}) for i in range(2)] + for resource in resources: + stage.add_client(resource.name, resource) + cleanup = asyncio.create_task(stage.discard()) + await asyncio.sleep(.05) + cleanup.cancel() + await asyncio.sleep(.05) + cleanup.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(cleanup, 10) + assert all(r.session._stop.is_set() and r.session._owner.done() for r in resources) + assert not stage._staged_clients + assert not live.tools diff --git a/src/backend/tests/llm/test_mcp_timeouts.py b/src/backend/tests/llm/test_mcp_timeouts.py index 0ecfe5e7..bcb13462 100644 --- a/src/backend/tests/llm/test_mcp_timeouts.py +++ b/src/backend/tests/llm/test_mcp_timeouts.py @@ -3,29 +3,22 @@ from types import SimpleNamespace -class _CapturingClient: - def __init__(self, **kwargs) -> None: - self.kwargs = kwargs - - def test_kb_client_gets_tighter_execution_and_transport_deadlines(monkeypatch): - from core.llm.mcp_pool import make_client + from core.llm.mcp.client import make_client monkeypatch.delenv("KB_MCP_EXECUTION_TIMEOUT_SECONDS", raising=False) client = make_client( "retrieve_dataset_content", {"transport": "streamable_http", "url": "http://mcp:9100/mcp/"}, - is_stateful=False, - client_cls=_CapturingClient, ) - assert client.kwargs["execution_timeout"] == 75.0 - assert client.kwargs["mcp_config"].timeout == 80.0 - assert client.kwargs["mcp_config"].url == "http://mcp:9100/mcp" + assert client.read_timeout_seconds == 75.0 + from core.llm.mcp.client import _http_transport_timeout + assert _http_transport_timeout("retrieve_dataset_content", {}, client.read_timeout_seconds) == 80.0 def test_mcp_timeout_config_overrides_defaults(): - from core.llm.mcp_pool import make_client + from core.llm.mcp.client import make_client client = make_client( "custom", @@ -35,33 +28,30 @@ def test_mcp_timeout_config_overrides_defaults(): "execution_timeout": 240, "transport_timeout": 250, }, - is_stateful=False, - client_cls=_CapturingClient, ) - assert client.kwargs["execution_timeout"] == 240.0 - assert client.kwargs["mcp_config"].timeout == 250.0 + assert client.read_timeout_seconds == 240.0 + from core.llm.mcp.client import _http_transport_timeout + assert _http_transport_timeout("custom", {"transport_timeout": 250}, client.read_timeout_seconds) == 250.0 def test_stdio_client_also_has_a_hard_execution_deadline(monkeypatch): - from core.llm.mcp_pool import make_client + from core.llm.mcp.client import make_client monkeypatch.setenv("MCP_TOOL_EXECUTION_TIMEOUT_SECONDS", "90") client = make_client( "stdio-tool", {"transport": "stdio", "command": "python", "args": []}, - client_cls=_CapturingClient, ) - assert client.kwargs["execution_timeout"] == 90.0 + assert client.read_timeout_seconds == 90.0 def test_database_extra_config_exposes_timeout_overrides(monkeypatch): - from core.services.mcp_service import McpServerConfigService + from core.capabilities.mcp.service import McpServerConfigService row = SimpleNamespace( transport="streamable_http", - is_stable=False, url="http://remote/mcp", headers={}, extra_config={"execution_timeout": 180, "transport_timeout": 190}, diff --git a/src/backend/tests/llm/test_migration_vendor_datasource.py b/src/backend/tests/llm/test_migration_vendor_datasource.py new file mode 100644 index 00000000..4b487704 --- /dev/null +++ b/src/backend/tests/llm/test_migration_vendor_datasource.py @@ -0,0 +1,114 @@ +"""Migration seams outside the main factory: LiteLLM and datasource probes.""" +import os +import sys +from types import SimpleNamespace + +import pytest +from agentscope.message import UserMsg +from agentscope.model import OpenAIChatModel +from openai.types.chat import ChatCompletionChunk +from mcp.server.mcpserver import MCPServer + +from core.models.providers.vendor_models import LiteLLMChatModel +from core.llm.mcp import lifetime +from features.datasources import introspection +from tests.llm.test_mcp_parallel_execution import probe_server + + +@pytest.mark.asyncio +async def test_litellm_stream_uses_native_parser(monkeypatch): + calls = [] + closed = [] + + async def completion(**kwargs): + calls.append(kwargs) + + async def stream(): + for delta, finish in [({"role": "assistant", "content": "vendor-ok"}, None), ({}, "stop")]: + yield ChatCompletionChunk.model_validate({ + "id": "litellm-stream", "object": "chat.completion.chunk", + "created": 1, "model": "vendor-probe", + "choices": [{"index": 0, "delta": delta, "finish_reason": finish}], + }) + class VendorStream: + def __aiter__(self): + return stream() + + async def __aenter__(self): + return self + + async def __aexit__(self, *exc_info): + closed.append(True) + + return VendorStream() + + monkeypatch.setitem(sys.modules, "litellm", SimpleNamespace(acompletion=completion)) + model = LiteLLMChatModel( + litellm_model="bedrock/vendor-probe", litellm_kwargs={}, model="vendor-probe", + parameters=OpenAIChatModel.Parameters(), stream=True, timeout=5, context_size=32768, + ) + response = await model._call_api("vendor-probe", [UserMsg(name="user", content="hello")]) + chunks = [chunk async for chunk in response] + assert calls[0]["stream"] is True + assert calls[0]["model"] == "bedrock/vendor-probe" + assert any("vendor-ok" in str(chunk.content) for chunk in chunks) + assert chunks[-1].finished_reason.value == "completed" + assert closed == [True] + + +TOOLS = ''' +@s.tool() +def search_objects(source: str = "", database: str = "") -> str: + return '{"tables":[{"schema":"public","table":"orders","columns":[{"name":"id","type":"integer"}]}]}' +@s.tool() +def list_indices(index_pattern: str) -> str: + return '["orders"]' +@s.tool() +def get_mappings(index: str) -> str: + return '{"orders":{"mappings":{"properties":{"id":{"type":"integer"}}}}}' +''' + + +@pytest.mark.asyncio +@pytest.mark.parametrize("transport", ["stdio", "streamable_http"]) +@pytest.mark.parametrize("kind", ["sql", "es"]) +async def test_real_datasource_probe_releases_resources(monkeypatch, tmp_path, transport, kind): + resources = [] + original = lifetime.discover + + async def record_discovery(*args, **kwargs): + resource = await original(*args, **kwargs) + resources.append(resource) + return resource + + monkeypatch.setattr(lifetime, "discover", record_discovery) + + async def run(cfg): + monkeypatch.setattr(introspection, "_get_mcp_cfg", lambda *_: (cfg, None)) + if kind == "sql": + rows, error = await introspection._introspect_sql("source", "database") + else: + rows, error = await introspection._introspect_es("source", "orders*") + assert error is None + assert rows[0]["table"] == "orders" + assert rows[0]["columns"] == [{"name": "id", "type": "integer"}] + assert len(resources) == 1 + assert resources[0].session._owner.done() + assert not resources[0].session._owner.cancelled() + + if transport == "streamable_http": + server = MCPServer("datasource-probe") + exec(TOOLS, {"s": server}) + async with probe_server(server) as (url, _): + await run({"transport": transport, "url": url}) + else: + pid_file = tmp_path / "pid" + script = tmp_path / "server.py" + script.write_text( + "import os\nfrom pathlib import Path\nfrom mcp.server.mcpserver import MCPServer\n" + + f"Path({str(pid_file)!r}).write_text(str(os.getpid()))\n" + + 's=MCPServer("datasource-probe")\n' + TOOLS + '\ns.run()\n' + ) + await run({"transport": transport, "command": sys.executable, "args": [str(script)]}) + with pytest.raises(ProcessLookupError): + os.kill(int(pid_file.read_text()), 0) diff --git a/src/backend/tests/llm/test_model_failover.py b/src/backend/tests/llm/test_model_failover.py index c4052e61..fe75ae5d 100644 --- a/src/backend/tests/llm/test_model_failover.py +++ b/src/backend/tests/llm/test_model_failover.py @@ -8,8 +8,8 @@ import httpx import openai import pytest -from core.llm import failover -from core.llm.failover import FailoverChatModel, provider_is_unusable +from core.models import failover +from core.models.failover import FailoverChatModel, provider_is_unusable def _status(code: int, cls: type = openai.APIStatusError) -> openai.APIStatusError: @@ -27,7 +27,8 @@ def __init__(self, provider_id: str, behaviour: Any, *, stream: bool = True): self.parameters = SimpleNamespace() self.stream = stream self.context_size = 32000 - self.formatter = SimpleNamespace() + from agentscope.formatter import OpenAIChatFormatter + self.formatter = OpenAIChatFormatter() self.wire_protocol = "openai_chat" self.structured_reasoning = False self._behaviour = behaviour @@ -240,17 +241,17 @@ def _cfg(pid: str, ctx: int = 32000): @pytest.mark.asyncio async def test_with_failover_switches_end_to_end(monkeypatch): """真实装配路径：主供应商 402 时整条链要把答案接住，而不是把错误抛给用户。""" - from core.llm import failover as mod + from core.models import failover as mod primary_cfg = _cfg("deepseek") backup = _Candidate("backup", ("stream", ["接住了"])) monkeypatch.setattr( - "core.services.model_config.ModelConfigService.get_instance", + 'core.models.config.ModelConfigService.get_instance', lambda: SimpleNamespace(resolve_failover_chain=lambda p: [primary_cfg, _cfg("backup")]), ) monkeypatch.setattr( - "core.llm.chat_models.build_model_for_mode", + 'core.models.chat_models.build_model_for_mode', lambda cfg, **kw: backup, ) @@ -263,11 +264,11 @@ async def test_with_failover_switches_end_to_end(monkeypatch): def test_with_failover_returns_bare_model_when_alone(monkeypatch): """只配了一家时不该平白多包一层。""" - from core.llm import failover as mod + from core.models import failover as mod only = _cfg("solo") monkeypatch.setattr( - "core.services.model_config.ModelConfigService.get_instance", + 'core.models.config.ModelConfigService.get_instance', lambda: SimpleNamespace(resolve_failover_chain=lambda p: [only]), ) primary = _Candidate("solo", ("stream", ["x"])) @@ -276,12 +277,12 @@ def test_with_failover_returns_bare_model_when_alone(monkeypatch): def test_with_failover_survives_a_broken_chain_lookup(monkeypatch): """取备选链本身失败时，至少要把主模型原样交出去，不能连累正常请求。""" - from core.llm import failover as mod + from core.models import failover as mod def _boom(): raise RuntimeError("db down") - monkeypatch.setattr("core.services.model_config.ModelConfigService.get_instance", _boom) + monkeypatch.setattr('core.models.config.ModelConfigService.get_instance', _boom) primary = _Candidate("p1", ("stream", ["x"])) assert mod.with_failover(primary, _cfg("p1"), mode="medium") is primary @@ -307,8 +308,8 @@ def __init__(self, pid: str, weight: int, ctx: int): def _chain_from(rows, monkeypatch, db_session): from sqlalchemy.orm import sessionmaker - from core.services import model_config as mc - from core.db.model_repository import create_provider + from core.models import config as mc + from core.models.repository import create_provider for row in rows: create_provider( @@ -359,7 +360,7 @@ def _build_seeing(primary: _Candidate, *pairs): def test_carries_media_sees_uploads_and_tool_results(): """图可能直接挂在消息上，也可能折在工具回执里——两处都得认出来。""" - from core.llm.failover import carries_media + from core.models.failover import carries_media assert carries_media([_msg({"type": "text", "text": "hi"})]) is False assert carries_media([_msg(_IMAGE)]) is True diff --git a/src/backend/tests/llm/test_model_steps.py b/src/backend/tests/llm/test_model_steps.py index 0a2f501e..ce8c19d2 100644 --- a/src/backend/tests/llm/test_model_steps.py +++ b/src/backend/tests/llm/test_model_steps.py @@ -10,14 +10,14 @@ import pytest from agentscope.message import TextBlock, ThinkingBlock, ToolCallBlock, ToolResultBlock -from core.llm import model_steps as M +from core.llm.runtime import model_steps as M @pytest.fixture def media_db(tmp_path, monkeypatch): """把媒体表指向一个临时 SQLite 库，并清掉取回缓存。""" from core.db.models import ToolMediaBlob - from core.llm import tool_media_store + from core.llm.runtime import tool_media_store from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker @@ -123,7 +123,7 @@ def test_replay_keeps_the_reference_so_snapshots_do_not_carry_pixels(media_db): def test_context_assembly_hands_the_model_the_same_pixels_a_later_turn(media_db): """装配上下文那一步把引用换回原图，模型下一轮看到的还是当时那张。""" - from core.llm.tool_media_store import hydrate_rows + from core.llm.runtime.tool_media_store import hydrate_rows media = hydrate_rows(M.replay_rows([_media_result()]))[0]["content"][0]["output"][1] @@ -137,7 +137,7 @@ def test_context_assembly_hands_the_model_the_same_pixels_a_later_turn(media_db) def test_hydrating_many_media_refs_costs_one_query(media_db): """整段历史只查一次库——每个媒体块查一次会把读历史变成 N+1。""" - from core.llm import tool_media_store + from core.llm.runtime import tool_media_store rows = [ row @@ -163,7 +163,7 @@ def test_hydrating_many_media_refs_costs_one_query(media_db): def test_history_without_media_is_returned_untouched(media_db): """没有媒体的历史不查库。""" - from core.llm import tool_media_store + from core.llm.runtime import tool_media_store rows = M.replay_rows([_result("c1", "plain text")]) original = tool_media_store._load @@ -188,7 +188,7 @@ def test_the_same_image_read_twice_is_stored_once(media_db): def test_lost_media_is_reported_as_unavailable_not_silently_described(media_db): """引用还在但原件没了：明说不可用，绝不退回成一段像内容的文字。""" - from core.llm import tool_media_store + from core.llm.runtime import tool_media_store from sqlalchemy import text rows = M.replay_rows([_media_result()]) @@ -240,7 +240,9 @@ def test_replay_reproduces_three_serial_steps_in_order_without_duplication(): "c1", "c2", ] - assert rows[-1]["content"][-1] == {"type": "text", "text": "done"} + assert rows[-1]["content"][-1]["type"] == "text" + assert rows[-1]["content"][-1]["text"] == "done" + assert rows[-1]["content"][-1]["created_at"] is not None def test_replay_keeps_two_parallel_calls_in_one_step_with_both_results_after_it(): @@ -306,7 +308,8 @@ def test_close_dangling_calls_balances_the_record_at_rest(): def test_cancelled_turn_is_marked_at_the_end_of_the_last_answer(): rows = M.replay_rows([_assistant(TextBlock(type="text", text="我先看看"))], cancelled=True) - assert rows[-1]["content"] == [{"type": "text", "text": "我先看看\n\n[本轮回答被用户中断]"}] + assert len(rows[-1]["content"]) == 1 + assert rows[-1]["content"][0]["text"] == "我先看看\n\n[本轮回答被用户中断]" rows = M.replay_rows( [ @@ -334,10 +337,9 @@ def test_replace_final_text_rewrites_only_the_last_answer_text(): M.replace_final_text(steps, "revised") - assert steps[-1]["blocks"] == [ - {"type": "thinking", "thinking": "B"}, - {"type": "text", "text": "revised"}, - ] + assert len(steps[-1]["blocks"]) == 2 + assert steps[-1]["blocks"][0]["thinking"] == "B" + assert steps[-1]["blocks"][1] == {"type": "text", "text": "revised"} assert steps[0]["blocks"][1]["type"] == "tool_call" diff --git a/src/backend/tests/llm/test_model_usage_extraction.py b/src/backend/tests/llm/test_model_usage_extraction.py index 9c4d3a28..6e77f9fe 100644 --- a/src/backend/tests/llm/test_model_usage_extraction.py +++ b/src/backend/tests/llm/test_model_usage_extraction.py @@ -12,7 +12,7 @@ from dataclasses import dataclass import pytest -from core.llm.model_usage import _usage_from_response, instrument_model_usage, model_usage_scope +from core.models.usage import _usage_from_response, instrument_model_usage, model_usage_scope class _DictMixinUsage(dict): diff --git a/src/backend/tests/llm/test_model_usage_scope_teardown.py b/src/backend/tests/llm/test_model_usage_scope_teardown.py index de47be4c..cae138f7 100644 --- a/src/backend/tests/llm/test_model_usage_scope_teardown.py +++ b/src/backend/tests/llm/test_model_usage_scope_teardown.py @@ -15,7 +15,7 @@ import asyncio import pytest -from core.llm.model_usage import CURRENT_MODEL_USAGE, model_usage_scope +from core.models.usage import CURRENT_MODEL_USAGE, model_usage_scope class _Recorder: diff --git a/src/backend/tests/llm/test_offloader.py b/src/backend/tests/llm/test_offloader.py index c2b599e0..8f246326 100644 --- a/src/backend/tests/llm/test_offloader.py +++ b/src/backend/tests/llm/test_offloader.py @@ -9,8 +9,8 @@ import pytest from agentscope.message import TextBlock, ToolResultBlock from core.config.settings import settings -from core.llm.offloader import SandboxOffloader -from core.llm.tools import _paths +from core.llm.context.offloader import SandboxOffloader +from core.sandbox.workspace import resolution as _paths from core.sandbox.runner import server @@ -79,8 +79,8 @@ async def test_failed_offload_is_not_presented_as_a_file(runner_workspace, strea from agentscope.message import UserMsg from agentscope.tool import Toolkit from agentscope.tool._response import ToolChunk - from core.llm.compacting_agent import CompactingAgent - from core.llm.tool_collector import ToolCollector + from core.llm.context.compacting_agent import CompactingAgent + from core.llm.factory.tools.definitions import ToolAssembly, function_tool from tests.llm.test_tool_result_images import CaptureModel class UnwritableFiles: @@ -91,14 +91,14 @@ async def logs(): """Read diagnostic logs.""" return ToolChunk(content=[TextBlock(text="BEGIN " + "log line " * 20_000 + " END")]) - collector = ToolCollector() - collector.register_tool_function(logs) + collector = ToolAssembly() + collector.append(function_tool(logs)) model = CaptureModel("logs", {}) agent = CompactingAgent( name="agent", system_prompt="Inspect logs.", model=model, - toolkit=Toolkit(tools=collector.function_tools), + toolkit=Toolkit(tools=list(collector)), context_config=ContextConfig(tool_result_limit=100), offloader=SandboxOffloader(UnwritableFiles(), "chat-a"), ) @@ -126,8 +126,8 @@ async def test_context_compaction_survives_offload_failure(monkeypatch, runner_w from agentscope.agent import ContextConfig from agentscope.message import Msg from agentscope.tool import Toolkit - from core.llm.compacting_agent import CompactingAgent - from core.services import compaction_service + from core.llm.context.compacting_agent import CompactingAgent + from core.chat.compaction import service as compaction_service from tests.llm.test_tool_result_images import CaptureModel class UnwritableFiles: @@ -148,7 +148,7 @@ async def compact(chat_id, history): sandbox=replace(settings.sandbox, provider="script_runner"), ), ) - monkeypatch.setattr(compaction_service, "run_mid_turn_compaction", compact) + monkeypatch.setattr(compaction_port, "run_mid_turn_compaction", compact) model = CaptureModel() model.context_size = 1_000 model.generate_structured_output = summary @@ -197,8 +197,8 @@ async def test_full_output_can_be_read_from_the_path_seen_by_model(runner_worksp from agentscope.message import UserMsg from agentscope.tool import Toolkit from agentscope.tool._response import ToolChunk - from core.llm.compacting_agent import CompactingAgent - from core.llm.tool_collector import ToolCollector + from core.llm.context.compacting_agent import CompactingAgent + from core.llm.factory.tools.definitions import ToolAssembly, function_tool from tests.llm.test_tool_result_images import CaptureModel text = 'BEGIN\n{"数据": "' + "line " * 30_000 + '"}\nEND' @@ -207,15 +207,15 @@ async def logs(): """Read diagnostic logs.""" return ToolChunk(content=[TextBlock(text=text)]) - collector = ToolCollector() - collector.register_tool_function(logs) + collector = ToolAssembly() + collector.append(function_tool(logs)) model = CaptureModel("logs", {}) files = RunnerFiles() agent = CompactingAgent( name="agent", system_prompt="Inspect logs.", model=model, - toolkit=Toolkit(tools=collector.function_tools), + toolkit=Toolkit(tools=list(collector)), context_config=ContextConfig(tool_result_limit=100), offloader=SandboxOffloader(files, "chat-a", user_id="u1"), ) @@ -329,9 +329,9 @@ async def test_offload_timeout_is_bounded_and_cancellation_propagates( ): import asyncio - from core.llm.offloader import OffloadError + from core.llm.context.offloader import OffloadError - monkeypatch.setattr("core.llm.offloader._WRITE_TIMEOUT_SEC", 0.01) + monkeypatch.setattr('core.llm.context.offloader._WRITE_TIMEOUT_SEC', 0.01) cancelled = asyncio.Event() class HangingFiles: @@ -358,7 +358,7 @@ async def put_file(self, *args, **kwargs): @pytest.mark.asyncio @pytest.mark.parametrize("session", [None, ""]) async def test_no_file_is_published_without_a_persistent_workspace(session): - from core.llm.offloader import OffloadError + from core.llm.context.offloader import OffloadError class Files: async def put_file(self, *args, **kwargs): @@ -379,7 +379,7 @@ async def test_disabled_compaction_preserves_context_without_summary_or_archive( from agentscope.message import UserMsg from agentscope.tool import Toolkit - from core.llm.compacting_agent import CompactingAgent + from core.llm.context.compacting_agent import CompactingAgent from tests.llm.test_tool_result_images import CaptureModel monkeypatch.setattr( @@ -419,3 +419,5 @@ async def test_disabled_compaction_preserves_context_without_summary_or_archive( assert agent.state.summary == "" model.generate_structured_output.assert_not_awaited() files.put_file.assert_not_awaited() + +from core.harness.registry import compaction as compaction_port diff --git a/src/backend/tests/llm/test_openai_provider.py b/src/backend/tests/llm/test_openai_provider.py index 88edb333..3e50d49e 100644 --- a/src/backend/tests/llm/test_openai_provider.py +++ b/src/backend/tests/llm/test_openai_provider.py @@ -1,7 +1,7 @@ """Tests for OpenAI-compatible model providers.""" -from core.llm.chat_models import make_chat_model -from core.llm.providers.registry import get_spec, to_frontend_schema +from core.models.chat_models import make_chat_model +from core.models.providers.registry import get_spec, to_frontend_schema def _make_model( @@ -55,7 +55,7 @@ def test_generic_compatible_provider_keeps_existing_reasoning_transport(): def test_unrecorded_protocol_defaults_to_responses(): """Responses is the default; chat completions is only for endpoints found to lack it.""" - from core.llm.responses_models import OpenAICompatResponsesModel + from core.models.responses import OpenAICompatResponsesModel model = _make_model("openai_compatible", None, api_protocol=None) @@ -66,7 +66,7 @@ def test_unrecorded_protocol_defaults_to_responses(): def test_probed_chat_only_endpoint_stays_on_chat_completions(): - from core.llm.chat_models import OpenAICompatChatModel + from core.models.chat_models import OpenAICompatChatModel model = _make_model("openai_compatible", None, api_protocol="chat_completions") @@ -76,7 +76,7 @@ def test_probed_chat_only_endpoint_stays_on_chat_completions(): def test_azure_never_uses_responses(): """Azure addresses Responses per deployment, which the shared client cannot build.""" - from core.llm.chat_models import OpenAICompatChatModel + from core.models.chat_models import OpenAICompatChatModel model = make_chat_model( model="test-model", @@ -96,8 +96,8 @@ def test_azure_never_uses_responses(): def test_config_lookup_failure_does_not_mask_original_failure(monkeypatch): - from core.llm.chat_models import _resolve_or_dummy - from core.services.model_config import ModelConfigService + from core.models.chat_models import _resolve_or_dummy + from core.models.config import ModelConfigService def unavailable(): raise RuntimeError("configuration unavailable") diff --git a/src/backend/tests/llm/test_permission_ticket_isolation.py b/src/backend/tests/llm/test_permission_ticket_isolation.py index a0315636..7f71e00d 100644 --- a/src/backend/tests/llm/test_permission_ticket_isolation.py +++ b/src/backend/tests/llm/test_permission_ticket_isolation.py @@ -8,6 +8,8 @@ """ from __future__ import annotations +from tests.orchestration.hook_support import permission_adapter +from types import SimpleNamespace import asyncio import json @@ -15,14 +17,9 @@ import pytest from agentscope.message import ToolCallBlock from agentscope.tool._response import ToolResponse -from core.llm.tool_permissions import ( - CURRENT_PERMISSION_TICKET, - PermissionRuntime, - ToolPermissionMiddleware, - ToolPermissionRegistry, - ToolPermissionService, - local_path_tool, -) +from core.sandbox.permissions.dispatch_models import CURRENT_PERMISSION_TICKET, PermissionRuntime +from core.sandbox.permissions.tool_specs import ToolPermissionRegistry, local_path_tool +from core.llm.tool_permissions import ToolPermissionService def _tool_call(path: str, call_id: str) -> ToolCallBlock: @@ -33,10 +30,10 @@ def _tool_call(path: str, call_id: str) -> ToolCallBlock: ) -def _middleware() -> ToolPermissionMiddleware: +def _middleware(): registry = ToolPermissionRegistry() registry.register("Read", local_path_tool("file_path", "read"), source="native") - return ToolPermissionMiddleware( + return permission_adapter( ToolPermissionService( registry, PermissionRuntime( @@ -60,7 +57,7 @@ async def handler(**_kwargs): yield ToolResponse(id=call_id) async for _item in middleware.on_acting( - None, {"tool_call": _tool_call(path, call_id)}, handler + SimpleNamespace(state=SimpleNamespace(run_id="permission-test", context=[])), {"tool_call": _tool_call(path, call_id)}, handler ): pass @@ -100,7 +97,7 @@ async def failing_handler(**_kwargs): with pytest.raises(RuntimeError): async for _item in middleware.on_acting( - None, {"tool_call": _tool_call("/workspace/a.txt", "call-a")}, failing_handler + SimpleNamespace(state=SimpleNamespace(run_id="permission-test", context=[])), {"tool_call": _tool_call("/workspace/a.txt", "call-a")}, failing_handler ): pass diff --git a/src/backend/tests/llm/test_plan_update_tool_description.py b/src/backend/tests/llm/test_plan_update_tool_description.py index 0e52c7fd..5a76176f 100644 --- a/src/backend/tests/llm/test_plan_update_tool_description.py +++ b/src/backend/tests/llm/test_plan_update_tool_description.py @@ -10,14 +10,16 @@ import pytest -from core.llm.plan_update_tool import register_plan_update_tool +from core.llm.tools.plan_update import register_plan_update_tool class _CapturingToolkit: def __init__(self) -> None: self.functions: dict = {} - def register_tool_function(self, fn, **_kwargs) -> None: + def append(self, tool): + fn = tool._func + _kwargs = {"func_description": tool.description, "permission": getattr(tool, "dispatch_spec", None)} self.functions[fn.__name__] = fn @@ -60,7 +62,7 @@ def test_explanation_is_a_real_parameter(): def test_explanation_never_reaches_the_plan_bar_payload(): """理由混进 plan_update 载荷就会污染用户可见的计划栏。""" - from core.llm.plan_update_tool import parse_plan_update_args + from core.llm.tools.plan_update import parse_plan_update_args parsed = parse_plan_update_args({ "title": "做 PPT", @@ -108,6 +110,6 @@ def test_description_forbids_resubmitting_an_unchanged_plan(update_plan_doc): def test_prompt_section_forbids_resubmitting_an_unchanged_plan(): """系统提示词那节和工具描述要给出同一条口径，否则两处互相抵消。""" - from core.llm.plan_update_tool import build_plan_update_prompt_section + from core.llm.tools.plan_update import build_plan_update_prompt_section assert "状态确实发生变化时才调用" in build_plan_update_prompt_section() diff --git a/src/backend/tests/llm/test_post_turn_measured_compaction.py b/src/backend/tests/llm/test_post_turn_measured_compaction.py index ee137ecb..bf980228 100644 --- a/src/backend/tests/llm/test_post_turn_measured_compaction.py +++ b/src/backend/tests/llm/test_post_turn_measured_compaction.py @@ -10,10 +10,10 @@ import core.db.engine as engine_mod from core.config import settings as config_settings from core.db.models import ChatSession -from core.llm import compaction as C -from core.services import compaction_budget as budget -from core.services.compaction_coordinator import CompactionCoordinator -from core.services.chat_service import ChatService +from core.llm.context import compaction as C +from core.llm.context import budget as budget +from core.chat.compaction.coordinator import CompactionCoordinator +from core.chat.service import ChatService INPUTS = { "system_prompt": "", @@ -61,7 +61,7 @@ async def summarize(history, *, timeout): called.append(history) return "synthetic summary" - monkeypatch.setattr("core.services.compaction_summary._summarize", summarize) + monkeypatch.setattr('core.chat.compaction.summary._summarize', summarize) result = asyncio.run( CompactionCoordinator("measured-compaction").compact( budget_inputs=INPUTS, token_limit=29520 @@ -110,7 +110,7 @@ def test_finalizing_covered_assistant_fences_only_mid_turn_checkpoint( db_session, monkeypatch, phase, pending ): from core.chat import inflight - from core.services.compaction_service import _load_history + from core.chat.compaction.service import _load_history chat = "covered-growth" db_session.add(ChatSession(chat_id=chat, user_id="u1", title="test")) @@ -175,7 +175,7 @@ def test_finalizing_covered_assistant_fences_only_mid_turn_checkpoint( async def summarize(history, *, timeout): return "complete summary" - monkeypatch.setattr("core.services.compaction_summary._summarize", summarize) + monkeypatch.setattr('core.chat.compaction.summary._summarize', summarize) assert asyncio.run(CompactionCoordinator(chat).compact(budget_inputs=INPUTS, token_limit=29520)) db_session.expire_all() final = svc.get_latest_compaction_checkpoint(chat) diff --git a/src/backend/tests/llm/test_project_environment_factory.py b/src/backend/tests/llm/test_project_environment_factory.py index 43d9cf0a..a74d41ae 100644 --- a/src/backend/tests/llm/test_project_environment_factory.py +++ b/src/backend/tests/llm/test_project_environment_factory.py @@ -10,12 +10,18 @@ def test_empty_environment_blocks_registration_and_keeps_one_run_frozen(tmp_path import asyncio from core.db.engine import Base, engine, SessionLocal from core.db.models import Project, UserShadow, ChatSession, UserAgent, AdminMcpServer, ChatRun -from core.db.model_repository import create_provider, assign_role +from core.models.repository import create_provider, assign_role +from features import register_product_tools, register_run_capabilities +register_product_tools() +register_run_capabilities() from core.llm.factory import create_agent_executor from core.llm.factory.request import AgentRequest -from core.llm.project_environment import bind_project_environment -from orchestration.workflow import _extract_project_ctx +from core.llm.runtime.project_environment import bind_project_environment +from core.agent_runtime.targets import _extract_project_ctx +from core.harness.registry import load_models +from features.installed import INSTALLED_FEATURES +load_models(INSTALLED_FEATURES) Base.metadata.create_all(engine) with SessionLocal() as db: model = create_provider(db, display_name="mock", provider_type="chat", base_url="http://127.0.0.1:1/v1", @@ -71,7 +77,7 @@ async def run(): assert outside.enabled_mcp_ids == ["internet_search"] # Exercise the actual custom-child dispatch, whose cloud path deliberately # drops chat_id and used to drop all project restrictions as well. - from core.llm.subagent_tool import _run_subagent_in_thread + from core.llm.subagents.tool import _run_subagent_in_thread success, result, _, _ = await asyncio.to_thread( _run_subagent_in_thread, "unbound", "Unbound", "test", "", "owner", parent_runtime={"project_ctx": request.project_ctx}, @@ -82,16 +88,16 @@ async def run(): # factory and Toolkit must expose only the explicitly bound private MCP, # including when its ambient switch is off. from unittest.mock import patch - from core.llm.mcp_pool import make_client + from core.llm.mcp.lifetime import discover connected = [] - def manifest_client(name, config, **kwargs): + async def manifest_client(name, config, **kwargs): connected.append(name) - return make_client(name, {**config, "schema_source": "cloud_manifest", + return await discover(name, {**config, "schema_source": "cloud_manifest", "gateway_invoke_url": "http://127.0.0.1:1/invoke", "schema_hash": "fixture", "manifest_tools": [{"name": "read_" + name, "description": "Read fixture", "inputSchema": {"type": "object", "properties": {}}}]}, **kwargs) selected_ctx = {"project_id": "selected"} - with patch("core.llm.mcp_pool.make_client", side_effect=manifest_client): + with patch("core.llm.mcp.lifetime.discover", side_effect=manifest_client): selected_agent, selected_clients = await create_agent_executor( current_user_id="owner", project_ctx=selected_ctx, max_iters=1, enabled_mcp_ids=["outside"], @@ -103,7 +109,7 @@ def manifest_client(name, config, **kwargs): assert (await bind_project_environment(following_step)).mcp_ids == frozenset({"allowed"}) selected_names = {s["function"]["name"] for s in selected_agent._jx_compaction_tool_schemas} assert "read_allowed" in selected_names and "read_outside" not in selected_names, selected_names - from core.llm.mcp_manager import close_clients + from core.llm.mcp.manager import close_clients await close_clients(selected_clients) revoked_request = AgentRequest(current_user_id="owner", project_ctx=selected_ctx) await bind_project_environment(revoked_request) @@ -111,13 +117,15 @@ def manifest_client(name, config, **kwargs): connector = db.get(AdminMcpServer, "allowed") connector.owner_user_id = "someone-else" db.commit() - from core.llm.project_environment_middleware import ProjectEnvironmentMiddleware + from core.llm.runtime.project_environment_hook import project_environment_hook + from tests.orchestration.hook_support import make_adapter + from types import SimpleNamespace executed = [] async def io(**kwargs): executed.append("tool I/O") yield "executed" - results = [item async for item in ProjectEnvironmentMiddleware(revoked_request).on_acting( - None, {}, io)] + results = [item async for item in make_adapter(project_environment_hook(revoked_request)).on_acting( + SimpleNamespace(state=SimpleNamespace(run_id="project-test", context=[])), {}, io)] assert executed == [], executed assert results[0].metadata["project_environment_denied"] is True diff --git a/src/backend/tests/llm/test_project_environment_policy.py b/src/backend/tests/llm/test_project_environment_policy.py index c865acf6..4e1123df 100644 --- a/src/backend/tests/llm/test_project_environment_policy.py +++ b/src/backend/tests/llm/test_project_environment_policy.py @@ -2,7 +2,7 @@ import pytest from core.llm.factory.request import AgentRequest -from core.llm.project_environment import ProjectEnvironmentPolicy +from core.llm.runtime.project_environment import ProjectEnvironmentPolicy from fastapi import HTTPException @@ -43,11 +43,13 @@ def test_outside_agent_cannot_execute_even_if_passed_directly(): def test_dynamic_plugin_loader_refuses_components_outside_frozen_environment(): import asyncio - from core.plugins.runtime.activation import register_load_plugin - from core.plugins.runtime.models import DeferredPlugin + from core.llm.tools.plugin_activation import register_load_plugin + from core.capabilities.plugins.runtime.models import DeferredPlugin class Collector: - def register_tool_function(self, fn, **_): + def append(self, tool): + fn = tool._func + _ = {"func_description": tool.description, "permission": getattr(tool, "dispatch_spec", None)} self.invoke = fn collector = Collector() @@ -60,7 +62,7 @@ def register_tool_function(self, fn, **_): def test_disallowed_plugin_sharing_a_loose_skill_does_not_hide_that_skill(): - from core.plugins.runtime.models import DeferredPlugin, ProgressiveResolution + from core.capabilities.plugins.runtime.models import DeferredPlugin, ProgressiveResolution deferred = DeferredPlugin( install_id="outside", slug="outside", name="Outside", description="", skill_ids=["plot"] @@ -74,7 +76,7 @@ def test_disallowed_plugin_sharing_a_loose_skill_does_not_hide_that_skill(): def test_custom_cloud_child_inherits_environment_without_file_context(): - from core.services.local_project_workspace import child_project_context + from core.projects.local_workspace import child_project_context context = { "project_id": "cloud-project", diff --git a/src/backend/tests/llm/test_protocol_probe.py b/src/backend/tests/llm/test_protocol_probe.py index 570c2d76..84dc6209 100644 --- a/src/backend/tests/llm/test_protocol_probe.py +++ b/src/backend/tests/llm/test_protocol_probe.py @@ -9,12 +9,8 @@ import httpx import pytest -from core.llm.providers import protocol_probe -from core.llm.providers.protocol_probe import ( - PROTOCOL_CHAT, - PROTOCOL_RESPONSES, - detect_api_protocol, -) +from core.models.providers import protocol_probe +from core.models.providers.protocol_probe import PROTOCOL_CHAT, PROTOCOL_RESPONSES, detect_api_protocol # 第二阶段（工具结果回灌）默认让它"通过"，这样只想测路由判定的用例不必关心它。 diff --git a/src/backend/tests/llm/test_read_image_native.py b/src/backend/tests/llm/test_read_image_native.py index 758c3c95..b7992f99 100644 --- a/src/backend/tests/llm/test_read_image_native.py +++ b/src/backend/tests/llm/test_read_image_native.py @@ -1,6 +1,7 @@ """多模态主模型直接看图：``Read`` 返回图片块、原生版 ``read_image``、看图方式判定。""" from __future__ import annotations +from core.files import artifact_bytes as shared_artifact_bytes import asyncio import base64 @@ -19,7 +20,9 @@ class _Toolkit: def __init__(self): self.fns = {} - def register_tool_function(self, fn, **kw): + def append(self, tool): + fn = tool._func + kw = {"func_description": tool.description, "permission": getattr(tool, "dispatch_spec", None)} self.fns[fn.__name__] = fn @@ -69,9 +72,8 @@ def _register_native_read_image(): def test_read_image_returns_pixels_by_file_id(monkeypatch): - from core.llm import hooks - monkeypatch.setattr(hooks, "_download_artifact_bytes", lambda *a, **kw: PNG_1X1) + monkeypatch.setattr(shared_artifact_bytes, 'download_artifact_bytes', lambda *a, **kw: PNG_1X1) fn = _register_native_read_image() meta, block = _image_response_parts(asyncio.run(fn(file_id="art-1"))) assert meta == { @@ -107,9 +109,8 @@ def test_read_image_requires_a_target(): def test_read_image_rejects_non_image_bytes(monkeypatch): - from core.llm import hooks - monkeypatch.setattr(hooks, "_download_artifact_bytes", lambda *a, **kw: b"PK\x03\x04zip") + monkeypatch.setattr(shared_artifact_bytes, 'download_artifact_bytes', lambda *a, **kw: b"PK\x03\x04zip") assert "不是可识别的图片格式" in _error( asyncio.run(_register_native_read_image()(file_id="art-1")) ) @@ -121,10 +122,9 @@ def test_read_image_rejects_paths_outside_workspace(): def test_read_image_refuses_oversized_image(monkeypatch): - from core.llm import hooks - from core.vision import service as svc + from core.models.vision import service as svc - monkeypatch.setattr(hooks, "_download_artifact_bytes", lambda *a, **kw: PNG_1X1) + monkeypatch.setattr(shared_artifact_bytes, 'download_artifact_bytes', lambda *a, **kw: PNG_1X1) monkeypatch.setattr(svc, "MAX_IMAGE_BYTES", len(PNG_1X1) - 1) assert "图片过大" in _error(asyncio.run(_register_native_read_image()(file_id="art-1"))) @@ -191,7 +191,7 @@ def test_read_falls_back_to_binary_without_vision(monkeypatch): def _model_service(monkeypatch, *, main_vision: bool, provider_vision=None): - from core.services.model_config import ModelConfigService + from core.models.config import ModelConfigService class _Service: @staticmethod @@ -210,7 +210,7 @@ def resolve_provider(pid): def test_vision_mode_prefers_selected_provider(monkeypatch): - from core.vision import resolve_vision_mode + from core.models.vision import resolve_vision_mode _model_service(monkeypatch, main_vision=False, provider_vision=True) assert resolve_vision_mode("prov-1") == "native" @@ -218,8 +218,8 @@ def test_vision_mode_prefers_selected_provider(monkeypatch): def test_vision_mode_probe_failure_means_none(monkeypatch): - from core.services.model_config import ModelConfigService - from core.vision import resolve_vision_mode + from core.models.config import ModelConfigService + from core.models.vision import resolve_vision_mode def _boom(): raise RuntimeError("db down") @@ -229,7 +229,7 @@ def _boom(): def test_read_image_is_governed_by_local_path_permission(): - from core.llm.tool_permissions import builtin_tool_permission + from core.sandbox.permissions.tool_specs import builtin_tool_permission spec = builtin_tool_permission("read_image") assert spec is not None diff --git a/src/backend/tests/llm/test_reasoning_budget.py b/src/backend/tests/llm/test_reasoning_budget.py index 46cbd160..7675b52f 100644 --- a/src/backend/tests/llm/test_reasoning_budget.py +++ b/src/backend/tests/llm/test_reasoning_budget.py @@ -7,9 +7,9 @@ model on every turn, what it costs has to be visible. """ -from core.llm.context_adapter import AgentScopeContextAdapter -from core.llm.context_ir import KIND_ASSISTANT, KIND_THINKING, KIND_TOOL_CALL -from core.llm.context_usage import BREAKDOWN_KEYS, _manifest_breakdown +from core.llm.context.adapter import AgentScopeContextAdapter +from core.llm.context.ir_constants import KIND_ASSISTANT, KIND_THINKING, KIND_TOOL_CALL +from core.llm.context.usage import BREAKDOWN_KEYS, _manifest_breakdown _REASONING = "先查论文再对比专利，" * 20 diff --git a/src/backend/tests/llm/test_reasoning_echo_formatter.py b/src/backend/tests/llm/test_reasoning_echo_formatter.py index 61d6f4a1..f335027f 100644 --- a/src/backend/tests/llm/test_reasoning_echo_formatter.py +++ b/src/backend/tests/llm/test_reasoning_echo_formatter.py @@ -12,7 +12,7 @@ from agentscope.formatter import OpenAIChatFormatter from agentscope.message import Msg, TextBlock, ThinkingBlock, ToolCallBlock, ToolResultBlock -from core.llm.chat_models import ReasoningEchoChatFormatter, make_chat_model +from core.models.chat_models import ReasoningEchoChatFormatter, make_chat_model def _assistant_with_thinking(): @@ -187,6 +187,6 @@ def test_every_openai_compatible_model_echoes_its_reasoning(): def test_responses_wire_replays_reasoning_through_its_own_formatter(): # Same guarantee on the other wire, by a different mechanism: reasoning rides back # as its own item rather than being folded into an assistant row. - from core.llm.responses_models import ResponsesReplayFormatter + from core.models.responses import ResponsesReplayFormatter assert isinstance(_mk(api_protocol="responses").formatter, ResponsesReplayFormatter) diff --git a/src/backend/tests/llm/test_reasoning_probe.py b/src/backend/tests/llm/test_reasoning_probe.py index dd365859..acb8965c 100644 --- a/src/backend/tests/llm/test_reasoning_probe.py +++ b/src/backend/tests/llm/test_reasoning_probe.py @@ -2,7 +2,7 @@ import httpx import pytest -from core.llm.providers.reasoning_probe import discover_reasoning_efforts +from core.models.providers.reasoning_probe import discover_reasoning_efforts @pytest.mark.asyncio diff --git a/src/backend/tests/llm/test_reasoning_replay_policy.py b/src/backend/tests/llm/test_reasoning_replay_policy.py index 6a36ed6b..80ee4480 100644 --- a/src/backend/tests/llm/test_reasoning_replay_policy.py +++ b/src/backend/tests/llm/test_reasoning_replay_policy.py @@ -1,7 +1,7 @@ import pytest from agentscope.message import Msg, TextBlock, ThinkingBlock -from core.llm.providers.registry import get_spec -from core.llm.providers.vendor_models import build_native_model +from core.models.providers.registry import get_spec +from core.models.providers.vendor_models import build_native_model def native(provider, model): @@ -91,7 +91,7 @@ async def test_same_anthropic_model_missing_signature_fails_explicitly(): @pytest.mark.asyncio @pytest.mark.parametrize("provider, expected_native", [("deepseek", True), ("anthropic", False)]) async def test_echo_formatter_preserves_only_target_model_reasoning(provider, expected_native): - from core.llm.chat_models import ReasoningEchoChatFormatter + from core.models.chat_models import ReasoningEchoChatFormatter formatter = ReasoningEchoChatFormatter( replay_provider="deepseek", replay_model="m", replay_protocol="openai_chat" @@ -114,7 +114,7 @@ async def test_echo_formatter_preserves_only_target_model_reasoning(provider, ex def test_new_output_is_stamped_for_live_replay(): from types import SimpleNamespace - from core.llm.reasoning_replay import stamp_reasoning_origin + from core.models.reasoning_replay import stamp_reasoning_origin block = ThinkingBlock(thinking="reason") stamp_reasoning_origin( @@ -125,7 +125,7 @@ def test_new_output_is_stamped_for_live_replay(): @pytest.mark.asyncio async def test_other_protocol_does_not_reuse_native_reasoning(): - from core.llm.chat_models import ReasoningEchoChatFormatter + from core.models.chat_models import ReasoningEchoChatFormatter formatter = ReasoningEchoChatFormatter( replay_provider="deepseek", replay_model="m", replay_protocol="openai_chat" diff --git a/src/backend/tests/llm/test_reasoning_retained_in_history.py b/src/backend/tests/llm/test_reasoning_retained_in_history.py index b219a584..dd937961 100644 --- a/src/backend/tests/llm/test_reasoning_retained_in_history.py +++ b/src/backend/tests/llm/test_reasoning_retained_in_history.py @@ -8,12 +8,8 @@ the canonical step record. """ -from core.llm.message_compat import dict_to_msg -from core.llm.model_steps import ( - assistant_row_replay, - record_assistant_step, - record_tool_result_step, -) +from core.llm.runtime.message_compat import dict_to_msg +from core.llm.runtime.model_steps import assistant_row_replay, record_assistant_step, record_tool_result_step def _thinking_texts(msg): diff --git a/src/backend/tests/llm/test_responses_reasoning_replay.py b/src/backend/tests/llm/test_responses_reasoning_replay.py index 0ee54b3d..b16bb44a 100644 --- a/src/backend/tests/llm/test_responses_reasoning_replay.py +++ b/src/backend/tests/llm/test_responses_reasoning_replay.py @@ -11,12 +11,7 @@ import pytest from agentscope.message import Msg, TextBlock, ThinkingBlock, ToolCallBlock -from core.llm.responses_models import ( - REASONING_ITEMS_ATTR, - ResponsesReplayFormatter, - WIRE_PROTOCOL, - _attach_reasoning_items, -) +from core.models.responses import REASONING_ITEMS_ATTR, ResponsesReplayFormatter, WIRE_PROTOCOL, _attach_reasoning_items PROVIDER = "openai_compatible" MODEL = "test-model" diff --git a/src/backend/tests/llm/test_responses_thinking_stream.py b/src/backend/tests/llm/test_responses_thinking_stream.py index a6eef711..22734b5d 100644 --- a/src/backend/tests/llm/test_responses_thinking_stream.py +++ b/src/backend/tests/llm/test_responses_thinking_stream.py @@ -1,10 +1,4 @@ -"""思考正文必须真的流到界面上。 - -Responses 有两条思考通道：``reasoning_summary_text`` 是删减过的摘要（OpenAI 返回的 -那条），``reasoning_text`` 是思考原文（自建推理服务返回的那条）。AgentScope 的解析器 -只认前者。实测自建 deepseekv4-flash-vision 一轮发 144 帧 ``reasoning_text.delta``， -全被丢掉——界面上是一个空思考块，用户看到的现象是「调到思考为高，却根本没有思考」。 -""" +"""Responses 思考事件原样交给实际解析器，保留条目身份、分段和增量。""" from __future__ import annotations @@ -13,11 +7,7 @@ import pytest from agentscope.message import TextBlock, ThinkingBlock -from core.llm.responses_models import ( - REASONING_ITEMS_ATTR, - _attach_reasoning_items, - _ReasoningItemCapture, -) +from core.models.responses import REASONING_ITEMS_ATTR, _attach_reasoning_items, _ReasoningItemCapture def _event(kind: str, **fields): @@ -35,20 +25,47 @@ async def source(): @pytest.mark.asyncio -async def test_reasoning_text_deltas_reach_the_parser(): - """原文通道的增量必须变成解析器认得的形状，否则思考一个字都到不了界面。""" - _, seen = await _drain( - [ - _event("response.reasoning_text.delta", delta="先"), - _event("response.reasoning_text.delta", delta="想想"), - _event("response.output_text.delta", delta="答案"), - ] - ) - - reasoning = [e for e in seen if e.type == "response.reasoning_summary_text.delta"] - assert [e.delta for e in reasoning] == ["先", "想想"] - # 原始事件不再重复下发，否则同一段思考会被累积两遍。 - assert not [e for e in seen if e.type == "response.reasoning_text.delta"] +async def test_reasoning_events_keep_identity_and_parts_in_real_parser(): + from datetime import datetime + from agentscope.model import OpenAIResponseModel + + events = [ + _event("response.reasoning_text.delta", item_id="rs_1", content_index=0, delta="先"), + _event("response.reasoning_text.delta", item_id="rs_1", content_index=0, delta="想想"), + _event("response.reasoning_text.delta", item_id="rs_1", content_index=1, delta="另一段"), + _event("response.reasoning_text.delta", item_id="rs_2", content_index=0, delta="另一条"), + _event("response.reasoning_summary_text.delta", item_id="rs_1", summary_index=0, delta="摘要"), + _event("response.output_text.delta", item_id="msg_1", content_index=0, delta="答案"), + ] + _, seen = await _drain(events) + assert all(actual is original for actual, original in zip(seen, events)) + assert len(seen) == len(events) + + class Stream: + async def __aenter__(self): + return self + + async def __aexit__(self, *args): + return False + + def __aiter__(self): + async def source(): + for event in events: + yield event + return source() + + model = object.__new__(OpenAIResponseModel) + chunks = [ + chunk async for chunk in model._parse_stream_response( + datetime.now(), _ReasoningItemCapture(Stream()) + ) + ] + blocks = [block for chunk in chunks for block in chunk.content] + thinking = [block for block in blocks if isinstance(block, ThinkingBlock)] + assert [block.thinking for block in thinking] == ["先", "想想", "另一段", "另一条", "摘要"] + assert thinking[0].id == thinking[1].id + assert len({block.id for block in thinking}) == 4 + assert [block.text for block in blocks if isinstance(block, TextBlock)] == ["答案"] @pytest.mark.asyncio @@ -64,10 +81,11 @@ async def test_other_events_pass_through_untouched(): @pytest.mark.asyncio -async def test_an_empty_delta_is_not_forwarded_as_thinking(): +async def test_an_empty_delta_is_preserved_for_native_parser(): _, seen = await _drain([_event("response.reasoning_text.delta", delta="")]) - assert seen == [] + assert len(seen) == 1 + assert seen[0].delta == "" @pytest.mark.asyncio @@ -115,3 +133,23 @@ def test_the_encrypted_blob_is_never_shown_as_thinking(): _attach_reasoning_items(blocks, items) assert blocks[0].thinking == "" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("interrupt", [False, True]) +async def test_capture_closes_upstream_on_completion_and_cancellation(interrupt): + import asyncio + from unittest.mock import AsyncMock + stream = SimpleNamespace(__aenter__=AsyncMock(), __aexit__=AsyncMock(return_value=False)) + capture = _ReasoningItemCapture(stream) + if interrupt: + with pytest.raises(asyncio.CancelledError): + async with capture as entered: + assert entered is capture + raise asyncio.CancelledError() + else: + async with capture as entered: + assert entered is capture + stream.__aenter__.assert_awaited_once() + stream.__aexit__.assert_awaited_once() + assert stream.__aexit__.call_args.args[0] is (asyncio.CancelledError if interrupt else None) diff --git a/src/backend/tests/llm/test_responses_tool_call_id.py b/src/backend/tests/llm/test_responses_tool_call_id.py index bce94a83..22a599c0 100644 --- a/src/backend/tests/llm/test_responses_tool_call_id.py +++ b/src/backend/tests/llm/test_responses_tool_call_id.py @@ -1,19 +1,11 @@ -"""Responses 线的工具结果配对：按调用 id 认人，不是按条目 id。 - -Responses 线上一次函数调用带两个互不相同的 id —— 条目 id（``function_call.id``）与 -调用 id（``function_call.call_id``），结果回传只认后者。AgentScope 建 -``ToolResultBlock`` 时只留了条目 id，格式化又拿它当 ``function_call_output.call_id`` -发出去；DeepSeek 官方端点两者恰好不同（条目 id 是 uuid，调用 id 形如 ``call_00_…``）， -于是只要模型调了工具，这一轮就被打回 -``No tool call found for tool output with call_id …``。这些用例钉住配对。 -""" +"""Native SDK call ids remain paired across Responses calls and outputs.""" from __future__ import annotations import pytest from agentscope.message import Msg, TextBlock, ToolCallBlock, ToolResultBlock -from core.llm.responses_models import ResponsesReplayFormatter, WIRE_PROTOCOL +from core.models.responses import ResponsesReplayFormatter, WIRE_PROTOCOL ITEM_ID = "c41e042e-2e51-4b7f-9476-d677e46eadc2" CALL_ID = "call_00_JsdbgnSxi85atlacyJeb5187" @@ -28,10 +20,10 @@ def _formatter() -> ResponsesReplayFormatter: def _turn(call_id: str | None) -> list[Msg]: - extra = {"call_id": call_id} if call_id else {} - call = ToolCallBlock(id=ITEM_ID, name="atlas_aggregate", input="{}", **extra) + canonical_id = call_id or ITEM_ID + call = ToolCallBlock(id=canonical_id, name="atlas_aggregate", input="{}") result = ToolResultBlock( - id=ITEM_ID, + id=canonical_id, name="atlas_aggregate", output=[TextBlock(text="ok")], state="success", @@ -72,7 +64,7 @@ async def test_message_items_have_explicit_type_without_changing_tool_items() -> messages = [item for item in items if "role" in item and "content" in item] assert messages assert all(item.get("type") == "message" for item in messages) - assert [item["content"][0]["type"] for item in messages] == ["input_text", "input_text"] + assert [item["content"][0]["type"] for item in messages] == ["input_text", "output_text"] assert [item["type"] for item in items if item.get("type", "").startswith("function_call")] == [ "function_call", "function_call_output", ] diff --git a/src/backend/tests/llm/test_sandbox_session_resolution.py b/src/backend/tests/llm/test_sandbox_session_resolution.py index 38a50710..c5bf9bf5 100644 --- a/src/backend/tests/llm/test_sandbox_session_resolution.py +++ b/src/backend/tests/llm/test_sandbox_session_resolution.py @@ -1,7 +1,7 @@ from types import SimpleNamespace import pytest -from core.llm.subagent_tool import _run_subagent_in_thread +from core.llm.subagents.tool import _run_subagent_in_thread from core.llm.tools._common import resolve_sandbox_session @@ -64,15 +64,15 @@ async def no_op(*args, **kwargs): monkeypatch.setattr("core.db.engine.SessionLocal", lambda: FakeDbContext()) monkeypatch.setattr( - "core.services.user_agent_service.UserAgentService", + 'core.capabilities.agents.edition.UserAgentService', FakeUserAgentService, ) - monkeypatch.setattr("core.llm.builtin_subagents.get_builtin_subagent", lambda _id: None) + monkeypatch.setattr('core.capabilities.agents.builtin.get_builtin_subagent', lambda _id: None) monkeypatch.setattr( "core.llm.factory.create_agent_executor", fake_create_agent_executor, ) - monkeypatch.setattr("core.llm.mcp_manager.close_clients", no_op) + monkeypatch.setattr('core.llm.mcp.manager.close_clients', no_op) monkeypatch.setattr("core.infra.redis.close_redis", no_op) ok, text, _pinned, _messages = _run_subagent_in_thread( "custom-1", diff --git a/src/backend/tests/llm/test_selected_model_context_window.py b/src/backend/tests/llm/test_selected_model_context_window.py index 1f1c75b5..ea2c2120 100644 --- a/src/backend/tests/llm/test_selected_model_context_window.py +++ b/src/backend/tests/llm/test_selected_model_context_window.py @@ -1,11 +1,13 @@ """Configured endpoint windows must survive model construction and compaction.""" from types import SimpleNamespace -from core.services.model_config import ResolvedModelConfig + +import core.chat.runtime.compaction as compaction +from core.models.config import ResolvedModelConfig def test_model_builder_uses_selected_provider_window(monkeypatch): - from core.llm import chat_models + from core.models import chat_models calls = [] monkeypatch.setattr( @@ -24,13 +26,13 @@ def test_model_builder_uses_selected_provider_window(monkeypatch): def test_post_turn_trigger_uses_frozen_window_not_same_name_provider(monkeypatch): - from orchestration import chat_run_executor as executor - from core.llm import context_manager - from core.services import compaction_service + pass + from core.models import context_window as context_manager + from core.chat.compaction import service as compaction_service monkeypatch.setattr(context_manager, "resolve_model_context_window", lambda name: 128000) monkeypatch.setattr(compaction_service, "resolve_trigger_ratio", lambda: 0.82) - assert executor._should_schedule_compaction( + assert compaction._should_schedule_compaction( model_name="shared-model", usage={"context_tokens": 40000}, budget_inputs={"context_window": 36000}, @@ -39,9 +41,10 @@ def test_post_turn_trigger_uses_frozen_window_not_same_name_provider(monkeypatch def test_background_compaction_receives_the_frozen_window(monkeypatch): import asyncio - from orchestration.post_turn_compaction import _spawn_compaction_task - from core.services import compaction_service - from core.llm import context_manager + + from core.chat.runtime.compaction import _spawn_compaction_task + from core.models import context_window as context_manager + from core.chat.compaction import service as compaction_service observed = [] monkeypatch.setattr(context_manager, "resolve_model_context_window", lambda name: 128000) @@ -69,8 +72,8 @@ async def run(): def test_default_model_identity_reaches_usage_and_frozen_budget(): - from core.llm.context_usage import build_context_usage_snapshot - from orchestration.workflow import _compaction_budget_inputs + from core.agent_runtime.budget import _compaction_budget_inputs + from core.llm.context.usage import build_context_usage_snapshot agent = SimpleNamespace( model=SimpleNamespace( @@ -87,7 +90,7 @@ def test_default_model_identity_reaches_usage_and_frozen_budget(): def test_failover_usage_uses_actual_endpoint_window_over_primary_manifest(): - from core.llm.context_usage import build_context_usage_snapshot + from core.llm.context.usage import build_context_usage_snapshot agent = SimpleNamespace( model=SimpleNamespace( diff --git a/src/backend/tests/llm/test_shared_human_interactions.py b/src/backend/tests/llm/test_shared_human_interactions.py index 87ec67fd..c6c6e9d7 100644 --- a/src/backend/tests/llm/test_shared_human_interactions.py +++ b/src/backend/tests/llm/test_shared_human_interactions.py @@ -7,7 +7,7 @@ from pathlib import Path import pytest -from core.llm.tools import user_questions, _tool_confirmation as confirms +from core.llm.tools import user_questions, confirmation as confirms @pytest.fixture(autouse=True, params=["local", "redis", "delayed"]) @@ -29,8 +29,8 @@ async def put(self, key, value, *, ttl): monkeypatch.setattr(ephemeral, "get_redis", lambda: client) store = ephemeral.RedisEphemeralState() monkeypatch.setattr(ephemeral, "get_ephemeral_state", lambda: store) - monkeypatch.setattr("core.llm.interaction_store.get_ephemeral_state", lambda: store) - monkeypatch.setattr("core.services.yida_service.get_ephemeral_state", lambda: store) + monkeypatch.setattr('core.llm.runtime.interaction_store.get_ephemeral_state', lambda: store) + monkeypatch.setattr('integrations.yida.service.get_ephemeral_state', lambda: store) return store @@ -287,7 +287,7 @@ async def test_same_path_different_permission_domains_are_not_deduplicated(monke @pytest.mark.asyncio async def test_yida_qr_and_organization_selection_cross_workers(tmp_path, monkeypatch): import json - from core.services import yida_service as original + from integrations.yida import service as original second = worker_copy(original) for module in [original, second]: diff --git a/src/backend/tests/llm/test_single_turn_protocol.py b/src/backend/tests/llm/test_single_turn_protocol.py index 8045b471..8261564c 100644 --- a/src/backend/tests/llm/test_single_turn_protocol.py +++ b/src/backend/tests/llm/test_single_turn_protocol.py @@ -10,7 +10,7 @@ import pytest -from core.llm.single_turn import Endpoint, build_request, parse_response, stream_delta, turns +from core.models.completions.single_turn import Endpoint, build_request, parse_response, stream_delta, turns RESPONSES = Endpoint( base_url="https://gw.example/api/v1/desktop/capability/gateway/models/p1", diff --git a/src/backend/tests/llm/test_stream_wrapper_cleanup.py b/src/backend/tests/llm/test_stream_wrapper_cleanup.py new file mode 100644 index 00000000..7abf4866 --- /dev/null +++ b/src/backend/tests/llm/test_stream_wrapper_cleanup.py @@ -0,0 +1,30 @@ +"""Closing the public stream synchronously unwinds its inner generator.""" +from contextvars import ContextVar +from types import SimpleNamespace +import pytest +from core.llm.runtime import stream_loop +from core.llm.runtime.streaming import StreamingAgent + + +@pytest.mark.asyncio +async def test_close_after_first_event_finishes_cleanup_in_same_context(monkeypatch): + active = ContextVar("active_stream", default=False) + cleaned = [] + + async def inner(*args, **kwargs): + token = active.set(True) + try: + yield ("text_delta", "first") + pytest.fail("Closing must not request another event") + finally: + active.reset(token) + cleaned.append(True) + + monkeypatch.setattr(stream_loop, "stream", inner) + wrapper = StreamingAgent(SimpleNamespace(), []) + stream = wrapper.stream([], {}) + assert await anext(stream) == ("text_delta", "first") + assert active.get() is True + await stream.aclose() + assert cleaned == [True] + assert active.get() is False diff --git a/src/backend/tests/llm/test_structured_reasoning_flag.py b/src/backend/tests/llm/test_structured_reasoning_flag.py index 2f9835e1..cd680745 100644 --- a/src/backend/tests/llm/test_structured_reasoning_flag.py +++ b/src/backend/tests/llm/test_structured_reasoning_flag.py @@ -13,8 +13,8 @@ These tests therefore pin the chat protocol explicitly. """ -from core.llm.chat_models import make_chat_model -from core.llm.providers.registry import get_spec +from core.models.chat_models import make_chat_model +from core.models.providers.registry import get_spec def _mk(**kw): diff --git a/src/backend/tests/llm/test_subagent_cancellation.py b/src/backend/tests/llm/test_subagent_cancellation.py index cb724d31..daedeb92 100644 --- a/src/backend/tests/llm/test_subagent_cancellation.py +++ b/src/backend/tests/llm/test_subagent_cancellation.py @@ -5,12 +5,17 @@ from types import SimpleNamespace from unittest.mock import AsyncMock +import core.agent_runtime.direct as agent_runtime_direct +import core.agent_runtime.runner as runner +import core.agent_runtime.targets as targets import pytest -from core.llm import subagent_tool, _subagent_stream +from core.llm.subagents import stream as _subagent_stream, tool as subagent_tool class CapturingToolkit: - def register_tool_function(self, function, **kwargs): + def append(self, tool): + function = tool._func + kwargs = {"func_description": tool.description, "permission": getattr(tool, "dispatch_spec", None)} self.call = function @@ -34,7 +39,7 @@ async def wait_for_cancel(): class Child: state = SimpleNamespace(context=[]) - async def _reply(self, **kwargs): + async def reply_stream(self, **kwargs): await wait_for_cancel() if False: yield None @@ -128,7 +133,7 @@ async def start(record): @pytest.mark.asyncio async def test_direct_conversation_cancel_during_log_creation(monkeypatch): - from orchestration import workflow + pass started, release = asyncio.Event(), asyncio.Event() @@ -139,11 +144,11 @@ async def start(record): finish = AsyncMock() monkeypatch.setattr( - workflow, "_load_direct_user_agent", lambda *args: SimpleNamespace(name="test") + targets, "_load_direct_user_agent", lambda *args: SimpleNamespace(name="test") ) monkeypatch.setattr(subagent_tool.log_writer, "start_subagent_log", start) monkeypatch.setattr(subagent_tool.log_writer, "finish_subagent_log", finish) - stream = workflow._astream_subagent_direct( + stream = agent_runtime_direct._astream_subagent_direct( agent_id="test", session_messages=[], user_message="wait", context={} ) task = asyncio.create_task(anext(stream)) @@ -158,7 +163,7 @@ async def start(record): @pytest.mark.asyncio async def test_public_workflow_close_waits_for_direct_child(monkeypatch): - from orchestration import workflow + pass stopped = asyncio.Event() async def direct(**kwargs): try: @@ -166,8 +171,8 @@ async def direct(**kwargs): finally: await asyncio.sleep(0.01) stopped.set() - monkeypatch.setattr(workflow, "_astream_subagent_direct", direct) - stream = workflow.astream_chat_workflow(session_messages=[], user_message="test", context={"agent_id": "test"}) + monkeypatch.setattr(agent_runtime_direct, "_astream_subagent_direct", direct) + stream = runner.astream_chat_workflow(session_messages=[], user_message="test", context={"agent_id": "test"}) assert (await anext(stream))["delta"] == "first" await stream.aclose() assert stopped.is_set() @@ -175,7 +180,7 @@ async def direct(**kwargs): @pytest.mark.asyncio async def test_mcp_cleanup_survives_repeat_cancel_in_owning_task(): - from core.llm.mcp_manager import close_clients + from core.llm.mcp.manager import close_clients started, release = asyncio.Event(), asyncio.Event() closed = [] owners = [] diff --git a/src/backend/tests/llm/test_subagent_model_config.py b/src/backend/tests/llm/test_subagent_model_config.py index f00b585f..90996edc 100644 --- a/src/backend/tests/llm/test_subagent_model_config.py +++ b/src/backend/tests/llm/test_subagent_model_config.py @@ -8,9 +8,9 @@ from api.deps import require_system_settings from api.routes.v1 import models as routes from core.db.engine import get_db -from core.db.model_repository import create_provider -from core.services import model_config -from core.services.model_config import ModelConfigService +from core.models.repository import create_provider +from core.models import config as model_config +from core.models.config import ModelConfigService @pytest.fixture @@ -89,9 +89,12 @@ def test_real_subagent_reply_uses_configured_or_inherited_model(tmp_path, scenar import core.db.models from agentscope.message import UserMsg from core.db.engine import Base, engine, SessionLocal -from core.db.model_repository import create_provider, assign_role, unassign_role, update_provider +from core.models.repository import create_provider, assign_role, unassign_role, update_provider +from features import register_product_tools, register_run_capabilities +register_product_tools() +register_run_capabilities() from core.llm.factory import create_agent_executor -from core.llm.builtin_subagents import build_builtin_runtime_profile, get_builtin_subagent +from core.capabilities.agents.builtin import build_builtin_runtime_profile, get_builtin_subagent Base.metadata.create_all(engine) requests = [] @@ -192,7 +195,7 @@ async def run(): try: asyncio.run(run()) if scenario in ("inherit_builtin", "shared_builtin", "system_default"): - from core.llm.subagent_tool import _run_subagent_in_thread + from core.llm.subagents.tool import _run_subagent_in_thread events = [] ok, output, _, _ = _run_subagent_in_thread( "builtin.worker", "worker", "Reply done", "", "", diff --git a/src/backend/tests/llm/test_subagent_resume.py b/src/backend/tests/llm/test_subagent_resume.py index 630ff14c..1a5ef9b2 100644 --- a/src/backend/tests/llm/test_subagent_resume.py +++ b/src/backend/tests/llm/test_subagent_resume.py @@ -4,13 +4,9 @@ import pytest from core.infra.ephemeral import LocalEphemeralState -from core.llm import subagent_sessions -from core.llm.builtin_subagents import ( - BUILTIN_SUBAGENTS, - list_builtin_subagents, - load_builtin_subagent_prompt, -) -from core.llm.subagent_tool import build_subagent_prompt_section +from core.llm.subagents import sessions as subagent_sessions +from core.capabilities.agents.builtin import BUILTIN_SUBAGENTS, list_builtin_subagents, load_builtin_subagent_prompt +from core.llm.subagents.tool import build_subagent_prompt_section @pytest.fixture(autouse=True) diff --git a/src/backend/tests/llm/test_subagent_tool_streaming.py b/src/backend/tests/llm/test_subagent_tool_streaming.py index d6e940fc..6ce071dd 100644 --- a/src/backend/tests/llm/test_subagent_tool_streaming.py +++ b/src/backend/tests/llm/test_subagent_tool_streaming.py @@ -1,12 +1,12 @@ """Sub-agent sidebar events reuse the main streaming event mapper.""" +from core.llm.runtime import streaming_constants as streaming_state_module from types import SimpleNamespace +import core.llm.runtime.streaming as streaming_mod import pytest - -from core.llm.subagent_tool import _SubMapper, _map_subagent_event -from orchestration.streaming import StreamingAgent -import orchestration.streaming as streaming_mod +from core.llm.runtime.streaming import StreamingAgent +from core.llm.subagents.tool import _map_subagent_event, _SubMapper class TextBlockDeltaEvent: @@ -57,8 +57,8 @@ async def test_structured_subagent_content_streams_before_model_call_end(): @pytest.mark.asyncio async def test_subagent_tool_arguments_use_main_mapper(monkeypatch): - monkeypatch.setattr(streaming_mod, "_TOOL_CALL_DELTA_FLUSH_CHARS", 8) - monkeypatch.setattr(streaming_mod, "_TOOL_CALL_DELTA_FLUSH_INTERVAL_S", 999.0) + monkeypatch.setattr(streaming_state_module, 'TOOL_CALL_DELTA_FLUSH_CHARS', 8) + monkeypatch.setattr(streaming_state_module, 'TOOL_CALL_DELTA_FLUSH_INTERVAL_S', 999.0) stream = _stream(structured=True) mapper = _SubMapper() diff --git a/src/backend/tests/llm/test_system_reminder.py b/src/backend/tests/llm/test_system_reminder.py index c3275f45..803b40c9 100644 --- a/src/backend/tests/llm/test_system_reminder.py +++ b/src/backend/tests/llm/test_system_reminder.py @@ -2,12 +2,7 @@ import pytest -from core.llm.system_reminder import ( - REMINDER_CLOSE, - REMINDER_OPEN, - inject_reminder, - wrap_reminder, -) +from core.llm.context.system_reminder import REMINDER_CLOSE, REMINDER_OPEN, inject_reminder, wrap_reminder class _FakeState: diff --git a/src/backend/tests/llm/test_thinking_modes_across_protocols.py b/src/backend/tests/llm/test_thinking_modes_across_protocols.py index 76eb4149..695bff12 100644 --- a/src/backend/tests/llm/test_thinking_modes_across_protocols.py +++ b/src/backend/tests/llm/test_thinking_modes_across_protocols.py @@ -12,9 +12,9 @@ import pytest -from core.llm.chat_models import build_model_for_mode -from core.llm.responses_models import native_reasoning_effort -from core.services.model_config import ResolvedModelConfig +from core.models.chat_models import build_model_for_mode +from core.models.responses import native_reasoning_effort +from core.models.config import ResolvedModelConfig # build_model_for_mode 认识的全部档位，外加“没选档位”。 MODES = ["fast", "turbo", "medium", "high", "max", None] @@ -25,7 +25,7 @@ def _window(monkeypatch): """构造模型要查上下文窗口，这里与档位无关，给个定值即可。""" monkeypatch.setattr( - "core.llm.context_manager.resolve_model_context_window", lambda name: 65536 + 'core.models.context_window.resolve_model_context_window', lambda name: 65536 ) @@ -92,7 +92,7 @@ def test_a_value_the_upstream_would_reject_is_translated_not_sent(): def test_the_vocabulary_is_read_from_the_sdk_declaration(): from agentscope.model import OpenAIResponseModel - from core.llm.responses_models import _wire_reasoning_efforts + from core.models.responses import _wire_reasoning_efforts declared = get_args( OpenAIResponseModel.Parameters.model_fields["reasoning_effort"].annotation diff --git a/src/backend/tests/llm/test_tool_call_identity.py b/src/backend/tests/llm/test_tool_call_identity.py index c4279a7c..5751e3c2 100644 --- a/src/backend/tests/llm/test_tool_call_identity.py +++ b/src/backend/tests/llm/test_tool_call_identity.py @@ -8,7 +8,7 @@ from agentscope.message import ToolCallBlock from agentscope.model import ChatResponse -from core.llm.tool_call_identity import ToolCallIdentity, ToolCallIdentityMixin +from core.models.tool_call_identity import ToolCallIdentity, ToolCallIdentityMixin def _response(*calls, is_last=True): diff --git a/src/backend/tests/llm/test_tool_media_no_disk.py b/src/backend/tests/llm/test_tool_media_no_disk.py index 674d9d5e..25345a3b 100644 --- a/src/backend/tests/llm/test_tool_media_no_disk.py +++ b/src/backend/tests/llm/test_tool_media_no_disk.py @@ -31,10 +31,10 @@ ToolResultBlock, ) from core.chat.tool_log import attach_tool_result, upsert_tool_call -from core.llm.chat_models import ReasoningEchoChatFormatter -from core.llm.compacting_agent import msg_to_history_dict -from core.llm.responses_models import ResponsesReplayFormatter -from core.llm.providers import vendor_models as vm +from core.models.chat_models import ReasoningEchoChatFormatter +from core.llm.context.compacting_agent import msg_to_history_dict +from core.models.responses import ResponsesReplayFormatter +from core.models.providers import vendor_models as vm from PIL import Image @@ -213,7 +213,7 @@ def forbidden(*args, **kwargs): @pytest.mark.asyncio async def test_media_the_wire_cannot_carry_is_reported_not_silently_dropped(caplog): """线路收不下的媒体：如实回执给模型 + 记 error 日志，但不中断这一轮。""" - with caplog.at_level(logging.ERROR, logger="core.llm.tool_result_media"): + with caplog.at_level(logging.ERROR, logger="core.models.tool_result_media"): rows = await ReasoningEchoChatFormatter().format( [image_result("call-1", png_bytes(), media_type="video/mp4")] ) diff --git a/src/backend/tests/llm/test_tool_permission_hardening.py b/src/backend/tests/llm/test_tool_permission_hardening.py index ec2a4a37..4c62b536 100644 --- a/src/backend/tests/llm/test_tool_permission_hardening.py +++ b/src/backend/tests/llm/test_tool_permission_hardening.py @@ -12,16 +12,9 @@ import pytest from agentscope.message import ToolCallBlock from core.llm.factory.runtime.evidence import _default_allow_builtin_tools -from core.llm.tool_permissions import ( - DOMAIN_LOCAL_COMMAND, - DOMAIN_LOCAL_PATH, - DOMAIN_MYSPACE, - PermissionEnforcementError, - PermissionRuntime, - ToolPermissionRegistry, - ToolPermissionService, - builtin_tool_permission, -) +from core.sandbox.permissions.dispatch_models import DOMAIN_LOCAL_COMMAND, DOMAIN_LOCAL_PATH, DOMAIN_MYSPACE, PermissionEnforcementError, PermissionRuntime +from core.sandbox.permissions.tool_specs import ToolPermissionRegistry, builtin_tool_permission +from core.llm.tool_permissions import ToolPermissionService def _runtime(**overrides) -> PermissionRuntime: @@ -187,9 +180,9 @@ async def test_trusted_unattended_run_still_obeys_a_policy_denial(): with ( patch("core.config.local_mode.local_mode_enabled", return_value=True), - patch("core.services.local_grant_service.grants_for_gate", return_value=[]), + patch('features.desktop.local.grants.grants_for_gate', return_value=[]), patch( - "core.services.local_grant_service.policy_for_gate", + 'features.desktop.local.grants.policy_for_gate', return_value=Policy(out_of_scope="block"), ), ): diff --git a/src/backend/tests/llm/test_tool_permissions.py b/src/backend/tests/llm/test_tool_permissions.py index 524532f2..38d1af2f 100644 --- a/src/backend/tests/llm/test_tool_permissions.py +++ b/src/backend/tests/llm/test_tool_permissions.py @@ -1,6 +1,8 @@ """Declarative tool-permission registry and pre-dispatch enforcement.""" from __future__ import annotations +from tests.orchestration.hook_support import permission_adapter +from types import SimpleNamespace import json from types import SimpleNamespace @@ -10,18 +12,11 @@ from agentscope.message import ToolCallBlock, ToolResultState from agentscope.state import AgentState from agentscope.tool._response import ToolChunk, ToolResponse -from core.llm.tool_collector import ToolCollector -from core.llm.tool_permissions import ( - CURRENT_PERMISSION_TICKET, - PermissionRuntime, - ToolPermissionMiddleware, - ToolPermissionRegistry, - ToolPermissionService, - allow_tool, - local_command_tool, - local_path_tool, -) -from core.ontology.toolkit import OntologyFilteredToolkit +from core.llm.factory.tools.definitions import ToolAssembly, function_tool +from core.sandbox.permissions.dispatch_models import CURRENT_PERMISSION_TICKET, PermissionRuntime +from core.sandbox.permissions.tool_specs import ToolPermissionRegistry, allow_tool, local_command_tool, local_path_tool +from core.llm.tool_permissions import ToolPermissionService +from core.llm.tools.ontology_toolkit import OntologyFilteredToolkit from core.sandbox.permissions.models import Grant from core.sandbox.permissions.models import Policy @@ -53,11 +48,11 @@ def test_native_tool_registration_does_not_require_a_declaration(): async def brand_new_tool() -> ToolResponse: return ToolResponse() - collector = ToolCollector() - collector.register_tool_function(brand_new_tool) + collector = ToolAssembly() + collector.append(function_tool(brand_new_tool)) assert "brand_new_tool" not in collector.permission_specs - collector.register_tool_function(brand_new_tool, permission=allow_tool()) + collector.append(function_tool(brand_new_tool, dispatch_spec=allow_tool())) assert collector.permission_specs["brand_new_tool"].key == "allow" @@ -84,7 +79,7 @@ async def test_unregistered_tool_passes_without_a_ticket(): @pytest.mark.asyncio async def test_middleware_dispatches_unregistered_tool_without_binding_ticket(): - middleware = ToolPermissionMiddleware( + middleware = permission_adapter( ToolPermissionService( ToolPermissionRegistry(), PermissionRuntime( @@ -151,11 +146,11 @@ async def test_local_path_decision_is_resolved_once_by_generic_service(): with ( patch("core.config.local_mode.local_mode_enabled", return_value=True), patch( - "core.services.local_grant_service.grants_for_gate", + 'features.desktop.local.grants.grants_for_gate', return_value=[Grant("/data/project", "read")], ), patch( - "core.services.local_grant_service.policy_for_gate", + 'features.desktop.local.grants.policy_for_gate', return_value=Policy(), ), ): @@ -170,7 +165,7 @@ async def test_local_path_decision_is_resolved_once_by_generic_service(): async def test_middleware_binds_ticket_only_around_matching_dispatch(): registry = ToolPermissionRegistry() registry.register("safe", allow_tool(), source="native") - middleware = ToolPermissionMiddleware( + middleware = permission_adapter( ToolPermissionService( registry, PermissionRuntime( @@ -240,9 +235,9 @@ async def brand_new_tool(value: int) -> ToolChunk: state=ToolResultState.SUCCESS, ) - collector = ToolCollector() - collector.register_tool_function(brand_new_tool) - toolkit = OntologyFilteredToolkit(tools=collector.function_tools) + collector = ToolAssembly() + collector.append(function_tool(brand_new_tool)) + toolkit = OntologyFilteredToolkit(tools=list(collector)) toolkit.set_tool_permission_service( ToolPermissionService( ToolPermissionRegistry(), @@ -283,11 +278,11 @@ async def test_local_command_ticket_carries_os_sandbox_constraints(): with ( patch("core.config.local_mode.local_mode_enabled", return_value=True), patch( - "core.services.local_grant_service.grants_for_gate", + 'features.desktop.local.grants.grants_for_gate', return_value=[Grant("/data/project", "readwrite")], ), patch( - "core.services.local_grant_service.policy_for_gate", + 'features.desktop.local.grants.policy_for_gate', return_value=Policy(), ), ): diff --git a/src/backend/tests/llm/test_tool_result_images.py b/src/backend/tests/llm/test_tool_result_images.py index 12c60a7a..c2eaeee3 100644 --- a/src/backend/tests/llm/test_tool_result_images.py +++ b/src/backend/tests/llm/test_tool_result_images.py @@ -21,11 +21,11 @@ from agentscope.model import ChatResponse from agentscope.tool import Toolkit from agentscope.tool._response import ToolChunk -from core.llm.chat_models import OpenAICompatChatModel, ReasoningEchoChatFormatter -from core.llm.compacting_agent import CompactingAgent -from core.llm.context_ir import IMAGE_TOKEN_RESERVE -from core.llm.offloader import SandboxOffloader -from core.llm.tool_collector import ToolCollector +from core.models.chat_models import OpenAICompatChatModel, ReasoningEchoChatFormatter +from core.llm.context.compacting_agent import CompactingAgent +from core.models.constants import IMAGE_TOKEN_RESERVE +from core.llm.context.offloader import SandboxOffloader +from core.llm.factory.tools.definitions import ToolAssembly, function_tool from core.llm.tools.read_image_tool import register_read_image from PIL import Image @@ -109,9 +109,9 @@ async def read_bytes(*args, **kwargs): return large_png monkeypatch.setattr("core.llm.tools.read_image_tool._read_path_bytes", read_bytes) - collector = ToolCollector() + collector = ToolAssembly() register_read_image(collector, chat_id="shared-workspace", user_id="u1", vision_mode="native") - toolkit = Toolkit(tools=collector.function_tools) + toolkit = Toolkit(tools=list(collector)) messages, files = await run_tool(toolkit, CaptureModel(), streaming=streaming) images = wire_images(messages) assert len(images) == 1, json.dumps(messages)[:2000] @@ -137,10 +137,10 @@ async def screenshots(): """Read screenshots with optional diagnostic output.""" return ToolChunk(content=[image, TextBlock(text=text), remote] if text else [image, remote]) - collector = ToolCollector() - collector.register_tool_function(screenshots) + collector = ToolAssembly() + collector.append(function_tool(screenshots)) messages, files = await run_tool( - Toolkit(tools=collector.function_tools), CaptureModel("screenshots", {}) + Toolkit(tools=list(collector)), CaptureModel("screenshots", {}) ) assert wire_images(messages) == [ "data:image/png;base64," + encoded, @@ -165,10 +165,10 @@ async def logs(): """Return large plain text output.""" return ToolChunk(content=[TextBlock(text=text)]) - collector = ToolCollector() - collector.register_tool_function(logs) + collector = ToolAssembly() + collector.append(function_tool(logs)) messages, files = await run_tool( - Toolkit(tools=collector.function_tools), CaptureModel("logs", {}) + Toolkit(tools=list(collector)), CaptureModel("logs", {}) ) assert wire_images(messages) == [] assert len(files) == 1 @@ -207,15 +207,15 @@ async def read_bytes(*args, **kwargs): "settings", replace(settings, compaction=replace(settings.compaction, enabled=compaction_enabled)), ) - monkeypatch.setattr("core.services.compaction_summary._summarize", unexpected_summary) + monkeypatch.setattr('core.chat.compaction.summary._summarize', unexpected_summary) monkeypatch.setattr("core.llm.tools.read_image_tool._read_path_bytes", read_bytes) model = CaptureModel() monkeypatch.setattr(model, "generate_structured_output", unexpected_summary) - collector = ToolCollector() + collector = ToolAssembly() register_read_image(collector, chat_id="shared-workspace", user_id="u1", vision_mode="native") messages, files = await run_tool( - Toolkit(tools=collector.function_tools), model, streaming=streaming + Toolkit(tools=list(collector)), model, streaming=streaming ) assert attempts == [] @@ -226,8 +226,8 @@ async def read_bytes(*args, **kwargs): def test_history_estimate_and_summary_do_not_treat_image_as_text(): - from core.llm import compaction as C - from core.services.compaction_budget import estimate_history_tokens + from core.llm.context import compaction as C + from core.llm.context.budget import estimate_history_tokens encoded = "A" * 800_000 history = [ @@ -281,10 +281,10 @@ def jx_chat_model_classes() -> list[type]: from agentscope.model import ChatModelBase for module in ( - "core.llm.chat_models", - "core.llm.failover", - "core.llm.providers.vendor_models", - "core.llm.responses_models", + "core.models.chat_models", + "core.models.failover", + "core.models.providers.vendor_models", + "core.models.responses", ): importlib.import_module(module) @@ -292,7 +292,7 @@ def jx_chat_model_classes() -> list[type]: def walk(cls: type) -> None: for sub in cls.__subclasses__(): - if sub.__module__.startswith("core.llm") and sub not in seen: + if sub.__module__.startswith("core.models") and sub not in seen: seen.append(sub) walk(sub) diff --git a/src/backend/tests/llm/test_vendor_timeouts.py b/src/backend/tests/llm/test_vendor_timeouts.py index 6bd98f2e..54410a86 100644 --- a/src/backend/tests/llm/test_vendor_timeouts.py +++ b/src/backend/tests/llm/test_vendor_timeouts.py @@ -2,8 +2,8 @@ import asyncio import pytest -from core.llm.providers.registry import get_spec -from core.llm.providers.vendor_models import build_native_model, build_litellm_model +from core.models.providers.registry import get_spec +from core.models.providers.vendor_models import build_native_model, build_litellm_model from agentscope.model import ( AnthropicChatModel, GeminiChatModel, diff --git a/src/backend/tests/llm/test_vision_bridge.py b/src/backend/tests/llm/test_vision_bridge.py index 9d6acaf4..cc647a1c 100644 --- a/src/backend/tests/llm/test_vision_bridge.py +++ b/src/backend/tests/llm/test_vision_bridge.py @@ -12,9 +12,9 @@ import pytest -from core.vision.render import render_evidence, render_many, render_unavailable -from core.vision.schema import VISION_JSON_SCHEMA, VisionEvidence, parse_evidence -from core.vision.service import sniff_mime +from core.models.vision.render import render_evidence, render_many, render_unavailable +from core.models.vision.schema import VISION_JSON_SCHEMA, VisionEvidence, parse_evidence +from core.models.vision.service import sniff_mime # 一张 1x1 的合法 PNG PNG_1X1 = base64.b64decode( @@ -158,8 +158,8 @@ class _FakeCfg: def _bridge_with(monkeypatch, responses: list[str]): """把 provider 换成按序返回给定文本的假实现，返回 (bridge, 调用计数器)。""" - from core.vision import service as svc - from core.vision.provider import VisionCallResult + from core.models.vision import service as svc + from core.models.vision.provider import VisionCallResult calls = {"n": 0} @@ -210,7 +210,7 @@ def test_describe_rejects_non_image_without_calling_model(monkeypatch): def test_describe_rejects_oversized_image(monkeypatch): - from core.vision import service as svc + from core.models.vision import service as svc bridge, calls = _bridge_with(monkeypatch, [json.dumps(GOOD_PAYLOAD)]) monkeypatch.setattr(svc, "MAX_IMAGE_BYTES", 10) @@ -219,7 +219,7 @@ def test_describe_rejects_oversized_image(monkeypatch): def test_describe_returns_none_when_unconfigured(monkeypatch): - from core.vision import service as svc + from core.models.vision import service as svc monkeypatch.setattr(svc, "resolve_vision_config", lambda: None) bridge = svc.VisionBridge() @@ -235,7 +235,7 @@ def test_describe_many_is_concurrent_and_order_preserving(monkeypatch): def test_cache_key_changes_with_model_and_focus(): - from core.vision.service import VisionBridge + from core.models.vision.service import VisionBridge class _Other(_FakeCfg): model_name = "another-vl" @@ -250,7 +250,7 @@ class _Other(_FakeCfg): def test_model_supports_vision_reads_explicit_flag(): - from core.vision import model_supports_vision + from core.models.vision import model_supports_vision class _Cfg(_FakeCfg): extra = {"supports_vision": True} @@ -261,7 +261,7 @@ class _Cfg(_FakeCfg): def test_vision_role_is_registered(): - from core.db.model_repository import ROLE_DEFINITIONS + from core.models.repository import ROLE_DEFINITIONS assert ROLE_DEFINITIONS["vision"]["type"] == "chat" diff --git a/src/backend/tests/llm/test_vision_bridge_e2e.py b/src/backend/tests/llm/test_vision_bridge_e2e.py index d6c3072b..d2e6e06e 100644 --- a/src/backend/tests/llm/test_vision_bridge_e2e.py +++ b/src/backend/tests/llm/test_vision_bridge_e2e.py @@ -152,8 +152,8 @@ def stub_endpoint(): @pytest.fixture() def bridge_against_stub(stub_endpoint, monkeypatch): - from core.services.model_config import ResolvedModelConfig - from core.vision import service as svc + from core.models.config import ResolvedModelConfig + from core.models.vision import service as svc cfg = ResolvedModelConfig( base_url=stub_endpoint, @@ -169,7 +169,7 @@ def bridge_against_stub(stub_endpoint, monkeypatch): def test_full_chain_over_real_http(bridge_against_stub): """图片经真实 HTTP 抵达对端、被解码、结果回来渲染成带围栏的证据。""" - from core.vision.render import render_evidence + from core.models.vision.render import render_evidence image = _make_image(width=320, height=180) result = asyncio.run(bridge_against_stub.describe(image, "image/png", use_cache=False)) diff --git a/src/backend/tests/llm/test_vision_injection.py b/src/backend/tests/llm/test_vision_injection.py index 08d591b5..3c9f3000 100644 --- a/src/backend/tests/llm/test_vision_injection.py +++ b/src/backend/tests/llm/test_vision_injection.py @@ -5,6 +5,7 @@ """ from __future__ import annotations +from core.files import artifact_bytes as shared_artifact_bytes import asyncio import base64 @@ -57,7 +58,7 @@ def _no_evidence_cache(monkeypatch): 跨测试文件互相串结果（缓存本身的行为由 test_vision_bridge_e2e 覆盖）。这里整体 关掉，让每个用例都真的走一次识别。 """ - from core.vision.service import VisionBridge + from core.models.vision.service import VisionBridge async def _miss(_key): return None @@ -72,8 +73,8 @@ async def _skip(_key, _result): @pytest.fixture() def working_bridge(monkeypatch): """让视觉桥可用，并总是返回一份固定证据。""" - from core.vision import service as svc - from core.vision.provider import VisionCallResult + from core.models.vision import service as svc + from core.models.vision.provider import VisionCallResult class _FakeProvider: def __init__(self, cfg): @@ -91,21 +92,19 @@ async def describe(self, *a, **kw): @pytest.fixture() def artifact_bytes(monkeypatch): - # 附件字节的读取已收归 core.vision.attachments，它在调用时才 from core.llm.hooks - # 导入，所以补丁要打在 hooks 上（打在 middlewares 上从此无效）。 - from core.llm import hooks + # 附件字节读取由共享文件模块负责，在该模块替换测试输入。 - monkeypatch.setattr(hooks, "_download_artifact_bytes", lambda *a, **kw: PNG_1X1) + monkeypatch.setattr(shared_artifact_bytes, 'download_artifact_bytes', lambda *a, **kw: PNG_1X1) # ── 主路径：纯文本模型 → 注入转写证据 ─────────────────────────────────────── def test_uploaded_image_becomes_text_evidence(working_bridge, artifact_bytes): - from core.llm.middlewares import FileContextMiddleware + from tests.orchestration.hook_support import inject_vision_context st = _stub_state() - asyncio.run(FileContextMiddleware._inject_vision_evidence(st, IMAGE_FILES, "u1")) + asyncio.run(inject_vision_context(st, IMAGE_FILES, "u1")) assert len(st.context) == 1 msg = st.context[0] @@ -121,12 +120,12 @@ def test_uploaded_image_becomes_text_evidence(working_bridge, artifact_bytes): def test_bridge_unavailable_says_so_instead_of_silently_dropping(monkeypatch, artifact_bytes): - from core.llm.middlewares import FileContextMiddleware - from core.vision import service as svc + from tests.orchestration.hook_support import inject_vision_context + from core.models.vision import service as svc monkeypatch.setattr(svc, "resolve_vision_config", lambda: None) st = _stub_state() - asyncio.run(FileContextMiddleware._inject_vision_evidence(st, IMAGE_FILES, "u1")) + asyncio.run(inject_vision_context(st, IMAGE_FILES, "u1")) text = st.context[0].content[0].text assert "不支持直接读图" in text @@ -135,18 +134,17 @@ def test_bridge_unavailable_says_so_instead_of_silently_dropping(monkeypatch, ar def test_all_reads_failing_is_reported(monkeypatch, working_bridge): """桥配了但每张图都识别失败时要明说，不能让模型当没收到图。""" - from core.llm import hooks - from core.llm.middlewares import FileContextMiddleware - from core.vision import service as svc + from tests.orchestration.hook_support import inject_vision_context + from core.models.vision import service as svc - monkeypatch.setattr(hooks, "_download_artifact_bytes", lambda *a, **kw: PNG_1X1) + monkeypatch.setattr(shared_artifact_bytes, 'download_artifact_bytes', lambda *a, **kw: PNG_1X1) class _DeadProvider: def __init__(self, cfg): pass async def describe(self, *a, **kw): - from core.vision.provider import VisionCallResult + from core.models.vision.provider import VisionCallResult return VisionCallResult(error="boom") @@ -154,18 +152,17 @@ async def describe(self, *a, **kw): svc.VisionBridge._instance = None st = _stub_state() - asyncio.run(FileContextMiddleware._inject_vision_evidence(st, IMAGE_FILES, "u1")) + asyncio.run(inject_vision_context(st, IMAGE_FILES, "u1")) text = st.context[0].content[0].text assert "视觉模型调用失败" in text def test_unreadable_attachment_injects_nothing(monkeypatch, working_bridge): - from core.llm import hooks - from core.llm.middlewares import FileContextMiddleware + from tests.orchestration.hook_support import inject_vision_context - monkeypatch.setattr(hooks, "_download_artifact_bytes", lambda *a, **kw: None) + monkeypatch.setattr(shared_artifact_bytes, 'download_artifact_bytes', lambda *a, **kw: None) st = _stub_state() - asyncio.run(FileContextMiddleware._inject_vision_evidence(st, IMAGE_FILES, "u1")) + asyncio.run(inject_vision_context(st, IMAGE_FILES, "u1")) assert st.context == [] @@ -173,8 +170,8 @@ def test_unreadable_attachment_injects_nothing(monkeypatch, working_bridge): def test_effective_capability_prefers_per_request_provider(monkeypatch): - from core.llm.middlewares import _effective_model_supports_vision - from core.services.model_config import ModelConfigService + from core.llm.context.attachment_hook import supports_native_images + from core.models.config import ModelConfigService class _Service: @staticmethod @@ -188,21 +185,21 @@ def resolve_provider(pid): monkeypatch.setattr(ModelConfigService, "get_instance", staticmethod(lambda: _Service())) # 用户本轮切到了一个多模态模型 → 图片直通，不走桥 - assert _effective_model_supports_vision(SimpleNamespace(model_provider_id="vl")) is True + assert supports_native_images("vl") is True # 没切模型 → 看 main_agent，它是纯文本 → 走桥 - assert _effective_model_supports_vision(SimpleNamespace(model_provider_id="")) is False + assert supports_native_images("") is False def test_capability_probe_failure_falls_back_to_bridge(monkeypatch): """探测能力位本身出错时，按「看不见」处理——宁可多转写一次，不可让图静默消失。""" - from core.llm.middlewares import _effective_model_supports_vision - from core.services.model_config import ModelConfigService + from core.llm.context.attachment_hook import supports_native_images + from core.models.config import ModelConfigService def _boom(): raise RuntimeError("db down") monkeypatch.setattr(ModelConfigService, "get_instance", staticmethod(_boom)) - assert _effective_model_supports_vision(SimpleNamespace(model_provider_id="")) is False + assert supports_native_images("") is False # ── read_tool 的图片分支 ──────────────────────────────────────────────────── @@ -227,7 +224,7 @@ def test_read_tool_leaves_non_images_alone(working_bridge): def test_read_tool_falls_through_when_vision_unconfigured(monkeypatch): from core.llm.tools.read_tool import _read_image_as_evidence - from core.vision import service as svc + from core.models.vision import service as svc monkeypatch.setattr(svc, "resolve_vision_config", lambda: None) assert asyncio.run(_read_image_as_evidence(PNG_1X1, "/workspace/shot.png")) is None @@ -244,8 +241,8 @@ def _register_bridge_read_image(): class _Toolkit: @staticmethod - def register_tool_function(fn, **kw): - captured["fn"] = fn + def append(tool): + captured["fn"] = tool._func register_read_image(_Toolkit(), chat_id="c1", user_id="u1", vision_mode="bridge") return captured["fn"] @@ -256,9 +253,8 @@ def _tool_json(response): def test_bridge_read_image_reads_by_file_id(working_bridge, monkeypatch): - from core.llm import hooks - monkeypatch.setattr(hooks, "_download_artifact_bytes", lambda *a, **kw: PNG_1X1) + monkeypatch.setattr(shared_artifact_bytes, 'download_artifact_bytes', lambda *a, **kw: PNG_1X1) fn = _register_bridge_read_image() payload = _tool_json(asyncio.run(fn(file_id="art-1", focus="发票号是多少"))) assert payload["type"] == "image_evidence" @@ -273,19 +269,17 @@ def test_bridge_read_image_requires_a_target(working_bridge): def test_bridge_read_image_rejects_non_image_bytes(working_bridge, monkeypatch): - from core.llm import hooks - monkeypatch.setattr(hooks, "_download_artifact_bytes", lambda *a, **kw: b"PK\x03\x04zip") + monkeypatch.setattr(shared_artifact_bytes, 'download_artifact_bytes', lambda *a, **kw: b"PK\x03\x04zip") fn = _register_bridge_read_image() payload = _tool_json(asyncio.run(fn(file_id="art-1"))) assert "不是可识别的图片格式" in payload["error"] def test_bridge_read_image_reports_missing_vision_config(monkeypatch): - from core.llm import hooks - from core.vision import service as svc + from core.models.vision import service as svc - monkeypatch.setattr(hooks, "_download_artifact_bytes", lambda *a, **kw: PNG_1X1) + monkeypatch.setattr(shared_artifact_bytes, 'download_artifact_bytes', lambda *a, **kw: PNG_1X1) monkeypatch.setattr(svc, "resolve_vision_config", lambda: None) fn = _register_bridge_read_image() payload = _tool_json(asyncio.run(fn(file_id="art-1"))) @@ -294,8 +288,8 @@ def test_bridge_read_image_reports_missing_vision_config(monkeypatch): def test_native_multimodal_model_gets_native_read_image(monkeypatch): """主模型自己能看图时注册原生版——经二道手只会掉精度。""" - from core.services.model_config import ModelConfigService - from core.vision import resolve_vision_mode + from core.models.config import ModelConfigService + from core.models.vision import resolve_vision_mode class _Service: @staticmethod diff --git a/src/backend/tests/llm/test_vision_progress_events.py b/src/backend/tests/llm/test_vision_progress_events.py index 015457f2..fa20aabc 100644 --- a/src/backend/tests/llm/test_vision_progress_events.py +++ b/src/backend/tests/llm/test_vision_progress_events.py @@ -9,6 +9,7 @@ """ from __future__ import annotations +from core.files import artifact_bytes as shared_artifact_bytes import asyncio import base64 @@ -64,8 +65,8 @@ def _state(**kw): @pytest.fixture() def working_bridge(monkeypatch): - from core.vision import service as svc - from core.vision.provider import VisionCallResult + from core.models.vision import service as svc + from core.models.vision.provider import VisionCallResult class _FakeProvider: def __init__(self, cfg): @@ -80,9 +81,8 @@ async def describe(self, *a, **kw): monkeypatch.setattr(svc.VisionBridge, "_cache_set", staticmethod(lambda k, r: _none())) svc.VisionBridge._instance = None - from core.llm import hooks - monkeypatch.setattr(hooks, "_download_artifact_bytes", lambda *a, **kw: PNG_1X1) + monkeypatch.setattr(shared_artifact_bytes, 'download_artifact_bytes', lambda *a, **kw: PNG_1X1) yield svc.VisionBridge._instance = None @@ -93,7 +93,7 @@ async def _none(): def _collect(state): """跑一遍 StreamingAgent 的识图前置，收集它 yield 出的事件。""" - from orchestration.streaming import StreamingAgent + from core.llm.runtime.streaming import StreamingAgent agent = SimpleNamespace(state=state) streaming = StreamingAgent.__new__(StreamingAgent) @@ -109,9 +109,9 @@ async def run(): def test_emits_running_then_done_around_the_wait(working_bridge, monkeypatch): - from core.llm import middlewares + from core.llm.context import attachment_hook - monkeypatch.setattr(middlewares, "_effective_model_supports_vision", lambda st: False) + monkeypatch.setattr(attachment_hook, "supports_native_images", lambda st: False) state = _state() events = _collect(state) @@ -126,9 +126,9 @@ def test_emits_running_then_done_around_the_wait(working_bridge, monkeypatch): def test_done_reports_failure_without_pretending_success(monkeypatch): """桥配了但读失败时，done 必须 ok=False —— 前端据此决定要不要提示。""" - from core.llm import hooks, middlewares - from core.vision import service as svc - from core.vision.provider import VisionCallResult + from core.llm.context import attachment_hook + from core.models.vision import service as svc + from core.models.vision.provider import VisionCallResult class _DeadProvider: def __init__(self, cfg): @@ -142,8 +142,8 @@ async def describe(self, *a, **kw): monkeypatch.setattr(svc.VisionBridge, "_cache_get", staticmethod(lambda k: _none())) monkeypatch.setattr(svc.VisionBridge, "_cache_set", staticmethod(lambda k, r: _none())) svc.VisionBridge._instance = None - monkeypatch.setattr(hooks, "_download_artifact_bytes", lambda *a, **kw: PNG_1X1) - monkeypatch.setattr(middlewares, "_effective_model_supports_vision", lambda st: False) + monkeypatch.setattr(shared_artifact_bytes, 'download_artifact_bytes', lambda *a, **kw: PNG_1X1) + monkeypatch.setattr(attachment_hook, "supports_native_images", lambda st: False) events = _collect(_state()) svc.VisionBridge._instance = None @@ -154,17 +154,17 @@ async def describe(self, *a, **kw): def test_silent_when_turn_has_no_images(working_bridge, monkeypatch): - from core.llm import middlewares + from core.llm.context import attachment_hook - monkeypatch.setattr(middlewares, "_effective_model_supports_vision", lambda st: False) + monkeypatch.setattr(attachment_hook, "supports_native_images", lambda st: False) assert _collect(_state(uploaded_files=[{"name": "a.txt", "mime_type": "text/plain"}])) == [] def test_silent_for_natively_multimodal_model(working_bridge, monkeypatch): """主模型自己能看图时压根不转写，也就没有「图像理解中」这回事。""" - from core.llm import middlewares + from core.llm.context import attachment_hook - monkeypatch.setattr(middlewares, "_effective_model_supports_vision", lambda st: True) + monkeypatch.setattr(attachment_hook, "supports_native_images", lambda st: True) state = _state() assert _collect(state) == [] assert state.vision_evidence_text == "" @@ -172,12 +172,12 @@ def test_silent_for_natively_multimodal_model(working_bridge, monkeypatch): def test_failure_in_pre_pass_still_closes_the_status(monkeypatch): """前置整体抛错也必须补一个 done，否则前端会永远停在「图像理解中」。""" - from core.llm import middlewares + from core.llm.context import attachment_hook def _boom(_st): raise RuntimeError("db down") - monkeypatch.setattr(middlewares, "_effective_model_supports_vision", _boom) + monkeypatch.setattr(attachment_hook, "supports_native_images", _boom) events = _collect(_state()) assert len(events) == 1 assert events[0][1]["status"] == "done" and events[0][1]["ok"] is False @@ -187,8 +187,8 @@ def _boom(_st): def test_middleware_uses_precomputed_text_without_calling_the_model(monkeypatch): - from core.llm.middlewares import FileContextMiddleware - from core.vision import service as svc + from tests.orchestration.hook_support import inject_vision_context + from core.models.vision import service as svc calls = {"n": 0} @@ -197,7 +197,7 @@ def __init__(self, cfg): calls["n"] += 1 async def describe(self, *a, **kw): - from core.vision.provider import VisionCallResult + from core.models.vision.provider import VisionCallResult return VisionCallResult(text=EVIDENCE_JSON, mode="json_object") @@ -206,7 +206,7 @@ async def describe(self, *a, **kw): svc.VisionBridge._instance = None state = _state(vision_evidence_text="预先转写好的") - asyncio.run(FileContextMiddleware._inject_vision_evidence(state, IMAGE_FILES, "u1")) + asyncio.run(inject_vision_context(state, IMAGE_FILES, "u1")) svc.VisionBridge._instance = None assert calls["n"] == 0, "已有预转写结果却又打了一次视觉模型" diff --git a/src/backend/tests/llm/test_vision_role_capability.py b/src/backend/tests/llm/test_vision_role_capability.py index 2115e9f5..17bd37a8 100644 --- a/src/backend/tests/llm/test_vision_role_capability.py +++ b/src/backend/tests/llm/test_vision_role_capability.py @@ -12,12 +12,7 @@ import pytest -from core.db.model_repository import ( - ROLE_DEFINITIONS, - list_role_assignments, - provider_has_capability, - role_required_capability, -) +from core.models.repository import ROLE_DEFINITIONS, list_role_assignments, provider_has_capability, role_required_capability def _provider(**extra_config): diff --git a/src/backend/tests/llm/test_workspace.py b/src/backend/tests/llm/test_workspace.py index 8e05036a..6cdc5141 100644 --- a/src/backend/tests/llm/test_workspace.py +++ b/src/backend/tests/llm/test_workspace.py @@ -2,7 +2,7 @@ import asyncio import contextvars -from core.llm import workspace +from core.llm.runtime import workspace def test_initial_state_is_empty_and_inactive(): @@ -77,7 +77,7 @@ def test_pin_to_workspace_tool_accepts_list(monkeypatch): import json as _json from agentscope.tool import Toolkit - from core.llm.tool_collector import ToolCollector + from core.llm.factory.tools.definitions import ToolAssembly from core.llm.tools.pin_tool import register_pin_to_workspace # Stub the artifact lookup so we don't need a real artifact store. @@ -89,13 +89,12 @@ def test_pin_to_workspace_tool_accepts_list(monkeypatch): def fake_get_artifact(fid): return fake_store.get(fid) - monkeypatch.setattr("core.artifacts.store.get_artifact", fake_get_artifact) + monkeypatch.setattr('core.files.artifacts.store.get_artifact', fake_get_artifact) workspace.init_state() - # AgentScope 2.0: register_* writes into the ToolCollector; schema goes through Toolkit.get_tool_schemas(). - c = ToolCollector() + c = ToolAssembly() register_pin_to_workspace(c) - tk = Toolkit(tools=c.function_tools) + tk = Toolkit(tools=list(c)) schemas = asyncio.run(tk.get_tool_schemas()) pin_schema = next(s for s in schemas if s["function"]["name"] == "pin_to_workspace") # Schema must declare file_ids as an array of strings (not a string). @@ -103,7 +102,7 @@ def fake_get_artifact(fid): assert fid_param["type"] == "array" assert fid_param["items"]["type"] == "string" - fn = c.get_tool("pin_to_workspace")._func + fn = next(t for t in c if t.name == "pin_to_workspace")._func resp = asyncio.run(fn(file_ids=["fid_a", "fid_b"])) payload = _json.loads(resp.content[0].text) # 2.0: block is a pydantic object diff --git a/src/backend/tests/llm/test_workspace_profiles.py b/src/backend/tests/llm/test_workspace_profiles.py index 71ef4240..fc2f0184 100644 --- a/src/backend/tests/llm/test_workspace_profiles.py +++ b/src/backend/tests/llm/test_workspace_profiles.py @@ -1,3 +1,5 @@ + +import core.jobs.files as jobs_files from tests.sandbox.runner_client import run_runner """Path identities must agree with the selected execution environment.""" @@ -9,7 +11,7 @@ import pytest from core.config import local_mode from core.config.settings import settings -from core.llm.tools import _paths +from core.sandbox.workspace import resolution as _paths @pytest.mark.parametrize("provider", ["cube", "opensandbox"]) @@ -55,7 +57,9 @@ async def test_skill_runtime_hint_runs_in_current_directory_with_spaces(monkeypa (skill / "run.py").write_text("from pathlib import Path; Path('output.txt').write_text('ok')") class Toolkit: - def register_tool_function(self, fn, **kwargs): + def append(self, tool): + fn = tool._func + kwargs = {"func_description": tool.description, "permission": getattr(tool, "dispatch_spec", None)} self.fn = fn loader = SimpleNamespace( @@ -122,12 +126,6 @@ async def put_file(self, session_id, path, content, user_id=None): ) ) - async def get_file_to_path(self, session_id, path, destination, **kwargs): - result = await server.get_file(server.GetFileRequest(session_id=session_id, path=path)) - data = base64.b64decode(result.content_b64) - destination.write_bytes(data) - return len(data) - monkeypatch.setattr("core.sandbox.get_sandbox_provider", lambda: Provider()) monkeypatch.setattr( "core.llm.tools.sandbox_artifact_tools._resolve_artifact_files", @@ -136,32 +134,47 @@ async def get_file_to_path(self, session_id, path, destination, **kwargs): None, ), ) - captured = [] - - def store(path, **kwargs): - captured.append(path.read_bytes()) - return dict( - file_id="result", - name="output.txt", - url="/files/result", - mime_type="text/plain", - size=10, - ) - - monkeypatch.setattr("core.llm.tools.sandbox_artifact_tools._store_generated_file_path", store) - - class Toolkit: - def register_tool_function(self, fn, **kwargs): - self.fn = fn - - put, get = Toolkit(), Toolkit() - sandbox_tool.register_sandbox_put_artifact(put, chat_id="chat", user_id="owner") - sandbox_tool.register_sandbox_get_artifact(get, chat_id="chat", user_id="owner") + from types import SimpleNamespace + from core.llm.factory.tools.definitions import ToolAssembly + from core.llm.tool_permissions import ToolPermissionService + from core.sandbox.permissions.tool_specs import ToolPermissionRegistry + from core.sandbox.permissions.dispatch_models import ( + CURRENT_PERMISSION_TICKET, PermissionRuntime, PermissionEnforcementError, + ) + from core.files.artifacts import store + from core.files.artifacts.local_reference import authorized_reference + from features.desktop.local.grants import add_grant + + monkeypatch.setenv("HUGAGENT_HOME", str(tmp_path / "home")) + monkeypatch.setattr(store, "_STORE_DIR", tmp_path / "artifacts") + root = Path(_paths.workspace_directory("chat")) + root.mkdir(parents=True) + add_grant(str(root)) + tools = ToolAssembly() + sandbox_tool.register_sandbox_put_artifact(tools, chat_id="chat", user_id="owner") + sandbox_tool.register_sandbox_get_artifact(tools, chat_id="chat", user_id="owner") + registry = ToolPermissionRegistry() + for name, spec in tools.permission_specs.items(): + registry.register(name, spec, source="test") + permissions = ToolPermissionService(registry, PermissionRuntime( + chat_id="chat", user_id="owner", interactive=False, approval_available=False, + )) + functions = {tool.name: tool._func for tool in tools} + + async def invoke(name, **args): + outcome = await permissions.authorize(SimpleNamespace(name=name, id="artifact-call", input=args)) + if not outcome.proceed: + raise PermissionError(str(outcome.payload)) + token = CURRENT_PERMISSION_TICKET.set(outcome.ticket) + try: + return await functions[name](**args) + finally: + CURRENT_PERMISSION_TICKET.reset(token) def payload(result): return json.loads(result.content[0].text) - assert payload(await put.fn("upload", "input.txt"))["ok"] + assert payload(await invoke("sandbox_put_artifact", artifact_id="upload", dest_path="input.txt"))["ok"] result = await run_runner( server.ProcessRequest( session_id="chat", @@ -171,9 +184,25 @@ def payload(result): ) ) assert result.exit_code == 0, result.stderr - assert payload(await get.fn("output.txt"))["ok"] - assert captured == [b"input-data"] - assert "error" in payload(await get.fn("../outside.txt")) + with pytest.raises(PermissionEnforcementError): + await functions["sandbox_get_artifact"]("output.txt") + result = payload(await invoke("sandbox_get_artifact", src_path="output.txt")) + assert result["ok"] + reference = authorized_reference(result["file_id"], "owner") + assert reference["path"] == str(root / "output.txt") + assert Path(reference["path"]).read_bytes() == b"input-data" + assert authorized_reference(result["file_id"], "other-user") is None + with pytest.raises(PermissionError): + await invoke("sandbox_get_artifact", src_path="../outside.txt") + other_root = Path(_paths.workspace_directory("other-chat")) + other_root.mkdir(parents=True) + other = other_root / "secret.txt" + other.write_text("another session") + with pytest.raises(PermissionError): + await invoke("sandbox_get_artifact", src_path=str(other)) + (root / "escape.txt").symlink_to(other) + with pytest.raises(PermissionError): + await invoke("sandbox_get_artifact", src_path="escape.txt") def test_desktop_runner_creates_no_aliases(monkeypatch, tmp_path): @@ -191,7 +220,7 @@ def test_desktop_runner_creates_no_aliases(monkeypatch, tmp_path): async def test_job_commands_preserve_paths_with_spaces(monkeypatch, tmp_path): import asyncio - from orchestration.jobs import files as job_runtime + import core.jobs.files as job_runtime monkeypatch.setattr(local_mode, "local_mode_enabled", lambda: True) root = tmp_path / "Application Support" / "张 三" @@ -212,7 +241,7 @@ async def bash(command, **kwargs): out, err = await proc.communicate() return proc.returncode, out.decode(), err.decode() - monkeypatch.setattr(job_runtime, "_sbx_bash", bash) + monkeypatch.setattr(jobs_files, "_sbx_bash", bash) workspace = Path(_paths.workspace_directory("chat")) output = workspace / "导出 数据.txt" ok, detail = await job_runtime.write_sandbox_file( diff --git a/src/backend/tests/llm/tool_call_debug.py b/src/backend/tests/llm/tool_call_debug.py index b355b2da..ecca17a0 100644 --- a/src/backend/tests/llm/tool_call_debug.py +++ b/src/backend/tests/llm/tool_call_debug.py @@ -9,6 +9,10 @@ async def main(): + from features import register_model_credentials, register_product_tools, register_run_capabilities + register_product_tools() + register_run_capabilities() + register_model_credentials() from core.llm.factory import create_agent_executor print("Creating agent...", flush=True) diff --git a/src/backend/tests/llm/vision_fullstack_check.py b/src/backend/tests/llm/vision_fullstack_check.py index 260eec8f..bc9714e6 100644 --- a/src/backend/tests/llm/vision_fullstack_check.py +++ b/src/backend/tests/llm/vision_fullstack_check.py @@ -199,7 +199,7 @@ def _run(client, headers, stub_url) -> tuple[bool, str]: ) ok &= _check("指派 vision 角色", assign.status_code == 200, assign.text[:160]) - from core.services.model_config import ModelConfigService + from core.models.config import ModelConfigService ModelConfigService.get_instance().invalidate_cache() @@ -291,7 +291,7 @@ def main() -> int: if provider_id: client.delete("/v1/models/roles/vision", headers=headers) client.delete(f"/v1/models/providers/{provider_id}", headers=headers) - from core.services.model_config import ModelConfigService + from core.models.config import ModelConfigService ModelConfigService.get_instance().invalidate_cache() print("cleanup: 临时视觉供应商已删除", flush=True) diff --git a/src/backend/tests/llm/vision_live_check.py b/src/backend/tests/llm/vision_live_check.py index 48fef29e..d4b651ea 100644 --- a/src/backend/tests/llm/vision_live_check.py +++ b/src/backend/tests/llm/vision_live_check.py @@ -43,7 +43,7 @@ async def list_models(role: str = "main_agent") -> None: """列出某个已配置 provider 所在网关的模型清单（用于挑一个多模态模型）。""" import httpx - from core.services.model_config import ModelConfigService + from core.models.config import ModelConfigService cfg = ModelConfigService.get_instance().resolve(role) if cfg is None: @@ -96,10 +96,10 @@ async def probe(model_name: str, role: str = "main_agent") -> None: """借用某个已配置 provider 的网关和凭据，试指定模型能否读图。""" from dataclasses import replace - from core.services.model_config import ModelConfigService - from core.vision.prompt import build_vision_prompt - from core.vision.provider import VisionProvider - from core.vision.schema import parse_evidence + from core.models.config import ModelConfigService + from core.models.vision.prompt import build_vision_prompt + from core.models.vision.provider import VisionProvider + from core.models.vision.schema import parse_evidence base = ModelConfigService.get_instance().resolve(role) if base is None: @@ -127,7 +127,7 @@ async def probe(model_name: str, role: str = "main_agent") -> None: async def run() -> int: """完整链路：视觉桥（含缓存/重试/渲染）→ 校验读到了图上的字。""" - from core.vision import get_vision_bridge, is_available, render_evidence + from core.models.vision import get_vision_bridge, is_available, render_evidence if not is_available(): print("FAIL: 未配置 vision 角色，且主模型不支持读图") diff --git a/src/backend/tests/llm/vision_role_live_check.py b/src/backend/tests/llm/vision_role_live_check.py index 3b190851..3a296511 100644 --- a/src/backend/tests/llm/vision_role_live_check.py +++ b/src/backend/tests/llm/vision_role_live_check.py @@ -182,8 +182,8 @@ def main() -> int: ok &= _check("把供应商指派给 vision 角色", assign.status_code == 200, assign.text[:200]) # 5) 能力接口 + 视觉桥自身都应认为可以读图了 - from core.services.model_config import ModelConfigService - from core.vision import is_available, resolve_vision_config + from core.models.config import ModelConfigService + from core.models.vision import is_available, resolve_vision_config ModelConfigService.get_instance().invalidate_cache() after = client.get("/v1/models/capabilities").json().get("data") or {} @@ -202,14 +202,14 @@ def main() -> int: ) # 6) 主模型是纯文本 → 应当挂上视觉桥版 read_image 工具 - from core.vision import resolve_vision_mode + from core.models.vision import resolve_vision_mode ok &= _check("纯文本主模型下会注册视觉桥版 read_image", resolve_vision_mode() == "bridge") # 7) 真的经这条角色配置识一次图 —— 证明「配得上」也「调得通」 import asyncio - from core.vision import get_vision_bridge, render_evidence + from core.models.vision import get_vision_bridge, render_evidence image = _sample_png() result = asyncio.run( @@ -235,7 +235,7 @@ def main() -> int: except Exception: # noqa: BLE001 pass client.delete(f"/v1/models/providers/{provider_id}", headers=headers) - from core.services.model_config import ModelConfigService + from core.models.config import ModelConfigService ModelConfigService.get_instance().invalidate_cache() restored = client.get("/v1/models/capabilities").json().get("data") or {} diff --git a/src/backend/tests/mcp/test_internet_search.py b/src/backend/tests/mcp/test_internet_search.py index acd7d670..2486924c 100644 --- a/src/backend/tests/mcp/test_internet_search.py +++ b/src/backend/tests/mcp/test_internet_search.py @@ -1,11 +1,15 @@ """Tests for the provider-neutral internet search MCP implementation.""" from __future__ import annotations +from core.config.store import config_keys +from core.config.store import system_config +from core.config.store import system_config_constants import asyncio import pytest -from core.services import service_probes, system_config +from core.infra.probes import service_probes +from core.config.store import system_config class _FakeResponse: @@ -106,13 +110,13 @@ async def _fake_test_langsearch(api_key: str) -> dict: def test_langsearch_has_an_independent_system_config_key() -> None: - seed_keys = {item[0] for item in system_config.SEED_CONFIGS} + seed_keys = {item[0] for item in system_config_constants.SEED_CONFIGS} assert "internet_search.langsearch_api_key" in seed_keys assert ( - system_config.get_config_key_for_env("LANGSEARCH_API_KEY") + config_keys.get_config_key_for_env("LANGSEARCH_API_KEY") == "internet_search.langsearch_api_key" ) - assert system_config.get_config_key_for_env("TAVILY_API_KEY") == ( + assert config_keys.get_config_key_for_env("TAVILY_API_KEY") == ( "internet_search.tavily_api_key" ) diff --git a/src/backend/tests/mcp/test_wiki_tool_gating.py b/src/backend/tests/mcp/test_wiki_tool_gating.py index 1ffddf0c..a3a63fcd 100644 --- a/src/backend/tests/mcp/test_wiki_tool_gating.py +++ b/src/backend/tests/mcp/test_wiki_tool_gating.py @@ -22,10 +22,9 @@ async def _list_tool_names() -> set[str]: - """走 lowlevel handler，覆盖到真实的 list_tools 链路而不是内部函数。""" - handler = srv.mcp._mcp_server.request_handlers[types.ListToolsRequest] - result = await handler(types.ListToolsRequest(method="tools/list")) - return {tool.name for tool in result.root.tools} + """The public override is also used by the SDK tools/list handler.""" + return {tool.name for tool in await srv.mcp.list_tools()} + @pytest.fixture @@ -120,7 +119,7 @@ def test_external_kb_skips_local_permission_check(monkeypatch): """外接库的白名单在其实现模块内把关，不该被本地权限判定误伤。""" monkeypatch.setattr(wiki_impl, "_accessible_local_kb_ids", lambda _uid: []) sentinel = object() - monkeypatch.setattr("core.kb.wiki_router.wiki_module_for", lambda kb_id: sentinel) + monkeypatch.setattr('features.knowledge.wiki_router.wiki_module_for', lambda kb_id: sentinel) assert wiki_impl._wiki_module("external-dataset", current_user_id="u1") is sentinel @@ -201,7 +200,7 @@ def test_resolve_dataset_id_prefers_request_allowlist(monkeypatch): def test_resolve_dataset_id_prefers_wiki_capable_local_kb(monkeypatch): """自建库排在外接库前面：用户问的多半是自己传的资料。""" monkeypatch.setattr( - "core.kb.wiki_router.wiki_capable_local_kb_ids", + 'features.knowledge.wiki_router.wiki_capable_local_kb_ids', lambda ids: [i for i in ids if i == "kb_wiki_one"], ) resolved = wiki_impl.resolve_dataset_id("", "external-a", allowed_kb_ids="kb_plain,kb_wiki_one") @@ -209,6 +208,6 @@ def test_resolve_dataset_id_prefers_wiki_capable_local_kb(monkeypatch): def test_resolve_dataset_id_falls_back_to_external_when_no_local_wiki(monkeypatch): - monkeypatch.setattr("core.kb.wiki_router.wiki_capable_local_kb_ids", lambda ids: []) + monkeypatch.setattr('features.knowledge.wiki_router.wiki_capable_local_kb_ids', lambda ids: []) resolved = wiki_impl.resolve_dataset_id("", "external-a", allowed_kb_ids="kb_plain") assert resolved == "external-a" diff --git a/src/backend/tests/mcp_servers/test_search_batch.py b/src/backend/tests/mcp_servers/test_search_batch.py index f79a6279..89f7526b 100644 --- a/src/backend/tests/mcp_servers/test_search_batch.py +++ b/src/backend/tests/mcp_servers/test_search_batch.py @@ -152,7 +152,8 @@ async def handler(request): def test_partial_failure_still_gets_citations(monkeypatch): import json - from orchestration.citation_anchor import AnchorAllocator, annotate_tool_result + from core.llm.citations.allocator import AnchorAllocator + from core.llm.citations.anchor import annotate_tool_result monkeypatch.setattr( "extensions.mcp_servers.internet_search_mcp.impl.get_runtime_value", @@ -297,8 +298,9 @@ def test_mcp_rejects_removed_argument(): def test_two_mcp_sessions_share_capacity_and_close_client(monkeypatch): import json - from mcp.shared.memory import create_connected_server_and_client_session from extensions.mcp_servers.internet_search_mcp import impl, server + from mcp import Client + from tests.llm.test_mcp_parallel_execution import probe_server monkeypatch.setattr( impl, @@ -336,8 +338,8 @@ def client_factory(**kwargs): monkeypatch.setattr(server.httpx, "AsyncClient", client_factory) async def run(): - async with create_connected_server_and_client_session(server.mcp) as first: - async with create_connected_server_and_client_session(server.mcp) as second: + async with probe_server(server.mcp) as (url, _): + async with Client(url) as first, Client(url) as second: assert len(clients) == 1 calls = [ asyncio.create_task( @@ -357,12 +359,12 @@ async def run(): assert started == 8 release.set() results = await asyncio.gather(*calls) - assert all(not result.isError for result in results) + assert all(not result.is_error for result in results) assert maximum == 8 data = json.loads(results[0].content[0].text) assert data["result"]["status"] == "complete" bad = await first.call_tool("internet_search", {"query": "a", "cn_only": True}) - assert bad.isError + assert bad.is_error assert not clients[0].is_closed assert clients[0].is_closed @@ -417,14 +419,6 @@ def test_round_robin_caps_at_eight_sources(): def test_search_transport_allows_structured_deadline_result(): - from core.llm.mcp_pool import make_client - - class Client: - def __init__(self, **kwargs): - self.kwargs = kwargs - - cfg = {"transport": "streamable_http", "url": "http://localhost:9102/mcp"} - client = make_client("internet_search", cfg, client_cls=Client) - assert client.kwargs["mcp_config"].timeout == 50.0 - client = make_client("internet_search", {**cfg, "transport_timeout": 12}, client_cls=Client) - assert client.kwargs["mcp_config"].timeout == 12.0 + from core.llm.mcp.client import _http_transport_timeout + assert _http_transport_timeout("internet_search", {}, 120) == 50.0 + assert _http_transport_timeout("internet_search", {"transport_timeout": 12}, 120) == 12.0 diff --git a/src/backend/tests/mcp_servers/test_web_fetch.py b/src/backend/tests/mcp_servers/test_web_fetch.py index 701b38d1..887129bb 100644 --- a/src/backend/tests/mcp_servers/test_web_fetch.py +++ b/src/backend/tests/mcp_servers/test_web_fetch.py @@ -30,7 +30,8 @@ async def handler(request): def test_download_budget_is_visible_and_source_is_citable(monkeypatch): import json - from orchestration.citation_anchor import AnchorAllocator, annotate_tool_result + from core.llm.citations.allocator import AnchorAllocator + from core.llm.citations.anchor import annotate_tool_result real_client = httpx.AsyncClient monkeypatch.setattr(impl, "MAX_RESPONSE_BYTES", 16) diff --git a/src/backend/tests/memory/outbox_test_support.py b/src/backend/tests/memory/outbox_test_support.py index b21ac2a3..9ab0a514 100644 --- a/src/backend/tests/memory/outbox_test_support.py +++ b/src/backend/tests/memory/outbox_test_support.py @@ -3,7 +3,7 @@ from sqlalchemy.orm import sessionmaker from core.db.models import ChatMessage, ChatSession -from core.evolution import settlement as evolution_settlement +from core.evolution.settlement import settlement as evolution_settlement from core.memory.extractors import writers as W from core.memory import outbox as O from core.memory.outbox import common as outbox_common diff --git a/src/backend/tests/memory/test_ce_overlay_memory_contracts.py b/src/backend/tests/memory/test_ce_overlay_memory_contracts.py index 51e1dbfa..474d4129 100644 --- a/src/backend/tests/memory/test_ce_overlay_memory_contracts.py +++ b/src/backend/tests/memory/test_ce_overlay_memory_contracts.py @@ -9,9 +9,9 @@ they never entered Base.metadata and the local SQLite bootstrap could not create their tables ("no such table" at runtime). -Both files live under ce/overlay/ (they replace same-named files when the CE -tree is derived); these tests load them by path from the FULL tree and are -skipped in a derived CE tree where ce/ does not exist. +The shared implementations now use a small CE audit policy. These tests load +that policy, exercise the real APIs and verify the shared models enter CE +metadata; they are skipped where the source overlay is absent. """ import importlib.util @@ -39,8 +39,8 @@ def _load(path: Path, name: str): return module -def test_ce_audit_stub_accepts_positional_calls_like_the_real_impl(): - stub = _load(_OVERLAY / "core" / "memory" / "audit.py", "_ce_audit_stub") +def test_ce_audit_stub_accepts_positional_calls_like_the_real_impl(ce_audit): + stub = ce_audit ctx = SimpleNamespace(user_id="u1", workspace_id=None) # Same positional call style as core/memory/profile.py::_audit_sync_safe. @@ -49,8 +49,8 @@ def test_ce_audit_stub_accepts_positional_calls_like_the_real_impl(): @pytest.mark.asyncio -async def test_ce_audit_stub_async_variants_accept_positional_calls(): - stub = _load(_OVERLAY / "core" / "memory" / "audit.py", "_ce_audit_stub_async") +async def test_ce_audit_stub_async_variants_accept_positional_calls(ce_audit): + stub = ce_audit ctx = SimpleNamespace(user_id="u1", workspace_id=None) await stub.record(ctx, "write", "L1", reason="r") @@ -58,7 +58,9 @@ async def test_ce_audit_stub_async_variants_accept_positional_calls(): def test_ce_models_facade_exports_evolution_and_ref_shadow_tables(): - src = (_OVERLAY / "core" / "db" / "models" / "__init__.py").read_text(encoding="utf-8") + from core.db import models + from core.db.personal_schema import ce_metadata + metadata = ce_metadata() required = [ "EvolutionAgentProfile", "EvolutionCandidate", @@ -72,7 +74,28 @@ def test_ce_models_facade_exports_evolution_and_ref_shadow_tables(): "EvolutionTraceEvent", "MemoryRefShadow", ] - missing = [name for name in required if name not in src] + missing = [name for name in required if not hasattr(models, name)] # A model missing from the facade never enters Base.metadata, so the local # bootstrap (create_all / reconcile) cannot create its table. assert not missing, f"CE models facade is missing imports: {missing}" + + for name in required: + assert getattr(models, name).__tablename__ in metadata.tables + + +@pytest.fixture +def ce_audit(monkeypatch): + from core.memory import audit, edition + policy = _load(_OVERLAY / "core/memory/edition.py", "_ce_memory_policy") + from unittest.mock import Mock + assert policy.audit_enabled() is False + enabled = Mock(wraps=policy.audit_enabled) + monkeypatch.setattr(edition, "audit_enabled", enabled) + monkeypatch.setattr(audit, "settings", SimpleNamespace(memory=SimpleNamespace(enabled=True))) + single, batch = Mock(), Mock() + monkeypatch.setattr(audit, "_sync_insert_one", single) + monkeypatch.setattr(audit, "_sync_insert_many", batch) + yield audit + assert enabled.call_count == 2 + single.assert_not_called() + batch.assert_not_called() diff --git a/src/backend/tests/memory/test_graph_memory.py b/src/backend/tests/memory/test_graph_memory.py index 32fcc678..b57f04b0 100644 --- a/src/backend/tests/memory/test_graph_memory.py +++ b/src/backend/tests/memory/test_graph_memory.py @@ -1,4 +1,6 @@ """L3 graph memory keeps declarative relations separate and actionable.""" +from core.memory import graph as graph_module +from core.memory import graph_contract as graph_contract_module from types import SimpleNamespace @@ -13,7 +15,7 @@ def test_relation_parser_accepts_only_the_bounded_graph_vocabulary(): - relation = G.parse_relation( + relation = graph_contract_module.parse_relation( { "source": "项目北斗", "source_type": "project", @@ -26,9 +28,9 @@ def test_relation_parser_accepts_only_the_bounded_graph_vocabulary(): assert relation is not None assert relation.relationship == "依赖" assert relation.confidence == 1.0 - assert G.parse_relation({"source": "A", "predicate": "invented", "target": "B"}) is None + assert graph_contract_module.parse_relation({"source": "A", "predicate": "invented", "target": "B"}) is None assert ( - G.parse_relation( + graph_contract_module.parse_relation( { "source": "项目北斗", "predicate": "depends_on", @@ -48,7 +50,7 @@ def test_relation_parser_deduplicates_one_turn(): "target": "数据中台", "target_type": "system", } - assert len(G.parse_relations([raw, dict(raw)])) == 1 + assert len(graph_contract_module.parse_relations([raw, dict(raw)])) == 1 def test_graph_read_uses_a_non_reserved_cypher_parameter(monkeypatch): @@ -69,9 +71,9 @@ class Driver: def session(): return Session() - monkeypatch.setattr(G, "_get_driver", lambda: Driver()) + monkeypatch.setattr(graph_module, 'get_driver', lambda: Driver()) assert ( - G._read_sync( + graph_module.read_sync( scope_user_id="u1", workspace_id="default", limit=5, @@ -118,8 +120,8 @@ class Driver: def session(): return Session() - monkeypatch.setattr(G, "_get_driver", lambda: Driver()) - relation = G.parse_relation( + monkeypatch.setattr(graph_module, 'get_driver', lambda: Driver()) + relation = graph_contract_module.parse_relation( { "source": "项目北斗", "source_type": "project", @@ -130,8 +132,8 @@ def session(): ) ctx = MemoryContext(user_id="u1", effect_id="mout-candidate-1") - first = G._write_sync(ctx, [relation]) - second = G._write_sync(ctx, [relation]) + first = graph_module.write_sync(ctx, [relation]) + second = graph_module.write_sync(ctx, [relation]) assert first[0]["seen_count"] == 1 assert first[0]["replayed"] is False diff --git a/src/backend/tests/memory/test_identity_frozen_block.py b/src/backend/tests/memory/test_identity_frozen_block.py index 411fb229..c8a7af25 100644 --- a/src/backend/tests/memory/test_identity_frozen_block.py +++ b/src/backend/tests/memory/test_identity_frozen_block.py @@ -15,10 +15,8 @@ import asyncio -from orchestration.memory_integration import ( - build_user_identity_block, - inject_session_blocks, -) +from core.agent_runtime.integrations.memory_identity import build_user_identity_block +from core.agent_runtime.integrations.memory_session import inject_session_blocks def _run(coro): diff --git a/src/backend/tests/memory/test_memory_outbox.py b/src/backend/tests/memory/test_memory_outbox.py index edf2eb5d..c3768842 100644 --- a/src/backend/tests/memory/test_memory_outbox.py +++ b/src/backend/tests/memory/test_memory_outbox.py @@ -3,8 +3,8 @@ import pytest from core.db.models import EvolutionEpisode, MemoryOutbox -from core.evolution import settlement_runner -from core.evolution import settlement as evolution_settlement +from core.evolution.settlement import settlement_runner +from core.evolution.settlement import settlement as evolution_settlement from core.memory.context import MemoryContext from core.memory.extractors.router import ExtractorType from core.memory.extractors import writers as W diff --git a/src/backend/tests/memory/test_memory_outbox_receipts.py b/src/backend/tests/memory/test_memory_outbox_receipts.py index b73c5a29..e4d9bde1 100644 --- a/src/backend/tests/memory/test_memory_outbox_receipts.py +++ b/src/backend/tests/memory/test_memory_outbox_receipts.py @@ -5,7 +5,7 @@ from sqlalchemy import event from core.db.models import ChatMessage, MemoryOutbox, ProfileMemory -from core.evolution import settlement as evolution_settlement +from core.evolution.settlement import settlement as evolution_settlement from core.memory.context import MemoryContext from core.memory.extractors.router import ExtractorType from core.memory.extractors import writers as W @@ -32,7 +32,7 @@ async def test_profile_effect_receipt_preserves_written_result_after_crash(db_se memory_settings = _memory_settings() monkeypatch.setattr(outbox_common, "settings", SimpleNamespace(memory=memory_settings)) monkeypatch.setattr(P, "settings", SimpleNamespace(memory=memory_settings)) - P._bg_semaphore = None + P.process_state().bg_semaphore = None monkeypatch.setattr("core.memory.profile._audit_sync_safe", lambda *_args: None) monkeypatch.setattr("core.memory.profile._schedule_compact", lambda *_args: None) ctx = MemoryContext(user_id="u1", message_id="m-profile-receipt", write_enabled=True) @@ -230,7 +230,7 @@ async def test_multi_rule_candidate_keeps_each_external_effect_receipt(db_sessio monkeypatch.setattr(outbox_common, "settings", SimpleNamespace(memory=memory_settings)) monkeypatch.setattr(P, "settings", SimpleNamespace(memory=memory_settings)) monkeypatch.setattr(memory_backend, "settings", SimpleNamespace(memory=memory_settings)) - P._bg_semaphore = None + P.process_state().bg_semaphore = None ctx = MemoryContext(user_id="u1", message_id="m-multi-effect", write_enabled=True) job_id = O.enqueue_candidate_job( "parent", @@ -320,7 +320,7 @@ async def test_later_l2_effect_defers_without_spending_an_attempt(db_session, mo memory_settings = _memory_settings() monkeypatch.setattr(outbox_common, "settings", SimpleNamespace(memory=memory_settings)) monkeypatch.setattr(P, "settings", SimpleNamespace(memory=memory_settings)) - P._bg_semaphore = None + P.process_state().bg_semaphore = None ctx = MemoryContext(user_id="u1", message_id="m-lane", write_enabled=True) older_id = O.enqueue_candidate_job( "parent-old", diff --git a/src/backend/tests/memory/test_memory_outbox_recovery.py b/src/backend/tests/memory/test_memory_outbox_recovery.py index 29f067e0..9ed45c0b 100644 --- a/src/backend/tests/memory/test_memory_outbox_recovery.py +++ b/src/backend/tests/memory/test_memory_outbox_recovery.py @@ -5,7 +5,7 @@ import pytest from core.db.models import MemoryOutbox -from core.evolution import settlement as evolution_settlement +from core.evolution.settlement import settlement as evolution_settlement from core.memory.context import MemoryContext from core.memory.extractors.router import ExtractorType from core.memory.extractors import writers as W @@ -34,7 +34,7 @@ async def test_worker_shutdown_cancels_woken_consumer_and_releases_lease(db_sess memory_settings = _memory_settings(outbox_poll_interval_s=30) monkeypatch.setattr(outbox_common, "settings", SimpleNamespace(memory=memory_settings)) monkeypatch.setattr(P, "settings", SimpleNamespace(memory=memory_settings)) - P._bg_semaphore = None + P.process_state().bg_semaphore = None ctx = MemoryContext(user_id="u1", message_id="m-stop", write_enabled=True) job_id = O.enqueue_candidate_job( "parent", @@ -67,7 +67,7 @@ async def test_lease_renewal_loss_cancels_the_old_writer(db_session, monkeypatch memory_settings = _memory_settings(outbox_lease_s=1) monkeypatch.setattr(outbox_common, "settings", SimpleNamespace(memory=memory_settings)) monkeypatch.setattr(P, "settings", SimpleNamespace(memory=memory_settings)) - P._bg_semaphore = None + P.process_state().bg_semaphore = None ctx = MemoryContext(user_id="u1", message_id="m-lease", write_enabled=True) job_id = O.enqueue_candidate_job( "parent", @@ -101,7 +101,7 @@ async def test_crash_before_ack_reuses_external_effect_receipt(db_session, monke memory_settings = _memory_settings() monkeypatch.setattr(outbox_common, "settings", SimpleNamespace(memory=memory_settings)) monkeypatch.setattr(P, "settings", SimpleNamespace(memory=memory_settings)) - P._bg_semaphore = None + P.process_state().bg_semaphore = None ctx = MemoryContext(user_id="u1", message_id="m-effect", write_enabled=True) job_id = O.enqueue_candidate_job( "parent", @@ -161,7 +161,7 @@ async def test_interactive_long_term_edits_are_admitted_before_store_effect( memory_settings = _memory_settings() monkeypatch.setattr(outbox_common, "settings", SimpleNamespace(memory=memory_settings)) monkeypatch.setattr(P, "settings", SimpleNamespace(memory=memory_settings)) - P._bg_semaphore = None + P.process_state().bg_semaphore = None ctx = MemoryContext(user_id="u1", message_id="m-edit", write_enabled=True) observed = [] @@ -211,7 +211,7 @@ async def test_shutdown_waits_for_real_executor_add_then_replay_uses_receipt( "settings", SimpleNamespace(memory=SimpleNamespace(enabled=True)), ) - P._bg_semaphore = None + P.process_state().bg_semaphore = None started = threading.Event() release = threading.Event() @@ -296,7 +296,7 @@ async def test_pipeline_reuses_atomic_extraction_checkpoint_after_crash(db_sessi memory_settings = _memory_settings() monkeypatch.setattr(outbox_common, "settings", SimpleNamespace(memory=memory_settings)) monkeypatch.setattr(P, "settings", SimpleNamespace(memory=memory_settings)) - P._bg_semaphore = None + P.process_state().bg_semaphore = None ctx = MemoryContext(user_id="u1", message_id="m-checkpoint", write_enabled=True) pipeline_id = O.enqueue_pipeline_job(ctx, "请记住我叫小明", "好的") _add_assistant_message(factory, "m-checkpoint") @@ -344,7 +344,7 @@ async def test_editing_back_to_an_old_value_creates_a_new_operation(db_session, memory_settings = _memory_settings() monkeypatch.setattr(outbox_common, "settings", SimpleNamespace(memory=memory_settings)) monkeypatch.setattr(P, "settings", SimpleNamespace(memory=memory_settings)) - P._bg_semaphore = None + P.process_state().bg_semaphore = None ctx = MemoryContext(user_id="u1", write_enabled=True) stored = {"text": "A"} effects = [] diff --git a/src/backend/tests/memory/test_memory_retrieval_lifecycle.py b/src/backend/tests/memory/test_memory_retrieval_lifecycle.py index 7c5a5dc6..a5c9e4cc 100644 --- a/src/backend/tests/memory/test_memory_retrieval_lifecycle.py +++ b/src/backend/tests/memory/test_memory_retrieval_lifecycle.py @@ -1,11 +1,13 @@ +from core.agent_runtime.integrations import memory as memory_module + import asyncio from types import SimpleNamespace +import core.agent_runtime.integrations.memory as memory +import core.agent_runtime.integrations.memory as M import pytest - +from core.evolution.settlement import settlement_runner from core.memory.retrieval_types import MemoryRetrievalResult -from core.evolution import settlement_runner -from orchestration import memory_integration as M @pytest.mark.asyncio @@ -21,26 +23,26 @@ async def empty_profile(*_args, **_kwargs): monkeypatch.setattr(M.profile, "get", empty_profile) monkeypatch.setattr( - M, + memory, "settings", SimpleNamespace(memory=SimpleNamespace(retrieval_budget_ms=1)), ) task = asyncio.create_task(slow_retrieval()) - M.track_memory_retrieval(task) + memory_module.track_memory_retrieval(task) - block = (await M.build_frozen_memory_block("u1", "default", task)).text + block = (await memory_module.build_frozen_memory_block("u1", "default", task)).text assert block == "" assert task.done() is False assert task.cancelled() is False - assert M.get_retrieval_state(task) == "timed_out_running" + assert memory_module.get_retrieval_state(task) == "timed_out_running" release.set() await task await asyncio.sleep(0) - assert M.get_retrieval_state(task) == "completed_after_timeout" - assert M.get_last_retrieval(task).degrade_reason == "late-result" + assert memory_module.get_retrieval_state(task) == "completed_after_timeout" + assert memory_module.get_last_retrieval(task).degrade_reason == "late-result" @pytest.mark.asyncio @@ -56,19 +58,19 @@ async def slow_retrieval(*_args, **kwargs): async def empty_profile(*_args, **_kwargs): return "" - monkeypatch.setattr(M, "retrieve_memories_structured", slow_retrieval) + monkeypatch.setattr(memory, "retrieve_memories_structured", slow_retrieval) monkeypatch.setattr(M.profile, "get", empty_profile) - task = await M.launch_memory_retrieval("u1", "hello", True, budget_ms=1) + task = await memory_module.launch_memory_retrieval("u1", "hello", True, budget_ms=1) - assert (await M.build_frozen_memory_block("u1", "default", task)).text == "" + assert (await memory_module.build_frozen_memory_block("u1", "default", task)).text == "" assert observed_timeouts == [None] assert task.done() is False - assert M.get_retrieval_state(task) == "timed_out_running" + assert memory_module.get_retrieval_state(task) == "timed_out_running" release.set() await task await asyncio.sleep(0) - assert M.get_retrieval_state(task) == "completed_after_timeout" + assert memory_module.get_retrieval_state(task) == "completed_after_timeout" @pytest.mark.asyncio @@ -77,13 +79,13 @@ async def never_finishes(): await asyncio.Event().wait() task = asyncio.create_task(never_finishes()) - M.track_memory_retrieval(task) + memory_module.track_memory_retrieval(task) task.cancel() with pytest.raises(asyncio.CancelledError): await task await asyncio.sleep(0) - assert M.get_retrieval_state(task) == "cancelled" + assert memory_module.get_retrieval_state(task) == "cancelled" def test_no_write_path_reports_empty_items_with_failure_state(monkeypatch): @@ -94,6 +96,6 @@ def test_no_write_path_reports_empty_items_with_failure_state(monkeypatch): lambda message_id, *, items, failed: reports.append((message_id, items, failed)), ) - M._report_no_memory_writes("m-no-write", failed=True) + memory_module.report_no_memory_writes("m-no-write", failed=True) assert reports == [("m-no-write", [], True)] diff --git a/src/backend/tests/memory/test_session_frozen_snapshot.py b/src/backend/tests/memory/test_session_frozen_snapshot.py index ab417ed5..dfeb300b 100644 --- a/src/backend/tests/memory/test_session_frozen_snapshot.py +++ b/src/backend/tests/memory/test_session_frozen_snapshot.py @@ -6,14 +6,16 @@ """ from __future__ import annotations +from core.agent_runtime.integrations import memory as memory_module +from core.agent_runtime.integrations import memory_session as memory_session_module import asyncio +import core.agent_runtime.integrations.memory as memory +import core.agent_runtime.integrations.memory as M import pytest - from core.memory.retrieval_types import MemoryRetrievalResult, RetrievedMemory, content_hash from core.memory.session_snapshot import SessionMemorySnapshot -from orchestration import memory_integration as M def _fact(text: str) -> MemoryRetrievalResult: @@ -57,8 +59,8 @@ def _save(chat_id, snapshot): saved[chat_id] = snapshot return snapshot - monkeypatch.setattr(M, "load_session_memory_snapshot", _load) - monkeypatch.setattr(M, "save_session_memory_snapshot", _save) + monkeypatch.setattr(memory_session_module, "load_session_memory_snapshot", _load) + monkeypatch.setattr(memory_session_module, "save_session_memory_snapshot", _save) return saved @@ -74,20 +76,20 @@ async def _retrieve(*, user_id, query, workspace_id, timeout_s): async def _profile(*_args, **_kwargs): return "- 用户偏好：简洁" - monkeypatch.setattr(M, "retrieve_memories_structured", _retrieve) + monkeypatch.setattr(memory, "retrieve_memories_structured", _retrieve) monkeypatch.setattr(M.profile, "get", _profile) return calls async def _turn(chat_id: str, message: str) -> str: - handle = await M.open_session_memory( + handle = await memory_session_module.open_session_memory( chat_id=chat_id, scope_user_id="u1", workspace_id="default", user_message=message, memory_enabled=True, ) - return await M.resolve_session_memory(handle) + return await memory_session_module.resolve_session_memory(handle) @pytest.mark.asyncio @@ -114,7 +116,7 @@ async def test_later_turns_replay_the_block_without_retrieving(store, retrievals @pytest.mark.asyncio async def test_replay_turn_exposes_no_retrieval_task(store, retrievals): await _turn("chat-1", "第一问") - handle = await M.open_session_memory( + handle = await memory_session_module.open_session_memory( chat_id="chat-1", scope_user_id="u1", workspace_id="default", @@ -124,7 +126,7 @@ async def test_replay_turn_exposes_no_retrieval_task(store, retrievals): # No read happened this turn, so the episode must not claim memory evidence. assert handle.retrieval_task is None - assert M.get_last_retrieval(handle.retrieval_task) is None + assert memory_module.get_last_retrieval(handle.retrieval_task) is None @pytest.mark.asyncio @@ -158,7 +160,7 @@ async def _retrieve(*, user_id, query, workspace_id, timeout_s): async def _profile(*_args, **_kwargs): return "" - monkeypatch.setattr(M, "retrieve_memories_structured", _retrieve) + monkeypatch.setattr(memory, "retrieve_memories_structured", _retrieve) monkeypatch.setattr(M.profile, "get", _profile) assert await _turn("chat-1", "第一问") == "" @@ -174,7 +176,7 @@ async def _degraded(*, user_id, query, workspace_id, timeout_s): async def _profile(*_args, **_kwargs): return "" - monkeypatch.setattr(M, "retrieve_memories_structured", _degraded) + monkeypatch.setattr(memory, "retrieve_memories_structured", _degraded) monkeypatch.setattr(M.profile, "get", _profile) await _turn("chat-1", "第一问") @@ -194,10 +196,10 @@ async def _slow(*, user_id, query, workspace_id, timeout_s): async def _profile(*_args, **_kwargs): return "" - monkeypatch.setattr(M, "retrieve_memories_structured", _slow) + monkeypatch.setattr(memory, "retrieve_memories_structured", _slow) monkeypatch.setattr(M.profile, "get", _profile) - handle = await M.open_session_memory( + handle = await memory_session_module.open_session_memory( chat_id="chat-1", scope_user_id="u1", workspace_id="default", @@ -205,7 +207,7 @@ async def _profile(*_args, **_kwargs): memory_enabled=True, budget_ms=1, ) - assert await M.resolve_session_memory(handle) == "" + assert await memory_session_module.resolve_session_memory(handle) == "" assert "chat-1" not in store release.set() @@ -214,7 +216,7 @@ async def _profile(*_args, **_kwargs): @pytest.mark.asyncio async def test_memory_off_neither_retrieves_nor_stores(store, retrievals): - handle = await M.open_session_memory( + handle = await memory_session_module.open_session_memory( chat_id="chat-1", scope_user_id="u1", workspace_id="default", @@ -222,7 +224,7 @@ async def test_memory_off_neither_retrieves_nor_stores(store, retrievals): memory_enabled=False, ) - assert await M.resolve_session_memory(handle) == "" + assert await memory_session_module.resolve_session_memory(handle) == "" assert handle.retrieval_task is None assert retrievals == [] assert store == {} diff --git a/src/backend/tests/misc/chat_metadata_merge_selftest.py b/src/backend/tests/misc/chat_metadata_merge_selftest.py index 153d09e3..68369588 100644 --- a/src/backend/tests/misc/chat_metadata_merge_selftest.py +++ b/src/backend/tests/misc/chat_metadata_merge_selftest.py @@ -7,7 +7,7 @@ project_root = Path(__file__).parent.parent.parent sys.path.insert(0, str(project_root)) -from core.services import ChatService +from core.chat.service import ChatService def test_update_session_merges_mode_flags_into_extra_data(db_session): diff --git a/src/backend/tests/misc/conftest.py b/src/backend/tests/misc/conftest.py index b892803c..cb61d967 100644 --- a/src/backend/tests/misc/conftest.py +++ b/src/backend/tests/misc/conftest.py @@ -1,13 +1,2 @@ -"""tests/misc: the desktop bridge now consults the capability index (name preferences).""" - -from __future__ import annotations - -import pytest -from tests._capability_index import bind_capability_index - - -@pytest.fixture(autouse=True) -def _capability_index(tmp_path, monkeypatch): - engine, _factory = bind_capability_index(tmp_path, monkeypatch) - yield - engine.dispose() +"""Shared capability-index isolation for miscellaneous tests.""" +from tests.features.desktop.index_fixtures import _capability_index diff --git a/src/backend/tests/misc/test_identity_dedup.py b/src/backend/tests/misc/test_identity_dedup.py index b01f9df7..ed386844 100644 --- a/src/backend/tests/misc/test_identity_dedup.py +++ b/src/backend/tests/misc/test_identity_dedup.py @@ -20,7 +20,7 @@ from core.db.identity_dedup import merge_duplicate_user_shadows from core.db.models import ChatSession, UserShadow from core.db.repository import UserRepository -from core.services.user_service import UserService +from core.auth.users import UserService CENTER_ID = "cloud:example.test:443:42" diff --git a/src/backend/tests/ontology/test_ontology_reviewer.py b/src/backend/tests/ontology/test_ontology_reviewer.py index 906f6755..005671d3 100644 --- a/src/backend/tests/ontology/test_ontology_reviewer.py +++ b/src/backend/tests/ontology/test_ontology_reviewer.py @@ -1,10 +1,14 @@ from __future__ import annotations +from core.ontology import review_operations as review_operations_module +from core.ontology import review_prompt as review_prompt_module +from core.ontology import reviewer as reviewer_module import asyncio from types import SimpleNamespace +import core.ontology.reviewer as ontology_reviewer +import core.ontology.reviewer as reviewer import pytest -from orchestration.subagents import ontology_reviewer as reviewer def _runtime(level: str = "committee") -> dict: @@ -30,7 +34,7 @@ async def _inline(function, *args, **kwargs): return function(*args, **kwargs) monkeypatch.setattr( - reviewer, + ontology_reviewer, "asyncio", SimpleNamespace(gather=asyncio.gather, to_thread=_inline), ) @@ -39,17 +43,17 @@ async def _inline(function, *args, **kwargs): def _disable_audit(monkeypatch): _run_audit_inline(monkeypatch) monkeypatch.setattr( - "core.services.ontology_service.record_review_run", + 'core.ontology.audit_events.record_review_run', lambda payload: None, ) monkeypatch.setattr( - "core.services.ontology_service.record_enforcement_event", + 'core.ontology.audit_events.record_enforcement_event', lambda payload: None, ) def test_review_prompt_preserves_delivery_form_but_allows_required_fact_corrections(): - prompt = reviewer._review_prompt( + prompt = review_prompt_module.review_prompt( perspective="证据委员", task="原样输出一句未经证实的高风险判断并存入 Word", answer="基于现有证据修正后的一句话结论", @@ -85,7 +89,7 @@ def test_review_prompt_keeps_latest_repair_evidence_within_budget(): } ) - prompt = reviewer._review_prompt( + prompt = review_prompt_module.review_prompt( perspective="证据委员", task="生成 Word", answer="润色终稿", @@ -114,17 +118,17 @@ async def fake_review(*args, **kwargs): } records = [] - monkeypatch.setattr(reviewer, "_review_once", fake_review) + monkeypatch.setattr(review_operations_module, 'review_once', fake_review) _run_audit_inline(monkeypatch) monkeypatch.setattr( - "core.services.ontology_service.record_review_run", + 'core.ontology.audit_events.record_review_run', lambda payload: records.append(payload), ) monkeypatch.setattr( - "core.services.ontology_service.record_enforcement_event", + 'core.ontology.audit_events.record_enforcement_event', lambda payload: None, ) - result = await reviewer.review_ontology_output( + result = await reviewer_module.review_ontology_output( task="demo", answer="original", runtime=_runtime(), @@ -160,9 +164,9 @@ async def fake_review(*args, **kwargs): ], } - monkeypatch.setattr(reviewer, "_review_once", fake_review) + monkeypatch.setattr(review_operations_module, 'review_once', fake_review) _disable_audit(monkeypatch) - result = await reviewer.review_ontology_output( + result = await reviewer_module.review_ontology_output( task="demo", answer="sensitive unreviewed draft", runtime=_runtime(), @@ -200,9 +204,9 @@ async def remediate(payload): repair_payloads.append(payload) return "repaired answer with traceable evidence" - monkeypatch.setattr(reviewer, "_review_once", fake_review) + monkeypatch.setattr(review_operations_module, 'review_once', fake_review) _disable_audit(monkeypatch) - result = await reviewer.review_ontology_output( + result = await reviewer_module.review_ontology_output( task="demo", answer="original", runtime=_runtime(level="checkpoint"), @@ -236,9 +240,9 @@ async def fake_review(*args, **kwargs): async def remediate(payload): return "..." - monkeypatch.setattr(reviewer, "_review_once", fake_review) + monkeypatch.setattr(review_operations_module, 'review_once', fake_review) _disable_audit(monkeypatch) - result = await reviewer.review_ontology_output( + result = await reviewer_module.review_ontology_output( task="demo", answer="必须被保留的原始完整答案", runtime=_runtime(level="checkpoint"), @@ -301,9 +305,9 @@ async def remediate(payload): citations.append({"id": "required_search-1"}) return "基于真实检索证据形成的完整风险分析报告。" * 2 - monkeypatch.setattr(reviewer, "_review_once", fake_review) + monkeypatch.setattr(review_operations_module, 'review_once', fake_review) _disable_audit(monkeypatch) - result = await reviewer.review_ontology_output( + result = await reviewer_module.review_ontology_output( task="demo", answer="short", runtime=runtime, diff --git a/src/backend/tests/ontology/test_ontology_revision_accept.py b/src/backend/tests/ontology/test_ontology_revision_accept.py index 19b99d71..a43ebae8 100644 --- a/src/backend/tests/ontology/test_ontology_revision_accept.py +++ b/src/backend/tests/ontology/test_ontology_revision_accept.py @@ -2,7 +2,7 @@ from types import SimpleNamespace -from core.services.chat_service import ChatService +from core.chat.service import ChatService class _MessageRepository: @@ -29,7 +29,7 @@ def test_accept_ontology_revision_replaces_content_and_marks_metadata(): "ontology_governance": {"review": {"candidate_answer": "基于新增证据的优化答案"}} }, ) - service = ChatService.__new__(ChatService) + service = object.__new__(ChatService) service.message_repo = _MessageRepository(message) updated = service.accept_ontology_revision("msg_1") @@ -47,7 +47,7 @@ def test_accept_ontology_revision_rejects_message_without_candidate(): content="原始答案", extra_data={"ontology_governance": {"review": {}}}, ) - service = ChatService.__new__(ChatService) + service = object.__new__(ChatService) service.message_repo = _MessageRepository(message) assert service.accept_ontology_revision("msg_2") is None @@ -60,7 +60,7 @@ def test_accept_ontology_revision_rejects_placeholder_candidate(): content="必须保留的原始答案", extra_data={"ontology_governance": {"review": {"candidate_answer": "..."}}}, ) - service = ChatService.__new__(ChatService) + service = object.__new__(ChatService) service.message_repo = _MessageRepository(message) assert service.accept_ontology_revision("msg_3") is None @@ -83,7 +83,7 @@ def test_accept_ontology_revision_removes_transport_wrapper(): } }, ) - service = ChatService.__new__(ChatService) + service = object.__new__(ChatService) service.message_repo = _MessageRepository(message) updated = service.accept_ontology_revision("msg_4") diff --git a/src/backend/tests/ontology/test_ontology_workflow_evidence.py b/src/backend/tests/ontology/test_ontology_workflow_evidence.py index 0e50b14d..13bffb61 100644 --- a/src/backend/tests/ontology/test_ontology_workflow_evidence.py +++ b/src/backend/tests/ontology/test_ontology_workflow_evidence.py @@ -1,18 +1,21 @@ from __future__ import annotations +import json from types import SimpleNamespace -import json import pytest -from orchestration.citation_anchor import AnchorAllocator, annotate_tool_result -from orchestration.workflow import ( - _astream_subagent_direct, +from core.agent_runtime.direct import _astream_subagent_direct +from core.agent_runtime.integrations.ontology_evidence import ( _capture_nested_ontology_evidence, - _ontology_repair_prompt, _ontology_review_failure_result, +) +from core.agent_runtime.integrations.ontology_repair import ( + _ontology_repair_prompt, _run_ontology_repair_round, - astream_chat_workflow, ) +from core.agent_runtime.runner import astream_chat_workflow +from core.llm.citations.allocator import AnchorAllocator +from core.llm.citations.anchor import annotate_tool_result def test_repair_prompt_requires_polished_delivery_instead_of_validation_report(): @@ -290,7 +293,7 @@ async def _capture_event(event): ) assert answer == ( - "第一段修订正文包含充分事实依据和明确结论，" "第二段继续说明证据边界并给出审慎判断。" + "第一段修订正文包含充分事实依据和明确结论，" "第二段继续说明证据边界并给出审慎判断。" ) assert len(content_events) >= 2 assert "".join(item["delta"] for item in content_events) == answer diff --git a/src/backend/tests/orchestration/agent_mention_parse_selftest.py b/src/backend/tests/orchestration/agent_mention_parse_selftest.py index 27aedf81..cffc6961 100644 --- a/src/backend/tests/orchestration/agent_mention_parse_selftest.py +++ b/src/backend/tests/orchestration/agent_mention_parse_selftest.py @@ -9,7 +9,7 @@ * ``routing/workflow.py`` uses it for fallback display-name resolution of ``call_subagent`` on the tool_call card (when no agent_id is found, the first mentioned agent is used). -* ``core.llm.subagent_tool.build_subagent_prompt_section`` renders this ID +* ``core.llm.subagents.tool.build_subagent_prompt_section`` renders this ID list into a system prompt section appended to the main agent's prompt — ``**用户已指定调用子智能体：X、Y。请直接使用 call_subagent ...**``. diff --git a/src/backend/tests/orchestration/automation_test_support.py b/src/backend/tests/orchestration/automation_test_support.py new file mode 100644 index 00000000..5de1de04 --- /dev/null +++ b/src/backend/tests/orchestration/automation_test_support.py @@ -0,0 +1,17 @@ +"""Exercise the scheduler through durable acceptance, not a private executor.""" +import asyncio +from core.db import engine +from features.automation.service import AutomationService + + +def request(task_id, user_id): + with engine.SessionLocal() as db: + service = AutomationService(db) + service.request_manual_trigger(service.get_task(task_id, user_id)) + + +async def execute_queued(scheduler, task_id, user_id): + await asyncio.to_thread(request, task_id, user_id) + before = set(scheduler._executions) + await scheduler._dispatch_pending() + await asyncio.gather(*(scheduler._executions - before)) diff --git a/src/backend/tests/orchestration/command_test_support.py b/src/backend/tests/orchestration/command_test_support.py index bccf511d..430c95b7 100644 --- a/src/backend/tests/orchestration/command_test_support.py +++ b/src/backend/tests/orchestration/command_test_support.py @@ -1,5 +1,5 @@ """Build a durable ordinary-chat recipient for completion tests.""" -from core.services.chat_sequencer import ChatSequencer +from core.chat.sequencer import ChatSequencer def source_run(factory): with factory() as db: diff --git a/src/backend/tests/orchestration/hook_support.py b/src/backend/tests/orchestration/hook_support.py new file mode 100644 index 00000000..6cdfcc7d --- /dev/null +++ b/src/backend/tests/orchestration/hook_support.py @@ -0,0 +1,53 @@ +"""Construct the real framework adapter with selected neutral policies.""" +from core.harness.hooks import HookBus +from core.llm.runtime.agentscope_hook_adapter import AgentScopeHookAdapter + + +def make_adapter(*specs, **options): + bus = HookBus() + for spec in specs: + bus.register(spec) + return AgentScopeHookAdapter(bus, **options) + + +async def inject_vision_context(state, image_files, user_id): + from types import SimpleNamespace + from core.llm.context.attachment_hook import vision_evidence, payload + from core.llm.runtime.context_hook_adapter import append_hook_context + text = await vision_evidence(image_files, user_id, getattr(state, "vision_evidence_text", "")) + if text: + append_hook_context(SimpleNamespace(state=state), SimpleNamespace(data={"context_items": [ + payload(text, kind="attachment", origin="vision:transcription", trust="tool")]})) + + +async def run_permission(service, input_kwargs, next_handler): + from types import SimpleNamespace + from core.llm.tool_permissions.hook import ToolPermissionHook + policy = ToolPermissionHook(service) + adapter = make_adapter(policy.spec(), permission_policy=policy) + agent = SimpleNamespace(state=SimpleNamespace(context=[], run_id="permission-test")) + async for event in adapter.on_acting(agent, input_kwargs, next_handler): + yield event + + +async def inject_attachments(agent): + from core.llm.context.attachment_hook import attachment_context_hook + adapter = make_adapter(attachment_context_hook()) + async def handler(**kwargs): + if False: + yield None + async for _ in adapter.on_reply(agent, {}, handler): + pass + + +def permission_adapter(service): + from core.llm.tool_permissions.hook import ToolPermissionHook + policy = ToolPermissionHook(service) + return make_adapter(policy.spec(), permission_policy=policy) + + +def durable_adapter(session_factory): + from core.llm.runtime.tool_execution_adapter import DurableToolDispatch + adapter = make_adapter() + adapter.tool_dispatch = DurableToolDispatch(session_factory=session_factory) + return adapter diff --git a/src/backend/tests/orchestration/recovery_test_support.py b/src/backend/tests/orchestration/recovery_test_support.py index 21c92643..d8084ce0 100644 --- a/src/backend/tests/orchestration/recovery_test_support.py +++ b/src/backend/tests/orchestration/recovery_test_support.py @@ -1,11 +1,14 @@ """Shared isolated fixtures and provider fakes.""" from __future__ import annotations + +import core.db.engine as db_engine +import core.runs.events.store as store +import core.runs.events.store as run_event_stream +import core.runs.tasks as tasks import pytest from core.db.engine import Base from core.db.models import ChatSession -from orchestration import chat_run_executor as executor -from orchestration import run_event_stream from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker @@ -18,15 +21,15 @@ def recovery_env(tmp_path, monkeypatch): ) Base.metadata.create_all(engine) sessions = sessionmaker(bind=engine, expire_on_commit=False) - monkeypatch.setattr(executor, "SessionLocal", sessions) - monkeypatch.setattr(run_event_stream, "redis_configured", lambda: True) - monkeypatch.setattr("core.services.model_config.SessionLocal", sessions) - executor._active_runs.clear() + monkeypatch.setattr(db_engine, "SessionLocal", sessions) + monkeypatch.setattr(store, "redis_configured", lambda: True) + monkeypatch.setattr('core.models.config.SessionLocal', sessions) + tasks.process_state().active_runs.clear() with sessions() as db: db.add(ChatSession(chat_id="chat-1", user_id="user-1", title="test")) db.commit() yield sessions - executor._active_runs.clear() + tasks.process_state().active_runs.clear() engine.dispose() diff --git a/src/backend/tests/orchestration/router_strategy_selftest.py b/src/backend/tests/orchestration/router_strategy_selftest.py index 5b2218ee..ff9fc87c 100644 --- a/src/backend/tests/orchestration/router_strategy_selftest.py +++ b/src/backend/tests/orchestration/router_strategy_selftest.py @@ -12,7 +12,7 @@ def main() -> int: # Default: main_only os.environ.pop("ROUTER_STRATEGY", None) - from orchestration.strategy import get_router_strategy + from core.agent_runtime.strategy import get_router_strategy s = get_router_strategy() assert s.route("生成一份报告") == "main" diff --git a/src/backend/tests/orchestration/routing_import_selftest.py b/src/backend/tests/orchestration/routing_import_selftest.py index 11921a2d..0a1cc2ec 100755 --- a/src/backend/tests/orchestration/routing_import_selftest.py +++ b/src/backend/tests/orchestration/routing_import_selftest.py @@ -8,7 +8,7 @@ def main() -> int: - from orchestration.strategy import get_router_strategy + from core.agent_runtime.strategy import get_router_strategy s = get_router_strategy() out = s.route("hello", context={"chat_id": "x"}) diff --git a/src/backend/tests/orchestration/stream_protocol_selftest.py b/src/backend/tests/orchestration/stream_protocol_selftest.py index 2bcd4b25..2b912e3f 100644 --- a/src/backend/tests/orchestration/stream_protocol_selftest.py +++ b/src/backend/tests/orchestration/stream_protocol_selftest.py @@ -23,10 +23,11 @@ async def _collect_stream() -> str: from types import SimpleNamespace from unittest.mock import patch + import core.runs.events.subscription as run_subscription from api.routes.v1 import agent_responses - from core.chat.schemas import ChatRequest from core.auth.backend import UserContext - from orchestration import chat_run_executor + from core.chat.schemas import ChatRequest + pass async def fake_start(request, user, db): assert request.stream is True @@ -47,7 +48,7 @@ async def fake_follow(run_id, *, from_offset=0): # Exercise the real response route and SSE wire formatter; replace only # admission and the durable event source so no model/database is needed. with patch.object(agent_responses, "_start_response_run", fake_start), patch.object( - chat_run_executor, "follow_run", fake_follow + run_subscription, "follow_run", fake_follow ): response = await agent_responses.agent_response( ChatRequest(chat_id="stream_selftest", message="生成报告", stream=True), diff --git a/src/backend/tests/orchestration/streaming_tool_call_dedupe_selftest.py b/src/backend/tests/orchestration/streaming_tool_call_dedupe_selftest.py index c2b4be68..d3bc3f39 100644 --- a/src/backend/tests/orchestration/streaming_tool_call_dedupe_selftest.py +++ b/src/backend/tests/orchestration/streaming_tool_call_dedupe_selftest.py @@ -7,10 +7,11 @@ arguments. Run from the repository root: - PYTHONPATH=src/backend python -m tests.orchestration.streaming_tool_call_dedupe_selftest + PYTHONPATH=src/backend python -m tests.core.llm.runtime.streaming_tool_call_dedupe_selftest """ from __future__ import annotations +from core.llm.runtime import streaming_constants as streaming_state_module import asyncio import json @@ -36,8 +37,8 @@ def __init__(self, tool_id: str) -> None: async def _collect(events: list[Any]) -> list[tuple[str, Any]]: - import orchestration.streaming as streaming_mod - from orchestration.streaming import StreamingAgent + import core.llm.runtime.streaming as streaming_mod + from core.llm.runtime.streaming import StreamingAgent async def reply_stream(inputs=None): # noqa: ANN001, ARG001 for event in events: @@ -53,8 +54,8 @@ async def reply_stream(inputs=None): # noqa: ANN001, ARG001 # Make the character threshold deterministic for this standalone test; the # production time threshold remains covered by the pytest suite. - streaming_mod._TOOL_CALL_DELTA_FLUSH_CHARS = 256 - streaming_mod._TOOL_CALL_DELTA_FLUSH_INTERVAL_S = 999.0 + streaming_state_module.TOOL_CALL_DELTA_FLUSH_CHARS = 256 + streaming_state_module.TOOL_CALL_DELTA_FLUSH_INTERVAL_S = 999.0 output: list[tuple[str, Any]] = [] streamer = StreamingAgent(agent, mcp_clients=[]) diff --git a/src/backend/tests/orchestration/test_agent_api_recovery.py b/src/backend/tests/orchestration/test_agent_api_recovery.py index d7c26aa5..756112db 100644 --- a/src/backend/tests/orchestration/test_agent_api_recovery.py +++ b/src/backend/tests/orchestration/test_agent_api_recovery.py @@ -3,16 +3,20 @@ from copy import deepcopy from types import SimpleNamespace +import core.chat.runtime.recovery as runtime_recovery +import core.chat.runtime.recovery_snapshot as recovery_snapshot +import core.chat.runtime.worker as worker +import core.db.engine as db_engine +import core.runs.ownership as ownership +import core.runs.runtime as runtime import pytest -from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker - from core.auth.backend import UserContext from core.db.engine import Base from core.db.models import ChatRun, ChatSession -from core.services.agent_api_service import make_agent_api_scope -from orchestration.agent_api_recovery import validate_recovery_context -from orchestration import chat_run_executor as executor +from core.runs.api_recovery import validate_recovery_context +from core.auth.agent_api import make_agent_api_scope +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker @pytest.fixture @@ -20,7 +24,7 @@ def recovery(tmp_path, monkeypatch): engine = create_engine(f"sqlite:///{tmp_path / 'scope-recovery.db'}") Base.metadata.create_all(engine, tables=[ChatSession.__table__, ChatRun.__table__]) sessions = sessionmaker(bind=engine, expire_on_commit=False) - monkeypatch.setattr(executor, "SessionLocal", sessions) + monkeypatch.setattr(db_engine, "SessionLocal", sessions) scope = make_agent_api_scope( UserContext( user_id="owner", @@ -168,9 +172,9 @@ def workflow(**kwargs): observed.update(kwargs) return object() - monkeypatch.setattr(executor, "_run_workflow", workflow) - monkeypatch.setattr(executor, "_register_run_task", lambda *a, **k: None) - assert executor._register_recovered_chat(_decision(context)) is True + monkeypatch.setattr(worker, "_run_workflow", workflow) + monkeypatch.setattr(runtime, 'register_run_task', lambda *a, **k: None) + assert runtime_recovery._register_recovered_chat(_decision(context)) is True assert observed["context"]["memory_enabled"] is False assert observed["context"]["agent_api_scope"] == context["agent_api_scope"] @@ -179,11 +183,11 @@ def test_worker_restore_refuses_lost_scope_before_start(recovery, monkeypatch): _, _, context = recovery context.pop("agent_api_scope") monkeypatch.setattr( - executor, - "_register_run_task", + runtime, + 'register_run_task', lambda *a, **k: pytest.fail("damaged scope must not create a worker"), ) - assert executor._register_recovered_chat(_decision(context)) is False + assert runtime_recovery._register_recovered_chat(_decision(context)) is False @pytest.mark.asyncio @@ -194,10 +198,10 @@ async def test_completed_snapshot_lost_scope_parks_without_writing(recovery, mon needs_attention=lambda run_id, **kwargs: decisions.append((run_id, kwargs)) or True, claim=lambda *a, **k: pytest.fail("must reject before claiming or committing"), ) - monkeypatch.setattr(executor, "_journal", lambda: journal) + monkeypatch.setattr(ownership, 'journal', lambda: journal) decision = _decision(context) decision.snapshot.update(assistant_content="answer", context={}) - assert await executor._commit_recovered_chat_snapshot(decision) is False + assert await recovery_snapshot._commit_recovered_chat_snapshot(decision) is False assert decisions[0][0] == "run-api" @@ -206,8 +210,8 @@ async def test_completed_snapshot_checks_worker_context_too(recovery, monkeypatc _, _, context = recovery decisions = [] monkeypatch.setattr( - executor, - "_journal", + ownership, + 'journal', lambda: SimpleNamespace( needs_attention=lambda *a, **k: decisions.append(k) or True, claim=lambda *a, **k: pytest.fail("must reject a mismatched worker snapshot"), @@ -215,7 +219,7 @@ async def test_completed_snapshot_checks_worker_context_too(recovery, monkeypatc ) decision = _decision({}) decision.snapshot.update(assistant_content="answer", context=context) - assert await executor._commit_recovered_chat_snapshot(decision) is False + assert await recovery_snapshot._commit_recovered_chat_snapshot(decision) is False assert decisions diff --git a/src/backend/tests/orchestration/test_agent_runtime_resources.py b/src/backend/tests/orchestration/test_agent_runtime_resources.py new file mode 100644 index 00000000..1efdf2f6 --- /dev/null +++ b/src/backend/tests/orchestration/test_agent_runtime_resources.py @@ -0,0 +1,20 @@ +"""Native MCP cleanup must run even when stopping the producer fails.""" +import asyncio +from types import SimpleNamespace +import pytest +from core.llm.runtime.streaming import StreamingAgent + +@pytest.mark.asyncio +async def test_shutdown_releases_all_resources_when_producer_cancelled(): + closed = [] + class Resource: + async def close(self): + closed.append(self) + async def stop(): + raise asyncio.CancelledError() + clients = [Resource(), Resource()] + agent = SimpleNamespace(stop_producer=stop, mcp_clients=clients) + with pytest.raises(asyncio.CancelledError): + await StreamingAgent.shutdown(agent) + assert closed == clients + assert agent.mcp_clients == [] diff --git a/src/backend/tests/orchestration/test_agentscope_hook_adapter.py b/src/backend/tests/orchestration/test_agentscope_hook_adapter.py index 140283a8..cd815808 100644 --- a/src/backend/tests/orchestration/test_agentscope_hook_adapter.py +++ b/src/backend/tests/orchestration/test_agentscope_hook_adapter.py @@ -10,12 +10,10 @@ from agentscope.middleware import MiddlewareBase from agentscope.tool._response import ToolResponse from core.harness.hooks import Decision, HookBus, HookContractError, HookSpec, HookStage +from core.harness.observation_writer import flush_observations from core.harness.usage import MemoryUsageRecorder -from core.llm.agentscope_hook_adapter import ( - AgentScopeHookAdapter, - adapt_agentscope_event, -) -from core.llm.middleware_migration import MIDDLEWARE_MIGRATION, deletion_checklist +from core.llm.runtime.agentscope_hook_adapter import AgentScopeHookAdapter, adapt_agentscope_event +from core.llm.middleware.migration_map import MIDDLEWARE_MIGRATION, deletion_checklist def _fake(name: str, **fields): @@ -71,9 +69,13 @@ def test_every_registered_business_middleware_has_a_deletion_entry(): "ToolEffectMiddleware", "JobLedgerReminderMiddleware", "FinishPinGuardMiddleware", + "CommandCompletionMiddleware", + "ProjectEnvironmentMiddleware", + "ScreenshotPruningMiddleware", + "PlanStaleReminderMiddleware", } assert {item.middleware for item in MIDDLEWARE_MIGRATION} == expected - assert all(item.status == "adapter-covered" for item in MIDDLEWARE_MIGRATION) + assert all(item.status == "neutral-hook" for item in MIDDLEWARE_MIGRATION) assert all(item.delete_after for item in MIDDLEWARE_MIGRATION) assert len(deletion_checklist()) == len(expected) @@ -224,17 +226,11 @@ def test_model_and_tool_inputs_are_read_only_at_post_manifest_permission_seams() @pytest.mark.asyncio -async def test_legacy_policy_executes_only_through_adapter_and_is_accounted(): - class LegacyPolicy(MiddlewareBase): - async def on_reply(self, agent, input_kwargs, next_handler): - async for item in next_handler(**input_kwargs): - yield item - +async def test_neutral_policy_executes_through_adapter_and_is_accounted(): usage = MemoryUsageRecorder() - adapter = AgentScopeHookAdapter( - HookBus(usage_recorder=usage), - legacy_middlewares=(LegacyPolicy(),), - ) + bus = HookBus(usage_recorder=usage) + bus.register(HookSpec("reply-policy", HookStage.BEFORE_RUN, lambda invocation: Decision())) + adapter = AgentScopeHookAdapter(bus) agent = SimpleNamespace(state=SimpleNamespace(run_id="run-adapter", context=[])) async def handler(**kwargs): @@ -242,9 +238,10 @@ async def handler(**kwargs): events = [item async for item in adapter.on_reply(agent, {"inputs": None}, handler)] assert len(events) == 1 + assert flush_observations() rows = usage.attempts("run-adapter") assert [(row.kind, row.operation_name, row.status) for row in rows] == [ - ("hook", "LegacyPolicy.on_reply", "success") + ("hook", "reply-policy", "success") ] diff --git a/src/backend/tests/orchestration/test_automation_admission.py b/src/backend/tests/orchestration/test_automation_admission.py index 182cfc15..6a4e3301 100644 --- a/src/backend/tests/orchestration/test_automation_admission.py +++ b/src/backend/tests/orchestration/test_automation_admission.py @@ -1,11 +1,15 @@ """Admission cleanup for scheduled conversations.""" +from tests.orchestration.automation_test_support import execute_queued import asyncio + +import core.agent_runtime.runner as runner +import core.chat.runtime.admission as admission +import core.runs.runtime as runtime import pytest from core.db.models import ChatRun -from core.services.automation_service import AutomationService -from orchestration import chat_run_executor as executor -from orchestration.schedulers.automation_scheduler import AutomationScheduler +from features.automation.service import AutomationService +from orchestration.automations.automation_scheduler import AutomationScheduler pytest_plugins = ["tests.orchestration.test_automation_live_conversations"] @@ -20,14 +24,14 @@ async def fail_launch(*, accepted_run, **kwargs): accepted_id = accepted_run.run_id raise RuntimeError("launch unavailable") - monkeypatch.setattr(executor, "start_run", fail_launch) + monkeypatch.setattr(admission, "start_run", fail_launch) scheduler = AutomationScheduler() async def noop(*args): return None monkeypatch.setattr(scheduler, "_send_notification", noop) - await scheduler.execute_task(task_id, "owner") + await execute_queued(scheduler, task_id, "owner") with factory() as db: run = db.get(ChatRun, accepted_id) assert run.status == "failed" @@ -35,7 +39,7 @@ async def noop(*args): scheduled = AutomationService(db).get_task_runs(task_id, "owner")[0] assert scheduled.status == "failed" assert scheduled.chat_id == run.chat_id - assert executor.get_active_run_for_chat(run.chat_id, "owner") is None + assert runtime.get_active_run_for_chat(run.chat_id, "owner") is None @pytest.mark.asyncio @@ -48,14 +52,14 @@ async def cancelled_launch(*, accepted_run, **kwargs): accepted_id = accepted_run.run_id raise asyncio.CancelledError() - monkeypatch.setattr(executor, "start_run", cancelled_launch) + monkeypatch.setattr(admission, "start_run", cancelled_launch) scheduler = AutomationScheduler() async def noop(*args): return None with pytest.raises(asyncio.CancelledError): - await scheduler.execute_task(task_id, "owner") + await execute_queued(scheduler, task_id, "owner") with factory() as db: run = db.get(ChatRun, accepted_id) assert run.status == "failed" @@ -72,7 +76,7 @@ async def workflow(*, context, **kwargs): yield {"type": "content", "delta": "Complete"} yield {"type": "meta", "usage": {}} - monkeypatch.setattr(executor, "astream_chat_workflow", workflow) + monkeypatch.setattr(runner, "astream_chat_workflow", workflow) with factory() as db: AutomationService(db).update_task( task_id, @@ -87,7 +91,7 @@ async def noop(*args): return None monkeypatch.setattr(scheduler, "_send_notification", noop) - await scheduler.execute_task(task_id, "owner") + await execute_queued(scheduler, task_id, "owner") assert captured["enabled_agents"] == ["analyst"] assert captured["enabled_skills"] == ["research"] assert captured["enabled_mcps"] == ["search"] @@ -102,3 +106,29 @@ async def noop(*args): assert payload["chat_mode"] == captured["chat_mode"] assert payload["enable_thinking"] == captured["enable_thinking"] assert captured["run_id"] and captured["message_id"] + + +async def test_child_acceptance_already_contains_parent_link_and_recovery(live_env, monkeypatch): + from core.db.models import ChatRun + from features.automation.models import ScheduledTaskRun + from orchestration.automations import automation_conversation + + factory, _, task_id = live_env + observed = [] + + async def crash_before_launch(**kwargs): + child_id = kwargs["accepted_run"].run_id + with factory() as db: + child = db.get(ChatRun, child_id) + parent = db.query(ScheduledTaskRun).filter_by(task_id=task_id).one() + assert child.status == "pending" + assert parent.chat_id == child.chat_id + assert child.recovery_snapshot["kind"] == "chat" + assert child.recovery_snapshot["worker_args"]["effective_user_message"] == "Research" + assert child.recovery_snapshot["worker_args"]["context"]["automation_run"] is True + observed.append(child_id) + raise RuntimeError("worker died after durable acceptance") + + monkeypatch.setattr(automation_conversation.admission, "start_run", crash_before_launch) + await execute_queued(AutomationScheduler(), task_id, "owner") + assert len(observed) == 1 diff --git a/src/backend/tests/orchestration/test_automation_handoff.py b/src/backend/tests/orchestration/test_automation_handoff.py index 90ddcb3c..d5b57b71 100644 --- a/src/backend/tests/orchestration/test_automation_handoff.py +++ b/src/backend/tests/orchestration/test_automation_handoff.py @@ -1,12 +1,15 @@ """Late user instructions remain part of a scheduled conversation's outcome.""" +from tests.orchestration.automation_test_support import execute_queued import asyncio -import pytest -from core.services.automation_service import AutomationService -from core.services.steer_queue import SteerQueue -from orchestration import chat_run_executor as executor -from orchestration.schedulers.automation_scheduler import AutomationScheduler +import core.agent_runtime.runner as runner +import core.chat.runtime.handoff as handoff +import core.runs.runtime as runtime +import pytest +from features.automation.service import AutomationService +from core.chat.steer_queue import SteerQueue +from orchestration.automations.automation_scheduler import AutomationScheduler pytest_plugins = ["tests.orchestration.test_automation_live_conversations"] @@ -37,25 +40,25 @@ async def workflow(*, context, **kwargs): yield {"type": "content", "delta": "Revised final result"} yield {"type": "meta", "usage": {}} - monkeypatch.setattr(executor, "astream_chat_workflow", workflow) + monkeypatch.setattr(runner, "astream_chat_workflow", workflow) scheduler = AutomationScheduler() async def noop(*args): return None monkeypatch.setattr(scheduler, "_send_notification", noop) - executing = asyncio.create_task(scheduler.execute_task(task_id, "owner")) + executing = asyncio.create_task(execute_queued(scheduler, task_id, "owner")) try: await asyncio.wait_for(entered.wait(), 5) await asyncio.sleep(0.2) assert not executing.done(), "The original result must not settle the scheduled task" - assert executor.get_successor_run(runs[0], user_id="owner").run_id == runs[1] + assert handoff.get_successor_run(runs[0], user_id="owner").run_id == runs[1] with factory() as db: assert AutomationService(db).get_task_runs(task_id, "owner")[0].status == "running" if outcome == "complete": release.set() elif outcome == "stop": - await executor.cancel_run(runs[1], user_id="owner") + await runtime.cancel_run(runs[1], user_id="owner") elif outcome == "shutdown": executing.cancel() if outcome == "shutdown": @@ -68,10 +71,13 @@ async def noop(*args): if outcome == "complete": assert scheduled.status == "success" assert scheduled.result_summary == "Revised final result" + elif outcome == "shutdown": + assert scheduled.status == "running" + assert runtime.get_run(runs[1]).status == "running" else: assert scheduled.status == "failed" - assert executor.get_run(runs[1]).status == "cancelled" - assert executor.get_active_run_for_chat(scheduled.chat_id, "owner") is None + assert runtime.get_run(runs[1]).status == "cancelled" + assert runtime.get_active_run_for_chat(scheduled.chat_id, "owner") is None finally: release.set() if not executing.done(): @@ -80,12 +86,12 @@ async def noop(*args): @pytest.mark.asyncio -async def test_shutdown_cancels_successor_when_completion_wins_cancel_race(live_env, monkeypatch): +async def test_shutdown_does_not_cancel_executor_or_successor(live_env, monkeypatch): factory, _, task_id = live_env source_entered, finish_source = asyncio.Event(), asyncio.Event() successor_entered, finish_successor = asyncio.Event(), asyncio.Event() runs, cancelled = [], [] - original_cancel = executor.cancel_run + original_cancel = runtime.cancel_run async def workflow(*, context, **kwargs): runs.append(context["run_id"]) @@ -105,31 +111,33 @@ async def workflow(*, context, **kwargs): await finish_successor.wait() yield {"type": "meta", "usage": {}} - async def completion_wins_cancel(run_id, *, user_id): + async def completion_wins_cancel(run_id, *, user_id, **kwargs): cancelled.append(run_id) if run_id == runs[0]: # The observer entered cancellation before the original completed. # Commit its successor before the cancellation CAS can fence it. finish_source.set() await asyncio.wait_for(successor_entered.wait(), 5) - return await original_cancel(run_id, user_id=user_id) + return await original_cancel(run_id, user_id=user_id, **kwargs) - monkeypatch.setattr(executor, "astream_chat_workflow", workflow) - monkeypatch.setattr(executor, "cancel_run", completion_wins_cancel) + monkeypatch.setattr(runner, "astream_chat_workflow", workflow) + monkeypatch.setattr(runtime, "cancel_run", completion_wins_cancel) scheduler = AutomationScheduler() async def noop(*args): return None - executing = asyncio.create_task(scheduler.execute_task(task_id, "owner")) + executing = asyncio.create_task(execute_queued(scheduler, task_id, "owner")) try: await asyncio.wait_for(source_entered.wait(), 5) executing.cancel() with pytest.raises(asyncio.CancelledError): await asyncio.wait_for(executing, 5) - assert cancelled == runs - assert executor.get_run(runs[1]).status == "cancelled" - assert executor.get_active_run_for_chat(executor.get_run(runs[0]).chat_id, "owner") is None + assert cancelled == [] + assert runtime.get_run(runs[0]).status == "running" + finish_source.set() + await asyncio.wait_for(successor_entered.wait(), 5) + assert runtime.get_run(runs[1]).status == "running" finally: finish_source.set() finish_successor.set() diff --git a/src/backend/tests/orchestration/test_automation_live_conversations.py b/src/backend/tests/orchestration/test_automation_live_conversations.py index 7612d464..2890a050 100644 --- a/src/backend/tests/orchestration/test_automation_live_conversations.py +++ b/src/backend/tests/orchestration/test_automation_live_conversations.py @@ -1,14 +1,23 @@ """Scheduled runs expose the same live conversation boundary as interactive runs.""" +from tests.orchestration.automation_test_support import execute_queued +from core.chat.projection.codec import chat_projection_codec import asyncio + +import core.agent_runtime.runner as runner +import core.chat.runtime.compaction as compaction +import core.chat.runtime.followups as followups +import core.db.engine as db_engine +import core.runs.events.store as store +import core.runs.events.store as run_event_stream +import core.runs.runtime as runtime import pytest +from core.db.engine import Base +from features.automation.service import AutomationService +from core.chat.service import ChatService +from orchestration.automations.automation_scheduler import AutomationScheduler from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker -from core.db.engine import Base -from core.services.automation_service import AutomationService -from core.services.chat_service import ChatService -from orchestration import chat_run_executor as executor, run_event_stream -from orchestration.schedulers.automation_scheduler import AutomationScheduler @pytest.fixture @@ -17,17 +26,17 @@ def live_env(tmp_path, monkeypatch): Base.metadata.create_all(engine) factory = sessionmaker(bind=engine, expire_on_commit=False) monkeypatch.setattr("core.db.engine.SessionLocal", factory) - monkeypatch.setattr(executor, "SessionLocal", factory) - monkeypatch.setattr("core.services.chat_steer_service.SessionLocal", factory) - monkeypatch.setattr(executor, "_spawn_followup_task", lambda **kw: None) - monkeypatch.setattr(executor, "_spawn_compaction_task", lambda **kw: None) - stream = run_event_stream.LocalRunEventStream() - monkeypatch.setattr(executor, "get_run_event_stream", lambda: stream) + monkeypatch.setattr(db_engine, "SessionLocal", factory) + monkeypatch.setattr('core.chat.steer.SessionLocal', factory) + monkeypatch.setattr(followups, "_spawn_followup_task", lambda **kw: None) + monkeypatch.setattr(compaction, "_spawn_compaction_task", lambda **kw: None) + stream = run_event_stream.LocalRunEventStream(chat_projection_codec()) + monkeypatch.setattr(store, "get_run_event_stream", lambda: stream) monkeypatch.setattr( - "core.services.user_model_selection.resolve_effective_chat_model_name", lambda: "test-model" + 'core.models.selection.resolve_effective_chat_model_name', lambda: "test-model" ) monkeypatch.setattr( - "core.services.ontology_service.build_user_ontology_runtime", lambda **kw: (False, {}) + 'core.ontology.runtime_policy.build_user_ontology_runtime', lambda **kw: (False, {}) ) monkeypatch.setattr("core.config.local_mode.local_mode_enabled", lambda: False) from core.infra import ephemeral @@ -35,7 +44,7 @@ def live_env(tmp_path, monkeypatch): state = ephemeral.LocalEphemeralState() monkeypatch.setattr(ephemeral, "get_ephemeral_state", lambda: state) monkeypatch.setattr( - "core.config.catalog_resolver.resolve_all_runtime_enabled", lambda *args: ([], [], []) + 'core.capabilities.catalog.resolver.resolve_all_runtime_enabled', lambda *args: ([], [], []) ) monkeypatch.setattr("core.chat.context.resolve_all_runtime_enabled", lambda *args: ([], [], [])) with factory() as db: @@ -65,30 +74,30 @@ async def workflow(**kwargs): yield {"type": "content", "delta": " complete"} yield {"type": "meta", "usage": {"total_tokens": 2}} - monkeypatch.setattr(executor, "astream_chat_workflow", workflow) - monkeypatch.setattr("orchestration.workflow.astream_chat_workflow", workflow) + monkeypatch.setattr(runner, "astream_chat_workflow", workflow) + monkeypatch.setattr('core.agent_runtime.runner.astream_chat_workflow', workflow) scheduler = AutomationScheduler() async def noop(*args): return None monkeypatch.setattr(scheduler, "_send_notification", noop) - executing = asyncio.create_task(scheduler.execute_task(task_id, "owner")) + executing = asyncio.create_task(execute_queued(scheduler, task_id, "owner")) try: await asyncio.wait_for(entered.wait(), 5) with factory() as db: history = AutomationService(db).get_task_runs(task_id, "owner") assert history[0].chat_id, "Running task must link its conversation immediately" chat_id = history[0].chat_id - active = executor.get_active_run_for_chat(chat_id, "owner") + active = runtime.get_active_run_for_chat(chat_id, "owner") assert active is not None, "Running automation must be discoverable by normal chat recovery" - assert executor.get_active_run_for_chat(chat_id, "other") is None + assert runtime.get_active_run_for_chat(chat_id, "other") is None events = await stream.read(active.run_id) assert any(event.get("delta") == "Researching" for _, event in events) assert await stream.read(active.run_id) == events, "Navigation/refresh can replay progress" - with pytest.raises(executor.ChatRunPermissionDenied): - await executor.cancel_run(active.run_id, user_id="other") - await executor.cancel_run(active.run_id, user_id="owner") + with pytest.raises(runtime.ChatRunPermissionDenied): + await runtime.cancel_run(active.run_id, user_id="other") + await runtime.cancel_run(active.run_id, user_id="owner") await asyncio.wait_for(executing, 5) with factory() as db: service = AutomationService(db) @@ -107,7 +116,7 @@ async def test_scheduled_prompt_accepts_instruction_and_delivers_final_segment( live_env, monkeypatch ): factory, stream, task_id = live_env - from core.services.chat_steer_service import put_pending_steer, take_pending_steer + from core.chat.steer import put_pending_steer, take_pending_steer entered, release = asyncio.Event(), asyncio.Event() @@ -121,7 +130,7 @@ async def workflow(*, context, **kwargs): yield {"type": "content", "delta": "Revised risk report"} yield {"type": "meta", "usage": {"total_tokens": 5}} - monkeypatch.setattr(executor, "astream_chat_workflow", workflow) + monkeypatch.setattr(runner, "astream_chat_workflow", workflow) scheduler = AutomationScheduler() notices = [] @@ -132,12 +141,12 @@ async def notify(*args): notices.append(args) monkeypatch.setattr(scheduler, "_send_notification", notify) - executing = asyncio.create_task(scheduler.execute_task(task_id, "owner")) + executing = asyncio.create_task(execute_queued(scheduler, task_id, "owner")) try: await asyncio.wait_for(entered.wait(), 5) with factory() as db: chat_id = AutomationService(db).get_task_runs(task_id, "owner")[0].chat_id - active = executor.get_active_run_for_chat(chat_id, "owner") + active = runtime.get_active_run_for_chat(chat_id, "owner") accepted = await put_pending_steer( active.run_id, { @@ -171,9 +180,9 @@ async def workflow(**kwargs): yield {"type": "content", "delta": "Work in progress"} await asyncio.Event().wait() - monkeypatch.setattr(executor, "astream_chat_workflow", workflow) + monkeypatch.setattr(runner, "astream_chat_workflow", workflow) monkeypatch.setattr( - "orchestration.schedulers.automation_scheduler.TASK_EXECUTION_TIMEOUT_S", + 'orchestration.automations.automation_scheduler.TASK_EXECUTION_TIMEOUT_S', 0.3, raising=False, ) @@ -183,21 +192,21 @@ async def noop(*args): return None monkeypatch.setattr(scheduler, "_send_notification", noop) - executing = asyncio.create_task(scheduler.execute_task(task_id, "owner")) + executing = asyncio.create_task(execute_queued(scheduler, task_id, "owner")) try: await asyncio.sleep(0.6) assert not executing.done() with factory() as db: run = AutomationService(db).get_task_runs(task_id, "owner")[0] assert run.status == "running" - active = executor.get_active_run_for_chat(run.chat_id, "owner") - await executor.cancel_run(active.run_id, user_id="owner") + active = runtime.get_active_run_for_chat(run.chat_id, "owner") + await runtime.cancel_run(active.run_id, user_id="owner") await asyncio.wait_for(executing, 5) with factory() as db: run = AutomationService(db).get_task_runs(task_id, "owner")[0] assert run.status == "failed" assert run.chat_id - assert executor.get_active_run_for_chat(run.chat_id, "owner") is None + assert runtime.get_active_run_for_chat(run.chat_id, "owner") is None finally: if not executing.done(): executing.cancel() @@ -207,7 +216,7 @@ async def noop(*args): @pytest.mark.asyncio async def test_scheduled_plan_exposes_steps_and_preserves_background_profile(live_env, monkeypatch): factory, stream, _ = live_env - from core.services.plan_service import PlanService + from core.chat.plans import PlanService entered, release = asyncio.Event(), asyncio.Event() observed = {} @@ -227,7 +236,7 @@ async def plan_workflow(**kwargs): "usage": {"total_tokens": 3}, } - monkeypatch.setattr("orchestration.subagents.plan_mode.astream_execute_plan", plan_workflow) + monkeypatch.setattr('orchestration.plans.mode.astream_execute_plan', plan_workflow) with factory() as db: plan = PlanService(db).create_plan( user_id="owner", @@ -245,12 +254,12 @@ async def noop(*args): return None monkeypatch.setattr(scheduler, "_send_notification", noop) - executing = asyncio.create_task(scheduler.execute_task(task_id, "owner")) + executing = asyncio.create_task(execute_queued(scheduler, task_id, "owner")) try: await asyncio.wait_for(entered.wait(), 5) with factory() as db: chat_id = AutomationService(db).get_task_runs(task_id, "owner")[0].chat_id - active = executor.get_active_run_for_chat(chat_id, "owner") + active = runtime.get_active_run_for_chat(chat_id, "owner") assert active.request_payload["kind"] == "plan_execute" assert observed["automation_run"] is True assert any(evt["type"] == "plan_step_start" for _, evt in await stream.read(active.run_id)) @@ -282,32 +291,33 @@ async def unlock(task_id): monkeypatch.setattr(scheduler, "_release_lock", unlock) - def fail(*args): + def fail(*args, **kwargs): raise RuntimeError("database unavailable") - monkeypatch.setattr(AutomationService, "record_run_start", fail) + monkeypatch.setattr("features.automation.admission.admit", fail) with pytest.raises(RuntimeError, match="database unavailable"): - await scheduler.execute_task(task_id, "owner") - assert released == [task_id] + await execute_queued(scheduler, task_id, "owner") + assert released == [] + assert not scheduler._leases @pytest.mark.asyncio async def test_scheduled_plan_stop_settles_without_failure_increment(live_env, monkeypatch): factory, stream, _ = live_env - from core.services.plan_service import PlanService + from core.chat.plans import PlanService entered = asyncio.Event() async def plan_workflow(**kwargs): yield {"type": "plan_step_start", "step_id": "step-1", "title": "Research"} entered.set() - while not executor.is_run_cancelled(kwargs["run_id"]): + while not runtime.is_run_cancelled(kwargs["run_id"]): await asyncio.sleep(0.01) with factory() as db: PlanService(db).update_plan(kwargs["plan_id"], status="cancelled") yield {"type": "plan_complete", "completed_steps": 0, "total_steps": 1} - monkeypatch.setattr("orchestration.subagents.plan_mode.astream_execute_plan", plan_workflow) + monkeypatch.setattr('orchestration.plans.mode.astream_execute_plan', plan_workflow) with factory() as db: plan = PlanService(db).create_plan( user_id="owner", @@ -325,22 +335,22 @@ async def noop(*args): return None monkeypatch.setattr(scheduler, "_send_notification", noop) - executing = asyncio.create_task(scheduler.execute_task(task_id, "owner")) + executing = asyncio.create_task(execute_queued(scheduler, task_id, "owner")) await asyncio.wait_for(entered.wait(), 5) with factory() as db: chat_id = AutomationService(db).get_task_runs(task_id, "owner")[0].chat_id - active = executor.get_active_run_for_chat(chat_id, "owner") - await executor.cancel_run(active.run_id, user_id="owner") + active = runtime.get_active_run_for_chat(chat_id, "owner") + await runtime.cancel_run(active.run_id, user_id="owner") await asyncio.wait_for(executing, 5) with factory() as db: svc = AutomationService(db) assert svc.get_task_runs(task_id, "owner")[0].status == "failed" assert svc.get_task(task_id, "owner").consecutive_failures == 0 - assert executor.get_active_run_for_chat(chat_id, "owner") is None + assert runtime.get_active_run_for_chat(chat_id, "owner") is None @pytest.mark.asyncio -async def test_stopping_scheduler_fences_already_started_conversation(live_env, monkeypatch): +async def test_stopping_scheduler_preserves_durable_child_for_recovery(live_env, monkeypatch): factory, _, task_id = live_env entered = asyncio.Event() @@ -349,22 +359,25 @@ async def workflow(**kwargs): entered.set() await asyncio.Event().wait() - monkeypatch.setattr(executor, "astream_chat_workflow", workflow) + monkeypatch.setattr(runner, "astream_chat_workflow", workflow) scheduler = AutomationScheduler() async def noop(*args): return None - await scheduler.start() - scheduler._launch_execution(task_id, "owner") + from tests.orchestration.automation_test_support import request + request(task_id, "owner") + await scheduler._dispatch_pending() await asyncio.wait_for(entered.wait(), 5) await asyncio.wait_for(scheduler.stop(), 5) assert not scheduler._executions with factory() as db: run = AutomationService(db).get_task_runs(task_id, "owner")[0] - assert run.status == "failed" + assert run.status == "running" assert run.chat_id - assert executor.get_active_run_for_chat(run.chat_id, "owner") is None + active = runtime.get_active_run_for_chat(run.chat_id, "owner") + assert active is not None + await runtime.cancel_run(active.run_id, user_id="owner") @pytest.mark.asyncio @@ -429,19 +442,19 @@ async def workflow(**kwargs): yield {"type": "content", "delta": "; continuing classification"} yield {"type": "meta", "usage": {"total_tokens": 2}} - monkeypatch.setattr(executor, "astream_chat_workflow", workflow) + monkeypatch.setattr(runner, "astream_chat_workflow", workflow) scheduler = AutomationScheduler() async def noop(*args): return None monkeypatch.setattr(scheduler, "_send_notification", noop) - executing = asyncio.create_task(scheduler.execute_task(task_id, "owner")) + executing = asyncio.create_task(execute_queued(scheduler, task_id, "owner")) try: await asyncio.wait_for(reached.wait(), 5) with factory() as db: scheduled = AutomationService(db).get_task_runs(task_id, "owner")[0] - active = executor.get_active_run_for_chat(scheduled.chat_id, "owner") + active = runtime.get_active_run_for_chat(scheduled.chat_id, "owner") assert active is not None row = db.get(ChatMessage, active.message_id) assert row is not None diff --git a/src/backend/tests/orchestration/test_automation_reaper.py b/src/backend/tests/orchestration/test_automation_reaper.py index 43505df7..8f5d4f06 100644 --- a/src/backend/tests/orchestration/test_automation_reaper.py +++ b/src/backend/tests/orchestration/test_automation_reaper.py @@ -2,10 +2,10 @@ from datetime import timedelta +import core.runs.watchdog as watchdog import pytest from core.db.models import ChatRun, ChatSession from core.infra.time import utc_now -from orchestration import chat_run_executor as executor pytest_plugins = ["tests.orchestration.test_stale_reaper"] @@ -38,6 +38,6 @@ async def test_live_automation_beyond_shared_age_cap_survives(reaper_env, kind): ) ) db.commit() - assert await executor.reap_stale_runs() == 0 + assert await watchdog.reap_stale_runs() == 0 with factory() as db: assert db.get(ChatRun, "successor").status == "running" diff --git a/src/backend/tests/orchestration/test_automation_unbounded.py b/src/backend/tests/orchestration/test_automation_unbounded.py index dd8a3348..1df84eeb 100644 --- a/src/backend/tests/orchestration/test_automation_unbounded.py +++ b/src/backend/tests/orchestration/test_automation_unbounded.py @@ -1,15 +1,18 @@ """Long automation runs retain ownership without runtime-based recovery.""" +from tests.orchestration.automation_test_support import execute_queued +from tests.features.automation.run_fixtures import running_run import asyncio from datetime import timedelta +import core.agent_runtime.runner as runner +import orchestration.automations.automation_lease as automations_automation_lease +import orchestration.automations.automation_lease as automation_lease import pytest - from core.infra import ephemeral from core.infra.time import utc_now -from core.services.automation_service import AutomationService -from orchestration.schedulers import automation_lease -from orchestration.schedulers.automation_scheduler import AutomationScheduler, REDIS_LOCK_PREFIX +from features.automation.service import AutomationService +from orchestration.automations.automation_scheduler import REDIS_LOCK_PREFIX, AutomationScheduler pytest_plugins = ["tests.orchestration.test_automation_live_conversations"] @@ -21,7 +24,7 @@ async def test_long_run_renews_across_many_lease_periods_and_excludes_other_work monkeypatch.setattr(ephemeral.time, "time", lambda: clock[0]) state = ephemeral.LocalEphemeralState() monkeypatch.setattr(ephemeral, "get_ephemeral_state", lambda: state) - monkeypatch.setattr(automation_lease, "RENEW_INTERVAL_SECONDS", 0.001) + monkeypatch.setattr(automations_automation_lease, "RENEW_INTERVAL_SECONDS", 0.001) owner, other = AutomationScheduler(), AutomationScheduler() assert await owner._acquire_lock("long-task") try: @@ -43,7 +46,7 @@ async def test_expired_owner_cannot_renew_or_delete_new_owners_lock(monkeypatch) monkeypatch.setattr(ephemeral.time, "time", lambda: clock[0]) state = ephemeral.LocalEphemeralState() monkeypatch.setattr(ephemeral, "get_ephemeral_state", lambda: state) - monkeypatch.setattr(automation_lease, "RENEW_INTERVAL_SECONDS", 0.001) + monkeypatch.setattr(automations_automation_lease, "RENEW_INTERVAL_SECONDS", 0.001) owner, other = AutomationScheduler(), AutomationScheduler() assert await owner._acquire_lock("task") old_lease = owner._leases["task"] @@ -59,7 +62,7 @@ async def test_expired_owner_cannot_renew_or_delete_new_owners_lock(monkeypatch) async def test_lease_loss_cancels_inflight_execution(monkeypatch): state = ephemeral.LocalEphemeralState() monkeypatch.setattr(ephemeral, "get_ephemeral_state", lambda: state) - monkeypatch.setattr(automation_lease, "RENEW_INTERVAL_SECONDS", 0.001) + monkeypatch.setattr(automations_automation_lease, "RENEW_INTERVAL_SECONDS", 0.001) lease = automation_lease.AutomationLease("test-loss") assert await lease.acquire() entered, cancelled = asyncio.Event(), asyncio.Event() @@ -87,21 +90,21 @@ async def test_recovery_uses_ownership_instead_of_run_age(live_env): factory, _, task_id = live_env with factory() as db: svc = AutomationService(db) - old = svc.record_run_start(task_id) + old = running_run(db, task_id) old.started_at = utc_now() - timedelta(days=3) db.commit() owner, recovering = AutomationScheduler(), AutomationScheduler() assert await owner._acquire_lock(task_id) try: - await recovering._recover_stuck_running_runs() + await recovering._dispatch_pending() with factory() as db: assert AutomationService(db).get_task_runs(task_id, "owner")[0].status == "running" finally: await owner._release_lock(task_id) # Even a newly started orphan is recoverable once ownership is absent. with factory() as db: - AutomationService(db).record_run_start(task_id) - await recovering._recover_stuck_running_runs() + running_run(db, task_id) + await recovering._dispatch_pending() await asyncio.gather(*recovering._executions) with factory() as db: runs = AutomationService(db).get_task_runs(task_id, "owner") @@ -121,10 +124,10 @@ async def test_missed_one_shot_cannot_bypass_another_workers_lock(live_env): owner, recovering = AutomationScheduler(), AutomationScheduler() assert await owner._acquire_lock(task_id) try: - await recovering._recover_missed_tasks() + await recovering._dispatch_pending() assert not recovering._executions with factory() as db: - assert AutomationService(db).get_task_runs(task_id, "owner") == [] + assert AutomationService(db).get_task_runs(task_id, "owner")[0].status == "pending" finally: await owner._release_lock(task_id) @@ -136,16 +139,16 @@ async def test_direct_execution_on_same_scheduler_cannot_share_ownership(live_en entered, release = asyncio.Event(), asyncio.Event() calls = [] - async def dispatch(*args): + async def dispatch(*args, **kwargs): calls.append(args) entered.set() await release.wait() - monkeypatch.setattr("orchestration.schedulers.automation_dispatch.execute_task", dispatch) - first = asyncio.create_task(scheduler.execute_task(task_id, "owner")) + monkeypatch.setattr('orchestration.automations.automation_dispatch.execute_task', dispatch) + first = asyncio.create_task(execute_queued(scheduler, task_id, "owner")) try: await entered.wait() - await scheduler.execute_task(task_id, "owner") + await execute_queued(scheduler, task_id, "owner") assert len(calls) == 1 finally: release.set() @@ -155,8 +158,8 @@ async def dispatch(*args): @pytest.mark.asyncio async def test_orphan_recovery_observes_child_without_creating_duplicate(live_env, monkeypatch): factory, _, task_id = live_env - from orchestration import chat_run_executor as executor - from orchestration.schedulers.automation_conversation import execute_prompt + pass + from orchestration.automations.automation_conversation import execute_prompt entered, release = asyncio.Event(), asyncio.Event() calls = [] @@ -168,9 +171,9 @@ async def workflow(**kwargs): yield {"type": "content", "delta": "Recovered report"} yield {"type": "meta", "usage": {}} - monkeypatch.setattr(executor, "astream_chat_workflow", workflow) + monkeypatch.setattr(runner, "astream_chat_workflow", workflow) with factory() as db: - run_id = AutomationService(db).record_run_start(task_id).run_id + run_id = running_run(db, task_id).run_id child_observer = asyncio.create_task( execute_prompt( user_id="owner", @@ -191,10 +194,10 @@ async def noop(*args): monkeypatch.setattr(scheduler, "_send_notification", noop) try: await asyncio.wait_for(entered.wait(), 5) - await scheduler._recover_stuck_running_runs() + await scheduler._dispatch_pending() await asyncio.sleep(0.1) assert scheduler._executions - await scheduler.execute_task(task_id, "owner") + await execute_queued(scheduler, task_id, "owner") assert len(calls) == 1 release.set() await asyncio.wait_for(asyncio.gather(*scheduler._executions), 5) @@ -213,17 +216,17 @@ async def noop(*args): @pytest.mark.asyncio async def test_recovery_delivers_original_result_before_later_user_message(live_env, monkeypatch): factory, _, task_id = live_env - from core.db.models import ChatRun, ChatMessage - from orchestration import chat_run_executor as executor - from orchestration.schedulers.automation_conversation import execute_prompt + from core.db.models import ChatMessage, ChatRun + pass + from orchestration.automations.automation_conversation import execute_prompt async def workflow(**kwargs): yield {"type": "content", "delta": "Scheduled report"} yield {"type": "meta", "usage": {}} - monkeypatch.setattr(executor, "astream_chat_workflow", workflow) + monkeypatch.setattr(runner, "astream_chat_workflow", workflow) with factory() as db: - run_id = AutomationService(db).record_run_start(task_id).run_id + run_id = running_run(db, task_id).run_id chat_id, _, _ = await execute_prompt( user_id="owner", task_name="Report", @@ -263,7 +266,7 @@ async def notify(*args): notices.append(args) monkeypatch.setattr(scheduler, "_send_notification", notify) - await scheduler._recover_stuck_running_runs() + await scheduler._dispatch_pending() await asyncio.wait_for(asyncio.gather(*scheduler._executions), 5) with factory() as db: runs = AutomationService(db).get_task_runs(task_id, "owner") @@ -277,7 +280,9 @@ async def test_stop_releases_lease_even_before_execution_coroutine_starts(live_e scheduler = AutomationScheduler() assert await scheduler._acquire_lock(task_id) lease = scheduler._leases[task_id] - scheduler._launch_execution(task_id, "owner", lease=lease) + task = asyncio.create_task(scheduler.execute_task(task_id)) + scheduler._executions.add(task) + task.add_done_callback(scheduler._executions.discard) await scheduler.stop() assert scheduler._leases == {} assert lease.renewal.done() diff --git a/src/backend/tests/orchestration/test_autonomous_loop_driver.py b/src/backend/tests/orchestration/test_autonomous_loop_driver.py index 405ca9d7..51bdfea0 100644 --- a/src/backend/tests/orchestration/test_autonomous_loop_driver.py +++ b/src/backend/tests/orchestration/test_autonomous_loop_driver.py @@ -13,16 +13,16 @@ 8. 收尾交付轮：部分完成收场时用现有成果跑一轮 wrap-up。 """ from __future__ import annotations +from core.ontology import runtime_policy import asyncio from types import SimpleNamespace from typing import Any, Dict, List +import orchestration.loops.driver as al import pytest - -import orchestration.autonomous_loop as al -from orchestration.autonomous_loop import LoopBudget, run_autonomous_loop -from orchestration.loop_evaluator import CONTINUE, DONE, GoalSpec +from orchestration.loops.driver import LoopBudget, run_autonomous_loop +from orchestration.loops.requirements import CONTINUE, DONE, GoalSpec def _run(coro): @@ -108,9 +108,9 @@ async def _criteria(**kw): sleep=self._fast_sleep, gather=asyncio.gather, CancelledError=asyncio.CancelledError)) # ontology 解析不碰 DB - import core.services.ontology_service as osvc + import core.ontology.service as osvc - monkeypatch.setattr(osvc, "build_user_ontology_runtime", + monkeypatch.setattr(runtime_policy, 'build_user_ontology_runtime', lambda **kw: (False, {"enabled": False, "packs": [], "review_level": "none"})) self.monkeypatch = monkeypatch diff --git a/src/backend/tests/orchestration/test_background_commands.py b/src/backend/tests/orchestration/test_background_commands.py index 8455a044..c178da7b 100644 --- a/src/backend/tests/orchestration/test_background_commands.py +++ b/src/backend/tests/orchestration/test_background_commands.py @@ -1,17 +1,25 @@ """Background command ownership, handoff and completion delivery.""" + import asyncio import json from types import SimpleNamespace from unittest.mock import AsyncMock + +import core.chat.runtime.worker as runtime_worker +import core.db.engine as db_engine +import core.sandbox.commands.events as commands_events +import core.sandbox.commands.notifications as commands_notifications +import core.sandbox.commands.runtime as commands_runtime import pytest +from core.db import models +from core.db.engine import Base +from core.sandbox.commands.runtime import CommandRuntime +from core.sandbox.commands import store from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker -from core.db.engine import Base -from core.db import models -from core.services.commands import store -from orchestration.commands.runtime import CommandRuntime from tests.orchestration.command_test_support import source_run + @pytest.fixture def command_db(tmp_path, monkeypatch): engine = create_engine("sqlite:///" + str(tmp_path / "commands.db"), @@ -19,9 +27,9 @@ def command_db(tmp_path, monkeypatch): Base.metadata.create_all(engine) factory = sessionmaker(bind=engine, expire_on_commit=False) monkeypatch.setattr(store, "SessionLocal", factory) - from orchestration.commands import events + import core.sandbox.commands.events as events isolated_events = events.MemoryCommandEvents() - monkeypatch.setattr(events, "get_command_events", lambda: isolated_events) + monkeypatch.setattr(commands_events, "get_command_events", lambda: isolated_events) with factory() as db: db.add(models.UserShadow(user_id="u", username="test")) db.flush() @@ -74,8 +82,8 @@ def test_terminal_notification_is_atomic_with_result(command_db): @pytest.fixture async def http_provider(tmp_path, monkeypatch): import httpx - from core.sandbox.runner import server from core.sandbox.providers.script_runner.provider import ScriptRunnerProvider + from core.sandbox.runner import server monkeypatch.setattr(server, "WORKSPACE_ROOT", str(tmp_path / "workspace")) monkeypatch.setattr(server, "_AUTH_TOKEN", "") monkeypatch.setenv("DEPLOY_PROFILE", "local") @@ -100,17 +108,22 @@ async def touch_session(session_id): await client.aclose() async def test_real_process_background_concurrency_and_cancel(command_db, http_provider, monkeypatch, tmp_path): + import core.sandbox.commands.runtime as module from core.llm.tools.sandbox_tool import register_bash - from orchestration.commands import runtime as module runtime = CommandRuntime() - monkeypatch.setattr(module, "runtime", runtime) + monkeypatch.setattr(commands_runtime, "runtime", runtime) + from core.llm.ports import process_state + monkeypatch.setattr(process_state(), "commands", runtime) monkeypatch.setattr("core.sandbox.get_sandbox_provider", lambda: http_provider) monkeypatch.setattr("core.config.local_mode.local_mode_enabled", lambda: False) class Tools: def __init__(self): self.functions = {} - def register_tool_function(self, fn, **_): self.functions[fn.__name__] = fn + def append(self, tool): + fn = tool._func + _ = {"func_description": tool.description, "permission": getattr(tool, "dispatch_spec", None)} + self.functions[fn.__name__] = fn tools = Tools() - register_bash(tools, loader=None, loaded_skill_ids=set(), chat_id="c", user_id="u") + register_bash(tools, loader=None, chat_id="c", user_id="u") def data(response): return json.loads(response.content[0].text) marker = tmp_path / "launches" command = f"printf x >> '{marker}'; printf first; sleep 1; printf second" @@ -175,16 +188,19 @@ async def test_soft_delete_stops_real_background_process(command_db, http_provid async def test_busy_react_delivery_and_final_output_race(command_db): run_id = source_run(command_db) from agentscope.agent import Agent - from agentscope.model import ChatModelBase, ChatResponse from agentscope.message import Msg, TextBlock - from core.llm.middlewares import AgentRuntimeState - from core.llm.command_middleware import CommandCompletionMiddleware + from agentscope.model import ChatModelBase, ChatResponse + from core.sandbox.commands.completion_hook import command_completion_hooks + from tests.orchestration.hook_support import make_adapter + from core.llm.runtime.state import AgentRuntimeState row = store.create(request(), "c", None) store.attach(row["task_id"], "owner", {"status":"running","session_id":"p"}, None) seen = [] class Model(ChatModelBase): def __init__(self): super().__init__(None, "deterministic", self.Parameters(), stream=False, max_retries=0) + from agentscope.formatter import OpenAIChatFormatter + self.formatter = OpenAIChatFormatter() async def count_tokens(self, **kwargs): return 10 async def _call_api(self, model_name, messages, **kwargs): seen.append([str(m.content) for m in messages]) @@ -194,7 +210,7 @@ async def _call_api(self, model_name, messages, **kwargs): return ChatResponse(content=[TextBlock(type="text", text="step")], is_last=True) agent = Agent(name="test", system_prompt="test", model=Model(), state=AgentRuntimeState(chat_id="c", user_id="u", run_id=run_id), - middlewares=[CommandCompletionMiddleware()]) + middlewares=[make_adapter(*command_completion_hooks())]) async for _ in agent.reply_stream(Msg(name="user", content=[TextBlock(type="text", text="start")], role="user")): pass assert len(seen) == 2 @@ -202,13 +218,13 @@ async def _call_api(self, model_name, messages, **kwargs): assert store.snapshot(row["task_id"])["notice_status"] == "delivered" async def test_idle_completion_uses_real_run_admission_and_executor(command_db, monkeypatch): - from core.services.chat_sequencer import ChatSequencer - from orchestration import chat_run_executor as executor - from orchestration.commands import notifications - from core.services.commands import delivery - from core.services.session_operations import session_context - from core.services import chat_service - monkeypatch.setattr(executor, "SessionLocal", command_db) + from core.chat.sequencer import ChatSequencer + pass + import core.sandbox.commands.notifications as notifications + from core.chat import service as chat_service + from core.sandbox.commands import delivery + from core.chat.operations import session_context + monkeypatch.setattr(db_engine, "SessionLocal", command_db) with command_db() as db: accepted = ChatSequencer(db).accept_main_run(chat_id="c", user_id="u", user_content="start", request_payload={}) @@ -229,7 +245,7 @@ async def worker(**kwargs): run = db.get(models.ChatRun, kwargs["run_id"]) run.status, run.writer_slot = "completed", None db.commit() - monkeypatch.setattr(executor, "_run_workflow", worker) + monkeypatch.setattr(runtime_worker, "_run_workflow", worker) await notifications.dispatch() await asyncio.sleep(0.1) await notifications.dispatch() @@ -250,14 +266,14 @@ def test_bounded_output_reports_omitted_characters(command_db): assert store.read(row["task_id"], "u", "c")["output_omitted_chars"] == 0 async def test_recovery_preserves_terminal_result_and_cursor(command_db, monkeypatch): - from orchestration.commands import runtime as module + import core.sandbox.commands.runtime as module row = store.create(request(), "c", None) store.attach(row["task_id"], "dead", {"status":"running","session_id":"old"}, {"provider":"opensandbox","cursor":7}) store.release_lease(row["task_id"], "dead") recover = AsyncMock(return_value={"status":"exited","session_id":None, "stdout":"remaining","stderr":"","exit_code":0,"execution_cursor":20}) - monkeypatch.setattr(module, "recover_execution", recover) + monkeypatch.setattr(commands_runtime, "recover_execution", recover) provider = SimpleNamespace(start_process=AsyncMock()) runtime = CommandRuntime() await runtime.recover(provider) @@ -282,7 +298,7 @@ async def test_deadline_stops_real_process_without_relaunch(command_db, http_pro await runtime.shutdown() async def test_boundary_event_is_durable_before_model_call(command_db): - from core.services.commands import delivery + from core.sandbox.commands import delivery row = store.create(request(), "c", None) store.attach(row["task_id"], "owner", {"status":"running","session_id":"p"}, None) store.collect(row["task_id"], "owner", {"status":"exited","exit_code":0}) @@ -297,8 +313,9 @@ async def test_boundary_event_is_durable_before_model_call(command_db): async def test_failed_model_request_keeps_durable_notice(command_db): run_id = source_run(command_db) - from core.services.commands import delivery - from core.llm.command_middleware import CommandCompletionMiddleware + from core.sandbox.commands.completion_hook import command_completion_hooks + from tests.orchestration.hook_support import make_adapter + from core.sandbox.commands import delivery row = store.create(request(), "c", None) store.attach(row["task_id"], "owner", {"status":"running","session_id":"p"}, None) store.collect(row["task_id"], "owner", {"status":"exited","exit_code":0}) @@ -307,16 +324,16 @@ async def broken(**kwargs): raise ConnectionError("model down") yield with pytest.raises(ConnectionError): - async for _ in CommandCompletionMiddleware().on_reasoning(agent, {}, broken): pass + async for _ in make_adapter(*command_completion_hooks()).on_reasoning(agent, {}, broken): pass assert store.snapshot(row["task_id"])["notice_status"] == "delivered" with command_db() as db: assert row["task_id"] in str(db.get(models.ChatRun, run_id).recovery_snapshot) async def test_command_http_authorization_and_nonconsuming_ui(command_db): import httpx - from fastapi import FastAPI - from core.auth.backend import get_current_user, UserContext from api.routes.v1.commands import router + from core.auth.backend import UserContext, get_current_user + from fastapi import FastAPI row = store.create(request(), "c", None) store.attach(row["task_id"], "owner", {"status":"exited","exit_code":0,"stdout":"result"}, None) app = FastAPI() @@ -346,7 +363,7 @@ async def test_immediate_shutdown_stops_owned_process(command_db): assert not runtime.providers async def test_cancel_does_not_publish_intermediate_terminal_notice(command_db, monkeypatch): - from orchestration.commands import notifications + import core.sandbox.commands.notifications as notifications stopped = asyncio.Event() release = asyncio.Event() async def poll(pid, **kwargs): @@ -391,8 +408,8 @@ async def poll(pid, **kwargs): await runtime.shutdown() async def test_notification_pages_pass_fifty_blocked_events(command_db, monkeypatch): - from core.services.chat_sequencer import ChatSequencer - from orchestration.commands import notifications + import core.sandbox.commands.notifications as notifications + from core.chat.sequencer import ChatSequencer with command_db() as db: source = ChatSequencer(db).accept_main_run(chat_id="c", user_id="u", user_content="source", request_payload={}).run @@ -409,7 +426,7 @@ async def test_notification_pages_pass_fifty_blocked_events(command_db, monkeypa db.get(models.CommandTask, tid).task_id = f"cmd_{index:03}" db.commit() launch = AsyncMock() - monkeypatch.setattr(notifications, "launch", launch) + monkeypatch.setattr(commands_notifications, "launch", launch) await notifications.dispatch() launch.assert_awaited_once() assert store.snapshot("cmd_050")["notice_status"] == "reserved" diff --git a/src/backend/tests/orchestration/test_chat_resume_checkpoint.py b/src/backend/tests/orchestration/test_chat_resume_checkpoint.py index 310109f8..5f79349c 100644 --- a/src/backend/tests/orchestration/test_chat_resume_checkpoint.py +++ b/src/backend/tests/orchestration/test_chat_resume_checkpoint.py @@ -1,14 +1,16 @@ """Public resume restores the event log's aligned snapshot after trimming.""" +from core.chat.projection.codec import chat_projection_codec + import asyncio import json +import core.runs.events.store as store +import core.runs.events.store as run_event_stream import fakeredis.aioredis import pytest - from api.routes.v1.chats import run_views from core.auth.backend import UserContext -from core.services.run_journal import RunJournal -from orchestration import run_event_stream +from core.runs.journal import RunJournal from tests.orchestration.recovery_test_support import recovery_env @@ -16,8 +18,8 @@ @pytest.mark.asyncio async def test_resume_restores_message_and_child_steps_after_prefix_is_trimmed(recovery_env, monkeypatch, mode): redis = fakeredis.aioredis.FakeRedis(decode_responses=True) - monkeypatch.setattr(run_event_stream, "get_redis", lambda **_: redis) - monkeypatch.setattr(run_event_stream, "MAXLEN", 8 if mode == "checkpoint_tail" else 2) + monkeypatch.setattr(store, "get_redis", lambda **_: redis) + monkeypatch.setattr(store, "MAXLEN", 8 if mode == "checkpoint_tail" else 2) await run_event_stream.get_run_event_stream().clear("run-trimmed") journal = RunJournal(recovery_env) journal.accept(run_id="run-trimmed", message_id="msg-trimmed", chat_id="chat-1", @@ -40,7 +42,7 @@ async def test_resume_restores_message_and_child_steps_after_prefix_is_trimmed(r if mode == "replacement": await stream.append("run-trimmed", {"type": "content_replace", "content": "修订答案。", "_offset": 6}) if mode == "resolved_confirm": - from orchestration import run_subscription + import core.chat.projection.subscription as run_subscription async def absent_confirmation(*_): return None monkeypatch.setattr(run_subscription.interaction_store, "read", absent_confirmation) @@ -53,7 +55,7 @@ async def absent_confirmation(*_): await stream.append("run-trimmed", {"type": "user_question", "request_id": "question-1", "questions": [{"id": "q1", "question": "继续吗？"}], "_offset": 6}) if mode == "two_writers": - other = run_event_stream.RedisRunEventStream() + other = run_event_stream.RedisRunEventStream(chat_projection_codec()) await other.append("run-trimmed", {"type": "content", "delta": "后续进展。", "_offset": 6}) await stream.append("run-trimmed", {"type": "content", "delta": "全部核对。", "_offset": 7}) await redis.xtrim(run_event_stream.redis_stream_key("run-trimmed"), maxlen=1, approximate=False) diff --git a/src/backend/tests/orchestration/test_chat_run_cancel_persistence.py b/src/backend/tests/orchestration/test_chat_run_cancel_persistence.py index ff66fce3..cff1643e 100644 --- a/src/backend/tests/orchestration/test_chat_run_cancel_persistence.py +++ b/src/backend/tests/orchestration/test_chat_run_cancel_persistence.py @@ -6,11 +6,18 @@ import contextlib from types import SimpleNamespace +import core.agent_runtime.runner as runner +import core.chat.runtime.admission as admission +import core.db.engine as db_engine +import core.runs.events.subscription as subscription +import core.runs.ownership as ownership +import core.runs.runtime as runtime +import core.runs.settings as settings +import core.runs.tasks as tasks import fakeredis.aioredis import pytest from core.db.engine import Base from core.db.models import ChatMessage, ChatRun, ChatSession -from orchestration import chat_run_executor as executor from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker @@ -23,21 +30,21 @@ def cancel_env(tmp_path, monkeypatch): ) Base.metadata.create_all(engine) sessions = sessionmaker(bind=engine, expire_on_commit=False) - monkeypatch.setattr(executor, "SessionLocal", sessions) + monkeypatch.setattr(db_engine, "SessionLocal", sessions) redis = fakeredis.aioredis.FakeRedis(decode_responses=True) - monkeypatch.setattr("orchestration.run_event_stream.get_redis", lambda **_: redis) - executor._active_runs.clear() + monkeypatch.setattr('core.runs.events.store.get_redis', lambda **_: redis) + tasks.process_state().active_runs.clear() with sessions() as db: db.add(ChatSession(chat_id="chat-1", user_id="user-1", title="test")) db.commit() yield sessions - executor._active_runs.clear() + tasks.process_state().active_runs.clear() engine.dispose() async def _start(workflow, monkeypatch, ready: asyncio.Event): - monkeypatch.setattr(executor, "astream_chat_workflow", workflow) - run = await executor.start_run( + monkeypatch.setattr(runner, "astream_chat_workflow", workflow) + run = await admission.start_run( chat_id="chat-1", user_id="user-1", session_messages=[{"role": "user", "content": "hello"}], @@ -58,8 +65,8 @@ async def _await_worker(worker) -> None: async def _start_and_cancel(workflow, monkeypatch, ready: asyncio.Event): run = await _start(workflow, monkeypatch, ready) - worker = executor._active_runs[run.run_id] - assert await executor.cancel_run(run.run_id, user_id="user-1") is True + worker = tasks.process_state().active_runs[run.run_id] + assert await runtime.cancel_run(run.run_id, user_id="user-1") is True await _await_worker(worker) return run @@ -126,7 +133,7 @@ async def silent_workflow(**_kwargs): def test_cancelled_turn_is_marked_in_replayed_context(): """继续会话时，被停掉那轮要带上"被中断"的标注，否则模型以为自己已经说完了。""" - from core.services.compaction_history import normalize_rows + from core.chat.compaction.history import normalize_rows rows = [ SimpleNamespace(role="user", content="帮我查一下", extra_data={}, tool_calls=None), @@ -148,7 +155,7 @@ def test_cancelled_turn_is_marked_in_replayed_context(): def test_normal_turn_carries_no_interruption_marker(): - from core.services.compaction_history import normalize_rows + from core.chat.compaction.history import normalize_rows rows = [SimpleNamespace(role="assistant", content="查完了", extra_data={}, tool_calls=None)] @@ -166,7 +173,7 @@ async def test_fenced_by_user_cancel_still_persists_partial(cancel_env, monkeypa 一个字都不写，用户刷新后只剩自己那句提问。 """ sessions = cancel_env - monkeypatch.setattr(executor, "_RUN_LEASE_HEARTBEAT_SEC", 0.01) + monkeypatch.setattr(settings, "_RUN_LEASE_HEARTBEAT_SEC", 0.01) streamed = asyncio.Event() async def half_written_workflow(**_kwargs): @@ -175,9 +182,9 @@ async def half_written_workflow(**_kwargs): await asyncio.Event().wait() run = await _start(half_written_workflow, monkeypatch, streamed) - worker = executor._active_runs[run.run_id] + worker = tasks.process_state().active_runs[run.run_id] # 只写终态 + 清租约，不碰 worker：让心跳去发现，复现被围栏的时序 - assert executor._journal().cancel(run.run_id, reason="user cancelled") is True + assert ownership.journal().cancel(run.run_id, reason="user cancelled") is True await _await_worker(worker) with sessions() as db: @@ -207,14 +214,14 @@ def test_fenced_by_takeover_stays_silent(cancel_env, monkeypatch): ) db.commit() - assert executor._cancelled_by_user("run-live") is False - assert executor._cancelled_by_user("run-missing") is False + assert ownership.cancelled_by_user("run-live") is False + assert ownership.cancelled_by_user("run-missing") is False @pytest.mark.asyncio @pytest.mark.parametrize("heartbeats", [True, False]) async def test_first_response_timeout_ends_silent_run(cancel_env, monkeypatch, heartbeats): - monkeypatch.setattr(executor, "_FIRST_RESPONSE_TIMEOUT_SEC", 0.1) + monkeypatch.setattr(settings, "_FIRST_RESPONSE_TIMEOUT_SEC", 0.1) entered = asyncio.Event() closed = asyncio.Event() @@ -232,12 +239,12 @@ async def silent_workflow(**_kwargs): closed.set() run = await _start(silent_workflow, monkeypatch, entered) - worker = executor._active_runs[run.run_id] + worker = tasks.process_state().active_runs[run.run_id] try: await asyncio.wait_for(asyncio.shield(worker), timeout=1) assert closed.is_set() - assert executor.get_run(run.run_id).status == "failed" - events = [event async for event in executor.follow_run(run.run_id)] + assert runtime.get_run(run.run_id).status == "failed" + events = [event async for event in subscription.follow_run(run.run_id)] errors = [event for event in events if event.get("type") == "error"] assert errors[0]["error"] == "当前模型调用量大，算力资源紧张，请稍后再试！" with cancel_env() as db: @@ -257,7 +264,7 @@ async def silent_workflow(**_kwargs): {"type": "plan_update", "title": "执行计划", "steps": []}, ]) async def test_visible_response_disarms_initial_deadline(cancel_env, monkeypatch, first_event): - monkeypatch.setattr(executor, "_FIRST_RESPONSE_TIMEOUT_SEC", 0.1) + monkeypatch.setattr(settings, "_FIRST_RESPONSE_TIMEOUT_SEC", 0.1) entered = asyncio.Event() continued = asyncio.Event() @@ -269,11 +276,11 @@ async def responding_workflow(**_kwargs): await asyncio.Event().wait() run = await _start(responding_workflow, monkeypatch, entered) - worker = executor._active_runs[run.run_id] + worker = tasks.process_state().active_runs[run.run_id] try: await asyncio.wait_for(continued.wait(), timeout=1) - assert executor.get_run(run.run_id).status == "running" + assert runtime.get_run(run.run_id).status == "running" finally: - await executor.cancel_run(run.run_id, user_id="user-1") + await runtime.cancel_run(run.run_id, user_id="user-1") await _await_worker(worker) - assert executor.get_run(run.run_id).status == "cancelled" + assert runtime.get_run(run.run_id).status == "cancelled" diff --git a/src/backend/tests/orchestration/test_chat_run_follow_recovery.py b/src/backend/tests/orchestration/test_chat_run_follow_recovery.py index 2ba0f019..a3c161a3 100644 --- a/src/backend/tests/orchestration/test_chat_run_follow_recovery.py +++ b/src/backend/tests/orchestration/test_chat_run_follow_recovery.py @@ -7,13 +7,25 @@ from datetime import datetime, timedelta, timezone import api.routes.v1.chats.run_views as chat_run_views +import core.agent_runtime.runner as runner +import core.chat.runtime.admission as admission +import core.chat.runtime.compaction as compaction +import core.chat.runtime.followups as followups +import core.chat.runtime.recovery as recovery +import core.runs.events.store as run_event_stream +import core.runs.events.subscription as subscription +import core.runs.events.writer as writer +import core.runs.ownership as ownership +import core.runs.runtime as runtime +import core.runs.settings as settings +import core.runs.tasks as tasks +import core.runs.watchdog as watchdog import fakeredis.aioredis +import orchestration.workers.run_recovery as _run_recovery import pytest from core.auth.backend import UserContext from core.db.models import ChatMessage, ChatRun -from core.services.run_journal import RunJournal -from orchestration import chat_run_executor as executor -from orchestration import run_event_stream +from core.runs.journal import RunJournal from tests.orchestration.recovery_test_support import recovery_env @@ -33,16 +45,16 @@ async def workflow(**_kwargs): "usage": {"input_tokens": 2, "output_tokens": 2}, } - monkeypatch.setattr("orchestration.run_event_stream.get_redis", lambda **_: redis) - monkeypatch.setattr(executor, "astream_chat_workflow", workflow) - monkeypatch.setattr(executor, "_spawn_followup_task", lambda **_kwargs: None) - monkeypatch.setattr(executor, "_spawn_compaction_task", lambda **_kwargs: None) + monkeypatch.setattr('core.runs.events.store.get_redis', lambda **_: redis) + monkeypatch.setattr(runner, "astream_chat_workflow", workflow) + monkeypatch.setattr(followups, "_spawn_followup_task", lambda **_kwargs: None) + monkeypatch.setattr(compaction, "_spawn_compaction_task", lambda **_kwargs: None) monkeypatch.setattr( - "core.services.artifact_service.persist_artifacts", + 'core.files.artifacts.service.persist_artifacts', lambda *_args, **_kwargs: None, ) - run = await executor.start_run( + run = await admission.start_run( chat_id="chat-1", user_id="user-1", session_messages=[{"role": "user", "content": "hello"}], @@ -54,7 +66,7 @@ async def workflow(**_kwargs): ) async def collect(): - return [event async for event in executor.follow_run(run.run_id)] + return [event async for event in subscription.follow_run(run.run_id)] events = await asyncio.wait_for(collect(), timeout=5) offsets = [event["_offset"] for event in events] @@ -75,7 +87,7 @@ async def test_public_follow_after_recovery_resets_partial_stream_and_keeps_offs ): sessions = recovery_env redis = fakeredis.aioredis.FakeRedis(decode_responses=True) - monkeypatch.setattr("orchestration.run_event_stream.get_redis", lambda **_: redis) + monkeypatch.setattr('core.runs.events.store.get_redis', lambda **_: redis) journal = RunJournal(sessions) row = journal.accept( run_id="run-partial-stream", @@ -104,7 +116,7 @@ async def test_public_follow_after_recovery_resets_partial_stream_and_keeps_offs ) for text in ("old partial one", "old partial two"): offset = journal.allocate_event_offset(row.run_id, owner="dead-worker") - await executor._xadd_event( + await writer.xadd_event( row.run_id, offset, {"type": "content", "delta": text, "chat_id": "chat-1"}, @@ -119,18 +131,18 @@ async def workflow(**_kwargs): yield {"type": "content", "delta": "recovered answer"} yield {"type": "meta", "route": "main", "usage": {}} - monkeypatch.setattr(executor, "astream_chat_workflow", workflow) - monkeypatch.setattr(executor, "_spawn_followup_task", lambda **_kwargs: None) - monkeypatch.setattr(executor, "_spawn_compaction_task", lambda **_kwargs: None) + monkeypatch.setattr(runner, "astream_chat_workflow", workflow) + monkeypatch.setattr(followups, "_spawn_followup_task", lambda **_kwargs: None) + monkeypatch.setattr(compaction, "_spawn_compaction_task", lambda **_kwargs: None) monkeypatch.setattr( - "core.services.artifact_service.persist_artifacts", + 'core.files.artifacts.service.persist_artifacts', lambda *_args, **_kwargs: None, ) - assert await executor.recover_orphan_runs() == 1 + assert await recovery.recover_orphan_runs() == 1 async def collect_from_crash_offset(): - return [event async for event in executor.follow_run(row.run_id, from_offset=2)] + return [event async for event in subscription.follow_run(row.run_id, from_offset=2)] events = await asyncio.wait_for(collect_from_crash_offset(), timeout=5) assert events[0]["_offset"] == 3 @@ -142,14 +154,14 @@ async def collect_from_crash_offset(): ) decoded = [json.loads(fields["data"]) for _entry_id, fields in raw_entries] assert all("old partial" not in str(event) for event in decoded) - assert decoded[-1]["type"] == executor._TERMINAL_TYPE + assert decoded[-1]["type"] == settings.TERMINAL_TYPE @pytest.mark.asyncio async def test_active_run_probe_replays_the_complete_existing_prefix(recovery_env, monkeypatch): sessions = recovery_env redis = fakeredis.aioredis.FakeRedis(decode_responses=True) - monkeypatch.setattr("orchestration.run_event_stream.get_redis", lambda **_: redis) + monkeypatch.setattr('core.runs.events.store.get_redis', lambda **_: redis) journal = RunJournal(sessions) row = journal.accept( run_id="run-refresh-replay", @@ -162,7 +174,7 @@ async def test_active_run_probe_replays_the_complete_existing_prefix(recovery_en assert journal.claim(row.run_id, owner="live-worker", lease_seconds=60) for text in ("already produced one", "already produced two"): offset = journal.allocate_event_offset(row.run_id, owner="live-worker") - await executor._xadd_event( + await writer.xadd_event( row.run_id, offset, {"type": "content", "delta": text, "chat_id": "chat-1"}, @@ -182,7 +194,7 @@ async def test_active_run_probe_replays_the_complete_existing_prefix(recovery_en replay_from = response["data"]["last_event_offset"] async def collect_existing_prefix(): - follower = executor.follow_run(row.run_id, from_offset=replay_from) + follower = subscription.follow_run(row.run_id, from_offset=replay_from) try: return [await anext(follower), await anext(follower)] finally: @@ -203,12 +215,12 @@ class ExplodingJournal: def renew(self, *_args, **_kwargs): raise RuntimeError("database unavailable") - monkeypatch.setattr(executor, "_journal", lambda: ExplodingJournal()) - monkeypatch.setattr(executor, "_RUN_LEASE_HEARTBEAT_SEC", 0.0) + monkeypatch.setattr(ownership, 'journal', lambda: ExplodingJournal()) + monkeypatch.setattr(settings, "_RUN_LEASE_HEARTBEAT_SEC", 0.0) lease_lost = asyncio.Event() worker = asyncio.create_task(asyncio.Event().wait()) heartbeat = asyncio.create_task( - executor._lease_heartbeat("run-heartbeat", "worker", worker, lease_lost) + ownership.lease_heartbeat("run-heartbeat", "worker", worker, lease_lost) ) await asyncio.wait_for(heartbeat, timeout=1) @@ -251,10 +263,10 @@ def register(_run_id, coro, *, name): coro.close() registered.set() - monkeypatch.setattr(executor, "_register_run_task", register) - monkeypatch.setattr(executor, "_RUN_RECOVERY_INTERVAL_SEC", 0.01) - monkeypatch.setattr(executor, "_STALE_REAPER_INTERVAL_SEC", 60.0) - loop_task = asyncio.create_task(executor.run_stale_reaper_loop()) + monkeypatch.setattr(runtime, 'register_run_task', register) + monkeypatch.setattr(settings, 'RUN_RECOVERY_INTERVAL_SEC', 0.01) + monkeypatch.setattr(settings, 'STALE_REAPER_INTERVAL_SEC', 60.0) + loop_task = asyncio.create_task(_run_recovery.run_stale_reaper_loop()) try: await asyncio.wait_for(registered.wait(), timeout=1) finally: @@ -276,9 +288,9 @@ async def blocked_workflow(**_kwargs): yield {"type": "model_progress"} await asyncio.Event().wait() - monkeypatch.setattr("orchestration.run_event_stream.get_redis", lambda **_: redis) - monkeypatch.setattr(executor, "astream_chat_workflow", blocked_workflow) - run = await executor.start_run( + monkeypatch.setattr('core.runs.events.store.get_redis', lambda **_: redis) + monkeypatch.setattr(runner, "astream_chat_workflow", blocked_workflow) + run = await admission.start_run( chat_id="chat-1", user_id="user-1", session_messages=[{"role": "user", "content": "hello"}], @@ -289,9 +301,9 @@ async def blocked_workflow(**_kwargs): model_name="test-model", ) await asyncio.wait_for(entered.wait(), timeout=1) - worker = executor._active_runs[run.run_id] + worker = tasks.process_state().active_runs[run.run_id] hard_expired_at = datetime.now(timezone.utc) - timedelta( - seconds=executor._HARD_MAX_AGE_SEC + 60 + seconds=settings._HARD_MAX_AGE_SEC + 60 ) with sessions() as db: db.query(ChatRun).filter(ChatRun.run_id == run.run_id).update( @@ -299,11 +311,11 @@ async def blocked_workflow(**_kwargs): ) db.commit() - assert await executor.reap_stale_runs() == 1 + assert await watchdog.reap_stale_runs() == 1 await asyncio.wait_for(worker, timeout=1) entries = await redis.xrange(run_event_stream.redis_stream_key(run.run_id), min="-", max="+") events = [json.loads(fields["data"]) for _entry_id, fields in entries] - assert sum(event.get("type") == executor._TERMINAL_TYPE for event in events) == 1 + assert sum(event.get("type") == settings.TERMINAL_TYPE for event in events) == 1 assert not any(event.get("_cancelled") for event in events) assert any("长时间无响应" in str(event.get("error") or "") for event in events) with sessions() as db: @@ -316,7 +328,7 @@ async def test_follow_run_survives_none_stream_reads(recovery_env, monkeypatch): follow_run must treat both as "no events yet" instead of failing the SSE stream with "流式响应中断". The Redis client now sits behind - orchestration.run_event_stream, so that is where the empty read enters. + core.runs.events.store, so that is where the empty read enters. """ class _NoneStreamRedis: @@ -326,7 +338,7 @@ async def xrange(self, *a, **k): async def xread(self, *a, **k): return None - monkeypatch.setattr("orchestration.run_event_stream.get_redis", lambda **_: _NoneStreamRedis()) + monkeypatch.setattr('core.runs.events.store.get_redis', lambda **_: _NoneStreamRedis()) journal = RunJournal(recovery_env) journal.accept( run_id="run-none-stream", @@ -340,7 +352,7 @@ async def xread(self, *a, **k): assert journal.complete("run-none-stream", owner="w", status="failed") events = [] - async for event in executor.follow_run("run-none-stream", from_offset=0): + async for event in subscription.follow_run("run-none-stream", from_offset=0): events.append(event) assert events == [] @@ -398,16 +410,16 @@ async def workflow(**_kwargs): "usage": {"input_tokens": 1, "output_tokens": 1}, } - monkeypatch.setattr("orchestration.run_event_stream.get_redis", lambda **_: client) - monkeypatch.setattr(executor, "astream_chat_workflow", workflow) - monkeypatch.setattr(executor, "_spawn_followup_task", lambda **_kwargs: None) - monkeypatch.setattr(executor, "_spawn_compaction_task", lambda **_kwargs: None) + monkeypatch.setattr('core.runs.events.store.get_redis', lambda **_: client) + monkeypatch.setattr(runner, "astream_chat_workflow", workflow) + monkeypatch.setattr(followups, "_spawn_followup_task", lambda **_kwargs: None) + monkeypatch.setattr(compaction, "_spawn_compaction_task", lambda **_kwargs: None) monkeypatch.setattr( - "core.services.artifact_service.persist_artifacts", + 'core.files.artifacts.service.persist_artifacts', lambda *_args, **_kwargs: None, ) - run = await executor.start_run( + run = await admission.start_run( chat_id="chat-1", user_id="user-1", session_messages=[{"role": "user", "content": "hello"}], @@ -419,7 +431,7 @@ async def workflow(**_kwargs): ) async def collect(): - return [event async for event in executor.follow_run(run.run_id)] + return [event async for event in subscription.follow_run(run.run_id)] follower = asyncio.create_task(collect()) await asyncio.wait_for(dropped.wait(), timeout=5) diff --git a/src/backend/tests/orchestration/test_chat_run_human_wait.py b/src/backend/tests/orchestration/test_chat_run_human_wait.py index e3cccc46..8a0abb6a 100644 --- a/src/backend/tests/orchestration/test_chat_run_human_wait.py +++ b/src/backend/tests/orchestration/test_chat_run_human_wait.py @@ -2,34 +2,34 @@ import asyncio +import core.runs.settings as settings +import core.runs.watchdog as watchdog import pytest - -from core.llm import human_interaction -from orchestration import chat_run_executor +from core.llm.runtime import human_interaction def test_stream_ttl_covers_maximum_human_wait_plus_recovery_grace(): assert ( - chat_run_executor._STREAM_TTL_SECONDS + settings._STREAM_TTL_SECONDS == human_interaction.MAX_WAIT_SECONDS + human_interaction.STREAM_RECOVERY_GRACE_SECONDS ) - assert chat_run_executor._STREAM_TTL_SECONDS > human_interaction.MAX_WAIT_SECONDS + assert settings._STREAM_TTL_SECONDS > human_interaction.MAX_WAIT_SECONDS def test_only_a_live_human_wait_turns_heartbeat_into_meaningful_activity(monkeypatch): monkeypatch.setattr(human_interaction, "has_pending", lambda _chat_id: False) - assert chat_run_executor._is_chat_run_activity( + assert watchdog._is_chat_run_activity( {"type": "heartbeat"}, "chat-1", ) is False - assert chat_run_executor._is_chat_run_activity( + assert watchdog._is_chat_run_activity( {"type": "content"}, "chat-1", ) is True monkeypatch.setattr(human_interaction, "has_pending", lambda _chat_id: True) - assert chat_run_executor._is_chat_run_activity( + assert watchdog._is_chat_run_activity( {"type": "heartbeat"}, "chat-1", ) is True @@ -45,10 +45,10 @@ async def delayed_content(): monkeypatch.setattr(human_interaction, "has_pending", lambda _chat_id: True) items = [] - async for item in chat_run_executor._aiter_with_inactivity_timeout( + async for item in watchdog._aiter_with_inactivity_timeout( delayed_content(), 0.02, - is_activity=lambda item: chat_run_executor._is_chat_run_activity( + is_activity=lambda item: watchdog._is_chat_run_activity( item, "chat-waiting", ), @@ -66,10 +66,10 @@ async def only_heartbeats(): monkeypatch.setattr(human_interaction, "has_pending", lambda _chat_id: False) with pytest.raises(TimeoutError, match="无有效输出"): - async for _ in chat_run_executor._aiter_with_inactivity_timeout( + async for _ in watchdog._aiter_with_inactivity_timeout( only_heartbeats(), 0.02, - is_activity=lambda item: chat_run_executor._is_chat_run_activity( + is_activity=lambda item: watchdog._is_chat_run_activity( item, "chat-not-waiting", ), diff --git a/src/backend/tests/orchestration/test_chat_run_recovery.py b/src/backend/tests/orchestration/test_chat_run_recovery.py index 357f7b77..ae4f030b 100644 --- a/src/backend/tests/orchestration/test_chat_run_recovery.py +++ b/src/backend/tests/orchestration/test_chat_run_recovery.py @@ -3,6 +3,10 @@ from __future__ import annotations import api.routes.v1.chats.run_views as chat_run_views +import core.agent_runtime.runner as runner +import core.chat.runtime.admission as admission +import core.chat.runtime.worker as worker +import core.runs.runtime as runtime import fakeredis.aioredis import pytest from agentscope.agent import Agent, ReActConfig @@ -14,10 +18,11 @@ from api.routes.v1 import chats as chat_routes from core.auth.backend import UserContext from core.db.models import ChatRun, ChatSession -from core.llm.middlewares import AgentRuntimeState, ToolEffectMiddleware -from core.services.run_journal import RunJournal -from core.services.tool_effect_ledger import ToolEffectJournal, recover_incomplete_tool_effects -from orchestration import chat_run_executor as executor +from core.llm.runtime.state import AgentRuntimeState +from tests.orchestration.hook_support import make_adapter, durable_adapter +from core.runs.journal import RunJournal +from core.runs.tool_effects.journal import ToolEffectJournal +from core.runs.tool_effects.recovery import recover_incomplete_tool_effects from tests.orchestration.recovery_test_support import recovery_env @@ -40,9 +45,9 @@ def register(run_id, coro, *, name): ) coro.close() - monkeypatch.setattr(executor, "_register_run_task", register) + monkeypatch.setattr(runtime, 'register_run_task', register) - run = await executor.start_run( + run = await admission.start_run( chat_id="chat-1", user_id="user-1", session_messages=[{"role": "user", "content": "hello"}], @@ -77,7 +82,7 @@ def test_active_run_probe_hides_internal_agent_rows(recovery_env): ) assert journal.claim(internal.run_id, owner="internal", lease_seconds=60) - assert executor.get_active_run_for_chat("chat-1", "user-1") is None + assert runtime.get_active_run_for_chat("chat-1", "user-1") is None public = journal.accept( run_id="run-public-visible", @@ -88,7 +93,7 @@ def test_active_run_probe_hides_internal_agent_rows(recovery_env): recovery_snapshot={"kind": "chat", "worker_args": {}}, ) assert journal.claim(public.run_id, owner="public", lease_seconds=60) - assert executor.get_active_run_for_chat("chat-1", "user-1").run_id == public.run_id + assert runtime.get_active_run_for_chat("chat-1", "user-1").run_id == public.run_id def test_list_active_runs_spans_chats_and_hides_background_kinds(recovery_env): @@ -141,9 +146,9 @@ def test_list_active_runs_spans_chats_and_hides_background_kinds(recovery_env): recovery_snapshot={"kind": "chat", "worker_args": {}}, ) - listed, _ = executor.list_active_runs_for_user("user-1") + listed, _ = runtime.list_active_runs_for_user("user-1") assert {row.chat_id for row in listed} == {"chat-1", "chat-2"} - assert "pa_chat_abc" not in executor.list_active_runs_for_user("user-1", 3600)[1] + assert "pa_chat_abc" not in runtime.list_active_runs_for_user("user-1", 3600)[1] assert {row.run_id for row in listed} == {live_one.run_id, live_two.run_id} # A finished run must stop lighting the sidebar dot, even while its writer @@ -152,7 +157,7 @@ def test_list_active_runs_spans_chats_and_hides_background_kinds(recovery_env): row = db.get(ChatRun, live_one.run_id) row.status = "completed" db.commit() - assert {row.chat_id for row in executor.list_active_runs_for_user("user-1")[0]} == {"chat-2"} + assert {row.chat_id for row in runtime.list_active_runs_for_user("user-1")[0]} == {"chat-2"} def test_active_runs_route_returns_chat_ids_and_outranks_the_detail_route(recovery_env): @@ -227,7 +232,7 @@ def test_active_runs_route_reports_recent_chats_whose_run_already_ended(recovery async def test_public_worker_keeps_ambiguous_agent_tool_call_recoverable(recovery_env, monkeypatch): sessions = recovery_env redis = fakeredis.aioredis.FakeRedis(decode_responses=True) - monkeypatch.setattr("orchestration.run_event_stream.get_redis", lambda **_: redis) + monkeypatch.setattr('core.runs.events.store.get_redis', lambda **_: redis) journal = RunJournal(sessions) row = journal.accept( run_id="run-agent-timeout", @@ -244,6 +249,8 @@ async def exploding_probe(value: int): raise DeveloperOrientedException("adapter response was lost") class ToolModel: + from agentscope.formatter import OpenAIChatFormatter + formatter = OpenAIChatFormatter() model = "ambiguous-tool" context_size = 32768 @@ -271,7 +278,7 @@ async def workflow(*, context, **_kwargs): system_prompt="Call the tool.", model=ToolModel(), toolkit=Toolkit(tools=[FunctionTool(exploding_probe)]), - middlewares=[ToolEffectMiddleware(session_factory=sessions)], + middlewares=[durable_adapter(session_factory=sessions)], state=AgentRuntimeState( run_id=context["run_id"], journal_owner=context["journal_owner"], @@ -283,8 +290,8 @@ async def workflow(*, context, **_kwargs): if False: yield {} - monkeypatch.setattr(executor, "astream_chat_workflow", workflow) - await executor._run_workflow( + monkeypatch.setattr(runner, "astream_chat_workflow", workflow) + await worker._run_workflow( run_id=row.run_id, chat_id=row.chat_id, user_id=row.user_id, diff --git a/src/backend/tests/orchestration/test_chat_run_restart_recovery.py b/src/backend/tests/orchestration/test_chat_run_restart_recovery.py index 5a3037c0..064ef537 100644 --- a/src/backend/tests/orchestration/test_chat_run_restart_recovery.py +++ b/src/backend/tests/orchestration/test_chat_run_restart_recovery.py @@ -1,18 +1,26 @@ """Behavioral regression coverage.""" from __future__ import annotations +from core.chat.projection.codec import chat_projection_codec +from core.ontology import runtime_policy + from datetime import datetime, timedelta, timezone + +import core.agent_runtime.runner as runner +import core.chat.runtime.compaction as compaction +import core.chat.runtime.followups as followups +import core.chat.runtime.recovery as recovery +import core.chat.runtime.worker as worker +import core.runs.events.store as store +import core.runs.events.store as run_event_stream +import core.runs.events.writer as writer +import core.runs.runtime as runtime +import core.runs.settings as settings import pytest from core.db.models import ChatMessage, ChatRun, ChatRunOperation, ChatSteerQueueItem -from core.services.run_journal import RunJournal -from core.services.steer_queue import SteerQueue -from orchestration import chat_run_executor as executor -from orchestration import run_event_stream - -from tests.orchestration.recovery_test_support import ( - recovery_env, - _async_none, -) +from core.runs.journal import RunJournal +from core.chat.steer_queue import SteerQueue +from tests.orchestration.recovery_test_support import _async_none, recovery_env @pytest.mark.asyncio @@ -21,9 +29,10 @@ async def test_automation_prompt_injects_durable_binding_and_recovery_surface( ): sessions = recovery_env from core.db import engine as db_engine - from core.services import ontology_service, user_model_selection - from orchestration import workflow - from orchestration.schedulers.automation_conversation import execute_prompt + from core.ontology import service as ontology_service + from core.models import selection as user_model_selection + pass + from orchestration.automations.automation_conversation import execute_prompt monkeypatch.setattr(db_engine, "SessionLocal", sessions) monkeypatch.setattr( @@ -32,8 +41,8 @@ async def test_automation_prompt_injects_durable_binding_and_recovery_surface( lambda: "test-model", ) monkeypatch.setattr( - ontology_service, - "build_user_ontology_runtime", + runtime_policy, + 'build_user_ontology_runtime', lambda **_kwargs: (False, {}), ) captured = {} @@ -43,11 +52,11 @@ async def bound_workflow(*, context, **_kwargs): yield {"type": "content", "delta": "automation result"} yield {"type": "meta", "usage": {"total_tokens": 2}} - monkeypatch.setattr(workflow, "astream_chat_workflow", bound_workflow) - monkeypatch.setattr(executor, "astream_chat_workflow", bound_workflow) - stream = run_event_stream.LocalRunEventStream() - monkeypatch.setattr(executor, "get_run_event_stream", lambda: stream) - from core.services.automation_service import AutomationService + monkeypatch.setattr(runner, "astream_chat_workflow", bound_workflow) + monkeypatch.setattr(runner, "astream_chat_workflow", bound_workflow) + stream = run_event_stream.LocalRunEventStream(chat_projection_codec()) + monkeypatch.setattr(store, "get_run_event_stream", lambda: stream) + from features.automation.service import AutomationService with sessions() as db: task = AutomationService(db).create_task( @@ -110,9 +119,9 @@ def register(run_id, coro, *, name): registered.append((run_id, name)) coro.close() - monkeypatch.setattr(executor, "_register_run_task", register) + monkeypatch.setattr(runtime, 'register_run_task', register) - assert await executor.recover_orphan_runs() == 1 + assert await recovery.recover_orphan_runs() == 1 assert registered == [(row.run_id, f"chat_run_recovery:{row.run_id}")] with sessions() as db: recovered = db.get(ChatRun, row.run_id) @@ -157,12 +166,12 @@ async def test_startup_recovery_commits_saved_model_output_without_second_model_ registered = [] monkeypatch.setattr( - executor, - "_register_run_task", + runtime, + 'register_run_task', lambda *args, **kwargs: registered.append((args, kwargs)), ) - assert await executor.recover_orphan_runs() == 1 + assert await recovery.recover_orphan_runs() == 1 assert registered == [] with sessions() as db: recovered = db.get(ChatRun, row.run_id) @@ -237,21 +246,21 @@ async def capture_event(_run_id, _offset, event): async def no_event_write(*_args, **_kwargs): return None - monkeypatch.setattr(executor, "_register_run_task", register) - monkeypatch.setattr(executor, "_xadd_event", capture_event) - monkeypatch.setattr(executor, "_expire_stream", no_event_write) + monkeypatch.setattr(runtime, 'register_run_task', register) + monkeypatch.setattr(writer, 'xadd_event', capture_event) + monkeypatch.setattr(writer, 'expire_stream', no_event_write) monkeypatch.setattr( - "core.services.artifact_service.persist_artifacts", + 'core.files.artifacts.service.persist_artifacts', lambda *_args, **_kwargs: None, ) - assert await executor.recover_orphan_runs() == 1 + assert await recovery.recover_orphan_runs() == 1 assert len(registered) == 1 assert registered[0][1].startswith("chat_run_handoff_recovery:") next_run_id = registered[0][0] assert [event["type"] for event in projected_events] == [ "queued_run_started", - executor._TERMINAL_TYPE, + settings.TERMINAL_TYPE, ] assert projected_events[0]["run_id"] == next_run_id assert projected_events[0]["message"] == "恢复后继续执行" @@ -303,9 +312,9 @@ async def test_unknown_tool_result_pauses_and_emits_compatible_terminal_projecti async def terminal(run_id, *, chat_id, error_text, cancelled=False): projected.append((run_id, chat_id, error_text, cancelled)) - monkeypatch.setattr(executor, "_write_terminal_to_stream", terminal) + monkeypatch.setattr(writer, "_write_terminal_to_stream", terminal) - assert await executor.recover_orphan_runs() == 1 + assert await recovery.recover_orphan_runs() == 1 with sessions() as db: recovered = db.get(ChatRun, row.run_id) assert recovered.status == "needs_attention" @@ -357,17 +366,17 @@ async def workflow(**_kwargs): async def emit(_run_id, offset, event): emitted.append((offset, dict(event))) - monkeypatch.setattr(executor, "astream_chat_workflow", workflow) - monkeypatch.setattr(executor, "_xadd_event", emit) - monkeypatch.setattr(executor, "_expire_stream", lambda _run_id: _async_none()) - monkeypatch.setattr(executor, "_spawn_followup_task", lambda **_kwargs: None) - monkeypatch.setattr(executor, "_spawn_compaction_task", lambda **_kwargs: None) + monkeypatch.setattr(runner, "astream_chat_workflow", workflow) + monkeypatch.setattr(writer, 'xadd_event', emit) + monkeypatch.setattr(writer, 'expire_stream', lambda _run_id: _async_none()) + monkeypatch.setattr(followups, "_spawn_followup_task", lambda **_kwargs: None) + monkeypatch.setattr(compaction, "_spawn_compaction_task", lambda **_kwargs: None) monkeypatch.setattr( - "core.services.artifact_service.persist_artifacts", + 'core.files.artifacts.service.persist_artifacts', lambda *_args, **_kwargs: None, ) - await executor._run_workflow( + await worker._run_workflow( run_id=row.run_id, chat_id=row.chat_id, user_id=row.user_id, @@ -401,4 +410,4 @@ async def emit(_run_id, offset, event): "run_completed", ] assert [item.operation_seq for item in operations] == [1, 2, 3, 4, 5] - assert emitted[-1][1]["type"] == executor._TERMINAL_TYPE + assert emitted[-1][1]["type"] == settings.TERMINAL_TYPE diff --git a/src/backend/tests/orchestration/test_chat_run_worker_recovery.py b/src/backend/tests/orchestration/test_chat_run_worker_recovery.py index a51154e4..0a93026a 100644 --- a/src/backend/tests/orchestration/test_chat_run_worker_recovery.py +++ b/src/backend/tests/orchestration/test_chat_run_worker_recovery.py @@ -4,13 +4,17 @@ from datetime import datetime, timedelta, timezone +import core.runs.events.store as run_event_stream import fakeredis.aioredis +import orchestration.loops.driver as driver +import orchestration.loops.run as loops_run +import orchestration.plans.execute_run as execute_run +import orchestration.plans.generate_run as generate_run +import orchestration.plans.mode as mode import pytest from core.db.models import ChatMessage, ChatRun, ChatRunOperation, ChatSession, Plan -from core.services.run_journal import RunJournal -from core.services.tool_effect_ledger import ToolOutcomeUnknown -from orchestration import chat_run_executor as executor -from orchestration import run_event_stream +from core.runs.journal import RunJournal +from core.runs.tool_effects.errors import ToolOutcomeUnknown from tests.orchestration.recovery_test_support import recovery_env @@ -18,7 +22,7 @@ async def test_plan_worker_pauses_nested_unknown_tool_outcome(recovery_env, monkeypatch): sessions = recovery_env redis = fakeredis.aioredis.FakeRedis(decode_responses=True) - monkeypatch.setattr("orchestration.run_event_stream.get_redis", lambda **_: redis) + monkeypatch.setattr('core.runs.events.store.get_redis', lambda **_: redis) journal = RunJournal(sessions) row = journal.accept( run_id="run-plan-tool-unknown", @@ -28,14 +32,14 @@ async def test_plan_worker_pauses_nested_unknown_tool_outcome(recovery_env, monk request_payload={"kind": "plan_execute", "plan_id": "plan-1"}, recovery_snapshot={"kind": "plan_execute", "worker_args": {"context": {}}}, ) - from orchestration.subagents import plan_mode + import orchestration.plans.mode as plan_mode async def broken_plan(**_kwargs): raise ExceptionGroup("plan tool failed", [ToolOutcomeUnknown("effect-plan")]) yield # pragma: no cover - monkeypatch.setattr(plan_mode, "astream_execute_plan", broken_plan) - await executor._run_plan_execute_workflow( + monkeypatch.setattr(mode, "astream_execute_plan", broken_plan) + await execute_run._run_plan_execute_workflow( run_id=row.run_id, plan_id="plan-1", chat_id=row.chat_id, @@ -59,7 +63,7 @@ async def broken_plan(**_kwargs): async def test_plan_generate_fences_late_message_after_lease_takeover(recovery_env, monkeypatch): sessions = recovery_env redis = fakeredis.aioredis.FakeRedis(decode_responses=True) - monkeypatch.setattr("orchestration.run_event_stream.get_redis", lambda **_: redis) + monkeypatch.setattr('core.runs.events.store.get_redis', lambda **_: redis) journal = RunJournal(sessions) row = journal.accept( run_id="run-plan-generate-fenced", @@ -69,7 +73,7 @@ async def test_plan_generate_fences_late_message_after_lease_takeover(recovery_e request_payload={"kind": "plan_generate"}, recovery_snapshot={"kind": "plan_generate", "worker_args": {}}, ) - from orchestration.subagents import plan_mode + import orchestration.plans.mode as plan_mode async def generated_then_taken_over(**_kwargs): yield { @@ -89,8 +93,8 @@ async def generated_then_taken_over(**_kwargs): ) db.commit() - monkeypatch.setattr(plan_mode, "astream_generate_plan", generated_then_taken_over) - await executor._run_plan_generate_workflow( + monkeypatch.setattr(mode, "astream_generate_plan", generated_then_taken_over) + await generate_run._run_plan_generate_workflow( run_id=row.run_id, chat_id=row.chat_id, user_id=row.user_id, @@ -123,7 +127,7 @@ async def test_plan_generate_commits_message_and_terminal_state_atomically( ): sessions = recovery_env redis = fakeredis.aioredis.FakeRedis(decode_responses=True) - monkeypatch.setattr("orchestration.run_event_stream.get_redis", lambda **_: redis) + monkeypatch.setattr('core.runs.events.store.get_redis', lambda **_: redis) journal = RunJournal(sessions) row = journal.accept( run_id="run-plan-generate-complete", @@ -133,7 +137,7 @@ async def test_plan_generate_commits_message_and_terminal_state_atomically( request_payload={"kind": "plan_generate"}, recovery_snapshot={"kind": "plan_generate", "worker_args": {}}, ) - from orchestration.subagents import plan_mode + import orchestration.plans.mode as plan_mode async def generated(**_kwargs): yield { @@ -145,8 +149,8 @@ async def generated(**_kwargs): "usage": {"total_tokens": 4}, } - monkeypatch.setattr(plan_mode, "astream_generate_plan", generated) - await executor._run_plan_generate_workflow( + monkeypatch.setattr(mode, "astream_generate_plan", generated) + await generate_run._run_plan_generate_workflow( run_id=row.run_id, chat_id=row.chat_id, user_id=row.user_id, @@ -188,7 +192,7 @@ async def test_autonomous_worker_pauses_nested_unknown_without_partial_stale_wri ): sessions = recovery_env redis = fakeredis.aioredis.FakeRedis(decode_responses=True) - monkeypatch.setattr("orchestration.run_event_stream.get_redis", lambda **_: redis) + monkeypatch.setattr('core.runs.events.store.get_redis', lambda **_: redis) journal = RunJournal(sessions) row = journal.accept( run_id="run-loop-tool-unknown", @@ -198,13 +202,13 @@ async def test_autonomous_worker_pauses_nested_unknown_without_partial_stale_wri request_payload={"kind": "autonomous_loop", "loop_id": "loop-1"}, recovery_snapshot={"kind": "autonomous_loop", "worker_args": {"context": {}}}, ) - from orchestration import autonomous_loop + import orchestration.loops.driver as autonomous_loop async def broken_loop(**_kwargs): raise ExceptionGroup("loop tool failed", [ToolOutcomeUnknown("effect-loop")]) - monkeypatch.setattr(autonomous_loop, "run_autonomous_loop", broken_loop) - await executor._run_autonomous_loop_workflow( + monkeypatch.setattr(driver, "run_autonomous_loop", broken_loop) + await loops_run._run_autonomous_loop_workflow( run_id=row.run_id, loop_id="loop-1", chat_id=row.chat_id, @@ -228,7 +232,7 @@ async def test_autonomous_project_binding_is_rejected_after_lease_takeover( ): sessions = recovery_env redis = fakeredis.aioredis.FakeRedis(decode_responses=True) - monkeypatch.setattr("orchestration.run_event_stream.get_redis", lambda **_: redis) + monkeypatch.setattr('core.runs.events.store.get_redis', lambda **_: redis) journal = RunJournal(sessions) row = journal.accept( run_id="run-loop-project-fenced", @@ -238,8 +242,8 @@ async def test_autonomous_project_binding_is_rejected_after_lease_takeover( request_payload={"kind": "autonomous_loop", "loop_id": "loop-project"}, recovery_snapshot={"kind": "autonomous_loop", "worker_args": {}}, ) - from core.services import project_scope - from orchestration import autonomous_loop + import orchestration.loops.driver as autonomous_loop + from core.projects import edition as project_scope monkeypatch.setattr( project_scope, @@ -270,8 +274,8 @@ async def must_not_run(**_kwargs): pytest.fail("stale autonomous worker reached model execution") monkeypatch.setattr(RunJournal, "append_operation", append_after_takeover) - monkeypatch.setattr(autonomous_loop, "run_autonomous_loop", must_not_run) - await executor._run_autonomous_loop_workflow( + monkeypatch.setattr(driver, "run_autonomous_loop", must_not_run) + await loops_run._run_autonomous_loop_workflow( run_id=row.run_id, loop_id="loop-project", chat_id=row.chat_id, @@ -294,7 +298,7 @@ async def must_not_run(**_kwargs): def test_retirement_removes_batch_run_ledger_without_losing_plan_recovery(recovery_env): from core.db.batch_retirement import retire_batch_execution - from core.db.models import ToolEffectLedger, ToolEffectLease, ToolEffectReceipt + from core.db.models import ToolEffectLease, ToolEffectLedger, ToolEffectReceipt sessions = recovery_env journal = RunJournal(sessions) diff --git a/src/backend/tests/orchestration/test_chat_steer.py b/src/backend/tests/orchestration/test_chat_steer.py index 9ac6f498..f47a388f 100644 --- a/src/backend/tests/orchestration/test_chat_steer.py +++ b/src/backend/tests/orchestration/test_chat_steer.py @@ -1,23 +1,35 @@ """Mid-run chat steering: Redis hand-off and ReAct middleware behavior.""" +from tests.orchestration.hook_support import make_adapter +from agentscope.message import ToolCallBlock + from datetime import datetime, timedelta, timezone from types import SimpleNamespace +import core.agent_runtime.runner as runner +import core.chat.runtime.compaction as compaction +import core.chat.runtime.followups as followups +import core.chat.runtime.handoff as handoff +import core.chat.runtime.worker as worker +import core.db.engine as db_engine +import core.runs.clock as clock +import core.runs.events.writer as writer +import core.runs.ownership as ownership +import core.runs.runtime as runtime +import core.chat.service as services_ChatService_module import fakeredis.aioredis import pytest from agentscope.message import ToolResultState from agentscope.tool._response import ToolChunk -from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker - from core.db.engine import Base from core.db.models import ChatMessage, ChatRun, ChatSession, ChatSteerQueueItem -from core.llm.steer_middleware import SteerMiddleware -from core.services import chat_steer_service -from core.services.chat_service import ChatService -from core.services.run_journal import RunJournal, RunLeaseLost -from core.services.steer_queue import SteerQueue -from orchestration import chat_run_executor as executor +from core.chat.steer_hook import SteeringHook +from core.chat import steer as chat_steer_service +from core.chat.service import ChatService +from core.runs.journal import RunJournal, RunLeaseLost +from core.chat.steer_queue import SteerQueue +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker @pytest.fixture() @@ -121,7 +133,7 @@ async def take_pending_steer(run_id: str): return delivery monkeypatch.setattr(chat_steer_service, "take_pending_steer", take_pending_steer) - middleware = SteerMiddleware() + middleware = make_adapter(*SteeringHook().specs()) agent = SimpleNamespace(state=SimpleNamespace(run_id="r1", context=[], steer_delivery=None)) tool_executed = False @@ -130,7 +142,7 @@ async def tool_handler(**_kwargs): tool_executed = True yield ToolChunk(content=[], state=ToolResultState.SUCCESS) - results = [item async for item in middleware.on_acting(agent, {}, tool_handler)] + results = [item async for item in middleware.on_acting(agent, {"tool_call": ToolCallBlock(id="steer-tool", name="test_tool", input="{}")}, tool_handler)] assert tool_executed is False assert results[-1].state == ToolResultState.INTERRUPTED @@ -160,7 +172,7 @@ async def take_pending_steer(run_id: str): return None if polls == 1 else delivery monkeypatch.setattr(chat_steer_service, "take_pending_steer", take_pending_steer) - middleware = SteerMiddleware() + middleware = make_adapter(*SteeringHook().specs()) agent = SimpleNamespace(state=SimpleNamespace(run_id="r1", context=[], steer_delivery=None)) tool_executed = False @@ -169,7 +181,7 @@ async def tool_handler(**_kwargs): tool_executed = True yield ToolChunk(content=[], state=ToolResultState.SUCCESS) - results = [item async for item in middleware.on_acting(agent, {}, tool_handler)] + results = [item async for item in middleware.on_acting(agent, {"tool_call": ToolCallBlock(id="steer-tool", name="test_tool", input="{}")}, tool_handler)] assert tool_executed is True assert results[-1].state == ToolResultState.SUCCESS @@ -192,13 +204,13 @@ async def take_pending_steer(_run_id: str): return None monkeypatch.setattr(chat_steer_service, "take_pending_steer", take_pending_steer) - middleware = SteerMiddleware() + middleware = make_adapter(*SteeringHook().specs()) agent = SimpleNamespace(state=SimpleNamespace(run_id="r1", context=[], steer_delivery=None)) async def tool_handler(**_kwargs): yield ToolChunk(content=[], state=ToolResultState.SUCCESS) - results = [item async for item in middleware.on_acting(agent, {}, tool_handler)] + results = [item async for item in middleware.on_acting(agent, {"tool_call": ToolCallBlock(id="steer-tool", name="test_tool", input="{}")}, tool_handler)] assert results[-1].state == ToolResultState.SUCCESS @@ -277,38 +289,38 @@ async def fake_workflow(**_kwargs): async def fake_xadd(_run_id, _offset, event): emitted.append(dict(event)) - monkeypatch.setattr(executor, "SessionLocal", lambda: FakeSession()) - monkeypatch.setattr(executor, "get_run", lambda _run_id: SimpleNamespace(request_payload={}, status="running")) - monkeypatch.setattr(executor, "ChatService", FakeChatService) - monkeypatch.setattr(executor, "_persisted_history", lambda *_args: []) + monkeypatch.setattr(db_engine, "SessionLocal", lambda: FakeSession()) + monkeypatch.setattr(runtime, "get_run", lambda _run_id: SimpleNamespace(request_payload={}, status="running")) + monkeypatch.setattr(services_ChatService_module, "ChatService", FakeChatService) + monkeypatch.setattr(handoff, "_persisted_history", lambda *_args: []) fake_journal = FakeJournal() - monkeypatch.setattr(executor, "_journal", lambda: fake_journal) - monkeypatch.setattr(executor, "astream_chat_workflow", fake_workflow) - monkeypatch.setattr(executor, "_xadd_event", fake_xadd) - monkeypatch.setattr(executor, "_update_run_status", lambda *_args, **_kwargs: None) - monkeypatch.setattr(executor, "_claim_run_execution", lambda _run_id: executor._utcnow()) - monkeypatch.setattr(executor, "_acknowledge_terminal_writer", lambda _run_id: False) - monkeypatch.setattr(executor, "_finalize_run", lambda *_args, **_kwargs: None) + monkeypatch.setattr(ownership, 'journal', lambda: fake_journal) + monkeypatch.setattr(runner, "astream_chat_workflow", fake_workflow) + monkeypatch.setattr(writer, 'xadd_event', fake_xadd) + monkeypatch.setattr(ownership, "_update_run_status", lambda *_args, **_kwargs: None) + monkeypatch.setattr(ownership, "_claim_run_execution", lambda _run_id: clock._utcnow()) + monkeypatch.setattr(ownership, "_acknowledge_terminal_writer", lambda _run_id: False) + monkeypatch.setattr(ownership, "_finalize_run", lambda *_args, **_kwargs: None) monkeypatch.setattr( - executor, + handoff, "_commit_queued_handoff_in_session", lambda *_args, **_kwargs: None, ) - monkeypatch.setattr(executor, "is_run_cancelled", lambda _run_id: False) - monkeypatch.setattr(executor, "_spawn_followup_task", lambda **_kwargs: None) - monkeypatch.setattr(executor, "_spawn_compaction_task", lambda **_kwargs: None) + monkeypatch.setattr(runtime, "is_run_cancelled", lambda _run_id: False) + monkeypatch.setattr(followups, "_spawn_followup_task", lambda **_kwargs: None) + monkeypatch.setattr(compaction, "_spawn_compaction_task", lambda **_kwargs: None) async def fake_expire(_run_id): return None - monkeypatch.setattr(executor, "_expire_stream", fake_expire) + monkeypatch.setattr(writer, 'expire_stream', fake_expire) monkeypatch.setattr( - "core.services.artifact_service.persist_artifacts", + 'core.files.artifacts.service.persist_artifacts', lambda *_args, **_kwargs: None, ) context = {"chat_id": "c1", "user_id": "u1"} - await executor._run_workflow( + await worker._run_workflow( run_id="r1", chat_id="c1", user_id="u1", @@ -361,18 +373,18 @@ async def fake_workflow(**_kwargs): async def no_event_write(*_args, **_kwargs): return None - monkeypatch.setattr(executor, "SessionLocal", sessions) - monkeypatch.setattr(executor, "astream_chat_workflow", fake_workflow) - monkeypatch.setattr(executor, "_xadd_event", no_event_write) - monkeypatch.setattr(executor, "_expire_stream", no_event_write) - monkeypatch.setattr(executor, "_spawn_followup_task", lambda **_kwargs: None) - monkeypatch.setattr(executor, "_spawn_compaction_task", lambda **_kwargs: None) + monkeypatch.setattr(db_engine, "SessionLocal", sessions) + monkeypatch.setattr(runner, "astream_chat_workflow", fake_workflow) + monkeypatch.setattr(writer, 'xadd_event', no_event_write) + monkeypatch.setattr(writer, 'expire_stream', no_event_write) + monkeypatch.setattr(followups, "_spawn_followup_task", lambda **_kwargs: None) + monkeypatch.setattr(compaction, "_spawn_compaction_task", lambda **_kwargs: None) monkeypatch.setattr( - "core.services.artifact_service.persist_artifacts", + 'core.files.artifacts.service.persist_artifacts', lambda *_args, **_kwargs: None, ) - await executor._run_workflow( + await worker._run_workflow( run_id="r1", chat_id="chat-1", user_id="user-1", @@ -430,16 +442,16 @@ async def crashing_workflow(**_kwargs): async def no_event_write(*_args, **_kwargs): return None - monkeypatch.setattr(executor, "SessionLocal", sessions) - monkeypatch.setattr(executor, "astream_chat_workflow", crashing_workflow) - monkeypatch.setattr(executor, "_xadd_event", no_event_write) - monkeypatch.setattr(executor, "_expire_stream", no_event_write) + monkeypatch.setattr(db_engine, "SessionLocal", sessions) + monkeypatch.setattr(runner, "astream_chat_workflow", crashing_workflow) + monkeypatch.setattr(writer, 'xadd_event', no_event_write) + monkeypatch.setattr(writer, 'expire_stream', no_event_write) # Admission persists the user's message before the worker starts; the # post-steer context is read back from those rows. with sessions() as db: ChatService(db).add_message(chat_id="chat-1", role="user", content="原问题") - await executor._run_workflow( + await worker._run_workflow( run_id="r1", chat_id="chat-1", user_id="user-1", diff --git a/src/backend/tests/orchestration/test_command_events.py b/src/backend/tests/orchestration/test_command_events.py index 3a0a59af..ff3b9dbf 100644 --- a/src/backend/tests/orchestration/test_command_events.py +++ b/src/backend/tests/orchestration/test_command_events.py @@ -1,4 +1,5 @@ """Command wake transport, snapshot races and authentication boundaries.""" + import asyncio import json import shutil @@ -6,15 +7,22 @@ import subprocess from types import SimpleNamespace from unittest.mock import AsyncMock + +import core.chat.runtime.admission as admission +import core.db.engine as engine +import core.sandbox.commands.events as commands_events +import core.sandbox.commands.events as events +import core.sandbox.commands.notifications as notifications +import core.sandbox.commands.subscription as commands_subscription +import core.sandbox.commands.subscription as subscription import pytest +from core.db import models +from core.chat.sequencer import ChatSequencer +from core.sandbox.commands import delivery, store from fastapi import HTTPException -from starlette.requests import Request from redis.asyncio import Redis +from starlette.requests import Request from tests.orchestration.test_background_commands import command_db, request -from orchestration.commands import events, subscription, notifications -from core.services.commands import store, delivery -from core.db import models -from core.services.chat_sequencer import ChatSequencer async def test_memory_event_wakes_immediately_and_isolates_topics(monkeypatch): @@ -28,13 +36,13 @@ async def test_memory_event_wakes_immediately_and_isolates_topics(monkeypatch): await bus.publish("u", "c", "wake") _, runs, _ = await asyncio.wait_for(waiter, .5) assert runs == ["wake"] - monkeypatch.setattr(events, "MAX_EVENTS", 2) + monkeypatch.setattr(commands_events, "MAX_EVENTS", 2) bus = events.MemoryCommandEvents() for index in range(3): await bus.publish("u", "c", str(index)) _, runs, resync = await bus.wait("u", "c", 0) assert runs == ["1", "2"] and resync - monkeypatch.setattr(events, "TTL_SECONDS", -.1) + monkeypatch.setattr(commands_events, "TTL_SECONDS", -.1) await bus.publish("u", "c", "expired") assert await bus.tail("u", "c") == 0 @@ -56,7 +64,7 @@ async def test_redis_two_instances_deliver_and_isolate(tmp_path, monkeypatch): break except Exception: await asyncio.sleep(.02) - monkeypatch.setattr(events, "get_redis", lambda blocking=False: clients[int(blocking)]) + monkeypatch.setattr(commands_events, "get_redis", lambda blocking=False: clients[int(blocking)]) publisher, subscriber = events.RedisCommandEvents(), events.RedisCommandEvents() cursor = await subscriber.tail("u", "c") waiting = asyncio.create_task(subscriber.wait("u", "c", cursor, timeout=1)) @@ -75,19 +83,19 @@ async def test_redis_two_instances_deliver_and_isolate(tmp_path, monkeypatch): async def test_snapshot_tail_race_and_auth_revocation(monkeypatch): bus = events.MemoryCommandEvents() - monkeypatch.setattr(subscription, "get_command_events", lambda: bus) - monkeypatch.setattr(subscription, "HEARTBEAT_SECONDS", .01) + monkeypatch.setattr(commands_subscription, "get_command_events", lambda: bus) + monkeypatch.setattr(commands_subscription, "HEARTBEAT_SECONDS", .01) authorized = True async def authorize(*args): if not authorized: raise HTTPException(401, "expired") return "u" - monkeypatch.setattr(subscription, "authorize", authorize) + monkeypatch.setattr(commands_subscription, "authorize", authorize) def snapshot(chat_id, user_id): # Publication happens after captured cursor, while DB snapshot is read. asyncio.run(bus.publish(user_id, chat_id, "during-snapshot")) return [] - monkeypatch.setattr(subscription, "snapshot", snapshot) + monkeypatch.setattr(commands_subscription, "snapshot", snapshot) stream = subscription.subscribe(None, "c", "u") assert json.loads((await anext(stream))[6:])["run_ids"] == [] assert json.loads((await anext(stream))[6:])["run_ids"] == ["during-snapshot"] @@ -99,7 +107,7 @@ def snapshot(chat_id, user_id): async def test_event_authorization_rechecks_identity_scope_and_edit(command_db, monkeypatch): req = Request({"type":"http", "headers":[]}) user = SimpleNamespace(user_id="u", api_key_id=None, public_chat_visitor=None) - monkeypatch.setattr(subscription, "get_current_user", AsyncMock(side_effect=lambda *a, **kw: user)) + monkeypatch.setattr(commands_subscription, "get_current_user", AsyncMock(side_effect=lambda *a, **kw: user)) assert await subscription.authorize(req, "c") == "u" user.user_id = "other" with pytest.raises(HTTPException) as error: @@ -123,7 +131,7 @@ async def test_event_authorization_rechecks_identity_scope_and_edit(command_db, async def test_publish_failure_retries_without_starting_run_again(command_db, monkeypatch): - from orchestration import chat_run_executor as executor + pass with command_db() as db: source = ChatSequencer(db).accept_main_run(chat_id="c", user_id="u", user_content="start", request_payload={}).run @@ -145,9 +153,9 @@ async def start_run(**kwargs): # published. That read must not cancel the accepted delivery. assert store.read(row["task_id"], "u", "c")["status"] == "exited" assert store.snapshot(row["task_id"])["notice_status"] == "reserved" - monkeypatch.setattr(executor, "start_run", start_run) + monkeypatch.setattr(admission, "start_run", start_run) publisher = SimpleNamespace(publish=AsyncMock(side_effect=[RuntimeError("Redis down"), None])) - monkeypatch.setattr(events, "get_command_events", lambda: publisher) + monkeypatch.setattr(commands_events, "get_command_events", lambda: publisher) with pytest.raises(RuntimeError, match="Redis down"): await notifications.launch(run_id) assert store.snapshot(row["task_id"])["notice_status"] == "reserved" @@ -163,20 +171,21 @@ async def start_run(**kwargs): async def test_real_http_sse_is_authenticated_and_pushes_without_polling(command_db, monkeypatch): import threading + import httpx import uvicorn - from fastapi import FastAPI from api.routes.v1.commands import router + from fastapi import FastAPI bus = events.MemoryCommandEvents() - monkeypatch.setattr(subscription, "get_command_events", lambda: bus) - monkeypatch.setattr(subscription, "HEARTBEAT_SECONDS", .03) + monkeypatch.setattr(commands_subscription, "get_command_events", lambda: bus) + monkeypatch.setattr(commands_subscription, "HEARTBEAT_SECONDS", .03) revoked = False async def authenticated(req, credentials, db): if credentials is None or revoked: raise HTTPException(401, "Authentication expired") return SimpleNamespace(user_id=credentials.credentials, api_key_id=None, public_chat_visitor=None) - monkeypatch.setattr(subscription, "get_current_user", authenticated) + monkeypatch.setattr(commands_subscription, "get_current_user", authenticated) app = FastAPI() app.include_router(router) with socket.socket() as listener: @@ -233,7 +242,7 @@ async def test_memory_publish_unregisters_closed_subscriber_loop(): def test_snapshot_is_bounded_and_returns_latest_runs(command_db, monkeypatch): from datetime import datetime, timezone - monkeypatch.setattr(subscription, "MAX_EVENTS", 2) + monkeypatch.setattr(commands_subscription, "MAX_EVENTS", 2) run_ids = [] for index in range(3): with command_db() as db: @@ -254,9 +263,9 @@ def test_snapshot_is_bounded_and_returns_latest_runs(command_db, monkeypatch): @pytest.mark.parametrize(("mode", "thinking"), [("fast", False), ("high", True)]) def test_followup_active_run_inherits_execution_mode(command_db, monkeypatch, mode, thinking): from core.auth.backend import UserContext - from core.services.session_operations.runs import chat_active_run - from orchestration import chat_run_executor as executor - monkeypatch.setattr(executor, "SessionLocal", command_db) + from core.chat.operations.runs import chat_active_run + pass + monkeypatch.setattr(engine, "SessionLocal", command_db) with command_db() as db: source = ChatSequencer(db).accept_main_run(chat_id="c", user_id="u", user_content="start", diff --git a/src/backend/tests/orchestration/test_command_recovery.py b/src/backend/tests/orchestration/test_command_recovery.py index 94cbe342..6537e3a5 100644 --- a/src/backend/tests/orchestration/test_command_recovery.py +++ b/src/backend/tests/orchestration/test_command_recovery.py @@ -1,17 +1,27 @@ """Crash and ownership races for command completion.""" + import asyncio from types import SimpleNamespace from unittest.mock import AsyncMock -from tests.orchestration.test_background_commands import command_db, request + +import core.chat.runtime.handoff as handoff +import core.chat.runtime.recovery as recovery +import core.chat.runtime.worker as worker +import core.db.engine as engine +import core.runs.api_recovery as api_recovery +import core.runs.runtime as runs_runtime +import core.sandbox.commands.runtime as commands_runtime from core.db import models -from core.services.commands import store, delivery -from core.services.chat_sequencer import ChatSequencer -from orchestration.commands.runtime import CommandRuntime +from core.sandbox.commands.runtime import CommandRuntime +from core.chat.sequencer import ChatSequencer +from core.sandbox.commands import delivery, store +from tests.orchestration.test_background_commands import command_db, request + def test_crash_before_context_injection_replays_durable_event(command_db, monkeypatch): - from core.services.run_journal import RecoveryDecision - from orchestration import chat_run_executor as executor - from orchestration import agent_api_recovery + from core.runs.journal import RecoveryDecision + pass + import core.runs.api_recovery as agent_api_recovery with command_db() as db: run = ChatSequencer(db).accept_main_run(chat_id="c", user_id="u", user_content="start", request_payload={}).run @@ -35,17 +45,17 @@ async def nothing(): pass def workflow(**kwargs): captured.append(kwargs) return nothing() - monkeypatch.setattr(executor, "SessionLocal", command_db) - monkeypatch.setattr(executor, "_run_workflow", workflow) - monkeypatch.setattr(executor, "_register_run_task", lambda rid, task, **kw: task.close()) - monkeypatch.setattr(agent_api_recovery, "validate_recovery_context", + monkeypatch.setattr(engine, "SessionLocal", command_db) + monkeypatch.setattr(worker, "_run_workflow", workflow) + monkeypatch.setattr(runs_runtime, 'register_run_task', lambda rid, task, **kw: task.close()) + monkeypatch.setattr(api_recovery, "validate_recovery_context", lambda db, run, context, **kw: context) - assert executor._register_recovered_chat(decision) + assert recovery._register_recovered_chat(decision) assert command["task_id"] in str(captured[0]["session_messages"]) assert store.snapshot(command["task_id"])["notice_status"] == "delivered" async def test_delayed_stop_renews_lease_and_cannot_be_recovered_twice(command_db, monkeypatch): - from orchestration.commands import runtime as module + import core.sandbox.commands.runtime as module monkeypatch.setattr(store, "LEASE_SECONDS", 1) stopping = False polls = 0 @@ -62,7 +72,7 @@ async def poll(pid, **kwargs): provider = SimpleNamespace(name="test", touch_session=AsyncMock(), start_process=AsyncMock(return_value={"status":"running","session_id":"p"}), write_stdin=poll) recover = AsyncMock() - monkeypatch.setattr(module, "recover_execution", recover) + monkeypatch.setattr(commands_runtime, "recover_execution", recover) owner, other = CommandRuntime(), CommandRuntime() result = await owner.start(provider, request(), chat_id="c", source_run_id=None, background=True) cancel = asyncio.create_task(owner.cancel(result["task_id"], "u", "c")) @@ -133,9 +143,9 @@ def test_list_never_hides_live_tasks_behind_display_limit(command_db): assert all(row["status"] == "starting" for row in rows) async def test_slow_launch_keeps_lease_without_execution_deadline(command_db, monkeypatch): - from orchestration.commands import runtime as module + import core.sandbox.commands.runtime as module monkeypatch.setattr(store, "LAUNCH_LEASE_SECONDS", 0.2) - monkeypatch.setattr(module, "LAUNCH_RENEW_SECONDS", 0.03) + monkeypatch.setattr(commands_runtime, "LAUNCH_RENEW_SECONDS", 0.03) called = asyncio.Event() async def start(req, **kwargs): called.set() @@ -162,9 +172,9 @@ def test_adoption_remains_idempotent_at_capacity(command_db): assert not repeated["created"] def test_steer_history_rebuild_preserves_command_sideband(command_db, monkeypatch): - from orchestration import chat_run_executor as executor - from core.services.chat_service import ChatService - from core.services.commands.context import append_active_completions + pass + from core.chat.service import ChatService + from core.sandbox.commands.context import append_active_completions with command_db() as db: run = ChatSequencer(db).accept_main_run(chat_id="c", user_id="u", user_content="start", request_payload={}).run @@ -183,7 +193,7 @@ def test_steer_history_rebuild_preserves_command_sideband(command_db, monkeypatc with command_db() as db: service = ChatService(db) for _ in range(3): - history = executor._persisted_history(service, "c") + history = handoff._persisted_history(service, "c") assert sum(m.get("message_id") == event_id for m in history) == 1 assert append_active_completions(db, "c", history) == history run = db.get(models.ChatRun, run_id) @@ -192,8 +202,8 @@ def test_steer_history_rebuild_preserves_command_sideband(command_db, monkeypatc "raw_user_message":"steer", "context":{"chat_mode":"medium","enable_thinking":True}}} db.commit() assert event_id in str(run.recovery_snapshot["worker_args"]["session_messages"]) - from core.services.run_journal import RecoveryDecision - from orchestration import agent_api_recovery + import core.runs.api_recovery as agent_api_recovery + from core.runs.journal import RecoveryDecision decision = RecoveryDecision(run_id=run_id, chat_id="c", user_id="u", message_id=run.message_id, action="resume", phase="running", snapshot=run.recovery_snapshot, request_payload={}) @@ -202,13 +212,13 @@ async def nothing(): pass def workflow(**kwargs): recovered_args.append(kwargs) return nothing() - monkeypatch.setattr(executor, "SessionLocal", command_db) - monkeypatch.setattr(executor, "_run_workflow", workflow) - monkeypatch.setattr(executor, "_register_run_task", + monkeypatch.setattr(engine, "SessionLocal", command_db) + monkeypatch.setattr(worker, "_run_workflow", workflow) + monkeypatch.setattr(runs_runtime, 'register_run_task', lambda rid, task, **kw: task.close()) - monkeypatch.setattr(agent_api_recovery, "validate_recovery_context", + monkeypatch.setattr(api_recovery, "validate_recovery_context", lambda db, run, context, **kw: context) - assert executor._register_recovered_chat(decision) + assert recovery._register_recovered_chat(decision) assert sum(m.get("message_id") == event_id for m in recovered_args[0]["session_messages"]) == 1 # Active events must not become trailing persisted user turns. @@ -216,11 +226,11 @@ def workflow(**kwargs): run.status, run.writer_slot = "completed", None db.commit() assert not any(m.get("message_id") == event_id - for m in executor._persisted_history(service, "c")) + for m in handoff._persisted_history(service, "c")) def test_command_sideband_does_not_cross_chat_or_user(command_db): - from core.services.commands.context import append_active_completions + from core.sandbox.commands.context import append_active_completions with command_db() as db: run = ChatSequencer(db).accept_main_run(chat_id="c", user_id="u", user_content="start", request_payload={}).run @@ -240,9 +250,9 @@ def test_command_sideband_does_not_cross_chat_or_user(command_db): def test_active_completion_context_retains_harness_provenance(): - from core.services.commands.context import completion_context - from core.llm.context_adapter import AgentScopeContextAdapter - from core.llm.context_ir import KIND_REMINDER, POLICY_NEVER, SESSION_CONTEXT_META_KEY + from core.llm.context.adapter import AgentScopeContextAdapter + from core.llm.context.ir_constants import KIND_REMINDER, POLICY_NEVER, SESSION_CONTEXT_META_KEY + from core.sandbox.commands.context import completion_context message = completion_context({"task_id":"cmd_event", "status":"completed", "exit_code":0}) assert SESSION_CONTEXT_META_KEY in message assert message["extra_data"]["hidden_in_chat"] is True @@ -255,9 +265,9 @@ def test_active_completion_context_retains_harness_provenance(): def test_idle_completion_message_retains_harness_provenance(command_db): - from core.services.compaction_history import normalize_rows - from core.llm.context_adapter import AgentScopeContextAdapter - from core.llm.context_ir import KIND_REMINDER, POLICY_NEVER, SESSION_CONTEXT_META_KEY + from core.llm.context.adapter import AgentScopeContextAdapter + from core.llm.context.ir_constants import KIND_REMINDER, POLICY_NEVER, SESSION_CONTEXT_META_KEY + from core.chat.compaction.history import normalize_rows with command_db() as db: source = ChatSequencer(db).accept_main_run(chat_id="c", user_id="u", user_content="start", request_payload={}).run @@ -280,3 +290,81 @@ def test_idle_completion_message_retains_harness_provenance(command_db): assert item.trust == "system" assert item.truncation_policy == POLICY_NEVER assert item.priority == 950 + + +async def test_observation_failure_preserves_cancellation_control(command_db): + polls = 0 + async def poll(pid, **kwargs): + nonlocal polls + polls += 1 + if polls == 1: + raise TimeoutError("transient output read") + if kwargs["chars"]: + return {"status": "exited", "exit_code": 130} + return {"status": "running", "session_id": "p"} + provider = SimpleNamespace(name="test", touch_session=AsyncMock(), + start_process=AsyncMock(return_value={"status":"running","session_id":"p"}), + write_stdin=poll) + owner = CommandRuntime() + result = await owner.start(provider, request(), chat_id="c", source_run_id=None, background=True) + await asyncio.sleep(0.1) + assert store.snapshot(result["task_id"])["status"] == "running" + await owner.cancel(result["task_id"], "u", "c") + assert store.snapshot(result["task_id"])["status"] == "cancelled" + assert provider.start_process.await_count == 1 + await owner.shutdown() + +async def test_failed_stop_keeps_shared_ownership_across_workers(command_db, monkeypatch): + monkeypatch.setattr(store, "LEASE_SECONDS", 0.3) + can_stop = False + async def poll(pid, **kwargs): + if can_stop and kwargs["chars"]: + return {"status": "exited", "exit_code": 130} + raise TimeoutError("process still alive; stop RPC unavailable") + provider = SimpleNamespace(name="test", touch_session=AsyncMock(), + start_process=AsyncMock(return_value={"status": "running", "session_id": "p"}), + write_stdin=poll) + owner, other = CommandRuntime(), CommandRuntime() + result = await owner.start(provider, request(), chat_id="c", source_run_id=None, background=True) + tid = result["task_id"] + # Force the observer down its failed-stop path, retaining the process handle. + task = owner.tasks[tid] + await asyncio.sleep(0.02) + task.cancel() + await asyncio.gather(task, return_exceptions=True) + for _ in range(20): + await owner.recover(provider) + await other.recover(provider) + await asyncio.sleep(0.04) + row = store.snapshot(tid) + assert row["status"] == "running" + assert row["owner"] == owner.owner + assert tid in owner.providers + can_stop = True + await owner.recover(provider) + await asyncio.sleep(0.1) + assert store.snapshot(tid)["status"] == "failed" + assert tid not in owner.providers + await owner.shutdown() + await other.shutdown() + +async def test_lost_owner_must_stop_local_process_before_forgetting_handle(command_db): + stopped = asyncio.Event() + provider = SimpleNamespace(name="test", touch_session=AsyncMock(), + start_process=AsyncMock(return_value={"status": "running", "session_id": "p"})) + async def poll(pid, **kwargs): + assert kwargs["chars"] == chr(3) + stopped.set() + return {"status": "exited", "exit_code": 130} + provider.write_stdin = poll + owner = CommandRuntime() + result = await owner.start(provider, request(), chat_id="c", source_run_id=None, background=True) + tid = result["task_id"] + with command_db() as db: + row = db.get(models.CommandTask, tid) + row.owner, row.status = "successor", "unknown" + db.commit() + await asyncio.wait_for(stopped.wait(), 2) + await asyncio.sleep(0.05) + assert tid not in owner.providers + await owner.shutdown() diff --git a/src/backend/tests/orchestration/test_command_worker_lifecycle.py b/src/backend/tests/orchestration/test_command_worker_lifecycle.py index 43d39e38..9230c76f 100644 --- a/src/backend/tests/orchestration/test_command_worker_lifecycle.py +++ b/src/backend/tests/orchestration/test_command_worker_lifecycle.py @@ -1,9 +1,13 @@ """Worker ownership and shutdown behavior of background commands.""" + import asyncio from types import SimpleNamespace from unittest.mock import AsyncMock + +import core.sandbox.commands.runtime as commands_runtime +from core.sandbox.commands.runtime import CommandRuntime from tests.orchestration.test_background_commands import command_db, request -from orchestration.commands.runtime import CommandRuntime + async def test_worker_initialization_survives_first_call_from_temporary_thread(command_db, monkeypatch): from api.startup import commands @@ -34,7 +38,7 @@ async def test_lifespan_starts_and_stops_commands_in_each_worker(command_db, mon from api import lifecycle from api.startup import commands from core.infra import leader - from core.services import desktop_gateway_observability + from features.desktop.gateway import observability as desktop_gateway_observability actual = next(row for row in lifecycle._startup_steps() if row[0] is lifecycle.startup_commands) assert actual[4] == lifecycle._PER_WORKER runtime = CommandRuntime() @@ -57,17 +61,19 @@ async def test_lifespan_starts_and_stops_commands_in_each_worker(command_db, mon assert runtime.loop is None async def test_managed_read_never_holds_model_for_long_wait(command_db, monkeypatch): - from core.services.commands import store + import core.sandbox.commands.runtime as module from core.llm.tools.sandbox_tool import register_bash - from orchestration.commands import runtime as module + from core.sandbox.commands import store from tests.llm.test_bash_process_tools import Toolkit row = store.create(request(), "c", None) store.attach(row["task_id"], "owner", {"status":"running","session_id":"p"}, None) instance = SimpleNamespace(wait=AsyncMock()) - monkeypatch.setattr(module, "runtime", instance) + monkeypatch.setattr(commands_runtime, "runtime", instance) + from core.llm.ports import process_state + monkeypatch.setattr(process_state(), "commands", instance) monkeypatch.setenv("SANDBOX_TOOLS_ENABLED", "true") tools = Toolkit() - register_bash(tools, loader=None, loaded_skill_ids=set(), chat_id="c", user_id="u") + register_bash(tools, loader=None, chat_id="c", user_id="u") await tools.functions["write_stdin"](row["task_id"], yield_time_ms=100000) instance.wait.assert_awaited_once_with(row["task_id"], "u", "c", 3000) instance.wait.reset_mock() diff --git a/src/backend/tests/orchestration/test_context_budget_stop.py b/src/backend/tests/orchestration/test_context_budget_stop.py index 21a4100f..052c1d92 100644 --- a/src/backend/tests/orchestration/test_context_budget_stop.py +++ b/src/backend/tests/orchestration/test_context_budget_stop.py @@ -1,14 +1,17 @@ + import asyncio +import core.runs.events.writer as writer +import core.runs.ownership as ownership +import core.runs.settings as settings import pytest from agentscope.message import UserMsg from core.db.models import ChatMessage, ChatRun -from core.llm.context_safety import ContextBudgetExceeded, request_admission_scope -from core.llm.manifest_agent import ManifestBoundAgent -from core.ontology.toolkit import OntologyFilteredToolkit -from core.services.context_budget_checkpoint import ContextBudgetJournal, ContextStopCheckpoint -from core.services.run_journal import RunLeaseLost -from orchestration import chat_run_executor as executor +from core.llm.context.safety import ContextBudgetExceeded, request_admission_scope +from core.llm.runtime.manifest_agent import ManifestBoundAgent +from core.llm.tools.ontology_toolkit import OntologyFilteredToolkit +from core.chat.compaction.budget_checkpoint import ContextBudgetJournal, ContextStopCheckpoint +from core.runs.journal import RunLeaseLost from tests.llm.test_context_safety import CaptureModel from tests.orchestration.test_model_steps_pipeline import _result, _run, _step from tests.orchestration.test_model_steps_pipeline import run_env as run_env @@ -18,7 +21,7 @@ @pytest.mark.parametrize("swallowed", [False, True]) async def test_budget_stop_saves_checkpoint_and_pauses_worker(run_env, monkeypatch, swallowed): monkeypatch.setattr( - "core.services.context_budget_checkpoint.ContextBudgetJournal", + 'core.chat.compaction.budget_checkpoint.ContextBudgetJournal', lambda *args: ContextBudgetJournal(run_env), ) model = CaptureModel() @@ -138,7 +141,7 @@ def test_checkpoint_preserves_post_steer_assistant_identity(): def test_lost_owner_cannot_save_budget_checkpoint(run_env, monkeypatch): - journal = executor._journal() + journal = ownership.journal() journal.accept( run_id="budget-run", message_id="budget-msg", @@ -149,7 +152,7 @@ def test_lost_owner_cannot_save_budget_checkpoint(run_env, monkeypatch): ) assert journal.claim("budget-run", owner="right-owner", lease_seconds=60) monkeypatch.setattr( - "core.services.context_budget_checkpoint.ContextBudgetJournal", + 'core.chat.compaction.budget_checkpoint.ContextBudgetJournal', lambda *args: ContextBudgetJournal(run_env), ) stopped = ContextBudgetExceeded( @@ -163,11 +166,11 @@ def test_lost_owner_cannot_save_budget_checkpoint(run_env, monkeypatch): @pytest.mark.asyncio async def test_non_chat_run_saves_and_preserves_budget_pause(run_env, monkeypatch): - from core.llm.middlewares import AgentRuntimeState - from core.services.run_journal import durable_run_binding + from core.llm.runtime.state import AgentRuntimeState + from core.runs.journal import durable_run_binding monkeypatch.setattr( - "core.services.context_budget_checkpoint.ContextBudgetJournal", + 'core.chat.compaction.budget_checkpoint.ContextBudgetJournal', lambda *args: ContextBudgetJournal(run_env), ) model = CaptureModel() @@ -198,7 +201,7 @@ async def test_non_chat_run_saves_and_preserves_budget_pause(run_env, monkeypatc @pytest.mark.parametrize("wrapped_error", [False, True]) async def test_queued_completed_events_are_drained_before_stop(run_env, monkeypatch, wrapped_error): monkeypatch.setattr( - "core.services.context_budget_checkpoint.ContextBudgetJournal", + 'core.chat.compaction.budget_checkpoint.ContextBudgetJournal', lambda *args: ContextBudgetJournal(run_env), ) model = CaptureModel() @@ -209,13 +212,13 @@ async def test_queued_completed_events_are_drained_before_stop(run_env, monkeypa toolkit=OntologyFilteredToolkit(), ) published = [] - original_emit = executor._xadd_event + original_emit = writer.xadd_event async def capture(run_id, offset, event, **kwargs): published.append(event) return await original_emit(run_id, offset, event, **kwargs) - monkeypatch.setattr(executor, "_xadd_event", capture) + monkeypatch.setattr(writer, 'xadd_event', capture) async def workflow(**kwargs): try: @@ -238,19 +241,19 @@ async def workflow(**kwargs): assert stop["assistant_content"] == "queued answer" assert db.get(ChatMessage, run.message_id).content == "queued answer" assert any(event.get("type") == "error" for event in published) - assert any(event.get("type") == executor._TERMINAL_TYPE for event in published) + assert any(event.get("type") == settings.TERMINAL_TYPE for event in published) assert not any(event.get("type") == "meta" for event in published) assert model.calls == [] @pytest.mark.asyncio async def test_non_chat_checkpoint_uses_actual_reply_and_completed_output(run_env, monkeypatch): - from core.llm.middlewares import AgentRuntimeState - from core.services.run_journal import durable_run_binding + from core.llm.runtime.state import AgentRuntimeState + from core.runs.journal import durable_run_binding from personal_agent.tasks.task_threads import StreamRecord monkeypatch.setattr( - "core.services.context_budget_checkpoint.ContextBudgetJournal", + 'core.chat.compaction.budget_checkpoint.ContextBudgetJournal', lambda *args: ContextBudgetJournal(run_env), ) model = CaptureModel() diff --git a/src/backend/tests/orchestration/test_context_provenance.py b/src/backend/tests/orchestration/test_context_provenance.py index 0a69bc35..c8d7931d 100644 --- a/src/backend/tests/orchestration/test_context_provenance.py +++ b/src/backend/tests/orchestration/test_context_provenance.py @@ -1,10 +1,11 @@ """System-provided orchestration hints must not masquerade as user input.""" -from core.llm.context_adapter import AgentScopeContextAdapter, render_session_input -from core.llm.context_ir import SESSION_CONTEXT_META_KEY -from core.llm.message_compat import session_to_msgs -from orchestration.subagents.plan_mode import _build_plan_generation_messages -from orchestration.workflow import _build_skill_injection +from core.agent_runtime.skills import _build_skill_injection +from core.llm.context.adapter import AgentScopeContextAdapter +from core.llm.context.rendering import render_session_input +from core.llm.context.ir_constants import SESSION_CONTEXT_META_KEY +from core.llm.runtime.message_compat import session_to_msgs +from orchestration.plans.mode import _build_plan_generation_messages def test_plan_generation_policy_attachment_and_task_keep_distinct_provenance(): diff --git a/src/backend/tests/orchestration/test_context_stats_finalization_race.py b/src/backend/tests/orchestration/test_context_stats_finalization_race.py index a5a9fbab..a208bc6a 100644 --- a/src/backend/tests/orchestration/test_context_stats_finalization_race.py +++ b/src/backend/tests/orchestration/test_context_stats_finalization_race.py @@ -10,9 +10,9 @@ from core.chat import inflight from core.db.models import ChatMessage, ChatSession, UserShadow from core.infra.time import utc_now -from core.services.chat_sequencer import ChatSequencer -from core.services.chat_service import ChatService -from core.services.run_journal import RunJournal +from core.chat.sequencer import ChatSequencer +from core.chat.service import ChatService +from core.runs.journal import RunJournal def verify_finalization_serializes_refresh(sessions, user_id): diff --git a/src/backend/tests/orchestration/test_direct_subagent_routing.py b/src/backend/tests/orchestration/test_direct_subagent_routing.py index 4e64ea40..376ad719 100644 --- a/src/backend/tests/orchestration/test_direct_subagent_routing.py +++ b/src/backend/tests/orchestration/test_direct_subagent_routing.py @@ -1,13 +1,20 @@ +from core.agent_runtime.integrations import memory_session as memory_session_module +from core.harness import log_context + from types import SimpleNamespace +import core.agent_runtime.direct as direct +import core.agent_runtime.events as events +import core.agent_runtime.integrations.memory as memory +import core.agent_runtime.runner as runner +import core.agent_runtime.targets as targets import pytest -from core.services.session_operations.agent_targets import _resolve_chat_agent_targets, _strip_direct_mention_prefix +from core.agent_runtime.integrations.memory_session import SessionMemory from core.chat.schemas import ChatRequest -from core.llm.subagent_tool import _shared_ontology_runtime, build_explicit_subagent_command_hint -from core.services.subagent_routing_service import parse_explicit_subagent_command +from core.llm.subagents.tool import _shared_ontology_runtime, build_explicit_subagent_command_hint +from core.chat.operations.agent_targets import _resolve_chat_agent_targets, _strip_direct_mention_prefix +from core.chat.subagent_routing import parse_explicit_subagent_command from fastapi import HTTPException -from orchestration import workflow -from orchestration.memory_integration import SessionMemory class _FakeUserAgentService: @@ -43,13 +50,13 @@ def get_disabled_builtin_subagent_ids(self, user_id: str): @pytest.fixture(autouse=True) def _stub_builtin_preferences(monkeypatch): - import core.services.user_service as user_service_module + import core.auth.users as user_service_module monkeypatch.setattr(user_service_module, "UserService", _FakeUserService) def test_explicit_mention_id_resolves_as_per_turn_target(monkeypatch): - import core.services.user_agent_service as service_module + import core.capabilities.agents.edition as service_module monkeypatch.setattr(service_module, "UserAgentService", _FakeUserAgentService) request = ChatRequest( @@ -72,7 +79,7 @@ def test_explicit_mention_id_resolves_as_per_turn_target(monkeypatch): def test_legacy_mention_name_resolves_only_unique_accessible_agent(monkeypatch): - import core.services.user_agent_service as service_module + import core.capabilities.agents.edition as service_module monkeypatch.setattr(service_module, "UserAgentService", _FakeUserAgentService) request = ChatRequest( @@ -91,7 +98,7 @@ def test_legacy_mention_name_resolves_only_unique_accessible_agent(monkeypatch): def test_unknown_mention_target_is_rejected(monkeypatch): - import core.services.user_agent_service as service_module + import core.capabilities.agents.edition as service_module monkeypatch.setattr(service_module, "UserAgentService", _FakeUserAgentService) request = ChatRequest( @@ -168,7 +175,7 @@ def test_duplicate_agent_name_does_not_trigger_ambiguous_direct_routing(): def test_natural_language_command_is_resolved_by_chat_route(monkeypatch): - import core.services.user_agent_service as service_module + import core.capabilities.agents.edition as service_module monkeypatch.setattr(service_module, "UserAgentService", _FakeUserAgentService) request = ChatRequest( @@ -189,7 +196,7 @@ def test_natural_language_command_is_resolved_by_chat_route(monkeypatch): def test_builtin_subagent_natural_language_command_is_resolved_without_db_row(monkeypatch): - import core.services.user_agent_service as service_module + import core.capabilities.agents.edition as service_module monkeypatch.setattr(service_module, "UserAgentService", _FakeUserAgentService) request = ChatRequest( @@ -209,8 +216,8 @@ def test_builtin_subagent_natural_language_command_is_resolved_without_db_row(mo def test_disabled_builtin_subagent_can_be_explicitly_targeted_for_one_turn(monkeypatch): - import core.services.user_agent_service as service_module - import core.services.user_service as user_service_module + import core.capabilities.agents.edition as service_module + import core.auth.users as user_service_module class _DisabledBuiltinUserService(_FakeUserService): def get_disabled_builtin_subagent_ids(self, user_id: str): @@ -239,13 +246,13 @@ def get_disabled_builtin_subagent_ids(self, user_id: str): def test_disabled_agent_is_absent_by_default_and_added_only_when_explicit(): custom = [{"agent_id": "ua_off", "name": "停用智能体", "is_enabled": False}] - default_visible = workflow._runtime_visible_subagents( + default_visible = targets._runtime_visible_subagents( custom, {}, {"builtin.explorer"}, [], ) - explicit_visible = workflow._runtime_visible_subagents( + explicit_visible = targets._runtime_visible_subagents( custom, {}, {"builtin.explorer"}, @@ -274,7 +281,7 @@ def test_parent_and_child_share_the_same_ontology_runtime_object(): def test_only_persistent_agent_chat_uses_direct_route(): assert ( - workflow._direct_agent_id_from_context( + targets._direct_agent_id_from_context( { "agent_id": None, "mention_agent_id": "ua_risk", @@ -286,7 +293,7 @@ def test_only_persistent_agent_chat_uses_direct_route(): is None ) assert ( - workflow._direct_agent_id_from_context( + targets._direct_agent_id_from_context( {"agent_id": "ua_risk", "direct_agent_id": "ua_risk"} ) == "ua_risk" @@ -294,10 +301,10 @@ def test_only_persistent_agent_chat_uses_direct_route(): def test_dedicated_builtin_chat_preserves_role_write_boundaries(): - assert workflow._direct_agent_execution_permissions("builtin.explorer") == (True, False) - assert workflow._direct_agent_execution_permissions("builtin.worker") == (False, True) - assert workflow._direct_agent_execution_permissions("builtin.reviewer") == (True, False) - assert workflow._direct_agent_execution_permissions("ua_risk") == (False, True) + assert targets._direct_agent_execution_permissions("builtin.explorer") == (True, False) + assert targets._direct_agent_execution_permissions("builtin.worker") == (False, True) + assert targets._direct_agent_execution_permissions("builtin.reviewer") == (True, False) + assert targets._direct_agent_execution_permissions("ua_risk") == (False, True) @pytest.mark.asyncio @@ -311,9 +318,9 @@ async def _unexpected_direct(**_kwargs): raise AssertionError("@mention must not bypass the main model") yield # pragma: no cover - monkeypatch.setattr(workflow, "open_session_memory", _fake_memory_retrieval) - monkeypatch.setattr(workflow, "_astream_subagent_direct", _unexpected_direct) - stream = workflow.astream_chat_workflow( + monkeypatch.setattr(memory_session_module, 'open_session_memory', _fake_memory_retrieval) + monkeypatch.setattr(direct, "_astream_subagent_direct", _unexpected_direct) + stream = runner.astream_chat_workflow( session_messages=[{"role": "user", "content": "分析企业风险"}], user_message="分析企业风险", context={ @@ -353,8 +360,8 @@ async def _fake_memory_retrieval(*_args, **_kwargs): chat_id="", scope_user_id="", workspace_id="default", memory_enabled=False ) - monkeypatch.setattr(workflow, "open_session_memory", _fake_memory_retrieval) - stream = workflow.astream_chat_workflow( + monkeypatch.setattr(memory_session_module, 'open_session_memory', _fake_memory_retrieval) + stream = runner.astream_chat_workflow( session_messages=[{"role": "user", "content": "分析杭州量知的风险"}], user_message="分析杭州量知的风险", context={ @@ -378,31 +385,31 @@ async def _fake_memory_retrieval(*_args, **_kwargs): @pytest.mark.asyncio async def test_subagent_log_scope_is_closed_before_outer_stream_yields(): - from core.services import log_service + from core.harness import log_writer as log_service async def source(): - assert log_service.current_subagent_log_id() == "sublog_1" + assert log_context.current_subagent_log_id() == "sublog_1" yield ("text_delta", "first") - assert log_service.current_subagent_log_id() == "sublog_1" + assert log_context.current_subagent_log_id() == "sublog_1" yield ("text_delta", "second") iterator = source().__aiter__() - first = await workflow._anext_in_subagent_log_scope(iterator, "sublog_1") + first = await events._anext_in_subagent_log_scope(iterator, "sublog_1") assert first == ("text_delta", "first") - assert log_service.current_subagent_log_id() is None + assert log_context.current_subagent_log_id() is None - second = await workflow._anext_in_subagent_log_scope(iterator, "sublog_1") + second = await events._anext_in_subagent_log_scope(iterator, "sublog_1") assert second == ("text_delta", "second") - assert log_service.current_subagent_log_id() is None + assert log_context.current_subagent_log_id() is None with pytest.raises(StopAsyncIteration): - await workflow._anext_in_subagent_log_scope(iterator, "sublog_1") - assert log_service.current_subagent_log_id() is None + await events._anext_in_subagent_log_scope(iterator, "sublog_1") + assert log_context.current_subagent_log_id() is None @pytest.mark.parametrize("message", ["你好", "分析这个问题", "讲解如何调用子智能体", " hello "]) def test_ordinary_message_does_not_load_agent_catalog(monkeypatch, message): - import core.services.user_agent_service as service_module + import core.capabilities.agents.edition as service_module class UnexpectedCatalog: def __init__(self, _db): diff --git a/src/backend/tests/orchestration/test_event_offset_fencing.py b/src/backend/tests/orchestration/test_event_offset_fencing.py index 61a1169d..bea9931c 100644 --- a/src/backend/tests/orchestration/test_event_offset_fencing.py +++ b/src/backend/tests/orchestration/test_event_offset_fencing.py @@ -22,12 +22,9 @@ import time import pytest -from core.services.run_journal import RunLeaseLost -from orchestration.chat_run_executor import ( - _CANCEL_VISIBILITY_SEC, - _OFFSET_RESERVATION, - _EventOffsets, -) +from core.runs.events.writer import EventOffsets +from core.runs.settings import _CANCEL_VISIBILITY_SEC, _OFFSET_RESERVATION +from core.runs.journal import RunLeaseLost class CountingJournal: @@ -53,7 +50,7 @@ def require_lease(self, run_id, owner): def _offsets(lease_lost=None): journal = CountingJournal() - return journal, _EventOffsets( + return journal, EventOffsets( journal, "run", "owner", lease_lost=lease_lost or asyncio.Event() ) diff --git a/src/backend/tests/orchestration/test_execution_manifest_binding.py b/src/backend/tests/orchestration/test_execution_manifest_binding.py index 4b66b374..27c625fb 100644 --- a/src/backend/tests/orchestration/test_execution_manifest_binding.py +++ b/src/backend/tests/orchestration/test_execution_manifest_binding.py @@ -1,6 +1,10 @@ """Main streaming route must bind its real run/workspace identity.""" from __future__ import annotations +from core.agent_runtime.integrations import memory_identity as memory_identity_module +from core.agent_runtime.integrations import memory_session as memory_session_module +from core.llm.runtime import streaming as streaming_module +from core.llm.citations import allocator as allocator_module import json import os @@ -8,8 +12,15 @@ import sys from types import SimpleNamespace +import core.agent_runtime.integrations.memory as memory +import core.agent_runtime.runner as runner +import core.agent_runtime.targets as targets +import core.capabilities.catalog.resolver as catalog_resolver +import core.llm.citations.anchor as citation_anchor +import core.llm.factory as factory +import core.llm.runtime.streaming as streaming import pytest -from orchestration.memory_integration import SessionMemory +from core.agent_runtime.integrations.memory_session import SessionMemory @pytest.mark.asyncio @@ -17,10 +28,12 @@ async def test_main_streaming_propagates_unknown_tool_outcome_after_partial_text monkeypatch, ): from core.db import engine as db_engine - from core.llm import builtin_subagents - from core.services import compaction_service, user_agent_service, user_service - from core.services.tool_effect_ledger import ToolOutcomeUnknown - from orchestration import workflow + from core.capabilities.agents import builtin as builtin_subagents + from core.chat.compaction import service as compaction_service + from core.capabilities.agents import edition as user_agent_service + from core.auth import users as user_service + from core.runs.tool_effects.errors import ToolOutcomeUnknown + pass class DummySession: def __enter__(self): @@ -76,18 +89,17 @@ async def no_compaction(_chat_id, messages, **_kwargs): ) monkeypatch.setattr(builtin_subagents, "merge_builtin_subagents", lambda *_a, **_kw: []) monkeypatch.setattr(compaction_service, "maybe_run_pre_turn_compaction", no_compaction) - monkeypatch.setattr(workflow, "create_agent_executor", create_agent) - monkeypatch.setattr(workflow, "open_session_memory", no_memory) - monkeypatch.setattr(workflow, "build_user_identity_block", no_identity) - monkeypatch.setattr(workflow, "anchor_start_for_chat", lambda _chat_id: 0) - monkeypatch.setattr(workflow, "enabled_skill_ids_from_context", lambda _ctx: []) - monkeypatch.setattr(workflow, "enabled_mcp_ids_from_context", lambda _ctx: []) - monkeypatch.setattr(workflow, "enabled_kb_ids_from_context", lambda _ctx: []) - monkeypatch.setattr(workflow, "_resolve_mode_spec", lambda _ctx: None) - monkeypatch.setattr(workflow, "StreamingAgent", FakeStreamingAgent) - monkeypatch.setattr(workflow, "_persistent_clients", []) - - stream = workflow.astream_chat_workflow( + monkeypatch.setattr(factory, "create_agent_executor", create_agent) + monkeypatch.setattr(memory_session_module, 'open_session_memory', no_memory) + monkeypatch.setattr(memory_identity_module, 'build_user_identity_block', no_identity) + monkeypatch.setattr(allocator_module, 'anchor_start_for_chat', lambda _chat_id: 0) + monkeypatch.setattr(catalog_resolver, "enabled_skill_ids_from_context", lambda _ctx: []) + monkeypatch.setattr(catalog_resolver, "enabled_mcp_ids_from_context", lambda _ctx: []) + monkeypatch.setattr(catalog_resolver, "enabled_kb_ids_from_context", lambda _ctx: []) + monkeypatch.setattr(targets, "_resolve_mode_spec", lambda _ctx: None) + monkeypatch.setattr(streaming_module, 'StreamingAgent', FakeStreamingAgent) + + stream = runner.astream_chat_workflow( session_messages=[{"role": "user", "content": "continue"}], user_message="continue", context={ @@ -108,8 +120,8 @@ async def no_compaction(_chat_id, messages, **_kwargs): @pytest.mark.asyncio async def test_main_streaming_forwards_run_and_workspace_to_agent_factory(monkeypatch): - from core.llm import builtin_subagents - from orchestration import workflow + from core.capabilities.agents import builtin as builtin_subagents + pass captured = {} @@ -125,16 +137,16 @@ async def _no_memory(*_args, **_kwargs): chat_id="", scope_user_id="", workspace_id="default", memory_enabled=False ) - monkeypatch.setattr(workflow, "create_agent_executor", _fake_create_agent_executor) - monkeypatch.setattr(workflow, "open_session_memory", _no_memory) - monkeypatch.setattr(workflow, "anchor_start_for_chat", lambda _chat_id: 0) - monkeypatch.setattr(workflow, "enabled_skill_ids_from_context", lambda _ctx: []) - monkeypatch.setattr(workflow, "enabled_mcp_ids_from_context", lambda _ctx: []) - monkeypatch.setattr(workflow, "enabled_kb_ids_from_context", lambda _ctx: []) - monkeypatch.setattr(workflow, "_resolve_mode_spec", lambda _ctx: None) + monkeypatch.setattr(factory, "create_agent_executor", _fake_create_agent_executor) + monkeypatch.setattr(memory_session_module, 'open_session_memory', _no_memory) + monkeypatch.setattr(allocator_module, 'anchor_start_for_chat', lambda _chat_id: 0) + monkeypatch.setattr(catalog_resolver, "enabled_skill_ids_from_context", lambda _ctx: []) + monkeypatch.setattr(catalog_resolver, "enabled_mcp_ids_from_context", lambda _ctx: []) + monkeypatch.setattr(catalog_resolver, "enabled_kb_ids_from_context", lambda _ctx: []) + monkeypatch.setattr(targets, "_resolve_mode_spec", lambda _ctx: None) monkeypatch.setattr(builtin_subagents, "merge_builtin_subagents", lambda *_a, **_kw: []) - stream = workflow.astream_chat_workflow( + stream = runner.astream_chat_workflow( session_messages=[{"role": "user", "content": "hello"}], user_message="hello", context={ @@ -157,18 +169,30 @@ async def _no_memory(*_args, **_kwargs): def test_main_streaming_executes_real_factory_and_binds_run_workspace(tmp_path): """Fresh process: public streaming path reaches the real factory/binder.""" database_url = f"sqlite:///{tmp_path / 'main-streaming.db'}" - script = r""" + script = """ import asyncio import json import core.db.models # register the complete metadata before create_all from core.db.engine import Base, engine -from core.evolution.runtime_binding import reset_for_tests, resolve_bundle_for_run -from core.llm import factory as agent_factory, builtin_subagents -import core.agent_skills.loader as skill_loader -from orchestration import workflow -from orchestration.memory_integration import SessionMemory - +from core.evolution.cycle.runtime_binding import reset_for_tests, resolve_bundle_for_run +from core.llm import factory as agent_factory +from core.capabilities.agents import builtin as builtin_subagents +import core.capabilities.skills.loader_factory as skill_loader +from core.agent_runtime import runner as workflow, targets +from core.agent_runtime.integrations import memory +from core.llm.citations import allocator as citation_anchor +from core.capabilities.catalog import resolver as catalog_resolver +from core.agent_runtime.integrations.memory_session import SessionMemory + +from features import register_product_tools, register_package_sources, register_run_capabilities, register_model_credentials +from core.harness.registry import load_models +from features.installed import INSTALLED_FEATURES +register_product_tools() +register_package_sources() +register_run_capabilities() +register_model_credentials() +load_models(INSTALLED_FEATURES) Base.metadata.create_all(engine) class EmptySkillLoader: @@ -180,7 +204,7 @@ def get_skill_dir(self, *_args, **_kwargs): return None skill_loader.get_skill_loader = lambda: EmptySkillLoader() -real_factory = workflow.create_agent_executor +real_factory = agent_factory.create_agent_executor class BoundAfterFactory(Exception): pass @@ -201,13 +225,13 @@ async def verifying_factory(**kwargs): async def no_memory(*_args, **_kwargs): return SessionMemory(chat_id="", scope_user_id="", workspace_id="default", memory_enabled=False) -workflow.create_agent_executor = verifying_factory -workflow.open_session_memory = no_memory -workflow.anchor_start_for_chat = lambda _chat_id: 0 -workflow.enabled_skill_ids_from_context = lambda _ctx: [] -workflow.enabled_mcp_ids_from_context = lambda _ctx: [] -workflow.enabled_kb_ids_from_context = lambda _ctx: [] -workflow._resolve_mode_spec = lambda _ctx: None +agent_factory.create_agent_executor = verifying_factory +memory.open_session_memory = no_memory +citation_anchor.anchor_start_for_chat = lambda _chat_id: 0 +catalog_resolver.enabled_skill_ids_from_context = lambda _ctx: [] +catalog_resolver.enabled_mcp_ids_from_context = lambda _ctx: [] +catalog_resolver.enabled_kb_ids_from_context = lambda _ctx: [] +targets._resolve_mode_spec = lambda _ctx: None builtin_subagents.merge_builtin_subagents = lambda *_args, **_kwargs: [] async def main(): @@ -267,12 +291,20 @@ def test_real_factory_public_reply_rebinds_or_fails_closed(tmp_path): from agentscope.model import ChatResponse, ChatUsage from core.db.engine import Base, engine, SessionLocal from core.db.models.evolution import EvolutionEpisode -from core.evolution import runtime_binding as rb -from core.evolution import trace_assembler +from core.evolution.cycle import runtime_binding as rb +from core.evolution.evidence import trace_assembler from core.llm import factory as agent_factory -import core.agent_skills.loader as skill_loader -from core.llm.agentscope_hook_adapter import AgentScopeHookAdapter - +import core.capabilities.skills.loader_factory as skill_loader +from core.llm.runtime.agentscope_hook_adapter import AgentScopeHookAdapter + +from features import register_product_tools, register_package_sources, register_run_capabilities, register_model_credentials +from core.harness.registry import load_models +from features.installed import INSTALLED_FEATURES +register_product_tools() +register_package_sources() +register_run_capabilities() +register_model_credentials() +load_models(INSTALLED_FEATURES) Base.metadata.create_all(engine) from core.db.models import Project, UserShadow with SessionLocal() as db: @@ -293,6 +325,8 @@ def get_skill_dir(self, *_args, **_kwargs): return None class CaptureModel: + from agentscope.formatter import OpenAIChatFormatter + formatter = OpenAIChatFormatter() model = "capture-model" context_size = 32768 def __init__(self): diff --git a/src/backend/tests/orchestration/test_finish_guard.py b/src/backend/tests/orchestration/test_finish_guard.py index d494f70a..54d15aef 100644 --- a/src/backend/tests/orchestration/test_finish_guard.py +++ b/src/backend/tests/orchestration/test_finish_guard.py @@ -1,26 +1,33 @@ -"""Tests for FinishPinGuardMiddleware (auto-pin on dumb-exit, no reminders). - -AgentScope 2.0: the former ``make_finish_pin_guard_hook`` is now -``core.llm.middlewares.FinishPinGuardMiddleware``. When a reasoning turn has **no -tool call at all** (``next_handler`` emits no ``ToolCallEndEvent``, i.e. the model is -about to finish), the middleware calls finish_guard's ``_collect_unpinned`` + -``_direct_pin`` to auto-pin. -""" +"""Finish decisions pin artifacts only at the completed-reply boundary.""" import pytest from agentscope.event import ToolCallEndEvent -from core.llm.middlewares import FinishPinGuardMiddleware +from core.llm.context.workspace_hooks import WorkspacePinHook +from core.harness.hooks import HookSpec, HookStage +from core.llm.runtime.context_hook_adapter import finish_context_hooks +from tests.orchestration.hook_support import make_adapter +from types import SimpleNamespace -async def _drive(mw, *, events=()): - """Drive one on_reasoning: next_handler emits the given events (empty = no tool call this turn).""" - async def next_handler(**kwargs): - for ev in events: - yield ev +def finish_adapter(ids): + policy = WorkspacePinHook() + policy.seen.update(ids) + return make_adapter(HookSpec("finish-pin", HookStage.BEFORE_FINISH, policy.finish, + mutable_fields=frozenset({"pin_file_ids"}))) - async for _ in mw.on_reasoning(object(), {}, next_handler): - pass + +async def _drive(adapter, *, events=()): + async def next_handler(**kwargs): + for event in events: + yield event + agent = SimpleNamespace(state=SimpleNamespace(run_id="finish-test", context=[])) + if events: + # A reasoning step that requests a tool is not a finish boundary. + async for _ in adapter.on_reasoning(agent, {}, next_handler): + pass + else: + await finish_context_hooks(adapter, agent, None, lambda value: value) def _tool_call_event(): @@ -30,15 +37,15 @@ def _tool_call_event(): @pytest.fixture(autouse=True) def reset_pin_state(monkeypatch): """Reset pin_hint ContextVar + stub workspace.pin/get_pinned/mark_active + - stub core.artifacts.store.get_artifact. Returns the collected list of pin calls.""" - from core.llm.hooks import reset_pin_hint_state + stub core.files.artifacts.store.get_artifact. Returns the collected list of pin calls.""" + from core.llm.runtime.turn_state import reset_pin_hint_state reset_pin_hint_state() - import core.llm.workspace as ws + import core.llm.runtime.workspace as ws monkeypatch.setattr(ws, "get_pinned_file_ids", lambda: []) pinned_calls: list[dict] = [] monkeypatch.setattr(ws, "pin", lambda fid, **kw: pinned_calls.append({"file_id": fid, **kw}) or True) monkeypatch.setattr(ws, "mark_active", lambda: None) - import core.artifacts.store as store + import core.files.artifacts.store as store monkeypatch.setattr(store, "get_artifact", lambda fid: { "name": f"{fid}.docx", "mime_type": "application/octet-stream", "size": 1234, }) @@ -48,9 +55,7 @@ def reset_pin_state(monkeypatch): @pytest.mark.asyncio async def test_no_op_when_has_tool_call(reset_pin_state): """This turn has a tool call (ToolCallEndEvent) → no intervention, no auto-pin.""" - from core.llm.hooks import _get_pin_hint_state - _get_pin_hint_state()["seen"].add("abc123") # an unpinned file present; would be pinned if wrongly triggered - mw = FinishPinGuardMiddleware() + mw = finish_adapter(["abc123"]) await _drive(mw, events=[_tool_call_event()]) assert reset_pin_state == [] @@ -58,9 +63,7 @@ async def test_no_op_when_has_tool_call(reset_pin_state): @pytest.mark.asyncio async def test_auto_pin_when_finish_with_unpinned(reset_pin_state): """No tool call + an unpinned file_id present → directly pin to workspace.""" - from core.llm.hooks import _get_pin_hint_state - _get_pin_hint_state()["seen"].update(["abc123", "def456"]) - mw = FinishPinGuardMiddleware() + mw = finish_adapter(["abc123", "def456"]) await _drive(mw, events=[]) assert sorted(p["file_id"] for p in reset_pin_state) == ["abc123", "def456"] @@ -68,9 +71,7 @@ async def test_auto_pin_when_finish_with_unpinned(reset_pin_state): @pytest.mark.asyncio async def test_auto_pin_only_fires_once_per_agent(reset_pin_state): """The same middleware instance dumb-exits repeatedly, auto-pinning only once (_fired guards against an infinite loop).""" - from core.llm.hooks import _get_pin_hint_state - _get_pin_hint_state()["seen"].add("abc123") - mw = FinishPinGuardMiddleware() + mw = finish_adapter(["abc123"]) await _drive(mw, events=[]) await _drive(mw, events=[]) assert len(reset_pin_state) == 1 @@ -78,12 +79,10 @@ async def test_auto_pin_only_fires_once_per_agent(reset_pin_state): @pytest.mark.asyncio async def test_failed_pin_does_not_disarm_guard(reset_pin_state, monkeypatch): - """When _direct_pin fails entirely, _fired is not set — so it can retry on the next chance.""" - import core.llm.workspace as ws + """When pinning fails entirely, completion is not recorded — so it can retry on the next chance.""" + import core.llm.runtime.workspace as ws monkeypatch.setattr(ws, "pin", lambda fid, **kw: False) # all pins fail - from core.llm.hooks import _get_pin_hint_state - _get_pin_hint_state()["seen"].add("abc123") - mw = FinishPinGuardMiddleware() + mw = finish_adapter(["abc123"]) await _drive(mw, events=[]) # Retry after failure — now pin is changed to succeed succeeded: list[str] = [] @@ -95,11 +94,9 @@ async def test_failed_pin_does_not_disarm_guard(reset_pin_state, monkeypatch): @pytest.mark.asyncio async def test_already_pinned_no_op(monkeypatch, reset_pin_state): """All file_ids are already pinned → do not pin again.""" - from core.llm.hooks import _get_pin_hint_state - _get_pin_hint_state()["seen"].add("abc123") - import core.llm.workspace as ws + import core.llm.runtime.workspace as ws monkeypatch.setattr(ws, "get_pinned_file_ids", lambda: ["abc123"]) - mw = FinishPinGuardMiddleware() + mw = finish_adapter(["abc123"]) await _drive(mw, events=[]) assert reset_pin_state == [] @@ -107,10 +104,8 @@ async def test_already_pinned_no_op(monkeypatch, reset_pin_state): @pytest.mark.asyncio async def test_missing_artifact_metadata_skipped(monkeypatch, reset_pin_state): """artifact metadata not found → skip that id, no crash, no pin.""" - from core.llm.hooks import _get_pin_hint_state - _get_pin_hint_state()["seen"].add("ghost123") - import core.artifacts.store as store + import core.files.artifacts.store as store monkeypatch.setattr(store, "get_artifact", lambda fid: None) - mw = FinishPinGuardMiddleware() + mw = finish_adapter(["abc123"]) await _drive(mw, events=[]) assert reset_pin_state == [] diff --git a/src/backend/tests/orchestration/test_harness_e2e_fault_matrix.py b/src/backend/tests/orchestration/test_harness_e2e_fault_matrix.py index 8579a396..12a40658 100644 --- a/src/backend/tests/orchestration/test_harness_e2e_fault_matrix.py +++ b/src/backend/tests/orchestration/test_harness_e2e_fault_matrix.py @@ -10,16 +10,16 @@ from datetime import datetime, timedelta, timezone +import core.chat.runtime.handoff as runtime_handoff +import core.runs.events.store as run_event_stream import fakeredis.aioredis import pytest from core.db.engine import Base from core.db.models import ChatMessage, ChatRun, ChatSession, ChatSteerQueueItem -from core.services import ChatService -from core.services.chat_sequencer import ChatBusyError, ChatSequencer -from core.services.run_journal import RunJournal, RunLeaseLost -from core.services.steer_queue import SteerQueue -from orchestration import chat_run_executor as executor -from orchestration import run_event_stream +from core.chat.service import ChatService +from core.chat.sequencer import ChatBusyError, ChatSequencer +from core.runs.journal import RunJournal, RunLeaseLost +from core.chat.steer_queue import SteerQueue from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker @@ -74,7 +74,7 @@ def commit_output_and_handoff(db) -> None: extra_data={}, commit=False, ) - handoff = executor._commit_queued_handoff_in_session( + handoff = runtime_handoff._commit_queued_handoff_in_session( db, source_run_id=accepted.run.run_id, chat_id="chat-1", @@ -152,7 +152,7 @@ def test_terminal_commit_hands_late_steer_to_successor_run(harness_sessions, mon prepared: dict = {} def commit_output_and_handoff(db) -> None: - handoff = executor._commit_queued_handoff_in_session( + handoff = runtime_handoff._commit_queued_handoff_in_session( db, source_run_id=accepted.run.run_id, chat_id="chat-1", @@ -180,7 +180,7 @@ def commit_output_and_handoff(db) -> None: assert queue_row.applied_run_id == successor.run_id assert successor.recovery_snapshot["worker_args"]["raw_user_message"] == queued.message - from core.services import chat_steer_service + from core.chat import steer as chat_steer_service monkeypatch.setattr(chat_steer_service, "SessionLocal", sessions) payload = chat_steer_service.list_run_steers(accepted.run.run_id)[0] diff --git a/src/backend/tests/orchestration/test_harness_ledger_postgres.py b/src/backend/tests/orchestration/test_harness_ledger_postgres.py index bfed6baa..9be05557 100644 --- a/src/backend/tests/orchestration/test_harness_ledger_postgres.py +++ b/src/backend/tests/orchestration/test_harness_ledger_postgres.py @@ -15,8 +15,8 @@ from core.db.models import ChatSession, UserShadow from core.harness.events import Event from core.harness.usage import UsageAttempt -from core.services.harness_ledger import DurableEventStore, HarnessUsageLedger -from core.services.run_journal import RunJournal +from core.harness.ledger import DurableEventStore, HarnessUsageLedger +from core.runs.journal import RunJournal from sqlalchemy import create_engine, inspect from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.orm import sessionmaker diff --git a/src/backend/tests/orchestration/test_harness_process_restart.py b/src/backend/tests/orchestration/test_harness_process_restart.py deleted file mode 100644 index 6d45fd02..00000000 --- a/src/backend/tests/orchestration/test_harness_process_restart.py +++ /dev/null @@ -1,52 +0,0 @@ -"""Black-box proof that every Harness safe point survives an OS process death.""" - -from __future__ import annotations - -import json -import signal -import subprocess -import sys -from pathlib import Path - -import pytest - - -@pytest.mark.skipif(not hasattr(signal, "SIGKILL"), reason="requires POSIX SIGKILL") -def test_sigkill_restart_matrix_covers_every_durable_safe_point(tmp_path): - backend_root = Path(__file__).resolve().parents[2] - script = backend_root / "scripts" / "harness_fault_injection.py" - state_dir = tmp_path / "fault-matrix" - - completed = subprocess.run( - [ - sys.executable, - str(script), - "matrix", - "--state-dir", - str(state_dir), - "--lease-seconds", - "1", - "--timeout", - "45", - ], - cwd=backend_root, - capture_output=True, - text=True, - timeout=180, - check=False, - ) - - assert completed.returncode == 0, completed.stderr or completed.stdout - payload = json.loads(completed.stdout.splitlines()[-1]) - assert payload["state"] == "matrix_passed" - assert payload["signal"] == "SIGKILL" - assert payload["killed_phases"] == [ - "pending", - "model_before", - "model_after", - "tool_intent", - "tool_unknown", - "message_committed", - "compacting", - "memory_outbox", - ] diff --git a/src/backend/tests/orchestration/test_harness_usage_ledger.py b/src/backend/tests/orchestration/test_harness_usage_ledger.py index a3463af3..e00e379c 100644 --- a/src/backend/tests/orchestration/test_harness_usage_ledger.py +++ b/src/backend/tests/orchestration/test_harness_usage_ledger.py @@ -4,23 +4,20 @@ from types import SimpleNamespace +import core.chat.followups as chat_followups import pytest from core.db.engine import Base from core.harness.events import Event, EventSink from core.harness.hooks import HookBus from core.harness.usage import AttemptUsage, MemoryUsageRecorder, UsageAttempt -from core.llm.agentscope_hook_adapter import AgentScopeHookAdapter -from core.llm.model_usage import ( - instrument_model_usage, - model_usage_scope, - record_provider_failure, -) -from core.services.harness_ledger import DurableEventStore, HarnessUsageLedger -from core.services.run_journal import RunJournal +from core.llm.runtime.agentscope_hook_adapter import AgentScopeHookAdapter +from core.llm.runtime.state import AgentRuntimeState +from core.models.usage import instrument_model_usage, model_usage_scope, record_provider_failure +from core.harness.ledger import DurableEventStore, HarnessUsageLedger +from core.runs.journal import RunJournal from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker - @pytest.fixture() def ledger_env(tmp_path): engine = create_engine( @@ -196,6 +193,28 @@ async def _call_api_with_structured_output(self, model_name, **kwargs): assert structured_rows[0].usage == AttemptUsage(7, 3) +def test_attempt_batch_reserves_consecutive_sequences_in_one_write(ledger_env): + ledger = HarnessUsageLedger(ledger_env) + ledger.record_attempt( + UsageAttempt(run_id="run-ledger", kind="model", operation_name="m", + status="success", latency_ms=1) + ) + batch = ledger.record_attempts( + UsageAttempt(run_id="run-ledger", kind="hook", operation_name=name, + status="success", latency_ms=1) + for name in ("a", "b", "c") + ) + assert [row.attempt_seq for row in batch] == [2, 3, 4] + assert [row.operation_name for row in ledger.attempts("run-ledger")] == ["m", "a", "b", "c"] + with pytest.raises(ValueError): + ledger.record_attempts([ + UsageAttempt(run_id="run-ledger", kind="hook", operation_name="x", + status="success", latency_ms=1), + UsageAttempt(run_id="other", kind="hook", operation_name="y", + status="success", latency_ms=1), + ]) + + @pytest.mark.asyncio async def test_durable_event_store_allocates_sequence_before_observers(ledger_env): sink = EventSink(DurableEventStore(ledger_env)) @@ -225,10 +244,10 @@ async def test_durable_event_store_allocates_sequence_before_observers(ledger_en async def test_compaction_http_retry_is_also_physical_model_usage( ledger_env, monkeypatch ): - import core.services.compaction_summary as compaction + import core.chat.compaction.summary as compaction ledger = HarnessUsageLedger(ledger_env) - from core.llm.single_turn import Endpoint + from core.models.completions.single_turn import Endpoint monkeypatch.setattr( compaction, @@ -310,7 +329,7 @@ async def _call_api(self, model_name, **kwargs): model = instrument_model_usage(Model()) adapter = AgentScopeHookAdapter(HookBus(usage_recorder=ledger)) agent = SimpleNamespace( - state=SimpleNamespace(run_id="run-ledger"), + state=AgentRuntimeState(run_id="run-ledger"), model=model, ) @@ -378,8 +397,8 @@ async def _call_api(self, model_name, **kwargs): @pytest.mark.asyncio async def test_followup_http_call_is_included_in_run_usage(ledger_env, monkeypatch): - from core.llm import single_turn - from orchestration import followups + import core.chat.followups as followups + from core.models.completions import single_turn endpoint = single_turn.Endpoint( base_url="http://fake", @@ -388,7 +407,7 @@ async def test_followup_http_call_is_included_in_run_usage(ledger_env, monkeypat provider="followup-provider", api_protocol="chat_completions", ) - monkeypatch.setattr(followups, "_resolve_followup_endpoint", lambda: endpoint) + monkeypatch.setattr(chat_followups, "_resolve_followup_endpoint", lambda: endpoint) class Response: status_code = 200 @@ -433,8 +452,8 @@ async def post(self, *args, **kwargs): @pytest.mark.asyncio async def test_vision_fallback_records_each_physical_model_attempt(monkeypatch): - from core.services.model_config import ResolvedModelConfig - from core.vision.provider import VisionCallResult, VisionProvider + from core.models.config import ResolvedModelConfig + from core.models.vision.provider import VisionCallResult, VisionProvider provider = VisionProvider( ResolvedModelConfig( @@ -479,8 +498,8 @@ async def fake_call(*args, **kwargs): @pytest.mark.asyncio async def test_stream_usage_aggregates_ledger_but_keeps_primary_context(monkeypatch): - import core.services.harness_ledger as ledger_module - from orchestration.streaming import StreamingAgent + import core.harness.ledger as ledger_module + from core.llm.runtime.streaming import StreamingAgent rows = ( UsageAttempt( diff --git a/src/backend/tests/orchestration/test_hook_bus.py b/src/backend/tests/orchestration/test_hook_bus.py index cedcc18c..12ad053c 100644 --- a/src/backend/tests/orchestration/test_hook_bus.py +++ b/src/backend/tests/orchestration/test_hook_bus.py @@ -10,6 +10,7 @@ import pytest from core.harness.events import Event, EventSink +from core.harness.observation_writer import flush_observations from core.harness.hooks import ( Decision, DecisionKind, @@ -255,8 +256,148 @@ async def test_hook_external_usage_is_recorded_as_its_own_attempt(): data={"model": "m", "messages": [], "parameters": {}}, ) ) + assert flush_observations() rows = usage.attempts("run-usage") assert len(rows) == 1 assert rows[0].kind == "hook" assert rows[0].operation_name == "safety_classifier" assert usage.aggregate("run-usage")["total_tokens"] == 14 + + +@pytest.mark.asyncio +async def test_interactive_policy_owns_wait_budget_and_preserves_cancellation(monkeypatch): + from core.llm.tool_permissions.hook import ToolPermissionHook + from dataclasses import replace + + entered = asyncio.Event() + approved = asyncio.Event() + cancelled = asyncio.Event() + + async def approval(invocation): + entered.set() + try: + await approved.wait() + except asyncio.CancelledError: + cancelled.set() + raise + return Decision() + + async def short_watchdog(coro, timeout): + coro.close() + raise AssertionError("Human approval must own its wait budget") + + bus = HookBus() + bus.register(replace(ToolPermissionHook(None).spec(), handler=approval)) + invocation = Invocation.create(run_id="approval", stage=HookStage.BEFORE_TOOL, + operation_name="write", data={"tool_args": {}, "metadata": {}}) + monkeypatch.setattr(asyncio, "wait_for", short_watchdog) + task = asyncio.create_task(bus.run(invocation)) + await entered.wait() + await asyncio.sleep(0) + assert not task.done() + approved.set() + assert (await task).decision.kind == DecisionKind.CONTINUE + entered.clear() + approved.clear() + task = asyncio.create_task(bus.run(invocation)) + await entered.wait() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert cancelled.is_set() + + +@pytest.mark.asyncio +async def test_sync_handlers_run_inline_without_executor_or_timeout(monkeypatch): + import threading + + async def saturated(*args, **kwargs): + raise AssertionError("sync hooks must not wait for the default executor") + + loop_thread = threading.get_ident() + seen = [] + + def inline(invocation): + seen.append(threading.get_ident()) + return Decision() + + bus = HookBus() + bus.register(HookSpec("durable", HookStage.BEFORE_TOOL, inline, timeout_ms=1)) + monkeypatch.setattr(asyncio, "to_thread", saturated) + outcome = await bus.run( + Invocation.create( + run_id="run-inline", + stage=HookStage.BEFORE_TOOL, + operation_name="write", + ) + ) + assert outcome.decision.kind == DecisionKind.CONTINUE + assert seen == [loop_thread] + + +@pytest.mark.asyncio +async def test_stage_writes_one_event_and_one_usage_batch(): + class BatchRecorder(MemoryUsageRecorder): + def __init__(self): + super().__init__() + self.batches = [] + + def record_attempts(self, attempts): + self.batches.append(len(attempts)) + return super().record_attempts(attempts) + + usage = BatchRecorder() + bus = HookBus(usage_recorder=usage) + for name, priority in (("one", 10), ("two", 20), ("three", 30)): + bus.register(HookSpec(name, HookStage.TRANSFORM_CONTEXT, lambda invocation: None, + priority=priority)) + await bus.run( + Invocation.create( + run_id="run-batch", + stage=HookStage.TRANSFORM_CONTEXT, + operation_name="agent_reasoning", + ) + ) + events = bus.event_sink.events() + assert [event.event_type for event in events] == ["hook_stage_finished"] + assert flush_observations() + assert [item["hook"] for item in events[0].payload["attempts"]] == ["one", "two", "three"] + assert usage.batches == [3] + assert [row.attempt_seq for row in usage.attempts("run-batch")] == [1, 2, 3] + + await bus.run( + Invocation.create( + run_id="run-batch", + stage=HookStage.BEFORE_MODEL, + operation_name="model", + ) + ) + assert len(bus.event_sink.events()) == 1 + assert flush_observations() + assert usage.batches == [3] + + +@pytest.mark.asyncio +async def test_cancelled_hook_still_flushes_the_stage_batch(): + usage = MemoryUsageRecorder() + bus = HookBus(usage_recorder=usage) + + async def cancelled(invocation): + raise asyncio.CancelledError + + bus.register(HookSpec("first", HookStage.BEFORE_TOOL, lambda invocation: None, priority=10)) + bus.register(HookSpec("cancel", HookStage.BEFORE_TOOL, cancelled, priority=20)) + bus.register(HookSpec("never", HookStage.BEFORE_TOOL, lambda invocation: None, priority=30)) + with pytest.raises(asyncio.CancelledError): + await bus.run( + Invocation.create( + run_id="run-cancel", + stage=HookStage.BEFORE_TOOL, + operation_name="write", + ) + ) + assert flush_observations() + assert [(row.operation_name, row.status) for row in usage.attempts("run-cancel")] == [ + ("first", "success"), + ("cancel", "cancelled"), + ] diff --git a/src/backend/tests/orchestration/test_inactivity_watchdog.py b/src/backend/tests/orchestration/test_inactivity_watchdog.py index 37f8dc11..944e6ba7 100644 --- a/src/backend/tests/orchestration/test_inactivity_watchdog.py +++ b/src/backend/tests/orchestration/test_inactivity_watchdog.py @@ -4,7 +4,7 @@ import time import pytest -from orchestration.chat_run_executor import _aiter_with_inactivity_timeout +from core.runs.watchdog import _aiter_with_inactivity_timeout def _is_meaningful(chunk: dict) -> bool: diff --git a/src/backend/tests/orchestration/test_inflight_message.py b/src/backend/tests/orchestration/test_inflight_message.py index c48ece81..1530b875 100644 --- a/src/backend/tests/orchestration/test_inflight_message.py +++ b/src/backend/tests/orchestration/test_inflight_message.py @@ -9,15 +9,19 @@ import contextlib from types import SimpleNamespace +import core.agent_runtime.runner as runner +import core.chat.runtime.admission as admission +import core.chat.runtime.followups as followups +import core.db.engine as db_engine +import core.runs.tasks as tasks import fakeredis.aioredis import pytest from core.chat import inflight from core.db.engine import Base from core.db.models import ChatMessage, ChatRun, ChatSession -from core.services.chat_sequencer import ChatSequencer -from core.services.compaction_history import normalize_rows -from core.services.run_journal import RunJournal -from orchestration import chat_run_executor as executor +from core.chat.sequencer import ChatSequencer +from core.chat.compaction.history import normalize_rows +from core.runs.journal import RunJournal from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker @@ -30,16 +34,16 @@ def env(tmp_path, monkeypatch): ) Base.metadata.create_all(engine) sessions = sessionmaker(bind=engine, expire_on_commit=False) - monkeypatch.setattr(executor, "SessionLocal", sessions) + monkeypatch.setattr(db_engine, "SessionLocal", sessions) redis = fakeredis.aioredis.FakeRedis(decode_responses=True) - monkeypatch.setattr("orchestration.run_event_stream.get_redis", lambda **_: redis) - monkeypatch.setattr(executor, "_spawn_followup_task", lambda **_kwargs: None) - executor._active_runs.clear() + monkeypatch.setattr('core.runs.events.store.get_redis', lambda **_: redis) + monkeypatch.setattr(followups, "_spawn_followup_task", lambda **_kwargs: None) + tasks.process_state().active_runs.clear() with sessions() as db: db.add(ChatSession(chat_id="chat-1", user_id="user-1", title="test")) db.commit() yield sessions - executor._active_runs.clear() + tasks.process_state().active_runs.clear() engine.dispose() @@ -54,7 +58,7 @@ def _assistant_rows(sessions): async def _start(workflow, monkeypatch, sessions): - monkeypatch.setattr(executor, "astream_chat_workflow", workflow) + monkeypatch.setattr(runner, "astream_chat_workflow", workflow) with sessions() as db: accepted = ChatSequencer(db).accept_main_run( chat_id="chat-1", @@ -62,7 +66,7 @@ async def _start(workflow, monkeypatch, sessions): user_content="hello", request_payload={"message": "hello"}, ) - return await executor.start_run( + return await admission.start_run( accepted_run=accepted.run, chat_id="chat-1", user_id="user-1", @@ -141,8 +145,8 @@ def _persisted(): assert marker["event_offset"] > 0 assert row.extra_data["segments"] - executor._active_runs[run.run_id].cancel() - await _finish(executor._active_runs[run.run_id]) + tasks.process_state().active_runs[run.run_id].cancel() + await _finish(tasks.process_state().active_runs[run.run_id]) @pytest.mark.asyncio @@ -154,7 +158,7 @@ async def workflow(**_kwargs): yield {"type": "meta", "route": "main", "is_markdown": False, "usage": {}} run = await _start(workflow, monkeypatch, sessions) - await _finish(executor._active_runs[run.run_id]) + await _finish(tasks.process_state().active_runs[run.run_id]) rows = _assistant_rows(sessions) assert len(rows) == 1 @@ -181,7 +185,7 @@ async def workflow(**_kwargs): raise RuntimeError("sandbox vanished") run = await _start(workflow, monkeypatch, sessions) - await _finish(executor._active_runs[run.run_id]) + await _finish(tasks.process_state().active_runs[run.run_id]) rows = _assistant_rows(sessions) assert len(rows) == 1 @@ -238,7 +242,7 @@ def test_fenced_worker_cannot_refresh_the_row(env): .run ) assert journal.claim(run.run_id, owner="worker-a", lease_seconds=60) is not None - from core.services.chat_service import ChatService + from core.chat.service import ChatService def _refresh(owner: str) -> bool: with sessions() as db: @@ -262,8 +266,8 @@ def _refresh(owner: str) -> bool: def test_a_row_reads_as_final_once_its_run_is_gone(env): """in-flight 不是存下来的状态，而是"归属的 run 是否还活着"：run 一停，行就是定稿。""" - from core.services.chat_service import ChatService - from core.services.compaction_service import _live_run_ids + from core.chat.service import ChatService + from core.chat.compaction.service import _live_run_ids sessions = env journal = RunJournal(sessions) diff --git a/src/backend/tests/orchestration/test_job_callback_transport.py b/src/backend/tests/orchestration/test_job_callback_transport.py index d806d392..2930775f 100644 --- a/src/backend/tests/orchestration/test_job_callback_transport.py +++ b/src/backend/tests/orchestration/test_job_callback_transport.py @@ -5,9 +5,10 @@ import threading from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import core.jobs.callback as jobs_callback +import core.jobs.callback as callback import pytest -from orchestration.jobs import callback -from orchestration.jobs.sdk import SDK_SOURCE +from core.jobs.sdk import SDK_SOURCE from core.sandbox.runner import server from tests.sandbox.runner_client import run_spawn @@ -119,7 +120,7 @@ def test_probe_execution_failure_is_not_reported_as_network_failure(monkeypatch) async def missing_interpreter(*args, **kwargs): return 127, "", "private interpreter missing" - monkeypatch.setattr(callback, "_sbx_bash", missing_interpreter) + monkeypatch.setattr(jobs_callback, "_sbx_bash", missing_interpreter) with pytest.raises(RuntimeError) as error: asyncio.run(callback.resolve_callback_base(session_id="test", user_id="test")) message = str(error.value) diff --git a/src/backend/tests/orchestration/test_job_conflict_policy.py b/src/backend/tests/orchestration/test_job_conflict_policy.py index b541a2c0..1aff0059 100644 --- a/src/backend/tests/orchestration/test_job_conflict_policy.py +++ b/src/backend/tests/orchestration/test_job_conflict_policy.py @@ -11,6 +11,8 @@ import json import core.db.engine as db_engine +import core.jobs.files as files +import core.jobs.runtime as runtime import pytest from core.db.engine import Base from core.db.models import Job @@ -39,7 +41,8 @@ class _Toolkit: def __init__(self): self.fn = None - def register_tool_function(self, fn): + def append(self, tool): + fn = tool._func self.fn = fn @@ -64,13 +67,13 @@ async def fake_sbx_bash(cmd, *, session_id, user_id, timeout=60): async def fake_start(**kw): return "job_new" - from orchestration import job_runtime + import core.jobs.runtime as job_runtime - monkeypatch.setattr(job_runtime, "cancel_job", fake_cancel) - from orchestration.jobs import files + monkeypatch.setattr(runtime, "cancel_job", fake_cancel) + import core.jobs.files as files monkeypatch.setattr(files, "_sbx_bash", fake_sbx_bash) - monkeypatch.setattr(job_runtime, "start_job", fake_start) + monkeypatch.setattr(runtime, "start_job", fake_start) tk = _Toolkit() job_tool.register_run_job( diff --git a/src/backend/tests/orchestration/test_job_derived_agent.py b/src/backend/tests/orchestration/test_job_derived_agent.py index 2ec189aa..69e13480 100644 --- a/src/backend/tests/orchestration/test_job_derived_agent.py +++ b/src/backend/tests/orchestration/test_job_derived_agent.py @@ -1,15 +1,19 @@ """Workflow forks keep the main-agent surface and independent run state.""" +from core.llm.runtime import streaming as streaming_module import asyncio import json from types import SimpleNamespace +import core.jobs.agent as agent +import core.jobs.files as files +import core.jobs.runtime as jobs_runtime import pytest from core.llm.factory.request import AgentRequest def test_snapshot_keeps_parent_and_deferred_capabilities_without_run_identity(): - from orchestration.jobs.agent import snapshot_parent_runtime + from core.jobs.agent import snapshot_parent_runtime request = AgentRequest( enabled_skill_ids=["research"], @@ -48,10 +52,10 @@ def test_snapshot_keeps_parent_and_deferred_capabilities_without_run_identity(): def test_derived_executor_uses_main_factory_and_independent_prompt(monkeypatch): + import core.jobs.agent as jobs import core.llm.factory as factory - import core.llm.mcp_manager as mcp - import orchestration.streaming as streaming - from orchestration.jobs import agent as jobs + import core.llm.mcp.manager as mcp + import core.llm.runtime.streaming as streaming calls, prompts, closed = [], [], [] @@ -72,8 +76,8 @@ async def stream(self, messages, context): monkeypatch.setattr(factory, "create_agent_executor", construct) monkeypatch.setattr(mcp, "close_clients", close) - monkeypatch.setattr(streaming, "StreamingAgent", Stream) - monkeypatch.setattr(jobs, "resolve_subagents", lambda *_args: [{"agent_id": "builtin.worker"}]) + monkeypatch.setattr(streaming_module, 'StreamingAgent', Stream) + monkeypatch.setattr(agent, "resolve_subagents", lambda *_args: [{"agent_id": "builtin.worker"}]) runtime = { "enabled_skill_ids": ["skill"], "enabled_mcp_ids": ["search"], @@ -112,10 +116,10 @@ async def stream(self, messages, context): def test_child_executor_inherits_full_approval(monkeypatch): - import core.llm.builtin_subagents as builtins + import core.capabilities.agents.builtin as builtins import core.llm.factory as factory from agentscope.message import AssistantMsg - from core.llm import subagent_tool + from core.llm.subagents import tool as subagent_tool captured = [] @@ -145,7 +149,7 @@ async def construct(**kwargs): @pytest.mark.asyncio async def test_job_fence_cancels_the_agent_and_waits_for_cleanup(): - from orchestration.jobs.agent import run_with_job_fence + from core.jobs.agent import run_with_job_fence started, stopped = asyncio.Event(), asyncio.Event() checks = 0 @@ -170,7 +174,7 @@ def validate(): def test_missing_parent_config_does_not_fall_back_to_light_executor(): - from orchestration.jobs.agent import run_derived_agent + from core.jobs.agent import run_derived_agent with pytest.raises(ValueError, match="重新提交"): asyncio.run( @@ -193,16 +197,19 @@ def test_real_main_factory_delegates_from_workflow_fork(tmp_path): import subprocess import sys - script = r""" + script = """ import asyncio, json, threading from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer import core.db.models from core.db.engine import Base, engine, SessionLocal -from core.db.model_repository import create_provider, assign_role -from orchestration.jobs.agent import run_derived_agent +from core.models.repository import create_provider, assign_role +from core.jobs.agent import run_derived_agent Base.metadata.create_all(engine) requests = [] # Inspect real permission services for both factory-created roles. +from features import register_product_tools, register_run_capabilities +register_product_tools() +register_run_capabilities() import core.llm.factory as factory from agentscope.message import ToolCallBlock from core.llm.tool_permissions import ToolPermissionSpec, PermissionIntent, DOMAIN_APPROVAL @@ -238,7 +245,7 @@ def do_POST(self): chunk = {'id': 'completion', 'object': 'chat.completion.chunk', 'created': 1, 'model': payload['model'], 'choices': [{'index': 0, 'delta': delta, 'finish_reason': None}]} end = dict(chunk, choices=[{'index': 0, 'delta': {}, 'finish_reason': finish}]) - data = ('data: '+json.dumps(chunk)+'\n\n'+'data: '+json.dumps(end)+'\n\n'+'data: [DONE]\n\n').encode() + data = ('data: '+json.dumps(chunk)+'\\n\\n'+'data: '+json.dumps(end)+'\\n\\n'+'data: [DONE]\\n\\n').encode() self.send_response(200); self.send_header('Content-Type','text/event-stream') self.send_header('Content-Length',str(len(data))); self.end_headers(); self.wfile.write(data) server = ThreadingHTTPServer(('127.0.0.1',0),Handler) @@ -250,7 +257,7 @@ def do_POST(self): provider_id = provider.provider_id assign_role(db,'main_agent',provider_id) async def main(): - from core.services.run_journal import durable_run_binding + from core.runs.journal import durable_run_binding async with durable_run_binding(user_id="test-user", chat_id="test-chat", kind="internal_job_agent", external_id="test", request_payload={}) as binding: result = await run_derived_agent('Delegate to the worker, then report.', @@ -292,16 +299,16 @@ def submitted_job(monkeypatch): from contextlib import asynccontextmanager import core.db.engine as db_engine + import core.jobs.files as files + import core.jobs.runtime as job_runtime from api.routes.v1 import internal_jobs, job_callback_session from core.db.engine import Base + from core.jobs.agent import snapshot_parent_runtime from core.llm.tools.job_tool import register_run_job - from core.services import run_journal - from core.services.job_service import JobService + from core.runs import journal as run_journal + from core.jobs.store import JobService from fastapi import FastAPI from fastapi.testclient import TestClient - from orchestration import job_runtime - from orchestration.jobs import files - from orchestration.jobs.agent import snapshot_parent_runtime from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker from sqlalchemy.pool import StaticPool @@ -338,12 +345,13 @@ async def bash(*_args, **_kwargs): async def binding(**_kwargs): yield SimpleNamespace(chat_id="chat", run_id="run", owner="owner") - monkeypatch.setattr(job_runtime, "start_job", start) + monkeypatch.setattr(jobs_runtime, "start_job", start) monkeypatch.setattr(files, "_sbx_bash", bash) monkeypatch.setattr(run_journal, "durable_run_binding", binding) class Toolkit: - def register_tool_function(self, fn): + def append(self, tool): + fn = tool._func self.call = fn runtime = snapshot_parent_runtime( @@ -385,7 +393,7 @@ def test_callback_uses_saved_main_snapshot_and_never_repeats_actions( submitted_job, monkeypatch, output, infra_failure, status ): from api.routes.v1 import internal_jobs - from core.services.job_service import JobService + from core.jobs.store import JobService client, (job_id, token), runtime, sessions = submitted_job calls = [] @@ -415,7 +423,7 @@ async def execute(prompt, **kwargs): def test_sdk_does_not_repeat_an_agent_after_network_result_is_unknown(): - from orchestration.jobs.sdk import SDK_SOURCE + from core.jobs.sdk import SDK_SOURCE ns = {} exec(compile(SDK_SOURCE, "", "exec"), ns) diff --git a/src/backend/tests/orchestration/test_job_failfast.py b/src/backend/tests/orchestration/test_job_failfast.py index 44b5af55..49023ce7 100644 --- a/src/backend/tests/orchestration/test_job_failfast.py +++ b/src/backend/tests/orchestration/test_job_failfast.py @@ -31,8 +31,9 @@ def classify(item): import asyncio +import core.jobs.supervisor as jobs_supervisor import pytest -from orchestration.jobs.sdk import SDK_SOURCE +from core.jobs.sdk import SDK_SOURCE # ── ① job.map 预检：脚本级 bug 必须当场中止 ────────────────────────── @@ -191,7 +192,7 @@ def _mk_items(db, job_id, statuses): ], ) def test_finish_downgrades_completed_when_nothing_was_produced(db_session, statuses, expected): - from core.services.job_service import JobService + from core.jobs.store import JobService _mk_job(db_session, "job_x") _mk_items(db_session, "job_x", statuses) @@ -207,7 +208,7 @@ def test_finish_downgrades_completed_when_nothing_was_produced(db_session, statu def test_finish_keeps_explicit_failure_reason(db_session): """真正的崩溃原因优先于兜底文案 —— 别把 traceback 覆盖掉。""" - from core.services.job_service import JobService + from core.jobs.store import JobService _mk_job(db_session, "job_y") _mk_items(db_session, "job_y", ["pending"]) @@ -219,7 +220,7 @@ def test_finish_keeps_explicit_failure_reason(db_session): def test_set_wake_on_finish_flips_start_params(db_session): """前台转后台后要有人叫醒会话播报，标记必须真的落库。""" - from core.services.job_service import JobService + from core.jobs.store import JobService job = _mk_job(db_session, "job_z") job.extra_data = {"start_params": {"wake_on_finish": False, "model_name": "m"}} @@ -241,8 +242,9 @@ async def test_run_and_wait_detaches_instead_of_blocking_forever(monkeypatch): """wait=True 撞上长作业时，等够上限就把对话还给用户，作业继续在后台跑。""" from types import SimpleNamespace - from orchestration import job_runtime - from orchestration.jobs import owner, supervisor + import core.jobs.owner as owner + import core.jobs.runtime as job_runtime + import core.jobs.supervisor as supervisor owner.bind() @@ -259,7 +261,7 @@ async def fake_drive(job_row_id, *, chat_id): return {"status": "completed"} task = asyncio.create_task(fake_drive("job_d", chat_id="c1")) - monkeypatch.setattr(supervisor, "_entries", {"job_d": SimpleNamespace(task=task)}) + monkeypatch.setattr(jobs_supervisor.process_state(), 'entries', {"job_d": SimpleNamespace(task=task)}) res = await job_runtime.run_and_wait("job_d", chat_id="c1", detach_after=0.05) @@ -267,7 +269,7 @@ async def fake_drive(job_row_id, *, chat_id): assert started.is_set() await asyncio.sleep(0.05) assert not cancelled.is_set(), "转后台 ≠ 取消作业，它必须继续跑" - assert "job_d" in supervisor._entries, "还在跑就得留在活跃表里，否则重复提交拦不住" + assert "job_d" in supervisor.process_state().entries, "还在跑就得留在活跃表里，否则重复提交拦不住" task.cancel() await asyncio.gather(task, return_exceptions=True) @@ -278,8 +280,9 @@ async def test_run_and_wait_returns_result_when_job_finishes_in_time(monkeypatch """作业按时收工时行为不变 —— 封顶不能把正常的前台等待也改坏。""" from types import SimpleNamespace - from orchestration import job_runtime - from orchestration.jobs import owner, supervisor + import core.jobs.owner as owner + import core.jobs.runtime as job_runtime + import core.jobs.supervisor as supervisor owner.bind() @@ -287,7 +290,7 @@ async def fake_drive(job_row_id, *, chat_id): return {"status": "completed", "stats": {"done": 3}} task = asyncio.create_task(fake_drive("job_e", chat_id="c1")) - monkeypatch.setattr(supervisor, "_entries", {"job_e": SimpleNamespace(task=task)}) + monkeypatch.setattr(jobs_supervisor.process_state(), 'entries', {"job_e": SimpleNamespace(task=task)}) res = await job_runtime.run_and_wait("job_e", chat_id="c1", detach_after=5) assert res["status"] == "completed" @@ -296,19 +299,19 @@ async def fake_drive(job_row_id, *, chat_id): def test_tool_caps_foreground_wait(): """上限得是个真常量，别又退回「让模型自己判断」。""" - from core.llm.tools.job_tool import _FOREGROUND_WAIT_CAP_S + from core.jobs.tool_operations import FOREGROUND_WAIT_CAP_S - assert 30 <= _FOREGROUND_WAIT_CAP_S <= 180 + assert 30 <= FOREGROUND_WAIT_CAP_S <= 180 def test_detached_payload_tells_the_model_to_wrap_up(monkeypatch): """转后台的返回体是模型唯一的行动依据：得说清「没中断」且「这轮就收掉」。""" - from core.llm.tools import job_tool + from core.jobs import tool_operations as job_tool marked = [] - monkeypatch.setattr(job_tool, "_mark_wake_on_finish", marked.append) + monkeypatch.setattr(job_tool, "mark_wake_on_finish", marked.append) - out = job_tool._detached_payload("job_q") + out = job_tool.detached_payload("job_q") assert marked == ["job_q"], "转后台必须打开唤醒标记，否则跑完没人播报" assert out["ok"] is True and out["job_id"] == "job_q" diff --git a/src/backend/tests/orchestration/test_job_inherited_grants.py b/src/backend/tests/orchestration/test_job_inherited_grants.py index 5669506a..af2e1eda 100644 --- a/src/backend/tests/orchestration/test_job_inherited_grants.py +++ b/src/backend/tests/orchestration/test_job_inherited_grants.py @@ -11,9 +11,9 @@ @pytest.mark.asyncio async def test_fork_reads_parent_grants_without_gaining_later_activations(monkeypatch): - from core.capabilities import paths - from core.llm import session_capabilities - from core.plugins import runtime as plugins + from features.desktop.capabilities import paths + from core.llm.runtime import session_capabilities + from core.capabilities.plugins import runtime as plugins seen = [] @@ -30,11 +30,11 @@ def direct(**kwargs): lambda **_: SimpleNamespace(install_ids=[], unavailable_ids=[], skill_ids=[], mcp_ids=[]), ) monkeypatch.setattr(paths, "capabilities_enabled", lambda: False) - # The imported flag in the phase is also disabled for a deterministic cloud path. + # Select the default cloud provider in the phase for deterministic inherited grants. import importlib phase = importlib.import_module("core.llm.factory.selection.sticky_capabilities") - monkeypatch.setattr(phase, "capabilities_enabled", lambda: False) + monkeypatch.setattr(phase, "has_run_capabilities", lambda: False) request = AgentRequest( derived_main=True, chat_id="fork-chat", @@ -63,8 +63,8 @@ def direct(**kwargs): # The grant reaches the real owned-server gate despite a personal disabled switch. from core.llm.factory.selection.server_selection import server_selection from core.llm.factory.tools import mcp_config - from core.services.desktop_cloud_bridge import cloud_gateway_mcp_configs - from core.services.mcp_service import McpServerConfigService + from features.desktop.bridge.gateway import cloud_gateway_mcp_configs + from core.capabilities.mcp.service import McpServerConfigService calls = [] @@ -75,11 +75,11 @@ def get_owned_servers(self, user, enabled_only=True): monkeypatch.setattr(McpServerConfigService, "get_instance", lambda: Config()) monkeypatch.setattr( - "core.services.desktop_cloud_bridge.cloud_gateway_mcp_configs", lambda *_a, **_k: {} + 'features.desktop.bridge.gateway.cloud_gateway_mcp_configs', lambda *_a, **_k: {} ) monkeypatch.setattr( mcp_config, - "_effective_mcp_server_keys", + 'effective_mcp_server_keys', lambda *_a, **k: [m for m in k["enabled_mcp_ids"] if m in k["owned_servers"]], ) result = await server_selection( diff --git a/src/backend/tests/orchestration/test_job_local_callback.py b/src/backend/tests/orchestration/test_job_local_callback.py index d1a910ca..7afdcc91 100644 --- a/src/backend/tests/orchestration/test_job_local_callback.py +++ b/src/backend/tests/orchestration/test_job_local_callback.py @@ -7,8 +7,9 @@ import threading from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import core.jobs.callback as jobs_callback +import core.jobs.callback as callback import pytest -from orchestration.jobs import callback @pytest.mark.parametrize("port_var", ["PORT", "BACKEND_PORT"]) @@ -57,7 +58,7 @@ async def host_bash(command, *, session_id, user_id, timeout): assert result.returncode == 0, result.stderr return result.returncode, result.stdout, result.stderr - monkeypatch.setattr(callback, "_sbx_bash", host_bash) + monkeypatch.setattr(jobs_callback, "_sbx_bash", host_bash) try: assert asyncio.run(callback.resolve_callback_base(session_id="local", user_id="test")) == ( f"http://127.0.0.1:{server.server_port}" diff --git a/src/backend/tests/orchestration/test_job_managed_runtime.py b/src/backend/tests/orchestration/test_job_managed_runtime.py index 97f2a2bc..f898e921 100644 --- a/src/backend/tests/orchestration/test_job_managed_runtime.py +++ b/src/backend/tests/orchestration/test_job_managed_runtime.py @@ -4,11 +4,19 @@ import json from pathlib import PurePosixPath +import core.jobs.files as files +import core.jobs.notifications as notifications +import core.jobs.owner as jobs_owner +import core.jobs.owner as owner +import core.jobs.process as process +import core.jobs.runtime as jobs_runtime +import core.jobs.runtime as runtime +import core.jobs.state as state +import core.jobs.supervisor as jobs_supervisor +import core.jobs.supervisor as supervisor import pytest from core.db.engine import Base -from core.services.job_service import JobService -from orchestration import job_runtime as runtime -from orchestration.jobs import files, notifications, owner, process, state, supervisor +from core.jobs.store import JobService from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker from sqlalchemy.pool import StaticPool @@ -72,8 +80,8 @@ def sandbox(monkeypatch): session = sessionmaker(bind=engine) for module in (runtime, state): monkeypatch.setattr(module, "SessionLocal", session) - monkeypatch.setattr(owner, "_loop", None) - monkeypatch.setattr(supervisor, "_entries", {}) + monkeypatch.setattr(jobs_owner.process_state(), 'loop', None) + monkeypatch.setattr(jobs_supervisor.process_state(), 'entries', {}) box = Sandbox() monkeypatch.setattr(process, "get_sandbox_provider", lambda: box) monkeypatch.setenv("JOB_CALLBACK_URL", "http://test.invalid") @@ -85,7 +93,7 @@ async def logs(*args, **kwargs): return 0, "python: startup failed", "" monkeypatch.setattr(files, "_sbx_bash", logs) - monkeypatch.setattr(supervisor, "_maybe_wake", no_wake) + monkeypatch.setattr(jobs_supervisor, "_maybe_wake", no_wake) monkeypatch.setattr(notifications, "_maybe_wake", no_wake) yield box engine.dispose() @@ -120,7 +128,7 @@ async def worker(): jid = await asyncio.to_thread(lambda: asyncio.run(worker())) assert state.snapshot(jid)["status"] == "running" - assert not supervisor._entries[jid].task.done() + assert not supervisor.process_state().entries[jid].task.done() sandbox.complete(jid) assert (await runtime.run_and_wait(jid))["status"] == "completed" @@ -139,7 +147,7 @@ async def test_start_failure_is_reported_and_process_exit_confirmed(sandbox): @pytest.mark.asyncio async def test_missing_handshake_stops_exact_command(sandbox, monkeypatch): sandbox.handshake = False - monkeypatch.setattr(supervisor, "STARTUP_SECONDS", 0.01) + monkeypatch.setattr(jobs_supervisor, "STARTUP_SECONDS", 0.01) with pytest.raises(RuntimeError, match="启动超时"): await start() row = state.snapshot(next(iter(sandbox.commands))) @@ -179,10 +187,10 @@ async def test_cancel_during_upload_never_launches(sandbox): sandbox.upload_gate = asyncio.Event() launching = asyncio.create_task(start()) for _ in range(50): - if runtime._launches: + if runtime.process_state().launches: break await asyncio.sleep(0.01) - jid = next(iter(runtime._launches)) + jid = next(iter(runtime.process_state().launches)) cancel = asyncio.create_task(runtime.cancel_job(jid, user_id="user")) await asyncio.sleep(0.05) sandbox.upload_gate.set() @@ -200,10 +208,10 @@ async def test_caller_cancellation_keeps_launch_owned(sandbox): sandbox.upload_gate = asyncio.Event() caller = asyncio.create_task(start()) for _ in range(50): - if runtime._launches: + if runtime.process_state().launches: break await asyncio.sleep(0.01) - jid = next(iter(runtime._launches)) + jid = next(iter(runtime.process_state().launches)) caller.cancel() await asyncio.gather(caller, return_exceptions=True) sandbox.upload_gate.set() @@ -229,7 +237,7 @@ async def test_old_monitor_cancellation_cannot_kill_new_owner(sandbox): jid = await start() row = state.snapshot(jid) attempt = row["meta"]["execution"]["attempt_id"] - task = supervisor._entries[jid].task + task = supervisor.process_state().entries[jid].task state.save_execution(jid, attempt, owner="another-worker") task.cancel() await asyncio.gather(task, return_exceptions=True) @@ -242,7 +250,7 @@ async def test_expired_lease_is_reattached_without_relaunch(sandbox, monkeypatch from datetime import timedelta jid = await start() - entry = supervisor._entries[jid] + entry = supervisor.process_state().entries[jid] state.save_execution( jid, entry.execution.attempt_id, @@ -277,7 +285,7 @@ async def slow(*args, **kwargs): entered.set() await asyncio.sleep(60) - monkeypatch.setattr(supervisor, "_maybe_wake_progress", slow) + monkeypatch.setattr(jobs_supervisor, "_maybe_wake_progress", slow) jid = await start(budget={"max_seconds": 2}, start_params={"progress_wake_sec": 0.001}) result = await asyncio.wait_for(runtime.run_and_wait(jid), 5) assert entered.is_set() @@ -295,7 +303,7 @@ async def test_cancel_route_reports_uncertain_stop_as_conflict(monkeypatch): async def uncertain(*args, **kwargs): raise RuntimeError("已撤销回调权限，但无法确认停止") - monkeypatch.setattr(runtime, "cancel_job", uncertain) + monkeypatch.setattr(jobs_runtime, "cancel_job", uncertain) with pytest.raises(HTTPException) as err: await jobs.cancel_job("job", user=SimpleNamespace(user_id="user")) assert err.value.status_code == 409 @@ -305,8 +313,8 @@ async def uncertain(*args, **kwargs): @pytest.mark.asyncio async def test_cancel_lost_ownership_never_confirms_exit(sandbox, monkeypatch): jid = await start() - entry = supervisor._entries[jid] - supervisor._entries.pop(jid) + entry = supervisor.process_state().entries[jid] + supervisor.process_state().entries.pop(jid) async def recover(row): state.save_execution(jid, entry.execution.attempt_id, owner="other-canceller") @@ -334,7 +342,7 @@ async def test_local_cancel_timeout_has_structured_conflict(sandbox, monkeypatch async def timeout(): raise asyncio.TimeoutError() - monkeypatch.setattr(supervisor._entries[jid].execution, "stop", timeout) + monkeypatch.setattr(supervisor.process_state().entries[jid].execution, "stop", timeout) with pytest.raises(HTTPException) as err: await jobs.cancel_job(jid, user=SimpleNamespace(user_id="user")) assert err.value.status_code == 409 diff --git a/src/backend/tests/orchestration/test_job_orphan_reaper.py b/src/backend/tests/orchestration/test_job_orphan_reaper.py index fa513e3b..0d39267c 100644 --- a/src/backend/tests/orchestration/test_job_orphan_reaper.py +++ b/src/backend/tests/orchestration/test_job_orphan_reaper.py @@ -16,6 +16,11 @@ import asyncio from datetime import datetime, timedelta, timezone +import core.jobs.callback as jobs_callback +import core.jobs.notifications as notifications +import core.jobs.owner as jobs_owner +import core.jobs.state as state +import core.jobs.supervisor as supervisor import pytest from core.db.engine import Base from core.db.models import Job @@ -32,8 +37,12 @@ async def run(): return asyncio.run(run()) -import orchestration.job_runtime as jr -from orchestration.jobs import callback, notifications, owner, state, supervisor +import core.jobs.callback as callback +import core.jobs.notifications as notifications +import core.jobs.owner as owner +import core.jobs.runtime as jr +import core.jobs.state as state +import core.jobs.supervisor as supervisor @pytest.fixture() @@ -45,8 +54,8 @@ def db_session(monkeypatch): Session = sessionmaker(bind=engine) monkeypatch.setattr(jr, "SessionLocal", Session) monkeypatch.setattr(state, "SessionLocal", Session) - monkeypatch.setattr(owner, "_loop", None) - monkeypatch.setattr(supervisor, "_entries", {}) + monkeypatch.setattr(jobs_owner.process_state(), 'loop', None) + monkeypatch.setattr(supervisor.process_state(), 'entries', {}) async def _no_wake(job_row_id): return None @@ -148,7 +157,7 @@ def test_explicit_env_wins_and_skips_probing(monkeypatch): async def _boom(*a, **k): # 手动指定即信任，不该再去沙箱里探 raise AssertionError("probe must not run when JOB_CALLBACK_URL is set") - monkeypatch.setattr(callback, "_sbx_bash", _boom) + monkeypatch.setattr(jobs_callback, "_sbx_bash", _boom) assert asyncio.run(callback.resolve_callback_base(session_id="s", user_id="u")) == ( "http://custom:9000/api" @@ -162,7 +171,7 @@ def test_probe_picks_the_reachable_base(monkeypatch): async def fake_bash(cmd, *, session_id, user_id, timeout=60): return 0, "PICK http://backend:8011\n", "" - monkeypatch.setattr(callback, "_sbx_bash", fake_bash) + monkeypatch.setattr(jobs_callback, "_sbx_bash", fake_bash) assert asyncio.run(callback.resolve_callback_base(session_id="s", user_id="u")) == ( "http://backend:8011" @@ -180,7 +189,7 @@ def test_unreachable_callback_refuses_to_start(monkeypatch): async def fake_bash(cmd, *, session_id, user_id, timeout=60): return 0, "NONE\n", "" - monkeypatch.setattr(callback, "_sbx_bash", fake_bash) + monkeypatch.setattr(jobs_callback, "_sbx_bash", fake_bash) with pytest.raises(RuntimeError) as exc: asyncio.run(callback.resolve_callback_base(session_id="s", user_id="u")) diff --git a/src/backend/tests/orchestration/test_job_resilience.py b/src/backend/tests/orchestration/test_job_resilience.py index 401484ab..03ae0a74 100644 --- a/src/backend/tests/orchestration/test_job_resilience.py +++ b/src/backend/tests/orchestration/test_job_resilience.py @@ -20,7 +20,7 @@ import pytest from api.routes.v1.internal_jobs import AgentBody, LedgerBody from core.chat.tool_log import _payload_carries_error -from orchestration.jobs.sdk import SDK_SOURCE +from core.jobs.sdk import SDK_SOURCE # ── ② 主键类型：int 必须能进门 ────────────────────────────────────── @@ -166,7 +166,7 @@ def test_runner_persists_receipt_even_when_callback_fails(tmp_path, script, star import subprocess import sys - from orchestration.jobs.runner import RUNNER_SOURCE + from core.jobs.runner import RUNNER_SOURCE (tmp_path / "_runner.py").write_text(RUNNER_SOURCE) (tmp_path / "execution.json").write_text(json.dumps({"JOB_ATTEMPT_ID": "attempt"})) diff --git a/src/backend/tests/orchestration/test_job_sdk_ledger.py b/src/backend/tests/orchestration/test_job_sdk_ledger.py index 7256788d..64b09a32 100644 --- a/src/backend/tests/orchestration/test_job_sdk_ledger.py +++ b/src/backend/tests/orchestration/test_job_sdk_ledger.py @@ -19,7 +19,7 @@ import re import pytest -from orchestration.jobs.sdk import SDK_SOURCE +from core.jobs.sdk import SDK_SOURCE @pytest.fixture() diff --git a/src/backend/tests/orchestration/test_job_ui_progress.py b/src/backend/tests/orchestration/test_job_ui_progress.py index 03e45475..c60a3d6d 100644 --- a/src/backend/tests/orchestration/test_job_ui_progress.py +++ b/src/backend/tests/orchestration/test_job_ui_progress.py @@ -22,7 +22,7 @@ @pytest.fixture() def pushed(monkeypatch) -> List[Dict[str, Any]]: """截获 _subagent_stream 旁路上真正被推出去的事件。""" - from core.llm import _subagent_stream + from core.llm.subagents import stream as _subagent_stream events: List[Dict[str, Any]] = [] monkeypatch.setattr(_subagent_stream, "is_active", lambda chat_id: True) @@ -31,7 +31,7 @@ def pushed(monkeypatch) -> List[Dict[str, Any]]: def test_ui_progress_carries_numbers_and_dodges_the_swallowed_sub_type(pushed): - from orchestration.jobs import notifications as job_runtime + import core.jobs.notifications as job_runtime job_runtime._emit_ui_progress( "chat_1", @@ -58,8 +58,8 @@ def test_ui_progress_carries_numbers_and_dodges_the_swallowed_sub_type(pushed): def test_ui_progress_binds_to_the_current_tool_call(pushed): - from core.llm.middlewares import CURRENT_TOOL_CALL_ID - from orchestration.jobs import notifications as job_runtime + import core.jobs.notifications as job_runtime + from core.harness.tool_call_context import CURRENT_TOOL_CALL_ID token = CURRENT_TOOL_CALL_ID.set("call_abc") try: @@ -72,8 +72,8 @@ def test_ui_progress_binds_to_the_current_tool_call(pushed): def test_ui_progress_silent_without_an_active_stream(monkeypatch): """后台作业跑到主对话已经收尾之后：没有监听者就什么都别推。""" - from core.llm import _subagent_stream - from orchestration.jobs import notifications as job_runtime + import core.jobs.notifications as job_runtime + from core.llm.subagents import stream as _subagent_stream pushes: List[Any] = [] monkeypatch.setattr(_subagent_stream, "is_active", lambda chat_id: False) diff --git a/src/backend/tests/orchestration/test_job_wake_delivery_format.py b/src/backend/tests/orchestration/test_job_wake_delivery_format.py index 5bda957e..73616876 100644 --- a/src/backend/tests/orchestration/test_job_wake_delivery_format.py +++ b/src/backend/tests/orchestration/test_job_wake_delivery_format.py @@ -18,8 +18,8 @@ 模型自由裁量。 """ +from core.chat.runtime.job_wakeup import _wake_prompt from core.db.models import Job as JobModel -from orchestration.job_wakeup import _wake_prompt _SETTLED = {"total": 265, "done": 261, "not_found": 4, "failed": 0, "pending": 0, "remaining": 0} diff --git a/src/backend/tests/orchestration/test_job_wake_message_visibility.py b/src/backend/tests/orchestration/test_job_wake_message_visibility.py index 20f40ba8..841121e2 100644 --- a/src/backend/tests/orchestration/test_job_wake_message_visibility.py +++ b/src/backend/tests/orchestration/test_job_wake_message_visibility.py @@ -14,11 +14,15 @@ 另外锁一条唤醒文案：作业丢后台之后没有任何一轮会再动 ``update_plan`` 的计划清单， 终态交付那一轮必须被要求给它收尾——否则计划栏永远停在"提交作业"那一步。 """ +from core.chat import service as service_module import asyncio -import core.services.session_operations.session_context as chat_session_context +import core.chat.runtime.admission as admission +import core.chat.runtime.job_wakeup as runtime_job_wakeup import core.db.engine as db_engine +import core.runs.runtime as runtime +import core.chat.operations.session_context as chat_session_context import pytest from core.db.engine import Base from core.db.models import ChatMessage, ChatSession, Job @@ -36,9 +40,9 @@ def db_session(monkeypatch): Session = sessionmaker(bind=engine) monkeypatch.setattr(db_engine, "SessionLocal", Session) - from orchestration import job_wakeup + import core.chat.runtime.job_wakeup as job_wakeup - monkeypatch.setattr(job_wakeup, "SessionLocal", Session, raising=False) + monkeypatch.setattr(runtime_job_wakeup, "SessionLocal", Session, raising=False) return Session @@ -85,23 +89,23 @@ def get_session(self, chat_id, user_id): async def fake_start_run(**kw): return "run_1" - import core.config.catalog_resolver as resolver_mod - import core.services.chat_service as chat_service_mod - from orchestration import chat_run_executor + import core.capabilities.catalog.resolver as resolver_mod + import core.chat.service as chat_service_mod + pass - monkeypatch.setattr(chat_session_context, "_load_session_messages", lambda *a, **k: []) + monkeypatch.setattr(chat_session_context, 'load_session_messages', lambda *a, **k: []) monkeypatch.setattr( resolver_mod, "resolve_all_runtime_enabled", lambda *a, **k: (None, None, None) ) - monkeypatch.setattr(chat_service_mod, "ChatService", _FakeChatService) - monkeypatch.setattr(chat_run_executor, "start_run", fake_start_run) + monkeypatch.setattr(service_module, 'ChatService', _FakeChatService) + monkeypatch.setattr(admission, "start_run", fake_start_run) def test_finish_wake_message_is_marked_hidden(monkeypatch, db_session): _seed(db_session) _stub_run_dependencies(monkeypatch, db_session) - from orchestration import job_wakeup + import core.chat.runtime.job_wakeup as job_wakeup assert asyncio.run(job_wakeup.wake_on_job_finish("job_1")) is True with db_session() as db: @@ -122,9 +126,9 @@ def test_progress_wake_message_is_marked_hidden(monkeypatch, db_session): _seed(db_session, status="running") _stub_run_dependencies(monkeypatch, db_session) - from orchestration import chat_run_executor, job_wakeup + import core.chat.runtime.job_wakeup as job_wakeup - monkeypatch.setattr(chat_run_executor, "get_active_run_for_chat", lambda *a, **k: None) + monkeypatch.setattr(runtime, "get_active_run_for_chat", lambda *a, **k: None) ok = asyncio.run( job_wakeup.wake_on_job_progress( @@ -155,7 +159,7 @@ def __init__(self, role, content, extra_data=None): def test_message_list_filters_wake_messages(): """消息列表接口的过滤谓词：带标记的、以及标记上线前落库的老消息，都不进聊天记录。""" - from core.services.session_operations.history import _is_internal_message + from core.chat.operations.history import _is_internal_message assert _is_internal_message(_Msg("user", "转述一下", {"hidden_in_chat": True})) # 老消息（没有标记，只有开头那句） @@ -174,8 +178,8 @@ def test_finish_wake_prompt_has_no_plan_noise_without_a_plan(): 有清单时"把清单念回去、要求收尾"的行为改由服务端快照驱动，锁在 ``test_plan_progress_closeout.py``——那里连清单内容和先后顺序一起验。 """ + from core.chat.runtime.job_wakeup import _wake_prompt from core.db.models import Job as JobModel - from orchestration.job_wakeup import _wake_prompt job = JobModel(job_id="job_1", name="问答记录分类", status="completed") prompt = _wake_prompt(job, {"total": 265, "done": 265, "remaining": 0}) diff --git a/src/backend/tests/orchestration/test_job_wake_mode.py b/src/backend/tests/orchestration/test_job_wake_mode.py index 57205987..f83983ec 100644 --- a/src/backend/tests/orchestration/test_job_wake_mode.py +++ b/src/backend/tests/orchestration/test_job_wake_mode.py @@ -13,12 +13,16 @@ 所以这里锁的是这条不变量：唤醒轮的 context 必须如实反映会话元数据里的 `workflow_chat`，进度唤醒和终态唤醒两条路都要带。 """ +from core.chat import service as service_module import asyncio from typing import Any, Dict, List -import core.services.session_operations.session_context as chat_session_context +import core.chat.runtime.admission as admission +import core.chat.runtime.job_wakeup as runtime_job_wakeup import core.db.engine as db_engine +import core.runs.runtime as runtime +import core.chat.operations.session_context as chat_session_context import pytest from core.db.engine import Base from core.db.models import ChatSession, Job @@ -36,9 +40,9 @@ def db_session(monkeypatch): Session = sessionmaker(bind=engine) monkeypatch.setattr(db_engine, "SessionLocal", Session) - from orchestration import job_wakeup + import core.chat.runtime.job_wakeup as job_wakeup - monkeypatch.setattr(job_wakeup, "SessionLocal", Session, raising=False) + monkeypatch.setattr(runtime_job_wakeup, "SessionLocal", Session, raising=False) return Session @@ -71,7 +75,7 @@ def _seed(db_session, *, chat_meta: Dict[str, Any], status: str = "completed") - def _capture_started_runs(monkeypatch, db_session) -> List[Dict[str, Any]]: """把唤醒真正入队的那一发拦下来，只留 context 供断言。""" - from orchestration import job_wakeup + import core.chat.runtime.job_wakeup as job_wakeup started: List[Dict[str, Any]] = [] @@ -99,16 +103,16 @@ async def fake_start_run(**kw): started.append(kw) return "run_1" - import core.config.catalog_resolver as resolver_mod - import core.services.chat_service as chat_service_mod - from orchestration import chat_run_executor + import core.capabilities.catalog.resolver as resolver_mod + import core.chat.service as chat_service_mod + pass - monkeypatch.setattr(chat_session_context, "_load_session_messages", fake_load_session_messages) + monkeypatch.setattr(chat_session_context, 'load_session_messages', fake_load_session_messages) monkeypatch.setattr(resolver_mod, "resolve_all_runtime_enabled", fake_resolve_all) - monkeypatch.setattr(chat_service_mod, "ChatService", _FakeChatService) - monkeypatch.setattr(chat_run_executor, "start_run", fake_start_run) + monkeypatch.setattr(service_module, 'ChatService', _FakeChatService) + monkeypatch.setattr(admission, "start_run", fake_start_run) # 唤醒文案本身不是本用例的关注点，但它要能生成——JobService 走的是同一个假库 - monkeypatch.setattr(job_wakeup, "SessionLocal", db_session, raising=False) + monkeypatch.setattr(runtime_job_wakeup, "SessionLocal", db_session, raising=False) return started @@ -118,7 +122,7 @@ def test_finish_wake_inherits_workflow_mode(monkeypatch, db_session, status): _seed(db_session, chat_meta={"workflow_chat": True}, status=status) started = _capture_started_runs(monkeypatch, db_session) - from orchestration import job_wakeup + import core.chat.runtime.job_wakeup as job_wakeup assert asyncio.run(job_wakeup.wake_on_job_finish("job_1")) is True assert len(started) == 1 @@ -133,7 +137,7 @@ def test_finish_wake_does_not_invent_workflow_mode(monkeypatch, db_session): _seed(db_session, chat_meta={}) started = _capture_started_runs(monkeypatch, db_session) - from orchestration import job_wakeup + import core.chat.runtime.job_wakeup as job_wakeup assert asyncio.run(job_wakeup.wake_on_job_finish("job_1")) is True assert started[0]["context"]["workflow_chat"] is False @@ -144,9 +148,9 @@ def test_progress_wake_inherits_workflow_mode(monkeypatch, db_session): _seed(db_session, chat_meta={"workflow_chat": True}, status="running") started = _capture_started_runs(monkeypatch, db_session) - from orchestration import chat_run_executor, job_wakeup + import core.chat.runtime.job_wakeup as job_wakeup - monkeypatch.setattr(chat_run_executor, "get_active_run_for_chat", lambda *a, **k: None) + monkeypatch.setattr(runtime, "get_active_run_for_chat", lambda *a, **k: None) ok = asyncio.run( job_wakeup.wake_on_job_progress( diff --git a/src/backend/tests/orchestration/test_job_windows_attempt_paths.py b/src/backend/tests/orchestration/test_job_windows_attempt_paths.py index f4912ddd..3afe0bec 100644 --- a/src/backend/tests/orchestration/test_job_windows_attempt_paths.py +++ b/src/backend/tests/orchestration/test_job_windows_attempt_paths.py @@ -1,11 +1,12 @@ """Windows attempts keep the full unique id while avoiding excess nesting.""" -from orchestration.jobs import process +import core.jobs.process as jobs_process +import core.jobs.process as process def test_native_windows_attempt_path_keeps_sdk_below_max_path(monkeypatch): workspace = "C:/" + "a" * 184 - monkeypatch.setattr(process, "_job_directory", lambda session, job: workspace + "/.job/" + job) + monkeypatch.setattr(jobs_process, "_job_directory", lambda session, job: workspace + "/.job/" + job) job = "job_" + "1" * 16 attempt = "2" * 32 old = process.attempt_directory("chat", job, attempt, windows_local=False) diff --git a/src/backend/tests/orchestration/test_loop_reviewer_registry_tool.py b/src/backend/tests/orchestration/test_loop_reviewer_registry_tool.py index 6b3635b3..53361098 100644 --- a/src/backend/tests/orchestration/test_loop_reviewer_registry_tool.py +++ b/src/backend/tests/orchestration/test_loop_reviewer_registry_tool.py @@ -8,8 +8,8 @@ registry-backed proof. """ -from core.llm.builtin_subagents import get_builtin_subagent -from orchestration.subagents.loop_reviewer import _build_review_prompt +from core.capabilities.agents.builtin import get_builtin_subagent +from orchestration.loops.reviewer import _build_review_prompt def test_review_prompt_points_registry_criteria_at_read_artifact(): diff --git a/src/backend/tests/orchestration/test_missing_tool_call_synthesis.py b/src/backend/tests/orchestration/test_missing_tool_call_synthesis.py index 1cdbf98b..a5b32c62 100644 --- a/src/backend/tests/orchestration/test_missing_tool_call_synthesis.py +++ b/src/backend/tests/orchestration/test_missing_tool_call_synthesis.py @@ -11,10 +11,10 @@ ``_synthesize_missing_tool_call`` closes that hole at the tool_result stage. """ +from core.agent_runtime.events import _synthesize_missing_tool_call from core.chat.tool_log import attach_tool_result, build_tool_call_event, upsert_tool_call +from core.chat.tool_payloads import _tool_args_ready from core.config.display_names import TOOL_DISPLAY_NAMES -from orchestration.tool_payloads import _tool_args_ready -from orchestration.workflow import _synthesize_missing_tool_call # The tool this was found on. Its display name is edition-dependent (the industry # MCP is trimmed out of CE), so the tests resolve it through the same map the diff --git a/src/backend/tests/orchestration/test_model_progress_signal.py b/src/backend/tests/orchestration/test_model_progress_signal.py index 69bdd4a5..14eecb1f 100644 --- a/src/backend/tests/orchestration/test_model_progress_signal.py +++ b/src/backend/tests/orchestration/test_model_progress_signal.py @@ -5,14 +5,14 @@ the run watchdog alive; flushed fragments become visible ``tool_call_delta`` events and the completed JSON is emitted exactly once as ``tool_call``. """ +from core.llm.runtime import streaming_constants as streaming_state_module import asyncio from types import SimpleNamespace +import core.llm.runtime.streaming as streaming_mod import pytest - -import orchestration.streaming as streaming_mod -from orchestration.streaming import StreamingAgent +from core.llm.runtime.streaming import StreamingAgent class ToolCallStartEvent: # noqa: D101 - name-dispatched by _map_event @@ -60,7 +60,7 @@ async def _collect(agent): @pytest.mark.asyncio async def test_small_buffered_deltas_emit_model_progress_until_flushed(monkeypatch): - monkeypatch.setattr(streaming_mod, "_MODEL_PROGRESS_MIN_INTERVAL_S", 0.0) + monkeypatch.setattr(streaming_state_module, 'MODEL_PROGRESS_MIN_INTERVAL_S', 0.0) events = [ToolCallStartEvent()] + [ToolCallDeltaEvent(delta="chunk") for _ in range(5)] out = await _collect(_fake_agent(events)) types = [t for t, _ in out] @@ -79,8 +79,8 @@ async def test_throttle_suppresses_model_progress(): @pytest.mark.asyncio async def test_tool_arguments_stream_as_bounded_deltas_then_one_final_call(monkeypatch): - monkeypatch.setattr(streaming_mod, "_TOOL_CALL_DELTA_FLUSH_CHARS", 8) - monkeypatch.setattr(streaming_mod, "_TOOL_CALL_DELTA_FLUSH_INTERVAL_S", 999.0) + monkeypatch.setattr(streaming_state_module, 'TOOL_CALL_DELTA_FLUSH_CHARS', 8) + monkeypatch.setattr(streaming_state_module, 'TOOL_CALL_DELTA_FLUSH_INTERVAL_S', 999.0) events = [ ToolCallStartEvent(), ToolCallDeltaEvent(delta='{"command"'), @@ -118,8 +118,8 @@ async def test_inflight_model_call_silence_emits_model_progress(monkeypatch): # A model call in flight (ModelCallStart seen, nothing since — hung/slow LLM # endpoint) must surface as model_progress, not heartbeat, so the run # inactivity watchdog does not kill a run that is merely waiting on the model. - monkeypatch.setattr(streaming_mod, "_MODEL_PROGRESS_MIN_INTERVAL_S", 0.0) - monkeypatch.setattr(streaming_mod, "_QUEUE_POLL_INTERVAL_S", 0.01) + monkeypatch.setattr(streaming_state_module, 'MODEL_PROGRESS_MIN_INTERVAL_S', 0.0) + monkeypatch.setattr(streaming_state_module, 'QUEUE_POLL_INTERVAL_S', 0.01) out = await _collect(_slow_model_agent([ModelCallStartEvent()], silence_s=0.1)) types = [t for t, _ in out] assert types.count("model_progress") >= 2 @@ -128,7 +128,7 @@ async def test_inflight_model_call_silence_emits_model_progress(monkeypatch): @pytest.mark.asyncio async def test_model_call_start_surfaces_the_real_dispatch_boundary(monkeypatch): - monkeypatch.setattr(streaming_mod, "_QUEUE_POLL_INTERVAL_S", 0.01) + monkeypatch.setattr(streaming_state_module, 'QUEUE_POLL_INTERVAL_S', 0.01) out = await _collect(_slow_model_agent([ModelCallStartEvent()], silence_s=0.01)) assert ("model_call_start", None) in out @@ -138,8 +138,8 @@ async def test_model_call_start_surfaces_the_real_dispatch_boundary(monkeypatch) async def test_silence_without_inflight_model_call_stays_heartbeat(monkeypatch): # Same silence but no model call in flight (e.g. a tool executing) keeps the # historical heartbeat semantics — excluded from watchdog activity. - monkeypatch.setattr(streaming_mod, "_MODEL_PROGRESS_MIN_INTERVAL_S", 0.0) - monkeypatch.setattr(streaming_mod, "_QUEUE_POLL_INTERVAL_S", 0.01) + monkeypatch.setattr(streaming_state_module, 'MODEL_PROGRESS_MIN_INTERVAL_S', 0.0) + monkeypatch.setattr(streaming_state_module, 'QUEUE_POLL_INTERVAL_S', 0.01) out = await _collect(_slow_model_agent([], silence_s=0.1)) types = [t for t, _ in out] assert types.count("heartbeat") >= 2 diff --git a/src/backend/tests/orchestration/test_model_steps_pipeline.py b/src/backend/tests/orchestration/test_model_steps_pipeline.py index cdee28f7..a65076fd 100644 --- a/src/backend/tests/orchestration/test_model_steps_pipeline.py +++ b/src/backend/tests/orchestration/test_model_steps_pipeline.py @@ -12,6 +12,10 @@ import contextlib from types import SimpleNamespace +import core.agent_runtime.runner as runner +import core.chat.runtime.admission as admission +import core.db.engine as db_engine +import core.runs.tasks as tasks import fakeredis.aioredis import pytest from agentscope.message import Msg, TextBlock, ThinkingBlock, ToolCallBlock, ToolResultBlock @@ -19,8 +23,7 @@ from core.db.engine import Base from core.db.models import ChatMessage, ChatSession from core.harness.hooks import HookBus -from core.llm.agentscope_hook_adapter import AgentScopeHookAdapter -from orchestration import chat_run_executor as executor +from core.llm.runtime.agentscope_hook_adapter import AgentScopeHookAdapter from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker @@ -93,7 +96,7 @@ def _fake(name: str, **fields): async def _stream_events(events, state): - from orchestration.streaming import StreamingAgent + from core.llm.runtime.streaming import StreamingAgent async def reply_stream(inputs=None): # noqa: ANN001, ARG001 for event in events: @@ -188,15 +191,15 @@ def run_env(tmp_path, monkeypatch): ) Base.metadata.create_all(engine) sessions = sessionmaker(bind=engine, expire_on_commit=False) - monkeypatch.setattr(executor, "SessionLocal", sessions) + monkeypatch.setattr(db_engine, "SessionLocal", sessions) redis = fakeredis.aioredis.FakeRedis(decode_responses=True) - monkeypatch.setattr("orchestration.run_event_stream.get_redis", lambda **_: redis) - executor._active_runs.clear() + monkeypatch.setattr('core.runs.events.store.get_redis', lambda **_: redis) + tasks.process_state().active_runs.clear() with sessions() as db: db.add(ChatSession(chat_id="chat-1", user_id="user-1", title="test")) db.commit() yield sessions - executor._active_runs.clear() + tasks.process_state().active_runs.clear() engine.dispose() @@ -241,8 +244,8 @@ async def gated(**kwargs): async for event in workflow(**kwargs): yield event - monkeypatch.setattr(executor, "astream_chat_workflow", gated) - run = await executor.start_run( + monkeypatch.setattr(runner, "astream_chat_workflow", gated) + run = await admission.start_run( chat_id="chat-1", user_id="user-1", session_messages=[{"role": "user", "content": "hello"}], @@ -253,7 +256,7 @@ async def gated(**kwargs): model_name="test-model", ) # The chat route reserves this row before the worker starts. - with executor.SessionLocal() as db: + with db_engine.SessionLocal() as db: db.add( ChatMessage( message_id=run.message_id, @@ -264,7 +267,7 @@ async def gated(**kwargs): ) db.commit() start.set() - worker = executor._active_runs.get(run.run_id) + worker = tasks.process_state().active_runs.get(run.run_id) if worker is not None: with contextlib.suppress(asyncio.TimeoutError, asyncio.CancelledError): await asyncio.wait_for(worker, timeout=10) diff --git a/src/backend/tests/orchestration/test_model_steps_plan_projection.py b/src/backend/tests/orchestration/test_model_steps_plan_projection.py index 5a6264f6..2207a5cf 100644 --- a/src/backend/tests/orchestration/test_model_steps_plan_projection.py +++ b/src/backend/tests/orchestration/test_model_steps_plan_projection.py @@ -7,16 +7,28 @@ """ from __future__ import annotations +from core.agent_runtime.integrations import memory_identity as memory_identity_module +from core.agent_runtime.integrations import memory_session as memory_session_module +from core.llm.citations import allocator as allocator_module import json from types import SimpleNamespace +import core.agent_runtime.integrations.memory as memory +import core.agent_runtime.runner as runner +import core.agent_runtime.targets as targets +import core.chat.plan_progress as plan_progress +import core.capabilities.catalog.resolver as catalog_resolver +import core.llm.citations.anchor as citation_anchor +import core.llm.factory as factory +import core.llm.runtime.streaming as streaming +import core.runs.events.writer as writer +import core.chat.service as services_ChatService_module import pytest from agentscope.message import Msg, ToolResultBlock +from core.agent_runtime.integrations.memory_session import SessionMemory from core.db.models import ChatMessage -from core.services.compaction_history import normalize_rows -from orchestration import chat_run_executor as executor -from orchestration.memory_integration import SessionMemory +from core.chat.compaction.history import normalize_rows from tests.orchestration.test_model_steps_pipeline import _fake, _run, _step, _stream_events from tests.orchestration.test_model_steps_pipeline import run_env as run_env @@ -27,16 +39,11 @@ async def test_plan_bar_and_canonical_results_survive_workflow_and_persistence( run_env, monkeypatch, persist_result ): from core.db import engine as db_engine - from core.llm import builtin_subagents - from core.services import compaction_service, user_agent_service, user_service - from orchestration import workflow - - class DummySession: - def __enter__(self): - return SimpleNamespace() - - def __exit__(self, *_args): - return False + from core.capabilities.agents import builtin as builtin_subagents + from core.chat.compaction import service as compaction_service + from core.capabilities.agents import edition as user_agent_service + from core.auth import users as user_service + pass class FakeStreamingAgent: def __init__(self, agent, _clients): @@ -55,6 +62,7 @@ async def stream(self, _messages, _context): state="success", ) state = SimpleNamespace( + pending_mcp_progress=[], user_id="", chat_id="", run_id="", @@ -120,7 +128,6 @@ async def no_identity(_user_id): async def no_compaction(_chat_id, messages, **_kwargs): return messages, None - monkeypatch.setattr(db_engine, "SessionLocal", lambda: DummySession()) monkeypatch.setattr( user_agent_service, "UserAgentService", @@ -133,39 +140,38 @@ async def no_compaction(_chat_id, messages, **_kwargs): ) monkeypatch.setattr(builtin_subagents, "merge_builtin_subagents", lambda *_a, **_kw: []) monkeypatch.setattr(compaction_service, "maybe_run_pre_turn_compaction", no_compaction) - monkeypatch.setattr(workflow, "create_agent_executor", create_agent) - monkeypatch.setattr(workflow, "open_session_memory", no_memory) - monkeypatch.setattr(workflow, "build_user_identity_block", no_identity) - monkeypatch.setattr(workflow, "anchor_start_for_chat", lambda _chat_id: 0) - monkeypatch.setattr(workflow, "enabled_skill_ids_from_context", lambda _ctx: []) - monkeypatch.setattr(workflow, "enabled_mcp_ids_from_context", lambda _ctx: []) - monkeypatch.setattr(workflow, "enabled_kb_ids_from_context", lambda _ctx: []) - monkeypatch.setattr(workflow, "_resolve_mode_spec", lambda _ctx: None) - monkeypatch.setattr(workflow, "StreamingAgent", FakeStreamingAgent) - monkeypatch.setattr(workflow, "_persistent_clients", []) + monkeypatch.setattr(factory, "create_agent_executor", create_agent) + monkeypatch.setattr(memory_session_module, 'open_session_memory', no_memory) + monkeypatch.setattr(memory_identity_module, 'build_user_identity_block', no_identity) + monkeypatch.setattr(allocator_module, 'anchor_start_for_chat', lambda _chat_id: 0) + monkeypatch.setattr(catalog_resolver, "enabled_skill_ids_from_context", lambda _ctx: []) + monkeypatch.setattr(catalog_resolver, "enabled_mcp_ids_from_context", lambda _ctx: []) + monkeypatch.setattr(catalog_resolver, "enabled_kb_ids_from_context", lambda _ctx: []) + monkeypatch.setattr(targets, "_resolve_mode_spec", lambda _ctx: None) + monkeypatch.setattr("core.agent_runtime.prepare_main.streaming", SimpleNamespace(StreamingAgent=FakeStreamingAgent)) if not persist_result: - original_refresh = executor.ChatService.refresh_streaming_message + original_refresh = services_ChatService_module.ChatService.refresh_streaming_message def fail_result(self, **kwargs): if any(s["kind"] == "tool_result" for s in kwargs.get("model_steps", [])): raise RuntimeError("injected result persistence failure") return original_refresh(self, **kwargs) - monkeypatch.setattr(executor.ChatService, "refresh_streaming_message", fail_result) + monkeypatch.setattr(services_ChatService_module.ChatService, "refresh_streaming_message", fail_result) saved_plans = [] monkeypatch.setattr( - workflow, "_save_plan_progress", lambda chat, plan: saved_plans.append(plan) + plan_progress, 'save_plan_progress', lambda chat, plan: saved_plans.append(plan) ) published = [] - original_emit = executor._xadd_event + original_emit = writer.xadd_event async def emit(run_id, offset, event): published.append(event) await original_emit(run_id, offset, event) - monkeypatch.setattr(executor, "_xadd_event", emit) - run = await _run(workflow.astream_chat_workflow, monkeypatch) + monkeypatch.setattr(writer, 'xadd_event', emit) + run = await _run(runner.astream_chat_workflow, monkeypatch) if not persist_result: assert not saved_plans assert not any(event["type"] == "plan_update" for event in published) diff --git a/src/backend/tests/orchestration/test_model_steps_worker.py b/src/backend/tests/orchestration/test_model_steps_worker.py index 8a07d959..24d28449 100644 --- a/src/backend/tests/orchestration/test_model_steps_worker.py +++ b/src/backend/tests/orchestration/test_model_steps_worker.py @@ -11,11 +11,16 @@ import asyncio import contextlib +import core.agent_runtime.runner as runner +import core.chat.runtime.admission as admission +import core.runs.events.writer as writer +import core.runs.runtime as runtime +import core.runs.tasks as tasks +import core.chat.service as services_ChatService_module import pytest from core.db.models import ChatMessage -from core.llm.model_steps import assistant_row_replay -from core.services.compaction_history import normalize_rows -from orchestration import chat_run_executor as executor +from core.llm.runtime.model_steps import assistant_row_replay +from core.chat.compaction.history import normalize_rows from tests.orchestration.test_model_steps_pipeline import _result, _run, _step from tests.orchestration.test_model_steps_pipeline import run_env as run_env @@ -106,8 +111,8 @@ async def workflow(**_kwargs): streamed.set() await asyncio.Event().wait() - monkeypatch.setattr(executor, "astream_chat_workflow", workflow) - run = await executor.start_run( + monkeypatch.setattr(runner, "astream_chat_workflow", workflow) + run = await admission.start_run( chat_id="chat-1", user_id="user-1", session_messages=[{"role": "user", "content": "hello"}], @@ -118,8 +123,8 @@ async def workflow(**_kwargs): model_name="test-model", ) await asyncio.wait_for(streamed.wait(), timeout=2) - worker = executor._active_runs.get(run.run_id) - assert await executor.cancel_run(run.run_id, user_id="user-1") + worker = tasks.process_state().active_runs.get(run.run_id) + assert await runtime.cancel_run(run.run_id, user_id="user-1") with contextlib.suppress(asyncio.CancelledError, asyncio.TimeoutError): await asyncio.wait_for(worker, timeout=3) @@ -139,7 +144,7 @@ async def workflow(**_kwargs): async def test_tool_success_checkpoint_contains_canonical_result( run_env, monkeypatch, combined, fail_once ): - original_emit = executor._xadd_event + original_emit = writer.xadd_event published_results = [] async def emit(run_id, offset, event): @@ -152,8 +157,8 @@ async def emit(run_id, offset, event): published_results.append(event) await original_emit(run_id, offset, event) - monkeypatch.setattr(executor, "_xadd_event", emit) - original_refresh = executor.ChatService.refresh_streaming_message + monkeypatch.setattr(writer, 'xadd_event', emit) + original_refresh = services_ChatService_module.ChatService.refresh_streaming_message failed = [] def refresh(self, **kwargs): @@ -166,7 +171,7 @@ def refresh(self, **kwargs): raise RuntimeError("injected checkpoint failure") return original_refresh(self, **kwargs) - monkeypatch.setattr(executor.ChatService, "refresh_streaming_message", refresh) + monkeypatch.setattr(services_ChatService_module.ChatService, "refresh_streaming_message", refresh) reached = asyncio.Event() release = asyncio.Event() start = asyncio.Event() @@ -194,8 +199,8 @@ async def workflow(**kwargs): await release.wait() yield {"type": "meta"} - monkeypatch.setattr(executor, "astream_chat_workflow", workflow) - run = await executor.start_run( + monkeypatch.setattr(runner, "astream_chat_workflow", workflow) + run = await admission.start_run( chat_id="chat-1", user_id="user-1", session_messages=[{"role": "user", "content": "q"}], @@ -233,7 +238,7 @@ async def workflow(**kwargs): assert published_results finally: release.set() - task = executor._active_runs.get(run.run_id) + task = tasks.process_state().active_runs.get(run.run_id) if task is not None: await asyncio.wait_for(task, 10) @@ -242,8 +247,8 @@ async def workflow(**kwargs): @pytest.mark.parametrize("combined", [False, True]) async def test_failed_tool_checkpoint_does_not_publish_success(run_env, monkeypatch, combined): start = asyncio.Event() - original_refresh = executor.ChatService.refresh_streaming_message - original_emit = executor._xadd_event + original_refresh = services_ChatService_module.ChatService.refresh_streaming_message + original_emit = writer.xadd_event published = [] def refresh(self, **kwargs): @@ -274,10 +279,10 @@ async def workflow(**kwargs): } pytest.fail("must stop after an undurable completed tool result") - monkeypatch.setattr(executor.ChatService, "refresh_streaming_message", refresh) - monkeypatch.setattr(executor, "_xadd_event", emit) - monkeypatch.setattr(executor, "astream_chat_workflow", workflow) - run = await executor.start_run( + monkeypatch.setattr(services_ChatService_module.ChatService, "refresh_streaming_message", refresh) + monkeypatch.setattr(writer, 'xadd_event', emit) + monkeypatch.setattr(runner, "astream_chat_workflow", workflow) + run = await admission.start_run( chat_id="chat-1", user_id="user-1", session_messages=[{"role": "user", "content": "q"}], @@ -297,7 +302,7 @@ async def workflow(**kwargs): ) ) db.commit() - task = executor._active_runs[run.run_id] + task = tasks.process_state().active_runs[run.run_id] start.set() await asyncio.wait_for(task, 10) assert not any(e["type"] == "tool_result" for e in published) diff --git a/src/backend/tests/orchestration/test_observation_writer.py b/src/backend/tests/orchestration/test_observation_writer.py new file mode 100644 index 00000000..a8d144ae --- /dev/null +++ b/src/backend/tests/orchestration/test_observation_writer.py @@ -0,0 +1,146 @@ +"""Hook-path observability is written in the background, bounded and lossy.""" + +from __future__ import annotations + +import asyncio +import logging +import threading +import time + +import pytest +from core.db.engine import Base +from core.harness.events import Event, EventSink +from core.harness.hooks import Decision, HookBus, HookSpec, HookStage, Invocation +from core.harness.ledger import DurableEventStore +from core.harness.observation_writer import ObservationWriter, flush_observations +from core.harness.usage import MemoryUsageRecorder +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + + +def _stage(run_id: str = "run-bg") -> Invocation: + return Invocation.create(run_id=run_id, stage=HookStage.BEFORE_MODEL, operation_name="model") + + +@pytest.mark.asyncio +async def test_slow_durable_writes_never_block_the_stage(): + release = threading.Event() + + class SlowRecorder(MemoryUsageRecorder): + def record_attempts(self, attempts): + release.wait(5) + return super().record_attempts(attempts) + + def slow_store(event): + release.wait(5) + return event + + usage = SlowRecorder() + bus = HookBus(event_sink=EventSink(slow_store), usage_recorder=usage) + bus.register(HookSpec("policy", HookStage.BEFORE_MODEL, lambda invocation: None)) + started = time.monotonic() + for _ in range(5): + await bus.run(_stage()) + assert time.monotonic() - started < 0.5 + # The in-memory view is available before anything reaches the store. + assert len(bus.event_sink.events()) == 5 + assert usage.attempts("run-bg") == () + + release.set() + assert flush_observations() + assert len(usage.attempts("run-bg")) == 5 + + +@pytest.mark.asyncio +async def test_failing_writes_do_not_change_the_outcome(caplog): + class BrokenRecorder: + def record_attempts(self, attempts): + raise RuntimeError("database down") + + def broken_store(event): + raise RuntimeError("database down") + + bus = HookBus(event_sink=EventSink(broken_store), usage_recorder=BrokenRecorder()) + bus.register(HookSpec("policy", HookStage.BEFORE_MODEL, lambda invocation: Decision.reject("no"))) + with caplog.at_level(logging.WARNING, logger="core.harness.observation_writer"): + outcome = await bus.run(_stage("run-broken")) + assert flush_observations() + assert outcome.decision.reason == "no" + assert "database down" in caplog.text + + +def test_full_queue_drops_without_blocking(caplog): + writer = ObservationWriter(max_pending=2) + gate = threading.Event() + written = [] + + def write(sink, records): + gate.wait(5) + written.extend(records) + + with caplog.at_level(logging.WARNING, logger="core.harness.observation_writer"): + assert writer.submit("sink", write, "run", ["first"]) + time.sleep(0.05) # the writer thread is now holding "first" + started = time.monotonic() + results = [writer.submit("sink", write, "run", [index]) for index in range(10)] + assert time.monotonic() - started < 0.1 + assert results[:2] == [True, True] and not any(results[2:]) + assert writer.dropped == 8 + assert "Observation queue full" in caplog.text + + gate.set() + assert writer.flush(5) + assert written == ["first", 0, 1] + + +def test_queued_records_are_batched_per_sink_and_run(): + writer = ObservationWriter() + gate = threading.Event() + calls = [] + + def write(sink, records): + gate.wait(5) + calls.append((sink, list(records))) + + writer.submit("a", write, "blocker", ["x"]) + time.sleep(0.05) + for record in range(3): + writer.submit("a", write, "run-1", [record]) + writer.submit("b", write, "run-1", [record]) + writer.submit("a", write, "run-2", ["y"]) + gate.set() + assert writer.flush(5) + assert calls == [ + ("a", ["x"]), + ("a", [0, 1, 2]), + ("b", [0, 1, 2]), + ("a", ["y"]), + ] + + +@pytest.mark.asyncio +async def test_published_events_reach_the_durable_store_in_order(tmp_path): + engine = create_engine(f"sqlite:///{tmp_path / 'events.sqlite'}") + Base.metadata.create_all(engine) + sessions = sessionmaker(bind=engine, expire_on_commit=False) + store = DurableEventStore(sessions) + sink = EventSink(store) + for index in range(4): + published = await sink.publish( + Event.create(run_id="run-events", event_type=f"e{index}", phase="before_run") + ) + assert published.event_seq is None + assert flush_observations() + batch = store.store_many( + [Event.create(run_id="run-events", event_type="late", phase="before_run")] + ) + assert batch[0].event_seq == 5 + from core.db.models import HarnessEventLog + + with sessions() as db: + rows = db.query(HarnessEventLog).order_by(HarnessEventLog.event_seq).all() + assert [(row.event_seq, row.event_type) for row in rows] == [ + (1, "e0"), (2, "e1"), (3, "e2"), (4, "e3"), (5, "late"), + ] + engine.dispose() + await asyncio.sleep(0) diff --git a/src/backend/tests/orchestration/test_pending_workflows.py b/src/backend/tests/orchestration/test_pending_workflows.py new file mode 100644 index 00000000..0863806d --- /dev/null +++ b/src/backend/tests/orchestration/test_pending_workflows.py @@ -0,0 +1,68 @@ +"""Restart between durable admission and local worker registration.""" +import asyncio +from datetime import datetime, timedelta, timezone + +import pytest +from core.db import engine, models +from core.runs import ownership, tasks, runtime +from orchestration import pending +from tests.orchestration.test_background_commands import command_db + + +@pytest.mark.parametrize("kind", ["plan_execute", "autonomous_loop"]) +async def test_pending_admission_restarts_once_but_running_never_replays(command_db, monkeypatch, kind): + monkeypatch.setattr(engine, "SessionLocal", command_db) + real_dispatch = pending.dispatch_pending + async def no_dispatch(run): + pass + monkeypatch.setattr(pending, "dispatch_pending", no_dispatch) + if kind == "plan_execute": + from orchestration.plans import admission, execute_run as module + worker_name = "_run_plan_execute_workflow" + run = await admission.start_plan_execute_run( + plan_id="plan", chat_id="c", user_id="u", + session_messages=[{"role": "user", "content": "original input"}], + enabled_agent_ids=["analyst"], model_name="accepted-model", + ) + else: + from orchestration.loops import admission, run as module + worker_name = "_run_autonomous_loop_workflow" + run = await admission.start_autonomous_loop_run( + loop_id="loop", chat_id="c", user_id="u", + goal_spec={"goal": "original goal"}, budget={"max_iterations": 7}, + worker_max_iters=23, model_name="accepted-model", + ) + executed = [] + async def worker(**args): + if ownership.journal().claim(args["run_id"], owner="worker", + lease_seconds=30, pending_only=True): + executed.append(args) + monkeypatch.setattr(module, worker_name, worker) + monkeypatch.setattr(pending, "dispatch_pending", real_dispatch) + # Fresh DB read represents the successor scheduler, without any local launch state. + with command_db() as db: + accepted = db.get(models.ChatRun, run.run_id) + assert accepted.status == "pending" + assert accepted.recovery_snapshot["initial_args"]["model_name"] == "accepted-model" + accepted = runtime.get_run(run.run_id) + assert await real_dispatch(accepted) + await real_dispatch(accepted) + await asyncio.sleep(0.05) + assert len(executed) == 1 + if kind == "plan_execute": + assert executed[0]["session_messages"][0]["content"] == "original input" + assert executed[0]["enabled_agent_ids"] == ["analyst"] + else: + assert executed[0]["worker_max_iters"] == 23 + assert executed[0]["goal_spec"]["goal"] == "original goal" + # Even a stale pending snapshot cannot replay an already-started execution. + with command_db() as db: + row = db.get(models.ChatRun, run.run_id) + row.lease_expires_at = datetime.now(timezone.utc) - timedelta(seconds=1) + db.commit() + assert not await real_dispatch(accepted) + await asyncio.sleep(0.05) + assert len(executed) == 1 + + assert not ownership.journal().claim(run.run_id, owner="late-worker", + lease_seconds=30, pending_only=True) diff --git a/src/backend/tests/orchestration/test_plan_early_cancel.py b/src/backend/tests/orchestration/test_plan_early_cancel.py new file mode 100644 index 00000000..8dcf526c --- /dev/null +++ b/src/backend/tests/orchestration/test_plan_early_cancel.py @@ -0,0 +1,84 @@ +"""Cancellation before Plan streaming must still settle partial persistence.""" + +import asyncio +from types import SimpleNamespace + +import pytest +from orchestration.plans import execute_run + + +@pytest.mark.asyncio +async def test_user_cancel_at_first_event_persists_interrupted_message(monkeypatch): + messages = [] + committed = [] + expired = [] + journal = SimpleNamespace(claim=lambda *a, **k: True) + + class DB: + def __enter__(self): + return self + + def __exit__(self, *args): + pass + + def get(self, *args): + return None + + def commit(self): + committed.append(True) + + class Offsets: + last = 0 + + def __init__(self, *args, lease_lost, **kwargs): + self.lease_lost = lease_lost + + async def emit(self, event, **kwargs): + self.lease_lost.set() + raise asyncio.CancelledError + + async def heartbeat(*args): + await asyncio.Event().wait() + + async def expire(run_id): + expired.append(run_id) + + from core.llm.runtime import workspace + from core.files.artifacts import service as artifact_service + from core.projects import edition as project_scope + + monkeypatch.setattr(execute_run.ownership, "journal", lambda: journal) + monkeypatch.setattr(execute_run.ownership, "cancelled_by_user", lambda _: True) + monkeypatch.setattr(execute_run.ownership, "lease_heartbeat", heartbeat) + monkeypatch.setattr(execute_run.writer, "EventOffsets", Offsets) + monkeypatch.setattr(execute_run.writer, "expire_stream", expire) + monkeypatch.setattr(execute_run.engine, "SessionLocal", DB) + monkeypatch.setattr( + "core.chat.service.ChatService", + lambda db: SimpleNamespace(add_message=lambda **kwargs: messages.append(kwargs)), + ) + monkeypatch.setattr(artifact_service, "persist_artifacts", lambda *a, **k: None) + monkeypatch.setattr(project_scope, "project_scope_from_chat_id", lambda *a: None) + monkeypatch.setattr(workspace, "init_state", lambda: None) + monkeypatch.setattr(workspace, "get_pinned", lambda: []) + monkeypatch.setattr(workspace, "get_pinned_file_ids", lambda: []) + + await execute_run._run_plan_execute_workflow( + run_id="run", + plan_id="plan", + chat_id="chat", + user_id="user", + message_id="message", + enabled_mcp_ids=[], + enabled_skill_ids=[], + enabled_kb_ids=[], + enabled_agent_ids=[], + session_messages=[], + model_name="test", + journal_owner="owner", + ) + assert len(messages) == 1 + assert "已中断" in messages[0]["content"] + assert messages[0]["extra_data"]["plan_snapshot"]["cancelled"] is True + assert committed == [True] + assert expired == ["run"] diff --git a/src/backend/tests/orchestration/test_plan_progress_closeout.py b/src/backend/tests/orchestration/test_plan_progress_closeout.py index 4429d288..a878c61f 100644 --- a/src/backend/tests/orchestration/test_plan_progress_closeout.py +++ b/src/backend/tests/orchestration/test_plan_progress_closeout.py @@ -14,6 +14,7 @@ **不伪造**步骤状态（停在 2/5 的计划收尾后仍是 2/5）。 """ +import core.chat.runtime.job_wakeup as job_wakeup import core.db.engine as db_engine import pytest from core.db.engine import Base @@ -44,11 +45,10 @@ def db_session(monkeypatch): monkeypatch.setattr(db_engine, "SessionLocal", Session) import core.chat.plan_progress as pp_mod - from orchestration import job_wakeup, workflow + import core.chat.runtime.job_wakeup as job_wakeup monkeypatch.setattr(pp_mod, "SessionLocal", Session, raising=False) monkeypatch.setattr(job_wakeup, "SessionLocal", Session, raising=False) - monkeypatch.setattr(workflow, "SessionLocal", Session, raising=False) with Session() as db: db.add(ChatSession(chat_id="chat_1", user_id="u1", title="批量分类", extra_data={})) @@ -101,7 +101,7 @@ def test_new_user_turn_clears_the_plan(db_session): def test_live_job_blocks_settling(db_session): """作业还在后台跑 → 本轮结束不收尾（交付轮还会来更新它）。""" - from orchestration.workflow import _chat_has_live_job + from core.agent_runtime.events import _chat_has_live_job assert _chat_has_live_job("chat_1") is False with db_session() as db: @@ -129,8 +129,8 @@ def test_live_job_blocks_settling(db_session): def test_finish_wake_prompt_reads_the_checklist_back(db_session): """终态唤醒必须把清单原样念给模型，并把收尾排在交付之前。""" from core.chat.plan_progress import save_plan_progress + from core.chat.runtime.job_wakeup import _wake_prompt from core.db.models import Job as JobModel - from orchestration.job_wakeup import _wake_prompt save_plan_progress("chat_1", PLAN_5) from core.chat.plan_progress import load_plan_progress @@ -146,8 +146,8 @@ def test_finish_wake_prompt_reads_the_checklist_back(db_session): def test_finished_plan_is_not_nagged_about(db_session): """清单已经全部结算（或压根没用过计划栏）→ 别在唤醒轮里凭空要求它调 update_plan。""" + from core.chat.runtime.job_wakeup import _wake_prompt from core.db.models import Job as JobModel - from orchestration.job_wakeup import _wake_prompt job = JobModel(job_id="job_1", name="问答记录分类", status="completed") done_plan = {"title": "x", "steps": [{"title": "a", "status": "completed"}]} diff --git a/src/backend/tests/orchestration/test_plan_stale_reminder.py b/src/backend/tests/orchestration/test_plan_stale_reminder.py index 8d43f1aa..9c0fb21c 100644 --- a/src/backend/tests/orchestration/test_plan_stale_reminder.py +++ b/src/backend/tests/orchestration/test_plan_stale_reminder.py @@ -17,7 +17,13 @@ import pytest -from core.llm.middlewares import _PLAN_REMINDER_INTERVAL, PlanStaleReminderMiddleware +from core.llm.context.workspace_hooks import PlanStaleHook, plan_stale_hook +from core.harness.hooks import Invocation, HookStage +from core.llm.runtime.context_hook_adapter import append_hook_context +from tests.orchestration.hook_support import make_adapter +from types import SimpleNamespace + +_PLAN_REMINDER_INTERVAL = PlanStaleHook().interval class _FakeState: @@ -31,24 +37,19 @@ def __init__(self) -> None: self.state = _FakeState() -class _Block: - def __init__(self, name: str, block_id: str, payload) -> None: - self.name = name - self.id = block_id - self.input = payload - +def _Block(name, block_id, payload): + return {"type": "tool_call", "name": name, "id": block_id, "input": payload} -class _ToolCallMsg: - """伪造一条带 tool_call 块的历史消息。""" - def __init__(self, *blocks: _Block) -> None: - self._blocks = list(blocks) +def _ToolCallMsg(*blocks): + return {"content": list(blocks)} - def has_content_blocks(self, block_type: str) -> bool: - return block_type == "tool_call" and bool(self._blocks) - def get_content_blocks(self, block_type: str): - return self._blocks if block_type == "tool_call" else [] +def _remind(policy, agent): + context = [m.model_dump(mode="json") if hasattr(m, "model_dump") else m for m in agent.state.context] + decision = policy(Invocation.create(run_id="plan-test", stage=HookStage.TRANSFORM_CONTEXT, + operation_name="agent_reasoning", data={"context": context, "context_items": []})) + append_hook_context(agent, SimpleNamespace(data=decision.patch)) def _plan(*statuses: str) -> dict: @@ -74,14 +75,14 @@ def _reminders(agent: _FakeAgent) -> list[str]: return out -def _spin(mw: PlanStaleReminderMiddleware, agent: _FakeAgent, rounds: int) -> None: +def _spin(mw: PlanStaleHook, agent: _FakeAgent, rounds: int) -> None: for _ in range(rounds): - mw._maybe_remind(agent) + _remind(mw, agent) def test_no_plan_never_reminds(): """模型压根没建过计划 —— 催建不是这个中间件的活，一个字都不该发。""" - mw = PlanStaleReminderMiddleware() + mw = PlanStaleHook() agent = _FakeAgent() agent.state.context.append(_ToolCallMsg(_Block("bash", "t1", {"command": "ls"}))) _spin(mw, agent, _PLAN_REMINDER_INTERVAL + 5) @@ -90,7 +91,7 @@ def test_no_plan_never_reminds(): def test_stale_plan_reminds_with_current_checklist(): """停在 1/5 不动满 N 轮 → 回灌一次，且把真实进度和每一步状态都念出来。""" - mw = PlanStaleReminderMiddleware() + mw = PlanStaleHook() agent = _FakeAgent() _plan_call( agent, @@ -109,7 +110,7 @@ def test_stale_plan_reminds_with_current_checklist(): def test_not_yet_stale_stays_quiet(): """没到停滞阈值就不打扰 —— 每轮都催会把上下文塞满、也会打断模型手上的活。""" - mw = PlanStaleReminderMiddleware() + mw = PlanStaleHook() agent = _FakeAgent() _plan_call(agent, "call_1", _plan("in_progress", "pending")) _spin(mw, agent, _PLAN_REMINDER_INTERVAL) # 含起点那一轮，还差一轮 @@ -118,20 +119,20 @@ def test_not_yet_stale_stays_quiet(): def test_model_updating_the_plan_resets_the_clock(): """模型真去更新了 —— 计时清零，不该紧接着再挨一次催。""" - mw = PlanStaleReminderMiddleware() + mw = PlanStaleHook() agent = _FakeAgent() _plan_call(agent, "call_1", _plan("in_progress", "pending", "pending")) _spin(mw, agent, _PLAN_REMINDER_INTERVAL) _plan_call(agent, "call_2", _plan("completed", "in_progress", "pending")) _spin(mw, agent, _PLAN_REMINDER_INTERVAL) assert _reminders(agent) == [] - mw._maybe_remind(agent) + _remind(mw, agent) assert len(_reminders(agent)) == 1 def test_finished_plan_is_left_alone(): """全部结算完 —— 收尾过的清单不该再被催。""" - mw = PlanStaleReminderMiddleware() + mw = PlanStaleHook() agent = _FakeAgent() _plan_call(agent, "call_1", _plan("completed", "completed")) _spin(mw, agent, _PLAN_REMINDER_INTERVAL + 5) @@ -140,7 +141,7 @@ def test_finished_plan_is_left_alone(): def test_json_string_args_are_parsed(): """有的供应商把 tool_call.input 当字符串回传 —— 解析不出来就等于永不催更。""" - mw = PlanStaleReminderMiddleware() + mw = PlanStaleHook() agent = _FakeAgent() _plan_call(agent, "call_1", _plan("in_progress", "pending"), as_json=True) _spin(mw, agent, 1 + _PLAN_REMINDER_INTERVAL) @@ -149,21 +150,21 @@ def test_json_string_args_are_parsed(): def test_reminder_clock_restarts_after_each_nudge(): """催过之后重新计时，不会连着两轮都催。""" - mw = PlanStaleReminderMiddleware() + mw = PlanStaleHook() agent = _FakeAgent() _plan_call(agent, "call_1", _plan("in_progress", "pending")) _spin(mw, agent, 1 + _PLAN_REMINDER_INTERVAL) assert len(_reminders(agent)) == 1 _spin(mw, agent, _PLAN_REMINDER_INTERVAL - 1) assert len(_reminders(agent)) == 1 - mw._maybe_remind(agent) + _remind(mw, agent) assert len(_reminders(agent)) == 2 @pytest.mark.asyncio async def test_on_reasoning_passes_events_through(): """中间件只做旁路提醒，不能吞掉推理事件。""" - mw = PlanStaleReminderMiddleware() + mw = make_adapter(plan_stale_hook()) agent = _FakeAgent() _plan_call(agent, "call_1", _plan("in_progress", "pending")) @@ -181,7 +182,7 @@ async def next_handler(**kwargs): def test_reminder_wording_stays_soft(): """硬祈使会让模型把改计划当成推进任务——文案必须明确允许忽略、且不许打断手上的活。""" - mw = PlanStaleReminderMiddleware() + mw = PlanStaleHook() agent = _FakeAgent() _plan_call(agent, "call_1", _plan("in_progress", "pending")) _spin(mw, agent, 1 + _PLAN_REMINDER_INTERVAL) diff --git a/src/backend/tests/orchestration/test_project_api_recovery.py b/src/backend/tests/orchestration/test_project_api_recovery.py index 3ee3a9a0..56d1587d 100644 --- a/src/backend/tests/orchestration/test_project_api_recovery.py +++ b/src/backend/tests/orchestration/test_project_api_recovery.py @@ -3,22 +3,25 @@ from copy import deepcopy from types import SimpleNamespace +import core.chat.runtime.recovery as runtime_recovery +import core.chat.runtime.recovery_snapshot as recovery_snapshot +import core.db.engine as db_engine +import core.runs.ownership as ownership +import core.runs.runtime as runtime import pytest +from core.db.engine import Base +from core.db.models import ChatRun, ChatSession +from core.runs.api_recovery import validate_recovery_context from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker -from core.db.engine import Base -from core.db.models import ChatSession, ChatRun -from orchestration.agent_api_recovery import validate_recovery_context -from orchestration import chat_run_executor as executor - @pytest.fixture def recovery(tmp_path, monkeypatch): engine = create_engine(f"sqlite:///{tmp_path / 'project-recovery.db'}") Base.metadata.create_all(engine, tables=[ChatSession.__table__, ChatRun.__table__]) sessions = sessionmaker(bind=engine, expire_on_commit=False) - monkeypatch.setattr(executor, "SessionLocal", sessions) + monkeypatch.setattr(db_engine, "SessionLocal", sessions) scope = { "version": 1, "owner_user_id": "owner", @@ -149,9 +152,9 @@ def test_worker_restore_rejects_drift_before_starting_worker(recovery, monkeypat _, _, context = recovery context["project_id"] = "two" monkeypatch.setattr( - executor, "_register_run_task", lambda *a, **k: pytest.fail("must not start") + runtime, 'register_run_task', lambda *a, **k: pytest.fail("must not start") ) - assert executor._register_recovered_chat(decision(context)) is False + assert runtime_recovery._register_recovered_chat(decision(context)) is False @pytest.mark.asyncio @@ -159,8 +162,8 @@ async def test_completed_snapshot_rejects_missing_scope_before_committing(recove _, _, context = recovery parked = [] monkeypatch.setattr( - executor, - "_journal", + ownership, + 'journal', lambda: SimpleNamespace( needs_attention=lambda *a, **k: parked.append(k) or True, claim=lambda *a, **k: pytest.fail("must not claim damaged snapshot"), @@ -175,5 +178,5 @@ async def test_completed_snapshot_rejects_missing_scope_before_committing(recove "project_id": "two", }, ) - assert await executor._commit_recovered_chat_snapshot(damaged) is False + assert await recovery_snapshot._commit_recovered_chat_snapshot(damaged) is False assert parked diff --git a/src/backend/tests/orchestration/test_project_init_stream_input.py b/src/backend/tests/orchestration/test_project_init_stream_input.py index 9bcc3699..6bdaacea 100644 --- a/src/backend/tests/orchestration/test_project_init_stream_input.py +++ b/src/backend/tests/orchestration/test_project_init_stream_input.py @@ -3,9 +3,8 @@ from types import SimpleNamespace import pytest - -from core.llm.context_ir import SESSION_CONTEXT_META_KEY -from orchestration.streaming import StreamingAgent +from core.llm.context.ir_constants import SESSION_CONTEXT_META_KEY +from core.llm.runtime.streaming import StreamingAgent @pytest.mark.asyncio diff --git a/src/backend/tests/orchestration/test_run_event_stream.py b/src/backend/tests/orchestration/test_run_event_stream.py index 2073ac2d..d4a78229 100644 --- a/src/backend/tests/orchestration/test_run_event_stream.py +++ b/src/backend/tests/orchestration/test_run_event_stream.py @@ -6,6 +6,7 @@ """ from __future__ import annotations +from core.chat.projection.codec import chat_projection_codec import asyncio import threading @@ -13,21 +14,16 @@ import fakeredis.aioredis import pytest - -from orchestration.run_event_stream import ( - START, - LocalRunEventStream, - RedisRunEventStream, -) +from core.runs.events.store import START, LocalRunEventStream, RedisRunEventStream @pytest.fixture(params=["local", "redis"]) def stream(request, monkeypatch): if request.param == "local": - return LocalRunEventStream() + return LocalRunEventStream(chat_projection_codec()) client = fakeredis.aioredis.FakeRedis(decode_responses=True, protocol=2) - monkeypatch.setattr("orchestration.run_event_stream.get_redis", lambda **_kwargs: client) - return RedisRunEventStream() + monkeypatch.setattr('core.runs.events.store.get_redis', lambda **_kwargs: client) + return RedisRunEventStream(chat_projection_codec()) async def _seed(stream, run_id, count): @@ -133,7 +129,7 @@ async def test_cursors_stay_ordered_across_many_appends(stream): @pytest.mark.asyncio async def test_local_wait_wakes_from_another_event_loop(): """Sub-agents run on their own loop, so the writer often is not the reader's.""" - stream = LocalRunEventStream() + stream = LocalRunEventStream(chat_projection_codec()) entries = await _seed(stream, "run", 1) def writer(): diff --git a/src/backend/tests/orchestration/test_run_journal.py b/src/backend/tests/orchestration/test_run_journal.py index a35b6de6..aecb212e 100644 --- a/src/backend/tests/orchestration/test_run_journal.py +++ b/src/backend/tests/orchestration/test_run_journal.py @@ -10,12 +10,7 @@ import pytest from core.db.engine import Base from core.db.models import ChatMessage, ChatRun, ChatRunOperation, ChatSession -from core.services.run_journal import ( - RunAlreadyExists, - RunJournal, - RunLeaseLost, - durable_run_binding, -) +from core.runs.journal import RunAlreadyExists, RunJournal, RunLeaseLost, durable_run_binding from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker @@ -576,7 +571,7 @@ async def test_durable_internal_binding_preserves_wrapped_unknown_tool_outcome( journal_env, ): _journal, sessions, _clock = journal_env - from core.services.tool_effect_ledger import ToolOutcomeUnknown + from core.runs.tool_effects.errors import ToolOutcomeUnknown run_id = "" with pytest.raises(ExceptionGroup): diff --git a/src/backend/tests/orchestration/test_run_journal_postgres.py b/src/backend/tests/orchestration/test_run_journal_postgres.py index 8417cb76..0c0c87c1 100644 --- a/src/backend/tests/orchestration/test_run_journal_postgres.py +++ b/src/backend/tests/orchestration/test_run_journal_postgres.py @@ -12,9 +12,7 @@ from uuid import uuid4 import pytest -from sqlalchemy import create_engine, inspect, text -from sqlalchemy.orm import sessionmaker - +from core.chat.runtime.handoff import _commit_queued_handoff_in_session from core.db.models import ( ChatMessage, ChatRun, @@ -24,11 +22,12 @@ ToolEffectLedger, UserShadow, ) -from core.services.chat_service import ChatService -from core.services.run_journal import RunJournal -from core.services.steer_queue import SteerQueue, SteerQueueConflict -from core.services.tool_effect_ledger import ToolEffectJournal -from orchestration.chat_run_executor import _commit_queued_handoff_in_session +from core.chat.service import ChatService +from core.runs.journal import RunJournal +from core.chat.steer_queue import SteerQueue, SteerQueueConflict +from core.runs.tool_effects.journal import ToolEffectJournal +from sqlalchemy import create_engine, inspect, text +from sqlalchemy.orm import sessionmaker POSTGRES_URL = os.getenv("RUN_JOURNAL_POSTGRES_URL", "").strip() pytestmark = pytest.mark.skipif( diff --git a/src/backend/tests/orchestration/test_runtime_phase_lifecycle.py b/src/backend/tests/orchestration/test_runtime_phase_lifecycle.py new file mode 100644 index 00000000..823e405c --- /dev/null +++ b/src/backend/tests/orchestration/test_runtime_phase_lifecycle.py @@ -0,0 +1,203 @@ +"""Exercise real preparation and stream adapters at consumer exit boundaries.""" +from core.agent_runtime.integrations import memory_identity as memory_identity_module +from core.agent_runtime.integrations import memory_session as memory_session_module +from core.ontology import output_review_state as output_review_state_module +from core.ontology import reviewer as reviewer_module +from core.llm.runtime import streaming as streaming_module +from core.llm.citations import allocator as allocator_module +from core.harness import log_writer + +import asyncio +from types import SimpleNamespace + +import pytest +from core.agent_runtime import budget, runner, targets +from core.agent_runtime.integrations import evolution, memory +from core.capabilities.catalog import resolver as catalog_resolver +from core.llm.citations import anchor as citation_anchor +from core.llm import factory +from core.llm.runtime import streaming +from core.ontology import reviewer, validator +from core.harness import log_writer as log_service + + +@pytest.fixture +def runtime(monkeypatch): + observed = SimpleNamespace( + messages=None, + created=0, + closed=0, + stream_closed=0, + claimed=False, + logs=[], + review_started=0, + ) + + async def empty(*args, **kwargs): + return "" + + async def session(**kwargs): + return memory_session_module.SessionMemory( + chat_id="", scope_user_id="", workspace_id="default", memory_enabled=False + ) + + async def create(**kwargs): + observed.created += 1 + return ( + SimpleNamespace( + state=SimpleNamespace(ontology_runtime=kwargs["ontology_runtime"]), model=None + ), + [], + ) + + class Adapter: + def __init__(self, agent, clients): + self.agent = agent + + async def stream(self, messages, context, **kwargs): + observed.messages = messages + try: + yield "text_delta", "Draft" + yield "user_question", {"id": "question"} + finally: + observed.stream_closed += 1 + + async def shutdown(self): + observed.closed += 1 + + async def aget_usage(self): + return {} + + def get_context_usage(self, usage): + return {} + + def claim(runtime, **kwargs): + observed.claimed = True + return True + + def release(runtime, **kwargs): + observed.claimed = False + + async def review(**kwargs): + observed.review_started += 1 + return {"verdict": "pass", "answer": kwargs["answer"], "attempts": 1} + + async def finish(*args, **kwargs): + observed.logs.append(kwargs["status"]) + + monkeypatch.setattr(factory, "create_agent_executor", create) + monkeypatch.setattr(streaming_module, 'StreamingAgent', Adapter) + monkeypatch.setattr(memory_session_module, 'open_session_memory', session) + monkeypatch.setattr(memory_session_module, 'resolve_session_memory', empty) + monkeypatch.setattr(memory_identity_module, 'build_user_identity_block', empty) + monkeypatch.setattr(allocator_module, 'anchor_start_for_chat', lambda *_: 0) + monkeypatch.setattr(targets, "_resolve_mode_spec", lambda *_: None) + monkeypatch.setattr(targets, "_runtime_visible_subagents", lambda *_: []) + monkeypatch.setattr( + targets, + "_load_direct_user_agent", + lambda *_: SimpleNamespace(name="Direct", extra_config={}), + ) + for name in ( + "enabled_skill_ids_from_context", + "enabled_mcp_ids_from_context", + "enabled_kb_ids_from_context", + ): + monkeypatch.setattr(catalog_resolver, name, lambda *_: []) + monkeypatch.setattr(budget, "_resolve_agent_model_runtime", lambda *_: ("test", 32000)) + monkeypatch.setattr(budget, "_compaction_budget_inputs", lambda *_: {}) + monkeypatch.setattr(output_review_state_module, 'claim_output_review', claim) + monkeypatch.setattr(output_review_state_module, 'release_output_review', release) + monkeypatch.setattr(output_review_state_module, 'complete_output_review', release) + monkeypatch.setattr(reviewer_module, 'review_ontology_output', review) + monkeypatch.setattr(log_writer, 'start_subagent_log', empty) + monkeypatch.setattr(log_writer, 'finish_subagent_log', finish) + monkeypatch.setattr(evolution, "_assemble_episode_background", lambda **_: None) + return observed + + +@pytest.mark.asyncio +@pytest.mark.parametrize("direct", [False, True]) +@pytest.mark.parametrize("stop", ["thinking", "user_question", "ontology_review", "meta", None]) +async def test_real_phases_close_stream_and_review_claim(runtime, direct, stop): + context = {"user_id": "phase-user", "memory_enabled": False} + if direct: + context["agent_id"] = "phase-agent" + stream = runner.astream_chat_workflow( + session_messages=[{"role": "user", "content": "Question"}], + user_message="Question", + context=context, + ) + events = [] + try: + async for event in stream: + events.append(event) + if event["type"] == stop: + break + finally: + await stream.aclose() + assert not runtime.claimed + if stop == "thinking": + assert runtime.created == 0 + assert runtime.closed == 0 + else: + assert runtime.created == 1 + assert runtime.messages[-1]["content"] == "Question" + assert runtime.stream_closed == 1 + assert runtime.closed >= 1 + if stop == "ontology_review": + assert runtime.review_started == 0 + assert events[-1]["status"] == "started" + if direct: + assert len(runtime.logs) == 1 + assert runtime.logs[0] == ("success" if stop in ("meta", None) else "cancelled") + if stop is None: + assert any(event["type"] == "meta" for event in events) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("direct", [False, True]) +async def test_cancel_while_review_is_running_joins_task(runtime, monkeypatch, direct): + started = asyncio.Event() + stopped = asyncio.Event() + + async def blocking_review(**kwargs): + started.set() + try: + await asyncio.Event().wait() + finally: + stopped.set() + + monkeypatch.setattr(reviewer_module, 'review_ontology_output', blocking_review) + context = {"user_id": "phase-user", "memory_enabled": False} + if direct: + context["agent_id"] = "phase-agent" + + async def consume(): + stream = runner.astream_chat_workflow( + session_messages=[{"role": "user", "content": "Question"}], + user_message="Question", + context=context, + ) + try: + async for _ in stream: + pass + finally: + await stream.aclose() + + consumer = asyncio.create_task(consume()) + try: + await asyncio.wait_for(started.wait(), 2) + consumer.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(consumer, 2) + assert stopped.is_set() + assert not runtime.claimed + assert runtime.stream_closed == 1 + assert runtime.closed >= 1 + if direct: + assert runtime.logs == ["cancelled"] + finally: + if not consumer.done(): + consumer.cancel() + await asyncio.gather(consumer, return_exceptions=True) diff --git a/src/backend/tests/orchestration/test_site_mode_prompt.py b/src/backend/tests/orchestration/test_site_mode_prompt.py index 0d67ca25..0f7e640d 100644 --- a/src/backend/tests/orchestration/test_site_mode_prompt.py +++ b/src/backend/tests/orchestration/test_site_mode_prompt.py @@ -3,8 +3,8 @@ import inspect from pathlib import Path -import core.services.session_operations.request_context as chat_request_context import core.chat.context as chat_context +import core.chat.operations.request_context as chat_request_context from core.chat.schemas import ChatRequest from core.llm.factory import create_agent_executor @@ -12,13 +12,14 @@ def test_site_flag_is_metadata_not_an_agent_prompt_switch(monkeypatch) -> None: monkeypatch.setattr(chat_context, "collect_historical_attachments", lambda **_kwargs: []) request = ChatRequest(chat_id="chat-1", message="做个公司官网", site_chat=True) - ctx = chat_request_context._build_ctx(request, "user-1", None, None, None) + ctx = chat_request_context.build_ctx(request, "user-1", None, None, None) assert ctx["site_chat"] is True assert "site_mode" not in inspect.signature(create_agent_executor).parameters - workflow_src = Path(__file__).resolve().parents[2].joinpath("orchestration/workflow.py") - assert "site_mode=" not in workflow_src.read_text() + runtime_dir = Path(__file__).resolve().parents[2] / "core/agent_runtime" + for module in runtime_dir.rglob("*.py"): + assert "site_mode=" not in module.read_text(), module def test_user_message_carries_no_site_rules() -> None: @@ -56,7 +57,7 @@ def test_history_repair_keeps_only_what_the_user_typed() -> None: for row in rows: conn.execute( text( - "INSERT INTO chat_messages (message_id, role, content) " "VALUES (:a, :b, :c)" + "INSERT INTO chat_messages (message_id, role, content) " "VALUES (:a, :b, :c)" ), dict(zip("abc", row)), ) diff --git a/src/backend/tests/orchestration/test_stale_reaper.py b/src/backend/tests/orchestration/test_stale_reaper.py index 3876f110..e68b9692 100644 --- a/src/backend/tests/orchestration/test_stale_reaper.py +++ b/src/backend/tests/orchestration/test_stale_reaper.py @@ -1,5 +1,6 @@ """Regression tests for the stale reaper's liveness awareness + terminal-state CAS. + Corresponds to a production incident: age-only reaping killed a 35-minute long task that was still running tools (the run was marked failed and the SSE was terminated), while the worker itself was not cancelled and kept running for another 1.5h, overwriting the terminal state @@ -9,16 +10,22 @@ import asyncio from datetime import datetime, timedelta, timezone +import core.db.engine as db_engine +import core.llm as llm +import core.llm.runtime.human_interaction as human_interaction +import core.runs.events.store as run_event_stream +import core.runs.ownership as ownership +import core.runs.runtime as runtime +import core.runs.settings as settings +import core.runs.tasks as tasks +import core.runs.watchdog as watchdog import pytest +from core.db.engine import Base +from core.db.models import ChatRun from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker from sqlalchemy.pool import StaticPool -from core.db.engine import Base -from core.db.models import ChatRun -import orchestration.chat_run_executor as executor -import orchestration.run_event_stream as run_event_stream - # ─── fakes / fixtures ────────────────────────────────────────────────── @@ -54,8 +61,8 @@ def reaper_env(monkeypatch): session_factory = sessionmaker(bind=engine) fake_redis = FakeRedis() monkeypatch.setattr(run_event_stream, "redis_configured", lambda: True) - monkeypatch.setattr(executor, "SessionLocal", session_factory) - monkeypatch.setattr("orchestration.run_event_stream.get_redis", lambda **_: fake_redis) + monkeypatch.setattr(db_engine, "SessionLocal", session_factory) + monkeypatch.setattr('core.runs.events.store.get_redis', lambda **_: fake_redis) yield session_factory, fake_redis engine.dispose() @@ -107,25 +114,25 @@ def _now_ms() -> int: async def test_active_over_age_run_survives(reaper_env): """An over-age long task whose stream is still producing must not be reaped (production incident scenario).""" session_factory, fake_redis = reaper_env - _insert_run(session_factory, "run_active", age_sec=executor._STALE_RUN_MAX_AGE_SEC + 300) + _insert_run(session_factory, "run_active", age_sec=settings._STALE_RUN_MAX_AGE_SEC + 300) fake_redis.seed( run_event_stream.redis_stream_key("run_active"), _now_ms() - 5_000 ) # just wrote 5s ago - assert await executor.reap_stale_runs() == 0 + assert await watchdog.reap_stale_runs() == 0 assert _get_run(session_factory, "run_active").status == "running" async def test_quiet_over_age_run_is_reaped(reaper_env): """Over-age and stream silent past the threshold → reaped to failed and a termination marker written.""" session_factory, fake_redis = reaper_env - _insert_run(session_factory, "run_quiet", age_sec=executor._STALE_RUN_MAX_AGE_SEC + 300) + _insert_run(session_factory, "run_quiet", age_sec=settings._STALE_RUN_MAX_AGE_SEC + 300) fake_redis.seed( run_event_stream.redis_stream_key("run_quiet"), - _now_ms() - int(executor._STALE_QUIET_SEC * 1000) - 60_000, + _now_ms() - int(settings._STALE_QUIET_SEC * 1000) - 60_000, ) - assert await executor.reap_stale_runs() == 1 + assert await watchdog.reap_stale_runs() == 1 run = _get_run(session_factory, "run_quiet") assert run.status == "failed" assert "stalled" in run.error_message @@ -136,19 +143,19 @@ async def test_quiet_over_age_run_is_reaped(reaper_env): async def test_over_age_run_without_stream_is_reaped(reaper_env): """Over-age with no stream (worker never wrote / redis lost it) → treated as a zombie and reaped.""" session_factory, _ = reaper_env - _insert_run(session_factory, "run_nostream", age_sec=executor._STALE_RUN_MAX_AGE_SEC + 300) + _insert_run(session_factory, "run_nostream", age_sec=settings._STALE_RUN_MAX_AGE_SEC + 300) - assert await executor.reap_stale_runs() == 1 + assert await watchdog.reap_stale_runs() == 1 assert _get_run(session_factory, "run_nostream").status == "failed" async def test_hard_max_age_reaps_even_active_run(reaper_env): """Past the absolute lifetime cap, force-reap even if the stream is still active.""" session_factory, fake_redis = reaper_env - _insert_run(session_factory, "run_forever", age_sec=executor._HARD_MAX_AGE_SEC + 300) + _insert_run(session_factory, "run_forever", age_sec=settings._HARD_MAX_AGE_SEC + 300) fake_redis.seed(run_event_stream.redis_stream_key("run_forever"), _now_ms() - 1_000) - assert await executor.reap_stale_runs() == 1 + assert await watchdog.reap_stale_runs() == 1 run = _get_run(session_factory, "run_forever") assert run.status == "failed" assert "hard max age" in run.error_message @@ -158,11 +165,11 @@ async def test_hard_max_age_preserves_bounded_human_wait(reaper_env, monkeypatch """A question opened near the run hard cap keeps its own full wait window.""" session_factory, fake_redis = reaper_env - _insert_run(session_factory, "run_human_wait", age_sec=executor._HARD_MAX_AGE_SEC + 300) + _insert_run(session_factory, "run_human_wait", age_sec=settings._HARD_MAX_AGE_SEC + 300) fake_redis.seed(run_event_stream.redis_stream_key("run_human_wait"), _now_ms() - 1_000) - monkeypatch.setattr(executor.human_interaction, "has_pending", lambda chat_id: bool(chat_id)) + monkeypatch.setattr(human_interaction, "has_pending", lambda chat_id: bool(chat_id)) - assert await executor.reap_stale_runs() == 0 + assert await watchdog.reap_stale_runs() == 0 assert _get_run(session_factory, "run_human_wait").status == "running" @@ -170,7 +177,7 @@ async def test_young_run_untouched(reaper_env): session_factory, _ = reaper_env _insert_run(session_factory, "run_young", age_sec=60) - assert await executor.reap_stale_runs() == 0 + assert await watchdog.reap_stale_runs() == 0 assert _get_run(session_factory, "run_young").status == "running" @@ -180,11 +187,11 @@ async def test_valid_cross_process_lease_shields_a_quiet_run(reaper_env): _insert_run( session_factory, "run_remote", - age_sec=executor._STALE_RUN_MAX_AGE_SEC + 300, + age_sec=settings._STALE_RUN_MAX_AGE_SEC + 300, lease_seconds=60, ) - assert await executor.reap_stale_runs() == 0 + assert await watchdog.reap_stale_runs() == 0 run = _get_run(session_factory, "run_remote") assert run.status == "running" assert run.lease_owner == "remote-worker" @@ -197,20 +204,20 @@ async def test_reaper_cas_cannot_kill_a_worker_claimed_after_candidate_scan( _insert_run( session_factory, "run_claim_race", - age_sec=executor._STALE_RUN_MAX_AGE_SEC + 300, + age_sec=settings._STALE_RUN_MAX_AGE_SEC + 300, ) async def claim_during_activity_probe(_run_id): - assert executor._journal().claim( + assert ownership.journal().claim( "run_claim_race", owner="new-worker", lease_seconds=60, ) return None - monkeypatch.setattr(executor, "_stream_last_write_ms", claim_during_activity_probe) + monkeypatch.setattr(watchdog, "_stream_last_write_ms", claim_during_activity_probe) - assert await executor.reap_stale_runs() == 0 + assert await watchdog.reap_stale_runs() == 0 run = _get_run(session_factory, "run_claim_race") assert run.status == "running" assert run.lease_owner == "new-worker" @@ -225,12 +232,12 @@ async def test_quiet_over_age_run_with_live_task_survives(reaper_env, monkeypatc whose model was streaming huge tool-call args (which map to no stream events); the quiet check now applies only to orphans.""" session_factory, _ = reaper_env - _insert_run(session_factory, "run_local", age_sec=executor._STALE_RUN_MAX_AGE_SEC + 300) + _insert_run(session_factory, "run_local", age_sec=settings._STALE_RUN_MAX_AGE_SEC + 300) task = asyncio.create_task(asyncio.sleep(3600)) - monkeypatch.setitem(executor._active_runs, "run_local", task) + monkeypatch.setitem(tasks.process_state().active_runs, "run_local", task) - assert await executor.reap_stale_runs() == 0 + assert await watchdog.reap_stale_runs() == 0 assert _get_run(session_factory, "run_local").status == "running" assert not task.cancelled() task.cancel() @@ -264,65 +271,65 @@ async def test_quiet_orphan_with_live_job_survives(reaper_env): run 杀了）。作业活性是跨进程可见的证据，必须能救下这个 run。 """ session_factory, _ = reaper_env - _insert_run(session_factory, "run_job_wait", age_sec=executor._STALE_RUN_MAX_AGE_SEC + 300) + _insert_run(session_factory, "run_job_wait", age_sec=settings._STALE_RUN_MAX_AGE_SEC + 300) _insert_job( session_factory, "job_live", chat_id="chat_test", status="running", updated_age_sec=5 ) - assert await executor.reap_stale_runs() == 0 + assert await watchdog.reap_stale_runs() == 0 assert _get_run(session_factory, "run_job_wait").status == "running" async def test_quiet_orphan_with_stalled_job_is_reaped(reaper_env): """作业本身也不动了（updated_at 落在静默窗口之外）→ 不是活性证据，照常回收。""" session_factory, _ = reaper_env - _insert_run(session_factory, "run_job_dead", age_sec=executor._STALE_RUN_MAX_AGE_SEC + 300) + _insert_run(session_factory, "run_job_dead", age_sec=settings._STALE_RUN_MAX_AGE_SEC + 300) _insert_job( session_factory, "job_stalled", chat_id="chat_test", status="running", - updated_age_sec=executor._STALE_QUIET_SEC + 120, + updated_age_sec=settings._STALE_QUIET_SEC + 120, ) - assert await executor.reap_stale_runs() == 1 + assert await watchdog.reap_stale_runs() == 1 assert _get_run(session_factory, "run_job_dead").status == "failed" async def test_terminal_job_does_not_shield_run(reaper_env): """作业已终态 → 不再是活性证据，run 该回收就回收（防止豁免变成永生通行证）。""" session_factory, _ = reaper_env - _insert_run(session_factory, "run_job_done", age_sec=executor._STALE_RUN_MAX_AGE_SEC + 300) + _insert_run(session_factory, "run_job_done", age_sec=settings._STALE_RUN_MAX_AGE_SEC + 300) _insert_job( session_factory, "job_done", chat_id="chat_test", status="completed", updated_age_sec=5 ) - assert await executor.reap_stale_runs() == 1 + assert await watchdog.reap_stale_runs() == 1 assert _get_run(session_factory, "run_job_done").status == "failed" async def test_orphan_with_done_task_is_reaped(reaper_env, monkeypatch): """A finished task left in _active_runs does not shield the run: quiet orphans are reaped.""" session_factory, _ = reaper_env - _insert_run(session_factory, "run_done", age_sec=executor._STALE_RUN_MAX_AGE_SEC + 300) + _insert_run(session_factory, "run_done", age_sec=settings._STALE_RUN_MAX_AGE_SEC + 300) task = asyncio.create_task(asyncio.sleep(0)) await task # completed task still parked in the registry - monkeypatch.setitem(executor._active_runs, "run_done", task) + monkeypatch.setitem(tasks.process_state().active_runs, "run_done", task) - assert await executor.reap_stale_runs() == 1 + assert await watchdog.reap_stale_runs() == 1 assert _get_run(session_factory, "run_done").status == "failed" async def test_hard_expired_cancels_local_worker_task(reaper_env, monkeypatch): """Past the absolute lifetime cap, even a live in-process task is reaped and cancelled.""" session_factory, _ = reaper_env - _insert_run(session_factory, "run_local_hard", age_sec=executor._HARD_MAX_AGE_SEC + 300) + _insert_run(session_factory, "run_local_hard", age_sec=settings._HARD_MAX_AGE_SEC + 300) task = asyncio.create_task(asyncio.sleep(3600)) - monkeypatch.setitem(executor._active_runs, "run_local_hard", task) + monkeypatch.setitem(tasks.process_state().active_runs, "run_local_hard", task) - assert await executor.reap_stale_runs() == 1 + assert await watchdog.reap_stale_runs() == 1 await asyncio.sleep(0) # let the cancel propagate assert task.cancelled() @@ -333,17 +340,17 @@ async def test_hard_expired_leaves_plan_execute_task_to_cooperative_stop(reaper_ _insert_run( session_factory, "run_plan", - age_sec=executor._HARD_MAX_AGE_SEC + 300, + age_sec=settings._HARD_MAX_AGE_SEC + 300, kind="plan_execute", ) task = asyncio.create_task(asyncio.sleep(3600)) - monkeypatch.setitem(executor._active_runs, "run_plan", task) + monkeypatch.setitem(tasks.process_state().active_runs, "run_plan", task) - assert await executor.reap_stale_runs() == 1 + assert await watchdog.reap_stale_runs() == 1 assert not task.cancelled() # DB already declared dead → a cooperative worker polling is_run_cancelled must get True - assert executor.is_run_cancelled("run_plan") is True + assert runtime.is_run_cancelled("run_plan") is True task.cancel() @@ -353,10 +360,10 @@ async def test_hard_expired_leaves_plan_execute_task_to_cooperative_stop(reaper_ async def test_late_worker_completion_cannot_overwrite_reaped_run(reaper_env): """Second half of the incident: after reaping, a worker's late completed write must be rejected.""" session_factory, _ = reaper_env - _insert_run(session_factory, "run_late", age_sec=executor._STALE_RUN_MAX_AGE_SEC + 300) - assert await executor.reap_stale_runs() == 1 + _insert_run(session_factory, "run_late", age_sec=settings._STALE_RUN_MAX_AGE_SEC + 300) + assert await watchdog.reap_stale_runs() == 1 - won = executor._finalize_run( + won = ownership._finalize_run( "run_late", status="completed", completed_at=datetime.now(timezone.utc), @@ -371,7 +378,7 @@ async def test_finalize_run_wins_on_live_run(reaper_env): session_factory, _ = reaper_env _insert_run(session_factory, "run_live", age_sec=10) - assert executor._finalize_run("run_live", status="completed") is True + assert ownership._finalize_run("run_live", status="completed") is True assert _get_run(session_factory, "run_live").status == "completed" @@ -386,14 +393,14 @@ async def test_hard_expired_loop_with_live_task_survives(reaper_env, monkeypatch _insert_run( session_factory, "run_loop_live", - age_sec=executor._HARD_MAX_AGE_SEC + 3600, + age_sec=settings._HARD_MAX_AGE_SEC + 3600, kind="autonomous_loop", ) task = asyncio.create_task(asyncio.sleep(3600)) - monkeypatch.setitem(executor._active_runs, "run_loop_live", task) + monkeypatch.setitem(tasks.process_state().active_runs, "run_loop_live", task) - assert await executor.reap_stale_runs() == 0 + assert await watchdog.reap_stale_runs() == 0 assert _get_run(session_factory, "run_loop_live").status == "running" assert not task.cancelled() task.cancel() @@ -405,20 +412,20 @@ async def test_hard_expired_orphan_loop_reaped_by_quiet_rule_only(reaper_env): _insert_run( session_factory, "run_loop_orphan", - age_sec=executor._HARD_MAX_AGE_SEC + 3600, + age_sec=settings._HARD_MAX_AGE_SEC + 3600, kind="autonomous_loop", ) # 流最近还在写 → 不是僵尸，跳过 fake_redis.seed(run_event_stream.redis_stream_key("run_loop_orphan"), _now_ms() - 5_000) - assert await executor.reap_stale_runs() == 0 + assert await watchdog.reap_stale_runs() == 0 assert _get_run(session_factory, "run_loop_orphan").status == "running" # 流静默超阈值 → 按「无活动」清理（而非年龄硬顶），错误文案是 stalled fake_redis.seed( run_event_stream.redis_stream_key("run_loop_orphan"), - _now_ms() - int(executor._STALE_QUIET_SEC * 1000) - 60_000, + _now_ms() - int(settings._STALE_QUIET_SEC * 1000) - 60_000, ) - assert await executor.reap_stale_runs() == 1 + assert await watchdog.reap_stale_runs() == 1 run = _get_run(session_factory, "run_loop_orphan") assert run.status == "failed" assert "stalled" in run.error_message @@ -434,4 +441,4 @@ async def test_is_run_cancelled_true_for_any_terminal_status(reaper_env): ]: rid = f"run_st_{status}" _insert_run(session_factory, rid, status=status) - assert executor.is_run_cancelled(rid) is expected + assert runtime.is_run_cancelled(rid) is expected diff --git a/src/backend/tests/orchestration/test_steer_queue.py b/src/backend/tests/orchestration/test_steer_queue.py index 1f8eda28..00dbde56 100644 --- a/src/backend/tests/orchestration/test_steer_queue.py +++ b/src/backend/tests/orchestration/test_steer_queue.py @@ -7,14 +7,13 @@ from threading import Barrier import pytest -from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker - +from core.chat.runtime.handoff import _commit_queued_handoff_in_session from core.db.engine import Base from core.db.models import ChatMessage, ChatRun, ChatSession, ChatSteerQueueItem -from core.services.run_journal import RunJournal -from core.services.steer_queue import SteerClaimLost, SteerQueue, SteerQueueConflict -from orchestration.chat_run_executor import _commit_queued_handoff_in_session +from core.runs.journal import RunJournal +from core.chat.steer_queue import SteerClaimLost, SteerQueue, SteerQueueConflict +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker class Clock: @@ -357,7 +356,7 @@ def commit_effect(db): assert queued_messages[0].content == item.message assert handoff["user_message_id"] == queued_messages[0].message_id - from core.services import chat_steer_service + from core.chat import steer as chat_steer_service monkeypatch.setattr(chat_steer_service, "SessionLocal", sessions) payload = chat_steer_service.list_run_steers("run-1")[0] diff --git a/src/backend/tests/orchestration/test_structured_reasoning_protocol.py b/src/backend/tests/orchestration/test_structured_reasoning_protocol.py index d1dce6b1..82955d88 100644 --- a/src/backend/tests/orchestration/test_structured_reasoning_protocol.py +++ b/src/backend/tests/orchestration/test_structured_reasoning_protocol.py @@ -3,7 +3,7 @@ from types import SimpleNamespace from core.chat.tool_log import build_thinking_event -from orchestration.streaming import StreamingAgent +from core.llm.runtime.streaming import StreamingAgent def test_streaming_agent_emits_structured_reasoning_protocol_once(): diff --git a/src/backend/tests/orchestration/test_thinking_block_persistence.py b/src/backend/tests/orchestration/test_thinking_block_persistence.py index b8a09675..5e0396d5 100644 --- a/src/backend/tests/orchestration/test_thinking_block_persistence.py +++ b/src/backend/tests/orchestration/test_thinking_block_persistence.py @@ -14,12 +14,16 @@ import asyncio import contextlib +import core.agent_runtime.runner as runner +import core.chat.runtime.admission as admission +import core.db.engine as db_engine +import core.runs.tasks as tasks import fakeredis.aioredis import pytest from core.db.engine import Base from core.db.models import ChatMessage, ChatSession -from core.services.chat_service import _CONTENT_MAX, _THINKING_MAX, clamp_thinking -from orchestration import chat_run_executor as executor +from core.chat.limits import CONTENT_MAX as _CONTENT_MAX, THINKING_MAX as _THINKING_MAX +from core.chat.limits import clamp_thinking from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker @@ -32,21 +36,21 @@ def run_env(tmp_path, monkeypatch): ) Base.metadata.create_all(engine) sessions = sessionmaker(bind=engine, expire_on_commit=False) - monkeypatch.setattr(executor, "SessionLocal", sessions) + monkeypatch.setattr(db_engine, "SessionLocal", sessions) redis = fakeredis.aioredis.FakeRedis(decode_responses=True) - monkeypatch.setattr("orchestration.run_event_stream.get_redis", lambda **_: redis) - executor._active_runs.clear() + monkeypatch.setattr('core.runs.events.store.get_redis', lambda **_: redis) + tasks.process_state().active_runs.clear() with sessions() as db: db.add(ChatSession(chat_id="chat-1", user_id="user-1", title="test")) db.commit() yield sessions - executor._active_runs.clear() + tasks.process_state().active_runs.clear() engine.dispose() async def _run_to_completion(workflow, monkeypatch) -> str: - monkeypatch.setattr(executor, "astream_chat_workflow", workflow) - run = await executor.start_run( + monkeypatch.setattr(runner, "astream_chat_workflow", workflow) + run = await admission.start_run( chat_id="chat-1", user_id="user-1", session_messages=[{"role": "user", "content": "hello"}], @@ -56,7 +60,7 @@ async def _run_to_completion(workflow, monkeypatch) -> str: request_payload={"message": "hello"}, model_name="test-model", ) - worker = executor._active_runs.get(run.run_id) + worker = tasks.process_state().active_runs.get(run.run_id) if worker is not None: with contextlib.suppress(asyncio.TimeoutError, asyncio.CancelledError): await asyncio.wait_for(worker, timeout=10) diff --git a/src/backend/tests/orchestration/test_thinking_suppression.py b/src/backend/tests/orchestration/test_thinking_suppression.py index c8518453..ec8b65d0 100644 --- a/src/backend/tests/orchestration/test_thinking_suppression.py +++ b/src/backend/tests/orchestration/test_thinking_suppression.py @@ -5,7 +5,7 @@ 所以模型仍然产出思考链）。抑制一旦漏，思考过程就直接落进自动化任务的输出正文。 """ -from orchestration.streaming import _strip_thinking_answer +from core.llm.runtime.stream_text import strip_thinking_answer as _strip_thinking_answer def strip(raw: str, in_thinking: bool = False): diff --git a/src/backend/tests/orchestration/test_tool_effect_ledger.py b/src/backend/tests/orchestration/test_tool_effect_ledger.py index faff603b..8e0b22cb 100644 --- a/src/backend/tests/orchestration/test_tool_effect_ledger.py +++ b/src/backend/tests/orchestration/test_tool_effect_ledger.py @@ -1,101 +1,58 @@ -"""Durable, idempotent tool side-effect execution and recovery.""" - -from __future__ import annotations - -import asyncio -import json -from concurrent.futures import ThreadPoolExecutor -from datetime import datetime, timedelta, timezone -from types import SimpleNamespace -from threading import Barrier - -import pytest -from agentscope.message import TextBlock, ToolCallBlock, ToolResultState -from agentscope.message import UserMsg -from agentscope.agent import Agent, ReActConfig -from agentscope.model import ChatResponse, ChatUsage -from agentscope.permission import PermissionContext, PermissionMode -from agentscope.tool import FunctionTool, Toolkit -from agentscope.tool._response import ToolChunk, ToolResponse -from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker -from sqlalchemy.pool import QueuePool - -from core.db.engine import Base -from core.db.models import ( +from tests.orchestration.tool_effect_support import ( + log_records, + asyncio, + json, + ThreadPoolExecutor, + datetime, + timedelta, + timezone, + Barrier, + SimpleNamespace, + pytest, + Agent, + ReActConfig, + TextBlock, + ToolCallBlock, + ToolResultState, + UserMsg, + ChatResponse, + ChatUsage, + PermissionContext, + PermissionMode, + FunctionTool, + Toolkit, + ToolChunk, + ToolResponse, + Base, ChatRun, ChatSession, - ScheduledTask, ToolCallLog, ToolEffectLease, ToolEffectLedger, ToolEffectReceipt, -) -from core.services.run_journal import RunJournal -from core.services.tool_effect_ledger import ( + ScheduledTask, + AgentRuntimeState, + make_adapter, + durable_adapter, + _plain_replay_args, + reconcile_scheduled_task_intent, + RunJournal, DEFAULT_TOOL_RECOVERY_REGISTRY, + ToolRecoveryRegistry, ToolEffectError, - ToolEffectGateway, - ToolEffectJournal, ToolIdempotencyConflict, - ToolIntent, ToolIntentCommitError, ToolOutcomeUnknown, - ToolRecoveryRegistry, + ToolEffectGateway, + ToolEffectJournal, + ToolIntent, recover_incomplete_tool_effects, + create_engine, + sessionmaker, + QueuePool, + MutableClock, + effect_env, ) -from core.llm.middlewares import AgentRuntimeState, ToolEffectMiddleware -from orchestration.tool_effect_recovery import ( - _plain_replay_args, - reconcile_scheduled_task_intent, -) - - -class MutableClock: - def __init__(self) -> None: - self.now = datetime.now(timezone.utc) - - def __call__(self) -> datetime: - return self.now - - def advance(self, seconds: float) -> None: - self.now += timedelta(seconds=seconds) - - -@pytest.fixture() -def effect_env(tmp_path): - engine = create_engine( - f"sqlite:///{tmp_path / 'tool-effects.sqlite'}", - connect_args={"check_same_thread": False}, - ) - Base.metadata.create_all(engine) - sessions = sessionmaker(bind=engine, expire_on_commit=False) - - def make_run( - run_id: str, - *, - owner: str = "worker", - user_id: str = "user-1", - chat_id: str = "chat-1", - ) -> None: - with sessions() as db: - if db.get(ChatSession, chat_id) is None: - db.add(ChatSession(chat_id=chat_id, user_id=user_id, title="test")) - db.commit() - journal = RunJournal(sessions) - journal.accept( - run_id=run_id, - message_id=f"msg-{run_id}", - chat_id=chat_id, - user_id=user_id, - request_payload={"kind": "chat"}, - recovery_snapshot={"kind": "chat", "worker_args": {}}, - ) - assert journal.claim(run_id, owner=owner, lease_seconds=300) - - yield sessions, make_run - engine.dispose() - @pytest.mark.asyncio async def test_intent_commit_failure_never_calls_tool_function(): @@ -514,1281 +471,3 @@ async def tool(): ) assert result == {"replayed": True} assert invoked == 1 - - -@pytest.mark.asyncio -async def test_startup_replayer_does_not_alias_a_later_same_args_provider_call(effect_env): - sessions, make_run = effect_env - make_run("run-startup-replay") - clock = MutableClock() - replayed = 0 - - async def replay(_intent): - nonlocal replayed - replayed += 1 - return {"from_adapter": True} - - registry = ToolRecoveryRegistry() - registry.register("safe_read", "replay_safe", replayer=replay) - ledger = ToolEffectJournal(sessions, clock=clock) - original = ledger.begin_intent( - run_id="run-startup-replay", - owner="worker", - claim_owner="crashed", - tool_call_id="old-provider-id", - tool_name="safe_read", - args={"query": "same"}, - recovery_policy="replay_safe", - lease_seconds=1, - ) - clock.advance(2) - with sessions() as db: - db.query(ChatRun).filter(ChatRun.run_id == "run-startup-replay").update( - {"lease_expires_at": clock() - timedelta(seconds=1)} - ) - db.commit() - decisions = await recover_incomplete_tool_effects(journal=ledger, registry=registry) - assert [(item.effect_id, item.action) for item in decisions] == [ - (original.effect_id, "replayed") - ] - assert RunJournal(sessions, clock=clock).claim( - "run-startup-replay", owner="new-worker", lease_seconds=300 - ) - invoked = 0 - - async def later_call(): - nonlocal invoked - invoked += 1 - return {"wrong": True} - - outcome = await ToolEffectGateway(ledger, registry).execute_outcome( - run_id="run-startup-replay", - owner="new-worker", - tool_call_id="new-provider-id", - tool_name="safe_read", - args={"query": "same"}, - invoke=later_call, - ) - assert outcome.result == {"wrong": True} - assert outcome.effect_id != original.effect_id - assert outcome.invoked is True - assert replayed == 1 - assert invoked == 1 - - -@pytest.mark.asyncio -async def test_invocation_heartbeat_keeps_long_call_claimed(effect_env): - sessions, make_run = effect_env - make_run("run-fenced", owner="old-worker") - registry = ToolRecoveryRegistry() - registry.register("safe", "replay_safe") - ledger = ToolEffectJournal(sessions) - gateway = ToolEffectGateway(ledger, registry, poll_interval=0.005, lease_seconds=1) - entered = asyncio.Event() - release = asyncio.Event() - invoked = 0 - - async def slow_tool(): - nonlocal invoked - invoked += 1 - entered.set() - await release.wait() - return {"ok": True} - - first = asyncio.create_task( - gateway.execute( - run_id="run-fenced", - owner="old-worker", - tool_call_id="call-old", - tool_name="safe", - args={"q": 1}, - idempotency_key="fenced-key", - invoke=slow_tool, - ) - ) - await entered.wait() - await asyncio.sleep(1.2) - assert not RunJournal(sessions).claim("run-fenced", owner="new-worker", lease_seconds=300) - assert invoked == 1 - release.set() - assert await first == {"ok": True} - assert invoked == 1 - - -def test_sqlite_atomic_recovery_claim_has_one_winner(effect_env): - sessions, make_run = effect_env - make_run("run-sqlite-cas") - ledger = ToolEffectJournal(sessions) - intent = ledger.begin_intent( - run_id="run-sqlite-cas", - owner="worker", - claim_owner="crashed", - tool_call_id="sqlite-cas-call", - tool_name="safe", - args={"q": 1}, - recovery_policy="replay_safe", - lease_seconds=1, - ) - expired = datetime.now(timezone.utc) - timedelta(seconds=1) - with sessions() as db: - db.query(ChatRun).filter(ChatRun.run_id == "run-sqlite-cas").update( - {"lease_expires_at": expired} - ) - db.query(ToolEffectLease).filter(ToolEffectLease.effect_id == intent.effect_id).update( - {"lease_expires_at": expired} - ) - db.commit() - - barrier = Barrier(2) - - def claim(owner: str): - barrier.wait() - return ToolEffectJournal(sessions).claim_recovery_intent( - intent.effect_id, - recovery_owner=owner, - lease_seconds=30, - ) - - with ThreadPoolExecutor(max_workers=2) as pool: - results = list(pool.map(claim, ("recovery-a", "recovery-b"))) - assert sum(result is not None for result in results) == 1 - - -@pytest.mark.asyncio -async def test_reconcile_intent_queries_adapter_before_execution(effect_env): - sessions, make_run = effect_env - make_run("run-reconcile") - clock = MutableClock() - reconciled = 0 - - async def reconcile(intent): - nonlocal reconciled - reconciled += 1 - assert intent.redacted_args == {"amount": 8} - return {"external_status": "already_applied"} - - registry = ToolRecoveryRegistry() - registry.register("charge_account", "reconcile", reconciler=reconcile) - ledger = ToolEffectJournal(sessions, clock=clock) - ledger.begin_intent( - run_id="run-reconcile", - owner="worker", - claim_owner="crashed-invocation", - tool_call_id="old-call", - tool_name="charge_account", - args={"amount": 8}, - recovery_policy="reconcile", - idempotency_key="charge-key", - lease_seconds=1, - ) - clock.advance(2) - with sessions() as db: - db.query(ChatRun).filter(ChatRun.run_id == "run-reconcile").update( - {"lease_expires_at": clock() - timedelta(seconds=1)} - ) - db.commit() - assert RunJournal(sessions, clock=clock).claim( - "run-reconcile", owner="recovery-worker", lease_seconds=300 - ) - invoked = 0 - - async def must_not_execute(): - nonlocal invoked - invoked += 1 - return {"wrong": True} - - result = await ToolEffectGateway(ledger, registry).execute( - run_id="run-reconcile", - owner="recovery-worker", - tool_call_id="new-call", - tool_name="charge_account", - args={"amount": 8}, - idempotency_key="charge-key", - invoke=must_not_execute, - ) - assert result == {"external_status": "already_applied"} - assert reconciled == 1 - assert invoked == 0 - - -@pytest.mark.asyncio -async def test_unknown_tool_defaults_never_replay_and_pauses_run(effect_env): - sessions, make_run = effect_env - make_run("run-unknown") - clock = MutableClock() - registry = ToolRecoveryRegistry() - assert registry.resolve("unregistered_write").policy == "never_replay" - ledger = ToolEffectJournal(sessions, clock=clock) - ledger.begin_intent( - run_id="run-unknown", - owner="worker", - claim_owner="crashed-invocation", - tool_call_id="old-call", - tool_name="unregistered_write", - args={"target": "outside"}, - recovery_policy="never_replay", - idempotency_key="unknown-key", - lease_seconds=1, - ) - clock.advance(2) - with sessions() as db: - db.query(ChatRun).filter(ChatRun.run_id == "run-unknown").update( - {"lease_expires_at": clock() - timedelta(seconds=1)} - ) - db.commit() - assert RunJournal(sessions, clock=clock).claim( - "run-unknown", owner="recovery-worker", lease_seconds=300 - ) - invoked = 0 - - async def must_not_execute(): - nonlocal invoked - invoked += 1 - return {"wrong": True} - - with pytest.raises(ToolOutcomeUnknown): - await ToolEffectGateway(ledger, registry).execute( - run_id="run-unknown", - owner="recovery-worker", - tool_call_id="new-call", - tool_name="unregistered_write", - args={"target": "outside"}, - idempotency_key="unknown-key", - invoke=must_not_execute, - ) - assert invoked == 0 - with sessions() as db: - assert db.get(ChatRun, "run-unknown").status == "needs_attention" - events = ( - db.query(ToolEffectLedger) - .filter(ToolEffectLedger.run_id == "run-unknown") - .order_by(ToolEffectLedger.created_at) - .all() - ) - assert [event.event_type for event in events] == [ - "intent", - "recovery_claim", - "unknown_outcome", - ] - with pytest.raises(ToolOutcomeUnknown): - await ToolEffectGateway(ledger, registry).execute( - run_id="run-unknown", - owner="recovery-worker", - tool_call_id="another-call", - tool_name="unregistered_write", - args={"target": "outside"}, - idempotency_key="unknown-key", - invoke=must_not_execute, - ) - - -def test_result_commit_retry_is_idempotent_and_args_are_redacted(effect_env): - sessions, make_run = effect_env - make_run("run-result") - ledger = ToolEffectJournal(sessions) - prepared = ledger.begin_intent( - run_id="run-result", - owner="worker", - claim_owner="invocation", - tool_call_id="call-result", - tool_name="write", - args={"path": "/tmp/a", "password": "super-secret"}, - recovery_policy="never_replay", - idempotency_key="result-key", - ) - first = ledger.commit_result( - prepared.effect_id, - run_owner="worker", - claim_owner="invocation", - result={"ok": True}, - ) - second = ledger.commit_result( - prepared.effect_id, - run_owner="worker", - claim_owner="invocation", - result={"ok": True}, - ) - assert first == second == {"ok": True} - with sessions() as db: - events = ( - db.query(ToolEffectLedger) - .filter(ToolEffectLedger.effect_id == prepared.effect_id) - .order_by(ToolEffectLedger.created_at) - .all() - ) - assert [event.event_type for event in events] == ["intent", "result"] - assert events[0].redacted_args["password"] != "super-secret" - assert "super-secret" not in str(events[0].redacted_args) - assert len(events[0].args_hash) == 64 - - -def test_concurrent_result_commit_retry_is_idempotent(effect_env): - sessions, make_run = effect_env - make_run("run-result-race") - ledger = ToolEffectJournal(sessions) - prepared = ledger.begin_intent( - run_id="run-result-race", - owner="worker", - claim_owner="same-claim", - tool_call_id="result-race", - tool_name="Read", - args={"path": "/a"}, - recovery_policy="replay_safe", - ) - barrier = Barrier(2) - - def commit(): - barrier.wait(timeout=5) - return ledger.commit_result( - prepared.effect_id, - run_owner="worker", - claim_owner="same-claim", - result={"ok": True}, - ) - - with ThreadPoolExecutor(max_workers=2) as pool: - assert list(pool.map(lambda _index: commit(), range(2))) == [ - {"ok": True}, - {"ok": True}, - ] - with sessions() as db: - assert ( - db.query(ToolEffectLedger) - .filter( - ToolEffectLedger.effect_id == prepared.effect_id, - ToolEffectLedger.event_type == "result", - ) - .count() - == 1 - ) - - -def test_default_recovery_policies_have_real_adapters(): - assert DEFAULT_TOOL_RECOVERY_REGISTRY.resolve("Read").replayer is not None - scheduled = DEFAULT_TOOL_RECOVERY_REGISTRY.resolve("create_scheduled_task") - assert scheduled.policy == "reconcile" - assert scheduled.reconciler is not None - assert scheduled.replayer is not None - assert DEFAULT_TOOL_RECOVERY_REGISTRY.resolve("unregistered_write").policy == "never_replay" - - -def test_recovery_rejects_nested_withheld_arguments(): - intent = ToolIntent( - effect_id="eff-nested-secret", - result_id="result-nested-secret", - run_id="run-nested-secret", - operation_seq=1, - tool_name="Read", - tool_call_id="call-nested-secret", - args_hash="hash", - redacted_args={ - "request": {"steps": [{"command": {"_withheld": True, "sha256": "abc", "length": 3}}]} - }, - idempotency_key="key", - recovery_policy="replay_safe", - ) - with pytest.raises(ToolEffectError, match="withheld sensitive free text"): - _plain_replay_args(intent) - - -@pytest.mark.asyncio -async def test_scheduled_create_reconciliation_uses_exact_atomic_receipt( - effect_env, - monkeypatch, -): - sessions, make_run = effect_env - make_run("run-scheduled-receipt") - ledger = ToolEffectJournal(sessions) - intent = ledger.begin_intent( - run_id="run-scheduled-receipt", - owner="worker", - claim_owner="scheduled-call", - tool_call_id="scheduled-call", - tool_name="create_scheduled_task", - args={ - "cron_expression": "0 9 * * *", - "prompt": "send report", - "name": "daily", - }, - recovery_policy="reconcile", - ).intent - with sessions() as db: - db.add( - ScheduledTask( - task_id="pre-existing-same-content", - user_id="user-1", - task_type="prompt", - prompt="send report", - cron_expression="0 9 * * *", - name="daily", - status="active", - ) - ) - db.commit() - - import core.db.engine as engine_module - import orchestration.tool_effect_recovery as recovery_module - from extensions.mcp_servers.automation_task_mcp import impl - - monkeypatch.setattr(engine_module, "SessionLocal", sessions) - monkeypatch.setattr(recovery_module, "SessionLocal", sessions) - before = await reconcile_scheduled_task_intent(intent) - assert before.outcome == "not_applied" - - result = impl.create_task( - user_id="user-1", - cron_expression="0 9 * * *", - prompt="send report", - name="daily", - tool_effect_id=intent.effect_id, - ) - assert result["ok"] is True - again = impl.create_task( - user_id="user-1", - cron_expression="0 9 * * *", - prompt="send report", - name="daily", - tool_effect_id=intent.effect_id, - ) - assert again == result - after = await reconcile_scheduled_task_intent(intent) - assert after.outcome == "applied" - assert after.result == result - with sessions() as db: - assert db.query(ScheduledTask).count() == 2 - assert db.get(ToolEffectReceipt, intent.effect_id).result_payload == result - - -def test_embedded_secrets_and_free_text_are_not_persisted(effect_env): - sessions, make_run = effect_env - make_run("run-secrets") - secret = "sk-123456789012345678901234567890" - ledger = ToolEffectJournal(sessions) - intent = ledger.begin_intent( - run_id="run-secrets", - owner="worker", - claim_owner="invocation", - tool_call_id="secret-call", - tool_name="Bash", - args={ - "command": f"curl -H 'Authorization: Bearer {secret}' https://x", - "url": f"https://example.test/?token={secret}", - }, - recovery_policy="never_replay", - ) - with sessions() as db: - row = ( - db.query(ToolEffectLedger) - .filter( - ToolEffectLedger.effect_id == intent.effect_id, - ToolEffectLedger.event_type == "intent", - ) - .one() - ) - assert secret not in json.dumps(row.redacted_args) - assert row.redacted_args["command"]["_withheld"] is True - assert "[REDACTED]" in row.redacted_args["url"] - - -@pytest.mark.asyncio -async def test_missing_run_binding_fails_closed_before_tool_invocation(effect_env): - sessions, _make_run = effect_env - middleware = ToolEffectMiddleware(session_factory=sessions) - agent = SimpleNamespace(state=SimpleNamespace(run_id=None, journal_owner=None)) - invoked = 0 - - async def tool(**_kwargs): - nonlocal invoked - invoked += 1 - yield ToolResponse(content=[TextBlock(text="wrong")]) - - with pytest.raises(ToolIntentCommitError): - _ = [ - item - async for item in middleware.on_acting( - agent, - {"tool_call": ToolCallBlock(id="x", name="Read", input="{}")}, - tool, - ) - ] - assert invoked == 0 - - -@pytest.mark.asyncio -async def test_middleware_leaves_adapter_exception_pending_for_recovery(effect_env): - sessions, make_run = effect_env - make_run("run-tool-failure") - middleware = ToolEffectMiddleware(session_factory=sessions) - agent = SimpleNamespace( - state=SimpleNamespace( - run_id="run-tool-failure", - journal_owner="worker", - tool_effect_links={}, - ) - ) - - async def broken(**_kwargs): - if False: - yield None - raise RuntimeError("adapter exploded") - - with pytest.raises(ToolOutcomeUnknown) as raised: - _ = [ - item - async for item in middleware.on_acting( - agent, - {"tool_call": ToolCallBlock(id="failure-call", name="Read", input="{}")}, - broken, - ) - ] - assert isinstance(raised.value.__cause__, RuntimeError) - with sessions() as db: - events = db.query(ToolEffectLedger).order_by(ToolEffectLedger.event_id).all() - assert [event.event_type for event in events] == ["intent"] - - -@pytest.mark.asyncio -async def test_cancelled_run_settles_pending_intent_without_delayed_replay(effect_env): - sessions, make_run = effect_env - make_run("run-cancelled-effect") - registry = ToolRecoveryRegistry() - invoked = 0 - - async def replay(_intent): - nonlocal invoked - invoked += 1 - return {"should_not": "run"} - - registry.register("Read", "replay_safe", replayer=replay) - ledger = ToolEffectJournal(sessions) - ledger.begin_intent( - run_id="run-cancelled-effect", - owner="worker", - claim_owner="dead-call", - tool_call_id="cancelled-call", - tool_name="Read", - args={"path": "/workspace/a.txt"}, - recovery_policy="replay_safe", - ) - assert RunJournal(sessions).cancel("run-cancelled-effect", reason="user cancelled") - - decisions = await recover_incomplete_tool_effects(journal=ledger, registry=registry) - - assert [item.action for item in decisions] == ["terminal_unknown"] - assert invoked == 0 - with sessions() as db: - run = db.get(ChatRun, "run-cancelled-effect") - events = ( - db.query(ToolEffectLedger) - .filter(ToolEffectLedger.run_id == run.run_id) - .order_by(ToolEffectLedger.event_id) - .all() - ) - assert run.status == "cancelled" - assert run.run_phase == "cancelled" - assert [item.event_type for item in events] == ["intent", "unknown_outcome"] - - -@pytest.mark.asyncio -async def test_middleware_commits_error_tool_response_as_failure(effect_env): - sessions, make_run = effect_env - make_run("run-tool-error-response") - middleware = ToolEffectMiddleware(session_factory=sessions) - agent = SimpleNamespace( - state=SimpleNamespace( - run_id="run-tool-error-response", - journal_owner="worker", - tool_effect_links={}, - ) - ) - - async def failed_response(**_kwargs): - yield ToolResponse( - content=[TextBlock(text="tool rejected input")], - state=ToolResultState.ERROR, - ) - - output = [ - item - async for item in middleware.on_acting( - agent, - {"tool_call": ToolCallBlock(id="error-response-call", name="Read", input="{}")}, - failed_response, - ) - ] - assert output[-1].state == ToolResultState.ERROR - with sessions() as db: - events = db.query(ToolEffectLedger).order_by(ToolEffectLedger.event_id).all() - assert [event.event_type for event in events] == ["intent", "failure"] - - -@pytest.mark.asyncio -async def test_middleware_returns_explicit_error_for_unregistered_tool(effect_env): - sessions, make_run = effect_env - make_run("run-unregistered-tool-error") - middleware = ToolEffectMiddleware(session_factory=sessions) - agent = SimpleNamespace( - state=SimpleNamespace( - run_id="run-unregistered-tool-error", - journal_owner="worker", - tool_effect_links={}, - ) - ) - original_error = "Remote API returned 404: resource not found or not authorized" - - async def failed_response(**_kwargs): - yield ToolResponse( - content=[TextBlock(text=original_error)], - state=ToolResultState.ERROR, - ) - - output = [ - item - async for item in middleware.on_acting( - agent, - { - "tool_call": ToolCallBlock( - id="unregistered-error-call", - name="remote_resource_lookup", - input='{"resource":"missing-or-private"}', - ) - }, - failed_response, - ) - ] - - assert output[-1].state == ToolResultState.ERROR - assert original_error in str(output[-1].content) - with sessions() as db: - events = db.query(ToolEffectLedger).order_by(ToolEffectLedger.event_id).all() - assert [event.event_type for event in events] == ["intent", "failure"] - assert events[0].recovery_policy == "never_replay" - assert original_error in str(events[-1].result_payload) - - -@pytest.mark.asyncio -async def test_agentscope_middleware_commits_intent_before_call_and_replays_same_shape( - effect_env, -): - sessions, make_run = effect_env - make_run("run-middleware") - middleware = ToolEffectMiddleware(session_factory=sessions) - agent = SimpleNamespace(state=SimpleNamespace(run_id="run-middleware", journal_owner="worker")) - invoked = 0 - saw_durable_intent = False - - async def execute_once(**_kwargs): - nonlocal invoked, saw_durable_intent - invoked += 1 - with sessions() as db: - saw_durable_intent = ( - db.query(ToolEffectLedger) - .filter( - ToolEffectLedger.run_id == "run-middleware", - ToolEffectLedger.event_type == "intent", - ) - .count() - == 1 - ) - block = TextBlock(text="same SSE-compatible result") - yield ToolChunk(content=[block], state=ToolResultState.RUNNING) - yield ToolResponse(content=[block], state=ToolResultState.SUCCESS) - - first_call = ToolCallBlock( - id="provider-call-1", - name="Read", - input='{"path":"/workspace/a.txt"}', - ) - first = [ - item - async for item in middleware.on_acting( - agent, - {"tool_call": first_call}, - execute_once, - ) - ] - - async def must_not_execute(**_kwargs): - nonlocal invoked - invoked += 1 - yield ToolResponse(content=[TextBlock(text="wrong")]) - - replay_call = ToolCallBlock( - id="provider-call-1", - name="Read", - input='{"path":"/workspace/a.txt"}', - ) - replayed = [ - item - async for item in middleware.on_acting( - agent, - {"tool_call": replay_call}, - must_not_execute, - ) - ] - - assert saw_durable_intent is True - assert invoked == 1 - assert [type(item) for item in first] == [ToolChunk, ToolResponse] - assert [type(item) for item in replayed] == [ToolChunk, ToolResponse] - assert replayed[-1].state == ToolResultState.SUCCESS - assert replayed[-1].content[0].text == "same SSE-compatible result" - - -@pytest.mark.asyncio -async def test_real_agentscope_public_reply_commits_intent_before_adapter(effect_env): - sessions, make_run = effect_env - make_run("run-public-agent") - saw_intent = False - invoked = 0 - - async def probe(value: int) -> ToolChunk: - """Return a test value.""" - nonlocal saw_intent, invoked - invoked += 1 - with sessions() as db: - saw_intent = ( - db.query(ToolEffectLedger) - .filter( - ToolEffectLedger.run_id == "run-public-agent", - ToolEffectLedger.event_type == "intent", - ) - .count() - == 1 - ) - return ToolChunk( - content=[TextBlock(text=f"value={value}")], - state=ToolResultState.SUCCESS, - ) - - class ToolThenAnswerModel: - model = "tool-then-answer" - context_size = 32768 - - def __init__(self): - self.calls = 0 - - async def __call__(self, messages, tools=None, **_kwargs): - del messages - self.calls += 1 - if self.calls == 1: - assert any(item["function"]["name"] == "probe" for item in tools) - return ChatResponse( - content=[ToolCallBlock(id="public-call", name="probe", input='{"value":7}')], - is_last=False, - usage=ChatUsage(input_tokens=1, output_tokens=1, time=0.01), - ) - return ChatResponse( - content=[TextBlock(text="done")], - is_last=True, - usage=ChatUsage(input_tokens=1, output_tokens=1, time=0.01), - ) - - async def count_tokens(self, messages, tools=None): - del messages, tools - return 1 - - state = AgentRuntimeState( - run_id="run-public-agent", - journal_owner="worker", - permission_context=PermissionContext(mode=PermissionMode.BYPASS), - ) - agent = Agent( - name="ledger-public-seam", - system_prompt="Use the probe tool once.", - model=ToolThenAnswerModel(), - toolkit=Toolkit(tools=[FunctionTool(probe)]), - middlewares=[ToolEffectMiddleware(session_factory=sessions)], - state=state, - react_config=ReActConfig(max_iters=2), - ) - reply = await agent.reply(UserMsg(name="user", content="run probe")) - assert reply.get_text_content() == "done" - assert invoked == 1 - assert saw_intent is True - with sessions() as db: - assert [ - row.event_type for row in db.query(ToolEffectLedger).order_by(ToolEffectLedger.event_id) - ] == ["intent", "result"] - - -@pytest.mark.asyncio -async def test_public_reply_cancellation_stops_tool_without_late_terminal_write(effect_env): - sessions, make_run = effect_env - make_run("run-public-cancel") - entered = asyncio.Event() - release = asyncio.Event() - side_effects = 0 - - async def blocking_probe(value: int) -> ToolChunk: - """Wait until the caller releases the test tool.""" - nonlocal side_effects - entered.set() - await release.wait() - side_effects += value - return ToolChunk( - content=[TextBlock(text=f"value={value}")], - state=ToolResultState.SUCCESS, - ) - - class BlockingToolModel: - model = "blocking-tool" - context_size = 32768 - - async def __call__(self, messages, tools=None, **_kwargs): - del messages - assert any(item["function"]["name"] == "blocking_probe" for item in tools) - return ChatResponse( - content=[ - ToolCallBlock( - id="cancel-call", - name="blocking_probe", - input='{"value":1}', - ) - ], - is_last=False, - usage=ChatUsage(input_tokens=1, output_tokens=1, time=0.01), - ) - - async def count_tokens(self, messages, tools=None): - del messages, tools - return 1 - - agent = Agent( - name="ledger-cancel-seam", - system_prompt="Use the blocking probe.", - model=BlockingToolModel(), - toolkit=Toolkit(tools=[FunctionTool(blocking_probe)]), - middlewares=[ToolEffectMiddleware(session_factory=sessions)], - state=AgentRuntimeState( - run_id="run-public-cancel", - journal_owner="worker", - permission_context=PermissionContext(mode=PermissionMode.BYPASS), - ), - react_config=ReActConfig(max_iters=2), - ) - reply_task = asyncio.create_task(agent.reply(UserMsg(name="user", content="start"))) - await entered.wait() - assert RunJournal(sessions).cancel("run-public-cancel", reason="test cancellation") - reply_task.cancel() - await asyncio.sleep(0) - release.set() - with pytest.raises(asyncio.CancelledError): - await reply_task - assert side_effects == 0 - with sessions() as db: - run = db.get(ChatRun, "run-public-cancel") - assert run.status == "cancelled" - assert run.run_phase == "cancelled" - assert [ - row.event_type - for row in db.query(ToolEffectLedger) - .filter(ToolEffectLedger.run_id == "run-public-cancel") - .order_by(ToolEffectLedger.event_id) - ] == ["intent"] - - -@pytest.mark.asyncio -async def test_restart_sweep_classifies_replay_reconcile_and_never_replay(effect_env): - sessions, make_run = effect_env - clock = MutableClock() - registry = ToolRecoveryRegistry() - - async def replay(_intent): - return {"replayed": True} - - registry.register("safe_read", "replay_safe", replayer=replay) - - async def reconcile(_intent): - return {"external_status": "committed"} - - registry.register("external_write", "reconcile", reconciler=reconcile) - ledger = ToolEffectJournal(sessions, clock=clock) - effects = {} - for run_id, tool_name, policy in ( - ("run-sweep-replay", "safe_read", "replay_safe"), - ("run-sweep-reconcile", "external_write", "reconcile"), - ("run-sweep-never", "unknown_write", "never_replay"), - ): - make_run(run_id) - effects[run_id] = ledger.begin_intent( - run_id=run_id, - owner="worker", - claim_owner="crashed-tool", - tool_call_id=f"call-{run_id}", - tool_name=tool_name, - args={"target": run_id}, - recovery_policy=policy, - idempotency_key=f"key-{run_id}", - lease_seconds=1, - ).effect_id - clock.advance(2) - with sessions() as db: - db.query(ChatRun).filter(ChatRun.run_id.in_(effects)).update( - {"lease_expires_at": clock() - timedelta(seconds=1)}, - synchronize_session=False, - ) - db.commit() - - decisions = await recover_incomplete_tool_effects(journal=ledger, registry=registry) - assert {(item.run_id, item.action) for item in decisions} == { - ("run-sweep-replay", "replayed"), - ("run-sweep-reconcile", "reconciled"), - ("run-sweep-never", "needs_attention"), - } - with sessions() as db: - assert db.get(ChatRun, "run-sweep-replay").status == "running" - assert db.get(ChatRun, "run-sweep-reconcile").run_phase == "tool_result_committed" - assert db.get(ChatRun, "run-sweep-never").status == "needs_attention" - reconciled = ( - db.query(ToolEffectLedger) - .filter( - ToolEffectLedger.effect_id == effects["run-sweep-reconcile"], - ToolEffectLedger.event_type == "result", - ) - .one() - ) - assert reconciled.result_payload == {"external_status": "committed"} - - -@pytest.mark.asyncio -async def test_recovery_adapter_failure_does_not_starve_later_intent(effect_env): - sessions, make_run = effect_env - clock = MutableClock() - registry = ToolRecoveryRegistry() - - async def broken(_intent): - raise RuntimeError("temporary adapter outage") - - async def healthy(_intent): - return {"ok": True} - - registry.register("broken_read", "replay_safe", replayer=broken) - registry.register("healthy_read", "replay_safe", replayer=healthy) - ledger = ToolEffectJournal(sessions, clock=clock) - effects = [] - for run_id, tool_name in (("run-broken", "broken_read"), ("run-healthy", "healthy_read")): - make_run(run_id) - effects.append( - ledger.begin_intent( - run_id=run_id, - owner="worker", - claim_owner="crashed", - tool_call_id=f"call-{run_id}", - tool_name=tool_name, - args={"q": run_id}, - recovery_policy="replay_safe", - lease_seconds=1, - ).effect_id - ) - clock.advance(2) - with sessions() as db: - db.query(ChatRun).filter(ChatRun.run_id.in_(["run-broken", "run-healthy"])).update( - {"lease_expires_at": clock() - timedelta(seconds=1)}, synchronize_session=False - ) - db.commit() - decisions = await recover_incomplete_tool_effects(journal=ledger, registry=registry) - assert {(item.run_id, item.action) for item in decisions} == { - ("run-broken", "retry"), - ("run-healthy", "replayed"), - } - assert ledger.terminal_decision(effects[0]) is None - assert ledger.terminal_decision(effects[1]).result == {"ok": True} - run_recovery = RunJournal(sessions, clock=clock).recover() - assert "run-broken" not in {item.run_id for item in run_recovery} - - -@pytest.mark.asyncio -async def test_recovery_heartbeat_prevents_second_sweep_from_replaying_long_adapter(effect_env): - sessions, make_run = effect_env - make_run("run-long-recovery") - ledger = ToolEffectJournal(sessions) - intent = ledger.begin_intent( - run_id="run-long-recovery", - owner="worker", - claim_owner="crashed", - tool_call_id="long-recovery-call", - tool_name="long_read", - args={"query": "slow"}, - recovery_policy="replay_safe", - lease_seconds=1, - ) - expired = datetime.now(timezone.utc) - timedelta(seconds=1) - with sessions() as db: - db.query(ChatRun).filter(ChatRun.run_id == "run-long-recovery").update( - {"lease_expires_at": expired} - ) - db.query(ToolEffectLease).filter(ToolEffectLease.effect_id == intent.effect_id).update( - {"lease_expires_at": expired} - ) - db.commit() - - started = asyncio.Event() - release = asyncio.Event() - replays = 0 - - async def replay(_intent): - nonlocal replays - replays += 1 - started.set() - await release.wait() - return {"ok": True} - - registry = ToolRecoveryRegistry() - registry.register("long_read", "replay_safe", replayer=replay) - first = asyncio.create_task( - recover_incomplete_tool_effects( - journal=ledger, - registry=registry, - lease_seconds=1, - heartbeat_interval=0.1, - ) - ) - await started.wait() - await asyncio.sleep(1.1) - second = await recover_incomplete_tool_effects( - journal=ledger, - registry=registry, - lease_seconds=1, - heartbeat_interval=0.1, - ) - assert second == [] - assert replays == 1 - release.set() - decisions = await first - assert [(item.effect_id, item.action) for item in decisions] == [(intent.effect_id, "replayed")] - - -def test_tool_call_log_is_a_projection_linked_to_authoritative_effect(effect_env, monkeypatch): - sessions, make_run = effect_env - make_run("run-projection") - ledger = ToolEffectJournal(sessions) - intent = ledger.begin_intent( - run_id="run-projection", - owner="worker", - claim_owner="projection-invocation", - tool_call_id="projection-call", - tool_name="Read", - args={"path": "/workspace/a.txt"}, - recovery_policy="replay_safe", - ) - ledger.commit_result( - intent.effect_id, - run_owner="worker", - claim_owner="projection-invocation", - result={"ok": True}, - ) - - from core.services import log_service - - monkeypatch.setattr(log_service, "SessionLocal", sessions) - projection_secret = "projection-secret-token-123456" - log_service._write_tool_call_sync( - { - "run_id": "run-projection", - "chat_id": "chat-1", - "user_id": "user-1", - "tool_name": "Read", - "tool_call_id": "projection-call", - "tool_args": { - "path": "/workspace/a.txt", - "command": f"curl -H 'Authorization: Bearer {projection_secret}' https://x", - "url": f"https://example.test/?token={projection_secret}", - "callback": f"https://example.test/cb?password={projection_secret}", - }, - "tool_result": "contents", - "status": "success", - } - ) - with sessions() as db: - projection = db.query(ToolCallLog).one() - assert projection.effect_id == intent.effect_id - assert projection.id == intent.intent.result_id - assert projection.tool_result == "contents" - assert projection_secret not in json.dumps(projection.tool_args) - assert projection.tool_args["command"]["_withheld"] is True - assert "[REDACTED]" in projection.tool_args["url"] - assert "[REDACTED]" in projection.tool_args["callback"] - - log_service._write_tool_call_sync( - { - "run_id": "run-projection", - "chat_id": "chat-1", - "user_id": "user-1", - "tool_name": "Read", - "tool_call_id": "new-provider-call-after-recovery", - "effect_id": intent.effect_id, - "result_id": intent.intent.result_id, - "tool_args": {"path": "/workspace/a.txt"}, - "tool_result": "replayed contents", - "status": "success", - } - ) - with sessions() as db: - assert db.query(ToolCallLog).count() == 1 - projection = db.query(ToolCallLog).one() - assert projection.id == intent.intent.result_id - assert projection.tool_call_id == "new-provider-call-after-recovery" - assert projection.tool_result == "replayed contents" - - -def test_prune_settled_drops_only_old_rows_of_settled_runs(effect_env): - sessions, make_run = effect_env - clock = MutableClock() - ledger = ToolEffectJournal(sessions, clock=clock) - make_run("run-old") - make_run("run-live", chat_id="chat-2") - old = ledger.begin_intent( - run_id="run-old", - owner="worker", - claim_owner="invocation", - tool_call_id="call-old", - tool_name="write", - args={"path": "/tmp/a"}, - recovery_policy="never_replay", - idempotency_key="prune-old", - ) - ledger.commit_result( - old.effect_id, - run_owner="worker", - claim_owner="invocation", - result={"ok": True}, - ) - live = ledger.begin_intent( - run_id="run-live", - owner="worker", - claim_owner="invocation", - tool_call_id="call-live", - tool_name="write", - args={"path": "/tmp/b"}, - recovery_policy="never_replay", - idempotency_key="prune-live", - ) - from core.db.models import ChatRunOperation, HarnessEventLog, HarnessUsageAttempt - - with sessions() as db: - run = db.get(ChatRun, "run-old") - run.status = "completed" - db.add( - HarnessUsageAttempt( - attempt_id="hat-old", - run_id="run-old", - attempt_seq=1, - kind="tool", - operation_name="write", - status="success", - ) - ) - db.add( - HarnessEventLog( - event_id="hev-old", - run_id="run-old", - event_seq=1, - event_type="tool_result", - phase="post", - payload={}, - ) - ) - db.commit() - clock.advance(8 * 24 * 3600) - assert ledger.prune_settled() >= 4 - with sessions() as db: - for model in ( - ToolEffectLedger, - HarnessUsageAttempt, - HarnessEventLog, - ChatRunOperation, - ): - assert db.query(model).filter(model.run_id == "run-old").count() == 0 - assert ( - db.query(ToolEffectLedger) - .filter(ToolEffectLedger.run_id == "run-live") - .count() - == 1 - ) - assert ( - db.query(ChatRunOperation) - .filter(ChatRunOperation.run_id == "run-live") - .count() - > 0 - ) - assert ledger.pending_effect_ids() == [live.effect_id] - - -def test_prune_settled_eventually_reclaims_needs_attention_runs(effect_env): - """needs_attention 保留得更久，但不是永远。 - - 原先它被整个排除在清理之外，等于永不回收：生产库里每一条超出保留期的 - journal 记录——四万条——都属于 needs_attention 的 run，被永久钉住，还要被 - 每五分钟一轮的清理反复全表扫。保留期该是"更长"，不是"无限"。 - """ - from core.db.models import HarnessEventLog - from core.services.tool_effect_ledger import _ATTENTION_RETENTION_FACTOR - - sessions, make_run = effect_env - clock = MutableClock() - ledger = ToolEffectJournal(sessions, clock=clock) - make_run("run-attention") - intent = ledger.begin_intent( - run_id="run-attention", - owner="worker", - claim_owner="invocation", - tool_call_id="call-attention", - tool_name="write", - args={"path": "/tmp/a"}, - recovery_policy="never_replay", - idempotency_key="prune-attention", - ) - ledger.commit_result( - intent.effect_id, - run_owner="worker", - claim_owner="invocation", - result={"ok": True}, - ) - with sessions() as db: - run = db.get(ChatRun, "run-attention") - run.status = "needs_attention" - # recovery_snapshot 是整轮对话的快照，库里最大的单个字段；它跟 journal - # 走同一套分档，不能因为状态是 needs_attention 就永久留着 - run.completed_at = clock.now - db.add( - HarnessEventLog( - event_id="hev-attention", - run_id="run-attention", - event_seq=1, - event_type="tool_result", - phase="post", - payload={}, - ) - ) - db.commit() - - # 保留期显式传入，两次推进都从它算——否则 prune_settled 的默认值一改， - # 这里的天数就悄悄测不到边界了 - retention_days = 7 - - # 刚过普通终态的保留期：记录必须还在 - clock.advance((retention_days + 1) * 24 * 3600) - ledger.prune_settled(retention_days=retention_days) - with sessions() as db: - assert ( - db.query(ToolEffectLedger).filter(ToolEffectLedger.run_id == "run-attention").count() - > 0 - ) - assert ( - db.query(HarnessEventLog).filter(HarnessEventLog.run_id == "run-attention").count() == 1 - ) - assert db.get(ChatRun, "run-attention").recovery_snapshot is not None - - # 过了加长保留期：回收，不能无限期占着库 - clock.advance(retention_days * _ATTENTION_RETENTION_FACTOR * 24 * 3600) - ledger.prune_settled(retention_days=retention_days) - with sessions() as db: - assert ( - db.query(ToolEffectLedger).filter(ToolEffectLedger.run_id == "run-attention").count() - == 0 - ) - assert ( - db.query(HarnessEventLog).filter(HarnessEventLog.run_id == "run-attention").count() == 0 - ) - assert db.get(ChatRun, "run-attention").recovery_snapshot is None diff --git a/src/backend/tests/orchestration/test_tool_effect_ledger_2.py b/src/backend/tests/orchestration/test_tool_effect_ledger_2.py new file mode 100644 index 00000000..45944831 --- /dev/null +++ b/src/backend/tests/orchestration/test_tool_effect_ledger_2.py @@ -0,0 +1,429 @@ +from tests.orchestration.tool_effect_support import ( + log_records, + asyncio, + json, + ThreadPoolExecutor, + datetime, + timedelta, + timezone, + Barrier, + SimpleNamespace, + pytest, + Agent, + ReActConfig, + TextBlock, + ToolCallBlock, + ToolResultState, + UserMsg, + ChatResponse, + ChatUsage, + PermissionContext, + PermissionMode, + FunctionTool, + Toolkit, + ToolChunk, + ToolResponse, + Base, + ChatRun, + ChatSession, + ToolCallLog, + ToolEffectLease, + ToolEffectLedger, + ToolEffectReceipt, + ScheduledTask, + AgentRuntimeState, + make_adapter, + durable_adapter, + _plain_replay_args, + reconcile_scheduled_task_intent, + RunJournal, + DEFAULT_TOOL_RECOVERY_REGISTRY, + ToolRecoveryRegistry, + ToolEffectError, + ToolIdempotencyConflict, + ToolIntentCommitError, + ToolOutcomeUnknown, + ToolEffectGateway, + ToolEffectJournal, + ToolIntent, + recover_incomplete_tool_effects, + create_engine, + sessionmaker, + QueuePool, + MutableClock, + effect_env, +) + +@pytest.mark.asyncio +async def test_startup_replayer_does_not_alias_a_later_same_args_provider_call(effect_env): + sessions, make_run = effect_env + make_run("run-startup-replay") + clock = MutableClock() + replayed = 0 + + async def replay(_intent): + nonlocal replayed + replayed += 1 + return {"from_adapter": True} + + registry = ToolRecoveryRegistry() + registry.register("safe_read", "replay_safe", replayer=replay) + ledger = ToolEffectJournal(sessions, clock=clock) + original = ledger.begin_intent( + run_id="run-startup-replay", + owner="worker", + claim_owner="crashed", + tool_call_id="old-provider-id", + tool_name="safe_read", + args={"query": "same"}, + recovery_policy="replay_safe", + lease_seconds=1, + ) + clock.advance(2) + with sessions() as db: + db.query(ChatRun).filter(ChatRun.run_id == "run-startup-replay").update( + {"lease_expires_at": clock() - timedelta(seconds=1)} + ) + db.commit() + decisions = await recover_incomplete_tool_effects(journal=ledger, registry=registry) + assert [(item.effect_id, item.action) for item in decisions] == [ + (original.effect_id, "replayed") + ] + assert RunJournal(sessions, clock=clock).claim( + "run-startup-replay", owner="new-worker", lease_seconds=300 + ) + invoked = 0 + + async def later_call(): + nonlocal invoked + invoked += 1 + return {"wrong": True} + + outcome = await ToolEffectGateway(ledger, registry).execute_outcome( + run_id="run-startup-replay", + owner="new-worker", + tool_call_id="new-provider-id", + tool_name="safe_read", + args={"query": "same"}, + invoke=later_call, + ) + assert outcome.result == {"wrong": True} + assert outcome.effect_id != original.effect_id + assert outcome.invoked is True + assert replayed == 1 + assert invoked == 1 + + +@pytest.mark.asyncio +async def test_invocation_heartbeat_keeps_long_call_claimed(effect_env): + sessions, make_run = effect_env + make_run("run-fenced", owner="old-worker") + registry = ToolRecoveryRegistry() + registry.register("safe", "replay_safe") + ledger = ToolEffectJournal(sessions) + gateway = ToolEffectGateway(ledger, registry, poll_interval=0.005, lease_seconds=1) + entered = asyncio.Event() + release = asyncio.Event() + invoked = 0 + + async def slow_tool(): + nonlocal invoked + invoked += 1 + entered.set() + await release.wait() + return {"ok": True} + + first = asyncio.create_task( + gateway.execute( + run_id="run-fenced", + owner="old-worker", + tool_call_id="call-old", + tool_name="safe", + args={"q": 1}, + idempotency_key="fenced-key", + invoke=slow_tool, + ) + ) + await entered.wait() + await asyncio.sleep(1.2) + assert not RunJournal(sessions).claim("run-fenced", owner="new-worker", lease_seconds=300) + assert invoked == 1 + release.set() + assert await first == {"ok": True} + assert invoked == 1 + + +def test_sqlite_atomic_recovery_claim_has_one_winner(effect_env): + sessions, make_run = effect_env + make_run("run-sqlite-cas") + ledger = ToolEffectJournal(sessions) + intent = ledger.begin_intent( + run_id="run-sqlite-cas", + owner="worker", + claim_owner="crashed", + tool_call_id="sqlite-cas-call", + tool_name="safe", + args={"q": 1}, + recovery_policy="replay_safe", + lease_seconds=1, + ) + expired = datetime.now(timezone.utc) - timedelta(seconds=1) + with sessions() as db: + db.query(ChatRun).filter(ChatRun.run_id == "run-sqlite-cas").update( + {"lease_expires_at": expired} + ) + db.query(ToolEffectLease).filter(ToolEffectLease.effect_id == intent.effect_id).update( + {"lease_expires_at": expired} + ) + db.commit() + + barrier = Barrier(2) + + def claim(owner: str): + barrier.wait() + return ToolEffectJournal(sessions).claim_recovery_intent( + intent.effect_id, + recovery_owner=owner, + lease_seconds=30, + ) + + with ThreadPoolExecutor(max_workers=2) as pool: + results = list(pool.map(claim, ("recovery-a", "recovery-b"))) + assert sum(result is not None for result in results) == 1 + + +@pytest.mark.asyncio +async def test_reconcile_intent_queries_adapter_before_execution(effect_env): + sessions, make_run = effect_env + make_run("run-reconcile") + clock = MutableClock() + reconciled = 0 + + async def reconcile(intent): + nonlocal reconciled + reconciled += 1 + assert intent.redacted_args == {"amount": 8} + return {"external_status": "already_applied"} + + registry = ToolRecoveryRegistry() + registry.register("charge_account", "reconcile", reconciler=reconcile) + ledger = ToolEffectJournal(sessions, clock=clock) + ledger.begin_intent( + run_id="run-reconcile", + owner="worker", + claim_owner="crashed-invocation", + tool_call_id="old-call", + tool_name="charge_account", + args={"amount": 8}, + recovery_policy="reconcile", + idempotency_key="charge-key", + lease_seconds=1, + ) + clock.advance(2) + with sessions() as db: + db.query(ChatRun).filter(ChatRun.run_id == "run-reconcile").update( + {"lease_expires_at": clock() - timedelta(seconds=1)} + ) + db.commit() + assert RunJournal(sessions, clock=clock).claim( + "run-reconcile", owner="recovery-worker", lease_seconds=300 + ) + invoked = 0 + + async def must_not_execute(): + nonlocal invoked + invoked += 1 + return {"wrong": True} + + result = await ToolEffectGateway(ledger, registry).execute( + run_id="run-reconcile", + owner="recovery-worker", + tool_call_id="new-call", + tool_name="charge_account", + args={"amount": 8}, + idempotency_key="charge-key", + invoke=must_not_execute, + ) + assert result == {"external_status": "already_applied"} + assert reconciled == 1 + assert invoked == 0 + + +@pytest.mark.asyncio +async def test_unknown_tool_defaults_never_replay_and_pauses_run(effect_env): + sessions, make_run = effect_env + make_run("run-unknown") + clock = MutableClock() + registry = ToolRecoveryRegistry() + assert registry.resolve("unregistered_write").policy == "never_replay" + ledger = ToolEffectJournal(sessions, clock=clock) + ledger.begin_intent( + run_id="run-unknown", + owner="worker", + claim_owner="crashed-invocation", + tool_call_id="old-call", + tool_name="unregistered_write", + args={"target": "outside"}, + recovery_policy="never_replay", + idempotency_key="unknown-key", + lease_seconds=1, + ) + clock.advance(2) + with sessions() as db: + db.query(ChatRun).filter(ChatRun.run_id == "run-unknown").update( + {"lease_expires_at": clock() - timedelta(seconds=1)} + ) + db.commit() + assert RunJournal(sessions, clock=clock).claim( + "run-unknown", owner="recovery-worker", lease_seconds=300 + ) + invoked = 0 + + async def must_not_execute(): + nonlocal invoked + invoked += 1 + return {"wrong": True} + + with pytest.raises(ToolOutcomeUnknown): + await ToolEffectGateway(ledger, registry).execute( + run_id="run-unknown", + owner="recovery-worker", + tool_call_id="new-call", + tool_name="unregistered_write", + args={"target": "outside"}, + idempotency_key="unknown-key", + invoke=must_not_execute, + ) + assert invoked == 0 + with sessions() as db: + assert db.get(ChatRun, "run-unknown").status == "needs_attention" + events = ( + db.query(ToolEffectLedger) + .filter(ToolEffectLedger.run_id == "run-unknown") + .order_by(ToolEffectLedger.created_at) + .all() + ) + assert [event.event_type for event in events] == [ + "intent", + "recovery_claim", + "unknown_outcome", + ] + with pytest.raises(ToolOutcomeUnknown): + await ToolEffectGateway(ledger, registry).execute( + run_id="run-unknown", + owner="recovery-worker", + tool_call_id="another-call", + tool_name="unregistered_write", + args={"target": "outside"}, + idempotency_key="unknown-key", + invoke=must_not_execute, + ) + + +def test_result_commit_retry_is_idempotent_and_args_are_redacted(effect_env): + sessions, make_run = effect_env + make_run("run-result") + ledger = ToolEffectJournal(sessions) + prepared = ledger.begin_intent( + run_id="run-result", + owner="worker", + claim_owner="invocation", + tool_call_id="call-result", + tool_name="write", + args={"path": "/tmp/a", "password": "super-secret"}, + recovery_policy="never_replay", + idempotency_key="result-key", + ) + first = ledger.commit_result( + prepared.effect_id, + run_owner="worker", + claim_owner="invocation", + result={"ok": True}, + ) + second = ledger.commit_result( + prepared.effect_id, + run_owner="worker", + claim_owner="invocation", + result={"ok": True}, + ) + assert first == second == {"ok": True} + with sessions() as db: + events = ( + db.query(ToolEffectLedger) + .filter(ToolEffectLedger.effect_id == prepared.effect_id) + .order_by(ToolEffectLedger.created_at) + .all() + ) + assert [event.event_type for event in events] == ["intent", "result"] + assert events[0].redacted_args["password"] != "super-secret" + assert "super-secret" not in str(events[0].redacted_args) + assert len(events[0].args_hash) == 64 + + +def test_concurrent_result_commit_retry_is_idempotent(effect_env): + sessions, make_run = effect_env + make_run("run-result-race") + ledger = ToolEffectJournal(sessions) + prepared = ledger.begin_intent( + run_id="run-result-race", + owner="worker", + claim_owner="same-claim", + tool_call_id="result-race", + tool_name="Read", + args={"path": "/a"}, + recovery_policy="replay_safe", + ) + barrier = Barrier(2) + + def commit(): + barrier.wait(timeout=5) + return ledger.commit_result( + prepared.effect_id, + run_owner="worker", + claim_owner="same-claim", + result={"ok": True}, + ) + + with ThreadPoolExecutor(max_workers=2) as pool: + assert list(pool.map(lambda _index: commit(), range(2))) == [ + {"ok": True}, + {"ok": True}, + ] + with sessions() as db: + assert ( + db.query(ToolEffectLedger) + .filter( + ToolEffectLedger.effect_id == prepared.effect_id, + ToolEffectLedger.event_type == "result", + ) + .count() + == 1 + ) + + +def test_default_recovery_policies_have_real_adapters(): + assert DEFAULT_TOOL_RECOVERY_REGISTRY.resolve("Read").replayer is not None + scheduled = DEFAULT_TOOL_RECOVERY_REGISTRY.resolve("create_scheduled_task") + assert scheduled.policy == "reconcile" + assert scheduled.reconciler is not None + assert scheduled.replayer is not None + assert DEFAULT_TOOL_RECOVERY_REGISTRY.resolve("unregistered_write").policy == "never_replay" + + +def test_recovery_rejects_nested_withheld_arguments(): + intent = ToolIntent( + effect_id="eff-nested-secret", + result_id="result-nested-secret", + run_id="run-nested-secret", + operation_seq=1, + tool_name="Read", + tool_call_id="call-nested-secret", + args_hash="hash", + redacted_args={ + "request": {"steps": [{"command": {"_withheld": True, "sha256": "abc", "length": 3}}]} + }, + idempotency_key="key", + recovery_policy="replay_safe", + ) + with pytest.raises(ToolEffectError, match="withheld sensitive free text"): + _plain_replay_args(intent) diff --git a/src/backend/tests/orchestration/test_tool_effect_ledger_3.py b/src/backend/tests/orchestration/test_tool_effect_ledger_3.py new file mode 100644 index 00000000..0dab7e11 --- /dev/null +++ b/src/backend/tests/orchestration/test_tool_effect_ledger_3.py @@ -0,0 +1,477 @@ +from tests.orchestration.tool_effect_support import ( + log_records, + asyncio, + json, + ThreadPoolExecutor, + datetime, + timedelta, + timezone, + Barrier, + SimpleNamespace, + pytest, + Agent, + ReActConfig, + TextBlock, + ToolCallBlock, + ToolResultState, + UserMsg, + ChatResponse, + ChatUsage, + PermissionContext, + PermissionMode, + FunctionTool, + Toolkit, + ToolChunk, + ToolResponse, + Base, + ChatRun, + ChatSession, + ToolCallLog, + ToolEffectLease, + ToolEffectLedger, + ToolEffectReceipt, + ScheduledTask, + AgentRuntimeState, + make_adapter, + durable_adapter, + _plain_replay_args, + reconcile_scheduled_task_intent, + RunJournal, + DEFAULT_TOOL_RECOVERY_REGISTRY, + ToolRecoveryRegistry, + ToolEffectError, + ToolIdempotencyConflict, + ToolIntentCommitError, + ToolOutcomeUnknown, + ToolEffectGateway, + ToolEffectJournal, + ToolIntent, + recover_incomplete_tool_effects, + create_engine, + sessionmaker, + QueuePool, + MutableClock, + effect_env, +) + +@pytest.mark.asyncio +async def test_scheduled_create_reconciliation_uses_exact_atomic_receipt( + effect_env, + monkeypatch, +): + sessions, make_run = effect_env + make_run("run-scheduled-receipt") + ledger = ToolEffectJournal(sessions) + intent = ledger.begin_intent( + run_id="run-scheduled-receipt", + owner="worker", + claim_owner="scheduled-call", + tool_call_id="scheduled-call", + tool_name="create_scheduled_task", + args={ + "cron_expression": "0 9 * * *", + "prompt": "send report", + "name": "daily", + }, + recovery_policy="reconcile", + ).intent + with sessions() as db: + db.add( + ScheduledTask( + task_id="pre-existing-same-content", + user_id="user-1", + task_type="prompt", + prompt="send report", + cron_expression="0 9 * * *", + name="daily", + status="active", + ) + ) + db.commit() + + import core.db.engine as engine_module + import core.runs.tool_effect_recovery as recovery_module + from extensions.mcp_servers.automation_task_mcp import impl + + monkeypatch.setattr(engine_module, "SessionLocal", sessions) + monkeypatch.setattr(recovery_module, "SessionLocal", sessions) + before = await reconcile_scheduled_task_intent(intent) + assert before.outcome == "not_applied" + + result = impl.create_task( + user_id="user-1", + cron_expression="0 9 * * *", + prompt="send report", + name="daily", + tool_effect_id=intent.effect_id, + ) + assert result["ok"] is True + again = impl.create_task( + user_id="user-1", + cron_expression="0 9 * * *", + prompt="send report", + name="daily", + tool_effect_id=intent.effect_id, + ) + assert again == result + after = await reconcile_scheduled_task_intent(intent) + assert after.outcome == "applied" + assert after.result == result + with sessions() as db: + assert db.query(ScheduledTask).count() == 2 + assert db.get(ToolEffectReceipt, intent.effect_id).result_payload == result + + +def test_embedded_secrets_and_free_text_are_not_persisted(effect_env): + sessions, make_run = effect_env + make_run("run-secrets") + secret = "sk-123456789012345678901234567890" + ledger = ToolEffectJournal(sessions) + intent = ledger.begin_intent( + run_id="run-secrets", + owner="worker", + claim_owner="invocation", + tool_call_id="secret-call", + tool_name="Bash", + args={ + "command": f"curl -H 'Authorization: Bearer {secret}' https://x", + "url": f"https://example.test/?token={secret}", + }, + recovery_policy="never_replay", + ) + with sessions() as db: + row = ( + db.query(ToolEffectLedger) + .filter( + ToolEffectLedger.effect_id == intent.effect_id, + ToolEffectLedger.event_type == "intent", + ) + .one() + ) + assert secret not in json.dumps(row.redacted_args) + assert row.redacted_args["command"]["_withheld"] is True + assert "[REDACTED]" in row.redacted_args["url"] + + +@pytest.mark.asyncio +async def test_missing_run_binding_fails_closed_before_tool_invocation(effect_env): + sessions, _make_run = effect_env + middleware = durable_adapter(sessions) + agent = SimpleNamespace(state=SimpleNamespace(run_id=None, journal_owner=None)) + invoked = 0 + + async def tool(**_kwargs): + nonlocal invoked + invoked += 1 + yield ToolResponse(content=[TextBlock(text="wrong")]) + + with pytest.raises(ToolIntentCommitError): + _ = [ + item + async for item in middleware.on_acting( + agent, + {"tool_call": ToolCallBlock(id="x", name="Read", input="{}")}, + tool, + ) + ] + assert invoked == 0 + + +@pytest.mark.asyncio +async def test_middleware_leaves_adapter_exception_pending_for_recovery(effect_env): + sessions, make_run = effect_env + make_run("run-tool-failure") + middleware = durable_adapter(sessions) + agent = SimpleNamespace( + state=SimpleNamespace( + run_id="run-tool-failure", + journal_owner="worker", + tool_effect_links={}, + ) + ) + + async def broken(**_kwargs): + if False: + yield None + raise RuntimeError("adapter exploded") + + with pytest.raises(ToolOutcomeUnknown) as raised: + _ = [ + item + async for item in middleware.on_acting( + agent, + {"tool_call": ToolCallBlock(id="failure-call", name="Read", input="{}")}, + broken, + ) + ] + assert isinstance(raised.value.__cause__, RuntimeError) + with sessions() as db: + events = db.query(ToolEffectLedger).order_by(ToolEffectLedger.event_id).all() + assert [event.event_type for event in events] == ["intent"] + + +@pytest.mark.asyncio +async def test_cancelled_run_settles_pending_intent_without_delayed_replay(effect_env): + sessions, make_run = effect_env + make_run("run-cancelled-effect") + registry = ToolRecoveryRegistry() + invoked = 0 + + async def replay(_intent): + nonlocal invoked + invoked += 1 + return {"should_not": "run"} + + registry.register("Read", "replay_safe", replayer=replay) + ledger = ToolEffectJournal(sessions) + ledger.begin_intent( + run_id="run-cancelled-effect", + owner="worker", + claim_owner="dead-call", + tool_call_id="cancelled-call", + tool_name="Read", + args={"path": "/workspace/a.txt"}, + recovery_policy="replay_safe", + ) + assert RunJournal(sessions).cancel("run-cancelled-effect", reason="user cancelled") + + decisions = await recover_incomplete_tool_effects(journal=ledger, registry=registry) + + assert [item.action for item in decisions] == ["terminal_unknown"] + assert invoked == 0 + with sessions() as db: + run = db.get(ChatRun, "run-cancelled-effect") + events = ( + db.query(ToolEffectLedger) + .filter(ToolEffectLedger.run_id == run.run_id) + .order_by(ToolEffectLedger.event_id) + .all() + ) + assert run.status == "cancelled" + assert run.run_phase == "cancelled" + assert [item.event_type for item in events] == ["intent", "unknown_outcome"] + + +@pytest.mark.asyncio +async def test_middleware_commits_error_tool_response_as_failure(effect_env): + sessions, make_run = effect_env + make_run("run-tool-error-response") + middleware = durable_adapter(sessions) + agent = SimpleNamespace( + state=SimpleNamespace( + run_id="run-tool-error-response", + journal_owner="worker", + tool_effect_links={}, + ) + ) + + async def failed_response(**_kwargs): + yield ToolResponse( + content=[TextBlock(text="tool rejected input")], + state=ToolResultState.ERROR, + ) + + output = [ + item + async for item in middleware.on_acting( + agent, + {"tool_call": ToolCallBlock(id="error-response-call", name="Read", input="{}")}, + failed_response, + ) + ] + assert output[-1].state == ToolResultState.ERROR + with sessions() as db: + events = db.query(ToolEffectLedger).order_by(ToolEffectLedger.event_id).all() + assert [event.event_type for event in events] == ["intent", "failure"] + + +@pytest.mark.asyncio +async def test_middleware_returns_explicit_error_for_unregistered_tool(effect_env): + sessions, make_run = effect_env + make_run("run-unregistered-tool-error") + middleware = durable_adapter(sessions) + agent = SimpleNamespace( + state=SimpleNamespace( + run_id="run-unregistered-tool-error", + journal_owner="worker", + tool_effect_links={}, + ) + ) + original_error = "Remote API returned 404: resource not found or not authorized" + + async def failed_response(**_kwargs): + yield ToolResponse( + content=[TextBlock(text=original_error)], + state=ToolResultState.ERROR, + ) + + output = [ + item + async for item in middleware.on_acting( + agent, + { + "tool_call": ToolCallBlock( + id="unregistered-error-call", + name="remote_resource_lookup", + input='{"resource":"missing-or-private"}', + ) + }, + failed_response, + ) + ] + + assert output[-1].state == ToolResultState.ERROR + assert original_error in str(output[-1].content) + with sessions() as db: + events = db.query(ToolEffectLedger).order_by(ToolEffectLedger.event_id).all() + assert [event.event_type for event in events] == ["intent", "failure"] + assert events[0].recovery_policy == "never_replay" + assert original_error in str(events[-1].result_payload) + + +@pytest.mark.asyncio +async def test_agentscope_middleware_commits_intent_before_call_and_replays_same_shape( + effect_env, +): + sessions, make_run = effect_env + make_run("run-middleware") + middleware = durable_adapter(sessions) + agent = SimpleNamespace(state=SimpleNamespace(run_id="run-middleware", journal_owner="worker")) + invoked = 0 + saw_durable_intent = False + + async def execute_once(**_kwargs): + nonlocal invoked, saw_durable_intent + invoked += 1 + with sessions() as db: + saw_durable_intent = ( + db.query(ToolEffectLedger) + .filter( + ToolEffectLedger.run_id == "run-middleware", + ToolEffectLedger.event_type == "intent", + ) + .count() + == 1 + ) + block = TextBlock(text="same SSE-compatible result") + yield ToolChunk(content=[block], state=ToolResultState.RUNNING) + yield ToolResponse(content=[block], state=ToolResultState.SUCCESS) + + first_call = ToolCallBlock( + id="provider-call-1", + name="Read", + input='{"path":"/workspace/a.txt"}', + ) + first = [ + item + async for item in middleware.on_acting( + agent, + {"tool_call": first_call}, + execute_once, + ) + ] + + async def must_not_execute(**_kwargs): + nonlocal invoked + invoked += 1 + yield ToolResponse(content=[TextBlock(text="wrong")]) + + replay_call = ToolCallBlock( + id="provider-call-1", + name="Read", + input='{"path":"/workspace/a.txt"}', + ) + replayed = [ + item + async for item in middleware.on_acting( + agent, + {"tool_call": replay_call}, + must_not_execute, + ) + ] + + assert saw_durable_intent is True + assert invoked == 1 + assert [type(item) for item in first] == [ToolChunk, ToolResponse] + assert [type(item) for item in replayed] == [ToolChunk, ToolResponse] + assert replayed[-1].state == ToolResultState.SUCCESS + assert replayed[-1].content[0].text == "same SSE-compatible result" + + +@pytest.mark.asyncio +async def test_real_agentscope_public_reply_commits_intent_before_adapter(effect_env): + sessions, make_run = effect_env + make_run("run-public-agent") + saw_intent = False + invoked = 0 + + async def probe(value: int) -> ToolChunk: + """Return a test value.""" + nonlocal saw_intent, invoked + invoked += 1 + with sessions() as db: + saw_intent = ( + db.query(ToolEffectLedger) + .filter( + ToolEffectLedger.run_id == "run-public-agent", + ToolEffectLedger.event_type == "intent", + ) + .count() + == 1 + ) + return ToolChunk( + content=[TextBlock(text=f"value={value}")], + state=ToolResultState.SUCCESS, + ) + + class ToolThenAnswerModel: + from agentscope.formatter import OpenAIChatFormatter + formatter = OpenAIChatFormatter() + model = "tool-then-answer" + context_size = 32768 + + def __init__(self): + self.calls = 0 + + async def __call__(self, messages, tools=None, **_kwargs): + del messages + self.calls += 1 + if self.calls == 1: + assert any(item["function"]["name"] == "probe" for item in tools) + return ChatResponse( + content=[ToolCallBlock(id="public-call", name="probe", input='{"value":7}')], + is_last=False, + usage=ChatUsage(input_tokens=1, output_tokens=1, time=0.01), + ) + return ChatResponse( + content=[TextBlock(text="done")], + is_last=True, + usage=ChatUsage(input_tokens=1, output_tokens=1, time=0.01), + ) + + async def count_tokens(self, messages, tools=None): + del messages, tools + return 1 + + state = AgentRuntimeState( + run_id="run-public-agent", + journal_owner="worker", + permission_context=PermissionContext(mode=PermissionMode.BYPASS), + ) + agent = Agent( + name="ledger-public-seam", + system_prompt="Use the probe tool once.", + model=ToolThenAnswerModel(), + toolkit=Toolkit(tools=[FunctionTool(probe)]), + middlewares=[durable_adapter(sessions)], + state=state, + react_config=ReActConfig(max_iters=2), + ) + reply = await agent.reply(UserMsg(name="user", content="run probe")) + assert reply.get_text_content() == "done" + assert invoked == 1 + assert saw_intent is True + with sessions() as db: + assert [ + row.event_type for row in db.query(ToolEffectLedger).order_by(ToolEffectLedger.event_id) + ] == ["intent", "result"] diff --git a/src/backend/tests/orchestration/test_tool_effect_ledger_4.py b/src/backend/tests/orchestration/test_tool_effect_ledger_4.py new file mode 100644 index 00000000..8d2b031c --- /dev/null +++ b/src/backend/tests/orchestration/test_tool_effect_ledger_4.py @@ -0,0 +1,459 @@ +from tests.orchestration.tool_effect_support import ( + log_records, + asyncio, + json, + ThreadPoolExecutor, + datetime, + timedelta, + timezone, + Barrier, + SimpleNamespace, + pytest, + Agent, + ReActConfig, + TextBlock, + ToolCallBlock, + ToolResultState, + UserMsg, + ChatResponse, + ChatUsage, + PermissionContext, + PermissionMode, + FunctionTool, + Toolkit, + ToolChunk, + ToolResponse, + Base, + ChatRun, + ChatSession, + ToolCallLog, + ToolEffectLease, + ToolEffectLedger, + ToolEffectReceipt, + ScheduledTask, + AgentRuntimeState, + make_adapter, + durable_adapter, + _plain_replay_args, + reconcile_scheduled_task_intent, + RunJournal, + DEFAULT_TOOL_RECOVERY_REGISTRY, + ToolRecoveryRegistry, + ToolEffectError, + ToolIdempotencyConflict, + ToolIntentCommitError, + ToolOutcomeUnknown, + ToolEffectGateway, + ToolEffectJournal, + ToolIntent, + recover_incomplete_tool_effects, + create_engine, + sessionmaker, + QueuePool, + MutableClock, + effect_env, +) + +@pytest.mark.asyncio +async def test_public_reply_cancellation_stops_tool_without_late_terminal_write(effect_env): + sessions, make_run = effect_env + make_run("run-public-cancel") + entered = asyncio.Event() + release = asyncio.Event() + side_effects = 0 + + async def blocking_probe(value: int) -> ToolChunk: + """Wait until the caller releases the test tool.""" + nonlocal side_effects + entered.set() + await release.wait() + side_effects += value + return ToolChunk( + content=[TextBlock(text=f"value={value}")], + state=ToolResultState.SUCCESS, + ) + + class BlockingToolModel: + from agentscope.formatter import OpenAIChatFormatter + formatter = OpenAIChatFormatter() + model = "blocking-tool" + context_size = 32768 + + async def __call__(self, messages, tools=None, **_kwargs): + del messages + assert any(item["function"]["name"] == "blocking_probe" for item in tools) + return ChatResponse( + content=[ + ToolCallBlock( + id="cancel-call", + name="blocking_probe", + input='{"value":1}', + ) + ], + is_last=False, + usage=ChatUsage(input_tokens=1, output_tokens=1, time=0.01), + ) + + async def count_tokens(self, messages, tools=None): + del messages, tools + return 1 + + agent = Agent( + name="ledger-cancel-seam", + system_prompt="Use the blocking probe.", + model=BlockingToolModel(), + toolkit=Toolkit(tools=[FunctionTool(blocking_probe, is_concurrency_safe=False)]), + middlewares=[durable_adapter(sessions)], + state=AgentRuntimeState( + run_id="run-public-cancel", + journal_owner="worker", + permission_context=PermissionContext(mode=PermissionMode.BYPASS), + ), + react_config=ReActConfig(max_iters=2, interruption_raise_cancelled_error=True), + ) + reply_task = asyncio.create_task(agent.reply(UserMsg(name="user", content="start"))) + await entered.wait() + assert RunJournal(sessions).cancel("run-public-cancel", reason="test cancellation") + reply_task.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(reply_task, timeout=5) + release.set() + await asyncio.sleep(0) + assert side_effects == 0 + with sessions() as db: + run = db.get(ChatRun, "run-public-cancel") + assert run.status == "cancelled" + assert run.run_phase == "cancelled" + assert [ + row.event_type + for row in db.query(ToolEffectLedger) + .filter(ToolEffectLedger.run_id == "run-public-cancel") + .order_by(ToolEffectLedger.event_id) + ] == ["intent"] + + +@pytest.mark.asyncio +async def test_restart_sweep_classifies_replay_reconcile_and_never_replay(effect_env): + sessions, make_run = effect_env + clock = MutableClock() + registry = ToolRecoveryRegistry() + + async def replay(_intent): + return {"replayed": True} + + registry.register("safe_read", "replay_safe", replayer=replay) + + async def reconcile(_intent): + return {"external_status": "committed"} + + registry.register("external_write", "reconcile", reconciler=reconcile) + ledger = ToolEffectJournal(sessions, clock=clock) + effects = {} + for run_id, tool_name, policy in ( + ("run-sweep-replay", "safe_read", "replay_safe"), + ("run-sweep-reconcile", "external_write", "reconcile"), + ("run-sweep-never", "unknown_write", "never_replay"), + ): + make_run(run_id) + effects[run_id] = ledger.begin_intent( + run_id=run_id, + owner="worker", + claim_owner="crashed-tool", + tool_call_id=f"call-{run_id}", + tool_name=tool_name, + args={"target": run_id}, + recovery_policy=policy, + idempotency_key=f"key-{run_id}", + lease_seconds=1, + ).effect_id + clock.advance(2) + with sessions() as db: + db.query(ChatRun).filter(ChatRun.run_id.in_(effects)).update( + {"lease_expires_at": clock() - timedelta(seconds=1)}, + synchronize_session=False, + ) + db.commit() + + decisions = await recover_incomplete_tool_effects(journal=ledger, registry=registry) + assert {(item.run_id, item.action) for item in decisions} == { + ("run-sweep-replay", "replayed"), + ("run-sweep-reconcile", "reconciled"), + ("run-sweep-never", "needs_attention"), + } + with sessions() as db: + assert db.get(ChatRun, "run-sweep-replay").status == "running" + assert db.get(ChatRun, "run-sweep-reconcile").run_phase == "tool_result_committed" + assert db.get(ChatRun, "run-sweep-never").status == "needs_attention" + reconciled = ( + db.query(ToolEffectLedger) + .filter( + ToolEffectLedger.effect_id == effects["run-sweep-reconcile"], + ToolEffectLedger.event_type == "result", + ) + .one() + ) + assert reconciled.result_payload == {"external_status": "committed"} + + +@pytest.mark.asyncio +async def test_recovery_adapter_failure_does_not_starve_later_intent(effect_env): + sessions, make_run = effect_env + clock = MutableClock() + registry = ToolRecoveryRegistry() + + async def broken(_intent): + raise RuntimeError("temporary adapter outage") + + async def healthy(_intent): + return {"ok": True} + + registry.register("broken_read", "replay_safe", replayer=broken) + registry.register("healthy_read", "replay_safe", replayer=healthy) + ledger = ToolEffectJournal(sessions, clock=clock) + effects = [] + for run_id, tool_name in (("run-broken", "broken_read"), ("run-healthy", "healthy_read")): + make_run(run_id) + effects.append( + ledger.begin_intent( + run_id=run_id, + owner="worker", + claim_owner="crashed", + tool_call_id=f"call-{run_id}", + tool_name=tool_name, + args={"q": run_id}, + recovery_policy="replay_safe", + lease_seconds=1, + ).effect_id + ) + clock.advance(2) + with sessions() as db: + db.query(ChatRun).filter(ChatRun.run_id.in_(["run-broken", "run-healthy"])).update( + {"lease_expires_at": clock() - timedelta(seconds=1)}, synchronize_session=False + ) + db.commit() + decisions = await recover_incomplete_tool_effects(journal=ledger, registry=registry) + assert {(item.run_id, item.action) for item in decisions} == { + ("run-broken", "retry"), + ("run-healthy", "replayed"), + } + assert ledger.terminal_decision(effects[0]) is None + assert ledger.terminal_decision(effects[1]).result == {"ok": True} + run_recovery = RunJournal(sessions, clock=clock).recover() + assert "run-broken" not in {item.run_id for item in run_recovery} + + +@pytest.mark.asyncio +async def test_recovery_heartbeat_prevents_second_sweep_from_replaying_long_adapter(effect_env): + sessions, make_run = effect_env + make_run("run-long-recovery") + ledger = ToolEffectJournal(sessions) + intent = ledger.begin_intent( + run_id="run-long-recovery", + owner="worker", + claim_owner="crashed", + tool_call_id="long-recovery-call", + tool_name="long_read", + args={"query": "slow"}, + recovery_policy="replay_safe", + lease_seconds=1, + ) + expired = datetime.now(timezone.utc) - timedelta(seconds=1) + with sessions() as db: + db.query(ChatRun).filter(ChatRun.run_id == "run-long-recovery").update( + {"lease_expires_at": expired} + ) + db.query(ToolEffectLease).filter(ToolEffectLease.effect_id == intent.effect_id).update( + {"lease_expires_at": expired} + ) + db.commit() + + started = asyncio.Event() + release = asyncio.Event() + replays = 0 + + async def replay(_intent): + nonlocal replays + replays += 1 + started.set() + await release.wait() + return {"ok": True} + + registry = ToolRecoveryRegistry() + registry.register("long_read", "replay_safe", replayer=replay) + first = asyncio.create_task( + recover_incomplete_tool_effects( + journal=ledger, + registry=registry, + lease_seconds=1, + heartbeat_interval=0.1, + ) + ) + await started.wait() + await asyncio.sleep(1.1) + second = await recover_incomplete_tool_effects( + journal=ledger, + registry=registry, + lease_seconds=1, + heartbeat_interval=0.1, + ) + assert second == [] + assert replays == 1 + release.set() + decisions = await first + assert [(item.effect_id, item.action) for item in decisions] == [(intent.effect_id, "replayed")] + + +def test_tool_call_log_is_a_projection_linked_to_authoritative_effect(effect_env, monkeypatch): + sessions, make_run = effect_env + make_run("run-projection") + ledger = ToolEffectJournal(sessions) + intent = ledger.begin_intent( + run_id="run-projection", + owner="worker", + claim_owner="projection-invocation", + tool_call_id="projection-call", + tool_name="Read", + args={"path": "/workspace/a.txt"}, + recovery_policy="replay_safe", + ) + ledger.commit_result( + intent.effect_id, + run_owner="worker", + claim_owner="projection-invocation", + result={"ok": True}, + ) + + from core.harness import log_writer as log_service + + monkeypatch.setattr(log_records, "SessionLocal", sessions) + projection_secret = "projection-secret-token-123456" + log_records.write_tool_call_sync( + { + "run_id": "run-projection", + "chat_id": "chat-1", + "user_id": "user-1", + "tool_name": "Read", + "tool_call_id": "projection-call", + "tool_args": { + "path": "/workspace/a.txt", + "command": f"curl -H 'Authorization: Bearer {projection_secret}' https://x", + "url": f"https://example.test/?token={projection_secret}", + "callback": f"https://example.test/cb?password={projection_secret}", + }, + "tool_result": "contents", + "status": "success", + } + ) + with sessions() as db: + projection = db.query(ToolCallLog).one() + assert projection.effect_id == intent.effect_id + assert projection.id == intent.intent.result_id + assert projection.tool_result == "contents" + assert projection_secret not in json.dumps(projection.tool_args) + assert projection.tool_args["command"]["_withheld"] is True + assert "[REDACTED]" in projection.tool_args["url"] + assert "[REDACTED]" in projection.tool_args["callback"] + + log_records.write_tool_call_sync( + { + "run_id": "run-projection", + "chat_id": "chat-1", + "user_id": "user-1", + "tool_name": "Read", + "tool_call_id": "new-provider-call-after-recovery", + "effect_id": intent.effect_id, + "result_id": intent.intent.result_id, + "tool_args": {"path": "/workspace/a.txt"}, + "tool_result": "replayed contents", + "status": "success", + } + ) + with sessions() as db: + assert db.query(ToolCallLog).count() == 1 + projection = db.query(ToolCallLog).one() + assert projection.id == intent.intent.result_id + assert projection.tool_call_id == "new-provider-call-after-recovery" + assert projection.tool_result == "replayed contents" + + +def test_prune_settled_drops_only_old_rows_of_settled_runs(effect_env): + sessions, make_run = effect_env + clock = MutableClock() + ledger = ToolEffectJournal(sessions, clock=clock) + make_run("run-old") + make_run("run-live", chat_id="chat-2") + old = ledger.begin_intent( + run_id="run-old", + owner="worker", + claim_owner="invocation", + tool_call_id="call-old", + tool_name="write", + args={"path": "/tmp/a"}, + recovery_policy="never_replay", + idempotency_key="prune-old", + ) + ledger.commit_result( + old.effect_id, + run_owner="worker", + claim_owner="invocation", + result={"ok": True}, + ) + live = ledger.begin_intent( + run_id="run-live", + owner="worker", + claim_owner="invocation", + tool_call_id="call-live", + tool_name="write", + args={"path": "/tmp/b"}, + recovery_policy="never_replay", + idempotency_key="prune-live", + ) + from core.db.models import ChatRunOperation, HarnessEventLog, HarnessUsageAttempt + + with sessions() as db: + run = db.get(ChatRun, "run-old") + run.status = "completed" + db.add( + HarnessUsageAttempt( + attempt_id="hat-old", + run_id="run-old", + attempt_seq=1, + kind="tool", + operation_name="write", + status="success", + ) + ) + db.add( + HarnessEventLog( + event_id="hev-old", + run_id="run-old", + event_seq=1, + event_type="tool_result", + phase="post", + payload={}, + ) + ) + db.commit() + clock.advance(8 * 24 * 3600) + assert ledger.prune_settled() >= 4 + with sessions() as db: + for model in ( + ToolEffectLedger, + HarnessUsageAttempt, + HarnessEventLog, + ChatRunOperation, + ): + assert db.query(model).filter(model.run_id == "run-old").count() == 0 + assert ( + db.query(ToolEffectLedger) + .filter(ToolEffectLedger.run_id == "run-live") + .count() + == 1 + ) + assert ( + db.query(ChatRunOperation) + .filter(ChatRunOperation.run_id == "run-live") + .count() + > 0 + ) + assert ledger.pending_effect_ids() == [live.effect_id] diff --git a/src/backend/tests/orchestration/test_tool_effect_ledger_5.py b/src/backend/tests/orchestration/test_tool_effect_ledger_5.py new file mode 100644 index 00000000..db62959a --- /dev/null +++ b/src/backend/tests/orchestration/test_tool_effect_ledger_5.py @@ -0,0 +1,133 @@ +from tests.orchestration.tool_effect_support import ( + log_records, + asyncio, + json, + ThreadPoolExecutor, + datetime, + timedelta, + timezone, + Barrier, + SimpleNamespace, + pytest, + Agent, + ReActConfig, + TextBlock, + ToolCallBlock, + ToolResultState, + UserMsg, + ChatResponse, + ChatUsage, + PermissionContext, + PermissionMode, + FunctionTool, + Toolkit, + ToolChunk, + ToolResponse, + Base, + ChatRun, + ChatSession, + ToolCallLog, + ToolEffectLease, + ToolEffectLedger, + ToolEffectReceipt, + ScheduledTask, + AgentRuntimeState, + make_adapter, + durable_adapter, + _plain_replay_args, + reconcile_scheduled_task_intent, + RunJournal, + DEFAULT_TOOL_RECOVERY_REGISTRY, + ToolRecoveryRegistry, + ToolEffectError, + ToolIdempotencyConflict, + ToolIntentCommitError, + ToolOutcomeUnknown, + ToolEffectGateway, + ToolEffectJournal, + ToolIntent, + recover_incomplete_tool_effects, + create_engine, + sessionmaker, + QueuePool, + MutableClock, + effect_env, +) + +def test_prune_settled_eventually_reclaims_needs_attention_runs(effect_env): + """needs_attention 保留得更久，但不是永远。 + + 原先它被整个排除在清理之外，等于永不回收：生产库里每一条超出保留期的 + journal 记录——四万条——都属于 needs_attention 的 run，被永久钉住，还要被 + 每五分钟一轮的清理反复全表扫。保留期该是"更长"，不是"无限"。 + """ + from core.db.models import HarnessEventLog + from core.runs.tool_effects.journal_constants import ATTENTION_RETENTION_FACTOR as _ATTENTION_RETENTION_FACTOR + + sessions, make_run = effect_env + clock = MutableClock() + ledger = ToolEffectJournal(sessions, clock=clock) + make_run("run-attention") + intent = ledger.begin_intent( + run_id="run-attention", + owner="worker", + claim_owner="invocation", + tool_call_id="call-attention", + tool_name="write", + args={"path": "/tmp/a"}, + recovery_policy="never_replay", + idempotency_key="prune-attention", + ) + ledger.commit_result( + intent.effect_id, + run_owner="worker", + claim_owner="invocation", + result={"ok": True}, + ) + with sessions() as db: + run = db.get(ChatRun, "run-attention") + run.status = "needs_attention" + # recovery_snapshot 是整轮对话的快照，库里最大的单个字段；它跟 journal + # 走同一套分档，不能因为状态是 needs_attention 就永久留着 + run.completed_at = clock.now + db.add( + HarnessEventLog( + event_id="hev-attention", + run_id="run-attention", + event_seq=1, + event_type="tool_result", + phase="post", + payload={}, + ) + ) + db.commit() + + # 保留期显式传入，两次推进都从它算——否则 prune_settled 的默认值一改， + # 这里的天数就悄悄测不到边界了 + retention_days = 7 + + # 刚过普通终态的保留期：记录必须还在 + clock.advance((retention_days + 1) * 24 * 3600) + ledger.prune_settled(retention_days=retention_days) + with sessions() as db: + assert ( + db.query(ToolEffectLedger).filter(ToolEffectLedger.run_id == "run-attention").count() + > 0 + ) + assert ( + db.query(HarnessEventLog).filter(HarnessEventLog.run_id == "run-attention").count() == 1 + ) + assert db.get(ChatRun, "run-attention").recovery_snapshot is not None + + # 过了加长保留期：回收，不能无限期占着库 + clock.advance(retention_days * _ATTENTION_RETENTION_FACTOR * 24 * 3600) + ledger.prune_settled(retention_days=retention_days) + with sessions() as db: + assert ( + db.query(ToolEffectLedger).filter(ToolEffectLedger.run_id == "run-attention").count() + == 0 + ) + assert ( + db.query(HarnessEventLog).filter(HarnessEventLog.run_id == "run-attention").count() == 0 + ) + assert db.get(ChatRun, "run-attention").recovery_snapshot is None diff --git a/src/backend/tests/orchestration/test_user_question_passthrough.py b/src/backend/tests/orchestration/test_user_question_passthrough.py index bc40f821..e579dc01 100644 --- a/src/backend/tests/orchestration/test_user_question_passthrough.py +++ b/src/backend/tests/orchestration/test_user_question_passthrough.py @@ -6,19 +6,32 @@ """ from __future__ import annotations +from core.agent_runtime.integrations import memory_identity as memory_identity_module +from core.agent_runtime.integrations import memory_session as memory_session_module +from core.llm.runtime import streaming as streaming_module +from core.llm.citations import allocator as allocator_module from types import SimpleNamespace +import core.agent_runtime.integrations.memory as memory +import core.agent_runtime.runner as runner +import core.agent_runtime.targets as targets +import core.capabilities.catalog.resolver as catalog_resolver +import core.llm.citations.anchor as citation_anchor +import core.llm.factory as factory +import core.llm.runtime.streaming as streaming import pytest -from orchestration.memory_integration import SessionMemory +from core.agent_runtime.integrations.memory_session import SessionMemory @pytest.mark.asyncio async def test_user_question_events_reach_the_stream(monkeypatch): from core.db import engine as db_engine - from core.llm import builtin_subagents - from core.services import compaction_service, user_agent_service, user_service - from orchestration import workflow + from core.capabilities.agents import builtin as builtin_subagents + from core.chat.compaction import service as compaction_service + from core.capabilities.agents import edition as user_agent_service + from core.auth import users as user_service + pass class DummySession: def __enter__(self): @@ -78,20 +91,19 @@ async def no_compaction(_chat_id, messages, **_kwargs): ) monkeypatch.setattr(builtin_subagents, "merge_builtin_subagents", lambda *_a, **_kw: []) monkeypatch.setattr(compaction_service, "maybe_run_pre_turn_compaction", no_compaction) - monkeypatch.setattr(workflow, "create_agent_executor", create_agent) - monkeypatch.setattr(workflow, "open_session_memory", no_memory) - monkeypatch.setattr(workflow, "build_user_identity_block", no_identity) - monkeypatch.setattr(workflow, "anchor_start_for_chat", lambda _chat_id: 0) - monkeypatch.setattr(workflow, "enabled_skill_ids_from_context", lambda _ctx: []) - monkeypatch.setattr(workflow, "enabled_mcp_ids_from_context", lambda _ctx: []) - monkeypatch.setattr(workflow, "enabled_kb_ids_from_context", lambda _ctx: []) - monkeypatch.setattr(workflow, "_resolve_mode_spec", lambda _ctx: None) - monkeypatch.setattr(workflow, "StreamingAgent", FakeStreamingAgent) - monkeypatch.setattr(workflow, "_persistent_clients", []) + monkeypatch.setattr(factory, "create_agent_executor", create_agent) + monkeypatch.setattr(memory_session_module, 'open_session_memory', no_memory) + monkeypatch.setattr(memory_identity_module, 'build_user_identity_block', no_identity) + monkeypatch.setattr(allocator_module, 'anchor_start_for_chat', lambda _chat_id: 0) + monkeypatch.setattr(catalog_resolver, "enabled_skill_ids_from_context", lambda _ctx: []) + monkeypatch.setattr(catalog_resolver, "enabled_mcp_ids_from_context", lambda _ctx: []) + monkeypatch.setattr(catalog_resolver, "enabled_kb_ids_from_context", lambda _ctx: []) + monkeypatch.setattr(targets, "_resolve_mode_spec", lambda _ctx: None) + monkeypatch.setattr(streaming_module, 'StreamingAgent', FakeStreamingAgent) chunks = [ chunk - async for chunk in workflow.astream_chat_workflow( + async for chunk in runner.astream_chat_workflow( session_messages=[{"role": "user", "content": "问我一个问题"}], user_message="问我一个问题", context={ diff --git a/src/backend/tests/orchestration/test_worker_base.py b/src/backend/tests/orchestration/test_worker_base.py index b4717881..732abde8 100644 --- a/src/backend/tests/orchestration/test_worker_base.py +++ b/src/backend/tests/orchestration/test_worker_base.py @@ -7,10 +7,9 @@ import asyncio +import orchestration.workers._worker_base as wb import pytest -from orchestration.schedulers import _worker_base as wb - @pytest.fixture(autouse=True) def redis_backend(monkeypatch): diff --git a/src/backend/tests/orchestration/test_workflow_hint_consistency.py b/src/backend/tests/orchestration/test_workflow_hint_consistency.py index 7db03577..b68235a1 100644 --- a/src/backend/tests/orchestration/test_workflow_hint_consistency.py +++ b/src/backend/tests/orchestration/test_workflow_hint_consistency.py @@ -83,7 +83,7 @@ def test_hint_states_the_real_wait_default(hint: str): def test_hint_progress_interval_matches_runtime(hint: str): """提示段里的唤醒间隔必须等于 job_runtime 的真值（曾漂移成 15 分钟）。""" - seconds = _module_constant("orchestration/jobs/supervisor.py", "PROGRESS_WAKE_SECONDS") + seconds = _module_constant('core/jobs/supervisor.py', "PROGRESS_WAKE_SECONDS") stated = re.search(r"默认\s*(\d+)\s*分钟[^）]*`progress_wake_sec`", hint) assert stated, "提示段应说明默认唤醒间隔（写成「默认 N 分钟，`progress_wake_sec` 可调」）" assert int(stated.group(1)) * 60 == int( diff --git a/src/backend/tests/orchestration/test_workflow_model_runtime.py b/src/backend/tests/orchestration/test_workflow_model_runtime.py index cb5ac698..cee7e9e4 100644 --- a/src/backend/tests/orchestration/test_workflow_model_runtime.py +++ b/src/backend/tests/orchestration/test_workflow_model_runtime.py @@ -1,6 +1,8 @@ + from types import SimpleNamespace -from orchestration import workflow +import core.agent_runtime.budget as budget +import core.models.context_window as context_manager def test_resolve_agent_model_runtime_prefers_baked_model_over_none_fallback(): @@ -9,7 +11,7 @@ def test_resolve_agent_model_runtime_prefers_baked_model_over_none_fallback(): state=SimpleNamespace(model_name=""), ) - assert workflow._resolve_agent_model_runtime(agent, None) == ( + assert budget._resolve_agent_model_runtime(agent, None) == ( "deepseekv4-flash", 393_216, ) @@ -18,7 +20,7 @@ def test_resolve_agent_model_runtime_prefers_baked_model_over_none_fallback(): def test_resolve_agent_model_runtime_supports_wrapper_name_and_window_lookup(monkeypatch): seen = [] monkeypatch.setattr( - workflow, + context_manager, "resolve_model_context_window", lambda model_name: seen.append(model_name) or 131_072, ) @@ -27,7 +29,7 @@ def test_resolve_agent_model_runtime_supports_wrapper_name_and_window_lookup(mon state=SimpleNamespace(model_name=""), ) - assert workflow._resolve_agent_model_runtime(agent, "None") == ( + assert budget._resolve_agent_model_runtime(agent, "None") == ( "qwen-max", 131_072, ) diff --git a/src/backend/tests/orchestration/tool_effect_support.py b/src/backend/tests/orchestration/tool_effect_support.py new file mode 100644 index 00000000..e3c69680 --- /dev/null +++ b/src/backend/tests/orchestration/tool_effect_support.py @@ -0,0 +1,81 @@ +"""Durable, idempotent tool side-effect execution and recovery.""" + +from __future__ import annotations +from core.harness import log_records + +import asyncio +import json +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timedelta, timezone +from threading import Barrier +from types import SimpleNamespace + +import pytest +from agentscope.agent import Agent, ReActConfig +from agentscope.message import TextBlock, ToolCallBlock, ToolResultState, UserMsg +from agentscope.model import ChatResponse, ChatUsage +from agentscope.permission import PermissionContext, PermissionMode +from agentscope.tool import FunctionTool, Toolkit +from agentscope.tool._response import ToolChunk, ToolResponse +from core.db.engine import Base +from core.db.models import ChatRun, ChatSession, ToolCallLog, ToolEffectLease, ToolEffectLedger, ToolEffectReceipt +from features.automation.models import ScheduledTask +from core.llm.runtime.state import AgentRuntimeState +from tests.orchestration.hook_support import make_adapter, durable_adapter +from core.runs.tool_effect_recovery import _plain_replay_args, reconcile_scheduled_task_intent +from core.runs.journal import RunJournal +from core.runs.tool_effects.registry import DEFAULT_TOOL_RECOVERY_REGISTRY, ToolRecoveryRegistry +from core.runs.tool_effects.errors import ToolEffectError, ToolIdempotencyConflict, ToolIntentCommitError, ToolOutcomeUnknown +from core.runs.tool_effects.gateway import ToolEffectGateway +from core.runs.tool_effects.journal import ToolEffectJournal +from core.runs.tool_effects.types import ToolIntent +from core.runs.tool_effects.recovery import recover_incomplete_tool_effects +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker +from sqlalchemy.pool import QueuePool + + +class MutableClock: + def __init__(self) -> None: + self.now = datetime.now(timezone.utc) + + def __call__(self) -> datetime: + return self.now + + def advance(self, seconds: float) -> None: + self.now += timedelta(seconds=seconds) + + +@pytest.fixture() +def effect_env(tmp_path): + engine = create_engine( + f"sqlite:///{tmp_path / 'tool-effects.sqlite'}", + connect_args={"check_same_thread": False}, + ) + Base.metadata.create_all(engine) + sessions = sessionmaker(bind=engine, expire_on_commit=False) + + def make_run( + run_id: str, + *, + owner: str = "worker", + user_id: str = "user-1", + chat_id: str = "chat-1", + ) -> None: + with sessions() as db: + if db.get(ChatSession, chat_id) is None: + db.add(ChatSession(chat_id=chat_id, user_id=user_id, title="test")) + db.commit() + journal = RunJournal(sessions) + journal.accept( + run_id=run_id, + message_id=f"msg-{run_id}", + chat_id=chat_id, + user_id=user_id, + request_payload={"kind": "chat"}, + recovery_snapshot={"kind": "chat", "worker_args": {}}, + ) + assert journal.claim(run_id, owner=owner, lease_seconds=300) + + yield sessions, make_run + engine.dispose() diff --git a/src/backend/tests/test_device_link.py b/src/backend/tests/personal_agent/test_device_link.py similarity index 97% rename from src/backend/tests/test_device_link.py rename to src/backend/tests/personal_agent/test_device_link.py index d1ddba1e..6b978365 100644 --- a/src/backend/tests/test_device_link.py +++ b/src/backend/tests/personal_agent/test_device_link.py @@ -6,8 +6,8 @@ import httpx import pytest -from core.device_link import runner as link_runner -from core.device_link import store +from personal_agent.device_link import runner as link_runner +from personal_agent.device_link import store ACCOUNT = "cloud:cloud.example:443:owner" CLOUD = "https://cloud.example" @@ -160,7 +160,7 @@ def test_links_are_private_and_per_account(home): def test_probe_never_reaches_the_cloud_vocabulary(): - from core.device_link import CAPABILITIES + from personal_agent.device_link import CAPABILITIES assert "probe" not in CAPABILITIES and "request_permissions" not in CAPABILITIES @@ -170,7 +170,7 @@ def api(monkeypatch): from fastapi import FastAPI from fastapi.testclient import TestClient - from api.routes.v1 import device_link + from features.desktop.api import device_link from core.auth.backend import UserContext, get_current_user async def probe(action): diff --git a/src/backend/tests/prompts/test_desktop_delivery_tools.py b/src/backend/tests/prompts/test_desktop_delivery_tools.py index aef87dc5..db9dd2c7 100644 --- a/src/backend/tests/prompts/test_desktop_delivery_tools.py +++ b/src/backend/tests/prompts/test_desktop_delivery_tools.py @@ -12,7 +12,7 @@ def test_factory_delivery_tools_match_execution_mode(tmp_path): from core.db.engine import Base, engine import core.db.models from core.llm import agent_factory -import core.agent_skills.loader as skill_loader +import core.capabilities.skills.loader as skill_loader from core.config import local_mode Base.metadata.create_all(engine) class EmptyLoader: diff --git a/src/backend/tests/prompts/test_desktop_environment.py b/src/backend/tests/prompts/test_desktop_environment.py index 2c12c021..3fac8e85 100644 --- a/src/backend/tests/prompts/test_desktop_environment.py +++ b/src/backend/tests/prompts/test_desktop_environment.py @@ -1,4 +1,5 @@ """Desktop environment facts must match the runner and stay out of cloud prompts.""" +from core.llm import prompt_versions as prompt_versions_module import xml.etree.ElementTree as ET import pytest @@ -10,15 +11,15 @@ @pytest.fixture(autouse=True) def isolated_prompt_sources(monkeypatch, tmp_path): from prompts import prompt_runtime - from core.services import prompt_version_service + from core.llm import prompt_versions as prompt_version_service from pathlib import Path stock = Path(prompt_runtime.__file__).parent / "prompt_text/default/system/40_format.system.md" monkeypatch.setattr(prompt_runtime, "_load_db_prompt_parts", lambda: {}) monkeypatch.setattr(prompt_runtime, "_get_db_prompt_version", lambda: "desktop-test") monkeypatch.setattr( - prompt_version_service, - "get_active_version", + prompt_versions_module, + 'get_active_version', lambda kind, **kwargs: ( { "id": "desktop-test", @@ -28,7 +29,7 @@ def isolated_prompt_sources(monkeypatch, tmp_path): else None ), ) - monkeypatch.setattr("core.services.local_grant_service._data_dir", lambda: tmp_path) + monkeypatch.setattr('features.desktop.local.grants._data_dir', lambda: tmp_path) prompt_runtime._prompt_cache.clear() @@ -74,7 +75,7 @@ def test_cloud_has_no_desktop_environment(monkeypatch): def test_environment_refreshes_date_permissions_and_workspace_without_stale_cache( monkeypatch, tmp_path ): - from core.services import local_grant_service as grants + from features.desktop.local import grants as grants monkeypatch.setattr("core.config.local_mode.local_mode_enabled", lambda: True) monkeypatch.setattr(grants, "_data_dir", lambda: tmp_path) @@ -133,7 +134,7 @@ def test_real_factory_modes_and_subagents_receive_environment(tmp_path): import core.db.models from core.db.engine import Base, engine from core.llm import agent_factory -import core.agent_skills.loader as skill_loader +import core.capabilities.skills.loader as skill_loader from core.config import local_mode Base.metadata.create_all(engine) local_mode.local_mode_enabled = lambda: True diff --git a/src/backend/tests/prompts/test_desktop_prompt_versions.py b/src/backend/tests/prompts/test_desktop_prompt_versions.py index 9cb1b19d..02943cb4 100644 --- a/src/backend/tests/prompts/test_desktop_prompt_versions.py +++ b/src/backend/tests/prompts/test_desktop_prompt_versions.py @@ -1,20 +1,22 @@ """Desktop versions use real CRUD; previews never activate drafts.""" +from core.llm import prompt_storage as prompt_storage_module +from core.llm import prompt_versions as prompt_versions_module import pytest from sqlalchemy import create_engine from sqlalchemy.orm import Session from core.db.models import ContentBlock -from core.services import prompt_version_service as pvs +from core.llm import prompt_storage as pvs from prompts.desktop_templates import render_desktop_part, desktop_version @pytest.fixture def db(monkeypatch): engine = create_engine("sqlite://") ContentBlock.__table__.create(engine) - pvs.invalidate_cache() + prompt_storage_module.invalidate_cache() with Session(engine) as session: monkeypatch.setattr(pvs, "SessionLocal", lambda: Session(engine)) yield session - pvs.invalidate_cache() + prompt_storage_module.invalidate_cache() engine.dispose() @@ -22,23 +24,23 @@ def test_seed_clone_activate_disable_and_preserve(db, monkeypatch): from prompts.prompt_runtime import build_system_prompt, invalidate_prompt_cache from prompts.prompt_config import PromptConfig monkeypatch.setattr("core.config.local_mode.local_mode_enabled", lambda: True) - pvs.seed_from_filesystem(db=db) - assert pvs.get_active_version("desktop", db=db)["id"] == "default" + prompt_versions_module.seed_from_filesystem(db=db) + assert prompt_versions_module.get_active_version("desktop", db=db)["id"] == "default" original = render_desktop_part("guidance") - pvs.upsert_version("desktop", "draft", from_id="default", db=db) - draft = pvs.get_version("desktop", "draft", db=db) + prompt_versions_module.upsert_version("desktop", "draft", from_id="default", db=db) + draft = prompt_versions_module.get_version("desktop", "draft", db=db) for part in draft["parts"]: if part["part_id"] == "guidance": part["content"] = "UNIQUE_DESKTOP_VERSION" - pvs.upsert_version("desktop", "draft", parts=draft["parts"], db=db) + prompt_versions_module.upsert_version("desktop", "draft", parts=draft["parts"], db=db) assert render_desktop_part("guidance") == original invalidate_prompt_cache() before = build_system_prompt(PromptConfig(), {"chat_id": "version-test"}) - pvs.activate_version("desktop", "draft", db=db) + prompt_versions_module.activate_version("desktop", "draft", db=db) after = build_system_prompt(PromptConfig(), {"chat_id": "version-test"}) assert "UNIQUE_DESKTOP_VERSION" not in before assert "UNIQUE_DESKTOP_VERSION" in after - pvs.seed_from_filesystem(db=db) + prompt_versions_module.seed_from_filesystem(db=db) assert render_desktop_part("guidance") == "UNIQUE_DESKTOP_VERSION" with desktop_version({"parts": [{"part_id": "guidance", "content": "hidden", "is_enabled": False}]}): assert render_desktop_part("guidance") == "" @@ -50,8 +52,8 @@ def test_seed_clone_activate_disable_and_preserve(db, monkeypatch): async def test_full_preview_does_not_activate_version(db, monkeypatch): from api.routes.v1.prompt_management import desktop_preview, DesktopPreviewRequest monkeypatch.setattr("core.config.local_mode.local_mode_enabled", lambda: False) - pvs.seed_from_filesystem(db=db) - pvs.upsert_version("desktop", "draft", from_id="default", db=db) + prompt_versions_module.seed_from_filesystem(db=db) + prompt_versions_module.upsert_version("desktop", "draft", from_id="default", db=db) data = (await desktop_preview(DesktopPreviewRequest(version_id="draft"), db=db))["data"] assert data["environment_source"] == "runtime_placeholders" assert "{runtime.os}" in data["prompt"] @@ -59,13 +61,13 @@ async def test_full_preview_does_not_activate_version(db, monkeypatch): assert "没 pin = 用户看不到" not in data["prompt"] assert "{runtime.cwd}" in data["bash_tool"] assert "project" in data["conditional_parts"] - assert pvs.get_active_version("desktop", db=db)["id"] == "default" + assert prompt_versions_module.get_active_version("desktop", db=db)["id"] == "default" def test_database_failure_uses_filesystem(monkeypatch): def fail(*args, **kwargs): raise RuntimeError("database unavailable") - monkeypatch.setattr(pvs, "get_active_version", fail) + monkeypatch.setattr(prompt_versions_module, 'get_active_version', fail) assert "本机文件与执行" in render_desktop_part("guidance") @@ -73,8 +75,8 @@ def fail(*args, **kwargs): async def test_local_preview_uses_runtime_assembly_with_selected_version(db, monkeypatch): from api.routes.v1 import prompt_management as admin_prompts monkeypatch.setattr("core.config.local_mode.local_mode_enabled", lambda: True) - pvs.seed_from_filesystem(db=db) - pvs.upsert_version("desktop", "draft", parts=[{"part_id": "guidance", "content": "SELECTED_DRAFT"}], db=db) + prompt_versions_module.seed_from_filesystem(db=db) + prompt_versions_module.upsert_version("desktop", "draft", parts=[{"part_id": "guidance", "content": "SELECTED_DRAFT"}], db=db) async def runtime(_, **kwargs): assert kwargs["approval_mode"] == "ask" return {"prompt": "RUNTIME_TOOLS\n" + render_desktop_part("guidance")} @@ -89,7 +91,7 @@ async def runtime(_, **kwargs): async def test_preview_unsaved_parts_and_snapshot_roundtrip(db, monkeypatch): from api.routes.v1 import prompt_management as api monkeypatch.setattr("core.config.local_mode.local_mode_enabled", lambda: False) - pvs.seed_from_filesystem(db=db) + prompt_versions_module.seed_from_filesystem(db=db) before = api.export_snapshot(db=db)["data"] data = (await api.desktop_preview(api.DesktopPreviewRequest(version_id="default", parts=[ api.PromptPartPayload(part_id="guidance", content="UNSAVED_PREVIEW")]), db=db))["data"] @@ -129,8 +131,8 @@ async def test_management_permission_gate(db, monkeypatch, user_id, allowed, sta def test_desktop_seed_has_no_site_workflow_parts(db): - pvs.seed_from_filesystem(db=db) - version = pvs.get_active_version("desktop", db=db) + prompt_versions_module.seed_from_filesystem(db=db) + version = prompt_versions_module.get_active_version("desktop", db=db) parts = {p["part_id"]: p["content"] for p in version["parts"]} assert not {"site_mode", "site_records", "site_lookup_failed"} & parts.keys() assert "publish_site" not in parts["project"] diff --git a/src/backend/tests/prompts/test_local_mode_drops_container_sections.py b/src/backend/tests/prompts/test_local_mode_drops_container_sections.py index bd7de55f..6817b428 100644 --- a/src/backend/tests/prompts/test_local_mode_drops_container_sections.py +++ b/src/backend/tests/prompts/test_local_mode_drops_container_sections.py @@ -1,6 +1,8 @@ """桌面端不注入容器时代的 code_exec 两段，路径也不做改写。""" from __future__ import annotations +from core.llm import prompt_rendering as prompt_rendering_module +from core.llm import prompt_versions as prompt_versions_module import pytest @@ -8,13 +10,13 @@ @pytest.mark.parametrize("local", [True, False]) def test_container_sections_only_ship_off_desktop(monkeypatch, local): import core.config.local_mode as local_mode - from core.services import prompt_version_service as pvs + from core.llm import prompt_versions as pvs monkeypatch.setattr(local_mode, "local_mode_enabled", lambda: local) - monkeypatch.setattr(pvs, "render_active_prompt", lambda *a, **k: None) - monkeypatch.setattr(pvs, "get_active_version", lambda *a, **k: None) + monkeypatch.setattr(prompt_rendering_module, 'render_active_prompt', lambda *a, **k: None) + monkeypatch.setattr(prompt_versions_module, 'get_active_version', lambda *a, **k: None) - seg = pvs.render_kind_segment("code_exec") + seg = prompt_rendering_module.render_kind_segment("code_exec") assert ("沙箱" in seg or "沙盒" in seg) or not local # 非本机时两段仍在 assert ("/workspace/" in seg) is (not local) @@ -23,9 +25,9 @@ def test_container_sections_only_ship_off_desktop(monkeypatch, local): def test_real_paths_are_not_rewritten_in_local_mode(monkeypatch, tmp_path): """本机模式下模型给什么路径就是什么路径，不再映射到会话目录。""" monkeypatch.setattr("core.sandbox.workspace.paths.WORKSPACE", str(tmp_path)) - monkeypatch.setattr("core.llm.tools._paths.WORKSPACE_ROOT", str(tmp_path)) + monkeypatch.setattr("core.sandbox.workspace.resolution.WORKSPACE_ROOT", str(tmp_path)) monkeypatch.setattr("core.config.local_mode.local_mode_enabled", lambda: True) - from core.llm.tools._paths import to_physical_path + from core.sandbox.workspace.resolution import to_physical_path real = str(tmp_path / ".sessions" / "abc" / "sites" / "demo") assert to_physical_path(real, "u1", session_id="chat-1") == real @@ -33,6 +35,6 @@ def test_real_paths_are_not_rewritten_in_local_mode(monkeypatch, tmp_path): def test_workspace_alias_is_gone(): """路径别名已删除。""" - from core.llm.tools import _paths + from core.sandbox.workspace import resolution as _paths assert not hasattr(_paths, "canonicalize_ws_path") diff --git a/src/backend/tests/prompts/test_local_project_section.py b/src/backend/tests/prompts/test_local_project_section.py index 119317d3..c2457d80 100644 --- a/src/backend/tests/prompts/test_local_project_section.py +++ b/src/backend/tests/prompts/test_local_project_section.py @@ -66,7 +66,7 @@ def test_local_section_handles_missing_path_and_slug(): def test_desktop_prompt_and_bash_describe_effective_session(monkeypatch): - from core.llm.tools import _paths + from core.sandbox.workspace import resolution as _paths from core.llm.tools.sandbox_tool import register_bash from prompts.prompt_config import PromptConfig from prompts.prompt_runtime import build_system_prompt @@ -82,10 +82,12 @@ def test_desktop_prompt_and_bash_describe_effective_session(monkeypatch): functions = {} class Toolkit: - def register_tool_function(self, fn, **kwargs): + def append(self, tool): + fn = tool._func + kwargs = {"func_description": tool.description, "permission": getattr(tool, "dispatch_spec", None)} functions[fn.__name__] = fn - register_bash(Toolkit(), loader=None, loaded_skill_ids=set(), **ctx) + register_bash(Toolkit(), loader=None, **ctx) description = functions["Bash"].__doc__ expected = session_root(root, "effective-session") assert expected in prompt and expected in description @@ -95,7 +97,7 @@ def register_tool_function(self, fn, **kwargs): def test_cloud_bash_description_keeps_container_paths(monkeypatch): - from core.llm.tools import _paths + from core.sandbox.workspace import resolution as _paths from core.llm.tools.sandbox_tool import register_bash monkeypatch.setattr("core.config.local_mode.local_mode_enabled", lambda: False) @@ -104,10 +106,12 @@ def test_cloud_bash_description_keeps_container_paths(monkeypatch): functions = {} class Toolkit: - def register_tool_function(self, fn, **kwargs): + def append(self, tool): + fn = tool._func + kwargs = {"func_description": tool.description, "permission": getattr(tool, "dispatch_spec", None)} functions[fn.__name__] = fn - register_bash(Toolkit(), loader=None, loaded_skill_ids=set(), chat_id="cloud-chat") + register_bash(Toolkit(), loader=None, chat_id="cloud-chat") doc = functions["Bash"].__doc__ assert "工作目录默认 /workspace" in doc assert "/workspace/skills" in doc diff --git a/src/backend/tests/prompts/test_project_isolation.py b/src/backend/tests/prompts/test_project_isolation.py index 9567701e..95dcbbcc 100644 --- a/src/backend/tests/prompts/test_project_isolation.py +++ b/src/backend/tests/prompts/test_project_isolation.py @@ -21,7 +21,7 @@ def _load(rel_path: str, name: str): @pytest.fixture(scope="module") def paths(): - return _load("core/llm/tools/_paths.py", "_paths_test_module") + return _load("core/sandbox/workspace/resolution.py", "_paths_test_module") class TestValidateProjectScopePath: diff --git a/src/backend/tests/prompts/test_prompt_cache_full_hash.py b/src/backend/tests/prompts/test_prompt_cache_full_hash.py index 6a47a351..54db0447 100644 --- a/src/backend/tests/prompts/test_prompt_cache_full_hash.py +++ b/src/backend/tests/prompts/test_prompt_cache_full_hash.py @@ -8,11 +8,11 @@ from core.db import engine as db_engine from core.db.models import AdminPromptPart, ContentBlock -from core.services import prompt_version_service as pvs +from core.llm import prompt_storage as pvs from prompts.prompt_config import PromptConfig, SystemPromptConfig from prompts import prompt_runtime from prompts import project_section -from core.llm.execution_manifest import PromptManifestBuilder +from core.harness.execution_manifest import PromptManifestBuilder @pytest.fixture(autouse=True) diff --git a/src/backend/tests/prompts/test_prompt_parts_from_filesystem.py b/src/backend/tests/prompts/test_prompt_parts_from_filesystem.py index 2cf101b5..0339e7ee 100644 --- a/src/backend/tests/prompts/test_prompt_parts_from_filesystem.py +++ b/src/backend/tests/prompts/test_prompt_parts_from_filesystem.py @@ -1,10 +1,16 @@ """提示词正文的真源是 prompt_text 下的 md 文件，不是 Python 里的字符串常量。""" from __future__ import annotations +from core.llm import prompt_files as prompt_files_module +from core.llm import prompt_kinds as prompt_kinds_module +from core.llm import prompt_rendering as prompt_rendering_module +from core.llm import prompt_storage as prompt_storage_module +from core.llm import prompt_versions as prompt_versions_module +from core.llm import prompt_versions_state as prompt_versions_state_module from pathlib import Path -from core.services import prompt_version_service as pvs +from core.llm import prompt_storage as pvs BACKEND_ROOT = Path(__file__).resolve().parents[2] PROMPT_TEXT = BACKEND_ROOT / "prompts" / "prompt_text" @@ -17,7 +23,7 @@ def test_system_reminder_convention_has_a_markdown_file(): def test_system_part_sort_order_comes_from_the_filename_prefix(): - by_id = {p["part_id"]: p["sort_order"] for p in pvs._read_fs_parts("system")} + by_id = {p["part_id"]: p["sort_order"] for p in prompt_files_module.read_fs_parts("system")} assert by_id["system/00_role"] == 0 # 新增的段落按文件名前缀落在 00_role 与 10_constraints 之间，不挤动相邻段。 assert by_id["system/05_system_reminder_convention"] == 5 @@ -29,17 +35,17 @@ def test_plan_tool_segment_is_rendered_from_its_markdown_file(monkeypatch): fp = PROMPT_TEXT / "plan_tool" / "plan_tool.system.md" assert fp.is_file() - from core.llm.plan_update_tool import build_plan_update_prompt_section + from core.llm.tools.plan_update import build_plan_update_prompt_section # 未建库/未播种时退回文件系统，拿到的就是这份 md。 - monkeypatch.setattr(pvs, "render_active_prompt", lambda kind, db=None: None) + monkeypatch.setattr(prompt_rendering_module, 'render_active_prompt', lambda kind, db=None: None) assert build_plan_update_prompt_section().strip() == fp.read_text(encoding="utf-8").strip() def test_plan_tool_segment_prefers_the_active_db_version(monkeypatch): - from core.llm.plan_update_tool import build_plan_update_prompt_section + from core.llm.tools.plan_update import build_plan_update_prompt_section - monkeypatch.setattr(pvs, "render_active_prompt", lambda kind, db=None: "管理员改过的正文") + monkeypatch.setattr(prompt_rendering_module, 'render_active_prompt', lambda kind, db=None: "管理员改过的正文") assert build_plan_update_prompt_section() == "管理员改过的正文" @@ -47,8 +53,8 @@ def test_plan_tool_is_a_builtin_kind_with_a_label(db_session, monkeypatch): from sqlalchemy.orm import sessionmaker monkeypatch.setattr(pvs, "SessionLocal", sessionmaker(bind=db_session.get_bind())) - assert "plan_tool" in pvs.BUILTIN_KINDS - assert any(k["key"] == "plan_tool" and k["builtin"] for k in pvs.all_kinds()) + assert "plan_tool" in prompt_kinds_module.BUILTIN_KINDS + assert any(k["key"] == "plan_tool" and k["builtin"] for k in prompt_kinds_module.all_kinds()) def test_new_filesystem_part_is_backfilled_into_an_existing_default_version(monkeypatch): @@ -65,12 +71,12 @@ def test_new_filesystem_part_is_backfilled_into_an_existing_default_version(monk } ], } - monkeypatch.setattr(pvs, "_load_payload", lambda db=None: payload) + monkeypatch.setattr(prompt_storage_module, 'load_payload', lambda db=None: payload) monkeypatch.setattr( - pvs, "_save_payload", lambda p, db=None, updated_by="": saved.update({"payload": p}) + prompt_storage_module, 'save_payload', lambda p, db=None, updated_by="": saved.update({"payload": p}) ) - result = pvs.seed_from_filesystem() + result = prompt_versions_module.seed_from_filesystem() parts = {p["part_id"] for p in saved["payload"]["versions"][0]["parts"]} assert "system/05_system_reminder_convention" in parts @@ -80,17 +86,17 @@ def test_new_filesystem_part_is_backfilled_into_an_existing_default_version(monk def test_no_prompt_body_is_hardcoded_in_python(): runtime_src = (BACKEND_ROOT / "prompts" / "prompt_runtime.py").read_text(encoding="utf-8") - plan_src = (BACKEND_ROOT / "core" / "llm" / "plan_update_tool.py").read_text(encoding="utf-8") + plan_src = (BACKEND_ROOT / "core" / "llm" / "tools" / "plan_update.py").read_text(encoding="utf-8") assert "_SYSTEM_REMINDER_CONVENTION_DEFAULT" not in runtime_src assert "## 任务计划清单" not in plan_src def test_every_builtin_kind_resolves_to_a_readable_directory(): """加一个内置 kind 只需在 KIND_SPECS 加一行——这里保证那一行是自洽的。""" - for kind, spec in pvs.KIND_SPECS.items(): + for kind, spec in prompt_kinds_module.KIND_SPECS.items(): assert spec.layout in {"concat", "independent", "single"} - assert pvs._fs_dir(kind).is_dir(), f"{kind} 的提示词目录不存在" - assert pvs._read_fs_parts(kind), f"{kind} 目录里没有可读的 md" + assert prompt_files_module.fs_dir(kind).is_dir(), f"{kind} 的提示词目录不存在" + assert prompt_files_module.read_fs_parts(kind), f"{kind} 目录里没有可读的 md" def test_fs_parts_are_cached_until_the_files_change(tmp_path, monkeypatch): @@ -98,28 +104,28 @@ def test_fs_parts_are_cached_until_the_files_change(tmp_path, monkeypatch): dirp = tmp_path / "plan_tool" dirp.mkdir() (dirp / "plan_tool.system.md").write_text("first", encoding="utf-8") - monkeypatch.setattr(pvs, "_fs_dir", lambda kind: dirp) - monkeypatch.setattr(pvs, "_fs_parts_cache", {}) + monkeypatch.setattr(prompt_files_module, 'fs_dir', lambda kind: dirp) + monkeypatch.setattr(prompt_versions_state_module.process_state(), 'fs_parts_cache', {}) reads: list[str] = [] - real_build = pvs._build_fs_parts + real_build = prompt_files_module.build_fs_parts monkeypatch.setattr( - pvs, - "_build_fs_parts", + prompt_files_module, + 'build_fs_parts', lambda kind, d, files: (reads.append(kind), real_build(kind, d, files))[1], ) - assert pvs._read_fs_parts("plan_tool")[0]["content"] == "first" - assert pvs._read_fs_parts("plan_tool")[0]["content"] == "first" + assert prompt_files_module.read_fs_parts("plan_tool")[0]["content"] == "first" + assert prompt_files_module.read_fs_parts("plan_tool")[0]["content"] == "first" assert len(reads) == 1, "文件没变却重读了磁盘" (dirp / "plan_tool.system.md").write_text("second content", encoding="utf-8") - assert pvs._read_fs_parts("plan_tool")[0]["content"] == "second content" + assert prompt_files_module.read_fs_parts("plan_tool")[0]["content"] == "second content" assert len(reads) == 2, "文件变了却没重读" def test_read_fs_parts_hands_back_copies(): """播种会把 part 直接塞进版本里，返回缓存本体会被就地改坏。""" - first = pvs._read_fs_parts("plan_tool") + first = prompt_files_module.read_fs_parts("plan_tool") first[0]["content"] = "mutated" - assert pvs._read_fs_parts("plan_tool")[0]["content"] != "mutated" + assert prompt_files_module.read_fs_parts("plan_tool")[0]["content"] != "mutated" diff --git a/src/backend/tests/prompts/test_prompt_version_freshness.py b/src/backend/tests/prompts/test_prompt_version_freshness.py index df22a4bc..8dbf9cad 100644 --- a/src/backend/tests/prompts/test_prompt_version_freshness.py +++ b/src/backend/tests/prompts/test_prompt_version_freshness.py @@ -1,6 +1,8 @@ """Saved prompts must be visible to already-warm backend workers.""" from __future__ import annotations +from core.llm import prompt_versions as prompt_versions_module +from core.config.store import system_config import os import subprocess @@ -12,7 +14,7 @@ from core.db import engine as db_engine from core.db.models import AdminPromptPart, ContentBlock, SystemConfig -from core.services import prompt_version_service as pvs +from core.llm import prompt_storage as pvs @pytest.fixture @@ -26,9 +28,9 @@ def prompt_db(tmp_path, monkeypatch): monkeypatch.setattr(pvs, "SessionLocal", sessions) monkeypatch.setattr(db_engine, "SessionLocal", sessions) from prompts import prompt_runtime - from core.services import system_config + from core.config.store import system_config - monkeypatch.setattr(system_config, "SessionLocal", sessions) + monkeypatch.setattr("core.db.system_configuration.SessionLocal", sessions) monkeypatch.setattr(system_config.SystemConfigService, "_instance", None) prompt_runtime.invalidate_prompt_cache() yield url, sessions @@ -42,7 +44,7 @@ def save_in_other_worker(url, kind, version_id, content): from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker from core.db import engine as db_engine -from core.services import prompt_version_service as pvs +from core.llm import prompt_storage as pvs sessions = sessionmaker(bind=create_engine(sys.argv[1])) pvs.SessionLocal = sessions db_engine.SessionLocal = sessions @@ -70,7 +72,7 @@ def test_saved_version_is_visible_to_another_warm_worker(prompt_db): from api.routes.v1.prompt_management import get_pool_version url, sessions = prompt_db - pvs.upsert_version( + prompt_versions_module.upsert_version( "system", "default", parts=[{"part_id": "system/00_role", "content": "BEFORE_SAVE"}] ) with sessions() as db: @@ -99,11 +101,11 @@ async def test_preview_and_runtime_use_saved_prompts_without_local_invalidation( url, sessions = prompt_db monkeypatch.setattr(local_mode, "local_mode_enabled", lambda: False) - pvs.seed_from_filesystem() - pvs.upsert_version( + prompt_versions_module.seed_from_filesystem() + prompt_versions_module.upsert_version( "system", "default", parts=[{"part_id": "system/00_role", "content": "SYSTEM_BEFORE"}] ) - pvs.upsert_version( + prompt_versions_module.upsert_version( "desktop", "default", parts=[{"part_id": "guidance", "content": "DESKTOP_BEFORE"}] ) config = PromptConfig() @@ -134,17 +136,17 @@ async def test_preview_and_runtime_use_saved_prompts_without_local_invalidation( def test_warm_worker_does_not_overwrite_other_workers_saved_version(prompt_db): url, _ = prompt_db - pvs.upsert_version( + prompt_versions_module.upsert_version( "system", "default", parts=[{"part_id": "system/00_role", "content": "FIRST_BEFORE"}] ) - pvs.upsert_version( + prompt_versions_module.upsert_version( "code_exec", "default", parts=[{"part_id": "system/00_role", "content": "SECOND_BEFORE"}] ) - assert pvs.get_version("system", "default")["parts"][0]["content"] == "FIRST_BEFORE" + assert prompt_versions_module.get_version("system", "default")["parts"][0]["content"] == "FIRST_BEFORE" save_in_other_worker(url, "system", "default", "FIRST_AFTER") - pvs.upsert_version( + prompt_versions_module.upsert_version( "code_exec", "default", parts=[{"part_id": "system/00_role", "content": "SECOND_AFTER"}] ) - assert pvs.get_version("system", "default")["parts"][0]["content"] == "FIRST_AFTER" - assert pvs.get_version("code_exec", "default")["parts"][0]["content"] == "SECOND_AFTER" + assert prompt_versions_module.get_version("system", "default")["parts"][0]["content"] == "FIRST_AFTER" + assert prompt_versions_module.get_version("code_exec", "default")["parts"][0]["content"] == "SECOND_AFTER" diff --git a/src/backend/tests/run_browser_opensandbox_e2e.py b/src/backend/tests/run_browser_opensandbox_e2e.py index 6ce074f7..d24ed51b 100644 --- a/src/backend/tests/run_browser_opensandbox_e2e.py +++ b/src/backend/tests/run_browser_opensandbox_e2e.py @@ -50,12 +50,12 @@ async def verify(): from core.db.engine import Base, engine from core.db import models Base.metadata.create_all(engine) - from core.plugins.resources.installation import Installation - from core.plugins.ui.contract import normalize_ui - from core.plugins.packaging.importer import manifest_extensions + from core.capabilities.plugins.resources.installation import Installation + from features.workbench.ui.contract import normalize_ui + from core.capabilities.plugins.packaging.importer import manifest_extensions from core.sandbox.providers.opensandbox.provider import OpenSandboxProvider from core.sandbox.processes.interactive import launch - from core.plugins.resources.stream import packets + from core.capabilities.plugins.resources.stream import packets import httpx package = ROOT / "src/backend/extensions/plugins/marketplace/browser-automation" manifest = json.loads((package / "plugin.json").read_text()) diff --git a/src/backend/tests/run_command_opensandbox_e2e.py b/src/backend/tests/run_command_opensandbox_e2e.py index 1bc562f4..a72101b0 100644 --- a/src/backend/tests/run_command_opensandbox_e2e.py +++ b/src/backend/tests/run_command_opensandbox_e2e.py @@ -10,12 +10,12 @@ ROOT = Path(__file__).resolve().parents[3] async def verify(): - from core.db.engine import Base, engine, SessionLocal + from core.db.engine import Base, SessionLocal, engine from core.db.models import ChatSession, UserShadow - from core.sandbox.providers.opensandbox.provider import OpenSandboxProvider + from core.sandbox.commands.runtime import CommandRuntime from core.sandbox.protocol import ProcessRequest - from core.services.commands import store - from orchestration.commands.runtime import CommandRuntime + from core.sandbox.providers.opensandbox.provider import OpenSandboxProvider + from core.sandbox.commands import store Base.metadata.create_all(engine) session = "command-e2e-" + uuid.uuid4().hex user = "api_ak_command_e2e" @@ -70,11 +70,11 @@ def main(): domain = os.environ["COMMAND_E2E_OPENSANDBOX_DOMAIN"] with tempfile.TemporaryDirectory(prefix="command-cloud-e2e-") as folder: env = {"DATABASE_URL":"sqlite:///" + str(Path(folder)/"test.db"), - "HUGAGENT_HOME":folder,"STORAGE_PATH":str(Path(folder)/"storage"), + "HUGAGENT_HOME":folder,"STORAGE_PATH":os.environ["COMMAND_E2E_STORAGE_PATH"], "DEPLOY_PROFILE":"team","AUTH_MODE":"mock","SANDBOX_PROVIDER":"opensandbox", "OPENSANDBOX_DOMAIN":domain,"OPENSANDBOX_IMAGE":image, "OPENSANDBOX_DIRECT_EXECD":"false","OPENSANDBOX_MYSPACE_BIND_MOUNT_ENABLED":"false", - "OPENSANDBOX_SNAPSHOT_ENABLED":"false","HOST_STORAGE_PATH":"", + "OPENSANDBOX_SNAPSHOT_ENABLED":"false","HOST_STORAGE_PATH":os.environ["COMMAND_E2E_HOST_STORAGE_PATH"], "SCRIPT_RUNNER_WORKSPACE":"/workspace","REDIS_URL":"", "NO_PROXY":"127.0.0.1,localhost","no_proxy":"127.0.0.1,localhost"} for name in ("DWS","LARK","EMAIL","YIDA"): diff --git a/src/backend/tests/run_personal_agent_cli_grant_e2e.py b/src/backend/tests/run_personal_agent_cli_grant_e2e.py index e2cc271a..a34d21fb 100644 --- a/src/backend/tests/run_personal_agent_cli_grant_e2e.py +++ b/src/backend/tests/run_personal_agent_cli_grant_e2e.py @@ -18,7 +18,8 @@ class Tools: def __init__(self): self.functions = {} - def register_tool_function(self, fn): + def append(self, tool): + fn = tool._func self.functions[fn.__name__] = fn diff --git a/src/backend/tests/run_personal_agent_e2e.py b/src/backend/tests/run_personal_agent_e2e.py index f32f0f7e..c9200c73 100644 --- a/src/backend/tests/run_personal_agent_e2e.py +++ b/src/backend/tests/run_personal_agent_e2e.py @@ -7,7 +7,7 @@ import time from sqlalchemy import select from core.db.engine import SessionLocal -from core.db.models import EmailConnection +from integrations.email.models import EmailConnection from core.infra.crypto import encrypt_secret from personal_agent.models import PersonalAgent, PersonalWorkItem from personal_agent.context import AgentContext @@ -15,7 +15,9 @@ class CaptureTools: def __init__(self): self.tools = {} - def register_tool_function(self, fn): self.tools[fn.__name__] = fn + def append(self, tool): + fn = tool._func + self.tools[fn.__name__] = fn def data(chunk): return json.loads(chunk.content[0].text) diff --git a/src/backend/tests/run_personal_agent_key_e2e.py b/src/backend/tests/run_personal_agent_key_e2e.py index 090c6c81..1685fd2b 100644 --- a/src/backend/tests/run_personal_agent_key_e2e.py +++ b/src/backend/tests/run_personal_agent_key_e2e.py @@ -3,7 +3,7 @@ import uuid from core.db.engine import SessionLocal from core.db.models import UserShadow -from core.services.api_key_service import ApiKeyService +from core.auth.api_keys import ApiKeyService from hugagent_cli.transport import Client uid = "cli-key-test-" + uuid.uuid4().hex diff --git a/src/backend/tests/run_personal_agent_project_sessions_e2e.py b/src/backend/tests/run_personal_agent_project_sessions_e2e.py index ab5c44a7..c51c260f 100644 --- a/src/backend/tests/run_personal_agent_project_sessions_e2e.py +++ b/src/backend/tests/run_personal_agent_project_sessions_e2e.py @@ -31,12 +31,13 @@ class Tools: def __init__(self): self.functions = {} - def register_tool_function(self, fn): + def append(self, tool): + fn = tool._func self.functions[fn.__name__] = fn def setup(uid, marker): - from core.services.project_service import ProjectService + from core.projects.edition import ProjectService from personal_agent.computer.devices import pair with SessionLocal() as db: diff --git a/src/backend/tests/sandbox/e2e_myspace_bulk_live.py b/src/backend/tests/sandbox/e2e_myspace_bulk_live.py index 230abe78..0e64384a 100644 --- a/src/backend/tests/sandbox/e2e_myspace_bulk_live.py +++ b/src/backend/tests/sandbox/e2e_myspace_bulk_live.py @@ -8,8 +8,8 @@ from core.db.models import Artifact, ChatSession, UserShadow from core.infra.logging import LogContext from core.llm.tools.sandbox_tool import register_bash -from core.services.api_key_service import ApiKeyService -from core.storage import get_storage +from core.auth.api_keys import ApiKeyService +from core.files.storage import get_storage from tests.sandbox.e2e_myspace_identity_live import Toolkit, decode @@ -23,7 +23,7 @@ async def main(): db.commit() _, raw = ApiKeyService(db).create_key(uid, "temporary bulk E2E", expires_in_days=1) tk = Toolkit() - register_bash(tk, loader=None, loaded_skill_ids=set(), chat_id=chat, + register_bash(tk, loader=None, chat_id=chat, sandbox_session_id=chat, user_id=uid) metrics = {"user": uid} diff --git a/src/backend/tests/sandbox/fixtures/egress_gateway.py b/src/backend/tests/sandbox/fixtures/egress_gateway.py index b4e36a3f..3498b41c 100644 --- a/src/backend/tests/sandbox/fixtures/egress_gateway.py +++ b/src/backend/tests/sandbox/fixtures/egress_gateway.py @@ -1,4 +1,7 @@ +from features.desktop.api import desktop_capability """Isolated E2E gateway: real production HTTP route, synthetic authorization/upstream.""" +from features.desktop.api import capability_dependencies +from features.desktop.api import desktop_capability_state import sys from pathlib import Path from types import ModuleType @@ -15,8 +18,9 @@ from fastapi import FastAPI import core.db.models # Initialize model extensions before standalone route imports. -from api.routes.v1 import desktop_capability as bridge -from core.services import desktop_capability, desktop_gateway_observability +from features.desktop.api import desktop_capability as bridge +from features.desktop.capability import service as desktop_capability +from features.desktop.gateway import observability as desktop_gateway_observability # Replace account/config storage only. HTTP forwarding, pooling, SSE and error handling # execute the production code; there are no real accounts, secrets or databases here. @@ -27,11 +31,11 @@ "provider_type": "chat", "api_key": "synthetic-e2e-key", } -bridge._stream_secrets = lambda *_: {"synthetic-e2e-key"} +capability_dependencies.stream_secrets = lambda *_: {"synthetic-e2e-key"} desktop_gateway_observability.submit = lambda *_: None app = FastAPI() -app.dependency_overrides[bridge._require_capability_user] = lambda: "e2e-user" -app.include_router(bridge.router) +app.dependency_overrides[capability_dependencies.require_capability_user] = lambda: "e2e-user" +app.include_router(desktop_capability.router) @app.get("/health") diff --git a/src/backend/tests/sandbox/provider_test_support.py b/src/backend/tests/sandbox/provider_test_support.py index caef8b20..42cc488e 100644 --- a/src/backend/tests/sandbox/provider_test_support.py +++ b/src/backend/tests/sandbox/provider_test_support.py @@ -146,10 +146,13 @@ def _fake_paginator(items): def patch_open_settings(monkeypatch, value): """Isolate the configuration of the concrete OpenSandbox components under test.""" from importlib import import_module - for name in ("creation", "lifecycle", "snapshots", "volumes", "account_volumes", "tracking", "reclamation"): + for name in ("creation", "lifecycle", "snapshots", "volumes", "tracking", "reclamation"): module = import_module("core.sandbox.providers.opensandbox." + name) if hasattr(module, "settings"): monkeypatch.setattr(module, "settings", value) + for owner in ("dingtalk", "lark", "email", "yida"): + module = import_module(f"integrations.{owner}.mounts") + monkeypatch.setattr(module, "settings", value) def install_session_scope(provider): from contextlib import asynccontextmanager diff --git a/src/backend/tests/sandbox/test_adopt_respects_session_binding.py b/src/backend/tests/sandbox/test_adopt_respects_session_binding.py index 7179f9ed..5198434c 100644 --- a/src/backend/tests/sandbox/test_adopt_respects_session_binding.py +++ b/src/backend/tests/sandbox/test_adopt_respects_session_binding.py @@ -144,9 +144,6 @@ def _server_info(sandbox_id: str, user_id: str = "u1"): def _provider(monkeypatch): """一个只装了认领这条路所需部件的 provider。""" - monkeypatch.setattr( - "core.sandbox.providers.opensandbox.provider._user_bound_sandbox_required", lambda: True - ) provider = object.__new__(OpenSandboxProvider) provider._make_config = lambda: None provider._repoint_execd_direct = AsyncMock(return_value=None) diff --git a/src/backend/tests/sandbox/test_artifact_streaming.py b/src/backend/tests/sandbox/test_artifact_streaming.py index 7d4c7964..5d0bb7f4 100644 --- a/src/backend/tests/sandbox/test_artifact_streaming.py +++ b/src/backend/tests/sandbox/test_artifact_streaming.py @@ -19,8 +19,8 @@ class _CaptureToolkit: - def register_tool_function(self, function, *, namesake_strategy: str): - assert namesake_strategy == "override" + def append(self, tool): + function = tool._func self.function = function @@ -315,7 +315,7 @@ async def _session_sandbox(session_id): def test_artifact_store_persists_from_file_without_byte_upload(monkeypatch, tmp_path): - from core.artifacts import store + from core.files.artifacts import store source = tmp_path / "source.pdf" source.write_bytes(b"streamed-pdf") @@ -336,7 +336,7 @@ def test_artifact_store_persists_from_file_without_byte_upload(monkeypatch, tmp_ def test_artifact_store_uses_file_upload_for_oss(monkeypatch, tmp_path): - from core.artifacts import store + from core.files.artifacts import store source = tmp_path / "source.pdf" source.write_bytes(b"streamed-pdf") diff --git a/src/backend/tests/sandbox/test_cloud_process_sessions.py b/src/backend/tests/sandbox/test_cloud_process_sessions.py index a2c1eaaa..9cb6ede8 100644 --- a/src/backend/tests/sandbox/test_cloud_process_sessions.py +++ b/src/backend/tests/sandbox/test_cloud_process_sessions.py @@ -160,7 +160,7 @@ async def run(cmd, **kwargs): async def test_runner_provider_checks_capability_integrity_before_returning_output(monkeypatch): import httpx - from core.capabilities.errors import IntegrityFailed + from core.harness.capability_errors import IntegrityFailed from core.sandbox.providers.script_runner.provider import ScriptRunnerProvider requests = [] @@ -186,12 +186,12 @@ async def request(req): "core.sandbox.providers.script_runner.provider.httpx.AsyncClient", lambda **kwargs: client_type(transport=httpx.MockTransport(request), **kwargs), ) - monkeypatch.setattr("core.capabilities.runtime.get", lambda *args, **kwargs: object()) + monkeypatch.setattr('features.desktop.capabilities.runtime.get', lambda *args, **kwargs: object()) def invalid(*args, **kwargs): raise IntegrityFailed("changed capability") - monkeypatch.setattr("core.capabilities.runtime.validate", invalid) + monkeypatch.setattr('features.desktop.capabilities.runtime.validate', invalid) with pytest.raises(IntegrityFailed, match="changed capability"): await ScriptRunnerProvider().write_stdin( "p1", sandbox_session_id="chat-a", user_id="alice", yield_time_ms=0 diff --git a/src/backend/tests/sandbox/test_controller_capacity.py b/src/backend/tests/sandbox/test_controller_capacity.py new file mode 100644 index 00000000..e614de42 --- /dev/null +++ b/src/backend/tests/sandbox/test_controller_capacity.py @@ -0,0 +1,37 @@ +"""Capacity includes all controller inventory and competing creation processes.""" +import importlib.util +from pathlib import Path +from types import SimpleNamespace +from concurrent.futures import ThreadPoolExecutor +from fastapi import HTTPException +import pytest + +ROOT = Path(__file__).resolve().parents[4] +spec = importlib.util.spec_from_file_location("sandbox_capacity", ROOT / "docker/opensandbox_capacity.py") +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) + + +def test_inventory_and_atomic_creation_share_one_budget(tmp_path, monkeypatch): + monkeypatch.setattr(module, "Path", lambda _: tmp_path) + monkeypatch.setenv("OPENSANDBOX_POOL_MAX_TOTAL", "1") + inventory = [] + first = SimpleNamespace(network_mode="compute", _owned_sandboxes=lambda: inventory) + second = SimpleNamespace(network_mode="compute", _owned_sandboxes=lambda: inventory) + def compete(): + with pytest.raises(HTTPException) as caught: + with module.sandbox_capacity(second): + pytest.fail("second creator admitted while first is pending") + return caught.value.status_code + with module.sandbox_capacity(first): + with ThreadPoolExecutor(max_workers=1) as pool: + assert pool.submit(compete).result(timeout=1) == 429 + inventory.append("sandbox") + # A restarted provider/controller sees inventory, without local reservations. + with pytest.raises(HTTPException) as caught: + with module.sandbox_capacity(second): + pytest.fail("capacity exceeded after first creation") + assert caught.value.status_code == 429 + inventory.clear() + with module.sandbox_capacity(second): + pass diff --git a/src/backend/tests/sandbox/test_creation_adoption_race.py b/src/backend/tests/sandbox/test_creation_adoption_race.py index 7ee7bbc7..af7683dd 100644 --- a/src/backend/tests/sandbox/test_creation_adoption_race.py +++ b/src/backend/tests/sandbox/test_creation_adoption_race.py @@ -23,7 +23,7 @@ def creator(): def stats(): - return dict.fromkeys(("adopted_jupyter", "adopted_light", "skipped_bound", "killed_legacy", "killed_dead", "killed_overflow"), 0) + return dict.fromkeys(("adopted_jupyter", "adopted_light", "skipped_bound", "killed_invalid", "killed_dead", "killed_overflow"), 0) def test_adopter_skips_container_while_creator_is_still_delivering_it(monkeypatch): @@ -67,7 +67,7 @@ def test_adoption_connect_timeout_preserves_live_container(monkeypatch): def test_startup_cleanup_cannot_kill_a_foreign_tracker_or_bound_instance(state, bound): b = creator() manager = SimpleNamespace(kill_sandbox=AsyncMock()) - # Missing pool-kind enters the legacy cleanup branch in the Running case. + # Missing pool-kind enters the invalid-metadata cleanup branch in the Running case. info = SimpleNamespace(id="foreign", status=SimpleNamespace(state=state), metadata={"hugagent-owner": "backend"}, entrypoint=[]) async def scenario(): if bound: @@ -79,9 +79,10 @@ async def scenario(): asyncio.run(scenario()) -def test_stopped_untagged_unknown_client_is_never_deleted(): +@pytest.mark.parametrize("entrypoint", [["foreign-agent"], ["sleep", "infinity"], ["/opt/opensandbox/code-interpreter.sh"]]) +def test_stopped_untagged_client_is_never_deleted(entrypoint): b = creator() manager = SimpleNamespace(kill_sandbox=AsyncMock()) - info = SimpleNamespace(id="another-client", status=SimpleNamespace(state="Stopped"), metadata={}, entrypoint=["foreign-agent"]) + info = SimpleNamespace(id="another-client", status=SimpleNamespace(state="Stopped"), metadata={}, entrypoint=entrypoint) asyncio.run(b._classify_and_adopt(manager, info, stats())) manager.kill_sandbox.assert_not_awaited() diff --git a/src/backend/tests/sandbox/test_credential_purge_preserves_mounts.py b/src/backend/tests/sandbox/test_credential_purge_preserves_mounts.py index 04f36e1d..37939310 100644 --- a/src/backend/tests/sandbox/test_credential_purge_preserves_mounts.py +++ b/src/backend/tests/sandbox/test_credential_purge_preserves_mounts.py @@ -11,6 +11,11 @@ """ from __future__ import annotations +from integrations.lark import process as lark_process +from integrations.lark import service as lark_service +import integrations.email.storage as integrations_email_storage +import integrations.lark.storage as integrations_lark_storage +import integrations.dingtalk.storage as integrations_dingtalk_storage import asyncio from pathlib import Path @@ -105,11 +110,11 @@ def test_purge_keeps_the_root_itself(tmp_path): def test_email_purge_config_preserves_the_himalaya_mount(db, monkeypatch, tmp_path): """The easiest path to trigger: this runs on **every failed mailbox verification**, not just on disconnect.""" - from core.services.email_service import EmailService + from integrations.email.service import EmailService root = tmp_path / "email_cache" / "u1" inodes = _seed_tree(root) - monkeypatch.setattr(common_credentials, "email_cache_dir", lambda uid: tmp_path / "email_cache" / uid) + monkeypatch.setattr(integrations_email_storage, 'email_cache_dir', lambda uid: tmp_path / "email_cache" / uid) EmailService(db)._purge_config("u1") @@ -117,18 +122,18 @@ def test_email_purge_config_preserves_the_himalaya_mount(db, monkeypatch, tmp_pa def test_lark_disconnect_preserves_the_lark_cli_mounts(db, monkeypatch, tmp_path): - from core.services import lark_service as ls + from integrations.lark import service as ls root = tmp_path / "lark_cache" / "u1" inodes = _seed_tree(root) - monkeypatch.setattr(common_credentials, "lark_cache_dir", lambda uid: tmp_path / "lark_cache" / uid) + monkeypatch.setattr(integrations_lark_storage, 'lark_cache_dir', lambda uid: tmp_path / "lark_cache" / uid) async def _fake_run(*_a, **_kw): return "", "", 0 - monkeypatch.setattr(ls, "_run_lark", _fake_run) + monkeypatch.setattr(lark_process, 'run_lark', _fake_run) - svc = ls.LarkService(db) + svc = lark_service.LarkService(db) svc.repo.ensure("u1") result = asyncio.run(svc.disconnect("u1")) @@ -137,11 +142,11 @@ async def _fake_run(*_a, **_kw): def test_dingtalk_disconnect_preserves_the_dws_mounts(db, monkeypatch, tmp_path): - from core.services import dingtalk_service as ds + from integrations.dingtalk import service as ds root = tmp_path / "dws_cache" / "u1" inodes = _seed_tree(root) - monkeypatch.setattr(common_credentials, "dws_cache_dir", lambda uid: tmp_path / "dws_cache" / uid) + monkeypatch.setattr(integrations_dingtalk_storage, 'dws_cache_dir', lambda uid: tmp_path / "dws_cache" / uid) async def _fake_run(*_a, **_kw): return "", "", 0 diff --git a/src/backend/tests/sandbox/test_cube_skill_push_ownership.py b/src/backend/tests/sandbox/test_cube_skill_push_ownership.py index 07de1116..dbdefb1c 100644 --- a/src/backend/tests/sandbox/test_cube_skill_push_ownership.py +++ b/src/backend/tests/sandbox/test_cube_skill_push_ownership.py @@ -7,6 +7,7 @@ copied into their own sandbox. """ from __future__ import annotations +from core.capabilities.skills import loader_factory as loader_factory_module import asyncio from types import SimpleNamespace @@ -48,15 +49,15 @@ def provider(monkeypatch, tmp_path): "bob-market-991dc1": "bob", } - import core.agent_skills.loader as loader_mod + import core.capabilities.skills.loader as loader_mod - monkeypatch.setattr(loader_mod, "get_skill_loader", lambda *a, **k: _Loader(owners, dirs)) + monkeypatch.setattr(loader_factory_module, 'get_skill_loader', lambda *a, **k: _Loader(owners, dirs)) def current_archive(sid, user_id=None): if owners.get(sid) and owners[sid] != user_id: return None return dirs.get(sid) - monkeypatch.setattr("core.agent_skills.skill_archive.build_current_skill_tar", current_archive) + monkeypatch.setattr('core.capabilities.skills.skill_archive.build_current_skill_tar', current_archive) return p diff --git a/src/backend/tests/sandbox/test_idle_reclaim_snapshot.py b/src/backend/tests/sandbox/test_idle_reclaim_snapshot.py index 937aaf8a..07925d6b 100644 --- a/src/backend/tests/sandbox/test_idle_reclaim_snapshot.py +++ b/src/backend/tests/sandbox/test_idle_reclaim_snapshot.py @@ -35,9 +35,8 @@ def _settings(**overrides): def _restore_provider(monkeypatch, *, snapshot_id, idle_hit): - monkeypatch.setattr("core.agent_skills.publication.prepare_skill_view", lambda uid: None) + monkeypatch.setattr('core.capabilities.skills.publication.prepare_skill_view', lambda uid: None) patch_open_settings(monkeypatch, _settings()) - monkeypatch.setattr(session_mod_runtime, "_user_bound_sandbox_required", lambda: True) provider = OpenSandboxProvider.__new__(OpenSandboxProvider) provider._lookup_snapshot = AsyncMock(return_value=snapshot_id) provider._jupyter_user_pool = SimpleNamespace(has_idle=lambda uid: idle_hit) diff --git a/src/backend/tests/sandbox/test_multiworker_lifecycle.py b/src/backend/tests/sandbox/test_multiworker_lifecycle.py index a709aca3..ae596080 100644 --- a/src/backend/tests/sandbox/test_multiworker_lifecycle.py +++ b/src/backend/tests/sandbox/test_multiworker_lifecycle.py @@ -3,7 +3,7 @@ from types import SimpleNamespace from unittest.mock import AsyncMock -from core.plugins.resources import prewarm +from core.capabilities.plugins.resources import prewarm import core.sandbox.sessions.registry as session_registry from core.sandbox.providers.opensandbox.provider import OpenSandboxProvider from tests.sandbox.test_session_affinity import _worker, _fake_opensandbox, _sandbox_settings diff --git a/src/backend/tests/sandbox/test_multiworker_transactions.py b/src/backend/tests/sandbox/test_multiworker_transactions.py index 33b8d7f1..98a46da6 100644 --- a/src/backend/tests/sandbox/test_multiworker_transactions.py +++ b/src/backend/tests/sandbox/test_multiworker_transactions.py @@ -5,7 +5,7 @@ import pytest -from core.plugins.resources import prewarm +from core.capabilities.plugins.resources import prewarm import core.sandbox.sessions.registry as session_registry from core.sandbox.providers.opensandbox.user_pool import _JupyterUserPool from core.sandbox.providers.opensandbox.pool import SandboxPool diff --git a/src/backend/tests/sandbox/test_opensandbox_expirations.py b/src/backend/tests/sandbox/test_opensandbox_expirations.py index 950d1034..d2d86ee5 100644 --- a/src/backend/tests/sandbox/test_opensandbox_expirations.py +++ b/src/backend/tests/sandbox/test_opensandbox_expirations.py @@ -112,6 +112,8 @@ def test_failed_initial_lease_rolls_back_container_and_mounts(failed_step): namespace[name] = name if failed_step == "mount": namespace["initialize_mounts"].side_effect = OSError("disk full") + from contextlib import nullcontext + namespace["sandbox_capacity"] = lambda service: nullcontext() provision = extract(source, "_provision_sandbox", namespace) with pytest.raises(OSError, match="disk full"): provision(service, "sandbox", request, datetime.now(timezone.utc), datetime(2099,1,1,tzinfo=timezone.utc)) diff --git a/src/backend/tests/sandbox/test_opensandbox_provider.py b/src/backend/tests/sandbox/test_opensandbox_provider.py index 5882b8e8..0060e89f 100644 --- a/src/backend/tests/sandbox/test_opensandbox_provider.py +++ b/src/backend/tests/sandbox/test_opensandbox_provider.py @@ -45,3 +45,33 @@ async def fake_renew(s): p._renew = fake_renew asyncio.run(p._renew_in_background("chat-x", sess)) assert sess.stale_marked is False + +@pytest.mark.parametrize("domain,expected", [ + ("127.0.0.1:46074", "http://127.0.0.1:46074"), + ("http://controller:8080/", "http://controller:8080"), + ("https://controller.example/prefix/", "https://controller.example/prefix"), +]) +def test_controller_url_shared_by_sdk_health_and_renew(monkeypatch, domain, expected): + from types import SimpleNamespace + from unittest.mock import AsyncMock + osp = _import_opensandbox_module() + original = osp.settings.sandbox + class Config: + opensandbox_domain = domain + def __getattr__(self, key): + return getattr(original, key) + monkeypatch.setattr(osp, "settings", SimpleNamespace(sandbox=Config())) + p = osp.OpenSandboxProvider() + client = SimpleNamespace( + post=AsyncMock(return_value=SimpleNamespace(status_code=200)), + get=AsyncMock(return_value=SimpleNamespace(status_code=200)), + ) + p._get_http_client = lambda: client + p._session_expiry_s = lambda: 600 + async def check(): + await p._renew(osp._Session(sandbox=SimpleNamespace(id="recovered"))) + assert await p.health() + asyncio.run(check()) + assert p._make_config().get_base_url() == expected + "/v1" + assert client.post.call_args.args[0] == expected + "/v1/sandboxes/recovered/renew-expiration" + assert client.get.call_args.args[0] == expected + "/health" diff --git a/src/backend/tests/sandbox/test_os_sandbox.py b/src/backend/tests/sandbox/test_os_sandbox.py index 7d1c775f..0262b490 100644 --- a/src/backend/tests/sandbox/test_os_sandbox.py +++ b/src/backend/tests/sandbox/test_os_sandbox.py @@ -403,7 +403,7 @@ def test_labels_follow_the_plan_and_leave_with_it(tmp_path, monkeypatch): for name in ("label_writable", "label_read_only", "remove_label"): setattr(stub, name, lambda path, _name=name: calls.append((_name, path))) monkeypatch.setitem(sys.modules, "core.sandbox.oslayer.win32.label", stub) - monkeypatch.setattr(windows_runtime, "_IN_USE", Counter()) + monkeypatch.setattr(windows_runtime.process_state(), 'IN_USE', Counter()) state = str(tmp_path / "state") workspace, once, busy = (str(tmp_path / name) for name in ("ws", "once", "busy")) @@ -426,7 +426,7 @@ def test_labels_follow_the_plan_and_leave_with_it(tmp_path, monkeypatch): assert LabeledRoots(state).load() == {root_key(p) for p in (workspace, once, busy)} calls.clear() - windows_runtime._IN_USE[root_key(busy)] += 1 + windows_runtime.process_state().IN_USE[root_key(busy)] += 1 windows_runtime.apply_labels( {"state_dir": state, "writable_roots": [{"path": workspace, "read_only": []}]} ) diff --git a/src/backend/tests/sandbox/test_pool_retirement_guards.py b/src/backend/tests/sandbox/test_pool_retirement_guards.py index e53c5433..55207a1a 100644 --- a/src/backend/tests/sandbox/test_pool_retirement_guards.py +++ b/src/backend/tests/sandbox/test_pool_retirement_guards.py @@ -120,21 +120,21 @@ async def kill(): def test_destruction_guard_renews_long_controller_calls(monkeypatch, sandbox_session_store): monkeypatch.setattr(session_registry, "_LOCK_RENEW_INTERVAL_S", 0.001) - original_hold = sandbox_session_store.hold + original_hold = sandbox_session_store.compare_exchange async def scenario(): renewed = asyncio.Event() renewals = 0 - async def hold(key, token, *, ttl): + async def hold(key, expected, token, *, ttl): nonlocal renewals if key == session_registry._TRACK_KEY.format(sandbox_id="long-kill"): renewals += 1 if renewals >= 2: renewed.set() - return await original_hold(key, token, ttl=ttl) + return await original_hold(key, expected, token, ttl=ttl) - monkeypatch.setattr(sandbox_session_store, "hold", hold) + monkeypatch.setattr(sandbox_session_store, "compare_exchange", hold) async with session_registry.destruction_guard("long-kill") as allowed: assert allowed await asyncio.wait_for(renewed.wait(), timeout=1) @@ -145,23 +145,23 @@ async def hold(key, token, *, ttl): def test_lost_destruction_guard_interrupts_kill_and_preserves_adopter(monkeypatch, sandbox_session_store): monkeypatch.setattr(session_registry, "_LOCK_RENEW_INTERVAL_S", 0.001) - original_hold = sandbox_session_store.hold + original_hold = sandbox_session_store.compare_exchange async def scenario(): calls = 0 key = session_registry._TRACK_KEY.format(sandbox_id="lost-kill") - async def hold(candidate, token, *, ttl): + async def hold(candidate, expected, token, *, ttl): nonlocal calls if candidate == key: calls += 1 - if calls > 1: + if calls >= 1: await sandbox_session_store.drop(key) await sandbox_session_store.put(key, "adopter", ttl=ttl) return False - return await original_hold(candidate, token, ttl=ttl) + return await original_hold(candidate, expected, token, ttl=ttl) - monkeypatch.setattr(sandbox_session_store, "hold", hold) + monkeypatch.setattr(sandbox_session_store, "compare_exchange", hold) with pytest.raises(RuntimeError, match="destruction lease lost"): async with session_registry.destruction_guard("lost-kill") as allowed: assert allowed diff --git a/src/backend/tests/sandbox/test_product_session_scope.py b/src/backend/tests/sandbox/test_product_session_scope.py index f852e374..9c279c99 100644 --- a/src/backend/tests/sandbox/test_product_session_scope.py +++ b/src/backend/tests/sandbox/test_product_session_scope.py @@ -11,7 +11,7 @@ from core.sandbox.processes.cloud import CloudProcesses from core.sandbox.providers.opensandbox.provider import OpenSandboxProvider from core.sandbox.protocol import ProcessRequest -from core.spaces.sandbox import SpaceSandboxMixin +from core.files.spaces.sandbox import SpaceSandboxMixin from personal_agent.computer.sandbox import PersonalSandboxProvider diff --git a/src/backend/tests/sandbox/test_runtime_ownership.py b/src/backend/tests/sandbox/test_runtime_ownership.py new file mode 100644 index 00000000..efe14771 --- /dev/null +++ b/src/backend/tests/sandbox/test_runtime_ownership.py @@ -0,0 +1,61 @@ +"""Shared maintenance discovers other workers and snapshot deletion is conditional.""" +import asyncio +import time +from datetime import timedelta +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker +from core.db.models import Base, ChatSandboxSnapshot +from core.infra.time import utc_now +from core.sandbox.providers.opensandbox.provider import OpenSandboxProvider +from core.sandbox.providers.opensandbox import snapshots, tracking +from core.sandbox.sessions import registry + + +async def test_missing_binding_does_not_block_other_worker_session(monkeypatch): + monkeypatch.setattr(registry, "bindings", AsyncMock(return_value=[ + ("missing", "gone", 0), ("other-worker", "live", 0)])) + monkeypatch.setattr(registry, "lookup", AsyncMock(side_effect=["gone", "live"])) + forget = AsyncMock() + monkeypatch.setattr(registry, "forget", forget) + response = SimpleNamespace(status_code=200, json=lambda: {"metadata": {}}, + raise_for_status=lambda: None) + missing = SimpleNamespace(status_code=404, json=lambda: {"code": "DOCKER::SANDBOX_NOT_FOUND"}) + client = SimpleNamespace(get=AsyncMock(side_effect=[missing, response])) + provider = object.__new__(OpenSandboxProvider) + provider._sessions = {} + provider._domain = "http://controller" + provider._get_http_client = lambda: client + provider._get_session_lock = AsyncMock(side_effect=lambda _: asyncio.Lock()) + session = SimpleNamespace(last_active_ts=time.monotonic()) + provider._attach_session = AsyncMock(return_value=session) + await provider._discover_idle_sessions() + forget.assert_awaited_once_with("missing", "gone") + assert provider._sessions["other-worker"] is session + assert session.last_active_ts < time.monotonic() - 100 + + +async def test_snapshot_gc_cannot_remove_concurrent_replacement(tmp_path, monkeypatch): + engine = create_engine(f"sqlite:///{tmp_path / 'snap.sqlite'}") + Base.metadata.create_all(engine, tables=[Base.metadata.tables["chat_sandbox_snapshots"]]) + factory = sessionmaker(bind=engine) + monkeypatch.setattr(snapshots, "_get_db_session_local", lambda: factory) + with factory() as db: + db.add(ChatSandboxSnapshot(chat_id="chat", snapshot_id="old", sandbox_id="sandbox", + expires_at=utc_now()-timedelta(hours=1))) + db.commit() + provider = object.__new__(OpenSandboxProvider) + async def delete_remote(snapshot_id): + with factory() as db: + row = db.get(ChatSandboxSnapshot, "chat") + row.snapshot_id = "new" + row.expires_at = utc_now() + timedelta(days=1) + db.commit() + provider._delete_snapshot = delete_remote + await provider._snapshot_gc_once() + with factory() as db: + assert db.get(ChatSandboxSnapshot, "chat").snapshot_id == "new" + engine.dispose() diff --git a/src/backend/tests/sandbox/test_sandbox_accounting.py b/src/backend/tests/sandbox/test_sandbox_accounting.py index c17b8c7d..c5dddabf 100644 --- a/src/backend/tests/sandbox/test_sandbox_accounting.py +++ b/src/backend/tests/sandbox/test_sandbox_accounting.py @@ -12,7 +12,7 @@ import core.sandbox.providers.opensandbox.lifecycle as runtime import core.sandbox.sessions.registry as session_registry -from core.sandbox.providers.opensandbox.models import _Session +from core.sandbox.providers.opensandbox.models import SandboxSessionRecord as _Session from core.sandbox.providers.opensandbox.user_pool import _JupyterUserPool from core.sandbox.providers.opensandbox.provider import OpenSandboxProvider @@ -254,17 +254,17 @@ async def blocked(*args,**kwargs): def test_lost_shared_claim_interrupts_owner_without_deleting_another_token(monkeypatch, sandbox_session_store): monkeypatch.setattr(session_registry,"_LOCK_RENEW_INTERVAL_S",0.001) - original_hold=sandbox_session_store.hold + original_hold=sandbox_session_store.compare_exchange calls=0 - async def hold(key,token,*,ttl): + async def hold(key,expected,token,*,ttl): nonlocal calls calls+=1 - if calls > 1: + if calls >= 1: await sandbox_session_store.drop(key) await sandbox_session_store.put(key,"new-owner",ttl=ttl) return False - return await original_hold(key,token,ttl=ttl) - monkeypatch.setattr(sandbox_session_store,"hold",hold) + return await original_hold(key,expected,token,ttl=ttl) + monkeypatch.setattr(sandbox_session_store,"compare_exchange",hold) async def scenario(): with pytest.raises(RuntimeError,match="claim lost"): async with session_registry.claim("chat"): @@ -290,7 +290,7 @@ async def scenario(): def test_cancelled_resource_stop_preserves_session_and_reservation(monkeypatch): - from core.plugins.resources import prewarm + from core.capabilities.plugins.resources import prewarm worker=_worker(monkeypatch,None) monkeypatch.setattr(prewarm,"stop",AsyncMock(side_effect=asyncio.CancelledError())) async def scenario(): diff --git a/src/backend/tests/sandbox/test_sandbox_packaging.py b/src/backend/tests/sandbox/test_sandbox_packaging.py index 661196e7..bb7d99b1 100644 --- a/src/backend/tests/sandbox/test_sandbox_packaging.py +++ b/src/backend/tests/sandbox/test_sandbox_packaging.py @@ -23,11 +23,11 @@ def test_opensandbox_controller_copy_sources_survive_restricted_context(): assert item.as_posix() in included, f"build context excludes {item}" -def test_builtin_skill_mount_fallback_uses_central_extension_root(tmp_path, monkeypatch): - backend = tmp_path / "src/backend" - skills = backend / "extensions/skills" - skills.mkdir(parents=True) - monkeypatch.delenv("HOST_REPO_PATH", raising=False) - monkeypatch.setattr(volumes, "SKILLS_DIR", skills) - monkeypatch.setattr(volumes, "_BUILTIN_SKILLS_HOST_PATH", None) - assert volumes._resolve_builtin_skills_host_path() == str(skills) +def test_skill_mount_requires_host_storage(monkeypatch): + from types import SimpleNamespace + import pytest + from core.sandbox.errors import SandboxError + monkeypatch.setattr(volumes, "settings", SimpleNamespace( + sandbox=SimpleNamespace(opensandbox_host_storage_path=""))) + with pytest.raises(SandboxError, match="HOST_STORAGE_PATH"): + volumes._make_skills_volumes("user") diff --git a/src/backend/tests/sandbox/test_service_loop_affinity.py b/src/backend/tests/sandbox/test_service_loop_affinity.py index 8d0d8f15..6f2b9978 100644 --- a/src/backend/tests/sandbox/test_service_loop_affinity.py +++ b/src/backend/tests/sandbox/test_service_loop_affinity.py @@ -4,7 +4,7 @@ - provider 是**进程级单例**，平台内置子智能体（explorer/worker/reviewer）**复用主对话 的 session_id**，为的是看同一份 workspace；但它们跑在 ``_subagent_pool`` 线程里的 - 独立事件循环上（``core/llm/subagent_tool.py`` 的 ``asyncio.new_event_loop()``）。 + 独立事件循环上（``core/llm/subagents/tool.py`` 的 ``asyncio.new_event_loop()``）。 - opensandbox SDK 的 Sandbox / CodeInterpreter、``SandboxPool`` 的锁、共享 httpx client 都在首次使用时绑定当时的运行循环，被另一个循环碰就抛 ``is bound to a different event loop``；子智能体线程 ``loop.close()`` 之后，主循环 @@ -54,7 +54,7 @@ async def _run_in_own_loop(coro_factory): 必须走 ``run_in_executor`` 等它，**不能**同步 ``join()``：转发的前提就是服务 循环在等待期间照常运转。生产里子智能体正是 ``await loop.run_in_executor(_subagent_pool, ...)`` 派出去的 - （``core/llm/subagent_tool.py``），同步阻塞服务循环会直接死锁。 + （``core/llm/subagents/tool.py``），同步阻塞服务循环会直接死锁。 """ box = {} diff --git a/src/backend/tests/sandbox/test_session_affinity.py b/src/backend/tests/sandbox/test_session_affinity.py index 4d421d13..ffc1912b 100644 --- a/src/backend/tests/sandbox/test_session_affinity.py +++ b/src/backend/tests/sandbox/test_session_affinity.py @@ -100,7 +100,7 @@ async def get_session_lock(session_id): async def create_session_for(session_id, user_id=None): sandbox_id = f"{name}-sbx-{len(worker.created) + 1}" worker.created.append(sandbox_id) - return session_mod_models._Session( + return session_mod_models.SandboxSessionRecord( sandbox=_FakeSandbox(sandbox_id), user_id=user_id, pool_source="user", diff --git a/src/backend/tests/sandbox/test_session_lock_cross_loop.py b/src/backend/tests/sandbox/test_session_lock_cross_loop.py index 5c9e332a..fccbd566 100644 --- a/src/backend/tests/sandbox/test_session_lock_cross_loop.py +++ b/src/backend/tests/sandbox/test_session_lock_cross_loop.py @@ -3,7 +3,7 @@ 事故形态特别隐蔽： - provider 是**进程级单例**，但调用方不止一个事件循环——子智能体跑在独立线程的独立 - 循环里（``core/llm/subagent_tool.py`` 的 ``asyncio.new_event_loop()``）。 + 循环里（``core/llm/subagents/tool.py`` 的 ``asyncio.new_event_loop()``）。 - ``asyncio.Lock`` 在**首次 await 时**绑定当时的 running loop。谁先碰 provider，锁就绑给谁。 - 之后另一个循环再用就抛 ``is bound to a different event loop``。而沙箱保活 ``touch_session`` 的调用方把异常 except 掉了 → **保活静默失效** → 沙箱空闲 600s 被 diff --git a/src/backend/tests/sandbox/test_user_bound_sandbox.py b/src/backend/tests/sandbox/test_user_bound_sandbox.py index 02f043d1..a397b14c 100644 --- a/src/backend/tests/sandbox/test_user_bound_sandbox.py +++ b/src/backend/tests/sandbox/test_user_bound_sandbox.py @@ -9,7 +9,6 @@ from unittest.mock import AsyncMock import pytest -import core.sandbox.providers.opensandbox.runtime as internals_runtime from core.sandbox.providers.opensandbox.provider import OpenSandboxProvider _FLAGS = ( @@ -29,26 +28,6 @@ def _settings(**overrides): return SimpleNamespace(sandbox=SimpleNamespace(**values)) -@pytest.mark.parametrize("enabled_flag", _FLAGS) -def test_each_private_mount_requires_user_bound_sandbox(monkeypatch, enabled_flag): - patch_open_settings(monkeypatch, _settings(**{enabled_flag: True})) - - assert internals_runtime._user_bound_sandbox_required() is True - - -def test_per_user_skill_view_requires_user_bound_sandbox(monkeypatch): - """Host storage alone is enough: /workspace/skills is that user's own view.""" - patch_open_settings(monkeypatch, _settings(opensandbox_host_storage_path="/srv/storage")) - - assert internals_runtime._user_bound_sandbox_required() is True - - -def test_space_mounts_require_user_bound_sandbox_without_credential_mounts(monkeypatch): - patch_open_settings(monkeypatch, _settings()) - - assert internals_runtime._user_bound_sandbox_required() is True - - def test_dingtalk_mount_uses_user_pool_when_myspace_mount_is_disabled(monkeypatch): """HugAgentOS regression: dws=true + myspace=false must still mount user credentials.""" patch_open_settings(monkeypatch, _settings(dws_creds_bind_mount_enabled=True)) diff --git a/src/backend/tests/services/project_instructions_support.py b/src/backend/tests/services/project_instructions_support.py index ac7a8f28..2db86240 100644 --- a/src/backend/tests/services/project_instructions_support.py +++ b/src/backend/tests/services/project_instructions_support.py @@ -1,11 +1,12 @@ """Isolated project-instruction fixtures.""" from __future__ import annotations +from core.files.storage.factory import process_state as state_core_files_storage_factory import pytest from core.db.engine import Base from core.db.models import UserShadow -from core.services.project_service import ProjectService +from core.projects.edition import ProjectService from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker @@ -21,7 +22,7 @@ def env(tmp_path, monkeypatch): monkeypatch.setattr("core.db.engine.SessionLocal", factory) monkeypatch.setenv("STORAGE_TYPE", "local") monkeypatch.setenv("STORAGE_PATH", str(tmp_path / "storage")) - monkeypatch.setattr("core.storage.factory._storage_instance", None) + monkeypatch.setattr(state_core_files_storage_factory(), 'storage_instance', None) monkeypatch.setattr("core.config.local_mode.local_mode_enabled", lambda: True) with factory() as db: db.add_all( diff --git a/src/backend/tests/services/test_artifact_reader.py b/src/backend/tests/services/test_artifact_reader.py index f9984116..d40815df 100644 --- a/src/backend/tests/services/test_artifact_reader.py +++ b/src/backend/tests/services/test_artifact_reader.py @@ -1,8 +1,9 @@ -"""Unit tests for core.content.artifact_reader. +"""Unit tests for core.files.artifacts.reader. Covers caching hit/miss, ownership check, parse error path. DB is mocked via an in-memory SQLite fixture from conftest.db_session pattern. """ +from core.files.content import file_parser from datetime import datetime from types import SimpleNamespace @@ -65,7 +66,7 @@ def __exit__(self, *exc): # ── fetch_parsed_text ──────────────────────────────────────────────────────── def test_fetch_parsed_text_cache_hit(monkeypatch): - from core.content import artifact_reader as ar + from core.files.artifacts import reader as ar row = _make_artifact_row(parsed_text="cached content") sess = _FakeSession(row) @@ -79,7 +80,7 @@ def test_fetch_parsed_text_cache_hit(monkeypatch): def test_fetch_parsed_text_miss_triggers_parse_and_cache(monkeypatch): - from core.content import artifact_reader as ar + from core.files.artifacts import reader as ar row = _make_artifact_row(parsed_text=None, filename="a.txt") sess = _FakeSession(row) @@ -91,10 +92,10 @@ def test_fetch_parsed_text_miss_triggers_parse_and_cache(monkeypatch): fake_storage = SimpleNamespace( download_bytes=lambda key: b"raw bytes" ) - import core.storage as storage_mod + import core.files.storage as storage_mod monkeypatch.setattr(storage_mod, "get_storage", lambda: fake_storage) - import core.content.file_parser as fp_mod - monkeypatch.setattr(fp_mod, "parse_file", lambda b, name: "parsed text content") + import core.files.content.file_parser as fp_mod + monkeypatch.setattr(file_parser, 'parse_file', lambda b, name: "parsed text content") result = ar.fetch_parsed_text("ua_test123") assert result == "parsed text content" @@ -105,7 +106,7 @@ def test_fetch_parsed_text_miss_triggers_parse_and_cache(monkeypatch): def test_fetch_parsed_text_permission_denied(monkeypatch): - from core.content import artifact_reader as ar + from core.files.artifacts import reader as ar row = _make_artifact_row(user_id="owner_a", parsed_text="secret") sess = _FakeSession(row) @@ -118,7 +119,7 @@ def test_fetch_parsed_text_permission_denied(monkeypatch): def test_fetch_parsed_text_permission_ok(monkeypatch): - from core.content import artifact_reader as ar + from core.files.artifacts import reader as ar row = _make_artifact_row(user_id="owner_a", parsed_text="ok content") sess = _FakeSession(row) @@ -131,7 +132,7 @@ def test_fetch_parsed_text_permission_ok(monkeypatch): def test_fetch_parsed_text_deleted_artifact(monkeypatch): - from core.content import artifact_reader as ar + from core.files.artifacts import reader as ar row = _make_artifact_row(parsed_text="content", deleted_at=datetime.utcnow()) sess = _FakeSession(row) @@ -144,7 +145,7 @@ def test_fetch_parsed_text_deleted_artifact(monkeypatch): def test_fetch_parsed_text_missing_artifact(monkeypatch): - from core.content import artifact_reader as ar + from core.files.artifacts import reader as ar sess = _FakeSession(None) import core.db.engine as engine_mod @@ -155,7 +156,7 @@ def test_fetch_parsed_text_missing_artifact(monkeypatch): def test_fetch_parsed_text_parse_error_records(monkeypatch): - from core.content import artifact_reader as ar + from core.files.artifacts import reader as ar from core.infra.exceptions import StorageError row = _make_artifact_row(parsed_text=None) @@ -168,7 +169,7 @@ def _raise(_key): raise StorageError("download", "disk err") fake_storage = SimpleNamespace(download_bytes=_raise) - import core.storage as storage_mod + import core.files.storage as storage_mod monkeypatch.setattr(storage_mod, "get_storage", lambda: fake_storage) result = ar.fetch_parsed_text("ua_test123") @@ -178,7 +179,7 @@ def _raise(_key): def test_fetch_parsed_text_empty_file_id(): - from core.content import artifact_reader as ar + from core.files.artifacts import reader as ar assert ar.fetch_parsed_text("") == "" assert ar.fetch_parsed_text(None) == "" # type: ignore[arg-type] @@ -186,7 +187,7 @@ def test_fetch_parsed_text_empty_file_id(): # ── load_artifact_meta ─────────────────────────────────────────────────────── def test_load_artifact_meta_returns_shape(monkeypatch): - from core.content import artifact_reader as ar + from core.files.artifacts import reader as ar row = _make_artifact_row( filename="data.xlsx", @@ -210,7 +211,7 @@ def test_load_artifact_meta_returns_shape(monkeypatch): def test_load_artifact_meta_permission_denied(monkeypatch): - from core.content import artifact_reader as ar + from core.files.artifacts import reader as ar row = _make_artifact_row(user_id="owner_a") sess = _FakeSession(row) import core.db.engine as engine_mod @@ -219,7 +220,7 @@ def test_load_artifact_meta_permission_denied(monkeypatch): def test_load_artifact_meta_deleted(monkeypatch): - from core.content import artifact_reader as ar + from core.files.artifacts import reader as ar row = _make_artifact_row(deleted_at=datetime.utcnow()) sess = _FakeSession(row) import core.db.engine as engine_mod @@ -230,9 +231,7 @@ def test_load_artifact_meta_deleted(monkeypatch): # ── infer_source ───────────────────────────────────────────────────────────── def test_infer_source_extra_data_wins(): - from core.content.artifact_reader import ( - SOURCE_AI_GENERATED, SOURCE_USER_UPLOAD, infer_source, - ) + from core.files.artifacts.reader import SOURCE_AI_GENERATED, SOURCE_USER_UPLOAD, infer_source assert infer_source(_make_artifact_row( artifact_id="ai_x", extra_data={"source": "user_upload"} )) == SOURCE_USER_UPLOAD @@ -242,9 +241,7 @@ def test_infer_source_extra_data_wins(): def test_infer_source_falls_back_to_id_prefix(): - from core.content.artifact_reader import ( - SOURCE_AI_GENERATED, SOURCE_USER_UPLOAD, infer_source, - ) + from core.files.artifacts.reader import SOURCE_AI_GENERATED, SOURCE_USER_UPLOAD, infer_source assert infer_source(_make_artifact_row( artifact_id="ua_abc", extra_data={} )) == SOURCE_USER_UPLOAD diff --git a/src/backend/tests/services/test_artifact_summary.py b/src/backend/tests/services/test_artifact_summary.py index b20c5484..7ef8ecc7 100644 --- a/src/backend/tests/services/test_artifact_summary.py +++ b/src/backend/tests/services/test_artifact_summary.py @@ -1,4 +1,4 @@ -"""Unit tests for core.content.artifact_summary. +"""Unit tests for core.files.artifacts.summary. Each test feeds bytes of a specific type and checks the summary shape. These are pure-logic tests — no DB, no storage. @@ -8,7 +8,7 @@ import pytest -from core.content.artifact_summary import build_summary +from core.files.artifacts.summary import build_summary # ── Plain text & markdown ───────────────────────────────────────────────────── @@ -154,7 +154,7 @@ def test_build_summary_never_raises(): # ── build_summary_from_text (the parsed-text cache path) ────────────────── def test_from_text_long_pdf_like_truncates_500(): - from core.content.artifact_summary import build_summary_from_text + from core.files.artifacts.summary import build_summary_from_text text = ("本文档为年度财报，涵盖 " * 200) out = build_summary_from_text(text, "report.pdf", "application/pdf") assert "共" in out and "字" in out @@ -162,14 +162,14 @@ def test_from_text_long_pdf_like_truncates_500(): def test_from_text_short_text_returned_directly(): - from core.content.artifact_summary import build_summary_from_text + from core.files.artifacts.summary import build_summary_from_text out = build_summary_from_text("hello", "note.txt", "text/plain") # short strings return full text; cap 600 chars triggers this path assert "hello" in out def test_from_text_markdown_uses_line_truncation(): - from core.content.artifact_summary import build_summary_from_text + from core.files.artifacts.summary import build_summary_from_text text = "\n".join(f"# H{i}" for i in range(50)) out = build_summary_from_text(text, "notes.md", "text/markdown") assert "# H0" in out @@ -179,7 +179,7 @@ def test_from_text_markdown_uses_line_truncation(): def test_from_text_code_uses_line_truncation(): - from core.content.artifact_summary import build_summary_from_text + from core.files.artifacts.summary import build_summary_from_text text = "\n".join(f"def f{i}(): pass" for i in range(100)) out = build_summary_from_text(text, "m.py", "text/x-python") assert "def f29" in out @@ -188,7 +188,7 @@ def test_from_text_code_uses_line_truncation(): def test_from_text_csv_shows_first_rows(): - from core.content.artifact_summary import build_summary_from_text + from core.files.artifacts.summary import build_summary_from_text text = "\n".join([f"r{i},c{i},d{i}" for i in range(20)]) out = build_summary_from_text(text, "data.csv", "text/csv") assert "r0" in out @@ -200,7 +200,7 @@ def test_from_text_csv_shows_first_rows(): def test_from_text_xlsx_preserves_sheet_markers(): """When the parsed markdown uses `### Sheet: N` headers, the truncation should preferentially cut at sheet boundaries.""" - from core.content.artifact_summary import build_summary_from_text + from core.files.artifacts.summary import build_summary_from_text text = ( "### Sheet 1: 员工\n| 姓名 | 部门 |\n|---|---|\n| 张三 | 工程 |\n" * 20 + "### Sheet 2: 薪酬\n| 工号 | 金额 |\n|---|---|\n| E001 | 10000 |\n" * 20 @@ -214,14 +214,14 @@ def test_from_text_xlsx_preserves_sheet_markers(): def test_from_text_empty_returns_metadata_only(): - from core.content.artifact_summary import build_summary_from_text + from core.files.artifacts.summary import build_summary_from_text out = build_summary_from_text("", "mystery.bin", "application/octet-stream") assert "mystery.bin" in out assert "octet-stream" in out def test_from_text_whitespace_only_returns_metadata_only(): - from core.content.artifact_summary import build_summary_from_text + from core.files.artifacts.summary import build_summary_from_text out = build_summary_from_text(" \n \t ", "blank.txt", "text/plain") assert "blank.txt" in out diff --git a/src/backend/tests/services/test_audio_live.py b/src/backend/tests/services/test_audio_live.py index c4ec40c4..e962f9ca 100644 --- a/src/backend/tests/services/test_audio_live.py +++ b/src/backend/tests/services/test_audio_live.py @@ -13,7 +13,7 @@ import pytest from fastapi.testclient import TestClient -from core.db.model_repository import assign_role, create_provider +from core.models.repository import assign_role, create_provider from tests.services.test_audio_routes import setup as setup BASE_URL = os.environ.get("HUGAGENT_TEST_ASR_URL") diff --git a/src/backend/tests/services/test_audio_models.py b/src/backend/tests/services/test_audio_models.py index 043c5a7f..f314777e 100644 --- a/src/backend/tests/services/test_audio_models.py +++ b/src/backend/tests/services/test_audio_models.py @@ -9,10 +9,10 @@ from core.db.engine import Base from core.db.audio_model_upgrade import reconcile_audio_model_type -from core.db.model_repository import create_provider, assign_role, get_provider, import_all, export_all -from core.llm.providers.registry import validate_payload -from core.services.model_config import ModelConfigService -from core.services import model_config +from core.models.repository import create_provider, assign_role, get_provider, import_all, export_all +from core.models.providers.registry import validate_payload +from core.models.config import ModelConfigService +from core.models import config as model_config @pytest.fixture diff --git a/src/backend/tests/services/test_audio_routes.py b/src/backend/tests/services/test_audio_routes.py index a642d6f1..0fd71983 100644 --- a/src/backend/tests/services/test_audio_routes.py +++ b/src/backend/tests/services/test_audio_routes.py @@ -12,8 +12,8 @@ from api.routes.v1 import audio from core.db.engine import Base, get_db from core.db.models import UserShadow -from core.db.model_repository import create_provider, assign_role, update_provider -from core.services.audio_transcription import resolve_audio_model +from core.models.repository import create_provider, assign_role, update_provider +from core.chat.audio_transcription import resolve_audio_model from tests.services.test_audio_transcription import wav diff --git a/src/backend/tests/services/test_audio_transcription.py b/src/backend/tests/services/test_audio_transcription.py index d0dc99ca..7815b640 100644 --- a/src/backend/tests/services/test_audio_transcription.py +++ b/src/backend/tests/services/test_audio_transcription.py @@ -5,7 +5,7 @@ import pytest from fastapi import HTTPException -from core.services.audio_transcription import AudioModel, transcribe, validate_wav +from core.chat.audio_transcription import AudioModel, transcribe, validate_wav def wav(seconds=0.5, rate=16000, channels=1): @@ -58,7 +58,7 @@ def test_hostile_wav_chunks_are_rejected(chunk): @pytest.mark.parametrize("url", ["http://[bad", "http://localhost:invalid/v1", "http://localhost:999999/v1", "http://localhost:0/v1"]) def test_malformed_model_urls_are_sanitized(url): - from core.services.audio_transcription import transcription_url + from core.chat.audio_transcription import transcription_url with pytest.raises(HTTPException) as error: transcription_url(url) assert error.value.status_code == 503 diff --git a/src/backend/tests/services/test_chat_mode_service.py b/src/backend/tests/services/test_chat_mode_service.py index 7e9c12ca..407137aa 100644 --- a/src/backend/tests/services/test_chat_mode_service.py +++ b/src/backend/tests/services/test_chat_mode_service.py @@ -13,7 +13,8 @@ from core.db.engine import Base from core.db.models.chat_mode import ChatMode from core.infra.exceptions import BadRequestError, ResourceNotFoundError -from core.services.chat_mode_service import SLUG_STANDARD, ChatModeService +from core.chat.mode_types import SLUG_STANDARD +from core.chat.modes import ChatModeService from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker diff --git a/src/backend/tests/services/test_chat_reference_service.py b/src/backend/tests/services/test_chat_reference_service.py index 93a30e6d..e804e7b0 100644 --- a/src/backend/tests/services/test_chat_reference_service.py +++ b/src/backend/tests/services/test_chat_reference_service.py @@ -10,16 +10,7 @@ import pytest from core.chat.context import build_effective_user_message from core.db.models import ChatMessage, ChatSession -from core.services.chat_reference_service import ( - OVERVIEW_MAX_CHARS, - READ_TOTAL_MAX_CHARS, - list_referencable_sessions, - read_session_messages, - render_reference_block, - resolve_reference_cards, - session_brief, - session_card, -) +from core.chat.references import OVERVIEW_MAX_CHARS, READ_TOTAL_MAX_CHARS, list_referencable_sessions, read_session_messages, render_reference_block, resolve_reference_cards, session_brief, session_card def _chat(db, chat_id: str, *, user_id: str = "u1", title: str = "会话", project_id=None): diff --git a/src/backend/tests/services/test_chat_sequencer.py b/src/backend/tests/services/test_chat_sequencer.py index e25e20c4..9435a660 100644 --- a/src/backend/tests/services/test_chat_sequencer.py +++ b/src/backend/tests/services/test_chat_sequencer.py @@ -7,13 +7,21 @@ from datetime import datetime from threading import Barrier +import core.agent_runtime.runner as runner +import core.chat.runtime.recovery as recovery +import core.chat.runtime.worker as runtime_worker +import core.db.engine as db_engine +import core.runs.events.writer as writer +import core.runs.ownership as ownership +import core.runs.runtime as runtime +import core.runs.tasks as tasks import pytest from core.chat import inflight from core.db.engine import Base from core.db.models import ChatMessage, ChatRun, ChatSession from core.db.models.chat import reserve_chat_sequences -from core.services.chat_sequencer import ChatBusyError, ChatSequencer -from core.services.chat_service import ChatService +from core.chat.sequencer import ChatBusyError, ChatSequencer +from core.chat.service import ChatService from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker @@ -111,17 +119,17 @@ async def test_restart_recovery_releases_writer_slot(db_session, monkeypatch): request_payload={"kind": "chat"}, ) - from orchestration import chat_run_executor + pass Session = sessionmaker(bind=db_session.get_bind()) async def fake_terminal(*_args, **_kwargs): return None - monkeypatch.setattr(chat_run_executor, "SessionLocal", Session) - monkeypatch.setattr(chat_run_executor, "_write_terminal_to_stream", fake_terminal) + monkeypatch.setattr(db_engine, "SessionLocal", Session) + monkeypatch.setattr(writer, "_write_terminal_to_stream", fake_terminal) - assert await chat_run_executor.recover_orphan_runs() == 1 + assert await recovery.recover_orphan_runs() == 1 db_session.expire_all() recovered = db_session.get(ChatRun, first.run.run_id) assert recovered.status == "failed" @@ -144,10 +152,10 @@ async def test_cancel_keeps_writer_fenced_until_registered_worker_stops(db_sessi user_content="first", request_payload={"kind": "chat"}, ) - from orchestration import chat_run_executor + pass Session = sessionmaker(bind=db_session.get_bind()) - monkeypatch.setattr(chat_run_executor, "SessionLocal", Session) + monkeypatch.setattr(db_engine, "SessionLocal", Session) cleaning_up = asyncio.Event() async def worker(): @@ -159,12 +167,12 @@ async def worker(): raise task = asyncio.create_task(worker()) - chat_run_executor._active_runs[accepted.run.run_id] = task + tasks.process_state().active_runs[accepted.run.run_id] = task task.add_done_callback( - lambda _task: chat_run_executor._active_runs.pop(accepted.run.run_id, None) + lambda _task: tasks.process_state().active_runs.pop(accepted.run.run_id, None) ) cancel_task = asyncio.create_task( - chat_run_executor.cancel_run(accepted.run.run_id, user_id="user-1") + runtime.cancel_run(accepted.run.run_id, user_id="user-1") ) await cleaning_up.wait() @@ -193,18 +201,18 @@ async def test_cross_process_cancel_retains_writer_fence_until_worker_acknowledg user_content="first", request_payload={"kind": "chat"}, ) - from orchestration import chat_run_executor + pass Session = sessionmaker(bind=db_session.get_bind()) - monkeypatch.setattr(chat_run_executor, "SessionLocal", Session) - chat_run_executor._active_runs.pop(accepted.run.run_id, None) + monkeypatch.setattr(db_engine, "SessionLocal", Session) + tasks.process_state().active_runs.pop(accepted.run.run_id, None) async def fake_terminal(*_args, **_kwargs): return None - monkeypatch.setattr(chat_run_executor, "_write_terminal_to_stream", fake_terminal) + monkeypatch.setattr(writer, "_write_terminal_to_stream", fake_terminal) - assert await chat_run_executor.cancel_run(accepted.run.run_id, user_id="user-1") is True + assert await runtime.cancel_run(accepted.run.run_id, user_id="user-1") is True with Session() as check_db: cancelled = check_db.get(ChatRun, accepted.run.run_id) assert (cancelled.status, cancelled.writer_slot) == ("cancelled", "main") @@ -228,17 +236,17 @@ async def test_cancelled_pending_run_cannot_be_revived_by_late_remote_worker( user_content="first", request_payload={"kind": "chat"}, ) - from orchestration import chat_run_executor + pass Session = sessionmaker(bind=db_session.get_bind()) - monkeypatch.setattr(chat_run_executor, "SessionLocal", Session) - chat_run_executor._active_runs.pop(accepted.run.run_id, None) + monkeypatch.setattr(db_engine, "SessionLocal", Session) + tasks.process_state().active_runs.pop(accepted.run.run_id, None) async def fake_terminal(*_args, **_kwargs): return None - monkeypatch.setattr(chat_run_executor, "_write_terminal_to_stream", fake_terminal) - assert await chat_run_executor.cancel_run(accepted.run.run_id, user_id="user-1") is True + monkeypatch.setattr(writer, "_write_terminal_to_stream", fake_terminal) + assert await runtime.cancel_run(accepted.run.run_id, user_id="user-1") is True workflow_started = False @@ -254,11 +262,11 @@ async def fake_xadd(*_args, **_kwargs): async def fake_expire(*_args, **_kwargs): return None - monkeypatch.setattr(chat_run_executor, "astream_chat_workflow", forbidden_workflow) - monkeypatch.setattr(chat_run_executor, "_xadd_event", fake_xadd) - monkeypatch.setattr(chat_run_executor, "_expire_stream", fake_expire) + monkeypatch.setattr(runner, "astream_chat_workflow", forbidden_workflow) + monkeypatch.setattr(writer, 'xadd_event', fake_xadd) + monkeypatch.setattr(writer, 'expire_stream', fake_expire) - await chat_run_executor._run_workflow( + await runtime_worker._run_workflow( run_id=accepted.run.run_id, chat_id="chat-1", user_id="user-1", @@ -287,19 +295,19 @@ async def test_losing_worker_cannot_release_concurrently_cancelled_winner_fence( user_content="first", request_payload={"kind": "chat"}, ) - from orchestration import chat_run_executor + pass Session = sessionmaker(bind=db_session.get_bind()) - monkeypatch.setattr(chat_run_executor, "SessionLocal", Session) + monkeypatch.setattr(db_engine, "SessionLocal", Session) # A successful claim hands back the instant the run was marked running; the # worker sends that on the first frame instead of reading the row again. - assert chat_run_executor._claim_run_execution(accepted.run.run_id) is not None + assert ownership._claim_run_execution(accepted.run.run_id) is not None - original_claim = chat_run_executor._claim_run_execution + original_claim = ownership._claim_run_execution def lose_claim_then_cancel(run_id: str): assert original_claim(run_id) is None - assert chat_run_executor._request_run_cancel(run_id) is True + assert ownership._request_run_cancel(run_id) is True return None workflow_started = False @@ -310,10 +318,10 @@ async def forbidden_workflow(**_kwargs): if False: # pragma: no cover yield None - monkeypatch.setattr(chat_run_executor, "_claim_run_execution", lose_claim_then_cancel) - monkeypatch.setattr(chat_run_executor, "astream_chat_workflow", forbidden_workflow) + monkeypatch.setattr(ownership, "_claim_run_execution", lose_claim_then_cancel) + monkeypatch.setattr(runner, "astream_chat_workflow", forbidden_workflow) - await chat_run_executor._run_workflow( + await runtime_worker._run_workflow( run_id=accepted.run.run_id, chat_id="chat-1", user_id="user-1", diff --git a/src/backend/tests/services/test_chat_titles.py b/src/backend/tests/services/test_chat_titles.py index 4a5c31f9..a0e5edbb 100644 --- a/src/backend/tests/services/test_chat_titles.py +++ b/src/backend/tests/services/test_chat_titles.py @@ -1,13 +1,15 @@ """First-message titles use an independent call and respect later user edits.""" + import asyncio from types import SimpleNamespace +import core.chat.runtime.admission as runtime_admission import pytest +from core.db.engine import Base +from core.db.models import ChatMessage, ChatSession, UserShadow +from core.chat import titles as chat_titles from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker -from core.db.engine import Base -from core.db.models import ChatSession, ChatMessage, UserShadow -from core.services import chat_titles @pytest.fixture @@ -132,9 +134,9 @@ async def generate(messages): def test_metadata_patch_refreshes_stale_state_and_keeps_a_manual_title(env, monkeypatch): - from core.services.chat_service import ChatService - from core.services.session_operations import sessions - from core.services.session_operations.schemas import UpdateChatRequest + from core.chat.service import ChatService + from core.chat.operations import sessions + from core.chat.operations.schemas import UpdateChatRequest # Use the public session operation with a session that cached old metadata. with env() as stale: @@ -147,7 +149,7 @@ def test_metadata_patch_refreshes_stale_state_and_keeps_a_manual_title(env, monk fresh.commit() monkeypatch.setattr(ChatService, "get_session_with_access", lambda self, cid, uid: (self.db.get(ChatSession, cid), "admin")) - monkeypatch.setattr(sessions, "_session_view_for_user", + monkeypatch.setattr(sessions, 'session_view_for_user', lambda db, row, uid, level: {"title": row.title, "metadata": row.extra_data}) def update(self, cid, fields, **kwargs): row = self.db.get(ChatSession, cid) @@ -161,18 +163,22 @@ def update(self, cid, fields, **kwargs): def test_real_admission_schedules_title_before_answer_completion(tmp_path, monkeypatch): - from tests.api.test_agent_response_admission import _route_database, _patch_common_chat_route, _user - from core.services.session_operations import admission, session_context from core.chat.schemas import ChatRequest - from orchestration import chat_run_executor + from core.chat.operations import admission, session_context + from tests.api.test_agent_response_admission import ( + _patch_common_chat_route, + _route_database, + _user, + ) + pass engine, factory = _route_database(tmp_path, "admission-title.db") monkeypatch.setattr(chat_titles.db_engine, "SessionLocal", factory) _patch_common_chat_route(monkeypatch) - monkeypatch.setattr(session_context, "_load_session_messages", lambda *args: []) + monkeypatch.setattr(session_context, 'load_session_messages', lambda *args: []) async def start(**kwargs): return kwargs["accepted_run"] - monkeypatch.setattr(chat_run_executor, "start_run", start) + monkeypatch.setattr(runtime_admission, "start_run", start) async def scenario(): entered, release = asyncio.Event(), asyncio.Event() async def generate(messages): @@ -181,7 +187,7 @@ async def generate(messages): return "首问标题" model(monkeypatch, generate) with factory() as db: - response = await admission._start_response_run( + response = await admission.start_response_run( ChatRequest(chat_id="chat-1", message="首条问题", stream=True), _user(), db) assert response.run_id await asyncio.wait_for(entered.wait(), 2) @@ -201,6 +207,7 @@ async def generate(messages): def test_title_write_keeps_concurrent_message_activity_time(env, monkeypatch): from datetime import datetime, timezone + from sqlalchemy import event async def scenario(): diff --git a/src/backend/tests/services/test_kb_multimodal_assets.py b/src/backend/tests/services/test_kb_multimodal_assets.py deleted file mode 100644 index c7ab4984..00000000 --- a/src/backend/tests/services/test_kb_multimodal_assets.py +++ /dev/null @@ -1,657 +0,0 @@ -"""知识库多模态（路径 A：图生文入索引）的单元测试。 - -盯的是那几处「错了不会报错、只会悄悄变差或变脏」的地方： - -* 图片上传曾被当纯文本 latin-1 硬解成乱码入库——静默污染向量库，必须回归； -* 富解析的开关必须是**显式**传给解析服务的，漏传就永远拿不到图，且没有任何报错； -* 图片字节是 ``data:image/...;base64,`` 数据 URI，少剥一层前缀就存进一堆坏文件； -* 资产必须落在**包含它的父块**上——挂错块，检索命中后回溯到的是别处的正文； -* 占位符改写必须同时改父块和子块，否则入库正文与被向量化的文本不一致； -* 未配置视觉模型是正常状态，不能因此让整篇文档索引失败； -* KB 的批量图像理解不许占满进程级的视觉桥信号量，否则对话看图会被排到后面。 -""" - -from __future__ import annotations - -import base64 -import json -from types import SimpleNamespace - -import pytest -from core.content import file_parser -from core.kb import kb_assets, kb_parser - -# ── 富解析：参数与解码 ──────────────────────────────────────────────────────── - - -_PNG_1PX = base64.b64decode( - "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==" -) - - -class _FakeResponse: - def __init__(self, payload: dict) -> None: - self._payload = payload - - def raise_for_status(self) -> None: # pragma: no cover - always OK in these tests - return None - - def json(self) -> dict: - return self._payload - - -def _install_fake_parser(monkeypatch, payload: dict) -> dict: - """Stub the parse service; returns a dict that captures the posted form data.""" - captured: dict = {} - - def _fake_post(url, files=None, data=None, timeout=None): - captured["url"] = url - captured["data"] = data - captured["filename"] = files["files"][0] if files else None - return _FakeResponse(payload) - - monkeypatch.setattr(file_parser, "_cfg_api_url", lambda: "http://parser.test/file_parse") - monkeypatch.setattr(file_parser.requests, "post", _fake_post) - return captured - - -_IMG_NAME = "abc123.jpg" -_RICH_PAYLOAD = { - "results": { - "doc": { - "md_content": f"# 标题\n\n![](images/{_IMG_NAME})\n\n图 1 季度营收趋势\n\n结论段落。", - "images": { - _IMG_NAME: "data:image/jpeg;base64," + base64.b64encode(b"JPEGBYTES").decode() - }, - "content_list": json.dumps( - [ - { - "type": "image", - "img_path": f"images/{_IMG_NAME}", - "image_caption": ["图 1 季度营收趋势"], - "image_footnote": [], - "bbox": [139, 178, 647, 431], - "page_idx": 3, - } - ] - ), - } - } -} - - -def test_plain_parse_does_not_request_media(monkeypatch): - """纯文本路径（对话附件）不能顺手把图片一起要回来：图片是 base64 内联在响应体里的， - 一篇图多的 PDF 会凭空多出几 MB 传输，而这条路径根本用不到。""" - captured = _install_fake_parser(monkeypatch, _RICH_PAYLOAD) - - file_parser.parse_pdf(b"%PDF-1.4", "a.pdf") - - assert "return_images" not in captured["data"] - assert "return_content_list" not in captured["data"] - - -def test_rich_parse_requests_media_and_decodes_data_uri(monkeypatch): - captured = _install_fake_parser(monkeypatch, _RICH_PAYLOAD) - - result = file_parser.parse_pdf_rich(b"%PDF-1.4", "a.pdf") - - assert captured["data"]["return_images"] == "true" - assert captured["data"]["return_content_list"] == "true" - # 数据 URI 前缀必须剥掉，否则存下来的是坏文件 - assert result.images[f"images/{_IMG_NAME}"] == b"JPEGBYTES" - # 裸文件名也要能查到：markdown 里带 images/ 前缀，images 字典按裸名索引 - assert result.images[_IMG_NAME] == b"JPEGBYTES" - # content_list 是 JSON **字符串**，不是数组 - assert result.blocks[0]["img_path"] == f"images/{_IMG_NAME}" - - -def test_rich_parse_survives_undecodable_image(monkeypatch): - payload = json.loads(json.dumps(_RICH_PAYLOAD)) - payload["results"]["doc"]["images"] = {_IMG_NAME: "data:image/jpeg;base64,!!!not-base64!!!"} - _install_fake_parser(monkeypatch, payload) - - result = file_parser.parse_pdf_rich(b"%PDF-1.4", "a.pdf") - - # 单张图坏掉不该让整篇文档解析失败——正文照常返回 - assert "标题" in result.markdown - - -# ── 解析：图片上传与资产抽取 ────────────────────────────────────────────────── - - -def test_uploaded_image_no_longer_indexed_as_binary_garbage(): - """上传白名单放行 png/jpg/webp/gif，但解析侧原先没有图片分支， - 会落到 latin-1 兜底把二进制解成乱码段落并向量化——静默污染，必须挡住。""" - paragraphs, assets = kb_parser.extract_text_with_assets(_PNG_1PX, "image/png") - - joined = "".join(p["text"] for p in paragraphs) - assert "\x89" not in joined and "PNG" not in joined - assert len(assets) == 1 - assert assets[0].kind == "image" - assert assets[0].data == _PNG_1PX - # 占位符必须真的出现在段落里，否则后面定位不到父块 - assert assets[0].placeholder in joined - - -def test_pdf_assets_carry_caption_and_locator(monkeypatch): - _install_fake_parser(monkeypatch, _RICH_PAYLOAD) - - paragraphs, assets = kb_parser.extract_text_with_assets(b"%PDF-1.4", "application/pdf") - - assert [p["text"] for p in paragraphs][0].startswith("# 标题") - assert len(assets) == 1 - asset = assets[0] - assert asset.text_content == "图 1 季度营收趋势" - assert asset.locator == {"page_idx": 3, "bbox": [139, 178, 647, 431]} - assert asset.mime_type == "image/jpeg" - - -def test_rich_parse_failure_degrades_to_text(monkeypatch): - """解析服务不支持这些参数（或直接抽风）时，必须退回今天的纯文本行为， - 而不是让整篇文档索引失败。""" - - def _boom(*_args, **_kwargs): - raise RuntimeError("parser exploded") - - monkeypatch.setattr(file_parser, "parse_pdf_rich", _boom) - monkeypatch.setattr(file_parser, "parse_file", lambda *_a, **_k: "# 纯文本兜底") - - paragraphs, assets = kb_parser.extract_text_with_assets(b"%PDF-1.4", "application/pdf") - - assert assets == [] - assert "纯文本兜底" in "".join(p["text"] for p in paragraphs) - - -def test_with_assets_false_keeps_legacy_path(monkeypatch): - """关掉多模态时不能再走重解析——parse_pdf_rich 一次都不该被调到。""" - calls: list = [] - - def _record(*_args, **_kwargs): - calls.append(1) - raise AssertionError("rich parse must not run when assets are disabled") - - monkeypatch.setattr(file_parser, "parse_pdf_rich", _record) - monkeypatch.setattr(file_parser, "parse_file", lambda *_a, **_k: "# 正文") - - paragraphs, assets = kb_parser.extract_text_with_assets( - b"%PDF-1.4", "application/pdf", with_assets=False - ) - - assert calls == [] - assert assets == [] - assert paragraphs - - -# ── 落库：父块定位与占位符改写 ──────────────────────────────────────────────── - - -class _FakeStorage: - def __init__(self) -> None: - self.objects: dict[str, bytes] = {} - self.deleted: list[str] = [] - - def upload_bytes(self, content: bytes, storage_key: str) -> str: - self.objects[storage_key] = content - return f"memory://{storage_key}" - - def download_bytes(self, storage_key: str) -> bytes: - return self.objects[storage_key] - - def delete(self, storage_key: str) -> None: - self.deleted.append(storage_key) - self.objects.pop(storage_key, None) - - -class _FakeSession: - """Just enough Session surface for persist_assets/generate_captions.""" - - def __init__(self) -> None: - self.added: list = [] - self.commits = 0 - - def add(self, obj) -> None: - self.added.append(obj) - - def commit(self) -> None: - self.commits += 1 - - def refresh(self, _obj) -> None: - return None - - -@pytest.fixture() -def fake_storage(monkeypatch): - storage = _FakeStorage() - monkeypatch.setattr("core.storage.get_storage", lambda: storage) - return storage - - -def _parent(parent_id: str, content: str): - child = kb_parser.ChildChunk(child_id=f"{parent_id}_0", content=content, index=0) - return kb_parser.ParentChunk(parent_id=parent_id, content=content, children=[child]) - - -def test_asset_lands_on_the_chunk_that_contains_it(fake_storage, monkeypatch): - """挂错父块的后果是检索命中后回溯到别处的正文——比丢图严重得多。""" - monkeypatch.setattr(kb_assets, "_MIN_ASSET_BYTES", 0) - parents = [ - _parent("p_first", "第一段没有图。"), - _parent("p_second", "第二段：![](images/x.jpg) 见上图。"), - ] - assets = [ - kb_parser.RawAsset(data=b"x" * 100, placeholder="images/x.jpg", mime_type="image/jpeg") - ] - db = _FakeSession() - - created = kb_assets.persist_assets( - db, kb_id="kb1", document_id="doc1", user_id="u1", assets=assets, parents=parents - ) - - assert len(created) == 1 - assert created[0].chunk_id == "p_second" - assert fake_storage.objects # 字节确实落盘了 - - -def test_placeholder_rewritten_in_both_parent_and_child(fake_storage, monkeypatch): - """入库正文与被向量化的子块文本必须是同一份；只改父块会让两者分叉。""" - monkeypatch.setattr(kb_assets, "_MIN_ASSET_BYTES", 0) - parents = [_parent("p1", "见 ![](images/x.jpg) 图。")] - assets = [ - kb_parser.RawAsset(data=b"x" * 100, placeholder="images/x.jpg", mime_type="image/jpeg") - ] - db = _FakeSession() - - created = kb_assets.persist_assets( - db, kb_id="kb1", document_id="doc1", user_id="u1", assets=assets, parents=parents - ) - - url = kb_assets.asset_public_url(created[0].asset_id) - assert url in parents[0].content - assert url in parents[0].children[0].content - assert "images/x.jpg" not in parents[0].content - - -def test_tiny_assets_are_dropped(fake_storage): - """版面切出来的分割线/项目符号图标不值一次 VLM 调用，也不该进检索。""" - parents = [_parent("p1", "![](images/tiny.jpg)")] - assets = [ - kb_parser.RawAsset(data=b"tiny", placeholder="images/tiny.jpg", mime_type="image/jpeg") - ] - - created = kb_assets.persist_assets( - _FakeSession(), kb_id="kb1", document_id="d1", user_id="u1", assets=assets, parents=parents - ) - - assert created == [] - assert fake_storage.objects == {} - - -def test_storage_failure_skips_one_asset_not_the_document(monkeypatch): - monkeypatch.setattr(kb_assets, "_MIN_ASSET_BYTES", 0) - - class _BrokenStorage(_FakeStorage): - def upload_bytes(self, content, storage_key): - raise RuntimeError("bucket on fire") - - monkeypatch.setattr("core.storage.get_storage", lambda: _BrokenStorage()) - parents = [_parent("p1", "![](images/x.jpg)")] - assets = [ - kb_parser.RawAsset(data=b"x" * 100, placeholder="images/x.jpg", mime_type="image/jpeg") - ] - - created = kb_assets.persist_assets( - _FakeSession(), kb_id="kb1", document_id="d1", user_id="u1", assets=assets, parents=parents - ) - - assert created == [] - assert "images/x.jpg" in parents[0].content # 没落盘就不该改写成死链 - - -# ── 图生文：走视觉桥 ───────────────────────────────────────────────────────── - - -def _asset_row(**kwargs): - base = dict( - asset_id="a1", - kind="image", - mime_type="image/jpeg", - storage_key="k1", - caption=None, - text_content="图 1 季度营收趋势", - caption_status="pending", - chunk_id="p1", - asset_index=0, - extra_data={}, - vector_state={}, - ) - base.update(kwargs) - return SimpleNamespace(**base) - - -def _evidence(summary="柱状图：四个季度营收，Q4 最高", ocr="Q1 Q2 Q3 Q4 营收(亿元)"): - """构造一份视觉桥证据（只用到 kb 侧真正读的字段）。""" - return SimpleNamespace( - summary=summary, - ocr=SimpleNamespace(full_text=ocr), - semantics=SimpleNamespace( - scene="财务图表", - intent="展示季度营收趋势", - entities=[SimpleNamespace(model_dump=lambda **_k: {"name": "Q4", "type": "季度"})], - relations=[], - ), - uncertainty=["坐标轴单位模糊"], - ) - - -def _vision_result(evidence=None): - return SimpleNamespace( - evidence=evidence if evidence is not None else _evidence(), - to_meta=lambda: {"model": "qwen-vl", "cached": False}, - ) - - -def _install_bridge(monkeypatch, results, *, available=True, calls=None): - """替换视觉桥：results 为按批返回的结果列表（长度与该批输入一致）。""" - monkeypatch.setattr(kb_assets, "vision_available", lambda: available) - - def _fake_describe(images, loop=None): - if calls is not None: - calls.append(len(images)) - return [results.pop(0) if results else None for _ in images] - - monkeypatch.setattr(kb_assets, "_describe_batch", _fake_describe) - - -def test_no_vision_model_is_a_degradation_not_a_failure(monkeypatch): - """未指派 vision 角色、主模型也不识图，是正常状态：图注照常入索引。 - 这正是「视觉模型可以晚点再配」的前提——之后回填不用重新解析文档。""" - monkeypatch.setattr(kb_assets, "vision_available", lambda: False) - rows = [_asset_row()] - db = _FakeSession() - - done = kb_assets.generate_captions(db, rows) - - assert done == 0 - assert rows[0].caption_status == "skipped" - assert "季度营收趋势" in kb_assets.asset_index_text(rows[0], title="年报") - - -def test_evidence_lands_in_caption_and_ocr_merges_into_text(fake_storage, monkeypatch): - """证据要落到正确的字段上：概述进 caption，OCR 并进 text_content（与图注共存）。 - 并错了检索文本就废了——图注是作者写的用途，OCR 是图里实际的字，两者都要。""" - fake_storage.objects["k1"] = b"x" * 100 - _install_bridge(monkeypatch, [_vision_result()]) - rows = [_asset_row()] - - done = kb_assets.generate_captions(_FakeSession(), rows) - - assert done == 1 - assert rows[0].caption_status == "completed" - assert rows[0].caption == "柱状图：四个季度营收，Q4 最高" - assert "图 1 季度营收趋势" in rows[0].text_content # 原图注保留 - assert "Q1 Q2 Q3 Q4" in rows[0].text_content # OCR 并入 - # 实体/不确定项留档，给后续本体层与人工复核用 - assert rows[0].extra_data["vision"]["entities"][0]["name"] == "Q4" - assert rows[0].extra_data["vision"]["uncertainty"] == ["坐标轴单位模糊"] - assert rows[0].extra_data["vision"]["meta"]["model"] == "qwen-vl" - - -def test_ocr_identical_to_caption_is_not_duplicated(fake_storage, monkeypatch): - fake_storage.objects["k1"] = b"x" * 100 - _install_bridge(monkeypatch, [_vision_result(_evidence(ocr="图 1 季度营收趋势"))]) - rows = [_asset_row()] - - kb_assets.generate_captions(_FakeSession(), rows) - - assert rows[0].text_content == "图 1 季度营收趋势" - - -def test_text_content_is_capped(fake_storage, monkeypatch): - """满是文字的扫描件能吐出几万字，直接入库会把行和向量文本一起撑爆。""" - fake_storage.objects["k1"] = b"x" * 100 - monkeypatch.setattr(kb_assets, "_MAX_TEXT_CONTENT_CHARS", 50) - _install_bridge(monkeypatch, [_vision_result(_evidence(ocr="字" * 500))]) - rows = [_asset_row()] - - kb_assets.generate_captions(_FakeSession(), rows) - - assert len(rows[0].text_content) == 50 - - -def test_single_image_failure_keeps_asset_indexable(fake_storage, monkeypatch): - """视觉桥对单张返回 None（任何失败都这样降级），资产仍要凭图注可检索。""" - fake_storage.objects["k1"] = b"x" * 100 - _install_bridge(monkeypatch, [None]) - rows = [_asset_row()] - - done = kb_assets.generate_captions(_FakeSession(), rows) - - assert done == 0 - assert rows[0].caption_status == "failed" - assert kb_assets.asset_index_text(rows[0]).strip() != "" - - -def test_whole_batch_exception_does_not_fail_the_document(fake_storage, monkeypatch): - fake_storage.objects["k1"] = b"x" * 100 - monkeypatch.setattr(kb_assets, "vision_available", lambda: True) - - def _boom(_images, _loop=None): - raise RuntimeError("gateway down") - - monkeypatch.setattr(kb_assets, "_describe_batch", _boom) - rows = [_asset_row()] - - assert kb_assets.generate_captions(_FakeSession(), rows) == 0 - assert rows[0].caption_status == "failed" - - -def test_kb_never_saturates_the_shared_vision_lane(fake_storage, monkeypatch): - """视觉桥的并发信号量是进程级的：一篇图海文档一次性 gather 过去会把对话里的看图 - 请求全排到后面。KB 必须按小批提交，任何时刻只占用其中几格。""" - for i in range(7): - fake_storage.objects[f"k{i}"] = b"x" * 100 - calls: list = [] - _install_bridge(monkeypatch, [], calls=calls) - monkeypatch.setattr(kb_assets, "_CAPTION_BATCH", 2) - rows = [_asset_row(asset_id=f"a{i}", storage_key=f"k{i}") for i in range(7)] - - kb_assets.generate_captions(_FakeSession(), rows) - - assert calls == [2, 2, 2, 1], f"批次切分不对: {calls}" - assert max(calls) <= 2 - - -def test_unreadable_asset_is_skipped_without_calling_vision(fake_storage, monkeypatch): - """存储里读不到的那张不该占用一次视觉调用，也不该拖垮同批其他图。""" - fake_storage.objects["k2"] = b"x" * 100 - calls: list = [] - _install_bridge(monkeypatch, [_vision_result()], calls=calls) - monkeypatch.setattr(kb_assets, "_CAPTION_BATCH", 5) - rows = [_asset_row(asset_id="a1", storage_key="missing"), _asset_row(asset_id="a2", storage_key="k2")] - - done = kb_assets.generate_captions(_FakeSession(), rows) - - assert calls == [1] # 只送了能读出来的那张 - assert rows[0].caption_status == "failed" - assert done == 1 - - -def test_per_document_cap_marks_the_rest_explicitly(fake_storage, monkeypatch): - """上限生效要在数据里看得见，否则和"视觉模型什么都没返回"分不开。""" - for i in range(4): - fake_storage.objects[f"k{i}"] = b"x" * 100 - _install_bridge(monkeypatch, []) - monkeypatch.setattr(kb_assets, "_MAX_CAPTIONS_PER_DOC", 2) - rows = [_asset_row(asset_id=f"a{i}", storage_key=f"k{i}") for i in range(4)] - - kb_assets.generate_captions(_FakeSession(), rows) - - assert [r.caption_status for r in rows[2:]] == ["skipped", "skipped"] - - -def test_index_text_prefers_caption_then_caption_text(): - row = _asset_row(caption="柱状图，Q4 最高，同比增长 32%") - text = kb_assets.asset_index_text(row, title="2026 年报") - assert text.splitlines() == ["柱状图，Q4 最高，同比增长 32%", "图 1 季度营收趋势", "2026 年报"] - - -# ── 检索行 ─────────────────────────────────────────────────────────────────── - - -def test_asset_rows_point_at_their_parent_chunk(monkeypatch): - """图片行必须挂在父块上：这样去重、回 PG 取父块正文、重排全都不用改。""" - written: list[dict] = [] - monkeypatch.setattr(kb_assets, "asset_index_text", lambda a, title="": "柱状图描述") - import core.kb.kb_vector as kb_vector - - monkeypatch.setattr(kb_vector, "embed_batch", lambda texts: [[0.1] * 8 for _ in texts]) - monkeypatch.setattr(kb_vector, "upsert_rows", lambda rows: written.extend(rows)) - - count = kb_assets.index_assets( - [_asset_row()], user_id="u1", kb_id="kb1", document_id="d1", title="年报" - ) - - assert count == 1 - row = written[0] - assert row["row_type"] == "image" - assert row["parent_chunk_id"] == "p1" - assert row["chunk_id"] != "p1" # 主键不能和分块行撞 - - -def test_orphan_asset_falls_back_to_itself(monkeypatch): - written: list[dict] = [] - import core.kb.kb_vector as kb_vector - - monkeypatch.setattr(kb_vector, "embed_batch", lambda texts: [[0.1] * 8 for _ in texts]) - monkeypatch.setattr(kb_vector, "upsert_rows", lambda rows: written.extend(rows)) - - kb_assets.index_assets( - [_asset_row(chunk_id=None)], user_id="u1", kb_id="kb1", document_id="d1", title="年报" - ) - - # 没有归属分块时指向自己，检索侧取不到父块正文会退回本行 content（描述本身） - assert written[0]["parent_chunk_id"] == written[0]["chunk_id"] - - -def test_multimodal_toggle_precedence(monkeypatch): - monkeypatch.delenv("KB_MULTIMODAL_INDEXING", raising=False) - assert kb_assets.multimodal_indexing_enabled(None) is True - assert kb_assets.multimodal_indexing_enabled({"multimodal_indexing": False}) is False - monkeypatch.setenv("KB_MULTIMODAL_INDEXING", "false") - assert kb_assets.multimodal_indexing_enabled(None) is False - # 单库显式打开要能盖过全局关闭 - assert kb_assets.multimodal_indexing_enabled({"multimodal_indexing": True}) is True - - -# ── 异步接缝：把协程交回宿主循环 ────────────────────────────────────────────── - - -def _stub_bridge(monkeypatch, seen: dict): - """替换 core.vision.get_vision_bridge，记录实际调用参数。""" - import sys - import types - - async def _describe_many(images, *, focus=None): - seen["describe_many"] = {"n": len(images), "focus": focus} - return [f"ok-{i}" for i in range(len(images))] - - async def _describe(data, mime, *, focus=None, use_cache=True): - seen.setdefault("describe", []).append({"use_cache": use_cache, "focus": focus}) - return f"ok-{mime}" - - bridge = SimpleNamespace(describe_many=_describe_many, describe=_describe) - module = types.ModuleType("core.vision") - module.get_vision_bridge = lambda: bridge - module.is_available = lambda: True - monkeypatch.setitem(sys.modules, "core.vision", module) - - -def test_describe_batch_runs_on_the_host_loop_when_there_is_one(monkeypatch): - """索引跑在 anyio 工作线程里时，协程必须交回宿主循环执行——只有走宿主循环， - 视觉桥的证据缓存与并发信号量才真的生效（也才不会各起各的循环）。""" - import anyio - import anyio.to_thread - - seen: dict = {} - _stub_bridge(monkeypatch, seen) - - async def _main(): - return await anyio.to_thread.run_sync( - lambda: kb_assets._describe_batch([(b"a", "image/png"), (b"b", "image/jpeg")]) - ) - - results = anyio.run(_main) - - assert results == ["ok-0", "ok-1"] - assert seen["describe_many"]["n"] == 2 - assert "文档中的插图" in seen["describe_many"]["focus"] - assert "describe" not in seen - - -def test_describe_batch_without_host_loop_disables_the_evidence_cache(monkeypatch): - """没有宿主循环时才起临时循环，而且必须关掉缓存：Redis 连接池是模块级全局、 - 绑定首次使用它的循环，让一个转瞬即逝的临时循环抢先创建它，会把整个进程的 Redis - 绑死在已销毁的循环上——比「这次没吃到缓存」严重得多。""" - seen: dict = {} - _stub_bridge(monkeypatch, seen) - - results = kb_assets._describe_batch([(b"a", "image/png")]) - - assert results == ["ok-image/png"] - assert seen["describe"][0]["use_cache"] is False - assert "describe_many" not in seen - - -def test_explicit_loop_wins_over_anyio(monkeypatch): - """索引现在跑在常驻 worker **自己的**线程池里（不是 anyio 的），线程里既没有运行中的 - 循环、也没有回宿主循环的通路——所以 worker 会把调度它的循环显式传下来，必须优先用它。 - 退化成临时循环的话，证据缓存就白配了。""" - import asyncio - import threading - - seen: dict = {} - _stub_bridge(monkeypatch, seen) - - loop = asyncio.new_event_loop() - thread = threading.Thread(target=loop.run_forever, daemon=True) - thread.start() - try: - # 在一个普通线程里调用（模拟 worker 的线程池），只有显式 loop 能救它 - box: dict = {} - worker = threading.Thread( - target=lambda: box.update( - r=kb_assets._describe_batch([(b"a", "image/png")], loop) - ) - ) - worker.start() - worker.join(timeout=10) - finally: - loop.call_soon_threadsafe(loop.stop) - thread.join(timeout=5) - loop.close() - - assert box["r"] == ["ok-0"] - assert seen["describe_many"]["n"] == 1 - assert "describe" not in seen # 没退化到不带缓存的临时循环 - - -# ── 检索结果取正文：纯图片父块 ──────────────────────────────────────────────── - - -def test_media_only_parent_falls_back_to_the_asset_row_content(): - """单独上传的图片，其父块正文就是一条 markdown 图片链接。父块无条件覆盖的话， - 调用方拿到的只是个 URL，而命中的图片行里恰好装着描述与转写。""" - from extensions.mcp_servers.retrieve_dataset_content_mcp.local_impl import _pick_content - - link = "![](/api/v1/catalog/kb/assets/abc)" - rich = "柱状图：Q4 达到峰值 42\nQuarterly Revenue 2026" - - assert _pick_content(link, rich) == rich - assert _pick_content(f" {link} \n", rich) == rich - # 正常文档：父块里除了图还有正文，必须原样保留父块 - normal = f"第四条 专项资金用于……\n{link}\n见上图。" - assert _pick_content(normal, rich) == normal - # 两边都空时不要凭空造内容 - assert _pick_content("", "") == "" - assert _pick_content(link, "") == link diff --git a/src/backend/tests/services/test_kb_wiki_end_to_end.py b/src/backend/tests/services/test_kb_wiki_end_to_end.py deleted file mode 100644 index 34c8d691..00000000 --- a/src/backend/tests/services/test_kb_wiki_end_to_end.py +++ /dev/null @@ -1,663 +0,0 @@ -"""Wiki 管线的端到端测试：喂真文档、桩掉模型、检查落库结果与读取面。 - -模型调用全部被桩掉——这里验的不是模型写得好不好，而是**管线接线对不对**： -候选是否落成页、引文是否变成可回溯的血缘、目录是否建起来、读取面返回的结构 -是否与前端/工具期望的一致。这类接线错误在真实环境里要跑几十分钟才暴露一次。 -""" - -from __future__ import annotations - -import json - -import pytest -from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker - -from core.db.engine import Base -from core.db.models import KBChunk, KBDocument, KBSpace, KBWikiPage, UserShadow -from core.kb.wiki import local_provider, pipeline -from core.kb.wiki.config import WikiConfig -from core.kb.wiki.finalize import INDEX_SLUG, finalize_kb_wiki -from core.kb.wiki.llm import LLMBudget - -KB_ID = "kb_wikitest" -USER_ID = "u_wikitest" -DOC_ID = "doc_alpha" - - -@pytest.fixture -def wiki_db(tmp_path): - """独立的 sqlite 库；local_provider 走自己的 SessionLocal，必须一并指向它。""" - engine = create_engine(f"sqlite:///{tmp_path}/wiki.db") - Base.metadata.create_all(engine) - Session = sessionmaker(bind=engine) - - session = Session() - session.add(UserShadow(user_id=USER_ID, username="wiki-tester")) - session.add( - KBSpace( - kb_id=KB_ID, - user_id=USER_ID, - name="测试知识库", - visibility="private", - extra_data={"index_modes": ["rag", "wiki"]}, - ) - ) - session.add( - KBDocument( - document_id=DOC_ID, - kb_id=KB_ID, - title="示例科技年度报告", - filename="report.md", - size_bytes=100, - mime_type="text/markdown", - storage_key="k", - indexing_status="completed", - ) - ) - for index, text in enumerate( - [ - "示例科技有限公司成立于 2020 年，主营企业级检索产品。", - "检索增强生成（RAG）先召回文档再交给大模型生成答案。", - "示例科技在 2024 年发布了基于 RAG 的问答平台。", - ] - ): - session.add( - KBChunk( - chunk_id=f"chunk-{index}", - kb_id=KB_ID, - document_id=DOC_ID, - chunk_index=index, - content=text, - ) - ) - session.commit() - yield session, Session - session.close() - - -@pytest.fixture -def stub_llm(monkeypatch): - """按提示词特征分派桩响应，模拟各阶段的模型输出。""" - calls: list[str] = [] - - def _fake_call(*, system, user, budget=None, purpose="wiki", **kwargs): - calls.append(purpose) - if budget is not None: - budget.check() - budget.record(len(user), 100) - - if purpose == "候选抽取": - return json.dumps( - { - "entities": [ - { - "name": "示例科技", - "slug": "entity/acme", - "aliases": ["示例科技有限公司"], - "description": "一家做企业级检索产品的公司。", - "details": "成立于 2020 年。", - } - ], - "concepts": [ - { - "name": "检索增强生成", - "slug": "concept/rag", - "aliases": ["RAG"], - "description": "先检索再生成的技术路线。", - "details": "召回文档后交给大模型。", - } - ], - } - ) - if purpose == "引文标注": - return json.dumps( - { - "citations": { - "entity/acme": ["c001", "c003"], - "concept/rag": ["c002"], - }, - "new_slugs": [], - } - ) - if purpose == "去重判定": - return json.dumps({"merges": {}}) - if purpose == "目录规划": - return json.dumps( - { - "assignments": [ - {"slug": "entity/acme", "path": ["组织"]}, - {"slug": "concept/rag", "path": ["技术", "检索"]}, - ] - } - ) - if purpose == "写页": - return "SUMMARY: 该条目的一句话摘要。\n# 条目\n\n正文内容，提到 检索增强生成。" - if purpose == "文档摘要": - return "SUMMARY: 这篇文档讲示例科技。\n# 示例科技年度报告\n\n概要正文。" - if purpose == "索引导语": - return "# 测试知识库\n\n本库覆盖企业与检索技术。" - return "{}" - - monkeypatch.setattr("core.kb.wiki.extract.call_llm_json", _wrap_json(_fake_call)) - monkeypatch.setattr("core.kb.wiki.cite.call_llm_json", _wrap_json(_fake_call)) - monkeypatch.setattr("core.kb.wiki.dedup.call_llm_json", _wrap_json(_fake_call)) - monkeypatch.setattr("core.kb.wiki.taxonomy.call_llm_json", _wrap_json(_fake_call)) - monkeypatch.setattr("core.kb.wiki.reduce.call_llm", _fake_call) - monkeypatch.setattr("core.kb.wiki.finalize.call_llm", _fake_call) - return calls - - -def _wrap_json(fake_call): - from core.kb.wiki.llm import parse_json_object - - def _inner(*, system, user, budget=None, purpose="wiki", **kwargs): - return parse_json_object( - fake_call(system=system, user=user, budget=budget, purpose=purpose) - ) - - return _inner - - -def _point_session_local(monkeypatch, Session): - """local_provider / store 用模块级 SessionLocal，测试里要一并指过去。""" - monkeypatch.setattr("core.kb.wiki.local_provider.SessionLocal", Session) - - -def test_ingest_creates_pages_with_lineage(wiki_db, stub_llm, monkeypatch): - session, Session = wiki_db - config = WikiConfig(map_parallel=1, reduce_parallel=1, citation_chunk_batch=8) - - result = pipeline.run_ingest( - session, KB_ID, [DOC_ID], config=config, budget=LLMBudget(max_calls=100) - ) - - assert result.documents == 1 - assert result.pages_written == 2 - assert result.summaries_written == 1 - - pages = {p.slug: p for p in session.query(KBWikiPage).all()} - assert set(pages) == {"entity/acme", "concept/rag", f"summary/{DOC_ID}"} - - acme = pages["entity/acme"] - # 血缘：引文标注给的短 ID 必须被映射回真实 chunk_id - assert acme.chunk_refs == ["chunk-0", "chunk-2"] - assert acme.source_refs == [DOC_ID] - assert acme.title == "示例科技" - assert "示例科技有限公司" in (acme.aliases or []) - # 目录规划落成真实目录 - assert acme.category_path == ["组织"] - assert acme.wiki_path == "组织" - assert acme.folder_id - - rag = pages["concept/rag"] - assert rag.chunk_refs == ["chunk-1"] - assert rag.category_path == ["技术", "检索"] - - -def test_ingest_links_pages_to_each_other(wiki_db, stub_llm): - """写页阶段漏链时，自动织链要把正文里的条目名补成 Wiki 链接。""" - session, _ = wiki_db - pipeline.run_ingest( - session, - KB_ID, - [DOC_ID], - config=WikiConfig(map_parallel=1, reduce_parallel=1), - budget=LLMBudget(max_calls=100), - ) - acme = session.query(KBWikiPage).filter_by(slug="entity/acme").one() - assert "[[concept/rag|检索增强生成]]" in acme.content - assert "concept/rag" in (acme.out_links or []) - - -def test_finalize_builds_index_and_backlinks(wiki_db, stub_llm): - session, _ = wiki_db - pipeline.run_ingest( - session, - KB_ID, - [DOC_ID], - config=WikiConfig(map_parallel=1, reduce_parallel=1), - budget=LLMBudget(max_calls=100), - ) - stats = finalize_kb_wiki(session, KB_ID, config=WikiConfig(), budget=LLMBudget(max_calls=10)) - session.commit() - - assert stats["pages"] == 3 - index = session.query(KBWikiPage).filter_by(slug=INDEX_SLUG).one() - assert "实体" in index.content and "概念" in index.content - - # 反向链接由全库 out_links 反推，而不是增量维护 - rag = session.query(KBWikiPage).filter_by(slug="concept/rag").one() - assert "entity/acme" in (rag.in_links or []) - - -def test_finalize_strips_dead_links(wiki_db, stub_llm): - """指向已不存在页面的链接必须降级成纯文本，否则 Wiki 里会积累死链。""" - session, _ = wiki_db - pipeline.run_ingest( - session, - KB_ID, - [DOC_ID], - config=WikiConfig(map_parallel=1, reduce_parallel=1), - budget=LLMBudget(max_calls=100), - ) - acme = session.query(KBWikiPage).filter_by(slug="entity/acme").one() - acme.content = acme.content + "\n参见 [[entity/ghost|幽灵页]]。" - acme.out_links = [*(acme.out_links or []), "entity/ghost"] - session.commit() - - stats = finalize_kb_wiki(session, KB_ID, config=WikiConfig(), budget=LLMBudget(max_calls=10)) - session.commit() - - assert stats["dead_links_removed"] >= 1 - refreshed = session.query(KBWikiPage).filter_by(slug="entity/acme").one() - assert "[[entity/ghost" not in refreshed.content - assert "幽灵页" in refreshed.content - assert "entity/ghost" not in (refreshed.out_links or []) - - -def test_retract_removes_document_lineage(wiki_db, stub_llm): - """文档删除后，只由它供源的页面要归档，悬空的分块引用要摘掉。""" - session, _ = wiki_db - pipeline.run_ingest( - session, - KB_ID, - [DOC_ID], - config=WikiConfig(map_parallel=1, reduce_parallel=1), - budget=LLMBudget(max_calls=100), - ) - session.query(KBChunk).filter_by(document_id=DOC_ID).delete() - session.commit() - - stats = pipeline.run_retract(session, KB_ID, [DOC_ID]) - - assert stats["archived"] == 3, "三个页面都只有这一个来源，应全部归档" - live = session.query(KBWikiPage).filter(KBWikiPage.deleted_at.is_(None)).all() - assert live == [] - - -def test_read_surface_matches_expected_shape(wiki_db, stub_llm, monkeypatch): - """读取面的字段是路由、MCP 工具与前端组件的共同契约，形状不能漂。""" - session, Session = wiki_db - pipeline.run_ingest( - session, - KB_ID, - [DOC_ID], - config=WikiConfig(map_parallel=1, reduce_parallel=1), - budget=LLMBudget(max_calls=100), - ) - finalize_kb_wiki(session, KB_ID, config=WikiConfig(), budget=LLMBudget(max_calls=10)) - session.commit() - _point_session_local(monkeypatch, Session) - - stats = local_provider.wiki_stats(KB_ID) - assert stats["total_pages"] == 4 # 2 条目 + 1 摘要 + 1 索引 - assert stats["pages_by_type"]["entity"] == 1 - - page = local_provider.wiki_read_page(KB_ID, "entity/acme") - for key in ( - "slug", - "title", - "page_type", - "type_label", - "summary", - "aliases", - "category_path", - "wiki_path", - "out_links", - "in_links", - "source_refs", - "chunk_refs", - "content", - "related_pages", - ): - assert key in page, f"读取面缺少字段 {key}" - assert page["type_label"] == "实体" - # 相关页要带上可读标题，否则模型和用户看到的都是一串拼音 - assert page["related_pages"][0]["title"] == "检索增强生成" - - -def test_source_chunks_resolve_by_id(wiki_db, stub_llm, monkeypatch): - """顺血缘取原文是按 ID 直取，且要保持页面记录的引用顺序。""" - session, Session = wiki_db - pipeline.run_ingest( - session, - KB_ID, - [DOC_ID], - config=WikiConfig(map_parallel=1, reduce_parallel=1), - budget=LLMBudget(max_calls=100), - ) - session.commit() - _point_session_local(monkeypatch, Session) - - result = local_provider.wiki_source_chunks(KB_ID, "entity/acme", max_chunks=6) - ids = [c["chunk_id"] for c in result["chunks"]] - assert ids == ["chunk-0", "chunk-2"] - assert result["chunks"][0]["document_title"] == "示例科技年度报告" - assert "示例科技有限公司成立于" in result["chunks"][0]["content"] - - -def test_source_chunks_fall_back_when_refs_are_stale(wiki_db, stub_llm, monkeypatch): - """页面被改写或文档重新索引后 chunk_refs 会对不上，此时退回该文档前几块。""" - session, Session = wiki_db - pipeline.run_ingest( - session, - KB_ID, - [DOC_ID], - config=WikiConfig(map_parallel=1, reduce_parallel=1), - budget=LLMBudget(max_calls=100), - ) - page = session.query(KBWikiPage).filter_by(slug="entity/acme").one() - page.chunk_refs = ["chunk-does-not-exist"] - session.commit() - _point_session_local(monkeypatch, Session) - - result = local_provider.wiki_source_chunks(KB_ID, "entity/acme", max_chunks=2) - assert len(result["chunks"]) == 2, "对不上时不该空手而归" - - -def test_search_ranks_title_hits_above_body(wiki_db, stub_llm, monkeypatch): - session, Session = wiki_db - pipeline.run_ingest( - session, - KB_ID, - [DOC_ID], - config=WikiConfig(map_parallel=1, reduce_parallel=1), - budget=LLMBudget(max_calls=100), - ) - session.commit() - _point_session_local(monkeypatch, Session) - - result = local_provider.wiki_search(KB_ID, "检索增强生成", limit=5) - assert result["pages"], "应命中概念页" - assert result["pages"][0]["slug"] == "concept/rag" - - -def test_search_supports_alternation(wiki_db, stub_llm, monkeypatch): - """口语说法常匹配不上书面术语，一次给多个说法要能命中。""" - session, Session = wiki_db - pipeline.run_ingest( - session, - KB_ID, - [DOC_ID], - config=WikiConfig(map_parallel=1, reduce_parallel=1), - budget=LLMBudget(max_calls=100), - ) - session.commit() - _point_session_local(monkeypatch, Session) - - slugs = {p["slug"] for p in local_provider.wiki_search(KB_ID, "示例科技|检索增强生成")["pages"]} - assert {"entity/acme", "concept/rag"} <= slugs - - -def test_folders_report_recursive_page_counts(wiki_db, stub_llm, monkeypatch): - session, Session = wiki_db - pipeline.run_ingest( - session, - KB_ID, - [DOC_ID], - config=WikiConfig(map_parallel=1, reduce_parallel=1), - budget=LLMBudget(max_calls=100), - ) - session.commit() - _point_session_local(monkeypatch, Session) - - roots = local_provider.wiki_folders(KB_ID) - by_name = {f["name"]: f for f in roots["folders"]} - assert {"组织", "技术"} <= set(by_name) - # 「技术」下的页面挂在子目录「检索」里，递归计数必须能看到它 - assert by_name["技术"]["page_count"] == 1 - assert by_name["技术"]["has_children"] is True - assert by_name["组织"]["has_children"] is False - - -def test_graph_ego_mode_centres_on_node(wiki_db, stub_llm, monkeypatch): - session, Session = wiki_db - pipeline.run_ingest( - session, - KB_ID, - [DOC_ID], - config=WikiConfig(map_parallel=1, reduce_parallel=1), - budget=LLMBudget(max_calls=100), - ) - finalize_kb_wiki(session, KB_ID, config=WikiConfig(), budget=LLMBudget(max_calls=10)) - session.commit() - _point_session_local(monkeypatch, Session) - - graph = local_provider.wiki_graph(KB_ID, mode="ego", center="entity/acme", depth=1) - slugs = {n["slug"] for n in graph["nodes"]} - assert "entity/acme" in slugs and "concept/rag" in slugs - assert graph["edges"], "两页之间有链接，应产出边" - - -def test_graph_ego_on_missing_node_is_empty(wiki_db, stub_llm, monkeypatch): - session, Session = wiki_db - _point_session_local(monkeypatch, Session) - graph = local_provider.wiki_graph(KB_ID, mode="ego", center="entity/nope") - assert graph["nodes"] == [] and graph["edges"] == [] - - -def test_ingest_is_idempotent_on_rerun(wiki_db, stub_llm): - """同一批文档重跑不该产生重复页——slug 复用规则要真的生效。""" - session, _ = wiki_db - config = WikiConfig(map_parallel=1, reduce_parallel=1) - pipeline.run_ingest(session, KB_ID, [DOC_ID], config=config, budget=LLMBudget(max_calls=100)) - first = session.query(KBWikiPage).count() - - pipeline.run_ingest(session, KB_ID, [DOC_ID], config=config, budget=LLMBudget(max_calls=100)) - assert session.query(KBWikiPage).count() == first - - acme = session.query(KBWikiPage).filter_by(slug="entity/acme").one() - assert acme.version >= 2, "重跑应写出新版本而不是新页面" - - -def test_concurrent_stages_never_touch_the_session(wiki_db, stub_llm): - """并发段绝不能碰 session。 - - SQLAlchemy 的 Session 不是线程安全的：从线程池里查库会炸成 - "concurrent operations are not permitted"，而且只在并发度 > 1 时才复现—— - 单线程跑一万遍都发现不了。这里用一个会记录调用线程的代理把它钉死。 - """ - import threading - - session, _ = wiki_db - main_thread = threading.get_ident() - offenders: list[str] = [] - - real_query = session.query - - def _tracking_query(*args, **kwargs): - if threading.get_ident() != main_thread: - offenders.append(threading.current_thread().name) - return real_query(*args, **kwargs) - - session.query = _tracking_query # type: ignore[method-assign] - try: - pipeline.run_ingest( - session, - KB_ID, - [DOC_ID], - config=WikiConfig(map_parallel=4, reduce_parallel=4), - budget=LLMBudget(max_calls=100), - ) - finally: - session.query = real_query # type: ignore[method-assign] - - assert offenders == [], f"这些线程在并发段里查了库：{offenders}" - - -def test_quota_stops_generation_without_failing(wiki_db, stub_llm): - """超额是设计好的刹车：已生成的部分保留，作业不算失败。""" - session, _ = wiki_db - result = pipeline.run_ingest( - session, - KB_ID, - [DOC_ID], - config=WikiConfig(map_parallel=1, reduce_parallel=1), - budget=LLMBudget(max_calls=1), - ) - assert result.quota_exceeded is True - - -def test_custom_instructions_reach_every_stage(wiki_db, monkeypatch): - """用户填的要求必须真的送达每个阶段的模型调用。 - - 配置字段加了却没接到提示词上，是这类功能最典型的"看起来做了"—— - 界面能填、能存库、生成结果毫无变化。这里逐阶段核实。 - """ - session, _ = wiki_db - seen: dict[str, str] = {} - - def _capture(*, system, user, budget=None, purpose="wiki", **kwargs): - seen[purpose] = f"{system or ''}\n{user}" - if budget is not None: - budget.record(len(user), 10) - if purpose == "候选抽取": - return json.dumps( - { - "entities": [ - {"name": "示例科技", "slug": "entity/acme", "aliases": [], - "description": "d", "details": "x"} - ], - "concepts": [], - } - ) - if purpose == "引文标注": - return json.dumps({"citations": {"entity/acme": ["c001"]}, "new_slugs": []}) - if purpose in ("去重判定",): - return json.dumps({"merges": {}}) - if purpose == "目录规划": - return json.dumps({"assignments": []}) - if purpose in ("写页", "文档摘要"): - return "SUMMARY: 摘要\n# 标题\n正文" - return "{}" - - for module in ("extract", "cite", "dedup", "taxonomy"): - monkeypatch.setattr(f"core.kb.wiki.{module}.call_llm_json", _wrap_json(_capture)) - monkeypatch.setattr("core.kb.wiki.reduce.call_llm", _capture) - - pipeline.run_ingest( - session, - KB_ID, - [DOC_ID], - config=WikiConfig( - map_parallel=1, - reduce_parallel=1, - extraction_instructions="重点关注补助金额与申报条件", - content_instructions="每页开头先给一句话结论", - ), - budget=LLMBudget(max_calls=100), - ) - - # 抽取侧两个阶段拿到抽取要求 - for stage in ("候选抽取", "引文标注"): - assert "重点关注补助金额与申报条件" in seen[stage], f"{stage} 未收到抽取要求" - assert "" in seen[stage] - - # 撰写侧两个阶段拿到撰写要求 - for stage in ("写页", "文档摘要"): - assert "每页开头先给一句话结论" in seen[stage], f"{stage} 未收到撰写要求" - assert "" in seen[stage] - - # 两侧不串味：抽取要求不该出现在写页提示词里 - assert "重点关注补助金额与申报条件" not in seen["写页"] - - -def test_worker_batches_sibling_ingest_jobs(wiki_db): - """同库的多个待跑摄入作业要被合并成一批。 - - 上传是一篇文档一个作业（入队点在索引完成回调里，天然单篇）。若处理时也一篇 - 一批，单 worker 串行跑 39 篇要数小时，而且 map_parallel 根本没有多篇文档可并行。 - """ - from core.kb.wiki import jobs as job_queue - - session, _ = wiki_db - first = job_queue.enqueue_ingest(session, KB_ID, ["doc-a"]) - job_queue.enqueue_ingest(session, KB_ID, ["doc-b"]) - job_queue.enqueue_ingest(session, KB_ID, ["doc-c"]) - - claimed = job_queue.claim_next_job(session, "worker-1") - assert claimed is not None and claimed.job_id == first.job_id - - siblings = job_queue.claim_sibling_ingest_jobs(session, "worker-1", KB_ID, limit=4) - assert [s.payload["document_ids"] for s in siblings] == [["doc-b"], ["doc-c"]] - - # 合批后全部一起结单，不留孤儿 - job_queue.finish_jobs(session, [claimed.job_id, *(s.job_id for s in siblings)]) - assert job_queue.kb_job_summary(session, KB_ID)["generating"] is False - - -def test_batching_never_crosses_knowledge_bases(wiki_db): - """跨库合批会让一个大库把其他库饿死。""" - from core.db.models import KBSpace - from core.kb.wiki import jobs as job_queue - - session, _ = wiki_db - session.add( - KBSpace(kb_id="kb_other", user_id=USER_ID, name="别的库", visibility="private", - extra_data={"index_modes": ["wiki"]}) - ) - session.commit() - job_queue.enqueue_ingest(session, KB_ID, ["mine"]) - job_queue.enqueue_ingest(session, "kb_other", ["theirs"]) - - siblings = job_queue.claim_sibling_ingest_jobs(session, "worker-1", KB_ID, limit=10) - assert all(s.kb_id == KB_ID for s in siblings) - - -def test_one_knowledge_base_runs_one_batch_at_a_time(wiki_db): - """同一个库同时只允许一批在跑。 - - 同库两批并发会丢更新：写页阶段先把既有页读成内存快照，再逐条基于快照重写、 - 最后整体 upsert，而不同文档抽出同名 slug 是常态——后写的会覆盖先写的。 - 生产上两个 uvicorn worker 各跑一个 wiki worker 同啃一个库，内存也随之翻倍。 - """ - from core.kb.wiki import jobs as job_queue - - session, _ = wiki_db - job_queue.enqueue_ingest(session, KB_ID, ["doc-a"]) - job_queue.enqueue_ingest(session, KB_ID, ["doc-b"]) - - assert job_queue.claim_next_job(session, "worker-1") is not None - assert job_queue.claim_next_job(session, "worker-2") is None - - -def test_a_busy_knowledge_base_does_not_block_other_ones(wiki_db): - """同库串行不能退化成全局串行，否则一个大库会把其他库全堵死。""" - from core.kb.wiki import jobs as job_queue - - session, _ = wiki_db - session.add( - KBSpace( - kb_id="kb_other", - user_id=USER_ID, - name="别的库", - visibility="private", - extra_data={"index_modes": ["wiki"]}, - ) - ) - session.commit() - job_queue.enqueue_ingest(session, KB_ID, ["mine"]) - job_queue.enqueue_ingest(session, "kb_other", ["theirs"]) - - first = job_queue.claim_next_job(session, "worker-1") - second = job_queue.claim_next_job(session, "worker-2") - assert first is not None and second is not None - assert {first.kb_id, second.kb_id} == {KB_ID, "kb_other"} - - -def test_a_dead_workers_knowledge_base_is_taken_over(wiki_db): - """持有者死了（心跳停摆）之后这个库必须能被接管，不能因为互斥而永远卡住。""" - from core.kb.wiki import jobs as job_queue - from core.kb.wiki.jobs import STALE_AFTER, _now - - session, _ = wiki_db - job_queue.enqueue_ingest(session, KB_ID, ["doc-a"]) - claimed = job_queue.claim_next_job(session, "worker-1") - assert claimed is not None - - claimed.heartbeat_at = _now() - STALE_AFTER * 2 - session.commit() - - retaken = job_queue.claim_next_job(session, "worker-2") - assert retaken is not None and retaken.job_id == claimed.job_id diff --git a/src/backend/tests/services/test_memory_settings_service.py b/src/backend/tests/services/test_memory_settings_service.py index c444c922..13243c32 100644 --- a/src/backend/tests/services/test_memory_settings_service.py +++ b/src/backend/tests/services/test_memory_settings_service.py @@ -1,11 +1,11 @@ """Tests for memory availability and default-on user settings.""" import pytest -from core.db import model_repository +from core.models import repository as model_repository from core.db.models import UserShadow from core.infra.exceptions import BadRequestError -from core.services.memory_settings_service import MemorySettingsService -from core.services.user_service import UserService +from core.memory.settings import MemorySettingsService +from core.auth.users import UserService def _create_user(db_session, *, extra_data=None) -> UserShadow: @@ -38,7 +38,7 @@ def test_memory_defaults_off_and_enable_is_rejected_without_embedding( ): _create_user(db_session) monkeypatch.setattr( - "core.services.memory_settings_service._memory_runtime_available", + 'core.memory.settings._memory_runtime_available', lambda: True, ) @@ -57,7 +57,7 @@ def test_memory_defaults_on_when_runtime_and_embedding_are_available( _create_user(db_session) _assign_embedding(db_session) monkeypatch.setattr( - "core.services.memory_settings_service._memory_runtime_available", + 'core.memory.settings._memory_runtime_available', lambda: True, ) @@ -77,7 +77,7 @@ def test_explicit_memory_off_is_preserved(db_session, monkeypatch): ) _assign_embedding(db_session) monkeypatch.setattr( - "core.services.memory_settings_service._memory_runtime_available", + 'core.memory.settings._memory_runtime_available', lambda: True, ) @@ -91,7 +91,7 @@ def test_inactive_embedding_provider_does_not_unlock_memory(db_session, monkeypa _create_user(db_session) _assign_embedding(db_session, active=False) monkeypatch.setattr( - "core.services.memory_settings_service._memory_runtime_available", + 'core.memory.settings._memory_runtime_available', lambda: True, ) @@ -108,7 +108,7 @@ def test_missing_memory_runtime_rejects_enable(db_session, monkeypatch): _create_user(db_session) _assign_embedding(db_session) monkeypatch.setattr( - "core.services.memory_settings_service._memory_runtime_available", + 'core.memory.settings._memory_runtime_available', lambda: False, ) diff --git a/src/backend/tests/services/test_model_config_consistency.py b/src/backend/tests/services/test_model_config_consistency.py index c9a57aac..2f143c5c 100644 --- a/src/backend/tests/services/test_model_config_consistency.py +++ b/src/backend/tests/services/test_model_config_consistency.py @@ -2,9 +2,9 @@ from sqlalchemy.orm import sessionmaker -from core.db.model_repository import assign_role, create_provider, unassign_role, update_provider -from core.services import model_config -from core.services.model_config import ModelConfigService +from core.models.repository import assign_role, create_provider, unassign_role, update_provider +from core.models import config as model_config +from core.models.config import ModelConfigService def test_independent_workers_observe_committed_role_and_provider_changes(db_session, monkeypatch): @@ -59,7 +59,7 @@ def test_separate_process_observes_commit_without_restart(tmp_path): assign_role(db, "subagent", provider.provider_id) script = """ import sys -from core.services.model_config import ModelConfigService +from core.models.config import ModelConfigService service = ModelConfigService() print(service.resolve('subagent').model_name, flush=True) sys.stdin.readline() diff --git a/src/backend/tests/services/test_model_config_export.py b/src/backend/tests/services/test_model_config_export.py index b7de9128..04ed5a93 100644 --- a/src/backend/tests/services/test_model_config_export.py +++ b/src/backend/tests/services/test_model_config_export.py @@ -7,13 +7,7 @@ from sqlalchemy.orm import sessionmaker from core.db.engine import Base -from core.db.model_repository import ( - assign_role, - create_provider, - export_all, - get_provider, - import_all, -) +from core.models.repository import assign_role, create_provider, export_all, get_provider, import_all @pytest.fixture diff --git a/src/backend/tests/services/test_plan_execution_snapshot.py b/src/backend/tests/services/test_plan_execution_snapshot.py index e6be92cb..1a7c571e 100644 --- a/src/backend/tests/services/test_plan_execution_snapshot.py +++ b/src/backend/tests/services/test_plan_execution_snapshot.py @@ -7,7 +7,7 @@ from types import SimpleNamespace -from core.services.plan_service import PlanService +from core.chat.plans import PlanService def _plan(status: str): diff --git a/src/backend/tests/services/test_project_context_no_inventory.py b/src/backend/tests/services/test_project_context_no_inventory.py index 98cc2404..f460cfe7 100644 --- a/src/backend/tests/services/test_project_context_no_inventory.py +++ b/src/backend/tests/services/test_project_context_no_inventory.py @@ -4,8 +4,8 @@ from pathlib import Path import pytest -from core.services.project_scope import build_project_ctx -from core.services.project_service import ProjectService +from core.projects.edition import build_project_ctx +from core.projects.edition import ProjectService from tests.services.project_instructions_support import env, local, update @@ -26,14 +26,14 @@ def forbidden_listing(*args, **kwargs): calls.append("directory enumeration") raise AssertionError("Sending a project message must not enumerate files") - monkeypatch.setattr("core.services.project_file_service.ProjectFileService.list_files", forbidden_listing) + monkeypatch.setattr('core.projects.files.ProjectFileService.list_files', forbidden_listing) monkeypatch.setattr("core.sandbox.workspace.mounts.list_local_files", forbidden_listing) monkeypatch.setattr("core.sandbox.workspace.mounts.ensure_local_project_link", lambda *args: None) build = build_project_ctx if edition == "ce": root = Path(__file__).resolve().parents[4] spec = importlib.util.spec_from_file_location( - "ce_project_scope_no_inventory", root / "ce/overlay/src/backend/core/services/project_scope.py" + "ce_project_scope_no_inventory", root / "src/backend/core/projects/personal_scope.py" ) module = importlib.util.module_from_spec(spec) monkeypatch.setitem(sys.modules, spec.name, module) diff --git a/src/backend/tests/services/test_project_environment.py b/src/backend/tests/services/test_project_environment.py index 668967bb..508ec857 100644 --- a/src/backend/tests/services/test_project_environment.py +++ b/src/backend/tests/services/test_project_environment.py @@ -1,18 +1,19 @@ +from core.capabilities.skills import loader_factory as loader_factory_module import pytest @pytest.fixture(autouse=True) def inventory_database(db_session, monkeypatch): - from core.agent_skills import loader + from core.capabilities.skills import loader from core.db import engine from sqlalchemy.orm import sessionmaker monkeypatch.setattr(engine, "SessionLocal", sessionmaker(bind=db_session.get_bind())) - monkeypatch.setattr(loader, "_global_loader", None) + monkeypatch.setattr(loader_factory_module, 'global_loader', None) from core.db.models import Project, UserShadow -from core.services.project_environment import ProjectEnvironmentService +from core.projects.environment.service import ProjectEnvironmentService def test_empty_custom_environment_is_distinct_from_all_and_persists(db_session): @@ -42,7 +43,7 @@ def test_empty_custom_environment_is_distinct_from_all_and_persists(db_session): def test_dependencies_are_expanded_without_granting_missing_resources(): - from core.project_environment.dependencies import expand_environment + from core.projects.environment.dependencies import expand_environment graph = { "agent_ids": {"analyst": {"name": "Analyst", "dependencies": {"plugin_ids": ["charts"]}}}, @@ -64,7 +65,7 @@ def test_dependencies_are_expanded_without_granting_missing_resources(): def test_binding_private_connector_cannot_grant_other_user_access(db_session): import pytest from core.db.models import AdminMcpServer - from core.services.project_environment_resources import environment_resources + from core.projects.environment_resources import environment_resources from fastapi import HTTPException db_session.add_all( @@ -93,7 +94,7 @@ def test_binding_private_connector_cannot_grant_other_user_access(db_session): def test_environment_is_saved_in_the_same_project_creation_transaction(db_session): - from core.services.project_service import ProjectService + from core.projects.edition import ProjectService db_session.add(UserShadow(user_id="creator", username="Creator")) db_session.commit() @@ -108,7 +109,7 @@ def test_environment_is_saved_in_the_same_project_creation_transaction(db_sessio def test_deep_dependency_cycles_resolve_without_recursion_or_duplicates(): - from core.project_environment.dependencies import expand_environment + from core.projects.environment.dependencies import expand_environment graph = {field: {} for field in ("agent_ids", "plugin_ids", "skill_ids", "mcp_ids")} for index in range(1200): @@ -123,7 +124,7 @@ def test_deep_dependency_cycles_resolve_without_recursion_or_duplicates(): def test_configuration_normalizes_identifiers_without_http_dependencies(): - from core.project_environment.config import ProjectEnvironment + from core.projects.environment.config import ProjectEnvironment from pydantic import ValidationError config = ProjectEnvironment(mode="custom", skill_ids=[" report ", "report"]) diff --git a/src/backend/tests/services/test_project_instructions.py b/src/backend/tests/services/test_project_instructions.py index 54473c30..75b8d6ce 100644 --- a/src/backend/tests/services/test_project_instructions.py +++ b/src/backend/tests/services/test_project_instructions.py @@ -9,11 +9,11 @@ from core.chat.schemas import ChatRequest from core.chat.context import generate_smart_title from core.db.models import Artifact, Project, UserFolder, UserShadow -from core.services.project_file_service import ProjectFileService -from core.services.project_init import resolve_project_init -from core.services.project_instructions import ProjectInstructionsService -from core.services.project_scope import build_project_ctx -from core.services.project_service import ProjectService +from core.projects.files import ProjectFileService +from core.chat.project_init import resolve_project_init +from core.projects.instructions import ProjectInstructionsService +from core.projects.edition import build_project_ctx +from core.projects.edition import ProjectService from fastapi import HTTPException from tests.services.project_instructions_support import env, local, update @@ -211,21 +211,20 @@ def test_init_requires_unadorned_project_chat(env, mode): @pytest.mark.asyncio async def test_registered_tools_save_real_cloud_content_with_revision_and_permissions(env): from agentscope.tool import Toolkit - from core.llm.tool_collector import ToolCollector + from core.llm.factory.tools.definitions import ToolAssembly from core.llm.tools.project_instructions_tool import register_project_instruction_tools db, _ = env p = ProjectService(db).create_personal("alice", "Tools") - collector = ToolCollector() + collector = ToolAssembly() register_project_instruction_tools(collector, project_id=p.project_id, user_id="alice") - schemas = await Toolkit(tools=collector.function_tools).get_tool_schemas() + schemas = await Toolkit(tools=list(collector)).get_tool_schemas() schema = next( s["function"] for s in schemas if s["function"]["name"] == "save_project_instructions" ) assert set(schema["parameters"]["required"]) == {"content", "expected_revision"} - # ToolCollector's documented get_tool exposes the registered function for direct execution. - read = collector.get_tool("read_project_instructions")._func - save = collector.get_tool("save_project_instructions")._func + read = next(t for t in collector if t.name == "read_project_instructions")._func + save = next(t for t in collector if t.name == "save_project_instructions")._func before = json.loads((await read()).content[0].text) result = json.loads( (await save("# Rules\nTest first.", before["instructions_revision"])).content[0].text @@ -243,7 +242,7 @@ async def test_registered_tools_save_real_cloud_content_with_revision_and_permis def test_saved_chat_binding_controls_initialization(env): - from core.services.chat_service import ChatService + from core.chat.service import ChatService db, _ = env svc = ProjectService(db) @@ -314,13 +313,13 @@ async def test_team_initialization_tool_and_viewer_denial(env, monkeypatch): if not hasattr(models, "Team"): pytest.skip("EE organization project") from core.db.models import Team, TeamMember - from core.llm.tool_collector import ToolCollector - from core.llm.tool_permissions import PermissionRuntime + from core.llm.factory.tools.definitions import ToolAssembly + from core.sandbox.permissions.dispatch_models import PermissionRuntime from core.llm.tools.project_instructions_tool import ( project_instruction_path, register_project_instruction_tools, ) - from core.services.project_scope import project_scope_from_context + from core.projects.edition import project_scope_from_context db, _ = env db.add(Team(team_id="team", name="Team", owner_user_id="alice", source="manual")) @@ -340,7 +339,7 @@ async def test_team_initialization_tool_and_viewer_denial(env, monkeypatch): target = project_instruction_path(ctx) assert target.startswith("/teamspace/team/") monkeypatch.setattr("core.config.local_mode.local_mode_enabled", lambda: False) - collector = ToolCollector() + collector = ToolAssembly() register_project_instruction_tools( collector, project_id=p.project_id, @@ -358,8 +357,8 @@ async def test_team_initialization_tool_and_viewer_denial(env, monkeypatch): intents = collector.permission_specs[tool_name].resolver({}, runtime) assert intents[0].target == "/workspace" + target assert all(intent.domain != "myspace" for intent in intents) - read = collector.get_tool("read_project_instructions")._func - save = collector.get_tool("save_project_instructions")._func + read = next(t for t in collector if t.name == "read_project_instructions")._func + save = next(t for t in collector if t.name == "save_project_instructions")._func snapshot = json.loads((await read()).content[0].text) assert snapshot["path"] == target result = json.loads( diff --git a/src/backend/tests/services/test_project_instructions_dispatch.py b/src/backend/tests/services/test_project_instructions_dispatch.py index c9d053fa..85eb8905 100644 --- a/src/backend/tests/services/test_project_instructions_dispatch.py +++ b/src/backend/tests/services/test_project_instructions_dispatch.py @@ -7,15 +7,15 @@ import pytest from core.chat.schemas import ChatRequest -from core.services.project_instructions import ProjectInstructionsService -from core.services.project_scope import build_project_ctx -from core.services.project_service import ProjectService +from core.projects.instructions import ProjectInstructionsService +from core.projects.edition import build_project_ctx +from core.projects.edition import ProjectService from tests.services.project_instructions_support import env, local, update def test_chat_dispatch_expands_command_and_loads_fresh_rules(env): - from core.services.session_operations.agent_targets import _resolve_chat_agent_targets - from core.services.session_operations.request_context import _build_ctx + from core.chat.operations.agent_targets import _resolve_chat_agent_targets + from core.chat.operations.request_context import build_ctx as _build_ctx db, _ = env p = ProjectService(db).create_personal("alice", "Dispatch") @@ -65,21 +65,21 @@ def test_combined_project_patch_preserves_metadata_with_production_session_setti @pytest.mark.asyncio async def test_normal_chat_can_read_canonical_rules_but_has_no_init_write_tool(env): - from core.llm.tool_collector import ToolCollector + from core.llm.factory.tools.definitions import ToolAssembly from core.llm.tools.project_instructions_tool import register_project_instruction_tools db, _ = env p = ProjectService(db).create_personal("alice", "Normal") update(db, p, "Current rules") - collector = ToolCollector() + collector = ToolAssembly() register_project_instruction_tools( collector, project_id=p.project_id, user_id="alice", allow_write=False, ) - assert collector.get_tool("save_project_instructions") is None - read = collector.get_tool("read_project_instructions")._func + assert not any(t.name == "save_project_instructions" for t in collector) + read = next(t for t in collector if t.name == "read_project_instructions")._func assert json.loads((await read()).content[0].text)["instructions"] == "Current rules" # Permission is checked at execution time, not only at registration. p.owner_user_id = "bob" @@ -89,10 +89,10 @@ async def test_normal_chat_can_read_canonical_rules_but_has_no_init_write_tool(e @pytest.mark.asyncio async def test_read_tool_ignores_stale_root_cache_without_changing_nested_rules(env, monkeypatch): - from core.llm.tool_collector import ToolCollector + from core.llm.factory.tools.definitions import ToolAssembly from core.llm.tools._state import ReadStateTracker from core.llm.tools.read_tool import register_read - from core.services.project_scope import project_scope_from_context + from core.projects.edition import project_scope_from_context db, _ = env monkeypatch.setattr("core.config.local_mode.local_mode_enabled", lambda: False) @@ -109,7 +109,7 @@ async def put_file(self, session, path, data, **kwargs): cache[path] = data monkeypatch.setattr("core.sandbox.get_sandbox_provider", lambda: Provider()) - collector = ToolCollector() + collector = ToolAssembly() register_read( collector, chat_id="read-init-test", @@ -118,7 +118,7 @@ async def put_file(self, session, path, data, **kwargs): project_folder_name=scope.folder_name, scope=scope, ) - read = collector.get_tool("Read")._func + read = next(t for t in collector if t.name == "Read")._func root = f"/myspace/{scope.folder_name}/AGENTS.md" result = json.loads((await read(root)).content[0].text) assert "FRESH-628" in result["content"] and "STALE" not in result["content"] diff --git a/src/backend/tests/services/test_retire_chart_mcp.py b/src/backend/tests/services/test_retire_chart_mcp.py index 9628f028..4b8036be 100644 --- a/src/backend/tests/services/test_retire_chart_mcp.py +++ b/src/backend/tests/services/test_retire_chart_mcp.py @@ -239,7 +239,7 @@ def test_retirement_rolls_back_atomically(chart_db): def test_chart_role_cannot_be_listed_assigned_or_imported(db_session): - from core.db import model_repository as repository + from core.models import repository as repository from core.db.models import ModelRoleAssignment assert "chart" not in { diff --git a/src/backend/tests/services/test_turbo_capability_config.py b/src/backend/tests/services/test_turbo_capability_config.py index 459077a8..5060abc7 100644 --- a/src/backend/tests/services/test_turbo_capability_config.py +++ b/src/backend/tests/services/test_turbo_capability_config.py @@ -4,9 +4,11 @@ 去重，配置层异常时按「没配」处理（不能因为读配置失败就把能力凭空塞进 极速模式）。EE 控制面选项接口的覆盖位于独立的 EE-only 测试文件中。 """ +from core.config.store import execution_options +from core.config.store import system_config import pytest -from core.services import system_config as sysconf +from core.config.store import system_config as sysconf class _StubConfigService: @@ -22,7 +24,7 @@ def get(self, key, default=None): def _patch_config(monkeypatch, values, *, raises=False): stub = _StubConfigService(values, raises=raises) - monkeypatch.setattr(sysconf.SystemConfigService, "get_instance", classmethod(lambda cls: stub)) + monkeypatch.setattr(system_config.SystemConfigService, "get_instance", classmethod(lambda cls: stub)) @pytest.mark.parametrize( @@ -36,7 +38,7 @@ def _patch_config(monkeypatch, values, *, raises=False): ) def test_turbo_skill_ids_parsing(monkeypatch, raw, expected): _patch_config(monkeypatch, {"turbo.skill_ids": raw}) - assert sysconf.turbo_skill_ids() == expected + assert execution_options.turbo_skill_ids() == expected @pytest.mark.parametrize( @@ -52,10 +54,10 @@ def test_turbo_skill_ids_parsing(monkeypatch, raw, expected): ) def test_turbo_plugin_ids_parsing(monkeypatch, raw, expected): _patch_config(monkeypatch, {"turbo.plugin_ids": raw}) - assert sysconf.turbo_plugin_ids() == expected + assert execution_options.turbo_plugin_ids() == expected def test_turbo_capability_ids_empty_when_config_layer_fails(monkeypatch): _patch_config(monkeypatch, {}, raises=True) - assert sysconf.turbo_skill_ids() == () - assert sysconf.turbo_plugin_ids() == () + assert execution_options.turbo_skill_ids() == () + assert execution_options.turbo_plugin_ids() == () diff --git a/src/backend/tests/services/test_user_model_selection.py b/src/backend/tests/services/test_user_model_selection.py index d0d4b483..cd2b3199 100644 --- a/src/backend/tests/services/test_user_model_selection.py +++ b/src/backend/tests/services/test_user_model_selection.py @@ -2,14 +2,9 @@ from sqlalchemy.orm import sessionmaker from core.db.models import ModelProvider, ModelRoleAssignment, UserShadow -from core.services.model_config import ModelConfigService -from core.services.user_model_selection import ( - UserModelSelectionError, - list_user_selectable_models, - resolve_effective_chat_model_name, - resolve_user_model_provider_id, - user_can_switch_model, -) +from core.models.config import ModelConfigService +from core.auth.model_selection import UserModelSelectionError, resolve_user_model_provider_id, user_can_switch_model +from core.models.selection import list_user_selectable_models, resolve_effective_chat_model_name def _provider( @@ -68,7 +63,7 @@ def test_resolve_user_model_provider_id_respects_switch_and_allowlist(db_session def test_resolve_effective_chat_model_name_uses_runtime_model_config(db_session, monkeypatch): - import core.services.model_config as model_config_module + import core.models.config as model_config_module db_session.add_all( [ diff --git a/src/backend/tests/services/user_folder_service_selftest.py b/src/backend/tests/services/user_folder_service_selftest.py index 5cbd9259..b46c09e8 100644 --- a/src/backend/tests/services/user_folder_service_selftest.py +++ b/src/backend/tests/services/user_folder_service_selftest.py @@ -15,11 +15,7 @@ def main() -> int: try: from core.db.engine import Base from core.db.models import Artifact, UserFolder, UserShadow - from core.services.user_folder_service import ( - MAX_FOLDER_DEPTH, - UserFolderService, - _sanitize_name, - ) + from core.projects.user_folders import MAX_FOLDER_DEPTH, UserFolderService, _sanitize_name from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker except ModuleNotFoundError as e: diff --git a/src/backend/tests/skills/agent_skills_selftest.py b/src/backend/tests/skills/agent_skills_selftest.py index 300de7f6..ac5933ed 100644 --- a/src/backend/tests/skills/agent_skills_selftest.py +++ b/src/backend/tests/skills/agent_skills_selftest.py @@ -2,8 +2,8 @@ from __future__ import annotations -from core.agent_skills.loader import get_skill_loader -from core.agent_skills.registry import render_skills_prompt +from core.capabilities.skills.loader_factory import get_skill_loader +from core.capabilities.skills.registry import render_skills_prompt def main() -> int: diff --git a/src/backend/tests/skills/backends_selftest.py b/src/backend/tests/skills/backends_selftest.py index 7be7e9d2..8bf67ed0 100644 --- a/src/backend/tests/skills/backends_selftest.py +++ b/src/backend/tests/skills/backends_selftest.py @@ -5,11 +5,7 @@ import tempfile from pathlib import Path -from core.agent_skills.backends import ( - CompositeBackend, - FilesystemBackend, - SkillFileInfo, -) +from core.capabilities.skills.backends import CompositeBackend, FilesystemBackend, SkillFileInfo def test_filesystem_backend_list(): diff --git a/src/backend/tests/skills/loader_selftest.py b/src/backend/tests/skills/loader_selftest.py index 1e90a440..6dc234c2 100644 --- a/src/backend/tests/skills/loader_selftest.py +++ b/src/backend/tests/skills/loader_selftest.py @@ -6,9 +6,10 @@ import tempfile from pathlib import Path -from core.agent_skills.backends import CompositeBackend, FilesystemBackend -from core.agent_skills.config import get_default_skill_sources, get_enabled_skill_sources -from core.agent_skills.loader import MultiSourceSkillLoader, get_skill_loader +from core.capabilities.skills.backends import CompositeBackend, FilesystemBackend +from core.capabilities.skills.config import get_default_skill_sources, get_enabled_skill_sources +from core.capabilities.skills.loader import MultiSourceSkillLoader +from core.capabilities.skills.loader_factory import get_skill_loader def test_config_default_sources(): diff --git a/src/backend/tests/skills/skill_loading_selftest.py b/src/backend/tests/skills/skill_loading_selftest.py index a0c679d3..2872c8c7 100644 --- a/src/backend/tests/skills/skill_loading_selftest.py +++ b/src/backend/tests/skills/skill_loading_selftest.py @@ -3,7 +3,7 @@ from __future__ import annotations import time -from core.agent_skills.loader import get_skill_loader +from core.capabilities.skills.loader_factory import get_skill_loader def test_metadata_loading(): diff --git a/src/backend/tests/skills/skill_selection_selftest.py b/src/backend/tests/skills/skill_selection_selftest.py index fdec3149..19511b15 100644 --- a/src/backend/tests/skills/skill_selection_selftest.py +++ b/src/backend/tests/skills/skill_selection_selftest.py @@ -2,8 +2,8 @@ from __future__ import annotations -from core.agent_skills.selector import select_skills_for_query -from core.config.catalog import get_enabled_ids +from core.llm.skill_selection import select_skills_for_query +from core.capabilities.catalog.api import get_enabled_ids def test_skill_selector_no_model(): @@ -33,7 +33,7 @@ def test_skill_selector_with_llm(): print("\nTesting skill selector with LLM...") try: - from core.llm.chat_models import get_summarize_model + from core.models.chat_models import get_summarize_model model = get_summarize_model() except Exception as e: @@ -80,7 +80,7 @@ def test_explicit_vs_implicit(): """Test explicit (required) vs implicit (available) skills via AgentSpec.""" print("\nTesting explicit vs implicit skills...") - from orchestration.registry import AgentSpec + from core.llm.factory.specs import AgentSpec spec = AgentSpec( name="test_agent", diff --git a/src/backend/tests/skills/test_database_backend_lazy.py b/src/backend/tests/skills/test_database_backend_lazy.py index 01c24def..e9d89efa 100644 --- a/src/backend/tests/skills/test_database_backend_lazy.py +++ b/src/backend/tests/skills/test_database_backend_lazy.py @@ -9,7 +9,7 @@ def test_database_skill_listing_projects_metadata_and_defers_full_content( db_session, monkeypatch, ): - from core.agent_skills.backends.database import DatabaseBackend + from core.capabilities.skills.backends.database import DatabaseBackend from core.db.models import AdminSkill full_content = ( @@ -76,9 +76,9 @@ def capture_sql(conn, cursor, statement, parameters, context, executemany): def test_loader_reads_full_database_skill_only_on_demand(): - from core.agent_skills.backends.composite import CompositeBackend - from core.agent_skills.backends.protocol import SkillFileInfo - from core.agent_skills.loader import MultiSourceSkillLoader + from core.capabilities.skills.backends.composite import CompositeBackend + from core.capabilities.skills.backends.protocol import SkillFileInfo + from core.capabilities.skills.loader import MultiSourceSkillLoader full_content = ( "---\n" @@ -136,7 +136,7 @@ def exists(self, skill_id): def test_disabled_installed_skill_is_loadable_on_explicit_selection(db_session, monkeypatch): - from core.agent_skills.backends.database import DatabaseBackend + from core.capabilities.skills.backends.database import DatabaseBackend from core.db.models import AdminSkill db_session.add( diff --git a/src/backend/tests/skills/test_deps_detector.py b/src/backend/tests/skills/test_deps_detector.py index b17096c0..be30dccc 100644 --- a/src/backend/tests/skills/test_deps_detector.py +++ b/src/backend/tests/skills/test_deps_detector.py @@ -3,7 +3,7 @@ import pytest -from core.agent_skills.deps_detector import detect_dependencies +from core.capabilities.skills.deps.detector import detect_dependencies def _names(deps: list[dict]) -> list[str]: diff --git a/src/backend/tests/skills/test_read_artifact_pptx_xlsx.py b/src/backend/tests/skills/test_read_artifact_pptx_xlsx.py index 26e7a328..0530a420 100644 --- a/src/backend/tests/skills/test_read_artifact_pptx_xlsx.py +++ b/src/backend/tests/skills/test_read_artifact_pptx_xlsx.py @@ -40,13 +40,13 @@ def _make_two_sheet_xlsx() -> bytes: def test_parse_xlsx_sheet_names_returns_workbook_order(): - from core.content.file_parser import parse_xlsx_sheet_names + from core.files.content.spreadsheet_parser import parse_xlsx_sheet_names names = parse_xlsx_sheet_names(_make_two_sheet_xlsx()) assert names == ["Alpha", "Beta"] def test_parse_xlsx_single_sheet_returns_only_that_sheet(): - from core.content.file_parser import parse_xlsx_single_sheet + from core.files.content.spreadsheet_parser import parse_xlsx_single_sheet md = parse_xlsx_single_sheet(_make_two_sheet_xlsx(), "Beta") assert "city" in md assert "pop" in md @@ -57,7 +57,7 @@ def test_parse_xlsx_single_sheet_returns_only_that_sheet(): def test_parse_xlsx_single_sheet_missing_raises(): - from core.content.file_parser import parse_xlsx_single_sheet + from core.files.content.spreadsheet_parser import parse_xlsx_single_sheet with pytest.raises(RuntimeError, match="sheet 不存在"): parse_xlsx_single_sheet(_make_two_sheet_xlsx(), "NonExistent") @@ -88,14 +88,12 @@ def test_is_pptx_meta_detects_by_mime_and_extension(): @pytest.fixture def registered_tool(): """Register read_artifact onto a fresh Toolkit and return the callable.""" - from core.llm.tool_collector import ToolCollector + from core.llm.factory.tools.definitions import ToolAssembly from core.llm.tools.read_artifact_tool import register_read_artifact - # AgentScope 2.0: register_* writes into the ToolCollector (duck-compatible with register_tool_function); - # the tool's original callable lives at AllowedFunctionTool._func. - c = ToolCollector() + c = ToolAssembly() register_read_artifact(c, user_id="u1") - return c.get_tool("read_artifact")._func + return next(t for t in c if t.name == "read_artifact")._func async def _call(func, **kwargs) -> Dict[str, Any]: @@ -115,7 +113,7 @@ async def test_mutual_exclusion_sheet_and_slide(registered_tool): @pytest.mark.asyncio async def test_missing_artifact_returns_error(registered_tool): - with patch("core.content.artifact_reader.load_artifact_meta", return_value=None): + with patch('core.files.artifacts.reader.load_artifact_meta', return_value=None): out = await _call(registered_tool, file_id="ua_missing") assert "error" in out assert "不存在" in out["error"] @@ -126,9 +124,9 @@ async def test_xlsx_default_call_includes_sheet_names(registered_tool): """Without sheet_name, returns concatenated text + sheet_names list.""" file_bytes = _make_two_sheet_xlsx() meta = {"filename": "two.xlsx", "mime_type": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"} - with patch("core.content.artifact_reader.load_artifact_meta", return_value=meta), \ - patch("core.content.artifact_reader.fetch_parsed_text", return_value="full concatenated md"), \ - patch("core.llm.hooks._download_artifact_bytes", return_value=file_bytes): + with patch('core.files.artifacts.reader.load_artifact_meta', return_value=meta), \ + patch('core.files.artifacts.reader.fetch_parsed_text', return_value="full concatenated md"), \ + patch('core.files.artifact_bytes.download_artifact_bytes', return_value=file_bytes): out = await _call(registered_tool, file_id="ua_xlsx") assert out.get("sheet_names") == ["Alpha", "Beta"] assert out["content"] == "full concatenated md" @@ -138,8 +136,8 @@ async def test_xlsx_default_call_includes_sheet_names(registered_tool): async def test_xlsx_with_sheet_name_returns_only_that_sheet(registered_tool): file_bytes = _make_two_sheet_xlsx() meta = {"filename": "two.xlsx", "mime_type": ""} - with patch("core.content.artifact_reader.load_artifact_meta", return_value=meta), \ - patch("core.llm.hooks._download_artifact_bytes", return_value=file_bytes): + with patch('core.files.artifacts.reader.load_artifact_meta', return_value=meta), \ + patch('core.files.artifact_bytes.download_artifact_bytes', return_value=file_bytes): out = await _call(registered_tool, file_id="ua_xlsx", sheet_name="Beta", limit=20000) assert out.get("sheet_names") == ["Alpha", "Beta"] assert "city" in out["content"] @@ -150,8 +148,8 @@ async def test_xlsx_with_sheet_name_returns_only_that_sheet(registered_tool): async def test_xlsx_unknown_sheet_returns_error_with_names(registered_tool): file_bytes = _make_two_sheet_xlsx() meta = {"filename": "two.xlsx", "mime_type": ""} - with patch("core.content.artifact_reader.load_artifact_meta", return_value=meta), \ - patch("core.llm.hooks._download_artifact_bytes", return_value=file_bytes): + with patch('core.files.artifacts.reader.load_artifact_meta', return_value=meta), \ + patch('core.files.artifact_bytes.download_artifact_bytes', return_value=file_bytes): out = await _call(registered_tool, file_id="ua_xlsx", sheet_name="Gamma") assert "error" in out assert "Gamma" in out["error"] @@ -172,26 +170,26 @@ def _make_three_slide_pptx() -> bytes: def test_parse_pptx_emits_one_section_per_slide(): - from core.content.file_parser import parse_pptx + from core.files.content.presentation_parser import parse_pptx md = parse_pptx(_make_three_slide_pptx()) assert md.count("## Slide ") == 3 assert "Hello" in md and "World" in md and "Goodbye" in md def test_parse_pptx_slide_count_returns_int(): - from core.content.file_parser import parse_pptx_slide_count + from core.files.content.presentation_parser import parse_pptx_slide_count assert parse_pptx_slide_count(_make_three_slide_pptx()) == 3 def test_parse_pptx_slide_extracts_single_slide(): - from core.content.file_parser import parse_pptx_slide + from core.files.content.presentation_parser import parse_pptx_slide text = parse_pptx_slide(_make_three_slide_pptx(), 1) assert "World" in text assert "Hello" not in text and "Goodbye" not in text def test_parse_pptx_slide_out_of_range_raises(): - from core.content.file_parser import parse_pptx_slide + from core.files.content.presentation_parser import parse_pptx_slide with pytest.raises(RuntimeError, match="越界"): parse_pptx_slide(_make_three_slide_pptx(), 10) @@ -203,9 +201,9 @@ async def test_pptx_default_returns_full_deck_with_slide_count(registered_tool): is reported from the in-process probe.""" file_bytes = _make_three_slide_pptx() meta = {"filename": "deck.pptx", "mime_type": ""} - with patch("core.content.artifact_reader.load_artifact_meta", return_value=meta), \ - patch("core.content.artifact_reader.fetch_parsed_text", return_value="full deck text"), \ - patch("core.llm.hooks._download_artifact_bytes", return_value=file_bytes): + with patch('core.files.artifacts.reader.load_artifact_meta', return_value=meta), \ + patch('core.files.artifacts.reader.fetch_parsed_text', return_value="full deck text"), \ + patch('core.files.artifact_bytes.download_artifact_bytes', return_value=file_bytes): out = await _call(registered_tool, file_id="ua_pptx", limit=20000) assert out["content"] == "full deck text" assert out["slide_count"] == 3 @@ -215,8 +213,8 @@ async def test_pptx_default_returns_full_deck_with_slide_count(registered_tool): async def test_pptx_with_slide_index_extracts_only_that_slide(registered_tool): file_bytes = _make_three_slide_pptx() meta = {"filename": "deck.pptx", "mime_type": ""} - with patch("core.content.artifact_reader.load_artifact_meta", return_value=meta), \ - patch("core.llm.hooks._download_artifact_bytes", return_value=file_bytes): + with patch('core.files.artifacts.reader.load_artifact_meta', return_value=meta), \ + patch('core.files.artifact_bytes.download_artifact_bytes', return_value=file_bytes): out = await _call(registered_tool, file_id="ua_pptx", slide_index=1, limit=2000) assert "World" in out["content"] assert "Hello" not in out["content"] @@ -228,8 +226,8 @@ async def test_pptx_with_slide_index_extracts_only_that_slide(registered_tool): async def test_pptx_slide_index_out_of_range_returns_error_with_count(registered_tool): file_bytes = _make_three_slide_pptx() meta = {"filename": "deck.pptx", "mime_type": ""} - with patch("core.content.artifact_reader.load_artifact_meta", return_value=meta), \ - patch("core.llm.hooks._download_artifact_bytes", return_value=file_bytes): + with patch('core.files.artifacts.reader.load_artifact_meta', return_value=meta), \ + patch('core.files.artifact_bytes.download_artifact_bytes', return_value=file_bytes): out = await _call(registered_tool, file_id="ua_pptx", slide_index=99) assert "error" in out assert "越界" in out["error"] @@ -239,7 +237,7 @@ async def test_pptx_slide_index_out_of_range_returns_error_with_count(registered @pytest.mark.asyncio async def test_non_office_with_sheet_or_slide_param_errors(registered_tool): meta = {"filename": "doc.pdf", "mime_type": "application/pdf"} - with patch("core.content.artifact_reader.load_artifact_meta", return_value=meta): + with patch('core.files.artifacts.reader.load_artifact_meta', return_value=meta): out = await _call(registered_tool, file_id="ua_pdf", sheet_name="Sheet1") assert "error" in out assert "xlsx" in out["error"] diff --git a/src/backend/tests/skills/test_registry.py b/src/backend/tests/skills/test_registry.py index d8c5e42f..15c94588 100644 --- a/src/backend/tests/skills/test_registry.py +++ b/src/backend/tests/skills/test_registry.py @@ -1,6 +1,6 @@ """Focused tests for the agent-skill SKILL.md parser.""" -from core.agent_skills.registry import _load_skill_metadata_from_str, _split_frontmatter +from core.capabilities.skills.registry import load_skill_metadata_from_str, split_frontmatter def test_frontmatter_accepts_windows_crlf_line_endings(): @@ -16,8 +16,8 @@ def test_frontmatter_accepts_windows_crlf_line_endings(): "Follow the instructions.\r\n" ) - frontmatter, body = _split_frontmatter(raw) - metadata = _load_skill_metadata_from_str(raw, "windows-skill") + frontmatter, body = split_frontmatter(raw) + metadata = load_skill_metadata_from_str(raw, "windows-skill") assert frontmatter["name"] == "windows-skill" assert body.startswith("\n# Windows skill\n") diff --git a/src/backend/tests/skills/test_sandbox_skill_purge.py b/src/backend/tests/skills/test_sandbox_skill_purge.py index 1c895423..2feeb8ab 100644 --- a/src/backend/tests/skills/test_sandbox_skill_purge.py +++ b/src/backend/tests/skills/test_sandbox_skill_purge.py @@ -10,16 +10,7 @@ import pytest -from core.agent_skills.config import ( - SHARED_LINK_NAME, - get_default_skill_sources, - get_sandbox_skills_dir, - get_user_skills_dir, - prune_orphan_sandbox_skill_dirs, - purge_skill_sandbox_files, - skill_files_dir, - sync_user_skill_view, -) +from core.capabilities.skills.config import SHARED_LINK_NAME, get_default_skill_sources, get_sandbox_skills_dir, get_user_skills_dir, prune_orphan_sandbox_skill_dirs, purge_skill_sandbox_files, skill_files_dir, sync_user_skill_view def _a_builtin_skill_id() -> str: @@ -158,9 +149,9 @@ def test_loader_materializes_a_private_skill_into_its_owners_dir(skills_dir) -> """The owner recorded on the row picks the dir, which is what keeps views correct.""" from pathlib import Path - from core.agent_skills.backends.composite import CompositeBackend - from core.agent_skills.backends.protocol import SkillFileInfo - from core.agent_skills.loader import MultiSourceSkillLoader + from core.capabilities.skills.backends.composite import CompositeBackend + from core.capabilities.skills.backends.protocol import SkillFileInfo + from core.capabilities.skills.loader import MultiSourceSkillLoader def _info(skill_id: str, owner): return SkillFileInfo( diff --git a/src/backend/tests/skills/test_skill_deps_aggregator.py b/src/backend/tests/skills/test_skill_deps_aggregator.py index 11120c6a..87ecbccb 100644 --- a/src/backend/tests/skills/test_skill_deps_aggregator.py +++ b/src/backend/tests/skills/test_skill_deps_aggregator.py @@ -1,4 +1,4 @@ -"""Tests for core.services.skill_deps_aggregator.""" +"""Tests for core.capabilities.skills.deps.aggregator.""" from __future__ import annotations import json @@ -11,13 +11,7 @@ from core.db.engine import Base from core.db.models import AdminSkill -from core.services.skill_deps_aggregator import ( - aggregate_all, - render_apt_packages, - render_npm_package_json, - render_pip_requirements, - write_manifests, -) +from core.capabilities.skills.deps.aggregator import aggregate_all, render_apt_packages, render_npm_package_json, render_pip_requirements, write_manifests @pytest.fixture diff --git a/src/backend/tests/skills/test_skill_publication.py b/src/backend/tests/skills/test_skill_publication.py index 619a7787..4905140e 100644 --- a/src/backend/tests/skills/test_skill_publication.py +++ b/src/backend/tests/skills/test_skill_publication.py @@ -1,12 +1,12 @@ """Regression coverage at skill loading, publication and sandbox preparation boundaries.""" - -from tests.sandbox.provider_test_support import patch_open_settings +from core.capabilities.skills import loader_factory as loader_factory_module from pathlib import Path -from core.agent_skills.backends.composite import CompositeBackend -from core.agent_skills.backends.protocol import SkillFileInfo -from core.agent_skills.loader import MultiSourceSkillLoader +from tests.sandbox.provider_test_support import patch_open_settings +from core.capabilities.skills.backends.composite import CompositeBackend +from core.capabilities.skills.backends.protocol import SkillFileInfo +from core.capabilities.skills.loader import MultiSourceSkillLoader class MutableSource: @@ -69,7 +69,7 @@ def test_replacement_publishes_exact_file_set_with_fresh_loader(tmp_path, monkey @pytest.fixture def cloud(tmp_path, monkeypatch): import core.db.engine as dbe - from core.agent_skills import config, loader + from core.capabilities.skills import config, loader from api.routes.v1 import me_capabilities, admin_skills from api.deps import require_admin @@ -92,7 +92,7 @@ def cloud(tmp_path, monkeypatch): dbe.Base.metadata.create_all(engine) factory = sessionmaker(bind=engine, expire_on_commit=False) monkeypatch.setattr(dbe, "SessionLocal", factory) - monkeypatch.setattr(loader, "_global_loader", None) + monkeypatch.setattr(loader_factory_module, 'global_loader', None) monkeypatch.setattr( me_capabilities, "resolve_user_capabilities", lambda *a: {"can_add_skill": True} ) @@ -113,7 +113,7 @@ def session(): yield SimpleNamespace( client=client, factory=factory, builtin=builtin, storage=tmp_path / "skills" ) - loader._global_loader = None + loader_factory_module.global_loader = None engine.dispose() @@ -131,7 +131,7 @@ def upload(client, files, prefix="/v1/me/skills/upload"): def test_web_delete_removes_materialized_skill(cloud): - from core.agent_skills.loader import get_skill_loader + from core.capabilities.skills.loader_factory import get_skill_loader upload(cloud.client, {"old.txt": "old"}) path = Path(get_skill_loader().get_skill_base_dir("sample")) @@ -141,7 +141,7 @@ def test_web_delete_removes_materialized_skill(cloud): def test_upload_replace_and_new_sandbox_view_are_exact(cloud): - from core.agent_skills.publication import prepare_skill_view + from core.capabilities.skills.publication import prepare_skill_view upload(cloud.client, {"old.txt": "old", "nested/run.txt": "v1"}) view = prepare_skill_view("alice") @@ -159,7 +159,7 @@ def test_upload_replace_and_new_sandbox_view_are_exact(cloud): def test_old_loader_cannot_republish_deleted_or_replaced_payload(cloud): - from core.agent_skills.loader import get_skill_loader + from core.capabilities.skills.loader_factory import get_skill_loader upload(cloud.client, {"old.txt": "v1"}) stale = get_skill_loader() @@ -180,8 +180,8 @@ def test_old_loader_cannot_republish_deleted_or_replaced_payload(cloud): def test_new_mounts_exclude_deleted_global_private_and_historical_dirs(cloud, monkeypatch): from core.db.models import AdminSkill - from core.agent_skills.publication import prepare_skill_view - from core.agent_skills.loader import get_skill_loader + from core.capabilities.skills.publication import prepare_skill_view + from core.capabilities.skills.loader_factory import get_skill_loader from core.sandbox.providers.opensandbox.volumes import _make_skills_volumes from core.config.settings import settings @@ -233,7 +233,7 @@ def test_new_mounts_exclude_deleted_global_private_and_historical_dirs(cloud, mo def test_shipped_package_replacement_removes_retired_files(cloud): - from core.agent_skills.publication import prepare_skill_view + from core.capabilities.skills.publication import prepare_skill_view bundle = cloud.builtin / "bundled" bundle.mkdir() @@ -249,7 +249,7 @@ def test_shipped_package_replacement_removes_retired_files(cloud): def test_mount_preparation_fails_if_authoritative_source_unavailable(cloud, monkeypatch): - from core.agent_skills.backends.database import DatabaseBackend + from core.capabilities.skills.backends.database import DatabaseBackend from core.sandbox.providers.opensandbox.volumes import _make_skills_volumes from core.config.settings import settings @@ -270,7 +270,7 @@ def unavailable(self): def test_single_file_delete_propagates_to_new_view(cloud): - from core.agent_skills.publication import prepare_skill_view + from core.capabilities.skills.publication import prepare_skill_view upload(cloud.client, {"nested/old.txt": "old", "nested/new.txt": "new"}) prepare_skill_view("alice") @@ -282,7 +282,7 @@ def test_single_file_delete_propagates_to_new_view(cloud): def test_invalid_replacement_preserves_previous_tree_and_cannot_escape(tmp_path, monkeypatch): - from core.agent_skills.publication import publish_tree + from core.capabilities.skills.publication import publish_tree monkeypatch.setenv("SANDBOX_SKILLS_DIR", str(tmp_path / "skills")) path = tmp_path / "skills/sample" @@ -295,7 +295,7 @@ def test_invalid_replacement_preserves_previous_tree_and_cannot_escape(tmp_path, def test_failed_swap_preserves_previous_tree(tmp_path, monkeypatch): import os - from core.agent_skills.publication import publish_tree + from core.capabilities.skills.publication import publish_tree monkeypatch.setenv("SANDBOX_SKILLS_DIR", str(tmp_path / "skills")) path = tmp_path / "skills/sample" @@ -317,7 +317,7 @@ def test_archive_for_new_sandbox_uses_current_files_without_local_cache_reset( tmp_path, monkeypatch ): import tarfile - from core.agent_skills.skill_archive import build_skill_tar + from core.capabilities.skills.skill_archive import build_skill_tar monkeypatch.setenv("SANDBOX_SKILLS_DIR", str(tmp_path / "skills")) source = tmp_path / "source" @@ -336,7 +336,7 @@ def test_archive_for_new_sandbox_uses_current_files_without_local_cache_reset( def test_startup_respects_db_override_and_bundle_executable_modes(cloud): from core.db.models import AdminSkill - from core.agent_skills.config import sync_builtin_skills_to_sandbox_dir + from core.capabilities.skills.config import sync_builtin_skills_to_sandbox_dir bundle = cloud.builtin / "bundled" bundle.mkdir() @@ -364,7 +364,7 @@ def test_startup_respects_db_override_and_bundle_executable_modes(cloud): def test_admin_delete_and_file_delete_use_the_same_publication_rules(cloud): - from core.agent_skills.publication import prepare_skill_view + from core.capabilities.skills.publication import prepare_skill_view upload(cloud.client, {"old.txt": "old", "new.txt": "new"}) response = cloud.client.delete("/v1/admin/skills/sample/files/old.txt") @@ -377,8 +377,8 @@ def test_admin_delete_and_file_delete_use_the_same_publication_rules(cloud): def test_prepared_view_repairs_stale_shared_link(cloud): - from core.agent_skills.publication import prepare_skill_view - from core.agent_skills.config import get_user_skills_root, SHARED_LINK_NAME + from core.capabilities.skills.publication import prepare_skill_view + from core.capabilities.skills.config import get_user_skills_root, SHARED_LINK_NAME link = get_user_skills_root() / SHARED_LINK_NAME link.unlink() @@ -391,7 +391,7 @@ def test_prepared_view_repairs_stale_shared_link(cloud): @pytest.mark.asyncio async def test_prewarmed_session_delivery_reconciles_before_pool_hit(cloud, monkeypatch): - from core.agent_skills.publication import prepare_skill_view + from core.capabilities.skills.publication import prepare_skill_view import core.sandbox.providers.opensandbox.lifecycle as sessions from unittest.mock import AsyncMock @@ -408,7 +408,6 @@ async def test_prewarmed_session_delivery_reconciles_before_pool_hit(cloud, monk return_value=SimpleNamespace(sandbox=SimpleNamespace(id="pooled")) ), ) - monkeypatch.setattr(sessions, "_user_bound_sandbox_required", lambda: True) await module_class._create_session_for(fake, "new-chat", user_id="alice") assert fake._create_session.await_count == 1 assert not (view / "sample/old.txt").exists() @@ -417,7 +416,7 @@ async def test_prewarmed_session_delivery_reconciles_before_pool_hit(cloud, monk def test_publication_lock_serializes_processes(tmp_path, monkeypatch): import multiprocessing - from core.agent_skills.publication import publication_lock + from core.capabilities.skills.publication import publication_lock monkeypatch.setenv("SANDBOX_SKILLS_DIR", str(tmp_path / "skills")) ctx = multiprocessing.get_context("fork") @@ -454,7 +453,7 @@ def waiting_writer(): def test_view_link_creation_failure_prevents_sandbox_preparation(cloud, monkeypatch): - from core.agent_skills.publication import prepare_skill_view + from core.capabilities.skills.publication import prepare_skill_view bundle = cloud.builtin / "bundled" bundle.mkdir() @@ -473,7 +472,7 @@ def fail(path, target, **kwargs): def test_unwritable_explicit_skill_root_never_falls_back(tmp_path, monkeypatch): - from core.agent_skills.config import get_sandbox_skills_dir + from core.capabilities.skills.config import get_sandbox_skills_dir occupied = tmp_path / "occupied" occupied.write_text("not a directory") @@ -483,7 +482,7 @@ def test_unwritable_explicit_skill_root_never_falls_back(tmp_path, monkeypatch): def test_private_collision_does_not_remove_other_users_builtin(cloud): - from core.agent_skills.publication import prepare_skill_view + from core.capabilities.skills.publication import prepare_skill_view package = cloud.builtin / "sample" package.mkdir() @@ -504,8 +503,8 @@ def test_private_collision_does_not_remove_other_users_builtin(cloud): def test_cube_current_archive_rechecks_owner_and_freezes_bytes(cloud): import tarfile from core.db.models import AdminSkill - from core.agent_skills.loader import get_skill_loader - from core.agent_skills.skill_archive import build_current_skill_tar + from core.capabilities.skills.loader_factory import get_skill_loader + from core.capabilities.skills.skill_archive import build_current_skill_tar upload(cloud.client, {"public.txt": "v1"}) with cloud.factory() as db: @@ -546,7 +545,7 @@ async def test_cube_prepush_uses_users_private_override(cloud, monkeypatch): provider._skill_prepush_concurrency = 1 provider._skill_prepush_max_bytes = 0 provider._push_skill_archive = AsyncMock() - monkeypatch.setattr("core.config.catalog.get_enabled_ids", lambda kind: {"sample"}) + monkeypatch.setattr("core.capabilities.catalog.api.get_enabled_ids", lambda kind: {"sample"}) sbx = SimpleNamespace(sandbox_id="fresh") await provider._prepush_skills(sbx, "alice") archive = provider._push_skill_archive.await_args.args[2] @@ -560,7 +559,7 @@ async def test_cube_prepush_uses_users_private_override(cloud, monkeypatch): def test_new_sandbox_accepts_legacy_windows_paths_and_excludes_corrupt_skill(cloud, caplog): from core.db.models import AdminSkill - from core.agent_skills.publication import prepare_skill_view + from core.capabilities.skills.publication import prepare_skill_view upload(cloud.client, {"scripts/run.py": "v1"}) old_view = prepare_skill_view("alice") @@ -583,7 +582,7 @@ def test_new_sandbox_accepts_legacy_windows_paths_and_excludes_corrupt_skill(clo def test_windows_zip_upload_is_stored_canonically_and_ready_for_sandbox(cloud): from core.db.models import AdminSkill - from core.agent_skills.publication import prepare_skill_view + from core.capabilities.skills.publication import prepare_skill_view upload(cloud.client, {"scripts\\rank_objects.py": "print(1)"}) with cloud.factory() as db: diff --git a/src/backend/tests/skills/test_skill_publication_docker.py b/src/backend/tests/skills/test_skill_publication_docker.py index 90920047..685b4f22 100644 --- a/src/backend/tests/skills/test_skill_publication_docker.py +++ b/src/backend/tests/skills/test_skill_publication_docker.py @@ -54,7 +54,7 @@ def inspect_new_sandbox(view, shared): def test_new_readonly_sandbox_after_replace_and_delete_has_exact_files(cloud): - from core.agent_skills.publication import prepare_skill_view + from core.capabilities.skills.publication import prepare_skill_view upload(cloud.client, {"old.txt": "old", "nested/run.txt": "v1"}) view = prepare_skill_view("alice") diff --git a/src/backend/tests/skills/test_skill_selector_context_ir.py b/src/backend/tests/skills/test_skill_selector_context_ir.py index 3ab86621..50a2ca0f 100644 --- a/src/backend/tests/skills/test_skill_selector_context_ir.py +++ b/src/backend/tests/skills/test_skill_selector_context_ir.py @@ -2,8 +2,8 @@ from types import SimpleNamespace -from core.agent_skills.registry import AgentSkillMetadata -from core.agent_skills.selector import _llm_select_skills +from core.capabilities.skills.registry import AgentSkillMetadata +from core.llm.skill_selection import rank_skill_candidates class CaptureSelectorModel: @@ -26,7 +26,7 @@ def test_selector_messages_carry_context_ir_provenance(): version="1", ) - selected = _llm_select_skills( + selected = rank_skill_candidates( user_query="analyze this", candidates=[skill], model=model, diff --git a/src/backend/tests/skills/test_skill_zip_export.py b/src/backend/tests/skills/test_skill_zip_export.py index 93d53ca8..5575af1e 100644 --- a/src/backend/tests/skills/test_skill_zip_export.py +++ b/src/backend/tests/skills/test_skill_zip_export.py @@ -6,17 +6,9 @@ import zipfile import pytest -from core.agent_skills.binary_files import ( - decode_binary, - encode_binary, - encode_upload, - is_binary_value, -) -from core.services.marketplace_service import ( - build_skill_zip, - build_skill_zip_from_dir, - parse_skill_zip, -) +from core.capabilities.skills.binary_files import decode_binary, encode_binary, encode_upload, is_binary_value +from features.marketplace.skill_archives import build_skill_zip, build_skill_zip_from_dir +from core.capabilities.skills.zip_reader import parse_skill_zip from fastapi import HTTPException SKILL_MD = """--- @@ -95,7 +87,7 @@ def test_encode_upload_text_and_binary(): def test_validate_skill_file_path(): - from core.services.skill_management_service import validate_skill_file_path + from core.capabilities.skills.management import validate_skill_file_path assert validate_skill_file_path("scripts/run.py") == "scripts/run.py" assert validate_skill_file_path("/config.json") == "config.json" @@ -137,7 +129,7 @@ def test_zip_rejects_normalized_name_collision(): def test_legacy_skill_paths_normalize_without_silent_overwrite(): - from core.agent_skills.publication import decode_files + from core.capabilities.skills.publication import decode_files assert decode_files("body", {"scripts\\run.py": "code"})["scripts/run.py"] == b"code" with pytest.raises(ValueError): @@ -147,7 +139,7 @@ def test_legacy_skill_paths_normalize_without_silent_overwrite(): def test_archive_and_legacy_db_reject_file_directory_conflicts(): - from core.agent_skills.publication import decode_files + from core.capabilities.skills.publication import decode_files with pytest.raises(ValueError): decode_files("body", {"scripts": "file", "scripts/run.py": "code"}) diff --git a/src/backend/tests/test_add_agent_permission.py b/src/backend/tests/test_add_agent_permission.py index 99308d0b..aa3f1794 100644 --- a/src/backend/tests/test_add_agent_permission.py +++ b/src/backend/tests/test_add_agent_permission.py @@ -36,7 +36,7 @@ def _mk_user(db, uid, meta=None): def test_agents_create_gate(db_session): - from api.routes.v1.agents import _require_can_add_agent + from api.edition import _require_can_add_agent _mk_user(db_session, "u_off", meta={}) with pytest.raises(AccessDeniedError): @@ -47,7 +47,7 @@ def test_agents_create_gate(db_session): def test_marketplace_install_gate(db_session): - from api.routes.v1.agent_marketplace import _require_can_add_agent + from features.marketplace.api.agent_marketplace import _require_can_add_agent _mk_user(db_session, "m_off", meta={}) with pytest.raises(AccessDeniedError): diff --git a/src/backend/tests/test_agent_manager_plugin.py b/src/backend/tests/test_agent_manager_plugin.py index 5efa21c4..9a68034e 100644 --- a/src/backend/tests/test_agent_manager_plugin.py +++ b/src/backend/tests/test_agent_manager_plugin.py @@ -74,7 +74,7 @@ def test_bundle_manifest_is_wellformed(): assert manifest["name"] == "agent-manager" ext_mcp = manifest["extensions"]["org.hugagent"]["mcp"] assert set(ext_mcp) == set(mcp_json["mcpServers"]), "扩展段的服务名必须与 mcp.json 一一对应" - assert mcp_json["mcpServers"]["agent_manager"]["url"] == "http://mcp:9115/mcp/" + assert mcp_json["mcpServers"]["agent_manager"]["url"] == "http://mcp:9115/mcp" # The declared tool list must match what the server actually exposes — a manifest that # advertises a tool the server doesn't have is a silently broken promise to the model. @@ -125,7 +125,7 @@ def test_port_is_registered(): # ── Plugin persistence + merging into the user's available set ────────────── def test_install_plugin_creates_rows_and_is_agent_visible(am_env): - from core.plugins import management as ps + from core.capabilities.plugins import management as ps with am_env.Session() as db: res = ps.install_plugin(db, "agent-manager", owner_user_id=OWNER, created_by=OWNER) @@ -141,15 +141,12 @@ def test_install_plugin_creates_rows_and_is_agent_visible(am_env): mcp = mcps[0] assert mcp.owner_user_id == OWNER assert mcp.transport == "streamable_http" - assert mcp.url == "http://mcp:9115/mcp/" + assert mcp.url == "http://mcp:9115/mcp" assert mcp.is_enabled is True # http MCP is not needs_runtime → enabled upon install assert len(mcp.tools_json) == 8 assert plugin is not None - from core.config.catalog_resolver import ( - invalidate_capability_cache, - resolve_all_runtime_enabled, - ) + from core.capabilities.catalog.resolver import invalidate_capability_cache, resolve_all_runtime_enabled invalidate_capability_cache() enabled_skills, _agents, enabled_mcps = resolve_all_runtime_enabled(db, OWNER) @@ -272,7 +269,7 @@ def test_cannot_touch_other_users_agents(am_env): def test_admin_agents_are_not_listed_as_mine(am_env): """list_for_user also returns admin-owned agents; the self-serve entry must not claim them.""" - from core.services.user_agent_service import UserAgentService + from core.capabilities.agents.edition import UserAgentService from extensions.mcp_servers.agent_manager_mcp import impl with am_env.Session() as db: diff --git a/src/backend/tests/test_agent_markdown.py b/src/backend/tests/test_agent_markdown.py index aac08c4a..169a4a68 100644 --- a/src/backend/tests/test_agent_markdown.py +++ b/src/backend/tests/test_agent_markdown.py @@ -6,12 +6,7 @@ from decimal import Decimal import pytest -from core.services.agent_markdown import ( - agent_to_markdown, - agents_to_zip, - parse_agent_markdown, - parse_agents_upload, -) +from core.capabilities.agents.markdown import agent_to_markdown, agents_to_zip, parse_agent_markdown, parse_agents_upload SAMPLE = { "name": "数据分析师", diff --git a/src/backend/tests/test_agent_version_history.py b/src/backend/tests/test_agent_version_history.py new file mode 100644 index 00000000..cf8a506c --- /dev/null +++ b/src/backend/tests/test_agent_version_history.py @@ -0,0 +1,15 @@ +"""Agent version helpers retain the base class normalization contract.""" +from core.capabilities.agents.base import UserAgentBaseService +from core.capabilities.agents import version_history + + +def test_version_operations_use_base_class_normalizer(monkeypatch): + monkeypatch.setattr(UserAgentBaseService, "_normalize_version", staticmethod(lambda value: "V9.9")) + assert version_history.read_version(UserAgentBaseService, {"version": "old"}) == "V9.9" + assert version_history.increment_version(UserAgentBaseService, "old") == "V9.10" + + +def test_history_uses_base_class_detail_normalizer(monkeypatch): + monkeypatch.setattr(UserAgentBaseService, "_normalize_change_details", staticmethod(lambda value: [{"field": "checked"}])) + rows = version_history.read_change_history(UserAgentBaseService, {"change_history": [{"timestamp": "now", "content": "change", "version": "V1.0"}]}) + assert rows[0]["details"] == [{"field": "checked"}] diff --git a/src/backend/tests/test_automation_manual_trigger.py b/src/backend/tests/test_automation_manual_trigger.py deleted file mode 100644 index 2e0787b8..00000000 --- a/src/backend/tests/test_automation_manual_trigger.py +++ /dev/null @@ -1,77 +0,0 @@ -"""Regression: 「立即执行」必须真的执行。 - -路由原本直接 `asyncio.create_task(scheduler.execute_task(...))`，在生产的多 worker + -同步路由下 100% 失败（503 或 `no running event loop`），而定时触发照常——所以这条路径 -必须单独有测试。改成经 `manual_trigger_at` 交接后，这里盯的就是交接本身。 -""" - -from core.services.automation_service import AutomationService - - -def _task(db, **kw): - svc = AutomationService(db) - return svc, svc.create_task( - user_id="u1", - task_type="prompt", - prompt="跑一次", - cron_expression="30 10 29 6 *", - **kw, - ) - - -def test_request_marks_task_and_drain_takes_it_once(db_session): - svc, task = _task(db_session) - assert task.manual_trigger_at is None - - svc.request_manual_trigger(task) - db_session.refresh(task) - assert task.manual_trigger_at is not None - - assert svc.take_manual_triggers() == [(task.task_id, "u1")] - - # 取走即清空：第二次排空不能把同一次点击再发一遍。 - db_session.refresh(task) - assert task.manual_trigger_at is None - assert svc.take_manual_triggers() == [] - - -def test_repeated_clicks_queue_once(db_session): - svc, task = _task(db_session) - svc.request_manual_trigger(task) - svc.request_manual_trigger(task) - svc.request_manual_trigger(task) - - assert svc.take_manual_triggers() == [(task.task_id, "u1")] - - -def test_paused_task_can_still_be_queued(db_session): - """暂停中的任务允许手动跑一次——状态校验在路由，这里确认交接本身不挑状态。""" - svc, task = _task(db_session) - svc.pause_task(task.task_id, "u1") - - svc.request_manual_trigger(task) - assert svc.take_manual_triggers() == [(task.task_id, "u1")] - - -def test_manual_trigger_does_not_move_the_schedule(db_session): - """手动跑一次不该打乱 cron 排期——next_run_at 由调度器推进，这条路径不碰它。""" - svc, task = _task(db_session) - before = task.next_run_at - - svc.request_manual_trigger(task) - svc.take_manual_triggers() - db_session.refresh(task) - - assert task.next_run_at == before - - -def test_bump_run_count_increments_in_place(db_session): - """排空时补的计数是原地自增，不是读-改-写，免得和 advance_next_run 互相覆盖。""" - svc, task = _task(db_session) - assert (task.run_count or 0) == 0 - - svc.bump_run_count(task.task_id) - svc.bump_run_count(task.task_id) - db_session.refresh(task) - - assert task.run_count == 2 diff --git a/src/backend/tests/test_automation_oneshot_finalize.py b/src/backend/tests/test_automation_oneshot_finalize.py deleted file mode 100644 index 1289beec..00000000 --- a/src/backend/tests/test_automation_oneshot_finalize.py +++ /dev/null @@ -1,109 +0,0 @@ -"""Regression: one-shot scheduled tasks must actually execute (and deliver). - -Bug: the scheduler pre-advances a task (run_count++, schedule moved on) BEFORE -firing execute_task, to avoid a death-spiral on crashed runs. For one-shot -tasks the pre-advance used to set status="completed". But execute_task's first -guard refuses any task whose status isn't active/paused — so the executor -skipped the very run it had just fired: no run record, no last_run_at, and the -channel delivery (Feishu, etc.) at the end never ran. The task *looked* executed -(run_count=1, status=completed) while its body never ran. - -Fix: advance_next_run only clears next_run_at for one-shot tasks (stops -re-firing); terminal "completed" status is set AFTER the run via -finalize_after_run(). -""" - -from core.services.automation_service import AutomationService - - -def _task(db, **kw): - svc = AutomationService(db) - return svc, svc.create_task( - user_id="u1", - task_type="prompt", - prompt="发送一条测试消息", - cron_expression="30 10 29 6 *", - **kw, - ) - - -def test_oneshot_pre_advance_keeps_task_runnable(db_session): - """After pre-advance a one-shot task must stay active (so execute_task's - guard passes) yet have next_run_at cleared (so it is never re-fired).""" - svc, task = _task(db_session, schedule_type="once") - assert task.status == "active" - - svc.advance_next_run(task.task_id) - db_session.refresh(task) - - # The exact pair the old code got wrong: - assert task.status == "active", "pre-advance must NOT mark one-shot completed" - assert task.next_run_at is None, "pre-advance must clear next_run_at" - assert task.run_count == 1 - - # ...and the guard execute_task uses would now let it run. - assert task.status in ("active", "paused") - - # get_due_tasks must NOT re-select it (next_run_at is NULL). - from datetime import datetime, timezone - due = svc.get_due_tasks(datetime.now(timezone.utc)) - assert task.task_id not in {t.task_id for t in due} - - -def test_finalize_marks_oneshot_completed_after_run(db_session): - svc, task = _task(db_session, schedule_type="once") - svc.advance_next_run(task.task_id) - - svc.finalize_after_run(task.task_id) - db_session.refresh(task) - - assert task.status == "completed" - assert task.next_run_at is None - - -def test_finalize_leaves_recurring_active(db_session): - svc, task = _task(db_session, schedule_type="recurring") - svc.advance_next_run(task.task_id) - db_session.refresh(task) - # recurring: schedule moved forward, still active - assert task.status == "active" - assert task.next_run_at is not None - - svc.finalize_after_run(task.task_id) - db_session.refresh(task) - assert task.status == "active", "recurring task must stay active after a run" - assert task.next_run_at is not None - - -def test_finalize_completes_when_max_runs_reached(db_session): - svc, task = _task(db_session, schedule_type="recurring", max_runs=2) - # first run - svc.advance_next_run(task.task_id) - svc.finalize_after_run(task.task_id) - db_session.refresh(task) - assert task.status == "active" and task.run_count == 1 - - # second run hits max_runs - svc.advance_next_run(task.task_id) - svc.finalize_after_run(task.task_id) - db_session.refresh(task) - assert task.run_count == 2 - assert task.status == "completed" - - -def test_finalize_never_revives_disabled(db_session): - svc, task = _task(db_session, schedule_type="once") - svc.advance_next_run(task.task_id) - svc.update_task_system(task.task_id, status="disabled") - - svc.finalize_after_run(task.task_id) - db_session.refresh(task) - assert task.status == "disabled", "auto-disabled task must not be overridden" - - -def test_manual_task_never_finalized(db_session): - svc, task = _task(db_session, schedule_type="manual") - svc.advance_next_run(task.task_id) - svc.finalize_after_run(task.task_id) - db_session.refresh(task) - assert task.status == "active", "manual tasks stay active across runs" diff --git a/src/backend/tests/test_browser_prewarm.py b/src/backend/tests/test_browser_prewarm.py index e45772f8..7c623051 100644 --- a/src/backend/tests/test_browser_prewarm.py +++ b/src/backend/tests/test_browser_prewarm.py @@ -5,7 +5,7 @@ import pytest import httpx from core.db.models import ChatSession -from core.plugins.resources import installation, service +from core.capabilities.plugins.resources import installation, service import core.sandbox.processes.interactive as interactive from core.sandbox import factory @@ -74,9 +74,9 @@ async def slow_launch(*args): assert len(launches) == 1 async def test_router_starts_warm_worker_in_background_and_close_reclaims_it(runtime, monkeypatch): - from core.plugins.resources import prewarm + from core.capabilities.plugins.resources import prewarm from core.db.models import InstalledPlugin - from core.capabilities import device_catalog + from features.desktop.capabilities import device_catalog from core.sandbox.sessions.router import SessionSandboxRouter from sqlalchemy.orm import sessionmaker db, launches, selected, provider = runtime @@ -105,7 +105,7 @@ async def close(*args): for _ in range(100): await asyncio.sleep(0.01) with prewarm.engine.SessionLocal() as check: - from core.plugins.resources import store + from core.capabilities.plugins.resources import store rows = check.query(store.PluginResource).filter_by(status="warm").all() if rows: resource_id = rows[0].resource_id @@ -136,8 +136,8 @@ async def test_warm_is_private_and_counts_against_quota(runtime, monkeypatch): assert active["resource_id"] == warm["resource_id"] async def test_checkpoint_opens_its_own_worker_and_never_consumes_blank_warm(runtime): - from core.plugins.resources import store - from core.plugins.resources.crypto import encrypt_secret + from core.capabilities.plugins.resources import store + from core.capabilities.plugins.resources.crypto import encrypt_secret db, launches, selected, _ = runtime warm = await service.create(db, "browser", "browser", "warm-user", "warm-chat", _prewarm=True) db.add(store.PluginResourceCheckpoint(checkpoint_id="saved", user_id="warm-user", @@ -149,7 +149,7 @@ async def test_checkpoint_opens_its_own_worker_and_never_consumes_blank_warm(run @pytest.mark.parametrize("change", ["expired", "sandbox", "revision"]) async def test_invalid_warm_is_never_reused(runtime, monkeypatch, change): - from core.plugins.resources import store + from core.capabilities.plugins.resources import store db, launches, selected, provider = runtime warm = await service.create(db, "browser", "browser", "warm-user", "warm-chat", _prewarm=True) if change == "expired": @@ -186,7 +186,7 @@ async def write(*args, **kwargs): assert cancelled == ["\x03"] async def test_initial_managed_start_cancellation_retains_handle_until_cleanup(tmp_path, monkeypatch): - from core.plugins.resources import confinement + from core.capabilities.plugins.resources import confinement runtime = tmp_path / "runtime" runtime.mkdir() (runtime / "worker.py").write_text("def main(config): pass") @@ -213,12 +213,12 @@ async def write(handle, **kwargs): assert cleaned == [("owned-start", "\x03")] async def test_close_fence_blocks_readmission_until_provider_close_finishes(runtime, monkeypatch): - from core.plugins.resources import prewarm + from core.capabilities.plugins.resources import prewarm from core.sandbox.sessions.router import SessionSandboxRouter from sqlalchemy.orm import sessionmaker db, launches, selected, provider = runtime from core.db.models import InstalledPlugin - from core.capabilities import device_catalog + from features.desktop.capabilities import device_catalog db.add(InstalledPlugin(install_id=selected.install_id, slug=selected.slug, name="Browser", owner_user_id="warm-user")) db.commit() monkeypatch.setattr(device_catalog, "active", lambda: False) @@ -278,7 +278,7 @@ def test_background_binding_accepts_its_live_sandbox(): require_available("bound") async def test_failed_warm_shutdown_reports_sandbox_dirty(runtime, monkeypatch): - from core.plugins.resources import prewarm + from core.capabilities.plugins.resources import prewarm from sqlalchemy.orm import sessionmaker db, _, _, _ = runtime monkeypatch.setattr(prewarm.engine, "SessionLocal", sessionmaker(bind=db.bind)) diff --git a/src/backend/tests/test_browser_remediation.py b/src/backend/tests/test_browser_remediation.py index d4aa7866..959903b0 100644 --- a/src/backend/tests/test_browser_remediation.py +++ b/src/backend/tests/test_browser_remediation.py @@ -26,7 +26,7 @@ async def dns_failure(url): @pytest.mark.asyncio async def test_non_json_runtime_error_is_safe(): - from core.plugins.resources.transport import RuntimeTransport + from core.capabilities.plugins.resources.transport import RuntimeTransport async with RuntimeTransport({"url": "http://probe", "headers": {}}) as transport: await transport.client.aclose() transport.client = httpx.AsyncClient(transport=httpx.MockTransport(lambda _: httpx.Response(500, text="private details"))) @@ -71,7 +71,7 @@ async def call(*args): async def test_chat_recovers_only_live_owned_resources(runtime, monkeypatch): from api.routes.v1.plugin_resources import chat_resources from types import SimpleNamespace - from core.plugins.resources import service + from core.capabilities.plugins.resources import service db, _, _, _ = runtime await service.create(db, "browser", "browser", "warm-user", "warm-chat") result = await chat_resources("warm-chat", SimpleNamespace(user_id="warm-user"), db) diff --git a/src/backend/tests/test_browser_resource_assets.py b/src/backend/tests/test_browser_resource_assets.py index 70a27423..8e130dc2 100644 --- a/src/backend/tests/test_browser_resource_assets.py +++ b/src/backend/tests/test_browser_resource_assets.py @@ -7,13 +7,13 @@ from sqlalchemy.orm import Session from core.db.engine import get_db from core.db.models.plugin_resource import PluginResource, PluginResourceAssetTicket -from core.plugins.resources import store +from core.capabilities.plugins.resources import store async def test_opaque_iframe_assets_use_revocable_ticket_without_session_cookie(monkeypatch, tmp_path): from api.routes.v1 import plugin_resources from core.auth import session from core.config.settings import settings - from core.plugins.resources import service, installation + from core.capabilities.plugins.resources import service, installation monkeypatch.setattr(session, "_use_memory_store", lambda: True) token = await session.create_session({"user_id": "owner", "user_center_id": "owner", "username": "test"}) engine = create_engine("sqlite://") diff --git a/src/backend/tests/test_browser_resource_security.py b/src/backend/tests/test_browser_resource_security.py index 26cdee3b..ae4031e2 100644 --- a/src/backend/tests/test_browser_resource_security.py +++ b/src/backend/tests/test_browser_resource_security.py @@ -8,8 +8,8 @@ from alembic.migration import MigrationContext from alembic.operations import Operations from core.db.models.plugin_resource import PluginResource, PluginResourceTicket, PluginResourceAssetTicket -from core.plugins.resources import store -from core.llm import mcp_invocation as proof +from core.capabilities.plugins.resources import store +from core.llm.mcp import invocation as proof def test_invocation_proof_rejects_impersonation_expiry_and_wrong_audience(monkeypatch): monkeypatch.setenv("BACKEND_INTERNAL_TOKEN", "isolated-test-signing-key") @@ -68,7 +68,7 @@ def test_migration_upgrade_and_downgrade_match_resource_tables(): async def test_viewer_authentication_is_revoked_with_real_session_store(monkeypatch): from core.auth import session from core.config.settings import settings - from core.plugins.resources import authentication + from core.capabilities.plugins.resources import authentication monkeypatch.setattr(session, "_use_memory_store", lambda: True) token = await session.create_session({"user_id": "owner", "user_center_id": "owner", "username": "test"}) context = {"headers": {"cookie": settings.session.cookie_name + "=" + token}, "path": "/api/v1/plugin-resources/resource/attach", "method": "POST"} @@ -80,7 +80,7 @@ async def test_viewer_authentication_is_revoked_with_real_session_store(monkeypa assert error.value.status_code == 401 async def test_expired_starting_reservation_is_cleaned_without_endpoint(monkeypatch): - from core.plugins.resources import lifecycle + from core.capabilities.plugins.resources import lifecycle engine = create_engine("sqlite://") for model in (PluginResource, PluginResourceTicket, PluginResourceAssetTicket): model.__table__.create(engine) @@ -118,15 +118,19 @@ async def test_opensandbox_endpoint_auth_is_confined_to_control_plane_origin(): assert headers == {"X-Endpoint-Route": "public"} -def test_opensandbox_template_allows_the_actual_builtin_skill_mount(monkeypatch, tmp_path): +def test_opensandbox_template_allows_the_actual_skill_mount(monkeypatch, tmp_path): import tomllib from pathlib import Path - import importlib import core.sandbox.providers.opensandbox.volumes as volumes + from core.capabilities.skills import config, publication root = Path(__file__).resolve().parents[3] monkeypatch.setenv("HOST_REPO_PATH", str(root)) - monkeypatch.setattr(volumes, "_BUILTIN_SKILLS_HOST_PATH", None) - source = Path(volumes._resolve_builtin_skills_host_path()) + from types import SimpleNamespace + monkeypatch.setattr(volumes, "settings", SimpleNamespace(sandbox=SimpleNamespace( + opensandbox_host_storage_path=str(tmp_path)))) + monkeypatch.setattr(config, "get_sandbox_skills_dir", lambda: tmp_path / "sandbox_skills") + monkeypatch.setattr(publication, "prepare_skill_view", lambda user_id: None) + source = Path(volumes._make_skills_volumes()[0].host.path) rendered = (root / "docker/opensandbox-config.toml.tpl").read_text().replace( "@@HOST_REPO_PATH@@", str(root)).replace("@@HOST_STORAGE_PATH@@", str(tmp_path)) allowed = tomllib.loads(rendered)["storage"]["allowed_host_paths"] diff --git a/src/backend/tests/test_browser_resource_transport.py b/src/backend/tests/test_browser_resource_transport.py index 5d9fc4b7..2f6fa3c9 100644 --- a/src/backend/tests/test_browser_resource_transport.py +++ b/src/backend/tests/test_browser_resource_transport.py @@ -6,8 +6,8 @@ from fastapi import HTTPException import pytest -from core.plugins.resources import service -from core.plugins.resources.transport import RuntimeTransport +from core.capabilities.plugins.resources import service +from core.capabilities.plugins.resources.transport import RuntimeTransport async def test_commands_reuse_one_tcp_connection_and_keep_runtime_authentication(): @@ -61,7 +61,7 @@ async def serve(reader, writer): async def test_websocket_revocation_stops_commands_and_closes_transport(monkeypatch): from starlette.websockets import WebSocketDisconnect - from core.plugins.resources import stream, authentication + from core.capabilities.plugins.resources import stream, authentication from contextlib import nullcontext bodies, writers, handlers, transports = [], [], set(), [] diff --git a/src/backend/tests/test_ce_agent_visibility.py b/src/backend/tests/test_ce_agent_visibility.py index bcde6090..64bda4e2 100644 --- a/src/backend/tests/test_ce_agent_visibility.py +++ b/src/backend/tests/test_ce_agent_visibility.py @@ -1,15 +1,15 @@ """CE sub-agent visibility regression tests.""" -from core.db.edition_tables import ce_create_all +from core.db.edition import ce_create_all from core.db.models import UserAgent, UserShadow -from core.services.user_agent_service import UserAgentService +from core.capabilities.agents.edition import UserAgentService from sqlalchemy import create_engine, inspect from sqlalchemy.orm import sessionmaker def test_ce_list_for_user_has_no_organization_repository_contracts(): """A fresh CE database and repository expose only personal/admin agents.""" - from core.db.repository.agent import UserAgentRepository + from core.db.edition import UserAgentRepository engine = create_engine("sqlite:///:memory:") ce_create_all(engine) diff --git a/src/backend/tests/test_ce_runtime_contracts.py b/src/backend/tests/test_ce_runtime_contracts.py index 7bd2bf8f..45c1a507 100644 --- a/src/backend/tests/test_ce_runtime_contracts.py +++ b/src/backend/tests/test_ce_runtime_contracts.py @@ -9,7 +9,7 @@ from api.routes.v1.auth import router as auth_router from api.routes.v1.users import router as users_router from cli import DEFAULT_LOCAL_CONTEXT_LENGTH, build_parser, configure_model -from core.db import model_repository +from core.models import repository as model_repository def test_ce_has_no_commercial_license_package(): @@ -25,7 +25,7 @@ def test_ce_has_no_enterprise_modules_or_model_exports(): for module_name in ( "edition_ee", - "core.kb.dify_kb", + "features.knowledge.dify_kb", "core.auth.team_permissions", "core.services.team_service", "core.db.repository.team", @@ -118,7 +118,7 @@ def test_ce_user_router_exposes_self_service_password_change(): def test_dingtalk_status_env_does_not_import_ee_sandbox_module(monkeypatch, tmp_path): import core.sandbox.providers.account_env as _common_account_env import core.sandbox.workspace.credentials as _common_credentials - from core.services.dingtalk_service import _dws_env + from integrations.dingtalk.service import _dws_env monkeypatch.setattr(_common_credentials, "dws_home_dir", lambda _user_id: tmp_path / "dws-home") monkeypatch.setattr( @@ -134,10 +134,10 @@ def test_dingtalk_status_env_does_not_import_ee_sandbox_module(monkeypatch, tmp_ def test_ce_ignores_stale_database_query_builtin(db_session, monkeypatch): - from core.config.catalog_runtime import get_runtime_catalog, invalidate_runtime_catalog_cache + from core.capabilities.catalog.runtime import get_runtime_catalog, invalidate_runtime_catalog_cache from core.db.models import AdminMcpServer - from core.services import mcp_service - from core.services.mcp_service import McpServerConfigService + from core.capabilities.mcp import service as mcp_service + from core.capabilities.mcp.service import McpServerConfigService from extensions.mcp_servers._ports import PORTS assert "query_database" not in PORTS @@ -196,7 +196,7 @@ def test_ce_local_launcher_covers_default_plugin_mcp_servers(): def test_ce_fresh_database_bootstraps_default_plugins(db_session): from core.db.models import AdminMcpServer, ContentBlock, InstalledPlugin - from core.services import plugin_service + from core.capabilities.plugins import service as plugin_service assert plugin_service.ensure_default_plugins_bootstrapped(db_session) is True installs = db_session.query(InstalledPlugin).filter(InstalledPlugin.owner_user_id.is_(None)) @@ -217,7 +217,7 @@ def test_ce_fresh_database_bootstraps_default_plugins(db_session): def test_ce_launcher_and_catalog_cover_core_mcp_servers(): - from core.services.mcp_service import BUILTIN_MCP_SERVERS + from core.capabilities.mcp.service import BUILTIN_MCP_SERVERS from extensions.mcp_servers._launcher import PORTS as LAUNCHER_PORTS from extensions.mcp_servers._ports import PORTS, package_name @@ -238,7 +238,7 @@ def test_ce_launcher_and_catalog_cover_core_mcp_servers(): def test_ce_fresh_database_seeds_core_mcp_servers_enabled(db_session): from core.db.models import AdminMcpServer - from core.services.mcp_service import seed_builtin_mcp_servers_if_empty + from core.capabilities.mcp.service import seed_builtin_mcp_servers_if_empty expected = { "retrieve_dataset_content", @@ -271,7 +271,7 @@ def test_serve_cli_accepts_explicit_public_bind(): def test_configure_chat_model_persists_context_window(monkeypatch): from core.db import engine - from core.services.model_config import ModelConfigService + from core.models.config import ModelConfigService class FakeDb: closed = False @@ -368,7 +368,7 @@ def test_ce_opensandbox_provider_is_shared_runtime(monkeypatch): from core.sandbox.providers.edition import PersistentSandboxProvider from core.sandbox.providers.opensandbox.provider import OpenSandboxProvider - from core.spaces.sandbox import SpaceSandboxMixin + from core.files.spaces.sandbox import SpaceSandboxMixin assert issubclass(PersistentSandboxProvider, OpenSandboxProvider) assert issubclass(PersistentSandboxProvider, SpaceSandboxMixin) @@ -404,7 +404,7 @@ def _personal_name_migration(): def test_ce_personal_name_migration_after_current_baseline(monkeypatch): import sqlalchemy as sa - from core.db.edition_tables import ce_create_all + from core.db.edition import ce_create_all from core.db.personal_file_names import INDEX_NAMES migration = _personal_name_migration() diff --git a/src/backend/tests/test_channel_bots.py b/src/backend/tests/test_channel_bots.py deleted file mode 100644 index 1c606726..00000000 --- a/src/backend/tests/test_channel_bots.py +++ /dev/null @@ -1,2109 +0,0 @@ -"""Unit tests for inbound channel bots (owner service-account model). - -Covers: capability flags, repository CRUD, service capability gating + token -lock, resource-whitelist normalization, and the Lark adapter's event -normalization / @-mention stripping / AES decryption / session keying. - -No network calls: create_bot's credential validation needs HTTP, so only -the pre-network branches are tested (capability gating / token lock). -""" - -import asyncio -import base64 -import hashlib -import json -import re - -import pytest - -from core.auth.capabilities import BOOL_CAPABILITY_DEFAULTS, resolve_capabilities -from core.channels.adapters.lark import LarkAdapter -from core.channels.protocol import InboundMsg, SendResult -from core.db.models import ChannelConnection, UserShadow -from core.db.repository.channel import ChannelConnectionRepository -from core.infra.exceptions import AccessDeniedError, BadRequestError -from core.services.channel_service import ChannelService, _clean_scope, bot_to_dict - - -# ── Capability flags ───────────────────────────────────────────────────── -def test_capability_bit_registered_default_false(): - assert "can_create_channel_bot" in BOOL_CAPABILITY_DEFAULTS - assert BOOL_CAPABILITY_DEFAULTS["can_create_channel_bot"] is False - - -def test_capability_personal_override_wins(): - caps = resolve_capabilities({"can_create_channel_bot": True}, {}) - assert caps["can_create_channel_bot"] is True - caps2 = resolve_capabilities({}, {"can_create_channel_bot": True}) # team default - assert caps2["can_create_channel_bot"] is True - - -# ── repository ────────────────────────────────────────────────────────── -def _mk_user(db, uid="u1", meta=None): - db.add(UserShadow(user_id=uid, username=uid, extra_data=meta or {})) - db.commit() - - -def test_repo_crud(db_session): - _mk_user(db_session) - repo = ChannelConnectionRepository(db_session) - conn = repo.create({ - "channel_id": "chan_a", "owner_user_id": "u1", "channel_type": "lark", - "app_id": "cli_x", "config": {"app_secret_enc": "x"}, "status": "pending", - }) - assert repo.get_by_id("chan_a") is conn - assert repo.get_by_app_id("lark", "cli_x").channel_id == "chan_a" - assert [c.channel_id for c in repo.list_by_owner("u1")] == ["chan_a"] - - repo.set_status("chan_a", "connected") - assert repo.get_by_id("chan_a").status == "connected" - repo.set_status("chan_a", "error", last_error="boom") - assert repo.get_by_id("chan_a").last_error == "boom" - - assert repo.delete("chan_a") is True - assert repo.get_by_id("chan_a") is None - - -# ── Resource-whitelist normalization ───────────────────────────────────── -def test_clean_scope(): - assert _clean_scope(None) is None - assert _clean_scope({}) is None - assert _clean_scope({"kb_ids": []}) is None - assert _clean_scope({"kb_ids": ["k1"], "junk": 1}) == {"kb_ids": ["k1"]} - assert _clean_scope({"skill_ids": [1, 2]}) == {"skill_ids": ["1", "2"]} - - -# ── service: capability gating + token lock (pre-network branches) ─────── -def test_create_bot_denied_without_capability(db_session): - _mk_user(db_session, "u_deny", meta={}) - svc = ChannelService(db_session) - with pytest.raises(AccessDeniedError): - asyncio.run(svc.create_bot( - "u_deny", channel_type="lark", app_id="cli_a", app_secret="s", - )) - - -def test_create_bot_token_lock(db_session): - _mk_user(db_session, "u_ok", meta={"can_create_channel_bot": True}) - # Pre-occupy the same app - ChannelConnectionRepository(db_session).create({ - "channel_id": "chan_pre", "owner_user_id": "u_ok", "channel_type": "lark", - "app_id": "cli_dup", "config": {}, "status": "connected", - }) - svc = ChannelService(db_session) - with pytest.raises(BadRequestError): - asyncio.run(svc.create_bot( - "u_ok", channel_type="lark", app_id="cli_dup", app_secret="s", - )) - - -def test_create_bot_rejects_unknown_channel(db_session): - _mk_user(db_session, "u_ch", meta={"can_create_channel_bot": True}) - svc = ChannelService(db_session) - with pytest.raises(BadRequestError): - asyncio.run(svc.create_bot( - "u_ch", channel_type="telegram", app_id="a", app_secret="s", - )) - - -def test_bot_to_dict_hides_secrets(db_session): - _mk_user(db_session, "u_d") - conn = ChannelConnectionRepository(db_session).create({ - "channel_id": "chan_d", "owner_user_id": "u_d", "channel_type": "lark", - "app_id": "cli_d", "config": {"app_secret_enc": "SECRET"}, "status": "connected", - }) - d = bot_to_dict(conn) - assert "config" not in d and "app_secret_enc" not in json.dumps(d) - assert d["app_id"] == "cli_d" and d["status"] == "connected" - assert d["agent_id"] is None # defaults to the main agent - - -# ── Sub-agent binding (agent_id) ────────────────────────────────────────── -def test_repo_list_filters_by_agent(db_session): - _mk_user(db_session, "u_f") - repo = ChannelConnectionRepository(db_session) - repo.create({ - "channel_id": "chan_main", "owner_user_id": "u_f", "channel_type": "lark", - "app_id": "cli_main", "config": {}, "status": "connected", "agent_id": None, - }) - repo.create({ - "channel_id": "chan_ag", "owner_user_id": "u_f", "channel_type": "lark", - "app_id": "cli_ag", "config": {}, "status": "connected", "agent_id": "ag_1", - }) - assert {c.channel_id for c in repo.list_by_owner("u_f")} == {"chan_main", "chan_ag"} - assert [c.channel_id for c in repo.list_by_owner("u_f", main_only=True)] == ["chan_main"] - assert [c.channel_id for c in repo.list_by_owner("u_f", agent_id="ag_1")] == ["chan_ag"] - - -def test_create_bot_rejects_unknown_agent(db_session): - _mk_user(db_session, "u_ag", meta={"can_create_channel_bot": True}) - svc = ChannelService(db_session) - # agent_id pointing to a nonexistent/inaccessible sub-agent → rejected before any network call - with pytest.raises(BadRequestError): - asyncio.run(svc.create_bot( - "u_ag", channel_type="lark", app_id="cli_ag2", app_secret="s", - agent_id="nope_agent", - )) - - -def test_inbound_resource_scope_ignored_marker_for_agent(db_session): - """A conn bound to a sub-agent: agent_id lands on the row, for inbound to set context['agent_id'] pinning that sub-agent. - - (Inbound's context assembly goes through the global SessionLocal; the - unit test only asserts at the row level that the binding is persisted; - the pinning branch `if conn.agent_id: context['agent_id']=conn.agent_id` - is a direct attribute read.) - """ - conn = _mk_conn(db_session) - assert conn.agent_id is None # _mk_conn defaults to the main agent - conn.agent_id = "ag_pinned" - db_session.commit() - reloaded = ChannelConnectionRepository(db_session).get_by_id("chan_in") - assert reloaded.agent_id == "ag_pinned" - - -# ── Lark adapter ────────────────────────────────────────────────────────── -class _FakeConn: - channel_id = "chan_lark" - channel_type = "lark" - app_id = "cli_z" - config = {} - - -def test_lark_extract_text_strips_mention(): - assert LarkAdapter._extract_text(json.dumps({"text": "@_user_1 你好"})) == "你好" - assert LarkAdapter._extract_text(json.dumps({"text": "纯文本"})) == "纯文本" - assert LarkAdapter._extract_text(None) == "" - - -def test_lark_parse_inbound_group(): - adapter = LarkAdapter() - payload = { - "header": {"event_type": "im.message.receive_v1"}, - "event": { - "message": { - "chat_id": "oc_group1", "chat_type": "group", - "message_id": "om_1", "message_type": "text", - "content": json.dumps({"text": "@_user_1 汇报一下"}), - }, - "sender": {"sender_id": {"open_id": "ou_alice"}}, - }, - } - msg = adapter.parse_inbound(_FakeConn(), payload) - assert isinstance(msg, InboundMsg) - assert msg.text == "汇报一下" - assert msg.chat_type == "group" - assert msg.external_conversation_id == "oc_group1" # groups are keyed by chat_id - assert msg.sender_id == "ou_alice" - assert msg.raw["lark_chat_id"] == "oc_group1" - - -def test_lark_parse_inbound_skips_non_message(): - adapter = LarkAdapter() - assert adapter.parse_inbound(_FakeConn(), {"type": "url_verification"}) is None - # Non-text messages are skipped - payload = { - "header": {"event_type": "im.message.receive_v1"}, - "event": {"message": {"message_type": "image", "chat_id": "x"}, "sender": {}}, - } - assert adapter.parse_inbound(_FakeConn(), payload) is None - - -def test_lark_aes_decrypt_roundtrip(): - from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes - - encrypt_key = "my-encrypt-key" - plain = json.dumps({"type": "url_verification", "challenge": "abc"}).encode("utf-8") - key = hashlib.sha256(encrypt_key.encode()).digest() - iv = b"\x00" * 16 - pad = 16 - (len(plain) % 16) - padded = plain + bytes([pad]) * pad - enc = Cipher(algorithms.AES(key), modes.CBC(iv)).encryptor() - ct = enc.update(padded) + enc.finalize() - token = base64.b64encode(iv + ct).decode() - - out = LarkAdapter._aes_decrypt(token, encrypt_key) - assert out["challenge"] == "abc" - - -def test_chunk_text(): - from core.channels.protocol import chunk_text - assert chunk_text("", 100) == [] - assert chunk_text("hello", 100) == ["hello"] - # Over-length → multiple chunks, each within the limit - parts = chunk_text("x" * 950, 200) - assert len(parts) == 5 and all(len(p) <= 200 for p in parts) - # Prefer breaking at a newline - parts2 = chunk_text("a" * 150 + "\n" + "b" * 50, 200) - assert parts2[0] == "a" * 150 # cut at the newline, without the b's - - -def test_speaker_label(): - from core.channels.inbound import _speaker_label - from core.channels.protocol import InboundMsg - m = InboundMsg(channel_id="c", channel_type="lark", text="hi", chat_type="group", - external_conversation_id="oc", sender_id="ou_xyz789", sender_name="张三") - assert _speaker_label(m) == "张三" - m2 = InboundMsg(channel_id="c", channel_type="lark", text="hi", chat_type="group", - external_conversation_id="oc", sender_id="ou_xyz789") - assert _speaker_label(m2) == "xyz789" # no name → tail segment of the open_id - - -def test_conv_lock_serializes_same_conversation(): - """#1: same-conversation locks serialize; different conversations run concurrently.""" - import asyncio - from core.channels import inbound as inb - - order = [] - - async def _fake(msg, tag, delay): - async with inb._conv_locks[inb._conv_key(msg)]: - order.append(f"{tag}-start") - await asyncio.sleep(delay) - order.append(f"{tag}-end") - - def _msg(conv): - from core.channels.protocol import InboundMsg - return InboundMsg(channel_id="c", channel_type="lark", text="x", chat_type="group", - external_conversation_id=conv) - - async def _run(): - same1, same2 = _msg("A"), _msg("A") - await asyncio.gather(_fake(same1, "a1", 0.05), _fake(same2, "a2", 0.01)) - - asyncio.run(_run()) - # Same conversation: a1 must fully finish before a2 starts (serial) - assert order == ["a1-start", "a1-end", "a2-start", "a2-end"] - - -def test_outbound_synthetic_msg_targets_conversation(): - """#7: the proactively delivered placeholder message lets the adapter locate the target chat_id.""" - from core.channels.outbound import _synthetic_msg - from core.channels.adapters.lark import LarkAdapter - m = _synthetic_msg("chan_x", "lark", "oc_group9") - assert m.external_conversation_id == "oc_group9" - # The adapter uses raw.lark_chat_id to locate the send target - assert LarkAdapter()._chat_id(m) == "oc_group9" - - -def test_send_result_helpers(): - ok = SendResult.ok("m1") - assert ok.success and ok.message_id == "m1" - bad = SendResult.fail("nonsense") - assert not bad.success and bad.error_kind == "unknown" - rl = SendResult.fail("rate_limited", "too fast") - assert rl.error_kind == "rate_limited" - - -# ── inbound session keying (multi-tenant core: p2p one per person / group one per group) ── -def _mk_conn(db, owner="u_o", scope=None): - _mk_user(db, owner, meta={"can_create_channel_bot": True}) - return ChannelConnectionRepository(db).create({ - "channel_id": "chan_in", "owner_user_id": owner, "channel_type": "lark", - "app_id": "cli_in", "config": {}, "status": "connected", "resource_scope": scope, - }) - - -def _inbound(conv_id, chat_type, text="hi", mid="m1"): - return InboundMsg( - channel_id="chan_in", channel_type="lark", text=text, chat_type=chat_type, - external_conversation_id=conv_id, sender_id="ou_x", message_id=mid, - ) - - -def test_inbound_session_keying(db_session): - from core.channels.inbound import _find_or_create_session - conn = _mk_conn(db_session) - - # Two messages in the same conversation → reuse the same chat_id, held by the owner - s1 = _find_or_create_session(db_session, conn, _inbound("oc_g1", "group")) - s2 = _find_or_create_session(db_session, conn, _inbound("oc_g1", "group", mid="m2")) - assert s1.chat_id == s2.chat_id - assert s1.user_id == "u_o" - assert s1.channel_id == "chan_in" and s1.external_conversation_id == "oc_g1" - - # A different conversation (another person's DM / another group) → separate chat_id - s3 = _find_or_create_session(db_session, conn, _inbound("oc_p2", "p2p")) - assert s3.chat_id != s1.chat_id - - -def test_lark_parse_file_and_image_messages(): - adapter = LarkAdapter() - base = {"header": {"event_type": "im.message.receive_v1"}, "event": { - "message": {"chat_id": "oc_1", "chat_type": "p2p", "message_id": "om_9"}, - "sender": {"sender_id": {"open_id": "ou_a"}}}} - # file message - import copy - fmsg = copy.deepcopy(base) - fmsg["event"]["message"]["message_type"] = "file" - fmsg["event"]["message"]["content"] = json.dumps({"file_key": "fk_1", "file_name": "数据.xlsx"}) - m = adapter.parse_inbound(_FakeConn(), fmsg) - assert m.text == "" and m.attachments == [{"kind": "file", "key": "fk_1", "name": "数据.xlsx"}] - assert m.raw["lark_message_id"] == "om_9" - # image message - imsg = copy.deepcopy(base) - imsg["event"]["message"]["message_type"] = "image" - imsg["event"]["message"]["content"] = json.dumps({"image_key": "ik_2"}) - m2 = adapter.parse_inbound(_FakeConn(), imsg) - assert m2.attachments[0]["kind"] == "image" and m2.attachments[0]["key"] == "ik_2" - - -def test_lark_parse_post_extracts_text_and_images(): - adapter = LarkAdapter() - post = {"header": {"event_type": "im.message.receive_v1"}, "event": { - "message": {"chat_id": "oc_1", "chat_type": "group", "message_id": "om_p", - "message_type": "post", - "content": json.dumps({"title": "周报", "content": [ - [{"tag": "text", "text": "完成了"}, {"tag": "img", "image_key": "ik_x"}], - ]})}, - "sender": {"sender_id": {"open_id": "ou_a"}}}} - m = adapter.parse_inbound(_FakeConn(), post) - assert "周报" in m.text and "完成了" in m.text - assert any(a["key"] == "ik_x" for a in m.attachments) - - -def test_inbound_ingest_attachments_stores_artifact(db_session): - import asyncio - from core.channels import inbound as inb - from core.db.models import Artifact - - conn = _mk_conn(db_session) - - class _Adapter: - async def download_resource(self, conn, msg, att): - return b"hello world bytes" - - # mock storage (avoid real disk writes) + parse_file - class _Storage: - def upload_bytes(self, content, key): - return f"/local/{key}" - import core.storage as cs - orig_get = cs.get_storage - cs.get_storage = lambda: _Storage() - try: - msg = _inbound("oc_1", "p2p", text="", mid="mf1") - msg.attachments = [{"kind": "file", "key": "fk_1", "name": "note.txt"}] - files = asyncio.run(inb._ingest_attachments(db_session, _Adapter(), conn, "u_o", "chat_x", msg)) - finally: - cs.get_storage = orig_get - - assert len(files) == 1 - f = files[0] - assert f["name"] == "note.txt" and f["file_id"].startswith("ua_") - assert f["download_url"] == f"/files/{f['file_id']}" - # Artifact persisted to DB - row = db_session.query(Artifact).filter(Artifact.artifact_id == f["file_id"]).first() - assert row is not None and row.user_id == "u_o" and row.chat_id == "chat_x" - assert row.extra_data["source"] == "channel_upload" - assert row.size_bytes == len(b"hello world bytes") - - -def test_inbound_resource_scope_narrows(db_session): - from core.channels.inbound import _resolve_enabled - # No whitelist → enabled_kbs=None (everything), enabled_skills resolved via the owner - conn = _mk_conn(db_session) - out = _resolve_enabled(db_session, conn, "u_o") - assert out["enabled_kbs"] is None - - # With a whitelist → narrowed to the specified kb/skill - conn.resource_scope = {"kb_ids": ["k1", "k2"], "skill_ids": ["sk1"]} - db_session.commit() - out2 = _resolve_enabled(db_session, conn, "u_o") - assert out2["enabled_kbs"] == ["k1", "k2"] - assert out2["enabled_skills"] == ["sk1"] - - -# ── DingTalk adapter ───────────────────────────────────────────────────── -class _FakeDTConn: - channel_id = "chan_dt" - channel_type = "dingtalk" - app_id = "dingkey" - config = {} - - -def test_dingtalk_parse_inbound_group_and_session_webhook(): - from core.channels.adapters.dingtalk import DingTalkAdapter - - adapter = DingTalkAdapter() - payload = { - "msgtype": "text", - "text": {"content": " 汇报一下 "}, - "conversationType": "2", - "conversationId": "cidGROUP", - "senderStaffId": "staff_a", - "senderNick": "Alice", - "msgId": "msg_1", - "sessionWebhook": "https://oapi.dingtalk.com/robot/send?session=xyz", - } - msg = adapter.parse_inbound(_FakeDTConn(), payload) - assert isinstance(msg, InboundMsg) - assert msg.text == "汇报一下" - assert msg.chat_type == "group" - assert msg.external_conversation_id == "cidGROUP" - assert msg.sender_name == "Alice" - assert msg.raw["dingtalk_session_webhook"].endswith("session=xyz") - - -def test_dingtalk_parse_inbound_p2p_and_skips_non_text(): - from core.channels.adapters.dingtalk import DingTalkAdapter - - adapter = DingTalkAdapter() - p2p = adapter.parse_inbound(_FakeDTConn(), { - "msgtype": "text", "text": {"content": "hi"}, "conversationType": "1", - "conversationId": "cidP2P", "msgId": "m2", - }) - assert p2p.chat_type == "p2p" - # Images and other non-text → None (v1 is text only) - assert adapter.parse_inbound(_FakeDTConn(), {"msgtype": "picture"}) is None - - -def test_dingtalk_edit_message_unsupported(): - from core.channels.adapters.dingtalk import DingTalkAdapter - - r = asyncio.run(DingTalkAdapter().edit_message(_FakeDTConn(), "mid", "x")) - assert r.success is False and r.error_kind == "bad_format" - - -def test_dingtalk_has_push_file(): - # Inbound orchestration decides whether to return generated files by hasattr(adapter, "push_file") — if missing, it is silently skipped - from core.channels.adapters.dingtalk import DingTalkAdapter - - assert callable(getattr(DingTalkAdapter(), "push_file", None)) - - -def test_dingtalk_robot_send_requires_target(): - from core.channels.adapters.dingtalk import DingTalkAdapter - - adapter = DingTalkAdapter() - group = InboundMsg(channel_id="chan_dt", channel_type="dingtalk", text="", - chat_type="group", external_conversation_id="") - r = asyncio.run(adapter._robot_send("tok", _FakeDTConn(), group, "sampleFile", {})) - assert r.success is False and r.error_kind == "bad_format" - - # p2p with neither staffId nor openConversationId → bad_format - # (with an openConversationId it goes through privateChatMessages/send, a network call, not covered by unit tests) - p2p = InboundMsg(channel_id="chan_dt", channel_type="dingtalk", text="", - chat_type="p2p", external_conversation_id="", sender_id="") - r2 = asyncio.run(adapter._robot_send("tok", _FakeDTConn(), p2p, "sampleFile", {})) - assert r2.success is False and r2.error_kind == "bad_format" - - -def test_dingtalk_caps_supports_markdown(): - from core.channels.adapters.dingtalk import DingTalkAdapter - - assert DingTalkAdapter.caps.supports_markdown is True - - -def test_dingtalk_send_markdown_via_session_webhook(monkeypatch): - """Reply scenario (with sessionWebhook): msgtype=markdown + title extraction + table downgrade.""" - from core.channels.adapters.dingtalk import DingTalkAdapter - - adapter = DingTalkAdapter() - captured = {} - - async def fake_post(webhook, payload): - captured.update(webhook=webhook, payload=payload) - return SendResult.ok() - - monkeypatch.setattr(adapter, "_post_webhook", fake_post) - msg = InboundMsg( - channel_id="chan_dt", channel_type="dingtalk", text="", chat_type="p2p", - external_conversation_id="cid1", - raw={"dingtalk_session_webhook": "https://oapi.dingtalk.com/robot/send?s=1"}, - ) - md = "# 日报\n\n| 指标 | 值 |\n| --- | --- |\n| 完成率 | 98% |" - r = asyncio.run(adapter.send_markdown(_FakeDTConn(), msg, md)) - assert r.success - assert captured["payload"]["msgtype"] == "markdown" - body = captured["payload"]["markdown"] - assert body["title"] == "日报" - assert "- 指标: 完成率｜值: 98%" in body["text"] # table downgraded - assert "| --- |" not in body["text"] - - -def test_dingtalk_send_markdown_proactive_uses_sample_markdown(monkeypatch): - """Proactive delivery (no sessionWebhook) → robot API sampleMarkdown.""" - from core.channels.adapters.dingtalk import DingTalkAdapter - - adapter = DingTalkAdapter() - captured = {} - - async def fake_token(app_id, secret): - return "tok" - - async def fake_robot_send(token, conn, inbound, msg_key, msg_param): - captured.update(msg_key=msg_key, msg_param=msg_param) - return SendResult.ok() - - monkeypatch.setattr(adapter, "_access_token", fake_token) - monkeypatch.setattr(adapter, "_robot_send", fake_robot_send) - msg = InboundMsg(channel_id="chan_dt", channel_type="dingtalk", text="", - chat_type="group", external_conversation_id="cidG") - r = asyncio.run(adapter.send_markdown(_FakeDTConn(), msg, "**每日简报**\n- 事项一")) - assert r.success - assert captured["msg_key"] == "sampleMarkdown" - assert captured["msg_param"]["title"] == "每日简报" - assert "**每日简报**" in captured["msg_param"]["text"] - - -def test_dingtalk_push_sends_markdown_chunks(monkeypatch): - """push (the automation proactive-delivery entry point) also sends via markdown.""" - from core.channels.adapters.dingtalk import DingTalkAdapter - - adapter = DingTalkAdapter() - sent = [] - - async def fake_send_md(conn, inbound, text): - sent.append(text) - return SendResult.ok() - - monkeypatch.setattr(adapter, "send_markdown", fake_send_md) - msg = InboundMsg(channel_id="chan_dt", channel_type="dingtalk", text="", - chat_type="group", external_conversation_id="cidG") - r = asyncio.run(adapter.push(_FakeDTConn(), msg, "## 标题\n正文")) - assert r.success and sent == ["## 标题\n正文"] - - -def test_deliver_reply_prefers_markdown_and_strips_refs(): - """_deliver_reply: channels that support markdown go through send_markdown, and [ref:...] markers are cleaned.""" - from core.channels.inbound import _deliver_reply - - calls = {"md": [], "text": []} - - class _MdAdapter: - class caps: - supports_markdown = True - max_message_len = 4000 - - @staticmethod - def prepare_markdown(text): - return text - - async def send_markdown(self, conn, msg, text): - calls["md"].append(text) - return SendResult.ok() - - async def send_text(self, conn, msg, text): - calls["text"].append(text) - return SendResult.ok() - - async def edit_message(self, conn, mid, text): - return SendResult.fail("bad_format", "不支持") - - msg = InboundMsg(channel_id="c", channel_type="dingtalk", text="", - chat_type="p2p", external_conversation_id="cid") - asyncio.run(_deliver_reply(_MdAdapter(), object(), msg, - "结论如下[ref:internet_search-1]，见上文。", "ph_1")) - assert calls["md"] == ["结论如下，见上文。"] # went via markdown + citation markers cleaned - assert calls["text"] == [] - - -def test_dingtalk_send_placeholder_prefers_robot_api(monkeypatch): - """Placeholder via the robot API: obtains the processQueryKey (used as message_id) → recallable afterwards.""" - from core.channels.adapters.dingtalk import DingTalkAdapter - - adapter = DingTalkAdapter() - - async def fake_token(app_id, secret): - return "tok" - - async def fake_robot_send(token, conn, inbound, msg_key, msg_param): - assert msg_key == "sampleText" - return SendResult.ok("pqk_123") - - monkeypatch.setattr(adapter, "_access_token", fake_token) - monkeypatch.setattr(adapter, "_robot_send", fake_robot_send) - msg = InboundMsg(channel_id="c", channel_type="dingtalk", text="", chat_type="p2p", - external_conversation_id="cid", sender_id="staff_1") - r = asyncio.run(adapter.send_placeholder(_FakeDTConn(), msg, "🤔 正在处理")) - assert r.success and r.message_id == "pqk_123" - - -def test_dingtalk_send_placeholder_falls_back_to_webhook(monkeypatch): - """Robot API unavailable (token failure) → fall back to sessionWebhook plain text (no id, not recallable).""" - from core.channels.adapters.dingtalk import DingTalkAdapter - - adapter = DingTalkAdapter() - captured = {} - - async def fake_token(app_id, secret): - raise RuntimeError("no permission") - - async def fake_post(webhook, payload): - captured.update(payload=payload) - return SendResult.ok() - - monkeypatch.setattr(adapter, "_access_token", fake_token) - monkeypatch.setattr(adapter, "_post_webhook", fake_post) - msg = InboundMsg(channel_id="c", channel_type="dingtalk", text="", chat_type="p2p", - external_conversation_id="cid", - raw={"dingtalk_session_webhook": "https://oapi.dingtalk.com/robot/send?s=1"}) - r = asyncio.run(adapter.send_placeholder(_FakeDTConn(), msg, "🤔 正在处理")) - assert r.success and r.message_id is None - assert captured["payload"]["msgtype"] == "text" - - -def test_dingtalk_recall_url_selection(): - """Recall endpoint routing: group chats use groupMessages/recall (requires openConversationId); one-on-one chats use batchRecall.""" - from core.channels.adapters.dingtalk import DingTalkAdapter - - adapter = DingTalkAdapter() - group = InboundMsg(channel_id="c", channel_type="dingtalk", text="", chat_type="group", - external_conversation_id="cidG") - url, body = adapter._recall_url_body(_FakeDTConn(), group, "pqk_1") - assert url.endswith("/v1.0/robot/groupMessages/recall") - assert body["openConversationId"] == "cidG" and body["processQueryKeys"] == ["pqk_1"] - - p2p = InboundMsg(channel_id="c", channel_type="dingtalk", text="", chat_type="p2p", - external_conversation_id="cidP") - url2, body2 = adapter._recall_url_body(_FakeDTConn(), p2p, "pqk_2") - assert url2.endswith("/v1.0/robot/otoMessages/batchRecall") - assert "openConversationId" not in body2 - - # Group chat missing openConversationId → None (recall_message returns bad_format based on this) - orphan = InboundMsg(channel_id="c", channel_type="dingtalk", text="", chat_type="group", - external_conversation_id="") - assert adapter._recall_url_body(_FakeDTConn(), orphan, "pqk_3") is None - r = asyncio.run(adapter.recall_message(_FakeDTConn(), orphan, "")) - assert r.success is False and r.error_kind == "bad_format" - - -def test_deliver_reply_recalls_placeholder_when_edit_unsupported(): - """Edit failure (DingTalk) → recall the placeholder then send the formal reply, visually equivalent to a "replace".""" - from core.channels.inbound import _deliver_reply - - calls = {"recalled": [], "md": []} - - class _Adapter: - class caps: - supports_markdown = True - max_message_len = 4000 - - @staticmethod - def prepare_markdown(text): - return text - - async def edit_message(self, conn, mid, text): - return SendResult.fail("bad_format", "不支持") - - async def recall_message(self, conn, msg, mid): - calls["recalled"].append(mid) - return SendResult.ok(mid) - - async def send_markdown(self, conn, msg, text): - calls["md"].append(text) - return SendResult.ok() - - async def send_text(self, conn, msg, text): - return SendResult.ok() - - msg = InboundMsg(channel_id="c", channel_type="dingtalk", text="", chat_type="p2p", - external_conversation_id="cid") - asyncio.run(_deliver_reply(_Adapter(), object(), msg, "正式回复", "pqk_ph")) - assert calls["recalled"] == ["pqk_ph"] # placeholder recalled - assert calls["md"] == ["正式回复"] # reply sent as a new message - - -def test_replace_placeholder_falls_back_to_recall_and_send(): - """Error / no-text receipt: edit failure → recall + send anew (the old logic only edited, so DingTalk users would be stuck on "processing" forever).""" - from core.channels.inbound import _replace_placeholder - - calls = {"recalled": [], "sent": []} - - class _Adapter: - async def edit_message(self, conn, mid, text): - return SendResult.fail("bad_format", "不支持") - - async def recall_message(self, conn, msg, mid): - calls["recalled"].append(mid) - return SendResult.ok(mid) - - async def send_text(self, conn, msg, text): - calls["sent"].append(text) - return SendResult.ok() - - msg = InboundMsg(channel_id="c", channel_type="dingtalk", text="", chat_type="p2p", - external_conversation_id="cid") - asyncio.run(_replace_placeholder(_Adapter(), object(), msg, "pqk_ph", "⚠️ 出错了")) - assert calls["recalled"] == ["pqk_ph"] and calls["sent"] == ["⚠️ 出错了"] - - -def test_replace_placeholder_without_message_id_sends_terminal_notice_directly(): - """WeChat returns no placeholder ID, but users must still receive the terminal message.""" - from core.channels.inbound import _replace_placeholder - - calls = {"edited": 0, "sent": []} - - class _Adapter: - async def edit_message(self, conn, mid, text): - calls["edited"] += 1 - return SendResult.fail("bad_format", "不支持") - - async def send_text(self, conn, msg, text): - calls["sent"].append(text) - return SendResult.ok() - - msg = InboundMsg( - channel_id="c", - channel_type="weixin", - text="", - chat_type="p2p", - external_conversation_id="wx-user", - ) - asyncio.run(_replace_placeholder(_Adapter(), object(), msg, None, "⚠️ 处理失败")) - assert calls == {"edited": 0, "sent": ["⚠️ 处理失败"]} - - -def test_collect_reply_raises_user_facing_stream_error(monkeypatch): - from core.channels.inbound import ChannelRunError, _collect_reply - from orchestration import chat_run_executor - - async def _events(): - yield {"type": "thinking", "delta": "内部思考"} - yield {"type": "error", "error": "模型暂时不可用"} - - monkeypatch.setattr(chat_run_executor, "follow_run", lambda run_id: _events()) - - with pytest.raises(ChannelRunError, match="模型暂时不可用"): - asyncio.run(_collect_reply("run_failed")) - - -# ── Channel markdown adaptation (core/channels/markdown.py) ─────────────── -def test_markdown_strip_citation_markers(): - from core.channels.markdown import strip_citation_markers - - assert strip_citation_markers("A[ref:internet_search-1]B[ref:query_database-12]") == "AB" - assert strip_citation_markers("无标记原样") == "无标记原样" - assert strip_citation_markers("") == "" - - -def test_markdown_strip_inline_thinking(): - """Inline chain of thought must never reach the channel as message text. - Semantics mirror the web frontend (segments.ts): body = whatever follows the LAST .""" - from core.channels.markdown import strip_inline_thinking - - # Full tag pair - assert strip_inline_thinking("推理过程\n最终回复") == "最终回复" - # Multi-round tool loop: one thinking block per round, opening often absent - # (server-side template pre-fill) — only the text after the last close tag survives - assert strip_inline_thinking("先想想A再想想B正式答复") == "正式答复" - # Dangling opener (stream cut mid-thinking) → drop the unclosed tail - assert strip_inline_thinking("正文没写完的思考") == "正文" - # All-thinking reply collapses to empty (caller then sends the no-text receipt) - assert strip_inline_thinking("只有思考") == "" - # No tags → untouched - assert strip_inline_thinking("普通回复") == "普通回复" - assert strip_inline_thinking("") == "" - - -def test_collect_reply_strips_inline_thinking(monkeypatch): - """_collect_reply: content deltas carrying inline spans (channel runs use - enable_thinking=True, so the streaming layer forwards them raw) must be stripped - before delivery; thinking events stay ignored as before.""" - from core.channels import inbound as inbound_mod - from orchestration import chat_run_executor - - events = [ - {"type": "thinking", "delta": "结构化思考,本就不进正文"}, - {"type": "content", "delta": "内联思"}, - {"type": "content", "delta": "考跨 delta你好,"}, - {"type": "content", "delta": "这是答复"}, - {"type": "meta", "artifacts": [{"file_id": "f1"}]}, - ] - - async def _fake_follow(run_id): - for ev in events: - yield ev - - monkeypatch.setattr(chat_run_executor, "follow_run", _fake_follow) - reply, artifacts = asyncio.run(inbound_mod._collect_reply("run_x")) - assert reply == "你好,这是答复" - assert artifacts == [{"file_id": "f1"}] - - -def test_markdown_derive_title(): - from core.channels.markdown import derive_title - - assert derive_title("# 每周产业简报\n正文") == "每周产业简报" - assert derive_title("**加粗开头** 后续") == "加粗开头 后续" - assert derive_title("- [官网](https://x.y) 入口") == "官网 入口" - assert derive_title("\n\n") == "新消息" - assert len(derive_title("很长" * 40)) == 20 - - -def test_markdown_downgrade_for_dingtalk(): - from core.channels.markdown import downgrade_for_dingtalk - - # Well-formed table → row-style list; supported syntax like headings/lists kept as-is - md = "# 标题\n| 姓名 | 部门 |\n| --- | :---: |\n| 张三 | 研发 |\n| 李四 | 市场 |\n- 列表项" - out = downgrade_for_dingtalk(md) - assert "# 标题" in out and "- 列表项" in out - assert "- 姓名: 张三｜部门: 研发" in out - assert "- 姓名: 李四｜部门: 市场" in out - assert "|" not in out.replace("｜", "") - - # Code fences: fence lines removed, content kept (embedded | lines are not misjudged as tables) - code = "```python\nprint('hi')\n| not a table\n```\n尾行" - out2 = downgrade_for_dingtalk(code) - assert "```" not in out2 - assert "print('hi')" in out2 and "| not a table" in out2 and "尾行" in out2 - - # Ill-formed (no separator row) pipe content untouched - raw = "|a|b|\n|1|2|" - assert downgrade_for_dingtalk(raw) == raw - # Idempotent - assert downgrade_for_dingtalk(out) == out - - -def test_outbound_synthetic_msg_carries_chat_type_and_peer(): - # The proactively delivered synthetic message must carry the conversation profile — DingTalk selects the robot endpoint by chat_type/sender_id - from core.channels.outbound import _synthetic_msg - - m = _synthetic_msg("chan_x", "dingtalk", "cidABC", chat_type="p2p", peer_id="staff_9") - assert m.chat_type == "p2p" and m.sender_id == "staff_9" - assert m.external_conversation_id == "cidABC" - # Default is still group (backward compatible with old callers) - assert _synthetic_msg("chan_x", "lark", "oc_1").chat_type == "group" - - -# ── WeCom (WeChat Work) adapter ─────────────────────────────────────────── -def _wecom_conn(token="tk", aes_key_b64=None): - """Build a fake conn with real Fernet-encrypted credentials.""" - from core.infra.crypto import encrypt_secret - - class _C: - channel_id = "chan_wc" - channel_type = "wecom" - app_id = "corp1" - config = { - "app_secret_enc": encrypt_secret("sec"), - "agent_id_enc": encrypt_secret("1000002"), - "token_enc": encrypt_secret(token), - "aes_key_enc": encrypt_secret(aes_key_b64), - } - return _C() - - -def _wecom_encrypt(key32: bytes, msg: str, receiveid: str = "corp1") -> str: - import struct - from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes - - body = msg.encode("utf-8") - plain = b"0123456789abcdef" + struct.pack(">I", len(body)) + body + receiveid.encode() - pad = 16 - (len(plain) % 16) - plain += bytes([pad]) * pad - enc = Cipher(algorithms.AES(key32), modes.CBC(key32[:16])).encryptor() - return base64.b64encode(enc.update(plain) + enc.finalize()).decode() - - -def test_wecom_signature_pure(): - from core.channels.adapters.wecom import WeComAdapter - - sig = WeComAdapter._signature("tok", "1700000000", "nonce1", "ENC") - # Matches hand computation: sha1(sorted concatenation) - import hashlib as _h - expect = _h.sha1("".join(sorted(["tok", "1700000000", "nonce1", "ENC"])).encode()).hexdigest() - assert sig == expect - - -def test_wecom_url_verify_roundtrip_and_signature(): - from core.channels.adapters.wecom import WeComAdapter - - key32 = bytes(range(32)) - aes_key_b64 = base64.b64encode(key32).decode().rstrip("=") # 43-character EncodingAESKey - conn = _wecom_conn(token="mytoken", aes_key_b64=aes_key_b64) - echostr = _wecom_encrypt(key32, "hello-echo") - sig = WeComAdapter._signature("mytoken", "ts", "nc", echostr) - params = {"echostr": echostr, "msg_signature": sig, "timestamp": "ts", "nonce": "nc"} - assert WeComAdapter().verify_url(conn, params) == "hello-echo" - # Wrong signature → raises - with pytest.raises(ValueError): - WeComAdapter().verify_url(conn, {**params, "msg_signature": "bad"}) - - -def test_wecom_decrypt_webhook_and_parse_inbound(): - from core.channels.adapters.wecom import WeComAdapter - - key32 = bytes(range(32)) - aes_key_b64 = base64.b64encode(key32).decode().rstrip("=") - conn = _wecom_conn(token="mytoken", aes_key_b64=aes_key_b64) - inner_xml = ( - "" - "" - "" - "" - "12345" - ) - encrypt = _wecom_encrypt(key32, inner_xml) - body = f"".encode() - headers = {} - payload = WeComAdapter().decrypt_webhook(conn, body, headers) - assert payload["Content"] == "你好" - assert headers["_encrypt"] == encrypt # backfilled for signature verification - msg = WeComAdapter().parse_inbound(conn, payload) - assert msg.text == "你好" - assert msg.chat_type == "p2p" - assert msg.external_conversation_id == "userA" - - -def test_wecom_edit_message_unsupported(): - from core.channels.adapters.wecom import WeComAdapter - - r = asyncio.run(WeComAdapter().edit_message(_wecom_conn(aes_key_b64="x"), "mid", "x")) - assert r.success is False - - -# ── WeChat iLink adapter ──────────────────────────────────────────────────── -class _FakeWXConn: - channel_id = "chan_wx" - channel_type = "weixin" - app_id = "wx_abc" - config = {} - - -def test_weixin_parse_inbound_text_and_context_token(): - from core.channels.adapters.weixin import WeixinAdapter - - adapter = WeixinAdapter() - payload = { - "from_user_id": "o9cq_xxx@im.wechat", - "message_type": 1, - "context_token": "CTX-TOKEN-123", - "item_list": [{"type": 1, "text_item": {"text": "你好微信"}}], - } - msg = adapter.parse_inbound(_FakeWXConn(), payload) - assert msg.text == "你好微信" - assert msg.chat_type == "p2p" - assert msg.external_conversation_id == "o9cq_xxx@im.wechat" - assert msg.raw["weixin_context_token"] == "CTX-TOKEN-123" - assert msg.raw["weixin_to_user_id"] == "o9cq_xxx@im.wechat" - # Non-text (image) → None - assert adapter.parse_inbound(_FakeWXConn(), {"message_type": 2}) is None - - -def test_weixin_uin_header_is_base64_decimal(): - from core.channels.adapters.weixin import _uin_header - - raw = base64.b64decode(_uin_header()).decode() - assert raw.isdigit() and 0 <= int(raw) <= 0xFFFFFFFF - - -def test_weixin_caps_qr_bind_mode(): - from core.channels.adapters.weixin import WeixinAdapter - - assert WeixinAdapter.caps.bind_mode == "qr" - assert WeixinAdapter.caps.credential_fields == () - - -def test_weixin_has_push_file(): - # Inbound orchestration decides whether to return generated files by hasattr(adapter, "push_file") — if missing, it is silently skipped - from core.channels.adapters.weixin import WeixinAdapter - - assert callable(getattr(WeixinAdapter(), "push_file", None)) - - -def test_weixin_push_file_requires_token_and_target(): - from core.channels.adapters.weixin import WeixinAdapter - - adapter = WeixinAdapter() - msg = InboundMsg(channel_id="chan_wx", channel_type="weixin", text="", - chat_type="p2p", external_conversation_id="o9cq_xxx@im.wechat") - # No bot_token → forbidden (_FakeWXConn.config is empty) - r = asyncio.run(adapter.push_file(_FakeWXConn(), msg, b"data", "a.docx", "application/x")) - assert r.success is False and r.error_kind == "forbidden" - - -def test_weixin_aes_ecb_encrypt_pkcs7(): - from core.channels.adapters.weixin import _aes_ecb_encrypt - - key = bytes(range(16)) - ct = _aes_ecb_encrypt(b"hello weixin cdn", key) # 16-byte plaintext → full PKCS7 padding block - assert len(ct) == 32 and ct != b"hello weixin cdn" - # Empty content must also be encryptable (1 padding block) - assert len(_aes_ecb_encrypt(b"", key)) == 16 - - -# ── Registry: all four channels present ──────────────────────────────────── -def test_registry_lists_all_four_channels(): - from core.channels.registry import list_adapters - - got = set(list_adapters()) - assert {"lark", "dingtalk", "wecom", "weixin"}.issubset(got) - - -def test_caps_bind_mode_defaults(): - from core.channels.adapters.lark import LarkAdapter - from core.channels.adapters.dingtalk import DingTalkAdapter - from core.channels.adapters.wecom import WeComAdapter - - assert LarkAdapter.caps.bind_mode == "credentials" - assert DingTalkAdapter.caps.bind_mode == "credentials" - assert WeComAdapter.caps.credential_fields == ("app_id", "app_secret", "agent_id", "token", "aes_key") - - -# ── Inbound history loading: preserve cross-turn tool calls/results (regression guard) ── -def test_inbound_history_preserves_tool_calls(db_session): - """_load_history goes through compaction_service.load_session_history (same - source as the web UI) and must preserve the assistant turns' tool_calls / - tool results, instead of the old approach of stripping them to plain text. - - Old bug: channel history kept only user/assistant text → across turns the - model could not see the tools it had called in its previous turn, redoing - work repeatedly and spinning idle. Here we assert the tool_call replay - (role="tool" carrier + tool name) is still present. - """ - from core.channels.inbound import _load_history - from core.services.chat_service import ChatService - - _mk_user(db_session, uid="owner1") - cs = ChatService(db_session) - session = cs.create_session(user_id="owner1", title="渠道会话") - chat_id = session.chat_id - - cs.add_message(chat_id=chat_id, role="user", content="生成一份周报") - cs.add_message( - chat_id=chat_id, role="assistant", content="好的，已生成。", - tool_calls=[{ - "tool_name": "word_create", "tool_id": "tc_1", "status": "success", - "tool_args": {"title": "周报"}, "tool_result": "已生成 weekly.docx", - }], - ) - - history = _load_history(db_session, chat_id, "owner1") - - # Tool replay is present: the assistant block list carries the tool name, and there is a role="tool" result carrier - roles = [m["role"] for m in history] - assert "tool" in roles, f"工具结果载体被丢弃，历史被剥成纯文本: {roles}" - dumped = json.dumps(history, default=str, ensure_ascii=False) - assert "word_create" in dumped, "工具调用信息未保留在跨轮历史中" - - -def test_inbound_history_empty_on_no_access(db_session): - """When the session does not exist / access is denied, load_session_history returns None → _load_history falls back to an empty list.""" - from core.channels.inbound import _load_history - - _mk_user(db_session, uid="owner2") - assert _load_history(db_session, "chat_does_not_exist", "owner2") == [] - - -# ── 群聊旁听（observe_all）───────────────────────────────────────────────────── - - -def test_lark_parse_extracts_mentions_and_defers_group_decision(): - """群消息的 @ 判定必须延后：飞书事件里没有机器人自己的 open_id，只能先收集 mentions。""" - adapter = LarkAdapter() - evt = {"header": {"event_type": "im.message.receive_v1"}, "event": { - "message": { - "chat_id": "oc_1", "chat_type": "group", "message_id": "om_1", - "message_type": "text", "content": json.dumps({"text": "@_user_1 帮我看下"}), - "mentions": [{"key": "@_user_1", "id": {"open_id": "ou_bot"}, "name": "小助手"}], - }, - "sender": {"sender_id": {"open_id": "ou_a"}}}} - m = adapter.parse_inbound(_FakeConn(), evt) - assert m.mentioned_ids == ["ou_bot"] - assert m.addressed_to_bot is None, "群消息不应在 parse 阶段就下结论" - - # 单聊永远是冲着机器人来的，不需要再解析 - evt["event"]["message"]["chat_type"] = "p2p" - assert adapter.parse_inbound(_FakeConn(), evt).addressed_to_bot is True - - -def test_dingtalk_parse_uses_is_in_at_list(): - """钉钉自带 isInAtList，可同步判定；字段缺失时按「已被 @」处理，保持老行为。""" - from core.channels.adapters.dingtalk import DingTalkAdapter - - adapter = DingTalkAdapter() - base = {"msgtype": "text", "text": {"content": "hi"}, "conversationType": "2", - "conversationId": "cid_1", "msgId": "m1"} - assert adapter.parse_inbound(_FakeConn(), {**base, "isInAtList": True}).addressed_to_bot is True - assert adapter.parse_inbound(_FakeConn(), {**base, "isInAtList": False}).addressed_to_bot is False - # 未开通群消息读取权限时钉钉不下发该字段 → 不能误判成「没 @ 」而静音 - assert adapter.parse_inbound(_FakeConn(), base).addressed_to_bot is True - # 单聊不受影响 - assert adapter.parse_inbound( - _FakeConn(), {**base, "conversationType": "1", "isInAtList": False} - ).addressed_to_bot is True - - -def test_resolve_addressed_fails_open(db_session): - """判定不了就当被 @ 了：宁可多答一句，也不能对着直接 @ 装死。""" - from core.channels.inbound import _resolve_addressed - - conn = _mk_conn(db_session) - msg = _inbound("oc_g1", "group") - - class _NoResolver: - pass - - class _Boom: - async def resolve_addressed(self, conn, inbound): - raise RuntimeError("bot/v3/info 挂了") - - class _Says: - def __init__(self, v): self.v = v - async def resolve_addressed(self, conn, inbound): return self.v - - # 渠道没实现该钩子（企微/微信）→ 行为与本特性上线前完全一致 - assert asyncio.run(_resolve_addressed(_NoResolver(), conn, msg)) is True - assert asyncio.run(_resolve_addressed(_Boom(), conn, msg)) is True - assert asyncio.run(_resolve_addressed(_Says(False), conn, msg)) is False - # adapter 已给出结论时不再回调钩子 - decided = _inbound("oc_g1", "group") - decided.addressed_to_bot = True - assert asyncio.run(_resolve_addressed(_Says(False), conn, decided)) is True - - -def test_observe_buffer_accumulates_caps_and_drains(db_session): - """旁听消息进缓冲区、有水位上限、被 @ 时一次性排空。""" - from core.channels.inbound import ( - _OBSERVE_MAX, _drain_observed, _find_or_create_session, _observe_message, - ) - - conn = _mk_conn(db_session) - for i in range(_OBSERVE_MAX + 5): - m = _inbound("oc_g1", "group", text=f"闲聊{i}", mid=f"m{i}") - m.sender_name = f"甲{i}" - _observe_message(db_session, conn, m) - - session = _find_or_create_session(db_session, conn, _inbound("oc_g1", "group")) - buf = (session.extra_data or {}).get("observed") - assert len(buf) == _OBSERVE_MAX, "旁听缓冲必须有水位上限，否则活跃群会把提示词撑爆" - assert buf[0]["t"] == "闲聊5", "应淘汰最旧的，保留最近的" - assert buf[0]["n"] == "甲5" - - drained = _drain_observed(db_session, session) - assert len(drained) == _OBSERVE_MAX - db_session.refresh(session) - assert (session.extra_data or {}).get("observed") == [], "排空后不能重复注入" - assert _drain_observed(db_session, session) == [] - - -def test_observe_text_truncates_and_keeps_attachment_keys(db_session): - """旁听正文截断；附件只留句柄不下载，但 key 必须保住——丢了 key 就永远取不回。""" - from core.channels.inbound import ( - _OBSERVE_TEXT_MAX, _observe_attachment_refs, _observe_text, - ) - - long_msg = _inbound("oc_g1", "group", text="x" * (_OBSERVE_TEXT_MAX + 50)) - assert _observe_text(long_msg) == "x" * _OBSERVE_TEXT_MAX + "…" - assert _observe_text(_inbound("oc_g1", "group", text=" ")) == "" - - withfile = _inbound("oc_g1", "group", text="看下这个") - withfile.attachments = [ - {"kind": "file", "key": "fk", "name": "报表.xlsx"}, - {"kind": "image", "key": "ik", "name": "p.png"}, - {"kind": "file", "name": "无 key 的不要"}, # 没有 key 就没法取回，直接丢弃 - ] - assert _observe_attachment_refs(withfile) == [ - {"kind": "file", "key": "fk", "name": "报表.xlsx"}, - {"kind": "image", "key": "ik", "name": "p.png"}, - ] - - -def test_observe_records_file_index_for_lazy_fetch(db_session): - """附件句柄进独立索引（含 message_id / raw），且有水位；纯附件消息也要记。""" - from core.channels.inbound import ( - _OBSERVE_FILE_INDEX_MAX, _drain_observed, _find_or_create_session, _observe_message, - ) - - conn = _mk_conn(db_session) - m = _inbound("oc_g1", "group", text="", mid="om_file") - m.sender_name = "张三" - m.attachments = [{"kind": "file", "key": "fk_1", "name": "Q3财报.xlsx"}] - m.raw = {"lark_message_id": "om_file"} - _observe_message(db_session, conn, m) - - session = _find_or_create_session(db_session, conn, _inbound("oc_g1", "group")) - idx = (session.extra_data or {})["observed_files"] - assert idx["fk_1"]["name"] == "Q3财报.xlsx" - assert idx["fk_1"]["message_id"] == "om_file", "缺 message_id 飞书就取不回资源" - assert idx["fk_1"]["raw"] == {"lark_message_id": "om_file"} - - # 索引在缓冲区排空后仍然存在——模型可能过几轮才决定要看这个文件 - _drain_observed(db_session, session) - db_session.refresh(session) - assert "fk_1" in (session.extra_data or {})["observed_files"] - - # 水位 - for i in range(_OBSERVE_FILE_INDEX_MAX + 3): - mi = _inbound("oc_g1", "group", text="", mid=f"om_{i}") - mi.attachments = [{"kind": "file", "key": f"k{i}", "name": f"f{i}.txt"}] - _observe_message(db_session, conn, mi) - db_session.refresh(session) - idx = (session.extra_data or {})["observed_files"] - assert len(idx) == _OBSERVE_FILE_INDEX_MAX - assert "fk_1" not in idx and f"k{_OBSERVE_FILE_INDEX_MAX + 2}" in idx - - -def test_render_observed_block_exposes_attachment_keys(): - """群聊记录里必须带出 key，否则模型看得到文件却拿不到句柄。""" - from core.channels.inbound import _render_observed_block - - block = _render_observed_block([ - {"n": "张三", "t": "看下这个", "a": [ - {"kind": "file", "key": "fk_1", "name": "Q3财报.xlsx"}, - ]}, - {"n": "李四", "t": "收到"}, - ]) - assert "[文件：Q3财报.xlsx｜key=fk_1]" in block - assert "channel_read_attachment" in block, "要告诉模型怎么取回" - assert "李四：收到" in block - - # 没有附件时不必挂取回提示，省 token - assert "channel_read_attachment" not in _render_observed_block([{"n": "李四", "t": "收到"}]) - - -def test_render_observed_block_marks_as_do_not_reply(): - """群聊记录必须显式标注为背景、不要逐条回复，否则模型会把群消息当成待办清单挨个答。""" - from core.channels.inbound import _render_observed_block - - block = _render_observed_block([{"n": "张三", "t": "明天几点"}, {"n": "李四", "t": "十点"}]) - assert "张三：明天几点" in block and "李四：十点" in block - assert "不要逐条回复" in block - assert block.endswith("\n"), "需与本次消息之间有分隔" - - -def test_group_listen_mode_validation(): - """observe_all 只能开在平台可能投递非 @ 群消息的渠道上。""" - from core.services.channel_service import _validate_group_listen - - assert _validate_group_listen("lark", "observe_all") == "observe_all" - assert _validate_group_listen("dingtalk", "observe_all") == "observe_all" - assert _validate_group_listen("lark", None) == "mention_only" - with pytest.raises(BadRequestError): - _validate_group_listen("lark", "listen_everything") - with pytest.raises(BadRequestError): - _validate_group_listen("weixin", "observe_all") - - -def test_default_group_listen_mode_is_mention_only(db_session): - """默认必须是 mention_only —— 旁听全群是隐私敏感行为，只能由 owner 显式打开。""" - conn = _mk_conn(db_session) - assert conn.group_listen_mode == "mention_only" - assert bot_to_dict(conn)["group_listen_mode"] == "mention_only" - - -def test_inbound_gate_routes_bystander_messages(db_session, monkeypatch): - """闸门路由：未 @ 的群消息在 observe_all 下只落缓冲、在 mention_only 下直接丢弃， - 两种情况都**绝不**起 run、绝不回复——旁听如果会触发回复，机器人会在群里刷屏。""" - import core.channels.inbound as inbound_mod - - chan_id = _mk_conn(db_session).channel_id - repo = ChannelConnectionRepository(db_session) - # 不 patch close：fixture 是共享文件 SQLite + teardown drop_all，压住 close 会留锁 - # 拖垮后续用例建表。但 _process_inbound 内部会 close，而 Session.close() 会把已加载 - # 对象全部 expunge —— 所以下面一律用 channel_id 重新取 conn，不复用旧引用。 - monkeypatch.setattr(inbound_mod, "SessionLocal", lambda: db_session) - - started = [] - - class _FakeAdapter: - caps = None - async def resolve_addressed(self, conn, inbound): return False - async def send_text(self, *a, **k): - started.append("sent"); return SendResult.ok("x") - - monkeypatch.setattr(inbound_mod, "get_adapter", lambda ct: _FakeAdapter()) - - def _boom(*a, **k): - started.append("run") - raise AssertionError("旁听消息不得触发 agent run") - - monkeypatch.setattr( - "orchestration.chat_run_executor.start_run", _boom, raising=False - ) - - msg = _inbound("oc_g1", "group", text="今天几点开会", mid="mo1") - msg.sender_name = "张三" - - # observe_all：进缓冲，不起 run、不回复 - repo.update(chan_id, {"group_listen_mode": "observe_all"}) - asyncio.run(inbound_mod._process_inbound(msg)) - assert started == [], "旁听消息既不应起 run 也不应回复" - session = inbound_mod._find_or_create_session(db_session, repo.get_by_id(chan_id), msg) - assert [it["t"] for it in (session.extra_data or {}).get("observed", [])] == ["今天几点开会"] - assert (session.extra_data or {})["observed"][0]["n"] == "张三" - - # mention_only：直接丢弃，缓冲区不动 - repo.update(chan_id, {"group_listen_mode": "mention_only"}) - asyncio.run(inbound_mod._process_inbound( - _inbound("oc_g1", "group", text="不该被记下", mid="mo2") - )) - # 同样：close() 已把 session 对象 expunge，重新查一次而不是 refresh - fresh = inbound_mod._find_or_create_session(db_session, repo.get_by_id(chan_id), msg) - assert [it["t"] for it in (fresh.extra_data or {}).get("observed", [])] == ["今天几点开会"] - assert started == [] - - -def test_lark_resolve_addressed_matches_bot_open_id(monkeypatch): - """飞书 @ 判定：只认机器人自己的 open_id。 - - 覆盖 include_bot 权限（`im:message.group_at_msg.include_bot:readonly`）下的场景—— - 群里 @ 的是**别的**机器人时，我们不该抢答；这在闸门上线前是会误答的。 - """ - adapter = LarkAdapter() - - async def _fake_id(conn): return "ou_me" - monkeypatch.setattr(LarkAdapter, "_bot_open_id", staticmethod(_fake_id)) - - def _msg(ids): - m = _inbound("oc_g1", "group") - m.mentioned_ids = ids - return m - - assert asyncio.run(adapter.resolve_addressed(_FakeConn(), _msg(["ou_me"]))) is True - assert asyncio.run(adapter.resolve_addressed(_FakeConn(), _msg(["ou_other_bot"]))) is False - assert asyncio.run(adapter.resolve_addressed(_FakeConn(), _msg([]))) is False - - # 取不到自身 open_id（接口抖动 / 空值）→ 一律按「已被 @」处理，不能在群里装死 - async def _boom(conn): raise RuntimeError("bot/v3/info 502") - monkeypatch.setattr(LarkAdapter, "_bot_open_id", staticmethod(_boom)) - assert asyncio.run(adapter.resolve_addressed(_FakeConn(), _msg(["ou_other"]))) is True - - async def _empty(conn): return "" - monkeypatch.setattr(LarkAdapter, "_bot_open_id", staticmethod(_empty)) - assert asyncio.run(adapter.resolve_addressed(_FakeConn(), _msg(["ou_other"]))) is True - - -# ── 钉钉附件接收 / 下载 ──────────────────────────────────────────────────────── - - -def test_dingtalk_extracts_attachments_by_msgtype(): - """钉钉各消息类型的 downloadCode 提取（字段位置依官方「接收的消息类型」）。""" - from core.channels.adapters.dingtalk import DingTalkAdapter - - a = DingTalkAdapter() - base = {"conversationType": "1", "conversationId": "cid", "msgId": "m1", - "robotCode": "ding_robot_1"} - - f = a.parse_inbound(_FakeConn(), {**base, "msgtype": "file", "content": { - "downloadCode": "dc_1", "fileName": "合同.pdf", "spaceId": "s", "fileId": "fi"}}) - assert f.attachments == [{"kind": "file", "key": "dc_1", "name": "合同.pdf"}] - # 纯附件、无正文的消息也必须成立——否则钉钉发文件永远收不到 - assert f.text == "" - assert f.raw["dingtalk_robot_code"] == "ding_robot_1" - - p = a.parse_inbound(_FakeConn(), {**base, "msgtype": "picture", "content": { - "downloadCode": "dc_2", "pictureDownloadCode": "pdc"}}) - assert p.attachments[0]["kind"] == "image" and p.attachments[0]["key"] == "dc_2" - # 只给 pictureDownloadCode 时回退取它 - p2 = a.parse_inbound(_FakeConn(), {**base, "msgtype": "picture", - "content": {"pictureDownloadCode": "pdc"}}) - assert p2.attachments[0]["key"] == "pdc" - - r = a.parse_inbound(_FakeConn(), {**base, "msgtype": "richText", "content": { - "richText": [{"text": "看图"}, {"downloadCode": "dc_3"}]}}) - assert r.text == "看图" and r.attachments[0]["key"] == "dc_3" - - # 字段缺失不能抛异常（payload 规格松散，抛了会打断长连接） - assert a.parse_inbound(_FakeConn(), {**base, "msgtype": "file", "content": {}}) is None - assert a.parse_inbound(_FakeConn(), {**base, "msgtype": "file"}) is None - - -def test_dingtalk_download_resource_two_hop(monkeypatch): - """downloadCode → downloadUrl → bytes；缺 robotCode 或过期一律返回 None 不抛。""" - import httpx as _httpx - - from core.channels.adapters.dingtalk import DingTalkAdapter - - a = DingTalkAdapter() - calls = [] - - class _Resp: - def __init__(self, code=200, payload=None, content=b""): - self.status_code, self._p, self.content = code, payload or {}, content - self.text = json.dumps(self._p) - def json(self): return self._p - - class _Client: - async def __aenter__(self): return self - async def __aexit__(self, *a): return False - async def post(self, url, headers=None, json=None): - calls.append(("post", url, json)) - return _Resp(200, {"downloadUrl": "https://dl.example/x.file"}) - async def get(self, url): - calls.append(("get", url, None)) - return _Resp(200, content=b"PDFBYTES") - - monkeypatch.setattr(_httpx, "AsyncClient", lambda **kw: _Client()) - async def _tok(app_id, secret): return "tk" - monkeypatch.setattr(DingTalkAdapter, "_access_token", staticmethod(_tok)) - - msg = InboundMsg(channel_id="c", channel_type="dingtalk", text="", chat_type="group", - external_conversation_id="cid", message_id="m1", - raw={"dingtalk_robot_code": "rc_1"}) - got = asyncio.run(a.download_resource(_FakeConn(), msg, {"kind": "file", "key": "dc_1"})) - assert got == b"PDFBYTES" - assert calls[0][1].endswith("/v1.0/robot/messageFiles/download") - assert calls[0][2] == {"downloadCode": "dc_1", "robotCode": "rc_1"} - - # 没有 robotCode 就调不动接口，直接放弃而不是发一个必失败的请求 - nomeat = InboundMsg(channel_id="c", channel_type="dingtalk", text="", chat_type="group", - external_conversation_id="cid", raw={}) - assert asyncio.run(a.download_resource(_FakeConn(), nomeat, {"key": "dc_1"})) is None - assert asyncio.run(a.download_resource(_FakeConn(), msg, {"key": ""})) is None - - -# ── channel_read_attachment 工具（按需取回旁听附件）──────────────────────────── - - -def _reg_channel_tool(user_id, chat_id): - """注册工具并取出被注册的函数。 - - 注意：register_* 系列拿到的不是 AgentScope 2.0 的 Toolkit，而是鸭子类型的 - ToolCollector（见 core/llm/tool_collector.py）——真 Toolkit 是一次性构造的。 - """ - from core.llm.tool_collector import ToolCollector - from core.llm.tools.channel_attachment_tool import register_channel_attachment - - tc = ToolCollector() - register_channel_attachment(tc, user_id=user_id, chat_id=chat_id) - ft = tc._tools.get("channel_read_attachment") - assert ft is not None, "channel_read_attachment 未注册成功" - for attr in ("original_func", "func", "_func"): - fn = getattr(ft, attr, None) - if callable(fn): - return fn - raise AssertionError(f"取不到底层函数，FunctionTool 属性：{dir(ft)}") - - -def _tool_json(resp): - block = resp.content[0] - text = block["text"] if isinstance(block, dict) else getattr(block, "text") - return json.loads(text) - - -def test_channel_read_attachment_fetches_and_stores(db_session, monkeypatch): - """按 key 取回 → 落 Artifact → 返回 file_id 供 read_artifact 读。""" - from core.channels.inbound import _find_or_create_session, _observe_message - - conn = _mk_conn(db_session) - m = _inbound("oc_g1", "group", text="", mid="om_f") - m.attachments = [{"kind": "file", "key": "fk_1", "name": "Q3财报.xlsx"}] - m.raw = {"lark_message_id": "om_f"} - _observe_message(db_session, conn, m) - session = _find_or_create_session(db_session, conn, _inbound("oc_g1", "group")) - - # 只替换 SessionLocal（工具内部是 `from core.db.engine import SessionLocal` 的函数内导入）。 - # 不要去 patch session.close —— fixture 用的是**共享文件** SQLite 且 teardown 会 drop_all， - # 把 close 变成空操作会让连接一直占着锁，下一个用例的 create_all 就建不出表了。 - # Session.close() 之后仍可继续使用（会开新事务），所以让 `with` 正常关闭即可。 - monkeypatch.setattr("core.db.engine.SessionLocal", lambda: db_session) - - seen = {} - - class _Ad: - async def download_resource(self, conn, inbound, att): - seen["message_id"] = inbound.message_id - seen["raw"] = dict(inbound.raw or {}) - seen["key"] = att.get("key") - return b"XLSXBYTES" - - monkeypatch.setattr("core.channels.registry.get_adapter", lambda ct: _Ad()) - - tool = _reg_channel_tool(str(session.user_id), session.chat_id) - out = _tool_json(asyncio.run(tool("fk_1"))) - - assert "error" not in out, out - assert out["filename"] == "Q3财报.xlsx" and out["size"] == len(b"XLSXBYTES") - assert out["file_id"] and "read_artifact" in out["next"] - # adapter 拿到了取回所需的上下文（飞书按 message_id 定位资源） - assert seen["key"] == "fk_1" and seen["message_id"] == "om_f" - assert seen["raw"] == {"lark_message_id": "om_f"} - - -def test_channel_read_attachment_rejects_unknown_and_foreign(db_session, monkeypatch): - """未知 key / 越权会话 / 空 key / 非渠道会话 都必须干净报错，不能抛。""" - from core.channels.inbound import _find_or_create_session, _observe_message - - conn = _mk_conn(db_session) - m = _inbound("oc_g1", "group", text="", mid="om_f") - m.attachments = [{"kind": "file", "key": "fk_1", "name": "a.xlsx"}] - _observe_message(db_session, conn, m) - session = _find_or_create_session(db_session, conn, _inbound("oc_g1", "group")) - - monkeypatch.setattr("core.db.engine.SessionLocal", lambda: db_session) - - tool = _reg_channel_tool(str(session.user_id), session.chat_id) - assert "不能为空" in _tool_json(asyncio.run(tool(" ")))["error"] - assert "未找到" in _tool_json(asyncio.run(tool("no_such_key")))["error"] - - # 别人的会话：拿到 chat_id 也不能越权取走群文件 - other = _reg_channel_tool("someone_else", session.chat_id) - assert "无权" in _tool_json(asyncio.run(other("fk_1")))["error"] - - # 不在渠道会话里时工具压根没有 chat_id - assert "渠道会话" in _tool_json(asyncio.run(_reg_channel_tool("u_o", None)("fk_1")))["error"] - - -# ── 渠道会话身份注入（让 agent 能去拉入群前的群历史）──────────────────────────── - - -def test_inbound_channel_origin_carries_channel_type(db_session, monkeypatch): - """channel_origin 必须带上 channel_type，否则提示里说不出「哪个平台」、也选不出 CLI。""" - import core.channels.inbound as inbound_mod - - chan_id = _mk_conn(db_session).channel_id - monkeypatch.setattr(inbound_mod, "SessionLocal", lambda: db_session) - - captured = {} - - class _Ad: - caps = None - async def resolve_addressed(self, conn, inbound): return True - async def send_text(self, *a, **k): return SendResult.ok("x") - async def send_placeholder(self, *a, **k): return SendResult.ok("ph") - - monkeypatch.setattr(inbound_mod, "get_adapter", lambda ct: _Ad()) - # 这个扫描器自己开 DB 会话（不走被 patch 的那个），测试库里没建表 → 直接短路掉， - # 本用例只关心 context 的组装。 - monkeypatch.setattr( - "core.chat.context.collect_historical_attachments", lambda *a, **k: [] - ) - - async def _fake_start_run(**kwargs): - captured.update(kwargs.get("context") or {}) - raise RuntimeError("stop here — 只验证 context 组装") - - monkeypatch.setattr("orchestration.chat_run_executor.start_run", _fake_start_run) - - asyncio.run(inbound_mod._process_inbound( - _inbound("oc_g1", "group", text="帮我看看群里发的文件", mid="mco1") - )) - - origin = captured.get("channel_origin") or {} - assert origin.get("channel_type") == "lark" - assert origin.get("chat_type") == "group" - assert origin.get("conversation_id") == "oc_g1" - assert origin.get("channel_id") == chan_id - - -# ── 确定性群历史拉取（代码触发、增量、带预算）────────────────────────────────── - - -class _HistAdapter: - """带 fetch_history 的假 adapter；记录每次调用的 since_ms 以验证增量。""" - - def __init__(self, batches): - self.batches = list(batches) - self.calls = [] - - async def fetch_history(self, conn, conversation_id, *, since_ms, limit, newest_first=False): - self.calls.append({ - "conv": conversation_id, "since_ms": since_ms, - "limit": limit, "newest_first": newest_first, - }) - return self.batches.pop(0) if self.batches else [] - - -def _hist(mid, text, ts_ms, name="群友", attachments=None): - from core.channels.protocol import HistoryItem - return HistoryItem( - message_id=mid, sender_name=name, text=text, ts_ms=ts_ms, - attachments=attachments or [], raw={"lark_message_id": mid}, - ) - - -def test_history_pull_is_incremental_and_dedupes(db_session): - """第二次拉取必须以游标为下界，且边界重复的消息不能重复入库。""" - from core.channels.inbound import _find_or_create_session, _pull_history - - conn = _mk_conn(db_session) - msg = _inbound("oc_g1", "group", mid="m_at1") - session = _find_or_create_session(db_session, conn, msg) - - # 用真实量级的 epoch ms：游标只进不退，比冷启动窗口更旧的时间戳不会让它倒退 - import time as _t - t0 = int(_t.time() * 1000) - 3_600_000 # 1 小时前，落在 24h 冷启动窗口内 - ad = _HistAdapter([ - [_hist("h1", "周三评审会", t0), _hist("h2", "地点 302", t0 + 1_000)], - # 第二批把 h2 又回了一遍（平台边界通常是闭区间）+ 一条新的 - [_hist("h2", "地点 302", t0 + 1_000), _hist("h3", "带上季度数据", t0 + 2_000)], - ]) - - assert asyncio.run(_pull_history(db_session, ad, conn, msg, session)) == 2 - assert ad.calls[0]["since_ms"] > 0, "首次拉取要有冷启动窗口下界，不能从 0 全量拉" - assert ad.calls[0]["limit"] == 80 - - assert asyncio.run(_pull_history(db_session, ad, conn, msg, session)) == 1, "重复的不该再记一次" - assert ad.calls[1]["since_ms"] == t0 + 1_000, "第二次必须从游标开始，而不是重新拉一遍窗口" - - db_session.refresh(session) - texts = [it["t"] for it in (session.extra_data or {})["observed"]] - assert texts == ["周三评审会", "地点 302", "带上季度数据"] - - -def test_history_pull_respects_budget_and_truncation(db_session): - """总字数预算与单条截断都要生效——活跃群不能把提示词撑爆。""" - from core.channels.inbound import ( - _HISTORY_MAX_TOTAL_CHARS, _OBSERVE_TEXT_MAX, _find_or_create_session, _pull_history, - ) - - conn = _mk_conn(db_session) - msg = _inbound("oc_g2", "group", mid="m_at2") - session = _find_or_create_session(db_session, conn, msg) - - import time as _t - _now = int(_t.time() * 1000) - big = [_hist(f"b{i}", "x" * 5000, _now - 60_000 + i) for i in range(10)] - ad = _HistAdapter([big]) - n = asyncio.run(_pull_history(db_session, ad, conn, msg, session)) - - db_session.refresh(session) - entries = (session.extra_data or {})["observed"] - assert n == len(entries) - assert all(len(e["t"]) <= _OBSERVE_TEXT_MAX + 1 for e in entries), "单条要截断" - assert sum(len(e["t"]) for e in entries) <= _HISTORY_MAX_TOTAL_CHARS, "总量要卡预算" - - -def test_history_pull_records_attachment_handles(db_session): - """历史里的文件同样只留句柄进索引，且带上 message_id 供后续按需取回。""" - from core.channels.inbound import _find_or_create_session, _pull_history - - conn = _mk_conn(db_session) - msg = _inbound("oc_g3", "group", mid="m_at3") - session = _find_or_create_session(db_session, conn, msg) - - ad = _HistAdapter([[ - _hist("hf1", "看下这个", int(__import__("time").time() * 1000) - 60_000, - attachments=[{"kind": "file", "key": "fk_h", "name": "周报.docx"}]), - ]]) - asyncio.run(_pull_history(db_session, ad, conn, msg, session)) - - db_session.refresh(session) - idx = (session.extra_data or {})["observed_files"] - assert idx["fk_h"]["name"] == "周报.docx" - assert idx["fk_h"]["message_id"] == "hf1" - assert idx["fk_h"]["raw"] == {"lark_message_id": "hf1"} - - -def test_history_pull_degrades_and_stops_retrying(db_session): - """拉取失败绝不能影响回复；连续失败到上限后不再重试，避免每条消息都白等一次超时。""" - from core.channels.inbound import _HISTORY_MAX_FAILURES, _find_or_create_session, _pull_history - - conn = _mk_conn(db_session) - msg = _inbound("oc_g4", "group", mid="m_at4") - session = _find_or_create_session(db_session, conn, msg) - - class _Broken: - calls = 0 - async def fetch_history(self, conn, conversation_id, *, since_ms, limit, newest_first=False): - type(self).calls += 1 - raise RuntimeError("接口 500") - - bad = _Broken() - for _ in range(_HISTORY_MAX_FAILURES + 3): - assert asyncio.run(_pull_history(db_session, bad, conn, msg, session)) == 0 - assert _Broken.calls == _HISTORY_MAX_FAILURES, "到上限后应停止重试" - - # 渠道没实现该钩子（钉钉当前状态）→ 直接跳过，不报错 - class _NoHook: - pass - assert asyncio.run(_pull_history(db_session, _NoHook(), conn, msg, session)) == 0 - - -def test_lark_history_item_parsing(): - """飞书 im/v1/messages 条目 → HistoryItem；已删除/不支持类型要跳过。""" - item = LarkAdapter._history_item({ - "message_id": "om_1", "msg_type": "text", "create_time": "1700000000000", - "sender": {"id": "ou_abcdef123"}, - "body": {"content": json.dumps({"text": "@_user_1 周三评审"})}, - }) - assert item.message_id == "om_1" and item.ts_ms == 1700000000000 - assert item.text == "周三评审", "@ 提及应被剥掉" - assert item.raw == {"lark_message_id": "om_1"} - - f = LarkAdapter._history_item({ - "message_id": "om_2", "msg_type": "file", "create_time": "1", - "sender": {"id": "ou_x"}, - "body": {"content": json.dumps({"file_key": "fk", "file_name": "a.pdf"})}, - }) - assert f.attachments == [{"kind": "file", "key": "fk", "name": "a.pdf"}] - - assert LarkAdapter._history_item({"message_id": "om_3", "deleted": True}) is None - assert LarkAdapter._history_item({"message_id": "om_4", "msg_type": "sticker", - "body": {"content": "{}"}}) is None - - -# ── 钉钉群历史拉取（走官方 dws CLI，复用已有登录态）────────────────────────── - - -def test_dingtalk_fetch_history_requires_owner(): - """没有 owner 或会话 ID 就直接跳过，不起子进程。""" - from core.channels.adapters.dingtalk import DingTalkAdapter - - class _C: - owner_user_id = "" - config = {} - app_id = "k" - channel_id = "c" - - a = DingTalkAdapter() - assert asyncio.run(a.fetch_history(_C(), "cid", since_ms=0, limit=5)) == [] - - -def test_dingtalk_fetch_history_dws_call_shape(monkeypatch): - """命令行形状：group / time（本地时间串）/ limit / json，且以 owner 身份运行。""" - from core.channels.adapters import dingtalk as dt - - seen = {} - - async def _fake_run(user_id, args, timeout=40): - seen["user_id"] = user_id - seen["args"] = args - seen["timeout"] = timeout - return (json.dumps({"success": True, "result": {"messages": [ - {"msgId": "m1", "senderNick": "张三", "content": "本周进展", "createTime": 1700000000000}, - {"msgId": "m2", "senderNick": "李四", "content": " "}, - ]}}), "", 0) - - monkeypatch.setattr("core.services.dingtalk_service._run_dws", _fake_run) - - class _C: - owner_user_id = "u_owner" - config = {} - app_id = "k" - channel_id = "c" - - items = asyncio.run( - dt.DingTalkAdapter().fetch_history(_C(), "cid_x", since_ms=1_700_000_000_000, limit=9) - ) - assert seen["user_id"] == "u_owner", "必须以连接 owner 的身份跑（凭据按 HOME 隔离）" - args = seen["args"] - assert args[:3] == ["chat", "message", "list"] - assert "--group" in args and args[args.index("--group") + 1] == "cid_x" - assert args[args.index("--limit") + 1] == "9" - assert "--format" in args and args[args.index("--format") + 1] == "json" - tval = args[args.index("--time") + 1] - assert re.match(r"^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$", tval), tval - assert seen["timeout"] <= 10, "子进程要有独立上限，不能吃满调用方的预算" - # 空白正文不入库 - assert [(i.message_id, i.sender_name, i.text) for i in items] == [("m1", "张三", "本周进展")] - - -def test_dingtalk_fetch_history_degrades(monkeypatch): - """未登录/超时/坏输出一律返回 []，不把异常抛回入站链路。""" - from core.channels.adapters import dingtalk as dt - - class _C: - owner_user_id = "u" - config = {} - app_id = "k" - channel_id = "c" - - a = dt.DingTalkAdapter() - - async def _not_logged_in(user_id, args, timeout=40): - return ("", "未登录，请先执行 dws auth login", 2) - - monkeypatch.setattr("core.services.dingtalk_service._run_dws", _not_logged_in) - assert asyncio.run(a.fetch_history(_C(), "cid", since_ms=0, limit=5)) == [] - - async def _garbage(user_id, args, timeout=40): - return ("not json at all", "", 0) - - monkeypatch.setattr("core.services.dingtalk_service._run_dws", _garbage) - assert asyncio.run(a.fetch_history(_C(), "cid", since_ms=0, limit=5)) == [] - - async def _boom(user_id, args, timeout=40): - raise RuntimeError("子进程炸了") - - monkeypatch.setattr("core.services.dingtalk_service._run_dws", _boom) - assert asyncio.run(a.fetch_history(_C(), "cid", since_ms=0, limit=5)) == [] - - -def test_dingtalk_parse_dws_history_real_shape(): - """按 dws 线上真实输出解析：openMessageId / sender / createTime 是本地时间**字符串**。 - - 这三个字段名跟机器人回调那套完全不同，踩过一次：把 createTime 当 epoch ms 解析会 - 静默得到 0，游标就永远停在冷启动窗口，每次 @ 都重拉同一个 24 小时。 - """ - from core.channels.adapters.dingtalk import DingTalkAdapter - - p = DingTalkAdapter._parse_dws_history - items = p({"errorCode": None, "result": {"hasMore": True, "messages": [{ - "content": "@HugAgentOS 你再看看 现在可以查到了吗", - "createTime": "2026-08-08 21:57:33", - "openConversationId": "cidaKkLSKoy6sVI9cklpK2LWw==", - "openMessageId": "msgpUQWt9mBVdjLj788Z/nnww==", - "sender": "朱路浩", - }]}}) - assert len(items) == 1 - it = items[0] - assert it.message_id == "msgpUQWt9mBVdjLj788Z/nnww==", "去重靠它，取错就重复入库" - assert it.sender_name == "朱路浩", "sender 已是显示名，不该退回「群成员」" - assert it.ts_ms > 1_700_000_000_000, f"时间戳没解析出来，游标会卡死: {it.ts_ms}" - - # 时间戳容错 - ts = DingTalkAdapter._dws_ts_ms - assert ts("2026-08-08 21:57:33") > 0 - assert ts(1786197795294) == 1786197795294 - assert ts("1786197795294") == 1786197795294 - assert ts("") == 0 and ts(None) == 0 and ts("不是时间") == 0 - - # 外壳兼容 + 失败态 - assert [i.text for i in p({"messages": [{"openMessageId": "2", "content": "b"}]})] == ["b"] - assert p({"success": False}) == [] and p({"errorCode": "40001"}) == [] - assert p({}) == [] and p(None) == [] - - -def test_history_coldstart_takes_newest_end(db_session): - """冷启动要取窗口里**最新**的一批，不是最旧的——7 天窗口 + 条数上限下，取旧端等于 - 把一周前的闲聊塞进来、而最近几天反而看不到。截断也必须砍旧的那头。""" - import time as _t - - from core.channels.inbound import ( - _HISTORY_COLDSTART_WINDOW_H, _find_or_create_session, _pull_history, - ) - - conn = _mk_conn(db_session) - msg = _inbound("oc_cold", "group", mid="m_cold") - session = _find_or_create_session(db_session, conn, msg) - - now = int(_t.time() * 1000) - # 单条先被截到 500 字，所以要够多条才会触到 12000 字总预算（约 24 条） - batch = [_hist(f"c{i}", "x" * 4000, now - i * 60_000, name=f"甲{i:02d}") for i in range(30)] - ad = _HistAdapter([batch]) - n = asyncio.run(_pull_history(db_session, ad, conn, msg, session)) - - assert ad.calls[0]["newest_first"] is True, "首次拉取必须取新端" - # 冷启动窗口 = 7 天 - assert _HISTORY_COLDSTART_WINDOW_H == 168 - assert now - ad.calls[0]["since_ms"] <= 168 * 3600 * 1000 + 60_000 - - db_session.refresh(session) - entries = (session.extra_data or {})["observed"] - assert len(entries) == n and n > 0 - names = [e["n"] for e in entries] - # 保留的是最新的几条（c0 最新），且落库顺序是时间正序 - assert len(entries) < 30, "总预算应当生效，砍掉一部分" - assert "甲00" in names, f"最新的一条被预算砍掉了: {names}" - assert "甲29" not in names, f"最旧的一条不该保留（应从旧端砍）: {names}" - assert names == sorted(names, key=lambda x: -int(x[1:])), f"落库应为时间正序: {names}" - - -def test_history_incremental_is_not_newest_first(db_session): - """有游标之后是增量：从游标往后取，方向必须是旧→新，否则会跳过消息。""" - import time as _t - - from core.channels.inbound import _find_or_create_session, _pull_history - - conn = _mk_conn(db_session) - msg = _inbound("oc_inc", "group", mid="m_inc") - session = _find_or_create_session(db_session, conn, msg) - t0 = int(_t.time() * 1000) - 3_600_000 - - ad = _HistAdapter([[_hist("i1", "第一条", t0)], [_hist("i2", "第二条", t0 + 1000)]]) - asyncio.run(_pull_history(db_session, ad, conn, msg, session)) - asyncio.run(_pull_history(db_session, ad, conn, msg, session)) - - assert ad.calls[0]["newest_first"] is True, "第一次是冷启动" - assert ad.calls[1]["newest_first"] is False, "第二次是增量，必须旧→新" - assert ad.calls[1]["since_ms"] == t0, "增量以游标为下界" - - -def test_dingtalk_history_extracts_file_handles(): - """dws 把文件消息渲染成**纯文本**，必须把 fileId 解析回附件句柄。 - - 不解析的话，拉回来的就只是一句「[文件] xx.docx fileId: ...」——模型看得到有文件， - 却没有任何句柄能打开它，正是实测踩到的现象。 - """ - from core.channels.adapters.dingtalk import DingTalkAdapter as D - - raw = ("[文件] 项目周报.docx fileId: X6GRezwJl2LLnwYYTgZMNMzbWdqbropQ " - "注意：如需下载使用dws drive download命令下载") - text, refs = D._extract_dws_files(raw) - assert refs == [{"kind": "file", "key": "X6GRezwJl2LLnwYYTgZMNMzbWdqbropQ", - "name": "项目周报.docx"}] - assert text == "", "匹配掉的整句要清掉，CLI 用法提示不该进提示词" - - img = "[图片] shot.png fileId: ABC123" - assert D._extract_dws_files(img)[1][0]["kind"] == "image" - - # 普通消息不受影响 - assert D._extract_dws_files("就是一句普通的话") == ("就是一句普通的话", []) - - items = D._parse_dws_history({"result": {"messages": [ - {"openMessageId": "m1", "sender": "朱路浩", "createTime": "2026-08-08 22:03:21", - "content": raw}, - ]}}) - assert items[0].attachments[0]["key"] == "X6GRezwJl2LLnwYYTgZMNMzbWdqbropQ" - assert items[0].raw == {"dws_drive": True}, "要标记走钉盘下载而不是机器人 downloadCode" - - -def test_dingtalk_download_routes_history_files_to_drive(monkeypatch, tmp_path): - """历史里的文件按 fileId 走 dws drive download；机器人附件仍走 downloadCode。""" - from core.channels.adapters import dingtalk as dt - from core.channels.protocol import InboundMsg - - calls = [] - - async def _fake_run(user_id, args, timeout=40): - calls.append(args) - # 模拟 CLI 写盘 - out_path = args[args.index("--output") + 1] - with open(out_path, "wb") as fh: - fh.write(b"DOCXBYTES") - return ('{"success": true}', "", 0) - - monkeypatch.setattr("core.services.dingtalk_service._run_dws", _fake_run) - - class _C: - owner_user_id = "u_owner" - config = {} - app_id = "k" - channel_id = "c" - - a = dt.DingTalkAdapter() - hist_msg = InboundMsg(channel_id="c", channel_type="dingtalk", text="", chat_type="group", - external_conversation_id="cid", raw={"dws_drive": True}) - got = asyncio.run(a.download_resource(_C(), hist_msg, {"kind": "file", "key": "FID1"})) - assert got == b"DOCXBYTES" - assert calls[0][:2] == ["drive", "download"] - assert calls[0][calls[0].index("--node") + 1] == "FID1" - - # 临时目录要清干净（下载跑在后端宿主上，不是一次性沙箱） - import glob as _g - assert not _g.glob("/tmp/dws_dl_*"), "临时目录未清理" - - # 没有 dws_drive 标记 → 仍走机器人 downloadCode 那条（缺 robotCode 时短路） - bot_msg = InboundMsg(channel_id="c", channel_type="dingtalk", text="", chat_type="group", - external_conversation_id="cid", raw={}) - assert asyncio.run(a.download_resource(_C(), bot_msg, {"key": "dc"})) is None - assert len(calls) == 1, "机器人附件不该走 dws" - - -def test_dingtalk_dws_download_degrades(monkeypatch): - """下载失败/未产出文件 → None，且不残留临时目录。""" - from core.channels.adapters import dingtalk as dt - - async def _fail(user_id, args, timeout=40): - return ("", "no permission", 1) - - monkeypatch.setattr("core.services.dingtalk_service._run_dws", _fail) - - class _C: - owner_user_id = "u" - config = {} - app_id = "k" - channel_id = "c" - - a = dt.DingTalkAdapter() - assert asyncio.run(a._download_via_dws(_C(), "FID")) is None - import glob as _g - assert not _g.glob("/tmp/dws_dl_*") - - class _NoOwner: - owner_user_id = "" - config = {} - app_id = "k" - channel_id = "c" - assert asyncio.run(a._download_via_dws(_NoOwner(), "FID")) is None - - -def test_dingtalk_history_extracts_online_doc_links(): - """钉钉**在线文档**是链接卡片，不是文件——也要解析成句柄，否则模型只看到一条没用的链接。""" - from core.channels.adapters.dingtalk import DingTalkAdapter as D - - card = ( - "季度工作台账\n创建者：张三\n" - "![image](https://img.alicdn.com/x.png)\n" - "[https://alidocs.dingtalk.com/i/nodes/NODEabc123XYZ?corpId=d1]" - "(https://alidocs.dingtalk.com/i/nodes/NODEabc123XYZ?corpId=d1)" - ) - text, refs = D._extract_dws_files(card) - assert refs == [{"kind": "doc", "key": "NODEabc123XYZ", - "name": "季度工作台账"}] - assert text == "", "卡片正文是标题+图标+两遍长 URL，留着纯属浪费提示词" - # 同一 URL 在卡片里出现两次（显示文本 + 链接），不能记成两个附件 - assert len(refs) == 1 - - -def test_dingtalk_doc_read_surfaces_real_reason(monkeypatch): - """平台给了明确理由（如跨组织被拒）时必须原样透出，不能笼统说“可能已过期”。 - - 实测踩到的就是这个：文档属于另一个组织，dws 明确说明了原因，但用户看到的是 - 模糊的“统一身份认证”，往完全错误的方向排查。 - """ - from core.channels.adapters import dingtalk as dt - from core.channels.protocol import ChannelResourceError, InboundMsg - - cross_org = json.dumps({"error": { - "category": "api", - "message": "出于安全考虑，不支持跨组织访问数据，该数据属于组织 A，而 MCP 工具配置的组织为 B", - "server_error_code": "forbidden.accessDenied", - }}, ensure_ascii=False) - - async def _refused(user_id, args, timeout=40): - return (cross_org, "", 1) - - monkeypatch.setattr("core.services.dingtalk_service._run_dws", _refused) - - class _C: - owner_user_id = "u" - config = {} - app_id = "k" - channel_id = "c" - - a = dt.DingTalkAdapter() - msg = InboundMsg(channel_id="c", channel_type="dingtalk", text="", chat_type="group", - external_conversation_id="cid", raw={}) - try: - asyncio.run(a.download_resource(_C(), msg, {"kind": "doc", "key": "NODE1"})) - raise AssertionError("应当抛出带原因的 ChannelResourceError") - except ChannelResourceError as exc: - assert "跨组织" in str(exc) - - # 正常读取：返回 markdown 字节 - async def _ok(user_id, args, timeout=40): - assert args[:2] == ["doc", "read"] - assert args[args.index("--content-format") + 1] == "markdown" - return (json.dumps({"content": "# 台账\n第一行"}, ensure_ascii=False), "", 0) - - monkeypatch.setattr("core.services.dingtalk_service._run_dws", _ok) - got = asyncio.run(a.download_resource(_C(), msg, {"kind": "doc", "key": "NODE1"})) - assert got.decode("utf-8").startswith("# 台账") - - # 业务错误判定不能误伤正常返回 - assert dt.DingTalkAdapter._dws_business_error('{"result":{"a":1}}') is None - assert dt.DingTalkAdapter._dws_business_error("not json") is None - - -def test_channel_read_attachment_names_docs_with_md_extension(db_session, monkeypatch): - """在线文档取回的是 markdown 正文，落库文件名必须带 .md。 - - read_artifact 的解析器是**纯按扩展名**分发的（file_parser.parse_file），标题类文件名 - 没有后缀就会被判成"不支持的格式"——文件下下来了却读不了，等于白取。 - """ - from core.channels.inbound import _find_or_create_session, _record_observed_batch - - conn = _mk_conn(db_session) - msg = _inbound("oc_doc", "group", mid="m_doc") - session = _find_or_create_session(db_session, conn, msg) - _record_observed_batch(db_session, session, [], { - "NODE1": {"name": "季度工作台账", "kind": "doc", - "message_id": "m1", "raw": {}}, - "FID2": {"name": "报表.xlsx", "kind": "file", "message_id": "m2", "raw": {}}, - }) - - monkeypatch.setattr("core.db.engine.SessionLocal", lambda: db_session) - - class _Ad: - async def download_resource(self, conn, inbound, att): - return b"# tai zhang" - - monkeypatch.setattr("core.channels.registry.get_adapter", lambda ct: _Ad()) - - tool = _reg_channel_tool(str(session.user_id), session.chat_id) - doc = _tool_json(asyncio.run(tool("NODE1"))) - assert doc["filename"].endswith(".md"), doc - assert doc["mime_type"].startswith("text/"), doc - - # 解析器确实认这个名字 - from core.content.file_parser import parse_file - assert parse_file(b"# tai zhang", doc["filename"]) == "# tai zhang" - assert parse_file(b"# tai zhang", "季度工作台账") is None, "无后缀本会被判不支持" - - # 普通文件名不受影响，不该被加后缀 - f = _tool_json(asyncio.run(tool("FID2"))) - assert f["filename"] == "报表.xlsx" diff --git a/src/backend/tests/test_citation_anchor.py b/src/backend/tests/test_citation_anchor.py index f3c42679..a07df80d 100644 --- a/src/backend/tests/test_citation_anchor.py +++ b/src/backend/tests/test_citation_anchor.py @@ -1,4 +1,4 @@ -"""统一证据锚点（orchestration/citation_anchor.py）单元测试。 +"""统一证据锚点（core/llm/citations/anchor.py）单元测试。 覆盖：发号器连续性、四层提取降级（__citations__ / 配置映射 / 启发式 / 整体兜底）、 JSON/纯文本回注、跳过名单、错误结果放行、collect_citation_dicts 双路径。 @@ -7,17 +7,9 @@ import json import pytest - -from orchestration.citation_anchor import ( - CITATION_ALLOCATOR, - SKIP_TOOLS, - AnchorAllocator, - anchor_start_for_chat, - annotate_tool_result, - attach_allocator, - collect_citation_dicts, - resolve_allocator, -) +from core.llm.citations.allocator import CITATION_ALLOCATOR, AnchorAllocator, anchor_start_for_chat, attach_allocator, collect_citation_dicts, resolve_allocator +from core.llm.citations.anchor_state import SKIP_TOOLS +from core.llm.citations.anchor import annotate_tool_result @pytest.fixture(autouse=True) diff --git a/src/backend/tests/test_cli_cold_bootstrap.py b/src/backend/tests/test_cli_cold_bootstrap.py new file mode 100644 index 00000000..6971b3c3 --- /dev/null +++ b/src/backend/tests/test_cli_cold_bootstrap.py @@ -0,0 +1,48 @@ +"""Exercise CLI plugin bootstrap in a fresh process before API composition.""" +from pathlib import Path +import os +import subprocess +import sys + + +def test_cli_default_plugins_bootstrap_before_api_import(tmp_path): + backend = Path(__file__).resolve().parents[1] + env = { + key: value for key, value in os.environ.items() + if key.upper() in {"PATH", "SYSTEMROOT", "WINDIR", "TMP", "TEMP", "TMPDIR"} + } + env.update( + PYTHONPATH=str(backend), + HOME=str(tmp_path), + USERPROFILE=str(tmp_path), + HUGAGENT_CAPS_ROOT=str(tmp_path / "caps"), + PYTHONUTF8="1", + JX_EDITION="ce", + REDIS_URL="", + ) + code = """ +import os +import sys +import cli +os.environ[cli._DEFAULT_PLUGIN_BOOTSTRAP_ENV] = "1" +cli.apply_local_env(33298) +from hugagent_cli.schema import ensure_schema_and_seed +ensure_schema_and_seed() +assert "api.app" not in sys.modules +cli.bootstrap_default_plugins_if_needed() +assert cli._default_plugins_marker().is_file() +assert cli.ensure_default_plugins_once() is False +assert "api.app" not in sys.modules +print("CLI_COLD_BOOTSTRAP_OK") +""" + result = subprocess.run( + [sys.executable, "-c", code], + cwd=tmp_path, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + encoding="utf-8", + timeout=90, + ) + assert result.returncode == 0, result.stdout + assert "CLI_COLD_BOOTSTRAP_OK" in result.stdout diff --git a/src/backend/tests/test_content_block_id_migration.py b/src/backend/tests/test_content_block_id_migration.py index 5322faa2..ff8909d3 100644 --- a/src/backend/tests/test_content_block_id_migration.py +++ b/src/backend/tests/test_content_block_id_migration.py @@ -8,7 +8,7 @@ from alembic.operations import Operations from core.db.models import ContentBlock -from core.services.capability_workcopies import _id +from features.desktop.capability.workcopies import record_id as _id def test_capability_record_ids_fit_model(): diff --git a/src/backend/tests/test_credential_registry.py b/src/backend/tests/test_credential_registry.py new file mode 100644 index 00000000..8e5a3a8c --- /dev/null +++ b/src/backend/tests/test_credential_registry.py @@ -0,0 +1,44 @@ +from types import SimpleNamespace +from unittest.mock import AsyncMock +import pytest + +from core.harness.registry import registry, SandboxCredentials, register_sandbox_credentials +from core.sandbox import credential_dispatch as dispatch +from integrations import register_credentials as register_defaults + + +@pytest.fixture(autouse=True) +def isolate_registry(monkeypatch): + monkeypatch.setattr(registry.registration_state(), 'sandbox_credentials', {}) + + +def test_environment_and_mount_order_preserve_registered_contributions(): + register_sandbox_credentials(SandboxCredentials("one", lambda: {"A": "1"}, lambda user: [(user, "one")])) + register_sandbox_credentials(SandboxCredentials("two", lambda: {"B": "2"}, lambda user: [(user, "two")])) + assert dispatch.credential_environment() == {"A": "1", "B": "2"} + assert dispatch.credential_mounts("owner") == [("owner", "one"), ("owner", "two")] + + +@pytest.mark.asyncio +async def test_builtin_command_order_identity_and_writeback_are_preserved(): + register_defaults() + names = ("_ensure_dingtalk_creds", "_ensure_email_creds", "_ensure_lark_creds", "_ensure_yida_creds", "_persist_yida_state") + calls = [] + def callback(name): + async def run(sandbox, user): + calls.append((name, sandbox, user)) + return run + states = {owner: SimpleNamespace(ensure=callback(names[index])) for owner, index in (("dingtalk", 0), ("email", 1), ("lark", 2), ("yida", 3))} + states["yida"].persist = callback(names[4]) + host = SimpleNamespace(credential_state=lambda name, factory: states[name]) + sandbox = object() + script = "dws himalaya lark-cli openyida" + await dispatch.prepare_credentials(host, sandbox, "owner", script) + await dispatch.persist_credentials(host, sandbox, "owner", script) + assert calls == [(name, sandbox, "owner") for name in names] + calls.clear() + await dispatch.prepare_credentials(host, sandbox, None, script) + await dispatch.persist_credentials(host, sandbox, None, script) + await dispatch.prepare_credentials(host, sandbox, "owner", "echo hello") + await dispatch.persist_credentials(host, sandbox, "owner", "echo hello") + assert calls == [] diff --git a/src/backend/tests/test_db_import_order.py b/src/backend/tests/test_db_import_order.py new file mode 100644 index 00000000..c0565a3f --- /dev/null +++ b/src/backend/tests/test_db_import_order.py @@ -0,0 +1,103 @@ +"""Database models load no matter which module a fresh process imports first.""" +from pathlib import Path +import os +import subprocess +import sys + +import pytest + +BACKEND = Path(__file__).resolve().parents[1] + +CHECK = """ +import core.db as db +import core.db.models as models +from core.db import ChatSession, Team, get_db +from core.db.models import * # noqa: F403 +from core.db.models import ProjectMember, Team as ModelTeam +assert ModelTeam is Team and ProjectMember is models.ProjectMember +assert {"Team", "ChatSessionUserState"} <= set(models.__all__) +registry = db.Base.registry._class_registry +assert {"Team", "ProjectMember", "ChatSession"} <= set(registry) +print("DB_IMPORT_ORDER_OK") +""" + + +@pytest.mark.parametrize( + "entry", + [ + "edition_ee.db.model_extensions", + "edition_ee.db.models", + "edition_ee.db.models.feedback", + "core.db.utc_datetime", + "core.auth.local_users", + "core.db.models", + ], +) +def test_models_import_from_any_entry_point(entry, tmp_path): + env = { + key: value for key, value in os.environ.items() + if key.upper() in {"PATH", "SYSTEMROOT", "WINDIR", "TMP", "TEMP", "TMPDIR"} + } + env.update(PYTHONPATH=str(BACKEND), HOME=str(tmp_path), PYTHONUTF8="1", REDIS_URL="") + result = subprocess.run( + [sys.executable, "-c", f"import {entry}\n{CHECK}"], + cwd=tmp_path, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + encoding="utf-8", + timeout=90, + ) + assert result.returncode == 0, result.stdout + assert "DB_IMPORT_ORDER_OK" in result.stdout + + +def test_db_leaf_modules_do_not_load_models(tmp_path): + code = "import sys, core.db.engine, core.db.utc_datetime\nassert 'core.db.models' not in sys.modules\n" + result = subprocess.run( + [sys.executable, "-c", code], + cwd=tmp_path, + env={**os.environ, "PYTHONPATH": str(BACKEND)}, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + encoding="utf-8", + timeout=90, + ) + assert result.returncode == 0, result.stdout + + +@pytest.mark.parametrize( + "entry", + [ + "features.automation.models", + "features.automation.service", + "extensions.mcp_servers.automation_task_mcp.server", + ], +) +def test_automation_relationships_resolve_in_a_cold_process(entry, tmp_path): + code = f""" +import {entry} +from features.automation.models import ScheduledTask +from sqlalchemy.orm import configure_mappers +configure_mappers() +assert ScheduledTask.user.property.mapper.class_.__name__ == "UserShadow" +assert ScheduledTask.plan.property.mapper.class_.__name__ == "Plan" +print("AUTOMATION_MODELS_OK") +""" + result = subprocess.run( + [sys.executable, "-c", code], + cwd=tmp_path, + env={ + **os.environ, + "PYTHONPATH": str(BACKEND), + "DATABASE_URL": f"sqlite:///{tmp_path / 'automation.db'}", + "SQLITE_FALLBACK_URL": f"sqlite:///{tmp_path / 'fallback.db'}", + "REDIS_URL": "", + }, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + encoding="utf-8", + timeout=30, + ) + assert result.returncode == 0, result.stdout + assert "AUTOMATION_MODELS_OK" in result.stdout diff --git a/src/backend/tests/test_default_plugin_bootstrap.py b/src/backend/tests/test_default_plugin_bootstrap.py index db9e52be..9b9533a6 100644 --- a/src/backend/tests/test_default_plugin_bootstrap.py +++ b/src/backend/tests/test_default_plugin_bootstrap.py @@ -5,16 +5,16 @@ import pytest from core.db.models import AdminMcpServer, ContentBlock, InstalledPlugin -from core.plugins import management as plugin_service -from core.plugins.ui import contributions as plugin_ui +from core.capabilities.plugins import management as plugin_service +from features.workbench.ui import contributions as plugin_ui @pytest.fixture(autouse=True) def isolated_catalog_caches(): # Each test uses a fresh DB; process-wide catalog snapshots must not leak # from earlier API/plugin tests into this bootstrap contract. - from core.config.catalog_runtime import invalidate_runtime_catalog_cache - from core.config.catalog_resolver import invalidate_capability_cache + from core.capabilities.catalog.runtime import invalidate_runtime_catalog_cache + from core.capabilities.catalog.resolver import invalidate_capability_cache invalidate_runtime_catalog_cache() invalidate_capability_cache() @@ -104,7 +104,7 @@ def close(self): lambda db: calls.append(db) or True, ) monkeypatch.setattr( - "core.plugins.management.defaults.ensure_visualize_plugin_bootstrapped", + 'core.capabilities.plugins.management.defaults.ensure_visualize_plugin_bootstrapped', lambda db: calls.append(("visualize", db)) or True, ) # 启动钩子同时负责把 ui_contributions 与 bundle 清单对齐（存量安装回填） diff --git a/src/backend/tests/test_desktop_observability.py b/src/backend/tests/test_desktop_observability.py deleted file mode 100644 index dbd22ce3..00000000 --- a/src/backend/tests/test_desktop_observability.py +++ /dev/null @@ -1,633 +0,0 @@ -"""Cloud management copies survive delivery retries and never become runnable chats.""" - -import pytest -from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker - - -@pytest.fixture -def store(tmp_path): - from core.db.models.observability import DesktopOutbox, DesktopRecord, DesktopSyncDevice - - engine = create_engine("sqlite:///" + str(tmp_path / "cloud.db")) - from core.db.models import UserShadow - - for model in (DesktopRecord, DesktopSyncDevice, DesktopOutbox, UserShadow): - model.__table__.create(engine) - return sessionmaker(bind=engine) - - -def test_duplicate_and_stale_delivery_keep_newest_chat(store): - from core.services.desktop_observability import ingest, list_records - - def batch(version, content): - return { - "events": [ - { - "kind": "message", - "object_id": "m1", - "revision": version, - "payload": {"chat_id": "c1", "role": "user", "content": content}, - } - ] - } - - with store() as db: - ingest(db, "alice", "laptop", batch(2, "latest")) - ingest(db, "alice", "laptop", batch(2, "latest")) - ingest(db, "alice", "laptop", batch(1, "old")) - rows, total = list_records(db, kind="message", user_id="alice") - assert total == 1 - assert rows[0]["payload"]["content"] == "latest" - assert list_records(db, kind="message", user_id="bob")[1] == 0 - ingest(db, "bob", "laptop", batch(1, "bob's message")) - assert ( - list_records(db, kind="message", user_id="bob")[0][0]["payload"]["content"] - == "bob's message" - ) - - -def test_capture_upload_and_conditional_ack_survive_retry(tmp_path): - import asyncio - - from core.db.models import ChatMessage, ChatSession, UserShadow - from core.db.models.observability import DesktopOutbox - from core.services.desktop_observability_sync import acknowledge, install_capture, prepare_batch - - engine = create_engine("sqlite:///" + str(tmp_path / "local.db")) - from core.db.models import ChatRun - - for m in (UserShadow, ChatSession, ChatMessage, ChatRun, DesktopOutbox): - m.__table__.create(engine) - factory = sessionmaker(bind=engine) - identity = { - "cloud_base": "https://cloud.example", - "subject": "alice", - "device_id": "device", - "shell_user_center_id": "cloud:cloud.example:443:center", - } - remove = install_capture(factory, lambda: identity) - try: - with factory() as db: - db.add( - UserShadow( - user_id="local-alice", - username="Alice", - user_center_id=identity["shell_user_center_id"], - ) - ) - db.add(ChatSession(chat_id="c", user_id="local-alice", title="Desktop task")) - db.add( - ChatMessage(message_id="m", chat_id="c", role="user", chat_seq=1, content="hello") - ) - db.commit() - with factory() as db: - first = prepare_batch(db, identity) - assert {e["kind"] for e in first["events"]} == {"session", "message"} - with factory() as db: - row = db.get(ChatMessage, "m") - row.content = "newer" - db.commit() - with factory() as db: - acknowledge(db, identity, {"acknowledged": first["events"]}) - retry = prepare_batch(db, identity) - assert len(retry["events"]) == 1 - assert retry["events"][0]["payload"]["content"] == "newer" - with factory() as db: - assert prepare_batch(db, {**identity, "subject": "bob"})["events"] == [] - finally: - remove() - - -def test_cloud_gateway_and_desktop_receipt_are_one_call(store): - from core.services.desktop_observability import ingest, list_records - - event = { - "kind": "tool", - "object_id": "result-1", - "revision": 1, - "payload": { - "tool_name": "search", - "chat_id": "c", - "status": "success", - "tool_args": {"api_key": "secret-value"}, - }, - } - with store() as db: - ingest(db, "alice", "device", {"events": [event]}) - ingest(db, "alice", "device", {"events": [event]}, observation="gateway") - rows, total = list_records(db, kind="tool", user_id="alice") - assert total == 1 - assert rows[0]["observation"] == "gateway" - assert rows[0]["payload"]["tool_args"]["api_key"] == "[redacted]" - - -def test_gateway_stream_arrives_before_log_writer_and_extracts_usage(monkeypatch): - import asyncio - - from core.services import desktop_gateway_observability as obs - from fastapi import FastAPI, Request - from fastapi.responses import StreamingResponse - from fastapi.testclient import TestClient - - captured = [] - monkeypatch.setattr(obs, "submit", lambda *args: captured.append(args)) - app = FastAPI() - app.router.route_class = obs.ObservedGatewayRoute - - @app.post("/gateway/models/model/chat/completions") - async def handler(request: Request): - request.state.desktop_observation_user = "alice" - - async def chunks(): - assert not captured - yield b'data: {"choices":[{"delta":{"content":"hello"}}]}\n\n' - assert not captured - yield b'data: {"usage":{"prompt_tokens":7,"completion_tokens":3}}\n\n' - - return StreamingResponse(chunks(), media_type="text/event-stream") - - response = TestClient(app).post( - "/gateway/models/model/chat/completions", - headers={"x-desktop-device-id": "d", "x-observation-call-id": "try-1"}, - ) - assert response.status_code == 200 - assert "hello" in response.text - assert len(captured) == 1 - assert captured[0][-1]["usage"] == {"prompt_tokens": 7, "completion_tokens": 3} - assert captured[0][-1]["status"] == "success" - - -@pytest.fixture -def local_store(tmp_path): - from core.db.models import UserShadow - from core.db.models.observability import DesktopOutbox - from core.services.desktop_observability_sync import SOURCES - - engine = create_engine("sqlite:///" + str(tmp_path / "desktop.db")) - for model in {UserShadow, DesktopOutbox, *(m for m, _ in SOURCES.values())}: - model.__table__.create(engine) - factory = sessionmaker(bind=engine) - return factory - - -def test_real_upload_endpoint_authorization_and_size_limit(store, monkeypatch): - from api.routes.v1 import desktop_observability as route - from core.db.engine import get_db - from core.services.desktop_observability import MAX_BATCH_BYTES - from fastapi import FastAPI - from fastapi.testclient import TestClient - - monkeypatch.setenv("CONFIG_TOKEN", "test-config") - monkeypatch.delenv("HUGAGENT_DESKTOP_BRIDGE_SECRET", raising=False) - monkeypatch.setattr(route, "SessionLocal", store) - - def database(): - with store() as db: - yield db - - app = FastAPI() - app.include_router(route.router, prefix="/api") - app.include_router(route.admin_router, prefix="/api") - app.dependency_overrides[get_db] = database - client = TestClient(app) - assert ( - client.post("/api/v1/desktop/observability/batch", json={"events": []}).status_code == 401 - ) - assert client.get("/api/v1/admin/desktop-observability/records").status_code == 401 - app.dependency_overrides[route._require_capability_user] = lambda: "alice" - event = { - "kind": "message", - "object_id": "m", - "revision": 1, - "payload": {"content": "private conversation", "user_id": "victim", "role": "user"}, - } - response = client.post( - "/api/v1/desktop/observability/batch", - headers={"x-desktop-device-id": "d"}, - json={"events": [event]}, - ) - assert response.status_code == 200, response.text - data = client.get( - "/api/v1/admin/desktop-observability/records", - headers={"Authorization": "Bearer test-config"}, - ).json()["data"] - assert data["pagination"]["total_items"] == 1 - assert data["items"][0]["user_id"] == "alice" - assert "user_id" not in data["items"][0]["payload"] - assert ( - client.post( - "/api/v1/desktop/observability/batch", - headers={"x-desktop-device-id": "d"}, - content=b"x" * (MAX_BATCH_BYTES + 1), - ).status_code - == 413 - ) - - -def test_offline_restart_and_http_delivery_to_cloud(local_store, store, monkeypatch): - import asyncio - - import httpx - from api.routes.v1 import desktop_observability as route - from core.db.models import ( - ChatMessage, - ChatSession, - SkillCallLog, - SubAgentCallLog, - ToolCallLog, - UserShadow, - ) - from core.services import desktop_cloud_bridge as bridge - from core.services.desktop_observability import list_records - from core.services.desktop_observability_sync import ( - DesktopSyncWorker, - install_capture, - prepare_batch, - ) - from fastapi import FastAPI - - identity = { - "cloud_base": "http://cloud", - "subject": "alice", - "device_id": "d", - "shell_user_center_id": "cloud:cloud:80:alice", - } - state = {"cloud_base": "http://cloud", "token": "test-credential", "device_id": "d"} - monkeypatch.setattr(bridge, "get_state", lambda: state) - monkeypatch.setattr(bridge, "get_identity_state", lambda: identity) - remove = install_capture(local_store, lambda: identity) - try: - with local_store() as db: - db.add( - UserShadow( - user_id="local", - username="Alice", - user_center_id=identity["shell_user_center_id"], - ) - ) - db.add(ChatSession(chat_id="c", user_id="local", title="Research")) - db.add( - ChatMessage(message_id="m", chat_id="c", chat_seq=1, role="user", content="Hello") - ) - db.add( - ToolCallLog( - id="t", user_id="local", chat_id="c", tool_name="search", status="success" - ) - ) - db.add( - SkillCallLog( - id="s", - user_id="local", - chat_id="c", - skill_id="cloud-skill", - skill_source="cloud", - invocation_type="view", - status="success", - ) - ) - db.add( - SubAgentCallLog( - id="a", - user_id="local", - chat_id="c", - subagent_name="Researcher", - status="success", - ) - ) - db.commit() - - async def offline(request): - raise httpx.ConnectError("offline") - - worker = DesktopSyncWorker(local_store, transport=httpx.MockTransport(offline)) - with pytest.raises(httpx.ConnectError): - asyncio.run(worker.sync_once()) - with local_store() as db: - assert len(prepare_batch(db, identity)["events"]) == 5 - monkeypatch.setattr(route, "SessionLocal", store) - app = FastAPI() - app.include_router(route.router, prefix="/api") - app.dependency_overrides[route._require_capability_user] = lambda: "alice" - restarted = DesktopSyncWorker(local_store, transport=httpx.ASGITransport(app=app)) - asyncio.run(restarted.sync_once()) - asyncio.run(restarted.sync_once()) - with local_store() as db: - assert prepare_batch(db, identity)["events"] == [] - with store() as db: - assert list_records(db, user_id="alice")[1] == 5 - assert list_records(db, kind="message")[0][0]["payload"]["content"] == "Hello" - assert list_records(db, kind="skill")[0][0]["payload"]["capability_origin"] == "cloud" - finally: - remove() - - -def test_unicode_large_records_and_secrets_are_safe(store): - import json - - from core.services.desktop_observability import ( - MAX_EVENT_BYTES, - ingest, - list_records, - public_payload, - ) - - payload = public_payload("message", {"content": "中" * 100000, "role": "assistant"}) - assert len(json.dumps(payload, ensure_ascii=False).encode()) < MAX_EVENT_BYTES - with store() as db: - ingest( - db, - "u", - "d", - {"events": [{"kind": "message", "object_id": "m", "revision": 1, "payload": payload}]}, - ) - assert list_records(db, kind="message")[0][0]["payload"]["content"] == "中" * 100000 - payload = public_payload( - "tool", - { - "tool_args": {"api_key": "hidden"}, - "tool_result": '{"api_key":"hidden"} Bearer abcdefghi -----BEGIN PRIVATE KEY-----abc-----END PRIVATE KEY-----', - }, - ) - assert "hidden" not in json.dumps(payload) - assert "abcdefghi" not in json.dumps(payload) - assert "BEGIN PRIVATE KEY" not in json.dumps(payload) - assert public_payload("message", {"content": "x" * 100001})["truncated"] is True - - -def test_gateway_unknown_usage_keeps_desktop_usage_and_parent(store): - from core.services.desktop_observability import ingest, list_records - - with store() as db: - ingest( - db, - "u", - "d", - { - "events": [ - { - "kind": "usage", - "object_id": "local-attempt", - "revision": 1, - "payload": { - "kind": "model", - "call_id": "attempt", - "prompt_tokens": 20, - "completion_tokens": 5, - }, - } - ] - }, - ) - ingest( - db, - "u", - "d", - { - "events": [ - { - "kind": "model", - "object_id": "attempt", - "revision": 2, - "payload": {"usage_known": False, "status": "success"}, - } - ] - }, - observation="gateway", - ) - rows, total = list_records(db, kind="model") - assert total == 1 - assert rows[0]["payload"]["usage"]["prompt_tokens"] == 20 - ingest( - db, - "u", - "d", - { - "events": [ - { - "kind": "tool", - "object_id": "t", - "revision": 1, - "payload": {"subagent_log_id": "parent", "status": "success"}, - } - ] - }, - ) - ingest( - db, - "u", - "d", - { - "events": [ - { - "kind": "tool", - "object_id": "t", - "revision": 2, - "payload": {"status": "success", "duration_ms": 10}, - } - ] - }, - observation="gateway", - ) - assert list_records(db, kind="tool")[0][0]["payload"]["subagent_log_id"] == "parent" - - -def test_bulk_updates_are_uploaded_and_rollback_is_not(local_store): - from core.db.models import ChatMessage, ChatSession, UserShadow - from core.services.desktop_observability_sync import acknowledge, install_capture, prepare_batch - from sqlalchemy import update - - identity = { - "cloud_base": "https://cloud", - "subject": "u", - "device_id": "d", - "shell_user_center_id": "cloud:center", - } - remove = install_capture(local_store, lambda: identity) - try: - with local_store() as db: - db.add(UserShadow(user_id="local", username="U", user_center_id="cloud:center")) - db.add(ChatSession(chat_id="c", user_id="local", title="Before")) - db.add( - ChatMessage( - message_id="m", chat_id="c", chat_seq=1, role="assistant", content="partial" - ) - ) - db.commit() - with local_store() as db: - acknowledge(db, identity, {"acknowledged": prepare_batch(db, identity)["events"]}) - with local_store() as db: - db.execute( - update(ChatMessage).where(ChatMessage.message_id == "m").values(content="final") - ) - db.commit() - with local_store() as db: - batch = prepare_batch(db, identity) - assert len(batch["events"]) == 1 - assert batch["events"][0]["payload"]["content"] == "final" - acknowledge(db, identity, {"acknowledged": batch["events"]}) - with local_store() as db: - db.get(ChatMessage, "m").content = "rolled back" - db.flush() - db.rollback() - with local_store() as db: - assert prepare_batch(db, identity)["events"] == [] - finally: - remove() - - -def test_account_switch_during_upload_does_not_ack_old_account(local_store, monkeypatch): - import asyncio - - import httpx - from core.capabilities.errors import CloudUnavailable - from core.db.models import ChatSession, UserShadow - from core.services import desktop_cloud_bridge as bridge - from core.services.desktop_observability_sync import ( - DesktopSyncWorker, - install_capture, - prepare_batch, - ) - - identity = { - "cloud_base": "http://cloud", - "subject": "alice", - "device_id": "d", - "shell_user_center_id": "cloud:alice", - } - state = {"cloud_base": "http://cloud", "token": "alice-credential", "device_id": "d"} - current = [state] - monkeypatch.setattr(bridge, "get_state", lambda: current[0]) - monkeypatch.setattr(bridge, "get_identity_state", lambda: identity) - remove = install_capture(local_store, lambda: identity) - try: - with local_store() as db: - db.add(UserShadow(user_id="local", username="Alice", user_center_id="cloud:alice")) - db.add(ChatSession(chat_id="c", user_id="local", title="Alice only")) - db.commit() - - async def switch(request): - import json - - body = json.loads(request.content) - assert request.headers["authorization"] == "Bearer alice-credential" - current[0] = {**state, "token": "bob-credential"} - return httpx.Response(200, json={"data": {"acknowledged": body["events"]}}) - - with pytest.raises(CloudUnavailable): - asyncio.run( - DesktopSyncWorker(local_store, transport=httpx.MockTransport(switch)).sync_once() - ) - with local_store() as db: - assert len(prepare_batch(db, identity)["events"]) == 1 - finally: - remove() - - -def test_blocked_upload_does_not_block_task_progress(local_store, monkeypatch): - import asyncio - - import httpx - from core.db.models import ChatSession, UserShadow - from core.services import desktop_cloud_bridge as bridge - from core.services.desktop_observability_sync import DesktopSyncWorker, install_capture - - identity = { - "cloud_base": "http://cloud", - "subject": "u", - "device_id": "d", - "shell_user_center_id": "cloud:u", - } - state = {"cloud_base": "http://cloud", "token": "test", "device_id": "d"} - monkeypatch.setattr(bridge, "get_state", lambda: state) - monkeypatch.setattr(bridge, "get_identity_state", lambda: identity) - remove = install_capture(local_store, lambda: identity) - try: - with local_store() as db: - db.add(UserShadow(user_id="local", username="U", user_center_id="cloud:u")) - db.add(ChatSession(chat_id="c", user_id="local", title="Initial")) - db.commit() - - async def scenario(): - entered, release = asyncio.Event(), asyncio.Event() - - async def slow_cloud(request): - entered.set() - await release.wait() - return httpx.Response(200, json={"data": {"acknowledged": []}}) - - worker = DesktopSyncWorker(local_store, transport=httpx.MockTransport(slow_cloud)) - upload = asyncio.create_task(worker.sync_once()) - await asyncio.wait_for(entered.wait(), timeout=2) - - def advance(): - with local_store() as db: - db.get(ChatSession, "c").title = "Task continued" - db.commit() - - await asyncio.wait_for(asyncio.to_thread(advance), timeout=1) - assert not upload.done() - release.set() - await upload - - asyncio.run(scenario()) - finally: - remove() - - -@pytest.mark.parametrize("finish", ["commit", "rollback"]) -@pytest.mark.parametrize("operation", ["flush", "update", "delete"]) -def test_capture_identity_is_stable_within_transaction_and_refreshes_on_reuse( - local_store, finish, operation -): - from core.db.models import ChatSession, UserShadow - from core.db.models.observability import DesktopOutbox - from core.services.desktop_observability_sync import install_capture - from sqlalchemy import update, delete - - alice = dict( - cloud_base="https://cloud", - subject="alice", - device_id="d", - shell_user_center_id="center-alice", - ) - bob = dict( - cloud_base="https://cloud", subject="bob", device_id="d", shell_user_center_id="center-bob" - ) - with local_store() as db: - for name in ("alice", "bob"): - db.add(UserShadow(user_id=name, username=name, user_center_id="center-" + name)) - db.add(ChatSession(chat_id="existing", user_id="alice", title="before")) - db.commit() - identity = alice - remove = install_capture(local_store, lambda: identity) - try: - with local_store() as db: - db.begin() - identity = bob # Source transaction still belongs to its original account. - if operation == "flush": - db.add(ChatSession(chat_id="new-alice", user_id="alice", title="Alice")) - db.flush() - else: - stmt = ( - update(ChatSession).values(title="after") - if operation == "update" - else delete(ChatSession) - ) - db.execute(stmt.where(ChatSession.chat_id == "existing")) - getattr(db, finish)() - # Reuse the same Session. Neither commit nor rollback may retain Alice. - with db.begin_nested(): - db.add(ChatSession(chat_id="new-bob", user_id="bob", title="Bob")) - db.flush() - db.commit() - with local_store() as db: - rows = db.query(DesktopOutbox).all() - assert [(r.subject, r.object_id) for r in rows if r.subject == "bob"] == [ - ("bob", "new-bob") - ] - assert len([r for r in rows if r.subject == "alice"]) == ( - 1 if finish == "commit" else 0 - ) - finally: - remove() diff --git a/src/backend/tests/test_explicit_capability_invocation.py b/src/backend/tests/test_explicit_capability_invocation.py index ef1048e6..f0910b81 100644 --- a/src/backend/tests/test_explicit_capability_invocation.py +++ b/src/backend/tests/test_explicit_capability_invocation.py @@ -1,13 +1,14 @@ +from core.capabilities.skills import loader_factory as loader_factory_module from types import SimpleNamespace import pytest -from core.services.session_operations.invocation import _resolve_explicit_capability_invocation +from core.agent_runtime.skills import _build_skill_injection from core.chat.schemas import ChatRequest -from core.config import catalog_resolver +from core.capabilities.catalog import resolver as catalog_resolver from core.db.models import AdminMcpServer, AdminSkill, CatalogOverride, InstalledPlugin, UserShadow -from core.plugins import management as plugin_service +from core.capabilities.plugins import management as plugin_service +from core.chat.operations.invocation import resolve_explicit_capability_invocation as _resolve_explicit_capability_invocation from fastapi import HTTPException -from orchestration.workflow import _build_skill_injection def _runtime_catalog(*_args, **_kwargs): @@ -128,7 +129,7 @@ def test_direct_selected_installed_skill_ignores_global_off(db_session, monkeypa def test_public_skill_selection_prefers_current_users_private_install(db_session, monkeypatch): - from core.services.marketplace_service import compute_install_id + from core.capabilities.skills.installation import compute_install_id as compute_install_id monkeypatch.setattr(catalog_resolver, "get_runtime_catalog", _runtime_catalog) private_id = compute_install_id("admin-disabled-skill", "user-a") @@ -209,13 +210,13 @@ def test_explicit_plugin_injection_guides_optional_usage(): def test_explicit_skill_injection_guides_skill_loading(monkeypatch): - import core.agent_skills.loader as skill_loader + import core.capabilities.skills.loader as skill_loader fake_loader = SimpleNamespace( load_all_metadata=lambda: {"security-overview": SimpleNamespace(name="安全概览")}, get_skill_dir=lambda _skill_id: "/tmp/security-overview", ) - monkeypatch.setattr(skill_loader, "get_skill_loader", lambda: fake_loader) + monkeypatch.setattr(loader_factory_module, 'get_skill_loader', lambda: fake_loader) hint = _build_skill_injection( { "skill_id": "security-overview", @@ -335,7 +336,7 @@ def test_installed_plugin_distinguishes_personal_enabled_from_hard_callability( def test_explicit_disabled_connector_is_loaded_but_not_ambient(monkeypatch): - import core.services.mcp_service as mcp_service + import core.capabilities.mcp.service as mcp_service from core.llm.factory.tools import mcp_config as factory_mcp_config class FakeMcpService: @@ -352,12 +353,12 @@ def get_all_servers(self, enabled_only=True): "get_instance", classmethod(lambda _cls: FakeMcpService()), ) - assert factory_mcp_config._effective_mcp_server_keys( + assert factory_mcp_config.effective_mcp_server_keys( None, None, enabled_mcp_ids=["installed-off"] ) == ["installed-off"] assert set(factory_mcp_config._filter_mcp_servers_by_keys(["installed-off"])) == { "installed-off" } - assert factory_mcp_config._effective_mcp_server_keys( + assert factory_mcp_config.effective_mcp_server_keys( None, None, enabled_mcp_ids=["enabled"] ) == ["enabled"] diff --git a/src/backend/tests/test_extensions_layout.py b/src/backend/tests/test_extensions_layout.py index a3f05fe5..8e8cdf68 100644 --- a/src/backend/tests/test_extensions_layout.py +++ b/src/backend/tests/test_extensions_layout.py @@ -1,4 +1,6 @@ """Distribution contracts for bundled extensions and platform agent defaults.""" +from core.llm import prompt_files as prompt_files_module +from core.llm import prompt_rendering as prompt_rendering_module import importlib.util import json @@ -18,10 +20,10 @@ def test_extension_roots_are_independent_of_working_directory(tmp_path, monkeypa assert EXTENSIONS_DIR == Path(__file__).resolve().parents[1] / "extensions" for directory in (SKILLS_DIR, AGENTS_DIR, PLUGINS_DIR, MCP_SERVERS_DIR): assert directory.is_dir() - from core.agent_skills.config import get_default_skill_sources - from core.plugins.packaging.definitions import PLUGIN_BUNDLES_DIR - from core.services.agent_market_service import AGENT_MARKET_DIR - from core.services.marketplace_service import MARKETPLACE_DIR + from core.capabilities.skills.config import get_default_skill_sources + from core.capabilities.plugins.packaging.definitions import PLUGIN_BUNDLES_DIR + from features.marketplace.agents_state import AGENT_MARKET_DIR + from features.marketplace.skills_state import MARKETPLACE_DIR assert get_default_skill_sources()[0].root_dir == SKILLS_DIR / "default" assert PLUGIN_BUNDLES_DIR == PLUGINS_DIR @@ -30,11 +32,11 @@ def test_extension_roots_are_independent_of_working_directory(tmp_path, monkeypa def test_builtin_agent_packages_preserve_identity_permissions_and_prompt_source(monkeypatch): - from core.llm.builtin_subagents import BUILTIN_SUBAGENTS, load_builtin_subagent_prompt - from core.services import prompt_version_service as pvs + from core.capabilities.agents.builtin import BUILTIN_SUBAGENTS, load_builtin_subagent_prompt + from core.llm import prompt_versions as pvs from extensions.paths import DEFAULT_AGENTS_DIR - monkeypatch.setattr(pvs, "render_active_prompt_part", lambda *args: None) + monkeypatch.setattr(model_services.registration_state(), 'prompt_renderer', lambda *args: None) for spec in BUILTIN_SUBAGENTS: package = DEFAULT_AGENTS_DIR / spec.role manifest = json.loads((package / "agent.json").read_text(encoding="utf-8")) @@ -47,16 +49,16 @@ def test_builtin_agent_packages_preserve_identity_permissions_and_prompt_source( load_builtin_subagent_prompt(spec) == (package / "system.md").read_text(encoding="utf-8").strip() ) - assert pvs._fs_dir("subagents") == DEFAULT_AGENTS_DIR - parts = {part["part_id"]: part["content"].strip() for part in pvs._read_fs_parts("subagents")} + assert prompt_files_module.fs_dir("subagents") == DEFAULT_AGENTS_DIR + parts = {part["part_id"]: part["content"].strip() for part in prompt_files_module.read_fs_parts("subagents")} assert parts == {spec.role: load_builtin_subagent_prompt(spec) for spec in BUILTIN_SUBAGENTS} def test_builtin_agent_prompt_still_prefers_active_database_version(monkeypatch): - from core.llm.builtin_subagents import get_builtin_subagent, load_builtin_subagent_prompt - from core.services import prompt_version_service as pvs + from core.capabilities.agents.builtin import get_builtin_subagent, load_builtin_subagent_prompt + from core.llm import prompt_versions as pvs - monkeypatch.setattr(pvs, "render_active_prompt_part", lambda kind, role: "active " + role) + monkeypatch.setattr(model_services.registration_state(), 'prompt_renderer', lambda kind, role: "active " + role) assert load_builtin_subagent_prompt(get_builtin_subagent("builtin.worker")) == "active worker" @@ -69,3 +71,5 @@ def test_mcp_registry_loads_new_package_and_old_module_is_absent(): assert module is not None assert Path(module.origin).is_relative_to(MCP_SERVERS_DIR) assert importlib.util.find_spec("mcp_servers") is None + +from core.harness.registry import model_services diff --git a/src/backend/tests/test_hook_contributions.py b/src/backend/tests/test_hook_contributions.py new file mode 100644 index 00000000..ad791dbe --- /dev/null +++ b/src/backend/tests/test_hook_contributions.py @@ -0,0 +1,47 @@ +"""Contributed hooks reach each assembled agent with isolated request state.""" +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest +from core.harness.registry import register_hook_factory +from core.harness.registry import hook_factories +from core.harness.hooks import Decision, HookRejected, HookSpec, HookStage, Invocation +from core.llm.factory.request import AgentRequest +from core.llm.factory.runtime.middleware_stack import middleware_stack + + +@pytest.mark.asyncio +async def test_factory_hooks_reach_each_agent(monkeypatch): + from unittest.mock import AsyncMock + monkeypatch.setattr('core.chat.steer.take_pending_steer', AsyncMock(return_value=None)) + monkeypatch.setattr(hook_factories.registration_state(), 'factories', []) + requests, called = [], [] + def factory(request): + requests.append(request.run_id) + def guard(invocation): + called.append(request.run_id) + return Decision.reject("policy denied") + yield HookSpec("module-guard", HookStage.BEFORE_TOOL, guard) + register_hook_factory(factory) + register_hook_factory(factory) + buses = [] + for run_id in ("first", "second"): + toolkit = Mock(function_tools=[], permission_specs={}) + _, adapters, _, _ = await middleware_stack( + AgentRequest(run_id=run_id), _api_scope=None, _eval_scope=None, + _build_toolkit=lambda: toolkit, _interactive=True, _ontology_runtime={}, + _plan_tool_enabled=False, _proj_scope=None, _sbx_sess=None, + _subagent_model_pinned=False, _tool_approval_available=False, + default_model=SimpleNamespace(model="test"), + profile=SimpleNamespace(intervention_rules=[]), tool_schemas=[], toolkit=toolkit) + bus = adapters[0].bus + # Isolate persistent event/usage recording; execute the real hook policy. + from core.harness.events import EventSink + bus.event_sink = EventSink() + bus.usage_recorder = None + with pytest.raises(HookRejected, match="policy denied"): + await bus.enforce(Invocation.create(run_id=run_id, stage=HookStage.BEFORE_TOOL, operation_name="test")) + buses.append(bus) + assert requests == ["first", "second"] + assert called == ["first", "second"] + assert buses[0] is not buses[1] diff --git a/src/backend/tests/test_iter_budget_middleware.py b/src/backend/tests/test_iter_budget_middleware.py index 281d052c..f70d78c9 100644 --- a/src/backend/tests/test_iter_budget_middleware.py +++ b/src/backend/tests/test_iter_budget_middleware.py @@ -1,15 +1,18 @@ """IterBudgetReminderMiddleware unit tests. -Tests only the pure logic _maybe_remind (synchronous), without starting a real agent: the middleware only depends on +Tests the neutral budget policy without starting a real agent. It consumes agent.react_config.max_iters / agent.state.{cur_iter, reply_id, context}, which can be faked with SimpleNamespace. """ +from core.evolution.profiles import agent_profile as agent_profile_module +from core.evolution.profiles import agent_profile_constants as agent_profile_state_module from types import SimpleNamespace import pytest -from core.llm.middlewares import IterBudgetReminderMiddleware +from core.llm.context.reasoning_hooks import IterationBudgetHook +from core.harness.hooks import Invocation, HookStage def _fake_agent(max_iters: int, cur_iter: int, reply_id: str = "r1"): @@ -27,24 +30,40 @@ def _reminder_texts(agent) -> list: return texts -def _remind(mw, agent): - mw._maybe_remind(agent, *mw._budget(agent)) +def _decision(policy, agent, tool_choice=None): + return policy(Invocation.create(run_id="budget-test", stage=HookStage.TRANSFORM_CONTEXT, + operation_name="agent_reasoning", data={"runtime": { + "max_iters": agent.react_config.max_iters, "cur_iter": agent.state.cur_iter, + "reply_id": agent.state.reply_id}, "context_items": [], "tool_choice": tool_choice})) -def _force(mw, agent, input_kwargs): - max_iters, _cur, remaining = mw._budget(agent) - return mw._maybe_force_text(input_kwargs, max_iters, remaining) +def _remind(policy, agent): + from core.llm.runtime.context_hook_adapter import append_hook_context + decision = _decision(policy, agent) + append_hook_context(agent, SimpleNamespace(data=decision.patch)) + + +def _force(policy, agent, input_kwargs): + from agentscope.tool import ToolChoice + # An explicit selection is never replaced; use a serializable marker for the snapshot. + existing = input_kwargs.get("tool_choice") + decision = _decision(policy, agent, {"mode": "auto"} if existing is not None else None) + choice = decision.patch.get("tool_choice") + if choice is not None: + from core.harness.events import thaw_value + return {**input_kwargs, "tool_choice": ToolChoice.model_validate(thaw_value(choice))} + return input_kwargs def test_no_remind_far_from_limit(): - mw = IterBudgetReminderMiddleware() + mw = IterationBudgetHook() agent = _fake_agent(max_iters=10, cur_iter=5) # 5 iterations left _remind(mw, agent) assert agent.state.context == [] def test_remind_at_threshold_mentions_remaining(): - mw = IterBudgetReminderMiddleware() + mw = IterationBudgetHook() agent = _fake_agent(max_iters=10, cur_iter=8) # 2 iterations left (including this one) _remind(mw, agent) texts = _reminder_texts(agent) @@ -54,7 +73,7 @@ def test_remind_at_threshold_mentions_remaining(): def test_last_round_forbids_tool_calls(): - mw = IterBudgetReminderMiddleware() + mw = IterationBudgetHook() agent = _fake_agent(max_iters=10, cur_iter=9) # last iteration _remind(mw, agent) texts = _reminder_texts(agent) @@ -64,7 +83,7 @@ def test_last_round_forbids_tool_calls(): def test_dedupe_same_round_reminds_once(): - mw = IterBudgetReminderMiddleware() + mw = IterationBudgetHook() agent = _fake_agent(max_iters=10, cur_iter=8) _remind(mw, agent) _remind(mw, agent) # same (reply_id, cur_iter) triggered again @@ -72,7 +91,7 @@ def test_dedupe_same_round_reminds_once(): def test_escalates_across_rounds(): - mw = IterBudgetReminderMiddleware() + mw = IterationBudgetHook() agent = _fake_agent(max_iters=10, cur_iter=8) _remind(mw, agent) agent.state.cur_iter = 9 # enter the next iteration @@ -84,7 +103,7 @@ def test_escalates_across_rounds(): def test_new_reply_resets_dedupe(): - mw = IterBudgetReminderMiddleware() + mw = IterationBudgetHook() agent = _fake_agent(max_iters=10, cur_iter=9, reply_id="r1") _remind(mw, agent) # New reply: cur_iter is reset to zero and then runs to the critical iteration again @@ -97,7 +116,7 @@ def test_new_reply_resets_dedupe(): @pytest.mark.parametrize("max_iters", [1, 2, 3]) def test_tiny_budget_never_reminds(max_iters): - mw = IterBudgetReminderMiddleware() # threshold=2 -> max_iters<=3 skipped + mw = IterationBudgetHook() # threshold=2 -> max_iters<=3 skipped agent = _fake_agent(max_iters=max_iters, cur_iter=max(0, max_iters - 1)) _remind(mw, agent) assert agent.state.context == [] @@ -107,7 +126,7 @@ def test_tiny_budget_never_reminds(max_iters): def test_force_text_on_final_round(): - mw = IterBudgetReminderMiddleware() + mw = IterationBudgetHook() agent = _fake_agent(max_iters=10, cur_iter=9) # final round out = _force(mw, agent, {"tool_choice": None}) tc = out.get("tool_choice") @@ -115,7 +134,7 @@ def test_force_text_on_final_round(): def test_no_force_text_before_final_round(): - mw = IterBudgetReminderMiddleware() + mw = IterationBudgetHook() agent = _fake_agent(max_iters=10, cur_iter=8) # 2 left out = _force(mw, agent, {"tool_choice": None}) assert out.get("tool_choice") is None @@ -123,7 +142,7 @@ def test_no_force_text_before_final_round(): def test_force_text_respects_explicit_tool_choice(): sentinel = object() - mw = IterBudgetReminderMiddleware() + mw = IterationBudgetHook() agent = _fake_agent(max_iters=10, cur_iter=9) out = _force(mw, agent, {"tool_choice": sentinel}) assert out["tool_choice"] is sentinel @@ -131,7 +150,7 @@ def test_force_text_respects_explicit_tool_choice(): @pytest.mark.parametrize("max_iters", [1, 2, 3]) def test_force_text_skips_tiny_budget(max_iters): - mw = IterBudgetReminderMiddleware() + mw = IterationBudgetHook() agent = _fake_agent(max_iters=max_iters, cur_iter=max(0, max_iters - 1)) out = _force(mw, agent, {"tool_choice": None}) assert out.get("tool_choice") is None @@ -139,7 +158,7 @@ def test_force_text_skips_tiny_budget(max_iters): def test_force_text_kill_switch(monkeypatch): monkeypatch.setenv("CHAT_FINAL_ITER_FORCE_TEXT", "false") - mw = IterBudgetReminderMiddleware() + mw = IterationBudgetHook() agent = _fake_agent(max_iters=10, cur_iter=9) out = _force(mw, agent, {"tool_choice": None}) assert out.get("tool_choice") is None @@ -167,15 +186,15 @@ def test_turn_budget_hint_is_byte_stable(): def test_unbounded_profile_is_valid_and_bad_bounds_are_not(): - from core.evolution import agent_profile as AP + from core.evolution.profiles import agent_profile as AP - unbounded = AP.builtin_profile() - assert unbounded.max_react_turns == AP.UNBOUNDED_REACT_TURNS - ok, problems = AP.validate_profile(unbounded) + unbounded = agent_profile_module.builtin_profile() + assert unbounded.max_react_turns == agent_profile_state_module.UNBOUNDED_REACT_TURNS + ok, problems = agent_profile_module.validate_profile(unbounded) assert ok, problems - fenced = AP.builtin_profile() + fenced = agent_profile_module.builtin_profile() fenced.max_react_turns = 1 # below MIN_REACT_TURNS, and not the 0 sentinel - ok, problems = AP.validate_profile(fenced) + ok, problems = agent_profile_module.validate_profile(fenced) assert ok is False assert any("最大轮数" in p for p in problems) diff --git a/src/backend/tests/test_leader_election.py b/src/backend/tests/test_leader_election.py index 12ad6162..f2283272 100644 --- a/src/backend/tests/test_leader_election.py +++ b/src/backend/tests/test_leader_election.py @@ -127,3 +127,51 @@ async def step() -> None: sink.append("x") return step + +async def test_shutdown_cannot_delete_successor_lease(shared_store): + holder = leader.Leadership("shutdown") + holder.start(on_elected=_noop, on_deposed=_noop) + await asyncio.sleep(0.01) + await shared_store.drop(holder._key) + successor = leader.Leadership("shutdown") + assert await successor._hold() + await holder.stop() + assert await shared_store.get(successor._key) == successor._token + + +async def test_shutdown_stops_work_before_releasing_role(shared_store): + observed = [] + holder = leader.Leadership("ordered") + async def stopped(): + observed.append(await shared_store.get(holder._key)) + holder.start(on_elected=_noop, on_deposed=stopped) + await asyncio.sleep(0.01) + await holder.stop() + assert observed == [holder._token] + + +async def test_failed_gate_cannot_admit_other_workers(shared_store): + async def fail(): + raise ValueError("required initialization failed") + with pytest.raises(Exception): + await leader.run_once("failure", fail) + with pytest.raises(Exception): + await leader.run_once("failure", _noop) + + +async def test_renewal_uncertainty_deposes_work(shared_store, monkeypatch): + stopped = asyncio.Event() + holder = leader.Leadership("failure", lease_seconds=1) + holder.start(on_elected=_noop, on_deposed=_recorder_event(stopped)) + await asyncio.sleep(0.02) + async def failed(*args, **kwargs): + raise ConnectionError("Redis lost") + monkeypatch.setattr(shared_store, "compare_exchange", failed) + await asyncio.wait_for(stopped.wait(), 1) + await holder.stop() + + +def _recorder_event(event): + async def record(): + event.set() + return record diff --git a/src/backend/tests/test_local_memory_defaults.py b/src/backend/tests/test_local_memory_defaults.py index e4290ce5..88aeae3a 100644 --- a/src/backend/tests/test_local_memory_defaults.py +++ b/src/backend/tests/test_local_memory_defaults.py @@ -146,6 +146,7 @@ def test_cli_reconfigures_redirected_gbk_stream_to_utf8(monkeypatch): [ "desktop/requirements-desktop-windows-py311.lock", "desktop/requirements-desktop-linux-x86_64-py311.lock", + "desktop/requirements-desktop-linux-aarch64-py311.lock", "desktop/requirements-desktop-macos-aarch64-py311.lock", "desktop/requirements-desktop-macos-x86_64-py311.lock", ], @@ -157,7 +158,7 @@ def test_desktop_offline_runtime_includes_persistent_memory_stack(relative_path) assert "mem0ai==" in lock assert "protobuf==" in lock assert "pymilvus==2.5.18" in lock - assert "milvus-lite==3.1.0" in lock + assert "milvus-lite==3.2.2" in lock def test_hybrid_does_not_install_local_default_plugins(monkeypatch): diff --git a/src/backend/tests/test_local_onboarding.py b/src/backend/tests/test_local_onboarding.py new file mode 100644 index 00000000..932b93a0 --- /dev/null +++ b/src/backend/tests/test_local_onboarding.py @@ -0,0 +1,95 @@ +"""CE local account bootstrap and web onboarding acceptance.""" +from types import SimpleNamespace +import pytest +from core.auth import local_user_bootstrap, local_user_policy, local_users + + +def configure_local_auth(monkeypatch): + settings = SimpleNamespace(edition=SimpleNamespace(edition="ce"), auth=SimpleNamespace(password_min_length=8)) + for module in (local_user_bootstrap, local_user_policy, local_users): + monkeypatch.setattr(module, "settings", settings) + return settings + + +def test_ce_default_admin_is_idempotent_and_requires_password_change(db_session, monkeypatch): + from core.auth.password import verify_password + from core.db.models import LocalUser, UserShadow + + configure_local_auth(monkeypatch) + + user_id, created = local_user_bootstrap.ensure_ce_default_admin(db_session) + assert (user_id, created) == ("user_ce_admin", True) + + shadow = db_session.query(UserShadow).filter_by(user_id=user_id).one() + local = db_session.query(LocalUser).filter_by(user_id=user_id).one() + assert shadow.username == "admin" + assert verify_password("admin", local.password_hash) + assert shadow.extra_data["role"] == "super_admin" + assert shadow.extra_data["can_add_skill"] is True + assert shadow.extra_data["can_add_mcp"] is True + assert shadow.extra_data["can_import_plugin"] is True + assert shadow.extra_data["can_create_private_kb"] is True + assert shadow.extra_data["can_create_public_kb"] is False + assert shadow.extra_data["can_create_channel_bot"] is True + assert shadow.extra_data["can_switch_model"] is True + assert shadow.extra_data["can_use_ontology_validation"] is True + assert shadow.extra_data["must_change_password"] is True + assert shadow.extra_data["onboarding_required"] is True + + assert local_user_bootstrap.ensure_ce_default_admin(db_session) == (user_id, False) + result = local_users.LocalUserService(db_session).change_password( + user_id, + old_password="admin", + new_password="new-password-123", + ) + assert result.ok is True + db_session.refresh(shadow) + assert "must_change_password" not in shadow.extra_data + assert shadow.extra_data["onboarding_required"] is True + login = local_users.LocalUserService(db_session).authenticate("admin", "new-password-123") + assert login.ok + assert login.user_info["onboarding_required"] is True + + + +def test_ce_web_onboarding_requires_main_model_and_clears_checkpoint(db_session, monkeypatch): + import api.routes.v1.users as user_routes + from core.auth.backend import UserContext + from core.models import repository as model_repository + from core.db.models import UserShadow + from core.infra.exceptions import ValidationError + + fake_settings = configure_local_auth(monkeypatch) + monkeypatch.setattr(user_routes, "settings", fake_settings) + + user_id, _ = local_user_bootstrap.ensure_ce_default_admin(db_session) + local_users.LocalUserService(db_session).change_password( + user_id, + old_password="admin", + new_password="new-password-123", + ) + user = UserContext( + user_id=user_id, + user_center_id=user_id, + username="admin", + ) + + with pytest.raises(ValidationError, match="主对话模型"): + user_routes.complete_my_onboarding(user=user, db=db_session) + + provider = model_repository.create_provider( + db_session, + display_name="Test chat", + provider_type="chat", + base_url="http://model.test/v1", + api_key="test-key", + model_name="test-model", + ) + model_repository.assign_role(db_session, "main_agent", provider.provider_id) + + response = user_routes.complete_my_onboarding(user=user, db=db_session) + + assert response["data"]["onboarding_required"] is False + shadow = db_session.query(UserShadow).filter_by(user_id=user_id).one() + assert "onboarding_required" not in shadow.extra_data + assert shadow.extra_data["onboarding_completed_version"] == 1 diff --git a/src/backend/tests/test_local_plugin_sidecars.py b/src/backend/tests/test_local_plugin_sidecars.py index 63e1c19f..ef79d44c 100644 --- a/src/backend/tests/test_local_plugin_sidecars.py +++ b/src/backend/tests/test_local_plugin_sidecars.py @@ -1,9 +1,16 @@ """Hybrid native services start only for an owned, selected local package.""" -from types import SimpleNamespace + import asyncio +from types import SimpleNamespace + +import core.infra.local_runtime.lifecycle as local_runtime_lifecycle +import core.infra.local_runtime.startup as startup +import core.infra.local_runtime.startup as host +import features.desktop.local.plugins.sidecars as sidecars +import features.desktop.local.plugins.sidecars as native import pytest -from orchestration import local_plugin_sidecars as native, local_subprocess as host -from core.config.mcp_config import _mcp_http_url +from core.config.mcp_config import mcp_http_url as _mcp_http_url + @pytest.mark.asyncio async def test_owned_bundled_server_is_supervised_once(monkeypatch): @@ -14,26 +21,26 @@ async def test_owned_bundled_server_is_supervised_once(monkeypatch): monkeypatch.setattr(config, "settings", SimpleNamespace( deploy=SimpleNamespace(is_local=True), server=SimpleNamespace(mcp_host="127.0.0.1") )) - monkeypatch.setattr(native, "_lock", asyncio.Lock()) - monkeypatch.setattr(host, "_cloud_serves_tools", lambda: True) - monkeypatch.setattr(host, "_WATCHDOG", None) - monkeypatch.setattr(host, "_PROCS", []) - monkeypatch.setattr(host, "_SPECS", {}) + monkeypatch.setattr("core.infra.local_runtime.dynamic_servers._lock", asyncio.Lock()) + monkeypatch.setattr(startup, "_cloud_serves_tools", lambda: True) + monkeypatch.setattr(startup, "_WATCHDOG", None) + monkeypatch.setattr(startup, "_PROCS", []) + monkeypatch.setattr(startup, "_SPECS", {}) async def port(*args): return False async def spawn(label, argv): calls.append((label, argv)); host._PROCS.append((label, proc)); return proc async def ready(*args, **kwargs): calls.append("ready") async def tools(*args): return {"browser_open"} - monkeypatch.setattr(host, "_tcp_port_ready", port) - monkeypatch.setattr(host, "_spawn", spawn) - monkeypatch.setattr(host, "_wait_for_mcp_ports", ready) - monkeypatch.setattr(host, "_list_mcp_tool_names", tools) - monkeypatch.setattr(host, "_start_watchdog", lambda: calls.append("watchdog")) + monkeypatch.setattr(startup, "_tcp_port_ready", port) + monkeypatch.setattr(startup, "_spawn", spawn) + monkeypatch.setattr(startup, "_wait_for_mcp_ports", ready) + monkeypatch.setattr(startup, "_list_mcp_tool_names", tools) + monkeypatch.setattr(startup, "_start_watchdog", lambda: calls.append("watchdog")) cfg = {"url": _mcp_http_url("browser_runtime"), "origin": "local_plugin", "execution_scope": "local", "owner_user_id": "owner"} await native.ensure_native_servers({"owned": cfg}) owner_loop = asyncio.get_running_loop() - monkeypatch.setattr(host, "_WATCHDOG", SimpleNamespace(get_loop=lambda: owner_loop)) + monkeypatch.setattr(startup, "_WATCHDOG", SimpleNamespace(get_loop=lambda: owner_loop)) await asyncio.to_thread(lambda: asyncio.run(native.ensure_native_servers({"owned": cfg}))) assert sum(isinstance(c, tuple) for c in calls) == 1 assert calls[0][1][2] == "extensions.mcp_servers.browser_runtime_mcp.server" @@ -44,20 +51,20 @@ async def tools(*args): return {"browser_open"} host._PROCS.clear() async def occupied(*args): return True - monkeypatch.setattr(host, "_tcp_port_ready", occupied) + monkeypatch.setattr(startup, "_tcp_port_ready", occupied) with pytest.raises(RuntimeError, match="another process"): await native.ensure_native_servers({"owned": cfg}) @pytest.mark.asyncio async def test_demand_spawn_and_watchdog_reuse_one_lifecycle_owner(monkeypatch): - from orchestration import local_sidecar_lifecycle as lifecycle - monkeypatch.setattr(lifecycle, "_spawn_lock", asyncio.Lock()) - monkeypatch.setattr(host, "_PROCS", []) - monkeypatch.setattr(host, "_SPECS", {}) - monkeypatch.setattr(host, "_SHUTTING_DOWN", False) - monkeypatch.setattr(host, "_child_env", lambda: {}) - monkeypatch.setattr(host, "no_window_kwargs", lambda: {}) + import core.infra.local_runtime.lifecycle as lifecycle + monkeypatch.setattr(local_runtime_lifecycle, "_spawn_lock", asyncio.Lock()) + monkeypatch.setattr(startup, "_PROCS", []) + monkeypatch.setattr(startup, "_SPECS", {}) + monkeypatch.setattr(startup, "_SHUTTING_DOWN", False) + monkeypatch.setattr(startup, "_child_env", lambda: {}) + monkeypatch.setattr(startup, "no_window_kwargs", lambda: {}) spawned = [] proc = SimpleNamespace(returncode=None, pid=55) async def create(*args, **kwargs): @@ -81,8 +88,8 @@ async def create(*args, **kwargs): @pytest.mark.asyncio async def test_failed_start_cleanup_has_kill_fallback_and_revokes_restart(monkeypatch): - from orchestration import local_sidecar_lifecycle as lifecycle - monkeypatch.setattr(lifecycle, "_spawn_lock", asyncio.Lock()) + import core.infra.local_runtime.lifecycle as lifecycle + monkeypatch.setattr(local_runtime_lifecycle, "_spawn_lock", asyncio.Lock()) calls = [] class Child: returncode = None @@ -90,8 +97,8 @@ def terminate(self): calls.append("terminate") def kill(self): calls.append("kill"); self.returncode = -9 async def wait(self): return self.returncode proc = Child() - monkeypatch.setattr(host, "_PROCS", [("plugin_mcp:test", proc)]) - monkeypatch.setattr(host, "_SPECS", {"plugin_mcp:test": ["test"]}) + monkeypatch.setattr(startup, "_PROCS", [("plugin_mcp:test", proc)]) + monkeypatch.setattr(startup, "_SPECS", {"plugin_mcp:test": ["test"]}) async def wait_for(awaitable, timeout): calls.append(timeout) if "kill" not in calls: @@ -106,14 +113,14 @@ async def wait_for(awaitable, timeout): @pytest.mark.asyncio async def test_shutdown_reaps_spawn_in_flight(monkeypatch): - from orchestration import local_sidecar_lifecycle as lifecycle - monkeypatch.setattr(lifecycle, "_spawn_lock", asyncio.Lock()) - monkeypatch.setattr(host, "_PROCS", []) - monkeypatch.setattr(host, "_SPECS", {}) - monkeypatch.setattr(host, "_SHUTTING_DOWN", False) - monkeypatch.setattr(host, "_WATCHDOG", None) - monkeypatch.setattr(host, "_child_env", lambda: {}) - monkeypatch.setattr(host, "no_window_kwargs", lambda: {}) + import core.infra.local_runtime.lifecycle as lifecycle + monkeypatch.setattr(local_runtime_lifecycle, "_spawn_lock", asyncio.Lock()) + monkeypatch.setattr(startup, "_PROCS", []) + monkeypatch.setattr(startup, "_SPECS", {}) + monkeypatch.setattr(startup, "_SHUTTING_DOWN", False) + monkeypatch.setattr(startup, "_WATCHDOG", None) + monkeypatch.setattr(startup, "_child_env", lambda: {}) + monkeypatch.setattr(startup, "no_window_kwargs", lambda: {}) monkeypatch.setattr(lifecycle.os, "name", "posix") entered, resume = asyncio.Event(), asyncio.Event() class Child: diff --git a/src/backend/tests/test_local_profile.py b/src/backend/tests/test_local_profile.py index 8f967e05..e180121d 100644 --- a/src/backend/tests/test_local_profile.py +++ b/src/backend/tests/test_local_profile.py @@ -9,6 +9,8 @@ - super_admin bootstrap writes ``users_shadow.extra_data.role`` - local single-origin hosting: ``/api`` prefix strip + SPA fallback """ +from core.chat.projection.codec import chat_projection_codec +from core.files.content import branding import asyncio import os @@ -20,7 +22,6 @@ # ── Redis-free local profile ───────────────────────────────────────────────── - def _local_redis_settings(monkeypatch): """Point core.infra.redis at an unconfigured (local-profile) deployment.""" import core.infra.redis as rmod @@ -55,7 +56,7 @@ async def _open_client(rmod): async def test_local_profile_picks_in_process_backends(monkeypatch): """Both store-agnostic seams follow the same predicate.""" import core.infra.ephemeral as ephemeral - import orchestration.run_event_stream as res + import core.runs.events.store as res _local_redis_settings(monkeypatch) monkeypatch.setattr(ephemeral, "redis_configured", lambda: False) @@ -72,9 +73,9 @@ async def test_local_event_log_blocks_then_wakes_on_append(monkeypatch): Busy-returning empty would spin the CPU; never waking would leave the UI on "starting task" for the whole run — the failure the imitation used to cause. """ - import orchestration.run_event_stream as res + import core.runs.events.store as res - stream = res.LocalRunEventStream() + stream = res.LocalRunEventStream(chat_projection_codec()) await stream.append("run-1", {"type": "content", "_offset": 1}) cursor = (await stream.read("run-1"))[-1][0] @@ -111,7 +112,7 @@ def test_bigint_pk_autoincrements_on_sqlite(db_session): def test_seed_builtin_mcp_is_idempotent_and_templates_url(db_session): from core.db.models import AdminMcpServer - from core.services.mcp_service import BUILTIN_MCP_SERVERS, seed_builtin_mcp_servers_if_empty + from core.capabilities.mcp.service import BUILTIN_MCP_SERVERS, seed_builtin_mcp_servers_if_empty from extensions.mcp_servers._ports import PORTS seeded = seed_builtin_mcp_servers_if_empty(db_session) @@ -125,12 +126,12 @@ def test_seed_builtin_mcp_is_idempotent_and_templates_url(db_session): sample_id = next(iter(expected_ids)) sample = db_session.query(AdminMcpServer).filter_by(server_id=sample_id).one() assert sample.transport == "streamable_http" - assert sample.url.endswith(f":{PORTS[sample_id]}/mcp/") + assert sample.url.endswith(f":{PORTS[sample_id]}/mcp") def test_seed_builtin_mcp_skips_when_catalog_present(db_session): from core.db.models import AdminMcpServer - from core.services.mcp_service import seed_builtin_mcp_servers_if_empty + from core.capabilities.mcp.service import seed_builtin_mcp_servers_if_empty # A pre-existing global built-in row means the catalog already exists. db_session.add( @@ -148,7 +149,7 @@ def test_seed_builtin_mcp_skips_when_catalog_present(db_session): def test_retired_report_export_mcp_is_not_seeded_and_legacy_row_is_pruned(db_session): from core.db.models import AdminMcpServer - from core.services.mcp_service import BUILTIN_MCP_SERVERS, prune_removed_builtin_mcp_servers + from core.capabilities.mcp.service import BUILTIN_MCP_SERVERS, prune_removed_builtin_mcp_servers from extensions.mcp_servers._ports import PORTS assert "report_export_mcp" not in PORTS @@ -174,8 +175,8 @@ def test_retired_report_export_mcp_is_not_seeded_and_legacy_row_is_pruned(db_ses def test_super_admin_bootstrap_writes_role(db_session): from core.db.models import UserShadow - from core.services.local_user_service import LocalUserService - from core.services.user_service import UserService + from core.auth.local_users import LocalUserService + from core.auth.users import UserService res = LocalUserService(db_session).create_by_admin(username="admin", password="pw-123456") assert res.ok and res.user_id @@ -190,118 +191,26 @@ def test_super_admin_bootstrap_writes_role(db_session): assert shadow.extra_data.get("auth_source") == "local" -def test_ce_default_admin_is_idempotent_and_requires_password_change(db_session, monkeypatch): - import core.services.local_user_service as local_users - from core.auth.password import verify_password - from core.db.models import LocalUser, UserShadow - monkeypatch.setattr( - local_users, - "settings", - SimpleNamespace( - edition=SimpleNamespace(edition="ce"), - auth=SimpleNamespace(password_min_length=8), - ), - ) - user_id, created = local_users.ensure_ce_default_admin(db_session) - assert (user_id, created) == ("user_ce_admin", True) - - shadow = db_session.query(UserShadow).filter_by(user_id=user_id).one() - local = db_session.query(LocalUser).filter_by(user_id=user_id).one() - assert shadow.username == "admin" - assert verify_password("admin", local.password_hash) - assert shadow.extra_data["role"] == "super_admin" - assert shadow.extra_data["can_add_skill"] is True - assert shadow.extra_data["can_add_mcp"] is True - assert shadow.extra_data["can_import_plugin"] is True - assert shadow.extra_data["can_create_private_kb"] is True - assert shadow.extra_data["can_create_public_kb"] is False - assert shadow.extra_data["can_create_channel_bot"] is True - assert shadow.extra_data["can_switch_model"] is True - assert shadow.extra_data["can_use_ontology_validation"] is True - assert shadow.extra_data["must_change_password"] is True - assert shadow.extra_data["onboarding_required"] is True - - assert local_users.ensure_ce_default_admin(db_session) == (user_id, False) - result = local_users.LocalUserService(db_session).change_password( - user_id, - old_password="admin", - new_password="new-password-123", - ) - assert result.ok is True - db_session.refresh(shadow) - assert "must_change_password" not in shadow.extra_data - assert shadow.extra_data["onboarding_required"] is True - login = local_users.LocalUserService(db_session).authenticate("admin", "new-password-123") - assert login.ok - assert login.user_info["onboarding_required"] is True - - -def test_ce_web_onboarding_requires_main_model_and_clears_checkpoint(db_session, monkeypatch): - import api.routes.v1.users as user_routes - import core.services.local_user_service as local_users - from core.auth.backend import UserContext - from core.db import model_repository - from core.db.models import UserShadow - from core.infra.exceptions import ValidationError - - fake_settings = SimpleNamespace( - edition=SimpleNamespace(edition="ce"), - auth=SimpleNamespace(password_min_length=8), - ) - monkeypatch.setattr(local_users, "settings", fake_settings) - monkeypatch.setattr(user_routes, "settings", fake_settings) - - user_id, _ = local_users.ensure_ce_default_admin(db_session) - local_users.LocalUserService(db_session).change_password( - user_id, - old_password="admin", - new_password="new-password-123", - ) - user = UserContext( - user_id=user_id, - user_center_id=user_id, - username="admin", - ) - - with pytest.raises(ValidationError, match="主对话模型"): - user_routes.complete_my_onboarding(user=user, db=db_session) - - provider = model_repository.create_provider( - db_session, - display_name="Test chat", - provider_type="chat", - base_url="http://model.test/v1", - api_key="test-key", - model_name="test-model", - ) - model_repository.assign_role(db_session, "main_agent", provider.provider_id) - - response = user_routes.complete_my_onboarding(user=user, db=db_session) - - assert response["data"]["onboarding_required"] is False - shadow = db_session.query(UserShadow).filter_by(user_id=user_id).one() - assert "onboarding_required" not in shadow.extra_data - assert shadow.extra_data["onboarding_completed_version"] == 1 def test_ce_has_no_optional_organization_permission_layers(): import importlib.util - from core.auth.edition_capabilities import default_capability_layers_for_user + from core.auth.edition import default_capability_layers_for_user - assert importlib.util.find_spec("core.auth.role_permissions") is None + assert importlib.util.find_spec("core.auth.edition") is None assert importlib.util.find_spec("core.auth.team_permissions") is None assert default_capability_layers_for_user(None, "user_ce_admin") == () def test_ce_branding_repairs_persistent_page_config(db_session, monkeypatch): - import core.content.content_blocks as blocks + import core.files.content.content_blocks as blocks from core.db.models import ContentBlock monkeypatch.setattr( - blocks, + branding, "settings", SimpleNamespace(edition=SimpleNamespace(edition="ce")), ) @@ -326,7 +235,7 @@ def test_ce_branding_repairs_persistent_page_config(db_session, monkeypatch): ) db_session.commit() - assert blocks.enforce_ce_branding(db_session) is True + assert branding.enforce_ce_branding(db_session) is True row = db_session.query(ContentBlock).filter_by(id="page_config").one() assert row.payload["branding"] == { "product_name": "HugAgentOS", @@ -338,19 +247,19 @@ def test_ce_branding_repairs_persistent_page_config(db_session, monkeypatch): assert row.payload["navigation"]["admin_header"]["title"] == "HugAgentOS — 后台管理" assert row.payload["navigation"]["admin_platform"]["product_name"] == "HugAgentOS" assert row.payload["auth"]["allow_register"] is False - assert blocks.enforce_ce_branding(db_session) is False + assert branding.enforce_ce_branding(db_session) is False def test_ce_registration_is_disabled_unconditionally(monkeypatch): - import core.content.content_blocks as blocks + import core.files.content.content_blocks as blocks monkeypatch.setattr( - blocks, + branding, "settings", SimpleNamespace(edition=SimpleNamespace(edition="ce")), ) - assert blocks.is_register_allowed() is False + assert branding.is_register_allowed() is False @pytest.mark.asyncio @@ -496,18 +405,18 @@ def test_kb_milvus_lite_dense_only(tmp_path, monkeypatch): monkeypatch.setenv("MILVUS_URL", str(tmp_path / "kb.db")) import importlib - - import core.kb.kb_vector as kv - - importlib.reload(kv) # re-read MILVUS_URL - + from features.knowledge import kb_vector, vector_client, vector_indexing + from core.models import embedding as vector_embedding, embedding_state + import features.knowledge.kb_vector_state as kv + importlib.reload(kv) + importlib.reload(embedding_state) # Reset relocated caches and environment configuration. DIM = 8 - monkeypatch.setattr(kv, "detect_embed_dim", lambda: DIM) - monkeypatch.setattr(kv, "embed_text", lambda t: [0.1] * DIM) + monkeypatch.setattr(vector_embedding, 'detect_embed_dim', lambda: DIM) + monkeypatch.setattr(vector_embedding, 'embed_text', lambda t: [0.1] * DIM) - assert kv._is_lite() is True - kv.get_or_create_collection() - kv.upsert_rows( + assert vector_client.is_lite() is True + vector_client.get_or_create_collection() + vector_indexing.upsert_rows( [ { "chunk_id": "c1", @@ -521,11 +430,11 @@ def test_kb_milvus_lite_dense_only(tmp_path, monkeypatch): "tags_text": "", "chunk_index": 0, "dense_embedding": [0.1] * DIM, - "sparse_embedding": kv.text_to_sparse("机器学习"), # must be stripped on Lite + "sparse_embedding": kb_vector.text_to_sparse("机器学习"), # must be stripped on Lite } ] ) - hits = kv.hybrid_search("u1", ["kb1"], "机器学习", [0.1] * DIM, top_k=5) + hits = kb_vector.hybrid_search("u1", ["kb1"], "机器学习", [0.1] * DIM, top_k=5) assert len(hits) == 1 and hits[0]["content"] == "机器学习" @@ -564,9 +473,10 @@ def test_local_env_merges_loopback_hosts_into_no_proxy(monkeypatch): @pytest.mark.parametrize("explicit", [None, "", "http://configured:8080"]) def test_windows_loopback_bypass_preserves_registry_proxy(monkeypatch, explicit): import os - import cli from urllib import request + import cli + for name in list(os.environ): if name.lower() in ("http_proxy", "https_proxy", "all_proxy", "no_proxy"): monkeypatch.delenv(name) diff --git a/src/backend/tests/test_local_subprocess.py b/src/backend/tests/test_local_subprocess.py index 360e8b7a..96b9caec 100644 --- a/src/backend/tests/test_local_subprocess.py +++ b/src/backend/tests/test_local_subprocess.py @@ -5,9 +5,10 @@ import tarfile from types import SimpleNamespace +import core.infra.local_runtime.startup as startup +import core.infra.local_runtime.startup as local_subprocess import pytest from extensions.mcp_servers import _serve -from orchestration import local_subprocess def test_child_env_binds_local_mcp_to_loopback(monkeypatch): @@ -38,7 +39,7 @@ def test_site_publish_callback_uses_local_listener_port(monkeypatch): @pytest.mark.asyncio async def test_local_site_pack_uses_portable_python_archive(monkeypatch): import core.sandbox as sandbox - from core.services import site_packaging + from features.sites import packaging as site_packaging archive = io.BytesIO() with tarfile.open(fileobj=archive, mode="w:gz") as tf: @@ -52,6 +53,7 @@ class FakeProvider: async def run_to_completion(self, request): requests.append(request) return SimpleNamespace(exit_code=0, stdout="", stderr="") + async def get_file(self, session_id, path, user_id=None): return archive.getvalue() @@ -103,7 +105,7 @@ async def fake_verify(ports): calls.append(("verify", dict(ports))) monkeypatch.setattr( - local_subprocess, + startup, "settings", SimpleNamespace( deploy=SimpleNamespace(is_local=True), @@ -113,9 +115,9 @@ async def fake_verify(ports): ), ), ) - monkeypatch.setattr(local_subprocess, "_spawn", fake_spawn) - monkeypatch.setattr(local_subprocess, "_wait_for_mcp_ports", fake_wait) - monkeypatch.setattr(local_subprocess, "_verify_required_plugin_tools", fake_verify) + monkeypatch.setattr(startup, "_spawn", fake_spawn) + monkeypatch.setattr(startup, "_wait_for_mcp_ports", fake_wait) + monkeypatch.setattr(startup, "_verify_required_plugin_tools", fake_verify) # 品牌隔离端口预检返回未占用；runner 就绪等待随后返回已就绪 port_probes = iter([False, True, True, True]) @@ -123,7 +125,7 @@ async def fake_verify(ports): async def fake_port_ready(host, port): return next(port_probes, True) - monkeypatch.setattr(local_subprocess, "_tcp_port_ready", fake_port_ready) + monkeypatch.setattr(startup, "_tcp_port_ready", fake_port_ready) await local_subprocess.start_local_sidecars() @@ -162,11 +164,9 @@ def test_mcp_port_namespace_offset_is_validated(): def test_script_runner_port_requires_loopback_url(monkeypatch): monkeypatch.setattr( - local_subprocess, + startup, "settings", - SimpleNamespace( - sandbox=SimpleNamespace(runner_url="http://127.0.0.1:32202") - ), + SimpleNamespace(sandbox=SimpleNamespace(runner_url="http://127.0.0.1:32202")), ) assert local_subprocess._script_runner_port() == 32202 @@ -227,9 +227,7 @@ def test_find_stale_sidecar_pids_matches_only_own_install(runner_module): def test_find_stale_sidecar_pids_excludes_self(): root = "/r/local-server" ps = f" 42 {root}/venv/bin/python -m extensions.mcp_servers._launcher" - assert ( - local_subprocess._find_stale_sidecar_pids(ps, root_marker=root, my_pid=42) == [] - ) + assert local_subprocess._find_stale_sidecar_pids(ps, root_marker=root, my_pid=42) == [] @pytest.mark.asyncio @@ -253,7 +251,7 @@ async def fake_stop(): return None monkeypatch.setattr( - local_subprocess, + startup, "settings", SimpleNamespace( deploy=SimpleNamespace(is_local=True), @@ -263,10 +261,10 @@ async def fake_stop(): ), ), ) - monkeypatch.setattr(local_subprocess, "_spawn", fake_spawn) - monkeypatch.setattr(local_subprocess, "_reap_stale_sidecars", fake_reap) - monkeypatch.setattr(local_subprocess, "_tcp_port_ready", fake_port_ready) - monkeypatch.setattr(local_subprocess, "stop_local_sidecars", fake_stop) + monkeypatch.setattr(startup, "_spawn", fake_spawn) + monkeypatch.setattr(startup, "_reap_stale_sidecars", fake_reap) + monkeypatch.setattr(startup, "_tcp_port_ready", fake_port_ready) + monkeypatch.setattr(startup, "stop_local_sidecars", fake_stop) with pytest.raises(RuntimeError, match="32202"): await local_subprocess.start_local_sidecars() @@ -328,14 +326,16 @@ async def fake_port_ready(host, port): return next(port_probe) monkeypatch.setattr(local_subprocess.os, "name", "nt") - monkeypatch.setattr(local_subprocess, "_packaged_install_root", lambda: r"C:\app\local-server") - monkeypatch.setattr(local_subprocess, "_process_table", fake_table) - monkeypatch.setattr(local_subprocess, "_terminate_process_trees", fake_kill) - monkeypatch.setattr(local_subprocess, "_tcp_port_ready", fake_port_ready) + monkeypatch.setattr(startup, "_packaged_install_root", lambda: r"C:\app\local-server") + monkeypatch.setattr(startup, "_process_table", fake_table) + monkeypatch.setattr(startup, "_terminate_process_trees", fake_kill) + monkeypatch.setattr(startup, "_tcp_port_ready", fake_port_ready) monkeypatch.setattr( - local_subprocess, + startup, "settings", - SimpleNamespace(sandbox=SimpleNamespace(provider="script_runner", runner_url="http://127.0.0.1:32202")), + SimpleNamespace( + sandbox=SimpleNamespace(provider="script_runner", runner_url="http://127.0.0.1:32202") + ), ) await local_subprocess._reap_stale_sidecars() assert killed == [77] @@ -356,8 +356,8 @@ async def wait(self): self.returncode = 0 monkeypatch.setattr(local_subprocess.os, "name", "nt") - monkeypatch.setattr(local_subprocess, "_terminate_process_trees", fake_kill) - monkeypatch.setattr(local_subprocess, "_PROCS", [("script_runner", DummyProcess())]) + monkeypatch.setattr(startup, "_terminate_process_trees", fake_kill) + monkeypatch.setattr(startup, "_PROCS", [("script_runner", DummyProcess())]) await local_subprocess.stop_local_sidecars() assert killed == [4242] @@ -386,16 +386,16 @@ async def fail_wait(*_a, **_k): async def fake_port_ready(host, port): return next(port_probes) - monkeypatch.setattr(local_subprocess, "_cloud_serves_tools", lambda: True) - monkeypatch.setattr(local_subprocess, "_reap_stale_sidecars", fake_reap) - monkeypatch.setattr(local_subprocess, "_spawn", fake_spawn) - monkeypatch.setattr(local_subprocess, "_wait_for_mcp_ports", fail_wait) - monkeypatch.setattr(local_subprocess, "_verify_required_plugin_tools", fail_wait) - monkeypatch.setattr(local_subprocess, "_tcp_port_ready", fake_port_ready) - monkeypatch.setattr(local_subprocess, "_start_watchdog", lambda: calls.append(("watchdog",))) - monkeypatch.setattr(local_subprocess, "_PROCS", []) + monkeypatch.setattr(startup, "_cloud_serves_tools", lambda: True) + monkeypatch.setattr(startup, "_reap_stale_sidecars", fake_reap) + monkeypatch.setattr(startup, "_spawn", fake_spawn) + monkeypatch.setattr(startup, "_wait_for_mcp_ports", fail_wait) + monkeypatch.setattr(startup, "_verify_required_plugin_tools", fail_wait) + monkeypatch.setattr(startup, "_tcp_port_ready", fake_port_ready) + monkeypatch.setattr(startup, "_start_watchdog", lambda: calls.append(("watchdog",))) + monkeypatch.setattr(startup, "_PROCS", []) monkeypatch.setattr( - local_subprocess, + startup, "settings", SimpleNamespace( deploy=SimpleNamespace(is_local=True), @@ -406,3 +406,26 @@ async def fake_port_ready(host, port): assert [c[1] for c in calls if c[0] == "spawn"] == ["script_runner"] assert ("watchdog",) in calls + +def test_child_pythonpath_resolves_relocated_backend_packages(monkeypatch, tmp_path): + import subprocess + import sys + + from core.infra.local_runtime import lifecycle as lifecycle + + monkeypatch.delenv("PYTHONPATH", raising=False) + env = lifecycle._child_env() + probe = ( + "from importlib.util import find_spec; " + "assert find_spec('extensions.mcp_servers._launcher') is not None; " + "assert find_spec('core.sandbox.runner.server') is not None" + ) + subprocess.run( + [sys.executable, "-c", probe], + cwd=tmp_path, + env=env, + check=True, + capture_output=True, + text=True, + timeout=15, + ) diff --git a/src/backend/tests/test_management_failures.py b/src/backend/tests/test_management_failures.py index 79af38d7..1d1992bf 100644 --- a/src/backend/tests/test_management_failures.py +++ b/src/backend/tests/test_management_failures.py @@ -6,7 +6,7 @@ from fastapi import HTTPException from sqlalchemy.exc import DataError, SQLAlchemyError -from core.services import cloud_management +from core.capabilities.plugins.management import cloud as cloud_management from extensions.mcp_servers.management_registration import ArtifactSource, register @@ -52,11 +52,9 @@ def test_unknown_database_error_is_not_empty_or_raw(monkeypatch): def test_local_management_uses_same_safe_error(monkeypatch): import json from agentscope.message import ToolResultState - from core.llm.management_tool import ManagementTool + from core.llm.tools.management import ManagementTool - tool = ManagementTool("manager", SimpleNamespace( - name="install_skill", description="Install", inputSchema={} - ), "skill-manager", {}, cloud_backed=False) + tool = ManagementTool("manager", dict(name="install_skill", description="Install", inputSchema={}), "skill-manager", {}, cloud_backed=False) original = RuntimeError("private payload") original.pgcode = "22001" def fail(arguments): diff --git a/src/backend/tests/test_mcp_canonical_url.py b/src/backend/tests/test_mcp_canonical_url.py new file mode 100644 index 00000000..d9c5a09b --- /dev/null +++ b/src/backend/tests/test_mcp_canonical_url.py @@ -0,0 +1,30 @@ +"""Built-in MCP requests must not incur an HTTP redirect.""" +from core.capabilities.mcp.service import rewrite_builtin_mcp_host +from core.config.mcp_config import mcp_http_url + + +def test_builtin_canonical_endpoint(): + assert mcp_http_url("web_fetch").endswith("/mcp") + assert rewrite_builtin_mcp_host("http://mcp:9106/mcp/").endswith("/mcp") + + +def test_external_paths_and_queries_are_preserved(): + external = "https://example.com/mcp/?tenant=a" + assert rewrite_builtin_mcp_host(external) == external + assert rewrite_builtin_mcp_host("http://mcp:9106/custom/?tenant=a").endswith("/custom/?tenant=a") + + +def test_local_seed_uses_canonical_endpoint(db_session, monkeypatch): + from core.config.settings import settings + from core.capabilities.mcp.service import seed_builtin_mcp_servers_if_empty, McpServerConfigService + from core.db.models import AdminMcpServer + + from dataclasses import replace + + monkeypatch.setattr("core.config.settings.settings", + replace(settings, server=replace(settings.server, mcp_host="127.0.0.1"))) + assert seed_builtin_mcp_servers_if_empty(db_session) + for row in db_session.query(AdminMcpServer).all(): + assert row.url.startswith("http://127.0.0.1:") + assert row.url.endswith("/mcp") + assert McpServerConfigService()._row_to_config(row)["url"] == row.url diff --git a/src/backend/tests/test_mcp_marketplace.py b/src/backend/tests/test_mcp_marketplace.py deleted file mode 100644 index 8983b261..00000000 --- a/src/backend/tests/test_mcp_marketplace.py +++ /dev/null @@ -1,1758 +0,0 @@ -"""End-to-end service tests for the reviewed MCP marketplace lifecycle.""" - -from __future__ import annotations - -import asyncio -import importlib.util -import json -import socket -from pathlib import Path -from types import SimpleNamespace - -import httpx -import pytest -from api.routes.v1 import admin_mcp_servers as admin_mcp_routes -from api.routes.v1 import me_capabilities as me_capability_routes -from core.db.engine import Base -from core.db.models import ( - AdminMcpServer, - McpMarketInstallation, - McpMarketItem, - McpMarketSubmission, - McpMarketVersion, -) -from core.infra.exceptions import BadRequestError, ResourceNotFoundError -from core.llm.mcp_pool import make_client -from core.services import mcp_management_service as management -from core.services import mcp_marketplace_service as market -from core.services import mcp_oauth_service as oauth_service -from core.services.mcp_management_service import ( - decrypt_mcp_headers, - encrypt_legacy_mcp_headers, - encrypt_mcp_headers, - mask_mcp_headers, - materialize_mcp_http_connection, - mcp_oauth_bundle_storage_key, - mcp_query_secret_storage_key, - tool_snapshot_hash, - validate_remote_mcp_url, -) -from core.services.mcp_oauth_service import OAuthBundleStorage -from core.services.mcp_service import McpServerConfigService -from mcp.shared.auth import OAuthToken -from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker - -_IS_CE_TREE = not (Path(__file__).resolve().parents[1] / "edition_ee").exists() - - -@pytest.fixture -def db(): - engine = create_engine("sqlite://", future=True) - Base.metadata.create_all(engine) - Session = sessionmaker(bind=engine, autoflush=False, future=True) - session = Session() - try: - yield session - finally: - session.close() - Base.metadata.drop_all(engine) - - -def _tools(*names: str): - return [ - { - "name": name, - "description": f"Tool {name}", - "inputSchema": {"type": "object", "properties": {"query": {"type": "string"}}}, - } - for name in names - ] - - -@pytest.mark.asyncio -async def test_http_probe_uses_stateless_client_without_explicit_close(db, monkeypatch): - calls = {"connected": 0, "closed": 0, "is_stateful": None} - - class FakeClient: - async def connect(self): - calls["connected"] += 1 - - async def list_tools(self): - return [SimpleNamespace(name="search", description="Search", inputSchema={})] - - async def close(self): - calls["closed"] += 1 - - def _make_client(name, cfg, *, is_stateful=True, **kwargs): - calls["is_stateful"] = is_stateful - return FakeClient() - - from core.llm import mcp_pool - - monkeypatch.setattr(mcp_pool, "make_client", _make_client) - row = AdminMcpServer( - server_id="stateless_probe", - display_name="Stateless Probe", - transport="streamable_http", - url="https://mcp.example.test/mcp", - headers={}, - is_enabled=True, - ) - - ok, error = await management.probe_mcp_connectivity(row) - - assert ok is True - assert error == "" - assert calls == {"connected": 0, "closed": 0, "is_stateful": False} - assert row.tools_json[0]["name"] == "search" - - -@pytest.mark.asyncio -async def test_http_probe_surfaces_401_instead_of_cancel_scope_noise(db, monkeypatch): - request = httpx.Request("POST", "https://mcp.example.test/mcp") - response = httpx.Response(401, request=request) - unauthorized = httpx.HTTPStatusError( - "401 Unauthorized", - request=request, - response=response, - ) - - class FakeClient: - async def list_tools(self): - raise BaseExceptionGroup( - "MCP request failed", - [ - unauthorized, - asyncio.CancelledError("Cancelled via cancel scope deadbeef"), - ], - ) - - def _make_client(name, cfg, *, is_stateful=True, **kwargs): - assert is_stateful is False - return FakeClient() - - from core.llm import mcp_pool - - monkeypatch.setattr(mcp_pool, "make_client", _make_client) - row = AdminMcpServer( - server_id="unauthorized_probe", - display_name="Unauthorized Probe", - transport="streamable_http", - url="https://mcp.example.test/mcp", - headers={}, - is_enabled=True, - ) - - ok, error = await management.probe_mcp_connectivity(row) - - assert ok is False - assert error == "远端 MCP 返回 401 Unauthorized，请检查鉴权设置和 Token" - assert "cancel scope" not in error.lower() - - -def _private_server(db, *, owner: str = "u1", server_id: str = "umcp_source"): - row = AdminMcpServer( - server_id=server_id, - display_name="Acme Search", - description="Search Acme documents", - transport="streamable_http", - url="https://mcp.example.test/mcp", - headers=encrypt_mcp_headers({"Authorization": "Bearer publisher-secret"}), - tools_json=_tools("search_documents"), - is_stable=False, - is_enabled=True, - owner_user_id=owner, - ) - db.add(row) - db.commit() - return row - - -def _approved_market(db, *, tools=None): - snapshot = tools or _tools("search_documents") - item = McpMarketItem( - slug="acme-search", - display_name="Acme Search", - description="Search Acme documents", - category="信息检索", - tags=["search"], - publisher_name="Acme", - source="community", - latest_version_id="mcpver_1", - status="active", - ) - version = McpMarketVersion( - version_id="mcpver_1", - slug="acme-search", - version="1.0.0", - transport="streamable_http", - url="https://mcp.example.test/mcp", - auth_schema=[ - { - "key": "Authorization", - "label": "Authorization", - "target": "header", - "required": True, - "secret": True, - } - ], - tools_json=snapshot, - tool_hash=tool_snapshot_hash(snapshot), - source_server_id="umcp_source", - ) - db.add_all([item, version]) - db.commit() - return item, version - - -def _patch_probe(monkeypatch, tools=None): - async def _validate(*args, **kwargs): - return None - - async def _probe(row, db=None): - if tools is not None: - row.tools_json = tools - return True, "" - - monkeypatch.setattr(market, "validate_remote_mcp_url", _validate) - monkeypatch.setattr(market, "probe_mcp_connectivity", _probe) - monkeypatch.setattr(market, "refresh_mcp_caches", lambda: None) - - -def test_ce_metadata_contains_mcp_market_auth_without_ee_dependency(): - project_root = Path(__file__).resolve().parents[3] - module_path = project_root / "ce/overlay/src/backend/core/db/edition_tables.py" - if not module_path.exists(): - module_path = project_root / "src/backend/core/db/edition_tables.py" - spec = importlib.util.spec_from_file_location("ce_edition_tables_test", module_path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - metadata = module._ce_metadata() - assert "mcp_market_items" in metadata.tables - assert "auth_config" in metadata.tables["mcp_market_versions"].columns - assert "auth_config" in metadata.tables["mcp_market_submissions"].columns - - -def test_header_credentials_are_encrypted_and_masked(): - encrypted = encrypt_mcp_headers({"Authorization": "Bearer top-secret"}) - assert encrypted["Authorization"].startswith("enc:v1:") - assert "top-secret" not in encrypted["Authorization"] - assert decrypt_mcp_headers(encrypted) == {"Authorization": "Bearer top-secret"} - assert mask_mcp_headers(encrypted) == {"Authorization": "***"} - - -def test_runtime_query_credentials_are_encrypted_and_never_sent_as_headers(): - storage_key = mcp_query_secret_storage_key("key") - encrypted = encrypt_mcp_headers({storage_key: "amap-secret"}) - assert "amap-secret" not in str(encrypted) - assert mask_mcp_headers(encrypted) == {} - - runtime_url, runtime_headers = materialize_mcp_http_connection( - "https://mcp.amap.com/mcp?locale=zh-CN", - decrypt_mcp_headers(encrypted), - ) - assert runtime_url == "https://mcp.amap.com/mcp?locale=zh-CN&key=amap-secret" - assert runtime_headers == {} - - row = AdminMcpServer( - server_id="amap_private", - display_name="Amap", - transport="streamable_http", - url="https://mcp.amap.com/mcp?locale=zh-CN", - headers=encrypted, - is_stable=False, - is_enabled=True, - ) - runtime_config = McpServerConfigService.get_instance()._row_to_config(row) - assert runtime_config["url"] == runtime_url - assert "headers" not in runtime_config - - -@pytest.mark.asyncio -async def test_oauth_bundle_is_provider_neutral_and_never_sent_as_header(): - storage = OAuthBundleStorage( - { - "client_metadata": { - "redirect_uris": ["https://app.example.test/api/v1/mcp-market/oauth/callback"], - "client_name": "HugAgentOS MCP Client", - }, - "client_info": { - "redirect_uris": ["https://app.example.test/api/v1/mcp-market/oauth/callback"], - "client_id": "client-id", - "token_endpoint_auth_method": "none", - }, - } - ) - await storage.set_tokens( - OAuthToken( - access_token="oauth-access-secret", - refresh_token="oauth-refresh-secret", - expires_in=3600, - ) - ) - bundle = storage.bundle - encrypted = encrypt_mcp_headers( - {mcp_oauth_bundle_storage_key(): json.dumps(bundle, separators=(",", ":"))} - ) - assert "oauth-access-secret" not in str(encrypted) - assert mask_mcp_headers(encrypted) == {} - - row = AdminMcpServer( - server_id="oauth_private", - display_name="OAuth MCP", - transport="streamable_http", - url="https://mcp.example.test/mcp", - headers=encrypted, - is_stable=False, - is_enabled=True, - ) - runtime_config = McpServerConfigService.get_instance()._row_to_config(row) - assert "oauth_provider" in runtime_config - assert "headers" not in runtime_config - client = make_client(row.server_id, runtime_config, is_stateful=False) - assert client.oauth_provider is runtime_config["oauth_provider"] - - -@pytest.mark.asyncio -async def test_oauth_callback_and_status_are_shared_without_plaintext_code(monkeypatch): - import fakeredis.aioredis - - fake_redis = fakeredis.aioredis.FakeRedis(decode_responses=True) - monkeypatch.setattr("core.infra.ephemeral.get_redis", lambda **_: fake_redis) - flow = oauth_service.OAuthInstallFlow( - flow_id="mcpoauth_test", - slug="gitlab-official", - owner_user_id="u1", - installed_by="u1", - auth_method="oauth2", - credentials={}, - callback_url="https://app.example.test/api/v1/mcp-market/oauth/callback", - status="waiting_for_user", - ) - await oauth_service._store_flow_status(flow) - oauth_service._flows.pop(flow.flow_id, None) # simulate callback landing on another worker - await oauth_service.complete_callback( - flow.flow_id, - code="short-lived-authorization-code", - state="oauth-state", - ) - callback_raw = await fake_redis.get(oauth_service._callback_key(flow.flow_id)) - assert "short-lived-authorization-code" not in callback_raw - status = await oauth_service.get_flow_status(flow.flow_id, owner_user_id="u1") - assert status["status"] == "processing_callback" - await fake_redis.aclose() - - -@pytest.mark.asyncio -async def test_oauth_start_hands_back_before_discovery_finishes(db, monkeypatch): - """Slow authorization-URL discovery must keep the flow pollable, never fail the install.""" - import fakeredis.aioredis - - fake_redis = fakeredis.aioredis.FakeRedis(decode_responses=True) - monkeypatch.setattr("core.infra.ephemeral.get_redis", lambda **_: fake_redis) - monkeypatch.setattr(oauth_service, "_DISCOVERY_GRACE_SECONDS", 0.05) - market.ensure_curated_market_items(db) - - async def slow_discovery(flow): - await asyncio.sleep(0.3) - flow.authorization_url = "https://provider.example.test/authorize?code_challenge=x" - flow.status = "waiting_for_user" - await oauth_service._store_flow_status(flow) - flow.ready.set() - - monkeypatch.setattr(oauth_service, "_run_flow", slow_discovery) - try: - started = await oauth_service.start_oauth_install( - db, - slug="gitlab-official", - owner_user_id="u1", - installed_by="u1", - auth_method="oauth2", - credentials={}, - callback_url_base="https://app.example.test/api/v1/mcp-market/oauth/callback", - ) - assert started["status"] == "starting" - assert started["authorization_url"] == "" - - await asyncio.sleep(0.4) - status = await oauth_service.get_flow_status(started["flow_id"], owner_user_id="u1") - assert status["status"] == "waiting_for_user" - assert status["authorization_url"].startswith("https://provider.example.test/authorize") - finally: - oauth_service._flows.clear() - await fake_redis.aclose() - - -@pytest.mark.asyncio -async def test_oauth_cancel_marks_flow_failed_and_wakes_worker(monkeypatch): - import fakeredis.aioredis - - fake_redis = fakeredis.aioredis.FakeRedis(decode_responses=True) - monkeypatch.setattr("core.infra.ephemeral.get_redis", lambda **_: fake_redis) - flow = oauth_service.OAuthInstallFlow( - flow_id="mcpoauth_cancel", - slug="gitlab-official", - owner_user_id="u1", - installed_by="u1", - auth_method="oauth2", - credentials={}, - callback_url="https://app.example.test/api/v1/mcp-market/oauth/callback", - status="waiting_for_user", - ) - oauth_service._flows[flow.flow_id] = flow - try: - await oauth_service._store_flow_status(flow) - status = await oauth_service.cancel_flow(flow.flow_id, owner_user_id="u1") - assert status["status"] == "failed" - assert status["error"] == "OAuth 登录已取消" - assert flow.callback_received.is_set() - with pytest.raises(ResourceNotFoundError): - await oauth_service.cancel_flow(flow.flow_id, owner_user_id="u2") - finally: - oauth_service._flows.pop(flow.flow_id, None) - await fake_redis.aclose() - - -def test_oauth_callback_uses_browser_origin_not_untrusted_host(monkeypatch): - monkeypatch.setattr( - oauth_service, - "settings", - SimpleNamespace(server=SimpleNamespace(mcp_oauth_public_base_url="", is_prod=False)), - ) - request = SimpleNamespace( - headers={"origin": "https://app.example.test", "host": "attacker.invalid"}, - url=SimpleNamespace(netloc="attacker.invalid"), - ) - assert oauth_service.public_oauth_callback_url(request) == ( - "https://app.example.test/api/v1/mcp-market/oauth/callback" - ) - - -def test_oauth_callback_allows_configured_http_in_production(monkeypatch): - monkeypatch.setattr( - oauth_service, - "settings", - SimpleNamespace( - server=SimpleNamespace( - mcp_oauth_public_base_url="http://app.example.test/api", - is_prod=True, - ) - ), - ) - request = SimpleNamespace(headers={}, url=SimpleNamespace(netloc="ignored.example.test")) - - assert oauth_service.public_oauth_callback_url(request) == ( - "http://app.example.test/api/v1/mcp-market/oauth/callback" - ) - - -def test_legacy_plaintext_headers_are_encrypted_once(db): - row = _private_server(db) - row.headers = {"Authorization": "Bearer legacy"} - db.commit() - assert encrypt_legacy_mcp_headers(db) == 1 - db.refresh(row) - assert decrypt_mcp_headers(row.headers) == {"Authorization": "Bearer legacy"} - assert "legacy" not in str(row.headers) - assert encrypt_legacy_mcp_headers(db) == 0 - - -@pytest.mark.asyncio -async def test_remote_url_security_blocks_private_and_accepts_public_dns(monkeypatch): - with pytest.raises(BadRequestError): - await validate_remote_mcp_url("https://127.0.0.1/mcp") - with pytest.raises(BadRequestError): - await validate_remote_mcp_url("http://example.com/mcp") - - monkeypatch.setattr( - socket, - "getaddrinfo", - lambda *args, **kwargs: [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("8.8.8.8", 443))], - ) - await validate_remote_mcp_url("https://example.com/mcp") - - -@pytest.mark.asyncio -async def test_remote_url_security_allows_public_http_when_requested(monkeypatch): - monkeypatch.setattr( - socket, - "getaddrinfo", - lambda *args, **kwargs: [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("8.8.8.8", 80))], - ) - - await validate_remote_mcp_url("http://example.com/mcp", require_https=False) - - with pytest.raises(BadRequestError, match="本机、内网或保留地址"): - await validate_remote_mcp_url("http://127.0.0.1/mcp", require_https=False) - - -@pytest.mark.asyncio -@pytest.mark.parametrize("address", ["127.0.0.1", "10.10.0.8", "172.18.0.12", "192.168.1.9"]) -async def test_remote_url_security_allows_trusted_private_networks_when_enabled(address): - await validate_remote_mcp_url( - f"http://{address}/mcp", - allow_private_network=True, - require_https=False, - ) - - -@pytest.mark.asyncio -@pytest.mark.parametrize("address", ["0.0.0.0", "169.254.169.254", "224.0.0.1"]) -async def test_remote_url_security_keeps_high_risk_ranges_blocked_when_private_enabled(address): - with pytest.raises(BadRequestError, match="本机、内网或保留地址"): - await validate_remote_mcp_url( - f"http://{address}/mcp", - allow_private_network=True, - require_https=False, - ) - - -@pytest.mark.asyncio -@pytest.mark.parametrize("allow_private_network", [False, True]) -async def test_self_service_mcp_uses_private_network_deployment_setting( - db, monkeypatch, allow_private_network -): - validation = {} - - async def _validate(url, **kwargs): - validation.update(url=url, **kwargs) - - async def _probe(row, _db): - row.tools_json = _tools("search") - return True, "" - - monkeypatch.setattr(me_capability_routes, "_require_flag", lambda *args: None) - monkeypatch.setattr(me_capability_routes, "validate_remote_mcp_url", _validate) - monkeypatch.setattr(me_capability_routes, "probe_mcp_connectivity", _probe) - monkeypatch.setattr( - me_capability_routes, - "settings", - SimpleNamespace( - server=SimpleNamespace( - mcp_self_service_allow_private_network=allow_private_network, - ) - ), - ) - - await me_capability_routes.create_my_mcp_server( - me_capability_routes.CreateUserMcpRequest( - display_name="Private MCP", - url="http://10.10.0.8/mcp", - ), - user=SimpleNamespace(user_id="u1"), - db=db, - ) - - assert validation == { - "url": "http://10.10.0.8/mcp", - "allow_private_network": allow_private_network, - "require_https": False, - } - - -@pytest.mark.asyncio -async def test_remote_url_security_rechecks_tun_fake_ip_dns(monkeypatch): - with pytest.raises(BadRequestError): - await validate_remote_mcp_url("https://198.18.0.1/mcp") - - monkeypatch.setattr( - socket, - "getaddrinfo", - lambda *args, **kwargs: [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("198.18.0.42", 443))], - ) - - async def _public_dns(_hostname: str): - return {"8.8.8.8", "2001:4860:4860::8888"} - - monkeypatch.setattr(management, "_resolve_public_dns_via_doh", _public_dns) - await validate_remote_mcp_url("https://example.com/mcp") - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - "answers", - [ - [ - ( - socket.AF_INET6, - socket.SOCK_STREAM, - 6, - "", - ("fc00::2", 443, 0, 0), - ) - ], - [ - ( - socket.AF_INET6, - socket.SOCK_STREAM, - 6, - "", - ("fdfe:dcba:9876::1b", 443, 0, 0), - ) - ], - [ - (socket.AF_INET, socket.SOCK_STREAM, 6, "", ("198.18.0.43", 443)), - ( - socket.AF_INET6, - socket.SOCK_STREAM, - 6, - "", - ("fdfe:dcba:9876::1b", 443, 0, 0), - ), - ], - ], - ids=["sing-box-ipv6", "tun-ipv6", "dual-stack"], -) -async def test_remote_url_security_rechecks_ipv6_tun_fake_ip_dns(monkeypatch, answers): - monkeypatch.setattr(socket, "getaddrinfo", lambda *args, **kwargs: answers) - - async def _public_dns(_hostname: str): - return {"140.82.112.4", "2606:50c0:8000::154"} - - monkeypatch.setattr(management, "_resolve_public_dns_via_doh", _public_dns) - await validate_remote_mcp_url("https://github.com/mcp") - - -@pytest.mark.asyncio -@pytest.mark.parametrize("address", ["fd00::1", "fdfe:dcba:9877::1"]) -async def test_remote_url_security_does_not_treat_arbitrary_ula_as_fake_ip(monkeypatch, address): - monkeypatch.setattr( - socket, - "getaddrinfo", - lambda *args, **kwargs: [ - (socket.AF_INET6, socket.SOCK_STREAM, 6, "", (address, 443, 0, 0)) - ], - ) - public_dns_called = False - - async def _public_dns(_hostname: str): - nonlocal public_dns_called - public_dns_called = True - return {"8.8.8.8"} - - monkeypatch.setattr(management, "_resolve_public_dns_via_doh", _public_dns) - with pytest.raises(BadRequestError, match="本机、内网或保留地址"): - await validate_remote_mcp_url("https://internal.example.com/mcp") - assert public_dns_called is False - - -@pytest.mark.asyncio -async def test_remote_url_security_rejects_fake_ip_when_public_dns_is_private(monkeypatch): - monkeypatch.setattr( - socket, - "getaddrinfo", - lambda *args, **kwargs: [ - ( - socket.AF_INET6, - socket.SOCK_STREAM, - 6, - "", - ("fdfe:dcba:9876::1b", 443, 0, 0), - ) - ], - ) - - async def _public_dns(_hostname: str): - return {"10.0.0.1"} - - monkeypatch.setattr(management, "_resolve_public_dns_via_doh", _public_dns) - with pytest.raises(BadRequestError, match="公共 DNS 安全复核失败"): - await validate_remote_mcp_url("https://example.com/mcp") - - -def test_tool_hash_is_order_independent(): - first = _tools("search", "delete_record") - second = list(reversed(first)) - assert tool_snapshot_hash(first) == tool_snapshot_hash(second) - - -@pytest.mark.asyncio -async def test_submit_snapshots_without_credentials_then_approve(db, monkeypatch): - source = _private_server(db) - _patch_probe(monkeypatch) - - result = await market.submit_to_marketplace( - db, - source.server_id, - owner_user_id="u1", - submitter_name="Alice", - category="信息检索", - version="1.0.0", - note="Tested", - ) - submission = ( - db.query(McpMarketSubmission).filter_by(submission_id=result["submission_id"]).one() - ) - assert submission.status == "pending" - assert submission.auth_schema == [ - { - "key": "Authorization", - "label": "Authorization", - "target": "header", - "required": True, - "secret": True, - } - ] - assert "publisher-secret" not in str(submission.auth_schema) - assert submission.tools_json == _tools("search_documents") - - reviewed = await market.review_submission( - db, - submission.submission_id, - approve=True, - reviewed_by="admin", - ) - assert reviewed["status"] == "approved" - item = db.query(McpMarketItem).filter_by(slug=submission.slug).one() - version = db.query(McpMarketVersion).filter_by(version_id=item.latest_version_id).one() - assert item.source == "community" - assert version.tool_hash == tool_snapshot_hash(_tools("search_documents")) - - -@pytest.mark.asyncio -async def test_public_http_marketplace_submit_review_and_install(db, monkeypatch): - source = _private_server(db) - source.url = "http://8.8.8.8:8082/mcp" - db.commit() - - async def _probe(row, db=None): - return True, "" - - monkeypatch.setattr(market, "probe_mcp_connectivity", _probe) - monkeypatch.setattr(market, "refresh_mcp_caches", lambda: None) - - submitted = await market.submit_to_marketplace( - db, - source.server_id, - owner_user_id="u1", - submitter_name="Alice", - category="信息检索", - version="1.0.0", - ) - reviewed = await market.review_submission( - db, - submitted["submission_id"], - approve=True, - reviewed_by="admin", - ) - installed = await market.install_market_item( - db, - reviewed["slug"], - owner_user_id="u2", - installed_by="u2", - credentials={"Authorization": "Bearer user-two"}, - ) - - row = db.query(AdminMcpServer).filter_by(server_id=installed["server_id"]).one() - assert row.url == "http://8.8.8.8:8082/mcp" - assert row.owner_user_id == "u2" - - -@pytest.mark.asyncio -async def test_market_installed_private_server_cannot_submit_again(db): - item, version = _approved_market(db) - source = _private_server(db) - db.add( - McpMarketInstallation( - install_id="mcpins_no_resubmit", - slug=item.slug, - version_id=version.version_id, - server_id=source.server_id, - owner_user_id="u1", - status="active", - ) - ) - db.commit() - - from api.routes.v1.catalog import _load_owned_capability_items - - _skills, mcps = _load_owned_capability_items(db, "u1") - catalog_item = next(row for row in mcps if row["id"] == source.server_id) - assert catalog_item["marketplace_installed"] is True - - with pytest.raises(BadRequestError, match="不能再次申请上架"): - await market.submit_to_marketplace( - db, - source.server_id, - owner_user_id="u1", - submitter_name="Alice", - category="信息检索", - version="1.0.0", - ) - - -@pytest.mark.asyncio -async def test_submission_strips_query_secret_into_install_schema(db, monkeypatch): - source = _private_server(db) - source.url = "https://mcp.example.test/mcp?key=publisher-secret&locale=zh-CN" - db.commit() - _patch_probe(monkeypatch) - - submitted = await market.submit_to_marketplace( - db, - source.server_id, - owner_user_id="u1", - submitter_name="Alice", - category="信息检索", - version="1.0.0", - ) - row = db.query(McpMarketSubmission).filter_by(submission_id=submitted["submission_id"]).one() - assert row.url == "https://mcp.example.test/mcp?locale=zh-CN" - assert row.auth_schema[-1] == { - "key": "QUERY_KEY", - "label": "key", - "target": "query", - "name": "key", - "required": True, - "secret": True, - } - assert "publisher-secret" not in str(row.url) - assert "publisher-secret" not in str(row.auth_schema) - - -@pytest.mark.asyncio -async def test_admin_publish_links_existing_global_instance_without_duplication(db, monkeypatch): - source = _private_server(db, owner="temporary", server_id="global_remote") - source.owner_user_id = None - db.commit() - _patch_probe(monkeypatch) - - published = await market.publish_admin_server( - db, - source.server_id, - category="信息检索", - version="1.0.0", - ) - assert published["installed"] is True - installation = db.query(McpMarketInstallation).one() - assert installation.server_id == source.server_id - assert installation.owner_user_id is None - assert db.query(AdminMcpServer).count() == 1 - listed = market.list_market_items( - db, - owner_user_id=None, - viewer_user_id=None, - include_disabled=True, - ) - assert listed["items"][0]["installed"] is True - - -@pytest.mark.asyncio -async def test_marketplace_first_admin_source_is_reused_on_global_install(db, monkeypatch): - source = _private_server(db, owner="temporary", server_id="market_source") - source.owner_user_id = None - db.commit() - _patch_probe(monkeypatch) - - published = await market.publish_admin_server( - db, - source.server_id, - category="信息检索", - version="1.0.0", - link_as_installation=False, - ) - db.refresh(source) - assert published["installed"] is False - assert source.is_enabled is False - assert source.extra_config["market_source_only"] is True - assert db.query(McpMarketInstallation).count() == 0 - assert published["requires_auth"] is True - assert published["credentials_managed_by_admin"] is True - assert published["requires_user_credentials"] is False - assert "publisher-secret" not in json.dumps(published) - - installed = await market.install_market_item( - db, - published["slug"], - owner_user_id=None, - installed_by="admin", - credentials={}, - ) - db.refresh(source) - assert installed["server_id"] == source.server_id - assert source.is_enabled is True - assert "market_source_only" not in source.extra_config - assert decrypt_mcp_headers(source.headers)["Authorization"] == "Bearer publisher-secret" - assert db.query(AdminMcpServer).count() == 1 - assert db.query(McpMarketInstallation).one().server_id == source.server_id - - -@pytest.mark.asyncio -async def test_user_install_reuses_admin_managed_credentials_without_exposing_them(db, monkeypatch): - source = _private_server(db, owner="temporary", server_id="shared_market_source") - source.owner_user_id = None - db.commit() - _patch_probe(monkeypatch) - - published = await market.publish_admin_server( - db, - source.server_id, - category="信息检索", - version="1.0.0", - link_as_installation=False, - ) - listed = market.get_market_item( - db, - published["slug"], - owner_user_id="u2", - viewer_user_id="u2", - ) - assert listed["credentials_managed_by_admin"] is True - assert listed["requires_user_credentials"] is False - assert "publisher-secret" not in json.dumps(listed) - - installed = await market.install_market_item( - db, - published["slug"], - owner_user_id="u2", - installed_by="u2", - credentials={}, - ) - private_server = db.query(AdminMcpServer).filter_by(server_id=installed["server_id"]).one() - assert private_server.owner_user_id == "u2" - assert decrypt_mcp_headers(private_server.headers)["Authorization"] == ( - "Bearer publisher-secret" - ) - assert "publisher-secret" not in str(private_server.headers) - db.refresh(source) - assert source.is_enabled is False - assert source.extra_config["market_source_only"] is True - - -@pytest.mark.asyncio -async def test_user_install_reuses_admin_managed_query_token(db, monkeypatch): - source = _private_server(db, owner="temporary", server_id="query_market_source") - source.owner_user_id = None - source.url = "https://mcp.example.test/mcp?key=publisher-query-secret&locale=zh-CN" - db.commit() - _patch_probe(monkeypatch) - - published = await market.publish_admin_server( - db, - source.server_id, - category="信息检索", - version="1.0.0", - link_as_installation=False, - ) - assert published["credentials_managed_by_admin"] is True - assert published["url_origin"] == "https://mcp.example.test" - assert "publisher-query-secret" not in json.dumps(published) - - installed = await market.install_market_item( - db, - published["slug"], - owner_user_id="u2", - installed_by="u2", - credentials={}, - ) - private_server = db.query(AdminMcpServer).filter_by(server_id=installed["server_id"]).one() - assert private_server.url == "https://mcp.example.test/mcp?locale=zh-CN" - assert "publisher-query-secret" not in str(private_server.headers) - runtime_url, runtime_headers = materialize_mcp_http_connection( - private_server.url, - decrypt_mcp_headers(private_server.headers), - ) - assert runtime_url == ("https://mcp.example.test/mcp?locale=zh-CN&key=publisher-query-secret") - assert runtime_headers == {"Authorization": "Bearer publisher-secret"} - - -@pytest.mark.asyncio -async def test_removed_admin_credential_falls_back_to_installer_input(db, monkeypatch): - source = _private_server(db, owner="temporary", server_id="unmanaged_market_source") - source.owner_user_id = None - db.commit() - _patch_probe(monkeypatch) - - published = await market.publish_admin_server( - db, - source.server_id, - category="信息检索", - version="1.0.0", - link_as_installation=False, - ) - source.headers = {} - db.commit() - listed = market.get_market_item( - db, - published["slug"], - owner_user_id="u2", - viewer_user_id="u2", - ) - assert listed["credentials_managed_by_admin"] is False - assert listed["requires_user_credentials"] is True - - with pytest.raises(BadRequestError, match="请填写安装凭据"): - await market.install_market_item( - db, - published["slug"], - owner_user_id="u2", - installed_by="u2", - credentials={}, - ) - - -@pytest.mark.asyncio -async def test_market_edit_can_require_installer_credentials_without_reusing_source( - db, monkeypatch -): - source = _private_server(db, owner="temporary", server_id="installer_auth_source") - source.owner_user_id = None - db.commit() - _patch_probe(monkeypatch) - - published = await market.publish_admin_server( - db, - source.server_id, - category="信息检索", - version="1.0.0", - link_as_installation=False, - ) - updated = market.update_market_item( - db, - published["slug"], - requires_auth=True, - credential_mode="installer", - ) - assert updated["requires_auth"] is True - assert updated["credentials_managed_by_admin"] is False - assert updated["requires_user_credentials"] is True - assert updated["auth_config"]["credential_mode"] == "installer" - assert decrypt_mcp_headers(source.headers)["Authorization"] == "Bearer publisher-secret" - - with pytest.raises(BadRequestError, match="请填写安装凭据"): - await market.install_market_item( - db, - published["slug"], - owner_user_id="u2", - installed_by="u2", - credentials={}, - ) - - -@pytest.mark.asyncio -async def test_market_edit_sets_admin_token_and_keeps_it_out_of_responses(db, monkeypatch): - source = _private_server(db, owner="temporary", server_id="edited_admin_auth_source") - source.owner_user_id = None - db.commit() - _patch_probe(monkeypatch) - - published = await market.publish_admin_server( - db, - source.server_id, - category="信息检索", - version="1.0.0", - link_as_installation=False, - ) - updated = market.update_market_item( - db, - published["slug"], - requires_auth=True, - credential_mode="admin", - managed_credentials={"Authorization": "Bearer centrally-managed"}, - ) - assert updated["credentials_managed_by_admin"] is True - assert updated["requires_user_credentials"] is False - assert updated["auth_config"]["credential_mode"] == "admin" - assert "centrally-managed" not in json.dumps(updated) - db.refresh(source) - assert decrypt_mcp_headers(source.headers)["Authorization"] == ("Bearer centrally-managed") - assert "centrally-managed" not in str(source.headers) - - installed = await market.install_market_item( - db, - published["slug"], - owner_user_id="u2", - installed_by="u2", - credentials={}, - ) - private_server = db.query(AdminMcpServer).filter_by(server_id=installed["server_id"]).one() - assert decrypt_mcp_headers(private_server.headers)["Authorization"] == ( - "Bearer centrally-managed" - ) - - -@pytest.mark.asyncio -async def test_create_market_mcp_with_blank_token_requires_each_installer(db, monkeypatch): - async def _validate(*args, **kwargs): - return None - - async def _fail_probe(*args, **kwargs): - return False, "401 Unauthorized" - - monkeypatch.setattr(market, "validate_remote_mcp_url", _validate) - monkeypatch.setattr(market, "probe_mcp_connectivity", _fail_probe) - monkeypatch.setattr(market, "refresh_mcp_caches", lambda: None) - monkeypatch.setattr(admin_mcp_routes, "_probe_connectivity", _fail_probe) - monkeypatch.setattr(admin_mcp_routes, "_sync_catalog_from_db", lambda db: None) - monkeypatch.setattr(admin_mcp_routes, "_refresh_caches", lambda: None) - - request = admin_mcp_routes.McpServerCreateRequest( - server_id="blank_token_mcp", - display_name="Blank Token MCP", - description="Discovers tools after the user authenticates", - transport="streamable_http", - url="https://mcp.example.test/mcp", - publish_to_marketplace=True, - market_requires_auth=True, - market_auth_type="token", - market_auth_header="Authorization", - market_auth_prefix="Bearer ", - market_admin_token="", - ) - await admin_mcp_routes.create_mcp_server(request, db) - - listed = market.list_market_items( - db, - owner_user_id="u2", - viewer_user_id="u2", - )[ - "items" - ][0] - assert listed["requires_auth"] is True - assert listed["requires_user_credentials"] is True - assert listed["credentials_managed_by_admin"] is False - assert listed["supports_admin_credentials"] is True - assert listed["auth_config"]["credential_mode"] == "installer" - assert listed["listing_notice"]["discovery_mode"] == "per_install" - assert listed["tool_count"] == 0 - - async def _authenticated_probe(row, db=None): - assert decrypt_mcp_headers(row.headers)["Authorization"] == "Bearer user-secret" - row.tools_json = _tools("search_documents") - return True, "" - - monkeypatch.setattr(market, "probe_mcp_connectivity", _authenticated_probe) - installed = await market.install_market_item( - db, - listed["slug"], - owner_user_id="u2", - installed_by="u2", - credentials={"Authorization": "user-secret"}, - ) - private_server = db.query(AdminMcpServer).filter_by(server_id=installed["server_id"]).one() - assert decrypt_mcp_headers(private_server.headers)["Authorization"] == "Bearer user-secret" - assert private_server.tools_json == _tools("search_documents") - - -@pytest.mark.asyncio -async def test_create_market_mcp_with_admin_token_makes_user_install_tokenless(db, monkeypatch): - discovered_tools = _tools("search_documents") - - async def _validate(*args, **kwargs): - return None - - async def _probe(row, db=None): - row.tools_json = discovered_tools - return True, "" - - monkeypatch.setattr(market, "validate_remote_mcp_url", _validate) - monkeypatch.setattr(market, "probe_mcp_connectivity", _probe) - monkeypatch.setattr(market, "refresh_mcp_caches", lambda: None) - monkeypatch.setattr(admin_mcp_routes, "_probe_connectivity", _probe) - monkeypatch.setattr(admin_mcp_routes, "_sync_catalog_from_db", lambda db: None) - monkeypatch.setattr(admin_mcp_routes, "_refresh_caches", lambda: None) - - request = admin_mcp_routes.McpServerCreateRequest( - server_id="admin_token_mcp", - display_name="Admin Token MCP", - description="Uses one administrator token", - transport="streamable_http", - url="https://mcp.example.test/mcp", - publish_to_marketplace=True, - market_requires_auth=True, - market_auth_type="token", - market_auth_header="Authorization", - market_auth_prefix="Bearer ", - market_admin_token="central-secret", - ) - await admin_mcp_routes.create_mcp_server(request, db) - - source = db.query(AdminMcpServer).filter_by(server_id="admin_token_mcp").one() - assert decrypt_mcp_headers(source.headers)["Authorization"] == "Bearer central-secret" - assert "central-secret" not in str(source.headers) - listed = market.list_market_items( - db, - owner_user_id="u2", - viewer_user_id="u2", - )[ - "items" - ][0] - assert listed["credentials_managed_by_admin"] is True - assert listed["requires_user_credentials"] is False - assert "central-secret" not in json.dumps(listed) - - -@pytest.mark.asyncio -async def test_create_market_mcp_with_oauth_defers_authorization_to_installer(db, monkeypatch): - async def _validate(*args, **kwargs): - return None - - async def _fail_probe(*args, **kwargs): - return False, "401 Unauthorized" - - monkeypatch.setattr(market, "validate_remote_mcp_url", _validate) - monkeypatch.setattr(market, "probe_mcp_connectivity", _fail_probe) - monkeypatch.setattr(market, "refresh_mcp_caches", lambda: None) - monkeypatch.setattr(admin_mcp_routes, "_probe_connectivity", _fail_probe) - monkeypatch.setattr(admin_mcp_routes, "_sync_catalog_from_db", lambda db: None) - monkeypatch.setattr(admin_mcp_routes, "_refresh_caches", lambda: None) - - request = admin_mcp_routes.McpServerCreateRequest( - server_id="oauth_market_mcp", - display_name="OAuth Market MCP", - transport="streamable_http", - url="https://mcp.example.test/mcp", - publish_to_marketplace=True, - market_requires_auth=True, - market_auth_type="oauth2", - ) - await admin_mcp_routes.create_mcp_server(request, db) - - listed = market.list_market_items( - db, - owner_user_id="u2", - viewer_user_id="u2", - )[ - "items" - ][0] - assert listed["requires_auth"] is True - assert listed["requires_user_credentials"] is True - assert listed["auth_config"]["default_method"] == "oauth2" - assert listed["auth_config"]["methods"][0]["type"] == "oauth2" - assert listed["listing_notice"]["discovery_mode"] == "per_install" - - -@pytest.mark.asyncio -async def test_move_existing_published_server_removes_global_installation(db, monkeypatch): - source = _private_server(db, owner="temporary", server_id="published_global") - source.owner_user_id = None - db.commit() - _patch_probe(monkeypatch) - - await market.publish_admin_server( - db, - source.server_id, - category="信息检索", - version="1.0.0", - ) - assert db.query(McpMarketInstallation).count() == 1 - - moved = await market.publish_admin_server( - db, - source.server_id, - category="信息检索", - version="1.0.1", - link_as_installation=False, - ) - - db.refresh(source) - assert moved["installed"] is False - assert source.is_enabled is False - assert source.extra_config["market_source_only"] is True - assert db.query(McpMarketInstallation).count() == 0 - - -def test_edit_market_metadata_propagates_to_existing_installations(db, monkeypatch): - item, version = _approved_market(db) - installed_server = AdminMcpServer( - server_id="installed_acme", - display_name=item.display_name, - description=item.description, - transport=version.transport, - url=version.url, - is_stable=False, - is_enabled=True, - owner_user_id="u2", - ) - db.add(installed_server) - db.flush() - db.add( - McpMarketInstallation( - install_id="mcpins_edit", - slug=item.slug, - version_id=version.version_id, - server_id=installed_server.server_id, - owner_user_id="u2", - status="active", - ) - ) - db.commit() - monkeypatch.setattr(market, "refresh_mcp_caches", lambda: None) - - result = market.update_market_item( - db, - item.slug, - display_name="Acme Knowledge Search", - description="Search the updated Acme knowledge base", - user_intro="## 用途\n检索 Acme 知识库", - category="通用工具", - tags=["search", "knowledge", "search"], - icon="/home/mcp/knowledge.svg", - ) - - db.refresh(installed_server) - assert result["display_name"] == "Acme Knowledge Search" - assert result["updated_installations"] == 1 - assert result["tags"] == ["search", "knowledge"] - assert installed_server.display_name == result["display_name"] - assert installed_server.description == result["description"] - assert installed_server.user_intro == result["user_intro"] - assert installed_server.icon == result["icon"] - assert db.query(McpMarketVersion).one().version == "1.0.0" - assert db.query(McpMarketVersion).one().tool_hash == version.tool_hash - - -@pytest.mark.asyncio -async def test_install_uses_installers_own_encrypted_credentials(db, monkeypatch): - _approved_market(db) - _patch_probe(monkeypatch) - - first = await market.install_market_item( - db, - "acme-search", - owner_user_id="u2", - installed_by="u2", - credentials={"Authorization": "Bearer user-two"}, - ) - second = await market.install_market_item( - db, - "acme-search", - owner_user_id="u3", - installed_by="u3", - credentials={"Authorization": "Bearer user-three"}, - ) - first_server = db.query(AdminMcpServer).filter_by(server_id=first["server_id"]).one() - second_server = db.query(AdminMcpServer).filter_by(server_id=second["server_id"]).one() - assert first_server.owner_user_id == "u2" - assert second_server.owner_user_id == "u3" - assert decrypt_mcp_headers(first_server.headers)["Authorization"] == "Bearer user-two" - assert decrypt_mcp_headers(second_server.headers)["Authorization"] == "Bearer user-three" - assert first_server.headers != second_server.headers - assert db.query(McpMarketInstallation).count() == 2 - - -@pytest.mark.asyncio -async def test_curated_templates_seed_once_and_materialize_user_query_key(db, monkeypatch): - assert set(market.ensure_curated_market_items(db)) == { - "amap-maps", - "metaso-search", - "github-official", - "gitlab-official", - "alibaba-cloud-observability", - "notion-official", - } - assert market.ensure_curated_market_items(db) == [] - gitlab_item = market.get_market_item( - db, - "gitlab-official", - owner_user_id="u2", - viewer_user_id="u2", - ) - assert gitlab_item["auth_config"]["default_method"] == "oauth2" - assert gitlab_item["supports_admin_credentials"] is False - assert {method["type"] for method in gitlab_item["auth_config"]["methods"]} == { - "oauth2", - "token", - } - - actual_tools = _tools("maps_geo", "maps_weather") - _patch_probe(monkeypatch, tools=actual_tools) - installed = await market.install_market_item( - db, - "amap-maps", - owner_user_id="u2", - installed_by="u2", - credentials={"AMAP_API_KEY": "user-amap-key"}, - ) - server = db.query(AdminMcpServer).filter_by(server_id=installed["server_id"]).one() - assert server.url == "https://mcp.amap.com/mcp" - assert "user-amap-key" not in str(server.headers) - runtime_url, runtime_headers = materialize_mcp_http_connection( - server.url, - decrypt_mcp_headers(server.headers), - ) - assert runtime_url == "https://mcp.amap.com/mcp?key=user-amap-key" - assert runtime_headers == {} - assert server.tools_json == actual_tools - assert server.owner_user_id == "u2" - assert server.icon == "/home/mcp/internet.svg" - - -@pytest.mark.asyncio -async def test_curated_bearer_token_gets_prefix_and_can_be_rotated(db, monkeypatch): - market.ensure_curated_market_items(db) - _patch_probe(monkeypatch, tools=_tools("metaso_web_search")) - - installed = await market.install_market_item( - db, - "metaso-search", - owner_user_id="u2", - installed_by="u2", - credentials={"METASO_API_KEY": "first-key"}, - ) - server = db.query(AdminMcpServer).filter_by(server_id=installed["server_id"]).one() - assert decrypt_mcp_headers(server.headers)["Authorization"] == "Bearer first-key" - - updated = await market.install_market_item( - db, - "metaso-search", - owner_user_id="u2", - installed_by="u2", - credentials={"METASO_API_KEY": "Bearer second-key"}, - ) - db.refresh(server) - assert updated["action"] == "existing" - assert decrypt_mcp_headers(server.headers)["Authorization"] == "Bearer second-key" - - -@pytest.mark.asyncio -async def test_tool_drift_pauses_new_installation(db, monkeypatch): - _approved_market(db) - _patch_probe(monkeypatch, tools=_tools("search_documents", "delete_documents")) - with pytest.raises(BadRequestError, match="发生变化"): - await market.install_market_item( - db, - "acme-search", - owner_user_id="u2", - installed_by="u2", - credentials={"Authorization": "Bearer u2"}, - ) - item = db.query(McpMarketItem).filter_by(slug="acme-search").one() - assert item.status == "changed" - assert db.query(McpMarketInstallation).count() == 0 - - -@pytest.mark.asyncio -async def test_failed_update_probe_never_commits_plaintext_credentials(db, monkeypatch): - _approved_market(db) - _patch_probe(monkeypatch) - installed = await market.install_market_item( - db, - "acme-search", - owner_user_id="u2", - installed_by="u2", - credentials={"Authorization": "Bearer original"}, - ) - _patch_probe(monkeypatch, tools=_tools("changed_tool")) - - with pytest.raises(BadRequestError, match="发生变化"): - await market.install_market_item( - db, - "acme-search", - owner_user_id="u2", - installed_by="u2", - credentials={"Authorization": "Bearer replacement"}, - ) - - server = db.query(AdminMcpServer).filter_by(server_id=installed["server_id"]).one() - assert decrypt_mcp_headers(server.headers)["Authorization"] == "Bearer original" - assert "original" not in str(server.headers) - assert "replacement" not in str(server.headers) - - -@pytest.mark.asyncio -async def test_security_suspension_disables_all_derived_servers(db, monkeypatch): - _approved_market(db) - _patch_probe(monkeypatch) - await market.install_market_item( - db, - "acme-search", - owner_user_id="u2", - installed_by="u2", - credentials={"Authorization": "Bearer u2"}, - ) - await market.install_market_item( - db, - "acme-search", - owner_user_id="u3", - installed_by="u3", - credentials={"Authorization": "Bearer u3"}, - ) - result = market.set_suspended(db, "acme-search", suspended=True, reason="malware") - assert result["disabled_installations"] == 2 - assert all(not row.is_enabled for row in db.query(AdminMcpServer).all()) - assert {row.status for row in db.query(McpMarketInstallation).all()} == {"suspended"} - - restored = market.set_suspended(db, "acme-search", suspended=False) - assert restored["restored_installations"] == 2 - assert all(row.is_enabled for row in db.query(AdminMcpServer).all()) - assert {row.status for row in db.query(McpMarketInstallation).all()} == {"active"} - - -@pytest.mark.asyncio -async def test_review_rechecks_live_snapshot(db, monkeypatch): - source = _private_server(db) - _patch_probe(monkeypatch) - submitted = await market.submit_to_marketplace( - db, - source.server_id, - owner_user_id="u1", - submitter_name="Alice", - category="信息检索", - version="1.0.0", - ) - _patch_probe(monkeypatch, tools=_tools("changed_after_submission")) - - with pytest.raises(BadRequestError, match="重新提交"): - await market.review_submission( - db, - submitted["submission_id"], - approve=True, - reviewed_by="admin", - ) - - assert db.query(McpMarketSubmission).one().status == "pending" - assert db.query(McpMarketItem).count() == 0 - - -@pytest.mark.asyncio -async def test_missing_source_revalidates_from_reviewed_version(db, monkeypatch): - _approved_market(db) - _patch_probe(monkeypatch) - result = await market.revalidate_market_item(db, "acme-search") - - assert result["status"] == "active" - assert result["status_reason"] == "" - - -@pytest.mark.asyncio -async def test_missing_source_still_detects_live_snapshot_drift(db, monkeypatch): - _approved_market(db) - _patch_probe(monkeypatch, tools=_tools("changed_after_review")) - - result = await market.revalidate_market_item(db, "acme-search") - - assert result["status"] == "changed" - assert "需发布新版本" in result["status_reason"] - - -@pytest.mark.asyncio -async def test_manual_admin_revalidation_accepts_drift_as_new_version(db, monkeypatch): - source = _private_server(db) - source.owner_user_id = None - item, old_version = _approved_market(db) - item.source = "admin" - db.commit() - changed_tools = _tools("search_documents", "list_accessible_projects") - _patch_probe(monkeypatch, tools=changed_tools) - - result = await market.revalidate_market_item( - db, - "acme-search", - accept_changes=True, - ) - - assert result["status"] == "active" - assert result["snapshot_accepted"] is True - assert result["previous_version"] == "1.0.0" - assert result["version"] == "1.0.1" - versions = { - row.version: row for row in db.query(McpMarketVersion).filter_by(slug="acme-search").all() - } - assert set(versions) == {"1.0.0", "1.0.1"} - assert versions["1.0.0"].tool_hash == old_version.tool_hash - assert versions["1.0.1"].tool_hash == tool_snapshot_hash(changed_tools) - assert versions["1.0.1"].approved_by == "admin-revalidation" - assert item.latest_version_id == versions["1.0.1"].version_id - - -@pytest.mark.asyncio -async def test_periodic_revalidation_never_auto_accepts_drift(db, monkeypatch): - source = _private_server(db) - source.owner_user_id = None - item, _ = _approved_market(db) - item.source = "admin" - db.commit() - _patch_probe(monkeypatch, tools=_tools("changed_after_review")) - - result = await market.revalidate_market_item(db, "acme-search") - - assert result["status"] == "changed" - assert db.query(McpMarketVersion).filter_by(slug="acme-search").count() == 1 - - -@pytest.mark.asyncio -async def test_manual_revalidation_rejects_empty_tool_snapshot(db, monkeypatch): - source = _private_server(db) - source.owner_user_id = None - item, _ = _approved_market(db) - item.source = "admin" - db.commit() - _patch_probe(monkeypatch, tools=[]) - - result = await market.revalidate_market_item( - db, - "acme-search", - accept_changes=True, - ) - - assert result["status"] == "changed" - assert "未返回任何工具" in result["status_reason"] - assert db.query(McpMarketVersion).filter_by(slug="acme-search").count() == 1 - - -@pytest.mark.skipif(_IS_CE_TREE, reason="CE marketplace listings are always public") -def test_mcp_visibility_uses_shared_marketplace_scope(db): - item, _ = _approved_market(db) - market.ml.set_listing_visibility( - db, - market.ml.KIND_MCP, - item.slug, - visibility="scoped", - grants=[{"principal_type": "user", "principal_id": "u2"}], - ) - visible = market.list_market_items( - db, - owner_user_id="u2", - viewer_user_id="u2", - ) - hidden = market.list_market_items( - db, - owner_user_id="u3", - viewer_user_id="u3", - ) - assert [row["slug"] for row in visible["items"]] == ["acme-search"] - assert hidden["items"] == [] - - -@pytest.mark.asyncio -async def test_delisted_item_blocks_user_install_but_not_admin_install(db, monkeypatch): - item, _ = _approved_market(db) - _patch_probe(monkeypatch) - market.ml.set_listing_enabled(db, market.ml.KIND_MCP, item.slug, False) - - with pytest.raises(ResourceNotFoundError): - await market.install_market_item( - db, - item.slug, - owner_user_id="u2", - installed_by="u2", - credentials={"Authorization": "Bearer u2"}, - ) - - installed = await market.install_market_item( - db, - item.slug, - owner_user_id=None, - installed_by="admin", - credentials={"Authorization": "Bearer global"}, - ) - assert installed["action"] == "installed" - - -def _admin_private_market(db, *, auth_schema=None, url: str = "http://mcp:9102/mcp/"): - """An admin-published item whose endpoint lives on the deployment's own network.""" - snapshot = _tools("search_documents") - item = McpMarketItem( - slug="builtin-search", - display_name="联网搜索", - description="Built-in search MCP", - category="信息检索", - tags=["search"], - publisher_name="平台", - source="admin", - latest_version_id="mcpver_builtin", - status="active", - ) - version = McpMarketVersion( - version_id="mcpver_builtin", - slug="builtin-search", - version="1.0.0", - transport="streamable_http", - url=url, - auth_schema=auth_schema or [], - tools_json=snapshot, - tool_hash=tool_snapshot_hash(snapshot), - ) - db.add_all([item, version]) - db.commit() - return item, version - - -def _patch_install_side_effects(monkeypatch, *, resolves_to: str = "172.27.0.4"): - """Keep the real URL guard in play while stubbing the network around it.""" - - async def _probe(row, db=None): - return True, "" - - monkeypatch.setattr( - socket, - "getaddrinfo", - lambda *args, **kwargs: [(socket.AF_INET, socket.SOCK_STREAM, 6, "", (resolves_to, 9102))], - ) - monkeypatch.setattr(market, "probe_mcp_connectivity", _probe) - monkeypatch.setattr(market, "refresh_mcp_caches", lambda: None) - - -@pytest.mark.asyncio -async def test_user_installs_admin_published_private_network_mcp(db, monkeypatch): - _admin_private_market(db) - _patch_install_side_effects(monkeypatch) - - installed = await market.install_market_item( - db, - "builtin-search", - owner_user_id="u2", - installed_by="u2", - credentials={}, - ) - - row = db.query(AdminMcpServer).filter(AdminMcpServer.owner_user_id == "u2").one() - assert installed["server_id"] == row.server_id - assert row.url == "http://mcp:9102/mcp/" - - -@pytest.mark.asyncio -async def test_installer_cannot_redirect_an_admin_item_to_another_private_host(db, monkeypatch): - _admin_private_market( - db, - auth_schema=[ - { - "key": "ENDPOINT", - "label": "Endpoint", - "target": "url", - "required": True, - "secret": True, - } - ], - ) - _patch_install_side_effects(monkeypatch) - - with pytest.raises(BadRequestError, match="本机、内网或保留地址"): - await market.install_market_item( - db, - "builtin-search", - owner_user_id="u2", - installed_by="u2", - credentials={"ENDPOINT": "http://127.0.0.1:9102/mcp/"}, - ) - assert db.query(AdminMcpServer).count() == 0 - - -@pytest.mark.asyncio -async def test_installer_may_keep_the_admin_host_when_overriding_the_endpoint(db, monkeypatch): - _admin_private_market( - db, - auth_schema=[ - { - "key": "ENDPOINT", - "label": "Endpoint", - "target": "url", - "required": True, - "secret": True, - } - ], - ) - _patch_install_side_effects(monkeypatch) - - await market.install_market_item( - db, - "builtin-search", - owner_user_id="u2", - installed_by="u2", - credentials={"ENDPOINT": "http://mcp:9102/mcp/tenant-a"}, - ) - - assert db.query(AdminMcpServer).filter(AdminMcpServer.owner_user_id == "u2").count() == 1 diff --git a/src/backend/tests/test_package_boundaries.py b/src/backend/tests/test_package_boundaries.py index 1a48ca65..f00187e6 100644 --- a/src/backend/tests/test_package_boundaries.py +++ b/src/backend/tests/test_package_boundaries.py @@ -25,10 +25,10 @@ def imported_modules(path): @pytest.mark.parametrize("layer", ["management", "packaging"]) def test_plugin_lifecycle_does_not_depend_on_agent_assembly(layer): - forbidden = ("core.llm.factory", "core.plugins.runtime") + forbidden = ("core.llm.factory", "core.capabilities.plugins.runtime") violations = [ f"{path.relative_to(BACKEND)}: {name}" - for path in (BACKEND / "core/plugins" / layer).rglob("*.py") + for path in (BACKEND / "core/capabilities/plugins" / layer).rglob("*.py") for name in imported_modules(path) if any(name == prefix or name.startswith(prefix + ".") for prefix in forbidden) ] @@ -36,24 +36,24 @@ def test_plugin_lifecycle_does_not_depend_on_agent_assembly(layer): def test_archive_and_manifest_parsing_do_not_depend_on_database_lifecycle(): - forbidden = ("core.db", "core.plugins.management", "sqlalchemy") + forbidden = ("core.db", "core.capabilities.plugins.management", "sqlalchemy") names = ("archive.py", "importer.py", "models.py") violations = [ f"{filename}: {name}" for filename in names - for name in imported_modules(BACKEND / "core/plugins/packaging" / filename) + for name in imported_modules(BACKEND / "core/capabilities/plugins/packaging" / filename) if any(name == prefix or name.startswith(prefix + ".") for prefix in forbidden) ] assert violations == [] def test_packaged_plugin_discovery_keeps_bundled_resources(): - from core.plugins.packaging.definitions import PLUGIN_BUNDLES_DIR - from core.plugins.packaging.importer import normalize_plugin_dir - from core.plugins.packaging.sources import _resolve_plugin_dir + from core.capabilities.plugins.packaging.definitions import PLUGIN_BUNDLES_DIR + from core.capabilities.plugins.packaging.importer import normalize_plugin_dir + from core.capabilities.plugins.packaging.sources import resolve_plugin_dir assert PLUGIN_BUNDLES_DIR == BACKEND / "extensions/plugins" - plugin_dir = _resolve_plugin_dir("skill-manager") + plugin_dir = resolve_plugin_dir("skill-manager") assert plugin_dir is not None normalized = normalize_plugin_dir(plugin_dir) assert normalized.slug == "skill-manager" diff --git a/src/backend/tests/test_package_sources.py b/src/backend/tests/test_package_sources.py new file mode 100644 index 00000000..5bd4d309 --- /dev/null +++ b/src/backend/tests/test_package_sources.py @@ -0,0 +1,23 @@ +"""CLI composes package providers before installing default or selected plugins.""" +from unittest.mock import Mock + + +def test_cli_install_initializes_package_sources(monkeypatch): + import cli + from core.db import engine + from core.capabilities.plugins import management + from core.harness.registry import package_sources + monkeypatch.setattr(package_sources.registration_state(), 'sources', {}) + monkeypatch.setattr(package_sources.registration_state(), 'upgrade_policy', None) + db = Mock() + monkeypatch.setattr(engine, "SessionLocal", lambda: db) + seen = [] + def install(session, slug, **kwargs): + assert session is db + assert package_sources.package_source("marketplace") is not None + assert package_sources.package_upgrade_policy() is not None + seen.append(slug) + monkeypatch.setattr(management, "install_plugin", install) + assert cli.install_plugins(["automation", "sites"]) == ["automation", "sites"] + assert seen == ["automation", "sites"] + db.close.assert_called_once() diff --git a/src/backend/tests/test_personal_agent.py b/src/backend/tests/test_personal_agent.py index 1e06ccbb..d8009c9b 100644 --- a/src/backend/tests/test_personal_agent.py +++ b/src/backend/tests/test_personal_agent.py @@ -11,10 +11,10 @@ @pytest.fixture(autouse=True) def personal_agent_on(monkeypatch): """These tests exercise the module itself: it is switched on and every user may use it.""" - from core.extensions import registry + from core.harness.registry import registry from personal_agent import access - monkeypatch.setattr(registry, "_enabled", {*registry._enabled, access.NAME}) + monkeypatch.setattr(registry.registration_state(), 'enabled', {*registry.registration_state().enabled, access.NAME}) monkeypatch.setattr(access, "user_has_capability", lambda db, user_id, flag: True) @@ -311,7 +311,7 @@ def test_explicit_attachments_reject_other_owner(sessions): def test_ordinary_chat_cannot_claim_personal_conversation(): from fastapi import HTTPException - from core.services.session_operations.session_context import _ensure_chat_session + from core.chat.operations.session_context import _ensure_chat_session with pytest.raises(HTTPException) as error: _ensure_chat_session(None, "pa_chat_reserved", "owner", "ordinary write") assert error.value.status_code == 403 @@ -378,7 +378,7 @@ def test_lifetime_history_starts_recent_and_supports_older_pages(sessions): def test_ordinary_search_and_fork_cannot_expose_personal_chat(sessions): from fastapi import HTTPException from core.db.chat_search import search_sessions - from core.services.chat_fork_service import ChatForkService + from core.chat.fork import ChatForkService from uuid import uuid4 with sessions() as db: chat=AgentContext(db,"owner").identity.ensure()["chat_id"] diff --git a/src/backend/tests/test_personal_agent_access.py b/src/backend/tests/test_personal_agent_access.py index 7953b9d3..e2a1c9fa 100644 --- a/src/backend/tests/test_personal_agent_access.py +++ b/src/backend/tests/test_personal_agent_access.py @@ -5,7 +5,7 @@ from core.db.models import ChatSession, UserShadow from core.db.reserved_chats import ordinary_chat_filter, reserved_chat_owner -from core.extensions import registry +from core.harness.registry import registry from personal_agent import access from personal_agent import extension from personal_agent.context import AgentContext @@ -16,7 +16,7 @@ def empty_registry(monkeypatch): for name, value in (("_enabled", set()), ("_agent_profiles", {}), ("_sandbox_routes", []), ("_channel_routes", []), ("_lifecycle", []), ("_routers", [])): - monkeypatch.setattr(registry, name, value) + monkeypatch.setattr(registry.registration_state(), name.lstrip("_"), value) def grant(db, user_id, value=True): @@ -99,8 +99,8 @@ def test_reserved_chats_stay_hidden_without_the_module(sessions, empty_registry) def test_reserved_chats_are_unreachable_through_ordinary_chat_access(sessions, empty_registry): # Every ordinary read (session detail, history, @-references, the history tool, # sidebar discovery) resolves chats through these lookups or the reference listing. - from core.services.chat_reference_service import list_referencable_sessions - from core.services.chat_service import ChatService + from core.chat.references import list_referencable_sessions + from core.chat.service import ChatService with sessions() as db: db.add_all([ChatSession(chat_id="pa_chat_1", user_id="owner"), ChatSession(chat_id="chat_1", user_id="owner")]) @@ -140,7 +140,7 @@ def test_community_access_requires_existing_user_and_enabled_module(sessions, em def test_history_filters_internal_wake_messages(): - from core.services.session_operations.history import _is_internal_message + from core.chat.operations.history import _is_internal_message assert not _is_internal_message(SimpleNamespace(role="user", content="Hello", extra_data={})) assert _is_internal_message(SimpleNamespace(role="user", content="[系统] 进度播报：running", extra_data={})) assert _is_internal_message(SimpleNamespace(role="assistant", content="hidden", extra_data={"hidden_in_chat": True})) diff --git a/src/backend/tests/test_personal_agent_capabilities.py b/src/backend/tests/test_personal_agent_capabilities.py index 37991fe2..05eebf93 100644 --- a/src/backend/tests/test_personal_agent_capabilities.py +++ b/src/backend/tests/test_personal_agent_capabilities.py @@ -13,10 +13,10 @@ def available(monkeypatch): "mcp": ["browser-automation-browser_runtime-u"]}, {"install_id": "sample@owner", "slug": "sample", "name": "Sample", "enabled": True, "skills": ["plugin-skill"], "mcp": ["plugin-tool"]}] - monkeypatch.setattr("core.config.catalog_resolver.resolve_all_runtime_enabled", + monkeypatch.setattr('core.capabilities.catalog.resolver.resolve_all_runtime_enabled', lambda db, uid: (permitted["skills"], [], permitted["mcps"])) - monkeypatch.setattr("core.plugins.management.list_installed", lambda db, **kw: plugins) - monkeypatch.setattr("core.capabilities.device_catalog.plugin_entries", lambda user_id=None: []) + monkeypatch.setattr('core.capabilities.plugins.management.list_installed', lambda db, **kw: plugins) + monkeypatch.setattr('features.desktop.capabilities.device_catalog.plugin_entries', lambda user_id=None: []) return permitted, plugins def test_browser_excluded_from_defaults_and_stale_runtime_config(sessions, available): @@ -82,10 +82,10 @@ async def test_explicit_profile_does_not_restore_history_or_skill_dependencies(m from core.llm.factory.selection.sticky_capabilities import sticky_capabilities from core.llm.factory.selection.progressive_bindings import progressive_bindings from core.llm.factory.models import RequiredCapabilities, StickyCapabilities - monkeypatch.setattr("core.extensions.agent_profile", lambda name: SimpleNamespace(inherit_default_mcps=False)) + monkeypatch.setattr('core.harness.registry.agent_profile', lambda name: SimpleNamespace(inherit_default_mcps=False)) def forbidden(**kwargs): pytest.fail("Explicit profile must not restore historical capability grants") - monkeypatch.setattr("core.plugins.runtime.resolve_sticky_plugin_capabilities", forbidden) + monkeypatch.setattr('core.capabilities.plugins.runtime.resolve_sticky_plugin_capabilities', forbidden) request = SimpleNamespace(enabled_skill_ids=["document"], enabled_mcp_ids=["web_fetch"], enabled_kb_ids=[], derived_main=False, chat_id="old-task", current_user_id="owner", disable_tools=False, turbo_mode=False, user_agent=None, profile="personal_agent") @@ -93,7 +93,7 @@ def forbidden(**kwargs): await sticky_capabilities(request, _elapsed=lambda: "", _eval_scope=None, _log=log, _mode_manual_invoke=False, _note_unavailable=lambda _: None, required=RequiredCapabilities(), sticky=StickyCapabilities()) - monkeypatch.setattr("core.llm.factory.selection.capabilities._filter_skill_ids_for_user", lambda ids, uid: ids) + monkeypatch.setattr('core.llm.factory.selection.capabilities.filter_skill_ids_for_user', lambda ids, uid: ids) monkeypatch.setattr("core.llm.factory.selection.capabilities._mcp_ids_bound_to_skills", lambda ids: ["browser_runtime", "web_fetch", "other"]) await progressive_bindings(request, _eval_scope=None, _log=log) @@ -164,7 +164,7 @@ def test_shadowed_private_plugin_components_are_not_standalone(sessions, availab global_plugin = {"install_id": "sample@global", "slug": "sample", "enabled": True, "skills": ["plugin-skill"], "mcp": ["plugin-tool"]} available[0]["skills"].append("private-component") - monkeypatch.setattr("core.plugins.management.list_installed", + monkeypatch.setattr('core.capabilities.plugins.management.list_installed', lambda db, **kw: [global_plugin] if kw.get("include_global") else [private]) with sessions() as db: svc = AgentContext(db, "owner") diff --git a/src/backend/tests/test_personal_agent_channel_files.py b/src/backend/tests/test_personal_agent_channel_files.py index f47f833a..3b26c2e3 100644 --- a/src/backend/tests/test_personal_agent_channel_files.py +++ b/src/backend/tests/test_personal_agent_channel_files.py @@ -3,7 +3,7 @@ import pytest -from core.channels.protocol import InboundMsg, SendResult +from integrations.channels.protocol import InboundMsg, SendResult from core.db.models import Artifact, ChatMessage from personal_agent.channels import channel_bridge from personal_agent.context import AgentContext @@ -43,8 +43,8 @@ async def push_file(self, conn, msg, content, name, mime): sent.append(("file", name, mime, content, msg.external_conversation_id)) return SendResult.ok() adapter = Adapter() - monkeypatch.setattr("core.channels.registry.get_adapter", lambda kind: adapter) - monkeypatch.setattr("core.storage.get_storage", lambda: SimpleNamespace(download_bytes=lambda key: b"data")) + monkeypatch.setattr('integrations.channels.registry.get_adapter', lambda kind: adapter) + monkeypatch.setattr('core.files.storage.get_storage', lambda: SimpleNamespace(download_bytes=lambda key: b"data")) return SimpleNamespace(conn=conn, adapter=adapter, sent=sent, task=task) @@ -186,7 +186,7 @@ async def test_artifact_deleted_after_queue_is_not_read(sessions, channel, monke db.commit() def unexpected_storage(): pytest.fail("Deleted artifact must not reach storage") - monkeypatch.setattr("core.storage.get_storage", unexpected_storage) + monkeypatch.setattr('core.files.storage.get_storage', unexpected_storage) await channel_bridge.deliver_pending() assert len(channel.sent) == 1 and channel.sent[0][0] == "text" diff --git a/src/backend/tests/test_personal_agent_channel_sessions.py b/src/backend/tests/test_personal_agent_channel_sessions.py index eb20fb34..bf87eda1 100644 --- a/src/backend/tests/test_personal_agent_channel_sessions.py +++ b/src/backend/tests/test_personal_agent_channel_sessions.py @@ -2,8 +2,9 @@ import pytest from types import SimpleNamespace from sqlalchemy import select -from core.channels.protocol import InboundMsg -from core.db.models import ChannelConnection, ChatMessage +from integrations.channels.protocol import InboundMsg +from integrations.channels.models import ChannelConnection +from core.db.models import ChatMessage from personal_agent.models import PersonalAgent, PersonalMessageInbox, PersonalWorkItem from personal_agent.context import AgentContext from personal_agent.identity.config import locked_agent @@ -129,7 +130,7 @@ def upload_bytes(self, content, key): return key def delete(self, key): pass monkeypatch.setattr(delivery, "SessionLocal", sessions) monkeypatch.setattr("personal_agent.tasks.guards.SessionLocal", sessions) - monkeypatch.setattr("core.storage.get_storage", lambda: Storage()) + monkeypatch.setattr('core.files.storage.get_storage', lambda: Storage()) state={"owner":"worker","run_id":"run","generation":generation} receipt = delivery._publish(task_id, "owner", state, "delivery-id", "artifact-id", "hash", b"report", "report.txt", "text/plain", "finished") diff --git a/src/backend/tests/test_personal_agent_cold_start.py b/src/backend/tests/test_personal_agent_cold_start.py index 265a6d34..e22507b7 100644 --- a/src/backend/tests/test_personal_agent_cold_start.py +++ b/src/backend/tests/test_personal_agent_cold_start.py @@ -1,4 +1,6 @@ """Cold SQLite initialization must allow catalog configuration to seed independently.""" +from core.config.store import system_config +from core.config.store import system_config_constants from types import SimpleNamespace from sqlalchemy import create_engine, select @@ -11,8 +13,8 @@ def test_first_identity_allows_catalog_seed_on_another_connection(tmp_path, monkeypatch): - from core.config import catalog_resolver - from core.services import system_config + from core.capabilities.catalog import resolver as catalog_resolver + from core.config.store import system_config from personal_agent.identity import capabilities # Separate physical connections expose SQLite's writer lock; an in-memory @@ -25,8 +27,8 @@ def test_first_identity_allows_catalog_seed_on_another_connection(tmp_path, monk factory = sessionmaker(bind=engine, expire_on_commit=False) Base.metadata.create_all(engine) config = system_config.SystemConfigService() - monkeypatch.setattr(system_config, "SessionLocal", factory) - monkeypatch.setattr(system_config, "SEED_CONFIGS", [ + monkeypatch.setattr("core.db.system_configuration.SessionLocal", factory) + monkeypatch.setattr(system_config_constants, 'SEED_CONFIGS', [ ("pa.cold_start", "seeded", "Cold start", "", "test", False), ]) monkeypatch.setattr(capabilities, "plugin_inventory", lambda db, user_id: []) diff --git a/src/backend/tests/test_personal_agent_computer_wait.py b/src/backend/tests/test_personal_agent_computer_wait.py index a40a642e..afdf70b8 100644 --- a/src/backend/tests/test_personal_agent_computer_wait.py +++ b/src/backend/tests/test_personal_agent_computer_wait.py @@ -77,7 +77,9 @@ async def driver(*args): monkeypatch.setattr(computer, "cloud_driver", driver) class Toolkit: def __init__(self): self.functions = {} - def register_tool_function(self, fn): self.functions[fn.__name__] = fn + def append(self, tool): + fn = tool._func + self.functions[fn.__name__] = fn with sessions() as db: svc = AgentContext(db, "owner") svc.identity.configure({"approval_mode": "full"}) @@ -134,7 +136,7 @@ async def test_worker_preserves_unknown_tool_outcome_despite_desktop_wait(sessio from contextlib import asynccontextmanager from types import SimpleNamespace from personal_agent.tasks import worker as module - from core.services.tool_effect_ledger import ToolOutcomeUnknown + from core.runs.tool_effects.errors import ToolOutcomeUnknown monkeypatch.setattr(module, "SessionLocal", sessions) monkeypatch.setattr("personal_agent.tasks.guards.SessionLocal", sessions) @asynccontextmanager diff --git a/src/backend/tests/test_personal_agent_delivery.py b/src/backend/tests/test_personal_agent_delivery.py index 6ad3e320..d16b93eb 100644 --- a/src/backend/tests/test_personal_agent_delivery.py +++ b/src/backend/tests/test_personal_agent_delivery.py @@ -48,15 +48,14 @@ def upload_bytes(self, content, key): self.files[key] = content; return key def download_bytes(self, key): return self.files[key] def delete(self, key): self.files.pop(key, None) storage = Storage() - monkeypatch.setattr("core.storage.get_storage", lambda: storage) + monkeypatch.setattr('core.files.storage.get_storage', lambda: storage) async def get_file(box, path, user_id=None): return files[path] monkeypatch.setattr(tools, "get_personal_sandbox_provider", lambda: SimpleNamespace(get_file=get_file)) cache = LocalEphemeralState() monkeypatch.setattr("core.infra.ephemeral.get_ephemeral_state", lambda: cache) - kit = SimpleNamespace(functions={}) - kit.register_tool_function = lambda fn: kit.functions.__setitem__(fn.__name__, fn) + kit = [] tools.register_personal_tools(kit, SimpleNamespace(profile_ref=task, current_user_id="owner")) - yield SimpleNamespace(task=task, functions=kit.functions, files=files, storage=storage, cache=cache) + yield SimpleNamespace(task=task, functions={tool.name: tool._func for tool in kit}, files=files, storage=storage, cache=cache) current_execution.set(None) @@ -240,7 +239,7 @@ async def test_observation_cache_is_bounded_and_receipt_survives_expiry(sessions async def test_screenshot_file_and_channel_end_to_end(sessions, setup, monkeypatch): - from core.channels.protocol import InboundMsg, SendResult + from integrations.channels.protocol import InboundMsg, SendResult from personal_agent.channels import channel_bridge monkeypatch.setattr(channel_bridge, "SessionLocal", sessions) monkeypatch.setattr("personal_agent.tasks.guards.SessionLocal", sessions) @@ -269,7 +268,7 @@ async def push_file(self, conn, msg, content, name, mime): sent.append(("file", name, mime)); return SendResult.ok() monkeypatch.setattr(channel_bridge, "connection", lambda ident: SimpleNamespace( channel_id="bot", channel_type="weixin", owner_user_id="owner")) - monkeypatch.setattr("core.channels.registry.get_adapter", lambda kind: Adapter()) + monkeypatch.setattr('integrations.channels.registry.get_adapter', lambda kind: Adapter()) await channel_bridge.deliver_pending() await channel_bridge.deliver_pending() with sessions() as db: diff --git a/src/backend/tests/test_personal_agent_fixes.py b/src/backend/tests/test_personal_agent_fixes.py index f8d50e82..92cc391f 100644 --- a/src/backend/tests/test_personal_agent_fixes.py +++ b/src/backend/tests/test_personal_agent_fixes.py @@ -1,4 +1,5 @@ """Personal Agent usability fixes: approvals, schedules, recovery, run limits and final answers.""" +from integrations.channels import inbound_attachments import personal_agent.tasks.guards as _pa_guards import asyncio import time @@ -198,7 +199,7 @@ def test_pending_approval_blocks_other_tools_until_user_decides(sessions, monkey def test_screen_match_tolerates_clock_but_not_navigation(): from PIL import Image, ImageDraw - from core.device_link import driver + from personal_agent.device_link import driver base = Image.new("RGB", (1280, 800), "white") clock = base.copy(); ImageDraw.Draw(clock).rectangle((1220, 0, 1280, 20), fill="black") page = base.copy(); ImageDraw.Draw(page).rectangle((0, 100, 1280, 700), fill="black") @@ -208,18 +209,30 @@ def test_screen_match_tolerates_clock_but_not_navigation(): assert not driver.same_screen({"observation_id": "old"}, {"observation_id": "new", "signature": driver.signature(base)}) -def test_old_screenshots_are_pruned_from_context(): +@pytest.mark.asyncio +async def test_old_screenshots_are_pruned_from_context(): from agentscope.message import DataBlock, Msg, TextBlock, ToolResultBlock - from personal_agent.computer.screenshot_pruning import prune, PLACEHOLDER + from personal_agent.computer.screenshot_pruning import screenshot_pruning_hook, PLACEHOLDER + from tests.orchestration.hook_support import make_adapter + adapter = make_adapter(screenshot_pruning_hook()) image = {"type": "base64", "media_type": "image/png", "data": "AAAA"} messages = [Msg(name="agent", role="assistant", content=[ToolResultBlock(id=str(i), name="pa_computer_observe", output=[TextBlock(text="observation " + str(i)), DataBlock(source=image)])]) for i in range(4)] - assert prune(messages, keep=2) == 2 + agent = SimpleNamespace(state=SimpleNamespace(context=messages)) + async def reasoning(**kwargs): + if False: + yield None + async for _ in adapter.on_reasoning(agent, {}, reasoning): + pass + messages = agent.state.context outputs = [m.content[0].output for m in messages] assert [o[1].text for o in outputs[:2]] == [PLACEHOLDER, PLACEHOLDER] assert all(isinstance(o[1], DataBlock) for o in outputs[2:]) assert outputs[0][0].text == "observation 0" - assert prune(messages, keep=2) == 0 + before = [m.model_dump() for m in messages] + async for _ in adapter.on_reasoning(agent, {}, reasoning): + pass + assert [m.model_dump() for m in agent.state.context] == before def test_final_answer_always_links_files_from_this_run(): @@ -232,15 +245,15 @@ def test_final_answer_always_links_files_from_this_run(): @pytest.mark.asyncio async def test_channel_message_enters_inbox_and_reply_returns_to_same_conversation(sessions, monkeypatch): - from core.channels.protocol import InboundMsg, SendResult + from integrations.channels.protocol import InboundMsg, SendResult from core.db.models import ChatMessage from personal_agent.channels import channel_bridge - import core.channels.inbound as inbound + import integrations.channels.inbound as inbound monkeypatch.setattr(channel_bridge, "SessionLocal", sessions) monkeypatch.setattr("personal_agent.tasks.guards.SessionLocal", sessions) async def no_files(*args): return [] - monkeypatch.setattr(inbound, "_ingest_attachments", no_files) + monkeypatch.setattr(inbound_attachments, 'ingest_attachments', no_files) sent = [] class Adapter: @@ -270,7 +283,7 @@ async def send_text(self, conn, msg, text): async def fake_send(conn, msg, text): delivered.append((msg.external_conversation_id, text)); return SendResult.ok() - monkeypatch.setattr("core.channels.registry.get_adapter", lambda kind: SimpleNamespace( + monkeypatch.setattr('integrations.channels.registry.get_adapter', lambda kind: SimpleNamespace( caps=SimpleNamespace(supports_markdown=False, max_message_len=0), send_text=fake_send)) monkeypatch.setattr(channel_bridge, "connection", lambda ident: conn) await channel_bridge.deliver_pending() @@ -308,7 +321,9 @@ def test_schedule_is_not_duplicated_and_scheduled_run_cannot_schedule(sessions, class Toolkit: def __init__(self): self.functions = {} - def register_tool_function(self, fn): self.functions[fn.__name__] = fn + def append(self, tool): + fn = tool._func + self.functions[fn.__name__] = fn toolkit = Toolkit() monkeypatch.setattr("personal_agent.tools.email.register_email_tools", lambda *a: None) monkeypatch.setattr("personal_agent.computer.computer.register_computer_tools", lambda *a: None) diff --git a/src/backend/tests/test_personal_agent_persona.py b/src/backend/tests/test_personal_agent_persona.py index 0b1af637..5e828fee 100644 --- a/src/backend/tests/test_personal_agent_persona.py +++ b/src/backend/tests/test_personal_agent_persona.py @@ -1,4 +1,7 @@ """Personal Agent workspace persona files and the assembled system prompt.""" +from core.llm import prompt_files as prompt_files_module +from core.llm import prompt_kinds as prompt_kinds_module +from core.llm import prompt_rendering as prompt_rendering_module import personal_agent.workspace.constants as _pa_constants import personal_agent.workspace.files as _pa_files import personal_agent.workspace.prompting as _pa_prompting @@ -98,9 +101,9 @@ def test_bootstrap_introduces_hugo_and_offers_a_rename(tmp_path, monkeypatch): def test_prompts_come_from_the_version_pool(workspace, monkeypatch): - from core.services import prompt_version_service as pvs + from core.llm import prompt_versions as pvs edited = {"platform_rules": "# Rules\nTask {task_id}; skills {skill_ids}.", "template_soul": "# SOUL.md - edited"} - monkeypatch.setattr(pvs, "render_active_prompt_part", lambda kind, part_id, db=None: edited.get(part_id) if kind == _pa_constants.PROMPT_KIND else None) + monkeypatch.setattr(prompt_rendering_module, 'render_active_prompt_part', lambda kind, part_id, db=None: edited.get(part_id) if kind == _pa_constants.PROMPT_KIND else None) text = _pa_prompting.render(_pa_prompting.sections(IDENT, task_id="t9", skill_ids=["a", "b"], seed_missing=False)) assert text.startswith("# Rules\nTask ; skills a, b.") # The skill list is generated, so an edited rules version still carries it. @@ -112,12 +115,12 @@ def test_prompts_come_from_the_version_pool(workspace, monkeypatch): def test_prompt_kind_ships_every_part(): - from core.services import prompt_version_service as pvs - pvs.register_kind(_pa_constants.PROMPT_KIND, _pa_prompting.prompt_spec()) + from core.llm import prompt_versions as pvs + prompt_kinds_module.register_kind(_pa_constants.PROMPT_KIND, _pa_prompting.prompt_spec()) try: - parts = {p["part_id"] for p in pvs._read_fs_parts(_pa_constants.PROMPT_KIND)} + parts = {p["part_id"] for p in prompt_files_module.read_fs_parts(_pa_constants.PROMPT_KIND)} finally: - pvs.KIND_SPECS.pop(_pa_constants.PROMPT_KIND) + prompt_kinds_module.KIND_SPECS.pop(_pa_constants.PROMPT_KIND) assert {"platform_rules", "platform_workspace_files", *_pa_constants.TEMPLATE_PARTS.values()} <= parts diff --git a/src/backend/tests/test_personal_agent_prompt_cache.py b/src/backend/tests/test_personal_agent_prompt_cache.py new file mode 100644 index 00000000..350a04ed --- /dev/null +++ b/src/backend/tests/test_personal_agent_prompt_cache.py @@ -0,0 +1,22 @@ +from tests.test_personal_agent_routing import sessions, AgentContext, admit +from tests.test_personal_agent import personal_agent_on + + +def test_the_turn_prefix_stays_cacheable(sessions, monkeypatch): + """The system prompt is identical across turns; the per-turn context follows the current message.""" + from core.db import engine as db_engine + from personal_agent.conversation import front + monkeypatch.setattr(db_engine, "SessionLocal", sessions) + monkeypatch.setattr("personal_agent.tasks.guards.SessionLocal", sessions) + with sessions() as db: + svc = AgentContext(db, "owner") + first = admit(svc, "你好", "hi") + svc.conversation.append(svc.identity.agent(), "assistant", "你好！", None, reply_to=first["message_id"]); db.commit() + second = admit(svc, "再见", "bye") + assert front.system_prompt(first["message_id"]) == front.system_prompt(second["message_id"]) + assert "Current time" not in front.system_prompt(second["message_id"]) + assert front.turn_context(second["message_id"]).startswith(front.CONTEXT_OPEN) + rows = front.with_context(front.history(db, second["message_id"]), "") + assert rows[-1]["content"] == "再见\n\n" + # Earlier messages are a prefix of what the previous turn sent. + assert front.history(db, second["message_id"])[0] == front.history(db, first["message_id"])[0] diff --git a/src/backend/tests/test_personal_agent_routing.py b/src/backend/tests/test_personal_agent_routing.py index fffca454..a3330a9d 100644 --- a/src/backend/tests/test_personal_agent_routing.py +++ b/src/backend/tests/test_personal_agent_routing.py @@ -31,7 +31,7 @@ def sessions(monkeypatch): from personal_agent.conversation import turn_worker for module in (front, front_tools, turn_worker): monkeypatch.setattr(module, "SessionLocal", factory) monkeypatch.setattr("personal_agent.tasks.guards.SessionLocal", factory) - from core.services import run_journal + from core.runs import journal as run_journal @asynccontextmanager async def binding(**options): @@ -44,10 +44,9 @@ async def binding(**options): def tools_for(ident, uid="owner"): from personal_agent.tools.front_tools import register_front_tools - kit = SimpleNamespace(functions={}) - kit.register_tool_function = lambda fn: kit.functions.__setitem__(fn.__name__, fn) + kit = [] register_front_tools(kit, SimpleNamespace(profile_ref=ident, current_user_id=uid)) - return kit.functions + return {tool.name: tool._func for tool in kit} def result(response): @@ -484,23 +483,3 @@ def test_an_agent_with_a_conversation_never_hatches(sessions, fresh): svc = AgentContext(db, "owner") admit(svc, "你好", "hi") assert not front.hatch(svc) and len(pending(svc)) == 1 - - -def test_the_turn_prefix_stays_cacheable(sessions, monkeypatch): - """The system prompt is identical across turns; the per-turn context follows the current message.""" - from core.db import engine as db_engine - from personal_agent.conversation import front - monkeypatch.setattr(db_engine, "SessionLocal", sessions) - monkeypatch.setattr("personal_agent.tasks.guards.SessionLocal", sessions) - with sessions() as db: - svc = AgentContext(db, "owner") - first = admit(svc, "你好", "hi") - svc.conversation.append(svc.identity.agent(), "assistant", "你好！", None, reply_to=first["message_id"]); db.commit() - second = admit(svc, "再见", "bye") - assert front.system_prompt(first["message_id"]) == front.system_prompt(second["message_id"]) - assert "Current time" not in front.system_prompt(second["message_id"]) - assert front.turn_context(second["message_id"]).startswith(front.CONTEXT_OPEN) - rows = front.with_context(front.history(db, second["message_id"]), "") - assert rows[-1]["content"] == "再见\n\n" - # Earlier messages are a prefix of what the previous turn sent. - assert front.history(db, second["message_id"])[0] == front.history(db, first["message_id"])[0] diff --git a/src/backend/tests/test_personal_agent_sessions.py b/src/backend/tests/test_personal_agent_sessions.py index e9c436b3..25d9c06b 100644 --- a/src/backend/tests/test_personal_agent_sessions.py +++ b/src/backend/tests/test_personal_agent_sessions.py @@ -204,7 +204,7 @@ def test_project_operations_map_to_fixed_endpoints(): def test_send_runs_in_the_chats_own_project(db): - from core.services.session_operations.admission import _check_project + from core.chat.operations.admission import _check_project from core.chat.schemas import ChatRequest from core.db.models import ChatSession, Project diff --git a/src/backend/tests/test_personal_agent_steer.py b/src/backend/tests/test_personal_agent_steer.py index f38cad9c..b08bc4ac 100644 --- a/src/backend/tests/test_personal_agent_steer.py +++ b/src/backend/tests/test_personal_agent_steer.py @@ -1,4 +1,7 @@ """Personal task steering through the same safe boundaries as ordinary chat.""" +from tests.orchestration.hook_support import make_adapter +from agentscope.message import ToolCallBlock + import time from types import SimpleNamespace import pytest @@ -21,20 +24,20 @@ async def test_steer_interrupts_old_tool_and_enters_context_without_status_poll( db.commit() inbox = steering.TaskSteering(row.id, "owner", "run", "worker", []) svc.tasks.steer(row.id, "Use a script instead", "update") - middleware = inbox.middleware() + middleware = make_adapter(*inbox.hook_policy().specs()) agent = SimpleNamespace(state=SimpleNamespace(run_id="run", context=[], steer_delivery=None)) called = [] async def tool(**kwargs): called.append(True) yield ToolChunk(content=[], state=ToolResultState.SUCCESS) - results = [x async for x in middleware.on_acting(agent, {}, tool)] + results = [x async for x in middleware.on_acting(agent, {"tool_call": ToolCallBlock(id="steer-tool", name="test_tool", input="{}")}, tool)] assert not called assert results[-1].state == ToolResultState.INTERRUPTED async def model(**kwargs): assert "Use a script instead" in agent.state.context[-1].content[0].text yield "continued" assert [x async for x in middleware.on_reasoning(agent, {}, model)] == ["continued"] - results = [x async for x in middleware.on_acting(agent, {}, tool)] + results = [x async for x in middleware.on_acting(agent, {"tool_call": ToolCallBlock(id="steer-tool", name="test_tool", input="{}")}, tool)] assert called == [True] assert results[-1].state == ToolResultState.SUCCESS @@ -50,7 +53,7 @@ async def test_inflight_result_is_preserved_and_multiple_inputs_are_delivered_on row.status, row.owner, row.run_id, row.lease_until = "running", "worker", "run", time.time() + 90 db.commit() inbox = steering.TaskSteering(task["id"], "owner", "run", "worker", []) - middleware = inbox.middleware() + middleware = make_adapter(*inbox.hook_policy().specs()) agent = SimpleNamespace(state=SimpleNamespace(run_id="run", context=[], steer_delivery=None)) async def tool(**kwargs): with sessions() as db: @@ -58,7 +61,7 @@ async def tool(**kwargs): svc.tasks.steer(task["id"], "Use a script", "one") svc.tasks.steer(task["id"], "Include the source", "two") yield ToolChunk(content=[], state=ToolResultState.SUCCESS) - results = [x async for x in middleware.on_acting(agent, {}, tool)] + results = [x async for x in middleware.on_acting(agent, {"tool_call": ToolCallBlock(id="steer-tool", name="test_tool", input="{}")}, tool)] assert results[-1].state == ToolResultState.SUCCESS async def model(**kwargs): text = agent.state.context[-1].content[0].text @@ -156,7 +159,7 @@ async def test_steer_invalidates_an_earlier_finish_before_model_continues(sessio db.commit() svc.tasks.steer(row.id, "Use a script", "update") inbox = steering.TaskSteering(task["id"], "owner", "run", "worker", []) - middleware = inbox.middleware() + middleware = make_adapter(*inbox.hook_policy().specs()) agent = SimpleNamespace(state=SimpleNamespace(run_id="run", context=[], steer_delivery=None)) async def model(**kwargs): with sessions() as db: @@ -176,7 +179,7 @@ class Sandbox: async def put_file(self, box, path, content, user_id): staged.append(path) monkeypatch.setattr("personal_agent.computer.sandbox.get_personal_sandbox_provider", lambda: Sandbox()) - monkeypatch.setattr("core.storage.get_storage", lambda: SimpleNamespace(download_bytes=lambda key: b"file")) + monkeypatch.setattr('core.files.storage.get_storage', lambda: SimpleNamespace(download_bytes=lambda key: b"file")) with sessions() as db: svc = AgentContext(db, "owner") task = svc.tasks.submit("Read files", "initial") diff --git a/src/backend/tests/test_personal_agent_tool_regressions.py b/src/backend/tests/test_personal_agent_tool_regressions.py index 57d1e8f8..82cb861e 100644 --- a/src/backend/tests/test_personal_agent_tool_regressions.py +++ b/src/backend/tests/test_personal_agent_tool_regressions.py @@ -12,14 +12,14 @@ async def test_exports_with_same_requested_name_create_distinct_current_files(se from personal_agent.tools import tools from personal_agent.models import PersonalWorkItem from core.db.models import Artifact - from core.storage import get_storage - from core.services import artifact_service + from core.files.storage import get_storage + from core.files.artifacts import service as artifact_service class Storage: def __init__(self): self.files = {} def upload_bytes(self, content, key): self.files[key] = content; return key storage = Storage() - import core.storage - monkeypatch.setattr(core.storage, "get_storage", lambda: storage) + import core.files.storage + monkeypatch.setattr(core.files.storage, "get_storage", lambda: storage) monkeypatch.setattr(tools, "SessionLocal", sessions) monkeypatch.setattr("personal_agent.tasks.guards.SessionLocal", sessions) from personal_agent.tools import space @@ -37,7 +37,9 @@ async def get_file(self, *args, **kwargs): return self.content monkeypatch.setattr(tools, "get_personal_sandbox_provider", lambda: provider) class Toolkit: def __init__(self): self.functions = {} - def register_tool_function(self, fn): self.functions[fn.__name__] = fn + def append(self, tool): + fn = tool._func + self.functions[fn.__name__] = fn with sessions() as db: task = AgentContext(db,"owner").tasks.submit("Export file","one") row = db.get(PersonalWorkItem, task["id"]) @@ -116,7 +118,9 @@ async def put_file(self, box, path, content, user_id=None): self.files[path] = c monkeypatch.setattr(tools, "get_personal_sandbox_provider", lambda: provider) class Toolkit: def __init__(self): self.functions = {} - def register_tool_function(self, fn): self.functions[fn.__name__] = fn + def append(self, tool): + fn = tool._func + self.functions[fn.__name__] = fn with sessions() as db: task = AgentContext(db, "owner").tasks.submit("Remember preference", "edit-one") row = db.get(PersonalWorkItem, task["id"]) @@ -197,7 +201,7 @@ def test_capabilities_follow_what_the_user_may_use(sessions, monkeypatch): from personal_agent import context as module from personal_agent.identity import config as identity_config permitted = {"skills": ["officecli-docx", "translate", "some-other-skill"], "mcps": ["internet_search", "web_fetch", "database_query"]} - monkeypatch.setattr("core.config.catalog_resolver.resolve_all_runtime_enabled", + monkeypatch.setattr('core.capabilities.catalog.resolver.resolve_all_runtime_enabled', lambda db, uid: (permitted["skills"], [], permitted["mcps"])) with sessions() as db: svc = module.AgentContext(db, "owner") @@ -215,15 +219,15 @@ def test_capabilities_follow_what_the_user_may_use(sessions, monkeypatch): def test_enabled_plugins_join_with_their_components(sessions, monkeypatch): from personal_agent import context as module from personal_agent.identity import config as identity_config - monkeypatch.setattr("core.config.catalog_resolver.resolve_all_runtime_enabled", + monkeypatch.setattr('core.capabilities.catalog.resolver.resolve_all_runtime_enabled', lambda db, uid: (["officecli-docx", "feishu-doc"], [], ["internet_search", "feishu_mcp"])) installed = [ {"install_id": "feishu@owner", "slug": "feishu", "name": "Feishu", "enabled": True, "callable": True, "skills": ["feishu-doc", "feishu-off"], "mcp": ["feishu_mcp"]}, {"install_id": "off@owner", "slug": "off", "name": "Off", "enabled": False, "callable": True, "skills": ["x"], "mcp": []}, ] - monkeypatch.setattr("core.plugins.management.list_installed", lambda db, **kw: installed) - monkeypatch.setattr("core.capabilities.device_catalog.plugin_entries", lambda user_id=None: []) + monkeypatch.setattr('core.capabilities.plugins.management.list_installed', lambda db, **kw: installed) + monkeypatch.setattr('features.desktop.capabilities.device_catalog.plugin_entries', lambda user_id=None: []) with sessions() as db: svc = module.AgentContext(db, "owner") config = svc.identity.ensure()["config"] @@ -231,9 +235,9 @@ def test_enabled_plugins_join_with_their_components(sessions, monkeypatch): assert config["plugins"] == [{"id": "feishu@owner", "name": "Feishu", "skill_ids": ["feishu-doc"], "mcp_ids": ["feishu_mcp"]}] assert identity_capabilities.capability_ids(config) == (["officecli-docx", "feishu-doc"], ["internet_search", "feishu_mcp"]) # An unreadable plugin listing keeps the last known set instead of breaking the agent. - monkeypatch.setattr("core.plugins.management.list_installed", lambda db, **kw: 1 / 0) + monkeypatch.setattr('core.capabilities.plugins.management.list_installed', lambda db, **kw: 1 / 0) assert svc.identity.ensure()["config"]["plugins"][0]["id"] == "feishu@owner" - monkeypatch.setattr("core.plugins.management.list_installed", lambda db, **kw: installed) + monkeypatch.setattr('core.capabilities.plugins.management.list_installed', lambda db, **kw: installed) # The conversation turn only answers and delegates: plugin tools stay with the tasks. assert identity_capabilities.front_mcp_ids(config) == ["internet_search"] installed[0]["enabled"] = False @@ -263,7 +267,7 @@ def test_model_choice_uses_the_chat_allowlist(sessions, monkeypatch): def resolve(db, pid, user_id=None): calls.append(pid) return pid if pid == "allowed" else None - monkeypatch.setattr("core.services.user_model_selection.resolve_user_model_provider_id", resolve) + monkeypatch.setattr('core.auth.model_selection.resolve_user_model_provider_id', resolve) with sessions() as db: svc = AgentContext(db, "owner") assert svc.identity.configure({"model_provider_id": "allowed"})["config"]["model_provider_id"] == "allowed" diff --git a/src/backend/tests/test_personal_canonical_history.py b/src/backend/tests/test_personal_canonical_history.py index 8715b101..021454cd 100644 --- a/src/backend/tests/test_personal_canonical_history.py +++ b/src/backend/tests/test_personal_canonical_history.py @@ -2,8 +2,8 @@ import pytest from core.db.models import ChatMessage -from core.llm.model_steps import record_assistant_step, record_tool_result_step -from core.services.compaction_history import normalize_rows +from core.llm.runtime.model_steps import record_assistant_step, record_tool_result_step +from core.chat.compaction.history import normalize_rows from tests.test_personal_agent_monitor import sessions as sessions from tests.test_personal_task_transcript_order import prepared as prepared from tests.test_personal_task_transcript_order import running as running diff --git a/src/backend/tests/test_personal_task_transcript_order.py b/src/backend/tests/test_personal_task_transcript_order.py index 78c01a02..d092ebca 100644 --- a/src/backend/tests/test_personal_task_transcript_order.py +++ b/src/backend/tests/test_personal_task_transcript_order.py @@ -1,9 +1,11 @@ """Task history preserves the position of instructions received during execution.""" +from tests.orchestration.hook_support import make_adapter +from agentscope.message import ToolCallBlock + import time from types import SimpleNamespace import pytest - from core.db.models import ChatRun from personal_agent.context import AgentContext from personal_agent.conversation.message_routing import admit @@ -165,11 +167,12 @@ def test_lost_execution_cannot_append_or_change_history(running, sessions, chang async def test_execute_task_consumes_midrun_instruction_and_finishes_in_order(prepared, sessions, monkeypatch): import asyncio import threading - from sqlalchemy import event + from personal_agent.tasks import runtime + from personal_agent.tasks.execution import current_execution from personal_agent.tasks.steering import current_steering from personal_agent.tools.tools import register_personal_tools - from personal_agent.tasks.execution import current_execution + from sqlalchemy import event ident, binding = prepared main_thread = threading.get_ident() @@ -189,8 +192,7 @@ async def input_loop_pulse(): monkeypatch.setattr("personal_agent.tools.tools.SessionLocal", sessions) monkeypatch.setattr("personal_agent.tools.space.SessionLocal", sessions) monkeypatch.setattr("personal_agent.tools.email.SessionLocal", sessions) - kit = SimpleNamespace(functions={}) - kit.register_tool_function = lambda fn: kit.functions.__setitem__(fn.__name__, fn) + kit = [] register_personal_tools(kit, SimpleNamespace(profile_ref=ident, current_user_id="owner")) # Substitute only the model/stream boundary; admission, task tools, transcript, @@ -207,19 +209,19 @@ async def stream(self, **options): assert database_started.is_set() and pulse.is_set() update(sessions, ident, "改用脚本", "runtime-update") from personal_agent.tasks.steering import current_steering - middleware = current_steering.get().middleware() + middleware = make_adapter(*current_steering.get().hook_policy().specs()) agent = SimpleNamespace(state=SimpleNamespace(run_id="run", context=[], steer_delivery=None)) async def continue_model(**kwargs): assert "改用脚本" in agent.state.context[-1].content[0].text yield "text_delta", "已按脚本要求完成" - await kit.functions["pa_finish"]("已按脚本要求完成") + await {tool.name: tool._func for tool in kit}["pa_finish"]("已按脚本要求完成") async for event in middleware.on_reasoning(agent, {}, continue_model): yield event monkeypatch.setattr("core.llm.factory.create_agent_executor", executor) - monkeypatch.setattr("orchestration.streaming.StreamingAgent", ModelStream) + monkeypatch.setattr('core.llm.runtime.streaming.StreamingAgent', ModelStream) token = current_execution.set(None) steering_token = current_steering.set(None) engine = sessions.kw["bind"] diff --git a/src/backend/tests/test_plugin_components.py b/src/backend/tests/test_plugin_components.py index aee4cb4e..c1c6b906 100644 --- a/src/backend/tests/test_plugin_components.py +++ b/src/backend/tests/test_plugin_components.py @@ -1,3 +1,4 @@ +from features.marketplace import plugin_market """Plugin system tests: import Claude Code / Codex / native plugin packages → persist to DB → uninstall. Covers the three-tier portability matrix: direct skill import (including @@ -5,6 +6,7 @@ disabled on install, and drop warnings for hooks/commands/agents. See internal design docs §10. """ +from integrations.lark import service as lark_service import json @@ -15,13 +17,13 @@ from core.infra.exceptions import BadRequestError -from core.plugins.packaging.archive import _extract_plugin_zip -from core.plugins.packaging import importer as pi +from core.capabilities.plugins.packaging.archive import extract_plugin_zip +from core.capabilities.plugins.packaging import importer as pi -from core.plugins import management as ps +from core.capabilities.plugins import management as ps -from core.plugins.management import packages as plugin_packages -from core.plugins.packaging import sources as plugin_sources +from core.capabilities.plugins.management import packages as plugin_packages +from core.capabilities.plugins.packaging import sources as plugin_sources OWNER = "test_user_123" @@ -41,7 +43,7 @@ def test_import_zip_rejects_backslash_path_traversal(): zf.writestr("..\\outside.txt", "blocked") with pytest.raises(BadRequestError, match="非法压缩包条目"): - with _extract_plugin_zip(buf.getvalue()): + with extract_plugin_zip(buf.getvalue()): pass @@ -49,7 +51,7 @@ def test_long_namespaced_component_ids_remain_unique(): """Long skill/MCP names with the same prefix must not overwrite one another.""" slug = "petrochemical-product-skills-v1-0" skill_ids = { - plugin_sources._make_skill_id(slug, name, OWNER) + plugin_sources.make_skill_id(slug, name, OWNER) for name in ( "petrochemical-prosperity-analysis", "petrochemical-price-trend", @@ -57,7 +59,7 @@ def test_long_namespaced_component_ids_remain_unique(): ) } server_ids = { - plugin_sources._make_server_id(slug, name, OWNER) + plugin_sources.make_server_id(slug, name, OWNER) for name in ( "petrochemical-product-market-data-provider-primary", "petrochemical-product-market-data-provider-secondary", @@ -94,7 +96,7 @@ def test_user_toggle_global_plugin_per_user(tmp_path, db_session): rather than the test db_session, which is unavailable in unit tests; per-user effectiveness is verified inside the container.) """ - from core.services.catalog_service import CatalogService + from core.capabilities.catalog.overrides import CatalogService zip_bytes = _zip_cc_plugin(tmp_path) res = ps.import_plugin_from_zip( @@ -126,7 +128,7 @@ def test_publish_zip_to_market_then_install(tmp_path, db_session): assert db_session.query(InstalledPlugin).count() == 0 # 2. The market list shows it, marked source=uploaded, not installed - market = ps.list_plugins(db_session, owner_user_id=None) + market = plugin_market.list_plugins(db_session, owner_user_id=None) m = next((p for p in market if p["slug"] == "hello-toolkit"), None) assert m is not None and m["source"] == "uploaded" and m["installed"] is False @@ -140,14 +142,14 @@ def test_publish_zip_to_market_then_install(tmp_path, db_session): ) assert inst["action"] == "installed" assert db_session.query(InstalledPlugin).count() == 1 - market2 = ps.list_plugins(db_session, owner_user_id=None) + market2 = plugin_market.list_plugins(db_session, owner_user_id=None) assert next(p for p in market2 if p["slug"] == "hello-toolkit")["installed"] is True # 5. Publishing again = update; deleting the market package does not affect installed instances res2 = ps.publish_plugin_zip_to_market(db_session, zip_bytes) assert res2["action"] == "updated" ps.delete_market_package(db_session, "hello-toolkit") - from core.db.models import PluginMarketPackage + from features.marketplace.models import PluginMarketPackage assert db_session.query(PluginMarketPackage).count() == 0 assert db_session.query(InstalledPlugin).count() == 1 # the installed instance is still there @@ -168,7 +170,7 @@ def test_app_registers_plugin_router(): @pytest.mark.asyncio async def test_feishu_plugin_app_routes_delegate_to_lark_service(monkeypatch): from api.routes.v1 import plugins as plugin_routes - from core.services import lark_service + from integrations.lark import service as lark_service calls = [] @@ -188,7 +190,7 @@ async def reset_app(self): calls.append("reset") return {"configured": False, "status": "idle"} - monkeypatch.setattr(lark_service, "LarkService", FakeLarkService) + monkeypatch.setattr(lark_service, 'LarkService', FakeLarkService) status = await plugin_routes.get_feishu_app_status(_="admin") started = await plugin_routes.init_feishu_app(_="admin") @@ -344,15 +346,15 @@ def test_legacy_topfield_manifest_still_imports(tmp_path): def test_market_meta_seed_override_and_install(db_session): # Seed applies without any override - meta = ps.resolve_market_meta(db_session, "automation") + meta = plugin_market.resolve_market_meta(db_session, "automation") assert meta["display_name"] == "定时任务管理" # Admin override wins over the seed and flows into the market list - ps.set_market_meta(db_session, "automation", display_name="自动化任务", category="效率") - meta = ps.resolve_market_meta(db_session, "automation") + plugin_market.set_market_meta(db_session, "automation", display_name="自动化任务", category="效率") + meta = plugin_market.resolve_market_meta(db_session, "automation") assert meta["display_name"] == "自动化任务" and meta["category"] == "效率" items = { it["slug"]: it - for it in ps.list_plugins(db_session, owner_user_id=None, include_disabled=True) + for it in plugin_market.list_plugins(db_session, owner_user_id=None, include_disabled=True) } assert items["automation"]["name"] == "自动化任务" # Install picks up the effective metadata for the installed record @@ -364,13 +366,13 @@ def test_market_meta_seed_override_and_install(db_session): ) assert row.name == "自动化任务" # Clearing the override falls back to the seed - ps.set_market_meta(db_session, "automation", display_name="", category="") - assert ps.resolve_market_meta(db_session, "automation")["display_name"] == "定时任务管理" + plugin_market.set_market_meta(db_session, "automation", display_name="", category="") + assert plugin_market.resolve_market_meta(db_session, "automation")["display_name"] == "定时任务管理" def test_market_meta_rejects_unknown_slug(db_session): with pytest.raises(Exception): - ps.set_market_meta(db_session, "no-such-plugin", display_name="x") + plugin_market.set_market_meta(db_session, "no-such-plugin", display_name="x") def test_installed_meta_owner_guard(tmp_path, db_session): @@ -389,26 +391,26 @@ def test_installed_meta_owner_guard(tmp_path, db_session): def test_market_meta_icon_validation(db_session): # Library path and uploaded data-URI are accepted - ps.set_market_meta(db_session, "automation", icon="/home/mcp/internet.svg") - assert ps.resolve_market_meta(db_session, "automation")["icon"] == "/home/mcp/internet.svg" - ps.set_market_meta(db_session, "automation", icon="data:image/svg+xml;base64,PHN2Zy8+") - assert ps.resolve_market_meta(db_session, "automation")["icon"].startswith("data:image/") + plugin_market.set_market_meta(db_session, "automation", icon="/home/mcp/internet.svg") + assert plugin_market.resolve_market_meta(db_session, "automation")["icon"] == "/home/mcp/internet.svg" + plugin_market.set_market_meta(db_session, "automation", icon="data:image/svg+xml;base64,PHN2Zy8+") + assert plugin_market.resolve_market_meta(db_session, "automation")["icon"].startswith("data:image/") # Non-image data URIs and arbitrary text are rejected with pytest.raises(BadRequestError): - ps.set_market_meta(db_session, "automation", icon="data:text/html;base64,eA==") + plugin_market.set_market_meta(db_session, "automation", icon="data:text/html;base64,eA==") with pytest.raises(BadRequestError): - ps.set_market_meta(db_session, "automation", icon="not-an-icon") + plugin_market.set_market_meta(db_session, "automation", icon="not-an-icon") # Oversized data URIs are rejected with pytest.raises(BadRequestError): - ps.set_market_meta(db_session, "automation", icon="data:image/png;base64," + "A" * 300_000) + plugin_market.set_market_meta(db_session, "automation", icon="data:image/png;base64," + "A" * 300_000) # Empty clears the override - ps.set_market_meta(db_session, "automation", icon="") - assert "icon" not in ps.resolve_market_meta(db_session, "automation") + plugin_market.set_market_meta(db_session, "automation", icon="") + assert "icon" not in plugin_market.resolve_market_meta(db_session, "automation") def test_capability_logo_tool_ownership(tmp_path, db_session): """Public/private connectors and installed plugins expose only tool names.""" - from core.config.catalog_runtime import _public_db_mcp_items + from core.capabilities.catalog.runtime import _public_db_mcp_items from api.routes.v1.catalog import _load_owned_capability_items ps.import_plugin_from_zip( @@ -443,7 +445,7 @@ def test_capability_logo_tool_ownership(tmp_path, db_session): def test_database_connector_inherits_child_tool_names(db_session, monkeypatch): - from core.config import catalog_runtime as runtime + from core.capabilities.catalog import runtime as runtime monkeypatch.setattr(runtime, "_database_query_capability_available", lambda: True) row = AdminMcpServer( diff --git a/src/backend/tests/test_plugin_manager_plugin.py b/src/backend/tests/test_plugin_manager_plugin.py index a92b055e..013f0a99 100644 --- a/src/backend/tests/test_plugin_manager_plugin.py +++ b/src/backend/tests/test_plugin_manager_plugin.py @@ -29,7 +29,7 @@ def pm_env(tmp_path, monkeypatch): TestSession = sessionmaker(bind=engine, expire_on_commit=False) monkeypatch.setattr(dbe, "SessionLocal", TestSession) - from core.artifacts import store + from core.files.artifacts import store art_dir = tmp_path / "artifacts" art_dir.mkdir(parents=True, exist_ok=True) @@ -76,7 +76,7 @@ def _make_skill_only_tar() -> bytes: return buf.getvalue() def _stash(tar_bytes: bytes, name: str = "plugin.tgz") -> str: - from core.artifacts import store + from core.files.artifacts import store return store.save_artifact_bytes(content=tar_bytes, name=name, extension="tgz")["file_id"] @@ -88,7 +88,7 @@ def test_bundle_manifest_is_wellformed(): assert manifest["name"] == "plugin-manager" ext_mcp = manifest["extensions"]["org.hugagent"]["mcp"] assert set(ext_mcp) == set(mcp_json["mcpServers"]), "扩展段的服务名必须与 mcp.json 一一对应" - assert mcp_json["mcpServers"]["plugin_manager"]["url"] == "http://mcp:9116/mcp/" + assert mcp_json["mcpServers"]["plugin_manager"]["url"] == "http://mcp:9116/mcp" from extensions.mcp_servers.plugin_manager_mcp import server @@ -124,7 +124,7 @@ def test_shipped_validator_accepts_all_real_bundles(): # ── Plugin persistence + merging into the user's available set ────────────── def test_install_plugin_creates_rows_and_is_agent_visible(pm_env): - from core.plugins import management as ps + from core.capabilities.plugins import management as ps with pm_env.Session() as db: res = ps.install_plugin(db, "plugin-manager", owner_user_id=OWNER, created_by=OWNER) @@ -138,15 +138,12 @@ def test_install_plugin_creates_rows_and_is_agent_visible(pm_env): assert len(skills) == 1 and skills[0].owner_user_id == OWNER assert len(mcps) == 1 mcp = mcps[0] - assert mcp.url == "http://mcp:9116/mcp/" + assert mcp.url == "http://mcp:9116/mcp" assert mcp.is_enabled is True assert len(mcp.tools_json) == 8 assert plugin is not None - from core.config.catalog_resolver import ( - invalidate_capability_cache, - resolve_all_runtime_enabled, - ) + from core.capabilities.catalog.resolver import invalidate_capability_cache, resolve_all_runtime_enabled invalidate_capability_cache() enabled_skills, _agents, enabled_mcps = resolve_all_runtime_enabled(db, OWNER) @@ -224,7 +221,7 @@ def _fake(user_id, kind): captured["user_id"] = user_id return {"ok": True, "plugins": [], "count": 0, "message": ""} - from core.services import cloud_management + from core.capabilities.plugins.management import cloud as cloud_management monkeypatch.setattr(cloud_management, "list_installed", _fake) handler = server.mcp._tool_manager.get_tool("list_plugins").fn diff --git a/src/backend/tests/test_plugin_progressive_loading.py b/src/backend/tests/test_plugin_progressive_loading.py index 5eb9ad31..3927fe23 100644 --- a/src/backend/tests/test_plugin_progressive_loading.py +++ b/src/backend/tests/test_plugin_progressive_loading.py @@ -6,6 +6,7 @@ """ from __future__ import annotations +from core.capabilities.skills import loader_factory as loader_factory_module import asyncio from types import SimpleNamespace @@ -13,8 +14,8 @@ import core.db.engine as dbe import pytest from core.db.models import ChatSession, InstalledPlugin, UserShadow -from core.plugins import runtime as plugin_loader -from core.llm.tool_collector import ToolCollector +from core.llm.factory.tools.definitions import ToolAssembly +from core.capabilities.plugins import runtime as plugin_loader from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker @@ -73,7 +74,7 @@ def ppl_env(tmp_path, monkeypatch): get_all_servers=lambda enabled_only=True: dict(server_cfgs), get_owned_servers=lambda uid, enabled_only=False: {}, ) - from core.services.mcp_service import McpServerConfigService + from core.capabilities.mcp.service import McpServerConfigService monkeypatch.setattr(McpServerConfigService, "get_instance", classmethod(lambda cls: svc)) @@ -85,9 +86,9 @@ def ppl_env(tmp_path, monkeypatch): load_all_metadata=lambda: dict(meta), get_skill_dir=lambda sid: None, ) - import core.agent_skills.loader as skills_loader_mod + import core.capabilities.skills.loader as skills_loader_mod - monkeypatch.setattr(skills_loader_mod, "get_skill_loader", lambda: loader_stub) + monkeypatch.setattr(loader_factory_module, 'get_skill_loader', lambda: loader_stub) return SimpleNamespace(Session=TestSession, loader=loader_stub, server_cfgs=server_cfgs) @@ -150,7 +151,7 @@ def test_sticky_activation_restores_personally_disabled_components(ppl_env, monk row.extra_data = {"activated_plugins": ["crawler@global"]} db.commit() - import core.config.catalog_resolver as resolver + import core.capabilities.catalog.resolver as resolver def allow_explicit(db, user_id, *, skill_ids=None, mcp_ids=None): return list(skill_ids or []), list(mcp_ids or []), [], [] @@ -170,7 +171,7 @@ def test_sticky_activation_accepts_existing_slug_rows(ppl_env, monkeypatch): row.extra_data = {"activated_plugins": ["crawler"]} db.commit() - import core.config.catalog_resolver as resolver + import core.capabilities.catalog.resolver as resolver monkeypatch.setattr( resolver, @@ -202,34 +203,30 @@ def test_directory_section_stable_and_empty_safe(ppl_env): assert plugin_loader.build_plugin_directory_section(res2.directory) == section -class _FakeMCPClient: - def __init__(self, name, cfg): - self.name = name - - async def list_tools(self): - return [ - SimpleNamespace( - name=name, - description=f"{name} tool", - input_schema={"type": "object", "properties": {}}, - ) - for name in ("crawl_url", "crawl_site") - ] +async def _discover(key, cfg, *, interactions, scope): + assert interactions.agent is not None + assert interactions.request.chat_id == CHAT + assert scope == (USER, CHAT, "") + from agentscope.tool import FunctionTool + from core.llm.mcp.lifetime import MCPResource + async def crawl(): + return "ok" + return MCPResource(key, tools=[FunctionTool(crawl, name=name) for name in ("crawl_url", "crawl_site")]) @pytest.mark.asyncio async def test_load_plugin_tool_activates_in_place(ppl_env, tmp_path, monkeypatch): - import core.llm.mcp_pool as mcp_pool + import core.llm.mcp.lifetime as mcp_pool + from core.agent_runtime.budget import _compaction_budget_inputs from core.llm.factory.runtime.evidence import cache_compaction_execution_surface - from core.ontology.toolkit import OntologyFilteredToolkit - from core.services import compaction_service as compaction - from core.services.chat_service import ChatService - from orchestration.workflow import _compaction_budget_inputs + from core.llm.tools.ontology_toolkit import OntologyFilteredToolkit + from core.chat.compaction import service as compaction + from core.chat.service import ChatService monkeypatch.setattr( mcp_pool, - "make_client", - lambda key, cfg, is_stateful=True, **kw: _FakeMCPClient(key, cfg), + "discover", + _discover, ) # 技能目录物化桩：含 SKILL.md 的临时目录。 @@ -242,7 +239,7 @@ async def test_load_plugin_tool_activates_in_place(ppl_env, tmp_path, monkeypatc ppl_env.loader.get_skill_dir = lambda sid: str(skill_dir) res = _resolve(ppl_env) - collector = ToolCollector() + collector = ToolAssembly() tk = OntologyFilteredToolkit(tools=[], mcps=[]) budget_agent = SimpleNamespace() tk.set_execution_surface_listener( @@ -253,10 +250,12 @@ async def test_load_plugin_tool_activates_in_place(ppl_env, tmp_path, monkeypatc initial_surface = await tk.freeze_execution_surface() close_list: list = [] permission_context = SimpleNamespace(allow_rules={}) - from core.llm.tool_permissions import ToolPermissionRegistry + from core.sandbox.permissions.tool_specs import ToolPermissionRegistry permission_registry = ToolPermissionRegistry() runtime = { + "request": SimpleNamespace(current_user_id=USER, chat_id=CHAT, capability_scope="", isolated=False), + "agent": SimpleNamespace(), "activated_slugs": set(), "connected_keys": {"internet_search"}, "toolkit": tk, @@ -272,8 +271,8 @@ async def test_load_plugin_tool_activates_in_place(ppl_env, tmp_path, monkeypatc "approval_available": False, "ontology_runtime": {}, } - plugin_loader.register_load_plugin(collector, res.deferred_by_slug(), runtime) - tool = collector.get_tool("load_plugin") + plugin_activation.register_load_plugin(collector, res.deferred_by_slug(), runtime) + tool = next(t for t in collector if t.name == "load_plugin") assert tool is not None resp = await tool._func(plugin="crawler") @@ -281,7 +280,7 @@ async def test_load_plugin_tool_activates_in_place(ppl_env, tmp_path, monkeypatc assert "已加载" in text and "crawl_url" in text and "crawler-scrape-a1" in text basic = tk.tool_groups[0] - assert len(basic.mcps) == 1 and len(close_list) == 1 + assert len(basic.tools) == 2 and len(close_list) == 1 assert {r for r in permission_context.allow_rules} == {"crawl_url", "crawl_site"} assert permission_registry.names == frozenset() assert len(basic.skills_or_loaders) == 1 @@ -301,7 +300,7 @@ async def test_load_plugin_tool_activates_in_place(ppl_env, tmp_path, monkeypatc async def fake_summary(history, *, timeout): return "plugin-aware summary" - monkeypatch.setattr("core.services.compaction_summary._summarize", fake_summary) + monkeypatch.setattr('core.chat.compaction.summary._summarize', fake_summary) with ppl_env.Session() as db: # Keep a real older turn beyond the protected verbatim tail. ChatService(db).add_message(CHAT, "user", "earlier context " * 20000) @@ -319,7 +318,7 @@ async def fake_summary(history, *, timeout): resp2 = await tool._func(plugin="crawler") text2 = "".join(getattr(b, "text", "") for b in resp2.content) assert "无需重复调用" in text2 - assert len(basic.mcps) == 1 + assert len(basic.tools) == 2 resp3 = await tool._func(plugin="nope") text3 = "".join(getattr(b, "text", "") for b in resp3.content) @@ -340,19 +339,21 @@ def test_resolve_bound_defers_http_plugins_and_applies_skill_filter(ppl_env): def test_load_plugin_no_persist_for_subagent_runtime(ppl_env, tmp_path, monkeypatch): - import core.llm.mcp_pool as mcp_pool + import core.llm.mcp.lifetime as mcp_pool from agentscope.tool import Toolkit monkeypatch.setattr( mcp_pool, - "make_client", - lambda key, cfg, is_stateful=True, **kw: _FakeMCPClient(key, cfg), + "discover", + _discover, ) res = plugin_loader.resolve_bound_progressive_plugins(["crawler@global"]) - collector = ToolCollector() - from core.llm.tool_permissions import ToolPermissionRegistry + collector = ToolAssembly() + from core.sandbox.permissions.tool_specs import ToolPermissionRegistry runtime = { + "request": SimpleNamespace(current_user_id=USER, chat_id=CHAT, capability_scope="", isolated=False), + "agent": SimpleNamespace(), "activated_slugs": set(), "connected_keys": set(), "toolkit": Toolkit(tools=[], mcps=[]), @@ -369,8 +370,8 @@ def test_load_plugin_no_persist_for_subagent_runtime(ppl_env, tmp_path, monkeypa "approval_available": False, "ontology_runtime": {}, } - plugin_loader.register_load_plugin(collector, res.deferred_by_slug(), runtime) - tool = collector.get_tool("load_plugin") + plugin_activation.register_load_plugin(collector, res.deferred_by_slug(), runtime) + tool = next(t for t in collector if t.name == "load_plugin") resp = asyncio.run(tool._func(plugin="crawler")) text = "".join(getattr(b, "text", "") for b in resp.content) assert "已加载" in text @@ -389,8 +390,8 @@ def test_env_kill_switch(monkeypatch): def test_pre_turn_compaction_inputs_match_the_service_signature(): from inspect import signature - from core.services.compaction_service import maybe_run_pre_turn_compaction - from orchestration.workflow import _pre_turn_compaction_inputs + from core.agent_runtime.budget import _pre_turn_compaction_inputs + from core.chat.compaction.service import maybe_run_pre_turn_compaction agent = SimpleNamespace( model=SimpleNamespace(model="model-a"), @@ -433,3 +434,5 @@ def test_components_shared_with_an_eager_plugin_are_not_withheld(ppl_env): assert res.activated_slugs == ["crawler"] assert res.deferred_skill_ids == set() assert res.deferred_mcp_ids == set() + +from core.llm.tools import plugin_activation diff --git a/src/backend/tests/test_plugin_resource_contract.py b/src/backend/tests/test_plugin_resource_contract.py index d846ab9a..1fb26dc4 100644 --- a/src/backend/tests/test_plugin_resource_contract.py +++ b/src/backend/tests/test_plugin_resource_contract.py @@ -1,4 +1,4 @@ -from core.plugins.ui.contract import normalize_ui, public_contributions +from features.workbench.ui.contract import normalize_ui, public_contributions def test_resource_module_retains_safe_runtime_and_binding(): ui, dropped = normalize_ui({"version": 2, "contributes": {"modules": [{ diff --git a/src/backend/tests/test_plugin_system.py b/src/backend/tests/test_plugin_system.py index 2a884ce6..ccc384ef 100644 --- a/src/backend/tests/test_plugin_system.py +++ b/src/backend/tests/test_plugin_system.py @@ -1,3 +1,4 @@ +from features.marketplace import plugin_market """Plugin system tests: import Claude Code / Codex / native plugin packages → persist to DB → uninstall. Covers the three-tier portability matrix: direct skill import (including @@ -14,13 +15,13 @@ from core.db.models import AdminMcpServer, AdminSkill, InstalledPlugin, UserShadow -from core.plugins.packaging import importer as pi +from core.capabilities.plugins.packaging import importer as pi -from core.plugins import management as ps +from core.capabilities.plugins import management as ps -from core.plugins.management import components as plugin_components +from core.capabilities.plugins.management import components as plugin_components -from core.services.ontology_policy import set_plugin_import_build_validation_forced +from core.ontology.policy import set_plugin_import_build_validation_forced OWNER = "test_user_123" @@ -184,7 +185,7 @@ def test_plugin_import_ontology_validation_follows_user_and_force_policy( def test_owned_skill_enters_runtime_set(tmp_path, db_session): """Imported private skill with is_enabled=True → selected by the owned merge of resolve_all_runtime_enabled.""" - from core.config.catalog_resolver import _owned_enabled_ids + from core.capabilities.catalog.resolver import _owned_enabled_ids pdir = _make_cc_plugin(tmp_path) ps.import_plugin(db_session, pdir, owner_user_id=OWNER, secrets={"api_token": "x"}) @@ -300,7 +301,7 @@ def test_reject_non_plugin_dir(tmp_path): def test_builtin_plugin_list_and_install(db_session): """The built-in sample plugin sample-translator should be discovered by list and be installable.""" - items = ps.list_plugins(db_session, owner_user_id=OWNER) + items = plugin_market.list_plugins(db_session, owner_user_id=OWNER) slugs = {it["slug"] for it in items} assert "sample-translator" in slugs sample = next(it for it in items if it["slug"] == "sample-translator") @@ -316,7 +317,7 @@ def test_builtin_plugin_list_and_install(db_session): assert any("glossary" in f for f in (sk[0].extra_files or {})) # List again → marked as installed - items2 = ps.list_plugins(db_session, owner_user_id=OWNER) + items2 = plugin_market.list_plugins(db_session, owner_user_id=OWNER) assert next(it for it in items2 if it["slug"] == "sample-translator")["installed"] is True @@ -350,7 +351,7 @@ def test_builtin_firecrawl_plugin_install(db_session): """Built-in firecrawl plugin (official CLI skill suite): discovered by list → installs 10 skills, all enabled, no MCP, no required_secrets (credentials are injected via system-config env) → clean uninstall.""" - items = ps.list_plugins(db_session, owner_user_id=OWNER) + items = plugin_market.list_plugins(db_session, owner_user_id=OWNER) fc = next((it for it in items if it["slug"] == "firecrawl"), None) assert fc is not None, "firecrawl 插件未被 list_plugins 发现" assert fc["installed"] is False diff --git a/src/backend/tests/test_redis_stream_pool.py b/src/backend/tests/test_redis_stream_pool.py index 8b515c21..afd763ed 100644 --- a/src/backend/tests/test_redis_stream_pool.py +++ b/src/backend/tests/test_redis_stream_pool.py @@ -55,7 +55,7 @@ def test_follower_asks_for_the_blocking_pool(): """Only the tailing read may park on a connection from the stream pool.""" import inspect - from orchestration import run_event_stream + import core.runs.events.store as run_event_stream backend = run_event_stream.RedisRunEventStream assert "get_redis(blocking=True)" in inspect.getsource(backend.wait) @@ -66,7 +66,7 @@ def test_follower_asks_for_the_blocking_pool(): def test_both_seams_use_redis_when_it_is_configured(monkeypatch): """A configured deployment keeps the Redis backends — nothing regresses.""" import core.infra.ephemeral as ephemeral - import orchestration.run_event_stream as res + import core.runs.events.store as res monkeypatch.setattr(ephemeral, "redis_configured", lambda: True) monkeypatch.setattr(res, "redis_configured", lambda: True) @@ -77,7 +77,7 @@ def test_both_seams_use_redis_when_it_is_configured(monkeypatch): def test_cursor_grammar_is_shared_by_both_backends(): """Callers parse the millisecond half, so both backends must mint it alike.""" - import orchestration.run_event_stream as res + import core.runs.events.store as res assert res.cursor_millis("1788745632008-0") == 1788745632008 assert res.cursor_millis("nonsense") is None diff --git a/src/backend/tests/test_retired_batch_chat_admission_http.py b/src/backend/tests/test_retired_batch_chat_admission_http.py index b124a515..c4dcfb2c 100644 --- a/src/backend/tests/test_retired_batch_chat_admission_http.py +++ b/src/backend/tests/test_retired_batch_chat_admission_http.py @@ -11,12 +11,12 @@ from datetime import datetime, timezone from types import SimpleNamespace +import api.transports.run_sse as run_sse +import core.chat.runtime.admission as admission +import core.runs.runtime as runtime import pytest -from fastapi.testclient import TestClient -from sqlalchemy.orm import object_session - -from core.chat.schemas import ChatRequest from core.auth import session +from core.chat.schemas import ChatRequest from core.config.settings import settings from core.db.engine import Base, SessionLocal, engine from core.db.models import ( @@ -28,18 +28,20 @@ Project, UserShadow, ) -from core.services.model_config import ModelConfigService +from core.models.config import ModelConfigService +from fastapi.testclient import TestClient +from sqlalchemy.orm import object_session @pytest.fixture() def boundary(monkeypatch): from api.app import app - from orchestration import chat_run_executor + pass assert settings.auth.mode == "session" assert session._use_memory_store() Base.metadata.create_all(engine) - session._MEMORY_SESSIONS.clear() + session.process_state().MEMORY_SESSIONS.clear() with SessionLocal() as db: db.add(UserShadow(user_id="retirement-user", username="Reader", extra_data={})) db.add( @@ -86,9 +88,9 @@ async def follow(run_id, *, chat_id): yield "event: run_started\ndata: " + json.dumps({"run_id": run_id}) + "\n\n" yield "data: [DONE]\n\n" - monkeypatch.setattr(chat_run_executor, "start_run", model_worker) - monkeypatch.setattr(chat_run_executor, "wait_run", wait) - monkeypatch.setattr(chat_run_executor, "follow_run_as_sse", follow) + monkeypatch.setattr(admission, "start_run", model_worker) + monkeypatch.setattr(runtime, "wait_run", wait) + monkeypatch.setattr(run_sse, "follow_run_as_sse", follow) client = TestClient(app) token = asyncio.run( session.create_session({"user_id": "retirement-user", "username": "Reader"}) @@ -96,7 +98,7 @@ async def follow(run_id, *, chat_id): client.cookies.set(settings.session.cookie_name, token) yield client, captured client.close() - session._MEMORY_SESSIONS.clear() + session.process_state().MEMORY_SESSIONS.clear() ModelConfigService.get_instance().invalidate_cache() Base.metadata.drop_all(engine) @@ -147,10 +149,10 @@ def test_ordinary_public_chat_and_project_init_survive_retirement(boundary, kind def test_scoped_runtime_wrappers_do_not_reintroduce_retired_fields(): - from core.services.session_operations.session_context import _ensure_chat_session - from core.llm.agent_api_runtime import apply_api_scope - from core.llm.evaluation_runtime import apply_evaluation_scope - from core.services.agent_api_service import _FORBIDDEN_INPUTS + from core.llm.api_runtime.agent import apply_api_scope + from features.evaluation.runtime.runtime import apply_evaluation_scope + from core.auth.agent_api import _FORBIDDEN_INPUTS + from core.chat.operations.session_context import _ensure_chat_session assert "batch_chat" not in ChatRequest.model_fields assert "batch_chat" not in inspect.signature(_ensure_chat_session).parameters diff --git a/src/backend/tests/test_route_import_order.py b/src/backend/tests/test_route_import_order.py new file mode 100644 index 00000000..6f114049 --- /dev/null +++ b/src/backend/tests/test_route_import_order.py @@ -0,0 +1,74 @@ +"""Route modules import on their own, without composing the application first.""" +from pathlib import Path +import importlib.util +import os +import subprocess +import sys + +import pytest + +BACKEND = Path(__file__).resolve().parents[1] + + +def _run(code, tmp_path): + env = { + key: value for key, value in os.environ.items() + if key.upper() in {"PATH", "SYSTEMROOT", "WINDIR", "TMP", "TEMP", "TMPDIR", + "DATABASE_URL", "SQLITE_FALLBACK_URL"} + } + env.update(PYTHONPATH=str(BACKEND), HOME=str(tmp_path), PYTHONUTF8="1", REDIS_URL="") + return subprocess.run( + [sys.executable, "-c", code], + cwd=tmp_path, + env=env, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + encoding="utf-8", + timeout=120, + ) + + +@pytest.mark.parametrize( + "entry", + [ + # Each of these once imported ``api`` (and with it the whole app), + # which re-imported the route module while it was half initialised. + "edition_ee.api.deps", + "edition_ee.api.openapi_data_schemas", + "edition_ee.routes.config_teams", + "edition_ee.routes.team_file_assets", + "features.marketplace.api.admin_marketplace", + "features.workbench.api.workbench_auth", + "features.datasources.api.data_sources", + # api.routes eagerly imported files -> local_file_actions -> api.routes. + "features.desktop.api.local_file_actions", + # device_catalog re-exports from device_plugin_catalog at import time. + "features.desktop.capabilities.device_plugin_catalog", + ], +) +def test_route_module_imports_first(entry, tmp_path): + if importlib.util.find_spec(entry.split(".")[0]) is None: + pytest.skip(f"{entry} is not part of this edition") + result = _run(f"import {entry}\nprint('ROUTE_IMPORT_OK')", tmp_path) + assert result.returncode == 0, result.stdout + assert "ROUTE_IMPORT_OK" in result.stdout + + +def test_api_submodules_do_not_compose_the_app_but_app_still_resolves(tmp_path): + code = """ +import sys +import api.deps +from api.routes import file_location +assert "api.app" not in sys.modules +from api import app +from api.routes import files_router +from fastapi import FastAPI +assert isinstance(app, FastAPI) +from features.desktop.capabilities import device_catalog, device_plugin_catalog +assert device_catalog.plugin_entries is device_plugin_catalog.plugin_entries +print("API_LAZY_OK") +""" + result = _run(code, tmp_path) + assert result.returncode == 0, result.stdout + assert "API_LAZY_OK" in result.stdout diff --git a/src/backend/tests/test_session_capabilities.py b/src/backend/tests/test_session_capabilities.py index 2aea5184..fb8013af 100644 --- a/src/backend/tests/test_session_capabilities.py +++ b/src/backend/tests/test_session_capabilities.py @@ -7,7 +7,7 @@ import core.db.engine as dbe import pytest from core.db.models import ChatSession, UserShadow -from core.llm import session_capabilities +from core.llm.runtime import session_capabilities from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker @@ -44,7 +44,7 @@ def test_record_and_restore_direct_capabilities(sticky_env, monkeypatch): assert row.extra_data["activated_skill_ids"] == ["skill-a", "skill-b"] assert row.extra_data["activated_mcp_ids"] == ["mcp-a", "mcp-b"] - import core.config.catalog_resolver as resolver + import core.capabilities.catalog.resolver as resolver def hard_gate(db, user_id, *, skill_ids=None, mcp_ids=None): assert user_id == USER diff --git a/src/backend/tests/test_session_database_isolation.py b/src/backend/tests/test_session_database_isolation.py new file mode 100644 index 00000000..fc309071 --- /dev/null +++ b/src/backend/tests/test_session_database_isolation.py @@ -0,0 +1,23 @@ +"""Concurrent pytest sessions never share a default SQLite database.""" +import os + +import pytest + +from tests import conftest + + +def test_default_databases_are_private_to_this_session(session_db_dir, test_database_url): + assert session_db_dir == conftest.SESSION_DB_DIR + assert f"hugagent-tests-{os.getpid()}-" in os.path.basename(session_db_dir) + if os.getenv("TEST_DATABASE_URL"): + pytest.skip("TEST_DATABASE_URL overrides the session database") + assert test_database_url == f"sqlite:///{session_db_dir}/hugagent_test.db" + + +def test_application_database_defaults_into_the_session_dir(session_db_dir): + if conftest.DATABASE_URL_PRESET: + pytest.skip("DATABASE_URL was set explicitly") + from core.config.settings import settings + + assert settings.db.url == f"sqlite:///{session_db_dir}/hugagent.db" + assert settings.db.sqlite_fallback_url.startswith(f"sqlite:///{session_db_dir}/") diff --git a/src/backend/tests/test_sidebar_task_status.py b/src/backend/tests/test_sidebar_task_status.py index eabff85e..ea479380 100644 --- a/src/backend/tests/test_sidebar_task_status.py +++ b/src/backend/tests/test_sidebar_task_status.py @@ -1,14 +1,15 @@ """Navigation reminders are journal verdicts, isolated by authenticated owner.""" + from datetime import datetime, timedelta, timezone +import core.runs.runtime as runtime import pytest +from core.db.engine import Base +from core.db.models import ChatMessage, ChatRun, ChatSession +from core.chat.sidebar_status import sidebar_run_statuses from sqlalchemy import create_engine from sqlalchemy.orm import Session -from core.db.engine import Base -from core.db.models import ChatRun, ChatSession, ChatMessage -from core.services.sidebar_status import sidebar_run_statuses - @pytest.fixture def db(): @@ -74,8 +75,8 @@ def test_new_run_supersedes_old_failure_and_old_live_tasks_are_still_visible(db) def test_active_run_route_only_adds_verdicts_when_requested(db,monkeypatch): from api.routes.v1.chats import run_views from core.auth.backend import UserContext - from orchestration import chat_run_executor - monkeypatch.setattr(chat_run_executor,"list_active_runs_for_user",lambda *args:([],[])) + pass + monkeypatch.setattr(runtime,"list_active_runs_for_user",lambda *args:([],[])) monkeypatch.setattr(run_views.chat_context,"resolve_db_user_id",lambda db,uid:uid) add_chat(db,"mine") add_run(db,"final","mine","failed",datetime.now(timezone.utc)) diff --git a/src/backend/tests/test_skill_manager_plugin.py b/src/backend/tests/test_skill_manager_plugin.py index 08fb4296..821726be 100644 --- a/src/backend/tests/test_skill_manager_plugin.py +++ b/src/backend/tests/test_skill_manager_plugin.py @@ -7,13 +7,13 @@ from pathlib import Path from types import SimpleNamespace +import core.db.engine as dbe import pytest +from core.db.models import AdminMcpServer, AdminSkill, InstalledPlugin +from features.marketplace.models import MarketplaceSubmission from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker -import core.db.engine as dbe -from core.db.models import AdminMcpServer, AdminSkill, InstalledPlugin, MarketplaceSubmission - OWNER = "sm_test_user" BUNDLE_DIR = Path(__file__).resolve().parents[1] / "extensions/plugins" / "marketplace" / "skill-manager" @@ -28,7 +28,7 @@ def sm_env(tmp_path, monkeypatch): monkeypatch.setattr(dbe, "SessionLocal", TestSession) # Artifact store goes to tmp (_STORE_DIR is fixed at import time; every other path derives from it) - from core.artifacts import store + from core.files.artifacts import store art_dir = tmp_path / "artifacts" art_dir.mkdir(parents=True, exist_ok=True) @@ -57,14 +57,14 @@ def _make_skill_tar(name: str, description: str, *, body: str = "做事。") -> def _stash_artifact(tar_bytes: bytes) -> str: """Write the tar into the shared artifact store and return the artifact_id (simulating a sandbox sandbox_get_artifact result).""" - from core.artifacts import store + from core.files.artifacts import store item = store.save_artifact_bytes(content=tar_bytes, name="skill.tgz", extension="tgz") return item["file_id"] # ── Plugin persistence + merging into the user's available set ────────────── def test_install_plugin_creates_rows_and_is_agent_visible(sm_env): - from core.plugins import management as ps + from core.capabilities.plugins import management as ps with sm_env.Session() as db: res = ps.install_plugin(db, "skill-manager", owner_user_id=OWNER, created_by=OWNER) @@ -80,13 +80,13 @@ def test_install_plugin_creates_rows_and_is_agent_visible(sm_env): mcp = mcps[0] assert mcp.owner_user_id == OWNER assert mcp.transport == "streamable_http" - assert mcp.url == "http://mcp:9112/mcp/" + assert mcp.url == "http://mcp:9112/mcp" assert mcp.is_enabled is True # http MCP is not needs_runtime → enabled upon install assert len(mcp.tools_json) == 8 assert plugin is not None # Key: merged into the user's available set via resolve_all_runtime_enabled (= the agent can really get the skill + mcp) - from core.config.catalog_resolver import resolve_all_runtime_enabled, invalidate_capability_cache + from core.capabilities.catalog.resolver import invalidate_capability_cache, resolve_all_runtime_enabled invalidate_capability_cache() enabled_skills, _agents, enabled_mcps = resolve_all_runtime_enabled(db, OWNER) @@ -99,11 +99,8 @@ def test_install_plugin_creates_rows_and_is_agent_visible(sm_env): def test_plan_mode_includes_global_plugin_components(sm_env): """Plan mode must also see plugin components globally installed by the admin.""" + from core.agent_runtime.plugin_visibility import all_plugin_component_ids, load_enabled_plugins from core.db.models import AdminMcpServer, InstalledPlugin - from orchestration.subagents.plugin_visibility import ( - all_plugin_component_ids, - load_enabled_plugins, - ) with sm_env.Session() as db: db.add( @@ -220,7 +217,7 @@ def test_runtime_skill_loader_reads_db_skills_without_route_specific_refresh( sm_env, tmp_path, monkeypatch ): """After a skill is written to AdminSkill, re-reading via the same loader instance must honor the DB as the source of truth.""" - from core.agent_skills.loader import get_skill_loader + from core.capabilities.skills.loader_factory import get_skill_loader monkeypatch.setenv("SANDBOX_SKILLS_DIR", str(tmp_path / "sandbox_skills")) skill_id = "runtime-db-skill" diff --git a/src/backend/tests/test_tool_contributions.py b/src/backend/tests/test_tool_contributions.py new file mode 100644 index 00000000..1ad3801c --- /dev/null +++ b/src/backend/tests/test_tool_contributions.py @@ -0,0 +1,90 @@ +"""Behavioral gates and native assembly order for contributed tools.""" +from features.datasources import metadata +from unittest.mock import Mock + +import pytest +from core.harness.registry import NativeToolContext, install_tools +from core.harness.registry import registry +from core.llm.factory.request import AgentRequest +from features import register_product_tools +from features.sites.tools import design_picker as design_picker_tool +from integrations.channels.tools import attachment as channel_attachment_tool +from features.datasources.tools import data_context as data_context_tool + + +@pytest.fixture +def contributions(monkeypatch): + monkeypatch.setattr(registry.registration_state(), 'tools', {}) + register_product_tools() + register_product_tools() # startup registration is idempotent + + +@pytest.mark.asyncio +@pytest.mark.parametrize("skills,channel,automation,interactive,expected", [ + ([], False, False, True, []), + (["site-builder"], False, False, True, [("design", True)]), + (["site-builder"], True, False, True, [("design", False), ("attachment", None)]), + (["site-builder"], False, True, True, [("design", False)]), + (["site-builder"], False, False, False, [("design", False)]), + ([], True, False, True, [("attachment", None)]), +]) +async def test_product_gates(contributions, monkeypatch, skills, channel, automation, interactive, expected): + calls = [] + monkeypatch.setattr(design_picker_tool, "register_choose_design", lambda toolkit, **kw: calls.append(("design", kw["interactive"]))) + monkeypatch.setattr(channel_attachment_tool, "register_channel_attachment", lambda toolkit, **kw: calls.append(("attachment", None))) + context = NativeToolContext(AgentRequest(automation_run=automation), interactive, channel, None, None, skills, [], Mock()) + for stage in ("after_sandbox", "after_image", "after_workflow"): + await install_tools(stage, object(), context) + assert calls == expected + + +@pytest.mark.asyncio +@pytest.mark.parametrize("servers,eligible,fail,expected", [ + (["query_database"], ["db"], False, []), + (["db_query"], [], False, []), + (["db_query"], ["db"], False, [["db"]]), + (["es_query"], ["es"], False, [["es"]]), + (["db_query"], [], True, []), +]) +async def test_data_dictionary_gates(contributions, monkeypatch, servers, eligible, fail, expected): + + lookup = Mock(side_effect=RuntimeError("unavailable")) if fail else Mock(return_value=eligible) + monkeypatch.setattr(metadata, 'eligible_datasource_ids', lookup) + calls = [] + monkeypatch.setattr(data_context_tool, "register_get_data_context", lambda toolkit, ids: calls.append(ids)) + context = NativeToolContext(AgentRequest(), True, False, None, None, [], servers, Mock()) + await install_tools("after_workflow", object(), context) + assert calls == expected + assert lookup.call_count == (0 if servers == ["query_database"] else 1) + assert context.logger.warning.call_count == int(fail) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("disabled,turbo,expected", [(False, False, ["after_sandbox", "after_image", "after_workflow"]), (True, False, []), (False, True, [])]) +async def test_native_assembly_gates_and_order(monkeypatch, disabled, turbo, expected): + import importlib + native = importlib.import_module("core.llm.factory.tools.native") + for name in vars(native): + if name.startswith("register_"): + monkeypatch.setattr(native, name, Mock()) + monkeypatch.setattr(native, "code_capability_enabled", lambda: False) + import core.models.vision + monkeypatch.setattr(core.models.vision, "resolve_vision_mode", lambda value: "none") + stages = [] + async def install(stage, toolkit, context): + stages.append(stage) + monkeypatch.setattr(native, "install_tools", install) + await native.native_tools(AgentRequest(disable_tools=disabled, turbo_mode=turbo), + _api_scope=None, _eval_scope=None, _interactive=True, _is_channel_run=False, + _log=Mock(), _proj_scope=None, _sbx_sess=None, _shared_subagent_cfg=None, + _turbo_code_exec=False, allowed_skill_dirs=[], enabled_mcp_keys=[], loaded_skill_ids=[], + loader=Mock(), skill_ids_to_register=[], toolkit=Mock()) + assert stages == expected + + +def test_headless_host_composes_tools_without_asgi(): + import subprocess + import sys + script = 'import asyncio\nimport logging\nimport sys\nfrom unittest.mock import patch\nfrom features import register_product_tools\nfrom core.harness.registry import NativeToolContext, install_tools\nfrom core.llm.factory.request import AgentRequest\nfrom core.llm.factory.tools.definitions import ToolAssembly\n\nassert "api.app" not in sys.modules\nregister_product_tools()\nregister_product_tools()\n\nasync def main():\n toolkit = ToolAssembly()\n context = NativeToolContext(AgentRequest(chat_id="test", current_user_id="test"), True, True, None, None, ["site-builder"], ["db_query"], logging.getLogger("test"))\n with patch("features.datasources.metadata.eligible_datasource_ids", return_value=["fixture"]):\n for stage in ("after_sandbox", "after_image", "after_workflow"):\n await install_tools(stage, toolkit, context)\n names = [tool.name for tool in list(toolkit)]\n assert names == ["choose_design", "channel_read_attachment", "get_data_context"], names\n assert "api.app" not in sys.modules\n print("headless tool composition passed", names)\n\nasyncio.run(main())\n' + result = subprocess.run([sys.executable, "-c", script], capture_output=True, text=True) + assert result.returncode == 0, result.stdout + result.stderr diff --git a/src/backend/tests/test_user_agent_resources.py b/src/backend/tests/test_user_agent_resources.py index 5d098e4b..0627858e 100644 --- a/src/backend/tests/test_user_agent_resources.py +++ b/src/backend/tests/test_user_agent_resources.py @@ -1,11 +1,11 @@ """Bindable resource coverage for user-created sub-agents.""" -from core.capabilities.agents import AgentDefinition +from features.desktop.capabilities.agents import AgentDefinition import pytest from core.db.models import AdminMcpServer -from core.services.user_agent_service import UserAgentService +from core.capabilities.agents.edition import UserAgentService def _mcp( @@ -43,15 +43,15 @@ def test_available_resources_include_personally_disabled_mcps( db_session.commit() monkeypatch.setattr( - "core.config.catalog_resolver.resolve_all_runtime_enabled", + 'core.capabilities.catalog.resolver.resolve_all_runtime_enabled', lambda _db, _user_id: ([], [], ["global_enabled"]), ) monkeypatch.setattr( - "core.plugins.management.builtin_plugin_component_ids", + 'core.capabilities.plugins.management.builtin_plugin_component_ids', lambda: (set(), {"builtin_plugin_tool"}), ) monkeypatch.setattr( - "core.config.catalog_runtime.get_runtime_catalog", + 'core.capabilities.catalog.runtime.get_runtime_catalog', lambda _db, include_runtime_details=False: { "mcp": [ { @@ -111,13 +111,13 @@ async def test_agent_factory_loads_disabled_private_mcp_only_for_explicit_subage ): from core.llm import factory as agent_factory from core.llm.factory.tools import mcp_config as factory_mcp_config - import core.services.mcp_service as mcp_service + import core.capabilities.mcp.service as mcp_service import prompts.prompt_config as prompt_config from sqlalchemy.orm import sessionmaker monkeypatch.setattr( - "core.services.model_config.SessionLocal", sessionmaker(bind=db_session.get_bind()) + 'core.models.config.SessionLocal', sessionmaker(bind=db_session.get_bind()) ) calls = [] @@ -137,7 +137,7 @@ class StopAfterOwnedMcpResolution(Exception): monkeypatch.setattr(prompt_config, "load_prompt_config", lambda: object()) monkeypatch.setattr( factory_mcp_config, - "_effective_mcp_server_keys", + 'effective_mcp_server_keys', lambda *_args, **_kwargs: (_ for _ in ()).throw(StopAfterOwnedMcpResolution()), ) diff --git a/src/backend/tests/test_user_cleanup.py b/src/backend/tests/test_user_cleanup.py new file mode 100644 index 00000000..3e398445 --- /dev/null +++ b/src/backend/tests/test_user_cleanup.py @@ -0,0 +1,34 @@ +"""User deletion removes only owned knowledge in the same transaction.""" +import pytest +from sqlalchemy import create_engine +from sqlalchemy.orm import Session +import core.db.models +from core.db.engine import Base +from core.db.models import UserShadow +from features.knowledge.models import KBSpace, KBDocument + + +@pytest.mark.parametrize("commit", [True, False]) +def test_delete_user_cleans_knowledge_and_preserves_rollback(commit): + engine = create_engine("sqlite://") + Base.metadata.create_all(engine) + with Session(engine) as session: + session.add_all([UserShadow(user_id="one", username="one"), UserShadow(user_id="two", username="two")]) + session.flush() + session.add_all([KBSpace(kb_id="a", user_id="one", name="A"), KBSpace(kb_id="b", user_id="two", name="B")]) + session.flush() + session.add(KBDocument(document_id="doc", kb_id="a", title="doc", filename="doc.txt", size_bytes=1, mime_type="text/plain", storage_key="doc")) + session.commit() + session.delete(session.get(UserShadow, "one")) + session.flush() + assert session.get(KBSpace, "a") is None + assert session.get(KBDocument, "doc") is None + assert session.get(KBSpace, "b") is not None + if commit: + session.commit() + else: + session.rollback() + assert (session.get(UserShadow, "one") is None) is commit + assert (session.get(KBSpace, "a") is None) is commit + assert (session.get(KBDocument, "doc") is None) is commit + engine.dispose() diff --git a/src/backend/tests/test_visualize_plugin.py b/src/backend/tests/test_visualize_plugin.py index ec94fc20..e0dc4062 100644 --- a/src/backend/tests/test_visualize_plugin.py +++ b/src/backend/tests/test_visualize_plugin.py @@ -6,9 +6,9 @@ import pytest from core.db.models import AdminSkill, ContentBlock, InstalledPlugin -from core.plugins import management as plugins -from core.plugins.management.defaults import ensure_visualize_plugin_bootstrapped -from core.plugins.packaging.importer import normalize_plugin_dir +from core.capabilities.plugins import management as plugins +from core.capabilities.plugins.management.defaults import ensure_visualize_plugin_bootstrapped +from core.capabilities.plugins.packaging.importer import normalize_plugin_dir BUNDLE = Path(__file__).resolve().parents[1] / "extensions/plugins/default/visualize" SKILL = BUNDLE / "skills/visualize" @@ -54,7 +54,7 @@ def test_native_bundle_includes_offline_resources(): text = plugin.skills[0].skill_content assert "pin_to_workspace" in text and "sandbox_get_artifact" in text for path in SKILL.rglob("*"): - if path.is_file(): + if path.is_file() and "__pycache__" not in path.parts and path.suffix not in {".pyc", ".pyo"}: assert "zcode" not in path.read_text().lower(), path @@ -89,10 +89,10 @@ async def test_startup_includes_visualize_in_all_profiles(monkeypatch, local, ed from api.startup import plugins as startup from core.db import engine - from core.services import edition_startup + from orchestration.workers import edition_startup monkeypatch.setenv("HUGAGENT_BOOTSTRAP_DEFAULT_PLUGINS", "1") - monkeypatch.setattr("core.capabilities.device_catalog.active", lambda: False) + monkeypatch.setattr('features.desktop.capabilities.device_catalog.active', lambda: False) calls = [] session = SimpleNamespace(close=lambda: None) monkeypatch.setattr( @@ -107,10 +107,10 @@ async def test_startup_includes_visualize_in_all_profiles(monkeypatch, local, ed monkeypatch.setattr(plugins, "ensure_default_plugins_bootstrapped", lambda db: False) monkeypatch.setattr(edition_startup, "bootstrap_edition_plugins", lambda db: ()) monkeypatch.setattr( - "core.plugins.ui.contributions.refresh_builtin_ui_contributions", lambda db: 0 + 'features.workbench.ui.contributions.refresh_builtin_ui_contributions', lambda db: 0 ) monkeypatch.setattr( - "core.plugins.management.defaults.ensure_visualize_plugin_bootstrapped", + 'core.capabilities.plugins.management.defaults.ensure_visualize_plugin_bootstrapped', lambda db: calls.append(db), ) await startup._startup_seed_default_plugins() @@ -134,7 +134,7 @@ async def test_local_bootstrap_respects_opt_out_and_hybrid(monkeypatch, opt_in, ), ) monkeypatch.setenv("HUGAGENT_BOOTSTRAP_DEFAULT_PLUGINS", "1" if opt_in else "0") - monkeypatch.setattr("core.capabilities.device_catalog.active", lambda: hybrid) + monkeypatch.setattr('features.desktop.capabilities.device_catalog.active', lambda: hybrid) def unexpected_session(): pytest.fail("Local opt-out and hybrid mode must not bootstrap global plugins") diff --git a/src/backend/tests/test_yida_local_login.py b/src/backend/tests/test_yida_local_login.py index 3877ac20..68e25fcd 100644 --- a/src/backend/tests/test_yida_local_login.py +++ b/src/backend/tests/test_yida_local_login.py @@ -6,7 +6,7 @@ import pytest from core.config.settings import settings -from core.services.yida_service import YidaService +from integrations.yida.service import YidaService @pytest.mark.skipif(os.name == "nt", reason="POSIX executable fixture; Windows uses bundled Bash") @@ -32,7 +32,7 @@ async def test_local_yida_login_persists_and_disconnects(tmp_path, monkeypatch): monkeypatch.setenv("PATH", str(binary.parent) + os.pathsep + os.environ["PATH"]) from core.infra.ephemeral import LocalEphemeralState memory = LocalEphemeralState() - monkeypatch.setattr("core.services.yida_service.get_ephemeral_state", lambda: memory) + monkeypatch.setattr('integrations.yida.service.get_ephemeral_state', lambda: memory) def no_remote_provider(): raise AssertionError("本机登录不应访问容器沙箱") monkeypatch.setattr("core.sandbox.factory.get_sandbox_provider", no_remote_provider) @@ -55,7 +55,9 @@ async def test_local_skill_uses_login_workspace(tmp_path, monkeypatch): file = tmp_path / "login.md" file.write_text("cd /home/ubuntu/yida-workspace && openyida login --agent-qr") class Tools: - def register_tool_function(self, function, **kwargs): + def append(self, tool): + function = tool._func + kwargs = {"func_description": tool.description, "permission": getattr(tool, "dispatch_spec", None)} self.read = function tools = Tools() register_sandboxed_view_text_file(tools, [str(tmp_path)], SimpleNamespace(), user_id="owner") @@ -67,8 +69,8 @@ def register_tool_function(self, function, **kwargs): @pytest.mark.skipif(not os.getenv("HUGAGENT_TEST_PLUGIN_RUNTIME"), reason="Requires staged offline runtime") async def test_bundled_real_openyida_qr_entry(tmp_path, monkeypatch): """Real locked OpenYida and private Node, using its offline QR fixture.""" - import core.services.yida_execution as execution - import core.services.yida_service as service + import integrations.yida.execution as execution + import integrations.yida.service as service from core.infra.ephemeral import LocalEphemeralState root = Path(os.environ["HUGAGENT_TEST_PLUGIN_RUNTIME"]).resolve() python = root / ("python/python.exe" if os.name == "nt" else "python/bin/python") diff --git a/src/frontend/Dockerfile b/src/frontend/Dockerfile index f2025767..b233cc0c 100644 --- a/src/frontend/Dockerfile +++ b/src/frontend/Dockerfile @@ -38,7 +38,7 @@ RUN npm ci COPY src/frontend/ . COPY scripts/build-chat-sdk.mjs /build/scripts/ COPY scripts/sync-workbench-schema.mjs scripts/build-workbench.mjs scripts/workbench-build-contract.mjs /build/scripts/ -COPY src/backend/core/applications/schemas /build/src/backend/core/applications/schemas +COPY src/backend/features/workbench/schemas /build/src/backend/features/workbench/schemas # Build production bundle RUN npm run build diff --git a/src/frontend/Dockerfile.dockerignore b/src/frontend/Dockerfile.dockerignore index 2fe215a0..7f09c28c 100644 --- a/src/frontend/Dockerfile.dockerignore +++ b/src/frontend/Dockerfile.dockerignore @@ -4,10 +4,10 @@ !src/frontend/ !src/frontend/** !src/backend/ -!src/backend/core/ -!src/backend/core/applications/ -!src/backend/core/applications/schemas/ -!src/backend/core/applications/schemas/** +!src/backend/features/ +!src/backend/features/workbench/ +!src/backend/features/workbench/schemas/ +!src/backend/features/workbench/schemas/** !packages/ !packages/workbench-protocol/ !packages/workbench-protocol/** diff --git a/src/frontend/scripts/archive-e2e-fixtures.py b/src/frontend/scripts/archive-e2e-fixtures.py index 6c02e238..2e931588 100644 --- a/src/frontend/scripts/archive-e2e-fixtures.py +++ b/src/frontend/scripts/archive-e2e-fixtures.py @@ -17,7 +17,7 @@ import json,sys from core.db.engine import SessionLocal from core.db.models import UserShadow -from core.services.local_user_service import LocalUserService +from core.auth.local_users import LocalUserService body=json.load(sys.stdin) created=[] with SessionLocal() as db: @@ -40,7 +40,7 @@ import asyncio,json,sys from core.db.engine import SessionLocal from core.db.models import UserShadow,Artifact -from core.storage import get_storage +from core.files.storage import get_storage from api.app import app from edition_ee.routes.config_users import delete_user body=json.load(sys.stdin) diff --git a/src/frontend/scripts/chat-customization-environment.py b/src/frontend/scripts/chat-customization-environment.py index 49cbcf7b..91c7374b 100644 --- a/src/frontend/scripts/chat-customization-environment.py +++ b/src/frontend/scripts/chat-customization-environment.py @@ -68,7 +68,7 @@ def up(): else: raise RuntimeError("Isolated backend did not become ready; inspect its logs.") seed = """from core.db.engine import SessionLocal -from core.db.model_repository import create_provider, assign_role +from core.models.repository import create_provider, assign_role with SessionLocal() as db: provider = create_provider(db, display_name='Component E2E', provider_type='chat', base_url='http://model:8081/v1', api_key='e2e-placeholder', model_name='component-test', extra_config={'api_protocol':'chat_completions','supports_thinking':True,'context_length':32000}) assert assign_role(db, 'main_agent', provider.provider_id, updated_by='component-e2e') diff --git a/src/frontend/scripts/component-e2e-environment.py b/src/frontend/scripts/component-e2e-environment.py index 028b1f2e..d95a66c0 100644 --- a/src/frontend/scripts/component-e2e-environment.py +++ b/src/frontend/scripts/component-e2e-environment.py @@ -61,7 +61,7 @@ def up(): else: raise RuntimeError("Isolated backend did not become ready; inspect its logs.") seed = """from core.db.engine import SessionLocal -from core.db.model_repository import create_provider, assign_role +from core.models.repository import create_provider, assign_role with SessionLocal() as db: provider = create_provider(db, display_name='Component E2E', provider_type='chat', base_url='http://model:8081/v1', api_key='e2e-placeholder', model_name='component-test', extra_config={'api_protocol':'chat_completions','supports_thinking':True,'context_length':32000}) assert assign_role(db, 'main_agent', provider.provider_id, updated_by='component-e2e') diff --git a/src/frontend/scripts/customer-devkit-environment.py b/src/frontend/scripts/customer-devkit-environment.py index ce4bb73a..c6a3d122 100644 --- a/src/frontend/scripts/customer-devkit-environment.py +++ b/src/frontend/scripts/customer-devkit-environment.py @@ -66,8 +66,8 @@ def up(): else: raise RuntimeError("Devkit backend not ready") seed = """from core.db.engine import SessionLocal -from core.db.model_repository import create_provider, assign_role -from core.services.local_user_service import LocalUserService +from core.models.repository import create_provider, assign_role +from core.auth.local_users import LocalUserService with SessionLocal() as db: result = LocalUserService(db).create_by_admin('devkit_user', 'Devkit-Test-Only-456', nickname='Devkit Tester') assert result.ok, result.message diff --git a/src/frontend/scripts/grapheme-e2e-environment.py b/src/frontend/scripts/grapheme-e2e-environment.py index 6154ec90..a2705048 100644 --- a/src/frontend/scripts/grapheme-e2e-environment.py +++ b/src/frontend/scripts/grapheme-e2e-environment.py @@ -61,7 +61,7 @@ def up(): else: raise RuntimeError("Isolated backend did not become ready; inspect its logs.") seed = """from core.db.engine import SessionLocal -from core.db.model_repository import create_provider, assign_role +from core.models.repository import create_provider, assign_role with SessionLocal() as db: provider = create_provider(db, display_name='Component E2E', provider_type='chat', base_url='http://model:8081/v1', api_key='e2e-placeholder', model_name='grapheme-test', extra_config={'api_protocol':'chat_completions','supports_thinking':True,'context_length':32000}) assert assign_role(db, 'main_agent', provider.provider_id, updated_by='component-e2e') diff --git a/src/frontend/scripts/test-desktop-login-browser.mjs b/src/frontend/scripts/test-desktop-login-browser.mjs index 21e2e552..378019a5 100644 --- a/src/frontend/scripts/test-desktop-login-browser.mjs +++ b/src/frontend/scripts/test-desktop-login-browser.mjs @@ -29,7 +29,7 @@ async function until(fn, message, ms = 30000) { } throw new Error(message); } -const server = child('.venv/bin/python', ['src/backend/tests/desktop_login_browser_server.py']); +const server = child('.venv/bin/python', ['src/backend/tests/features/desktop/desktop_login_browser_server.py']); let browser; try { const base = await until(() => server.output.match(/FIXTURE_URL (\S+)/)?.[1], 'fixture startup: ' + server.errors); diff --git a/src/frontend/scripts/test-sidebar-task-status-browser.mjs b/src/frontend/scripts/test-sidebar-task-status-browser.mjs index 7b850761..cd10bc4f 100644 --- a/src/frontend/scripts/test-sidebar-task-status-browser.mjs +++ b/src/frontend/scripts/test-sidebar-task-status-browser.mjs @@ -85,7 +85,6 @@ const evidence=resolve('../../.scratch/sidebar-task-status');await mkdir(evidenc try{ const context=await browser.newContext({viewport:{width:960,height:820}}); const page=await context.newPage();page.on('pageerror',e=>{errors.push(e.message);console.error(e.stack);});page.on('console',message=>{if(message.type()==='error')console.error('BROWSER',message.text());}); - await page.addInitScript(() => localStorage.setItem('jx_lang', 'zh-CN')); await page.goto('http://127.0.0.1:'+server.address().port);await page.bringToFront(); const row=id=>page.locator('#ordinary .jx-historyItem').filter({has:page.locator('[data-task-status="'+id+'"]')}); await page.locator('#ordinary [data-task-status="running"]').waitFor({timeout:6000}).catch(async error=>{console.error('FIXTURE DOM',await page.locator('#root').innerHTML());console.error('ERRORS',errors);await page.screenshot({path:resolve(evidence,'failure.png'),fullPage:true});throw error;}); diff --git a/src/frontend/scripts/test-workbench-capabilities-browser.mjs b/src/frontend/scripts/test-workbench-capabilities-browser.mjs index 4110adbd..83911457 100644 --- a/src/frontend/scripts/test-workbench-capabilities-browser.mjs +++ b/src/frontend/scripts/test-workbench-capabilities-browser.mjs @@ -53,7 +53,7 @@ async function run(method,params){ } async function result(){const output=page.frameLocator('iframe').locator('output');await output.filter({hasNotText:'pending'}).waitFor();return output.innerText();} try{ - server=spawn(python,['-m','uvicorn','tests.workbench_capabilities_http_fixture:app','--host','127.0.0.1','--port','5299'],{cwd:root,env,stdio:['ignore','pipe','pipe']}); + server=spawn(python,['-m','uvicorn','tests.features.workbench.workbench_capabilities_http_fixture:app','--host','127.0.0.1','--port','5299'],{cwd:root,env,stdio:['ignore','pipe','pipe']}); server.stdout.on('data',b=>log+=b);server.stderr.on('data',b=>log+=b); for(let n=0;n<100;n++){try{if((await fetch(origin+'/ready')).ok)break;}catch{}await new Promise(r=>setTimeout(r,100));if(n===99)throw new Error(log);} const login=await fetch(origin+'/fixture/session',{method:'POST'}),cookie=login.headers.get('set-cookie').split(';')[0]; diff --git a/src/frontend/scripts/test-workbench-diagnostics-browser.mjs b/src/frontend/scripts/test-workbench-diagnostics-browser.mjs index 87645a42..078caa4f 100644 --- a/src/frontend/scripts/test-workbench-diagnostics-browser.mjs +++ b/src/frontend/scripts/test-workbench-diagnostics-browser.mjs @@ -29,7 +29,7 @@ createRoot(document.getElementById('root')).render(); bundle:true,format:'esm',jsx:'automatic',define:{'import.meta.env':'{}'},loader:{'.css':'empty'}}); await writeFile(resolve(assets,'index.html'),'
'); const server=spawn((process.env.WB_TEST_PYTHON || resolve(root, '.venv/bin/python')), - ['-m','uvicorn','tests.workbench_diagnostic_http_fixture:app','--host','127.0.0.1','--port','5219'], + ['-m','uvicorn','tests.features.workbench.workbench_diagnostic_http_fixture:app','--host','127.0.0.1','--port','5219'], {cwd:root,env:{...process.env,PYTHONPATH:'src/backend',AUTH_MODE:'mock',CONFIG_TOKEN:'wb19-http-config', DATABASE_URL:'sqlite:///'+resolve(folder,'db.sqlite'),STORAGE_PATH:resolve(folder,'storage'), TMPDIR:folder,WB19_ASSET_PATH:assets},stdio:['ignore','pipe','pipe']}); diff --git a/src/frontend/scripts/test-workbench-embed-browser.mjs b/src/frontend/scripts/test-workbench-embed-browser.mjs index 9e15b3cc..e64f5d3c 100644 --- a/src/frontend/scripts/test-workbench-embed-browser.mjs +++ b/src/frontend/scripts/test-workbench-embed-browser.mjs @@ -72,7 +72,7 @@ const site=createServer(siteHandler), attacker=createServer(siteHandler); await new Promise(r=>attacker.listen(5256,'127.0.0.1',r)); await new Promise(r=>site.listen(5255,'127.0.0.1',r)); const server=spawn((process.env.WB_TEST_PYTHON || resolve(root, '.venv/bin/python')), - ['-m','uvicorn','tests.workbench_embed_http_fixture:app','--host','127.0.0.1','--port','5254'], + ['-m','uvicorn','tests.features.workbench.workbench_embed_http_fixture:app','--host','127.0.0.1','--port','5254'], {cwd:root,env:{...process.env,PYTHONPATH:'src/backend',AUTH_MODE:'mock',CONFIG_TOKEN:'wb14-test-config', DATABASE_URL:'sqlite:///'+resolve(scratch,'http.db'),STORAGE_PATH:resolve(scratch,'storage'), TMPDIR:scratch,WB14_BROWSER_OUTPUT:output},stdio:['ignore','pipe','pipe']}); diff --git a/src/frontend/scripts/test-workbench-plugin-browser.mjs b/src/frontend/scripts/test-workbench-plugin-browser.mjs index 24a5a36a..2c32d55f 100644 --- a/src/frontend/scripts/test-workbench-plugin-browser.mjs +++ b/src/frontend/scripts/test-workbench-plugin-browser.mjs @@ -59,7 +59,7 @@ window.addEventListener('unload',()=>c.dispose()); await build({entryPoints:[resolve(consumer,'page.js')],outfile:resolve(consumer,'page.bundle.js'),bundle:true,format:'iife',target:'es2022'}); function zip(html,entry='web/index.html'){ - const script="from tests.workbench_plugin_support import package; import sys; sys.stdout.buffer.write(package(html=sys.argv[1],entry=sys.argv[2],assets={'web/page.js':open(sys.argv[3]).read(),'web/style.css':'h1{color:rgb(1,2,3)}'}))"; + const script="from tests.features.workbench.workbench_plugin_support import package; import sys; sys.stdout.buffer.write(package(html=sys.argv[1],entry=sys.argv[2],assets={'web/page.js':open(sys.argv[3]).read(),'web/style.css':'h1{color:rgb(1,2,3)}'}))"; const r=spawnSync(python,['-c',script,html,entry,resolve(consumer,'page.bundle.js')],{cwd:root,env,maxBuffer:2**24}); assert.equal(r.status,0,r.stderr?.toString() || String(r.error));return r.stdout; } @@ -71,7 +71,7 @@ async function login(){ await page.context().addCookies([{name:cookie.slice(0,split),value:cookie.slice(split+1),url:origin,httpOnly:true,sameSite:'Lax'}]); } async function start(){ - server=spawn(python,['-m','uvicorn','tests.workbench_routing_http_fixture:app','--host','127.0.0.1','--port','5217'],{cwd:root,env,stdio:['ignore','pipe','pipe']}); + server=spawn(python,['-m','uvicorn','tests.features.workbench.workbench_routing_http_fixture:app','--host','127.0.0.1','--port','5217'],{cwd:root,env,stdio:['ignore','pipe','pipe']}); server.stdout.on('data',b=>log+=b);server.stderr.on('data',b=>log+=b); for(let n=0;n<100;n++){if(server.exitCode!==null)throw Error(log); try{if((await fetch(serviceOrigin+'/ready')).ok)return;}catch{} diff --git a/src/frontend/scripts/test-workspace-http-browser.mjs b/src/frontend/scripts/test-workspace-http-browser.mjs index 08bae0b1..e826d864 100644 --- a/src/frontend/scripts/test-workspace-http-browser.mjs +++ b/src/frontend/scripts/test-workspace-http-browser.mjs @@ -58,7 +58,7 @@ window.addEventListener('unload',()=>c.dispose()); await build({entryPoints:[resolve(consumer,'page.js')],outfile:resolve(consumer,'page.bundle.js'),bundle:true,format:'iife',target:'es2022'}); function zip(html,entry='web/index.html'){ - const script="from tests.workbench_plugin_support import package; import sys,io,json,zipfile; raw=package(html=sys.argv[1],entry=sys.argv[2],assets={'web/page.js':open(sys.argv[3]).read(),'web/style.css':'h1{color:rgb(1,2,3)}'}); old=zipfile.ZipFile(io.BytesIO(raw)); out=io.BytesIO(); z=zipfile.ZipFile(out,'w'); manifest=json.loads(old.read('plugin.json')); manifest['extensions']['org.hugagent']['ui']['contributes']['applications'][0]['views']=[{'id':'reports','path':'reports','title':'Reports'},{'id':'encoded','path':'encoded','title':'Encoded view'}]; [z.writestr(n,json.dumps(manifest) if n=='plugin.json' else old.read(n)) for n in old.namelist()]; z.close(); sys.stdout.buffer.write(out.getvalue())"; + const script="from tests.features.workbench.workbench_plugin_support import package; import sys,io,json,zipfile; raw=package(html=sys.argv[1],entry=sys.argv[2],assets={'web/page.js':open(sys.argv[3]).read(),'web/style.css':'h1{color:rgb(1,2,3)}'}); old=zipfile.ZipFile(io.BytesIO(raw)); out=io.BytesIO(); z=zipfile.ZipFile(out,'w'); manifest=json.loads(old.read('plugin.json')); manifest['extensions']['org.hugagent']['ui']['contributes']['applications'][0]['views']=[{'id':'reports','path':'reports','title':'Reports'},{'id':'encoded','path':'encoded','title':'Encoded view'}]; [z.writestr(n,json.dumps(manifest) if n=='plugin.json' else old.read(n)) for n in old.namelist()]; z.close(); sys.stdout.buffer.write(out.getvalue())"; const r=spawnSync(python,['-c',script,html,entry,resolve(consumer,'page.bundle.js')],{cwd:root,env,maxBuffer:2**24}); assert.equal(r.status,0,r.stderr?.toString() || String(r.error));return r.stdout; } @@ -70,7 +70,7 @@ async function login(){ await page.context().addCookies([{name:cookie.slice(0,split),value:cookie.slice(split+1),url:origin,httpOnly:true,sameSite:'Lax'}]); } async function start(){ - server=spawn(python,['-m','uvicorn','tests.workbench_routing_http_fixture:app','--host','127.0.0.1','--port','5288'],{cwd:root,env,stdio:['ignore','pipe','pipe']}); + server=spawn(python,['-m','uvicorn','tests.features.workbench.workbench_routing_http_fixture:app','--host','127.0.0.1','--port','5288'],{cwd:root,env,stdio:['ignore','pipe','pipe']}); server.stdout.on('data',b=>log+=b);server.stderr.on('data',b=>log+=b); for(let n=0;n<600;n++){if(server.exitCode!==null)throw Error(log); try{if((await fetch(serviceOrigin+'/ready')).ok)return;}catch{} diff --git a/src/frontend/src/components/automation/RunTimelinePanel.tsx b/src/frontend/src/components/automation/RunTimelinePanel.tsx index 76b263cc..1a45d5b9 100644 --- a/src/frontend/src/components/automation/RunTimelinePanel.tsx +++ b/src/frontend/src/components/automation/RunTimelinePanel.tsx @@ -39,7 +39,7 @@ function formatRunOrdinal(n: number): string { } function summarizeRun(run: AutomationRun): string { - if (run.status === 'running') return t('任务正在执行，结果生成后会自动更新到对话区。'); + if ((run.status === 'running' || run.status === 'pending')) return t('任务正在执行，结果生成后会自动更新到对话区。'); if (run.status === 'failed') { return String(run.error_message || run.result_summary || t('执行失败，请进入详情查看错误信息。')).replace(/\s+/g, ' ').trim(); } @@ -112,7 +112,7 @@ export function RunTimelinePanel() { let latestSuccessRun: AutomationRun | undefined; let latestFinishedRun: AutomationRun | undefined; for (const run of numberedRuns) { - if (run.status === 'running') running += 1; + if ((run.status === 'running' || run.status === 'pending')) running += 1; else if (run.status === 'success') { success += 1; if (!latestSuccessRun) latestSuccessRun = run; @@ -274,7 +274,7 @@ export function RunTimelinePanel() {
{group.runs.map((run) => { const isActive = run.run_id === selectedRunId; - const isRunning = run.status === 'running'; + const isRunning = (run.status === 'running' || run.status === 'pending'); const isClickable = !isRunning && !!run.chat_id; const summary = summarizeRun(run); diff --git a/src/frontend/src/components/automation/automationUtils.ts b/src/frontend/src/components/automation/automationUtils.ts index d21de847..e43fd645 100644 --- a/src/frontend/src/components/automation/automationUtils.ts +++ b/src/frontend/src/components/automation/automationUtils.ts @@ -12,6 +12,7 @@ export function channelConversationLabel(c: ChannelConversation): string { /** Labels for a single automation run's status. Kept here so they aren't redeclared across multiple components. */ export const RUN_STATUS_LABEL: Record = { + pending: t('待执行'), running: t('执行中'), success: t('成功'), failed: t('失败'), @@ -21,6 +22,7 @@ export const RUN_STATUS_LABEL: Record = { * detail page and the aggregated run list. The running state additionally carries the * `.jx-anim-ripple` motion primitive; colour/spread live in automation.css. */ export const RUN_STATUS_CLASS: Record = { + pending: 'is-running', running: 'is-running jx-anim-ripple', success: 'is-success', failed: 'is-failed', diff --git a/src/frontend/src/hooks/chatStream/lifecycleAndTools.ts b/src/frontend/src/hooks/chatStream/lifecycleAndTools.ts index b770342e..fd9516c0 100644 --- a/src/frontend/src/hooks/chatStream/lifecycleAndTools.ts +++ b/src/frontend/src/hooks/chatStream/lifecycleAndTools.ts @@ -311,6 +311,20 @@ if (eventType === 'tool_use' || eventType === 'tool_call_start' || eventType === appendOrUpdate(s, true); return true; } +if (eventType === 'tool_progress') { + const toolId = getEventToolId(s, eventObj); + if (!toolId) return true; + const index = toolCardIndexById(s.toolCalls, toolId); + if (index >= 0 && s.toolCalls[index].status === 'running') { + const value = typeof eventObj.progress === 'number' ? eventObj.progress : undefined; + const total = typeof eventObj.total === 'number' ? eventObj.total : undefined; + const message = typeof eventObj.message === 'string' ? eventObj.message.slice(0, 500) : ''; + s.toolCalls[index] = { ...s.toolCalls[index], progressNote: + [message, value === undefined ? '' : total ? `${value}/${total}` : String(value)].filter(Boolean).join(' · ') }; + appendOrUpdate(s, true); + } + return true; +} if (eventType === 'tool_call_delta') { const eventToolId = getEventToolId(s, eventObj); const delta = typeof eventObj.arguments_delta === 'string' ? eventObj.arguments_delta : ''; diff --git a/src/frontend/src/plugin-ui/README.md b/src/frontend/src/plugin-ui/README.md index 76772585..3b7e6ad9 100644 --- a/src/frontend/src/plugin-ui/README.md +++ b/src/frontend/src/plugin-ui/README.md @@ -12,7 +12,7 @@ plugin-ui/ ├── README.md ← 本文件：素材清单 + 如何扩展 ├── index.ts ← 对外出口（宿主只从这里 import） -├── types.ts ← 契约类型，与后端 core/plugins/ui/contract.py 一一对应 +├── types.ts ← 契约类型，与后端 features/workbench/ui/contract.py 一一对应 ├── pointer.ts ← 受限 pointer 求值器（白名单解析，无 eval） ├── i18n.ts ← 贡献文案解析（字符串或 {zh-CN, en} 映射） ├── registry.ts ← ★ view kind → 组件 的唯一注册表 @@ -60,7 +60,7 @@ plugin-ui/ 只读 `map` 指定的字段（用 `pointer.ts` 的 `readText` / `readRecords` 等）， 要触发动作就调 `ctx.runAction`——**不要自己发请求**，数据一律走 L1 代理。 2. **登记**：在 `registry.ts` 的 `VIEW_REGISTRY` 里加一行。 -3. **放行**：在后端 `core/plugins/ui/contract.py` 的 `VIEW_KINDS` 里加上同名字符串， +3. **放行**：在后端 `features/workbench/ui/contract.py` 的 `VIEW_KINDS` 里加上同名字符串， 否则安装时会被当作未知类型丢弃（这是有意的：拼错的 view 名会出现在导入报告里， 而不是运行时变成一张空卡片）。 diff --git a/src/frontend/src/types/automations.ts b/src/frontend/src/types/automations.ts index 75bbd103..57bc2cb6 100644 --- a/src/frontend/src/types/automations.ts +++ b/src/frontend/src/types/automations.ts @@ -12,7 +12,7 @@ export type AutomationTaskType = 'prompt' | 'plan'; export type AutomationStatus = 'active' | 'paused' | 'disabled' | 'completed' | 'expired'; -export type AutomationRunStatus = 'running' | 'success' | 'failed'; +export type AutomationRunStatus = 'pending' | 'running' | 'success' | 'failed'; export type AutomationScheduleType = 'recurring' | 'once' | 'manual'; From 8fbfb5afd68656fa3cbdcbf47d223fbc026a0815 Mon Sep 17 00:00:00 2001 From: Luhaozhu Date: Thu, 22 Oct 2026 17:57:37 -0700 Subject: [PATCH 2/4] test(ce): align release contracts with current module layout --- .../tests/test_ce_runtime_contracts.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/backend/tests/test_ce_runtime_contracts.py b/src/backend/tests/test_ce_runtime_contracts.py index 45c1a507..5e4fe1e7 100644 --- a/src/backend/tests/test_ce_runtime_contracts.py +++ b/src/backend/tests/test_ce_runtime_contracts.py @@ -30,7 +30,13 @@ def test_ce_has_no_enterprise_modules_or_model_exports(): "core.services.team_service", "core.db.repository.team", ): - assert importlib.util.find_spec(module_name) is None + try: + spec = importlib.util.find_spec(module_name) + except ModuleNotFoundError as exc: + # A removed parent package also makes this enterprise path unavailable. + assert exc.name and (module_name == exc.name or module_name.startswith(exc.name + ".")) + spec = None + assert spec is None for model_name in ( "Team", "TeamMember", @@ -116,8 +122,8 @@ def test_ce_user_router_exposes_self_service_password_change(): def test_dingtalk_status_env_does_not_import_ee_sandbox_module(monkeypatch, tmp_path): - import core.sandbox.providers.account_env as _common_account_env - import core.sandbox.workspace.credentials as _common_credentials + import integrations.dingtalk.environment as _common_account_env + import integrations.dingtalk.storage as _common_credentials from integrations.dingtalk.service import _dws_env monkeypatch.setattr(_common_credentials, "dws_home_dir", lambda _user_id: tmp_path / "dws-home") @@ -184,7 +190,7 @@ def test_ce_local_launcher_covers_default_plugin_mcp_servers(): from extensions.mcp_servers._launcher import PORTS as LAUNCHER_PORTS from extensions.mcp_servers._ports import PORTS, package_name - assert _default_plugins() == ["automation", "skill-manager", "sites"] + assert _default_plugins() == ["automation", "skill-manager", "sites", "visualize"] assert { "automation_task": 9108, "skill_manager": 9112, @@ -196,7 +202,7 @@ def test_ce_local_launcher_covers_default_plugin_mcp_servers(): def test_ce_fresh_database_bootstraps_default_plugins(db_session): from core.db.models import AdminMcpServer, ContentBlock, InstalledPlugin - from core.capabilities.plugins import service as plugin_service + from core.capabilities.plugins import management as plugin_service assert plugin_service.ensure_default_plugins_bootstrapped(db_session) is True installs = db_session.query(InstalledPlugin).filter(InstalledPlugin.owner_user_id.is_(None)) @@ -204,6 +210,7 @@ def test_ce_fresh_database_bootstraps_default_plugins(db_session): "automation", "skill-manager", "sites", + "visualize", } assert { row.source_plugin @@ -212,7 +219,7 @@ def test_ce_fresh_database_bootstraps_default_plugins(db_session): .all() } == {"automation", "skill-manager", "sites"} marker = db_session.get(ContentBlock, plugin_service.DEFAULT_BOOTSTRAP_MARKER_ID) - assert marker.payload["plugins"] == ["automation", "skill-manager", "sites"] + assert marker.payload["plugins"] == ["automation", "skill-manager", "sites", "visualize"] assert plugin_service.ensure_default_plugins_bootstrapped(db_session) is False From bc3a58253d75d69f1173d28b8be6eadd15d461ce Mon Sep 17 00:00:00 2001 From: Luhaozhu Date: Thu, 22 Oct 2026 18:31:30 -0700 Subject: [PATCH 3/4] fix(desktop): finish HTTP probes before disconnecting MCP listeners --- .../core/infra/local_runtime/startup.py | 29 +++++++++++++++- .../tests/test_local_mcp_http_readiness.py | 33 +++++++++++++++++++ 2 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 src/backend/tests/test_local_mcp_http_readiness.py diff --git a/src/backend/core/infra/local_runtime/startup.py b/src/backend/core/infra/local_runtime/startup.py index 11fdb3fd..35d92eb1 100644 --- a/src/backend/core/infra/local_runtime/startup.py +++ b/src/backend/core/infra/local_runtime/startup.py @@ -317,6 +317,33 @@ async def _tcp_port_ready(host: str, port: int) -> bool: except (OSError, asyncio.TimeoutError): return False +async def _http_port_ready(host: str, port: int) -> bool: + """Wait for an HTTP response before disconnecting a new MCP listener. + + A connect-and-close probe can abort Windows Proactor's pending AcceptEx and + permanently stop that listener. Receiving response headers proves the server + accepted the connection; a missing root route (404/405) is still ready. + """ + writer = None + try: + async with asyncio.timeout(2.0): + reader, writer = await asyncio.open_connection(host, port) + writer.write( + f"HEAD / HTTP/1.1\r\nHost: {host}:{port}\r\n" + "Connection: close\r\n\r\n".encode("ascii") + ) + await writer.drain() + headers = await reader.readuntil(b"\r\n\r\n") + return headers.startswith((b"HTTP/1.0 ", b"HTTP/1.1 ")) + except (OSError, TimeoutError, asyncio.IncompleteReadError, asyncio.LimitOverrunError): + return False + finally: + if writer is not None: + writer.close() + with contextlib.suppress(OSError, TimeoutError): + await asyncio.wait_for(writer.wait_closed(), timeout=0.5) + + async def _wait_for_mcp_ports( launcher: "asyncio.subprocess.Process", ports: dict[str, int], @@ -330,7 +357,7 @@ async def _wait_for_mcp_ports( if launcher.returncode is not None: raise RuntimeError(f"MCP 启动器提前退出（exit={launcher.returncode}）") checks = await asyncio.gather( - *(_tcp_port_ready("127.0.0.1", port) for port in pending.values()) + *(_http_port_ready("127.0.0.1", port) for port in pending.values()) ) pending = { server_id: port diff --git a/src/backend/tests/test_local_mcp_http_readiness.py b/src/backend/tests/test_local_mcp_http_readiness.py new file mode 100644 index 00000000..2ec421f9 --- /dev/null +++ b/src/backend/tests/test_local_mcp_http_readiness.py @@ -0,0 +1,33 @@ +"""MCP readiness waits for the listener to accept and answer the connection.""" +import asyncio + +import pytest + +from core.infra.local_runtime.startup import _http_port_ready + + +@pytest.mark.asyncio +async def test_http_probe_waits_for_response_even_when_root_route_is_missing(): + requests = [] + async def serve(reader, writer): + requests.append(await reader.readuntil(b"\r\n\r\n")) + await asyncio.sleep(0.05) + writer.write(b"HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\nConnection: close\r\n\r\n") + await writer.drain() + writer.close() + await writer.wait_closed() + server = await asyncio.start_server(serve, "127.0.0.1", 0) + async with server: + assert await _http_port_ready("127.0.0.1", server.sockets[0].getsockname()[1]) + assert requests[0].startswith(b"HEAD / HTTP/1.1\r\n") + + +@pytest.mark.asyncio +async def test_tcp_accept_without_an_http_response_is_not_ready(): + async def serve(reader, writer): + await reader.readuntil(b"\r\n\r\n") + writer.close() + await writer.wait_closed() + server = await asyncio.start_server(serve, "127.0.0.1", 0) + async with server: + assert not await _http_port_ready("127.0.0.1", server.sockets[0].getsockname()[1]) From 72ba37e3fc415cdc153dd4a673e39e10efc331f5 Mon Sep 17 00:00:00 2001 From: Luhaozhu Date: Fri, 23 Oct 2026 03:24:22 -0700 Subject: [PATCH 4/4] Sync CE database threading, execution status, and Docker layout --- .../references/api-envelope.md | 6 +- .../references/architecture.md | 265 ++-- .../references/error-codes.md | 2 + .../scripts/scaffold_feature.sh | 145 +- .../templates/extension.py | 38 + .../hugagent-backend-dev/templates/model.py | 78 +- .../templates/repository.py | 70 +- .../hugagent-backend-dev/templates/route.py | 156 +-- .../hugagent-backend-dev/templates/service.py | 138 +- .../hugagent-backend-dev/templates/test.py | 171 +-- .../assets/design-v2/icons-ui/attachment.svg | 3 + .../assets/design-v2/icons-ui/automation.svg | 3 + .../assets/design-v2/icons-ui/check.svg | 3 + .../design-v2/icons-ui/chevron-right.svg | 3 + .../assets/design-v2/icons-ui/close.svg | 3 + .../assets/design-v2/icons-ui/knowledge.svg | 3 + .../assets/design-v2/icons-ui/my-space.svg | 3 + .../assets/design-v2/icons-ui/new-chat.svg | 3 + .../assets/design-v2/icons-ui/search.svg | 3 + .../assets/design-v2/icons-ui/send.svg | 3 + .../assets/design-v2/icons-ui/settings.svg | 3 + .../assets/design-v2/icons-ui/skill.svg | 3 + .../assets/design-v2/logo/favicon.svg | 5 + .../design-v2/logo/hugagentos-glyph-black.svg | 4 + .../design-v2/logo/hugagentos-glyph-blue.svg | 5 + .../design-v2/logo/hugagentos-glyph-white.svg | 4 + .../logo/hugagentos-mark-mono-black.svg | 5 + .../logo/hugagentos-mark-mono-white.svg | 5 + .../logo/hugagentos-mark-reverse.svg | 6 + .../design-v2/logo/hugagentos-mark-small.svg | 5 + .../assets/design-v2/logo/hugagentos-mark.svg | 6 + .../assets/design-v2/tokens/appTheme.v2.ts | 39 + .../design-v2/tokens/hugagentos-tokens.v2.css | 123 ++ .../tokens/hugagentos-tokens.v2.json | 223 +++ .../design-v2/tokens/motionTokens.v2.ts | 32 + .../references/css-variables.md | 277 ++-- .../references/icon-assets.md | 46 +- .../references/sse-events.md | 294 ++-- .../references/ui-design-spec.md | 667 +++++---- .../scripts/scaffold_component.sh | 5 +- .../templates/component.tsx | 115 +- .../templates/css-module.css | 114 +- .../hugagent-frontend-dev/templates/hook.ts | 98 +- .../hugagent-frontend-dev/templates/store.ts | 121 +- .env.example | 6 +- .gitignore | 10 +- README.md | 363 +++-- README_CN.md | 290 ++-- docker-compose.yml | 14 +- ...Dockerfile.opensandbox-server.dockerignore | 17 - docker/Dockerfile.personal-agent-test | 6 - .../compose.application-hosting.yml | 2 +- .../Dockerfile.object-storage | 0 .../compose.object-storage.yml | 4 +- .../init-object-storage.py | 0 .../Dockerfile.opensandbox-server | 12 +- ...Dockerfile.opensandbox-server.dockerignore | 18 + .../opensandbox-config.toml.tpl | 0 .../opensandbox_capacity.py | 0 .../opensandbox_expirations.py | 0 .../opensandbox_mount_recovery.py | 0 .../opensandbox_mount_worker.py | 0 .../opensandbox_object_mounts.py | 0 .../patch_opensandbox_server.py | 0 .../render-opensandbox-config.py | 0 .../Dockerfile.personal-agent | 6 +- .../compose.personal-agent.yml | 4 +- .../desktop}/helpers.rc | 0 .../desktop}/mimeapps.list | 0 .../desktop}/pa-browser | 0 .../desktop}/pa-browser-helper.desktop | 0 .../desktop}/pa-browser.desktop | 0 .../desktop}/pa-stream.py | 0 .../hugagent-cli-entrypoint.sh | 0 .../personal-agent-entrypoint.sh | 0 .../render-personal-agent-config.py | 0 docker/{ => sandbox}/Dockerfile.opensandbox | 18 +- docker/{ => sandbox}/Dockerfile.script-runner | 8 +- docker/{ => sandbox}/apt-skills.txt | 0 docker/{ => sandbox}/package-skills.json | 0 .../requirements-script-runner.txt | 0 docker/{ => sandbox}/requirements-skills.txt | 0 .../site-template/init-react-site.sh | 0 .../site-template/react-vite/.npmrc | 0 .../site-template/react-vite/index.html | 0 .../react-vite/package-lock.json | 0 .../site-template/react-vite/package.json | 0 .../site-template/react-vite/src/App.jsx | 0 .../site-template/react-vite/src/index.css | 0 .../react-vite/src/lib/siteApi.js | 0 .../site-template/react-vite/src/main.jsx | 0 .../react-vite/src/pages/Home.jsx | 0 .../site-template/react-vite/vite.config.mjs | 0 .../{ => sandbox}/verify-browser-runtime.py | 0 document/README.md | 173 +-- document/en/README.md | 129 +- document/en/api/overview.md | 6 +- document/en/architecture/backend.md | 6 +- document/en/architecture/data-model.md | 4 +- document/en/architecture/overview.md | 109 +- document/en/deployment/README.md | 4 - .../deployment/artifact-storage-key-repair.md | 55 - document/en/deployment/desktop-local-mode.md | 2 +- document/en/deployment/docker-compose.md | 16 +- .../en/deployment/environment-variables.md | 10 +- document/en/deployment/offline-production.md | 80 -- document/en/deployment/quick-install.md | 4 +- document/en/deployment/space-files.md | 6 +- document/en/deployment/uos1070-desktop.md | 101 -- document/en/development/backend.md | 10 +- document/en/development/container-layout.md | 28 - document/en/development/frontend.md | 9 + document/en/development/workbench-routing.md | 2 +- document/en/editions/build-ce.md | 8 +- document/en/getting-started/introduction.md | 170 ++- document/en/getting-started/quick-start.md | 119 +- document/en/modules/admin-console.md | 16 +- document/en/modules/agent-skills.md | 18 +- document/en/modules/application-hosting.md | 32 +- document/en/modules/auth.md | 4 +- document/en/modules/automation.md | 14 +- document/en/modules/autonomous-loop.md | 6 +- document/en/modules/browser-automation.md | 2 +- document/en/modules/canvas-artifacts.md | 6 +- document/en/modules/catalog.md | 6 +- document/en/modules/chat.md | 42 +- document/en/modules/knowledge-base-wiki.md | 14 +- document/en/modules/knowledge-base.md | 32 +- document/en/modules/mcp-tools.md | 16 +- document/en/modules/memory.md | 8 +- document/en/modules/model-providers.md | 22 +- document/en/modules/personal-agent.md | 10 +- document/en/modules/projects-myspace.md | 20 +- document/en/modules/prompts.md | 6 +- document/en/modules/sandbox.md | 24 +- document/en/modules/sites.md | 14 +- document/en/modules/storage.md | 2 +- document/en/modules/workflow-jobs.md | 8 - document/zh-CN/README.md | 132 +- document/zh-CN/api/overview.md | 6 +- document/zh-CN/architecture/backend.md | 6 +- document/zh-CN/architecture/data-model.md | 4 +- document/zh-CN/architecture/overview.md | 97 +- document/zh-CN/deployment/README.md | 4 - .../deployment/artifact-storage-key-repair.md | 48 - .../zh-CN/deployment/desktop-local-mode.md | 3 +- document/zh-CN/deployment/docker-compose.md | 20 +- .../zh-CN/deployment/environment-variables.md | 10 +- .../zh-CN/deployment/offline-production.md | 79 -- document/zh-CN/deployment/quick-install.md | 4 +- document/zh-CN/deployment/space-files.md | 12 +- document/zh-CN/deployment/uos1070-desktop.md | 87 -- document/zh-CN/development/backend.md | 10 +- .../zh-CN/development/container-layout.md | 24 - document/zh-CN/development/frontend.md | 7 + .../zh-CN/development/workbench-routing.md | 2 +- document/zh-CN/editions/build-ce.md | 8 +- .../zh-CN/getting-started/introduction.md | 159 ++- document/zh-CN/getting-started/quick-start.md | 108 +- document/zh-CN/modules/admin-console.md | 16 +- document/zh-CN/modules/agent-skills.md | 18 +- document/zh-CN/modules/application-hosting.md | 32 +- document/zh-CN/modules/auth.md | 4 +- document/zh-CN/modules/automation.md | 14 +- document/zh-CN/modules/autonomous-loop.md | 6 +- document/zh-CN/modules/browser-automation.md | 2 +- document/zh-CN/modules/canvas-artifacts.md | 6 +- document/zh-CN/modules/catalog.md | 6 +- document/zh-CN/modules/chat.md | 42 +- document/zh-CN/modules/knowledge-base-wiki.md | 14 +- document/zh-CN/modules/knowledge-base.md | 32 +- document/zh-CN/modules/mcp-tools.md | 18 +- document/zh-CN/modules/memory.md | 8 +- document/zh-CN/modules/model-providers.md | 22 +- document/zh-CN/modules/personal-agent.md | 10 +- document/zh-CN/modules/projects-myspace.md | 20 +- document/zh-CN/modules/prompts.md | 6 +- document/zh-CN/modules/sandbox.md | 24 +- document/zh-CN/modules/sites.md | 14 +- document/zh-CN/modules/storage.md | 2 +- document/zh-CN/modules/workflow-jobs.md | 8 - examples/customer-chat/.env.example | 6 - examples/customer-chat/.gitignore | 8 - examples/customer-chat/README.md | 37 - examples/customer-chat/debug.html | 1 - examples/customer-chat/index.html | 9 - examples/customer-chat/package-lock.json | 1224 ----------------- examples/customer-chat/package.json | 41 - examples/customer-chat/preview.html | 1 - .../customer-chat/scripts/build-component.mjs | 14 - .../customer-chat/scripts/developer-proxy.mjs | 43 - .../customer-chat/scripts/test-package.mjs | 30 - examples/customer-chat/scripts/test-proxy.mjs | 26 - examples/customer-chat/session.html | 1 - examples/customer-chat/src/controls.ts | 76 - examples/customer-chat/src/css.d.ts | 1 - examples/customer-chat/src/customization.tsx | 29 - examples/customer-chat/src/debug.ts | 32 - examples/customer-chat/src/developer.css | 3 - examples/customer-chat/src/login.ts | 11 - examples/customer-chat/src/preview.tsx | 5 - examples/customer-chat/src/session.ts | 20 - examples/customer-chat/src/standalone.css | 6 - examples/customer-chat/src/standalone.ts | 142 -- examples/customer-chat/src/theme.css | 19 - examples/customer-chat/tsconfig.json | 25 - examples/customer-chat/vite.config.mjs | 30 - install.sh | 2 +- packages/cli/DESIGN.md | 164 +++ src/backend/api/deps.py | 201 +-- src/backend/api/health.py | 9 +- src/backend/api/lifecycle.py | 2 + src/backend/api/personal_agents.py | 18 +- src/backend/api/routes/v1/agent_api_keys.py | 22 +- src/backend/api/routes/v1/agent_responses.py | 39 +- src/backend/api/routes/v1/api_keys.py | 10 +- src/backend/api/routes/v1/artifacts.py | 84 +- src/backend/api/routes/v1/audio.py | 13 +- src/backend/api/routes/v1/auth.py | 52 +- src/backend/api/routes/v1/catalog.py | 6 +- src/backend/api/routes/v1/chats/confirms.py | 19 +- .../api/routes/v1/chats/pending_access.py | 6 +- src/backend/api/routes/v1/chats/questions.py | 26 +- src/backend/api/routes/v1/chats/reruns.py | 36 +- src/backend/api/routes/v1/chats/run_views.py | 22 +- src/backend/api/routes/v1/content.py | 4 +- src/backend/api/routes/v1/file_upload.py | 4 +- src/backend/api/routes/v1/internal_jobs.py | 5 +- src/backend/api/routes/v1/loops.py | 59 +- src/backend/api/routes/v1/me_capabilities.py | 24 +- src/backend/api/routes/v1/me_skill_files.py | 6 +- src/backend/api/routes/v1/memories.py | 69 +- src/backend/api/routes/v1/mock_sso.py | 25 +- src/backend/api/routes/v1/model_reasoning.py | 6 +- src/backend/api/routes/v1/models.py | 119 +- src/backend/api/routes/v1/ontologies.py | 5 +- src/backend/api/routes/v1/plans.py | 160 +-- src/backend/api/routes/v1/plugin_resources.py | 68 +- src/backend/api/routes/v1/plugins.py | 12 +- src/backend/api/routes/v1/project_api_keys.py | 22 +- src/backend/api/routes/v1/projects.py | 26 +- .../api/routes/v1/prompt_management.py | 37 +- src/backend/api/routes/v1/public_chats.py | 17 +- src/backend/api/routes/v1/users.py | 4 +- src/backend/api/startup/bootstrap.py | 59 +- src/backend/api/startup/plugins.py | 14 +- src/backend/api/startup/runtime.py | 15 +- src/backend/cli.py | 3 - .../core/agent_runtime/prepare_main.py | 29 +- src/backend/core/auth/__init__.py | 2 +- src/backend/core/auth/api_key_runs.py | 23 +- src/backend/core/auth/backend.py | 267 ++-- src/backend/core/auth/bearer.py | 149 ++ .../core/auth/personal_permissions_iface.py | 2 +- src/backend/core/auth/session.py | 5 +- .../core/capabilities/mcp/management.py | 16 +- src/backend/core/capabilities/mcp/oauth.py | 83 +- .../capabilities/mcp/tool_schema_backfill.py | 110 +- .../plugins/resources/authentication.py | 4 +- .../plugins/resources/deadlines.py | 31 +- .../plugins/resources/lifecycle.py | 47 +- .../capabilities/plugins/resources/prewarm.py | 75 +- .../plugins/resources/reservation.py | 85 +- .../capabilities/plugins/resources/service.py | 401 ++++-- .../capabilities/plugins/resources/stream.py | 54 +- .../capabilities/skills/deps/aggregator.py | 16 +- .../core/capabilities/skills/deps/prober.py | 4 +- .../core/capabilities/skills/deps/requests.py | 106 +- src/backend/core/chat/operations/admission.py | 335 +++-- .../core/chat/operations/session_context.py | 27 - src/backend/core/chat/runtime/followups.py | 7 +- src/backend/core/chat/runtime/job_wakeup.py | 204 +-- src/backend/core/chat/runtime/recovery.py | 5 +- .../core/chat/runtime/recovery_snapshot.py | 9 +- src/backend/core/chat/runtime/worker.py | 27 +- src/backend/core/chat/sequencer.py | 33 - src/backend/core/config/identity_settings.py | 8 +- src/backend/core/config/sandbox_settings.py | 2 +- src/backend/core/db/engine.py | 39 +- src/backend/core/db/threads.py | 99 ++ .../core/harness/registry/authentication.py | 21 +- .../harness/registry/marketplace_catalog.py | 2 +- src/backend/core/infra/worker_count.py | 7 + src/backend/core/jobs/runtime.py | 9 +- src/backend/core/jobs/tool_operations.py | 161 ++- .../core/llm/runtime/delta_coalescer.py | 106 ++ src/backend/core/llm/runtime/stream_events.py | 32 +- src/backend/core/llm/runtime/stream_loop.py | 43 +- src/backend/core/llm/runtime/streaming.py | 15 +- .../core/llm/runtime/streaming_constants.py | 7 +- src/backend/core/llm/subagents/execution.py | 14 +- src/backend/core/llm/tools/pin_tool.py | 10 +- src/backend/core/memory/access.py | 48 +- src/backend/core/memory/effect_lane.py | 54 +- src/backend/core/memory/outbox/leases.py | 3 +- src/backend/core/memory/outbox/worker.py | 37 +- .../core/models/api_protocol_backfill.py | 124 +- src/backend/core/models/config.py | 203 ++- src/backend/core/ontology/evolution.py | 79 +- src/backend/core/runs/binding.py | 92 +- src/backend/core/runs/runtime.py | 65 +- src/backend/core/runs/tool_effect_recovery.py | 60 +- src/backend/core/runs/watchdog.py | 5 +- .../core/sandbox/commands/subscription.py | 4 +- .../sandbox/providers/opensandbox/runtime.py | 2 +- .../core/sandbox/workspace/identity.py | 2 +- src/backend/features/__init__.py | 2 +- .../features/automation/tool_routing.py | 94 +- src/backend/features/desktop/auth/bridge.py | 5 + .../features/desktop/local/plugins/assets.py | 6 +- src/backend/features/evaluation/sandboxes.py | 8 +- .../hosted_apps/api/application_transports.py | 36 +- .../features/hosted_apps/api/applications.py | 17 +- .../features/hosted_apps/publication.py | 2 +- .../hosted_apps/resources/admission.py | 129 +- .../features/hosted_apps/resources/cleanup.py | 49 +- .../features/knowledge/api/kb_documents.py | 7 +- .../marketplace/api/mcp_marketplace.py | 24 +- .../features/marketplace/core_catalog.py | 4 +- .../features/marketplace/mcp_installation.py | 81 +- .../features/marketplace/mcp_monitor.py | 37 +- .../features/marketplace/mcp_publishing.py | 76 +- .../features/marketplace/mcp_revalidation.py | 162 ++- .../features/marketplace/mcp_submissions.py | 131 +- src/backend/features/sites/api/sites_serve.py | 54 +- .../features/workbench/api/plugin_ui.py | 4 +- .../features/workbench/api/workbench.py | 23 +- .../workbench/api/workbench_activity.py | 36 +- .../api/workbench_application_config.py | 6 +- .../features/workbench/api/workbench_auth.py | 17 +- .../workbench/api/workbench_capabilities.py | 92 +- .../workbench/api/workbench_events.py | 7 +- .../workbench/api/workbench_resources.py | 42 +- src/backend/features/workbench/background.py | 118 +- .../features/workbench/resource_deadlines.py | 65 +- src/backend/features/workbench/revocation.py | 36 +- .../features/workbench/revocation_sweep.py | 28 +- src/backend/integrations/api/integrations.py | 41 +- .../channels/api/channel_desktop.py | 25 +- .../integrations/channels/api/channels.py | 36 +- .../integrations/channels/desktop_worker.py | 78 +- src/backend/integrations/channels/inbound.py | 269 ++-- .../channels/inbound_attachments.py | 25 +- .../integrations/channels/inbound_history.py | 26 +- src/backend/integrations/channels/manager.py | 4 +- src/backend/integrations/channels/outbound.py | 64 +- .../integrations/channels/relay_dispatch.py | 86 +- .../integrations/channels/relay_rpc.py | 105 +- src/backend/integrations/channels/service.py | 224 +-- .../integrations/channels/tools/attachment.py | 127 +- .../integrations/dingtalk/cube_credentials.py | 6 +- src/backend/integrations/dingtalk/service.py | 44 +- src/backend/integrations/email/service.py | 53 +- src/backend/integrations/lark/service.py | 46 +- .../automations/automation_conversation.py | 199 +-- .../automations/automation_dispatch.py | 288 ++-- src/backend/orchestration/loops/admission.py | 5 +- src/backend/orchestration/loops/recovery.py | 23 +- src/backend/orchestration/loops/run.py | 26 +- .../orchestration/plans/execute_run.py | 132 +- .../orchestration/plans/generate_run.py | 115 +- src/backend/orchestration/plans/mode.py | 200 ++- .../orchestration/workers/_worker_base.py | 5 +- src/backend/personal_agent/api/sessions.py | 11 +- src/backend/personal_agent/background.py | 8 +- .../personal_agent/channels/channel_bridge.py | 43 +- .../personal_agent/computer/devices.py | 43 +- .../personal_agent/computer/live_stream.py | 27 +- .../personal_agent/computer/sandbox.py | 28 +- .../personal_agent/conversation/front.py | 5 +- .../personal_agent/sessions/session_auth.py | 43 +- .../personal_agent/sessions/session_cloud.py | 72 +- .../sessions/session_manager.py | 151 +- .../personal_agent/sessions/session_tool.py | 37 +- .../personal_agent/tasks/attachments.py | 6 +- src/backend/personal_agent/tasks/runtime.py | 80 +- src/backend/personal_agent/tasks/worker.py | 168 ++- .../personal_agent/tools/front_tools.py | 43 +- src/backend/personal_agent/tools/tools.py | 25 +- .../personal_agent/workspace/dreaming.py | 8 +- src/backend/tests/api/chat_fork_fixtures.py | 6 +- src/backend/tests/api/test_agent_api_keys.py | 2 + .../api/test_agent_response_admission.py | 29 +- .../tests/api/test_agent_response_scope.py | 1 - src/backend/tests/api/test_agent_responses.py | 2 +- src/backend/tests/api/test_archive_files.py | 2 + .../tests/api/test_artifacts_backfill_once.py | 4 +- .../tests/api/test_catalog_event_loop.py | 27 +- .../api/test_chat_event_loop_performance.py | 4 +- .../tests/api/test_chat_rerun_options.py | 4 +- .../tests/api/test_chat_stream_sequencer.py | 7 +- .../tests/api/test_file_open_browser_e2e.py | 2 + .../tests/api/test_loop_cancel_scope.py | 4 +- .../api/test_model_reasoning_settings.py | 2 + .../tests/api/test_plugin_delete_isolation.py | 2 + .../tests/api/test_project_environments.py | 2 + src/backend/tests/api/test_public_chats.py | 8 +- .../auth/test_auth_dependencies_off_loop.py | 176 +++ .../tests/auth/test_bearer_identity.py | 180 +++ .../ce_release/test_ce_release_regressions.py | 146 +- .../tests/chat/test_user_questions_api.py | 2 - .../tests/content/security_selftest.py | 24 +- .../tests/content/test_local_file_location.py | 2 + src/backend/tests/db_support.py | 43 + .../features/desktop/capabilities/conftest.py | 2 + .../capabilities/test_agent_rerun_identity.py | 25 +- .../capabilities/test_plugin_connections.py | 2 + .../capabilities/test_runtime_scopes.py | 10 +- .../features/desktop/observability_cases_1.py | 2 + .../desktop/test_desktop_sync_locking.py | 3 +- .../evaluation/test_evaluation_service.py | 1 + .../application_hosting_support.py | 2 + .../test_application_agent_contract.py | 2 + .../hosted_apps/test_application_desktop.py | 4 +- .../hosted_apps/test_application_hosting.py | 2 + .../test_application_personal_mcp.py | 2 + .../marketplace/mcp_marketplace_cases_1.py | 4 +- .../marketplace/mcp_marketplace_cases_2.py | 46 +- .../marketplace/mcp_marketplace_cases_3.py | 59 +- .../marketplace/mcp_marketplace_cases_4.py | 23 +- .../marketplace/mcp_marketplace_support.py | 14 +- .../marketplace/test_oauth_sdk2_contract.py | 5 +- .../sites/test_site_access_password.py | 5 +- .../workbench/test_plugin_ui_routes.py | 4 +- .../test_workbench_activity_routes.py | 7 +- .../workbench/test_workbench_auth_routes.py | 4 +- .../test_workbench_background_subpage_http.py | 2 + .../test_workbench_capability_routes.py | 5 +- .../test_workbench_diagnostic_failure_http.py | 2 + .../workbench/test_workbench_diagnostics.py | 5 +- .../test_workbench_external_routes.py | 5 +- .../workbench/test_workbench_plugin_routes.py | 5 +- .../test_workbench_preferences_routes.py | 7 +- .../workbench/test_workbench_resources.py | 27 +- .../test_workbench_revocation_failure_http.py | 2 +- .../test_workbench_state_concurrency.py | 4 +- .../workbench/test_workbench_state_routes.py | 4 +- .../infra/test_db_off_loop_call_sites.py | 115 ++ .../tests/infra/test_db_thread_budget.py | 120 ++ .../infra/test_no_session_on_event_loop.py | 100 ++ .../integrations/channels/bot_cases_1.py | 32 +- .../integrations/channels/bot_cases_4.py | 5 +- .../integrations/channels/bot_cases_5.py | 37 +- .../integrations/channels/bot_support.py | 10 +- .../channels/test_channel_bots_api.py | 2 + .../channels/test_channel_desktop_e2e.py | 14 +- .../channels/test_channel_desktop_http.py | 21 +- .../test_channel_desktop_relay_api.py | 17 + .../channels/test_channel_desktop_worker.py | 2 - .../channels/test_weixin_connection_health.py | 3 +- .../channels/test_weixin_qr_protocol.py | 3 +- .../dingtalk/test_dingtalk_integration.py | 17 +- .../email/test_email_integration.py | 28 +- .../integrations/lark/test_app_lifecycle.py | 6 +- src/backend/tests/llm/test_delta_coalescer.py | 145 ++ .../tests/llm/test_local_project_cwd.py | 2 + .../tests/llm/test_local_project_pin.py | 6 +- .../tests/llm/test_local_session_reference.py | 8 +- .../tests/llm/test_subagent_model_config.py | 2 + .../tests/llm/test_subagent_tool_streaming.py | 9 +- .../tests/memory/outbox_test_support.py | 1 + .../tests/memory/test_mem0_real_lifecycle.py | 1 + .../streaming_tool_call_dedupe_selftest.py | 8 +- .../test_chat_resume_checkpoint.py | 2 +- .../orchestration/test_command_events.py | 4 +- .../orchestration/test_job_managed_runtime.py | 4 +- .../orchestration/test_job_orphan_reaper.py | 2 +- .../test_job_wake_message_visibility.py | 8 +- .../tests/orchestration/test_job_wake_mode.py | 12 +- .../test_model_progress_signal.py | 42 +- .../test_plan_progress_closeout.py | 3 - .../test_tool_effect_ledger_3.py | 2 - .../prompts/test_desktop_prompt_versions.py | 17 +- .../prompts/test_prompt_version_freshness.py | 14 +- .../tests/sandbox/test_controller_capacity.py | 2 +- .../test_credential_purge_preserves_mounts.py | 18 +- .../test_opensandbox_controller_patch.py | 2 +- .../sandbox/test_opensandbox_expirations.py | 2 +- .../tests/sandbox/test_runner_package.py | 2 +- .../tests/sandbox/test_sandbox_packaging.py | 2 +- .../tests/services/test_audio_models.py | 8 +- .../tests/services/test_audio_routes.py | 2 + .../tests/services/test_chat_titles.py | 2 +- .../services/test_model_config_consistency.py | 42 + .../services/test_project_instructions.py | 4 +- .../skills/test_skill_deps_aggregator.py | 10 +- src/backend/tests/test_browser_prewarm.py | 55 +- src/backend/tests/test_browser_remediation.py | 2 +- .../tests/test_browser_resource_assets.py | 2 + .../tests/test_browser_resource_security.py | 19 +- .../tests/test_browser_resource_transport.py | 10 +- .../tests/test_ce_runtime_contracts.py | 8 - src/backend/tests/test_personal_agent.py | 8 +- .../test_personal_agent_channel_sessions.py | 4 +- .../tests/test_personal_agent_dreaming.py | 4 +- .../tests/test_personal_agent_monitor.py | 3 +- .../tests/test_personal_agent_routing.py | 1 + .../tests/test_personal_agent_session_tool.py | 5 +- .../tests/test_personal_agent_sessions.py | 10 +- .../tests/test_personal_agent_steer.py | 4 +- src/backend/tests/test_plugin_components.py | 7 +- .../test_script_runner_process_limits.py | 4 +- src/frontend/Dockerfile | 1 - .../scripts/test-reference-motion-browser.mjs | 34 +- .../test-sidebar-task-status-browser.mjs | 15 +- .../scripts/test-subagent-canvas-browser.mjs | 3 +- .../scripts/test-turn-status-alignment.mjs | 2 +- .../automation/AutomationTaskSections.tsx | 6 +- .../automation/RunTimelinePanel.tsx | 64 +- .../components/automation/automationUtils.ts | 7 +- .../src/components/chat/AgentCursor.tsx | 72 + .../src/components/chat/MessageBody.tsx | 107 +- .../src/components/chat/MessageBubble.tsx | 1 - .../components/chat/OntologyRevisionPanel.tsx | 7 - .../components/chat/StreamWaitIndicator.tsx | 4 +- .../components/chat/TurnStatusIndicator.tsx | 56 +- .../src/components/common/AgentMark.tsx | 48 + .../src/components/common/ElapsedTimer.tsx | 8 +- .../src/components/common/RollingText.tsx | 105 ++ .../shared/PersonalAgentNavStatus.tsx | 9 +- .../personal-agent/shared/notifications.ts | 15 +- .../src/components/tool/CompactionStepRow.tsx | 10 +- .../src/components/tool/PendingStepRow.tsx | 34 - .../src/components/tool/ThinkingStepRow.tsx | 7 +- .../src/components/tool/ToolCallRow.tsx | 72 +- .../components/tool/ToolCapabilityIcon.tsx | 8 +- .../src/components/tool/ToolKindStack.tsx | 69 + .../src/components/tool/ToolRunShell.tsx | 151 +- .../src/components/tool/toolIcons.tsx | 68 + src/frontend/src/hooks/chatStream/consume.ts | 1 - .../src/hooks/chatStream/lifecycleAndTools.ts | 32 +- .../src/hooks/chatStream/messageStore.ts | 6 +- src/frontend/src/hooks/chatStream/snapshot.ts | 1 - src/frontend/src/hooks/chatStream/state.ts | 1 - .../hooks/chatStream/toolResultsAndMeta.ts | 1 - src/frontend/src/hooks/useActivityAge.ts | 47 + src/frontend/src/hooks/useStallDetector.ts | 6 +- src/frontend/src/hooks/useToolCapability.ts | 18 +- src/frontend/src/i18n/en/chat.ts | 2 - src/frontend/src/i18n/en/lab.ts | 4 +- src/frontend/src/i18n/en/tool.ts | 27 +- .../src/stores/automationChatStore.ts | 35 +- src/frontend/src/styles/chat.css | 1 + .../src/styles/chat/agent-activity.css | 226 +++ src/frontend/src/styles/chat/messages.css | 7 + src/frontend/src/styles/chat/turn-status.css | 28 +- src/frontend/src/styles/tool.css | 144 +- src/frontend/src/styles/variables.css | 3 + src/frontend/src/types/conversations.ts | 7 +- src/frontend/src/types/messages.ts | 1 - src/frontend/src/utils/elapsed.ts | 5 + src/frontend/src/utils/toolActionLabel.ts | 84 +- src/frontend/src/utils/toolIconKind.ts | 12 + src/frontend/src/utils/turnActivity.ts | 69 + 554 files changed, 11704 insertions(+), 9990 deletions(-) create mode 100644 .claude/skills/hugagent-backend-dev/templates/extension.py create mode 100644 .claude/skills/hugagent-frontend-dev/assets/design-v2/icons-ui/attachment.svg create mode 100644 .claude/skills/hugagent-frontend-dev/assets/design-v2/icons-ui/automation.svg create mode 100644 .claude/skills/hugagent-frontend-dev/assets/design-v2/icons-ui/check.svg create mode 100644 .claude/skills/hugagent-frontend-dev/assets/design-v2/icons-ui/chevron-right.svg create mode 100644 .claude/skills/hugagent-frontend-dev/assets/design-v2/icons-ui/close.svg create mode 100644 .claude/skills/hugagent-frontend-dev/assets/design-v2/icons-ui/knowledge.svg create mode 100644 .claude/skills/hugagent-frontend-dev/assets/design-v2/icons-ui/my-space.svg create mode 100644 .claude/skills/hugagent-frontend-dev/assets/design-v2/icons-ui/new-chat.svg create mode 100644 .claude/skills/hugagent-frontend-dev/assets/design-v2/icons-ui/search.svg create mode 100644 .claude/skills/hugagent-frontend-dev/assets/design-v2/icons-ui/send.svg create mode 100644 .claude/skills/hugagent-frontend-dev/assets/design-v2/icons-ui/settings.svg create mode 100644 .claude/skills/hugagent-frontend-dev/assets/design-v2/icons-ui/skill.svg create mode 100644 .claude/skills/hugagent-frontend-dev/assets/design-v2/logo/favicon.svg create mode 100644 .claude/skills/hugagent-frontend-dev/assets/design-v2/logo/hugagentos-glyph-black.svg create mode 100644 .claude/skills/hugagent-frontend-dev/assets/design-v2/logo/hugagentos-glyph-blue.svg create mode 100644 .claude/skills/hugagent-frontend-dev/assets/design-v2/logo/hugagentos-glyph-white.svg create mode 100644 .claude/skills/hugagent-frontend-dev/assets/design-v2/logo/hugagentos-mark-mono-black.svg create mode 100644 .claude/skills/hugagent-frontend-dev/assets/design-v2/logo/hugagentos-mark-mono-white.svg create mode 100644 .claude/skills/hugagent-frontend-dev/assets/design-v2/logo/hugagentos-mark-reverse.svg create mode 100644 .claude/skills/hugagent-frontend-dev/assets/design-v2/logo/hugagentos-mark-small.svg create mode 100644 .claude/skills/hugagent-frontend-dev/assets/design-v2/logo/hugagentos-mark.svg create mode 100644 .claude/skills/hugagent-frontend-dev/assets/design-v2/tokens/appTheme.v2.ts create mode 100644 .claude/skills/hugagent-frontend-dev/assets/design-v2/tokens/hugagentos-tokens.v2.css create mode 100644 .claude/skills/hugagent-frontend-dev/assets/design-v2/tokens/hugagentos-tokens.v2.json create mode 100644 .claude/skills/hugagent-frontend-dev/assets/design-v2/tokens/motionTokens.v2.ts delete mode 100644 docker/Dockerfile.opensandbox-server.dockerignore delete mode 100644 docker/Dockerfile.personal-agent-test rename docker-compose.application-hosting.yml => docker/application-hosting/compose.application-hosting.yml (94%) rename docker/{ => object-storage}/Dockerfile.object-storage (100%) rename docker/{ => object-storage}/compose.object-storage.yml (91%) rename docker/{ => object-storage}/init-object-storage.py (100%) rename docker/{ => opensandbox-server}/Dockerfile.opensandbox-server (52%) create mode 100644 docker/opensandbox-server/Dockerfile.opensandbox-server.dockerignore rename docker/{ => opensandbox-server}/opensandbox-config.toml.tpl (100%) rename docker/{ => opensandbox-server}/opensandbox_capacity.py (100%) rename docker/{ => opensandbox-server}/opensandbox_expirations.py (100%) rename docker/{ => opensandbox-server}/opensandbox_mount_recovery.py (100%) rename docker/{ => opensandbox-server}/opensandbox_mount_worker.py (100%) rename docker/{ => opensandbox-server}/opensandbox_object_mounts.py (100%) rename docker/{ => opensandbox-server}/patch_opensandbox_server.py (100%) rename docker/{ => opensandbox-server}/render-opensandbox-config.py (100%) rename docker/{ => personal-agent}/Dockerfile.personal-agent (90%) rename docker/{ => personal-agent}/compose.personal-agent.yml (91%) rename docker/{personal-agent-desktop => personal-agent/desktop}/helpers.rc (100%) rename docker/{personal-agent-desktop => personal-agent/desktop}/mimeapps.list (100%) rename docker/{personal-agent-desktop => personal-agent/desktop}/pa-browser (100%) rename docker/{personal-agent-desktop => personal-agent/desktop}/pa-browser-helper.desktop (100%) rename docker/{personal-agent-desktop => personal-agent/desktop}/pa-browser.desktop (100%) rename docker/{personal-agent-desktop => personal-agent/desktop}/pa-stream.py (100%) rename docker/{ => personal-agent}/hugagent-cli-entrypoint.sh (100%) rename docker/{ => personal-agent}/personal-agent-entrypoint.sh (100%) rename docker/{ => personal-agent}/render-personal-agent-config.py (100%) rename docker/{ => sandbox}/Dockerfile.opensandbox (95%) rename docker/{ => sandbox}/Dockerfile.script-runner (96%) rename docker/{ => sandbox}/apt-skills.txt (100%) rename docker/{ => sandbox}/package-skills.json (100%) rename docker/{ => sandbox}/requirements-script-runner.txt (100%) rename docker/{ => sandbox}/requirements-skills.txt (100%) rename docker/{ => sandbox}/site-template/init-react-site.sh (100%) rename docker/{ => sandbox}/site-template/react-vite/.npmrc (100%) rename docker/{ => sandbox}/site-template/react-vite/index.html (100%) rename docker/{ => sandbox}/site-template/react-vite/package-lock.json (100%) rename docker/{ => sandbox}/site-template/react-vite/package.json (100%) rename docker/{ => sandbox}/site-template/react-vite/src/App.jsx (100%) rename docker/{ => sandbox}/site-template/react-vite/src/index.css (100%) rename docker/{ => sandbox}/site-template/react-vite/src/lib/siteApi.js (100%) rename docker/{ => sandbox}/site-template/react-vite/src/main.jsx (100%) rename docker/{ => sandbox}/site-template/react-vite/src/pages/Home.jsx (100%) rename docker/{ => sandbox}/site-template/react-vite/vite.config.mjs (100%) rename docker/{ => sandbox}/verify-browser-runtime.py (100%) delete mode 100644 document/en/deployment/artifact-storage-key-repair.md delete mode 100644 document/en/deployment/offline-production.md delete mode 100644 document/en/deployment/uos1070-desktop.md delete mode 100644 document/en/development/container-layout.md delete mode 100644 document/zh-CN/deployment/artifact-storage-key-repair.md delete mode 100644 document/zh-CN/deployment/offline-production.md delete mode 100644 document/zh-CN/deployment/uos1070-desktop.md delete mode 100644 document/zh-CN/development/container-layout.md delete mode 100644 examples/customer-chat/.env.example delete mode 100644 examples/customer-chat/.gitignore delete mode 100644 examples/customer-chat/README.md delete mode 100644 examples/customer-chat/debug.html delete mode 100644 examples/customer-chat/index.html delete mode 100644 examples/customer-chat/package-lock.json delete mode 100644 examples/customer-chat/package.json delete mode 100644 examples/customer-chat/preview.html delete mode 100644 examples/customer-chat/scripts/build-component.mjs delete mode 100644 examples/customer-chat/scripts/developer-proxy.mjs delete mode 100644 examples/customer-chat/scripts/test-package.mjs delete mode 100644 examples/customer-chat/scripts/test-proxy.mjs delete mode 100644 examples/customer-chat/session.html delete mode 100644 examples/customer-chat/src/controls.ts delete mode 100644 examples/customer-chat/src/css.d.ts delete mode 100644 examples/customer-chat/src/customization.tsx delete mode 100644 examples/customer-chat/src/debug.ts delete mode 100644 examples/customer-chat/src/developer.css delete mode 100644 examples/customer-chat/src/login.ts delete mode 100644 examples/customer-chat/src/preview.tsx delete mode 100644 examples/customer-chat/src/session.ts delete mode 100644 examples/customer-chat/src/standalone.css delete mode 100644 examples/customer-chat/src/standalone.ts delete mode 100644 examples/customer-chat/src/theme.css delete mode 100644 examples/customer-chat/tsconfig.json delete mode 100644 examples/customer-chat/vite.config.mjs create mode 100644 packages/cli/DESIGN.md create mode 100644 src/backend/core/auth/bearer.py create mode 100644 src/backend/core/db/threads.py create mode 100644 src/backend/core/llm/runtime/delta_coalescer.py create mode 100644 src/backend/tests/auth/test_auth_dependencies_off_loop.py create mode 100644 src/backend/tests/auth/test_bearer_identity.py create mode 100644 src/backend/tests/db_support.py create mode 100644 src/backend/tests/infra/test_db_off_loop_call_sites.py create mode 100644 src/backend/tests/infra/test_db_thread_budget.py create mode 100644 src/backend/tests/infra/test_no_session_on_event_loop.py create mode 100644 src/backend/tests/llm/test_delta_coalescer.py create mode 100644 src/frontend/src/components/chat/AgentCursor.tsx create mode 100644 src/frontend/src/components/common/AgentMark.tsx create mode 100644 src/frontend/src/components/common/RollingText.tsx delete mode 100644 src/frontend/src/components/tool/PendingStepRow.tsx create mode 100644 src/frontend/src/components/tool/ToolKindStack.tsx create mode 100644 src/frontend/src/components/tool/toolIcons.tsx create mode 100644 src/frontend/src/hooks/useActivityAge.ts create mode 100644 src/frontend/src/styles/chat/agent-activity.css create mode 100644 src/frontend/src/utils/elapsed.ts create mode 100644 src/frontend/src/utils/toolIconKind.ts create mode 100644 src/frontend/src/utils/turnActivity.ts diff --git a/.claude/skills/hugagent-backend-dev/references/api-envelope.md b/.claude/skills/hugagent-backend-dev/references/api-envelope.md index 841012ce..a0d9525c 100644 --- a/.claude/skills/hugagent-backend-dev/references/api-envelope.md +++ b/.claude/skills/hugagent-backend-dev/references/api-envelope.md @@ -78,4 +78,8 @@ return error_response(code=20001, message="Bad request", status_code=400) 1. **路由中只用 `success_response` / `created_response` / `paginated_response`** 2. **错误通过抛 `AppException` 子类，由全局 handler 转 `error_response`** 3. **trace_id 默认自动生成，无需手动传** -4. **SSE 端点（流式聊天）不使用信封，直接发 SSE 事件** +4. **SSE 端点不使用信封**：用 `sse_response(generator, extra_headers=...)` 返回（自带 `Cache-Control: no-cache`、 + `X-Accel-Buffering: no` 等 `SSE_HEADERS`）；聊天回合的 SSE 输出统一走 `api/transports/run_sse.py`，不要自写一套 +5. **License 402** 只来自 `edition_ee/licensing/features.py` 的 `FeatureNotLicensed`（40201）/ `SeatLimitExceeded`（40202）， + 路由与服务层不要手搓 402 +6. 时间字段序列化用 `.isoformat()`（模型存的是带时区的 UTC），不要输出无时区字符串 diff --git a/.claude/skills/hugagent-backend-dev/references/architecture.md b/.claude/skills/hugagent-backend-dev/references/architecture.md index 5fda6ac0..306f4283 100644 --- a/.claude/skills/hugagent-backend-dev/references/architecture.md +++ b/.claude/skills/hugagent-backend-dev/references/architecture.md @@ -1,129 +1,164 @@ # 后端架构参考 -## 请求流转 +## 1. 顶层布局与依赖方向 ``` -Browser → Nginx (:3000 /api/ proxy) → FastAPI (api/app.py) - → api/middleware/ (CORS → Logging → Error Handler → license_gate) - → api/routes/v1/*.py (路由层；CE 注册表 api/routes/v1/__init__.py，EE 注册表 edition_ee/routes/registry.py) - → core/services/*.py (服务层 — 业务逻辑) - → core/db/repository/ (仓库层 — 数据访问，按领域分文件) - → core/db/models/ (ORM 包 → PostgreSQL/SQLite) +src/backend/ +├── api/ HTTP 协议层：app.py（组装）、deps.py、middleware/、routes/v1/（CE_ROUTERS）、transports/run_sse.py、startup/ +├── core/ 内核：领域逻辑与可复用运行时（不导入 features / integrations / personal_agent / orchestration） +├── orchestration/ 模式编排：plans/、loops/、automations/、workers/（周期任务与恢复） +├── features/ 可安装的产品功能模块（各自 README + extension.py + models.py + api/） +├── integrations/ 外部系统集成：channels（IM 机器人框架）、dingtalk、lark、email、yida、api +├── personal_agent/ Personal Agent 模块（同样经 extension.py 接入） +├── edition_ee/ 商业版专属：路由注册表、License、Team/RBAC、EE 表、KB provider、EE 服务（CE 树物理删除） +├── extensions/ 随包资源：skills/、agents/、plugins/（default + marketplace）、mcp_servers/；根路径唯一来源 paths.py +├── prompts/ 提示词装配运行时 + prompt_text/ 文件兜底 +├── hugagent_cli/ 命令行客户端 +├── alembic/ 迁移（EE 主链；CE 用独立 ce_0001 基线） +└── tests/ pytest（按领域子目录：api/ chat/ core/ features// infra/ llm/ orchestration/ …） ``` -## 分层职责 +``` +api ──► orchestration ──► core + │ │ ▲ + └──► features / integrations / personal_agent ──┘ （通过 core.harness.registry 注册钩子接入） +edition_ee ──► core / features （CE 派生树中不存在） +``` + +- **core 永远不导入功能模块的实现**。功能模块在 `extension.setup()` 里调用 `core.harness.registry.register_*` 把能力挂进来； + 关闭时什么也不注册，core 的行为等同于未安装。 +- **不保留转发垫片**：模块搬家时直接改所有调用方的 import，不留旧路径 re-export（ADR 20261021）。 +- 单个模块 ≤ 500 行（存量超限文件已登记为例外，不要新增）。 + +## 2. core/ 子包职责 + +| 子包 | 职责 | +|---|---| +| `agent_runtime/` | 一次普通智能体回合：`runner.astream_chat_workflow` 选主智能体 / 直连路径；`prepare_*`（建 agent、组装输入）、`stream_*`（模型与工具事件 → 既有事件协议）、`*_state`（单轮有类型状态）、`integrations/`（记忆、压缩、本体证据/修复、演化） | +| `runs/` | 持久化执行所有权：租约 / 心跳、取消、事件日志与续播（`events/`）、看门狗、工具副作用账本（`tool_effects/`）、恢复 | +| `chat/` | 会话与消息读写、模式（modes）、分叉、引用、追问、压缩；`runtime/`（聊天 worker：消费回合事件、在 runs 租约下持久化输出、排队 / 插话 / 暂停 / 恢复）；`operations/`、`projection/` | +| `llm/` | AgentScope 适配与智能体装配：`factory/`（build / assembly / tools / prompts）、`runtime/`（流式循环、钩子适配、工具执行、人机交互）、`citations/`（锚点发号）、`mcp/`、`subagents/`、`tools/`、`tool_permissions/`、提示词版本池 | +| `models/` | 模型供应商与选择：chat_models、providers、failover、reasoning effort、embedding / reranking、vision、用量 | +| `capabilities/` | 能力生命周期：`catalog/`、`skills/`、`plugins/`（含插件资源）、`mcp/`、`agents/`、manifest 协议 | +| `harness/` | 宿主接缝：`registry/`（功能模块注册钩子的唯一入口）、执行清单、观测事件、日志写入、用量 | +| `auth/` | `backend.py`（`get_current_user` / `UserContext`）、`bearer.py`（mock/remote Bearer 异步校验 + 共享缓存）、session、本地账号、API Key、能力位、权限接口 | +| `db/` | `engine.py`（Base / SessionLocal / `get_db` / 线程预算）、`threads.py`（`run_db` / `run_in_session` / `off_loop`）、`models/`（核心 ORM 包）、`repository/`、`utc_datetime.py`、各类升级 / 修复脚本 | +| `config/` | `settings.py`（`settings` 冻结 dataclass：deploy / auth / sso / db / llm / memory / storage / kb / redis / server / sandbox / edition / branding …）、`catalog.json`、`mcp_config.py`、`display_names.py`、`runtime_env.py` | +| `infra/` | `responses.py`、`exceptions.py`、`logging.py`、`time.py`（`utc_now`）、`background.py`（`spawn`）、redis、metrics、限流 / 熔断、leader 选举、加密、脱敏 | +| `files/` | 产物（artifacts）、内容解析（content）、空间文件（spaces）、存储协议（storage：local / s3 / oss） | +| `projects/` | 项目、项目环境、指令、本地工作区、个人项目作用域 | +| `memory/` | 分层记忆（L1 画像 / L2 向量 / L3 图谱）、抽取流水线、outbox、脱敏、TTL | +| `ontology/` | 本体治理：schema、校验、运行时闸门、输出评审、修订 | +| `evolution/` | 协同进化共享契约与证据（控制面在 EE） | +| `jobs/` | 后台作业运行时（作业 SDK、监督、通知、文件） | +| `sandbox/` | 沙箱协议 / 工厂 / providers（script_runner、opensandbox、cube[EE]）、runner 进程服务、进程 / 会话 / 工作区 / 权限 / OS 隔离 | + +## 3. 功能模块（features / integrations / personal_agent） + +`features/installed.py::INSTALLED_FEATURES` 是全量安装清单（CE 由 `ce/overlay/src/backend/features/installed.py` 覆盖）。 + +| 模块 | 内容 | +|---|---| +| `features/automation` | 定时任务：调度、执行、投递目标、通知、远程副作用 | +| `features/knowledge` | 自建知识库：解析、分块、向量、资产索引、Wiki；外部 provider 接缝 | +| `features/marketplace` | 技能 / 智能体 / MCP 市场：目录、安装、发布、版本、导出 | +| `features/workbench` | 可扩展工作台：应用目录、外部应用 / 嵌入 / 外部认证、插件 UI、资源实例、偏好、诊断、迁移 | +| `features/hosted_apps` | 托管业务应用（SQL / MCP）：部署、数据、发布、资源 | +| `features/sites` | 站点托管与发布（静态服务、访问密码、打包、本机发布） | +| `features/desktop` | 桌面双端：能力桥、网关、本机能力、登录桥接、可观测性 | +| `features/evaluation` | 评测沙箱（严格 OpenSandbox） | +| `features/distillation` · `security_console` · `datasources` · `sandbox_ops` | EE：技能 / 人格蒸馏、安全台、数据源、沙箱运维 | +| `integrations/channels` + `dingtalk` / `lark` / `email` / `yida` | 入站渠道机器人框架与各平台适配 | +| `personal_agent` | Personal Agent：会话、渠道、调度、投递、设备连接、工作区人格 | + +模块约定： ``` -┌─────────────────────────────────────────┐ -│ Routes (api/routes/v1/) │ ← HTTP 入口、参数校验、响应包装 -│ - 依赖注入: auth, db │ -│ - Pydantic 请求模型 │ -│ - _to_dict() 转换函数 │ -│ - 调用 Service │ -├─────────────────────────────────────────┤ -│ Services (core/services/) │ ← 业务逻辑、权限校验、审计 -│ - 构造函数接收 db: Session │ -│ - 内部创建 Repository │ -│ - 幂等操作 (ensure_xxx) │ -│ - 抛 AppException │ -├─────────────────────────────────────────┤ -│ Repository (core/db/repository/) │ ← 数据访问抽象（包，10 个领域文件） -│ - CRUD 操作 │ -│ - 分页查询 │ -│ - 软删除过滤 │ -│ - 不含业务逻辑 │ -├─────────────────────────────────────────┤ -│ Models (core/db/models/) │ ← ORM 定义（包，17 个领域文件） -│ - SQLAlchemy declarative │ -│ - 索引、约束、关系 │ -│ - EE 专属表登记 edition_ee/db/edition_tables.py │ -│ - Alembic 迁移 │ -└─────────────────────────────────────────┘ +features// +├── README.md 负责什么、公开入口、依赖方向、版本差异 +├── extension.py NAME + setup() -> bool；可选 register_tools / register_gateway_contributions / +│ register_catalog_options / register_retrieval_context +├── models.py ORM（按安装清单加载，与运行开关无关） +├── edition.py （可选）CE/EE 差异唯一入口，CE overlay 只覆盖它 +├── service.py / repository.py / <领域>.py +└── api/ 路由文件（多数进 CE_ROUTERS / EE_ROUTERS，可关闭的由 setup() 内 register_router 挂载） ``` -## 聊天流式请求流转 +## 4. 启动与路由装配（api/app.py） + +1. `orchestration.configuration.register_configuration()`、`features.register_run_capabilities()`、 + `register_model_credentials()`、`integrations.register_credentials()` +2. 挂 health、`api/routes` 文件下载路由 +3. `iter_edition_routers(CE_ROUTERS)` 全部 include；`iter_edition_routers(EE_ROUTERS)` include 并附加 + `edition_router_dependencies(policy_key)`（License 能力位，未授权 402） +4. `register_product_tools()` / `register_catalog_options()` / `register_package_sources()` / `register_resource_owners()` +5. `load_extensions(INSTALLED_FEATURES)`：先 `load_models`，再逐个 `.extension.setup()`；然后挂 + `core.harness.registry.routers()` 里登记的模块路由 +6. `sites_serve_router`（静态站点兜底）→ 登录路由 → mock SSO（按登录模式）→ 本地模式前端静态托管（最后） + +注册表规则： + +- `CE_ROUTERS`（`api/routes/v1/__init__.py`）：二元组 `(module, attr)`；短名解析为 `api.routes.v1.`， + 含点号的写绝对路径（`"features.sites.api.sites"`）。**顺序即注册顺序**，同前缀族的先后有语义（例：`plugin_ui` 必须在 + `plugins` 之前，否则 `/v1/plugins/{slug}` 会吞掉 `/v1/plugins/ui-contributions`） +- `EE_ROUTERS`（`edition_ee/routes/registry.py`）：三元组 `(module, attr, policy_key)`；`policy_key` 取 + `edition_ee/licensing/features.py::Feature` 的值（`content_admin` / `audit` / `multi_tenancy` / `system_config` / + `billing` / `model_gateway` / `evolution_control` …）；`None` 仅用于 License 失效时也必须可达的入口（登录、换证） +- `api.edition.EE_ROUTERS` 是懒加载接缝；CE overlay 把它换成空表 +- **不要**在 `app.py` 手工 `include_router()` + +## 5. 一次聊天回合的流转 ``` -POST /v1/agents/responses - → api/routes/v1/chats/__init__.py - → orchestration/chat_run_executor.py # ChatRun + Redis Stream（后台 run，SSE 跟随，断线续播） - → orchestration/workflow.py # 流式编排主入口 - → core/chat/modes.py # 解析对话模式 ChatModeSpec（standard/turbo/市场模式；装配契约） - → orchestration/memory_integration.py # 检索分层记忆并注入（user-role 冻结块，600ms 预算） - → core/llm/factory/build.py # 构建 AgentScope 2.0 ReActAgent（按 ChatModeSpec 收窄工具面） - → core/llm/mcp/manager.py # MCP 客户端池（streamable-http → mcp 容器） - → core/config/mcp_config.py # MCP server 定义（端口真源 mcp_servers/_ports.py） - → core/llm/middlewares.py # AS2 中间件（动态模型、文件上下文、CitationAnchorMiddleware、OntologyGateMiddleware） - → prompts/prompt_runtime.py # 装配系统提示词（DB 版本池优先，prompt_text/ 兜底） - → orchestration/streaming.py # StreamingAgent：把 AS2 事件流映射为内部块事件 - → orchestration/citation_anchor.py # 引用锚点唯一发号（AnchorAllocator，e1/e2/…，经中间件回注工具结果） - → core/memory/service.py # 流结束后后台保存记忆（L1/L2/L3） - → orchestration/followups.py # 流结束后后台生成追问建议（写入消息 extra_data） +POST /v1/agents/responses api/routes/v1/agent_responses.py（+ chats/ 子路由） + → 准入与回合登记 core/chat/runtime/admission.py · core/runs（租约、事件日志） + → 聊天 worker core/chat/runtime/worker.py（消费事件、按顺序持久化输出） + → core/agent_runtime/runner.astream_chat_workflow + → prepare_main / prepare_direct 建 agent、组装输入（模式 core/chat/modes.py、记忆 integrations/memory.py） + → core/llm/factory/build.py AgentScope agent：模型选择、工具集、MCP（core/llm/mcp）、技能、提示词 + → stream_main / stream_direct 模型与工具事件 → 事件协议（core/llm/runtime/streaming.py、stream_events.py） + → core/llm/citations/ 引用锚点发号（e1/e2…） + → 结束后：本体评审、记忆 / 演化结算、追问（core/chat/runtime/followups.py） + → SSE 输出 api/transports/run_sse.py（跟随事件日志；续播 GET /v1/chats/stream/{run_id}/subscription） ``` -> `orchestration/strategy.py` 仍在但已退化为占位符（两个分支都返回 MainOnlyStrategy），不再是主链路环节。 - -**平级编排入口**（不走普通 chat run 的独立驱动）： - -| 入口 | 文件 | 说明 | -|------|------|------| -| 自主循环 | `orchestration/autonomous_loop.py` + `loop_planner.py` / `loop_evaluator.py` + `subagents/loop_reviewer.py` | driver 持有需求台账 feature_list.json，逐条注入 + 只读 reviewer 验收；路由 `v1/loops.py` | -| 批量执行 | `orchestration/batch_orchestrator.py` | batch_confirm 确认后逐行执行 | -| 计划模式 | `orchestration/subagents/plan_mode.py` | plan_generate / plan_execute run | -| 渠道入站 | `core/channels/`（protocol/registry/manager/inbound/outbound） | IM 消息归一为 InboundMsg 后复用聊天编排，回复经 markdown.py 降级回发 | -| 定时任务 | `orchestration/schedulers/`（automation / distillation_cron / evolution / memory_ttl） | croniter 调度 | - -SSE 事件（前端解析真源 `src/frontend/src/hooks/chatStream.ts`）： - -- 生命周期：`run_started`（首帧，携带续播 run_id）、`meta`、`error`、`end` / `data: [DONE]`（终止）、心跳为 SSE 注释行 -- 正文/思考：`content`（delta 增量；别名 ai_message/text/delta）、`content_replace`、`thinking`、`compaction_notice`、`steer_applied` -- 工具：`tool_call_start`、`tool_call_delta`（参数流式）、`tool_call`、`tool_result`、`tool_pending`、`model_progress`（活性信号） -- 交互确认：`batch_confirm`、`file_confirm`、`design_pick` -- 计划/子智能体：`plan_update`、`subagent_event` -- 本体治理：`ontology_activation` / `ontology_gate` / `ontology_review` / `ontology_repair` / `ontology_revision` / `ontology_revision_thinking` - -断线续播：`GET /v1/chats/stream/{run_id}`。追问建议（follow_up_questions）流结束后后台生成，也可能以 `follow_up` 事件流内直送。 - -## 模块索引 - -| 模块 | 路径 | 职责 | -|------|------|------| -| App 入口 | `api/app.py` | FastAPI 实例、中间件；路由按注册表自动注册 | -| CE 路由注册表 | `api/routes/v1/__init__.py` | **CE_ROUTERS 单一真源**（二元组）；re-export EE_ROUTERS | -| EE 路由注册表 | `edition_ee/routes/registry.py` | **EE_ROUTERS 单一真源**（三元组，带 license 能力位） | -| 依赖注入 | `api/deps.py` | require_admin（ADMIN_TOKEN）、require_config（CONFIG_TOKEN）、require_admin_or_config、require_system_settings、require_super_admin | -| 用户认证 | `core/auth/backend.py` | get_current_user / UserContext（桌面桥接 → session → API-Key → mock/remote） | -| 健康检查 | `api/health.py` | /health, /ready, /live | -| Schema | `api/schemas.py` | 请求/响应 Pydantic 模型 | -| 路由 | `api/routes/v1/` | 66 个路由文件 | -| 技能引擎 | `core/capabilities/skills/` | SKILL.md 解析、多源加载、{dir} 沙箱路径注入 | -| 生成物 | `core/files/artifacts/` | 注册与下载（store.py，local/oss 双模） | -| 认证 | `core/auth/` | backend.py, session.py, sso.py, permissions_iface.py（CE/EE 接缝） | -| 渠道 | `core/channels/` | 入站机器人框架：钉钉/飞书/企微/微信 adapter；加渠道 = 写 adapter + registry.py 注册 | -| 聊天 | `core/chat/` | context.py, tool_log.py | -| 配置 | `core/config/` | settings.py + catalog 五件套（catalog.json/catalog.py/loader/resolver/migration）+ mcp_config.py | -| 内容 | `core/files/content/` | 内容块、file_parser.py | -| 数据库 | `core/db/` | engine.py, models/（17 领域文件）, repository/（10 领域文件）, model_extensions.py, edition_tables.py（垫片） | -| 进化 | `core/evolution/` | GCE 共享契约（Episode/候选/结算）；控制面在 edition_ee/evolution | -| 基础设施 | `core/infra/` | exceptions, responses, logging, metrics, 限流, Redis, crypto | -| 知识库 | `core/kb/` | 分块、向量化、混合检索 + wiki/（LLM-Wiki 管线）+ wiki_router.py + external_provider.py（外接接缝） | -| License | `core/licensing/` | features.py（能力位+402）、manager.py（状态机） | -| LLM | `core/llm/` | agent_factory（含 turbo 参数）, chat_models, middlewares, mcp_manager/mcp_pool, offloader, subagent_tool, tools/ | -| 记忆 | `core/memory/` | service.py（mem0）、pipeline.py、profile.py（L1）、sanitizer.py、audit.py | -| 本体 | `core/ontology/` | Domain Pack 校验、运行时选取、工具闸（schemas/validator/revision/toolkit） | -| 沙箱 | `core/sandbox/` | protocol.py + script_runner / opensandbox / cube 三实现 | -| 服务 | `core/services/` | 70+ 业务服务（user/chat/chat_mode/channel/loop/kb/plugin/prompt_version/marketplace…） | -| 存储 | `core/files/storage/` | protocol, local, s3, oss | -| 编排 | `orchestration/` | workflow, chat_run_executor, streaming, citation_anchor, autonomous_loop, followups, batch_orchestrator, schedulers/, subagents/ | -| 提示词 | `prompts/` | prompt_runtime + provider + prompt_text/{default,code_exec,distillation,plan_mode,subagents,turbo}/ | -| MCP 服务器 | `mcp_servers/` | 12 个 server 目录、11 个在役（端口 9100–9114，真源 _ports.py；9105/9109-9111 为退役保留位） | -| 技能资产 | `skill_bundles/` | default/（5 内置）+ marketplace/（40+ 可安装） | -| EE 树 | `edition_ee/` | EE 专属路由/模型/KB provider（dify/fastgpt/weknora）/evolution 控制面；CE 派生树物理删除 | - -## 新路由注册步骤 - -1. 创建 `api/routes/v1/my_feature.py`（EE 专属则放 `edition_ee/routes/`），文件内定义 `router = APIRouter(prefix="/v1/xxx", tags=["Xxx"])` -2. 注册进对应注册表： - - 通用功能 → `api/routes/v1/__init__.py` 的 `CE_ROUTERS` 加 `("my_feature", "router")` - - 企业版功能 → `edition_ee/routes/registry.py` 的 `EE_ROUTERS` 加 `("edition_ee.routes.my_feature", "router", "")`（能力位 `None` 表示显式豁免 feature 守卫） -3. **不要**在 `api/app.py` 手工 `include_router()` —— app 启动时经 `iter_edition_routers()` 按表自动注册，模块缺失（CE 派生树物理删除了 EE 文件）会被静默跳过 +平级编排入口（不走普通回合）： + +| 入口 | 位置 | +|---|---| +| 计划模式 | `orchestration/plans/`（generate_run / execute_run / mode） | +| 自主循环 | `orchestration/loops/`（driver / planner / reviewer / requirements / recovery） | +| 定时任务派发 | `orchestration/automations/` + `features/automation` | +| 周期任务 / 恢复 | `orchestration/workers/`（`_worker_base.py`、run_recovery、memory_ttl、evolution、蒸馏 cron） | +| 渠道入站 | `integrations/channels/`（归一为入站消息后复用聊天回合） | + +## 6. 数据访问与事件循环 + +- 同步 SQLAlchemy Session 的每次查询都是阻塞 I/O。 +- **只用 Session 不 await 的路由写 `def`**（FastAPI 线程池执行）；`def` 路由里没有事件循环，不能 `create_task` / `get_running_loop`。 +- **要 await 网络的 async 代码不持有 Session**：每段 DB 工作写成同步函数，用 `await run_in_session(fn, ...)`（新开并关闭会话） + 或 `await run_db(fn, ...)`（已有会话）在线程里跑；需要 async 接口的纯 DB 函数用 `@off_loop` 包装。 +- 线程池与连接池按 `core.db.engine.thread_budget()` 同一预算配置（`apply_thread_budget()`）。 +- 跨请求存活的工作：落库交给 worker（`orchestration/workers/_worker_base.py`）；事件循环上的临时任务用 `core.infra.background.spawn(coro, name=...)`。 +- 守卫测试：`tests/test_no_blocking_async_routes.py`、`tests/test_no_event_loop_in_sync_routes.py`、 + `tests/infra/test_no_session_on_event_loop.py`（全仓零例外：任何 async 函数都不收 `db` / `Session` / `Depends(get_db)`、 + 不调 `SessionLocal()`、不 `with … as db`；测试里覆盖 `get_db` 时同时用 `tests/db_support.py` 让 `run_in_session` 指向同一个库）。 + +## 7. 模块索引（常用入口） + +| 主题 | 路径 | +|---|---| +| 应用组装 | `api/app.py`、`api/lifecycle.py`、`api/startup/` | +| 路由注册表 | `api/routes/v1/__init__.py`（CE）、`edition_ee/routes/registry.py`（EE）、`core/harness/registry`（模块路由） | +| 依赖注入 | `api/deps.py`（require_admin / require_config / require_admin_or_config / require_system_settings / require_ontology_governance / require_super_admin / console_actor） | +| 认证 | `core/auth/backend.py`（get_current_user / UserContext）、`core/auth/bearer.py` | +| DB | `core/db/engine.py`（get_db）、`core/db/threads.py`、`core/db/models/`、`core/db/repository/` | +| 信封 / 异常 | `core/infra/responses.py`、`core/infra/exceptions.py`、`api/middleware/error_handler.py` | +| 配置 | `core/config/settings.py`、`core/config/catalog.json` | +| 智能体回合 | `core/agent_runtime/`、`core/llm/factory/`、`core/llm/runtime/` | +| 运行与续播 | `core/runs/`、`core/chat/runtime/`、`api/transports/run_sse.py` | +| MCP 服务 | `extensions/mcp_servers/`（`registry.py`、`_ports.py` 端口稳定契约、`_launcher.py`）、`core/config/mcp_config.py` | +| 技能 / 插件 / 智能体资源 | `extensions/{skills,plugins,agents}/{default,marketplace}`、`core/capabilities/` | +| 提示词 | `prompts/prompt_runtime.py`（DB 版本池优先，`prompt_text/` 兜底）、`core/llm/prompt_*.py` | +| License | `edition_ee/licensing/`（features / deps / gate / manager）、`api/middleware/edition.py` | +| CE 派生 | `ce/manifest.yaml`（排除）、`ce/overlay/`（覆盖接缝文件） | diff --git a/.claude/skills/hugagent-backend-dev/references/error-codes.md b/.claude/skills/hugagent-backend-dev/references/error-codes.md index 18149924..73d26851 100644 --- a/.claude/skills/hugagent-backend-dev/references/error-codes.md +++ b/.claude/skills/hugagent-backend-dev/references/error-codes.md @@ -35,6 +35,8 @@ |------|------|----------------|------| | 40001 | 404 | `ResourceNotFoundError` | 资源未找到 | | 40002 | 404 | `EndpointNotFoundError` | API 端点不存在 | +| 40201 | 402 | `FeatureNotLicensed`（`edition_ee/licensing/features.py`） | License 未授权该能力（EE 路由能力位守卫） | +| 40202 | 402 | `SeatLimitExceeded`（同上） | 席位不足 / License 无效，无法新增用户 | | 41001 | 409 | `ResourceAlreadyExistsError` | 资源已存在 | | 41002 | 409 | `ConcurrentModificationError` | 并发修改冲突 | | 42001 | 429 | `RateLimitExceededError` | 速率限制 | diff --git a/.claude/skills/hugagent-backend-dev/scripts/scaffold_feature.sh b/.claude/skills/hugagent-backend-dev/scripts/scaffold_feature.sh index 313654df..ff2ca623 100755 --- a/.claude/skills/hugagent-backend-dev/scripts/scaffold_feature.sh +++ b/.claude/skills/hugagent-backend-dev/scripts/scaffold_feature.sh @@ -1,83 +1,108 @@ #!/usr/bin/env bash # -# 后端新功能脚手架生成器 +# 后端功能模块脚手架：生成 features// 完整骨架 # # 用法: -# bash .claude/skills/hugagent-backend-dev/scripts/scaffold_feature.sh +# bash .claude/skills/hugagent-backend-dev/scripts/scaffold_feature.sh # # 示例: # bash .claude/skills/hugagent-backend-dev/scripts/scaffold_feature.sh bookmark Bookmark bookmarks # -# 生成文件: -# - src/backend/api/routes/v1/s.py (路由) -# - src/backend/core/services/_service.py (服务) -# - src/backend/tests/test_.py (测试) +# 生成: +# src/backend/features//__init__.py +# src/backend/features//README.md 职责、公开入口、依赖方向 +# src/backend/features//extension.py NAME + setup() +# src/backend/features//models.py ORM（UTCDateTime + utc_now + JSONType） +# src/backend/features//repository.py +# src/backend/features//service.py +# src/backend/features//api/__init__.py +# src/backend/features//api/s.py 路由（同步 def + run_in_session 示例） +# src/backend/tests/features//__init__.py +# src/backend/tests/features//test_.py # -# 注意: model 和 repository 需手动合并到对应领域文件中 -# - core/db/models/<领域>.py ← 添加 ORM 类（并在包 __init__.py re-export） -# - core/db/repository/<领域>.py ← 添加 Repository 类 -# - api/routes/v1/__init__.py ← 注册进 CE_ROUTERS / EE_ROUTERS 注册表 -# - 运行: alembic revision --autogenerate -m "add ${table_name} table" +# 只建新文件，已存在的文件跳过；不改任何注册表（手动步骤见脚本末尾输出）。 +# 不是所有新功能都该进 features/：先按 SKILL.md「放哪里」判断。 set -euo pipefail -FEATURE="${1:?Usage: $0 }" -FEATURE_CAP="${2:?Usage: $0 }" -TABLE="${3:?Usage: $0 }" +FEATURE="${1:?Usage: $0 }" +FEATURE_CAP="${2:?Usage: $0 }" +TABLE="${3:?Usage: $0 }" +FEATURE_UPPER="$(echo "$FEATURE" | tr '[:lower:]' '[:upper:]')" SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" TEMPLATE_DIR="$SCRIPT_DIR/../templates" PROJECT_ROOT="$(cd "$SCRIPT_DIR/../../../../" && pwd)" BACKEND="$PROJECT_ROOT/src/backend" +PKG="$BACKEND/features/$FEATURE" +TESTS="$BACKEND/tests/features/$FEATURE" -echo "=== HugAgentOS Backend Scaffold ===" -echo "Feature: $FEATURE / $FEATURE_CAP" -echo "Table: $TABLE" -echo "" - -# Route file -ROUTE_FILE="$BACKEND/api/routes/v1/${FEATURE}s.py" -if [ ! -f "$ROUTE_FILE" ]; then +render() { # render